├── metroid ├── py.typed ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── metroid.py ├── migrations │ ├── __init__.py │ ├── 0002_failedpublishmessage.py │ └── 0001_initial.py ├── __init__.py ├── apps.py ├── typing.py ├── utils.py ├── models.py ├── republish.py ├── rq.py ├── celery.py ├── publish.py ├── subscribe.py ├── admin.py └── config.py ├── tests ├── __init__.py ├── unit │ ├── __init__.py │ ├── test_subject_pattern_validation.py │ └── test_config.py ├── functional │ ├── __init__.py │ ├── test_rq │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── test_rq.py │ │ ├── test_admin_rq.py │ │ └── test_subscribe_rq.py │ ├── test_celery │ │ ├── __init__.py │ │ ├── test_celery.py │ │ ├── test_subscribe_celery.py │ │ └── test_admin_celery.py │ ├── test_management_command.py │ ├── test_publish.py │ ├── test_republish.py │ └── mock_service_bus.py └── conftest.py ├── demoproj ├── __init__.py ├── demoapp │ ├── __init__.py │ ├── apps.py │ └── services.py ├── asgi.py ├── wsgi.py ├── celery.py ├── urls.py ├── tasks.py └── settings.py ├── .github ├── images │ └── intility.png └── workflows │ ├── publish_to_pypi.yml │ ├── coverage.yml │ └── testing.yml ├── .env_example ├── docker-compose.yml ├── .gitignore ├── mypy.ini ├── CONTRIBUTING.md ├── manage.py ├── LICENSE ├── .flake8 ├── .pre-commit-config.yaml ├── pyproject.toml └── README.md /metroid/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demoproj/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /demoproj/demoapp/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/functional/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metroid/management/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metroid/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/functional/test_rq/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /metroid/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/functional/test_celery/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/images/intility.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intility/metroid/HEAD/.github/images/intility.png -------------------------------------------------------------------------------- /.env_example: -------------------------------------------------------------------------------- 1 | CONNECTION_STRING_METRO_DEMO=Endpoint=sb://..... 2 | PUBLISH_METRO_KEY=asd... 3 | PUBLISH_METRO_URL=https://... 4 | -------------------------------------------------------------------------------- /demoproj/demoapp/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class DemoappConfig(AppConfig): 5 | name = 'demoapp' 6 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | 5 | redis: 6 | image: redis:latest 7 | ports: 8 | - '127.0.0.1:6378:6379' 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .idea/* 3 | env/ 4 | .env 5 | venv/ 6 | .venv/ 7 | build/ 8 | dist/ 9 | *.egg-info/ 10 | notes 11 | .pytest_cache 12 | .coverage 13 | htmlcov/ 14 | db.sqlite3 15 | coverage.xml 16 | 17 | # celery 18 | celerybeat-* 19 | -------------------------------------------------------------------------------- /tests/functional/test_rq/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from redislite import Redis 3 | 4 | 5 | @pytest.fixture(autouse=True) 6 | def mock_redis(monkeypatch): 7 | monkeypatch.setattr('django_rq.queues.get_redis_connection', lambda _, strict: Redis('/tmp/test.rdb')) 8 | -------------------------------------------------------------------------------- /metroid/__init__.py: -------------------------------------------------------------------------------- 1 | from metroid.config import settings 2 | 3 | if settings.worker_type == 'celery': 4 | from metroid.celery import MetroidTask # noqa F401 5 | from metroid.publish import publish_event # noqa F401 6 | 7 | __version__ = '1.4.0' 8 | default_app_config = 'metroid.apps.MetroidConfig' 9 | -------------------------------------------------------------------------------- /metroid/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class MetroidConfig(AppConfig): 5 | name = 'metroid' 6 | 7 | def ready(self) -> None: 8 | """ 9 | Validate settings on app ready 10 | """ 11 | from metroid.config import settings 12 | 13 | settings.validate() 14 | -------------------------------------------------------------------------------- /demoproj/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for demoproj project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.1/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'demoproj.settings') 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /demoproj/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for demoproj project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.1/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'demoproj.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | # Global options 2 | 3 | [mypy] 4 | python_version = 3.10 5 | # flake8-mypy expects the two following for sensible formatting 6 | show_column_numbers = True 7 | show_error_context = False 8 | # Enables or disables strict Optional checks: https://readthedocs.org/projects/mypy/downloads/pdf/stable/ 9 | strict_optional = True 10 | # Allow no return statement 11 | warn_no_return = False 12 | 13 | # Per module options: 14 | 15 | [mypy-metroid.migrations.*] 16 | ignore_errors = True 17 | -------------------------------------------------------------------------------- /demoproj/celery.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from celery import Celery 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | if os.name == 'nt': 9 | # Windows configuration to make celery run ok on Windows 10 | os.environ.setdefault('FORKED_BY_MULTIPROCESSING', '1') 11 | 12 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'demoproj.settings') 13 | 14 | app = Celery('metroid') 15 | app.config_from_object('django.conf:settings', namespace='CELERY') 16 | app.autodiscover_tasks() 17 | -------------------------------------------------------------------------------- /.github/workflows/publish_to_pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPI 2 | 3 | on: 4 | release: 5 | types: [published] 6 | 7 | jobs: 8 | build-and-publish: 9 | name: Build and publish 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: actions/setup-python@v2 14 | with: 15 | python-version: 3.9 16 | - uses: snok/install-poetry@v1 17 | - name: Publish to pypi 18 | run: | 19 | poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }} 20 | poetry publish --build --no-interaction 21 | -------------------------------------------------------------------------------- /metroid/typing.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Callable 2 | from typing import Literal, TypedDict 3 | 4 | 5 | class Handler(TypedDict): 6 | subject: str 7 | regex: bool 8 | handler_function: Callable 9 | 10 | 11 | class Subscription(TypedDict): 12 | topic_name: str 13 | subscription_name: str 14 | connection_string: str 15 | handlers: list[Handler] 16 | 17 | 18 | class TopicPublishSettings(TypedDict): 19 | topic_name: str 20 | x_metro_key: str 21 | 22 | 23 | class MetroidSettings(TypedDict): 24 | subscriptions: list[Subscription] 25 | publish_settings: list[TopicPublishSettings] 26 | worker_type: Literal['rq', 'celery'] 27 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing to Metroid 2 | ======================= 3 | 4 | 1. Create an issue explaining the bug you'd like to fix/the feature you'd like to implement 5 | 6 | 2. Get it approved 7 | 8 | 3. Fork the upstream repository into a personal account 9 | 10 | 4. Install poetry running ``pip install poetry`` or ``curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -`` 11 | 12 | 5. Install dependencies by running ``poetry install`` 13 | 14 | 6. Install pre-commit (for linting) by running ``pre-commit install`` 15 | 16 | 7. Create a new branch for you changes 17 | 18 | 8. Push the topic branch to your personal fork 19 | 20 | 9. Create a pull request to the main repository 21 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | """Run administrative tasks.""" 9 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'demoproj.settings') 10 | try: 11 | from django.core.management import execute_from_command_line 12 | except ImportError as exc: 13 | raise ImportError( 14 | "Couldn't import Django. Are you sure it's installed and " 15 | 'available on your PYTHONPATH environment variable? Did you ' 16 | 'forget to activate a virtual environment?' 17 | ) from exc 18 | execute_from_command_line(sys.argv) 19 | 20 | 21 | if __name__ == '__main__': 22 | main() 23 | -------------------------------------------------------------------------------- /tests/functional/test_celery/test_celery.py: -------------------------------------------------------------------------------- 1 | from django.test import override_settings 2 | from django.utils.module_loading import import_string 3 | 4 | import pytest 5 | 6 | from metroid.models import FailedMessage 7 | 8 | 9 | @pytest.mark.django_db 10 | def test_faulty_metro_data(): 11 | assert FailedMessage.objects.count() == 0 12 | with override_settings(CELERY_TASK_ALWAYS_EAGER=True): 13 | a_random_task = import_string('demoproj.tasks.a_random_task') 14 | a_random_task.apply_async( 15 | kwargs={ 16 | 'message': {'hello': 'world'}, 17 | 'topic_name': 'mocked_topic', 18 | 'subscription_name': 'mocked_subscription', 19 | 'subject': 'mocked_subject', 20 | } 21 | ) 22 | assert FailedMessage.objects.count() == 1 23 | -------------------------------------------------------------------------------- /demoproj/urls.py: -------------------------------------------------------------------------------- 1 | """demoproj URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/3.1/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.contrib import admin 17 | from django.urls import path 18 | 19 | urlpatterns = [ 20 | path('admin/', admin.site.urls), 21 | ] 22 | -------------------------------------------------------------------------------- /metroid/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import re 3 | 4 | from django.core.exceptions import ImproperlyConfigured 5 | 6 | logger = logging.getLogger('metroid') 7 | 8 | 9 | def match_handler_subject( 10 | subject: str, 11 | message_subject: str, 12 | is_regex: bool, 13 | ) -> bool: 14 | """ 15 | Checks if the provided message subject matches the handler's subject. Performs a match by using the regular 16 | expression, or compares strings based on handler settings defined in settings.py. 17 | 18 | """ 19 | if is_regex: 20 | try: 21 | pattern = re.compile(subject) 22 | return bool(re.match(pattern=pattern, string=message_subject)) 23 | except re.error: 24 | raise ImproperlyConfigured(f'Provided regex pattern: {subject} is invalid.') 25 | else: 26 | return subject == message_subject 27 | -------------------------------------------------------------------------------- /metroid/migrations/0002_failedpublishmessage.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2021-10-04 11:32 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('metroid', '0001_initial'), 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='FailedPublishMessage', 15 | fields=[ 16 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 17 | ('topic_name', models.CharField(max_length=255)), 18 | ('event_type', models.CharField(max_length=255)), 19 | ('subject', models.CharField(max_length=255)), 20 | ('data_version', models.CharField(max_length=255)), 21 | ('event_time', models.DateTimeField()), 22 | ('data', models.JSONField()), 23 | ('correlation_id', models.CharField(blank=True, max_length=36)), 24 | ], 25 | ), 26 | ] 27 | -------------------------------------------------------------------------------- /metroid/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.1.5 on 2021-02-01 18:06 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | initial = True 9 | 10 | dependencies = [] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='FailedMessage', 15 | fields=[ 16 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 17 | ('created', models.DateTimeField(auto_now_add=True)), 18 | ('topic_name', models.CharField(max_length=255)), 19 | ('subscription_name', models.CharField(max_length=255)), 20 | ('subject', models.CharField(max_length=255)), 21 | ('message', models.JSONField()), 22 | ('exception_str', models.TextField()), 23 | ('traceback', models.TextField()), 24 | ('correlation_id', models.CharField(blank=True, max_length=36)), 25 | ], 26 | ), 27 | ] 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Intility AS 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: coverage 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | jobs: 9 | codecov: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - uses: actions/setup-python@v2 14 | with: 15 | python-version: 3.11 16 | - name: Install poetry 17 | uses: snok/install-poetry@v1 18 | with: 19 | virtualenvs-create: true 20 | virtualenvs-in-project: true 21 | - name: Load cached venv 22 | id: cached-poetry-dependencies 23 | uses: actions/cache@v2 24 | with: 25 | path: .venv 26 | key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }} 27 | - name: Install dependencies 28 | run: poetry install 29 | if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' 30 | - name: Test with pytest 31 | run: poetry run pytest --cov=metroid tests/ --verbose --assert=plain --cov-report=xml 32 | - name: Upload coverage 33 | uses: codecov/codecov-action@v2 34 | with: 35 | file: ./coverage.xml 36 | fail_ci_if_error: true 37 | token: ${{ secrets.CODECOV_TOKEN }} 38 | -------------------------------------------------------------------------------- /demoproj/tasks.py: -------------------------------------------------------------------------------- 1 | from demoproj.celery import app 2 | 3 | from metroid.celery import MetroidTask 4 | 5 | 6 | @app.task(base=MetroidTask) 7 | def a_random_task(*, message: dict, topic_name: str, subscription_name: str, subject: str) -> Exception: 8 | """ 9 | Mocked function for tests. Matches on the subject 'MockedTask'. 10 | """ 11 | raise ValueError('My mocked error :)') 12 | 13 | 14 | def error_task(*, message: dict, topic_name: str, subscription_name: str, subject: str) -> None: 15 | """ 16 | Mocked function for tests, is to be ran on the subject 'ErrorTask'. This function does nothing, and is meant to 17 | tests tasks without the Celery annotation. 18 | """ 19 | lambda x: None 20 | 21 | 22 | @app.task(base=MetroidTask) 23 | def my_task(**kwargs) -> None: 24 | """ 25 | Function used for testing, returns nothing. 26 | """ 27 | return None 28 | 29 | 30 | @app.task(base=MetroidTask) 31 | def example_celery_task(*, message: dict, topic_name: str, subscription_name: str, subject: str) -> None: 32 | """ 33 | Celery Example Task 34 | """ 35 | print('Do Something') # noqa: T201 36 | 37 | 38 | def example_rq_task(*, message: dict, topic_name: str, subscription_name: str, subject: str) -> None: 39 | """ 40 | RQ Example Task 41 | """ 42 | print('Do Something') # noqa: T201 43 | -------------------------------------------------------------------------------- /metroid/models.py: -------------------------------------------------------------------------------- 1 | from django.db.models import CharField, DateTimeField, JSONField, Model, TextField 2 | 3 | 4 | class FailedMessage(Model): 5 | created = DateTimeField(auto_now_add=True) 6 | topic_name = CharField(max_length=255) 7 | subscription_name = CharField(max_length=255) 8 | subject = CharField(max_length=255) 9 | message = JSONField() 10 | 11 | exception_str = TextField() 12 | traceback = TextField() 13 | 14 | # If there is a correlation ID (Requires Django GUID + Celery Integration), save it. Makes fetching logs easy 15 | correlation_id = CharField(max_length=36, blank=True) 16 | 17 | def __str__(self) -> str: 18 | """ 19 | String representation 20 | """ 21 | return f'{self.topic_name}-{self.subject}. Correlation ID: {self.correlation_id}' # pragma: no cover 22 | 23 | 24 | class FailedPublishMessage(Model): 25 | topic_name = CharField(max_length=255) 26 | event_type = CharField(max_length=255) 27 | subject = CharField(max_length=255) 28 | data_version = CharField(max_length=255) 29 | event_time = DateTimeField() 30 | data = JSONField() 31 | correlation_id = CharField(max_length=36, blank=True) 32 | 33 | def __str__(self) -> str: 34 | """ 35 | String representation 36 | """ 37 | return f'{self.topic_name}-{self.subject}. Correlation ID: {self.correlation_id}' # pragma: no cover 38 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | 4 | ignore= 5 | # E501: Line length 6 | E501 7 | # Docstring at the top of a public module 8 | D100 9 | # Docstring at the top of a public class (method is enough) 10 | D101 11 | # Make docstrings one line if it can fit. 12 | D200 13 | # Imperative docstring declarations 14 | D401 15 | # Type annotation for `self` 16 | TYP101 17 | # for cls 18 | TYP102 19 | # Missing docstring in __init__ 20 | D107 21 | # Missing docstring in public package 22 | D104 23 | # Missing type annotations for `**kwargs` 24 | TYP003 25 | # Whitespace before ':'. Black formats code this way. 26 | E203 27 | # 1 blank line required between summary line and description 28 | D205 29 | # First line should end with a period - here we have a few cases where the first line is too long, and 30 | # this issue can't be fixed without using noqa notation 31 | D400 32 | # Missing type annotations for self 33 | ANN101 34 | # Missing type annotation for cls in classmethod 35 | ANN102 36 | # Missing type annotations for **args 37 | ANN002 38 | # Missing type annotations for **kwargs 39 | ANN003 40 | # W503 line break before binary operator - conflicts with black 41 | W503 42 | 43 | exclude = 44 | .git, 45 | .idea, 46 | __pycache__, 47 | tests/*, 48 | venv, 49 | manage.py 50 | -------------------------------------------------------------------------------- /demoproj/demoapp/services.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | from django.utils import timezone 5 | 6 | import requests 7 | from decouple import config 8 | from demoproj.celery import app 9 | 10 | from metroid import MetroidTask 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | @app.task(base=MetroidTask) 16 | def my_func(*, message: dict, topic_name: str, subscription_name: str, subject: str) -> None: 17 | """ 18 | Demo task 19 | """ 20 | logger.info( 21 | 'Message %s, topic %s, subscription_name %s, subject: %s', message, topic_name, subscription_name, subject 22 | ) 23 | if subject == 'Test/Django/Module': 24 | response = requests.post( 25 | url=config('PUBLISH_METRO_URL'), 26 | headers={'content-type': 'application/json', 'x-metro-key': config('PUBLISH_METRO_KEY')}, 27 | data=json.dumps( 28 | { 29 | 'eventType': 'Intility.Jonas.Testing', 30 | 'eventTime': timezone.now().isoformat(), 31 | 'dataVersion': '1.0', 32 | 'data': {'content': 'Yo, Metro is awesome'}, 33 | 'subject': 'Test/Django/Module', 34 | } 35 | ), 36 | ) 37 | response.raise_for_status() 38 | logger.info('POSTED! %s', response.status_code) 39 | return 40 | 41 | 42 | @app.task(base=MetroidTask) 43 | def my_broken_task(*, message: dict, topic_name: str, subscription_name: str, subject: str) -> None: 44 | """ 45 | Broken demo task 46 | """ 47 | raise ValueError('Oops, an exception happened! You can retry this task in the admin Dashboard :-)') 48 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: | 2 | (?x)^( 3 | .*README.md| 4 | .*migrations/.*| 5 | .*static/.*| 6 | )$ 7 | repos: 8 | - repo: https://github.com/ambv/black 9 | rev: 23.12.1 10 | hooks: 11 | - id: black 12 | args: 13 | - --quiet 14 | - repo: https://github.com/pre-commit/pre-commit-hooks 15 | rev: v4.5.0 16 | hooks: 17 | - id: check-case-conflict 18 | - id: end-of-file-fixer 19 | - id: trailing-whitespace 20 | - id: check-ast 21 | - id: check-json 22 | - id: check-merge-conflict 23 | - id: detect-private-key 24 | - id: double-quote-string-fixer 25 | - repo: https://github.com/pycqa/flake8 26 | rev: 6.1.0 27 | hooks: 28 | - id: flake8 29 | additional_dependencies: 30 | - flake8-annotations==3.0.1 # Enforces type annotation 31 | - flake8-bugbear==23.9.16 # Looks for likely bugs and design problems 32 | - flake8-comprehensions==3.14.0 # Looks for unnecessary generator functions that can be converted to list comprehensions 33 | - flake8-deprecated==2.2.1 # Looks for method deprecations 34 | - flake8-docstrings==1.7.0 # Verifies that all functions/methods have docstrings 35 | - flake8-print==5.0.0 # Checks for print statements 36 | - flake8-use-fstring==1.4 # Enforces use of f-strings over .format and %s 37 | args: 38 | - --enable-extensions=G 39 | - repo: https://github.com/asottile/pyupgrade 40 | rev: v3.15.0 41 | hooks: 42 | - id: pyupgrade 43 | args: 44 | - --py310-plus 45 | - repo: https://github.com/pycqa/isort 46 | rev: 5.13.2 47 | hooks: 48 | - id: isort 49 | - repo: https://github.com/pre-commit/mirrors-mypy 50 | rev: v1.8.0 51 | hooks: 52 | - id: mypy 53 | additional_dependencies: 54 | - types-requests 55 | -------------------------------------------------------------------------------- /metroid/republish.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | from django.utils import timezone 5 | 6 | import requests 7 | 8 | from metroid.config import settings 9 | from metroid.models import FailedPublishMessage 10 | 11 | logger = logging.getLogger('metroid') 12 | 13 | 14 | def retry_failed_published_events() -> None: 15 | """ 16 | Trys to publish all previously failed messages again. 17 | 18 | It makes a wrapper around this function based on your needs. If you implement posting to Metro at the end 19 | of your API logic you might want to still return a 200 to the API user, even if a post to Metro should fail. 20 | 21 | :return: None - Metro gives empty response on valid posts 22 | :raises: requests.exceptions.HTTPError 23 | """ 24 | for message in FailedPublishMessage.objects.all(): 25 | formatted_data = { 26 | 'eventType': message.event_type, 27 | 'eventTime': message.event_time.isoformat() or timezone.now().isoformat(), 28 | 'dataVersion': message.data_version, 29 | 'data': message.data, 30 | 'subject': message.subject, 31 | } 32 | logger.info('Posting event to Metro topic %s. Data: %s', message.topic_name, formatted_data) 33 | try: 34 | metro_response = requests.post( 35 | url=f'https://api.intility.no/metro/{message.topic_name}', 36 | headers={ 37 | 'content-type': 'application/json', 38 | 'x-metro-key': settings.get_x_metro_key(topic_name=message.topic_name), 39 | }, 40 | data=json.dumps(formatted_data), 41 | ) 42 | logger.info('Posted to metro') 43 | metro_response.raise_for_status() 44 | message.delete() 45 | except Exception as error: 46 | logger.info('Failed to post to metro. %s', error) 47 | -------------------------------------------------------------------------------- /.github/workflows/testing.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: pull_request 4 | 5 | jobs: 6 | linting: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - uses: actions/setup-python@v2 11 | with: 12 | python-version: 3.11 13 | - run: python -m pip install pre-commit 14 | - run: pre-commit run --all-files 15 | test: 16 | needs: linting 17 | runs-on: ubuntu-latest 18 | strategy: 19 | fail-fast: false 20 | matrix: 21 | python-version: [ "3.10", "3.11", "3.12"] 22 | django-version: [ "4.2", "5.0" ] 23 | steps: 24 | - name: Check out repository 25 | uses: actions/checkout@v2 26 | - name: Set up python ${{ matrix.python-version }} 27 | uses: actions/setup-python@v2 28 | with: 29 | python-version: ${{ matrix.python-version }} 30 | - name: Install poetry 31 | uses: snok/install-poetry@v1 32 | with: 33 | virtualenvs-create: true 34 | virtualenvs-in-project: true 35 | version: latest 36 | - name: Load cached venv 37 | uses: actions/cache@v2 38 | id: cache-venv 39 | with: 40 | path: .venv 41 | key: ${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}-2 42 | - name: Install dependencies 43 | run: poetry install --no-interaction --no-root 44 | if: steps.cache-venv.outputs.cache-hit != 'true' 45 | - name: Install package 46 | run: poetry install --no-interaction 47 | - name: Install django ${{ matrix.django-version }} 48 | run: | 49 | source .venv/bin/activate 50 | poetry add "Django==${{ matrix.django-version }}" 51 | - name: Run tests 52 | run: | 53 | source .venv/bin/activate 54 | poetry run pytest --cov=metroid --verbose --assert=plain 55 | poetry run coverage report 56 | -------------------------------------------------------------------------------- /metroid/rq.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django_guid import get_guid 4 | 5 | from rq.job import Job 6 | 7 | logger = logging.getLogger('metroid') 8 | 9 | 10 | def on_failure(job: Job, *exc_info) -> bool: 11 | """ 12 | Custom exception handler for Metro RQ tasks. 13 | This function must be added as a custom exception handler in django-rq RQ_EXCEPTION_HANDLERS settings 14 | 15 | :param job: RQ Job that has failed 16 | :param exc_info: Exception Info, tuple of exception type, value and traceback 17 | """ 18 | if job.origin == 'metroid': 19 | topic_name = job.kwargs.get('topic_name') 20 | subscription_name = job.kwargs.get('subscription_name') 21 | subject = job.kwargs.get('subject') 22 | message = job.kwargs.get('message') 23 | correlation_id = get_guid() 24 | logger.critical( 25 | 'Metro task exception. Message: %s, exception: %s, traceback: %s', 26 | message, 27 | str(exc_info[1]), 28 | exc_info, 29 | ) 30 | try: 31 | from metroid.models import FailedMessage 32 | 33 | FailedMessage.objects.create( 34 | topic_name=topic_name, 35 | subscription_name=subscription_name, 36 | subject=subject, 37 | message=message, 38 | exception_str=str(exc_info[1]), 39 | traceback=str(exc_info), 40 | correlation_id=correlation_id or '', 41 | ) 42 | logger.info('Saved failed message to database.') 43 | except Exception as error: # pragma: no cover 44 | # Should be impossible for this to happen (famous last words), but a nice failsafe. 45 | logger.exception('Unable to save Metro message. Error: %s', error) 46 | # Return false to stop processing exception 47 | return False 48 | else: 49 | # Return true to send exception to next handler 50 | return True 51 | -------------------------------------------------------------------------------- /metroid/celery.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Any 3 | 4 | from billiard.einfo import ExceptionInfo 5 | from django_guid import get_guid 6 | 7 | from celery import Task 8 | 9 | logger = logging.getLogger('metroid') 10 | 11 | 12 | class MetroidTask(Task): 13 | def on_failure( 14 | self, exc: Exception, task_id: str, args: tuple, kwargs: dict[str, Any], einfo: ExceptionInfo 15 | ) -> None: 16 | """ 17 | Custom error handler for Metro Celery tasks. 18 | This function is automatically run by the worker when the task fails. 19 | 20 | :param exc: The exception raised by the task. 21 | :param task_id: Unique ID of the failed task. 22 | :param args: Original arguments for the task that failed. 23 | :param kwargs: Original keyword arguments for the task that failed 24 | :param einfo: Exception information 25 | :return: The return value of this handler is ignored, so it should not return anything 26 | """ 27 | # pass 28 | topic_name = kwargs.get('topic_name') 29 | subscription_name = kwargs.get('subscription_name') 30 | subject = kwargs.get('subject') 31 | message = kwargs.get('message') 32 | correlation_id = get_guid() 33 | logger.critical('Metro task exception. Message: %s, exception: %s, traceback: %s', message, str(exc), einfo) 34 | try: 35 | from metroid.models import FailedMessage 36 | 37 | FailedMessage.objects.create( 38 | topic_name=topic_name, 39 | subscription_name=subscription_name, 40 | subject=subject, 41 | message=message, 42 | exception_str=str(exc), 43 | traceback=str(einfo), 44 | correlation_id=correlation_id or '', 45 | ) 46 | logger.info('Saved failed message to database.') 47 | except Exception as error: # pragma: no cover 48 | # Should be impossible for this to happen (famous last words), but a nice failsafe. 49 | logger.exception('Unable to save Metro message. Error: %s', error) 50 | -------------------------------------------------------------------------------- /tests/functional/test_rq/test_rq.py: -------------------------------------------------------------------------------- 1 | from django.test import override_settings 2 | from django.utils.module_loading import import_string 3 | 4 | import django_rq 5 | import pytest 6 | 7 | from metroid.models import FailedMessage 8 | from metroid.rq import on_failure 9 | from rq import SimpleWorker 10 | 11 | 12 | @pytest.mark.django_db 13 | def test_faulty_metro_data(): 14 | assert FailedMessage.objects.count() == 0 15 | with override_settings(RQ_QUEUES={'metroid': {}, 'fake': {}}): 16 | a_random_task = import_string('demoproj.tasks.a_random_task') 17 | queue = django_rq.get_queue('metroid') 18 | queue.enqueue( 19 | a_random_task, 20 | job_id='5F9914AB-2CC1-4A57-9A67-8586ADC2D8B6', 21 | kwargs={ 22 | 'message': {'hello': 'world'}, 23 | 'topic_name': 'mocked_topic', 24 | 'subscription_name': 'mocked_subscription', 25 | 'subject': 'mocked_subject', 26 | }, 27 | ) 28 | worker = SimpleWorker([queue], connection=queue.connection, exception_handlers=[on_failure]) 29 | worker.work(burst=True) 30 | 31 | assert FailedMessage.objects.count() == 1 32 | 33 | 34 | @pytest.mark.django_db 35 | def test_non_metroid_task(): 36 | assert FailedMessage.objects.count() == 0 37 | with override_settings(RQ_QUEUES={'metroid': {}, 'fake': {}}): 38 | a_random_task = import_string('demoproj.tasks.a_random_task') 39 | queue = django_rq.get_queue('fake') 40 | queue.enqueue( 41 | a_random_task, 42 | job_id='5F9914AB-2CC1-4A57-9A67-8586ADC2D8B6', 43 | kwargs={ 44 | 'message': {'hello': 'world'}, 45 | 'topic_name': 'mocked_topic', 46 | 'subscription_name': 'mocked_subscription', 47 | 'subject': 'mocked_subject', 48 | }, 49 | ) 50 | worker = SimpleWorker([queue], connection=queue.connection, exception_handlers=[on_failure]) 51 | worker.work(burst=True) 52 | 53 | assert FailedMessage.objects.count() == 0 54 | -------------------------------------------------------------------------------- /tests/functional/test_management_command.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from django.core.management import call_command 4 | 5 | import pytest 6 | 7 | 8 | @pytest.mark.asyncio 9 | async def mock_subscription_return(**kwargs): 10 | if kwargs.get('topic_name') == 'test-two': 11 | await asyncio.sleep(2) 12 | else: 13 | await asyncio.sleep(20) 14 | 15 | 16 | @pytest.mark.asyncio 17 | async def mock_subscription_exception(**kwargs): 18 | if kwargs.get('topic_name') == 'test-two': 19 | raise ValueError('Mocked error') 20 | else: 21 | await asyncio.sleep(20) 22 | 23 | 24 | @pytest.fixture 25 | def subscriptions_return(mocker): 26 | """ 27 | Make one task exit quickly, while the rest runs for a long time. 28 | This triggers `return_when=asyncio.FIRST_COMPLETED` 29 | """ 30 | mocker.patch('metroid.management.commands.metroid.subscribe_to_topic', mock_subscription_return) 31 | 32 | 33 | @pytest.fixture 34 | def subscriptions_exception(mocker): 35 | """ 36 | Make one task exit quickly, while the rest runs for a long time. 37 | This triggers `return_when=asyncio.FIRST_COMPLETED` 38 | """ 39 | mocker.patch('metroid.management.commands.metroid.subscribe_to_topic', mock_subscription_exception) 40 | 41 | 42 | def test_command_early_return(subscriptions_return, caplog): 43 | with pytest.raises(SystemExit): 44 | call_command('metroid') 45 | assert [x for x in caplog.records if 'ended early without an exception' in x.message] 46 | assert [x for x in caplog.records if 'Cancelling pending task' in x.message] 47 | assert not [x for x in caplog.records if 'Exception in subscription' in x.message] 48 | assert [x for x in caplog.records if 'All tasks cancelled' in x.message] 49 | 50 | 51 | def test_command_exception(subscriptions_exception, caplog): 52 | with pytest.raises(SystemExit): 53 | call_command('metroid') 54 | assert [x for x in caplog.records if 'Exception in subscription' in x.message] 55 | assert [x for x in caplog.records if 'Cancelling pending task' in x.message] 56 | assert not [x for x in caplog.records if 'ended early without an exception' in x.message] 57 | assert [x for x in caplog.records if 'All tasks cancelled' in x.message] 58 | -------------------------------------------------------------------------------- /tests/unit/test_subject_pattern_validation.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import ImproperlyConfigured 2 | 3 | import pytest 4 | 5 | from metroid.subscribe import match_handler_subject 6 | 7 | 8 | def test_valid_pattern() -> None: 9 | """ 10 | Tests if a valid pattern matches the provided subject. 11 | """ 12 | subject = r'^something\/tests\/haha.*$' 13 | subject_in_message = 'something/tests/haha/asd-123' 14 | is_match = match_handler_subject(subject=subject, message_subject=subject_in_message, is_regex=True) 15 | assert is_match is True 16 | 17 | 18 | def test_wrong_subject_match_on_pattern() -> None: 19 | """ 20 | Tests if the validation fails if not a matching reggex is provided. 21 | """ 22 | subject = r'^something\/tests\/haha.*$' 23 | subject_in_message = 'tests/haha/asd-123' 24 | is_match = match_handler_subject(subject=subject, message_subject=subject_in_message, is_regex=True) 25 | assert is_match is False 26 | 27 | 28 | def test_match_on_string() -> None: 29 | """ 30 | Tests if the pattern matches the subject provided in string format. 31 | """ 32 | subject = 'tests/haha.Create' 33 | subject_in_message = 'tests/haha.Create' 34 | is_match = match_handler_subject(subject=subject, message_subject=subject_in_message, is_regex=False) 35 | assert is_match is True 36 | 37 | 38 | def test_bogus_string() -> None: 39 | """ 40 | Tests if a very weird string with special characters matches . 41 | """ 42 | subject = 'tests/haha$somethi#ngth"atshoul,-,.dn/otbehere' 43 | subject_in_message = 'tests/haha$somethi#ngth"atshoul,-,.dn/otbehere' 44 | is_match = match_handler_subject(subject=subject, message_subject=subject_in_message, is_regex=False) 45 | assert is_match is True 46 | 47 | 48 | def test_if_exception_is_thrown() -> None: 49 | """ 50 | Tests if the correct exception is thrown upon providing an invalid regex. 51 | """ 52 | with pytest.raises(ImproperlyConfigured) as e: 53 | subject = 'tests/invalid[' 54 | subject_in_message = 'tests/invalid[' 55 | is_match = match_handler_subject(subject=subject, message_subject=subject_in_message, is_regex=True) 56 | assert str(e.value) == f'Provided regex pattern: {subject} is invalid.' 57 | -------------------------------------------------------------------------------- /tests/functional/test_publish.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.utils import timezone 4 | 5 | import pytest 6 | from urllib3.exceptions import HTTPError 7 | 8 | from metroid.publish import publish_event 9 | 10 | 11 | @pytest.fixture 12 | def mock_response_ok(mocker): 13 | return_mock = mocker.Mock() 14 | mocker.patch('metroid.publish.requests', return_mock) 15 | return return_mock 16 | 17 | 18 | @pytest.fixture 19 | def mock_response_error(mocker): 20 | return_mock = mocker.Mock() 21 | return_mock.post.side_effect = HTTPError(mocker.Mock(status_code=500)) 22 | 23 | mocker.patch('metroid.publish.requests', return_mock) 24 | return return_mock 25 | 26 | 27 | def test_no_event_time_provided(mock_response_ok, freezer): 28 | now = timezone.now().isoformat() # freezegun ensures time don't pass 29 | publish_event( 30 | topic_name='test123', 31 | event_type='Intility.MyTopic', 32 | data_version='1.0', 33 | data={'hello': 'world'}, 34 | subject='my/test/subject', 35 | ) 36 | 37 | mock_response_ok.post.assert_called_with( 38 | url='https://api.intility.no/metro/test123', 39 | headers={'content-type': 'application/json', 'x-metro-key': 'my-metro-key'}, 40 | data=json.dumps( 41 | { 42 | 'eventType': 'Intility.MyTopic', 43 | 'eventTime': now, 44 | 'dataVersion': '1.0', 45 | 'data': {'hello': 'world'}, 46 | 'subject': 'my/test/subject', 47 | } 48 | ), 49 | ) 50 | 51 | 52 | def test_event_time_provided(mock_response_ok): 53 | now = timezone.now().isoformat() 54 | publish_event( 55 | topic_name='test123', 56 | event_type='Intility.MyTopic', 57 | data_version='1.0', 58 | data={'hello': 'world'}, 59 | subject='my/test/subject', 60 | event_time=now, 61 | ) 62 | mock_response_ok.post.assert_called_with( 63 | url='https://api.intility.no/metro/test123', 64 | headers={'content-type': 'application/json', 'x-metro-key': 'my-metro-key'}, 65 | data=json.dumps( 66 | { 67 | 'eventType': 'Intility.MyTopic', 68 | 'eventTime': now, 69 | 'dataVersion': '1.0', 70 | 'data': {'hello': 'world'}, 71 | 'subject': 'my/test/subject', 72 | } 73 | ), 74 | ) 75 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from django.test import override_settings 2 | 3 | import pytest 4 | from tests.functional.mock_service_bus import instance_service_mock_error, instance_service_mock_ok 5 | 6 | from metroid.config import Settings 7 | 8 | 9 | @pytest.fixture 10 | def mock_service_bus_client_ok(mocker): 11 | return mocker.patch('metroid.subscribe.ServiceBusClient', instance_service_mock_ok()) 12 | 13 | 14 | @pytest.fixture 15 | def mock_service_bus_client_failure(mocker): 16 | return mocker.patch('metroid.subscribe.ServiceBusClient', instance_service_mock_error()) 17 | 18 | 19 | @pytest.fixture(autouse=True) 20 | def mock_subscriptions(monkeypatch): 21 | """ 22 | Default settings 23 | """ 24 | with override_settings( 25 | METROID={ 26 | 'subscriptions': [ 27 | { 28 | 'topic_name': 'test', 29 | 'subscription_name': 'sub-test-djangomoduletest', 30 | 'connection_string': 'my long connection string', 31 | 'handlers': [ 32 | {'subject': 'Test/Django/Module', 'handler_function': 'demoproj.demoapp.services.my_func'}, 33 | { 34 | 'subject': 'Exception/Django/Module', 35 | 'handler_function': 'demoproj.demoapp.services.my_broken_task', 36 | }, 37 | ], 38 | }, 39 | { 40 | 'topic_name': 'test-two', 41 | 'subscription_name': 'sub-test-test2', 42 | 'connection_string': 'my long connection string', 43 | 'handlers': [ 44 | { 45 | 'subject': 'Exception/Django/Module', 46 | 'handler_function': 'demoproj.demoapp.services.my_broken_task', 47 | }, 48 | ], 49 | }, 50 | ], 51 | 'publish_settings': [ 52 | { 53 | 'topic_name': 'test123', 54 | 'x_metro_key': 'my-metro-key', 55 | } 56 | ], 57 | }, 58 | ): 59 | settings = Settings() 60 | monkeypatch.setattr('metroid.management.commands.metroid.settings', settings) 61 | monkeypatch.setattr('metroid.publish.settings', settings) 62 | monkeypatch.setattr('metroid.republish.settings', settings) 63 | monkeypatch.setattr('metroid.admin.settings', settings) 64 | -------------------------------------------------------------------------------- /metroid/management/commands/metroid.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import sys 4 | import time 5 | from asyncio.tasks import Task 6 | 7 | from django.core.management.base import BaseCommand 8 | 9 | from metroid.config import settings 10 | from metroid.subscribe import subscribe_to_topic 11 | 12 | logger = logging.getLogger('metroid') 13 | 14 | 15 | class Command(BaseCommand): 16 | help = ( 17 | 'Starts a subscription asyncio loop for each subscription configured in your settings.' 18 | 'When a message we expect is received, a Celery Worker task will be spawned to handle that message.' 19 | 'The message will be marked as deferred, and the Celery task must then complete the message.' 20 | ) 21 | 22 | @staticmethod 23 | async def start_tasks() -> None: 24 | """ 25 | Creates background tasks to subscribe to events 26 | """ 27 | if not settings.subscriptions: # pragma: no cover 28 | logger.info('No subscriptions found. Sleeping forever to avoid crash loops.') 29 | while True: 30 | time.sleep(60 * 10) # Keeps CPU usage to a minimum 31 | 32 | tasks: list[Task] = [ 33 | asyncio.create_task( 34 | subscribe_to_topic( 35 | connection_string=subscription['connection_string'], 36 | topic_name=subscription['topic_name'], 37 | subscription_name=subscription['subscription_name'], 38 | handlers=subscription['handlers'], 39 | ) 40 | ) 41 | for subscription in settings.subscriptions 42 | ] 43 | done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) # Also covers FIRST_EXCEPTION 44 | 45 | # Log why the task ended 46 | for task in done: 47 | try: 48 | raise task.exception() # type: ignore # We do this to use `logger.exception` which enables trace nicely 49 | except TypeError: 50 | logger.critical('Task %s ended early without an exception', task) 51 | except Exception as error: 52 | logger.exception('Exception in subscription task %s. Exception: %s', task, error) 53 | 54 | for task in pending: 55 | # Cancel all remaining running tasks. This kills the service (and container) 56 | logger.info('Cancelling pending task %s', task) 57 | task.cancel() 58 | logger.info('All tasks cancelled') 59 | sys.exit('Exiting process') 60 | 61 | def handle(self, *args: None, **options) -> None: 62 | """ 63 | This function is called when `manage.py metroid` is run from the terminal. 64 | """ 65 | logger.info('Starting Metro subscriptions') 66 | asyncio.run(self.start_tasks()) 67 | -------------------------------------------------------------------------------- /tests/functional/test_republish.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.utils import timezone 4 | 5 | import pytest 6 | from urllib3.exceptions import HTTPError 7 | 8 | from metroid.models import FailedPublishMessage 9 | from metroid.publish import publish_event 10 | from metroid.republish import retry_failed_published_events 11 | 12 | 13 | @pytest.fixture 14 | def mock_republish_error(mocker): 15 | return_mock = mocker.Mock() 16 | return_mock.post.side_effect = HTTPError(mocker.Mock(status_code=500)) 17 | 18 | mocker.patch('metroid.republish.requests', return_mock) 19 | return return_mock 20 | 21 | 22 | @pytest.fixture 23 | def mock_republish_ok(mocker): 24 | return_mock = mocker.Mock() 25 | mocker.patch('metroid.republish.requests', return_mock) 26 | return return_mock 27 | 28 | 29 | @pytest.mark.django_db 30 | def test_failed_message_saved(mock_republish_error): 31 | now = timezone.now().isoformat() 32 | publish_event( 33 | topic_name='test123', 34 | event_type='Intility.MyTopic', 35 | data_version='1.0', 36 | data={'hello': 'world'}, 37 | subject='my/test/subject', 38 | event_time=now, 39 | ) 40 | 41 | assert len(FailedPublishMessage.objects.all()) == 1 42 | 43 | 44 | @pytest.mark.django_db 45 | def test_retry_failed_messages(mock_republish_ok): 46 | now = timezone.now().isoformat() 47 | FailedPublishMessage.objects.create( 48 | event_type='Intility.MyTopic', 49 | event_time=now, 50 | data_version='1.0', 51 | data={'hello': 'world'}, 52 | subject='my/test/subject', 53 | topic_name='test123', 54 | ) 55 | 56 | assert len(FailedPublishMessage.objects.all()) == 1 57 | retry_failed_published_events() 58 | 59 | mock_republish_ok.post.assert_called_with( 60 | url='https://api.intility.no/metro/test123', 61 | headers={'content-type': 'application/json', 'x-metro-key': 'my-metro-key'}, 62 | data=json.dumps( 63 | { 64 | 'eventType': 'Intility.MyTopic', 65 | 'eventTime': now, 66 | 'dataVersion': '1.0', 67 | 'data': {'hello': 'world'}, 68 | 'subject': 'my/test/subject', 69 | } 70 | ), 71 | ) 72 | assert len(FailedPublishMessage.objects.all()) == 0 73 | 74 | 75 | @pytest.mark.django_db 76 | def test_retry_failed_messages_fail(mock_republish_error): 77 | now = timezone.now().isoformat() 78 | FailedPublishMessage.objects.create( 79 | event_type='Intility.MyTopic', 80 | event_time=now, 81 | data_version='1.0', 82 | data={'hello': 'world'}, 83 | subject='my/test/subject', 84 | topic_name='test123', 85 | ) 86 | 87 | assert len(FailedPublishMessage.objects.all()) == 1 88 | retry_failed_published_events() 89 | assert len(FailedPublishMessage.objects.all()) == 1 90 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "metroid" 3 | version = "1.4.0" # Remember to change in __init__.py as well 4 | description = "Metroid - Metro for Django" 5 | authors = ["Jonas Krüger Svensson "] 6 | maintainers = [ 7 | "Ali Arfan ", 8 | "Ingvald Lorentzen ", 10 | ] 11 | readme = "README.md" 12 | homepage = "https://github.com/intility/metroid" 13 | repository = "https://github.com/intility/metroid" 14 | documentation = "https://github.com/intility/metroid" 15 | keywords = [ 16 | 'async', 'django', 'servicebus', 'task', 'celery', 'worker', 'rq', 17 | ] 18 | classifiers = [ 19 | 'Development Status :: 5 - Production/Stable', 20 | 'Environment :: Web Environment', 21 | 'Framework :: Django', 22 | 'Framework :: Django :: 4.2', 23 | 'Framework :: Django :: 5.0', 24 | 'Intended Audience :: Developers', 25 | 'License :: OSI Approved :: BSD License', 26 | 'Operating System :: OS Independent', 27 | 'Programming Language :: Python', 28 | 'Programming Language :: Python :: 3.10', 29 | 'Programming Language :: Python :: 3.11', 30 | 'Programming Language :: Python :: 3.12', 31 | 'Topic :: Software Development', 32 | 'Topic :: Software Development :: Libraries', 33 | 'Topic :: Software Development :: Libraries :: Application Frameworks', 34 | 'Topic :: Software Development :: Libraries :: Python Modules', 35 | ] 36 | 37 | [tool.poetry.dependencies] 38 | python = "^3.10" 39 | aiohttp = "^3.8.4" 40 | azure-servicebus = "^7.10.0" 41 | Django = "^4.0 || ^5.0" 42 | django-guid = "^3.2.0" 43 | 44 | [tool.poetry.dev-dependencies] 45 | pre-commit = "^3.3.2" 46 | python-decouple = "^3.4" 47 | redis = "^4.0.0 || ^5.0.0" 48 | black = "^23.3.0" 49 | pytest = "^7.3.1" 50 | pytest-cov = "^4.0.0" 51 | pytest-django = "^4.1.0" 52 | pytest-asyncio = "^0.21.0" 53 | pytest-mock = "^3.5.1" 54 | requests-mock = "^1.8.0" 55 | pytest-freezegun = "^0.4.2" 56 | celery = "^5.3.0" 57 | django-rq = "^2.4.1" 58 | redislite = "^6.0.674960" 59 | 60 | [tool.black] 61 | line-length = 120 62 | skip-string-normalization = true 63 | target-version = ['py310'] 64 | include = '\.pyi?$' 65 | exclude = ''' 66 | ( 67 | (\.eggs|\.git|\.hg|\.mypy_cache|\.tox|\.venv|\venv|\.github|\docs|\tests|\__pycache__) 68 | ) 69 | ''' 70 | 71 | [tool.isort] 72 | profile = "black" 73 | src_paths = ["metroid"] 74 | combine_as_imports = true 75 | line_length = 120 76 | sections = [ 77 | 'FUTURE', 78 | 'STDLIB', 79 | 'DJANGO', 80 | 'THIRDPARTY', 81 | 'FIRSTPARTY', 82 | 'LOCALFOLDER' 83 | ] 84 | known_django = ['django'] 85 | 86 | [tool.pytest.ini_options] 87 | DJANGO_SETTINGS_MODULE = 'demoproj.settings' 88 | log_cli_format = '%(levelname)s %(asctime)s [%(correlation_id)s] %(name)s %(message)s' 89 | log_cli = true 90 | log_cli_level = 'DEBUG' 91 | 92 | 93 | [build-system] 94 | requires = ["poetry-core>=1.0.0"] 95 | build-backend = "poetry.core.masonry.api" 96 | -------------------------------------------------------------------------------- /metroid/publish.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | from django.utils import timezone 5 | 6 | import requests 7 | 8 | from metroid.config import settings 9 | 10 | logger = logging.getLogger('metroid') 11 | 12 | 13 | def publish_event( 14 | *, 15 | topic_name: str, 16 | event_type: str, 17 | data: list | dict, 18 | subject: str, 19 | data_version: str, 20 | event_time: str | None = None, 21 | ) -> None: 22 | """ 23 | Sync helper function to publish metro events based on a topic name. 24 | Please read the metro docs before publishing event. 25 | 26 | It make a wrapper around this function based on your needs. If you implement posting to Metro at the end 27 | of your API logic you might want to still return a 200 to the API user, even if a post to Metro should fail. 28 | 29 | :param topic_name: str 30 | 'Intility.MyTopic' 31 | :param event_type: str 32 | 'My.Event.Created' 33 | :param data: Union[list, dict] - Any JSON serializable data 34 | {'hello': 'world} 35 | :param subject: str 36 | 'test/subject' 37 | :param data_version: str 38 | '1.0' 39 | :param event_time: Optional[str] - A valid ISO-8601 timestamp (YYYY-MM-DDThh:mm:ssZ/YYYY-MM-DDThh:mm:ss±hh:mm..) 40 | '2021-02-22T12:34:18.216747+00:00' 41 | 42 | :return: None - Metro gives empty response on valid posts 43 | :raises: requests.exceptions.HTTPError 44 | """ 45 | formatted_data = { 46 | 'eventType': event_type, 47 | 'eventTime': event_time or timezone.now().isoformat(), 48 | 'dataVersion': data_version, 49 | 'data': data, 50 | 'subject': subject, 51 | } 52 | logger.info('Posting event to Metro topic %s. Data: %s', topic_name, formatted_data) 53 | 54 | try: 55 | metro_response = requests.post( 56 | url=f'https://api.intility.no/metro/{topic_name}', 57 | headers={ 58 | 'content-type': 'application/json', 59 | 'x-metro-key': settings.get_x_metro_key(topic_name=topic_name), 60 | }, 61 | data=json.dumps(formatted_data), 62 | ) 63 | metro_response.raise_for_status() 64 | logger.info('Posted to metro') 65 | except Exception as error: 66 | from metroid.models import FailedPublishMessage 67 | 68 | logger.info('Failed to post to metro. %s', error) 69 | 70 | try: 71 | FailedPublishMessage.objects.create( 72 | event_type=event_type, 73 | event_time=event_time or timezone.now().isoformat(), 74 | data_version=data_version, 75 | data=data, 76 | subject=subject, 77 | topic_name=topic_name, 78 | ) 79 | logger.info('Saved failed message to database.') 80 | # failsafe just in case 81 | except Exception as error: # pragma: no cover 82 | logger.exception('Unable to save Metro message. Error: %s', error) 83 | return 84 | -------------------------------------------------------------------------------- /tests/functional/test_rq/test_admin_rq.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.test import override_settings 3 | from django.urls import reverse 4 | from django.utils.module_loading import import_string 5 | 6 | import django_rq 7 | import pytest 8 | 9 | from metroid.config import Settings 10 | from metroid.models import FailedMessage 11 | from metroid.rq import on_failure 12 | from rq import SimpleWorker 13 | 14 | 15 | @pytest.fixture 16 | def create_and_sign_in_user(client): 17 | user = User.objects.create_superuser('testuser', 'test@test.com', 'testpw') 18 | client.login(username='testuser', password='testpw') 19 | return 20 | 21 | 22 | @pytest.fixture 23 | def mock_subscriptions_admin(monkeypatch): 24 | with override_settings( 25 | METROID={ 26 | 'subscriptions': [ 27 | { 28 | 'topic_name': 'test', 29 | 'subscription_name': 'sub-test-djangomoduletest', 30 | 'connection_string': 'my long connection string', 31 | 'handlers': [ 32 | {'subject': 'MockedTask', 'handler_function': 'demoproj.tasks.a_random_task'}, 33 | {'subject': 'ErrorTask', 'handler_function': 'demoproj.tasks.error_task'}, 34 | ], 35 | }, 36 | ], 37 | 'worker_type': 'rq', 38 | }, 39 | ): 40 | settings = Settings() 41 | monkeypatch.setattr('metroid.admin.settings', settings) 42 | 43 | 44 | @pytest.mark.django_db 45 | def test_admin_action_handler_found_rq(client, caplog, create_and_sign_in_user, mock_subscriptions_admin): 46 | with override_settings(RQ_QUEUES={'metroid': {}, 'fake': {}}): 47 | # Create a failed task 48 | a_random_task = import_string('demoproj.tasks.a_random_task') 49 | queue = django_rq.get_queue('metroid') 50 | queue.enqueue( 51 | a_random_task, 52 | job_id='5F9914AB-2CC1-4A57-9A67-8586ADC2D8B6', 53 | kwargs={ 54 | 'message': {'id': '5F9914AB-2CC1-4A57-9A67-8586ADC2D8B6'}, 55 | 'topic_name': 'test', 56 | 'subscription_name': 'sub-test-djangomoduletest', 57 | 'subject': 'MockedTask', 58 | }, 59 | ) 60 | worker = SimpleWorker([queue], connection=queue.connection, exception_handlers=[on_failure]) 61 | worker.work(burst=True) 62 | 63 | change_url = reverse('admin:metroid_failedmessage_changelist') 64 | data = {'action': 'retry', '_selected_action': [1]} 65 | response = client.post(change_url, data, follow=True) 66 | 67 | # Run the retry task 68 | worker.work(burst=True) 69 | 70 | assert response.status_code == 200 71 | assert len([x.message for x in caplog.records if x.message == 'Attempting to retry id 1']) == 1 72 | assert FailedMessage.objects.get(id=2) # Prev message should fail 73 | with pytest.raises(FailedMessage.DoesNotExist): 74 | FailedMessage.objects.get(id=1) # message we created above should be deleted 75 | -------------------------------------------------------------------------------- /tests/functional/test_celery/test_subscribe_celery.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.test import override_settings 4 | 5 | import pytest 6 | from azure.servicebus import TransportType 7 | 8 | from metroid.subscribe import subscribe_to_topic 9 | 10 | 11 | @pytest.mark.asyncio 12 | async def test_subscription_celery(caplog, mock_service_bus_client_ok): 13 | with override_settings(CELERY_TASK_ALWAYS_EAGER=True): # Runs celery in same thread without workers and stuff 14 | caplog.set_level(logging.INFO) 15 | await subscribe_to_topic( 16 | **{ 17 | 'topic_name': 'test', 18 | 'subscription_name': 'sub-test-mocktest', 19 | 'connection_string': 'my long connection string', 20 | 'handlers': [ 21 | { 22 | 'subject': 'Test/Django/Module', 23 | 'regex': False, 24 | 'handler_function': 'demoproj.tasks.my_task', 25 | }, 26 | { 27 | 'subject': 'Exception/Django/Module', 28 | 'regex': False, 29 | 'handler_function': 'demoproj.tasks.my_task', 30 | }, 31 | ], 32 | } 33 | ) 34 | log_messages = [x.message for x in caplog.records] 35 | mock_service_bus_client_ok.from_connection_string.assert_called_with( 36 | conn_str='my long connection string', transport_type=TransportType.AmqpOverWebsocket 37 | ) 38 | # This is kinda dumb, as it makes us have to rewrite test if order of logs changes, but it's the easiest way to 39 | # actually confirm logic. 40 | # We expect two tasks to be started with the two subjects, and three messages to be ignored 41 | assert 'Started subscription for topic test and subscription sub-test-mocktest' == log_messages[0] 42 | 43 | assert 'Subject matching: Test/Django/Module' == log_messages[2] 44 | assert 'Task demoproj.tasks.my_task' in log_messages[5] 45 | assert 'Celery task started' in log_messages[7] 46 | 47 | assert 'Subject matching: Exception/Django/Module' == log_messages[10] 48 | assert 'Task demoproj.tasks.my_task' in log_messages[13] 49 | assert 'Celery task started' in log_messages[15] 50 | 51 | assert len([message for message in log_messages if message == 'No handler found, completing message']) == 3 52 | 53 | 54 | @pytest.mark.asyncio 55 | async def test_faulty_metro_data_celery(caplog, mock_service_bus_client_failure): 56 | with override_settings(CELERY_TASK_ALWAYS_EAGER=True): 57 | await subscribe_to_topic( 58 | **{ 59 | 'topic_name': 'error', # This triggers an error in the metro data 60 | 'subscription_name': 'sub-test-mocktest', 61 | 'connection_string': 'my long connection string', 62 | 'handlers': [ 63 | { 64 | 'subject': 'Test/Django/Module', 65 | 'regex': False, 66 | 'handler_function': 'demoproj.tasks.my_task', 67 | }, 68 | { 69 | 'subject': 'Exception/Django/Module', 70 | 'regex': False, 71 | 'handler_function': 'demoproj.tasks.my_task', 72 | }, 73 | ], 74 | } 75 | ) 76 | mock_service_bus_client_failure.from_connection_string.assert_called_with( 77 | conn_str='my long connection string', transport_type=TransportType.AmqpOverWebsocket 78 | ) 79 | log_messages = [x.message for x in caplog.records] 80 | assert len([message for message in log_messages if 'Unable to decode message' in message]) == 1 81 | -------------------------------------------------------------------------------- /tests/functional/test_rq/test_subscribe_rq.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.test import override_settings 4 | 5 | import pytest 6 | from azure.servicebus import TransportType 7 | 8 | from metroid.config import Settings 9 | from metroid.subscribe import subscribe_to_topic 10 | 11 | 12 | @pytest.fixture(autouse=True) 13 | def mock_rq_worker(monkeypatch): 14 | with override_settings(METROID={'worker_type': 'rq'}): 15 | settings = Settings() 16 | monkeypatch.setattr('metroid.subscribe.settings', settings) 17 | 18 | 19 | @pytest.mark.asyncio 20 | async def test_subscription_rq(caplog, mock_service_bus_client_ok): 21 | with override_settings(RQ_QUEUES={'metroid': {'ASYNC': False}, 'fake': {}}): 22 | caplog.set_level(logging.INFO) 23 | await subscribe_to_topic( 24 | **{ 25 | 'topic_name': 'test', 26 | 'subscription_name': 'sub-test-mocktest', 27 | 'connection_string': 'my long connection string', 28 | 'handlers': [ 29 | { 30 | 'subject': 'Test/Django/Module', 31 | 'regex': False, 32 | 'handler_function': 'demoproj.tasks.my_task', 33 | }, 34 | { 35 | 'subject': 'Exception/Django/Module', 36 | 'regex': False, 37 | 'handler_function': 'demoproj.tasks.my_task', 38 | }, 39 | ], 40 | } 41 | ) 42 | log_messages = [x.message for x in caplog.records] 43 | mock_service_bus_client_ok.from_connection_string.assert_called_with( 44 | conn_str='my long connection string', transport_type=TransportType.AmqpOverWebsocket 45 | ) 46 | # This is kinda dumb, as it makes us have to rewrite test if order of logs changes, but it's the easiest way to 47 | # actually confirm logic. 48 | # We expect two tasks to be started with the two subjects, and three messages to be ignored 49 | assert 'Started subscription for topic test and subscription sub-test-mocktest' == log_messages[0] 50 | 51 | assert 'Subject matching: Test/Django/Module' == log_messages[2] 52 | assert 'RQ task started' in log_messages[3] 53 | 54 | assert 'Subject matching: Exception/Django/Module' == log_messages[6] 55 | assert 'RQ task started' in log_messages[7] 56 | assert len([message for message in log_messages if message == 'No handler found, completing message']) == 3 57 | 58 | 59 | @pytest.mark.asyncio 60 | async def test_faulty_metro_data_rq(caplog, mock_service_bus_client_failure): 61 | with override_settings(RQ_QUEUES={'metroid': {'ASYNC': False}, 'fake': {}}): 62 | await subscribe_to_topic( 63 | **{ 64 | 'topic_name': 'error', # This triggers an error in the metro data 65 | 'subscription_name': 'sub-test-mocktest', 66 | 'connection_string': 'my long connection string', 67 | 'handlers': [ 68 | { 69 | 'subject': 'Test/Django/Module', 70 | 'regex': False, 71 | 'handler_function': 'demoproj.tasks.my_task', 72 | }, 73 | { 74 | 'subject': 'Exception/Django/Module', 75 | 'regex': False, 76 | 'handler_function': 'demoproj.tasks.my_task', 77 | }, 78 | ], 79 | } 80 | ) 81 | mock_service_bus_client_failure.from_connection_string.assert_called_with( 82 | conn_str='my long connection string', transport_type=TransportType.AmqpOverWebsocket 83 | ) 84 | log_messages = [x.message for x in caplog.records] 85 | assert len([message for message in log_messages if 'Unable to decode message' in message]) == 1 86 | -------------------------------------------------------------------------------- /tests/functional/mock_service_bus.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import json 3 | from unittest.mock import AsyncMock, MagicMock 4 | 5 | 6 | class Message: 7 | def __init__(self, i, **kwargs): 8 | self.error = kwargs.get('error', False) 9 | self.i = i 10 | if self.i == 0: 11 | self.message = { 12 | 'eventType': 'Intility.Jonas.Testing', 13 | 'eventTime': '2021-02-02T12:50:39.611290+00:00', 14 | 'dataVersion': '1.0', 15 | 'data': f'Mocked - Yo, Metro is awesome', 16 | 'subject': 'Test/Django/Module', 17 | } 18 | elif self.i == 1: 19 | self.message = { 20 | 'eventType': 'Intility.Jonas.Testing', 21 | 'eventTime': '2021-02-02T12:50:39.611290+00:00', 22 | 'dataVersion': '1.0', 23 | 'data': {'content': 'Mocked - Yo, Metro is awesome'}, 24 | 'subject': 'Exception/Django/Module', 25 | } 26 | else: 27 | self.message = { 28 | 'eventType': 'Intility.Jonas.Testing', 29 | 'eventTime': '2021-02-02T12:50:39.611290+00:00', 30 | 'dataVersion': '1.0', 31 | 'data': {'content': 'Mocked - Yo, Metro is awesome'}, 32 | 'subject': 'Ignore me', 33 | } 34 | 35 | @property 36 | def sequence_number(self): 37 | return self.i 38 | 39 | def __str__(self): 40 | if not self.error: 41 | return json.dumps(self.message) 42 | else: 43 | return str(self.message) # Not something we can json.loads() 44 | 45 | 46 | class ReceiverMock: 47 | def __init__(self, *args, **kwargs): 48 | self.error = kwargs.get('error', False) 49 | self.i = 0 50 | 51 | async def __aenter__(self): 52 | return self 53 | 54 | async def __aexit__(self, *args): 55 | return None 56 | 57 | async def __anext__(self): 58 | """ 59 | Returns something 5 times with a 1 sec sleep in between each one 60 | """ 61 | i = self.i 62 | if i >= 5: 63 | raise StopAsyncIteration 64 | self.i += 1 65 | await asyncio.sleep(0.2) 66 | if i == 4 and self.error: 67 | return Message(4, error=True) 68 | return Message(i) 69 | 70 | def __aiter__(self): 71 | return self 72 | 73 | async def complete_message(self, message): 74 | return True 75 | 76 | 77 | class ServiceBusMock: 78 | def __init__(self, *args, **kwargs): 79 | pass 80 | 81 | async def __aenter__(self): 82 | return self 83 | 84 | async def __aexit__(self, *args): 85 | return None 86 | 87 | @classmethod 88 | def from_connection_string(cls, conn_str, transport_type): 89 | return cls(conn_str, transport_type) 90 | 91 | def get_subscription_receiver(self, topic_name, subscription_name): 92 | if topic_name == 'error': 93 | return AsyncMock(ReceiverMock(error=True)) 94 | return AsyncMock(ReceiverMock()) 95 | 96 | 97 | # Create MagicMock with specs of our instances 98 | # If this is confusing, it's because it is - and it's not really documented. 99 | # https://bugs.python.org/issue40406 100 | def instance_service_mock_ok() -> ServiceBusMock: 101 | service_mock = MagicMock() 102 | service_mock.configure_mock( 103 | **{ 104 | 'from_connection_string.return_value.__aenter__.return_value': MagicMock( 105 | ServiceBusMock(), **{'get_subscription_receiver.return_value.__aenter__.return_value': ReceiverMock()} 106 | ) 107 | } 108 | ) 109 | return service_mock 110 | 111 | 112 | def instance_service_mock_error() -> ServiceBusMock: 113 | service_mock_error = MagicMock() 114 | service_mock_error.configure_mock( 115 | **{ 116 | 'from_connection_string.return_value.__aenter__.return_value': MagicMock( 117 | ServiceBusMock(), 118 | **{'get_subscription_receiver.return_value.__aenter__.return_value': ReceiverMock(error=True)}, 119 | ) 120 | } 121 | ) 122 | return service_mock_error 123 | -------------------------------------------------------------------------------- /metroid/subscribe.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | from django.utils.module_loading import import_string 5 | 6 | from asgiref.sync import sync_to_async 7 | from azure.servicebus import ServiceBusReceivedMessage, TransportType 8 | from azure.servicebus.aio import ServiceBusClient, ServiceBusReceiver 9 | 10 | from metroid.config import settings 11 | from metroid.typing import Handler 12 | from metroid.utils import match_handler_subject 13 | 14 | logger = logging.getLogger('metroid') 15 | 16 | 17 | async def subscribe_to_topic( 18 | connection_string: str, 19 | topic_name: str, 20 | subscription_name: str, 21 | handlers: list[Handler], 22 | ) -> None: 23 | """ 24 | Subscribe to a topic, with a connection string 25 | """ 26 | # Create a connection to Metro 27 | metro_client: ServiceBusClient 28 | async with ServiceBusClient.from_connection_string( 29 | conn_str=connection_string, transport_type=TransportType.AmqpOverWebsocket 30 | ) as metro_client: 31 | # Subscribe to a topic with through our subscription name 32 | receiver: ServiceBusReceiver 33 | async with metro_client.get_subscription_receiver( 34 | topic_name=topic_name, 35 | subscription_name=subscription_name, 36 | ) as receiver: 37 | logger.info('Started subscription for topic %s and subscription %s', topic_name, subscription_name) 38 | # We now have a receiver, we can use this to talk with Metro 39 | message: ServiceBusReceivedMessage 40 | async for message in receiver: 41 | sequence_number: int = message.sequence_number 42 | loaded_message: dict = {} 43 | try: 44 | loaded_message = json.loads(str(message)) 45 | except Exception as error: 46 | # We defer messages with a faulty body, we do not crash. 47 | logger.exception( 48 | 'Unable to decode message %s. Sequence number %s. Error: %s', 49 | message, 50 | sequence_number, 51 | error, 52 | ) 53 | # Check how to handle this message 54 | logger.info( 55 | '%s: Received message, sequence number %s. Content: %s', 56 | subscription_name, 57 | sequence_number, 58 | loaded_message, 59 | ) 60 | handled_message = False 61 | for handler in handlers: 62 | subject = handler['subject'] 63 | subject_is_regex = handler['regex'] 64 | message_subject = loaded_message.get('subject', '') 65 | 66 | if match_handler_subject( 67 | subject=subject, message_subject=message_subject, is_regex=subject_is_regex 68 | ): 69 | logger.info('Subject matching: %s', handler.get('subject')) 70 | handler_function = import_string(handler.get('handler_function')) 71 | 72 | if settings.worker_type == 'celery': 73 | await sync_to_async(handler_function.apply_async)( # type: ignore 74 | kwargs={ 75 | 'message': loaded_message, 76 | 'topic_name': topic_name, 77 | 'subscription_name': subscription_name, 78 | 'subject': subject, 79 | } 80 | ) 81 | logger.info('Celery task started') 82 | 83 | elif settings.worker_type == 'rq': 84 | import django_rq 85 | 86 | queue = django_rq.get_queue('metroid') 87 | await sync_to_async(queue.enqueue)( 88 | handler_function, 89 | job_id=loaded_message.get('id'), 90 | kwargs={ 91 | 'message': loaded_message, 92 | 'topic_name': topic_name, 93 | 'subscription_name': subscription_name, 94 | 'subject': subject, 95 | }, 96 | ) 97 | logger.info('RQ task started') 98 | 99 | await receiver.complete_message(message=message) 100 | handled_message = True 101 | logger.info('Message with sequence number %s completed', sequence_number) 102 | if not handled_message: 103 | logger.info('No handler found, completing message') 104 | await receiver.complete_message(message=message) 105 | -------------------------------------------------------------------------------- /metroid/admin.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.contrib import admin, messages 4 | from django.db.models import QuerySet 5 | from django.http import HttpRequest 6 | 7 | import requests 8 | 9 | from metroid.config import settings 10 | from metroid.models import FailedMessage, FailedPublishMessage 11 | 12 | logger = logging.getLogger('metroid') 13 | 14 | 15 | @admin.register(FailedMessage) 16 | class FailedMessageAdmin(admin.ModelAdmin): 17 | readonly_fields = ( 18 | 'created', 19 | 'correlation_id', 20 | 'topic_name', 21 | 'subject', 22 | 'subscription_name', 23 | 'message', 24 | 'exception_str', 25 | 'traceback', 26 | ) 27 | list_display = ['id', 'correlation_id', 'topic_name', 'subject', 'created'] 28 | 29 | actions = ['retry'] 30 | 31 | def retry(self, request: HttpRequest, queryset: QuerySet) -> None: 32 | """ 33 | Retry a failed message. 34 | """ 35 | for message in queryset: 36 | handler = settings.get_handler_function( 37 | topic_name=message.topic_name, subscription_name=message.subscription_name, subject=message.subject 38 | ) 39 | if handler: 40 | try: 41 | logger.info('Attempting to retry id %s', message.id) 42 | 43 | if settings.worker_type == 'celery': 44 | handler.apply_async( # type: ignore 45 | kwargs={ 46 | 'message': message.message, 47 | 'topic_name': message.topic_name, 48 | 'subscription_name': message.subscription_name, 49 | 'subject': message.subject, 50 | } 51 | ) 52 | elif settings.worker_type == 'rq': 53 | import django_rq 54 | 55 | failed_job_registry = django_rq.get_queue('metroid').failed_job_registry 56 | failed_job_registry.requeue(message.message.get('id')) 57 | 58 | logger.info('Deleting %s from database', message.id) 59 | message.delete() 60 | logger.info('Returning') 61 | self.message_user( 62 | request=request, 63 | message='Task has been retried.', 64 | level=messages.SUCCESS, 65 | ) 66 | except Exception as error: 67 | logger.exception('Unable to retry Metro message. Error: %s', error) 68 | self.message_user( 69 | request=request, 70 | message=f'Unable to retry Metro message. Error: {error}', 71 | level=messages.ERROR, 72 | ) 73 | else: 74 | logger.warning('No handler found for %s', message.id) 75 | self.message_user( 76 | request=request, 77 | message=f'No handler function found for id {message.id}.', 78 | level=messages.WARNING, 79 | ) 80 | return 81 | 82 | 83 | @admin.register(FailedPublishMessage) 84 | class FailedPublishMessageAdmin(admin.ModelAdmin): 85 | readonly_fields = ( 86 | 'correlation_id', 87 | 'data', 88 | 'data_version', 89 | 'event_time', 90 | 'event_type', 91 | 'topic_name', 92 | 'subject', 93 | ) 94 | list_display = ['id', 'correlation_id', 'topic_name', 'subject', 'event_time'] 95 | 96 | actions = ['retry_publish'] 97 | 98 | def retry_publish(self, request: HttpRequest, queryset: QuerySet) -> None: 99 | """ 100 | Retry messages that failed to publish. 101 | """ 102 | for message in queryset: 103 | try: 104 | metro_response = requests.post( 105 | url=f'https://api.intility.no/metro/{message.topic_name}', 106 | headers={ 107 | 'content-type': 'application/json', 108 | 'x-metro-key': settings.get_x_metro_key(topic_name=message.topic_name), 109 | }, 110 | data=message.data, 111 | ) 112 | metro_response.raise_for_status() 113 | logger.info('Deleting %s from database', message.id) 114 | message.delete() 115 | logger.info('Posted to Metro') 116 | self.message_user( 117 | request=request, 118 | message='Message has been republished', 119 | level=messages.SUCCESS, 120 | ) 121 | except Exception as error: 122 | logger.exception('Unable to republish Metro message. Error: %s', error) 123 | self.message_user( 124 | request=request, 125 | message=f'Unable to republish Metro message. Error: {error}', 126 | level=messages.ERROR, 127 | ) 128 | return 129 | -------------------------------------------------------------------------------- /demoproj/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for demoproj project. 3 | 4 | Generated by 'django-admin startproject' using Django 3.1.5. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.1/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/3.1/ref/settings/ 11 | """ 12 | 13 | from pathlib import Path 14 | 15 | # Build paths inside the project like this: BASE_DIR / 'subdir'. 16 | BASE_DIR = Path(__file__).resolve().parent.parent 17 | 18 | # Quick-start development settings - unsuitable for production 19 | # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/ 20 | 21 | # SECURITY WARNING: keep the secret key used in production secret! 22 | SECRET_KEY = '*4sjc%a@ht^yx3x$&e4gyr1+-iv=@()cm$@_s%jxmw(&_x*u@e' 23 | 24 | # SECURITY WARNING: don't run with debug turned on in production! 25 | DEBUG = True 26 | 27 | ALLOWED_HOSTS: list = [] 28 | 29 | # Application definition 30 | 31 | INSTALLED_APPS = [ 32 | 'django.contrib.admin', 33 | 'django.contrib.auth', 34 | 'django.contrib.contenttypes', 35 | 'django.contrib.sessions', 36 | 'django.contrib.messages', 37 | 'django.contrib.staticfiles', 38 | 'metroid', 39 | 'django_guid', 40 | ] 41 | 42 | MIDDLEWARE = [ 43 | 'django_guid.middleware.guid_middleware', 44 | 'django.middleware.security.SecurityMiddleware', 45 | 'django.contrib.sessions.middleware.SessionMiddleware', 46 | 'django.middleware.common.CommonMiddleware', 47 | 'django.middleware.csrf.CsrfViewMiddleware', 48 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 49 | 'django.contrib.messages.middleware.MessageMiddleware', 50 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 51 | ] 52 | 53 | ROOT_URLCONF = 'demoproj.urls' 54 | 55 | TEMPLATES = [ 56 | { 57 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 58 | 'DIRS': [], 59 | 'APP_DIRS': True, 60 | 'OPTIONS': { 61 | 'context_processors': [ 62 | 'django.template.context_processors.debug', 63 | 'django.template.context_processors.request', 64 | 'django.contrib.auth.context_processors.auth', 65 | 'django.contrib.messages.context_processors.messages', 66 | ], 67 | }, 68 | }, 69 | ] 70 | 71 | WSGI_APPLICATION = 'demoproj.wsgi.application' 72 | 73 | # Database 74 | # https://docs.djangoproject.com/en/3.1/ref/settings/#databases 75 | 76 | DATABASES = { 77 | 'default': { 78 | 'ENGINE': 'django.db.backends.sqlite3', 79 | 'NAME': BASE_DIR / 'db.sqlite3', 80 | } 81 | } 82 | 83 | # Password validation 84 | # https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators 85 | 86 | AUTH_PASSWORD_VALIDATORS = [ 87 | { 88 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 89 | }, 90 | { 91 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 92 | }, 93 | { 94 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 95 | }, 96 | { 97 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 98 | }, 99 | ] 100 | 101 | # Internationalization 102 | # https://docs.djangoproject.com/en/3.1/topics/i18n/ 103 | 104 | LANGUAGE_CODE = 'en-us' 105 | 106 | TIME_ZONE = 'UTC' 107 | 108 | USE_I18N = True 109 | 110 | USE_L10N = True 111 | 112 | USE_TZ = True 113 | 114 | # Static files (CSS, JavaScript, Images) 115 | # https://docs.djangoproject.com/en/3.1/howto/static-files/ 116 | 117 | STATIC_URL = '/static/' 118 | 119 | from django_guid.integrations import CeleryIntegration # noqa 120 | 121 | DJANGO_GUID = { 122 | 'INTEGRATIONS': [ 123 | CeleryIntegration( 124 | use_django_logging=True, 125 | log_parent=True, 126 | ) 127 | ], 128 | } 129 | 130 | 131 | METROID = { 132 | 'subscriptions': [ 133 | { 134 | 'topic_name': 'test', 135 | 'subscription_name': 'sub-test-djangomoduletest', 136 | 'connection_string': 'Endpoint=sb://...', 137 | 'handlers': [ 138 | { 139 | 'subject': 'Test/Django/Module', 140 | 'regex': False, 141 | 'handler_function': 'demoproj.demoapp.services.my_func', 142 | }, 143 | { 144 | 'subject': r'^Exception\/.*$', 145 | 'regex': True, 146 | 'handler_function': 'demoproj.demoapp.services.my_broken_task', 147 | }, 148 | ], 149 | }, 150 | ] 151 | } 152 | 153 | LOGGING = { 154 | 'version': 1, 155 | 'disable_existing_loggers': False, 156 | 'formatters': { 157 | # Basic log format without django-guid filters 158 | 'basic_format': {'format': '%(levelname)s %(asctime)s [%(correlation_id)s] %(name)s - %(message)s'}, 159 | }, 160 | 'filters': {'correlation_id': {'()': 'django_guid.log_filters.CorrelationId'}}, 161 | 'handlers': { 162 | 'default': { 163 | 'class': 'logging.StreamHandler', 164 | 'formatter': 'basic_format', 165 | 'filters': ['correlation_id'], 166 | }, 167 | }, 168 | 'loggers': { 169 | 'demoproj': {'handlers': ['default'], 'level': 'DEBUG'}, 170 | 'metroid': { 171 | 'handlers': ['default'], 172 | 'level': 'DEBUG', 173 | 'propagate': True, 174 | }, 175 | }, 176 | } 177 | 178 | CELERY_BROKER_URL = 'redis://:@127.0.0.1:6378' 179 | CELERY_RESULT_BACKEND = CELERY_BROKER_URL 180 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 |
3 | Metroid 4 |

5 | 6 |

7 | Subscribe, act, publish. 8 |

9 |

10 | 11 | Python version 12 | 13 | 14 | Django version 15 | 16 | 17 | Celery version 18 | 19 | 20 | ServiceBus version 21 | 22 | 23 | Django GUID version 24 | 25 |

26 |

27 | 28 | Codecov 29 | 30 | 31 | Pre-commit 32 | 33 | 34 | Black 35 | 36 | 37 | mypy 38 | 39 | 40 | isort 41 | 42 |

43 | 44 | 45 | # Metroid - Metro for Django 46 | 47 | This app is intended to streamline integration with Metro for all Django+Celery users by: 48 | 49 | * Asynchronous handling of subscriptions and messages with one command 50 | * Execute Celery tasks based on message topics, defined in `settings.py` 51 | * Retry failed tasks through your admin dashboard when using the `MetroidTask` base 52 | 53 | ## Overview 54 | * `python` >= 3.10 55 | * `django` >= 4.2 - For `asgiref`, settings 56 | * `django-guid` >= 3.2.0 - Storing correlation IDs for failed tasks in the database, making debugging easy 57 | * Choose one: 58 | * `celery` >= 5.3.0 - Execute tasks based on a subject 59 | * `django-rq` >= 2.4.1 - Execute tasks based on a subject 60 | 61 | ### Implementation 62 | 63 | The `python manage.py metroid` app is fully asynchronous, and has no blocking code. It utilizes `Celery` to execute tasks. 64 | 65 | It works by: 66 | 1. Going through all your configured subscriptions and start a new async connection for each one of them 67 | 2. Metro sends messages on the subscriptions 68 | 3. This app filters out messages matching subjects you have defined, and queues a celery task to execute 69 | the function as specified for that subject 70 | 3.1. If no task is found for that subject, the message is marked as complete 71 | 4. The message is marked as complete after the Celery task has successfully been queued 72 | 5. If the task is failed, an entry is automatically created in your database 73 | 6. All failed tasks can be retried manually through the admin dashboard 74 | 75 | 76 | ### Configure and install this package 77 | 78 | 79 | > **_Note_** 80 | > For a complete example, have a look in `demoproj/settings.py`. 81 | 82 | 1. Create a `METROID` key in `settings.py` with all your subscriptions and handlers. 83 | Example settings: 84 | ```python 85 | METROID = { 86 | 'subscriptions': [ 87 | { 88 | 'topic_name': 'metro-demo', 89 | 'subscription_name': 'sub-metrodemo-metrodemoerfett', 90 | 'connection_string': config('CONNECTION_STRING_METRO_DEMO', None), 91 | 'handlers': [ 92 | { 93 | 'subject': 'MetroDemo/Type/GeekJokes', 94 | 'regex': False, 95 | 'handler_function': 'demoproj.demoapp.services.my_func' 96 | } 97 | ], 98 | }, 99 | ], 100 | 'worker_type': 'celery', # default 101 | } 102 | ``` 103 | 104 | The `handler_function` is defined by providing the full dotted path as a string. For example,`from demoproj.demoapp.services import my_func` is provided as `'demoproj.demoapp.services.my_func'`. 105 | 106 | The handlers subject can be a regular expression or a string. If a regular expression is provided, the variable regex must be set to True. Example: 107 | ```python 108 | 'handlers': [{'subject': r'^MetroDemo/Type/.*$','regex':True,'handler_function': my_func}], 109 | ``` 110 | 111 | 112 | 113 | 2. Configure `Django-GUID` by adding the app to your installed apps, to your middlewares and configuring logging 114 | as described [here](https://github.com/snok/django-guid#configuration). 115 | Make sure you enable the [`CeleryIntegration`](https://django-guid.readthedocs.io/en/latest/integrations.html#celery): 116 | ```python 117 | from django_guid.integrations import CeleryIntegration 118 | 119 | DJANGO_GUID = { 120 | 'INTEGRATIONS': [ 121 | CeleryIntegration( 122 | use_django_logging=True, 123 | log_parent=True, 124 | ) 125 | ], 126 | } 127 | ``` 128 | 129 | 130 | #### Creating your own handler functions 131 | 132 | Your functions will be called with keyword arguments for 133 | 134 | 135 | `message`, `topic_name`, `subscription_name` and `subject`. You function should in other words 136 | look something like this: 137 | 138 | ##### Celery 139 | ```python 140 | @app.task(base=MetroidTask) 141 | def my_func(*, message: dict, topic_name: str, subscription_name: str, subject: str) -> None: 142 | ``` 143 | 144 | ##### rq 145 | ```python 146 | def my_func(*, message: dict, topic_name: str, subscription_name: str, subject: str) -> None: 147 | ``` 148 | 149 | 150 | ### Running the project 151 | 1. Ensure you have redis running: 152 | ```bash 153 | docker-compose up 154 | ``` 155 | 2. Run migrations 156 | ```bash 157 | python manage.py migrate 158 | ``` 159 | 3. Create an admin account 160 | ```bash 161 | python manage.py createsuperuser 162 | ``` 163 | 4. Start a worker: 164 | ```python 165 | celery -A demoproj worker -l info 166 | ``` 167 | 5. Run the subscriber: 168 | ```python 169 | python manage.py metroid 170 | ``` 171 | 6. Send messages to Metro. Example code can be found in [`demoproj/demoapp/services.py`](demoproj/demoapp/services.py) 172 | 7. Run the webserver: 173 | ```python 174 | python manage.py runserver 8000 175 | ``` 176 | 8. See failed messages under `http://localhost:8080/admin` 177 | 178 | To contribute, please see [`CONTRIBUTING.md`](CONTRIBUTING.md) 179 | -------------------------------------------------------------------------------- /tests/functional/test_celery/test_admin_celery.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.test import override_settings 3 | from django.urls import reverse 4 | from django.utils import timezone 5 | 6 | import pytest 7 | from urllib3.exceptions import HTTPError 8 | 9 | from metroid.config import Settings 10 | from metroid.models import FailedMessage, FailedPublishMessage 11 | 12 | 13 | @pytest.fixture 14 | def create_and_sign_in_user(client): 15 | user = User.objects.create_superuser('testuser', 'test@test.com', 'testpw') 16 | client.login(username='testuser', password='testpw') 17 | return 18 | 19 | 20 | @pytest.fixture 21 | def mock_subscriptions_admin(monkeypatch): 22 | with override_settings( 23 | METROID={ 24 | 'subscriptions': [ 25 | { 26 | 'topic_name': 'test', 27 | 'subscription_name': 'sub-test-djangomoduletest', 28 | 'connection_string': 'my long connection string', 29 | 'handlers': [ 30 | {'subject': 'MockedTask', 'handler_function': 'demoproj.tasks.a_random_task'}, 31 | {'subject': 'ErrorTask', 'handler_function': 'demoproj.tasks.error_task'}, 32 | ], 33 | }, 34 | ], 35 | } 36 | ): 37 | settings = Settings() 38 | monkeypatch.setattr('metroid.admin.settings', settings) 39 | 40 | 41 | @pytest.fixture 42 | def mock_republish_error(mocker): 43 | return_mock = mocker.Mock() 44 | return_mock.post.side_effect = HTTPError(mocker.Mock(status_code=500)) 45 | 46 | mocker.patch('metroid.admin.requests', return_mock) 47 | return return_mock 48 | 49 | 50 | @pytest.mark.django_db 51 | def test_admin_action_no_handler_celery(client, caplog, create_and_sign_in_user): 52 | content = FailedMessage.objects.create( 53 | topic_name='test_topic', 54 | subscription_name='test_sub', 55 | subject='test subj', 56 | message='my message', 57 | exception_str='exc', 58 | traceback='long trace', 59 | correlation_id='', 60 | ) 61 | change_url = reverse('admin:metroid_failedmessage_changelist') 62 | data = {'action': 'retry', '_selected_action': [content.id]} 63 | response = client.post(change_url, data, follow=True) 64 | assert response.status_code == 200 65 | assert len([x.message for x in caplog.records if x.message == 'No handler found for 1']) == 1 66 | 67 | 68 | @pytest.mark.django_db 69 | def test_admin_action_handler_found_celery(client, caplog, create_and_sign_in_user, mock_subscriptions_admin): 70 | with override_settings(CELERY_TASK_ALWAYS_EAGER=True): 71 | content = FailedMessage.objects.create( 72 | topic_name='test', 73 | subscription_name='sub-test-djangomoduletest', 74 | subject='MockedTask', 75 | message='my message', 76 | exception_str='exc', 77 | traceback='long trace', 78 | correlation_id='', 79 | ) 80 | change_url = reverse('admin:metroid_failedmessage_changelist') 81 | data = {'action': 'retry', '_selected_action': [content.id]} 82 | response = client.post(change_url, data, follow=True) 83 | assert response.status_code == 200 84 | assert len([x.message for x in caplog.records if x.message == 'Attempting to retry id 1']) == 1 85 | assert FailedMessage.objects.get(id=2) # Prev message should fail 86 | with pytest.raises(FailedMessage.DoesNotExist): 87 | FailedMessage.objects.get(id=1) # message we created above should be deleted 88 | 89 | 90 | @pytest.mark.django_db 91 | def test_admin_action_failed_retry_celery(client, caplog, create_and_sign_in_user, mock_subscriptions_admin): 92 | with override_settings(CELERY_TASK_ALWAYS_EAGER=True): 93 | content = FailedMessage.objects.create( 94 | topic_name='test', 95 | subscription_name='sub-test-djangomoduletest', 96 | subject='ErrorTask', 97 | message='my message', 98 | exception_str='exc', 99 | traceback='long trace', 100 | correlation_id='', 101 | ) 102 | change_url = reverse('admin:metroid_failedmessage_changelist') 103 | data = {'action': 'retry', '_selected_action': [content.id]} 104 | response = client.post(change_url, data, follow=True) 105 | assert response.status_code == 200 106 | assert len([x.message for x in caplog.records if x.message == 'Attempting to retry id 1']) == 1 107 | assert ( 108 | len( 109 | [ 110 | x.message 111 | for x in caplog.records 112 | if x.message 113 | == "Unable to retry Metro message. Error: 'function' object has no attribute 'apply_async'" 114 | ] 115 | ) 116 | == 1 117 | ) 118 | 119 | 120 | @pytest.mark.django_db 121 | def test_admin_action_retry_publish(client, caplog, create_and_sign_in_user, requests_mock): 122 | now = timezone.now().isoformat() 123 | with override_settings(CELERY_TASK_ALWAYS_EAGER=True): 124 | content = FailedPublishMessage.objects.create( 125 | event_type='Intility.MyTopic', 126 | event_time=now, 127 | data_version='1.0', 128 | data={'hello': 'world'}, 129 | subject='my/test/subject', 130 | topic_name='test123', 131 | ) 132 | 133 | assert FailedPublishMessage.objects.get(id=1) 134 | requests_mock.post('https://api.intility.no/metro/test123', status_code=200) 135 | change_url = reverse('admin:metroid_failedpublishmessage_changelist') 136 | data = {'action': 'retry_publish', '_selected_action': [content.id]} 137 | response = client.post(change_url, data, follow=True) 138 | 139 | assert response.status_code == 200 140 | with pytest.raises(FailedPublishMessage.DoesNotExist): 141 | FailedPublishMessage.objects.get(id=1) # message we created above should be deleted 142 | 143 | 144 | @pytest.mark.django_db 145 | def test_admin_action_retry_publish_error(client, caplog, create_and_sign_in_user, requests_mock): 146 | now = timezone.now().isoformat() 147 | with override_settings(CELERY_TASK_ALWAYS_EAGER=True): 148 | content = FailedPublishMessage.objects.create( 149 | event_type='Intility.MyTopic', 150 | event_time=now, 151 | data_version='1.0', 152 | data={'hello': 'world'}, 153 | subject='my/test/subject', 154 | topic_name='test123', 155 | ) 156 | 157 | assert FailedPublishMessage.objects.get(id=1) 158 | requests_mock.post('https://api.intility.no/metro/test123', status_code=500) 159 | change_url = reverse('admin:metroid_failedpublishmessage_changelist') 160 | data = {'action': 'retry_publish', '_selected_action': [content.id]} 161 | response = client.post(change_url, data, follow=True) 162 | assert response.status_code == 200 163 | assert FailedPublishMessage.objects.get(id=1) # message we created above should not be deleted 164 | -------------------------------------------------------------------------------- /metroid/config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from collections.abc import Callable 3 | 4 | from django.conf import settings as django_settings 5 | from django.core.exceptions import ImproperlyConfigured 6 | from django.utils.module_loading import import_string 7 | 8 | from metroid.typing import MetroidSettings, Subscription, TopicPublishSettings 9 | 10 | logger = logging.getLogger('metroid') 11 | 12 | 13 | class Settings: 14 | """ 15 | Subscriptions should be in this format: 16 | 17 | METROID = { 18 | 'subscriptions': [ 19 | { 20 | 'topic_name': 'test', 21 | 'subscription_name': 'sub-test-helloworld', 22 | 'connection_string': 'Endpoint=sb://...', 23 | 'handlers': [ 24 | { 25 | 'subject': '^MetroDemo/Type/.*', 26 | 'regex': True, 27 | 'handler_function': function_to_call, 28 | }, 29 | { 30 | 'subject': 'MetroDemo/Type/DadJokes.Created', 31 | 'regex': False, 32 | 'handler_function': another_func_to_call 33 | } 34 | ], 35 | }, 36 | ], 37 | 'publish_settings': [ 38 | { 39 | 'topic_name': 'test', 40 | 'x_metro_key': 'my-metro-key', 41 | }, 42 | { 43 | 'topic_name': 'another_topic', 44 | 'x_metro_key': 'my-other-metro-key', 45 | }, 46 | ], 47 | 'worker_type': 'celery' # default 48 | } 49 | """ 50 | 51 | def __init__(self) -> None: 52 | if hasattr(django_settings, 'METROID'): 53 | self.settings: MetroidSettings = django_settings.METROID 54 | else: 55 | raise ImproperlyConfigured('`METROID` settings must be defined in settings.py') 56 | 57 | @property 58 | def subscriptions(self) -> list[Subscription]: 59 | """ 60 | Returns all subscriptions 61 | """ 62 | return self.settings.get('subscriptions', []) 63 | 64 | @property 65 | def publish_settings(self) -> list[TopicPublishSettings]: 66 | """ 67 | Returns all publish to metro settings 68 | """ 69 | return self.settings.get('publish_settings', []) 70 | 71 | @property 72 | def worker_type(self) -> str: 73 | """ 74 | Returns the worker type. 75 | """ 76 | return self.settings.get('worker_type', 'celery') 77 | 78 | def get_x_metro_key(self, *, topic_name: str) -> str: 79 | """ 80 | Fetches the x-metro-key based on topic 81 | """ 82 | for topic in self.publish_settings: 83 | if topic['topic_name'] == topic_name: 84 | return topic['x_metro_key'] 85 | logger.critical('Unable to find a x-metro-key for %s', topic_name) 86 | raise ImproperlyConfigured(f'No x-metro-key found for {topic_name}') 87 | 88 | def get_handler_function(self, *, topic_name: str, subscription_name: str, subject: str) -> Callable | None: 89 | """ 90 | Intended to be used by retry-log. 91 | It finds the handler function based on information we have stored in the database. 92 | """ 93 | for subscription in self.subscriptions: 94 | if ( 95 | subscription.get('topic_name') == topic_name 96 | and subscription.get('subscription_name') == subscription_name 97 | ): 98 | for handler in subscription['handlers']: 99 | if handler.get('subject') == subject: 100 | return import_string(handler.get('handler_function')) 101 | return None 102 | 103 | def _validate_import_strings(self) -> None: 104 | """ 105 | Validates the handler_function string for handlers specified in the settings. 106 | """ 107 | for subscription in self.subscriptions: 108 | topic_name = subscription.get('topic_name') 109 | handlers = subscription.get('handlers', []) 110 | for handler in handlers: 111 | handler_function = handler['handler_function'] 112 | if not isinstance(handler_function, str): 113 | raise ImproperlyConfigured(f'Handler function:{handler_function} for {topic_name} must be a string') 114 | 115 | try: 116 | import_string(handler_function) 117 | except ModuleNotFoundError: 118 | raise ImproperlyConfigured( 119 | f'Handler function:{handler_function} for {topic_name} cannot find module' 120 | ) 121 | except ImportError as error: 122 | if f"{handler_function} doesn't look like a module path" in str(error): 123 | raise ImproperlyConfigured( 124 | f'Handler function:{handler_function} for {topic_name} is not a dotted function path' 125 | ) 126 | else: 127 | raise ImproperlyConfigured( 128 | f'Handler function:{handler_function} ' 129 | f'for {topic_name} cannot be imported. Verify that the dotted path points to a function' 130 | ) 131 | 132 | def validate(self) -> None: 133 | """ 134 | Validates all settings 135 | """ 136 | if self.worker_type == 'celery': 137 | try: 138 | import celery # noqa: F401 139 | except ModuleNotFoundError: 140 | raise ImproperlyConfigured( 141 | 'The package `celery` is required when using `celery` as worker type. ' 142 | 'Please run `pip install celery` if you with to use celery as the workers.' 143 | ) 144 | elif self.worker_type == 'rq': 145 | try: 146 | import django_rq # noqa: F401 147 | except ModuleNotFoundError: 148 | raise ImproperlyConfigured( 149 | 'The package `django-rq` is required when using `rq` as worker type. ' 150 | 'Please run `pip install django-rq` if you with to use rq as the workers.' 151 | ) 152 | else: 153 | raise ImproperlyConfigured("Worker type must be 'celery' or 'rq'") 154 | if not isinstance(self.subscriptions, list): 155 | raise ImproperlyConfigured('Subscriptions must be a list') 156 | if not isinstance(self.publish_settings, list): 157 | raise ImproperlyConfigured('Publish settings must be a list') 158 | for subscription in self.subscriptions: 159 | topic_name = subscription['topic_name'] 160 | subscription_name = subscription['subscription_name'] 161 | connection_string = subscription['connection_string'] 162 | handlers = subscription['handlers'] 163 | if not isinstance(topic_name, str): 164 | raise ImproperlyConfigured(f'Topic name {topic_name} must be a string') 165 | if not isinstance(subscription_name, str): 166 | raise ImproperlyConfigured(f'Subscription name {subscription_name} must be a string') 167 | if not isinstance(connection_string, str): 168 | raise ImproperlyConfigured(f'Connection string {connection_string} must be a string') 169 | if not connection_string.startswith('Endpoint=sb://'): 170 | raise ImproperlyConfigured( 171 | f'Invalid connection string: {connection_string}. Must start with Endpoint=sb://' 172 | ) 173 | if not isinstance(handlers, list): 174 | raise ImproperlyConfigured(f'Handler function {handlers} must be a list') 175 | for handler in handlers: 176 | if not isinstance(handler, dict): 177 | raise ImproperlyConfigured(f'{handlers} must contain dict values, got: {handler}') 178 | subject = handler['subject'] 179 | if not isinstance(subject, str): 180 | raise ImproperlyConfigured(f'Handler subject {subject} for {topic_name} must be a string') 181 | 182 | for topic in self.publish_settings: 183 | if not isinstance(topic['topic_name'], str): 184 | raise ImproperlyConfigured('Topic name must be a string') 185 | if not isinstance(topic['x_metro_key'], str): 186 | raise ImproperlyConfigured('x_metro_key must be a string') 187 | self._validate_import_strings() 188 | 189 | 190 | settings = Settings() 191 | -------------------------------------------------------------------------------- /tests/unit/test_config.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from django.conf import settings as django_settings 4 | from django.core.exceptions import ImproperlyConfigured 5 | from django.test import override_settings 6 | 7 | import pytest 8 | 9 | from metroid.config import Settings 10 | 11 | 12 | def test_metro_is_not_in_settings_exception(): 13 | """ 14 | Deletes Metro from settings, and checks if the correct exception is thrown. 15 | """ 16 | 17 | with pytest.raises(ImproperlyConfigured) as e: 18 | delattr(django_settings, 'METROID') 19 | Settings() 20 | 21 | assert str(e.value) == '`METROID` settings must be defined in settings.py' 22 | 23 | 24 | def test_get_x_metro_key_raises_correctly(): 25 | """ 26 | Tests if the method get_x_metro_key raises an exception when no key is found. 27 | """ 28 | with override_settings(METROID={'subscriptions': [], 'publish_settings': []}): 29 | mock_settings = Settings() 30 | with pytest.raises(ImproperlyConfigured): 31 | mock_settings.get_x_metro_key( 32 | topic_name='test2', 33 | ) 34 | 35 | 36 | def test_publish_topic_name_settings_not_string(): 37 | """ 38 | Tests if the method get_x_metro_key raises an exception when no key is found. 39 | """ 40 | with override_settings(METROID={'publish_settings': [{'topic_name': 123}]}): 41 | with pytest.raises(ImproperlyConfigured) as e: 42 | mock_settings = Settings() 43 | mock_settings.validate() 44 | assert str(e.value) == 'Topic name must be a string' 45 | 46 | 47 | def test_publish_metro_key_settings_not_string(): 48 | """ 49 | Tests if the method get_x_metro_key raises an exception when no key is found. 50 | """ 51 | with override_settings(METROID={'publish_settings': [{'topic_name': 'topicName', 'x_metro_key': 123}]}): 52 | with pytest.raises(ImproperlyConfigured) as e: 53 | mock_settings = Settings() 54 | mock_settings.validate() 55 | assert str(e.value) == 'x_metro_key must be a string' 56 | 57 | 58 | def test_get_x_metro_key_returns_when_found(): 59 | with override_settings( 60 | METROID={'subscriptions': [], 'publish_settings': [{'topic_name': 'test2', 'x_metro_key': 'asdf'}]} 61 | ): 62 | mock_settings = Settings() 63 | metro_key = mock_settings.get_x_metro_key(topic_name='test2') 64 | assert metro_key == 'asdf' 65 | 66 | 67 | def test_publish_settings_is_not_list_exception(): 68 | """ 69 | Provides subscriptions as a string instead of a list, and checks if the correct exception is thrown. 70 | """ 71 | with override_settings(METROID={'publish_settings': 'i am string'}): 72 | with pytest.raises(ImproperlyConfigured) as e: 73 | invalid_settings = Settings() 74 | invalid_settings.validate() 75 | 76 | assert str(e.value) == 'Publish settings must be a list' 77 | 78 | 79 | def test_subscriptions_is_not_list_exception(): 80 | """ 81 | Provides subscriptions as a string instead of a list, and checks if the correct exception is thrown. 82 | """ 83 | with override_settings(METROID={'subscriptions': 'i am string'}): 84 | with pytest.raises(ImproperlyConfigured) as e: 85 | invalid_settings = Settings() 86 | invalid_settings.validate() 87 | 88 | assert str(e.value) == 'Subscriptions must be a list' 89 | 90 | 91 | def test_worker_type_is_not_valid(): 92 | """ 93 | Provides worker type an invalid string, and checks if the correct exception is thrown. 94 | """ 95 | with override_settings(METROID={'worker_type': 'coolio'}): 96 | with pytest.raises(ImproperlyConfigured) as e: 97 | invalid_settings = Settings() 98 | invalid_settings.validate() 99 | 100 | assert str(e.value) == "Worker type must be 'celery' or 'rq'" 101 | 102 | 103 | def test_worker_type_is_celery_not_installed(): 104 | """ 105 | Provides worker type an invalid string, and checks if the correct exception is thrown. 106 | """ 107 | import celery # noqa: F401 108 | 109 | # Mock away the celery dependency 110 | backup = None 111 | if 'celery' in sys.modules: 112 | backup = sys.modules['celery'] 113 | sys.modules['celery'] = None 114 | 115 | with override_settings(METROID={'worker_type': 'celery'}): 116 | with pytest.raises(ImproperlyConfigured) as e: 117 | invalid_settings = Settings() 118 | invalid_settings.validate() 119 | 120 | assert ( 121 | str(e.value) == 'The package `celery` is required when using `celery` as worker type. ' 122 | 'Please run `pip install celery` if you with to use celery as the workers.' 123 | ) 124 | 125 | # Put it back in - otherwise a bunch of downstream tests break 126 | if backup: 127 | sys.modules['celery'] = backup 128 | 129 | 130 | def test_worker_type_is_rq_not_installed(): 131 | """ 132 | Provides worker type an invalid string, and checks if the correct exception is thrown. 133 | """ 134 | import django_rq # noqa: F401 135 | 136 | # Mock away the django-rq dependency 137 | backup = None 138 | if 'django_rq' in sys.modules: 139 | backup = sys.modules['django_rq'] 140 | sys.modules['django_rq'] = None 141 | 142 | with override_settings(METROID={'worker_type': 'rq'}): 143 | with pytest.raises(ImproperlyConfigured) as e: 144 | invalid_settings = Settings() 145 | invalid_settings.validate() 146 | 147 | assert ( 148 | str(e.value) == 'The package `django-rq` is required when using `rq` as worker type. ' 149 | 'Please run `pip install django-rq` if you with to use rq as the workers.' 150 | ) 151 | 152 | # Put it back in - otherwise a bunch of downstream tests break 153 | if backup: 154 | sys.modules['django_rq'] = backup 155 | 156 | 157 | def test_topic_name_is_not_str_exception(): 158 | """ 159 | Provides topic_name as an integer instead of a string, and checks if the correct exception is thrown. 160 | """ 161 | topic_name_value = 123 162 | with override_settings( 163 | METROID={ 164 | 'subscriptions': [ 165 | { 166 | 'topic_name': topic_name_value, 167 | 'subscription_name': 'hello', 168 | 'handlers': [], 169 | 'connection_string': 'Endpoint=sb://...', 170 | } 171 | ] 172 | } 173 | ): 174 | with pytest.raises(ImproperlyConfigured) as e: 175 | invalid_settings = Settings() 176 | invalid_settings.validate() 177 | assert str(e.value) == f'Topic name {topic_name_value} must be a string' 178 | 179 | 180 | def test_subscription_name_is_not_str_exception(): 181 | """ 182 | Provides subscription_name as None instead of a string, and checks if the correct exception is thrown. 183 | """ 184 | subscription_name = None 185 | with override_settings( 186 | METROID={ 187 | 'subscriptions': [ 188 | { 189 | 'topic_name': 'test', 190 | 'subscription_name': subscription_name, 191 | 'handlers': [], 192 | 'connection_string': 'Endpoint=sb://...', 193 | } 194 | ] 195 | } 196 | ): 197 | with pytest.raises(ImproperlyConfigured) as e: 198 | invalid_settings = Settings() 199 | invalid_settings.validate() 200 | 201 | assert str(e.value) == f'Subscription name {subscription_name} must be a string' 202 | 203 | 204 | def test_connection_string_is_not_str_exception(): 205 | """ 206 | Provides connection_string as bool instead of a string, and checks if the correct exception is thrown. 207 | """ 208 | connection_string = bool 209 | with override_settings( 210 | METROID={ 211 | 'subscriptions': [ 212 | { 213 | 'topic_name': 'test', 214 | 'subscription_name': 'coolest/sub/ever', 215 | 'connection_string': connection_string, 216 | 'handlers': [], 217 | } 218 | ] 219 | } 220 | ): 221 | with pytest.raises(ImproperlyConfigured) as e: 222 | invalid_settings = Settings() 223 | invalid_settings.validate() 224 | 225 | assert str(e.value) == f'Connection string {connection_string} must be a string' 226 | 227 | 228 | def test_connection_string_does_not_start_with_endpoint(): 229 | """ 230 | Provides connection_string in an invalid format, and checks if the correct exception is thrown. 231 | """ 232 | connection_string = 'Where is my endpoint!?' 233 | with override_settings( 234 | METROID={ 235 | 'subscriptions': [ 236 | { 237 | 'topic_name': 'test', 238 | 'subscription_name': 'coolest/sub/ever', 239 | 'connection_string': connection_string, 240 | 'handlers': [], 241 | } 242 | ] 243 | } 244 | ): 245 | with pytest.raises(ImproperlyConfigured) as e: 246 | invalid_settings = Settings() 247 | invalid_settings.validate() 248 | 249 | assert str(e.value) == f'Invalid connection string: {connection_string}. Must start with Endpoint=sb://' 250 | 251 | 252 | def test_handlers_is_not_list_exception(): 253 | """ 254 | Provides handlers as a dict, and checks if the correct exception is thrown. 255 | """ 256 | handlers = {} 257 | with override_settings( 258 | METROID={ 259 | 'subscriptions': [ 260 | { 261 | 'topic_name': 'test', 262 | 'subscription_name': 'coolest/sub/ever', 263 | 'connection_string': 'Endpoint=sb://cool', 264 | 'handlers': handlers, 265 | } 266 | ] 267 | } 268 | ): 269 | with pytest.raises(ImproperlyConfigured) as e: 270 | invalid_settings = Settings() 271 | invalid_settings.validate() 272 | 273 | assert str(e.value) == f'Handler function {handlers} must be a list' 274 | 275 | 276 | def test_handler_in_handlers_is_not_dict_exception(): 277 | """ 278 | Provides the handler_function in an invalid format, and checks if the correct exception is thrown. 279 | """ 280 | handler = 'subject' 281 | handlers = [handler] 282 | with override_settings( 283 | METROID={ 284 | 'subscriptions': [ 285 | { 286 | 'topic_name': 'test', 287 | 'subscription_name': 'coolest/sub/ever', 288 | 'connection_string': 'Endpoint=sb://cool', 289 | 'handlers': ['subject'], 290 | } 291 | ] 292 | } 293 | ): 294 | with pytest.raises(ImproperlyConfigured) as e: 295 | invalid_settings = Settings() 296 | invalid_settings.validate() 297 | 298 | assert str(e.value) == f'{handlers} must contain dict values, got: {handler}' 299 | 300 | 301 | def test_subject_is_not_str_exception(): 302 | """ 303 | Provides connection_string in an invalid format, and checks if the correct exception is thrown. 304 | """ 305 | topic_name = 'test' 306 | subject = 123 307 | with override_settings( 308 | METROID={ 309 | 'subscriptions': [ 310 | { 311 | 'topic_name': topic_name, 312 | 'subscription_name': 'coolest/sub/ever', 313 | 'connection_string': 'Endpoint=sb://cool', 314 | 'handlers': [{'subject': subject}], 315 | } 316 | ] 317 | } 318 | ): 319 | with pytest.raises(ImproperlyConfigured) as e: 320 | invalid_settings = Settings() 321 | invalid_settings.validate() 322 | 323 | assert str(e.value) == f'Handler subject {subject} for {topic_name} must be a string' 324 | 325 | 326 | def test_handler_function_is_not_str_exception(): 327 | """ 328 | Provides handler_function in an invalid format, and checks if the correct exception is thrown. 329 | """ 330 | topic_name = 'test' 331 | subject = 'test/banonza' 332 | handler_function = None 333 | with override_settings( 334 | METROID={ 335 | 'subscriptions': [ 336 | { 337 | 'topic_name': topic_name, 338 | 'subscription_name': 'coolest/sub/ever', 339 | 'connection_string': 'Endpoint=sb://cool', 340 | 'handlers': [{'subject': subject, 'handler_function': handler_function}], 341 | } 342 | ] 343 | } 344 | ): 345 | with pytest.raises(ImproperlyConfigured) as e: 346 | invalid_settings = Settings() 347 | invalid_settings._validate_import_strings() 348 | assert str(e.value) == f'Handler function:{handler_function} for {topic_name} must be a string' 349 | 350 | 351 | def test_handler_function_is_not_dotted_path_exception(): 352 | """ 353 | Provides handler_function in an invalid dotted path format, and checks if the correct exception is thrown. 354 | """ 355 | topic_name = 'test' 356 | subject = 'test/banonza' 357 | handler_function = 'testtesttest' 358 | with override_settings( 359 | METROID={ 360 | 'subscriptions': [ 361 | { 362 | 'topic_name': topic_name, 363 | 'subscription_name': 'coolest/sub/ever', 364 | 'connection_string': 'Endpoint=sb://cool', 365 | 'handlers': [{'subject': subject, 'handler_function': handler_function}], 366 | } 367 | ] 368 | } 369 | ): 370 | with pytest.raises(ImproperlyConfigured) as e: 371 | invalid_settings = Settings() 372 | invalid_settings._validate_import_strings() 373 | 374 | assert str(e.value) == f'Handler function:{handler_function} for {topic_name} is not a dotted function path' 375 | 376 | 377 | def test_handler_function_module_not_found_exception(): 378 | """ 379 | Provides handler_function with a non-existing module, and checks if the correct exception is thrown. 380 | """ 381 | topic_name = 'test' 382 | subject = 'test/banonza' 383 | handler_function = 'test.test.test' 384 | with override_settings( 385 | METROID={ 386 | 'subscriptions': [ 387 | { 388 | 'topic_name': topic_name, 389 | 'subscription_name': 'coolest/sub/ever', 390 | 'connection_string': 'Endpoint=sb://cool', 391 | 'handlers': [{'subject': subject, 'handler_function': handler_function}], 392 | } 393 | ] 394 | } 395 | ): 396 | with pytest.raises(ImproperlyConfigured) as e: 397 | invalid_settings = Settings() 398 | invalid_settings._validate_import_strings() 399 | 400 | assert str(e.value) == f'Handler function:{handler_function} for {topic_name} cannot find module' 401 | 402 | 403 | def test_handler_function_method_not_found_exception(): 404 | """ 405 | Provides handler_function with a non-existing method, and checks if the correct exception is thrown. 406 | """ 407 | topic_name = 'test' 408 | subject = 'test/banonza' 409 | handler_function = 'demoproj.tasks.notfound' 410 | with override_settings( 411 | METROID={ 412 | 'subscriptions': [ 413 | { 414 | 'topic_name': topic_name, 415 | 'subscription_name': 'coolest/sub/ever', 416 | 'connection_string': 'Endpoint=sb://cool', 417 | 'handlers': [{'subject': subject, 'handler_function': handler_function}], 418 | } 419 | ] 420 | } 421 | ): 422 | with pytest.raises(ImproperlyConfigured) as e: 423 | invalid_settings = Settings() 424 | invalid_settings._validate_import_strings() 425 | 426 | assert ( 427 | str(e.value) == f'Handler function:{handler_function}' 428 | f' for {topic_name} cannot be imported. Verify that the dotted path points to a function' 429 | ) 430 | 431 | 432 | def test_no_exception_is_thrown(): 433 | """ 434 | Provides mock_subscriptions, which is in valid format and checks if no exception is thrown. 435 | """ 436 | try: 437 | with override_settings( 438 | METROID={ 439 | 'subscriptions': [ 440 | { 441 | 'topic_name': 'test', 442 | 'subscription_name': 'sub-test-djangomoduletest', 443 | 'connection_string': 'Endpoint=sb://cool', 444 | 'handlers': [ 445 | { 446 | 'subject': 'Test/Django/Module', 447 | 'handler_function': 'demoproj.tasks.a_random_task', 448 | }, 449 | ], 450 | } 451 | ], 452 | 'worker_type': 'rq', 453 | }, 454 | ): 455 | mock_settings = Settings() 456 | mock_settings.validate() 457 | except Exception as e: 458 | pytest.fail('Settings validation should not throw an exception with correct mock data') 459 | --------------------------------------------------------------------------------