├── remoulade ├── py.typed ├── cli │ ├── __init__.py │ ├── remoulade_ls.py │ ├── remoulade_run.py │ └── remoulade_scheduler.py ├── api │ ├── __init__.py │ └── apispec.py ├── state │ ├── __init__.py │ ├── errors.py │ ├── backends │ │ ├── __init__.py │ │ ├── stub.py │ │ └── redis.py │ └── middleware.py ├── helpers │ ├── actor_arguments.py │ ├── __init__.py │ ├── redis_client.py │ ├── reduce.py │ └── queues.py ├── brokers │ └── __init__.py ├── scheduler │ └── __init__.py ├── cancel │ ├── __init__.py │ ├── errors.py │ ├── backends │ │ ├── __init__.py │ │ ├── stub.py │ │ └── redis.py │ ├── backend.py │ └── middleware.py ├── logging.py ├── middleware │ ├── current_message.py │ ├── max_tasks.py │ ├── catch_error.py │ ├── age_limit.py │ ├── max_memory.py │ ├── worker_thread_logging.py │ ├── __init__.py │ ├── threading.py │ ├── logging_metadata.py │ ├── heartbeat.py │ └── shutdown.py ├── results │ ├── __init__.py │ ├── backends │ │ ├── __init__.py │ │ ├── local.py │ │ └── stub.py │ └── errors.py ├── rate_limits │ ├── backends │ │ ├── __init__.py │ │ └── stub.py │ ├── __init__.py │ ├── concurrent.py │ ├── window.py │ ├── rate_limiter.py │ ├── bucket.py │ └── backend.py ├── utils.py ├── common.py ├── __init__.py └── errors.py ├── tests ├── __init__.py ├── benchmarks │ ├── __init__.py │ └── test_rabbitmq_cli.py ├── middleware │ ├── __init__.py │ ├── test_max_memory.py │ ├── test_logging_metadata.py │ ├── test_prometheus.py │ ├── test_current_message.py │ ├── test_max_tasks.py │ ├── test_time_limit.py │ ├── test_threading.py │ ├── test_heartbeat.py │ └── test_catch_error.py ├── docker-compose.yml ├── common.py ├── test_bucket_rate_limiter.py ├── test_common.py ├── test_cli.py ├── test_window_rate_limiter.py ├── state │ ├── test_progress_message.py │ ├── test_schemes_api.py │ ├── test_state_objects.py │ ├── test_postgres.py │ └── test_backend.py ├── test_concurrent_rate_limiter.py ├── test_stub_broker.py ├── test_channel_pool.py ├── test_pidfile.py ├── test_helpers.py └── test_worker.py ├── benchmarks ├── __init__.py ├── requirements.txt └── README.md ├── examples ├── basic │ ├── __init__.py │ ├── README.md │ └── example.py ├── crawler │ ├── __init__.py │ ├── requirements.txt │ ├── docker-compose.yaml │ ├── README.md │ └── example.py ├── results │ ├── __init__.py │ ├── docker-compose.yaml │ ├── README.md │ └── example.py ├── long_running │ ├── __init__.py │ ├── docker-compose.yaml │ ├── README.md │ └── example.py ├── persistent │ ├── __init__.py │ ├── .gitignore │ ├── docker-compose.yaml │ ├── README.md │ └── example.py ├── scheduling │ ├── __init__.py │ ├── docker-compose.yaml │ ├── README.md │ └── example.py ├── time_limit │ ├── __init__.py │ ├── docker-compose.yaml │ ├── README.md │ └── example.py └── composition │ ├── composition │ ├── __init__.py │ ├── serve.py │ ├── worker.py │ ├── count_words.py │ └── actors.py │ ├── Dockerfile │ ├── setup.py │ └── README.md ├── docs ├── .gitignore ├── source │ ├── _static │ │ ├── custom.css │ │ ├── logo.png │ │ └── worker_architecture.mmd │ ├── _templates │ │ ├── codefund.html │ │ ├── sidebarlogo.html │ │ └── versions.html │ ├── license.rst │ ├── best_practices.rst │ ├── sitemap.py │ ├── installation.rst │ └── index.rst ├── app.yaml └── Makefile ├── artwork ├── logo.png └── LICENSE ├── MANIFEST.in ├── bin └── remoulade-gevent ├── .github ├── workflows │ ├── validate_commits.yml │ ├── pythonpublish.yml │ ├── lint.yml │ ├── tests.yml │ └── example-container.yml └── ISSUE_TEMPLATE.md ├── compose └── compose.yaml ├── .readthedocs.yaml ├── .pre-commit-config.yaml ├── CONTRIBUTING.md ├── .gitignore └── CONTRIBUTORS.md /remoulade/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /benchmarks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/basic/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/crawler/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/results/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /remoulade/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/benchmarks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/middleware/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/long_running/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/persistent/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/scheduling/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/time_limit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/persistent/.gitignore: -------------------------------------------------------------------------------- 1 | states -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | .coverage 2 | build 3 | _build -------------------------------------------------------------------------------- /examples/composition/composition/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /benchmarks/requirements.txt: -------------------------------------------------------------------------------- 1 | celery 2 | pylibmc 3 | -------------------------------------------------------------------------------- /examples/crawler/requirements.txt: -------------------------------------------------------------------------------- 1 | pylibmc 2 | requests[security] 3 | -------------------------------------------------------------------------------- /remoulade/api/__init__.py: -------------------------------------------------------------------------------- 1 | from .main import app 2 | 3 | __all__ = ["app"] 4 | -------------------------------------------------------------------------------- /artwork/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wiremind/remoulade/HEAD/artwork/logo.png -------------------------------------------------------------------------------- /docs/source/_static/custom.css: -------------------------------------------------------------------------------- 1 | button.copybtn img { 2 | box-sizing: border-box; 3 | } 4 | -------------------------------------------------------------------------------- /docs/source/_static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wiremind/remoulade/HEAD/docs/source/_static/logo.png -------------------------------------------------------------------------------- /examples/persistent/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | rabbitmq: 3 | image: rabbitmq 4 | ports: 5 | - "5672:5672" 6 | -------------------------------------------------------------------------------- /examples/time_limit/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | rabbitmq: 3 | image: rabbitmq 4 | ports: 5 | - "5672:5672" 6 | -------------------------------------------------------------------------------- /examples/long_running/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | rabbitmq: 3 | image: rabbitmq 4 | ports: 5 | - "5672:5672" 6 | -------------------------------------------------------------------------------- /examples/crawler/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | redis: 3 | image: redis 4 | ports: 5 | - "6379:6379" 6 | rabbitmq: 7 | image: rabbitmq 8 | ports: 9 | - "5672:5672" 10 | -------------------------------------------------------------------------------- /examples/results/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | redis: 3 | image: redis 4 | ports: 5 | - "6379:6379" 6 | rabbitmq: 7 | image: rabbitmq 8 | ports: 9 | - "5672:5672" 10 | -------------------------------------------------------------------------------- /examples/scheduling/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | redis: 3 | image: redis 4 | ports: 5 | - "6379:6379" 6 | rabbitmq: 7 | image: rabbitmq 8 | ports: 9 | - "5672:5672" 10 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | prune artwork 2 | prune *benchmarks 3 | prune compose 4 | prune docs 5 | prune examples 6 | prune .git* 7 | exclude .git* 8 | exclude .readthedocs.yaml 9 | prune tests 10 | exclude .pre-commit-config.yaml 11 | -------------------------------------------------------------------------------- /remoulade/state/__init__.py: -------------------------------------------------------------------------------- 1 | from .backend import State, StateBackend, StateStatusesEnum 2 | from .errors import InvalidStateError 3 | from .middleware import MessageState 4 | 5 | __all__ = ["InvalidStateError", "MessageState", "State", "StateBackend", "StateStatusesEnum"] 6 | -------------------------------------------------------------------------------- /remoulade/state/errors.py: -------------------------------------------------------------------------------- 1 | from ..errors import RemouladeError 2 | 3 | 4 | class StateError(RemouladeError): 5 | """Base class for State Errors""" 6 | 7 | 8 | class InvalidStateError(StateError): 9 | """Raised when you try to declare an state 10 | that is not defined. 11 | """ 12 | -------------------------------------------------------------------------------- /docs/source/_templates/codefund.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | 5 | 11 | -------------------------------------------------------------------------------- /bin/remoulade-gevent: -------------------------------------------------------------------------------- 1 | #!python 2 | try: 3 | from gevent import monkey 4 | 5 | monkey.patch_all() 6 | except ImportError: 7 | import sys 8 | 9 | sys.stderr.write("error: gevent is missing. Run `pip install gevent`.") 10 | sys.exit(1) 11 | 12 | import sys 13 | 14 | import remoulade.__main__ 15 | 16 | sys.exit(remoulade.__main__.main()) 17 | -------------------------------------------------------------------------------- /.github/workflows/validate_commits.yml: -------------------------------------------------------------------------------- 1 | name: validate-commit 2 | 3 | on: push 4 | 5 | jobs: 6 | validate-commits: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Check out code 10 | uses: actions/checkout@v4.1.1 11 | with: 12 | fetch-depth: 0 13 | - name: Commitsar check 14 | uses: docker://aevea/commitsar:0.20.2 15 | -------------------------------------------------------------------------------- /examples/composition/Dockerfile: -------------------------------------------------------------------------------- 1 | # Please run docker build from remoulade main directory 2 | 3 | FROM python:3.12 4 | 5 | WORKDIR /usr/src/app 6 | 7 | COPY . . 8 | ARG VERSION=4 9 | RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_REMOULADE=${VERSION} pip install -e .[all] 10 | RUN pip install -e examples/composition 11 | 12 | WORKDIR /usr/src/app/examples/composition 13 | CMD composition_worker 14 | -------------------------------------------------------------------------------- /tests/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | redis: 3 | image: redis 4 | ports: 5 | - "6481:6379" 6 | rabbitmq: 7 | image: rabbitmq 8 | ports: 9 | - "5784:5672" 10 | postgres: 11 | image: postgres 12 | ports: 13 | - "5544:5432" 14 | environment: 15 | POSTGRES_USER: remoulade 16 | POSTGRES_HOST_AUTH_METHOD: trust 17 | POSTGRES_DB: test -------------------------------------------------------------------------------- /tests/common.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | 3 | from remoulade import Worker 4 | 5 | 6 | @contextmanager 7 | def worker(*args, **kwargs): 8 | try: 9 | worker = Worker(*args, **kwargs) 10 | worker.start() 11 | yield worker 12 | finally: 13 | worker.stop() 14 | 15 | 16 | def get_logs(caplog, msg): 17 | return [record for record in caplog.records if msg in record.message] 18 | -------------------------------------------------------------------------------- /compose/compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | redis: 3 | image: redis:7 4 | ports: 5 | - "127.0.0.1:6379:6379" 6 | rabbitmq: 7 | image: rabbitmq:4.1 8 | ports: 9 | - "127.0.0.1:5672:5672" 10 | postgres: 11 | image: postgres:16 12 | ports: 13 | - "127.0.0.1:5432:5432" 14 | environment: 15 | POSTGRES_USER: remoulade 16 | POSTGRES_HOST_AUTH_METHOD: trust 17 | POSTGRES_DB: remoulade 18 | -------------------------------------------------------------------------------- /examples/composition/composition/serve.py: -------------------------------------------------------------------------------- 1 | # Here, we run the remoulade API server written in Flask, which can be used to manage 2 | # the broker or to get stats (through the super-bowl frontend) 3 | 4 | from remoulade.api import app 5 | 6 | from .actors import broker # noqa: F401 7 | 8 | 9 | # Run backend API server (used for superbowl for instance) 10 | def serve(): 11 | app.run(debug=True, host="0.0.0.0", port=5000, reloader_type="stat") # noqa: S104 12 | -------------------------------------------------------------------------------- /docs/app.yaml: -------------------------------------------------------------------------------- 1 | runtime: python27 2 | api_version: 1 3 | threadsafe: true 4 | 5 | handlers: 6 | - url: / 7 | static_files: build/html/index.html 8 | upload: build/html/index.html 9 | secure: always 10 | 11 | - url: /(.*) 12 | static_files: build/html/\1 13 | upload: build/html/(.*) 14 | secure: always 15 | 16 | skip_files: 17 | - ^(.*/)?#.*#$ 18 | - ^(.*/)?.*~$ 19 | - ^(.*/)?.*\.py[co]$ 20 | - ^(.*/)?.*/RCS/.*$ 21 | - ^(.*/)?\..*$ 22 | - ^(.*/)?\.bak$ 23 | - ^(.*/)?(?!build/html)/.*$ 24 | -------------------------------------------------------------------------------- /examples/time_limit/README.md: -------------------------------------------------------------------------------- 1 | # Remoulade Time Limit Example 2 | 3 | This example demonstrates the time limit feature of Remoulade. 4 | 5 | ## Running the Example 6 | 7 | 1. Install [Docker][docker]. 8 | 1. In a terminal, run `docker compose up`. 9 | 1. Install remoulade: `pip install remoulade[rabbitmq]` 10 | 1. In a separate terminal window, run the workers: `remoulade example`. 11 | 1. In another terminal, run `python -m example`. 12 | 13 | 14 | [docker]: https://docs.docker.com/engine/install/ 15 | 16 | -------------------------------------------------------------------------------- /examples/results/README.md: -------------------------------------------------------------------------------- 1 | # Remoulade Results Example 2 | 3 | This example demonstrates how to use Remoulade actors to store and 4 | retrieve task results. 5 | 6 | ## Running the Example 7 | 8 | 1. Install [Docker][docker]. 9 | 1. In a terminal, run `docker compose up`. 10 | 1. Install remoulade: `pip install remoulade[rabbitmq]` 11 | 1. In a separate terminal window, run the workers: `remoulade example`. 12 | 1. In another terminal, run `python -m example`. 13 | 14 | 15 | [docker]: https://docs.docker.com/engine/install/ 16 | -------------------------------------------------------------------------------- /examples/long_running/README.md: -------------------------------------------------------------------------------- 1 | # Remoulade Long Running Example 2 | 3 | This example demonstrates extremely long-running tasks in Remoulade. 4 | 5 | ## Running the Example 6 | 7 | 1. Install [Docker][docker]. 8 | 1. In a terminal, run `docker compose up`. 9 | 1. Install remoulade: `pip install remoulade[rabbitmq]` 10 | 1. In a separate terminal window, run the workers: `remoulade example`. 11 | 1. In another terminal, run `python -m example` to enqueue a task. 12 | 13 | 14 | [docker]: https://docs.docker.com/engine/install/ 15 | 16 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Build documentation in the docs/ directory with Sphinx 9 | sphinx: 10 | configuration: docs/source/conf.py 11 | 12 | # Optionally set the version of Python and requirements required to build your docs 13 | python: 14 | version: 3.8 15 | install: 16 | - method: pip 17 | path: . 18 | extra_requirements: 19 | - dev 20 | -------------------------------------------------------------------------------- /docs/source/_templates/sidebarlogo.html: -------------------------------------------------------------------------------- 1 | 7 | 8 |

9 | 11 |
12 |

13 | -------------------------------------------------------------------------------- /examples/composition/composition/worker.py: -------------------------------------------------------------------------------- 1 | # Here, we launch the worker, which will process tasks 2 | 3 | from prometheus_client.registry import REGISTRY 4 | 5 | from remoulade import Middleware 6 | from remoulade.__main__ import main 7 | from remoulade.middleware import Prometheus 8 | 9 | from .actors import broker 10 | 11 | prometheus: Middleware = Prometheus( 12 | http_host="0.0.0.0", # noqa: S104 13 | http_port=9191, 14 | registry=REGISTRY, 15 | ) 16 | broker.add_middleware(prometheus) 17 | 18 | 19 | # Run worker 20 | def run(): 21 | main() 22 | -------------------------------------------------------------------------------- /examples/basic/README.md: -------------------------------------------------------------------------------- 1 | # Remoulade Basic Example 2 | 3 | This example demonstrates how easy it is to get started with Remoulade. 4 | 5 | ## Running the Example 6 | 7 | 1. Install [Docker][docker]. 8 | 1. In a terminal, run `docker run --name basic_rabbitmq -p 5672:5672 rabbitmq`. 9 | 1. Install remoulade: `pip install remoulade[rabbitmq]`. 10 | 1. In a separate terminal window, run the workers: `remoulade example`. 11 | 1. In another terminal, run `python -m example 100` to enqueue 100 12 | `add` tasks. 13 | 14 | [docker]: https://docs.docker.com/engine/install/ 15 | -------------------------------------------------------------------------------- /examples/basic/example.py: -------------------------------------------------------------------------------- 1 | import random 2 | import sys 3 | 4 | import remoulade 5 | from remoulade.brokers.rabbitmq import RabbitmqBroker 6 | 7 | broker = RabbitmqBroker() 8 | remoulade.set_broker(broker) 9 | 10 | 11 | @remoulade.actor 12 | def add(x, y): 13 | add.logger.info("The sum of %d and %d is %d.", x, y, x + y) 14 | 15 | 16 | broker.declare_actor(add) 17 | 18 | 19 | def main(count): 20 | count = int(count) 21 | for _ in range(count): 22 | add.send(random.randint(0, 1000), random.randint(0, 1000)) 23 | 24 | 25 | if __name__ == "__main__": 26 | sys.exit(main(sys.argv[1])) 27 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_language_version: 2 | python: python3.12 3 | exclude: ^env/ 4 | repos: 5 | - repo: https://github.com/pre-commit/mirrors-mypy 6 | rev: v1.18.2 7 | hooks: 8 | - id: mypy 9 | additional_dependencies: 10 | - sqlalchemy[mypy] 11 | - types-redis 12 | - types-python-dateutil 13 | - types-simplejson 14 | - types-requests 15 | - attrs 16 | - repo: https://github.com/astral-sh/ruff-pre-commit 17 | rev: v0.14.0 18 | hooks: 19 | - id: ruff-check 20 | args: 21 | - --fix 22 | - id: ruff-format 23 | -------------------------------------------------------------------------------- /remoulade/helpers/actor_arguments.py: -------------------------------------------------------------------------------- 1 | from inspect import Parameter, signature 2 | 3 | 4 | def get_actor_arguments(actor): 5 | params = signature(actor.fn).parameters 6 | 7 | args = [] 8 | for param in params.values(): 9 | arg = {"name": param.name} 10 | if param.annotation != Parameter.empty: 11 | arg["type"] = str(param.annotation) 12 | if param.default != Parameter.empty: 13 | if param.default is None: 14 | arg["default"] = "" 15 | else: 16 | arg["default"] = str(param.default) 17 | 18 | args.append(arg) 19 | 20 | return args 21 | -------------------------------------------------------------------------------- /examples/crawler/README.md: -------------------------------------------------------------------------------- 1 | # Remoulade Web Crawler Example 2 | 3 | This example implements a very simple distributed web crawler using 4 | Remoulade. 5 | 6 | ## Running the Example 7 | 8 | 1. Install [Docker][docker]. 9 | 1. In a terminal, run `docker compose up`. 10 | 1. Install remoulade: `pip install remoulade[rabbitmq]` 11 | 1. Install the example's dependencies: `pip install -r requirements.txt` 12 | 1. In a separate terminal, run `remoulade example` to run the workers. 13 | 1. Finally, run `python example.py https://example.com` to begin 14 | crawling http://example.com. 15 | 16 | 17 | [docker]: https://docs.docker.com/engine/install/ 18 | -------------------------------------------------------------------------------- /examples/persistent/README.md: -------------------------------------------------------------------------------- 1 | # Remoulade Persistent Example 2 | 3 | This example demonstrates long-running tasks in Remoulade that are durable to 4 | worker shutdowns by using simple state persistence. 5 | 6 | ## Running the Example 7 | 8 | 1. Install [Docker][docker]. 9 | 1. In a terminal, run `docker compose up`. 10 | 1. Install remoulade: `pip install remoulade[rabbitmq]` 11 | 1. In a separate terminal window, run the workers: `remoulade example`. 12 | 1. In another terminal, run `python -m example ` to enqueue a task. 13 | 1. To test the persistence, terminate the worker process with `crtl-c`, then 14 | restart with the same `n` value. 15 | 16 | 17 | [docker]: https://docs.docker.com/engine/install/ 18 | -------------------------------------------------------------------------------- /examples/scheduling/README.md: -------------------------------------------------------------------------------- 1 | # Remoulade Scheduling Example 2 | 3 | This example demonstrates how to use the scheduler of remoulade. 4 | 5 | ### Jobs 6 | In this example, the scheduler will be used to run the following tasks : 7 | - Count the words of https://github.com every second 8 | - Count the words of https://gitlab.com every ten seconds 9 | 10 | ### Instructions 11 | 12 | 1. Install [Docker][docker]. 13 | 1. In a terminal, run `docker compose up`. 14 | 1. Install remoulade: `pip install remoulade[rabbitmq]` 15 | 1. In a separate terminal window, run the workers: `remoulade example`. 16 | 1. In another terminal, run `python -m example`. 17 | 18 | 19 | [docker]: https://docs.docker.com/engine/install/ 20 | -------------------------------------------------------------------------------- /examples/composition/setup.py: -------------------------------------------------------------------------------- 1 | # Dummy python package to show composition 2 | 3 | from setuptools import setup 4 | 5 | setup( 6 | name="composition", 7 | version="0.0.1", 8 | packages=[ 9 | "composition", 10 | ], 11 | package_dir={"composition": "."}, 12 | install_requires=[ 13 | "remoulade[redis,rabbitmq,postgres,server]", 14 | "prometheus_client", 15 | "requests", 16 | ], 17 | entry_points={ 18 | "console_scripts": [ 19 | "composition_worker=composition.worker:run", 20 | "composition_ask_count_words=composition.count_words:ask_count_words", 21 | "composition_serve=composition.serve:serve", 22 | ] 23 | }, 24 | ) 25 | -------------------------------------------------------------------------------- /examples/composition/README.md: -------------------------------------------------------------------------------- 1 | # Remoulade Composition Example 2 | 3 | __ 4 | This example demonstrates how to use Remoulade's high level composition 5 | abstractions. 6 | 7 | ## Running the Example 8 | 9 | - Install [Docker](https://docs.docker.com/engine/install/). 10 | - In a terminal window, run `docker compose -f compose/compose.yaml up` to run the RabbitMQ and Redis services. 11 | - Install this example, ideally in a virtual env: `cd examples/composition && pip install -e .`. 12 | - In a separate terminal window, run the worker (defined as entry_point): `composition_worker`. 13 | - In another terminal, run the example with multiple URLs as argument, for example : `composition_ask_count_words https://google.com https://rabbitmq.com https://redis.io`. 14 | -------------------------------------------------------------------------------- /docs/source/license.rst: -------------------------------------------------------------------------------- 1 | Project License 2 | =============== 3 | 4 | Copyright (C) 2017,2018 CLEARTYPE SRL 5 | 6 | Remoulade is free software; you can redistribute it and/or modify it 7 | under the terms of the GNU Lesser General Public License as published by 8 | the Free Software Foundation, either version 3 of the License, or (at 9 | your option) any later version. 10 | 11 | Remoulade is distributed in the hope that it will be useful, but WITHOUT 12 | ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 | FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 14 | License for more details. 15 | 16 | You should have received a copy of the GNU Lesser General Public License 17 | along with this program. If not, see http://www.gnu.org/licenses/. 18 | -------------------------------------------------------------------------------- /remoulade/state/backends/__init__.py: -------------------------------------------------------------------------------- 1 | try: 2 | from .postgres import PostgresBackend 3 | except ImportError: # pragma: no cover 4 | import warnings 5 | 6 | warnings.warn( 7 | "PostgresBackend is not available. Run `pip install remoulade[postgres]` to add support for that backend.", 8 | ImportWarning, 9 | stacklevel=2, 10 | ) 11 | 12 | try: 13 | from .redis import RedisBackend 14 | from .stub import StubBackend 15 | except ImportError: # pragma: no cover 16 | import warnings 17 | 18 | warnings.warn( 19 | "RedisBackend is not available. Run `pip install remoulade[redis]` to add support for that backend.", 20 | ImportWarning, 21 | stacklevel=2, 22 | ) 23 | 24 | __all__ = ["PostgresBackend", "RedisBackend", "StubBackend"] 25 | -------------------------------------------------------------------------------- /examples/composition/composition/count_words.py: -------------------------------------------------------------------------------- 1 | # Here, we enqueue a new task that will be processed by workers. 2 | # For the sake of the example, we are going to run synchronously 3 | 4 | import argparse 5 | 6 | from remoulade import group 7 | 8 | from .actors import count_words, request 9 | 10 | 11 | def ask_count_words(): 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument("uri", nargs="*", help="A website URI.") 14 | arguments = parser.parse_args() 15 | if not arguments.uri: 16 | raise Exception("Please specify URI(s) to count") 17 | 18 | jobs = group([request.message(uri) | count_words.message() for uri in arguments.uri]).run() 19 | for uri, count in zip(arguments.uri, jobs.results.get(block=True), strict=False): 20 | print(f" * {uri} has {count} words") 21 | -------------------------------------------------------------------------------- /remoulade/brokers/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | -------------------------------------------------------------------------------- /tests/test_bucket_rate_limiter.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from remoulade.rate_limits import BucketRateLimiter 4 | 5 | 6 | def test_bucket_rate_limiter_limits_per_bucket(rate_limiter_backend): 7 | # Given that I have a bucket rate limiter and a call database 8 | limiter = BucketRateLimiter(rate_limiter_backend, "sequential-test", limit=2) 9 | calls = 0 10 | 11 | for _ in range(2): 12 | # And I wait until the next second starts 13 | now = time.time() 14 | time.sleep(1 - (now - int(now))) 15 | 16 | # And I acquire it multiple times sequentially 17 | for _ in range(8): 18 | with limiter.acquire(raise_on_failure=False) as acquired: 19 | if not acquired: 20 | continue 21 | 22 | calls += 1 23 | 24 | # I expect it to have succeeded four times 25 | assert calls == 4 26 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = remoulade 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | 12 | .PHONY: all 13 | all: 14 | sphinx-versioning build -a -b -W 'v[1-9]+.[0-9]+.[0-9]+' "./docs/$(SOURCEDIR)" "$(BUILDDIR)/html/" 15 | 16 | 17 | .PHONY: deploy 18 | deploy: all 19 | gcloud app deploy --project remoulade-docs app.yaml 20 | 21 | 22 | .PHONY: help 23 | help: 24 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 25 | 26 | 27 | .PHONY: Makefile 28 | # Catch-all target: route all unknown targets to Sphinx using the new 29 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 30 | %: Makefile 31 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 32 | -------------------------------------------------------------------------------- /tests/test_common.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from remoulade.helpers.queues import dq_name, q_name, xq_name 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "given,expected", [("default", "default"), ("default.DQ", "default"), ("default.XQ", "default")] 8 | ) 9 | def test_q_name_returns_canonical_names(given, expected): 10 | assert q_name(given) == expected 11 | 12 | 13 | @pytest.mark.parametrize( 14 | "given,expected", [("default", "default.DQ"), ("default.DQ", "default.DQ"), ("default.XQ", "default.DQ")] 15 | ) 16 | def test_dq_name_returns_canonical_delay_names(given, expected): 17 | assert dq_name(given) == expected 18 | 19 | 20 | @pytest.mark.parametrize( 21 | "given,expected", [("default", "default.XQ"), ("default.DQ", "default.XQ"), ("default.XQ", "default.XQ")] 22 | ) 23 | def test_xq_name_returns_delay_names(given, expected): 24 | assert xq_name(given) == expected 25 | -------------------------------------------------------------------------------- /artwork/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018 WIREMIND SAS 2 | 3 | All rights reserved. 4 | 5 | This logo or a modified version may be used by anyone to refer to the 6 | Remoulade project, but does not indicate endorsement by the project. 7 | 8 | Redistribution and use in source (the AI file) and binary forms 9 | (rendered PNG files, etc.) of the image, with or without modification, 10 | are permitted provided that the following conditions are met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice and this list of conditions. 14 | 15 | * The names of the contributors to the Remoulade software (see 16 | CONTRIBUTORS.md) may not be used to endorse or promote products 17 | derived from this software without specific prior written 18 | permission. 19 | 20 | We would appreciate it if you make the image a link to 21 | https://remoulade.readthedocs.io/ if you use it on a web page. -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | from subprocess import PIPE 2 | 3 | fakebroker = object() 4 | 5 | 6 | def test_cli_fails_to_start_given_an_invalid_broker_name(start_cli): 7 | # Given that this module doesn't define a broker called "idontexist" 8 | # When I start the cli and point it at that broker 9 | proc = start_cli("tests.test_cli:idontexist", stdout=PIPE, stderr=PIPE) 10 | proc.wait(5) 11 | 12 | # Then the process return code should be 2 13 | assert proc.returncode == 2 14 | 15 | 16 | def test_cli_fails_to_start_given_an_invalid_broker_instance(start_cli): 17 | # Given that this module defines a "fakebroker" variable that's not a Broker 18 | # When I start the cli and point it at that broker 19 | proc = start_cli("tests.test_cli:fakebroker", stdout=PIPE, stderr=PIPE) 20 | proc.wait(5) 21 | 22 | # Then the process return code should be 2 23 | assert proc.returncode == 2 24 | -------------------------------------------------------------------------------- /remoulade/scheduler/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | from .scheduler import ScheduledJob, Scheduler 19 | 20 | __all__ = ["ScheduledJob", "Scheduler"] 21 | -------------------------------------------------------------------------------- /examples/scheduling/example.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | import remoulade 4 | from remoulade.brokers.rabbitmq import RabbitmqBroker 5 | from remoulade.scheduler import ScheduledJob, Scheduler 6 | 7 | broker = RabbitmqBroker() 8 | remoulade.set_broker(broker) 9 | 10 | 11 | @remoulade.actor() 12 | def count_words(url): 13 | response = requests.get(url).text # noqa: S113 14 | print(f"There are {len(response)} words in {url}") 15 | 16 | 17 | broker.declare_actor(count_words) 18 | 19 | if __name__ == "__main__": 20 | scheduler = Scheduler( 21 | broker, 22 | [ 23 | ScheduledJob(actor_name="count_words", kwargs={"url": "https://github.com"}, interval=1), 24 | ScheduledJob(actor_name="count_words", kwargs={"url": "https://gitlab.com"}, interval=10), 25 | ], 26 | period=0.1, # scheduler run each 0.1 (second) 27 | ) 28 | remoulade.set_scheduler(scheduler) 29 | scheduler.start() 30 | -------------------------------------------------------------------------------- /.github/workflows/pythonpublish.yml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | jobs: 9 | deploy: 10 | name: Upload release to PyPI 11 | runs-on: ubuntu-latest 12 | environment: 13 | name: release 14 | url: https://pypi.org/project/remoulade/ 15 | permissions: 16 | id-token: write # IMPORTANT: this permission is mandatory for trusted publishing 17 | steps: 18 | - uses: actions/checkout@v5 19 | - name: Install uv and set the Python version 20 | uses: astral-sh/setup-uv@v6 21 | with: 22 | python-version: "3.12" 23 | activate-environment: true 24 | - name: Install dependencies 25 | run: uv pip install twine 26 | - name: Build package 27 | run: | 28 | uv build 29 | twine check --strict dist/* 30 | - name: Publish package 31 | uses: pypa/gh-action-pypi-publish@release/v1 32 | -------------------------------------------------------------------------------- /remoulade/cli/remoulade_ls.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import importlib 3 | import sys 4 | 5 | from remoulade import get_broker 6 | 7 | 8 | def parse_arguments(): 9 | parser = argparse.ArgumentParser( 10 | prog="remoulade-ls", 11 | description="List of remoulade actors", 12 | formatter_class=argparse.RawDescriptionHelpFormatter, 13 | ) 14 | parser.add_argument("modules", metavar="module", nargs="*", help="additional python modules to import") 15 | parser.add_argument("--path", "-P", default=".", nargs="*", type=str, help="the module import path (default: .)") 16 | return parser.parse_args() 17 | 18 | 19 | def main(): 20 | args = parse_arguments() 21 | 22 | for path in args.path: 23 | sys.path.insert(0, path) 24 | 25 | for module in args.modules: 26 | importlib.import_module(module) 27 | 28 | print("List of available actors:") 29 | for actor in get_broker().actors: 30 | print(actor) 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # Issues 2 | 3 | GitHub issues are for bugs. If you have questions, please ask them on stackoverflow. 4 | 5 | ## Checklist 6 | 7 | * [ ] Does your title concisely summarize the problem? 8 | * [ ] Did you include a minimal, reproducible example? 9 | * [ ] What OS are you using? 10 | * [ ] What version of Remoulade are you using? 11 | * [ ] What did you do? 12 | * [ ] What did you expect would happen? 13 | * [ ] What happened? 14 | 15 | 16 | ## What OS are you using? 17 | 18 | 19 | 20 | 21 | ## What version of Remoulade are you using? 22 | 23 | 24 | 25 | 26 | ## What did you do? 27 | 28 | 29 | 30 | 31 | ## What did you expect would happen? 32 | 33 | 34 | 35 | 36 | ## What happened? 37 | 38 | 39 | -------------------------------------------------------------------------------- /benchmarks/README.md: -------------------------------------------------------------------------------- 1 | # Celery vs Remoulade benchmarks 2 | 3 | ## Setup 4 | 5 | 1. Install [RabbitMQ][rabbitmq] and [Redis][redis] 6 | 1. Install Remoulade: `pip install remoulade[rabbitmq]` or `pip install remoulade[redis]` 7 | 1. Install the dependencies: `pip install -r requirements.txt` 8 | 1. Run `redis-server` or `rabbitmq-server` in a terminal window. 9 | 1. In a separate terminal window, run `redis-server`. 10 | 11 | ## Running the benchmarks 12 | 13 | Run `python bench.py` to run the Remoulade benchmark, and `python 14 | bench.py --use-celery` to run the Celery benchmark. Prepend `env 15 | REDIS=1` to each command to run the benchmarks against Redis. 16 | 17 | Run `python bench.py --help` to see all the available options. 18 | 19 | ## Caveats 20 | 21 | As with any benchmark, take it with a grain of salt. Remoulade has an 22 | advantage over Celery in most of these benchmarks by design. 23 | 24 | 25 | [rabbitmq]: https://www.rabbitmq.com 26 | [redis]: https://redis.io 27 | -------------------------------------------------------------------------------- /remoulade/cancel/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | from .backend import CancelBackend 19 | from .errors import MessageCanceled 20 | from .middleware import Cancel 21 | 22 | __all__ = ["Cancel", "CancelBackend", "MessageCanceled"] 23 | -------------------------------------------------------------------------------- /remoulade/cancel/errors.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | from ..middleware.middleware import MiddlewareError 18 | 19 | 20 | class MessageCanceled(MiddlewareError): 21 | """Raised when a message has been canceled before it was processed""" 22 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Python Linting 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | paths: 7 | - "**/*.py" 8 | - ".github/workflows/*.yml" 9 | pull_request: 10 | paths: 11 | - "**/*.py" 12 | - ".github/workflows/*.yml" 13 | 14 | jobs: 15 | lint: 16 | runs-on: ubuntu-latest 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | python-version: ["3.12", "3.13", "3.14"] 21 | steps: 22 | - uses: actions/checkout@v5 23 | - name: Install uv and set the Python version 24 | uses: astral-sh/setup-uv@v6 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | activate-environment: true 28 | - name: Install dependencies 29 | run: uv pip install -e '.[dev]' 30 | - name: Lint with ruff 31 | run: ruff check 32 | - name: Format with ruff 33 | run: ruff format --check --diff 34 | - name: Check typing with mypy 35 | run: mypy --show-traceback 36 | -------------------------------------------------------------------------------- /docs/source/_templates/versions.html: -------------------------------------------------------------------------------- 1 | {% if versions %} 2 |
3 |

Versions

4 | 5 | 22 | 23 | 29 |
30 | {% endif %} 31 | -------------------------------------------------------------------------------- /remoulade/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | from .actor_arguments import get_actor_arguments 19 | from .backoff import compute_backoff 20 | from .reduce import reduce 21 | 22 | __all__ = ["compute_backoff", "get_actor_arguments", "reduce"] 23 | -------------------------------------------------------------------------------- /examples/composition/composition/actors.py: -------------------------------------------------------------------------------- 1 | # Here, we define all the actors and the remoulade broker 2 | 3 | import requests 4 | 5 | import remoulade 6 | from remoulade.brokers.rabbitmq import RabbitmqBroker 7 | from remoulade.encoder import PickleEncoder 8 | from remoulade.results import Results 9 | from remoulade.results.backends import RedisBackend 10 | from remoulade.state import MessageState 11 | from remoulade.state.backends import PostgresBackend 12 | 13 | encoder = PickleEncoder() 14 | backend = RedisBackend(encoder=encoder) 15 | broker = RabbitmqBroker() 16 | broker.add_middleware(Results(backend=backend)) 17 | remoulade.set_broker(broker) 18 | remoulade.set_encoder(encoder) 19 | remoulade.get_broker().add_middleware(MessageState(backend=PostgresBackend())) 20 | 21 | 22 | @remoulade.actor(store_results=True) 23 | def request(uri): 24 | return requests.get(uri).text # noqa: S113 25 | 26 | 27 | @remoulade.actor(store_results=True) 28 | def count_words(response): 29 | return len(response.split(" ")) 30 | 31 | 32 | remoulade.declare_actors([request, count_words]) 33 | -------------------------------------------------------------------------------- /examples/time_limit/example.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | import time 4 | 5 | import remoulade 6 | from remoulade.brokers.rabbitmq import RabbitmqBroker 7 | 8 | broker = RabbitmqBroker() 9 | remoulade.set_broker(broker) 10 | 11 | 12 | @remoulade.actor(time_limit=5000) 13 | def long_running(): 14 | logger = logging.getLogger("long_running") 15 | 16 | while True: 17 | logger.info("Sleeping...") 18 | time.sleep(1) 19 | 20 | 21 | @remoulade.actor(time_limit=5000) 22 | def long_running_with_catch(): 23 | logger = logging.getLogger("long_running_with_catch") 24 | 25 | try: 26 | while True: 27 | logger.info("Sleeping...") 28 | time.sleep(1) 29 | except remoulade.middleware.time_limit.TimeLimitExceeded: 30 | logger.warning("Time limit exceeded. Aborting...") 31 | 32 | 33 | remoulade.declare_actors([long_running, long_running_with_catch]) 34 | 35 | 36 | def main(): 37 | long_running.send() 38 | long_running_with_catch.send() 39 | 40 | 41 | if __name__ == "__main__": 42 | sys.exit(main()) 43 | -------------------------------------------------------------------------------- /remoulade/logging.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | import inspect 19 | import logging 20 | 21 | 22 | def get_logger(module, name=None): 23 | logger_fqn = module 24 | if name is not None: 25 | if inspect.isclass(name): 26 | name = name.__name__ 27 | logger_fqn += "." + name 28 | 29 | return logging.getLogger(logger_fqn) 30 | -------------------------------------------------------------------------------- /remoulade/middleware/current_message.py: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: LPGL-3.0 2 | # Copyright (C) 2019 CLEARTYPE SRL 3 | # Based in current_message.py from the dramatiq project 4 | 5 | from threading import local 6 | from typing import TYPE_CHECKING, Any, Optional 7 | 8 | from .middleware import Middleware 9 | 10 | if TYPE_CHECKING: 11 | from .. import Broker, Message 12 | 13 | 14 | class CurrentMessage(Middleware): 15 | local_data = local() 16 | 17 | @classmethod 18 | def get_current_message(cls) -> Optional["Message[Any]"]: 19 | """Get the message that triggered the current actor. Messages 20 | are thread local so this returns ``None`` when called outside 21 | of actor code. 22 | """ 23 | return getattr(cls.local_data, "current_message", None) 24 | 25 | def before_process_message(self, broker: "Broker", message: "Message") -> None: 26 | self.local_data.current_message = message 27 | 28 | def after_process_message(self, broker: "Broker", message: "Message", *, result=None, exception=None) -> None: 29 | self.local_data.current_message = None 30 | -------------------------------------------------------------------------------- /remoulade/middleware/max_tasks.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | from ..logging import get_logger 4 | from .middleware import Middleware 5 | 6 | 7 | class MaxTasks(Middleware): 8 | """Middleware that stop a worker if its amount of tasks completed reach max_tasks. 9 | If a task causes a worker to reach this limit, all other worker threads will complete their task, and the worker 10 | will be stopped afterwards. 11 | 12 | Parameters: 13 | max_tasks(int): The amount of tasks to complete before stopping the worker 14 | """ 15 | 16 | def __init__(self, *, max_tasks: int): 17 | self.logger = get_logger(__name__, type(self)) 18 | self.max_tasks = max_tasks 19 | self.tasks_count = 0 20 | self.lock = threading.Lock() 21 | 22 | def after_worker_thread_process_message(self, broker, thread): 23 | with self.lock: 24 | self.tasks_count += 1 25 | if self.tasks_count == self.max_tasks: 26 | self.logger.info( 27 | f"Stopping worker thread as completed tasks ({self.tasks_count}) > max_tasks ({self.max_tasks})" 28 | ) 29 | thread.stop() 30 | -------------------------------------------------------------------------------- /tests/middleware/test_max_memory.py: -------------------------------------------------------------------------------- 1 | import time 2 | from unittest import mock 3 | 4 | from remoulade.middleware import MaxMemory 5 | 6 | 7 | @mock.patch("remoulade.middleware.max_memory.resource.getrusage") 8 | def test_below_max_memory(getrusage, stub_broker, stub_worker, do_work): 9 | getrusage.return_value = mock.MagicMock(ru_maxrss=50) 10 | 11 | # Given a broker with a MaxMemory middleware 12 | stub_broker.add_middleware(MaxMemory(max_memory=100)) 13 | 14 | do_work.send() 15 | stub_broker.join(do_work.queue_name) 16 | stub_worker.join() 17 | time.sleep(0.01) 18 | 19 | assert not stub_worker.worker_stopped 20 | 21 | 22 | @mock.patch("remoulade.middleware.max_memory.resource.getrusage") 23 | def test_above_max_memory(getrusage, stub_broker, stub_worker, do_work): 24 | getrusage.return_value = mock.MagicMock(ru_maxrss=200) 25 | 26 | # Given a broker with a MaxMemory middleware 27 | stub_broker.add_middleware(MaxMemory(max_memory=100)) 28 | 29 | do_work.send() 30 | stub_broker.join(do_work.queue_name) 31 | stub_worker.join() 32 | time.sleep(0.01) # wait a bit for worker thread to finish 33 | 34 | assert stub_worker.worker_stopped 35 | -------------------------------------------------------------------------------- /remoulade/results/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | from .backend import Missing, ResultBackend 19 | from .errors import ErrorStored, MessageIdsMissing, ResultError, ResultMissing, ResultTimeout 20 | from .middleware import Results 21 | 22 | __all__ = [ 23 | "ErrorStored", 24 | "MessageIdsMissing", 25 | "Missing", 26 | "ResultBackend", 27 | "ResultError", 28 | "ResultMissing", 29 | "ResultTimeout", 30 | "Results", 31 | ] 32 | -------------------------------------------------------------------------------- /remoulade/cancel/backends/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | import warnings 18 | 19 | from .stub import StubBackend 20 | 21 | try: 22 | from .redis import RedisBackend 23 | except ImportError: # pragma: no cover 24 | warnings.warn( 25 | "RedisBackend is not available. Run `pip install remoulade[redis]` to add support for that backend.", 26 | ImportWarning, 27 | stacklevel=2, 28 | ) 29 | 30 | __all__ = ["RedisBackend", "StubBackend"] 31 | -------------------------------------------------------------------------------- /tests/test_window_rate_limiter.py: -------------------------------------------------------------------------------- 1 | import time 2 | from collections import Counter 3 | from concurrent.futures import ThreadPoolExecutor 4 | 5 | from remoulade.rate_limits import WindowRateLimiter 6 | 7 | 8 | def test_window_rate_limiter_limits_per_window(rate_limiter_backend): 9 | # Given that I have a bucket rate limiter and a call database 10 | limiter = WindowRateLimiter(rate_limiter_backend, "window-test", limit=2, window=1) 11 | calls = Counter() 12 | 13 | # And a function that increments keys over the span of 3 seconds 14 | def work(): 15 | for _ in range(15): 16 | for _ in range(8): 17 | with limiter.acquire(raise_on_failure=False) as acquired: 18 | if not acquired: 19 | continue 20 | 21 | calls[int(time.time())] += 1 22 | 23 | time.sleep(0.2) 24 | 25 | # If I run that function multiple times concurrently 26 | with ThreadPoolExecutor(max_workers=8) as e: 27 | futures = [e.submit(work) for _ in range(8)] 28 | 29 | for future in futures: 30 | future.result() 31 | 32 | # I expect between 6 and 10 calls to have been made in total 33 | assert 6 <= sum(calls.values()) <= 10 34 | -------------------------------------------------------------------------------- /examples/long_running/example.py: -------------------------------------------------------------------------------- 1 | import random 2 | import sys 3 | import time 4 | 5 | import remoulade 6 | from remoulade.brokers.rabbitmq import RabbitmqBroker 7 | 8 | broker = RabbitmqBroker() 9 | remoulade.set_broker(broker) 10 | 11 | 12 | def fib(n): 13 | x, y = 1, 1 14 | while n > 2: 15 | x, y = x + y, x 16 | n -= 1 17 | return x 18 | 19 | 20 | @remoulade.actor(time_limit=86400000) 21 | def long_running(duration): 22 | deadline = time.monotonic() + duration 23 | while time.monotonic() < deadline: 24 | long_running.logger.info("%d seconds remaining.", deadline - time.monotonic()) 25 | 26 | n = random.randint(1000, 1000000) 27 | long_running.logger.info("Computing fib(%d).", n) 28 | 29 | fib(n) 30 | long_running.logger.info("Computed fib(%d).", n) 31 | 32 | sleep = random.randint(1, 30) 33 | long_running.logger.info("Sleeping for %d seconds...", sleep) 34 | time.sleep(sleep) 35 | 36 | 37 | broker.declare_actor(long_running) 38 | 39 | 40 | def main(): 41 | for _ in range(1000): 42 | long_running.send(random.randint(3600, 14400)) 43 | time.sleep(random.randint(60, 3600)) 44 | 45 | 46 | if __name__ == "__main__": 47 | sys.exit(main()) 48 | -------------------------------------------------------------------------------- /remoulade/rate_limits/backends/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | import warnings 19 | 20 | from .stub import StubBackend 21 | 22 | try: 23 | from .redis import RedisBackend 24 | except ImportError: # pragma: no cover 25 | warnings.warn( 26 | "RedisBackend is not available. Run `pip install remoulade[redis]` to add support for that backend.", 27 | ImportWarning, 28 | stacklevel=2, 29 | ) 30 | 31 | 32 | __all__ = ["RedisBackend", "StubBackend"] 33 | -------------------------------------------------------------------------------- /remoulade/rate_limits/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | from .backend import RateLimiterBackend 19 | from .bucket import BucketRateLimiter 20 | from .concurrent import ConcurrentRateLimiter 21 | from .rate_limiter import RateLimiter, RateLimitExceeded 22 | from .window import WindowRateLimiter 23 | 24 | __all__ = [ 25 | "BucketRateLimiter", 26 | "ConcurrentRateLimiter", 27 | "RateLimitExceeded", 28 | "RateLimiter", 29 | "RateLimiterBackend", 30 | "WindowRateLimiter", 31 | ] 32 | -------------------------------------------------------------------------------- /examples/results/example.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import random 3 | import sys 4 | import time 5 | 6 | import remoulade 7 | from remoulade.brokers.rabbitmq import RabbitmqBroker 8 | from remoulade.encoder import PickleEncoder 9 | from remoulade.results import Results 10 | from remoulade.results.backends import RedisBackend 11 | 12 | result_backend = RedisBackend(encoder=PickleEncoder()) 13 | broker = RabbitmqBroker() 14 | broker.add_middleware(Results(backend=result_backend)) 15 | remoulade.set_broker(broker) 16 | 17 | 18 | @remoulade.actor(store_results=True) 19 | def sleep_then_add(t, x, y): 20 | time.sleep(t) 21 | return x + y 22 | 23 | 24 | broker.declare_actor(sleep_then_add) 25 | 26 | 27 | def main(): 28 | parser = argparse.ArgumentParser() 29 | parser.add_argument("count", type=int, help="the number of messages to enqueue") 30 | args = parser.parse_args() 31 | 32 | messages = [ 33 | sleep_then_add.send( 34 | random.randint(1, 5), 35 | random.randint(0, 1000), 36 | random.randint(0, 1000), 37 | ) 38 | for _ in range(args.count) 39 | ] 40 | 41 | for message in messages: 42 | print(message.result.get(block=True)) 43 | 44 | 45 | if __name__ == "__main__": 46 | sys.exit(main()) 47 | -------------------------------------------------------------------------------- /tests/middleware/test_logging_metadata.py: -------------------------------------------------------------------------------- 1 | import remoulade 2 | from remoulade.middleware import LoggingMetadata 3 | 4 | 5 | def test_callback(stub_broker): 6 | # Given that I have two callbacks 7 | def callback_middleware(): 8 | return {"correlation_id_middleware": "id_middleware", "correlation_id": "id_middleware"} 9 | 10 | def callback_message(): 11 | return {"correlation_id": "id_message"} 12 | 13 | # and a LoggingMetadata Middleware 14 | stub_broker.add_middleware(LoggingMetadata(logging_metadata_getter=callback_middleware)) 15 | 16 | # and a declared actor 17 | @remoulade.actor(logging_metadata={"correlation_id_actor": "id_actor", "correlation_id": "id_actor"}) 18 | def do_work(): 19 | return 1 20 | 21 | stub_broker.declare_actor(do_work) 22 | 23 | # I build a message with the logging_metadata and logging_metadata_getter options 24 | message = do_work.message_with_options( 25 | logging_metadata={"correlation_id": "id_logging_metadata"}, logging_metadata_getter=callback_message 26 | ) 27 | 28 | assert message.options["logging_metadata"] == { 29 | "correlation_id": "id_message", 30 | "correlation_id_actor": "id_actor", 31 | "correlation_id_middleware": "id_middleware", 32 | } 33 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Python Test 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | paths: 7 | - "**/*.py" 8 | - ".github/workflows/*.yml" 9 | pull_request: 10 | paths: 11 | - "**/*.py" 12 | - ".github/workflows/*.yml" 13 | 14 | jobs: 15 | test: 16 | runs-on: ubuntu-latest 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | python-version: ["3.12", "3.13", "3.14"] 21 | services: 22 | redis: 23 | image: redis:7 24 | ports: 25 | - 6481:6379 26 | rabbitmq: 27 | image: rabbitmq:4.1 28 | ports: 29 | - 5784:5672 30 | postgres: 31 | image: postgres:16 32 | ports: 33 | - 5544:5432 34 | env: 35 | POSTGRES_USER: remoulade 36 | POSTGRES_HOST_AUTH_METHOD: trust 37 | POSTGRES_DB: test 38 | 39 | steps: 40 | - uses: actions/checkout@v5 41 | - name: Install uv and set the Python version 42 | uses: astral-sh/setup-uv@v6 43 | with: 44 | python-version: ${{ matrix.python-version }} 45 | activate-environment: true 46 | - name: Install dependencies 47 | run: uv pip install -e '.[dev]' 48 | - name: Test 49 | run: pytest --benchmark-skip 50 | -------------------------------------------------------------------------------- /remoulade/api/apispec.py: -------------------------------------------------------------------------------- 1 | from functools import update_wrapper 2 | 3 | from apispec import APISpec 4 | from apispec.ext.marshmallow import MarshmallowPlugin 5 | from flask import request 6 | from flask_apispec import FlaskApiSpec, use_kwargs 7 | 8 | 9 | def validate_schema(schema): 10 | def decorator(func): 11 | def wrapper(*args, **kwargs): 12 | if request.is_json and request.content_length and request.content_length > 0: 13 | body = request.get_json() 14 | else: 15 | body = {} 16 | res = schema().load(body) 17 | return func(*args, **res, **kwargs) 18 | 19 | return use_kwargs(schema, apply=False)(update_wrapper(wrapper, func)) 20 | 21 | return decorator 22 | 23 | 24 | def add_swagger(app, routes_dict): 25 | app.config.update( 26 | { 27 | "APISPEC_SPEC": APISpec( 28 | title="Remoulade API", version="v1", plugins=[MarshmallowPlugin()], openapi_version="3.0.3" 29 | ) 30 | } 31 | ) 32 | api_spec = FlaskApiSpec(document_options=False) 33 | 34 | for name, route_list in routes_dict.items(): 35 | for route in route_list: 36 | api_spec.register(route, blueprint=name if name != "main" else None) 37 | api_spec.init_app(app) 38 | -------------------------------------------------------------------------------- /remoulade/results/backends/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | import warnings 19 | 20 | from .local import LocalBackend 21 | from .stub import StubBackend 22 | 23 | try: 24 | from .redis import RedisBackend 25 | except ImportError: # pragma: no cover 26 | warnings.warn( 27 | "RedisBackend is not available. Run `pip install remoulade[redis]` to add support for that backend.", 28 | ImportWarning, 29 | stacklevel=2, 30 | ) 31 | 32 | __all__ = ["LocalBackend", "RedisBackend", "StubBackend"] 33 | -------------------------------------------------------------------------------- /docs/source/_static/worker_architecture.mmd: -------------------------------------------------------------------------------- 1 | sequenceDiagram 2 | participant CT as ConsumerThread("default") 3 | participant RMQ as RabbitMQ 4 | participant WorkQueue as WorkQueue in memory 5 | participant AcksQueue as AcksQueue in memory 6 | participant WT1 as WorkerThread1 7 | participant WT2 as WorkerThread2 8 | 9 | loop while running, read messages off the broker and put them on the work queue 10 | CT->>+RMQ: get_message(queue_name="default", block=True) 11 | RMQ-->>CT: Message(id=1, {...}) 12 | CT->>WorkQueue: put_work(Message(id=1, {...})) 13 | 14 | loop while pending acks 15 | CT->>AcksQueue: get_pending_ack() 16 | AcksQueue-->>CT: Message(id=1, {...}) 17 | CT->>RMQ: ack_message(id=1) 18 | end 19 | end 20 | 21 | alt concurrently 22 | loop while running, process work 23 | WT1->>WorkQueue: get_work(block=True) 24 | WorkQueue-->>WT1: Message(id=2, { ... }) 25 | WT1->>WT1: process_message(Message(id=2, {...})) 26 | WT1->>AcksQueue: ack_message(Message(id=2, { ... })) 27 | end 28 | else 29 | loop while running, process work 30 | WT2->>WorkQueue: get_work(block=True) 31 | WorkQueue-->>WT2: Message(id=1, { ... }) 32 | WT2->>WT2: process_message(Message(id=1, {...})) 33 | WT2->>AcksQueue: ack_message(Message(id=1, { ... })) 34 | end 35 | end -------------------------------------------------------------------------------- /remoulade/cli/remoulade_run.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import importlib 3 | import json 4 | import sys 5 | 6 | from remoulade import get_broker 7 | 8 | 9 | def parse_arguments(): 10 | parser = argparse.ArgumentParser( 11 | prog="remoulade-run", description="Runs a remoulade actor", formatter_class=argparse.RawDescriptionHelpFormatter 12 | ) 13 | parser.add_argument("modules", metavar="module", nargs="*", help="additional python modules to import") 14 | parser.add_argument("--path", "-P", default=".", nargs="*", type=str, help="the module import path (default: .)") 15 | parser.add_argument("--actor-name", "-N", type=str, help="The actor to be ran") 16 | parser.add_argument("--args", "-A", type=json.loads, help="The actor's args") 17 | parser.add_argument("--kwargs", "-K", type=json.loads, help="The actor's kwargs") 18 | return parser.parse_args() 19 | 20 | 21 | def main(): 22 | args = parse_arguments() 23 | 24 | for path in args.path: 25 | sys.path.insert(0, path) 26 | 27 | for module in args.modules: 28 | importlib.import_module(module) 29 | 30 | if args.actor_name not in get_broker().actors: 31 | print(f"{args.actor_name} is not an available actor") 32 | return 33 | 34 | args_ = args.args or [] 35 | kwargs_ = args.kwargs or {} 36 | 37 | print(get_broker().actors[args.actor_name](*args_, **kwargs_)) 38 | -------------------------------------------------------------------------------- /remoulade/utils.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | from typing import TYPE_CHECKING 18 | 19 | from .errors import NoScheduler 20 | 21 | if TYPE_CHECKING: 22 | from .scheduler import Scheduler 23 | 24 | global_scheduler: "Scheduler | None" = None 25 | 26 | 27 | def get_scheduler() -> "Scheduler": 28 | global global_scheduler 29 | if global_scheduler is None: 30 | raise NoScheduler("Scheduler not found, are you sure you called set_scheduler(scheduler) ?") 31 | return global_scheduler 32 | 33 | 34 | def set_scheduler(scheduler: "Scheduler") -> None: 35 | global global_scheduler 36 | global_scheduler = scheduler 37 | -------------------------------------------------------------------------------- /tests/middleware/test_prometheus.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | from unittest import mock 3 | from urllib import request as request 4 | 5 | import pytest 6 | 7 | from remoulade.brokers.stub import StubBroker 8 | 9 | prometheus = pytest.importorskip("remoulade.middleware.prometheus") 10 | 11 | 12 | def test_prometheus_middleware_exposes_metrics(): 13 | try: 14 | # Given a broker 15 | broker = StubBroker() 16 | 17 | # And an instance of the prometheus middleware 18 | prom = prometheus.Prometheus() 19 | prom.before_worker_boot(broker, None) 20 | 21 | # If I wait for the server to start 22 | sleep(0.01) 23 | 24 | # When I request metrics via HTTP 25 | with request.urlopen("http://127.0.0.1:9191") as resp: 26 | # Then the response should be successful 27 | assert resp.getcode() == 200 28 | finally: 29 | prom.after_worker_shutdown(broker, None) 30 | 31 | 32 | @mock.patch("prometheus_client.start_http_server") 33 | def test_prometheus_process_message(_, stub_broker, do_work): 34 | prom = prometheus.Prometheus() 35 | 36 | stub_broker.emit_before("worker_boot", mock.Mock()) 37 | prom.before_worker_boot(stub_broker, mock.Mock(consumer_whitelist=None)) 38 | 39 | message = do_work.message() 40 | 41 | assert not prom.message_start_times 42 | prom.before_process_message(stub_broker, message) 43 | assert prom.message_start_times 44 | prom.after_process_message(stub_broker, message) 45 | assert not prom.message_start_times 46 | -------------------------------------------------------------------------------- /remoulade/results/errors.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | from ..errors import RemouladeError 19 | 20 | 21 | class ResultError(RemouladeError): 22 | """Base class for result errors.""" 23 | 24 | 25 | class ResultTimeout(ResultError): 26 | """Raised when waiting for a result times out.""" 27 | 28 | 29 | class ResultMissing(ResultError): 30 | """Raised when a result can't be found.""" 31 | 32 | 33 | class ErrorStored(ResultError): 34 | """Raised when an error is stored in the result backend and raise_on_error is True""" 35 | 36 | 37 | class ParentFailed(ResultError): 38 | """Error stored when a parent actor in the pipeline failed""" 39 | 40 | 41 | class MessageIdsMissing(ResultError): 42 | """Raised when message_ids linked to a group_id can't be found""" 43 | -------------------------------------------------------------------------------- /tests/middleware/test_current_message.py: -------------------------------------------------------------------------------- 1 | import remoulade 2 | from remoulade.middleware import CurrentMessage 3 | 4 | 5 | class TestCurrentMessage: 6 | """Class to test the middleware 7 | CurrentMessage""" 8 | 9 | def test_middleware_in_actor(self, stub_broker, stub_worker): 10 | message_ids = set() 11 | 12 | @remoulade.actor 13 | def do_work(): 14 | msg = CurrentMessage.get_current_message() 15 | message_ids.add(msg.message_id) 16 | 17 | stub_broker.declare_actor(do_work) 18 | messages = [do_work.send() for _ in range(20)] 19 | 20 | stub_broker.join(do_work.queue_name) 21 | stub_worker.join() 22 | 23 | assert message_ids == {m.message_id for m in messages} 24 | # verify no current message outside of actor 25 | assert CurrentMessage.get_current_message() is None 26 | 27 | def test_middleware_not_in_actor(self, stub_broker, stub_worker): 28 | stub_broker.middleware.remove(next(m for m in stub_broker.middleware if isinstance(m, CurrentMessage))) 29 | message_ids = set() 30 | 31 | @remoulade.actor 32 | def do_work(): 33 | msg = CurrentMessage.get_current_message() 34 | if msg is not None: 35 | message_ids.add(msg.message_id) 36 | 37 | stub_broker.declare_actor(do_work) 38 | for _ in range(10): 39 | do_work.send() 40 | 41 | stub_broker.join(do_work.queue_name) 42 | stub_worker.join() 43 | 44 | assert len(message_ids) == 0 45 | -------------------------------------------------------------------------------- /.github/workflows/example-container.yml: -------------------------------------------------------------------------------- 1 | # According to https://docs.github.com/en/packages/managing-github-packages-using-github-actions-workflows/publishing-and-installing-a-package-with-github-actions#upgrading-a-workflow-that-accesses-ghcrio 2 | name: Create and publish a Container image 3 | 4 | on: 5 | push: 6 | tags: 7 | - 'v*' 8 | branches: 9 | - master 10 | - docker 11 | 12 | env: 13 | REGISTRY: ghcr.io 14 | IMAGE_NAME: ${{ github.repository }} 15 | 16 | jobs: 17 | build-and-push-image: 18 | runs-on: ubuntu-latest 19 | permissions: 20 | contents: read 21 | packages: write 22 | 23 | steps: 24 | - name: Checkout repository 25 | uses: actions/checkout@v4.1.1 26 | 27 | - name: Log in to the Container registry 28 | uses: docker/login-action@v3.0.0 29 | with: 30 | registry: ${{ env.REGISTRY }} 31 | username: ${{ github.actor }} 32 | password: ${{ secrets.GITHUB_TOKEN }} 33 | 34 | - name: Extract metadata (tags, labels) for Docker 35 | id: meta 36 | uses: docker/metadata-action@v5.0.0 37 | with: 38 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 39 | flavor: | 40 | suffix=-example-composition 41 | 42 | - name: Build and push Docker image 43 | uses: docker/build-push-action@v5.0.0 44 | with: 45 | push: true 46 | file: ./examples/composition/Dockerfile 47 | tags: ${{ steps.meta.outputs.tags }} 48 | labels: ${{ steps.meta.outputs.labels }} 49 | -------------------------------------------------------------------------------- /remoulade/cli/remoulade_scheduler.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import importlib 3 | import logging 4 | import signal 5 | import sys 6 | 7 | from remoulade import get_logger, get_scheduler 8 | 9 | logformat = "[%(asctime)s] [PID %(process)d] [%(threadName)s] [%(name)s] [%(levelname)s] %(message)s" 10 | 11 | 12 | def parse_arguments(): 13 | parser = argparse.ArgumentParser( 14 | prog="remoulade-scheduler", 15 | description="Run remoulade scheduler", 16 | formatter_class=argparse.RawDescriptionHelpFormatter, 17 | ) 18 | parser.add_argument("modules", metavar="module", nargs="*", help="additional python modules to import") 19 | parser.add_argument("--path", "-P", default=".", nargs="*", type=str, help="the module import path (default: .)") 20 | parser.add_argument("--verbose", "-v", action="count", default=0, help="turn on verbose log output") 21 | return parser.parse_args() 22 | 23 | 24 | def main(): 25 | args = parse_arguments() 26 | 27 | for path in args.path: 28 | sys.path.insert(0, path) 29 | 30 | for module in args.modules: 31 | importlib.import_module(module) 32 | 33 | logging.basicConfig(level=logging.INFO if not args.verbose else logging.DEBUG, format=logformat) 34 | logger = get_logger("remoulade", "Scheduler") 35 | 36 | def signal_handler(signal, frame): 37 | logger.debug("Remoulade scheduler is shutting down") 38 | sys.exit(0) 39 | 40 | signal.signal(signal.SIGINT, signal_handler) 41 | signal.signal(signal.SIGTERM, signal_handler) 42 | 43 | logger.debug("Remoulade scheduler start") 44 | sys.exit(get_scheduler().start()) 45 | -------------------------------------------------------------------------------- /tests/middleware/test_max_tasks.py: -------------------------------------------------------------------------------- 1 | import time 2 | from unittest import mock 3 | 4 | import pytest 5 | 6 | import remoulade 7 | from remoulade import Worker 8 | from remoulade.__main__ import main 9 | from remoulade.middleware import MaxTasks 10 | 11 | 12 | def test_max_tasks(stub_broker): 13 | worker = Worker(stub_broker, worker_timeout=100, worker_threads=1) 14 | worker.start() 15 | stub_broker.add_middleware(MaxTasks(max_tasks=20)) 16 | 17 | result = 0 18 | 19 | @remoulade.actor 20 | def do_work(): 21 | nonlocal result 22 | result += 1 23 | 24 | stub_broker.declare_actor(do_work) 25 | 26 | for _ in range(30): 27 | do_work.send() 28 | 29 | time.sleep(0.1) 30 | 31 | # The worker should have run tasks until he reached max_tasks and then stopped 32 | assert result == 20 33 | assert worker.worker_stopped 34 | 35 | 36 | @pytest.mark.timeout(5) 37 | @mock.patch("sys.argv", ["", "tests.middleware.test_max_tasks"]) 38 | def test_max_tasks_multiple_threads(stub_broker): 39 | stub_broker.add_middleware(MaxTasks(max_tasks=1)) 40 | 41 | result = 0 42 | 43 | @remoulade.actor 44 | def do_work(): 45 | nonlocal result 46 | result += 1 47 | time.sleep(1) 48 | 49 | stub_broker.declare_actor(do_work) 50 | 51 | for _ in range(300): 52 | do_work.send() 53 | 54 | ret_code = main() 55 | 56 | # There are 8 worker threads 57 | # All worker threads should stop after the first one is finished 58 | # The amount of message processed may go up to 15 since every thread except the stopped one might start processing 59 | # another message before the main function stops them 60 | assert 8 <= result <= 15 61 | assert ret_code == 0 62 | -------------------------------------------------------------------------------- /tests/state/test_progress_message.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | import pytest 4 | 5 | import remoulade 6 | from remoulade.errors import InvalidProgress 7 | from remoulade.middleware import CurrentMessage 8 | 9 | 10 | class TestProgressMessage: 11 | """Class to test the actions related the 12 | progress of the message""" 13 | 14 | def test_set_progress_message(self, stub_broker, stub_worker, state_middleware): 15 | progress_messages: dict[str, float] = {} 16 | 17 | @remoulade.actor 18 | def do_work(): 19 | msg = CurrentMessage.get_current_message() 20 | progress = random.uniform(0, 1) 21 | msg.set_progress(progress) 22 | progress_messages[msg.message_id] = progress 23 | 24 | stub_broker.declare_actor(do_work) 25 | 26 | messages = [do_work.send() for _ in range(10)] 27 | stub_broker.join(do_work.queue_name) 28 | stub_worker.join() 29 | for m in messages: 30 | state = state_middleware.backend.get_state(m.message_id) 31 | assert state.progress == progress_messages[m.message_id] 32 | 33 | @pytest.mark.parametrize("progress", [-1, 2]) 34 | def test_invalid_progress(self, progress, stub_broker, stub_worker, state_middleware): 35 | error = [] 36 | 37 | @remoulade.actor 38 | def do_work(): 39 | try: 40 | msg = CurrentMessage.get_current_message() 41 | msg.set_progress(progress) 42 | except InvalidProgress: 43 | error.append(True) 44 | 45 | stub_broker.declare_actor(do_work) 46 | do_work.send() 47 | stub_broker.join(do_work.queue_name) 48 | stub_worker.join() 49 | assert error == [True] 50 | -------------------------------------------------------------------------------- /remoulade/cancel/backend.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | from collections.abc import Iterable 18 | 19 | DEFAULT_CANCELLATION_TTL = 3600 20 | 21 | 22 | class CancelBackend: 23 | """ABC for cancel backends. 24 | 25 | Parameters: 26 | cancellation_ttl(int): The minimal amount of seconds message cancellations 27 | should be kept in the backend (default 1h). 28 | """ 29 | 30 | def __init__(self, *, cancellation_ttl: int | None = None) -> None: 31 | self.cancellation_ttl = cancellation_ttl or DEFAULT_CANCELLATION_TTL 32 | 33 | def is_canceled(self, message_id: str, composition_id: str | None) -> bool: 34 | """Return true if the message has been canceled""" 35 | raise NotImplementedError(f"{type(self).__name__!r} does not implement is_canceled") 36 | 37 | def cancel(self, message_ids: Iterable[str]) -> None: 38 | """Mark a message as canceled""" 39 | raise NotImplementedError(f"{type(self).__name__!r} does not implement cancel") 40 | -------------------------------------------------------------------------------- /remoulade/common.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | import uuid 18 | from collections.abc import Iterable, Iterator 19 | from itertools import islice 20 | from time import time 21 | 22 | 23 | def current_millis(): 24 | """Returns the current UNIX time in milliseconds.""" 25 | return int(time() * 1000) 26 | 27 | 28 | def generate_unique_id() -> str: 29 | """Generate a globally-unique message id.""" 30 | return str(uuid.uuid4()) 31 | 32 | 33 | def flatten(iterable): 34 | """Flatten deep an iterable""" 35 | for el in iterable: 36 | if isinstance(el, Iterable) and not isinstance(el, (str, bytes)): 37 | yield from flatten(el) 38 | else: 39 | yield el 40 | 41 | 42 | def chunk(iterable: Iterable, size: int) -> Iterator[list]: 43 | """Returns an iterator of a list of length size""" 44 | i = iter(iterable) 45 | piece = list(islice(i, size)) 46 | while piece: 47 | yield piece 48 | piece = list(islice(i, size)) 49 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## Code 4 | 5 | [Start a discussion] before attempting to make a contribution. Any 6 | contribution that doesn't fit my design goals for the project will be 7 | rejected so it's always better to start a discussion first! 8 | 9 | By submitting contributions, you disavow any rights or claims to any 10 | changes submitted to the Remoulade project and assign the copyright of 11 | those changes to WIREMIND SAS. If you cannot or do not want to 12 | reassign those rights, you shouldn't submit a PR. Instead, you should 13 | open an issue and let someone else do that work. 14 | 15 | ### Pull Requests 16 | 17 | * Make sure any code changes are covered by tests. 18 | * Run [pre-commit] hooks. 19 | * If this is your first contribution, add yourself to the [CONTRIBUTORS] file. 20 | * If your branch is behind master, [rebase] on top of it. 21 | 22 | Run the test suite with `pytest`. 23 | The tests require running [RabbitMQ] and [Redis], the easiest way is to use [docker compose]. 24 | ```bash 25 | cd tests/ 26 | docker compose up 27 | ``` 28 | 29 | [CONTRIBUTORS]: https://github.com/wiremind/remoulade/blob/master/CONTRIBUTORS.md 30 | [RabbitMQ]: https://www.rabbitmq.com/ 31 | [Redis]: https://redis.io 32 | [rebase]: https://github.com/edx/edx-platform/wiki/How-to-Rebase-a-Pull-Request 33 | [pre-commit]: (https://pre-commit.com) 34 | [docker compose]: (https://docs.docker.com/compose/) 35 | 36 | ## Issues 37 | 38 | When you open an issue make sure you include the full stack trace and 39 | that you list all pertinent information (operating system, message 40 | broker, Python implementation) as part of the issue description. 41 | 42 | Please include a minimal, reproducible test case with every bug 43 | report. If the issue is actually a question, consider asking it on 44 | Stack Overflow first. 45 | -------------------------------------------------------------------------------- /remoulade/results/backends/local.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | from typing import Any, ClassVar 3 | 4 | from ..backend import ForgottenResult, Missing, ResultBackend 5 | 6 | 7 | class LocalBackend(ResultBackend): 8 | """An in-memory result backend. For use with LocalBroker only. 9 | 10 | We need to be careful here: if an actor store its results and never forget it, we may store all its results 11 | and never delete it. Resulting in a memory leak. 12 | """ 13 | 14 | results: ClassVar[dict[str, Any]] = {} 15 | group_completions: ClassVar[dict[str, set[str]]] = {} 16 | forgotten_results: ClassVar[set[str]] = set() 17 | 18 | def _get(self, message_key: str, forget: bool = False): 19 | if message_key in self.forgotten_results: 20 | return ForgottenResult.asdict() 21 | 22 | try: 23 | if forget: 24 | data = self.results.pop(message_key) 25 | self.forgotten_results.add(message_key) 26 | return data 27 | else: 28 | return self.results[message_key] 29 | except KeyError: 30 | return Missing 31 | 32 | def _store(self, message_keys, results, _): 33 | for message_key, result in zip(message_keys, results, strict=False): 34 | self.results[message_key] = result 35 | 36 | def _delete(self, key: str): 37 | with contextlib.suppress(KeyError): 38 | del self.results[key] 39 | 40 | def increment_group_completion(self, group_id: str, message_id: str, ttl: int) -> int: 41 | group_completion_key = self.build_group_completion_key(group_id) 42 | completed = self.group_completions.get(group_completion_key, set()) | {message_id} 43 | self.group_completions[group_completion_key] = completed 44 | return len(completed) 45 | -------------------------------------------------------------------------------- /docs/source/best_practices.rst: -------------------------------------------------------------------------------- 1 | .. include:: global.rst 2 | 3 | Best Practices 4 | ============== 5 | 6 | Concurrent Actors 7 | ----------------- 8 | 9 | Your actor will run concurrently with other actors in the system. You 10 | need to be mindful of the impact this has on your database, any third 11 | party services you might be calling and the resources available on the 12 | systems running your workers. Additionally, you need to be mindful of 13 | data races between actors that manipulate the same objects in your 14 | database. 15 | 16 | 17 | Retriable Actors 18 | ---------------- 19 | 20 | Remoulade actors may receive the same message multiple times in the 21 | event of a worker failure (hardware, network, or power failure). This 22 | means that, for any given message, running your actor multiple times 23 | must be safe. This is also known as being "idempotent". 24 | 25 | 26 | Simple Messages 27 | --------------- 28 | 29 | Attempting to send an actor any object that can't be encoded to JSON 30 | by the standard ``json`` package will immediately fail so you'll want 31 | to limit your actor parameters to the following object types: `bool`, 32 | `int`, `float`, `bytes`, `string`, `list` and `dict`. 33 | 34 | Additionally, since messages are sent over the wire you'll want to 35 | keep them as short as possible. For example, if you've got an actor 36 | that operates over ``User`` objects in your system, send that actor 37 | the user's id rather than the serialized user. 38 | 39 | 40 | Error Reporting 41 | --------------- 42 | 43 | Invariably, you're probably going to introduce issues in production 44 | every now and then and some of those issues are going to affect your 45 | tasks. You should use an error reporting service such as Sentry_ so 46 | you get notified of these errors as soon as they occur. 47 | 48 | 49 | .. _Sentry: https://sentry.io/welcome/ 50 | -------------------------------------------------------------------------------- /docs/source/sitemap.py: -------------------------------------------------------------------------------- 1 | import os 2 | import xml.etree.ElementTree as ET 3 | from datetime import datetime 4 | 5 | 6 | def setup(app): 7 | # app.connect("build-finished", build_sitemap) 8 | return {"version": "1.0"} 9 | 10 | 11 | def collect_pages(basedir): 12 | for root, _, files in os.walk(basedir): 13 | for filename in files: 14 | if not filename.endswith(".html"): 15 | continue 16 | 17 | if filename == "index.html": 18 | filename = "" 19 | 20 | priority = 0.8 21 | if "genindex" in filename or "modindex" in filename: 22 | priority = 0.4 23 | elif "_modules" in root: 24 | priority = 0.6 25 | elif filename == "": 26 | priority = 1 27 | 28 | path = os.path.join(root, filename) 29 | stat = os.stat(path) 30 | last_mod = datetime.fromtimestamp(stat.st_mtime).date().isoformat() 31 | 32 | page_path = path[len(basedir) :].lstrip("/") 33 | yield page_path, last_mod, priority 34 | 35 | 36 | def build_sitemap(app, exception): 37 | pages = collect_pages(app.outdir) 38 | root = ET.Element("urlset") 39 | root.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9") 40 | for _, mod, _ in sorted(pages, key=lambda t: -t[2]): 41 | url = ET.SubElement(root, "url") 42 | loc = ET.SubElement(url, "loc") 43 | lastmod = ET.SubElement(url, "lastmod") 44 | priority = ET.SubElement(url, "priority") 45 | loc.text = "https://remoulade.readthedocs.io/{page}" 46 | lastmod.text = mod 47 | priority.text = "{prio:.02f}" 48 | 49 | filename = "{app.outdir}/sitemap.xml" 50 | tree = ET.ElementTree(root) 51 | tree.write(filename, xml_declaration=True, encoding="utf-8", method="xml") 52 | -------------------------------------------------------------------------------- /tests/state/test_schemes_api.py: -------------------------------------------------------------------------------- 1 | from remoulade.api.state import StatesParamsSchema 2 | 3 | 4 | class TestSchemeAPI: 5 | def test_valid_page_scheme(self): 6 | page = { 7 | "sort_column": "message_id", 8 | "sort_direction": "asc", 9 | "size": 100, 10 | "offset": 0, 11 | "selected_actors": ["actor"], 12 | "selected_statuses": ["Success"], 13 | "selected_message_ids": ["id"], 14 | "start_datetime": "2021-08-16 10:00:00", 15 | "end_datetime": "2021-08-16 10:00:00", 16 | } 17 | result = StatesParamsSchema().validate(page) 18 | assert len(result) == 0 19 | 20 | def test_invalid_page_scheme(self): 21 | page = { 22 | "sort_column": 1, 23 | "sort_direction": "up", 24 | "size": -10, 25 | "offset": "offset", 26 | "selected_actors": [1], 27 | "selected_statuses": ["status"], 28 | "selected_message_ids": [123], 29 | "start_datetime": 100000, 30 | "end_datetime": 100000, 31 | } 32 | result = StatesParamsSchema().validate(page) 33 | assert result == { 34 | "end_datetime": ["Not a valid datetime."], 35 | "offset": ["Not a valid integer."], 36 | "selected_actors": {0: ["Not a valid string."]}, 37 | "selected_message_ids": {0: ["Not a valid string."]}, 38 | "selected_statuses": {0: ["Must be one of: Started, Pending, Skipped, Canceled, Failure, Success."]}, 39 | "size": ["Must be greater than or equal to 1 and less than or equal to 1000."], 40 | "sort_column": ["Not a valid string."], 41 | "sort_direction": ["Must be one of: asc, desc."], 42 | "start_datetime": ["Not a valid datetime."], 43 | } 44 | -------------------------------------------------------------------------------- /remoulade/cancel/backends/stub.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | import time 18 | from collections.abc import Iterable 19 | 20 | from ..backend import CancelBackend 21 | 22 | 23 | class StubBackend(CancelBackend): 24 | """An in-memory cancel backend. For use in unit tests. 25 | 26 | Parameters: 27 | cancellation_ttl(int): The minimal amount of seconds message cancellations 28 | should be kept in the backend. 29 | """ 30 | 31 | def __init__(self, *, cancellation_ttl: int | None = None) -> None: 32 | super().__init__(cancellation_ttl=cancellation_ttl) 33 | self.cancellations: dict[str, float] = {} 34 | 35 | def is_canceled(self, message_id: str, composition_id: str) -> bool: 36 | return any( 37 | self.cancellations.get(key, -float("inf")) > time.time() - self.cancellation_ttl 38 | for key in [message_id, composition_id] 39 | if key 40 | ) 41 | 42 | def cancel(self, message_ids: Iterable[str]) -> None: 43 | timestamp = time.time() 44 | for message_id in message_ids: 45 | self.cancellations[message_id] = timestamp 46 | -------------------------------------------------------------------------------- /remoulade/middleware/catch_error.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from .middleware import Middleware 4 | 5 | 6 | class CatchError(Middleware): 7 | """Middleware that lets you enqueue another actor or message on message failure. 8 | 9 | Parameters: 10 | on_failure(Message|Actor|str): A Message, Actor or Actor name to enqueue on failure. 11 | """ 12 | 13 | @property 14 | def actor_options(self): 15 | return {"on_failure"} 16 | 17 | def after_process_message(self, broker, message, *, result=None, exception=None): 18 | from .. import Message 19 | 20 | if message.failed: 21 | on_failure = self.get_option("on_failure", broker=broker, message=message) 22 | if isinstance(on_failure, str): 23 | actor = broker.get_actor(on_failure) 24 | actor.send(message.actor_name, type(exception).__name__, message.args, message.kwargs) 25 | elif isinstance(on_failure, dict): 26 | on_failure_message = Message[Any](**on_failure) 27 | on_failure_message = on_failure_message.copy( 28 | args=[message.actor_name, type(exception).__name__, message.args, message.kwargs] 29 | ) 30 | broker.enqueue(on_failure_message) 31 | 32 | def update_options_before_create_message(self, options, broker, actor_name): 33 | from .. import Message 34 | from ..actor import Actor 35 | 36 | on_failure = options.get("on_failure") 37 | if isinstance(on_failure, Message): 38 | options["on_failure"] = on_failure.asdict() 39 | elif isinstance(on_failure, Actor): 40 | options["on_failure"] = on_failure.actor_name 41 | elif on_failure is not None and not isinstance(on_failure, str): 42 | raise TypeError(f"on_failure must be an Message, an Actor or a string, got {type(on_failure)} instead") 43 | 44 | return options 45 | -------------------------------------------------------------------------------- /examples/crawler/example.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | import re 4 | import sys 5 | from contextlib import closing 6 | from threading import local 7 | 8 | import requests 9 | 10 | import remoulade 11 | from remoulade.brokers.rabbitmq import RabbitmqBroker 12 | 13 | logger = logging.getLogger("example") 14 | anchor_re = re.compile(rb'') 15 | state = local() 16 | 17 | broker = RabbitmqBroker() 18 | remoulade.set_broker(broker) 19 | 20 | urls = [] 21 | 22 | 23 | def get_session(): 24 | session = getattr(state, "session", None) 25 | if session is None: 26 | session = state.session = requests.Session() 27 | return session 28 | 29 | 30 | @remoulade.actor(max_retries=3, time_limit=10000) 31 | def crawl(url): 32 | if url in urls: 33 | logger.warning("URL %r has already been visited. Skipping...", url) 34 | return 35 | 36 | urls.append(url) 37 | logger.info("Crawling %r...", url) 38 | matches = 0 39 | session = get_session() 40 | with closing(session.get(url, timeout=(3.05, 5), stream=True)) as response: 41 | if not response.headers.get("content-type", "").startswith("text/html"): 42 | logger.warning("Skipping URL %r since it's not HTML.", url) 43 | return 44 | 45 | for match in anchor_re.finditer(response.content): 46 | anchor = match.group(1).decode("utf-8") 47 | if anchor.startswith("http://") or anchor.startswith("https://"): 48 | crawl.send(anchor) 49 | matches += 1 50 | 51 | logger.info("Done crawling %r. Found %d anchors.", url, matches) 52 | 53 | 54 | broker.declare_actor(crawl) 55 | 56 | 57 | def main(): 58 | parser = argparse.ArgumentParser() 59 | parser.add_argument("url", type=str, help="a URL to crawl") 60 | args = parser.parse_args() 61 | crawl.send(args.url) 62 | return 0 63 | 64 | 65 | if __name__ == "__main__": 66 | sys.exit(main()) 67 | -------------------------------------------------------------------------------- /tests/middleware/test_time_limit.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | import time 18 | from unittest import mock 19 | 20 | import remoulade 21 | from remoulade.middleware import TimeLimit, TimeLimitExceeded 22 | 23 | 24 | @mock.patch("os._exit") 25 | @mock.patch("remoulade.middleware.time_limit.raise_thread_exception") 26 | def test_time_limit_soft(raise_thread_exception, exit, stub_broker, stub_worker, do_work): # noqa: A002 27 | @remoulade.actor(time_limit=1) 28 | def do_work(): 29 | time.sleep(1) 30 | 31 | stub_broker.declare_actor(do_work) 32 | 33 | # With a TimeLimit middleware 34 | for middleware in stub_broker.middleware: 35 | if isinstance(middleware, TimeLimit): 36 | # That emit a signal every ms and stop after 15ms 37 | middleware.interval = 1 38 | middleware.time_limit = 15 39 | middleware.exit_delay = 0 40 | 41 | # When I send it a message 42 | do_work.send() 43 | 44 | # And join on the queue 45 | stub_broker.join(do_work.queue_name) 46 | stub_worker.join() 47 | 48 | assert raise_thread_exception.called 49 | assert raise_thread_exception.call_args[0][1] == TimeLimitExceeded 50 | assert exit.called 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | uv.lock 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .nox/ 42 | .coverage 43 | .coverage.* 44 | .cache 45 | nosetests.xml 46 | coverage.xml 47 | *.cover 48 | .hypothesis/ 49 | .pytest_cache/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | local_settings.py 58 | db.sqlite3 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | 70 | # PyBuilder 71 | target/ 72 | 73 | # Jupyter Notebook 74 | .ipynb_checkpoints 75 | 76 | # IPython 77 | profile_default/ 78 | ipython_config.py 79 | 80 | # pyenv 81 | .python-version 82 | 83 | # celery beat schedule file 84 | celerybeat-schedule 85 | 86 | # SageMath parsed files 87 | *.sage.py 88 | 89 | # Environments 90 | .env 91 | .venv 92 | env/ 93 | venv/ 94 | ENV/ 95 | env.bak/ 96 | venv.bak/ 97 | 98 | # Spyder project settings 99 | .spyderproject 100 | .spyproject 101 | 102 | # Rope project settings 103 | .ropeproject 104 | 105 | # mkdocs documentation 106 | /site 107 | .run 108 | 109 | # mypy 110 | .mypy_cache/ 111 | .dmypy.json 112 | dmypy.json 113 | 114 | #Others 115 | /.benchmarks 116 | /benchmark*.svg 117 | /.idea 118 | dump.rdb 119 | 120 | # Redis 121 | *.rdb 122 | 123 | # Vim 124 | *.swp 125 | -------------------------------------------------------------------------------- /remoulade/rate_limits/concurrent.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | from .rate_limiter import RateLimiter 19 | 20 | 21 | class ConcurrentRateLimiter(RateLimiter): 22 | """A rate limiter that ensures that only `limit` concurrent 23 | operations may happen at the same time. 24 | 25 | Note: 26 | 27 | You can use a concurrent rate limiter of size 1 to get a 28 | distributed mutex. 29 | 30 | Parameters: 31 | backend(RateLimiterBackend): The backend to use. 32 | key(str): The key to rate limit on. 33 | limit(int): The maximum number of concurrent operations per key. 34 | ttl(int): The time in milliseconds that keys may live for. 35 | """ 36 | 37 | def __init__(self, backend, key, *, limit=1, ttl=900000): 38 | if not limit >= 1: 39 | raise ValueError("limit must be positive") 40 | 41 | super().__init__(backend, key) 42 | self.limit = limit 43 | self.ttl = ttl 44 | 45 | def _acquire(self): 46 | added = self.backend.add(self.key, 1, ttl=self.ttl) 47 | if added: 48 | return True 49 | 50 | return self.backend.incr(self.key, 1, maximum=self.limit, ttl=self.ttl) 51 | 52 | def _release(self): 53 | return self.backend.decr(self.key, 1, minimum=0, ttl=self.ttl) 54 | -------------------------------------------------------------------------------- /remoulade/middleware/age_limit.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | from ..common import current_millis 19 | from ..logging import get_logger 20 | from .middleware import Middleware, MiddlewareError 21 | 22 | 23 | class AgeLimitException(MiddlewareError): 24 | pass 25 | 26 | 27 | class AgeLimit(Middleware): 28 | """Middleware that drops messages that have been in the queue for 29 | too long. 30 | 31 | Parameters: 32 | max_age(int): The default message age limit in millseconds. 33 | Defaults to ``None``, meaning that messages can exist 34 | indefinitely. 35 | """ 36 | 37 | def __init__(self, *, max_age=None): 38 | self.logger = get_logger(__name__, type(self)) 39 | self.max_age = max_age 40 | 41 | @property 42 | def actor_options(self): 43 | return {"max_age"} 44 | 45 | def before_process_message(self, broker, message): 46 | max_age = self.get_option("max_age", broker=broker, message=message) 47 | if not max_age: 48 | return 49 | 50 | if current_millis() - message.message_timestamp >= max_age: 51 | self.logger.warning("Message %r has exceeded its age limit.", message.message_id) 52 | message.fail() 53 | raise AgeLimitException(f"Message {message.message_id} has exceeded its age limit {max_age}ms.") 54 | -------------------------------------------------------------------------------- /remoulade/middleware/max_memory.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | import resource 19 | 20 | from ..logging import get_logger 21 | from .middleware import Middleware 22 | 23 | 24 | class MaxMemory(Middleware): 25 | """Middleware that stop a worker if its amount of resident memory exceed max_memory (in kilobytes) 26 | If a task causes a worker to exceed this limit, the task will be completed, and the worker will be 27 | stopped afterwards. 28 | 29 | Parameters: 30 | max_memory(int): The maximum amount of resident memory (in kilobytes) 31 | """ 32 | 33 | def __init__(self, *, max_memory: int): 34 | self.logger = get_logger(__name__, type(self)) 35 | self.max_memory = max_memory 36 | 37 | def after_worker_thread_process_message(self, broker, thread): 38 | used_memory = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss 39 | if used_memory <= 0: 40 | self.logger.error("Worker unable to determine memory usage") 41 | if used_memory > self.max_memory: 42 | self.logger.warning( 43 | f"Stopping worker thread as used_memory ({used_memory:.2E}Kb) > max_memory ({self.max_memory:.2E}Kb)" 44 | ) 45 | # stopping the worker thread will in time stop the worker process which check if all workers are still 46 | # running (via Worker.worker_stopped) 47 | thread.stop() 48 | -------------------------------------------------------------------------------- /docs/source/installation.rst: -------------------------------------------------------------------------------- 1 | .. include:: global.rst 2 | 3 | Installation 4 | ============ 5 | 6 | Remoulade supports Python versions 3.12 and up and is installable via 7 | `pip`_ or from source. 8 | 9 | 10 | Via pip 11 | ------- 12 | 13 | To install remoulade, simply run the following command in a terminal:: 14 | 15 | $ pip install -U 'remoulade[rabbitmq]' 16 | 17 | Remoulade use RabbitMQ_ as message broker. 18 | 19 | If you would like to use it with Redis_ to store the results then run: 20 | 21 | $ pip install -U 'remoulade[rabbitmq, redis]' 22 | 23 | If you don't have `pip`_ installed, check out `this guide`_. 24 | 25 | Extra Requirements 26 | ^^^^^^^^^^^^^^^^^^ 27 | 28 | When installing the package via pip you can specify the following 29 | extra requirements: 30 | 31 | ============= ======================================================================================= 32 | Name Description 33 | ============= ======================================================================================= 34 | ``rabbitmq`` Installs the required dependencies for using Remoulade with RabbitMQ. 35 | ``redis`` Installs the required dependencies for using Remoulade with Redis. 36 | ============= ======================================================================================= 37 | 38 | If you want to install Remoulade with all available features, run:: 39 | 40 | $ pip install -U 'remoulade[all]' 41 | 42 | Optional Requirements 43 | ^^^^^^^^^^^^^^^^^^^^^ 44 | 45 | If you're using Redis as your broker and aren't planning on using PyPy 46 | then you should additionally install the ``hiredis`` package to get an 47 | increase in throughput. 48 | 49 | 50 | From Source 51 | ----------- 52 | 53 | To install the latest development version of remoulade from source, 54 | clone the repo from `GitHub`_ 55 | 56 | :: 57 | 58 | $ git clone https://github.com/wiremind/remoulade 59 | 60 | then install it to your local site-packages by running 61 | 62 | :: 63 | 64 | $ python -m pip install . 65 | 66 | in the cloned directory. 67 | 68 | 69 | .. _GitHub: https://github.com/wiremind/remoulade 70 | .. _pip: https://pip.pypa.io/en/stable/ 71 | .. _this guide: http://docs.python-guide.org/en/latest/starting/installation/ 72 | -------------------------------------------------------------------------------- /tests/state/test_state_objects.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from remoulade.state.backend import State, StateStatusesEnum 4 | from remoulade.state.errors import InvalidStateError 5 | 6 | 7 | class TestState: 8 | """Class to test State Objects, a data transfer object 9 | that represents the state of a message""" 10 | 11 | @pytest.mark.parametrize("defined_state", StateStatusesEnum) 12 | def test_create_valid_state(self, defined_state): 13 | assert State("id", defined_state, args=[], kwargs={}) 14 | 15 | @pytest.mark.parametrize("undefined_state", ["UndefinedState", "pending"]) 16 | def test_raise_exception_when_invalid_state(self, undefined_state): 17 | # if I send a state not defined in StateStatusesEnum 18 | # it should raise an exception 19 | with pytest.raises(InvalidStateError): 20 | State("id", undefined_state, args=[], kwargs={}) 21 | 22 | def test_check_conversion_object_to_dict(self): 23 | dict_state = State("id", StateStatusesEnum.Success, args=[1, 2, 3], kwargs={"key": "value"}).as_dict() 24 | assert dict_state["status"] == StateStatusesEnum.Success.name 25 | assert dict_state["args"] == [1, 2, 3] 26 | assert dict_state["kwargs"] == {"key": "value"} 27 | 28 | def test_check_conversion_dict_to_object(self): 29 | dict_state = { 30 | "status": "Success", 31 | "args": [1, 2, 3], 32 | "kwargs": {"key": "value"}, 33 | "message_id": "idtest", 34 | "options": {"onsuccess": "save", "time_limit": 1000}, 35 | } 36 | stateobj = State.from_dict(dict_state) 37 | assert stateobj.message_id == dict_state["message_id"] 38 | assert stateobj.status == dict_state["status"] 39 | assert stateobj.kwargs == dict_state["kwargs"] 40 | assert stateobj.args == dict_state["args"] 41 | assert stateobj.options == dict_state["options"] 42 | 43 | def test_exclude_keys_from_serialization(self): 44 | dict_state = State("id", StateStatusesEnum.Success, args=[1, 2, 3], kwargs={"key": "value"}).as_dict( 45 | exclude_keys=("kwargs", "id") 46 | ) 47 | assert "kwargs" not in dict_state 48 | assert "id" not in dict_state 49 | -------------------------------------------------------------------------------- /examples/persistent/example.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import contextlib 3 | import json 4 | import os 5 | import sys 6 | import time 7 | 8 | import remoulade 9 | from remoulade.brokers.rabbitmq import RabbitmqBroker 10 | from remoulade.middleware import Shutdown 11 | 12 | broker = RabbitmqBroker() 13 | remoulade.set_broker(broker) 14 | 15 | 16 | def path_to(*xs): 17 | return os.path.abspath(os.path.join(os.path.dirname(__file__), *(str(x) for x in xs))) 18 | 19 | 20 | def load_state(n): 21 | try: 22 | with open(path_to("states", n)) as f: 23 | data = json.load(f) 24 | except Exception: 25 | fib.logger.info("Could not read state file, using defaults.") 26 | return 1, 1, 0 27 | 28 | i, x2, x1 = data["i"], data["x2"], data["x1"] 29 | fib.logger.info("Resuming fib(%d) from iteration %d.", n, i) 30 | return i, x2, x1 31 | 32 | 33 | def dump_state(n, state): 34 | os.makedirs("states", exist_ok=True) 35 | with open(path_to("states", n), "w") as f: 36 | json.dump(state, f) 37 | 38 | fib.logger.info("Dumped fib(%d) state for iteration %d.", n, state["i"]) 39 | 40 | 41 | def remove_state(n): 42 | with contextlib.suppress(OSError): 43 | os.remove(path_to("states", n)) 44 | 45 | fib.logger.info("Deleted state for fib(%d).", n) 46 | 47 | 48 | @remoulade.actor(time_limit=float("inf"), notify_shutdown=True, max_retries=0) 49 | def fib(n): 50 | i, x2, x1 = load_state(n) 51 | 52 | try: 53 | for j in range(i, n + 1): 54 | state = { 55 | "i": j, 56 | "x2": x2, 57 | "x1": x1, 58 | } 59 | 60 | x2, x1 = x1, x2 + x1 61 | fib.logger.info("fib(%d): %d", j, x1) 62 | time.sleep(0.1) 63 | 64 | remove_state(n) 65 | fib.logger.info("Done!") 66 | except Shutdown: 67 | dump_state(n, state) 68 | 69 | 70 | broker.declare_actor(fib) 71 | 72 | 73 | def main(): 74 | parser = argparse.ArgumentParser(description="Calculates fib(n)") 75 | parser.add_argument("n", type=int, help="must be a positive number") 76 | args = parser.parse_args() 77 | fib.send(args.n) 78 | return 0 79 | 80 | 81 | if __name__ == "__main__": 82 | sys.exit(main()) 83 | -------------------------------------------------------------------------------- /tests/middleware/test_threading.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | from threading import Thread 4 | 5 | import pytest 6 | 7 | from remoulade.middleware import threading 8 | 9 | not_supported = threading.current_platform not in threading.supported_platforms 10 | 11 | 12 | @pytest.mark.skipif(not_supported, reason="Threading not supported on this platform.") 13 | def test_raise_thread_exception(): 14 | # Given that I have a database 15 | caught = [] 16 | 17 | # And a function that waits for an interrupt 18 | def work(): 19 | try: 20 | for _ in range(10): 21 | time.sleep(0.1) 22 | except threading.Interrupt: 23 | caught.append(1) 24 | 25 | # When I start the thread 26 | t = Thread(target=work) 27 | t.start() 28 | time.sleep(0.1) 29 | 30 | # And raise the interrupt and join on the thread 31 | threading.raise_thread_exception(t.ident, threading.Interrupt) 32 | t.join() 33 | 34 | # I expect the interrupt to have been caught 35 | assert sum(caught) == 1 36 | 37 | 38 | @pytest.mark.skipif(not_supported, reason="Threading not supported on this platform.") 39 | def test_raise_thread_exception_on_nonexistent_thread(caplog): 40 | # When an interrupt is raised on a nonexistent thread 41 | threading.raise_thread_exception(-1, threading.Interrupt) 42 | 43 | # I expect a 'failed to set exception' critical message to be logged 44 | assert caplog.record_tuples == [ 45 | ("remoulade.middleware.threading", logging.CRITICAL, ("Failed to set exception (Interrupt) in thread -1.")), 46 | ] 47 | 48 | 49 | def test_raise_thread_exception_unsupported_platform(caplog, monkeypatch): 50 | # monkeypatch fake platform to test logging. 51 | monkeypatch.setattr(threading, "current_platform", "not supported") 52 | 53 | # When raising a thread exception on an unsupported platform 54 | threading.raise_thread_exception(1, threading.Interrupt) 55 | 56 | # I expect a 'platform not supported' critical message to be logged 57 | assert caplog.record_tuples == [ 58 | ( 59 | "remoulade.middleware.threading", 60 | logging.CRITICAL, 61 | ("Setting thread exceptions (Interrupt) is not supported for your current platform ('not supported')."), 62 | ), 63 | ] 64 | -------------------------------------------------------------------------------- /remoulade/helpers/redis_client.py: -------------------------------------------------------------------------------- 1 | import os 2 | from urllib.parse import urlparse 3 | 4 | import redis 5 | import redis.asyncio as redis_async 6 | 7 | 8 | def redis_client(url: str | None, socket_timeout: float | None = None, **parameters): 9 | socket_parameters = {} 10 | if socket_timeout is not None: 11 | socket_parameters = { 12 | "socket_timeout": socket_timeout, 13 | "socket_connect_timeout": socket_timeout, 14 | "socket_keepalive": True, 15 | } 16 | if url: 17 | url_parsed = urlparse(url) 18 | if url_parsed.scheme == "sentinel": 19 | sentinel_kwargs = {"password": url_parsed.password, **socket_parameters} 20 | sentinel = redis.Sentinel([(url_parsed.hostname, url_parsed.port)], sentinel_kwargs=sentinel_kwargs) 21 | return sentinel.master_for( 22 | service_name=os.path.normpath(url_parsed.path).split("/")[1], 23 | password=url_parsed.password, 24 | **socket_parameters, 25 | ) 26 | else: 27 | parameters["connection_pool"] = redis.ConnectionPool.from_url(url, **socket_parameters) # type: ignore 28 | return redis.Redis(**parameters) 29 | 30 | 31 | def async_redis_client(url: str | None, socket_timeout: float | None = None, **parameters): 32 | socket_parameters = {} 33 | if socket_timeout is not None: 34 | socket_parameters = { 35 | "socket_timeout": socket_timeout, 36 | "socket_connect_timeout": socket_timeout, 37 | "socket_keepalive": True, 38 | } 39 | if url: 40 | url_parsed = urlparse(url) 41 | if url_parsed.scheme == "sentinel": 42 | sentinel_kwargs = {"password": url_parsed.password, **socket_parameters} 43 | sentinel = redis_async.Sentinel([(url_parsed.hostname, url_parsed.port)], sentinel_kwargs=sentinel_kwargs) 44 | return sentinel.master_for( # type: ignore 45 | service_name=os.path.normpath(url_parsed.path).split("/")[1], 46 | password=url_parsed.password, 47 | **socket_parameters, 48 | ) 49 | else: 50 | parameters["connection_pool"] = redis_async.ConnectionPool.from_url(url, **socket_parameters) 51 | return redis_async.Redis(**parameters) 52 | -------------------------------------------------------------------------------- /remoulade/helpers/reduce.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | from ..composition import group 18 | 19 | 20 | def reduce(messages, merge_actor, cancel_on_error=False, size=2, merge_kwargs=None): 21 | """Recursively merge messages 22 | 23 | Parameters: 24 | messages(Iterator[Message|pipeline]): A sequence of messages or pipelines that needs to be merged. 25 | merge_actor(Actor): The actor that will be responsible for the merge of two messages. 26 | cancel_on_error(boolean): True if you want to cancel all messages of a group if one of 27 | the actor fails, this is only possible with a Cancel middleware. 28 | size(int): Number of messages that are reduced at once. 29 | merge_kwargs: kwargs to be passed to each merge message. 30 | 31 | Returns: 32 | Message|pipeline: a message or a pipeline that will return the reduced result of all the given messages. 33 | 34 | Raise: 35 | NoCancelBackend: if no cancel middleware is set 36 | """ 37 | if merge_kwargs is None: 38 | merge_kwargs = {} 39 | messages = list(messages) 40 | while len(messages) > 1: 41 | reduced_messages = [] 42 | for i in range(0, len(messages), size): 43 | if i == len(messages) - (size - 1): 44 | reduced_messages.append(messages[i]) 45 | else: 46 | grouped_message = group(messages[i : i + size], cancel_on_error=cancel_on_error) 47 | reduced_messages.append(grouped_message | merge_actor.message(**merge_kwargs)) 48 | messages = reduced_messages 49 | 50 | return messages[0] 51 | -------------------------------------------------------------------------------- /remoulade/middleware/worker_thread_logging.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from ..errors import RateLimitExceeded 4 | from ..logging import get_logger 5 | from .middleware import Middleware 6 | 7 | 8 | class WorkerThreadLogging(Middleware): 9 | def __init__(self): 10 | self.logger = get_logger(__name__, type(self)) 11 | 12 | def build_extra(self, message, max_input_size: int = 1000): 13 | return { 14 | **message.options.get("logging_metadata", {}), 15 | "message_id": message.message_id, 16 | "input": {"args": str(message.args)[:max_input_size], "kwargs": str(message.kwargs)[:max_input_size]}, 17 | } 18 | 19 | def before_actor_execution(self, broker, message): 20 | self.logger.info("Started Actor %s", message, extra=self.build_extra(message)) 21 | 22 | def after_actor_execution(self, broker, message, *, runtime=0): 23 | extra = self.build_extra(message) 24 | extra["runtime"] = runtime 25 | self.logger.info("Finished Actor %s after %.02fms.", message, runtime, extra=extra) 26 | 27 | def before_process_message(self, broker, message): 28 | self.logger.debug( 29 | "Received message %s with id %r.", message, message.message_id, extra=self.build_extra(message) 30 | ) 31 | 32 | def after_process_message(self, broker, message, *, result=None, exception=None): 33 | if exception is None: 34 | return 35 | if isinstance(exception, RateLimitExceeded): 36 | self.logger.warning( 37 | "Rate limit exceeded in message %s: %s.", message, exception, extra=self.build_extra(message) 38 | ) 39 | else: 40 | self.logger.log( 41 | logging.ERROR if message.failed else logging.WARNING, 42 | "Failed to process message %s with unhandled %s", 43 | message, 44 | exception.__class__.__name__, 45 | exc_info=True, 46 | extra=self.build_extra(message, 5000), 47 | ) 48 | 49 | def after_skip_message(self, broker, message): 50 | self.logger.warning("Message %s was skipped.", message, extra=self.build_extra(message)) 51 | 52 | def after_message_canceled(self, broker, message): 53 | self.logger.warning("Message %s has been canceled", message, extra=self.build_extra(message)) 54 | -------------------------------------------------------------------------------- /remoulade/rate_limits/window.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | import time 19 | 20 | from .rate_limiter import RateLimiter 21 | 22 | 23 | class WindowRateLimiter(RateLimiter): 24 | """A rate limiter that ensures that only `limit` operations may 25 | happen over some sliding window. 26 | 27 | Note: 28 | 29 | Windows are in seconds rather that milliseconds. This is 30 | different from most durations and intervals used in Remoulade, 31 | because keeping metadata at the millisecond level is far too 32 | expensive for most use cases. 33 | 34 | Parameters: 35 | backend(RateLimiterBackend): The backend to use. 36 | key(str): The key to rate limit on. 37 | limit(int): The maximum number of operations per window per key. 38 | window(int): The window size in *seconds*. The wider the 39 | window, the more expensive it is to maintain. 40 | """ 41 | 42 | def __init__(self, backend, key, *, limit=1, window=1): 43 | if not limit >= 1: 44 | raise ValueError("limit must be positive") 45 | if not window >= 1: 46 | raise ValueError("window must be positive") 47 | 48 | super().__init__(backend, key) 49 | self.limit = limit 50 | self.window = window 51 | self.window_millis = window * 1000 52 | 53 | def _get_keys(self): 54 | timestamp = int(time.time()) 55 | return [f"{self.key}@{timestamp - i}" for i in range(self.window)] 56 | 57 | def _acquire(self): 58 | keys = self._get_keys() 59 | return self.backend.incr_and_sum(keys[0], self._get_keys, 1, maximum=self.limit, ttl=self.window_millis) 60 | 61 | def _release(self): 62 | pass 63 | -------------------------------------------------------------------------------- /tests/state/test_postgres.py: -------------------------------------------------------------------------------- 1 | from remoulade.state.backends.postgres import DB_VERSION, StateVersion, StoredState 2 | from tests.conftest import check_postgres 3 | 4 | 5 | def test_no_changes(stub_broker, postgres_state_middleware, check_postgres_begin): 6 | backend = postgres_state_middleware.backend 7 | client = backend.client 8 | with client.begin() as session: 9 | session.add(StoredState(message_id="id")) 10 | 11 | backend.init_db() 12 | assert check_postgres(client) 13 | with client.begin() as session: 14 | assert len(session.query(StoredState).all()) == 1 15 | 16 | 17 | def test_create_tables(stub_broker, postgres_state_middleware, check_postgres_begin): 18 | backend = postgres_state_middleware.backend 19 | client = backend.client 20 | with client.begin() as session: 21 | engine = session.get_bind() 22 | StoredState.__table__.drop(bind=engine) 23 | StateVersion.__table__.drop(bind=engine) 24 | 25 | backend.init_db() 26 | assert check_postgres(client) 27 | 28 | 29 | def test_change_version(stub_broker, postgres_state_middleware, check_postgres_begin): 30 | backend = postgres_state_middleware.backend 31 | client = backend.client 32 | with client.begin() as session: 33 | version = session.query(StateVersion).first() 34 | version.version = DB_VERSION + 1 35 | session.add(StoredState(message_id="id")) 36 | 37 | backend.init_db() 38 | assert check_postgres(client) 39 | with client.begin() as session: 40 | assert len(session.query(StoredState).all()) == 0 41 | 42 | 43 | def test_no_version(stub_broker, postgres_state_middleware, check_postgres_begin): 44 | backend = postgres_state_middleware.backend 45 | client = backend.client 46 | with client.begin() as session: 47 | engine = session.get_bind() 48 | StateVersion.__table__.drop(bind=engine) 49 | session.add(StoredState(message_id="id")) 50 | 51 | backend.init_db() 52 | assert check_postgres(client) 53 | with client.begin() as session: 54 | assert len(session.query(StoredState).all()) == 0 55 | 56 | 57 | def test_no_states(stub_broker, postgres_state_middleware, check_postgres_begin): 58 | backend = postgres_state_middleware.backend 59 | client = backend.client 60 | with client.begin() as session: 61 | engine = session.get_bind() 62 | StoredState.__table__.drop(bind=engine) 63 | 64 | backend.init_db() 65 | assert check_postgres(client) 66 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. include:: global.rst 2 | 3 | Remoulade: simple task processing 4 | ================================= 5 | 6 | Release v\ |release|. (:doc:`installation`, :doc:`changelog`, `Source Code`_) 7 | 8 | .. _Source Code: https://github.com/wiremind/remoulade 9 | 10 | .. image:: https://img.shields.io/badge/license-LGPL-blue.svg 11 | :target: license.html 12 | .. image:: https://circleci.com/gh/wiremind/remoulade.svg?style=svg 13 | :target: https://circleci.com/gh/wiremind/remoulade 14 | .. image:: https://badge.fury.io/py/remoulade.svg 15 | :target: https://badge.fury.io/py/remoulade 16 | 17 | **Remoulade** is a distributed task processing library for Python with 18 | a focus on simplicity, reliability and performance. 19 | This is a fork of Dramatiq_. 20 | 21 | Here's what it looks like: 22 | 23 | :: 24 | 25 | import remoulade 26 | import requests 27 | 28 | @remoulade.actor 29 | def count_words(url): 30 | response = requests.get(url) 31 | count = len(response.text.split(" ")) 32 | print(f"There are {count} words at {url!r}.") 33 | 34 | # Synchronously count the words on example.com in the current process 35 | count_words("http://example.com") 36 | 37 | # or send the actor a message so that it may perform the count 38 | # later, in a separate process. 39 | count_words.send("http://example.com") 40 | 41 | **Remoulade** is :doc:`licensed` under the LGPL and it 42 | officially supports Python 3.12 and later. 43 | 44 | 45 | Get It Now 46 | ---------- 47 | 48 | If you want to use it with RabbitMQ_:: 49 | 50 | $ pip install -U 'remoulade[rabbitmq]' 51 | 52 | Or if you want to use it with Redis_:: 53 | 54 | $ pip install -U 'remoulade[redis]' 55 | 56 | Read the :doc:`guide` if you're ready to get started. 57 | 58 | 59 | User Guide 60 | ---------- 61 | 62 | This part of the documentation is focused primarily on teaching you 63 | how to use Remoulade. 64 | 65 | .. toctree:: 66 | :maxdepth: 2 67 | 68 | installation 69 | getting_started 70 | guide 71 | best_practices 72 | advanced 73 | cookbook 74 | 75 | 76 | API Reference 77 | ------------- 78 | 79 | This part of the documentation is focused on detailing the various 80 | bits and pieces of the Remoulade developer interface. 81 | 82 | .. toctree:: 83 | :maxdepth: 2 84 | 85 | reference 86 | 87 | 88 | Project Info 89 | ------------ 90 | 91 | .. toctree:: 92 | :maxdepth: 1 93 | 94 | Source Code 95 | changelog 96 | Contributing 97 | license 98 | -------------------------------------------------------------------------------- /tests/test_concurrent_rate_limiter.py: -------------------------------------------------------------------------------- 1 | import time 2 | from concurrent.futures import ThreadPoolExecutor 3 | 4 | from remoulade.rate_limits import ConcurrentRateLimiter, RateLimitExceeded 5 | 6 | 7 | def test_concurrent_rate_limiter_releases_the_lock_after_each_call(rate_limiter_backend): 8 | # Given that I have a distributed mutex and a call database 9 | mutex = ConcurrentRateLimiter(rate_limiter_backend, "sequential-test", limit=1) 10 | calls = 0 11 | 12 | # And I acquire it multiple times sequentially 13 | for _ in range(8): 14 | with mutex.acquire(raise_on_failure=False) as acquired: 15 | if not acquired: 16 | continue 17 | 18 | calls += 1 19 | 20 | # I expect it to have succeeded that number of times 21 | assert calls == 8 22 | 23 | 24 | def test_concurrent_rate_limiter_can_act_as_a_mutex(rate_limiter_backend): 25 | # Given that I have a distributed mutex and a call database 26 | mutex = ConcurrentRateLimiter(rate_limiter_backend, "concurrent-test", limit=1) 27 | calls = [] 28 | 29 | # And a function that adds calls to the database after acquiring the mutex 30 | def work(): 31 | with mutex.acquire(raise_on_failure=False) as acquired: 32 | if not acquired: 33 | return 34 | 35 | calls.append(1) 36 | time.sleep(0.3) 37 | 38 | # If I execute multiple workers concurrently 39 | with ThreadPoolExecutor(max_workers=8) as e: 40 | futures = [e.submit(work) for _ in range(8)] 41 | 42 | for future in futures: 43 | future.result() 44 | 45 | # I exepct only one call to have succeeded 46 | assert sum(calls) == 1 47 | 48 | 49 | def test_concurrent_rate_limiter_limits_concurrency(rate_limiter_backend): 50 | # Given that I have a distributed rate limiter and a call database 51 | mutex = ConcurrentRateLimiter(rate_limiter_backend, "concurrent-test", limit=4) 52 | calls = [] 53 | 54 | # And a function that adds calls to the database after acquiring the rate limit 55 | def work(): 56 | try: 57 | with mutex.acquire(): 58 | calls.append(1) 59 | time.sleep(0.3) 60 | except RateLimitExceeded: 61 | pass 62 | 63 | # If I execute multiple workers concurrently 64 | with ThreadPoolExecutor(max_workers=32) as e: 65 | futures = [e.submit(work) for _ in range(32)] 66 | 67 | for future in futures: 68 | future.result() 69 | 70 | # I expect at most 4 calls to have succeeded 71 | assert 3 <= sum(calls) <= 4 72 | -------------------------------------------------------------------------------- /tests/state/test_backend.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from remoulade.state import State, StateStatusesEnum 4 | from remoulade.state.backends import PostgresBackend 5 | 6 | 7 | class TestStateBackend: 8 | """This class test the different methods of state 9 | backend""" 10 | 11 | def test_change_existence_state(self, state_backend): 12 | message_id = "14271" 13 | state = State( 14 | message_id, StateStatusesEnum.Pending, actor_name="do_work", args=[1, 2], kwargs={"status": "work"} 15 | ) 16 | state_backend.set_state(state, ttl=100) 17 | # the state status changes 18 | state = State(message_id, StateStatusesEnum.Success) 19 | state_backend.set_state(state, ttl=100) 20 | # if the states exist, then it update the new fields 21 | # then args and kwargs should hold 22 | # and the state status should change 23 | state = state_backend.get_state(message_id) 24 | assert state.status == StateStatusesEnum.Success 25 | assert state.args == [1, 2] 26 | assert state.kwargs == {"status": "work"} 27 | 28 | def test_count_messages(self, stub_broker, state_middleware): 29 | backend = state_middleware.backend 30 | for i in range(3): 31 | backend.set_state(State(f"id{i}")) 32 | 33 | assert backend.get_states_count() == 3 34 | 35 | def test_count_compositions(self, stub_broker, state_middleware): 36 | if not isinstance(state_middleware.backend, PostgresBackend): 37 | pytest.skip() 38 | 39 | backend = state_middleware.backend 40 | 41 | for i in range(3): 42 | for j in range(2): 43 | backend.set_state(State(f"id{i * j}", composition_id=f"id{j}")) 44 | 45 | assert backend.get_states_count() == 2 46 | 47 | def test_sort_with_offset(self, stub_broker, state_middleware): 48 | if not isinstance(state_middleware.backend, PostgresBackend): 49 | pytest.skip() 50 | backend = state_middleware.backend 51 | for i in range(8): 52 | backend.set_state(State(f"id{i}", actor_name=f"{3 + 4 * (i // 4) - i % 4}")) 53 | 54 | res = backend.get_states(size=3, sort_column="actor_name", sort_direction="desc") 55 | assert res[0].actor_name == "7" 56 | assert res[1].actor_name == "6" 57 | assert res[2].actor_name == "5" 58 | res = backend.get_states(size=3, sort_column="actor_name", sort_direction="asc") 59 | assert res[0].actor_name == "0" 60 | assert res[1].actor_name == "1" 61 | assert res[2].actor_name == "2" 62 | -------------------------------------------------------------------------------- /tests/test_stub_broker.py: -------------------------------------------------------------------------------- 1 | import time 2 | from unittest.mock import Mock 3 | 4 | import pytest 5 | 6 | import remoulade 7 | from remoulade import QueueJoinTimeout, QueueNotFound 8 | 9 | 10 | def test_stub_broker_raises_queue_error_when_consuming_undeclared_queues(stub_broker): 11 | # Given that I have a stub broker 12 | # If I attempt to consume a queue that wasn't declared 13 | # I expect a QueueNotFound error to be raised 14 | with pytest.raises(QueueNotFound): 15 | stub_broker.consume("idontexist") 16 | 17 | 18 | def test_stub_broker_raises_queue_error_when_enqueueing_messages_on_undeclared_queues(stub_broker): 19 | # Given that I have a stub broker 20 | # If I attempt to enqueue a message on a queue that wasn't declared 21 | # I expect a QueueNotFound error to be raised 22 | with pytest.raises(QueueNotFound): 23 | stub_broker.enqueue(Mock(queue_name="idontexist")) 24 | 25 | 26 | def test_stub_broker_raises_queue_error_when_joining_on_undeclared_queues(stub_broker): 27 | # Given that I have a stub broker 28 | # If I attempt to join on a queue that wasn't declared 29 | # I expect a QueueNotFound error to be raised 30 | with pytest.raises(QueueNotFound): 31 | stub_broker.join("idontexist") 32 | 33 | 34 | def test_stub_broker_can_be_flushed(stub_broker): 35 | # Given that I have an actor 36 | @remoulade.actor 37 | def do_work(): 38 | pass 39 | 40 | # And this actor is declared 41 | stub_broker.declare_actor(do_work) 42 | 43 | # When I send that actor a message 44 | do_work.send() 45 | 46 | # Then its queue should contain a message 47 | assert stub_broker.queues[do_work.queue_name].qsize() == 1 48 | 49 | # When I flush all the queues in the broker 50 | stub_broker.flush_all() 51 | 52 | # Then the queue should be empty and it should contain no in-progress tasks 53 | assert stub_broker.queues[do_work.queue_name].qsize() == 0 54 | assert stub_broker.queues[do_work.queue_name].unfinished_tasks == 0 55 | 56 | 57 | def test_stub_broker_can_join_with_timeout(stub_broker, stub_worker): 58 | # Given that I have an actor that takes a long time to run 59 | @remoulade.actor 60 | def do_work(): 61 | time.sleep(0.5) 62 | 63 | # And this actor is declared 64 | stub_broker.declare_actor(do_work) 65 | 66 | # When I send that actor a message 67 | do_work.send() 68 | 69 | # And join on its queue with a timeout 70 | # Then I expect a QueueJoinTimeout to be raised 71 | with pytest.raises(QueueJoinTimeout): 72 | stub_broker.join(do_work.queue_name, timeout=200) 73 | -------------------------------------------------------------------------------- /tests/test_channel_pool.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from remoulade.brokers.rabbitmq import ChannelPool 4 | from remoulade.errors import ChannelPoolTimeout 5 | 6 | 7 | def test_channel_pool_acquire(mock_channel_factory): 8 | channel_pool = ChannelPool(channel_factory=mock_channel_factory, pool_size=10) 9 | 10 | with channel_pool.acquire() as channel_1: 11 | assert channel_1.id == 0 12 | assert len(channel_pool) == 9 13 | 14 | with channel_pool.acquire() as channel_2: 15 | assert channel_2.id == 1 16 | assert len(channel_pool) == 8 17 | 18 | assert len(channel_pool) == 10 19 | items_in_pool = [] 20 | while len(channel_pool): 21 | channel = channel_pool.get() 22 | if channel is not None: 23 | items_in_pool.append(channel.id) 24 | else: 25 | items_in_pool.append(channel) 26 | 27 | assert items_in_pool == [0, 1, None, None, None, None, None, None, None, None] 28 | 29 | 30 | def test_channel_pool_acquire_one_used(mock_channel_factory): 31 | channel_pool = ChannelPool(channel_factory=mock_channel_factory, pool_size=10) 32 | 33 | for _ in range(10): 34 | with channel_pool.acquire() as channel_1: 35 | assert channel_1.id == 0 36 | assert len(channel_pool) == 9 37 | 38 | assert len(channel_pool) == 10 39 | items_in_pool = [] 40 | while len(channel_pool): 41 | channel = channel_pool.get() 42 | if channel is not None: 43 | items_in_pool.append(channel.id) 44 | else: 45 | items_in_pool.append(channel) 46 | 47 | assert items_in_pool == [0, None, None, None, None, None, None, None, None, None] 48 | 49 | 50 | def test_raise_channel_pool_timeout(mock_channel_factory): 51 | channel_pool = ChannelPool(channel_factory=mock_channel_factory, pool_size=1) 52 | with channel_pool.acquire(): 53 | assert len(channel_pool) == 0 54 | with pytest.raises(ChannelPoolTimeout), channel_pool.acquire(timeout=1): 55 | pass 56 | 57 | 58 | def test_channel_pool_clear(mock_channel_factory): 59 | channel_pool = ChannelPool(channel_factory=mock_channel_factory, pool_size=5) 60 | with ( 61 | channel_pool.acquire(), 62 | channel_pool.acquire(), 63 | channel_pool.acquire(), 64 | channel_pool.acquire(), 65 | channel_pool.acquire(), 66 | ): 67 | pass 68 | 69 | channel_pool.clear() 70 | items_in_pool = [] 71 | while len(channel_pool): 72 | items_in_pool.append(channel_pool.get()) 73 | 74 | assert items_in_pool == [None, None, None, None, None] 75 | -------------------------------------------------------------------------------- /remoulade/middleware/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | import platform 19 | 20 | from .age_limit import AgeLimit, AgeLimitException 21 | from .catch_error import CatchError 22 | from .current_message import CurrentMessage 23 | from .heartbeat import Heartbeat 24 | from .logging_metadata import LoggingMetadata 25 | from .max_memory import MaxMemory 26 | from .max_tasks import MaxTasks 27 | from .middleware import Middleware, MiddlewareError, SkipMessage 28 | from .pipelines import Pipelines 29 | from .retries import Retries 30 | from .shutdown import Shutdown, ShutdownNotifications 31 | from .threading import Interrupt, raise_thread_exception 32 | from .time_limit import TimeLimit, TimeLimitExceeded 33 | from .worker_thread_logging import WorkerThreadLogging 34 | 35 | CURRENT_OS = platform.system() 36 | 37 | if CURRENT_OS != "Windows": 38 | from .prometheus import Prometheus # noqa: F401 39 | 40 | __all__ = [ 41 | # Middlewares 42 | "AgeLimit", 43 | "AgeLimitException", 44 | "CatchError", 45 | "CurrentMessage", 46 | "Heartbeat", 47 | # Threading 48 | "Interrupt", 49 | "LoggingMetadata", 50 | "MaxMemory", 51 | "MaxTasks", 52 | # Basics 53 | "Middleware", 54 | "MiddlewareError", 55 | "Pipelines", 56 | "Retries", 57 | "Shutdown", 58 | "ShutdownNotifications", 59 | "SkipMessage", 60 | "TimeLimit", 61 | "TimeLimitExceeded", 62 | "WorkerThreadLogging", 63 | "default_middleware", 64 | "raise_thread_exception", 65 | ] 66 | 67 | if CURRENT_OS != "Windows": 68 | __all__.append("Prometheus") 69 | 70 | #: The list of middleware that are enabled by default. 71 | default_middleware = [ 72 | WorkerThreadLogging, 73 | AgeLimit, 74 | TimeLimit, 75 | ShutdownNotifications, 76 | Pipelines, 77 | Retries, 78 | CatchError, 79 | CurrentMessage, 80 | ] 81 | -------------------------------------------------------------------------------- /remoulade/results/backends/stub.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | import contextlib 18 | import time 19 | from typing import Any, ClassVar 20 | 21 | from ..backend import ForgottenResult, Missing, ResultBackend 22 | 23 | 24 | class StubBackend(ResultBackend): 25 | """An in-memory result backend. For use in unit tests. 26 | 27 | Parameters: 28 | namespace(str): A string with which to prefix result keys. 29 | encoder(Encoder): The encoder to use when storing and retrieving 30 | result data. Defaults to :class:`.JSONEncoder`. 31 | """ 32 | 33 | results: ClassVar[dict[str, Any]] = {} 34 | 35 | def _get(self, message_key: str, forget: bool = False): 36 | if forget: 37 | data, expiration = self.results.get(message_key, (None, None)) 38 | if data is not None: 39 | self.results[message_key] = self.encoder.encode(ForgottenResult.asdict()), expiration 40 | else: 41 | data, expiration = self.results.get(message_key, (None, None)) 42 | 43 | if data is not None and time.monotonic() < expiration: 44 | return self.encoder.decode(data) 45 | return Missing 46 | 47 | def _store(self, message_keys, results, ttl): 48 | for message_key, result in zip(message_keys, results, strict=False): 49 | result_data = self.encoder.encode(result) 50 | expiration = time.monotonic() + int(ttl / 1000) 51 | self.results[message_key] = (result_data, expiration) 52 | 53 | def _delete(self, key: str): 54 | with contextlib.suppress(KeyError): 55 | del self.results[key] 56 | 57 | def increment_group_completion(self, group_id: str, message_id: str, ttl: int) -> int: 58 | group_completion_key = self.build_group_completion_key(group_id) 59 | completed = self.results.get(group_completion_key, set()) | {message_id} 60 | self.results[group_completion_key] = completed 61 | return len(completed) 62 | -------------------------------------------------------------------------------- /tests/test_pidfile.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import os 3 | import signal 4 | import time 5 | 6 | import remoulade 7 | from remoulade.brokers.stub import StubBroker 8 | 9 | broker = StubBroker() 10 | remoulade.set_broker(broker) 11 | 12 | 13 | def remove(filename): 14 | with contextlib.suppress(OSError): 15 | os.remove(filename) 16 | 17 | 18 | def test_cli_scrubs_stale_pid_files(start_cli): 19 | try: 20 | # Given that I have an existing file containing an old pid 21 | filename = "test_scrub.pid" 22 | with open(filename, "w") as f: 23 | f.write("999999") 24 | 25 | # When I try to start the cli and pass that file as a PID file 26 | proc = start_cli("tests.test_pidfile", extra_args=["--pid-file", filename]) 27 | 28 | # And I wait for it to write the pid file 29 | time.sleep(1) 30 | 31 | # Then the process should write its pid to the file 32 | with open(filename) as f: 33 | pid = int(f.read()) 34 | 35 | assert pid == proc.pid 36 | 37 | # When I stop the process 38 | proc.terminate() 39 | proc.wait() 40 | 41 | # Then the process should exit with return code 0 42 | assert proc.returncode == 0 43 | 44 | # And the file should be removed 45 | assert not os.path.exists(filename) 46 | finally: 47 | remove(filename) 48 | 49 | 50 | def test_cli_aborts_when_pidfile_contains_garbage(start_cli): 51 | try: 52 | # Given that I have an existing file containing important information 53 | filename = "test_garbage.pid" 54 | with open(filename, "w") as f: 55 | f.write("important!") 56 | 57 | # When I try to start the cli and pass that file as a PID file 58 | proc = start_cli("tests.test_pidfile:broker", extra_args=["--pid-file", filename]) 59 | proc.wait() 60 | 61 | # Then the process should exit with return code 4 62 | assert proc.returncode == 4 63 | finally: 64 | remove(filename) 65 | 66 | 67 | def test_cli_with_pidfile_can_be_reloaded(start_cli): 68 | try: 69 | # Given that I have a PID file 70 | filename = "test_reload.pid" 71 | 72 | # When I try to start the cli and pass that file as a PID file 73 | proc = start_cli("tests.test_pidfile", extra_args=["--pid-file", filename]) 74 | time.sleep(1) 75 | 76 | # And send the proc a HUP signal 77 | proc.send_signal(signal.SIGHUP) 78 | time.sleep(3) 79 | 80 | # And then terminate the process 81 | proc.terminate() 82 | proc.wait() 83 | 84 | # Then the process should exit with return code 0 85 | assert proc.returncode == 0 86 | finally: 87 | remove(filename) 88 | -------------------------------------------------------------------------------- /remoulade/helpers/queues.py: -------------------------------------------------------------------------------- 1 | from remoulade import QueueJoinTimeout 2 | from remoulade.common import current_millis 3 | 4 | 5 | def iter_queue(queue): 6 | """Iterate over the messages on a queue until it's empty. Does 7 | not block while waiting for messages. 8 | 9 | Parameters: 10 | queue(Queue): The queue to iterate over. 11 | 12 | Returns: 13 | generator[object]: The iterator. 14 | """ 15 | while not queue.empty(): 16 | yield queue.get_nowait() 17 | 18 | 19 | def join_queue(queue, timeout=None): 20 | """The join() method of standard queues in Python doesn't support 21 | timeouts. This implements the same functionality as that method, 22 | with optional timeout support, by depending the internals of 23 | Queue. 24 | 25 | Raises: 26 | QueueJoinTimeout: When the timeout is reached. 27 | 28 | Parameters: 29 | timeout(Optional[float]) 30 | """ 31 | with queue.all_tasks_done: 32 | while queue.unfinished_tasks: 33 | finished_in_time = queue.all_tasks_done.wait(timeout=timeout) 34 | if not finished_in_time: 35 | raise QueueJoinTimeout(f"timed out after {timeout:.2f} seconds") 36 | 37 | 38 | def join_all(joinables, timeout): 39 | """Wait on a list of objects that can be joined with a total 40 | timeout represented by ``timeout``. 41 | 42 | Parameters: 43 | joinables(object): Objects with a join method. 44 | timeout(int): The total timeout in milliseconds. 45 | """ 46 | started, elapsed = current_millis(), 0 47 | for ob in joinables: 48 | ob.join(timeout=timeout / 1000) 49 | elapsed = current_millis() - started 50 | timeout = max(0, timeout - elapsed) 51 | 52 | 53 | def q_name(queue_name): 54 | """Returns the canonical queue name for a given queue.""" 55 | if queue_name.endswith(".DQ") or queue_name.endswith(".XQ"): 56 | return queue_name[:-3] 57 | return queue_name 58 | 59 | 60 | def dq_name(queue_name): 61 | """Returns the delayed queue name for a given queue. If the given 62 | queue name already belongs to a delayed queue, then it is returned 63 | unchanged. 64 | """ 65 | if queue_name.endswith(".DQ"): 66 | return queue_name 67 | 68 | if queue_name.endswith(".XQ"): 69 | queue_name = queue_name[:-3] 70 | return queue_name + ".DQ" 71 | 72 | 73 | def xq_name(queue_name): 74 | """Returns the dead letter queue name for a given queue. If the 75 | given queue name belongs to a delayed queue, the dead letter queue 76 | name for the original queue is generated. 77 | """ 78 | if queue_name.endswith(".XQ"): 79 | return queue_name 80 | 81 | if queue_name.endswith(".DQ"): 82 | queue_name = queue_name[:-3] 83 | return queue_name + ".XQ" 84 | -------------------------------------------------------------------------------- /remoulade/rate_limits/rate_limiter.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | from contextlib import contextmanager 19 | 20 | from ..errors import RateLimitExceeded 21 | 22 | 23 | class RateLimiter: 24 | """ABC for rate limiters. 25 | 26 | Examples: 27 | 28 | >>> from remoulade.rate_limits.backends import RedisBackend 29 | 30 | >>> backend = RedisBackend() 31 | >>> limiter = ConcurrentRateLimiter(backend, "distributed-mutex", limit=1) 32 | 33 | >>> with limiter.acquire(raise_on_failure=False) as acquired: 34 | ... if not acquired: 35 | ... print("Mutex not acquired.") 36 | ... return 37 | ... 38 | ... print("Mutex acquired.") 39 | 40 | Parameters: 41 | backend(RateLimiterBackend): The rate limiting backend to use. 42 | key(str): The key to rate limit on. 43 | """ 44 | 45 | def __init__(self, backend, key): 46 | self.backend = backend 47 | self.key = key 48 | 49 | def _acquire(self): # pragma: no cover 50 | raise NotImplementedError 51 | 52 | def _release(self): # pragma: no cover 53 | raise NotImplementedError 54 | 55 | @contextmanager 56 | def acquire(self, *, raise_on_failure=True): 57 | """Attempt to acquire a slot under this rate limiter. 58 | 59 | Parameters: 60 | raise_on_failure(bool): Whether or not failures should raise an 61 | exception. If this is false, the context manager will instead 62 | return a boolean value representing whether or not the rate 63 | limit slot was acquired. 64 | 65 | Returns: 66 | bool: Whether or not the slot could be acquired. 67 | """ 68 | acquired = False 69 | 70 | try: 71 | acquired = self._acquire() 72 | if raise_on_failure and not acquired: 73 | raise RateLimitExceeded("rate limit exceeded for key {key!r}".format(**vars(self))) 74 | 75 | yield acquired 76 | finally: 77 | if acquired: 78 | self._release() 79 | -------------------------------------------------------------------------------- /remoulade/__init__.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | import importlib.metadata 19 | 20 | from .actor import Actor, actor 21 | from .broker import Broker, Consumer, MessageProxy, change_broker, declare_actors, get_broker, set_broker 22 | from .collection_results import CollectionResults 23 | from .composition import group, pipeline 24 | from .encoder import Encoder, JSONEncoder, PickleEncoder 25 | from .errors import ( 26 | ActorNotFound, 27 | BrokerError, 28 | ChannelPoolTimeout, 29 | ConnectionClosed, 30 | ConnectionError, # noqa: A004 31 | ConnectionFailed, 32 | NoResultBackend, 33 | QueueJoinTimeout, 34 | QueueNotFound, 35 | RateLimitExceeded, 36 | RemouladeError, 37 | ) 38 | from .generic import GenericActor 39 | from .logging import get_logger 40 | from .message import Message, get_encoder, set_encoder 41 | from .middleware import Middleware 42 | from .result import Result 43 | from .utils import get_scheduler, set_scheduler 44 | from .worker import Worker 45 | 46 | __all__ = [ 47 | # Actors 48 | "Actor", 49 | "ActorNotFound", 50 | # Brokers 51 | "Broker", 52 | "BrokerError", 53 | "ChannelPoolTimeout", 54 | "CollectionResults", 55 | "ConnectionClosed", 56 | "ConnectionError", 57 | "ConnectionFailed", 58 | "Consumer", 59 | # Encoding 60 | "Encoder", 61 | "GenericActor", 62 | "JSONEncoder", 63 | # Messages 64 | "Message", 65 | "MessageProxy", 66 | # Middleware 67 | "Middleware", 68 | "NoResultBackend", 69 | "PickleEncoder", 70 | "QueueJoinTimeout", 71 | "QueueNotFound", 72 | "RateLimitExceeded", 73 | # Errors 74 | "RemouladeError", 75 | "Result", 76 | # Workers 77 | "Worker", 78 | "actor", 79 | "change_broker", 80 | "declare_actors", 81 | "get_broker", 82 | "get_encoder", 83 | # Logging 84 | "get_logger", 85 | # Scheduler 86 | "get_scheduler", 87 | # Composition 88 | "group", 89 | "pipeline", 90 | "set_broker", 91 | "set_encoder", 92 | "set_scheduler", 93 | ] 94 | 95 | __version__ = importlib.metadata.version(__package__) 96 | -------------------------------------------------------------------------------- /tests/benchmarks/test_rabbitmq_cli.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import time 4 | 5 | import pytest 6 | 7 | import remoulade 8 | 9 | 10 | @remoulade.actor(queue_name="benchmark-throughput") 11 | def throughput(): 12 | pass 13 | 14 | 15 | @remoulade.actor(queue_name="benchmark-fib") 16 | def fib(n): 17 | x, y = 1, 1 18 | while n > 2: 19 | x, y = x + y, x 20 | n -= 1 21 | return x 22 | 23 | 24 | @remoulade.actor(queue_name="benchmark-latency") 25 | def latency(): 26 | p = random.randint(1, 100) 27 | if p == 1: 28 | durations = [3, 3, 3, 1] 29 | elif p <= 10: 30 | durations = [2, 3] 31 | elif p <= 40: 32 | durations = [1, 2] 33 | else: 34 | durations = [1] 35 | 36 | for duration in durations: 37 | time.sleep(duration) 38 | 39 | 40 | @pytest.mark.skipif(os.getenv("CI") == "true", reason="test skipped on CI") 41 | @pytest.mark.benchmark(group="rabbitmq-100k-throughput") 42 | def test_rabbitmq_process_100k_messages_with_cli(benchmark, info_logging, start_cli, rabbitmq_broker): 43 | remoulade.declare_actors([throughput]) 44 | 45 | # Given that I've loaded 100k messages into RabbitMQ 46 | def setup(): 47 | for _ in range(100000): 48 | throughput.send() 49 | 50 | start_cli("tests.benchmarks.test_rabbitmq_cli") 51 | 52 | # I expect processing those messages with the CLI to be consistently fast 53 | benchmark.pedantic(rabbitmq_broker.join, args=(throughput.queue_name,), setup=setup) 54 | 55 | 56 | @pytest.mark.skipif(os.getenv("CI") == "true", reason="test skipped on CI") 57 | @pytest.mark.benchmark(group="rabbitmq-10k-fib") 58 | def test_rabbitmq_process_10k_fib_with_cli(benchmark, info_logging, start_cli, rabbitmq_broker): 59 | remoulade.declare_actors([fib]) 60 | 61 | # Given that I've loaded 10k messages into RabbitMQ 62 | def setup(): 63 | for _ in range(10000): 64 | fib.send(random.choice([1, 512, 1024, 2048, 4096, 8192])) 65 | 66 | start_cli("tests.benchmarks.test_rabbitmq_cli") 67 | 68 | # I expect processing those messages with the CLI to be consistently fast 69 | benchmark.pedantic(rabbitmq_broker.join, args=(fib.queue_name,), setup=setup) 70 | 71 | 72 | @pytest.mark.skipif(os.getenv("CI") == "true", reason="test skipped on CI") 73 | @pytest.mark.benchmark(group="rabbitmq-1k-latency") 74 | def test_rabbitmq_process_1k_latency_with_cli(benchmark, info_logging, start_cli, rabbitmq_broker): 75 | remoulade.declare_actors([latency]) 76 | 77 | # Given that I've loaded 1k messages into RabbitMQ 78 | def setup(): 79 | for _ in range(1000): 80 | latency.send() 81 | 82 | start_cli("tests.benchmarks.test_rabbitmq_cli") 83 | 84 | # I expect processing those messages with the CLI to be consistently fast 85 | benchmark.pedantic(rabbitmq_broker.join, args=(latency.queue_name,), setup=setup) 86 | -------------------------------------------------------------------------------- /remoulade/state/middleware.py: -------------------------------------------------------------------------------- 1 | from datetime import UTC, datetime 2 | 3 | from ..middleware import Middleware 4 | from .backend import State, StateStatusesEnum 5 | 6 | 7 | class MessageState(Middleware): 8 | """Middleware use to storage and update the state 9 | of the messages. 10 | Parameters 11 | state_ttl(int):Time(seconds) that the state will be storage 12 | in the database 13 | """ 14 | 15 | def __init__(self, backend, state_ttl=3600): 16 | self.backend = backend 17 | self.state_ttl = state_ttl 18 | 19 | def save(self, message, status, priority=None, **kwargs): 20 | if self.state_ttl is None or self.state_ttl <= 0: 21 | return 22 | args = message.args 23 | options = message.options 24 | kwargs_state = message.kwargs 25 | message_id = message.message_id 26 | actor_name = message.actor_name 27 | queue_name = message.queue_name 28 | self.backend.set_state( 29 | State( 30 | message_id, 31 | status, 32 | actor_name=actor_name, 33 | args=args, 34 | priority=priority, 35 | kwargs=kwargs_state, 36 | options=options, 37 | queue_name=queue_name, 38 | **kwargs, 39 | ), 40 | self.state_ttl, 41 | ) 42 | 43 | def _get_current_time(self): 44 | return datetime.now(UTC) 45 | 46 | def before_enqueue(self, broker, message, delay): 47 | priority = broker.get_actor(message.actor_name).priority 48 | composition_id = self.get_option("composition_id", broker=broker, message=message) 49 | self.save( 50 | message, 51 | status=StateStatusesEnum.Pending, 52 | enqueued_datetime=self._get_current_time(), 53 | priority=priority, 54 | composition_id=composition_id, 55 | ) 56 | 57 | def after_enqueue(self, broker, message, delay, exception=None): 58 | if exception is not None: 59 | self.save(message, status=StateStatusesEnum.Failure, end_datetime=self._get_current_time()) 60 | 61 | def after_skip_message(self, broker, message): 62 | self.save(message, status=StateStatusesEnum.Skipped) 63 | 64 | def after_message_canceled(self, broker, message): 65 | self.save(message, status=StateStatusesEnum.Canceled) 66 | 67 | def after_process_message(self, broker, message, *, result=None, exception=None): 68 | self.save( 69 | message, 70 | status=StateStatusesEnum.Success if exception is None else StateStatusesEnum.Failure, 71 | end_datetime=self._get_current_time(), 72 | ) 73 | 74 | def before_process_message(self, broker, message): 75 | self.save(message, status=StateStatusesEnum.Started, started_datetime=self._get_current_time()) 76 | -------------------------------------------------------------------------------- /remoulade/rate_limits/bucket.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | import time 19 | 20 | from .rate_limiter import RateLimiter 21 | 22 | 23 | class BucketRateLimiter(RateLimiter): 24 | """A rate limiter that ensures that only up to `limit` operations 25 | may happen over some time interval. 26 | 27 | Examples: 28 | 29 | Up to 10 operations every second: 30 | 31 | >>> BucketRateLimiter(backend, "some-key", limit=10, bucket=1_000) 32 | 33 | Up to 1 operation every minute: 34 | 35 | >>> BucketRateLimiter(backend, "some-key", limit=1, bucket=60_000) 36 | 37 | Warning: 38 | 39 | Bucket rate limits are cheap to maintain but are susceptible to 40 | burst "attacks". Given a bucket rate limit of 100 per minute, 41 | an attacker could make a burst of 100 calls in the last second 42 | of a minute and then another 100 calls in the first second of 43 | the subsequent minute. 44 | 45 | For a rate limiter that doesn't have this problem (but is more 46 | expensive to maintain), see |WindowRateLimiter|. 47 | 48 | Parameters: 49 | backend(RateLimiterBackend): The backend to use. 50 | key(str): The key to rate limit on. 51 | limit(int): The maximum number of operations per bucket per key. 52 | bucket(int): The bucket interval in milliseconds. 53 | 54 | .. |WindowRateLimiter| replace:: :class:`WindowRateLimiter` 55 | """ 56 | 57 | def __init__(self, backend, key, *, limit=1, bucket=1000): 58 | if not limit >= 1: 59 | raise ValueError("limit must be positive") 60 | 61 | super().__init__(backend, key) 62 | self.limit = limit 63 | self.bucket = bucket 64 | 65 | def _acquire(self): 66 | timestamp = int(time.time() * 1000) 67 | current_timestamp = timestamp - (timestamp % self.bucket) 68 | current_key = f"{self.key}@{current_timestamp}" 69 | added = self.backend.add(current_key, 1, ttl=self.bucket) 70 | if added: 71 | return True 72 | 73 | return self.backend.incr(current_key, 1, maximum=self.limit, ttl=self.bucket) 74 | 75 | def _release(self): 76 | pass 77 | -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | # Contributors 2 | 3 | This file lists the contributors to the Remoulade project. 4 | 5 | By adding your name to the list below you disavow any rights or claims 6 | to any changes submitted to the Remoulade project and assign copyright 7 | of those changes to CLEARTYPE SRL. 8 | 9 | | Username | Name | 10 | |:-----------------------------------------------------------------|:-----------------------| 11 | | [@bendemaree](https://github.com/bendemaree) | Ben Demaree | 12 | | [@whalesalad](https://github.com/whalesalad) | Michael Whalen | 13 | | [@rakanalh](https://github.com/rakanalh) | Rakan Alhneiti | 14 | | [@jssuzanne](https://github.com/jssuzanne) | Jean-Sébastien Suzanne | 15 | | [@chen2aaron](https://github.com/chen2aaron) | xixijun | 16 | | [@aequitas](https://github.com/aequitas) | Johan Bloemberg | 17 | | [@najamansari](https://github.com/najamansari) | Najam Ahmed Ansari | 18 | | [@rpkilby](https://github.com/rpkilby) | Ryan P Kilby | 19 | | [@2miksyn](https://github.com/2miksyn) | Mikhail Smirnov | 20 | | [@gdvalle](https://github.com/gdvalle) | Greg Dallavalle | 21 | | [@viiicky](https://github.com/viiicky) | Vikas Prasad | 22 | | [@xdmiodz](https://github.com/xdmiodz) | Dmitry Odzerikho | 23 | | [@ryansm1](https://github.com/ryansm1) | Ryan Smith | 24 | | [@aericson](https://github.com/aericson) | André Ericson | 25 | | [@xdanielsb](https://github.com/xdanielsb) | Daniel Santos | 26 | | [@simondosda](https://github.com/simondosda) | Simon Dosda | 27 | | [@ben-natan](https://github.com/ben-natan) | Adrien Bennatan | 28 | | [@Corentin-Br](https://github.com/Corentin-Br) | Corentin Bravo | 29 | | [@antoinerabany](https://github.com/antoinerabany) | Antoine Rabany | 30 | | [@thomasLeMeur](https://github.com/thomasLeMeur) | Thomas Le Meur | 31 | | [@fregogui](https://github.com/fregogui) | Guillaume Fregosi | 32 | | [@pgitips](https://github.com/pgitips) | Pierre Giraud | 33 | | [@williampollet](https://github.com/williampollet) | William Pollet | 34 | | [@mehdithez](https://github.com/mehdithez) | Zeroual Mehdi | 35 | | [@julien-duponchelle](https://github.com/julien-duponchelle) | Julien Duponchelle | 36 | | [@alisterd51](https://github.com/alisterd51) | Antoine Clarman | 37 | -------------------------------------------------------------------------------- /remoulade/middleware/threading.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | import ctypes 19 | import inspect 20 | import platform 21 | 22 | from ..logging import get_logger 23 | 24 | __all__ = ["Interrupt", "raise_thread_exception"] 25 | 26 | 27 | logger = get_logger(__name__) 28 | 29 | current_platform = platform.python_implementation() 30 | supported_platforms = {"CPython"} 31 | 32 | 33 | class Interrupt(BaseException): 34 | """Base class for exceptions used to asynchronously interrupt a 35 | thread's execution. An actor may catch these exceptions in order 36 | to respond gracefully, such as performing any necessary cleanup. 37 | 38 | This is *not* a subclass of ``RemouladeError`` to avoid it being 39 | caught unintentionally. 40 | """ 41 | 42 | 43 | def raise_thread_exception(thread_id, exception): 44 | """Raise an exception in a thread. 45 | 46 | Currently, this is only available on CPython. 47 | 48 | Note: 49 | This works by setting an async exception in the thread. This means 50 | that the exception will only get called the next time that thread 51 | acquires the GIL. Concretely, this means that this middleware can't 52 | cancel system calls. 53 | """ 54 | if current_platform == "CPython": 55 | _raise_thread_exception_cpython(thread_id, exception) 56 | else: 57 | message = "Setting thread exceptions (%s) is not supported for your current platform (%r)." 58 | exctype = (exception if inspect.isclass(exception) else type(exception)).__name__ 59 | logger.critical(message, exctype, current_platform) 60 | 61 | 62 | def _raise_thread_exception_cpython(thread_id, exception): 63 | exctype = (exception if inspect.isclass(exception) else type(exception)).__name__ 64 | thread_id = ctypes.c_long(thread_id) 65 | exception = ctypes.py_object(exception) 66 | count = ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id, exception) 67 | if count == 0: 68 | logger.critical("Failed to set exception (%s) in thread %r.", exctype, thread_id.value) 69 | elif count > 1: # pragma: no cover 70 | logger.critical("Exception (%s) was set in multiple threads. Undoing...", exctype) 71 | ctypes.pythonapi.PyThreadState_SetAsyncExc(thread_id, ctypes.c_long(0)) 72 | -------------------------------------------------------------------------------- /remoulade/rate_limits/backends/stub.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | import time 19 | from collections.abc import Callable 20 | from threading import Lock 21 | 22 | from ..backend import RateLimiterBackend 23 | 24 | 25 | class StubBackend(RateLimiterBackend): 26 | """An in-memory rate limiter backend. For use in unit tests.""" 27 | 28 | def __init__(self): 29 | self.mutex = Lock() 30 | self.db: dict[str, tuple[int, float]] = {} 31 | 32 | def add(self, key: str, value: int, ttl: int) -> bool: 33 | with self.mutex: 34 | res = self._get(key) 35 | if res is not None: 36 | return False 37 | 38 | return self._put(key, value, ttl) 39 | 40 | def incr(self, key: str, amount: int, maximum: int, ttl: int) -> bool: 41 | with self.mutex: 42 | value = self._get(key, default=0) + amount 43 | if value > maximum: 44 | return False 45 | 46 | return self._put(key, value, ttl) 47 | 48 | def decr(self, key: str, amount: int, minimum: int, ttl: int) -> bool: 49 | with self.mutex: 50 | value = self._get(key, default=0) - amount 51 | if value < minimum: 52 | return False 53 | 54 | return self._put(key, value, ttl) 55 | 56 | def incr_and_sum(self, key: str, keys: Callable[[], list[str]], amount: int, maximum: int, ttl: int) -> bool: 57 | self.add(key, 0, ttl) 58 | with self.mutex: 59 | value = self._get(key, default=0) + amount 60 | if value > maximum: 61 | return False 62 | 63 | # TODO: Drop non-callable keys in Remoulade v2. 64 | key_list = keys() if callable(keys) else keys 65 | values = sum(self._get(k, default=0) for k in key_list) 66 | total = amount + values 67 | if total > maximum: 68 | return False 69 | 70 | return self._put(key, value, ttl) 71 | 72 | def _get(self, key: str, *, default: int | None = None) -> int | None: 73 | value, expiration = self.db.get(key, (None, None)) 74 | if expiration and time.monotonic() < expiration: 75 | return value 76 | return default 77 | 78 | def _put(self, key: str, value: int, ttl: int) -> bool: 79 | self.db[key] = (value, time.monotonic() + ttl / 1000) 80 | return True 81 | -------------------------------------------------------------------------------- /remoulade/middleware/logging_metadata.py: -------------------------------------------------------------------------------- 1 | from collections.abc import Callable 2 | 3 | from ..logging import get_logger 4 | from .middleware import Middleware 5 | 6 | 7 | class LoggingMetadata(Middleware): 8 | """Middleware that lets you add logging_metadata to messages. 9 | 10 | Parameters: 11 | logging_metadata(dict): The metadata to add to messages 12 | logging_metadata_getter(Callable[[], dict]): A callback that returns 13 | """ 14 | 15 | def __init__( 16 | self, *, logging_metadata: dict | None = None, logging_metadata_getter: Callable[[], dict] | None = None 17 | ): 18 | self.logger = get_logger(__name__, type(self)) 19 | self.logging_metadata = logging_metadata 20 | self.logging_metadata_getter = logging_metadata_getter 21 | 22 | @property 23 | def actor_options(self): 24 | return { 25 | "logging_metadata", 26 | "logging_metadata_getter", 27 | } 28 | 29 | def merge_metadata(self, total_logging_metadata, logging_metadata, logging_metadata_getter): 30 | if logging_metadata is not None: 31 | total_logging_metadata = {**total_logging_metadata, **logging_metadata} 32 | if logging_metadata_getter is not None: 33 | if callable(logging_metadata_getter): 34 | total_logging_metadata = {**total_logging_metadata, **logging_metadata_getter()} 35 | else: 36 | raise TypeError("logging_metadata_getter must be callable.") 37 | 38 | return total_logging_metadata 39 | 40 | def update_options_before_create_message(self, options, broker, actor_name): 41 | total_logging_metadata: dict = {} 42 | 43 | # getting logging_metadata at middleware level 44 | logging_metadata = self.logging_metadata 45 | logging_metadata_getter = self.logging_metadata_getter 46 | total_logging_metadata = self.merge_metadata(total_logging_metadata, logging_metadata, logging_metadata_getter) 47 | 48 | # getting logging_metadata at actor level 49 | actor_options = broker.get_actor(actor_name).options 50 | logging_metadata = actor_options.get("logging_metadata", None) 51 | logging_metadata_getter = actor_options.get("logging_metadata_getter", None) 52 | total_logging_metadata = self.merge_metadata(total_logging_metadata, logging_metadata, logging_metadata_getter) 53 | 54 | # getting logging_metadata at message level 55 | logging_metadata = options.get("logging_metadata", None) 56 | logging_metadata_getter = options.get("logging_metadata_getter", None) 57 | total_logging_metadata = self.merge_metadata(total_logging_metadata, logging_metadata, logging_metadata_getter) 58 | if logging_metadata_getter is not None: 59 | del options["logging_metadata_getter"] 60 | 61 | if total_logging_metadata != {}: 62 | options["logging_metadata"] = total_logging_metadata 63 | 64 | for name in ["message_id", "input"]: 65 | if name in total_logging_metadata: 66 | self.logger.error(f"'{name}' cannot be used as a logging_metadata key. It will be overwritten.") 67 | 68 | return options 69 | -------------------------------------------------------------------------------- /tests/test_helpers.py: -------------------------------------------------------------------------------- 1 | import remoulade 2 | from remoulade.helpers import compute_backoff, get_actor_arguments 3 | from remoulade.helpers.reduce import reduce 4 | from remoulade.results import Results 5 | 6 | 7 | def test_reduce_messages(stub_broker, stub_worker, result_backend): 8 | # Given a result backend 9 | # And a broker with the results middleware 10 | stub_broker.add_middleware(Results(backend=result_backend)) 11 | 12 | # Given an actor that stores results 13 | @remoulade.actor(store_results=True) 14 | def do_work(): 15 | return 1 16 | 17 | # Given an actor that stores results 18 | @remoulade.actor(store_results=True) 19 | def merge(results): 20 | return sum(results) 21 | 22 | # And this actor is declared 23 | stub_broker.declare_actor(do_work) 24 | stub_broker.declare_actor(merge) 25 | 26 | merged_message = reduce((do_work.message() for _ in range(10)), merge) 27 | merged_message.run() 28 | 29 | result = merged_message.result.get(block=True) 30 | 31 | assert result == 10 32 | 33 | 34 | def test_actor_arguments(): 35 | @remoulade.actor 36 | def do_work(a: int | None = None): 37 | return 1 38 | 39 | assert get_actor_arguments(do_work) == [{"default": "", "name": "a", "type": "int | None"}] 40 | 41 | 42 | def test_compute_backoff_exponential(): 43 | assert compute_backoff(4, min_backoff=10, jitter=False, max_backoff=50, max_retries=4) == (5, 50) 44 | assert compute_backoff(2, min_backoff=10, jitter=False, max_backoff=100, max_retries=4) == (3, 40) 45 | assert compute_backoff(5, min_backoff=10, jitter=False, max_backoff=100, max_retries=3) == (6, 40) 46 | 47 | 48 | def test_compute_backoff_constant(): 49 | for retry in range(5): 50 | assert compute_backoff(retry, min_backoff=10, jitter=False, backoff_strategy="constant") == (retry + 1, 10) 51 | 52 | 53 | def test_compute_backoff_linear(): 54 | assert compute_backoff(3, min_backoff=10, jitter=False, backoff_strategy="linear") == (4, 40) 55 | assert compute_backoff(4, min_backoff=10, jitter=False, backoff_strategy="linear") == (5, 50) 56 | 57 | 58 | def test_compute_backoff_spread_linear(): 59 | assert compute_backoff( 60 | 2, min_backoff=10, max_backoff=210, max_retries=5, jitter=False, backoff_strategy="spread_linear" 61 | ) == (3, 110) 62 | assert compute_backoff( 63 | 3, min_backoff=10, max_backoff=210, max_retries=5, jitter=False, backoff_strategy="spread_linear" 64 | ) == (4, 160) 65 | assert compute_backoff( 66 | 3, min_backoff=10, max_backoff=410, max_retries=5, jitter=False, backoff_strategy="spread_linear" 67 | ) == (4, 310) 68 | 69 | 70 | def test_compute_backoff_spread_exponential(): 71 | assert compute_backoff( 72 | 0, min_backoff=10, jitter=False, max_backoff=100, max_retries=2, backoff_strategy="spread_exponential" 73 | ) == (1, 10) 74 | assert compute_backoff( 75 | 1, min_backoff=10, jitter=False, max_backoff=160, max_retries=3, backoff_strategy="spread_exponential" 76 | ) == (2, 40) 77 | assert compute_backoff( 78 | 4, min_backoff=10, jitter=False, max_backoff=160, max_retries=3, backoff_strategy="spread_exponential" 79 | ) == (5, 160) 80 | -------------------------------------------------------------------------------- /remoulade/errors.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | 19 | class RemouladeError(Exception): # pragma: no cover 20 | """Base class for all remoulade errors.""" 21 | 22 | def __init__(self, message: str) -> None: 23 | self.message = message 24 | 25 | def __str__(self) -> str: 26 | return str(self.message) or repr(self.message) 27 | 28 | 29 | class BrokerError(RemouladeError): 30 | """Base class for broker-related errors.""" 31 | 32 | 33 | class ActorNotFound(BrokerError): 34 | """Raised when a message is sent to an actor that hasn't been declared.""" 35 | 36 | 37 | class QueueNotFound(BrokerError): 38 | """Raised when a message is sent to an queue that hasn't been declared.""" 39 | 40 | 41 | class QueueJoinTimeout(RemouladeError): 42 | """Raised by brokers that support joining on queues when the join 43 | operation times out. 44 | """ 45 | 46 | 47 | class UnknownStrategy(RemouladeError): 48 | """Raised when the backoff_strategy option is set to an unexpected value.""" 49 | 50 | 51 | class ConnectionError(BrokerError): # noqa: A001 52 | """Base class for broker connection-related errors.""" 53 | 54 | 55 | class ConnectionFailed(ConnectionError): 56 | """Raised when a broker connection could not be opened.""" 57 | 58 | 59 | class ConnectionClosed(ConnectionError): 60 | """Raised when a broker connection is suddenly closed.""" 61 | 62 | 63 | class MessageNotDelivered(ConnectionError): 64 | """Raised when a message has not been delivered.""" 65 | 66 | 67 | class RateLimitExceeded(RemouladeError): 68 | """Raised when a rate limit has been exceeded.""" 69 | 70 | 71 | class NoResultBackend(BrokerError): 72 | """Raised when trying to access the result backend on a broker without it""" 73 | 74 | 75 | class NoCancelBackend(BrokerError): 76 | """Raised when trying to access the cancel backend on a broker without it""" 77 | 78 | 79 | class NoStateBackend(BrokerError): 80 | """Raised when trying to access the state backend on a broker without it""" 81 | 82 | 83 | class ChannelPoolTimeout(BrokerError): 84 | """Raised when the broker has wait for tool long to fetch a channel from its channel pool.""" 85 | 86 | 87 | class InvalidProgress(RemouladeError): 88 | """Raised when trying to set a progress that is greater than 1 or less than 0""" 89 | 90 | 91 | class NoScheduler(RemouladeError): 92 | """Raised when trying to get a scheduler when there is not it""" 93 | -------------------------------------------------------------------------------- /tests/test_worker.py: -------------------------------------------------------------------------------- 1 | import remoulade 2 | from remoulade.worker import build_extra 3 | 4 | from .common import get_logs, worker 5 | 6 | 7 | def test_workers_dont_register_queues_that_arent_whitelisted(stub_broker): 8 | # Given that I have a worker object with a restricted set of queues 9 | with worker(stub_broker, queues={"a", "b"}) as stub_worker: 10 | # When I try to register a consumer for a queue that hasn't been whitelisted 11 | stub_broker.declare_queue("c") 12 | stub_broker.declare_queue("c.DQ") 13 | 14 | # Then a consumer should not get spun up for that queue 15 | assert "c" not in stub_worker.consumers 16 | assert "c.DQ" not in stub_worker.consumers 17 | 18 | 19 | def test_build_extra(stub_broker): 20 | # Given that I have an actor 21 | @remoulade.actor 22 | def do_work(): 23 | return 1 24 | 25 | # and the actor is declared 26 | stub_broker.declare_actor(do_work) 27 | 28 | # I build a message 29 | message = do_work.message() 30 | 31 | # I build the extra 32 | extra = build_extra(message) 33 | 34 | # I expect the extra to look like this 35 | assert extra == {"message_id": message.message_id, "input": {"args": "()", "kwargs": "{}"}} 36 | 37 | 38 | def test_build_extra_metadata(stub_broker): 39 | # Given that I have an actor 40 | @remoulade.actor 41 | def do_work(): 42 | return 1 43 | 44 | # and the actor is declared 45 | stub_broker.declare_actor(do_work) 46 | 47 | # I build a message with logging_metadata 48 | message = do_work.message_with_options(logging_metadata={"id": "1"}) 49 | 50 | # I build the extra 51 | extra = build_extra(message) 52 | 53 | # I expect the find the logging_metadata in the extra 54 | assert extra == {"id": "1", "message_id": message.message_id, "input": {"args": "()", "kwargs": "{}"}} 55 | 56 | 57 | def test_build_extra_override(stub_broker): 58 | # Given that I have an actor 59 | @remoulade.actor 60 | def do_work(): 61 | return 1 62 | 63 | # and the actor is declared 64 | stub_broker.declare_actor(do_work) 65 | 66 | # I build a message with logging_metadata that has a key already used in the extra 67 | message = do_work.message_with_options(logging_metadata={"message_id": "1"}) 68 | 69 | # I build the extra 70 | extra = build_extra(message) 71 | 72 | # I expect the 'message_id' value to be the message.message_id 73 | assert extra == {"message_id": message.message_id, "input": {"args": "()", "kwargs": "{}"}} 74 | 75 | 76 | def test_logging_metadata_logs(stub_broker, stub_worker, caplog): 77 | # Given that I have an actor 78 | @remoulade.actor 79 | def do_work(): 80 | return 1 81 | 82 | # and the actor is declared 83 | stub_broker.declare_actor(do_work) 84 | 85 | # I build a message with logging_metadata 86 | do_work.send_with_options(logging_metadata={"id": "1"}) 87 | 88 | # And join on the queue 89 | stub_broker.join(do_work.queue_name) 90 | stub_worker.join() 91 | 92 | # I expect to find the metadata in the logs extra 93 | records = get_logs(caplog, "Started Actor") 94 | assert len(records) == 1 95 | assert records[0].levelname == "INFO" 96 | assert records[0].__dict__["id"] == "1" 97 | -------------------------------------------------------------------------------- /remoulade/state/backends/stub.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import time 3 | 4 | from ..backend import State, StateBackend 5 | 6 | 7 | class StubBackend(StateBackend): 8 | """An in-memory state backend. For use in unit tests. 9 | Parameters: 10 | namespace(str): A string with which to prefix result keys. 11 | encoder(Encoder): The encoder to use when storing and retrieving 12 | result data. Defaults to :class:`.JSONEncoder`. 13 | """ 14 | 15 | def __init__(self, *args, **kwargs): 16 | super().__init__(*args, **kwargs) 17 | self.states: dict[str, dict[str, str]] = {} 18 | 19 | def get_state(self, message_id): 20 | message_key = self._build_message_key(message_id) 21 | data = self.states.get(message_key) 22 | state = None 23 | if data: 24 | if time.monotonic() > float(data["expiration"]): 25 | self._delete(message_key) 26 | else: 27 | state = State.from_dict(self._decode_dict(data["state"])) 28 | return state 29 | 30 | def set_state(self, state, ttl=3600): 31 | message_key = self._build_message_key(state.message_id) 32 | ttl = ttl + time.monotonic() 33 | encoded_state = self._encode_dict(state.as_dict()) 34 | if message_key not in self.states: 35 | payload = {"state": encoded_state, "expiration": ttl} 36 | self.states[message_key] = payload 37 | else: 38 | state = self.states[message_key]["state"] 39 | for key, value in encoded_state.items(): 40 | state[key] = value 41 | self.states[message_key]["state"] = state 42 | self.states[message_key]["expiration"] = ttl 43 | 44 | def _delete(self, message_key): 45 | del self.states[message_key] 46 | 47 | def get_states( 48 | self, 49 | *, 50 | size: int | None = None, 51 | offset: int = 0, 52 | selected_actors: list[str] | None = None, 53 | selected_statuses: list[str] | None = None, 54 | selected_message_ids: list[str] | None = None, 55 | selected_composition_ids: list[str] | None = None, 56 | start_datetime: datetime.datetime | None = None, 57 | end_datetime: datetime.datetime | None = None, 58 | sort_column: str | None = None, 59 | sort_direction: str | None = None, 60 | ): 61 | time_now = time.monotonic() 62 | states = [] 63 | for message_key in list(self.states.keys()): 64 | data = self.states[message_key] 65 | if time_now > float(data["expiration"]): 66 | self._delete(message_key) 67 | continue 68 | state = State.from_dict(self._decode_dict(data["state"])) 69 | states.append(state) 70 | if size is None: 71 | return states[offset:] 72 | return states[offset : size + offset] 73 | 74 | def get_states_count( 75 | self, 76 | *, 77 | selected_actors: list[str] | None = None, 78 | selected_statuses: list[str] | None = None, 79 | selected_messages_ids: list[str] | None = None, 80 | selected_composition_ids: list[str] | None = None, 81 | start_datetime: datetime.datetime | None = None, 82 | end_datetime: datetime.datetime | None = None, 83 | **kwargs, 84 | ) -> int: 85 | return len(self.states) 86 | -------------------------------------------------------------------------------- /remoulade/cancel/backends/redis.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | import time 18 | from collections.abc import Iterable 19 | 20 | import redis 21 | 22 | from ...helpers.redis_client import redis_client 23 | from ..backend import CancelBackend 24 | 25 | 26 | class RedisBackend(CancelBackend): 27 | """A cancel backend for Redis_. 28 | 29 | It uses a sorted set with the message_id as member and the timestamp of the addition as a score. 30 | We can check if message has been canceled if it belongs to the set. 31 | And on each message cancel we delete all the cancellations that are older than cancellation_ttl (ZREMRANGEBYSCORE). 32 | This prevents unlimited set growth. 33 | 34 | Parameters: 35 | cancellation_ttl(int): The minimal amount of seconds message cancellations 36 | should be kept in the backend. 37 | key(str): A string to serve as key for the sorted set. 38 | client(Redis): An optional client. If this is passed, 39 | then all other parameters are ignored. 40 | url(str): An optional connection URL. If both a URL and 41 | connection parameters are provided, the URL is used. 42 | **parameters(dict): Connection parameters are passed directly 43 | to :class:`redis.Redis`. 44 | 45 | .. _redis: https://redis.io 46 | """ 47 | 48 | def __init__( 49 | self, 50 | *, 51 | cancellation_ttl: int | None = None, 52 | key: str = "remoulade-cancellations", 53 | client: redis.Redis | None = None, 54 | url: str | None = None, 55 | socket_timeout: float = 5.0, 56 | **parameters, 57 | ) -> None: 58 | super().__init__(cancellation_ttl=cancellation_ttl) 59 | 60 | self.client = client or redis_client(url=url, socket_timeout=socket_timeout, **parameters) 61 | self.key = key 62 | 63 | def is_canceled(self, message_id: str, composition_id: str | None) -> bool: 64 | try: 65 | with self.client.pipeline() as pipe: 66 | [pipe.zscore(self.key, key) for key in [message_id, composition_id] if key] 67 | results = pipe.execute() 68 | return any(result is not None for result in results) 69 | except redis.exceptions.RedisError: 70 | return False # if connection to redis fail for any reason, consider the message as not cancelled 71 | 72 | def cancel(self, message_ids: Iterable[str]) -> None: 73 | timestamp = time.time() 74 | with self.client.pipeline() as pipe: 75 | pipe.zadd(self.key, dict.fromkeys(message_ids, timestamp)) 76 | pipe.zremrangebyscore(self.key, "-inf", timestamp - self.cancellation_ttl) 77 | pipe.execute() 78 | -------------------------------------------------------------------------------- /remoulade/middleware/heartbeat.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | import os 18 | import tempfile 19 | import threading 20 | import time 21 | 22 | from ..logging import get_logger 23 | from .middleware import Middleware 24 | 25 | 26 | class Heartbeat(Middleware): 27 | """Make Remoulade's heart beats. 28 | 29 | This middleware makes each worker thread writes a file 30 | in the specified directory containing the timestamp they started their latest task. 31 | You can specify a minimal interval in seconds (default to 1 minute) between each write. 32 | The directory is created when starting if it does not exist 33 | (default to your temporary directory+/remouladebeat). 34 | 35 | You should use an out-of-process probe to check the beats and react in consequence 36 | if the threads lock themselves. 37 | """ 38 | 39 | class ThreadBeat(threading.local): 40 | ts: float 41 | file: str 42 | 43 | # This implementation is rather simple and naive. 44 | # There is nothing to ensure the actual state of the filesystem corresponds 45 | # to what we think about it here. 46 | # In particular, if something is killed unexpectedly, some files may be left 47 | # and you could think that a thread is stuck because a file is not updated anymore. 48 | 49 | def __init__(self, directory: str | None = None, interval: int = 60): 50 | super().__init__() 51 | self.basedir = directory 52 | self.interval = interval 53 | self.log = get_logger(__name__, "HeartbeatMiddleware") 54 | # thread-specific 55 | self.beat = Heartbeat.ThreadBeat() 56 | 57 | def after_process_boot(self, broker): 58 | if not self.basedir: 59 | self.basedir = tempfile.gettempdir() + "/remouladebeat" 60 | os.makedirs(self.basedir, exist_ok=True) 61 | self.log.debug("Created directory %s", self.basedir) 62 | 63 | def after_worker_thread_boot(self, broker, thread): 64 | fd, self.beat.file = tempfile.mkstemp(dir=self.basedir, prefix=f"th-{thread.ident}-") 65 | os.close(fd) 66 | self.beat.ts = 0 67 | 68 | def heartbeat(self): 69 | if self.beat.ts + self.interval < (beat := time.time()): 70 | with open(self.beat.file, "w") as f: 71 | f.write(f"{beat}") 72 | self.beat.ts = beat 73 | 74 | def before_process_message(self, broker, message): 75 | self.heartbeat() 76 | 77 | def after_worker_thread_empty(self, broker, thread): 78 | self.heartbeat() 79 | 80 | def before_worker_thread_shutdown(self, broker, thread): 81 | try: 82 | os.remove(self.beat.file) 83 | except FileNotFoundError: 84 | pass 85 | -------------------------------------------------------------------------------- /remoulade/middleware/shutdown.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | 18 | import threading 19 | import warnings 20 | 21 | from ..logging import get_logger 22 | from .middleware import Middleware 23 | from .threading import Interrupt, current_platform, raise_thread_exception, supported_platforms 24 | 25 | 26 | class Shutdown(Interrupt): 27 | """Exception used to interrupt worker threads when their worker 28 | processes have been signaled for termination. 29 | """ 30 | 31 | 32 | class ShutdownNotifications(Middleware): 33 | """Middleware that interrupts actors whose worker process has been 34 | signaled for termination. 35 | Currently, this is only available on CPython. 36 | 37 | Note: 38 | This works by setting an async exception in the worker thread 39 | that runs the actor. This means that the exception will only get 40 | called the next time that thread acquires the GIL. Concretely, 41 | this means that this middleware can't cancel system calls. 42 | 43 | Parameters: 44 | notify_shutdown(bool): When true, the actor will be interrupted 45 | if the worker process was terminated. 46 | """ 47 | 48 | def __init__(self, notify_shutdown=False): 49 | self.logger = get_logger(__name__, type(self)) 50 | self.notify_shutdown = notify_shutdown 51 | self.notifications: set[int] = set() 52 | 53 | @property 54 | def actor_options(self): 55 | return {"notify_shutdown"} 56 | 57 | def after_process_boot(self, broker): 58 | if current_platform not in supported_platforms: # pragma: no cover 59 | msg = "ShutdownNotifications cannot kill threads on your current platform (%r)." 60 | warnings.warn(msg % current_platform, category=RuntimeWarning, stacklevel=2) 61 | 62 | def before_worker_shutdown(self, broker, worker): 63 | self.logger.debug("Sending shutdown notification to worker threads...") 64 | for thread_id in self.notifications: 65 | self.logger.info("Worker shutdown notification. Raising exception in worker thread %r.", thread_id) 66 | raise_thread_exception(thread_id, Shutdown) 67 | 68 | def before_process_message(self, broker, message): 69 | if self.get_option("notify_shutdown", broker=broker, message=message): 70 | self.notifications.add(threading.get_ident()) 71 | 72 | def after_process_message(self, broker, message, *, result=None, exception=None): 73 | thread_id = threading.get_ident() 74 | 75 | if thread_id in self.notifications: 76 | self.notifications.remove(thread_id) 77 | 78 | after_skip_message = after_process_message 79 | after_message_canceled = after_process_message 80 | -------------------------------------------------------------------------------- /remoulade/rate_limits/backend.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 CLEARTYPE SRL 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | from collections.abc import Callable 18 | 19 | 20 | class RateLimiterBackend: 21 | """ABC for rate limiter backends.""" 22 | 23 | def add(self, key: str, value: int, ttl: int) -> bool: # pragma: no cover 24 | """Add a key to the backend iff it doesn't exist. 25 | 26 | Parameters: 27 | key(str): The key to add. 28 | value(int): The value to add. 29 | ttl(int): The max amount of time in milliseconds the key can 30 | live in the backend for. 31 | """ 32 | raise NotImplementedError 33 | 34 | def incr(self, key: str, amount: int, maximum: int, ttl: int) -> bool: # pragma: no cover 35 | """Atomically increment a key in the backend up to the given maximum. 36 | 37 | Parameters: 38 | key(str): The key to increment. 39 | amount(int): The amount to increment the value by. 40 | maximum(int): The maximum amount the value can have. 41 | ttl(int): The max amount of time in milliseconds the key can 42 | live in the backend for. 43 | 44 | Returns: 45 | bool: True if the key was successfully incremented. 46 | """ 47 | raise NotImplementedError 48 | 49 | def decr(self, key: str, amount: int, minimum: int, ttl: int) -> bool: # pragma: no cover 50 | """Atomically decrement a key in the backend up to the given maximum. 51 | 52 | Parameters: 53 | key(str): The key to decrement. 54 | amount(int): The amount to decrement the value by. 55 | minimum(int): The minimum amount the value can have. 56 | ttl(int): The max amount of time in milliseconds the key can 57 | live in the backend for. 58 | 59 | Returns: 60 | bool: True if the key was successfully decremented. 61 | """ 62 | raise NotImplementedError 63 | 64 | def incr_and_sum( 65 | self, key: str, keys: Callable[[], list[str]], amount: int, maximum: int, ttl: int 66 | ): # pragma: no cover 67 | """Atomically increment a key unless the sum of keys is greater than the given maximum. 68 | 69 | Parameters: 70 | key(str): The key to increment. 71 | keys(callable): A callable to return the list of keys to be 72 | summed over. 73 | amount(int): The amount to decrement the value by. 74 | maximum(int): The maximum sum of the keys. 75 | ttl(int): The max amount of time in milliseconds the key can 76 | live in the backend for. 77 | 78 | Returns: 79 | bool: True if the key was successfully incremented. 80 | """ 81 | raise NotImplementedError 82 | -------------------------------------------------------------------------------- /remoulade/cancel/middleware.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | # This file is a part of Remoulade. 18 | # 19 | # Copyright (C) 2017,2018 CLEARTYPE SRL 20 | # 21 | # Remoulade is free software; you can redistribute it and/or modify it 22 | # under the terms of the GNU Lesser General Public License as published by 23 | # the Free Software Foundation, either version 3 of the License, or (at 24 | # your option) any later version. 25 | # 26 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 27 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 28 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 29 | # License for more details. 30 | # 31 | # You should have received a copy of the GNU Lesser General Public License 32 | # along with this program. If not, see . 33 | from ..logging import get_logger 34 | from ..middleware import Middleware 35 | from .errors import MessageCanceled 36 | 37 | 38 | class Cancel(Middleware): 39 | """Middleware that check if a message has been canceled before processing it 40 | If the message has been canceled raise a MessageCanceled to prevent message processing. 41 | 42 | Example: 43 | 44 | >>> from remoulade.cancel import Cancel 45 | >>> from remoulade.cancel.backends import RedisBackend 46 | >>> backend = RedisBackend() 47 | >>> broker.add_middleware(Cancel(backend=backend)) 48 | 49 | >>> @remoulade.actor(store_results=True) 50 | ... def add(x, y): 51 | ... return x + y 52 | 53 | >>> broker.declare_actor(add) 54 | >>> message = add.send(1, 2) 55 | >>> message.cancel() 56 | 3 57 | 58 | Parameters: 59 | backend(CancelBackend): The cancel backend to use to check 60 | cancellations. 61 | Defaults to False and can be set on a per-actor basis. 62 | """ 63 | 64 | def __init__(self, *, backend=None): 65 | self.logger = get_logger(__name__, type(self)) 66 | self.backend = backend 67 | 68 | def before_process_message(self, broker, message): 69 | composition_id = message.options.get("composition_id") 70 | 71 | if self.backend.is_canceled(message.message_id, composition_id): 72 | raise MessageCanceled(f"Message {message.message_id} has been canceled") 73 | 74 | def after_process_message(self, broker, message, *, result=None, exception=None): 75 | """Cancel all the messages in the composition if one of the message of the composition fails""" 76 | 77 | if exception is None: 78 | return 79 | 80 | cancel_on_error = message.options.get("cancel_on_error") 81 | 82 | if cancel_on_error: 83 | self.backend.cancel([message.options["composition_id"]]) 84 | -------------------------------------------------------------------------------- /tests/middleware/test_heartbeat.py: -------------------------------------------------------------------------------- 1 | # This file is a part of Remoulade. 2 | # 3 | # Copyright (C) 2017,2018 WIREMIND SAS 4 | # 5 | # Remoulade is free software; you can redistribute it and/or modify it 6 | # under the terms of the GNU Lesser General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or (at 8 | # your option) any later version. 9 | # 10 | # Remoulade is distributed in the hope that it will be useful, but WITHOUT 11 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 13 | # License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public License 16 | # along with this program. If not, see . 17 | import time 18 | from pathlib import Path 19 | 20 | from freezegun import freeze_time 21 | 22 | from remoulade.brokers.stub import StubBroker 23 | from remoulade.middleware import Heartbeat 24 | from remoulade.worker import Worker 25 | 26 | 27 | def test_heartbeat(stub_broker: StubBroker, do_work, tmp_path: Path): 28 | beatdir = tmp_path 29 | stub_broker.add_middleware(Heartbeat(directory=str(beatdir), interval=1)) 30 | stub_broker.emit_after("process_boot") 31 | worker = Worker(stub_broker, worker_timeout=100, worker_threads=1) 32 | worker.start() 33 | with freeze_time("1998-07-12 21:00"): 34 | do_work.send() 35 | stub_broker.join(do_work.queue_name) 36 | worker.join() 37 | 38 | assert beatdir.is_dir() 39 | # one thread file 40 | assert len(list(beatdir.iterdir())) == 1 41 | beatfile = next(beatdir.iterdir()) 42 | beat = float(beatfile.read_text()) 43 | 44 | with freeze_time("1998-07-12 21:00"): 45 | do_work.send() 46 | stub_broker.join(do_work.queue_name) 47 | worker.join() 48 | assert float(beatfile.read_text()) == beat 49 | 50 | with freeze_time("1998-07-12 21:27"): 51 | do_work.send() 52 | stub_broker.join(do_work.queue_name) 53 | worker.join() 54 | assert (beat2 := float(beatfile.read_text())) > beat 55 | 56 | # Test it beats even if the queue is empty 57 | with freeze_time("1998-07-12 23:00"): 58 | time.sleep(2) 59 | assert float(beatfile.read_text()) > beat2 60 | 61 | worker.workers[0].stop() 62 | worker.workers[0].join() 63 | assert not beatfile.is_file() 64 | worker.stop() 65 | 66 | 67 | def test_multith_heartbeat(stub_broker: StubBroker, do_work, tmp_path: Path): 68 | beatdir = tmp_path 69 | stub_broker.add_middleware(Heartbeat(directory=str(beatdir), interval=1)) 70 | stub_broker.emit_after("process_boot") 71 | worker = Worker(stub_broker, worker_timeout=100, worker_threads=2) 72 | worker.start() 73 | t0 = worker.workers[0] 74 | t1 = worker.workers[1] 75 | t0.pause() 76 | t0.paused_event.wait() 77 | do_work.send() 78 | stub_broker.join(do_work.queue_name) 79 | worker.join() 80 | 81 | t1.pause() 82 | t1.paused_event.wait() 83 | t0.resume() 84 | do_work.send() 85 | stub_broker.join(do_work.queue_name) 86 | worker.join() 87 | 88 | assert beatdir.is_dir() 89 | # two thread files 90 | assert len(list(beatdir.iterdir())) == 2 91 | assert all(float(f.read_text()) for f in beatdir.iterdir()) 92 | 93 | t0.stop() 94 | t0.join() 95 | assert not any(str(t0.ident) in str(p) for p in beatdir.iterdir()) 96 | assert len(list(beatdir.iterdir())) == 1 97 | worker.stop() 98 | -------------------------------------------------------------------------------- /remoulade/state/backends/redis.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from remoulade.common import chunk 4 | 5 | from ...helpers.redis_client import redis_client 6 | from ..backend import State, StateBackend 7 | 8 | 9 | class RedisBackend(StateBackend): 10 | """A state backend for Redis_. 11 | Parameters: 12 | namespace(str): A string with which to prefix result keys. 13 | encoder(Encoder): The encoder to use when storing and retrieving 14 | result data. Defaults to :class:`.JSONEncoder`. 15 | client(Redis): An optional client. If this is passed, 16 | then all other parameters are ignored. 17 | url(str): An optional connection URL. If both a URL and 18 | connection parameters are provided, the URL is used. 19 | **parameters(dict): Connection parameters are passed directly 20 | to :class:`redis.Redis`. 21 | .. _redis: https://redis.io 22 | """ 23 | 24 | def __init__( 25 | self, *, namespace="remoulade-state", encoder=None, client=None, url=None, socket_timeout=5.0, **parameters 26 | ): 27 | super().__init__(namespace=namespace, encoder=encoder) 28 | self.client = client or redis_client(url=url, socket_timeout=socket_timeout, **parameters) 29 | 30 | def get_state(self, message_id): 31 | key = self._build_message_key(message_id) 32 | data = self.client.hgetall(key) 33 | if not data: 34 | return None 35 | return self._parse_state(data) 36 | 37 | def set_state(self, state, ttl=3600): 38 | message_key = self._build_message_key(state.message_id) 39 | with self.client.pipeline() as pipe: 40 | encoded_state = self._encode_dict(state.as_dict()) 41 | pipe.hset(message_key, mapping=encoded_state) 42 | pipe.expire(message_key, ttl) 43 | pipe.execute() 44 | 45 | def get_states( 46 | self, 47 | *, 48 | size: int | None = None, 49 | offset: int = 0, 50 | selected_actors: list[str] | None = None, 51 | selected_statuses: list[str] | None = None, 52 | selected_message_ids: list[str] | None = None, 53 | selected_composition_ids: list[str] | None = None, 54 | start_datetime: datetime.datetime | None = None, 55 | end_datetime: datetime.datetime | None = None, 56 | sort_column: str | None = None, 57 | sort_direction: str | None = None, 58 | ): 59 | states: list[State] = [] 60 | for keys in chunk(self.client.scan_iter(match=f"{StateBackend.namespace}*", count=size), size): 61 | with self.client.pipeline() as pipe: 62 | for key in keys: 63 | pipe.hgetall(key) 64 | data = pipe.execute() 65 | states.extend(self._parse_state(state_dict) for state_dict in data if state_dict) 66 | 67 | if size is None: 68 | return states[offset:] 69 | 70 | return states[offset : size + offset] 71 | 72 | def get_states_count( 73 | self, 74 | *, 75 | selected_actors: list[str] | None = None, 76 | selected_statuses: list[str] | None = None, 77 | selected_messages_ids: list[str] | None = None, 78 | selected_composition_ids: list[str] | None = None, 79 | start_datetime: datetime.datetime | None = None, 80 | end_datetime: datetime.datetime | None = None, 81 | **kwargs, 82 | ) -> int: 83 | return len(list(self.client.scan_iter(match=f"{StateBackend.namespace}*"))) 84 | 85 | def _parse_state(self, data): 86 | decoded_state = self._decode_dict(data) 87 | return State.from_dict(decoded_state) 88 | -------------------------------------------------------------------------------- /tests/middleware/test_catch_error.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import remoulade 4 | 5 | 6 | def test_on_failure(stub_broker, stub_worker): 7 | on_failure_count = 0 8 | 9 | @remoulade.actor 10 | def on_failure(actor_name, exception_name, args, kwargs): 11 | nonlocal on_failure_count 12 | on_failure_count += 1 13 | 14 | @remoulade.actor 15 | def fail_actor(): 16 | raise Exception() 17 | 18 | remoulade.declare_actors([on_failure, fail_actor]) 19 | 20 | fail_actor.send_with_options(on_failure=on_failure) 21 | 22 | stub_broker.join(fail_actor.queue_name) 23 | stub_broker.join(on_failure.queue_name) 24 | stub_worker.join() 25 | 26 | # The on_failure should have run 27 | assert on_failure_count == 1 28 | 29 | 30 | def test_on_failure_message(stub_broker, stub_worker): 31 | on_failure_count = 0 32 | 33 | @remoulade.actor 34 | def on_failure(actor_name, exception_name, args, kwargs): 35 | nonlocal on_failure_count 36 | on_failure_count += 1 37 | 38 | @remoulade.actor 39 | def fail_actor(): 40 | raise Exception() 41 | 42 | remoulade.declare_actors([on_failure, fail_actor]) 43 | 44 | fail_actor.send_with_options(on_failure=on_failure.message()) 45 | 46 | stub_broker.join(fail_actor.queue_name) 47 | stub_broker.join(on_failure.queue_name) 48 | stub_worker.join() 49 | 50 | # The on_failure should have run 51 | assert on_failure_count == 1 52 | 53 | 54 | def test_on_failure_message_in_actor_options(stub_broker, stub_worker): 55 | on_failure_count = 0 56 | 57 | @remoulade.actor 58 | def on_failure(actor_name, exception_name, args, kwargs): 59 | nonlocal on_failure_count 60 | on_failure_count += 1 61 | 62 | stub_broker.declare_actor(on_failure) 63 | 64 | @remoulade.actor(on_failure=on_failure.message()) 65 | def fail_actor(): 66 | raise Exception() 67 | 68 | stub_broker.declare_actor(fail_actor) 69 | 70 | fail_actor.send() 71 | stub_broker.join(fail_actor.queue_name) 72 | stub_broker.join(on_failure.queue_name) 73 | stub_worker.join() 74 | 75 | # The message should not have run 76 | assert on_failure_count == 0 77 | 78 | 79 | def test_on_failure_runs_only_once(stub_broker, stub_worker): 80 | on_failure_count = 0 81 | 82 | @remoulade.actor 83 | def on_failure(actor_name, exception_name, args, kwargs): 84 | nonlocal on_failure_count 85 | on_failure_count += 1 86 | 87 | @remoulade.actor 88 | def fail_actor(): 89 | raise Exception() 90 | 91 | remoulade.declare_actors([on_failure, fail_actor]) 92 | 93 | fail_actor.send_with_options(on_failure=on_failure, max_retries=3, min_backoff=1, backoff_strategy="constant") 94 | 95 | stub_broker.join(fail_actor.queue_name) 96 | stub_broker.join(on_failure.queue_name) 97 | stub_worker.join() 98 | 99 | # The on_failure should have run only once 100 | assert on_failure_count == 1 101 | 102 | 103 | def test_clean_runs_on_timeout(stub_broker, stub_worker): 104 | on_failure_count = 0 105 | 106 | @remoulade.actor 107 | def on_failure(actor_name, exception_name, args, kwargs): 108 | nonlocal on_failure_count 109 | on_failure_count += 1 110 | 111 | @remoulade.actor 112 | def do_work(): 113 | time.sleep(1) 114 | 115 | remoulade.declare_actors([on_failure, do_work]) 116 | 117 | do_work.send_with_options(on_failure=on_failure, time_limit=1) 118 | 119 | stub_broker.join(do_work.queue_name) 120 | stub_broker.join(on_failure.queue_name) 121 | stub_worker.join() 122 | 123 | # The on_failure should have run as messages should not be passed in on_failure as actor options 124 | assert on_failure_count == 1 125 | --------------------------------------------------------------------------------