├── resonate ├── py.typed ├── models │ ├── __init__.py │ ├── retry_policy.py │ ├── clock.py │ ├── encoder.py │ ├── router.py │ ├── result.py │ ├── message_source.py │ ├── logger.py │ ├── callback.py │ ├── context.py │ ├── convention.py │ ├── task.py │ ├── message.py │ ├── handle.py │ ├── schedules.py │ ├── store.py │ ├── commands.py │ └── durable_promise.py ├── clocks │ ├── __init__.py │ └── step.py ├── routers │ ├── __init__.py │ └── tag.py ├── loggers │ ├── __init__.py │ ├── context.py │ └── dst.py ├── stores │ ├── __init__.py │ └── remote.py ├── message_sources │ ├── __init__.py │ ├── local.py │ └── poller.py ├── conventions │ ├── __init__.py │ ├── base.py │ ├── sleep.py │ ├── local.py │ └── remote.py ├── __init__.py ├── encoders │ ├── noop.py │ ├── base64.py │ ├── jsonpickle.py │ ├── __init__.py │ ├── combined.py │ ├── header.py │ ├── pair.py │ └── json.py ├── retry_policies │ ├── __init__.py │ ├── never.py │ ├── linear.py │ ├── constant.py │ └── exponential.py ├── errors │ ├── __init__.py │ └── errors.py ├── dependencies.py ├── delay_q.py ├── processor.py ├── utils.py ├── registry.py ├── graph.py ├── options.py ├── coroutine.py └── bridge.py ├── tests ├── __init__.py ├── test_delay_q.py ├── test_retry_policies.py ├── test_processor.py ├── test_store_schedules.py ├── conftest.py ├── test_equivalencies.py ├── test_encoders.py ├── test_store_task.py ├── test_retries.py ├── runners.py ├── test_dst.py └── test_bridge.py ├── assets └── resonate-component.png ├── .github ├── dependabot.yml └── workflows │ ├── cd.yml │ ├── api_ref.yml │ ├── ci.yml │ └── dst.yml ├── CONTRIBUTING.md ├── .gitignore ├── ruff.toml ├── scripts └── new-release.py ├── pyproject.toml ├── README.md └── LICENSE /resonate/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resonate/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assets/resonate-component.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/resonatehq/resonate-sdk-py/HEAD/assets/resonate-component.png -------------------------------------------------------------------------------- /resonate/clocks/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from .step import StepClock 4 | 5 | __all__ = ["StepClock"] 6 | -------------------------------------------------------------------------------- /resonate/routers/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from .tag import TagRouter 4 | 5 | __all__ = ["TagRouter"] 6 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - package-ecosystem: "uv" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | -------------------------------------------------------------------------------- /resonate/loggers/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from .context import ContextLogger 4 | from .dst import DSTLogger 5 | 6 | __all__ = ["ContextLogger", "DSTLogger"] 7 | -------------------------------------------------------------------------------- /resonate/stores/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from .local import LocalStore 4 | from .remote import RemoteStore 5 | 6 | __all__ = ["LocalStore", "RemoteStore"] 7 | -------------------------------------------------------------------------------- /resonate/message_sources/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from .local import LocalMessageSource 4 | from .poller import Poller 5 | 6 | __all__ = ["LocalMessageSource", "Poller"] 7 | -------------------------------------------------------------------------------- /resonate/conventions/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from .base import Base 4 | from .local import Local 5 | from .remote import Remote 6 | from .sleep import Sleep 7 | 8 | __all__ = ["Base", "Local", "Remote", "Sleep"] 9 | -------------------------------------------------------------------------------- /resonate/models/retry_policy.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Protocol, runtime_checkable 4 | 5 | 6 | @runtime_checkable 7 | class RetryPolicy(Protocol): 8 | def next(self, attempt: int) -> float | None: ... 9 | -------------------------------------------------------------------------------- /resonate/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from .coroutine import Promise, Yieldable 4 | from .models.handle import Handle 5 | from .resonate import Context, Resonate 6 | 7 | __all__ = ["Context", "Handle", "Promise", "Resonate", "Yieldable"] 8 | -------------------------------------------------------------------------------- /resonate/encoders/noop.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Any 4 | 5 | 6 | class NoopEncoder: 7 | def encode(self, obj: Any) -> Any: 8 | return None 9 | 10 | def decode(self, obj: Any) -> Any: 11 | return None 12 | -------------------------------------------------------------------------------- /resonate/models/clock.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Protocol, runtime_checkable 4 | 5 | 6 | @runtime_checkable 7 | class Clock(Protocol): 8 | def time(self) -> float: ... 9 | def strftime(self, format: str, /) -> str: ... 10 | -------------------------------------------------------------------------------- /resonate/retry_policies/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from .constant import Constant 4 | from .exponential import Exponential 5 | from .linear import Linear 6 | from .never import Never 7 | 8 | __all__ = ["Constant", "Exponential", "Linear", "Never"] 9 | -------------------------------------------------------------------------------- /resonate/models/encoder.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Protocol, runtime_checkable 4 | 5 | 6 | @runtime_checkable 7 | class Encoder[I, O](Protocol): 8 | def encode(self, obj: I, /) -> O: ... 9 | def decode(self, obj: O, /) -> I: ... 10 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contribute to the Resonate Python SDK 2 | 3 | Please [open a Github Issue](https://github.com/resonatehq/resonate-sdk-py/issues) prior to submitting a Pull Request. 4 | 5 | Join the #resonate-engineering channel in the [community Discord](https://www.resonatehq.io/discord) to discuss your changes. 6 | -------------------------------------------------------------------------------- /resonate/models/router.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING, Protocol 4 | 5 | if TYPE_CHECKING: 6 | from resonate.stores.local import DurablePromiseRecord 7 | 8 | 9 | class Router(Protocol): 10 | def route(self, promise: DurablePromiseRecord) -> str: ... 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | # Python-generated files 4 | __pycache__/ 5 | *.py[oc] 6 | build/ 7 | dist/ 8 | wheels/ 9 | *.egg-info 10 | 11 | # Virtual environments 12 | .venv 13 | 14 | # cache 15 | .mypy_cache/ 16 | .ruff_cache/ 17 | .pytest_cache/ 18 | 19 | # cov 20 | .coverage 21 | 22 | # docs 23 | apidocs/ 24 | -------------------------------------------------------------------------------- /resonate/errors/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from .errors import ResonateCanceledError, ResonateError, ResonateShutdownError, ResonateStoreError, ResonateTimedoutError 4 | 5 | __all__ = ["ResonateCanceledError", "ResonateError", "ResonateShutdownError", "ResonateStoreError", "ResonateTimedoutError"] 6 | -------------------------------------------------------------------------------- /resonate/models/result.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass 4 | from typing import Final 5 | 6 | type Result[T] = Ok[T] | Ko 7 | 8 | 9 | @dataclass 10 | class Ok[T]: 11 | value: Final[T] 12 | 13 | 14 | @dataclass 15 | class Ko: 16 | value: Final[BaseException] 17 | -------------------------------------------------------------------------------- /resonate/retry_policies/never.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass 4 | from typing import final 5 | 6 | 7 | @final 8 | @dataclass(frozen=True) 9 | class Never: 10 | def next(self, attempt: int) -> float | None: 11 | assert attempt >= 0, "attempt must be greater than or equal to 0" 12 | return 0 if attempt == 0 else None 13 | -------------------------------------------------------------------------------- /resonate/encoders/base64.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import base64 4 | 5 | 6 | class Base64Encoder: 7 | def encode(self, obj: str | None) -> str | None: 8 | return base64.b64encode(obj.encode()).decode() if obj is not None else None 9 | 10 | def decode(self, obj: str | None) -> str | None: 11 | return base64.b64decode(obj).decode() if obj is not None else None 12 | -------------------------------------------------------------------------------- /resonate/dependencies.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Any 4 | 5 | 6 | class Dependencies: 7 | def __init__(self) -> None: 8 | self._deps: dict[str, Any] = {} 9 | 10 | def add(self, key: str, obj: Any) -> None: 11 | self._deps[key] = obj 12 | 13 | def get[T](self, key: str, default: T) -> Any | T: 14 | return self._deps.get(key, default) 15 | -------------------------------------------------------------------------------- /resonate/encoders/jsonpickle.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Any 4 | 5 | import jsonpickle 6 | 7 | 8 | class JsonPickleEncoder: 9 | def encode(self, obj: Any) -> str: 10 | data = jsonpickle.encode(obj, unpicklable=True) 11 | assert data 12 | return data 13 | 14 | def decode(self, obj: str | None) -> Any: 15 | return jsonpickle.decode(obj) # noqa: S301 16 | -------------------------------------------------------------------------------- /resonate/routers/tag.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING, Any 4 | 5 | if TYPE_CHECKING: 6 | from resonate.stores.local import DurablePromiseRecord 7 | 8 | 9 | class TagRouter: 10 | def __init__(self, tag: str = "resonate:invoke") -> None: 11 | self.tag = tag 12 | 13 | def route(self, promise: DurablePromiseRecord) -> Any: 14 | return (promise.tags or {}).get(self.tag) 15 | -------------------------------------------------------------------------------- /resonate/encoders/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from .base64 import Base64Encoder 4 | from .combined import CombinedEncoder 5 | from .header import HeaderEncoder 6 | from .json import JsonEncoder 7 | from .jsonpickle import JsonPickleEncoder 8 | from .noop import NoopEncoder 9 | from .pair import PairEncoder 10 | 11 | __all__ = ["Base64Encoder", "CombinedEncoder", "CombinedEncoder", "HeaderEncoder", "JsonEncoder", "JsonPickleEncoder", "NoopEncoder", "PairEncoder"] 12 | -------------------------------------------------------------------------------- /resonate/encoders/combined.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | if TYPE_CHECKING: 6 | from resonate.models.encoder import Encoder 7 | 8 | 9 | class CombinedEncoder[T, U, V]: 10 | def __init__(self, l: Encoder[T, U], r: Encoder[U, V]) -> None: 11 | self._l = l 12 | self._r = r 13 | 14 | def encode(self, obj: T) -> V: 15 | return self._r.encode(self._l.encode(obj)) 16 | 17 | def decode(self, obj: V) -> T: 18 | return self._l.decode(self._r.decode(obj)) 19 | -------------------------------------------------------------------------------- /resonate/retry_policies/linear.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import sys 4 | from dataclasses import dataclass 5 | from typing import final 6 | 7 | 8 | @final 9 | @dataclass(frozen=True) 10 | class Linear: 11 | delay: float = 1 12 | max_retries: int = sys.maxsize 13 | 14 | def next(self, attempt: int) -> float | None: 15 | assert attempt >= 0, "attempt must be greater than or equal to 0" 16 | 17 | if attempt > self.max_retries: 18 | return None 19 | 20 | return self.delay * attempt 21 | -------------------------------------------------------------------------------- /resonate/models/message_source.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING, Protocol, runtime_checkable 4 | 5 | if TYPE_CHECKING: 6 | from resonate.models.message import Mesg 7 | 8 | 9 | @runtime_checkable 10 | class MessageSource(Protocol): 11 | @property 12 | def unicast(self) -> str: ... 13 | @property 14 | def anycast(self) -> str: ... 15 | 16 | def start(self) -> None: ... 17 | def stop(self) -> None: ... 18 | def next(self) -> Mesg | None: ... 19 | def enqueue(self, mesg: Mesg) -> None: ... 20 | -------------------------------------------------------------------------------- /resonate/clocks/step.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import time 4 | 5 | 6 | class StepClock: 7 | def __init__(self) -> None: 8 | self._time = 0.0 9 | 10 | def step(self, time: float) -> None: 11 | assert time >= self._time, "The arrow of time only flows forward." 12 | self._time = time 13 | 14 | def time(self) -> float: 15 | """Return the current time in seconds.""" 16 | return self._time 17 | 18 | def strftime(self, format: str, /) -> str: 19 | return time.strftime(format, time.gmtime(self._time)) 20 | -------------------------------------------------------------------------------- /resonate/models/logger.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Any, Protocol 4 | 5 | 6 | class Logger(Protocol): 7 | def log(self, level: int, msg: Any, *args: Any, **kwargs: Any) -> None: ... 8 | def debug(self, msg: Any, *args: Any, **kwargs: Any) -> None: ... 9 | def info(self, msg: Any, *args: Any, **kwargs: Any) -> None: ... 10 | def warning(self, msg: Any, *args: Any, **kwargs: Any) -> None: ... 11 | def error(self, msg: Any, *args: Any, **kwargs: Any) -> None: ... 12 | def critical(self, msg: Any, *args: Any, **kwargs: Any) -> None: ... 13 | -------------------------------------------------------------------------------- /resonate/retry_policies/constant.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import sys 4 | from dataclasses import dataclass 5 | from typing import final 6 | 7 | 8 | @final 9 | @dataclass(frozen=True) 10 | class Constant: 11 | delay: float = 1 12 | max_retries: int = sys.maxsize 13 | 14 | def next(self, attempt: int) -> float | None: 15 | assert attempt >= 0, "attempt must be greater than or equal to 0" 16 | 17 | if attempt > self.max_retries: 18 | return None 19 | 20 | if attempt == 0: 21 | return 0 22 | 23 | return self.delay 24 | -------------------------------------------------------------------------------- /resonate/models/callback.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass 4 | from typing import TYPE_CHECKING, Any, Self 5 | 6 | if TYPE_CHECKING: 7 | from collections.abc import Mapping 8 | 9 | 10 | @dataclass 11 | class Callback: 12 | id: str 13 | promise_id: str 14 | timeout: int 15 | created_on: int 16 | 17 | @classmethod 18 | def from_dict(cls, data: Mapping[str, Any]) -> Self: 19 | return cls( 20 | id=data["id"], 21 | promise_id=data["promiseId"], 22 | timeout=data["timeout"], 23 | created_on=data["createdOn"], 24 | ) 25 | -------------------------------------------------------------------------------- /resonate/encoders/header.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | if TYPE_CHECKING: 6 | from resonate.models.encoder import Encoder 7 | 8 | 9 | class HeaderEncoder[T, U]: 10 | def __init__(self, key: str, encoder: Encoder[T, U]) -> None: 11 | self._key = key 12 | self._enc = encoder 13 | 14 | def encode(self, obj: T) -> dict[str, U] | None: 15 | return {self._key: self._enc.encode(obj)} 16 | 17 | def decode(self, obj: dict[str, U] | None) -> T | None: 18 | if obj and self._key in obj: 19 | return self._enc.decode(obj[self._key]) 20 | 21 | return None 22 | -------------------------------------------------------------------------------- /tests/test_delay_q.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from resonate.delay_q import DelayQ 4 | 5 | 6 | def test_delay_queue() -> None: 7 | dq = DelayQ[int]() 8 | dq.add(1, 10) 9 | dq.add(1, 10) 10 | dq.add(1, 10) 11 | assert dq.get(11)[0] == [1, 1, 1] 12 | assert dq.get(11)[0] == [] 13 | dq.add(1, 2) 14 | dq.add(2, 1) 15 | item, next_time = dq.get(1) 16 | assert item == [2] 17 | assert next_time == 2 18 | 19 | item, next_time = dq.get(2) 20 | assert item == [1] 21 | assert next_time == 0 22 | 23 | dq.add(1, 2) 24 | dq.add(1, 2) 25 | dq.add(1, 2) 26 | assert dq.get(2)[0] == [1, 1, 1] 27 | -------------------------------------------------------------------------------- /resonate/retry_policies/exponential.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import sys 4 | from dataclasses import dataclass 5 | from typing import final 6 | 7 | 8 | @final 9 | @dataclass(frozen=True) 10 | class Exponential: 11 | delay: float = 1 12 | max_retries: int = sys.maxsize 13 | factor: float = 2 14 | max_delay: float = 30 15 | 16 | def next(self, attempt: int) -> float | None: 17 | assert attempt >= 0, "attempt must be greater than or equal to 0" 18 | 19 | if attempt > self.max_retries: 20 | return None 21 | 22 | if attempt == 0: 23 | return 0 24 | 25 | return min(self.delay * (self.factor**attempt), self.max_delay) 26 | -------------------------------------------------------------------------------- /.github/workflows/cd.yml: -------------------------------------------------------------------------------- 1 | name: cd 2 | 3 | on: 4 | release: 5 | types: [released] 6 | 7 | jobs: 8 | run: 9 | name: release 10 | 11 | permissions: 12 | id-token: write 13 | 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | 19 | - name: install uv 20 | uses: astral-sh/setup-uv@v5 21 | with: 22 | version: 0.5.23 23 | 24 | - name: set up Python 25 | uses: actions/setup-python@v5 26 | with: 27 | python-version: "3.12" 28 | 29 | - name: build library 30 | run: uv build 31 | 32 | - name: push build artifacts to PyPI 33 | uses: pypa/gh-action-pypi-publish@v1.13.0 34 | -------------------------------------------------------------------------------- /resonate/encoders/pair.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | if TYPE_CHECKING: 6 | from resonate.models.encoder import Encoder 7 | 8 | 9 | class PairEncoder[T, U, V]: 10 | def __init__(self, l: Encoder[T, U], r: Encoder[T, V]) -> None: 11 | self._l = l 12 | self._r = r 13 | 14 | def encode(self, obj: T) -> tuple[U, V]: 15 | return self._l.encode(obj), self._r.encode(obj) 16 | 17 | def decode(self, obj: tuple[U, V]) -> T | None: 18 | u, v = obj 19 | 20 | if (t := self._l.decode(u)) is not None: 21 | return t 22 | 23 | if (t := self._r.decode(v)) is not None: 24 | return t 25 | 26 | return None 27 | -------------------------------------------------------------------------------- /resonate/delay_q.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import heapq 4 | 5 | 6 | class DelayQ[T]: 7 | def __init__(self) -> None: 8 | self._delayed: list[tuple[float, T]] = [] 9 | 10 | def add(self, item: T, delay: float) -> None: 11 | heapq.heappush(self._delayed, (delay, item)) 12 | 13 | def get(self, time: float) -> tuple[list[T], float]: 14 | items: list[T] = [] 15 | while self._delayed and self._delayed[0][0] <= time: 16 | _, item = heapq.heappop(self._delayed) 17 | items.append(item) 18 | 19 | next_time = self._delayed[0][0] if self._delayed else 0 20 | return items, next_time 21 | 22 | def empty(self) -> bool: 23 | return bool(self._delayed) 24 | -------------------------------------------------------------------------------- /ruff.toml: -------------------------------------------------------------------------------- 1 | line-length = 200 2 | 3 | [lint] 4 | ignore = [ 5 | "A001", 6 | "A002", 7 | "A005", 8 | "A006", 9 | "ANN401", 10 | "ARG001", 11 | "ARG002", 12 | "BLE001", 13 | "C901", 14 | "COM812", 15 | "D100", 16 | "D101", 17 | "D102", 18 | "D103", 19 | "D104", 20 | "D105", 21 | "D107", 22 | "D203", 23 | "D211", 24 | "D213", 25 | "E501", 26 | "E741", 27 | "ERA001", 28 | "FBT001", 29 | "FIX002", 30 | "INP001", 31 | "PLR0911", 32 | "PLR0912", 33 | "PLR0913", 34 | "PLR0915", 35 | "PLR2004", 36 | "S101", 37 | "S311", 38 | "TD003", 39 | ] 40 | select = ["ALL"] 41 | 42 | [lint.isort] 43 | combine-as-imports = true 44 | required-imports = ["from __future__ import annotations"] 45 | -------------------------------------------------------------------------------- /resonate/models/context.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING, Any, Protocol 4 | 5 | if TYPE_CHECKING: 6 | from resonate.models.logger import Logger 7 | 8 | 9 | class Context(Protocol): 10 | @property 11 | def id(self) -> str: ... 12 | @property 13 | def info(self) -> Info: ... 14 | @property 15 | def logger(self) -> Logger: ... 16 | 17 | def get_dependency(self, key: str, default: Any = None) -> Any: ... 18 | 19 | 20 | class Info(Protocol): 21 | @property 22 | def attempt(self) -> int: ... 23 | @property 24 | def idempotency_key(self) -> str | None: ... 25 | @property 26 | def tags(self) -> dict[str, str] | None: ... 27 | @property 28 | def timeout(self) -> float: ... 29 | @property 30 | def version(self) -> int: ... 31 | -------------------------------------------------------------------------------- /resonate/encoders/json.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | from typing import Any 5 | 6 | 7 | class JsonEncoder: 8 | def encode(self, obj: Any) -> str | None: 9 | if obj is None: 10 | return None 11 | 12 | return json.dumps(obj, default=_encode_exception) 13 | 14 | def decode(self, obj: str | None) -> Any: 15 | if obj is None: 16 | return None 17 | 18 | return json.loads(obj, object_hook=_decode_exception) 19 | 20 | 21 | def _encode_exception(obj: Any) -> dict[str, Any]: 22 | if isinstance(obj, BaseException): 23 | return {"__error__": str(obj)} 24 | return {} # ignore unencodable objects 25 | 26 | 27 | def _decode_exception(obj: dict[str, Any]) -> Any: 28 | if "__error__" in obj: 29 | return Exception(obj["__error__"]) 30 | return obj 31 | -------------------------------------------------------------------------------- /resonate/models/convention.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING, Any, Protocol 4 | 5 | if TYPE_CHECKING: 6 | from collections.abc import Callable 7 | 8 | 9 | class Convention(Protocol): 10 | @property 11 | def id(self) -> str: ... 12 | @property 13 | def idempotency_key(self) -> str | None: ... 14 | @property 15 | def data(self) -> Any: ... 16 | @property 17 | def timeout(self) -> float: ... # relative time in seconds 18 | @property 19 | def tags(self) -> dict[str, str] | None: ... 20 | 21 | def options( 22 | self, 23 | id: str | None = None, 24 | idempotency_key: str | Callable[[str], str] | None = None, 25 | tags: dict[str, str] | None = None, 26 | target: str | None = None, 27 | timeout: float | None = None, 28 | version: int | None = None, 29 | ) -> Convention: ... 30 | -------------------------------------------------------------------------------- /resonate/models/task.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass, field 4 | from typing import TYPE_CHECKING, Any 5 | 6 | if TYPE_CHECKING: 7 | from collections.abc import Mapping 8 | 9 | from resonate.models.durable_promise import DurablePromise 10 | from resonate.models.store import Store 11 | 12 | 13 | @dataclass 14 | class Task: 15 | id: str 16 | counter: int 17 | store: Store = field(repr=False) 18 | 19 | def claim(self, pid: str, ttl: int) -> tuple[DurablePromise, DurablePromise | None]: 20 | return self.store.tasks.claim(id=self.id, counter=self.counter, pid=pid, ttl=ttl) 21 | 22 | def complete(self) -> None: 23 | self._completed = self.store.tasks.complete(id=self.id, counter=self.counter) 24 | 25 | @classmethod 26 | def from_dict(cls, store: Store, data: Mapping[str, Any]) -> Task: 27 | return cls( 28 | id=data["id"], 29 | counter=data["counter"], 30 | store=store, 31 | ) 32 | -------------------------------------------------------------------------------- /scripts/new-release.py: -------------------------------------------------------------------------------- 1 | """New release.""" 2 | 3 | from __future__ import annotations 4 | 5 | import pathlib 6 | import tomllib 7 | import webbrowser 8 | from typing import Any 9 | from urllib.parse import urlencode 10 | 11 | 12 | def project_project(cwd: pathlib.Path) -> dict[str, Any]: 13 | """Read `pyproject.toml` project.""" 14 | pyproject = tomllib.loads( 15 | cwd.joinpath( 16 | "pyproject.toml", 17 | ).read_text(encoding="utf-8"), 18 | ) 19 | 20 | return pyproject["project"] 21 | 22 | 23 | def main() -> None: 24 | """Prepare new release.""" 25 | cwd = pathlib.Path().cwd() 26 | project = project_project(cwd=cwd) 27 | version = project["version"] 28 | source = project["urls"]["Source"] 29 | params = urlencode( 30 | query={ 31 | "title": f"v{version}", 32 | "tag": f"v{version}", 33 | }, 34 | ) 35 | webbrowser.open_new_tab(url=f"{source}/releases/new?{params}") 36 | 37 | 38 | if __name__ == "__main__": 39 | main() 40 | -------------------------------------------------------------------------------- /resonate/conventions/base.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass 4 | from typing import TYPE_CHECKING, Any 5 | 6 | if TYPE_CHECKING: 7 | from collections.abc import Callable 8 | 9 | 10 | @dataclass 11 | class Base: 12 | id: str 13 | timeout: float 14 | idempotency_key: str | None = None 15 | data: Any = None 16 | tags: dict[str, str] | None = None 17 | 18 | def options( 19 | self, 20 | id: str | None = None, 21 | idempotency_key: str | Callable[[str], str] | None = None, 22 | tags: dict[str, str] | None = None, 23 | target: str | None = None, 24 | timeout: float | None = None, 25 | version: int | None = None, 26 | ) -> Base: 27 | self.id = id or self.id 28 | self.idempotency_key = idempotency_key(self.id) if callable(idempotency_key) else (idempotency_key or self.idempotency_key) 29 | self.timeout = timeout or self.timeout 30 | self.tags = tags or self.tags 31 | 32 | return self 33 | -------------------------------------------------------------------------------- /resonate/models/message.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import Literal, TypedDict 4 | 5 | type Mesg = InvokeMesg | ResumeMesg | NotifyMesg 6 | 7 | 8 | class InvokeMesg(TypedDict): 9 | type: Literal["invoke"] 10 | task: TaskMesg 11 | 12 | 13 | class ResumeMesg(TypedDict): 14 | type: Literal["resume"] 15 | task: TaskMesg 16 | 17 | 18 | class NotifyMesg(TypedDict): 19 | type: Literal["notify"] 20 | promise: DurablePromiseMesg 21 | 22 | 23 | class TaskMesg(TypedDict): 24 | id: str 25 | counter: int 26 | 27 | 28 | class DurablePromiseMesg(TypedDict): 29 | id: str 30 | state: str 31 | timeout: int 32 | idempotencyKeyForCreate: str | None 33 | idempotencyKeyForComplete: str | None 34 | param: DurablePromiseValueMesg 35 | value: DurablePromiseValueMesg 36 | tags: dict[str, str] | None 37 | createdOn: int 38 | completedOn: int | None 39 | 40 | 41 | class DurablePromiseValueMesg(TypedDict): 42 | headers: dict[str, str] | None 43 | data: str | None 44 | -------------------------------------------------------------------------------- /resonate/conventions/sleep.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass 4 | from typing import TYPE_CHECKING, Any 5 | 6 | if TYPE_CHECKING: 7 | from collections.abc import Callable 8 | 9 | 10 | @dataclass 11 | class Sleep: 12 | id: str 13 | secs: float 14 | 15 | @property 16 | def idempotency_key(self) -> str: 17 | return self.id 18 | 19 | @property 20 | def data(self) -> Any: 21 | return None 22 | 23 | @property 24 | def timeout(self) -> float: 25 | return self.secs 26 | 27 | @property 28 | def tags(self) -> dict[str, str]: 29 | return {"resonate:timeout": "true"} 30 | 31 | def options( 32 | self, 33 | id: str | None = None, 34 | idempotency_key: str | Callable[[str], str] | None = None, 35 | tags: dict[str, str] | None = None, 36 | target: str | None = None, 37 | timeout: float | None = None, 38 | version: int | None = None, 39 | ) -> Sleep: 40 | self.id = id or self.id 41 | return self 42 | -------------------------------------------------------------------------------- /tests/test_retry_policies.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | import pytest 6 | 7 | from resonate.retry_policies import Constant, Exponential, Linear, Never 8 | 9 | if TYPE_CHECKING: 10 | from resonate.models.retry_policy import RetryPolicy 11 | 12 | 13 | @pytest.mark.parametrize( 14 | ("policy", "progression"), 15 | [ 16 | (Never(), None), 17 | ( 18 | Constant(delay=1, max_retries=2), 19 | [1, 1, None], 20 | ), 21 | ( 22 | Linear(delay=1, max_retries=2), 23 | [1, 2, None], 24 | ), 25 | ( 26 | Exponential(delay=1, factor=2, max_retries=5, max_delay=8), 27 | [2, 4, 8, 8, 8, None], 28 | ), 29 | ], 30 | ) 31 | def test_delay_progression(policy: RetryPolicy, progression: list[float | None] | None) -> None: 32 | if isinstance(policy, Never): 33 | return 34 | 35 | i: int = 1 36 | delays: list[float | None] = [] 37 | while True: 38 | delays.append(policy.next(i)) 39 | i += 1 40 | if delays[-1] is None: 41 | break 42 | 43 | assert delays == progression 44 | -------------------------------------------------------------------------------- /tests/test_processor.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from queue import Queue 4 | 5 | import pytest 6 | 7 | from resonate.models.result import Ok, Result 8 | from resonate.processor import Processor 9 | 10 | 11 | def greet(name: str) -> str: 12 | return f"Hi {name}" 13 | 14 | 15 | def callback(q: Queue[tuple[str, str]], expected: str, result: Result[str]) -> None: 16 | assert isinstance(result, Ok) 17 | q.put((result.value, expected)) 18 | 19 | 20 | @pytest.mark.parametrize("workers", [1, 2, 3]) 21 | def test_processor(workers: int) -> None: 22 | q = Queue[tuple[str, str]]() 23 | 24 | p = Processor(workers) 25 | assert len(p.threads) == workers 26 | 27 | names = ["A", "B"] 28 | expected_greet = [greet("A"), greet("B")] 29 | p.start() 30 | for name, expected in zip(names, expected_greet, strict=False): 31 | p.enqueue(lambda name=name: greet(name), lambda r, expected=expected: callback(q, expected, r)) 32 | 33 | p.stop() 34 | assert q.qsize() == len(names) 35 | 36 | for _ in range(q.qsize()): 37 | actual, expected = q.get() 38 | assert actual == expected 39 | q.task_done() 40 | 41 | q.join() 42 | assert q.empty() 43 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "resonate-sdk" 3 | version = "0.6.7" 4 | description = "Distributed Async Await by Resonate HQ, Inc" 5 | readme = "README.md" 6 | authors = [{ name = "Resonate HQ, Inc", email = "contact@resonatehq.io" }] 7 | requires-python = ">=3.12" 8 | dependencies = [ 9 | "croniter >= 6.0.0, < 7", 10 | "jsonpickle >= 4, < 5", 11 | "requests >= 2, < 3", 12 | ] 13 | 14 | [project.urls] 15 | Documentation = "https://github.com/resonatehq/resonate-sdk-py#readme" 16 | Issues = "https://github.com/resonatehq/resonate-sdk-py/issues" 17 | Source = "https://github.com/resonatehq/resonate-sdk-py" 18 | 19 | [build-system] 20 | requires = ["hatchling"] 21 | build-backend = "hatchling.build" 22 | 23 | [dependency-groups] 24 | dev = [ 25 | "docutils>=0.21.2", 26 | "pydoctor>=24.11.2", 27 | "pyright>=1.1.396", 28 | "pytest-cov>=6.1.1", 29 | "pytest>=8.3.5", 30 | "ruff>=0.11.0", 31 | "tabulate>=0.9.0", 32 | "types-requests>=2.32.0.20250306", 33 | ] 34 | 35 | [tool.pytest.ini_options] 36 | testpaths = ["tests"] 37 | addopts = ["--import-mode=importlib"] 38 | 39 | [tool.hatch.build.targets.wheel] 40 | packages = ["resonate"] 41 | 42 | [tool.pyright] 43 | venvPath = "." 44 | venv = ".venv" 45 | -------------------------------------------------------------------------------- /.github/workflows/api_ref.yml: -------------------------------------------------------------------------------- 1 | name: api ref 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | permissions: 9 | contents: write 10 | pages: write 11 | id-token: write 12 | 13 | jobs: 14 | deploy: 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: install uv 20 | uses: astral-sh/setup-uv@v5 21 | with: 22 | version: 0.5.23 23 | 24 | - name: set up python 25 | uses: actions/setup-python@v5 26 | with: 27 | python-version: 3.12 28 | 29 | - name: install 30 | run: uv sync --dev 31 | 32 | - name: generate api docs 33 | run: uv run pydoctor resonate --docformat=google --project-name "Resonate Python SDK" --project-url "https://github.com/resonatehq/resonate-sdk-py" --html-output docs/build 34 | 35 | - name: publish api docs to gh pages 36 | uses: peaceiris/actions-gh-pages@v3 37 | with: 38 | github_token: ${{ secrets.GITHUB_TOKEN }} 39 | publish_dir: ./docs/build 40 | publish_branch: gh-pages 41 | force_orphan: true # Ensure this creates a fresh gh-pages branch if needed 42 | commit_message: generate API docs 43 | -------------------------------------------------------------------------------- /resonate/models/handle.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | from typing import TYPE_CHECKING 5 | 6 | if TYPE_CHECKING: 7 | from collections.abc import Callable, Generator 8 | from concurrent.futures import Future 9 | 10 | 11 | class Handle[T]: 12 | def __init__(self, id: str, f: Future[T], cb: Callable[[str, Future[T]], None]) -> None: 13 | self._id = id 14 | self._f = f 15 | self._should_subscribe = not self._f.done() 16 | self._cb = cb 17 | 18 | @property 19 | def id(self) -> str: 20 | return self._id 21 | 22 | @property 23 | def future(self) -> Future[T]: 24 | return self._f 25 | 26 | def done(self) -> bool: 27 | self._subscribe() 28 | return self._f.done() 29 | 30 | def result(self, timeout: float | None = None) -> T: 31 | self._subscribe() 32 | return self._f.result(timeout) 33 | 34 | def __await__(self) -> Generator[None, None, T]: 35 | self._subscribe() 36 | return asyncio.wrap_future(self._f).__await__() 37 | 38 | def _subscribe(self) -> None: 39 | if self._should_subscribe: 40 | self._cb(self._id, self._f) 41 | self._should_subscribe = False 42 | -------------------------------------------------------------------------------- /tests/test_store_schedules.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import sys 4 | from typing import TYPE_CHECKING 5 | 6 | import pytest 7 | 8 | from resonate.errors.errors import ResonateStoreError 9 | 10 | if TYPE_CHECKING: 11 | from collections.abc import Generator 12 | 13 | from resonate.models.store import Store 14 | 15 | 16 | COUNTER = 0 17 | 18 | 19 | @pytest.fixture 20 | def sid(store: Store) -> Generator[str]: 21 | global COUNTER # noqa: PLW0603 22 | 23 | id = f"sid{COUNTER}" 24 | COUNTER += 1 25 | 26 | yield id 27 | 28 | store.schedules.delete(id) 29 | 30 | 31 | def test_create_read_delete(store: Store, sid: str) -> None: 32 | schedule = store.schedules.create(sid, "0 * * * *", "foo", sys.maxsize) 33 | assert schedule == store.schedules.get(sid) 34 | 35 | 36 | def test_create_twice_without_ikey(store: Store, sid: str) -> None: 37 | store.schedules.create(sid, "* * * * *", "foo", 10) 38 | with pytest.raises(ResonateStoreError): 39 | store.schedules.create(sid, "* * * * *", "foo", 10) 40 | 41 | 42 | def test_create_twice_with_ikey(store: Store, sid: str) -> None: 43 | schedule = store.schedules.create(sid, "* * * * *", "foo", 10, ikey="foo") 44 | store.schedules.create(sid, "0 * 2 * *", "bar", 10, ikey="foo") 45 | assert schedule == store.schedules.get(sid) 46 | -------------------------------------------------------------------------------- /resonate/conventions/local.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass, field 4 | from typing import TYPE_CHECKING, Any 5 | 6 | from resonate.options import Options 7 | 8 | if TYPE_CHECKING: 9 | from collections.abc import Callable 10 | 11 | 12 | @dataclass 13 | class Local: 14 | id: str 15 | r_id: str 16 | p_id: str 17 | opts: Options = field(default_factory=Options, repr=False) 18 | 19 | @property 20 | def idempotency_key(self) -> str | None: 21 | return self.opts.get_idempotency_key(self.id) 22 | 23 | @property 24 | def data(self) -> Any: 25 | return None 26 | 27 | @property 28 | def timeout(self) -> float: 29 | return self.opts.timeout 30 | 31 | @property 32 | def tags(self) -> dict[str, str]: 33 | return {**self.opts.tags, "resonate:root": self.r_id, "resonate:parent": self.p_id, "resonate:scope": "local"} 34 | 35 | def options( 36 | self, 37 | id: str | None = None, 38 | idempotency_key: str | Callable[[str], str] | None = None, 39 | tags: dict[str, str] | None = None, 40 | target: str | None = None, 41 | timeout: float | None = None, 42 | version: int | None = None, 43 | ) -> Local: 44 | self.id = id or self.id 45 | 46 | # delibrately ignore target and version 47 | self.opts = self.opts.merge(id=id, idempotency_key=idempotency_key, tags=tags, timeout=timeout) 48 | return self 49 | -------------------------------------------------------------------------------- /resonate/message_sources/local.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import queue 4 | from typing import TYPE_CHECKING 5 | 6 | from resonate.models.message import Mesg 7 | 8 | if TYPE_CHECKING: 9 | from resonate.stores import LocalStore 10 | 11 | 12 | class LocalMessageSource: 13 | def __init__(self, store: LocalStore, group: str, id: str, scheme: str = "local") -> None: 14 | self._messages = queue.Queue[Mesg | None]() 15 | self._store = store 16 | self._scheme = scheme 17 | self._group = group 18 | self._id = id 19 | 20 | @property 21 | def group(self) -> str: 22 | return self._group 23 | 24 | @property 25 | def id(self) -> str: 26 | return self._id 27 | 28 | @property 29 | def unicast(self) -> str: 30 | return f"{self._scheme}://uni@{self._group}/{self._id}" 31 | 32 | @property 33 | def anycast(self) -> str: 34 | return f"{self._scheme}://any@{self._group}/{self._id}" 35 | 36 | def start(self) -> None: 37 | # idempotently connect to the store 38 | self._store.connect(self) 39 | 40 | # idempotently start the store 41 | self._store.start() 42 | 43 | def stop(self) -> None: 44 | # disconnect from the store 45 | self._store.disconnect(self) 46 | 47 | # signal to consumers to disconnect 48 | self._messages.put(None) 49 | 50 | def enqueue(self, mesg: Mesg) -> None: 51 | self._messages.put(mesg) 52 | 53 | def next(self) -> Mesg | None: 54 | return self._messages.get() 55 | -------------------------------------------------------------------------------- /resonate/processor.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import os 4 | import queue 5 | from collections.abc import Callable 6 | from threading import Thread 7 | from typing import Any 8 | 9 | from resonate.models.result import Ko, Ok, Result 10 | from resonate.utils import exit_on_exception 11 | 12 | 13 | class Processor: 14 | def __init__(self, workers: int | None = None) -> None: 15 | self.threads = set[Thread]() 16 | for _ in range(min(32, workers or (os.cpu_count() or 1))): 17 | self.threads.add(Thread(target=self._run, daemon=True)) 18 | 19 | self.sq = queue.Queue[tuple[Callable[[], Any], Callable[[Result[Any]], None]] | None]() 20 | 21 | @exit_on_exception 22 | def _run(self) -> None: 23 | while sqe := self.sq.get(): 24 | func, callback = sqe 25 | 26 | try: 27 | r = Ok(func()) 28 | except Exception as e: 29 | r = Ko(e) 30 | 31 | callback(r) 32 | 33 | def enqueue(self, func: Callable[[], Any], callback: Callable[[Result[Any]], None]) -> None: 34 | self.sq.put((func, callback)) 35 | 36 | def start(self) -> None: 37 | for t in self.threads: 38 | if not t.is_alive(): 39 | t.start() 40 | 41 | def stop(self) -> None: 42 | for _ in self.threads: 43 | self.sq.put(None) 44 | 45 | for t in self.threads: 46 | # we might want to consider specifying a timeout 47 | # in the case the user has a long-running function blocking 48 | t.join() 49 | -------------------------------------------------------------------------------- /resonate/models/schedules.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass, field 4 | from typing import TYPE_CHECKING, Any 5 | 6 | from resonate.models.durable_promise import DurablePromiseValue 7 | 8 | if TYPE_CHECKING: 9 | from collections.abc import Mapping 10 | 11 | from resonate.models.store import Store 12 | 13 | 14 | @dataclass 15 | class Schedule: 16 | id: str 17 | description: str | None 18 | cron: str 19 | tags: dict[str, str] | None 20 | promise_id: str 21 | promise_timeout: int 22 | promise_param: DurablePromiseValue 23 | promise_tags: dict[str, str] | None 24 | last_runtime: int | None 25 | next_runtime: int 26 | ikey: str | None 27 | created_on: int 28 | 29 | store: Store = field(repr=False) 30 | 31 | def delete(self) -> None: 32 | self.store.schedules.delete(self.id) 33 | 34 | @classmethod 35 | def from_dict(cls, store: Store, data: Mapping[str, Any]) -> Schedule: 36 | return cls( 37 | id=data["id"], 38 | description=data.get("description"), 39 | cron=data["cron"], 40 | tags=data.get("tags", {}), 41 | promise_id=data["promiseId"], 42 | promise_timeout=data["promiseTimeout"], 43 | promise_param=DurablePromiseValue.from_dict(store, data.get("param", {})), 44 | promise_tags=data.get("promiseTags", {}), 45 | last_runtime=data.get("lastRunTime"), 46 | next_runtime=data["nextRunTime"], 47 | ikey=data.get("idempotencyKey"), 48 | created_on=data["createdOn"], 49 | store=store, 50 | ) 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![resonate component banner](/assets/resonate-component.png) 2 | 3 | # Resonate Python SDK 4 | 5 | [![ci](https://github.com/resonatehq/resonate-sdk-py/actions/workflows/ci.yml/badge.svg)](https://github.com/resonatehq/resonate-sdk-py/actions/workflows/ci.yml) 6 | [![codecov](https://codecov.io/gh/resonatehq/resonate-sdk-py/graph/badge.svg?token=61GYC3DXID)](https://codecov.io/gh/resonatehq/resonate-sdk-py) 7 | [![dst](https://github.com/resonatehq/resonate-sdk-py/actions/workflows/dst.yml/badge.svg)](https://github.com/resonatehq/resonate-sdk-py/actions/workflows/dst.yml) 8 | [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 9 | 10 | ## About this component 11 | 12 | The Resonate Python SDK enables developers to build reliable and scalable cloud applications across a wide variety of use cases. 13 | 14 | ### [How to contribute to this SDK](./CONTRIBUTING.md) 15 | 16 | ### [How to use this SDK](https://docs.resonatehq.io/develop/python) 17 | 18 | ### [Get started with Resonate](https://docs.resonatehq.io/get-started/) 19 | 20 | ### [Evaluate Resonate for your next project](https://docs.resonatehq.io/evaluate/) 21 | 22 | ### [Example application library](https://github.com/resonatehq-examples) 23 | 24 | ### [The concepts that power Resonate](https://www.distributed-async-await.io/) 25 | 26 | ### [Join the Discord](https://resonatehq.io/discord) 27 | 28 | ### [Subscribe to the Blog](https://journal.resonatehq.io/subscribe) 29 | 30 | ### [Follow on Twitter](https://twitter.com/resonatehqio) 31 | 32 | ### [Follow on LinkedIn](https://www.linkedin.com/company/resonatehqio) 33 | 34 | ### [Subscribe on YouTube](https://www.youtube.com/@resonatehqio) 35 | -------------------------------------------------------------------------------- /resonate/loggers/context.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | from typing import Any 5 | 6 | 7 | class ContextLogger: 8 | def __init__(self, cid: str, id: str, level: int | str = logging.NOTSET) -> None: 9 | self.cid = cid 10 | self.id = id 11 | 12 | self._logger = logging.getLogger(f"resonate:{cid}:{id}") 13 | self._logger.setLevel(level) 14 | self._logger.propagate = False 15 | 16 | if not self._logger.handlers: 17 | formatter = logging.Formatter( 18 | "[%(asctime)s] [%(levelname)s] [%(cid)s] [%(id)s] %(message)s", 19 | datefmt="%Y-%m-%d %H:%M:%S", 20 | ) 21 | handler = logging.StreamHandler() 22 | handler.setFormatter(formatter) 23 | self._logger.addHandler(handler) 24 | 25 | def _log(self, level: int, msg: Any, *args: Any, **kwargs: Any) -> None: 26 | self._logger.log(level, msg, *args, **{**kwargs, "extra": {"cid": self.cid, "id": self.id}}) 27 | 28 | def log(self, level: int, msg: Any, *args: Any, **kwargs: Any) -> None: 29 | self._log(level, msg, *args, **kwargs) 30 | 31 | def debug(self, msg: Any, *args: Any, **kwargs: Any) -> None: 32 | self._log(logging.DEBUG, msg, *args, **kwargs) 33 | 34 | def info(self, msg: Any, *args: Any, **kwargs: Any) -> None: 35 | self._log(logging.INFO, msg, *args, **kwargs) 36 | 37 | def warning(self, msg: Any, *args: Any, **kwargs: Any) -> None: 38 | self._log(logging.WARNING, msg, *args, **kwargs) 39 | 40 | def error(self, msg: Any, *args: Any, **kwargs: Any) -> None: 41 | self._log(logging.ERROR, msg, *args, **kwargs) 42 | 43 | def critical(self, msg: Any, *args: Any, **kwargs: Any) -> None: 44 | self._log(logging.CRITICAL, msg, *args, **kwargs) 45 | -------------------------------------------------------------------------------- /resonate/conventions/remote.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass, field 4 | from typing import TYPE_CHECKING, Any 5 | 6 | from resonate import utils 7 | from resonate.options import Options 8 | 9 | if TYPE_CHECKING: 10 | from collections.abc import Callable 11 | 12 | 13 | @dataclass 14 | class Remote: 15 | id: str 16 | r_id: str 17 | p_id: str 18 | name: str 19 | args: tuple[Any, ...] = field(default_factory=tuple) 20 | kwargs: dict[str, Any] = field(default_factory=dict) 21 | opts: Options = field(default_factory=Options, repr=False) 22 | 23 | @property 24 | def idempotency_key(self) -> str | None: 25 | return self.opts.get_idempotency_key(self.id) 26 | 27 | @property 28 | def data(self) -> dict[str, Any]: 29 | return {"func": self.name, "args": self.args, "kwargs": self.kwargs, "version": self.opts.version} 30 | 31 | @property 32 | def timeout(self) -> float: 33 | return self.opts.timeout 34 | 35 | @property 36 | def tags(self) -> dict[str, str]: 37 | return { 38 | **self.opts.tags, 39 | "resonate:root": self.r_id, 40 | "resonate:parent": self.p_id, 41 | "resonate:scope": "global", 42 | "resonate:invoke": self.match(self.opts.target), 43 | } 44 | 45 | def options( 46 | self, 47 | id: str | None = None, 48 | idempotency_key: str | Callable[[str], str] | None = None, 49 | tags: dict[str, str] | None = None, 50 | target: str | None = None, 51 | timeout: float | None = None, 52 | version: int | None = None, 53 | ) -> Remote: 54 | self.id = id or self.id 55 | self.opts = self.opts.merge(id=id, idempotency_key=idempotency_key, target=target, tags=tags, timeout=timeout, version=version) 56 | 57 | return self 58 | 59 | def match(self, target: str) -> str: 60 | # can be refactored to be configurable 61 | return target if utils.is_url(target) else f"poll://any@{target}" 62 | -------------------------------------------------------------------------------- /resonate/loggers/dst.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | from typing import TYPE_CHECKING, Any 5 | 6 | if TYPE_CHECKING: 7 | from resonate.models.clock import Clock 8 | 9 | 10 | class DSTFormatter(logging.Formatter): 11 | def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str: # noqa: N802 12 | time: float = getattr(record, "time", 0.0) 13 | return f"{time:09.0f}" 14 | 15 | 16 | class DSTLogger: 17 | def __init__(self, cid: str, id: str, clock: Clock, level: int = logging.NOTSET) -> None: 18 | self.cid = cid 19 | self.id = id 20 | self.clock = clock 21 | 22 | self._logger = logging.getLogger(f"resonate:{cid}:{id}") 23 | self._logger.setLevel(level) 24 | self._logger.propagate = False 25 | 26 | if not self._logger.handlers: 27 | formatter = DSTFormatter("[%(asctime)s] [%(levelname)s] [%(cid)s] [%(id)s] %(message)s") 28 | handler = logging.StreamHandler() 29 | handler.setFormatter(formatter) 30 | self._logger.addHandler(handler) 31 | 32 | def _log(self, level: int, msg: Any, *args: Any, **kwargs: Any) -> None: 33 | self._logger.log(level, msg, *args, **{**kwargs, "extra": {"cid": self.cid, "id": self.id, "time": self.clock.time()}}) 34 | 35 | def log(self, level: int, msg: Any, *args: Any, **kwargs: Any) -> None: 36 | self._log(level, msg, *args, **kwargs) 37 | 38 | def debug(self, msg: Any, *args: Any, **kwargs: Any) -> None: 39 | self._log(logging.DEBUG, msg, *args, **kwargs) 40 | 41 | def info(self, msg: Any, *args: Any, **kwargs: Any) -> None: 42 | self._log(logging.INFO, msg, *args, **kwargs) 43 | 44 | def warning(self, msg: Any, *args: Any, **kwargs: Any) -> None: 45 | self._log(logging.WARNING, msg, *args, **kwargs) 46 | 47 | def error(self, msg: Any, *args: Any, **kwargs: Any) -> None: 48 | self._log(logging.ERROR, msg, *args, **kwargs) 49 | 50 | def critical(self, msg: Any, *args: Any, **kwargs: Any) -> None: 51 | self._log(logging.CRITICAL, msg, *args, **kwargs) 52 | -------------------------------------------------------------------------------- /resonate/errors/errors.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | from typing import Any 5 | 6 | 7 | class ResonateError(Exception): 8 | def __init__(self, mesg: str, code: float, details: Any = None) -> None: 9 | super().__init__(mesg) 10 | self.mesg = mesg 11 | self.code = code 12 | try: 13 | self.details = json.dumps(details, indent=2) if details else None 14 | except Exception: 15 | self.details = details 16 | 17 | def __str__(self) -> str: 18 | return f"[{self.code:09.5f}] {self.mesg}{'\n' + self.details if self.details else ''}" 19 | 20 | def __reduce__(self) -> str | tuple[Any, ...]: 21 | return (self.__class__, (self.mesg, self.code, self.details)) 22 | 23 | 24 | # Error codes 100-199 25 | 26 | 27 | class ResonateStoreError(ResonateError): 28 | def __init__(self, mesg: str, code: int, details: Any = None) -> None: 29 | super().__init__(mesg, float(f"{100}.{code}"), details) 30 | 31 | def __reduce__(self) -> str | tuple[Any, ...]: 32 | return (self.__class__, (self.mesg, self.code, self.details)) 33 | 34 | 35 | # Error codes 200-299 36 | 37 | 38 | class ResonateCanceledError(ResonateError): 39 | def __init__(self, promise_id: str) -> None: 40 | super().__init__(f"Promise {promise_id} canceled", 200) 41 | self.promise_id = promise_id 42 | 43 | def __reduce__(self) -> str | tuple[Any, ...]: 44 | return (self.__class__, (self.promise_id,)) 45 | 46 | 47 | class ResonateTimedoutError(ResonateError): 48 | def __init__(self, promise_id: str, timeout: float) -> None: 49 | super().__init__(f"Promise {promise_id} timedout at {timeout}", 201) 50 | self.promise_id = promise_id 51 | self.timeout = timeout 52 | 53 | def __reduce__(self) -> str | tuple[Any, ...]: 54 | return (self.__class__, (self.promise_id, self.timeout)) 55 | 56 | 57 | # Error codes 300-399 58 | 59 | 60 | class ResonateShutdownError(ResonateError): 61 | def __init__(self, mesg: str) -> None: 62 | super().__init__(mesg, 300) 63 | 64 | def __reduce__(self) -> str | tuple[Any, ...]: 65 | return (self.__class__, (self.mesg,)) 66 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import os 5 | import random 6 | import sys 7 | from typing import TYPE_CHECKING 8 | 9 | import pytest 10 | 11 | from resonate.message_sources import Poller 12 | from resonate.stores import LocalStore, RemoteStore 13 | 14 | if TYPE_CHECKING: 15 | from collections.abc import Generator 16 | 17 | from resonate.models.message_source import MessageSource 18 | from resonate.models.store import Store 19 | 20 | 21 | def pytest_configure() -> None: 22 | logging.basicConfig(level=logging.ERROR) # set log levels very high for tests 23 | 24 | 25 | def pytest_addoption(parser: pytest.Parser) -> None: 26 | parser.addoption("--seed", action="store") 27 | parser.addoption("--steps", action="store") 28 | 29 | 30 | # DST fixtures 31 | 32 | 33 | @pytest.fixture 34 | def seed(request: pytest.FixtureRequest) -> str: 35 | seed = request.config.getoption("--seed") 36 | 37 | if not isinstance(seed, str): 38 | return str(random.randint(0, sys.maxsize)) 39 | 40 | return seed 41 | 42 | 43 | @pytest.fixture 44 | def steps(request: pytest.FixtureRequest) -> int: 45 | steps = request.config.getoption("--steps") 46 | 47 | if isinstance(steps, str): 48 | try: 49 | return int(steps) 50 | except ValueError: 51 | pass 52 | 53 | return 10000 54 | 55 | 56 | @pytest.fixture 57 | def log_level(request: pytest.FixtureRequest) -> int: 58 | level = request.config.getoption("--log-level") 59 | 60 | if isinstance(level, str) and level.isdigit(): 61 | return int(level) 62 | 63 | match str(level).lower(): 64 | case "critical": 65 | return logging.CRITICAL 66 | case "error": 67 | return logging.ERROR 68 | case "warning" | "warn": 69 | return logging.WARNING 70 | case "info": 71 | return logging.INFO 72 | case "debug": 73 | return logging.DEBUG 74 | case _: 75 | return logging.NOTSET 76 | 77 | 78 | # Store fixtures 79 | 80 | 81 | @pytest.fixture( 82 | scope="module", 83 | params=[LocalStore, RemoteStore] if "RESONATE_HOST" in os.environ else [LocalStore], 84 | ) 85 | def store(request: pytest.FixtureRequest) -> Store: 86 | return request.param() 87 | 88 | 89 | @pytest.fixture 90 | def message_source(store: Store) -> Generator[MessageSource]: 91 | match store: 92 | case LocalStore(): 93 | ms = store.message_source(group="default", id="test") 94 | case _: 95 | assert isinstance(store, RemoteStore) 96 | ms = Poller(group="default", id="test") 97 | 98 | # start the message source 99 | ms.start() 100 | 101 | yield ms 102 | 103 | # stop the message source 104 | ms.stop() 105 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | permissions: 4 | contents: read 5 | 6 | on: 7 | workflow_dispatch: 8 | push: 9 | branches: [main] 10 | paths-ignore: 11 | - README.md 12 | pull_request: 13 | branches: [main] 14 | paths-ignore: 15 | - README.md 16 | 17 | jobs: 18 | run: 19 | runs-on: ${{ matrix.os }} 20 | timeout-minutes: 45 21 | 22 | env: 23 | RESONATE_HOST: http://localhost 24 | 25 | strategy: 26 | fail-fast: true 27 | matrix: 28 | os: [ubuntu-latest, macos-latest] 29 | python-version: [3.12, 3.13] 30 | server-version: [main, latest] 31 | 32 | steps: 33 | - uses: actions/checkout@v4 34 | 35 | - name: install uv 36 | uses: astral-sh/setup-uv@v5 37 | with: 38 | version: 0.5.23 39 | 40 | - name: set up python 41 | uses: actions/setup-python@v5 42 | with: 43 | python-version: ${{matrix.python-version}} 44 | 45 | - name: install 46 | run: uv sync --dev 47 | 48 | - name: check linting 49 | run: uv run ruff check 50 | 51 | - name: check types 52 | run: uv run pyright 53 | 54 | - name: set up go 55 | uses: actions/setup-go@v5 56 | with: 57 | go-version: "1.23.0" 58 | cache: false # turn caching off to avoid warning 59 | 60 | - name: Get latest Resonate release tag (if needed) 61 | if: matrix.server-version == 'latest' 62 | id: get-resonate-tag 63 | run: | 64 | LATEST_TAG=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ 65 | https://api.github.com/repos/resonatehq/resonate/releases/latest | jq -r .tag_name) 66 | echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT 67 | 68 | - name: checkout resonate repository 69 | uses: actions/checkout@v4 70 | with: 71 | repository: resonatehq/resonate 72 | path: server 73 | ref: ${{ matrix.server-version == 'latest' && steps.get-resonate-tag.outputs.tag || 'main' }} 74 | 75 | - name: build resonate 76 | run: go build -o resonate 77 | working-directory: server 78 | 79 | - name: start resonate server 80 | run: ./resonate serve --system-signal-timeout 0.1s & 81 | working-directory: server 82 | 83 | - name: run tests with coverage 84 | id: coverage 85 | if: ${{ matrix.server-version == 'latest' && matrix.python-version == '3.13' && matrix.os == 'ubuntu-latest' && github.ref == 'refs/heads/main' }} 86 | run: uv run pytest --cov --cov-branch --cov-report=xml 87 | 88 | - name: Upload coverage reports to Codecov 89 | if: ${{ steps.coverage.outcome != 'skipped' }} 90 | uses: codecov/codecov-action@v5 91 | with: 92 | token: ${{ secrets.CODECOV_TOKEN }} 93 | 94 | - name: run tests 95 | if: ${{ steps.coverage.outcome == 'skipped' }} 96 | run: uv run pytest 97 | -------------------------------------------------------------------------------- /resonate/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import os 5 | import threading 6 | import traceback 7 | import urllib.parse 8 | from functools import wraps 9 | from importlib.metadata import version 10 | from typing import TYPE_CHECKING, Any 11 | 12 | if TYPE_CHECKING: 13 | from collections.abc import Callable 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def exit_on_exception[**P, R](func: Callable[P, R]) -> Callable[P, R]: 19 | @wraps(func) 20 | def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: 21 | try: 22 | return func(*args, **kwargs) 23 | except Exception as e: 24 | body = f""" 25 | An exception occurred in the resonate python sdk. 26 | 27 | **Version** 28 | ``` 29 | {resonate_version()} 30 | ``` 31 | 32 | **Thread** 33 | ``` 34 | {threading.current_thread().name} 35 | ``` 36 | 37 | **Exception** 38 | ``` 39 | {e!r} 40 | ``` 41 | 42 | **Stacktrace** 43 | ``` 44 | {traceback.format_exc()} 45 | ``` 46 | 47 | **Additional context** 48 | Please provide any additional context that might help us debug this issue. 49 | """ 50 | 51 | format = """ 52 | Resonate encountered an unexpected exception and had to shut down. 53 | 54 | 📦 Version: %s 55 | 🧵 Thread: %s 56 | ❌ Exception: %s 57 | 📄 Stacktrace: 58 | ───────────────────────────────────────────────────────────────────── 59 | %s 60 | ───────────────────────────────────────────────────────────────────── 61 | 62 | 🔗 Please help us make resonate better by reporting this issue: 63 | https://github.com/resonatehq/resonate-sdk-py/issues/new?body=%s 64 | """ 65 | logger.critical( 66 | format, 67 | resonate_version(), 68 | threading.current_thread().name, 69 | repr(e), 70 | traceback.format_exc(), 71 | urllib.parse.quote(body), 72 | ) 73 | 74 | # Exit the process with a non-zero exit code, this kills all 75 | # threads 76 | os._exit(1) 77 | 78 | return wrapper 79 | 80 | 81 | def resonate_version() -> str: 82 | try: 83 | return version("resonate-sdk") 84 | except Exception: 85 | return "unknown" 86 | 87 | 88 | def format_args_and_kwargs(args: tuple[Any, ...], kwargs: dict[str, Any]) -> str: 89 | parts = [repr(arg) for arg in args] 90 | parts += [f"{k}={v!r}" for k, v in kwargs.items()] 91 | return ", ".join(parts) 92 | 93 | 94 | def truncate(s: str, n: int) -> str: 95 | if len(s) > n: 96 | return s[:n] + "..." 97 | return s 98 | 99 | 100 | def is_url(string: str) -> bool: 101 | try: 102 | result = urllib.parse.urlparse(string) 103 | # A valid URL usually has at least a scheme and netloc 104 | return all([result.scheme, result.netloc]) 105 | except ValueError: 106 | return False 107 | -------------------------------------------------------------------------------- /resonate/registry.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import inspect 4 | from collections.abc import Callable 5 | from typing import overload 6 | 7 | 8 | class Registry: 9 | def __init__(self) -> None: 10 | self._forward_registry: dict[str, dict[int, tuple[str, Callable, int]]] = {} 11 | self._reverse_registry: dict[Callable, tuple[str, Callable, int]] = {} 12 | 13 | def add(self, func: Callable, name: str | None = None, version: int = 1) -> None: 14 | if not inspect.isfunction(func): 15 | msg = "provided callable must be a function" 16 | raise ValueError(msg) 17 | 18 | if not name and func.__name__ == "": 19 | msg = "name required when registering a lambda function" 20 | raise ValueError(msg) 21 | 22 | if not version > 0: 23 | msg = "provided version must be greater than zero" 24 | raise ValueError(msg) 25 | 26 | name = name or func.__name__ 27 | if version in self._forward_registry.get(name, {}) or func in self._reverse_registry: 28 | msg = f"function {name} already registered" 29 | raise ValueError(msg) 30 | 31 | item = (name, func, version) 32 | self._forward_registry.setdefault(name, {})[version] = item 33 | self._reverse_registry[func] = item 34 | 35 | @overload 36 | def get(self, func: str, version: int = 0) -> tuple[str, Callable, int]: ... 37 | @overload 38 | def get(self, func: Callable, version: int = 0) -> tuple[str, Callable, int]: ... 39 | def get(self, func: str | Callable, version: int = 0) -> tuple[str, Callable, int]: 40 | if func not in (self._forward_registry if isinstance(func, str) else self._reverse_registry): 41 | msg = f"function {func if isinstance(func, str) else getattr(func, '__name__', 'unknown')} not found in registry" 42 | raise ValueError(msg) 43 | 44 | if version != 0 and version not in (self._forward_registry[func] if isinstance(func, str) else (self._reverse_registry[func][2],)): 45 | msg = f"function {func if isinstance(func, str) else getattr(func, '__name__', 'unknown')} version {version} not found in registry" 46 | raise ValueError(msg) 47 | 48 | match func: 49 | case str(): 50 | vers = max(self._forward_registry[func]) if version == 0 else version 51 | return self._forward_registry[func][vers] 52 | 53 | case Callable(): 54 | return self._reverse_registry[func] 55 | 56 | @overload 57 | def latest(self, func: str, default: int = 1) -> int: ... 58 | @overload 59 | def latest(self, func: Callable, default: int = 1) -> int: ... 60 | def latest(self, func: str | Callable, default: int = 1) -> int: 61 | match func: 62 | case str(): 63 | return max(self._forward_registry.get(func, [default])) 64 | case Callable(): 65 | _, _, version = self._reverse_registry.get(func, (None, None, default)) 66 | return version 67 | -------------------------------------------------------------------------------- /tests/test_equivalencies.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import itertools 4 | from typing import TYPE_CHECKING, Any 5 | 6 | import pytest 7 | 8 | from resonate.registry import Registry 9 | from tests.runners import LocalContext, RemoteContext, ResonateLFXRunner, ResonateRFXRunner, ResonateRunner, Runner, SimpleRunner 10 | 11 | if TYPE_CHECKING: 12 | from collections.abc import Callable, Generator 13 | 14 | 15 | # Functions 16 | 17 | 18 | def fib(ctx: LocalContext | RemoteContext, n: int) -> Generator[Any, Any, int]: 19 | if n <= 1: 20 | return n 21 | 22 | p1 = yield ctx.rfi(fib, n - 1).options(id=f"fib({n - 1})") 23 | p2 = yield ctx.rfi(fib, n - 2).options(id=f"fib({n - 2})") 24 | 25 | v1 = yield p1 26 | v2 = yield p2 27 | 28 | return v1 + v2 29 | 30 | 31 | def fac(ctx: LocalContext | RemoteContext, n: int) -> Generator[Any, Any, int]: 32 | if n <= 1: 33 | return 1 34 | 35 | return n * (yield ctx.rfc(fac, n - 1).options(id=f"fac({n - 1})")) 36 | 37 | 38 | def gcd(ctx: LocalContext | RemoteContext, a: int, b: int) -> Generator[Any, Any, int]: 39 | if b == 0: 40 | return a 41 | 42 | return (yield ctx.rfc(gcd, b, a % b).options(id=f"gcd({b},{a % b})")) 43 | 44 | 45 | def rpt(ctx: LocalContext | RemoteContext, s: str, n: int) -> Generator[Any, Any, Any]: 46 | v = "" 47 | p = yield ctx.lfi(idv, s) 48 | 49 | for _ in range(n): 50 | v += yield ctx.lfc(idp, p) 51 | 52 | return v 53 | 54 | 55 | def idv(c: LocalContext | RemoteContext, x: Any) -> Any: 56 | return x 57 | 58 | 59 | def idp(c: LocalContext | RemoteContext, p: Any) -> Any: 60 | return (yield p) 61 | 62 | 63 | # Tests 64 | 65 | 66 | @pytest.fixture(scope="module") 67 | def registry() -> Registry: 68 | registry = Registry() 69 | registry.add(fib, "fib") 70 | registry.add(fac, "fac") 71 | registry.add(gcd, "gcd") 72 | registry.add(rpt, "rpt") 73 | registry.add(idv, "idv") 74 | registry.add(idp, "idp") 75 | 76 | return registry 77 | 78 | 79 | @pytest.fixture 80 | def runners(registry: Registry) -> tuple[Runner, ...]: 81 | return ( 82 | SimpleRunner(), 83 | ResonateRunner(registry), 84 | ResonateLFXRunner(registry), 85 | ResonateRFXRunner(registry), 86 | ) 87 | 88 | 89 | @pytest.mark.parametrize( 90 | ("id", "func", "args", "kwargs"), 91 | [ 92 | *((f"fib({n})", fib, (n,), {}) for n in range(20)), 93 | *((f"fac({n})", fac, (n,), {}) for n in range(20)), 94 | *((f"gcd({a},{b})", gcd, (a, b), {}) for a, b in itertools.product(range(10), repeat=2)), 95 | *((f"rpt({s},{n})", rpt, (s, n), {}) for s, n in itertools.product("abc", range(10))), 96 | ], 97 | ) 98 | def test_equivalencies(runners: tuple[Runner, ...], id: str, func: Callable, args: Any, kwargs: Any) -> None: 99 | # a promise is not json serializable so we must skip ResonateRFXRunner and rpt 100 | results = [r.run(id, func, *args, **kwargs) for r in runners if not (isinstance(r, ResonateRFXRunner) and func == rpt)] 101 | assert all(x == results[0] for x in results[1:]) 102 | -------------------------------------------------------------------------------- /resonate/graph.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | if TYPE_CHECKING: 6 | from collections.abc import Callable, Generator 7 | 8 | 9 | class Graph[T]: 10 | def __init__(self, id: str, root: T) -> None: 11 | self.id = id 12 | self.root = Node(id, root) 13 | 14 | def find(self, func: Callable[[Node[T]], bool], edge: str = "default") -> Node[T] | None: 15 | return self.root.find(func, edge) 16 | 17 | def filter(self, func: Callable[[Node[T]], bool], edge: str = "default") -> Generator[Node[T], None, None]: 18 | return self.root.filter(func, edge) 19 | 20 | def traverse(self, edge: str = "default") -> Generator[Node[T], None, None]: 21 | return self.root.traverse(edge) 22 | 23 | def traverse_with_level(self, edge: str = "default") -> Generator[tuple[Node[T], int], None, None]: 24 | return self.root.traverse_with_level(edge) 25 | 26 | 27 | class Node[T]: 28 | def __init__(self, id: str, value: T) -> None: 29 | self.id = id 30 | self._value = value 31 | self._edges: dict[str, list[Node[T]]] = {} 32 | 33 | def __repr__(self) -> str: 34 | edges = {e: [v.id for v in v] for e, v in self._edges.items()} 35 | return f"Node({self.id}, {self.value}, {edges})" 36 | 37 | @property 38 | def value(self) -> T: 39 | return self._value 40 | 41 | def transition(self, value: T) -> None: 42 | self._value = value 43 | 44 | def add_edge(self, node: Node[T], edge: str = "default") -> None: 45 | self._edges.setdefault(edge, []).append(node) 46 | 47 | def rmv_edge(self, node: Node[T], edge: str = "default") -> None: 48 | self._edges[edge].remove(node) 49 | 50 | def has_edge(self, node: Node[T], edge: str = "default") -> bool: 51 | return node in self._edges.get(edge, []) 52 | 53 | def get_edge(self, edge: str = "default") -> list[Node]: 54 | return self._edges.get(edge, []) 55 | 56 | def find(self, func: Callable[[Node[T]], bool], edge: str = "default") -> Node[T] | None: 57 | for node in self.traverse(edge): 58 | if func(node): 59 | return node 60 | 61 | return None 62 | 63 | def filter(self, func: Callable[[Node[T]], bool], edge: str = "default") -> Generator[Node[T], None, None]: 64 | for node in self.traverse(edge): 65 | if func(node): 66 | yield node 67 | 68 | def traverse(self, edge: str = "default") -> Generator[Node[T], None, None]: 69 | for node, _ in self._traverse(edge): 70 | yield node 71 | 72 | def traverse_with_level(self, edge: str = "default") -> Generator[tuple[Node[T], int], None, None]: 73 | return self._traverse(edge) 74 | 75 | def _traverse(self, edge: str, visited: set[str] | None = None, level: int = 0) -> Generator[tuple[Node[T], int], None, None]: 76 | if visited is None: 77 | visited = set() 78 | 79 | if self.id in visited: 80 | return 81 | 82 | visited.add(self.id) 83 | yield self, level 84 | 85 | for node in self._edges.get(edge, []): 86 | yield from node._traverse(edge, visited, level + 1) # noqa: SLF001 87 | -------------------------------------------------------------------------------- /resonate/models/store.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import TYPE_CHECKING, Protocol, runtime_checkable 4 | 5 | if TYPE_CHECKING: 6 | from resonate.models.callback import Callback 7 | from resonate.models.durable_promise import DurablePromise 8 | from resonate.models.encoder import Encoder 9 | from resonate.models.schedules import Schedule 10 | from resonate.models.task import Task 11 | 12 | 13 | @runtime_checkable 14 | class Store(Protocol): 15 | @property 16 | def encoder(self) -> Encoder[str | None, str | None]: ... 17 | 18 | @property 19 | def promises(self) -> PromiseStore: ... 20 | 21 | @property 22 | def tasks(self) -> TaskStore: ... 23 | 24 | @property 25 | def schedules(self) -> ScheduleStore: ... 26 | 27 | 28 | class PromiseStore(Protocol): 29 | def get( 30 | self, 31 | id: str, 32 | ) -> DurablePromise: ... 33 | 34 | def create( 35 | self, 36 | id: str, 37 | timeout: int, 38 | *, 39 | ikey: str | None = None, 40 | strict: bool = False, 41 | headers: dict[str, str] | None = None, 42 | data: str | None = None, 43 | tags: dict[str, str] | None = None, 44 | ) -> DurablePromise: ... 45 | 46 | def create_with_task( 47 | self, 48 | id: str, 49 | timeout: int, 50 | pid: str, 51 | ttl: int, 52 | *, 53 | ikey: str | None = None, 54 | strict: bool = False, 55 | headers: dict[str, str] | None = None, 56 | data: str | None = None, 57 | tags: dict[str, str] | None = None, 58 | ) -> tuple[DurablePromise, Task | None]: ... 59 | 60 | def resolve( 61 | self, 62 | id: str, 63 | *, 64 | ikey: str | None = None, 65 | strict: bool = False, 66 | headers: dict[str, str] | None = None, 67 | data: str | None = None, 68 | ) -> DurablePromise: ... 69 | 70 | def reject( 71 | self, 72 | id: str, 73 | *, 74 | ikey: str | None = None, 75 | strict: bool = False, 76 | headers: dict[str, str] | None = None, 77 | data: str | None = None, 78 | ) -> DurablePromise: ... 79 | 80 | def cancel( 81 | self, 82 | id: str, 83 | *, 84 | ikey: str | None = None, 85 | strict: bool = False, 86 | headers: dict[str, str] | None = None, 87 | data: str | None = None, 88 | ) -> DurablePromise: ... 89 | 90 | def callback( 91 | self, 92 | promise_id: str, 93 | root_promise_id: str, 94 | recv: str, 95 | timeout: int, 96 | ) -> tuple[DurablePromise, Callback | None]: ... 97 | 98 | def subscribe( 99 | self, 100 | id: str, 101 | promise_id: str, 102 | recv: str, 103 | timeout: int, 104 | ) -> tuple[DurablePromise, Callback | None]: ... 105 | 106 | 107 | class TaskStore(Protocol): 108 | def claim( 109 | self, 110 | id: str, 111 | counter: int, 112 | pid: str, 113 | ttl: int, 114 | ) -> tuple[DurablePromise, DurablePromise | None]: ... 115 | 116 | def complete( 117 | self, 118 | id: str, 119 | counter: int, 120 | ) -> bool: ... 121 | 122 | def heartbeat( 123 | self, 124 | pid: str, 125 | ) -> int: ... 126 | 127 | 128 | class ScheduleStore(Protocol): 129 | def create( 130 | self, 131 | id: str, 132 | cron: str, 133 | promise_id: str, 134 | promise_timeout: int, 135 | *, 136 | ikey: str | None = None, 137 | description: str | None = None, 138 | tags: dict[str, str] | None = None, 139 | promise_headers: dict[str, str] | None = None, 140 | promise_data: str | None = None, 141 | promise_tags: dict[str, str] | None = None, 142 | ) -> Schedule: ... 143 | 144 | def get(self, id: str) -> Schedule: ... 145 | 146 | def delete(self, id: str) -> None: ... 147 | -------------------------------------------------------------------------------- /tests/test_encoders.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from types import NoneType 4 | from typing import Any 5 | 6 | import pytest 7 | 8 | from resonate.encoders import Base64Encoder, HeaderEncoder, JsonEncoder, JsonPickleEncoder, PairEncoder 9 | 10 | 11 | @pytest.mark.parametrize( 12 | "value", 13 | ["", "foo", "bar", "baz", None], 14 | ) 15 | def test_base64_enconder(value: str) -> None: 16 | encoder = Base64Encoder() 17 | encoded = encoder.encode(value) 18 | assert isinstance(encoded, (str, NoneType)) 19 | 20 | decoded = encoder.decode(encoded) 21 | assert type(decoded) is type(value) 22 | assert decoded == value 23 | 24 | 25 | @pytest.mark.parametrize( 26 | "value", 27 | [ 28 | "", 29 | "foo", 30 | "bar", 31 | "baz", 32 | 0, 33 | 1, 34 | 2, 35 | [], 36 | ["foo", "bar", "baz"], 37 | [0, 1, 2], 38 | {}, 39 | {"foo": "foo", "bar": "bar", "baz": "baz"}, 40 | {"foo": 0, "bar": 1, "baz": 2}, 41 | None, 42 | (), 43 | ("foo", "bar", "baz"), 44 | (0, 1, 2), 45 | Exception("foo"), 46 | ValueError("bar"), 47 | BaseException("baz"), 48 | ], 49 | ) 50 | def test_json_encoder(value: Any) -> None: 51 | encoder = JsonEncoder() 52 | encoded = encoder.encode(value) 53 | assert isinstance(encoded, (str, NoneType)) 54 | 55 | match decoded := encoder.decode(encoded): 56 | case BaseException() as e: 57 | # all exceptions are flattened to Exception 58 | assert isinstance(decoded, Exception) 59 | assert str(e) == str(value) 60 | case list(items): 61 | # tuples are converted to lists 62 | assert isinstance(value, (list, tuple)) 63 | assert items == list(value) 64 | case _: 65 | assert type(decoded) is type(value) 66 | assert decoded == value 67 | 68 | 69 | @pytest.mark.parametrize( 70 | "value", 71 | [ 72 | "", 73 | "foo", 74 | "bar", 75 | "baz", 76 | 0, 77 | 1, 78 | 2, 79 | [], 80 | ["foo", "bar", "baz"], 81 | [0, 1, 2], 82 | {}, 83 | {"foo": "foo", "bar": "bar", "baz": "baz"}, 84 | {"foo": 0, "bar": 1, "baz": 2}, 85 | None, 86 | (), 87 | ("foo", "bar", "baz"), 88 | (0, 1, 2), 89 | Exception("foo"), 90 | ValueError("bar"), 91 | BaseException("baz"), 92 | ], 93 | ) 94 | def test_jsonpickle_encoder(value: Any) -> None: 95 | encoder = JsonPickleEncoder() 96 | encoded = encoder.encode(value) 97 | assert isinstance(encoded, (str, NoneType)) 98 | 99 | decoded = encoder.decode(encoded) 100 | assert type(decoded) is type(value) 101 | 102 | if not isinstance(value, BaseException): 103 | assert decoded == value 104 | 105 | 106 | @pytest.mark.parametrize( 107 | "value", 108 | [ 109 | "", 110 | "foo", 111 | "bar", 112 | "baz", 113 | 0, 114 | 1, 115 | 2, 116 | [], 117 | ["foo", "bar", "baz"], 118 | [0, 1, 2], 119 | {}, 120 | {"foo": "foo", "bar": "bar", "baz": "baz"}, 121 | {"foo": 0, "bar": 1, "baz": 2}, 122 | None, 123 | (), 124 | ("foo", "bar", "baz"), 125 | (0, 1, 2), 126 | Exception("foo"), 127 | ValueError("bar"), 128 | ], 129 | ) 130 | def test_default_encoder(value: Any) -> None: 131 | encoder = PairEncoder(HeaderEncoder("resonate:format-py", JsonPickleEncoder()), JsonEncoder()) 132 | headers, encoded = encoder.encode(value) 133 | assert isinstance(headers, dict) 134 | assert "resonate:format-py" in headers 135 | assert isinstance(headers["resonate:format-py"], str) 136 | assert isinstance(encoded, (str, NoneType)) 137 | 138 | # primary decoder 139 | for decoded in (encoder.decode((headers, encoded)), JsonPickleEncoder().decode(headers["resonate:format-py"])): 140 | assert type(decoded) is type(value) 141 | if not isinstance(value, Exception): 142 | assert decoded == value 143 | 144 | # backup decoder 145 | match decoded := JsonEncoder().decode(encoded): 146 | case BaseException() as e: 147 | # all exceptions are flattened to Exception 148 | assert isinstance(decoded, Exception) 149 | assert str(e) == str(value) 150 | case list(items): 151 | # tuples are converted to lists 152 | assert isinstance(value, (list, tuple)) 153 | assert items == list(value) 154 | case _: 155 | assert type(decoded) is type(value) 156 | assert decoded == value 157 | -------------------------------------------------------------------------------- /resonate/message_sources/poller.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import os 5 | import queue 6 | import time 7 | from threading import Thread 8 | from typing import TYPE_CHECKING, Any 9 | 10 | import requests 11 | 12 | from resonate.encoders import JsonEncoder 13 | from resonate.models.message import InvokeMesg, Mesg, NotifyMesg, ResumeMesg 14 | from resonate.utils import exit_on_exception 15 | 16 | if TYPE_CHECKING: 17 | from resonate.models.encoder import Encoder 18 | 19 | logger = logging.getLogger(__name__) 20 | 21 | 22 | class Poller: 23 | def __init__( 24 | self, 25 | group: str, 26 | id: str, 27 | host: str | None = None, 28 | port: str | None = None, 29 | auth: tuple[str, str] | None = None, 30 | timeout: float | None = None, 31 | encoder: Encoder[Any, str] | None = None, 32 | ) -> None: 33 | self._messages = queue.Queue[Mesg | None]() 34 | self._group = group 35 | self._id = id 36 | self._host = host or os.getenv("RESONATE_HOST_MESSAGE_SOURCE", os.getenv("RESONATE_HOST", "http://localhost")) 37 | self._port = port or os.getenv("RESONATE_PORT_MESSAGE_SOURCE", "8002") 38 | self._auth = auth or ((os.getenv("RESONATE_USERNAME", ""), os.getenv("RESONATE_PASSWORD", "")) if "RESONATE_USERNAME" in os.environ else None) 39 | self._timeout = timeout 40 | self._encoder = encoder or JsonEncoder() 41 | self._thread = Thread(name="message-source::poller", target=self.loop, daemon=True) 42 | self._stopped = False 43 | 44 | @property 45 | def url(self) -> str: 46 | return f"{self._host}:{self._port}/{self._group}/{self._id}" 47 | 48 | @property 49 | def unicast(self) -> str: 50 | return f"poll://uni@{self._group}/{self._id}" 51 | 52 | @property 53 | def anycast(self) -> str: 54 | return f"poll://any@{self._group}/{self._id}" 55 | 56 | def start(self) -> None: 57 | if not self._thread.is_alive(): 58 | self._thread.start() 59 | 60 | def stop(self) -> None: 61 | # signal to consumer to disconnect 62 | self._messages.put(None) 63 | 64 | # TODO(avillega): Couldn't come up with a nice way of stoping this thread 65 | # iter_lines is blocking and request.get is also blocking, this makes it so 66 | # the only way to stop it is waiting for a timeout on the request itself 67 | # which could never happen. 68 | 69 | # This shutdown is only respected when the poller is instantiated with a timeout 70 | # value, which is not the default. This is still useful for tests. 71 | self._stopped = True 72 | 73 | def enqueue(self, mesg: Mesg) -> None: 74 | self._messages.put(mesg) 75 | 76 | def next(self) -> Mesg | None: 77 | return self._messages.get() 78 | 79 | @exit_on_exception 80 | def loop(self) -> None: 81 | delay = 5 82 | while not self._stopped: 83 | try: 84 | with requests.get(self.url, auth=self._auth, stream=True, timeout=self._timeout) as res: 85 | res.raise_for_status() 86 | 87 | for line in res.iter_lines(chunk_size=None, decode_unicode=True): 88 | assert isinstance(line, str), "line must be a string" 89 | if msg := self._process_line(line): 90 | self._messages.put(msg) 91 | 92 | except requests.exceptions.Timeout: 93 | logger.warning("Networking. Cannot connect to %s:%s. Retrying in %s sec.", self._host, self._port, delay) 94 | time.sleep(delay) 95 | continue 96 | except requests.exceptions.RequestException: 97 | logger.warning("Networking. Cannot connect to %s:%s. Retrying in %s sec.", self._host, self._port, delay) 98 | time.sleep(delay) 99 | continue 100 | except Exception: 101 | logger.warning("Networking. Cannot connect to %s:%s. Retrying in %s sec.", self._host, self._port, delay) 102 | time.sleep(delay) 103 | continue 104 | 105 | def _process_line(self, line: str) -> Mesg | None: 106 | if not line: 107 | return None 108 | 109 | stripped = line.strip() 110 | if not stripped.startswith("data:"): 111 | return None 112 | 113 | d = self._encoder.decode(stripped[5:]) 114 | match d["type"]: 115 | case "invoke": 116 | return InvokeMesg(type="invoke", task=d["task"]) 117 | case "resume": 118 | return ResumeMesg(type="resume", task=d["task"]) 119 | case "notify": 120 | return NotifyMesg(type="notify", promise=d["promise"]) 121 | case _: 122 | # Unknown message type 123 | return None 124 | -------------------------------------------------------------------------------- /.github/workflows/dst.yml: -------------------------------------------------------------------------------- 1 | name: dst 2 | 3 | permissions: 4 | contents: read 5 | issues: write 6 | 7 | on: 8 | workflow_dispatch: 9 | inputs: 10 | seed: 11 | description: "seed" 12 | type: number 13 | steps: 14 | description: "steps" 15 | type: number 16 | schedule: 17 | - cron: "*/20 * * * *" # every 20 mins 18 | 19 | jobs: 20 | values: 21 | runs-on: ubuntu-22.04 22 | steps: 23 | - id: seed 24 | name: Set random seed 25 | run: echo seed=$RANDOM >> $GITHUB_OUTPUT 26 | outputs: 27 | seed: ${{ inputs.seed || steps.seed.outputs.seed }} 28 | steps: ${{ inputs.steps || 10000 }} 29 | 30 | dst: 31 | runs-on: ${{ matrix.os }} 32 | needs: [values] 33 | timeout-minutes: 15 34 | 35 | strategy: 36 | fail-fast: true 37 | matrix: 38 | os: [ubuntu-latest, windows-latest, macos-latest] 39 | python-version: ["3.10", "3.11", "3.12", "3.13"] 40 | run: [1, 2] 41 | 42 | steps: 43 | - uses: actions/checkout@v4 44 | 45 | - name: Install uv 46 | uses: astral-sh/setup-uv@v5 47 | 48 | - name: Set up python 49 | uses: actions/setup-python@v5 50 | with: 51 | python-version: ${{matrix.python-version}} 52 | 53 | - name: Install resonate 54 | run: uv sync --dev 55 | 56 | - name: Run dst (seed=${{ needs.values.outputs.seed }}, steps=${{ needs.values.outputs.steps }}) 57 | run: uv run pytest -s --seed ${{ needs.values.outputs.seed }} --steps ${{ needs.values.outputs.steps }} tests/test_dst.py 2> logs.txt 58 | 59 | - name: Create issue if dst failed 60 | if: ${{ failure() && matrix.run == 1 }} 61 | uses: actions/github-script@v8 62 | with: 63 | script: | 64 | github.rest.issues.create({ 65 | owner: context.repo.owner, 66 | repo: context.repo.repo, 67 | title: `DST: ${{ needs.values.outputs.seed }}`, 68 | labels: ['dst'], 69 | body: `# DST Failed 70 | DST run failed for seed=${{ needs.values.outputs.seed }}, steps=${{ needs.values.outputs.steps }}, os=${{ matrix.os }}, version=${{ matrix.python-version }}. 71 | 72 | **Seed** 73 | ~~~ 74 | ${{ needs.values.outputs.seed }} 75 | ~~~ 76 | 77 | **Commit** 78 | ~~~ 79 | ${ process.env.GITHUB_SHA } 80 | ~~~ 81 | 82 | **Command** 83 | ~~~ 84 | uv run pytest -s --seed ${{ needs.values.outputs.seed }} --steps ${{ needs.values.outputs.steps }} tests/test_dst.py 85 | ~~~ 86 | 87 | [more details](${ process.env.GITHUB_SERVER_URL }/${ process.env.GITHUB_REPOSITORY }/actions/runs/${ process.env.GITHUB_RUN_ID })`, 88 | }) 89 | 90 | - uses: actions/upload-artifact@v4 91 | if: ${{ always() }} 92 | with: 93 | name: ${{ matrix.os }}-${{ matrix.python-version }}-logs-${{ matrix.run }} 94 | path: logs.txt 95 | 96 | diff: 97 | runs-on: ubuntu-22.04 98 | needs: [values, dst] 99 | 100 | strategy: 101 | fail-fast: false 102 | matrix: 103 | os: [ubuntu-latest, windows-latest, macos-latest] 104 | python-version: ["3.10", "3.11", "3.12", "3.13"] 105 | 106 | steps: 107 | - name: Download logs from run 1 108 | uses: actions/download-artifact@v4 109 | with: 110 | name: ${{ matrix.os }}-${{ matrix.python-version }}-logs-1 111 | path: logs-1.txt 112 | 113 | - name: Download logs from run 2 114 | uses: actions/download-artifact@v4 115 | with: 116 | name: ${{ matrix.os }}-${{ matrix.python-version }}-logs-2 117 | path: logs-2.txt 118 | 119 | - name: Diff 120 | run: diff logs-1.txt logs-2.txt 121 | 122 | - name: Create issue if diff 123 | if: ${{ failure() }} 124 | uses: actions/github-script@v8 125 | with: 126 | script: | 127 | github.rest.issues.create({ 128 | owner: context.repo.owner, 129 | repo: context.repo.repo, 130 | title: `DST: ${{ needs.values.outputs.seed }}`, 131 | labels: ['dst'], 132 | body: `# DST Failed 133 | Two DST runs produced different results for seed=${{ needs.values.outputs.seed }}, steps=${{ needs.values.outputs.steps }}, os=${{ matrix.os }}, version=${{ matrix.python-version }}. 134 | 135 | **Seed** 136 | ~~~ 137 | ${{ needs.values.outputs.seed }} 138 | ~~~ 139 | 140 | **Commit** 141 | ~~~ 142 | ${ process.env.GITHUB_SHA } 143 | ~~~ 144 | 145 | **Command** 146 | ~~~ 147 | uv run pytest -s --seed ${{ needs.values.outputs.seed }} --steps ${{ needs.values.outputs.steps }} tests/test_dst.py 148 | ~~~ 149 | 150 | [more details](${ process.env.GITHUB_SERVER_URL }/${ process.env.GITHUB_REPOSITORY }/actions/runs/${ process.env.GITHUB_RUN_ID })`, 151 | }) 152 | -------------------------------------------------------------------------------- /resonate/models/commands.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass, field 4 | from typing import TYPE_CHECKING, Any 5 | 6 | from resonate.options import Options 7 | 8 | if TYPE_CHECKING: 9 | from collections.abc import Callable 10 | 11 | from resonate.models.callback import Callback 12 | from resonate.models.convention import Convention 13 | from resonate.models.durable_promise import DurablePromise 14 | from resonate.models.result import Result 15 | from resonate.models.task import Task 16 | 17 | 18 | # Commands 19 | 20 | type Command = Invoke | Resume | Return | Receive | Retry | Listen | Notify 21 | 22 | 23 | @dataclass 24 | class Invoke: 25 | id: str 26 | conv: Convention 27 | timeout: float # absolute time in seconds 28 | func: Callable[..., Any] = field(repr=False) 29 | args: tuple[Any, ...] = field(default_factory=tuple) 30 | kwargs: dict[str, Any] = field(default_factory=dict) 31 | opts: Options = field(default_factory=Options, repr=False) 32 | promise: DurablePromise | None = None 33 | 34 | @property 35 | def cid(self) -> str: 36 | return self.id 37 | 38 | 39 | @dataclass 40 | class Resume: 41 | id: str 42 | cid: str 43 | promise: DurablePromise 44 | invoke: Invoke | None = None 45 | 46 | 47 | @dataclass 48 | class Return: 49 | id: str 50 | cid: str 51 | res: Result 52 | 53 | 54 | @dataclass 55 | class Receive: 56 | id: str 57 | cid: str 58 | res: CreatePromiseRes | CreatePromiseWithTaskRes | ResolvePromiseRes | RejectPromiseRes | CancelPromiseRes | CreateCallbackRes 59 | 60 | 61 | @dataclass 62 | class Listen: 63 | id: str 64 | 65 | @property 66 | def cid(self) -> str: 67 | return self.id 68 | 69 | 70 | @dataclass 71 | class Notify: 72 | id: str 73 | promise: DurablePromise 74 | 75 | @property 76 | def cid(self) -> str: 77 | return self.id 78 | 79 | 80 | @dataclass 81 | class Retry: 82 | id: str 83 | cid: str 84 | 85 | 86 | # Requests 87 | 88 | type Request = Network | Function | Delayed 89 | 90 | 91 | @dataclass 92 | class Network[T: CreatePromiseReq | ResolvePromiseReq | RejectPromiseReq | CancelPromiseReq | CreateCallbackReq | CreateSubscriptionReq]: 93 | id: str 94 | cid: str 95 | req: T 96 | 97 | 98 | @dataclass 99 | class Function: 100 | id: str 101 | cid: str 102 | func: Callable[[], Any] 103 | 104 | 105 | @dataclass 106 | class Delayed[T: Function | Retry]: 107 | item: T 108 | delay: float 109 | 110 | @property 111 | def id(self) -> str: 112 | return self.item.id 113 | 114 | 115 | @dataclass 116 | class CreatePromiseReq: 117 | id: str 118 | timeout: int 119 | ikey: str | None = None 120 | strict: bool = False 121 | headers: dict[str, str] | None = None 122 | data: str | None = None 123 | tags: dict[str, str] | None = None 124 | 125 | 126 | @dataclass 127 | class CreatePromiseRes: 128 | promise: DurablePromise 129 | 130 | 131 | @dataclass 132 | class CreatePromiseWithTaskReq: 133 | id: str 134 | timeout: int 135 | pid: str 136 | ttl: int 137 | ikey: str | None = None 138 | strict: bool = False 139 | headers: dict[str, str] | None = None 140 | data: str | None = None 141 | tags: dict[str, str] | None = None 142 | 143 | 144 | @dataclass 145 | class CreatePromiseWithTaskRes: 146 | promise: DurablePromise 147 | task: Task | None 148 | 149 | 150 | @dataclass 151 | class ResolvePromiseReq: 152 | id: str 153 | ikey: str | None = None 154 | strict: bool = False 155 | headers: dict[str, str] | None = None 156 | data: str | None = None 157 | 158 | 159 | @dataclass 160 | class ResolvePromiseRes: 161 | promise: DurablePromise 162 | 163 | 164 | @dataclass 165 | class RejectPromiseReq: 166 | id: str 167 | ikey: str | None = None 168 | strict: bool = False 169 | headers: dict[str, str] | None = None 170 | data: str | None = None 171 | 172 | 173 | @dataclass 174 | class RejectPromiseRes: 175 | promise: DurablePromise 176 | 177 | 178 | @dataclass 179 | class CancelPromiseReq: 180 | id: str 181 | ikey: str | None = None 182 | strict: bool = False 183 | headers: dict[str, str] | None = None 184 | data: str | None = None 185 | 186 | 187 | @dataclass 188 | class CancelPromiseRes: 189 | promise: DurablePromise 190 | 191 | 192 | @dataclass 193 | class CreateCallbackReq: 194 | promise_id: str 195 | root_promise_id: str 196 | timeout: int 197 | recv: str 198 | 199 | 200 | @dataclass 201 | class CreateCallbackRes: 202 | promise: DurablePromise 203 | callback: Callback | None 204 | 205 | 206 | @dataclass 207 | class CreateSubscriptionReq: 208 | id: str 209 | promise_id: str 210 | timeout: int 211 | recv: str 212 | 213 | 214 | @dataclass 215 | class CreateSubscriptionRes: 216 | promise: DurablePromise 217 | callback: Callback | None 218 | 219 | 220 | @dataclass 221 | class ClaimTaskReq: 222 | id: str 223 | counter: int 224 | pid: str 225 | ttl: int 226 | 227 | 228 | @dataclass 229 | class ClaimTaskRes: 230 | root: DurablePromise 231 | leaf: DurablePromise | None 232 | task: Task 233 | 234 | 235 | @dataclass 236 | class CompleteTaskReq: 237 | id: str 238 | counter: int 239 | 240 | 241 | @dataclass 242 | class CompleteTaskRes: 243 | pass 244 | 245 | 246 | @dataclass 247 | class HeartbeatTasksReq: 248 | pid: str 249 | 250 | 251 | @dataclass 252 | class HeartbeatTasksRes: 253 | affected: int 254 | -------------------------------------------------------------------------------- /resonate/options.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass, field 4 | from inspect import isgeneratorfunction 5 | from typing import TYPE_CHECKING, Any 6 | 7 | from resonate.encoders import HeaderEncoder, JsonEncoder, JsonPickleEncoder, NoopEncoder, PairEncoder 8 | from resonate.models.encoder import Encoder 9 | from resonate.models.retry_policy import RetryPolicy 10 | from resonate.retry_policies import Exponential, Never 11 | 12 | if TYPE_CHECKING: 13 | from collections.abc import Callable 14 | 15 | 16 | @dataclass(frozen=True) 17 | class Options: 18 | durable: bool = True 19 | encoder: Encoder[Any, str | None] | None = None 20 | id: str | None = None 21 | idempotency_key: str | Callable[[str], str] | None = lambda id: id 22 | non_retryable_exceptions: tuple[type[Exception], ...] = () 23 | retry_policy: RetryPolicy | Callable[[Callable], RetryPolicy] = lambda f: Never() if isgeneratorfunction(f) else Exponential() 24 | target: str = "default" 25 | tags: dict[str, str] = field(default_factory=dict) 26 | timeout: float = 31536000 # relative time in seconds, default 1 year 27 | version: int = 0 28 | 29 | def __post_init__(self) -> None: 30 | if not isinstance(self.durable, bool): 31 | msg = f"durable must be `bool`, got {type(self.durable).__name__}" 32 | raise TypeError(msg) 33 | 34 | if self.encoder is not None and not isinstance(self.encoder, Encoder): 35 | msg = f"encoder must be `Encoder | None`, got {type(self.encoder).__name__}" 36 | raise TypeError(msg) 37 | 38 | if self.id is not None and not isinstance(self.id, str): 39 | msg = f"id must be `str | None`, got {type(self.id).__name__}" 40 | raise TypeError(msg) 41 | 42 | if self.idempotency_key is not None and not (isinstance(self.idempotency_key, str) or callable(self.idempotency_key)): 43 | msg = f"idempotency_key must be `Callable | str | None`, got {type(self.idempotency_key).__name__}" 44 | raise TypeError(msg) 45 | 46 | if not isinstance(self.non_retryable_exceptions, tuple): 47 | msg = f"non_retryable_exceptions must be `tuple`, got {type(self.non_retryable_exceptions).__name__}" 48 | raise TypeError(msg) 49 | 50 | if not (isinstance(self.retry_policy, RetryPolicy) or callable(self.retry_policy)): 51 | msg = f"retry_policy must be `Callable | RetryPolicy | None`, got {type(self.retry_policy).__name__}" 52 | raise TypeError(msg) 53 | 54 | if not isinstance(self.target, str): 55 | msg = f"target must be `str`, got {type(self.target).__name__}" 56 | raise TypeError(msg) 57 | 58 | if not isinstance(self.tags, dict): 59 | msg = f"tags must be `dict`, got {type(self.tags).__name__}" 60 | raise TypeError(msg) 61 | 62 | if not isinstance(self.timeout, int | float): 63 | msg = f"timeout must be `float`, got {type(self.timeout).__name__}" 64 | raise TypeError(msg) 65 | 66 | if not isinstance(self.version, int): 67 | msg = f"version must be `int`, got {type(self.version).__name__}" 68 | raise TypeError(msg) 69 | 70 | if not (self.version >= 0): 71 | msg = "version must be greater than or equal to zero" 72 | raise ValueError(msg) 73 | if not (self.timeout >= 0): 74 | msg = "timeout must be greater than or equal to zero" 75 | raise ValueError(msg) 76 | 77 | def merge( 78 | self, 79 | *, 80 | durable: bool | None = None, 81 | encoder: Encoder[Any, str | None] | None = None, 82 | id: str | None = None, 83 | idempotency_key: str | Callable[[str], str] | None = None, 84 | non_retryable_exceptions: tuple[type[Exception], ...] | None = None, 85 | retry_policy: RetryPolicy | Callable[[Callable], RetryPolicy] | None = None, 86 | target: str | None = None, 87 | tags: dict[str, str] | None = None, 88 | timeout: float | None = None, 89 | version: int | None = None, 90 | ) -> Options: 91 | return Options( 92 | durable=durable if durable is not None else self.durable, 93 | encoder=encoder if encoder is not None else self.encoder, 94 | id=id if id is not None else self.id, 95 | idempotency_key=idempotency_key if idempotency_key is not None else self.idempotency_key, 96 | non_retryable_exceptions=non_retryable_exceptions if non_retryable_exceptions is not None else self.non_retryable_exceptions, 97 | retry_policy=retry_policy if retry_policy is not None else self.retry_policy, 98 | target=target if target is not None else self.target, 99 | tags=tags if tags is not None else self.tags, 100 | timeout=timeout if timeout is not None else self.timeout, 101 | version=version if version is not None else self.version, 102 | ) 103 | 104 | def get_encoder(self) -> Encoder[Any, tuple[dict[str, str] | None, str | None]]: 105 | l = NoopEncoder() if self.encoder else HeaderEncoder("resonate:format-py", JsonPickleEncoder()) 106 | r = self.encoder or JsonEncoder() 107 | return PairEncoder(l, r) 108 | 109 | def get_idempotency_key(self, id: str) -> str | None: 110 | return self.idempotency_key(id) if callable(self.idempotency_key) else self.idempotency_key 111 | 112 | def get_retry_policy(self, func: Callable) -> RetryPolicy: 113 | return self.retry_policy(func) if callable(self.retry_policy) else self.retry_policy 114 | -------------------------------------------------------------------------------- /tests/test_store_task.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import sys 4 | import time 5 | from typing import TYPE_CHECKING 6 | 7 | import pytest 8 | 9 | from resonate.errors import ResonateStoreError 10 | 11 | if TYPE_CHECKING: 12 | from collections.abc import Generator 13 | 14 | from resonate.models.message import TaskMesg 15 | from resonate.models.message_source import MessageSource 16 | from resonate.models.store import Store 17 | 18 | TICK_TIME = 1 19 | COUNTER = 0 20 | 21 | 22 | @pytest.fixture 23 | def task(store: Store, message_source: MessageSource) -> Generator[TaskMesg]: 24 | global COUNTER # noqa: PLW0603 25 | 26 | id = f"tid{COUNTER}" 27 | COUNTER += 1 28 | 29 | store.promises.create( 30 | id=id, 31 | timeout=sys.maxsize, 32 | tags={"resonate:invoke": "default"}, 33 | ) 34 | 35 | mesg = message_source.next() 36 | assert mesg 37 | assert mesg["type"] == "invoke" 38 | 39 | yield mesg["task"] 40 | store.promises.resolve(id=id) 41 | 42 | 43 | def test_case_5_transition_from_enqueue_to_claimed_via_claim(store: Store, task: TaskMesg) -> None: 44 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task5", ttl=sys.maxsize) 45 | 46 | 47 | def test_case_6_transition_from_enqueue_to_enqueue_via_claim(store: Store, task: TaskMesg) -> None: 48 | with pytest.raises(ResonateStoreError): 49 | store.tasks.claim( 50 | id=task["id"], 51 | counter=task["counter"] + 1, 52 | pid="task6", 53 | ttl=sys.maxsize, 54 | ) 55 | 56 | 57 | def test_case_8_transition_from_enqueue_to_enqueue_via_complete(store: Store, task: TaskMesg) -> None: 58 | with pytest.raises(ResonateStoreError): 59 | store.tasks.complete( 60 | id=task["id"], 61 | counter=task["counter"], 62 | ) 63 | 64 | 65 | def test_case_10_transition_from_enqueue_to_enqueue_via_hearbeat(store: Store, task: TaskMesg) -> None: 66 | assert store.tasks.heartbeat(pid="task10") == 0 67 | 68 | 69 | def test_case_12_transition_from_claimed_to_claimed_via_claim(store: Store, task: TaskMesg) -> None: 70 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task12", ttl=sys.maxsize) 71 | with pytest.raises(ResonateStoreError): 72 | store.tasks.claim( 73 | id=task["id"], 74 | counter=task["counter"], 75 | pid="task12", 76 | ttl=sys.maxsize, 77 | ) 78 | 79 | 80 | def test_case_13_transition_from_claimed_to_init_via_claim(store: Store, task: TaskMesg) -> None: 81 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task13", ttl=0) 82 | with pytest.raises(ResonateStoreError): 83 | store.tasks.claim( 84 | id=task["id"], 85 | counter=task["counter"], 86 | pid="task12", 87 | ttl=sys.maxsize, 88 | ) 89 | 90 | 91 | def test_case_14_transition_from_claimed_to_completed_via_complete(store: Store, task: TaskMesg) -> None: 92 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task14", ttl=sys.maxsize) 93 | store.tasks.complete(id=task["id"], counter=task["counter"]) 94 | 95 | 96 | def test_case_15_transition_from_claimed_to_init_via_complete(store: Store, task: TaskMesg) -> None: 97 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task15", ttl=0) 98 | time.sleep(TICK_TIME) 99 | with pytest.raises(ResonateStoreError): 100 | store.tasks.complete(id=task["id"], counter=task["counter"]) 101 | 102 | 103 | def test_case_16_transition_from_claimed_to_claimed_via_complete(store: Store, task: TaskMesg) -> None: 104 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task16", ttl=sys.maxsize) 105 | with pytest.raises(ResonateStoreError): 106 | store.tasks.complete(id=task["id"], counter=task["counter"] + 1) 107 | 108 | 109 | def test_case_17_transition_from_claimed_to_init_via_complete(store: Store, task: TaskMesg) -> None: 110 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task17", ttl=0) 111 | time.sleep(TICK_TIME) 112 | with pytest.raises(ResonateStoreError): 113 | store.tasks.complete(id=task["id"], counter=task["counter"]) 114 | 115 | 116 | def test_case_18_transition_from_claimed_to_claimed_via_heartbeat(store: Store, task: TaskMesg) -> None: 117 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task18", ttl=sys.maxsize) 118 | assert store.tasks.heartbeat(pid="task18") == 1 119 | 120 | 121 | def test_case_19_transition_from_claimed_to_init_via_heartbeat(store: Store, task: TaskMesg) -> None: 122 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task19", ttl=0) 123 | assert store.tasks.heartbeat(pid="task19") == 1 124 | 125 | 126 | def test_case_20_transition_from_completed_to_completed_via_claim(store: Store, task: TaskMesg) -> None: 127 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task20", ttl=sys.maxsize) 128 | store.tasks.complete(id=task["id"], counter=task["counter"]) 129 | with pytest.raises(ResonateStoreError): 130 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task20", ttl=0) 131 | 132 | 133 | def test_case_21_transition_from_completed_to_completed_via_complete(store: Store, task: TaskMesg) -> None: 134 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task21", ttl=sys.maxsize) 135 | store.tasks.complete(id=task["id"], counter=task["counter"]) 136 | store.tasks.complete(id=task["id"], counter=task["counter"]) 137 | 138 | 139 | def test_case_22_transition_from_completed_to_completed_via_heartbeat(store: Store, task: TaskMesg) -> None: 140 | store.tasks.claim(id=task["id"], counter=task["counter"], pid="task22", ttl=sys.maxsize) 141 | store.tasks.complete(id=task["id"], counter=task["counter"]) 142 | assert store.tasks.heartbeat(pid="task22") == 0 143 | -------------------------------------------------------------------------------- /tests/test_retries.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import sys 4 | from typing import TYPE_CHECKING 5 | 6 | import pytest 7 | 8 | from resonate.conventions import Base 9 | from resonate.dependencies import Dependencies 10 | from resonate.loggers import ContextLogger 11 | from resonate.models.commands import Delayed, Function, Invoke, Retry, Return 12 | from resonate.models.result import Ko, Ok 13 | from resonate.options import Options 14 | from resonate.registry import Registry 15 | from resonate.resonate import Context 16 | from resonate.retry_policies import Constant, Exponential, Linear, Never 17 | from resonate.scheduler import Done, More, Scheduler 18 | 19 | if TYPE_CHECKING: 20 | from resonate.models.retry_policy import RetryPolicy 21 | 22 | 23 | def foo(ctx: Context): # noqa: ANN201 24 | return "foo" 25 | 26 | 27 | def bar_ok(ctx: Context): # noqa: ANN201 28 | return "bar" 29 | yield ctx.lfi(foo) 30 | 31 | 32 | def bar_ko(ctx: Context): # noqa: ANN201 33 | raise ValueError 34 | yield ctx.lfi(foo) 35 | 36 | 37 | @pytest.fixture 38 | def scheduler() -> Scheduler: 39 | return Scheduler(lambda id, cid, info: Context(id, cid, info, Registry(), Dependencies(), ContextLogger("f", "f"))) 40 | 41 | 42 | @pytest.mark.parametrize( 43 | "retry_policy", 44 | [ 45 | Never(), 46 | Constant(), 47 | Linear(), 48 | Exponential(), 49 | ], 50 | ) 51 | def test_function_happy_path(scheduler: Scheduler, retry_policy: RetryPolicy) -> None: 52 | next = scheduler.step(Invoke("foo", Base("foo", sys.maxsize), sys.maxsize, foo, opts=Options(durable=False, retry_policy=retry_policy))) 53 | assert isinstance(next, More) 54 | assert len(next.reqs) == 1 55 | req = next.reqs[0] 56 | assert isinstance(req, Function) 57 | next = scheduler.step(Return("foo", "foo", Ok("foo"))) 58 | assert isinstance(next, Done) 59 | assert scheduler.computations["foo"].result() == Ok("foo") 60 | 61 | 62 | @pytest.mark.parametrize( 63 | ("retry_policy", "retries"), 64 | [ 65 | (Never(), 0), 66 | (Constant(max_retries=2), 2), 67 | (Linear(max_retries=3), 3), 68 | (Exponential(max_retries=2), 2), 69 | ], 70 | ) 71 | def test_function_sad_path(scheduler: Scheduler, retry_policy: RetryPolicy, retries: int) -> None: 72 | e = ValueError("something went wrong") 73 | 74 | next = scheduler.step(Invoke("foo", Base("foo", sys.maxsize), sys.maxsize, foo, opts=Options(durable=False, retry_policy=retry_policy))) 75 | assert isinstance(next, More) 76 | assert len(next.reqs) == 1 77 | req = next.reqs[0] 78 | assert isinstance(req, Function) 79 | 80 | for _ in range(retries): 81 | next = scheduler.step(Return("foo", "foo", Ko(e))) 82 | assert isinstance(next, More) 83 | assert len(next.reqs) == 1 84 | req = next.reqs[0] 85 | assert isinstance(req, Delayed) 86 | 87 | next = scheduler.step(Return("foo", "foo", Ko(e))) 88 | assert isinstance(next, Done) 89 | assert scheduler.computations["foo"].result() == Ko(e) 90 | 91 | 92 | @pytest.mark.parametrize( 93 | "retry_policy", 94 | [ 95 | Never(), 96 | Constant(), 97 | Linear(), 98 | Exponential(), 99 | ], 100 | ) 101 | def test_generator_happy_path(scheduler: Scheduler, retry_policy: RetryPolicy) -> None: 102 | next = scheduler.step(Invoke("bar", Base("bar", sys.maxsize), sys.maxsize, bar_ok, opts=Options(durable=False, retry_policy=retry_policy))) 103 | assert isinstance(next, Done) 104 | assert scheduler.computations["bar"].result() == Ok("bar") 105 | 106 | 107 | @pytest.mark.parametrize( 108 | ("retry_policy", "retries"), 109 | [ 110 | (Never(), 0), 111 | (Constant(max_retries=2), 2), 112 | (Linear(max_retries=3), 3), 113 | (Exponential(max_retries=1), 1), 114 | ], 115 | ) 116 | def test_generator_sad_path(scheduler: Scheduler, retry_policy: RetryPolicy, retries: int) -> None: 117 | next = scheduler.step(Invoke("bar", Base("bar", sys.maxsize), sys.maxsize, bar_ko, opts=Options(durable=False, retry_policy=retry_policy))) 118 | 119 | for _ in range(retries): 120 | assert isinstance(next, More) 121 | assert len(next.reqs) == 1 122 | req = next.reqs[0] 123 | assert isinstance(req, Delayed) 124 | assert isinstance(req.item, Retry) 125 | next = scheduler.step(req.item) 126 | 127 | assert isinstance(next, Done) 128 | assert isinstance(scheduler.computations["bar"].result(), Ko) 129 | 130 | 131 | @pytest.mark.parametrize( 132 | ("retry_policy", "non_retryable_exceptions"), 133 | [ 134 | (Never(), (ValueError,)), 135 | (Constant(max_retries=2), (ValueError,)), 136 | (Linear(max_retries=3), (ValueError,)), 137 | (Exponential(max_retries=1), (ValueError,)), 138 | ], 139 | ) 140 | def test_non_retriable_errors(scheduler: Scheduler, retry_policy: RetryPolicy, non_retryable_exceptions: tuple[type[Exception], ...]) -> None: 141 | opts = Options(durable=False, retry_policy=retry_policy, non_retryable_exceptions=non_retryable_exceptions) 142 | next = scheduler.step(Invoke("bar", Base("bar", sys.maxsize), sys.maxsize, bar_ko, opts=opts)) 143 | assert isinstance(next, Done) 144 | assert isinstance(scheduler.computations["bar"].result(), Ko) 145 | 146 | 147 | @pytest.mark.parametrize( 148 | "retry_policy", 149 | [ 150 | Never(), 151 | Constant(max_retries=2), 152 | Linear(max_retries=3), 153 | Exponential(max_retries=1), 154 | ], 155 | ) 156 | def test_timeout_over_delay(scheduler: Scheduler, retry_policy: RetryPolicy) -> None: 157 | opts = Options(durable=False, retry_policy=retry_policy) 158 | next = scheduler.step(Invoke("bar", Base("bar", 0), 0, bar_ko, opts=opts)) 159 | assert isinstance(next, Done) 160 | assert isinstance(scheduler.computations["bar"].result(), Ko) 161 | -------------------------------------------------------------------------------- /resonate/models/durable_promise.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass, field 4 | from typing import TYPE_CHECKING, Any, Literal 5 | 6 | from resonate.errors import ResonateCanceledError, ResonateTimedoutError 7 | from resonate.models.result import Ko, Ok, Result 8 | 9 | if TYPE_CHECKING: 10 | from collections.abc import Mapping 11 | 12 | from resonate.models.callback import Callback 13 | from resonate.models.encoder import Encoder 14 | from resonate.models.store import Store 15 | 16 | 17 | @dataclass 18 | class DurablePromise: 19 | id: str 20 | state: Literal["PENDING", "RESOLVED", "REJECTED", "REJECTED_CANCELED", "REJECTED_TIMEDOUT"] 21 | timeout: int 22 | ikey_for_create: str | None 23 | ikey_for_complete: str | None 24 | param: DurablePromiseValue 25 | value: DurablePromiseValue 26 | tags: dict[str, str] 27 | created_on: int 28 | completed_on: int | None 29 | 30 | store: Store = field(repr=False) 31 | 32 | @property 33 | def pending(self) -> bool: 34 | return self.state == "PENDING" 35 | 36 | @property 37 | def completed(self) -> bool: 38 | return not self.pending 39 | 40 | @property 41 | def resolved(self) -> bool: 42 | return self.state == "RESOLVED" 43 | 44 | @property 45 | def rejected(self) -> bool: 46 | return self.state == "REJECTED" 47 | 48 | @property 49 | def canceled(self) -> bool: 50 | return self.state == "REJECTED_CANCELED" 51 | 52 | @property 53 | def timedout(self) -> bool: 54 | return self.state == "REJECTED_TIMEDOUT" 55 | 56 | @property 57 | def abs_timeout(self) -> float: 58 | return self.timeout / 1000 59 | 60 | @property 61 | def rel_timeout(self) -> float: 62 | return (self.timeout - self.created_on) / 1000 63 | 64 | def resolve(self, data: str | None, *, ikey: str | None = None, strict: bool = False, headers: dict[str, str] | None = None) -> None: 65 | promise = self.store.promises.resolve( 66 | id=self.id, 67 | ikey=ikey, 68 | strict=strict, 69 | headers=headers, 70 | data=data, 71 | ) 72 | self._complete(promise) 73 | 74 | def reject(self, data: str | None, *, ikey: str | None = None, strict: bool = False, headers: dict[str, str] | None = None) -> None: 75 | promise = self.store.promises.reject( 76 | id=self.id, 77 | ikey=ikey, 78 | strict=strict, 79 | headers=headers, 80 | data=data, 81 | ) 82 | self._complete(promise) 83 | 84 | def cancel(self, data: str | None, *, ikey: str | None = None, strict: bool = False, headers: dict[str, str] | None = None) -> None: 85 | promise = self.store.promises.cancel( 86 | id=self.id, 87 | ikey=ikey, 88 | strict=strict, 89 | headers=headers, 90 | data=data, 91 | ) 92 | self._complete(promise) 93 | 94 | def callback(self, id: str, root_promise_id: str, recv: str) -> Callback | None: 95 | promise, callback = self.store.promises.callback( 96 | promise_id=self.id, 97 | root_promise_id=root_promise_id, 98 | recv=recv, 99 | timeout=self.timeout, 100 | ) 101 | if callback is None: 102 | self._complete(promise) 103 | return callback 104 | 105 | def subscribe(self, id: str, recv: str) -> Callback | None: 106 | promise, callback = self.store.promises.subscribe( 107 | id=id, 108 | promise_id=self.id, 109 | recv=recv, 110 | timeout=self.timeout, 111 | ) 112 | if callback is None: 113 | self._complete(promise) 114 | return callback 115 | 116 | def _complete(self, promise: DurablePromise) -> None: 117 | assert promise.completed 118 | self.state = promise.state 119 | self.ikey_for_complete = promise.ikey_for_complete 120 | self.value = promise.value 121 | self.completed_on = promise.completed_on 122 | 123 | def result(self, encoder: Encoder[Any, tuple[dict[str, str] | None, str | None]]) -> Result[Any]: 124 | assert self.completed, "Promise must be completed" 125 | 126 | if self.rejected: 127 | v = encoder.decode(self.value.to_tuple()) 128 | 129 | # In python, only exceptions may be raised. Here, we are converting 130 | # a value that is not an exception into an exception. 131 | return Ko(v) if isinstance(v, BaseException) else Ko(Exception(v)) 132 | if self.canceled: 133 | return Ko(ResonateCanceledError(self.id)) 134 | if self.timedout: 135 | return Ko(ResonateTimedoutError(self.id, self.abs_timeout)) 136 | 137 | return Ok(encoder.decode(self.value.to_tuple())) 138 | 139 | @classmethod 140 | def from_dict(cls, store: Store, data: Mapping[str, Any]) -> DurablePromise: 141 | return cls( 142 | id=data["id"], 143 | state=data["state"], 144 | timeout=data["timeout"], 145 | ikey_for_create=data.get("idempotencyKeyForCreate"), 146 | ikey_for_complete=data.get("idempotencyKeyForComplete"), 147 | param=DurablePromiseValue.from_dict(store, data.get("param", {})), 148 | value=DurablePromiseValue.from_dict(store, data.get("value", {})), 149 | tags=data.get("tags", {}), 150 | created_on=data["createdOn"], 151 | completed_on=data.get("completedOn"), 152 | store=store, 153 | ) 154 | 155 | 156 | @dataclass 157 | class DurablePromiseValue: 158 | headers: dict[str, str] | None 159 | data: str | None 160 | 161 | def to_tuple(self) -> tuple[dict[str, str] | None, Any]: 162 | return self.headers, self.data 163 | 164 | @classmethod 165 | def from_dict(cls, store: Store, data: Mapping[str, Any]) -> DurablePromiseValue: 166 | return cls( 167 | headers=data.get("headers"), 168 | data=store.encoder.decode(data.get("data")), 169 | ) 170 | -------------------------------------------------------------------------------- /resonate/coroutine.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from dataclasses import dataclass, field 4 | from typing import TYPE_CHECKING, Any, Literal, Self 5 | 6 | from resonate.models.result import Ko, Ok, Result 7 | from resonate.options import Options 8 | 9 | if TYPE_CHECKING: 10 | from collections.abc import Callable, Generator 11 | 12 | from resonate.models.convention import Convention 13 | from resonate.models.encoder import Encoder 14 | from resonate.models.retry_policy import RetryPolicy 15 | 16 | 17 | @dataclass 18 | class LFX[T]: 19 | conv: Convention 20 | func: Callable[..., Generator[Any, Any, T] | T] 21 | args: tuple[Any, ...] 22 | kwargs: dict[str, Any] 23 | opts: Options = field(default_factory=Options) 24 | 25 | @property 26 | def id(self) -> str: 27 | return self.conv.id 28 | 29 | def options( 30 | self, 31 | *, 32 | durable: bool | None = None, 33 | encoder: Encoder[Any, str | None] | None = None, 34 | id: str | None = None, 35 | idempotency_key: str | Callable[[str], str] | None = None, 36 | non_retryable_exceptions: tuple[type[Exception], ...] | None = None, 37 | retry_policy: RetryPolicy | Callable[[Callable], RetryPolicy] | None = None, 38 | tags: dict[str, str] | None = None, 39 | timeout: float | None = None, 40 | ) -> Self: 41 | """Configure options for the function. 42 | 43 | - durable: Whether or not you want this function to be durable. 44 | - encoder: Configure your own data encoder. 45 | - id: Override the id for the function invocation. 46 | - idempotency_key: Define the idempotency key invocation or a function that 47 | receives the promise id and creates an idempotency key. 48 | - non_retryable_exceptions: Exceptions that ignore the retry policy of the invocation. 49 | - retry_policy: Define the retry policy exponential | constant | linear | never 50 | - tags: Add custom tags to the durable promise representing the invocation. 51 | - target: Target to distribute the invocation. 52 | - timeout: Number of seconds before the invocation times out. 53 | """ 54 | # Note: we deliberately ignore the version for LFX 55 | self.conv = self.conv.options( 56 | id=id, 57 | idempotency_key=idempotency_key, 58 | tags=tags, 59 | timeout=timeout, 60 | ) 61 | self.opts = self.opts.merge( 62 | durable=durable, 63 | encoder=encoder, 64 | non_retryable_exceptions=non_retryable_exceptions, 65 | retry_policy=retry_policy, 66 | ) 67 | return self 68 | 69 | 70 | @dataclass 71 | class LFI[T](LFX[T]): 72 | pass 73 | 74 | 75 | @dataclass 76 | class LFC[T](LFX[T]): 77 | pass 78 | 79 | 80 | @dataclass 81 | class RFX[T]: 82 | conv: Convention 83 | opts: Options = field(default_factory=Options) 84 | 85 | @property 86 | def id(self) -> str: 87 | return self.conv.id 88 | 89 | def options( 90 | self, 91 | *, 92 | encoder: Encoder[Any, str | None] | None = None, 93 | id: str | None = None, 94 | idempotency_key: str | Callable[[str], str] | None = None, 95 | tags: dict[str, str] | None = None, 96 | target: str | None = None, 97 | timeout: float | None = None, 98 | version: int | None = None, 99 | ) -> Self: 100 | """Configure options for the function. 101 | 102 | - encoder: Configure your own data encoder. 103 | - id: Override the id for the function invocation. 104 | - idempotency_key: Define the idempotency key invocation or a function that 105 | receives the promise id and creates an idempotency key. 106 | - tags: Add custom tags to the durable promise representing the invocation. 107 | - target: Target to distribute the invocation. 108 | - timeout: Number of seconds before the invocation times out. 109 | - version: Version of the function to invoke. 110 | """ 111 | self.conv = self.conv.options( 112 | id=id, 113 | idempotency_key=idempotency_key, 114 | target=target, 115 | tags=tags, 116 | timeout=timeout, 117 | version=version, 118 | ) 119 | self.opts = self.opts.merge( 120 | encoder=encoder, 121 | ) 122 | return self 123 | 124 | 125 | @dataclass 126 | class RFI[T](RFX[T]): 127 | mode: Literal["attached", "detached"] = "attached" 128 | 129 | 130 | @dataclass 131 | class RFC[T](RFX[T]): 132 | pass 133 | 134 | 135 | @dataclass 136 | class AWT: 137 | id: str 138 | 139 | 140 | @dataclass 141 | class TRM: 142 | id: str 143 | result: Result 144 | 145 | 146 | @dataclass(frozen=True) 147 | class Promise[T]: 148 | id: str 149 | cid: str 150 | 151 | 152 | type Yieldable = LFI | LFC | RFI | RFC | Promise 153 | 154 | 155 | class Coroutine: 156 | def __init__(self, id: str, cid: str, gen: Generator[Yieldable, Any, Any]) -> None: 157 | self.id = id 158 | self.cid = cid 159 | self.gen = gen 160 | 161 | self.done = False 162 | self.skip = False 163 | self.next: type[None | AWT] | tuple[type[Result], ...] = type(None) 164 | self.unyielded: list[AWT | TRM] = [] 165 | 166 | def __repr__(self) -> str: 167 | return f"Coroutine(done={self.done})" 168 | 169 | def send(self, value: None | AWT | Result) -> LFI | RFI | AWT | TRM: 170 | assert self.done or isinstance(value, self.next), "AWT must follow LFI/RFI. Value must follow AWT." 171 | assert not self.skip or isinstance(value, AWT), "If skipped, value must be an AWT." 172 | 173 | if self.done: 174 | # When done, yield all unyielded values to enforce structured concurrency, the final 175 | # value must be a TRM. 176 | 177 | match self.unyielded: 178 | case []: 179 | raise StopIteration 180 | case [trm]: 181 | assert isinstance(trm, TRM), "Last unyielded value must be a TRM." 182 | self.unyielded = [] 183 | return trm 184 | case [head, *tail]: 185 | self.unyielded = tail 186 | return head 187 | try: 188 | match value, self.skip: 189 | case None, _: 190 | yielded = next(self.gen) 191 | case Ok(v), _: 192 | yielded = self.gen.send(v) 193 | case Ko(e), _: 194 | yielded = self.gen.throw(e) 195 | case awt, True: 196 | # When skipped, pretend as if the generator yielded a promise 197 | self.skip = False 198 | yielded = Promise(awt.id, self.cid) 199 | case awt, False: 200 | yielded = self.gen.send(Promise(awt.id, self.cid)) 201 | 202 | match yielded: 203 | case LFI(conv) | RFI(conv, mode="attached"): 204 | # LFIs and attached RFIs require an AWT 205 | self.next = AWT 206 | self.unyielded.append(AWT(conv.id)) 207 | command = yielded 208 | case LFC(conv, func, args, kwargs, opts): 209 | # LFCs can be converted to an LFI+AWT 210 | self.next = AWT 211 | self.skip = True 212 | command = LFI(conv, func, args, kwargs, opts) 213 | case RFC(conv): 214 | # RFCs can be converted to an RFI+AWT 215 | self.next = AWT 216 | self.skip = True 217 | command = RFI(conv) 218 | case Promise(id): 219 | # When a promise is yielded we can remove it from unyielded 220 | self.next = (Ok, Ko) 221 | self.unyielded = [y for y in self.unyielded if y.id != id] 222 | command = AWT(id) 223 | case _: 224 | assert isinstance(yielded, RFI), "Yielded must be an RFI." 225 | assert yielded.mode == "detached", "RFI must be detached." 226 | self.next = AWT 227 | command = yielded 228 | 229 | except StopIteration as e: 230 | self.done = True 231 | self.unyielded.append(TRM(self.id, Ok(e.value))) 232 | return self.unyielded.pop(0) 233 | except Exception as e: 234 | self.done = True 235 | self.unyielded.append(TRM(self.id, Ko(e))) 236 | return self.unyielded.pop(0) 237 | else: 238 | assert not isinstance(yielded, Promise) or yielded.cid == self.cid, "If promise, cids must match." 239 | return command 240 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /tests/runners.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import sys 5 | import time 6 | import uuid 7 | from concurrent.futures import Future 8 | from inspect import isgeneratorfunction 9 | from typing import TYPE_CHECKING, Any, Protocol 10 | 11 | from resonate.conventions import Base, Local 12 | from resonate.coroutine import LFC, LFI, RFC, RFI 13 | from resonate.encoders import JsonEncoder, NoopEncoder, PairEncoder 14 | from resonate.models.commands import ( 15 | CancelPromiseReq, 16 | CancelPromiseRes, 17 | Command, 18 | CreateCallbackReq, 19 | CreatePromiseReq, 20 | CreatePromiseRes, 21 | CreatePromiseWithTaskReq, 22 | CreatePromiseWithTaskRes, 23 | Function, 24 | Invoke, 25 | Network, 26 | Receive, 27 | RejectPromiseReq, 28 | RejectPromiseRes, 29 | ResolvePromiseReq, 30 | ResolvePromiseRes, 31 | Resume, 32 | Return, 33 | ) 34 | from resonate.models.result import Ko, Ok 35 | from resonate.models.task import Task 36 | from resonate.options import Options 37 | from resonate.resonate import Remote 38 | from resonate.scheduler import Scheduler 39 | from resonate.stores import LocalStore 40 | 41 | if TYPE_CHECKING: 42 | from collections.abc import Callable 43 | 44 | from resonate.models.context import Info 45 | from resonate.models.logger import Logger 46 | from resonate.registry import Registry 47 | 48 | 49 | # Context 50 | class Context: 51 | def __init__(self, id: str, cid: str) -> None: 52 | self.id = id 53 | self.cid = cid 54 | 55 | @property 56 | def info(self) -> Info: 57 | raise NotImplementedError 58 | 59 | @property 60 | def logger(self) -> Logger: 61 | return logging.getLogger("test") 62 | 63 | def get_dependency(self, key: str, default: Any = None) -> Any: 64 | return default 65 | 66 | def lfi(self, func: str | Callable, *args: Any, **kwargs: Any) -> LFI: 67 | assert not isinstance(func, str) 68 | return LFI(Local(uuid.uuid4().hex, self.cid, self.id), func, args, kwargs) 69 | 70 | def lfc(self, func: str | Callable, *args: Any, **kwargs: Any) -> LFC: 71 | assert not isinstance(func, str) 72 | return LFC(Local(uuid.uuid4().hex, self.cid, self.id), func, args, kwargs) 73 | 74 | def rfi(self, func: str | Callable, *args: Any, **kwargs: Any) -> RFI: 75 | assert not isinstance(func, str) 76 | return RFI(Remote(uuid.uuid4().hex, self.cid, self.id, func.__name__, args, kwargs)) 77 | 78 | def rfc(self, func: str | Callable, *args: Any, **kwargs: Any) -> RFC: 79 | assert not isinstance(func, str) 80 | return RFC(Remote(uuid.uuid4().hex, self.cid, self.id, func.__name__, args, kwargs)) 81 | 82 | def detached(self, func: str | Callable, *args: Any, **kwargs: Any) -> RFI: 83 | assert not isinstance(func, str) 84 | return RFI(Remote(uuid.uuid4().hex, self.cid, self.id, func.__name__, args, kwargs), mode="detached") 85 | 86 | 87 | class LocalContext: 88 | def __init__(self, id: str, cid: str) -> None: 89 | self.id = id 90 | self.cid = cid 91 | 92 | @property 93 | def info(self) -> Info: 94 | raise NotImplementedError 95 | 96 | @property 97 | def logger(self) -> Logger: 98 | return logging.getLogger("test") 99 | 100 | def get_dependency(self, key: str, default: Any = None) -> Any: 101 | return default 102 | 103 | def lfi(self, func: str | Callable, *args: Any, **kwargs: Any) -> LFI: 104 | assert not isinstance(func, str) 105 | return LFI(Local(uuid.uuid4().hex, self.cid, self.id), func, args, kwargs) 106 | 107 | def lfc(self, func: str | Callable, *args: Any, **kwargs: Any) -> LFC: 108 | assert not isinstance(func, str) 109 | return LFC(Local(uuid.uuid4().hex, self.cid, self.id), func, args, kwargs) 110 | 111 | def rfi(self, func: str | Callable, *args: Any, **kwargs: Any) -> LFI: 112 | assert not isinstance(func, str) 113 | return LFI(Local(uuid.uuid4().hex, self.cid, self.id), func, args, kwargs) 114 | 115 | def rfc(self, func: str | Callable, *args: Any, **kwargs: Any) -> LFC: 116 | assert not isinstance(func, str) 117 | return LFC(Local(uuid.uuid4().hex, self.cid, self.id), func, args, kwargs) 118 | 119 | def detached(self, func: str | Callable, *args: Any, **kwargs: Any) -> LFI: 120 | assert not isinstance(func, str) 121 | return LFI(Local(uuid.uuid4().hex, self.cid, self.id), func, args, kwargs) 122 | 123 | 124 | class RemoteContext: 125 | def __init__(self, id: str, cid: str) -> None: 126 | self.id = id 127 | self.cid = cid 128 | 129 | @property 130 | def info(self) -> Info: 131 | raise NotImplementedError 132 | 133 | @property 134 | def logger(self) -> Logger: 135 | return logging.getLogger("test") 136 | 137 | def get_dependency(self, key: str, default: Any = None) -> Any: 138 | return default 139 | 140 | def lfi(self, func: str | Callable, *args: Any, **kwargs: Any) -> RFI: 141 | assert not isinstance(func, str) 142 | return RFI(Remote(uuid.uuid4().hex, self.cid, self.id, func.__name__, args, kwargs)) 143 | 144 | def lfc(self, func: str | Callable, *args: Any, **kwargs: Any) -> RFC: 145 | assert not isinstance(func, str) 146 | return RFC(Remote(uuid.uuid4().hex, self.cid, self.id, func.__name__, args, kwargs)) 147 | 148 | def rfi(self, func: str | Callable, *args: Any, **kwargs: Any) -> RFI: 149 | assert not isinstance(func, str) 150 | return RFI(Remote(uuid.uuid4().hex, self.cid, self.id, func.__name__, args, kwargs)) 151 | 152 | def rfc(self, func: str | Callable, *args: Any, **kwargs: Any) -> RFC: 153 | assert not isinstance(func, str) 154 | return RFC(Remote(uuid.uuid4().hex, self.cid, self.id, func.__name__, args, kwargs)) 155 | 156 | def detached(self, func: str | Callable, *args: Any, **kwargs: Any) -> RFI: 157 | assert not isinstance(func, str) 158 | return RFI(Remote(uuid.uuid4().hex, self.cid, self.id, func.__name__, args, kwargs), mode="detached") 159 | 160 | 161 | # Runners 162 | 163 | 164 | class Runner(Protocol): 165 | def run[**P, R](self, id: str, func: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R: ... 166 | 167 | 168 | class SimpleRunner: 169 | def run[**P, R](self, id: str, func: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R: 170 | return self._run(id, func, args, kwargs) 171 | 172 | def _run(self, id: str, func: Callable, args: tuple, kwargs: dict) -> Any: 173 | if not isgeneratorfunction(func): 174 | return func(None, *args, **kwargs) 175 | 176 | g = func(LocalContext(id, id), *args, **kwargs) 177 | v = None 178 | 179 | try: 180 | while True: 181 | match g.send(v): 182 | case LFI(conv, func, args, kwargs): 183 | v = (conv.id, func, args, kwargs) 184 | case LFC(conv, func, args, kwargs): 185 | v = self._run(conv.id, func, args, kwargs) 186 | case (id, func, args, kwargs): 187 | v = self._run(id, func, args, kwargs) 188 | except StopIteration as e: 189 | return e.value 190 | 191 | 192 | class ResonateRunner: 193 | def __init__(self, registry: Registry) -> None: 194 | # registry 195 | self.registry = registry 196 | 197 | # store 198 | self.store = LocalStore() 199 | 200 | # encoder 201 | self.encoder = PairEncoder(NoopEncoder(), JsonEncoder()) 202 | 203 | # create scheduler and connect store 204 | self.scheduler = Scheduler(ctx=lambda id, cid, *_: Context(id, cid)) 205 | 206 | def run[**P, R](self, id: str, func: Callable[P, R], *args: P.args, **kwargs: P.kwargs) -> R: 207 | cmds: list[Command] = [] 208 | init = True 209 | conv = Remote(id, id, id, func.__name__, args, kwargs) 210 | future = Future[R]() 211 | 212 | headers, data = self.encoder.encode(conv.data) 213 | assert headers is None 214 | 215 | promise, _ = self.store.promises.create_with_task( 216 | id=conv.id, 217 | ikey=conv.idempotency_key, 218 | timeout=int((time.time() + conv.timeout) * 1000), 219 | data=data, 220 | tags=conv.tags, 221 | pid=self.scheduler.pid, 222 | ttl=sys.maxsize, 223 | ) 224 | 225 | cmds.append(Invoke(id, conv, promise.abs_timeout, func, args, kwargs, promise=promise)) 226 | 227 | while cmds: 228 | next = self.scheduler.step(cmds.pop(0), future if init else None) 229 | init = False 230 | 231 | for req in next.reqs: 232 | match req: 233 | case Function(_id, cid, f): 234 | try: 235 | r = Ok(f()) 236 | except Exception as e: 237 | r = Ko(e) 238 | cmds.append(Return(_id, cid, r)) 239 | 240 | case Network(_id, cid, CreatePromiseReq(id, timeout, ikey, strict, headers, data, tags)): 241 | promise = self.store.promises.create( 242 | id=id, 243 | timeout=timeout, 244 | ikey=ikey, 245 | strict=strict, 246 | headers=headers, 247 | data=data, 248 | tags=tags, 249 | ) 250 | cmds.append(Receive(_id, cid, CreatePromiseRes(promise))) 251 | 252 | case Network(_id, cid, CreatePromiseWithTaskReq(id, timeout, pid, ttl, ikey, strict, headers, data, tags)): 253 | promise, task = self.store.promises.create_with_task( 254 | id=id, 255 | timeout=timeout, 256 | pid=pid, 257 | ttl=ttl, 258 | ikey=ikey, 259 | strict=strict, 260 | headers=headers, 261 | data=data, 262 | tags=tags, 263 | ) 264 | cmds.append(Receive(_id, cid, CreatePromiseWithTaskRes(promise, task))) 265 | 266 | case Network(_id, cid, ResolvePromiseReq(id, ikey, strict, headers, data)): 267 | promise = self.store.promises.resolve( 268 | id=id, 269 | ikey=ikey, 270 | strict=strict, 271 | headers=headers, 272 | data=data, 273 | ) 274 | cmds.append(Receive(_id, cid, ResolvePromiseRes(promise))) 275 | 276 | case Network(_id, cid, RejectPromiseReq(id, ikey, strict, headers, data)): 277 | promise = self.store.promises.reject( 278 | id=id, 279 | ikey=ikey, 280 | strict=strict, 281 | headers=headers, 282 | data=data, 283 | ) 284 | cmds.append(Receive(_id, cid, RejectPromiseRes(promise))) 285 | 286 | case Network(_id, cid, CancelPromiseReq(id, ikey, strict, headers, data)): 287 | promise = self.store.promises.cancel( 288 | id=id, 289 | ikey=ikey, 290 | strict=strict, 291 | headers=headers, 292 | data=data, 293 | ) 294 | cmds.append(Receive(_id, cid, CancelPromiseRes(promise))) 295 | 296 | case Network(_id, cid, CreateCallbackReq(promise_id, root_promise_id, timeout, recv)): 297 | promise, callback = self.store.promises.callback( 298 | promise_id=promise_id, 299 | root_promise_id=root_promise_id, 300 | recv=recv, 301 | timeout=timeout, 302 | ) 303 | if promise.completed: 304 | assert not callback 305 | cmds.append(Resume(_id, cid, promise)) 306 | 307 | case _: 308 | raise NotImplementedError 309 | 310 | for _, msg in self.store.step(): 311 | match msg: 312 | case {"type": "invoke", "task": {"id": id, "counter": counter}}: 313 | task = Task(id=id, counter=counter, store=self.store) 314 | root, leaf = task.claim(pid=self.scheduler.pid, ttl=sys.maxsize) 315 | assert root.pending 316 | assert not leaf 317 | 318 | data = self.encoder.decode(root.param.to_tuple()) 319 | assert isinstance(data, dict) 320 | assert "func" in data 321 | assert "args" in data 322 | assert "kwargs" in data 323 | 324 | _, func, version = self.registry.get(data["func"]) 325 | 326 | cmds.append( 327 | Invoke( 328 | root.id, 329 | Base( 330 | root.id, 331 | root.rel_timeout, 332 | root.ikey_for_create, 333 | root.param.data, 334 | root.tags, 335 | ), 336 | root.abs_timeout, 337 | func, 338 | data["args"], 339 | data["kwargs"], 340 | Options(version=version), 341 | root, 342 | ) 343 | ) 344 | 345 | case {"type": "resume", "task": {"id": id, "counter": counter}}: 346 | task = Task(id=id, counter=counter, store=self.store) 347 | root, leaf = task.claim(pid=self.scheduler.pid, ttl=sys.maxsize) 348 | assert root.pending 349 | assert leaf 350 | assert leaf.completed 351 | 352 | cmds.append( 353 | Resume( 354 | id=leaf.id, 355 | cid=root.id, 356 | promise=leaf, 357 | ) 358 | ) 359 | 360 | case _: 361 | raise NotImplementedError 362 | 363 | return future.result() 364 | 365 | 366 | class ResonateLFXRunner(ResonateRunner): 367 | def __init__(self, registry: Registry) -> None: 368 | self.registry = registry 369 | 370 | # create store 371 | self.store = LocalStore() 372 | 373 | # create encoder 374 | self.encoder = PairEncoder(NoopEncoder(), JsonEncoder()) 375 | 376 | # create scheduler 377 | self.scheduler = Scheduler(ctx=lambda id, cid, *_: LocalContext(id, cid)) 378 | 379 | 380 | class ResonateRFXRunner(ResonateRunner): 381 | def __init__(self, registry: Registry) -> None: 382 | self.registry = registry 383 | 384 | # create store 385 | self.store = LocalStore() 386 | 387 | # create encoder 388 | self.encoder = PairEncoder(NoopEncoder(), JsonEncoder()) 389 | 390 | # create scheduler and connect store 391 | self.scheduler = Scheduler(ctx=lambda id, cid, *_: RemoteContext(id, cid)) 392 | -------------------------------------------------------------------------------- /tests/test_dst.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import json 4 | import logging 5 | import random 6 | import re 7 | from typing import TYPE_CHECKING, Any 8 | 9 | from tabulate import tabulate 10 | 11 | from resonate.clocks import StepClock 12 | from resonate.conventions import Remote 13 | from resonate.dependencies import Dependencies 14 | from resonate.models.commands import Invoke, Listen 15 | from resonate.models.result import Ko, Ok, Result 16 | from resonate.options import Options 17 | from resonate.registry import Registry 18 | from resonate.scheduler import Coro, Init, Lfnc, Rfnc 19 | from resonate.simulator import Server, Simulator, Worker 20 | 21 | if TYPE_CHECKING: 22 | from collections.abc import Generator 23 | 24 | from resonate import Context, Promise 25 | from resonate.models.context import Info 26 | 27 | 28 | logger = logging.getLogger(__name__) 29 | 30 | 31 | def foo(ctx: Context) -> Generator[Any, Any, Any]: 32 | p1 = yield ctx.lfi(bar) 33 | p2 = yield ctx.rfi(bar) 34 | yield ctx.lfi(bar) 35 | yield ctx.rfi(bar) 36 | yield ctx.lfc(bar) 37 | yield ctx.rfc(bar) 38 | yield ctx.detached(bar) 39 | 40 | return (yield p1), (yield p2) 41 | 42 | 43 | def bar(ctx: Context) -> Generator[Any, Any, Any]: 44 | p1 = yield ctx.lfi(baz) 45 | p2 = yield ctx.rfi(baz) 46 | yield ctx.lfi(baz) 47 | yield ctx.rfi(baz) 48 | yield ctx.lfc(baz) 49 | yield ctx.rfc(baz) 50 | yield ctx.detached(baz) 51 | 52 | return (yield p1), (yield p2) 53 | 54 | 55 | def baz(ctx: Context) -> str: 56 | return "baz" 57 | 58 | 59 | def foo_lfi(ctx: Context) -> Generator[Any, Any, Any]: 60 | p = yield ctx.lfi(bar_lfi) 61 | v = yield p 62 | return v 63 | 64 | 65 | def bar_lfi(ctx: Context) -> Generator[Any, Any, Any]: 66 | p = yield ctx.lfi(baz) 67 | v = yield p 68 | return v 69 | 70 | 71 | def foo_lfc(ctx: Context) -> Generator[Any, Any, Any]: 72 | v = yield ctx.lfc(bar_lfc) 73 | return v 74 | 75 | 76 | def bar_lfc(ctx: Context) -> Generator[Any, Any, Any]: 77 | v = yield ctx.lfc(baz) 78 | return v 79 | 80 | 81 | def foo_rfi(ctx: Context) -> Generator[Any, Any, Any]: 82 | p = yield ctx.rfi(bar_rfi) 83 | v = yield p 84 | return v 85 | 86 | 87 | def bar_rfi(ctx: Context) -> Generator[Any, Any, Any]: 88 | p = yield ctx.rfi(baz) 89 | v = yield p 90 | return v 91 | 92 | 93 | def foo_rfc(ctx: Context) -> Generator[Any, Any, Any]: 94 | v = yield ctx.lfc(bar_rfc) 95 | return v 96 | 97 | 98 | def bar_rfc(ctx: Context) -> Generator[Any, Any, Any]: 99 | v = yield ctx.rfc(baz) 100 | return v 101 | 102 | 103 | def foo_detached(ctx: Context) -> Generator[Any, Any, Any]: 104 | p = yield ctx.detached(bar_detached) 105 | return p.id 106 | 107 | 108 | def bar_detached(ctx: Context) -> Generator[Any, Any, Any]: 109 | p = yield ctx.detached(baz) 110 | return p.id 111 | 112 | 113 | def structured_concurrency_lfi(ctx: Context) -> Generator[Any, Any, Any]: 114 | p1 = yield ctx.lfi(baz) 115 | p2 = yield ctx.lfi(baz) 116 | p3 = yield ctx.lfi(baz) 117 | return p1.id, p2.id, p3.id 118 | 119 | 120 | def structured_concurrency_rfi(ctx: Context) -> Generator[Any, Any, Any]: 121 | p1 = yield ctx.rfi(baz) 122 | p2 = yield ctx.rfi(baz) 123 | p3 = yield ctx.rfi(baz) 124 | return p1.id, p2.id, p3.id 125 | 126 | 127 | def same_p_lfi(ctx: Context, n: int) -> Generator[Any, Any, None]: 128 | for _ in range(n): 129 | yield ctx.lfi(_same_p_lfi, f"{ctx.id}:common") 130 | 131 | 132 | def _same_p_lfi(ctx: Context, id: str) -> Generator[Any, Any, None]: 133 | yield ctx.lfi(baz).options(id=id) 134 | yield ctx.lfi(baz) 135 | 136 | 137 | def same_p_rfi(ctx: Context, n: int) -> Generator[Any, Any, None]: 138 | for _ in range(n): 139 | yield ctx.lfi(_same_p_rfi, f"{ctx.id}:common") 140 | 141 | 142 | def _same_p_rfi(ctx: Context, id: str) -> Generator[Any, Any, None]: 143 | yield ctx.rfi(baz).options(id=id) 144 | yield ctx.lfi(baz) 145 | 146 | 147 | def same_v_lfi(ctx: Context, n: int) -> Generator[Any, Any, None]: 148 | p = yield ctx.lfi(baz) 149 | for _ in range(n): 150 | yield ctx.lfi(_same_v, p) 151 | 152 | 153 | def same_v_rfi(ctx: Context, n: int) -> Generator[Any, Any, None]: 154 | p = yield ctx.rfi(baz) 155 | for _ in range(n): 156 | yield ctx.lfi(_same_v, p) 157 | 158 | 159 | def _same_v(ctx: Context, p: Promise) -> Generator[Any, Any, None]: 160 | yield p 161 | yield ctx.lfi(baz) 162 | 163 | 164 | def fail_25(ctx: Context) -> str: 165 | r = ctx.get_dependency("resonate:random") 166 | assert isinstance(r, random.Random) 167 | 168 | if r.random() < 0.25: 169 | msg = f"ko — {ctx.info.attempt} attempt(s)" 170 | raise RuntimeError(msg) 171 | 172 | return f"ok — {ctx.info.attempt} attempt(s)" 173 | 174 | 175 | def fail_50(ctx: Context) -> str: 176 | r = ctx.get_dependency("resonate:random") 177 | assert isinstance(r, random.Random) 178 | 179 | if r.random() < 0.50: 180 | msg = f"ko — {ctx.info.attempt} attempt(s)" 181 | raise RuntimeError(msg) 182 | 183 | return f"ok — {ctx.info.attempt} attempt(s)" 184 | 185 | 186 | def fail_75(ctx: Context) -> str: 187 | r = ctx.get_dependency("resonate:random") 188 | assert isinstance(r, random.Random) 189 | 190 | if r.random() < 0.75: 191 | msg = f"ko — {ctx.info.attempt} attempt(s)" 192 | raise RuntimeError(msg) 193 | 194 | return f"ok — {ctx.info.attempt} attempt(s)" 195 | 196 | 197 | def fail_99(ctx: Context) -> str: 198 | r = ctx.get_dependency("resonate:random") 199 | assert isinstance(r, random.Random) 200 | 201 | if r.random() < 0.99: 202 | msg = f"ko — {ctx.info.attempt} attempt(s)" 203 | raise RuntimeError(msg) 204 | 205 | return f"ok — {ctx.info.attempt} attempt(s)" 206 | 207 | 208 | def fib(ctx: Context, n: int) -> Generator[Any, Any, int]: 209 | if n <= 1: 210 | return n 211 | 212 | ps = [] 213 | vs = [] 214 | for i in range(1, 3): 215 | match (yield ctx.random.choice(["lfi", "rfi", "lfc", "rfc"]).options(id=f"fib:{n - i}")): 216 | case "lfi": 217 | p = yield ctx.lfi(fib, n - i).options(id=f"fib-{n - i}") 218 | ps.append(p) 219 | case "rfi": 220 | p = yield ctx.rfi(fib, n - i).options(id=f"fib-{n - i}") 221 | ps.append(p) 222 | case "lfc": 223 | v = yield ctx.lfc(fib, n - i).options(id=f"fib-{n - i}") 224 | vs.append(v) 225 | case "rfc": 226 | v = yield ctx.rfc(fib, n - i).options(id=f"fib-{n - i}") 227 | vs.append(v) 228 | 229 | for p in ps: 230 | v = yield p 231 | vs.append(v) 232 | 233 | assert len(vs) == 2 234 | return sum(vs) 235 | 236 | 237 | def test_dst(seed: str, steps: int, log_level: int) -> None: 238 | logger.setLevel(log_level or logging.INFO) # if log level is not set use INFO for dst 239 | logger.info("DST(seed=%s, steps=%s, log_level=%s)", seed, steps, log_level) 240 | 241 | # create seeded random number generator 242 | r = random.Random(seed) 243 | 244 | # create a step clock 245 | clock = StepClock() 246 | 247 | # create a registry 248 | registry = Registry() 249 | registry.add(foo, "foo") 250 | registry.add(bar, "bar") 251 | registry.add(baz, "baz") 252 | registry.add(foo_lfi, "foo_lfi") 253 | registry.add(bar_lfi, "bar_lfi") 254 | registry.add(foo_lfc, "foo_lfc") 255 | registry.add(bar_lfc, "bar_lfc") 256 | registry.add(foo_rfi, "foo_rfi") 257 | registry.add(bar_rfi, "bar_rfi") 258 | registry.add(foo_rfc, "foo_rfc") 259 | registry.add(bar_rfc, "bar_rfc") 260 | registry.add(foo_detached, "foo_detached") 261 | registry.add(bar_detached, "bar_detached") 262 | registry.add(structured_concurrency_lfi, "structured_concurrency_lfi") 263 | registry.add(structured_concurrency_rfi, "structured_concurrency_rfi") 264 | registry.add(same_p_lfi, "same_p_lfi") 265 | registry.add(same_p_rfi, "same_p_rfi") 266 | registry.add(same_v_lfi, "same_v_lfi") 267 | registry.add(same_v_rfi, "same_v_rfi") 268 | registry.add(fail_25, "fail_25") 269 | registry.add(fail_50, "fail_50") 270 | registry.add(fail_75, "fail_75") 271 | registry.add(fail_99, "fail_99") 272 | registry.add(fib, "fib") 273 | 274 | # create dependencies 275 | dependencies = Dependencies() 276 | dependencies.add("resonate:random", r) 277 | dependencies.add("resonate:time", clock) 278 | 279 | # create a simulator 280 | sim = Simulator(r, clock) 281 | servers: list[Server] = [ 282 | Server( 283 | r, 284 | "sim://uni@server", 285 | "sim://any@server", 286 | clock=clock, 287 | ), 288 | ] 289 | workers: list[Worker] = [ 290 | Worker( 291 | r, 292 | f"sim://uni@default/{n}", 293 | f"sim://any@default/{n}", 294 | clock=clock, 295 | registry=registry, 296 | dependencies=dependencies, 297 | log_level=log_level, 298 | ) 299 | for n in range(10) 300 | ] 301 | 302 | # add components to simlator 303 | for c in servers + workers: 304 | sim.add_component(c) 305 | 306 | # step the simlator 307 | for _ in range(steps): 308 | # step 309 | sim.step() 310 | 311 | # only generate command 10% of the time 312 | if r.random() > 0.1: 313 | continue 314 | 315 | # id set 316 | n = r.randint(0, 99) 317 | id = str(n) 318 | 319 | # opts 320 | opts = Options( 321 | timeout=r.randint(0, steps), 322 | ) 323 | 324 | # generate commands 325 | match r.randint(0, 24): 326 | case 0: 327 | sim.send_msg("sim://any@default", Listen(id)) 328 | case 1: 329 | conv = Remote(id, id, id, "foo", opts=opts) 330 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, foo)) 331 | case 2: 332 | conv = Remote(id, id, id, "bar", opts=opts) 333 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, bar)) 334 | case 3: 335 | conv = Remote(id, id, id, "baz", opts=opts) 336 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, baz)) 337 | case 4: 338 | conv = Remote(id, id, id, "foo_lfi", opts=opts) 339 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, foo_lfi)) 340 | case 5: 341 | conv = Remote(id, id, id, "bar_lfi", opts=opts) 342 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, bar_lfi)) 343 | case 6: 344 | conv = Remote(id, id, id, "foo_lfc", opts=opts) 345 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, foo_lfc)) 346 | case 7: 347 | conv = Remote(id, id, id, "bar_lfc", opts=opts) 348 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, bar_lfc)) 349 | case 8: 350 | conv = Remote(id, id, id, "foo_rfi", opts=opts) 351 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, foo_rfi)) 352 | case 9: 353 | conv = Remote(id, id, id, "bar_rfi", opts=opts) 354 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, bar_rfi)) 355 | case 10: 356 | conv = Remote(id, id, id, "foo_rfc", opts=opts) 357 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, foo_rfc)) 358 | case 11: 359 | conv = Remote(id, id, id, "bar_rfc", opts=opts) 360 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, bar_rfc)) 361 | case 12: 362 | conv = Remote(id, id, id, "foo_detached", opts=opts) 363 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, foo_detached)) 364 | case 13: 365 | conv = Remote(id, id, id, "bar_detached", opts=opts) 366 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, bar_detached)) 367 | case 14: 368 | conv = Remote(id, id, id, "structured_concurrency_lfi", opts=opts) 369 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, structured_concurrency_lfi)) 370 | case 15: 371 | conv = Remote(id, id, id, "same_p_lfi", (n,), opts=opts) 372 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, same_p_lfi, (n,))) 373 | case 16: 374 | conv = Remote(id, id, id, "same_p_rfi", (n,), opts=opts) 375 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, same_p_rfi, (n,))) 376 | case 17: 377 | conv = Remote(id, id, id, "same_v_lfi", (n,), opts=opts) 378 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, same_v_lfi, (n,))) 379 | case 18: 380 | conv = Remote(id, id, id, "same_v_rfi", (n,), opts=opts) 381 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, same_v_rfi, (n,))) 382 | case 19: 383 | conv = Remote(id, id, id, "structured_concurrency_rfi", opts=opts) 384 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, structured_concurrency_rfi)) 385 | case 20: 386 | conv = Remote(id, id, id, "fail_25", opts=opts) 387 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, fail_25)) 388 | case 21: 389 | conv = Remote(id, id, id, "fail_50", opts=opts) 390 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, fail_50)) 391 | case 22: 392 | conv = Remote(id, id, id, "fail_75", opts=opts) 393 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, fail_75)) 394 | case 23: 395 | conv = Remote(id, id, id, "fail_99", opts=opts) 396 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, fail_99)) 397 | case 24: 398 | conv = Remote(id, id, id, "fib", (n,), opts=opts) 399 | sim.send_msg("sim://any@default", Invoke(id, conv, 0, fib, (n,))) 400 | 401 | # log 402 | for log in sim.logs: 403 | logger.info(log) 404 | 405 | print_worker_computations(workers) 406 | 407 | 408 | def print_worker_computations(workers: list[Worker]) -> None: 409 | head = ["id", "worker", "func", "result", "attempts", "timeout"] 410 | data = sorted( 411 | [ 412 | [c.id, w.uni, func(c.graph.root.value.func), traffic_light(c.result()), info(c.graph.root.value.func).attempt, f"{info(c.graph.root.value.func).timeout:,.0f}"] 413 | for w in workers 414 | for c in w.scheduler.computations.values() 415 | ], 416 | key=lambda row: natsort(row[0]), # sort by computation id 417 | ) 418 | logger.debug("\n%s", tabulate(data, head, tablefmt="outline", colalign=("left", "left", "left", "left", "right", "right"))) 419 | 420 | 421 | def func(f: Init | Lfnc | Rfnc | Coro | None) -> str: 422 | match f: 423 | case Init(next): 424 | return func(next) 425 | case Lfnc(func=fn) | Coro(func=fn): 426 | return fn.__name__ 427 | case Rfnc(conv=conv): 428 | return json.loads(conv.data)["func"] 429 | case None: 430 | return "" 431 | 432 | 433 | def info(f: Init | Lfnc | Rfnc | Coro | None) -> Info: 434 | class InfoPlaceholder: 435 | attempt = 0 436 | idempotency_key = None 437 | tags = None 438 | timeout = 0 439 | version = 0 440 | 441 | match f: 442 | case Lfnc(ctx=ctx) | Coro(ctx=ctx): 443 | return ctx.info 444 | case _: 445 | return InfoPlaceholder() 446 | 447 | 448 | def traffic_light(r: Result | None) -> str: 449 | match r: 450 | case Ok(v): 451 | return f"🟢 {v}" 452 | case None: 453 | return "🟡" 454 | case Ko(e): 455 | return f"🔴 {e}" 456 | 457 | 458 | def natsort(s: str | int) -> list[str | int]: 459 | return [int(t) if t.isdigit() else t.lower() for t in re.split(r"(\d+)", str(s))] 460 | -------------------------------------------------------------------------------- /resonate/stores/remote.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import logging 4 | import os 5 | import sys 6 | import time 7 | from typing import TYPE_CHECKING, Any 8 | 9 | import requests 10 | from requests import PreparedRequest, Request, Session 11 | 12 | from resonate.encoders import Base64Encoder 13 | from resonate.errors import ResonateStoreError 14 | from resonate.models.callback import Callback 15 | from resonate.models.durable_promise import DurablePromise 16 | from resonate.models.schedules import Schedule 17 | from resonate.models.task import Task 18 | from resonate.retry_policies import Constant 19 | 20 | if TYPE_CHECKING: 21 | from resonate.models.encoder import Encoder 22 | from resonate.models.retry_policy import RetryPolicy 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | 27 | class RemoteStore: 28 | def __init__( 29 | self, 30 | host: str | None = None, 31 | port: str | None = None, 32 | auth: tuple[str, str] | None = None, 33 | encoder: Encoder[str | None, str | None] | None = None, 34 | timeout: float | tuple[float, float] = 5, 35 | retry_policy: RetryPolicy | None = None, 36 | ) -> None: 37 | self._host = host or os.getenv("RESONATE_HOST_STORE", os.getenv("RESONATE_HOST", "http://localhost")) 38 | self._port = port or os.getenv("RESONATE_PORT_STORE", "8001") 39 | self._auth = auth or ((os.getenv("RESONATE_USERNAME", ""), os.getenv("RESONATE_PASSWORD", "")) if "RESONATE_USERNAME" in os.environ else None) 40 | self._encoder = encoder or Base64Encoder() 41 | self._timeout = timeout 42 | self._retry_policy = retry_policy or Constant(delay=1, max_retries=sys.maxsize) # We will retry forever until we implement the drop task strategy 43 | 44 | self._promises = RemotePromiseStore(self) 45 | self._tasks = RemoteTaskStore(self) 46 | self._schedules = RemoteScheduleStore(self) 47 | 48 | @property 49 | def url(self) -> str: 50 | return f"{self._host}:{self._port}" 51 | 52 | @property 53 | def encoder(self) -> Encoder[str | None, str | None]: 54 | return self._encoder 55 | 56 | @property 57 | def promises(self) -> RemotePromiseStore: 58 | return self._promises 59 | 60 | @property 61 | def tasks(self) -> RemoteTaskStore: 62 | return self._tasks 63 | 64 | @property 65 | def schedules(self) -> RemoteScheduleStore: 66 | return self._schedules 67 | 68 | def call(self, req: PreparedRequest) -> Any: 69 | attempt = 0 70 | req.prepare_auth(self._auth) 71 | 72 | with Session() as s: 73 | while True: 74 | delay = self._retry_policy.next(attempt) 75 | attempt += 1 76 | 77 | try: 78 | res = s.send(req, timeout=self._timeout) 79 | if res.status_code == 204: 80 | return None 81 | 82 | res.raise_for_status() 83 | data = res.json() 84 | except requests.exceptions.HTTPError as e: 85 | try: 86 | error = e.response.json()["error"] 87 | except Exception: 88 | error = {"message": e.response.text, "code": 0} 89 | 90 | # Only a 500 response code should be retried 91 | if delay is None or e.response.status_code != 500: 92 | mesg = error.get("message", "Unknown exception") 93 | code = error.get("code", 0) 94 | details = error.get("details") 95 | raise ResonateStoreError(mesg=mesg, code=code, details=details) from e 96 | except requests.exceptions.Timeout as e: 97 | if delay is None: 98 | raise ResonateStoreError(mesg="Request timed out", code=0) from e 99 | 100 | logger.warning("Networking. Cannot connect to %s. Retrying in %s sec", self.url, delay) 101 | except requests.exceptions.ConnectionError as e: 102 | if delay is None: 103 | raise ResonateStoreError(mesg="Failed to connect", code=0) from e 104 | 105 | logger.warning("Networking. Cannot connect to %s. Retrying in %s sec", self.url, delay) 106 | except Exception as e: 107 | if delay is None: 108 | raise ResonateStoreError(mesg="Unknown exception", code=0) from e 109 | 110 | logger.warning("Networking. Cannot connect to %s. Retrying in %s sec", self.url, delay) 111 | else: 112 | return data 113 | 114 | time.sleep(delay) 115 | 116 | 117 | class RemotePromiseStore: 118 | def __init__(self, store: RemoteStore) -> None: 119 | self._store = store 120 | 121 | def _headers(self, *, strict: bool, ikey: str | None) -> dict[str, str]: 122 | headers: dict[str, str] = {"strict": str(strict)} 123 | if ikey is not None: 124 | headers["idempotency-Key"] = ikey 125 | return headers 126 | 127 | def get(self, id: str) -> DurablePromise: 128 | req = Request( 129 | method="get", 130 | url=f"{self._store.url}/promises/{id}", 131 | ) 132 | 133 | res = self._store.call(req.prepare()) 134 | return DurablePromise.from_dict(self._store, res) 135 | 136 | def create( 137 | self, 138 | id: str, 139 | timeout: int, 140 | *, 141 | ikey: str | None = None, 142 | strict: bool = False, 143 | headers: dict[str, str] | None = None, 144 | data: str | None = None, 145 | tags: dict[str, str] | None = None, 146 | ) -> DurablePromise: 147 | param = {} 148 | if headers is not None: 149 | param["headers"] = headers 150 | if data is not None: 151 | param["data"] = self._store.encoder.encode(data) 152 | 153 | req = Request( 154 | method="post", 155 | url=f"{self._store.url}/promises", 156 | headers=self._headers(strict=strict, ikey=ikey), 157 | json={ 158 | "id": id, 159 | "param": param, 160 | "timeout": timeout, 161 | "tags": tags or {}, 162 | }, 163 | ) 164 | res = self._store.call(req.prepare()) 165 | return DurablePromise.from_dict(self._store, res) 166 | 167 | def create_with_task( 168 | self, 169 | id: str, 170 | timeout: int, 171 | pid: str, 172 | ttl: int, 173 | *, 174 | ikey: str | None = None, 175 | strict: bool = False, 176 | headers: dict[str, str] | None = None, 177 | data: str | None = None, 178 | tags: dict[str, str] | None = None, 179 | ) -> tuple[DurablePromise, Task | None]: 180 | param = {} 181 | if headers is not None: 182 | param["headers"] = headers 183 | if data is not None: 184 | param["data"] = self._store.encoder.encode(data) 185 | 186 | req = Request( 187 | method="post", 188 | url=f"{self._store.url}/promises/task", 189 | headers=self._headers(strict=strict, ikey=ikey), 190 | json={ 191 | "promise": { 192 | "id": id, 193 | "param": param, 194 | "timeout": timeout, 195 | "tags": tags or {}, 196 | }, 197 | "task": { 198 | "processId": pid, 199 | "ttl": ttl, 200 | }, 201 | }, 202 | ) 203 | 204 | res = self._store.call(req.prepare()) 205 | promise = res["promise"] 206 | task = res.get("task") 207 | 208 | return ( 209 | DurablePromise.from_dict(self._store, promise), 210 | Task.from_dict(self._store, task) if task else None, 211 | ) 212 | 213 | def resolve( 214 | self, 215 | id: str, 216 | *, 217 | ikey: str | None = None, 218 | strict: bool = False, 219 | headers: dict[str, str] | None = None, 220 | data: str | None = None, 221 | ) -> DurablePromise: 222 | value = {} 223 | if headers is not None: 224 | value["headers"] = headers 225 | if data is not None: 226 | value["data"] = self._store.encoder.encode(data) 227 | 228 | req = Request( 229 | method="patch", 230 | url=f"{self._store.url}/promises/{id}", 231 | headers=self._headers(strict=strict, ikey=ikey), 232 | json={ 233 | "state": "RESOLVED", 234 | "value": value, 235 | }, 236 | ) 237 | 238 | res = self._store.call(req.prepare()) 239 | return DurablePromise.from_dict(self._store, res) 240 | 241 | def reject( 242 | self, 243 | id: str, 244 | *, 245 | ikey: str | None = None, 246 | strict: bool = False, 247 | headers: dict[str, str] | None = None, 248 | data: str | None = None, 249 | ) -> DurablePromise: 250 | value = {} 251 | if headers is not None: 252 | value["headers"] = headers 253 | if data is not None: 254 | value["data"] = self._store.encoder.encode(data) 255 | 256 | req = Request( 257 | method="patch", 258 | url=f"{self._store.url}/promises/{id}", 259 | headers=self._headers(strict=strict, ikey=ikey), 260 | json={ 261 | "state": "REJECTED", 262 | "value": value, 263 | }, 264 | ) 265 | 266 | res = self._store.call(req.prepare()) 267 | return DurablePromise.from_dict(self._store, res) 268 | 269 | def cancel( 270 | self, 271 | id: str, 272 | *, 273 | ikey: str | None = None, 274 | strict: bool = False, 275 | headers: dict[str, str] | None = None, 276 | data: str | None = None, 277 | ) -> DurablePromise: 278 | value = {} 279 | if headers is not None: 280 | value["headers"] = headers 281 | if data is not None: 282 | value["data"] = self._store.encoder.encode(data) 283 | 284 | req = Request( 285 | method="patch", 286 | url=f"{self._store.url}/promises/{id}", 287 | headers=self._headers(strict=strict, ikey=ikey), 288 | json={ 289 | "state": "REJECTED_CANCELED", 290 | "value": value, 291 | }, 292 | ) 293 | 294 | res = self._store.call(req.prepare()) 295 | return DurablePromise.from_dict(self._store, res) 296 | 297 | def callback( 298 | self, 299 | promise_id: str, 300 | root_promise_id: str, 301 | recv: str, 302 | timeout: int, 303 | ) -> tuple[DurablePromise, Callback | None]: 304 | req = Request( 305 | method="post", 306 | url=f"{self._store.url}/promises/callback/{promise_id}", 307 | json={ 308 | "rootPromiseId": root_promise_id, 309 | "timeout": timeout, 310 | "recv": recv, 311 | }, 312 | ) 313 | 314 | res = self._store.call(req.prepare()) 315 | promise = res["promise"] 316 | callback = res.get("callback") 317 | 318 | return ( 319 | DurablePromise.from_dict(self._store, promise), 320 | Callback.from_dict(callback) if callback else None, 321 | ) 322 | 323 | def subscribe( 324 | self, 325 | id: str, 326 | promise_id: str, 327 | recv: str, 328 | timeout: int, 329 | ) -> tuple[DurablePromise, Callback | None]: 330 | req = Request( 331 | method="post", 332 | url=f"{self._store.url}/promises/subscribe/{promise_id}", 333 | json={ 334 | "id": id, 335 | "timeout": timeout, 336 | "recv": recv, 337 | }, 338 | ) 339 | 340 | res = self._store.call(req.prepare()) 341 | promise = res["promise"] 342 | callback = res.get("callback") 343 | 344 | return ( 345 | DurablePromise.from_dict(self._store, promise), 346 | Callback.from_dict(callback) if callback else None, 347 | ) 348 | 349 | 350 | class RemoteTaskStore: 351 | def __init__(self, store: RemoteStore) -> None: 352 | self._store = store 353 | 354 | def claim( 355 | self, 356 | id: str, 357 | counter: int, 358 | pid: str, 359 | ttl: int, 360 | ) -> tuple[DurablePromise, DurablePromise | None]: 361 | req = Request( 362 | method="post", 363 | url=f"{self._store.url}/tasks/claim", 364 | json={ 365 | "id": id, 366 | "counter": counter, 367 | "processId": pid, 368 | "ttl": ttl, 369 | }, 370 | ) 371 | 372 | res = self._store.call(req.prepare()) 373 | root = res["promises"]["root"]["data"] 374 | leaf = res["promises"].get("leaf", {}).get("data") 375 | 376 | return ( 377 | DurablePromise.from_dict(self._store, root), 378 | DurablePromise.from_dict(self._store, leaf) if leaf else None, 379 | ) 380 | 381 | def complete( 382 | self, 383 | id: str, 384 | counter: int, 385 | ) -> bool: 386 | req = Request( 387 | method="post", 388 | url=f"{self._store.url}/tasks/complete", 389 | json={ 390 | "id": id, 391 | "counter": counter, 392 | }, 393 | ) 394 | 395 | self._store.call(req.prepare()) 396 | return True 397 | 398 | def heartbeat( 399 | self, 400 | pid: str, 401 | ) -> int: 402 | req = Request( 403 | method="post", 404 | url=f"{self._store.url}/tasks/heartbeat", 405 | json={ 406 | "processId": pid, 407 | }, 408 | ) 409 | 410 | res = self._store.call(req.prepare()) 411 | return res["tasksAffected"] 412 | 413 | 414 | class RemoteScheduleStore: 415 | def __init__(self, store: RemoteStore) -> None: 416 | self._store = store 417 | 418 | def create( 419 | self, 420 | id: str, 421 | cron: str, 422 | promise_id: str, 423 | promise_timeout: int, 424 | *, 425 | ikey: str | None = None, 426 | description: str | None = None, 427 | tags: dict[str, str] | None = None, 428 | promise_headers: dict[str, str] | None = None, 429 | promise_data: str | None = None, 430 | promise_tags: dict[str, str] | None = None, 431 | ) -> Schedule: 432 | promise_param = {} 433 | if promise_headers is not None: 434 | promise_param["headers"] = promise_headers 435 | if promise_data is not None: 436 | promise_param["data"] = self._store.encoder.encode(promise_data) 437 | 438 | req = Request( 439 | method="post", 440 | url=f"{self._store.url}/schedules", 441 | headers=self._headers(ikey=ikey), 442 | json={ 443 | "id": id, 444 | "description": description or "", 445 | "cron": cron, 446 | "tags": tags or {}, 447 | "promiseId": promise_id, 448 | "promiseTimeout": promise_timeout, 449 | "promiseParam": promise_param, 450 | "promiseTags": promise_tags or {}, 451 | }, 452 | ) 453 | res = self._store.call(req.prepare()) 454 | return Schedule.from_dict(self._store, res) 455 | 456 | def get(self, id: str) -> Schedule: 457 | req = Request(method="get", url=f"{self._store.url}/schedules/{id}") 458 | res = self._store.call(req.prepare()) 459 | return Schedule.from_dict(self._store, res) 460 | 461 | def delete(self, id: str) -> None: 462 | req = Request(method="delete", url=f"{self._store.url}/schedules/{id}") 463 | self._store.call(req.prepare()) 464 | 465 | def _headers(self, *, ikey: str | None) -> dict[str, str]: 466 | headers: dict[str, str] = {} 467 | if ikey is not None: 468 | headers["idempotency-Key"] = ikey 469 | return headers 470 | -------------------------------------------------------------------------------- /tests/test_bridge.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import asyncio 4 | import sys 5 | import threading 6 | import time 7 | import uuid 8 | from typing import TYPE_CHECKING, Any, Literal 9 | from unittest.mock import patch 10 | 11 | import pytest 12 | 13 | from resonate.errors import ResonateShutdownError 14 | from resonate.errors.errors import ResonateStoreError 15 | from resonate.resonate import Resonate 16 | from resonate.retry_policies import Constant, Never 17 | from resonate.stores import LocalStore 18 | 19 | if TYPE_CHECKING: 20 | from collections.abc import Generator 21 | 22 | from resonate.coroutine import Yieldable 23 | from resonate.models.message_source import MessageSource 24 | from resonate.models.store import Store 25 | from resonate.resonate import Context 26 | 27 | 28 | def foo_lfi(ctx: Context) -> Generator: 29 | p = yield ctx.lfi(bar_lfi) 30 | v = yield p 31 | return v 32 | 33 | 34 | def bar_lfi(ctx: Context) -> Generator: 35 | p = yield ctx.lfi(baz) 36 | v = yield p 37 | return v 38 | 39 | 40 | def foo_rfi(ctx: Context) -> Generator: 41 | p = yield ctx.rfi(bar_rfi) 42 | v = yield p 43 | return v 44 | 45 | 46 | def bar_rfi(ctx: Context) -> Generator: 47 | p = yield ctx.rfi(baz) 48 | v = yield p 49 | return v 50 | 51 | 52 | def baz(ctx: Context) -> str: 53 | return "baz" 54 | 55 | 56 | def fib_lfi(ctx: Context, n: int) -> Generator[Any, Any, int]: 57 | if n <= 1: 58 | return n 59 | 60 | p1 = yield ctx.lfi(fib_lfi, n - 1).options(id=f"fli{n - 1}") 61 | p2 = yield ctx.lfi(fib_lfi, n - 2).options(id=f"fli{n - 2}") 62 | 63 | v1 = yield p1 64 | v2 = yield p2 65 | 66 | return v1 + v2 67 | 68 | 69 | def fib_lfc(ctx: Context, n: int) -> Generator[Any, Any, int]: 70 | if n <= 1: 71 | return n 72 | 73 | v1 = yield ctx.lfc(fib_lfc, n - 1).options(id=f"flc{n - 1}") 74 | v2 = yield ctx.lfc(fib_lfc, n - 2).options(id=f"flc{n - 2}") 75 | 76 | return v1 + v2 77 | 78 | 79 | def fib_rfi(ctx: Context, n: int) -> Generator[Any, Any, int]: 80 | if n <= 1: 81 | return n 82 | 83 | p1 = yield ctx.rfi(fib_rfi, n - 1).options(id=f"fri{n - 1}") 84 | p2 = yield ctx.rfi(fib_rfi, n - 2).options(id=f"fri{n - 2}") 85 | 86 | v1 = yield p1 87 | v2 = yield p2 88 | 89 | return v1 + v2 90 | 91 | 92 | def fib_rfc(ctx: Context, n: int) -> Generator[Any, Any, int]: 93 | if n <= 1: 94 | return n 95 | 96 | v1 = yield ctx.rfc(fib_rfc, n - 1).options(id=f"frc{n - 1}") 97 | v2 = yield ctx.rfc(fib_rfc, n - 2).options(id=f"frc{n - 2}") 98 | 99 | return v1 + v2 100 | 101 | 102 | def sleep(ctx: Context, n: int) -> Generator[Yieldable, Any, int]: 103 | yield ctx.sleep(n) 104 | return 1 105 | 106 | 107 | def add_one(ctx: Context, n: int) -> int: 108 | return n + 1 109 | 110 | 111 | def get_dependency(ctx: Context) -> int: 112 | dep = ctx.get_dependency("foo") 113 | assert dep is not None 114 | return dep + 1 115 | 116 | 117 | def rfi_add_one_by_name(ctx: Context, n: int) -> Generator[Any, Any, int]: 118 | v = yield ctx.rfc("add_one", n) 119 | return v 120 | 121 | 122 | def hitl(ctx: Context, id: str | None) -> Generator[Yieldable, Any, int]: 123 | p = yield ctx.promise().options(id=id, idempotency_key=id) 124 | v = yield p 125 | return v 126 | 127 | 128 | def random_generation(ctx: Context) -> Generator[Yieldable, Any, float]: 129 | return (yield ctx.random.randint(0, 10)) 130 | 131 | 132 | def info1(ctx: Context, idempotency_key: str, tags: dict[str, str], version: int) -> None: 133 | assert ctx.info.attempt == 1 134 | assert ctx.info.idempotency_key == idempotency_key 135 | assert ctx.info.tags == tags 136 | assert ctx.info.version == version 137 | 138 | 139 | def info2(ctx: Context, *args: Any, **kwargs: Any) -> Generator[Yieldable, Any, None]: 140 | info1(ctx, *args, **kwargs) 141 | yield ctx.lfc(info1, f"{ctx.id}.1", {"resonate:root": ctx.id, "resonate:parent": ctx.id, "resonate:scope": "local"}, 1) 142 | yield ctx.rfc(info1, f"{ctx.id}.2", {"resonate:root": ctx.id, "resonate:parent": ctx.id, "resonate:scope": "global", "resonate:invoke": "poll://any@default"}, 1) 143 | yield (yield ctx.lfi(info1, f"{ctx.id}.3", {"resonate:root": ctx.id, "resonate:parent": ctx.id, "resonate:scope": "local"}, 1)) 144 | yield (yield ctx.rfi(info1, f"{ctx.id}.4", {"resonate:root": ctx.id, "resonate:parent": ctx.id, "resonate:scope": "global", "resonate:invoke": "poll://any@default"}, 1)) 145 | yield (yield ctx.detached(info1, f"{ctx.id}.5", {"resonate:root": ctx.id, "resonate:parent": ctx.id, "resonate:scope": "global", "resonate:invoke": "poll://any@default"}, 1)) 146 | 147 | 148 | def parent_bound(ctx: Context, child_timeout_rel: float, mode: Literal["rfc", "lfc"]) -> Generator[Yieldable, Any, None]: 149 | match mode: 150 | case "lfc": 151 | yield ctx.lfc(child_bounded, ctx.info.timeout).options(timeout=child_timeout_rel) 152 | case "rfc": 153 | yield ctx.rfc(child_bounded, ctx.info.timeout).options(timeout=child_timeout_rel) 154 | 155 | 156 | def child_bounded(ctx: Context, parent_timeout_abs: float) -> None: 157 | assert not (ctx.info.timeout > parent_timeout_abs) # child timeout never exceeds parent timeout 158 | 159 | 160 | def unbound_detached( 161 | ctx: Context, 162 | parent_timeout_rel: float, 163 | child_timeout_rel: float, 164 | ) -> Generator[Yieldable, Any, None]: 165 | p = yield ctx.detached(child_unbounded, parent_timeout_rel, child_timeout_rel, ctx.info.timeout).options(timeout=child_timeout_rel) 166 | yield p 167 | 168 | 169 | def child_unbounded(ctx: Context, parent_timeout_rel: float, child_timeout_rel: float, parent_timeout_abs: float) -> None: 170 | if parent_timeout_rel < child_timeout_rel: 171 | assert ctx.info.timeout > parent_timeout_abs 172 | elif parent_timeout_rel > child_timeout_rel: 173 | assert ctx.info.timeout < parent_timeout_abs 174 | else: 175 | assert pytest.approx(ctx.info.timeout) == parent_timeout_abs 176 | 177 | 178 | def wkflw(ctx: Context, durable: bool) -> Generator[Yieldable, Any, None]: 179 | yield ctx.lfc(failure_fn).options(timeout=1, retry_policy=Constant(delay=10, max_retries=1_000_000), durable=durable) 180 | 181 | 182 | def failure_fn(ctx: Context) -> None: 183 | raise RuntimeError 184 | 185 | 186 | def failure_wkflw(ctx: Context) -> Generator[Yieldable, Any, None]: 187 | yield ctx.lfc(add_one, 1) 188 | raise RuntimeError 189 | 190 | 191 | @pytest.fixture 192 | def resonate(store: Store, message_source: MessageSource) -> Generator[Resonate, None, None]: 193 | resonate = Resonate(store=store, message_source=message_source) 194 | resonate.register(foo_lfi) 195 | resonate.register(bar_lfi) 196 | resonate.register(foo_rfi) 197 | resonate.register(bar_rfi) 198 | resonate.register(baz) 199 | resonate.register(fib_lfi) 200 | resonate.register(fib_lfc) 201 | resonate.register(fib_rfi) 202 | resonate.register(fib_rfc) 203 | resonate.register(sleep) 204 | resonate.register(add_one) 205 | resonate.register(rfi_add_one_by_name) 206 | resonate.register(get_dependency) 207 | resonate.register(hitl) 208 | resonate.register(random_generation) 209 | resonate.register(info1, name="info", version=1) 210 | resonate.register(info2, name="info", version=2) 211 | resonate.register(parent_bound) 212 | resonate.register(child_bounded) 213 | resonate.register(unbound_detached) 214 | resonate.register(child_unbounded) 215 | resonate.register(wkflw) 216 | resonate.register(failure_wkflw) 217 | 218 | # start resonate (this starts the bridge) 219 | resonate.start() 220 | 221 | yield resonate 222 | 223 | # stop resonate (and the bridge) 224 | resonate.stop() 225 | 226 | 227 | def test_await_keyword(resonate: Resonate) -> None: 228 | async def run() -> None: 229 | v = await resonate.begin_run(uuid.uuid4().hex, add_one, 2) 230 | assert v == 3 231 | v = await resonate.begin_rpc(uuid.uuid4().hex, add_one, 2) 232 | assert v == 3 233 | 234 | asyncio.run(run()) 235 | 236 | 237 | def test_local_invocations_with_registered_functions(resonate: Resonate) -> None: 238 | @resonate.register 239 | def recursive(ctx: Context, n: int) -> Generator[Yieldable, Any, int]: 240 | if n == 1: 241 | return 1 242 | elif n % 2 == 0: 243 | return (yield ctx.lfc(recursive, n - 1)) 244 | else: 245 | return (yield (yield ctx.lfi(recursive, n - 1))) 246 | 247 | assert recursive.run("recursive", 5) == 1 248 | 249 | 250 | @pytest.mark.parametrize("durable", [True, False]) 251 | def test_fail_immediately_fn(resonate: Resonate, durable: bool) -> None: 252 | with pytest.raises(RuntimeError): 253 | resonate.run(f"fail-immediately-fn-{uuid.uuid4()}", wkflw, durable) 254 | 255 | 256 | def test_fail_immediately_coro(resonate: Resonate) -> None: 257 | with pytest.raises(RuntimeError): 258 | resonate.options(timeout=1, retry_policy=Constant(delay=10, max_retries=1_000_000)).run(f"fail-immediately-coro-{uuid.uuid4()}", failure_wkflw) 259 | 260 | 261 | @pytest.mark.parametrize("mode", ["rfc", "lfc"]) 262 | @pytest.mark.parametrize(("parent_timeout", "child_timeout"), [(1100, 10), (10, 1100), (10, 10), (10, 11), (11, 10)]) 263 | def test_timeout_bound_by_parent(resonate: Resonate, mode: Literal["rfc", "lfc"], parent_timeout: float, child_timeout: float) -> None: 264 | resonate.options(timeout=parent_timeout).run(f"parent-bound-timeout-{uuid.uuid4()}", parent_bound, child_timeout, mode) 265 | 266 | 267 | @pytest.mark.parametrize( 268 | ("parent_timeout", "child_timeout"), 269 | [ 270 | (1100, 10), 271 | (10, 1100), 272 | (10, 10), 273 | (10, 11), 274 | (11, 10), 275 | ], 276 | ) 277 | def test_timeout_unbound_by_parent_detached(resonate: Resonate, parent_timeout: float, child_timeout: float) -> None: 278 | resonate.options(timeout=parent_timeout).run(f"parent-bound-timeout-{uuid.uuid4()}", unbound_detached, parent_timeout, child_timeout) 279 | 280 | 281 | def test_random_generation(resonate: Resonate) -> None: 282 | timestamp = int(time.time()) 283 | assert resonate.run(f"random-gen-{timestamp}", random_generation) == resonate.run(f"random-gen-{timestamp}", random_generation) 284 | 285 | 286 | @pytest.mark.parametrize("id", ["foo", None]) 287 | def test_hitl(resonate: Resonate, id: str) -> None: 288 | uid = uuid.uuid4().hex 289 | id = id or f"hitl-{uid}.1" 290 | p = resonate.promises.create(id, sys.maxsize, ikey=id, strict=False) 291 | handle = resonate.begin_run(f"hitl-{uid}", hitl, p.id) 292 | p_done = resonate.promises.resolve(id=p.id, data="1") 293 | assert p_done.state == "RESOLVED" 294 | assert handle.result() == 1 295 | 296 | 297 | def test_get_dependency(resonate: Resonate) -> None: 298 | timestamp = int(time.time()) 299 | resonate.set_dependency("foo", 1) 300 | assert resonate.run(f"get-dependency-{timestamp}", get_dependency) == 2 301 | 302 | 303 | def test_basic_lfi(resonate: Resonate) -> None: 304 | timestamp = int(time.time()) 305 | assert resonate.run(f"foo-lfi-{timestamp}", foo_lfi) == "baz" 306 | 307 | 308 | def test_basic_rfi(resonate: Resonate) -> None: 309 | timestamp = int(time.time()) 310 | assert resonate.run(f"foo-rfi-{timestamp}", foo_rfi) == "baz" 311 | 312 | 313 | def test_rfi_by_name(resonate: Resonate) -> None: 314 | timestamp = int(time.time()) 315 | assert resonate.rpc(f"add_one_by_name_rfi-{timestamp}", "rfi_add_one_by_name", 42) == 43 316 | 317 | 318 | def test_fib_lfi(resonate: Resonate) -> None: 319 | timestamp = int(time.time()) 320 | fib_10 = 55 321 | assert resonate.run(f"fib_lfi-{timestamp}", fib_lfi, 10) == fib_10 322 | 323 | 324 | def test_fib_rfi(resonate: Resonate) -> None: 325 | timestamp = int(time.time()) 326 | fib_10 = 55 327 | assert resonate.run(f"fib_rfi-{timestamp}", fib_rfi, 10) == fib_10 328 | 329 | 330 | def test_fib_lfc(resonate: Resonate) -> None: 331 | timestamp = int(time.time()) 332 | fib_10 = 55 333 | assert resonate.run(f"fib_lfc-{timestamp}", fib_lfc, 10) == fib_10 334 | 335 | 336 | def test_fib_rfc(resonate: Resonate) -> None: 337 | timestamp = int(time.time()) 338 | fib_10 = 55 339 | assert resonate.run(f"fib_rfc-{timestamp}", fib_rfc, 10) == fib_10 340 | 341 | 342 | def test_sleep(resonate: Resonate) -> None: 343 | timestamp = int(time.time()) 344 | assert resonate.run(f"sleep-{timestamp}", sleep, 0) == 1 345 | 346 | 347 | def test_handle_timeout(resonate: Resonate) -> None: 348 | timestamp = int(time.time()) 349 | handle = resonate.begin_run(f"handle-timeout-{timestamp}", sleep, 1) 350 | with pytest.raises(TimeoutError): 351 | handle.result(timeout=0.1) 352 | assert handle.result() == 1 353 | 354 | 355 | def test_basic_retries() -> None: 356 | # Use a different instance that only do local store 357 | resonate = Resonate() 358 | 359 | def retriable(ctx: Context) -> int: 360 | if ctx.info.attempt == 4: 361 | return ctx.info.attempt 362 | raise RuntimeError 363 | 364 | f = resonate.register(retriable) 365 | resonate.start() 366 | 367 | start_time = time.time() 368 | result = f.options(retry_policy=Constant(delay=1, max_retries=3)).run(f"retriable-{int(start_time)}") 369 | end_time = time.time() 370 | 371 | assert result == 4 372 | delta = end_time - start_time 373 | assert delta >= 3.0 374 | assert delta < 4.0 # This is kind of arbitrary, if it is failing feel free to increase the number 375 | 376 | resonate.stop() 377 | 378 | 379 | def test_listen(resonate: Resonate) -> None: 380 | timestamp = int(time.time()) 381 | assert resonate.rpc(f"add_one_{timestamp}", "add_one", 42) == 43 382 | 383 | 384 | def test_implicit_resonate_start() -> None: 385 | resonate = Resonate() 386 | 387 | def f(ctx: Context, n: int) -> Generator[Any, Any, int]: 388 | if n == 0: 389 | return 1 390 | 391 | v = yield ctx.rfc(f, n - 1) 392 | return v + n 393 | 394 | r = resonate.register(f) 395 | 396 | timestamp = int(time.time()) 397 | assert r.run(f"r-implicit-start-{timestamp}", 1) == 2 398 | 399 | 400 | @pytest.mark.parametrize("idempotency_key", ["foo", None]) 401 | @pytest.mark.parametrize("tags", [{"foo": "bar"}, None]) 402 | @pytest.mark.parametrize("target", ["foo", "bar", None]) 403 | @pytest.mark.parametrize("version", [1, 2]) 404 | def test_info( 405 | idempotency_key: str | None, 406 | resonate: Resonate, 407 | tags: dict[str, str] | None, 408 | target: str | None, 409 | version: int, 410 | ) -> None: 411 | id = f"info-{uuid.uuid4()}" 412 | 413 | resonate = resonate.options( 414 | idempotency_key=idempotency_key, 415 | retry_policy=Never(), 416 | tags=tags, 417 | target=target, 418 | timeout=10, 419 | version=version, 420 | ) 421 | 422 | handle = resonate.begin_run( 423 | id, 424 | "info", 425 | idempotency_key or id, 426 | {**(tags or {}), "resonate:root": id, "resonate:parent": id, "resonate:scope": "global", "resonate:invoke": f"poll://any@{target or 'default'}"}, 427 | version, 428 | ) 429 | 430 | handle.result() 431 | 432 | 433 | def test_resonate_get(resonate: Resonate) -> None: 434 | def resolve_promise_slow(id: str) -> None: 435 | time.sleep(1) 436 | resonate.promises.resolve(id=id, data="42") 437 | 438 | timestamp = int(time.time()) 439 | id = f"get.{timestamp}" 440 | resonate.promises.create(id=id, timeout=sys.maxsize) 441 | thread = threading.Thread(target=resolve_promise_slow, args=(id,), daemon=True) # Do this in a different thread to simulate concurrency 442 | 443 | handle = resonate.get(id) 444 | 445 | thread.start() 446 | res = handle.result() 447 | assert res == 42 448 | thread.join() 449 | 450 | 451 | def test_resonate_platform_errors() -> None: 452 | # If you look at this test and you think: "This is horrible" 453 | # You are right, this test is cursed. But it needed to be done. 454 | local_store = LocalStore() 455 | resonate = Resonate( 456 | store=local_store, 457 | message_source=local_store.message_source("default", "default"), 458 | ) 459 | 460 | original_transition = local_store.promises.transition 461 | raise_flag = [False] # Use mutable container for flag 462 | 463 | def side_effect(*args: Any, **kwargs: Any) -> Any: 464 | if raise_flag[0]: 465 | msg = "Got an error from server" 466 | raise ResonateStoreError(msg, 0) 467 | 468 | return original_transition(*args[1:], **kwargs) 469 | 470 | def g(_: Context) -> int: 471 | return 42 472 | 473 | def f(ctx: Context, flag: bool) -> Generator[Any, Any, None]: 474 | raise_flag[0] = flag # Update mutable flag 475 | val = yield ctx.rfc(g) 476 | return val 477 | 478 | with patch.object( 479 | local_store.promises, 480 | "transition", 481 | side_effect=side_effect, 482 | ): 483 | resonate.register(f) 484 | resonate.register(g) 485 | 486 | # First test normal behavior 487 | assert resonate.run("f-no-err", f, flag=False) == 42 488 | 489 | # Now trigger errors 490 | with pytest.raises(ResonateShutdownError): 491 | resonate.run("f-err", f, flag=True) 492 | -------------------------------------------------------------------------------- /resonate/bridge.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import queue 4 | import threading 5 | import time 6 | from concurrent.futures import Future 7 | from typing import TYPE_CHECKING 8 | 9 | from resonate.conventions import Base 10 | from resonate.delay_q import DelayQ 11 | from resonate.errors import ResonateShutdownError 12 | from resonate.models.commands import ( 13 | CancelPromiseReq, 14 | CancelPromiseRes, 15 | Command, 16 | CreateCallbackReq, 17 | CreateCallbackRes, 18 | CreatePromiseReq, 19 | CreatePromiseRes, 20 | CreateSubscriptionReq, 21 | Delayed, 22 | Function, 23 | Invoke, 24 | Listen, 25 | Network, 26 | Notify, 27 | Receive, 28 | RejectPromiseReq, 29 | RejectPromiseRes, 30 | ResolvePromiseReq, 31 | ResolvePromiseRes, 32 | Resume, 33 | Retry, 34 | Return, 35 | ) 36 | from resonate.models.durable_promise import DurablePromise 37 | from resonate.models.result import Ko, Ok 38 | from resonate.models.task import Task 39 | from resonate.options import Options 40 | from resonate.processor import Processor 41 | from resonate.scheduler import Done, Info, More, Scheduler 42 | from resonate.utils import exit_on_exception 43 | 44 | if TYPE_CHECKING: 45 | from collections.abc import Callable 46 | 47 | from resonate.models.convention import Convention 48 | from resonate.models.message_source import MessageSource 49 | from resonate.models.store import Store 50 | from resonate.registry import Registry 51 | from resonate.resonate import Context 52 | 53 | 54 | class Bridge: 55 | def __init__( 56 | self, 57 | ctx: Callable[[str, str, Info], Context], 58 | pid: str, 59 | ttl: int, 60 | opts: Options, 61 | workers: int | None, 62 | unicast: str, 63 | anycast: str, 64 | registry: Registry, 65 | store: Store, 66 | message_source: MessageSource, 67 | ) -> None: 68 | self._cq = queue.Queue[Command | tuple[Command, Future] | None]() 69 | self._promise_id_to_task: dict[str, Task] = {} 70 | 71 | self._ctx = ctx 72 | self._pid = pid 73 | self._ttl = ttl 74 | self._opts = opts 75 | self._unicast = unicast 76 | self._anycast = anycast 77 | 78 | self._registry = registry 79 | self._store = store 80 | self._message_source = message_source 81 | self._delay_q = DelayQ[Function | Retry]() 82 | 83 | self._scheduler = Scheduler( 84 | ctx=self._ctx, 85 | pid=self._pid, 86 | unicast=self._unicast, 87 | anycast=self._anycast, 88 | ) 89 | self._processor = Processor(workers) 90 | 91 | self._bridge_thread = threading.Thread(target=self._process_cq, name="bridge", daemon=True) 92 | self._message_source_thread = threading.Thread(target=self._process_msgs, name="message-source", daemon=True) 93 | 94 | self._delay_q_thread = threading.Thread(target=self._process_delayed_events, name="delay-q", daemon=True) 95 | self._delay_q_condition = threading.Condition() 96 | 97 | self._heartbeat_thread = threading.Thread(target=self._heartbeat, name="heartbeat", daemon=True) 98 | self._heartbeat_active = threading.Event() 99 | 100 | self._shutdown = threading.Event() 101 | 102 | def run(self, conv: Convention, func: Callable, args: tuple, kwargs: dict, opts: Options, future: Future) -> DurablePromise: 103 | encoder = opts.get_encoder() 104 | 105 | headers, data = encoder.encode(conv.data) 106 | promise, task = self._store.promises.create_with_task( 107 | id=conv.id, 108 | ikey=conv.idempotency_key, 109 | timeout=int((time.time() + conv.timeout) * 1000), 110 | headers=headers, 111 | data=data, 112 | tags=conv.tags, 113 | pid=self._pid, 114 | ttl=self._ttl * 1000, 115 | ) 116 | if promise.completed: 117 | assert not task 118 | match promise.result(encoder): 119 | case Ok(v): 120 | future.set_result(v) 121 | case Ko(e): 122 | future.set_exception(e) 123 | elif task is not None: 124 | self._promise_id_to_task[promise.id] = task 125 | self.start_heartbeat() 126 | self._cq.put_nowait(Invoke(conv.id, conv, promise.abs_timeout, func, args, kwargs, opts, promise)) 127 | 128 | return promise 129 | 130 | def rpc(self, conv: Convention, opts: Options, future: Future) -> DurablePromise: 131 | encoder = opts.get_encoder() 132 | 133 | headers, data = encoder.encode(conv.data) 134 | promise = self._store.promises.create( 135 | id=conv.id, 136 | ikey=conv.idempotency_key, 137 | timeout=int((time.time() + conv.timeout) * 1000), 138 | headers=headers, 139 | data=data, 140 | tags=conv.tags, 141 | ) 142 | 143 | if promise.completed: 144 | match promise.result(encoder): 145 | case Ok(v): 146 | future.set_result(v) 147 | case Ko(e): 148 | future.set_exception(e) 149 | 150 | return promise 151 | 152 | def get(self, id: str, opts: Options, future: Future) -> DurablePromise: 153 | promise = self._store.promises.get(id=id) 154 | 155 | if promise.completed: 156 | match promise.result(opts.get_encoder()): 157 | case Ok(v): 158 | future.set_result(v) 159 | case Ko(e): 160 | future.set_exception(e) 161 | 162 | return promise 163 | 164 | def start(self) -> None: 165 | self._processor.start() 166 | 167 | if not self._message_source_thread.is_alive(): 168 | self._message_source.start() 169 | self._message_source_thread.start() 170 | 171 | if not self._bridge_thread.is_alive(): 172 | self._bridge_thread.start() 173 | 174 | if not self._heartbeat_thread.is_alive(): 175 | self._heartbeat_thread.start() 176 | 177 | if not self._delay_q_thread.is_alive(): 178 | self._delay_q_thread.start() 179 | 180 | def stop(self) -> None: 181 | """Stop internal components and threads. Intended for use only within the resonate class.""" 182 | self._stop_no_join() 183 | if self._bridge_thread.is_alive(): 184 | self._bridge_thread.join() 185 | if self._message_source_thread.is_alive(): 186 | self._message_source_thread.join() 187 | if self._heartbeat_thread.is_alive(): 188 | self._heartbeat_thread.join() 189 | 190 | def _stop_no_join(self) -> None: 191 | """Stop internal components and threads. Does not join the threads, to be able to call it from the bridge itself.""" 192 | self._processor.stop() 193 | self._message_source.stop() 194 | self._cq.put_nowait(None) 195 | self._heartbeat_active.clear() 196 | self._shutdown.set() 197 | 198 | @exit_on_exception 199 | def _process_cq(self) -> None: 200 | shutdown_error: ResonateShutdownError | None = None 201 | while item := self._cq.get(): 202 | cmd, future = item if isinstance(item, tuple) else (item, None) 203 | if shutdown_error is not None and future is not None: 204 | future.set_exception(shutdown_error) 205 | continue 206 | 207 | match self._scheduler.step(cmd, future): 208 | case More(reqs): 209 | for req in reqs: 210 | match req: 211 | case Network(id, cid, n_req): 212 | try: 213 | cmd = self._handle_network_request(id, cid, n_req) 214 | self._cq.put_nowait(cmd) 215 | except Exception as e: 216 | shutdown_error = ResonateShutdownError(mesg="An unexpected store error has occurred, shutting down") 217 | shutdown_error.__cause__ = e # bind original error 218 | 219 | # bypass the cq and shutdown right away 220 | self._scheduler.shutdown(shutdown_error) 221 | self._stop_no_join() 222 | 223 | case Function(id, cid, func): 224 | self._processor.enqueue(func, lambda r, id=id, cid=cid: self._cq.put_nowait(Return(id, cid, r))) 225 | case Delayed() as item: 226 | self._handle_delay(item) 227 | 228 | case Done(reqs): 229 | cid = cmd.cid 230 | task = self._promise_id_to_task.get(cid, None) 231 | match reqs: 232 | case [Network(_, cid, CreateSubscriptionReq(id, promise_id, timeout, recv))]: 233 | # Current implementation returns a single CreateSubscriptionReq in the list 234 | # if we get more than one element they are all CreateCallbackReq 235 | durable_promise, _ = self._store.promises.subscribe( 236 | id=id, 237 | promise_id=promise_id, 238 | timeout=timeout, 239 | recv=recv, 240 | ) 241 | assert durable_promise.id == cid 242 | 243 | if durable_promise.completed: 244 | self._cq.put_nowait(Notify(cid, durable_promise)) 245 | 246 | case _: 247 | got_resume = False 248 | for req in reqs: 249 | assert isinstance(req, Network) 250 | assert isinstance(req.req, CreateCallbackReq) 251 | 252 | res_cmd = self._handle_network_request(req.id, req.cid, req.req) 253 | if isinstance(res_cmd, Resume): 254 | # if we get a resume here we can bail the rest of the callback requests 255 | # and continue with the rest of the work in the cq. 256 | self._cq.put_nowait(res_cmd) 257 | got_resume = True 258 | break 259 | 260 | if got_resume: 261 | continue 262 | 263 | if task is not None: 264 | self._store.tasks.complete(id=task.id, counter=task.counter) 265 | 266 | @exit_on_exception 267 | def _process_msgs(self) -> None: 268 | encoder = self._opts.get_encoder() 269 | 270 | def _invoke(root: DurablePromise) -> Invoke: 271 | data = encoder.decode(root.param.to_tuple()) 272 | assert isinstance(data, dict) 273 | 274 | assert "func" in data 275 | assert "version" in data 276 | assert isinstance(data["func"], str) 277 | assert isinstance(data["version"], int) 278 | 279 | _, func, version = self._registry.get(data["func"], data["version"]) 280 | return Invoke( 281 | root.id, 282 | Base( 283 | root.id, 284 | root.rel_timeout, 285 | root.ikey_for_create, 286 | root.param.data, 287 | root.tags, 288 | ), 289 | root.abs_timeout, 290 | func, 291 | data.get("args", ()), 292 | data.get("kwargs", {}), 293 | Options(version=version), 294 | root, 295 | ) 296 | 297 | while msg := self._message_source.next(): 298 | match msg: 299 | case {"type": "invoke", "task": {"id": id, "counter": counter}}: 300 | task = Task(id, counter, self._store) 301 | root, _ = self._store.tasks.claim(id=task.id, counter=task.counter, pid=self._pid, ttl=self._ttl * 1000) 302 | self.start_heartbeat() 303 | self._promise_id_to_task[root.id] = task 304 | self._cq.put_nowait(_invoke(root)) 305 | 306 | case {"type": "resume", "task": {"id": id, "counter": counter}}: 307 | task = Task(id, counter, self._store) 308 | root, leaf = self._store.tasks.claim(id=task.id, counter=task.counter, pid=self._pid, ttl=self._ttl * 1000) 309 | self.start_heartbeat() 310 | assert leaf is not None, "leaf must not be None" 311 | cmd = Resume( 312 | id=leaf.id, 313 | cid=root.id, 314 | promise=leaf, 315 | invoke=_invoke(root), 316 | ) 317 | self._promise_id_to_task[root.id] = task 318 | self._cq.put_nowait(cmd) 319 | 320 | case {"type": "notify", "promise": promise}: 321 | durable_promise = DurablePromise.from_dict(self._store, promise) 322 | self._cq.put_nowait(Notify(durable_promise.id, durable_promise)) 323 | 324 | @exit_on_exception 325 | def _process_delayed_events(self) -> None: 326 | while not self._shutdown.is_set(): 327 | with self._delay_q_condition: 328 | while not self._delay_q.empty(): 329 | if self._shutdown.is_set(): 330 | self._delay_q_condition.release() 331 | return 332 | 333 | self._delay_q_condition.wait() 334 | 335 | now = time.time() 336 | events, next_time = self._delay_q.get(now) 337 | 338 | # Release the lock so more event can be added to the delay queue while 339 | # the ones just pulled get processed. 340 | self._delay_q_condition.release() 341 | 342 | for item in events: 343 | match item: 344 | case Function(id, cid, func): 345 | self._processor.enqueue(func, lambda r, id=id, cid=cid: self._cq.put_nowait(Return(id, cid, r))) 346 | case retry: 347 | self._cq.put_nowait(retry) 348 | 349 | if self._shutdown.is_set(): 350 | return 351 | 352 | timeout = max(0.0, next_time - now) 353 | self._delay_q_condition.acquire() 354 | self._delay_q_condition.wait(timeout=timeout) 355 | 356 | def start_heartbeat(self) -> None: 357 | self._heartbeat_active.set() 358 | 359 | @exit_on_exception 360 | def _heartbeat(self) -> None: 361 | while not self._shutdown.is_set(): 362 | # If this timeout don't execute the heartbeat 363 | if self._heartbeat_active.wait(0.3): 364 | heartbeated = self._store.tasks.heartbeat(pid=self._pid) 365 | if heartbeated == 0: 366 | self._heartbeat_active.clear() 367 | else: 368 | self._shutdown.wait(self._ttl * 0.5) 369 | 370 | def _handle_delay(self, delay: Delayed) -> None: 371 | """Add a command to the delay queue. 372 | 373 | Uses a threading.condition to synchronize access to the underlaying delay_q. 374 | """ 375 | with self._delay_q_condition: 376 | self._delay_q.add(delay.item, time.time() + delay.delay) 377 | self._delay_q_condition.notify() 378 | 379 | def _handle_network_request(self, cmd_id: str, cid: str, req: CreatePromiseReq | ResolvePromiseReq | RejectPromiseReq | CancelPromiseReq | CreateCallbackReq) -> Command: 380 | match req: 381 | case CreatePromiseReq(id, timeout, ikey, strict, headers, data, tags): 382 | promise = self._store.promises.create( 383 | id=id, 384 | timeout=timeout, 385 | ikey=ikey, 386 | strict=strict, 387 | headers=headers, 388 | data=data, 389 | tags=tags, 390 | ) 391 | return Receive(cmd_id, cid, CreatePromiseRes(promise)) 392 | 393 | case ResolvePromiseReq(id, ikey, strict, headers, data): 394 | promise = self._store.promises.resolve( 395 | id=id, 396 | ikey=ikey, 397 | strict=strict, 398 | headers=headers, 399 | data=data, 400 | ) 401 | return Receive(cmd_id, cid, ResolvePromiseRes(promise)) 402 | 403 | case RejectPromiseReq(id, ikey, strict, headers, data): 404 | promise = self._store.promises.reject( 405 | id=id, 406 | ikey=ikey, 407 | strict=strict, 408 | headers=headers, 409 | data=data, 410 | ) 411 | return Receive(cmd_id, cid, RejectPromiseRes(promise)) 412 | 413 | case CancelPromiseReq(id, ikey, strict, headers, data): 414 | promise = self._store.promises.cancel( 415 | id=id, 416 | ikey=ikey, 417 | strict=strict, 418 | headers=headers, 419 | data=data, 420 | ) 421 | return Receive(cmd_id, cid, CancelPromiseRes(promise)) 422 | 423 | case CreateCallbackReq(promise_id, root_promise_id, timeout, recv): 424 | promise, callback = self._store.promises.callback( 425 | promise_id=promise_id, 426 | root_promise_id=root_promise_id, 427 | timeout=timeout, 428 | recv=recv, 429 | ) 430 | 431 | if promise.completed: 432 | return Resume(cmd_id, cid, promise) 433 | 434 | return Receive(cmd_id, cid, CreateCallbackRes(promise, callback)) 435 | 436 | case _: 437 | raise NotImplementedError 438 | 439 | def subscribe(self, id: str, future: Future) -> None: 440 | self._cq.put_nowait((Listen(id), future)) 441 | --------------------------------------------------------------------------------