├── test ├── __init__.py ├── test_callback.py ├── test_basic.py └── test_multiprocess.py ├── earlgrey ├── version.py ├── patterns │ ├── __init__.py │ ├── rpc │ │ ├── __init__.py │ │ ├── server.py │ │ ├── client_sync.py │ │ └── client_async.py │ └── worker │ │ ├── __init__.py │ │ ├── server.py │ │ ├── client_async.py │ │ └── client_sync.py ├── __init__.py ├── message_queue_task.py ├── message_queue_info.py ├── message_queue_connection.py ├── message_queue_service.py └── message_queue_stub.py ├── requirements.txt ├── README.md ├── setup.py ├── .gitignore ├── .travis.yml └── LICENSE /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /earlgrey/version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.2.2" 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aio-pika~=6.7.1 2 | pika~=1.1.0 -------------------------------------------------------------------------------- /earlgrey/patterns/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .rpc import * 16 | from .worker import * 17 | -------------------------------------------------------------------------------- /test/test_callback.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class CallbackCollection: 5 | def __init__(self): 6 | self._cb: list = [] 7 | self._sender = "sender" 8 | 9 | def add(self, cb): 10 | self._cb.append(cb) 11 | 12 | def __call__(self, *args, **kwargs): 13 | for cb in self._cb: 14 | cb(self._sender, *args, **kwargs) 15 | 16 | 17 | class CallbackObj: 18 | def callback(self, sender, msg): 19 | print(sender, msg) 20 | 21 | 22 | class TestCallback(unittest.TestCase): 23 | def test_func(self): 24 | collection = CallbackCollection() 25 | obj = CallbackObj() 26 | collection.add(obj.callback) 27 | collection("msg") 28 | -------------------------------------------------------------------------------- /earlgrey/patterns/rpc/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .client_async import ClientAsync 16 | from .client_sync import ClientSync 17 | from .server import Server 18 | -------------------------------------------------------------------------------- /earlgrey/patterns/worker/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .client_async import ClientAsync 16 | from .client_sync import ClientSync 17 | from .server import Server 18 | -------------------------------------------------------------------------------- /earlgrey/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from .version import __version__ 16 | 17 | from .message_queue_info import * 18 | from .message_queue_task import * 19 | from .message_queue_connection import * 20 | from .message_queue_service import * 21 | from .message_queue_stub import * 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Earlgrey 2 | 3 | Earlgrey is a python library which provides a convenient way to publish and consume messages between processes using RabbitMQ. It is abstracted to RPC pattern. 4 | 5 | ## How to use 6 | ```python 7 | # RPC methods 8 | class Task: 9 | @message_queue_task 10 | async def echo(self, value): 11 | return value 12 | 13 | # Client stub 14 | class Stub(MessageQueueStub[Task]): 15 | TaskType = Task 16 | 17 | # Server service 18 | class Service(MessageQueueService[Task]): 19 | TaskType = Task 20 | 21 | async def run(): 22 | route_key = 'any same string between processes' 23 | 24 | client = Stub('localhost', route_key) 25 | server = Service('localhost', route_key) 26 | 27 | await client.connect() 28 | await server.connect() 29 | 30 | result = await client.async_task().echo('any value') 31 | print(result) # 'any value' 32 | 33 | loop = asyncio.get_event_loop() 34 | loop.run_until_complete(run()) 35 | 36 | ``` 37 | 38 | 39 | #### Caution 40 | Actually `MessageQueueStub` does not need exact `Task` class which has full implementation of methods. It just needs signature of methods. 41 | ```python 42 | # client side. 43 | class Task: 44 | @message_queue_task 45 | async def echo(self, value): 46 | # Just signature. It is okay. Do not need implemetation. 47 | # But server must have its implementation 48 | pass 49 | ``` 50 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from importlib import import_module 3 | from pathlib import Path 4 | 5 | from setuptools import setup, find_packages 6 | 7 | sys.path.insert(0, str(Path.cwd() / 'earlgrey')) 8 | try: 9 | module = import_module('version') 10 | version = getattr(module, '__version__') 11 | finally: 12 | sys.path = sys.path[1:] 13 | 14 | extras_requires = { 15 | 'test': ['pytest~=5.4.2', 'mock~=4.0.1'] 16 | } 17 | 18 | setup_options = { 19 | 'name': 'earlgrey', 20 | 'description': 'Python AMQP RPC library', 21 | 'long_description': open('README.md').read(), 22 | 'long_description_content_type': 'text/markdown', 23 | 'url': 'https://github.com/icon-project/earlgrey', 24 | 'version': version, 25 | 'author': 'ICON foundation', 26 | 'packages': find_packages(), 27 | 'license': "Apache License 2.0", 28 | 'install_requires': list(open('requirements.txt')), 29 | 'extras_require': extras_requires, 30 | 'classifiers': [ 31 | 'Development Status :: 5 - Production/Stable', 32 | 'Intended Audience :: Developers', 33 | 'Intended Audience :: System Administrators', 34 | 'Natural Language :: English', 35 | 'License :: OSI Approved :: Apache Software License', 36 | 'Programming Language :: Python', 37 | 'Programming Language :: Python :: 3.6', 38 | 'Programming Language :: Python :: 3.7' 39 | ] 40 | } 41 | 42 | setup(**setup_options) 43 | -------------------------------------------------------------------------------- /earlgrey/message_queue_task.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import asyncio 16 | import functools 17 | import logging 18 | import traceback 19 | 20 | from enum import IntEnum 21 | 22 | TASK_ATTR_DICT = "task_attr" 23 | MESSAGE_QUEUE_TYPE_KEY = "type" 24 | MESSAGE_QUEUE_PRIORITY_KEY = "priority" 25 | 26 | 27 | class MessageQueueType(IntEnum): 28 | Worker = 0, 29 | RPC = 1, 30 | 31 | 32 | class MessageQueueException(Exception): 33 | pass 34 | 35 | 36 | def message_queue_task(func=None, *, type_=MessageQueueType.RPC, priority=128): 37 | if func is None: 38 | return functools.partial(message_queue_task, type_=type_, priority=priority) 39 | 40 | @functools.wraps(func) 41 | async def _wrapper(*args, **kwargs): 42 | try: 43 | return await asyncio.coroutine(func)(*args, **kwargs) 44 | except Exception as e: 45 | logging.error(e) 46 | traceback.print_exc() 47 | return MessageQueueException(str(e)) 48 | 49 | task_attr = { 50 | MESSAGE_QUEUE_TYPE_KEY: type_, 51 | MESSAGE_QUEUE_PRIORITY_KEY: priority 52 | } 53 | setattr(_wrapper, TASK_ATTR_DICT, task_attr) 54 | return _wrapper 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # IDE 107 | .idea 108 | 109 | -------------------------------------------------------------------------------- /earlgrey/message_queue_info.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from aio_pika.robust_channel import RobustChannel 16 | from pika.adapters.blocking_connection import BlockingChannel 17 | 18 | 19 | class MessageQueueInfoAsync: 20 | def __init__(self, channel: RobustChannel, route_key: str=None): 21 | self._channel = channel 22 | self._route_key = route_key 23 | 24 | async def queue_info(self, name: str=None): 25 | if name is None: 26 | name = self._route_key 27 | 28 | return await self._channel.declare_queue(name, passive=True) 29 | 30 | async def exchange_info(self, name: str=None): 31 | return await self._channel.declare_exchange(name, passive=True) 32 | 33 | 34 | class MessageQueueInfoSync: 35 | def __init__(self, channel: BlockingChannel, route_key: str): 36 | self._channel = channel 37 | self._route_key = route_key 38 | 39 | def queue_info(self, name: str=None): 40 | if name is None: 41 | name = self._route_key 42 | 43 | return self._channel.queue_declare(queue=name, passive=True) 44 | 45 | def exchange_info(self, name: str=None): 46 | if name is None: 47 | name = self._route_key 48 | 49 | return self._channel.exchange_declare(exchange=name, passive=True) 50 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | dist: bionic 3 | 4 | addons: 5 | apt: 6 | packages: 7 | - rabbitmq-server 8 | 9 | python: 10 | - "3.7" 11 | 12 | branches: 13 | only: 14 | - master 15 | - develop 16 | - /^(release|hotfix)[/-].*$/ 17 | - /^(\d+)(\.\d+){1,2}((a|b|rc)?\d*)?(\.?(post|dev)\d*)?/ 18 | - deploy-test 19 | 20 | services: 21 | - rabbitmq 22 | 23 | install: 24 | - virtualenv venv 25 | - source venv/bin/activate 26 | - pip install -e .[test] 27 | 28 | script: 29 | - python -m pytest -ra 30 | 31 | deploy: 32 | - provider: pypi 33 | edge: true 34 | distributions: sdist bdist_wheel 35 | user: $PYPI_USER 36 | password: $PYPI_PW 37 | on: 38 | tags: true 39 | all_branches: true 40 | condition: $TRAVIS_TAG =~ ^([0-9]+)(\.[0-9]+){1,2}$ 41 | - provider: pypi 42 | edge: true 43 | server: https://test.pypi.org/legacy/ 44 | distributions: sdist bdist_wheel 45 | user: $PYPI_USER 46 | password: $PYPI_PW 47 | skip_existing: true 48 | on: 49 | tags: true 50 | all_branches: true 51 | condition: $TRAVIS_TAG =~ ^([0-9]+)(\.[0-9]+){1,2}((a|b|rc)+[0-9]*)+(\.?(post|dev)[0-9]*)? 52 | 53 | notifications: 54 | slack: 55 | rooms: 56 | - secure: gKVeRcXlQHJ/L/j1jiYNr2QOyQWUa+iw6RIEl0VJHFYFet0lQhGBkqKqnonQlmR9z+6P3/s1eWX15ZFwQYgtvwX+4ch03j1rop8n1inLbyA4IAfCThWn0IjKcFf6SsEI54tB/XC099uX3TwPKlLD1fxlawxwxbN0RCKx/vCVIcDDcPQmg9ICkiU0LO14rAsnUrtZua6gKgFAevppURwD1n7GEz2oc/kXG/aLKBOqJuIpeaS5FYpV2uuPevlOs5geyRsA5TiJuldCqfzXbDNL+TAR9NUlBym7RuJL763Q3ywPMziwaWV/u0EHEQfcCjP7ifdPNgriwwvJqQ0VU1RkPNuYDuY/QEEcSelGS0yD6onPfh8ggJIyPxNJgoZbYwN/+KzJHd1hpCC4xc2xIJo3lF9DOODJ+pL36CRBlluKJDYXWsBaksGd3LSrmTbfSHsSAU53QmWIMSQ8XzL1dp6Vjoksz22mBLUL5J967ZwodLJVTOhZc0t9KMZ+8EryBDFHjuLvvUrXfVHDMKrqh4xBCrsLXptokAx8Yv0ehTt0hHCdzMe7Q0bOopa4/p8BV7CZB+8aG6J+5K04w7tJbC6QhFUgOEpLkngI5VU8Q+nFfc92Ucu3iUeSmJVuidoDN7TKebdo/qn0TheD8+LSjiY+TA+lq1MQATj3HkS0L1R1b1Q= 57 | on_success: change 58 | on_failure: always 59 | -------------------------------------------------------------------------------- /earlgrey/patterns/worker/server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # The original codes exist in aio_pika.patterns.master 16 | 17 | from typing import TYPE_CHECKING 18 | 19 | import asyncio 20 | from typing import Callable 21 | from aio_pika.channel import Channel 22 | from aio_pika.message import DeliveryMode 23 | from aio_pika.patterns.base import Base 24 | 25 | if TYPE_CHECKING: 26 | from aio_pika.message import IncomingMessage 27 | 28 | 29 | class Server(Base): 30 | CONTENT_TYPE = 'application/python-pickle' 31 | DELIVERY_MODE = DeliveryMode.PERSISTENT 32 | 33 | def __init__(self, channel: Channel, queue_name): 34 | self.channel = channel 35 | self.queue_name = queue_name 36 | 37 | self.routes = {} 38 | self.queue = None 39 | 40 | @asyncio.coroutine 41 | def initialize_queue(self, **kwargs): 42 | self.queue = yield from self.channel.declare_queue(name=self.queue_name, **kwargs) 43 | 44 | def create_callback(self, func_name: str, func: Callable): 45 | self.routes[func_name] = func 46 | 47 | @asyncio.coroutine 48 | def consume(self): 49 | yield from self.queue.consume(self.on_callback) 50 | 51 | @asyncio.coroutine 52 | def on_callback(self, message: 'IncomingMessage'): 53 | func_name = message.headers['FuncName'] 54 | func = self.routes.get(func_name) 55 | if func: 56 | with message.process(requeue=True, ignore_processed=True): 57 | data = self.deserialize(message.body) 58 | yield from self._execute(func, data) 59 | 60 | @classmethod 61 | @asyncio.coroutine 62 | def _execute(cls, func, kwargs): 63 | kwargs = kwargs or {} 64 | result = yield from func(**kwargs) 65 | return result 66 | -------------------------------------------------------------------------------- /earlgrey/patterns/worker/client_async.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # The original codes exist in aio_pika.patterns.master 16 | 17 | import asyncio 18 | import logging 19 | from typing import TYPE_CHECKING 20 | 21 | from aio_pika.channel import Channel 22 | from aio_pika.message import Message, DeliveryMode 23 | from aio_pika.patterns.base import Base 24 | 25 | if TYPE_CHECKING: 26 | from aiormq.types import DeliveredMessage 27 | 28 | 29 | class ClientAsync(Base): 30 | CONTENT_TYPE = 'application/python-pickle' 31 | DELIVERY_MODE = DeliveryMode.PERSISTENT 32 | 33 | def __init__(self, channel: Channel, queue_name): 34 | self.channel = channel 35 | self.queue_name = queue_name 36 | self.queue = None 37 | 38 | self.channel.add_on_return_callback(self._on_message_returned) 39 | 40 | @asyncio.coroutine 41 | def initialize_queue(self, **kwargs): 42 | self.queue = yield from self.channel.declare_queue(name=self.queue_name, **kwargs) 43 | 44 | @asyncio.coroutine 45 | def call(self, func_name: str, kwargs=None, priority=128): 46 | message = Message( 47 | body=self.serialize(kwargs or {}), 48 | content_type=self.CONTENT_TYPE, 49 | delivery_mode=self.DELIVERY_MODE, 50 | priority=priority, 51 | headers={ 52 | 'FuncName': func_name 53 | } 54 | ) 55 | 56 | yield from self.channel.default_exchange.publish( 57 | message, self.queue_name, mandatory=True 58 | ) 59 | 60 | @classmethod 61 | def _on_message_returned(cls, sender, message: 'DeliveredMessage', *args, **kwargs): 62 | logging.warning( 63 | f"Message returned. Probably destination queue does not exists: ({sender}) {message}" 64 | ) 65 | -------------------------------------------------------------------------------- /earlgrey/patterns/worker/client_sync.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # The original codes exist in aio_pika.patterns.master 16 | 17 | from aio_pika.message import DeliveryMode, Message 18 | from aio_pika.patterns.base import Base 19 | from pika.adapters.blocking_connection import BlockingChannel 20 | from pika.spec import BasicProperties 21 | 22 | 23 | class ClientSync(Base): 24 | CONTENT_TYPE = 'application/python-pickle' 25 | DELIVERY_MODE = DeliveryMode.PERSISTENT 26 | 27 | def __init__(self, channel: BlockingChannel, queue_name): 28 | self.channel = channel 29 | self.queue_name = queue_name 30 | 31 | def initialize_queue(self, **kwargs): 32 | self.channel.queue_declare(queue=self.queue_name, **kwargs) 33 | 34 | def call(self, func_name: str, kwargs=None, priority=128): 35 | message = Message( 36 | body=self.serialize(kwargs or {}), 37 | content_type=self.CONTENT_TYPE, 38 | delivery_mode=self.DELIVERY_MODE, 39 | priority=priority, 40 | headers={ 41 | 'FuncName': func_name 42 | } 43 | ) 44 | 45 | properties = BasicProperties( 46 | message.properties.content_type, 47 | message.content_encoding, 48 | message.headers, 49 | message.delivery_mode, 50 | message.priority, 51 | message.correlation_id, 52 | message.reply_to, 53 | message.expiration, 54 | message.message_id, 55 | None, 56 | message.type, 57 | message.user_id, 58 | message.app_id, 59 | ) 60 | # noinspection PyProtectedMember 61 | self.channel.basic_publish( 62 | '', 63 | self.queue_name, 64 | message.body, 65 | properties 66 | ) 67 | -------------------------------------------------------------------------------- /earlgrey/message_queue_connection.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import os 15 | from typing import Optional, TYPE_CHECKING 16 | 17 | import aio_pika 18 | 19 | from .message_queue_info import MessageQueueInfoAsync 20 | 21 | if TYPE_CHECKING: 22 | from aio_pika.robust_connection import RobustChannel, RobustConnection 23 | 24 | 25 | class MessageQueueConnection: 26 | def __init__(self, amqp_target, route_key, username=None, password=None): 27 | self._amqp_target = amqp_target 28 | self._route_key = route_key 29 | 30 | self._username = username or os.getenv("AMQP_USERNAME", "guest") 31 | self._password = password or os.getenv("AMQP_PASSWORD", "guest") 32 | 33 | self._connection: 'RobustConnection' = None 34 | self._channel: 'RobustChannel' = None 35 | 36 | self._async_info: MessageQueueInfoAsync = None 37 | 38 | async def connect(self, connection_attempts=None, retry_delay=None): 39 | kwargs = {} 40 | if connection_attempts is not None: 41 | kwargs['connection_attempts'] = connection_attempts 42 | if retry_delay is not None: 43 | kwargs['retry_delay'] = retry_delay 44 | 45 | self._connection: 'RobustConnection' = await aio_pika.connect_robust( 46 | host=self._amqp_target, 47 | login=self._username, 48 | password=self._password, 49 | **kwargs) 50 | 51 | self._connection.add_close_callback(self._callback_connection_close) 52 | self._connection.add_reconnect_callback(self._callback_connection_reconnect_callback) 53 | 54 | self._channel: 'RobustChannel' = await self._connection.channel() 55 | 56 | self._async_info = MessageQueueInfoAsync(self._channel, self._route_key) 57 | 58 | def async_info(self): 59 | return self._async_info 60 | 61 | def _callback_connection_close(self, sender, exc: Optional[BaseException], *args, **kwargs): 62 | pass 63 | 64 | def _callback_connection_reconnect_callback(self, sender, connection: 'RobustConnection', *args, **kwargs): 65 | pass 66 | -------------------------------------------------------------------------------- /earlgrey/message_queue_service.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import asyncio 16 | from typing import TypeVar, Generic 17 | 18 | from . import MessageQueueConnection, MessageQueueType, MESSAGE_QUEUE_TYPE_KEY, TASK_ATTR_DICT 19 | from .patterns import worker, rpc 20 | 21 | T = TypeVar('T') 22 | 23 | 24 | class MessageQueueService(MessageQueueConnection, Generic[T]): 25 | TaskType: type = object 26 | 27 | loop = asyncio.get_event_loop() or asyncio.new_event_loop() 28 | asyncio.set_event_loop(loop) 29 | 30 | def __init__(self, amqp_target, route_key, username=None, password=None, **task_kwargs): 31 | super().__init__(amqp_target, route_key, username, password) 32 | 33 | if self.TaskType is object and type(self) is not MessageQueueService: 34 | raise RuntimeError("MessageQueueTasks is not specified.") 35 | 36 | self._worker_server: worker.Server = None 37 | self._rpc_server: rpc.Server = None 38 | 39 | self._task = self.__class__.TaskType(**task_kwargs) 40 | 41 | async def connect(self, connection_attempts=None, retry_delay=None, **kwargs): 42 | await super().connect(connection_attempts, retry_delay) 43 | 44 | self._worker_server = worker.Server(self._channel, self._route_key) 45 | self._rpc_server = rpc.Server(self._channel, self._route_key) 46 | 47 | queue = await self._channel.declare_queue(self._route_key, auto_delete=True) 48 | await queue.consume(self._consume, **kwargs) 49 | 50 | await self._serve_tasks() 51 | 52 | async def _serve_tasks(self): 53 | for attribute_name in dir(self._task): 54 | try: 55 | attribute = getattr(self._task, attribute_name) 56 | task_attr: dict = getattr(attribute, TASK_ATTR_DICT) 57 | except AttributeError: 58 | pass 59 | else: 60 | func_name = f"{type(self._task).__name__}.{attribute_name}" 61 | 62 | message_queue_type = task_attr[MESSAGE_QUEUE_TYPE_KEY] 63 | if message_queue_type == MessageQueueType.Worker: 64 | self._worker_server.create_callback(func_name, attribute) 65 | elif message_queue_type == MessageQueueType.RPC: 66 | self._rpc_server.create_callback(func_name, attribute) 67 | else: 68 | raise RuntimeError(f"MessageQueueType invalid. {func_name}, {message_queue_type}") 69 | 70 | async def _consume(self, message): 71 | await self._worker_server.on_callback(message) 72 | await self._rpc_server.on_callback(message) 73 | 74 | def serve(self, **kwargs): 75 | self.loop.create_task(self.connect(**kwargs)) 76 | 77 | @classmethod 78 | def serve_all(cls): 79 | cls.loop.run_forever() 80 | -------------------------------------------------------------------------------- /earlgrey/patterns/rpc/server.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # The original codes exist in aio_pika.patterns.rpc 16 | 17 | import asyncio 18 | import time 19 | from typing import Callable 20 | 21 | from aio_pika.exchange import ExchangeType 22 | from aio_pika.channel import Channel 23 | from aio_pika.message import Message, IncomingMessage 24 | from aio_pika.patterns.base import Base 25 | 26 | 27 | class Server(Base): 28 | DLX_NAME = 'rpc.dlx' 29 | 30 | def __init__(self, channel: Channel, queue_name): 31 | self.channel = channel 32 | 33 | self.func_names = {} 34 | self.routes = {} 35 | 36 | self.queue_name = queue_name 37 | self.queue = None 38 | 39 | self.dlx_exchange = None 40 | 41 | @asyncio.coroutine 42 | def initialize_exchange(self): 43 | self.dlx_exchange = yield from self.channel.declare_exchange( 44 | self.DLX_NAME, 45 | type=ExchangeType.HEADERS, 46 | auto_delete=True, 47 | ) 48 | 49 | @asyncio.coroutine 50 | def initialize_queue(self, **kwargs): 51 | arguments = kwargs.pop('arguments', {}).update({ 52 | 'x-dead-letter-exchange': self.DLX_NAME, 53 | }) 54 | 55 | kwargs['arguments'] = arguments 56 | 57 | self.queue = yield from self.channel.declare_queue(name=self.queue_name, **kwargs) 58 | 59 | def create_callback(self, func_name, func): 60 | if func_name in self.routes: 61 | raise RuntimeError( 62 | 'Method name already used for %r' % self.routes[func_name] 63 | ) 64 | 65 | self.func_names[func] = func_name 66 | self.routes[func_name] = func 67 | 68 | @asyncio.coroutine 69 | def consume(self): 70 | yield from self.queue.consume(self.on_callback) 71 | 72 | @asyncio.coroutine 73 | def on_callback(self, message: IncomingMessage): 74 | func_name = message.headers['FuncName'] 75 | if func_name not in self.routes: 76 | return 77 | 78 | payload = self.deserialize(message.body) 79 | func = self.routes[func_name] 80 | 81 | try: 82 | result = yield from self._execute(func, payload) 83 | result = self.serialize(result) 84 | message_type = 'result' 85 | except Exception as e: 86 | result = self.serialize(e) 87 | message_type = 'error' 88 | 89 | result_message = Message( 90 | result, 91 | delivery_mode=message.delivery_mode, 92 | correlation_id=message.correlation_id, 93 | timestamp=time.time(), 94 | type=message_type, 95 | ) 96 | 97 | yield from self.channel.default_exchange.publish( 98 | result_message, 99 | message.reply_to, 100 | mandatory=False 101 | ) 102 | 103 | message.ack() 104 | 105 | @asyncio.coroutine 106 | def _execute(self, func, payload): 107 | return (yield from func(**payload)) 108 | 109 | 110 | -------------------------------------------------------------------------------- /test/test_basic.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import unittest 3 | import random 4 | import string 5 | 6 | from earlgrey import MessageQueueStub, MessageQueueService, MessageQueueType, message_queue_task 7 | 8 | 9 | class TestBasic(unittest.TestCase): 10 | def test_order(self): 11 | order = list(reversed(range(8))) 12 | 13 | def _assert_order(num): 14 | curr_order = order.pop() 15 | print(f"curr_order : {curr_order}") 16 | self.assertEqual(num, curr_order) 17 | 18 | class Task: 19 | @message_queue_task(type_=MessageQueueType.RPC) 20 | async def long_time_rpc(self, x, y): 21 | _assert_order(1) 22 | await asyncio.sleep(0.5) 23 | _assert_order(2) 24 | return x + y 25 | 26 | @message_queue_task(type_=MessageQueueType.Worker) 27 | async def long_time_work(self): 28 | await asyncio.sleep(0.5) 29 | _assert_order(6) 30 | 31 | class Stub(MessageQueueStub[Task]): 32 | TaskType = Task 33 | 34 | class Service(MessageQueueService[Task]): 35 | TaskType = Task 36 | 37 | async def _run(): 38 | route_key = 'any same string between processes' 39 | 40 | client = Stub('localhost', route_key) 41 | server = Service('localhost', route_key) 42 | 43 | await client.connect() 44 | await server.connect() 45 | 46 | _assert_order(0) 47 | result = await client.async_task().long_time_rpc(10, 20) 48 | _assert_order(3) 49 | self.assertEqual(result, 30) 50 | 51 | _assert_order(4) 52 | await client.async_task().long_time_work() 53 | _assert_order(5) 54 | 55 | await asyncio.sleep(1.5) 56 | _assert_order(7) 57 | 58 | self.assertEqual(0, len(order)) 59 | 60 | loop = asyncio.get_event_loop() 61 | loop.run_until_complete(_run()) 62 | 63 | def test_mq_info(self): 64 | class Task: 65 | @message_queue_task(type_=MessageQueueType.Worker) 66 | async def work_async(self): 67 | pass 68 | 69 | @message_queue_task(type_=MessageQueueType.Worker) 70 | def work_sync(self): 71 | pass 72 | 73 | class Stub(MessageQueueStub[Task]): 74 | TaskType = Task 75 | 76 | class Service(MessageQueueService[Task]): 77 | TaskType = Task 78 | 79 | async def _run(): 80 | route_key_length = random.randint(5, 10) 81 | route_key = ''.join(random.choice(string.ascii_letters) for _ in range(route_key_length)) 82 | 83 | client = Stub('localhost', route_key) 84 | await client.connect() 85 | 86 | await client.async_task().work_async() 87 | client.sync_task().work_sync() 88 | await asyncio.sleep(1) 89 | 90 | info = await client.async_info().queue_info() 91 | self.assertEqual(info.declaration_result.message_count, 2) 92 | 93 | info = client.sync_info().queue_info() 94 | self.assertEqual(info.method.message_count, 2) 95 | 96 | server = Service('localhost', route_key) 97 | await server.connect() 98 | await asyncio.sleep(1) 99 | 100 | info = await client.async_info().queue_info() 101 | self.assertEqual(info.declaration_result.message_count, 0) 102 | 103 | info = client.sync_info().queue_info() 104 | self.assertEqual(info.method.message_count, 0) 105 | 106 | loop = asyncio.get_event_loop() 107 | loop.run_until_complete(_run()) 108 | 109 | 110 | -------------------------------------------------------------------------------- /earlgrey/patterns/rpc/client_sync.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # The original codes exist in aio_pika.patterns.rpc 16 | 17 | import time 18 | from concurrent import futures 19 | 20 | from aio_pika import ExchangeType, Message, DeliveryMode 21 | from aio_pika.patterns.base import Base 22 | from pika import BasicProperties 23 | 24 | 25 | class ClientSync(Base): 26 | DLX_NAME = 'rpc.dlx' 27 | 28 | def __init__(self, channel, queue_name): 29 | self.channel = channel 30 | self.queue_name = queue_name 31 | self.result_queue_name = None 32 | 33 | self.futures = {} 34 | 35 | self.func_names = {} 36 | self.routes = {} 37 | 38 | self.dlx_exchange = None 39 | 40 | def initialize_exchange(self): 41 | self.dlx_exchange = self.channel.exchange_declare( 42 | exchange=self.DLX_NAME, 43 | exchange_type=ExchangeType.HEADERS.value, 44 | auto_delete=True, 45 | ) 46 | 47 | def initialize_queue(self, **kwargs): 48 | arguments = kwargs.pop('arguments', {}).update({ 49 | 'x-dead-letter-exchange': self.DLX_NAME, 50 | }) 51 | 52 | kwargs['arguments'] = arguments 53 | 54 | self.channel.queue_declare(queue=self.queue_name, **kwargs) 55 | self.result_queue_name = self.channel.queue_declare(queue='', exclusive=True, auto_delete=True).method.queue 56 | 57 | self.channel.queue_bind( 58 | queue=self.queue_name, 59 | exchange=self.DLX_NAME, 60 | arguments={ 61 | "From": self.result_queue_name, 62 | 'x-match': 'any', 63 | } 64 | ) 65 | self.channel.basic_consume( 66 | on_message_callback=self._on_result_message, 67 | queue=self.result_queue_name, 68 | auto_ack=True 69 | ) 70 | 71 | def _on_result_message(self, channel, method, properties, body): 72 | correlation_id = int(properties.correlation_id) 73 | try: 74 | future = self.futures[correlation_id] 75 | except KeyError: 76 | pass 77 | else: 78 | payload = self.deserialize(body) 79 | future.set_result(payload) 80 | 81 | def call(self, func_name, kwargs: dict=None, *, expiration: int=None, 82 | priority: int=128, delivery_mode: DeliveryMode=DeliveryMode.NOT_PERSISTENT): 83 | future = self._create_future() 84 | correlation_id = id(future) 85 | 86 | message = Message( 87 | body=self.serialize(kwargs or {}), 88 | type='call', 89 | timestamp=time.time(), 90 | expiration=expiration, 91 | priority=priority, 92 | correlation_id=correlation_id, 93 | delivery_mode=delivery_mode, 94 | reply_to=self.result_queue_name, 95 | headers={ 96 | 'From': self.result_queue_name, 97 | 'FuncName': func_name 98 | }, 99 | ) 100 | 101 | properties = BasicProperties( 102 | message.properties.content_type, 103 | message.content_encoding, 104 | message.headers, 105 | message.delivery_mode, 106 | message.priority, 107 | message.correlation_id, 108 | message.reply_to, 109 | message.expiration, 110 | message.message_id, 111 | None, 112 | message.type, 113 | message.user_id, 114 | message.app_id, 115 | ) 116 | 117 | # noinspection PyProtectedMember 118 | self.channel.basic_publish( 119 | '', self.queue_name, message.body, properties, mandatory=True 120 | ) 121 | 122 | while not future.done(): 123 | self.channel.connection.process_data_events() 124 | 125 | if future.exception(): 126 | return future.exception() 127 | else: 128 | return future.result() 129 | 130 | def _create_future(self) -> futures.Future: 131 | future = futures.Future() 132 | future_id = id(future) 133 | self.futures[future_id] = future 134 | future.add_done_callback(lambda f: self.futures.pop(future_id, None)) 135 | return future 136 | -------------------------------------------------------------------------------- /earlgrey/patterns/rpc/client_async.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # The original codes exist in aio_pika.patterns.rpc 16 | 17 | 18 | import asyncio 19 | import logging 20 | import time 21 | from typing import TYPE_CHECKING 22 | 23 | from aio_pika.channel import Channel 24 | from aio_pika.exceptions import DeliveryError 25 | from aio_pika.exchange import ExchangeType 26 | from aio_pika.message import Message, DeliveryMode 27 | from aio_pika.patterns.base import Base 28 | 29 | if TYPE_CHECKING: 30 | from aiormq.types import DeliveredMessage 31 | from aio_pika.message import IncomingMessage 32 | 33 | 34 | class ClientAsync(Base): 35 | DLX_NAME = 'rpc.dlx' 36 | 37 | def __init__(self, channel: Channel, queue_name): 38 | self.channel = channel 39 | self.queue_name = queue_name 40 | self.queue = None 41 | self.result_queue = None 42 | 43 | self.async_futures = {} 44 | self.concurrent_futures = {} 45 | 46 | self.func_names = {} 47 | self.routes = {} 48 | 49 | self.dlx_exchange = None 50 | 51 | @asyncio.coroutine 52 | def initialize_exchange(self): 53 | self.dlx_exchange = yield from self.channel.declare_exchange( 54 | self.DLX_NAME, 55 | type=ExchangeType.HEADERS, 56 | auto_delete=True, 57 | ) 58 | 59 | @asyncio.coroutine 60 | def initialize_queue(self, **kwargs): 61 | arguments = kwargs.pop('arguments', {}).update({ 62 | 'x-dead-letter-exchange': self.DLX_NAME, 63 | }) 64 | 65 | kwargs['arguments'] = arguments 66 | 67 | self.queue = yield from self.channel.declare_queue(name=self.queue_name, **kwargs) 68 | 69 | self.result_queue = yield from self.channel.declare_queue(None, exclusive=True, auto_delete=True) 70 | yield from self.result_queue.bind( 71 | self.dlx_exchange, "", 72 | arguments={ 73 | "From": self.result_queue.name, 74 | 'x-match': 'any', 75 | } 76 | ) 77 | 78 | yield from self.result_queue.consume( 79 | self._on_result_message, no_ack=True 80 | ) 81 | 82 | self.channel.add_on_return_callback(self._on_message_returned) 83 | 84 | def _on_message_returned(self, sender, message: 'DeliveredMessage', *args, **kwargs): 85 | correlation_id = int(message.correlation_id) if message.correlation_id else None 86 | 87 | future = self.async_futures.pop(correlation_id, None) or self.concurrent_futures.pop(correlation_id, None) 88 | if future and future.done(): 89 | logging.warning(f"Unknown message was returned: ({sender}) {message}") 90 | else: 91 | future.set_exception(DeliveryError(message, None)) 92 | 93 | @asyncio.coroutine 94 | def _on_result_message(self, message: 'IncomingMessage'): 95 | correlation_id = int(message.correlation_id) if message.correlation_id else None 96 | try: 97 | future = self.async_futures[correlation_id] # type: asyncio.Future 98 | except KeyError: 99 | pass 100 | else: 101 | payload = self.deserialize(message.body) 102 | 103 | if message.type == 'result': 104 | future.set_result(payload) 105 | elif message.type == 'error': 106 | future.set_exception(payload) 107 | elif message.type == 'call': 108 | future.set_exception(asyncio.TimeoutError("Message timed-out", message)) 109 | else: 110 | future.set_exception(RuntimeError("Unknown message type %r" % message.type)) 111 | 112 | @asyncio.coroutine 113 | def call(self, func_name, kwargs: dict=None, *, expiration: int=None, 114 | priority: int=128, delivery_mode: DeliveryMode=DeliveryMode.NOT_PERSISTENT): 115 | future = self._create_future() 116 | message = Message( 117 | body=self.serialize(kwargs or {}), 118 | type='call', 119 | timestamp=time.time(), 120 | expiration=expiration, 121 | priority=priority, 122 | correlation_id=id(future), 123 | delivery_mode=delivery_mode, 124 | reply_to=self.result_queue.name, 125 | headers={ 126 | 'From': self.result_queue.name, 127 | 'FuncName': func_name 128 | }, 129 | ) 130 | 131 | yield from self.channel.default_exchange.publish( 132 | message, routing_key=self.queue_name, mandatory=True 133 | ) 134 | 135 | return (yield from future) 136 | 137 | def _create_future(self) -> asyncio.Future: 138 | future = self.channel.loop.create_future() 139 | future_id = id(future) 140 | self.async_futures[future_id] = future 141 | future.add_done_callback(lambda f: self.async_futures.pop(future_id, None)) 142 | return future 143 | -------------------------------------------------------------------------------- /test/test_multiprocess.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import unittest 3 | import multiprocessing 4 | import random 5 | import string 6 | import subprocess 7 | import traceback 8 | import os 9 | 10 | from earlgrey import MessageQueueStub, MessageQueueService, MessageQueueType, message_queue_task 11 | 12 | 13 | def available_credential(username, password): 14 | try: 15 | subprocess.check_output(["rabbitmqctl", "authenticate_user", username, password]) 16 | result = subprocess.check_output(["rabbitmqctl", "list_user_permissions", username]) 17 | return b"/ .* .* .*" in result 18 | except Exception as e: 19 | traceback.print_exc() 20 | return False 21 | 22 | 23 | class TestMultiprocess(unittest.TestCase): 24 | def test_basic(self): 25 | server = multiprocessing.Process(target=self._run_server) 26 | server.daemon = True 27 | server.start() 28 | 29 | client = multiprocessing.Process(target=self._run_client) 30 | client.daemon = True 31 | client.start() 32 | 33 | server.join() 34 | client.join() 35 | 36 | self.assertEqual(server.exitcode, 0) 37 | self.assertEqual(client.exitcode, 0) 38 | 39 | def test_credential_failure(self): 40 | random_length = random.randrange(5, 10) 41 | random_string = "".join(random.choice(string.ascii_letters) for _ in range(random_length)) 42 | 43 | print(f"username : {random_string}") 44 | print(f"password : {random_string}") 45 | 46 | server = multiprocessing.Process(target=self._run_server, args=(random_string, random_string)) 47 | server.daemon = True 48 | server.start() 49 | 50 | client = multiprocessing.Process(target=self._run_client, args=(random_string, random_string)) 51 | client.daemon = True 52 | client.start() 53 | 54 | server.join() 55 | client.join() 56 | 57 | self.assertEqual(server.exitcode, 1) 58 | self.assertEqual(client.exitcode, 1) 59 | 60 | test_username = "test" 61 | test_password = "test" 62 | 63 | @unittest.skipUnless(available_credential(test_username, test_password), 64 | f"{test_username} / {test_password} invalid") 65 | def test_credential_success(self): 66 | print(f"username : {self.test_username}") 67 | print(f"password : {self.test_password}") 68 | 69 | server = multiprocessing.Process(target=self._run_server, args=(self.test_username, self.test_password)) 70 | server.daemon = True 71 | server.start() 72 | 73 | client = multiprocessing.Process(target=self._run_client, args=(self.test_username, self.test_password)) 74 | client.daemon = True 75 | client.start() 76 | 77 | server.join() 78 | client.join() 79 | 80 | self.assertEqual(server.exitcode, 0) 81 | self.assertEqual(client.exitcode, 0) 82 | 83 | @unittest.skipUnless(available_credential(test_username, test_password), 84 | f"{test_username} / {test_password} invalid") 85 | def test_credential_success_environment_variable(self): 86 | print(f"username : {self.test_username}") 87 | print(f"password : {self.test_password}") 88 | 89 | os.environ["AMQP_USERNAME"] = self.test_username 90 | os.environ["AMQP_PASSWORD"] = self.test_password 91 | 92 | server = multiprocessing.Process(target=self._run_server) 93 | server.daemon = True 94 | server.start() 95 | 96 | client = multiprocessing.Process(target=self._run_client) 97 | client.daemon = True 98 | client.start() 99 | 100 | server.join() 101 | client.join() 102 | 103 | self.assertEqual(server.exitcode, 0) 104 | self.assertEqual(client.exitcode, 0) 105 | 106 | def _run_server(self, username=None, password=None): 107 | async def _run(): 108 | try: 109 | message_queue_service = Service('localhost', route_key, username, password) 110 | await message_queue_service.connect() 111 | except: 112 | loop.stop() 113 | raise 114 | 115 | loop = asyncio.new_event_loop() 116 | asyncio.set_event_loop(loop) 117 | 118 | task = loop.create_task(_run()) 119 | loop.run_forever() 120 | 121 | if task.exception(): 122 | raise task.exception() 123 | 124 | print("run_server finished.") 125 | 126 | def _run_client(self, username=None, password=None): 127 | async def _run(): 128 | try: 129 | message_queue_stub = Stub('localhost', route_key, username, password) 130 | await message_queue_stub.connect() 131 | 132 | result = await message_queue_stub.async_task().sum(10, 20) 133 | self.assertEqual(result, 30) 134 | 135 | result = message_queue_stub.sync_task().multiply(10, 20) 136 | self.assertEqual(result, 200) 137 | 138 | message_queue_stub.sync_task().ping(123) 139 | 140 | await message_queue_stub.async_task().stop() 141 | except: 142 | loop.stop() 143 | raise 144 | 145 | loop = asyncio.new_event_loop() 146 | asyncio.set_event_loop(loop) 147 | 148 | loop.run_until_complete(_run()) 149 | print("run_client finished.") 150 | 151 | 152 | class Task: 153 | @message_queue_task 154 | async def sum(self, x, y): 155 | return x + y 156 | 157 | @message_queue_task 158 | def multiply(self, x, y): 159 | return x * y 160 | 161 | @message_queue_task(type_=MessageQueueType.Worker) 162 | def ping(self, value): 163 | print(f'value : {value}') 164 | assert value == 123 165 | 166 | @message_queue_task(type_=MessageQueueType.Worker) 167 | async def stop(self): 168 | print('stop') 169 | asyncio.get_event_loop().stop() 170 | 171 | 172 | class Stub(MessageQueueStub[Task]): 173 | TaskType = Task 174 | 175 | 176 | class Service(MessageQueueService[Task]): 177 | TaskType = Task 178 | 179 | 180 | route_key = 'something same you want' -------------------------------------------------------------------------------- /earlgrey/message_queue_stub.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 theloop Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import functools 16 | import inspect 17 | import logging 18 | import threading 19 | from typing import TypeVar, Generic 20 | 21 | import pika 22 | 23 | from . import (MessageQueueConnection, MessageQueueInfoSync, MessageQueueType, MessageQueueException, worker, rpc, 24 | MESSAGE_QUEUE_TYPE_KEY, MESSAGE_QUEUE_PRIORITY_KEY, TASK_ATTR_DICT) 25 | 26 | T = TypeVar('T') 27 | 28 | 29 | class MessageQueueStub(MessageQueueConnection, Generic[T]): 30 | TaskType: type = object 31 | 32 | def __init__(self, amqp_target, route_key, username=None, password=None): 33 | super().__init__(amqp_target, route_key, username, password) 34 | 35 | if self.TaskType is object and type(self) is not MessageQueueStub: 36 | raise RuntimeError("MessageQueueTasks is not specified.") 37 | 38 | self._worker_client_async: worker.ClientAsync = None 39 | self._rpc_client_async: rpc.ClientAsync = None 40 | 41 | self._async_task = object.__new__(self.__class__.TaskType) # not calling __init__ 42 | self._thread_local = _Local() 43 | 44 | async def connect(self, connection_attempts=None, retry_delay=None): 45 | await super().connect(connection_attempts, retry_delay) 46 | await self._connect_async() 47 | self._register_tasks_async() 48 | 49 | async def _connect_async(self): 50 | self._worker_client_async = worker.ClientAsync(self._channel, self._route_key) 51 | await self._worker_client_async.initialize_queue(auto_delete=True) 52 | 53 | self._rpc_client_async = rpc.ClientAsync(self._channel, self._route_key) 54 | await self._rpc_client_async.initialize_exchange() 55 | await self._rpc_client_async.initialize_queue(auto_delete=True) 56 | 57 | def _connect_sync(self): 58 | credential_params = pika.PlainCredentials(self._username, self._password) 59 | connection_params = pika.ConnectionParameters( 60 | host=f'{self._amqp_target}', 61 | heartbeat=0, 62 | credentials=credential_params) 63 | 64 | connection = pika.BlockingConnection(connection_params) 65 | channel = connection.channel() 66 | 67 | self._thread_local.worker_client_sync = worker.ClientSync(channel, self._route_key) 68 | self._thread_local.worker_client_sync.initialize_queue(auto_delete=True) 69 | 70 | self._thread_local.rpc_client_sync: rpc.ClientSync = rpc.ClientSync(channel, self._route_key) 71 | self._thread_local.rpc_client_sync.initialize_exchange() 72 | self._thread_local.rpc_client_sync.initialize_queue(auto_delete=True) 73 | 74 | self._thread_local.sync_task = object.__new__(self.__class__.TaskType) # not calling __init__ 75 | self._thread_local.sync_info = MessageQueueInfoSync(channel, self._route_key) 76 | 77 | def _register_tasks_async(self): 78 | for attribute_name in dir(self._async_task): 79 | try: 80 | attribute = getattr(self._async_task, attribute_name) 81 | task_attr: dict = getattr(attribute, TASK_ATTR_DICT) 82 | except AttributeError: 83 | pass 84 | else: 85 | func_name = f"{type(self._async_task).__name__}.{attribute_name}" 86 | 87 | message_queue_type = task_attr[MESSAGE_QUEUE_TYPE_KEY] 88 | message_queue_priority = task_attr[MESSAGE_QUEUE_PRIORITY_KEY] 89 | if message_queue_type == MessageQueueType.Worker: 90 | binding_async_method = self._call_async_worker 91 | elif message_queue_type == MessageQueueType.RPC: 92 | binding_async_method = self._call_async_rpc 93 | else: 94 | raise RuntimeError(f"MessageQueueType invalid. {func_name}, {message_queue_type}") 95 | 96 | stub = functools.partial(binding_async_method, func_name, attribute, message_queue_priority) 97 | setattr(self._async_task, attribute_name, stub) 98 | 99 | def _register_tasks_sync(self): 100 | for attribute_name in dir(self._thread_local.sync_task): 101 | try: 102 | attribute = getattr(self._thread_local.sync_task, attribute_name) 103 | task_attr: dict = getattr(attribute, TASK_ATTR_DICT) 104 | except AttributeError: 105 | pass 106 | else: 107 | func_name = f"{type(self._thread_local.sync_task).__name__}.{attribute_name}" 108 | 109 | message_queue_type = task_attr[MESSAGE_QUEUE_TYPE_KEY] 110 | message_queue_priority = task_attr[MESSAGE_QUEUE_PRIORITY_KEY] 111 | if message_queue_type == MessageQueueType.Worker: 112 | binding_sync_method = self._call_sync_worker 113 | elif message_queue_type == MessageQueueType.RPC: 114 | binding_sync_method = self._call_sync_rpc 115 | else: 116 | raise RuntimeError(f"MessageQueueType invalid. {func_name}, {message_queue_type}") 117 | 118 | stub = functools.partial(binding_sync_method, func_name, attribute, message_queue_priority) 119 | setattr(self._thread_local.sync_task, attribute_name, stub) 120 | 121 | async def _call_async_worker(self, func_name, func, priority, *args, **kwargs): 122 | params = inspect.signature(func).bind(*args, **kwargs) 123 | params.apply_defaults() 124 | await self._worker_client_async.call(func_name, kwargs=params.arguments, priority=priority) 125 | 126 | async def _call_async_rpc(self, func_name, func, priority, *args, **kwargs): 127 | params = inspect.signature(func).bind(*args, **kwargs) 128 | params.apply_defaults() 129 | result = await self._rpc_client_async.call(func_name, kwargs=params.arguments, priority=priority) 130 | if isinstance(result, MessageQueueException): 131 | logging.error(result) 132 | raise result 133 | return result 134 | 135 | def _call_sync_worker(self, func_name, func, priority, *args, **kwargs): 136 | params = inspect.signature(func).bind(*args, **kwargs) 137 | params.apply_defaults() 138 | self._thread_local.worker_client_sync.call(func_name, kwargs=params.arguments, priority=priority) 139 | 140 | def _call_sync_rpc(self, func_name, func, priority, *args, **kwargs): 141 | params = inspect.signature(func).bind(*args, **kwargs) 142 | params.apply_defaults() 143 | result = self._thread_local.rpc_client_sync.call(func_name, kwargs=params.arguments, priority=priority) 144 | if isinstance(result, MessageQueueException): 145 | logging.error(result) 146 | raise result 147 | return result 148 | 149 | def async_task(self) -> T: 150 | return self._async_task 151 | 152 | def sync_task(self) -> T: 153 | if self._thread_local.sync_task is None: 154 | self._connect_sync() 155 | self._register_tasks_sync() 156 | 157 | return self._thread_local.sync_task 158 | 159 | def sync_info(self): 160 | if self._thread_local.sync_info is None: 161 | self._connect_sync() 162 | self._register_tasks_sync() 163 | 164 | return self._thread_local.sync_info 165 | 166 | 167 | class _Local(threading.local): 168 | worker_client_sync: worker.ClientSync = None 169 | rpc_client_sync: rpc.ClientSync = None 170 | sync_task = None 171 | sync_info: MessageQueueInfoSync = None 172 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018-current ICONLOOP 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------