├── .editorconfig ├── .gitignore ├── .travis.yml ├── AUTHORS.md ├── CONTRIBUTING.md ├── HISTORY.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── contrib └── hermes_mock.py ├── docs ├── configuration.md ├── django.md ├── flask.md ├── hermes_mock.md ├── index.md ├── installation.md ├── publishing.md └── subscribing.md ├── mkdocs.yml ├── pyhermes ├── __init__.py ├── apps │ ├── __init__.py │ ├── django │ │ ├── __init__.py │ │ ├── config.py │ │ ├── management │ │ │ ├── __init__.py │ │ │ └── commands │ │ │ │ ├── __init__.py │ │ │ │ └── hermes_test.py │ │ ├── models.py │ │ ├── urls.py │ │ └── views.py │ └── flask │ │ ├── __init__.py │ │ ├── blueprints.py │ │ ├── command.py │ │ └── config.py ├── decorators.py ├── exceptions.py ├── management.py ├── publishing.py ├── registry.py ├── settings.py ├── subscription.py └── utils.py ├── pyproject.toml ├── requirements ├── base.txt ├── dev.txt └── test.txt ├── runtests_django.py ├── tests ├── __init__.py ├── test_apps │ ├── __init__.py │ ├── test_django │ │ ├── __init__.py │ │ └── test_subscriber.py │ └── test_flask │ │ ├── __init__.py │ │ └── test_subscriber.py └── test_internal │ ├── __init__.py │ ├── test_decorators.py │ ├── test_publishing.py │ └── test_utils.py └── tox.ini /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.{py,rst,ini}] 12 | indent_style = space 13 | indent_size = 4 14 | 15 | [*.{html,css,scss,json,yml}] 16 | indent_style = space 17 | indent_size = 2 18 | 19 | [*.md] 20 | trim_trailing_whitespace = false 21 | 22 | [Makefile] 23 | indent_style = tab 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | __pycache__ 3 | 4 | # C extensions 5 | *.so 6 | 7 | # Packages 8 | *.egg 9 | *.egg-info 10 | dist 11 | build 12 | eggs 13 | parts 14 | bin 15 | var 16 | sdist 17 | develop-eggs 18 | .installed.cfg 19 | lib 20 | lib64 21 | 22 | # Installer logs 23 | pip-log.txt 24 | 25 | # Unit test / coverage reports 26 | .coverage 27 | .tox 28 | nosetests.xml 29 | htmlcov 30 | 31 | # Translations 32 | *.mo 33 | 34 | # Mr Developer 35 | .mr.developer.cfg 36 | .project 37 | .pydevproject 38 | 39 | # Pycharm/Intellij 40 | .idea 41 | 42 | # Complexity 43 | output/*.html 44 | output/*/index.html 45 | 46 | # Sphinx 47 | docs/_build 48 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Config file for automatic testing at travis-ci.org 2 | 3 | language: python 4 | 5 | python: 6 | - "3.9" 7 | 8 | sudo: false 9 | 10 | env: 11 | - TOX_ENV=py27 12 | - TOX_ENV=py39 13 | 14 | - TOX_ENV=py39-django315 15 | 16 | - TOX_ENV=py27-django111 17 | - TOX_ENV=py39-django111 18 | 19 | - TOX_ENV=py27-django110 20 | - TOX_ENV=py39-django110 21 | 22 | - TOX_ENV=py27-django19 23 | - TOX_ENV=py39-django19 24 | 25 | - TOX_ENV=py27-django18 26 | - TOX_ENV=py39-django18 27 | 28 | - TOX_ENV=py27-djangodev 29 | - TOX_ENV=py39-djangodev 30 | 31 | - TOX_ENV=py27-django17 32 | - TOX_ENV=py39-django17 33 | 34 | - TOX_ENV=py27-flask10 35 | - TOX_ENV=py39-flask10 36 | 37 | - TOX_ENV=py27-flask012 38 | - TOX_ENV=py39-flask012 39 | 40 | - TOX_ENV=py39-flask112 41 | 42 | - TOX_ENV=py27-flaskdev 43 | - TOX_ENV=py39-flaskdev 44 | 45 | matrix: 46 | fast_finish: true 47 | # python 3.5 and 3.6 are installed on demand, so to not create additional 48 | # dimension in build matrix, python3.9 build are specified directly here 49 | include: 50 | - python: "3.9" 51 | env: TOX_ENV=py39-django315 52 | - python: "3.9" 53 | env: TOX_ENV=py39-djangodev 54 | - python: "3.9" 55 | env: TOX_ENV=py39-django111 56 | - python: "3.9" 57 | env: TOX_ENV=py39-django110 58 | - python: "3.9" 59 | env: TOX_ENV=py39-django19 60 | - python: "3.9" 61 | env: TOX_ENV=py39-django18 62 | 63 | - python: "3.9" 64 | env: TOX_ENV=py39-flask10 65 | - python: "3.9" 66 | env: TOX_ENV=py39-flask012 67 | - python: "3.9" 68 | env: TOX_ENV=py39-flask112 69 | - python: "3.9" 70 | env: TOX_ENV=py39-flaskdev 71 | allow_failures: 72 | # Django dev is Django 2.0 now, removing python2 compatibility - allow 73 | # failuers temporary 74 | - env: TOX_ENV=py35-djangodev 75 | - env: TOX_ENV=py36-djangodev 76 | - env: TOX_ENV=py39-djangodev 77 | 78 | install: 79 | - pip install tox flake8 80 | 81 | script: 82 | - tox -e $TOX_ENV 83 | - make lint 84 | 85 | after_success: 86 | - pip install codecov 87 | - codecov -e TOX_ENV 88 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | # Credits 2 | 3 | ## Development Lead 4 | 5 | * pyLabs 6 | 7 | ## Contributors 8 | 9 | None yet. Why not be the first? 10 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions are welcome, and they are greatly appreciated! Every 4 | little bit helps, and credit will always be given. 5 | 6 | You can contribute in many ways: 7 | 8 | ## Types of Contributions 9 | 10 | ### Report Bugs 11 | 12 | Report bugs at https://github.com/allegro/pyhermes/issues. 13 | 14 | If you are reporting a bug, please include: 15 | 16 | * Your operating system name and version. 17 | * Any details about your local setup that might be helpful in troubleshooting. 18 | * Detailed steps to reproduce the bug. 19 | 20 | ### Fix Bugs 21 | 22 | Look through the GitHub issues for bugs. Anything tagged with "bug" 23 | is open to whoever wants to implement it. 24 | 25 | ### Implement Features 26 | 27 | Look through the GitHub issues for features. Anything tagged with "feature" 28 | is open to whoever wants to implement it. 29 | 30 | ### Write Documentation 31 | 32 | pyhermes could always use more documentation, whether as part of the 33 | official pyhermes docs, in docstrings, or even on the web in blog posts, 34 | articles, and such. 35 | 36 | ### Submit Feedback 37 | 38 | The best way to send feedback is to file an issue at https://github.com/allegro/pyhermes/issues. 39 | 40 | If you are proposing a feature: 41 | 42 | * Explain in detail how it would work. 43 | * Keep the scope as narrow as possible, to make it easier to implement. 44 | * Remember that this is a volunteer-driven project, and that contributions 45 | are welcome :) 46 | 47 | ## Get Started! 48 | 49 | Ready to contribute? Here's how to set up `pyhermes` for local development. 50 | 51 | 1. Fork the `pyhermes` repo on GitHub. 52 | 2. Clone your fork locally: 53 | 54 | ``` 55 | $ git clone git@github.com:your_name_here/pyhermes.git 56 | ``` 57 | 58 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development: 59 | 60 | ``` 61 | $ mkvirtualenv pyhermes 62 | $ cd pyhermes/ 63 | $ python setup.py develop 64 | ``` 65 | 66 | 4. Create a branch for local development: 67 | 68 | ``` 69 | $ git checkout -b name-of-your-bugfix-or-feature 70 | ``` 71 | 72 | Now you can make your changes locally. 73 | 74 | 5. When you're done making changes, check that your changes pass flake8 and the 75 | tests, including testing other Python versions with tox: 76 | 77 | ``` 78 | $ flake8 pyhermes tests 79 | $ python setup.py test 80 | $ tox 81 | ``` 82 | 83 | To get flake8 and tox, just pip install them into your virtualenv. 84 | 85 | 6. Commit your changes and push your branch to GitHub:: 86 | 87 | ``` 88 | $ git add . 89 | $ git commit -m "Your detailed description of your changes." 90 | $ git push origin name-of-your-bugfix-or-feature 91 | ``` 92 | 93 | 7. Submit a pull request through the GitHub website. 94 | 95 | ## Pull Request Guidelines 96 | 97 | Before you submit a pull request, check that it meets these guidelines: 98 | 99 | 1. The pull request should include tests. 100 | 2. If the pull request adds functionality, the docs should be updated. Put 101 | your new functionality into a function with a docstring, and add the 102 | feature to the list in README.rst. 103 | 3. The pull request should work for Python 2.6, 2.7, and 3.3, and for PyPy. Check 104 | https://travis-ci.org/allegro/pyhermes/pull_requests 105 | and make sure that the tests pass for all supported Python versions. 106 | 107 | ## Tips 108 | 109 | To run a subset of tests: 110 | 111 | ``` 112 | $ python -m unittest tests.test_pyhermes 113 | ``` 114 | -------------------------------------------------------------------------------- /HISTORY.md: -------------------------------------------------------------------------------- 1 | 2 | # History 3 | 4 | ## 0.6.0 (2023-13-02) 5 | 6 | * Compatibility with Django 4.1 7 | 8 | ## 0.5.0 (2022-09-02) 9 | 10 | * Rename django app label from `pyhermes.django` to `pyhermes_django` 11 | * Compatibility with Django 3.2 12 | 13 | 14 | ## 0.3.0 (2016-12-29) 15 | 16 | * Retry publishing to hermes in case of failure (default: 3x) 17 | * Support for Python3.6, Django 1.10 and Django development version in tests 18 | 19 | 20 | ## 0.2.1 (2016-12-12) 21 | 22 | * Configure custom label for django app #11 23 | 24 | 25 | ## 0.2.0 (2016-11-03) 26 | 27 | * Fix ambiguity with pyhermes.decorators.subscriber (rename subscriber module to subscription) 28 | 29 | 30 | ## 0.1.3 (2016-06-21) 31 | 32 | * Allow for custom wrapper around subcriber function 33 | * Additional logging for event id and retry count 34 | * Added support for Django <= 1.7 35 | * Raw data is dumped only to debug logs. 36 | 37 | 38 | ## 0.1.2 (2016-04-20) 39 | 40 | * New management command for testing Hermes connection 41 | 42 | 43 | ## 0.1.0 (2016-04-13) 44 | 45 | * First release on PyPI. 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Allegro Group 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.md 2 | include CONTRIBUTING.md 3 | include HISTORY.md 4 | include LICENSE 5 | include README.md 6 | recursive-include pyhermes *.html *.png *.gif *js *.css *jpg *jpeg *svg *py 7 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean-pyc clean-build docs 2 | 3 | help: 4 | @echo "clean-build - remove build artifacts" 5 | @echo "clean-pyc - remove Python file artifacts" 6 | @echo "lint - check style with flake8" 7 | @echo "test - run tests quickly with the default Python" 8 | @echo "test-all - run tests on every Python version with tox" 9 | @echo "coverage - check code coverage quickly with the default Python" 10 | @echo "docs - generate Sphinx HTML documentation, including API docs" 11 | @echo "release - package and upload a release" 12 | @echo "sdist - package" 13 | 14 | clean: clean-build clean-pyc 15 | 16 | clean-build: 17 | rm -fr build/ 18 | rm -fr dist/ 19 | rm -fr *.egg-info 20 | 21 | clean-pyc: 22 | find . -name '*.pyc' -exec rm -f {} + 23 | find . -name '*.pyo' -exec rm -f {} + 24 | find . -name '*~' -exec rm -f {} + 25 | 26 | lint: 27 | flake8 pyhermes tests 28 | 29 | test: 30 | python runtests_django.py tests 31 | 32 | test-all: 33 | tox 34 | 35 | coverage: 36 | coverage run --source pyhermes runtests_django.py tests 37 | coverage report -m 38 | coverage html 39 | open htmlcov/index.html 40 | 41 | docs: 42 | mkdocs build 43 | 44 | release: clean 45 | python setup.py sdist upload 46 | python setup.py bdist_wheel upload 47 | 48 | sdist: clean 49 | python setup.py sdist 50 | ls -l dist 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## pyhermes 2 | 3 | [![Version Badge](https://badge.fury.io/py/pyhermes.png)](https://badge.fury.io/py/pyhermes.png) 4 | [![Build Status](https://travis-ci.org/allegro/pyhermes.png?branch=master)](https://travis-ci.org/allegro/pyhermes) 5 | 6 | The Python interface to the [Hermes](http://hermes.allegro.tech) message broker. 7 | 8 | ## Documentation 9 | 10 | The full documentation is at https://pyhermes.readthedocs.org. 11 | 12 | ## Installation 13 | 14 | To install pyhermes, simply: 15 | 16 | ```python 17 | pip install pyhermes 18 | ``` 19 | 20 | Then use it in a project: 21 | 22 | ```python 23 | import pyhermes 24 | ``` 25 | 26 | ## Features 27 | 28 | * TODO 29 | 30 | ## Quickstart 31 | 32 | ### Subscriber 33 | 34 | To create handler for particular subscription topic decorate your function using `subscribe` decorator: 35 | 36 | ```python 37 | import pyhermes 38 | 39 | @pyhermes.subscriber(topic='pl.allegro.pyhermes.sample-topic') 40 | def handler(data): 41 | # process data 42 | ``` 43 | 44 | This function will be called every time there is new message published to the selected topic. 45 | 46 | ### Publisher 47 | Use `publish` function to publish data to some topic in hermes: 48 | 49 | ```python 50 | import pyhermes 51 | 52 | @pyhermes.publisher(topic='pl.allegro.pyhermes.sample-topic') 53 | def my_complex_function(a, b, c): 54 | result = a + b + c 55 | publish(my_complex_function._topic, {'complex_result': result}) 56 | ``` 57 | 58 | You could publish directly result of the function as well: 59 | 60 | ```python 61 | import pyhermes 62 | 63 | @pyhermes.publisher(topic='pl.allegro.pyhermes.sample-topic', auto_publish_result=True) 64 | def my_complex_function(a, b, c): 65 | return {'complex_result': a + b + c} 66 | ``` 67 | 68 | Result of decorated function is automatically published to selected topic in hermes. 69 | 70 | ## Running Tests 71 | 72 | Does the code actually work? 73 | 74 | ```python 75 | source /bin/activate 76 | (myenv) $ pip install -r requirements/test.txt 77 | (myenv) $ python runtests.py 78 | ``` 79 | 80 | ## Credits 81 | 82 | Tools used in rendering this package: 83 | 84 | * [Cookiecutter](https://github.com/audreyr/cookiecutter) 85 | * [cookiecutter-djangopackage](https://github.com/pydanny/cookiecutter-djangopackage) 86 | -------------------------------------------------------------------------------- /contrib/hermes_mock.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Simple mock of Hermes publisher, useful for local testing of communication 4 | between services using Hermes events. Incoming message would be "proxied" to 5 | publishing url with the same topic name. 6 | """ 7 | 8 | import argparse 9 | import concurrent.futures 10 | import uuid 11 | from http.server import BaseHTTPRequestHandler, HTTPServer 12 | from urllib.request import Request, urlopen 13 | 14 | 15 | SUCCESS_CODE = 201 16 | TIMEOUT = 10 # in seconds 17 | MAX_WORKERS = 2 18 | MESSAGE_ID_HEADER = 'Hermes-Message-Id' 19 | RETRY_COUNT_HEADER = 'Hermes-Retry-Count' 20 | 21 | parser = argparse.ArgumentParser(description='Hermes mock server.') 22 | parser.add_argument( 23 | '-p', '--port', dest='port', default=8888, type=int, 24 | help='Port of Hermes mock server.' 25 | ) 26 | parser.add_argument('-u', '--publish-url', dest='publish_url', required=True) 27 | args = parser.parse_args() 28 | 29 | 30 | def _publish(path, data, msg_id): 31 | topic = path.split('/')[-1] 32 | url = args.publish_url + topic + '/' 33 | request = Request( 34 | url, 35 | data=data, 36 | headers={ 37 | MESSAGE_ID_HEADER: msg_id, 38 | RETRY_COUNT_HEADER: 0, 39 | } 40 | ) 41 | print('Publishing {} to {} (msg_id: {})'.format(data, url, msg_id)) 42 | try: 43 | urlopen(request, timeout=TIMEOUT) 44 | except Exception as e: 45 | print('Exception during publishing data: {}'.format(e)) 46 | 47 | 48 | class HermesMockHandler(BaseHTTPRequestHandler): 49 | def do_POST(self): 50 | print('Get request on {}'.format(self.path)) 51 | content_len = int(self.headers.get('Content-Length', 0)) 52 | data = self.rfile.read(content_len) 53 | msg_id = str(uuid.uuid4()) 54 | self.tp.submit(_publish, self.path, data, msg_id) 55 | self.send_response(SUCCESS_CODE) 56 | self.send_header(MESSAGE_ID_HEADER, msg_id) 57 | self.end_headers() 58 | return 59 | 60 | 61 | def main(): 62 | with concurrent.futures.ThreadPoolExecutor( 63 | max_workers=MAX_WORKERS 64 | ) as executor: 65 | handler = type('HermesHandler', (HermesMockHandler,), {'tp': executor}) 66 | server = HTTPServer(('', args.port), handler) 67 | try: 68 | server.serve_forever() 69 | except KeyboardInterrupt: 70 | server.socket.close() 71 | 72 | if __name__ == '__main__': 73 | main() 74 | -------------------------------------------------------------------------------- /docs/configuration.md: -------------------------------------------------------------------------------- 1 | # Configuration 2 | 3 | Possible configuration options for `pyhermes` are listed below. 4 | 5 | ## `BASE_URL` 6 | 7 | Specifies base URL of Hermes. Example: 8 | ```python 9 | BASE_URL = 'http://my-hermes.local' 10 | ``` 11 | 12 | ## `URL_ADAPTER` 13 | 14 | Set this to any callable, if you want to add additional adapter for hermes requests (it's usefull when you're using service discovery, like [Consul](https://www.consul.io/)). Example usage (with [requests-consul](https://github.com/RulersOfAsgard/requests-consul) library): 15 | ```python 16 | def consul_adapter(): 17 | from requests_consul.adapters.service import ConsulServiceAdapter 18 | return 'service://', ConsulServiceAdapter(**CONSUL) 19 | 20 | BASE_URL = 'service://hermes' 21 | URL_ADAPTER = consul_adapter 22 | ``` 23 | 24 | ## `RETRY_MAX_ATTEMTPS` 25 | 26 | Specify how many times pyhermes should retry communication with Hermes. Default: 3 27 | 28 | ## `PUBLISHING_GROUP` 29 | Configuration of Hermes group to which you application will publish messages. Keys of this dictionary are the same as in [Hermes creating group](http://hermes-pubsub.readthedocs.org/en/latest/user/publishing/#creating-group) request body. Example: 30 | ```python 31 | PUBLISHING_GROUP = { 32 | 'groupName': 'pl.allegro.pyhermes', 33 | 'supportTeam': 'pyLabs', 34 | 'owner': 'pyLabs', 35 | 'contact': 'pylabs@allegro.pl' 36 | } 37 | ``` 38 | 39 | ## `PUBLISHING_TOPICS` 40 | Configuration of topics to which messages will be published from your application. Key of the main dictionary is name of the topic. Keys of single topic configuration are the same as in [Hermes creating topic](http://hermes-pubsub.readthedocs.org/en/latest/user/publishing/#creating-topic) request body. Example: 41 | ```python 42 | PUBLISHING_TOPICS = { 43 | 'test1': { 44 | 'description': "test topic", 45 | 'ack': 'LEADER', 46 | 'retentionTime': 1, 47 | 'trackingEnabled': False, 48 | 'contentType': 'JSON', 49 | 'validationEnabled': False, 50 | } 51 | } 52 | ``` 53 | 54 | > You don't have to specify all properties for single topic, just like in [Hermes creating topic](http://hermes-pubsub.readthedocs.org/en/latest/user/publishing/#creating-topic) request body. 55 | 56 | > Note that full topic name for publishing will be combination of publish group name (`PUBLISHING_GROUP['groupName']`) and topic name, for example `pl.allegro.pyhermes.test1`. 57 | 58 | ## `SUBSCRIBERS_MAPPING` 59 | Here you could define mapping for subscribed topics. Example: 60 | 61 | ```python 62 | SUBSCRIBERS_MAPPING = { 63 | 'pl.allegro.pyhermes.test1': 'my-topic' 64 | } 65 | ``` 66 | 67 | Then you could use mapped topic for your subscriptions: 68 | 69 | ```python 70 | import pyhermes 71 | 72 | @pyhermes.subscriber(topic='my-topic') 73 | def my_handler(data): 74 | pass 75 | ``` 76 | 77 | In this case, `my_handler` will be used for every message coming for `pl.allegro.pyhermes.test1` topic. 78 | -------------------------------------------------------------------------------- /docs/django.md: -------------------------------------------------------------------------------- 1 | # Django integration 2 | 3 | `pyhermes` has built-in [Django](https://www.djangoproject.com/) integration. 4 | 5 | To use `pyhermes` together with Django, follow these steps: 6 | 7 | 1. Add `pyhermes.apps.django` to `INSTALLED_APPS` in `settings.py`: 8 | 9 | ```python 10 | INSTALLED_APPS = ( 11 | # other apps 12 | 'pyhermes.apps.django', 13 | ) 14 | ``` 15 | 16 | 2. Configure `pyhermes` in `settings.py`, for example: 17 | 18 | ```python 19 | HERMES = { 20 | 'BASE_URL': 'http://hermes.local', 21 | 'PUBLISHING_GROUP': { 22 | 'groupName': 'pl.allegro.pyhermes', 23 | 'supportTeam': 'pyLabs', 24 | 'owner': 'pyLabs', 25 | 'contact': 'pylabs@allegro.pl' 26 | }, 27 | 'PUBLISHING_TOPICS': { 28 | 'test1': { 29 | 'description': "test topic", 30 | 'ack': 'LEADER', 31 | 'retentionTime': 1, 32 | 'trackingEnabled': False, 33 | 'contentType': 'JSON', 34 | 'validationEnabled': False, 35 | } 36 | } 37 | } 38 | ``` 39 | 40 | 3. Include `pyhermes.apps.django.urls` in your `urls.py`: 41 | 42 | ```python 43 | urlpatterns += patterns('', 44 | url(r'^hermes/', include('pyhermes.apps.django.urls')), 45 | ) 46 | ``` 47 | 48 | > Use `/hermes/events/` (for example `http://my-django-app.local/hermes/events/pl.allegro.pyhermes.test1`) to subscribe to particular topic in Hermes. 49 | 50 | 51 | ## Test command 52 | After instalation you can test your configuration by following command: 53 | 54 | ```bash 55 | ./manage.py hermes_test 56 | ``` 57 | 58 | Command send message to all topics defined in settings to Hermes. 59 | -------------------------------------------------------------------------------- /docs/flask.md: -------------------------------------------------------------------------------- 1 | # Flask integration 2 | 3 | `pyhermes` has built-in [Flask](http://flask.pocoo.org/) integration. 4 | 5 | To use `pyhermes` together with Flask, follow these steps: 6 | 7 | 1. Add following code to your app: 8 | 9 | ``` 10 | from pyhermes.apps.flask import configure_pyhermes 11 | 12 | configure_pyhermes(app, url_prefix='/hermes') 13 | ``` 14 | 15 | 2. Configure `pyhermes` in flask config, for example: 16 | 17 | ```python 18 | app.config['HERMES'] = { 19 | 'BASE_URL': 'http://hermes.local', 20 | 'PUBLISHING_GROUP': { 21 | 'groupName': 'pl.allegro.pyhermes', 22 | 'supportTeam': 'pyLabs', 23 | 'owner': 'pyLabs', 24 | 'contact': 'pylabs@allegro.pl' 25 | }, 26 | 'PUBLISHING_TOPICS': { 27 | 'test1': { 28 | 'description': "test topic", 29 | 'ack': 'LEADER', 30 | 'retentionTime': 1, 31 | 'trackingEnabled': False, 32 | 'contentType': 'JSON', 33 | 'validationEnabled': False, 34 | } 35 | } 36 | } 37 | ``` 38 | 39 | > Use `/hermes/events/` (for example `http://my-flask-app.local/hermes/events/pl.allegro.pyhermes.test1`) to subscribe to particular topic in Hermes. 40 | 41 | 42 | ## Test command 43 | After instalation you can test your configuration by following command: 44 | 45 | ```bash 46 | flask hermes test 47 | ``` 48 | 49 | Command send message to all topics defined in settings to Hermes. 50 | -------------------------------------------------------------------------------- /docs/hermes_mock.md: -------------------------------------------------------------------------------- 1 | # Hermes mock 2 | 3 | Pyhermes provides simple mock of Hermes publisher. It could be useful for local 4 | testing of interaction between services through Hermes events. 5 | 6 | This mock is simple http server, which forwards every request it gets to the 7 | subscriber (there could be only one subsriber, for now). Messages are proxied 8 | with the same topic as received. 9 | 10 | ## Usage 11 | 12 | ### Obtaining the mock 13 | 14 | To get the mock, either go to the `contrib` directory, or download it directly 15 | from [github](https://github.com/allegro/pyhermes/tree/master/contrib/hermes_mock.py). 16 | 17 | ### Running 18 | 19 | Type `python3 hermes_mock.py --help` for possible options: 20 | 21 | ``` 22 | $ python3 hermes_mock.py --help 23 | usage: hermes_mock.py [-h] [-p PORT] -u PUBLISH_URL 24 | 25 | Hermes mock server. 26 | 27 | optional arguments: 28 | -h, --help show this help message and exit 29 | -p PORT, --port PORT Port of Hermes mock server. 30 | -u PUBLISH_URL, --publish-url PUBLISH_URL 31 | ``` 32 | 33 | You could specify port, on which Hermes mock will be listening and URL of the 34 | subscriber. 35 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | ../README.md -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | 4 | At the command line type: 5 | ``` 6 | $ pip install pyhermes 7 | ``` 8 | -------------------------------------------------------------------------------- /docs/publishing.md: -------------------------------------------------------------------------------- 1 | # Publishing 2 | 3 | ## `publish` function 4 | Use `pyhermes.publish` function to publish messages to Hermes. 5 | 6 | ```python 7 | import pyhermes 8 | 9 | @pyhermes.publisher(topic='sample-topic') 10 | def my_function(): 11 | # processing 12 | pyhermes.publish(my_function._topic, {'result': result}) 13 | ``` 14 | 15 | > Note that after using `pyhermes.publisher` decorator, `_topic` attribute with the name of the topic is assigned to your function. 16 | 17 | ## autopublishing result of the function 18 | You could use `auto_publish_result` param to automatically publish result of the function to the given topic. 19 | 20 | ```python 21 | @pyhermes.publisher(topic='sample-topic', auto_publish_result=True) 22 | def my_function(): 23 | return {'result': 'abc'} 24 | ``` 25 | 26 | In this case, `{'result': 'abc'}` will be published with `sample-topic` topic after execution of the function. 27 | 28 | > Note that result of the function has to be JSON-serializable. 29 | -------------------------------------------------------------------------------- /docs/subscribing.md: -------------------------------------------------------------------------------- 1 | # Subscribing 2 | 3 | Use `pyhermes.subscriber` decorator to mark your function as incoming message handlers. 4 | 5 | ```python 6 | from pyhermes import subscriber 7 | 8 | @subscriber(topic='sample-topic') 9 | def handler(data): 10 | # process data 11 | ... 12 | ``` 13 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: pyhermes 2 | repo_url: https://github.com/allegro/pyhermes/ 3 | pages: 4 | - Quickstart: index.md 5 | - Installation: installation.md 6 | - Configuration: configuration.md 7 | - Subscribing: subscribing.md 8 | - Publishing: publishing.md 9 | - Django integration: django.md 10 | - Mock client: hermes_mock.md 11 | theme: 12 | name: readthedocs 13 | strict: true 14 | dev_addr: 0.0.0.0:8003 15 | markdown_extensions: 16 | - toc: 17 | permalink: True 18 | site_dir: docs/_build/html # sphinx compatible 19 | -------------------------------------------------------------------------------- /pyhermes/__init__.py: -------------------------------------------------------------------------------- 1 | from pyhermes.decorators import publisher, subscriber 2 | from pyhermes.publishing import publish 3 | 4 | __version__ = '0.7.1' 5 | 6 | __all__ = [ 7 | 'publish', 8 | 'publisher', 9 | 'subscriber' 10 | ] 11 | -------------------------------------------------------------------------------- /pyhermes/apps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allegro/pyhermes/17561042794ff930e0302bddf8fef3ab11f5071a/pyhermes/apps/__init__.py -------------------------------------------------------------------------------- /pyhermes/apps/django/__init__.py: -------------------------------------------------------------------------------- 1 | default_app_config = 'pyhermes.apps.django.config.PyhermesConfig' 2 | -------------------------------------------------------------------------------- /pyhermes/apps/django/config.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class PyhermesConfig(AppConfig): 5 | name = 'pyhermes.apps.django' 6 | label = 'pyhermes_django' 7 | -------------------------------------------------------------------------------- /pyhermes/apps/django/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allegro/pyhermes/17561042794ff930e0302bddf8fef3ab11f5071a/pyhermes/apps/django/management/__init__.py -------------------------------------------------------------------------------- /pyhermes/apps/django/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allegro/pyhermes/17561042794ff930e0302bddf8fef3ab11f5071a/pyhermes/apps/django/management/commands/__init__.py -------------------------------------------------------------------------------- /pyhermes/apps/django/management/commands/hermes_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from django.core.management.base import BaseCommand 3 | 4 | from pyhermes.management import integrations_command_handler, TOPICS_ALL 5 | 6 | 7 | class Command(BaseCommand): 8 | 9 | help = "Testing integration with Hermes" 10 | 11 | def add_arguments(self, parser): 12 | parser.add_argument( 13 | '-m', '--message', 14 | help='test message', 15 | default='From pyhermes With Love' 16 | ) 17 | parser.add_argument( 18 | '-t', '--topic', 19 | help='topic', 20 | default=TOPICS_ALL, 21 | ) 22 | 23 | def handle(self, *args, **options): 24 | topic = options.get('topic') 25 | message = options.get('message') 26 | integrations_command_handler(topic, message) 27 | -------------------------------------------------------------------------------- /pyhermes/apps/django/models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /pyhermes/apps/django/urls.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import django 3 | 4 | from pyhermes.apps.django.views import subscriber_view 5 | 6 | 7 | if django.VERSION < (2, 0, 0): 8 | from django.conf.urls import url 9 | urlpatterns = [ 10 | url( 11 | r'^events/(?P[a-zA-Z0-9_\.-]+)/$', 12 | subscriber_view, 13 | name='hermes-event-subscriber', 14 | ), 15 | ] 16 | 17 | if django.VERSION <= (1, 7): 18 | from django.conf.urls import patterns 19 | urlpatterns = patterns('', *urlpatterns) 20 | 21 | else: 22 | from django.urls import re_path 23 | urlpatterns = [ 24 | re_path( 25 | r'^events/(?P[a-zA-Z0-9_\.-]+)/$', 26 | subscriber_view, 27 | name='hermes-event-subscriber', 28 | ), 29 | ] 30 | -------------------------------------------------------------------------------- /pyhermes/apps/django/views.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import logging 3 | 4 | from django.http import HttpResponse 5 | from django.views.decorators.csrf import csrf_exempt 6 | from django.views.decorators.http import require_POST 7 | 8 | from pyhermes.exceptions import TopicHandlersNotFoundError 9 | from pyhermes.subscription import handle_subscription 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | @csrf_exempt 15 | @require_POST 16 | def subscriber_view(request, subscriber_name): 17 | raw_data = request.read().decode('utf-8') 18 | event_id = request.META.get('HTTP_HERMES_MESSAGE_ID') 19 | retry_count = request.META.get('HTTP_HERMES_RETRY_COUNT') 20 | try: 21 | handle_subscription(subscriber_name, raw_data, event_id, retry_count) 22 | except TopicHandlersNotFoundError: 23 | logger.error('subscriber `{}` does not exist.'.format(subscriber_name)) 24 | return HttpResponse(status=404) 25 | except ValueError: 26 | # json loading error 27 | # TODO: better handling 28 | return HttpResponse(status=400) 29 | else: 30 | return HttpResponse(status=204) 31 | -------------------------------------------------------------------------------- /pyhermes/apps/flask/__init__.py: -------------------------------------------------------------------------------- 1 | from pyhermes.apps.flask.config import configure_pyhermes 2 | 3 | __all__ = [ 4 | 'configure_pyhermes', 5 | ] 6 | -------------------------------------------------------------------------------- /pyhermes/apps/flask/blueprints.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from flask import abort, request, Blueprint 3 | from pyhermes.exceptions import TopicHandlersNotFoundError 4 | from pyhermes.subscription import handle_subscription 5 | 6 | logger = logging.getLogger(__name__) 7 | subscriber_handler = Blueprint('pyhermes', __name__) 8 | 9 | 10 | @subscriber_handler.route( 11 | '/events//', methods=['POST'] 12 | ) 13 | def subscriber_view(subscriber_name): 14 | raw_data = request.get_data().decode('utf-8') 15 | event_id = request.headers.get('HTTP_HERMES_MESSAGE_ID') 16 | retry_count = request.headers.get('HTTP_HERMES_RETRY_COUNT') 17 | try: 18 | handle_subscription(subscriber_name, raw_data, event_id, retry_count) 19 | except TopicHandlersNotFoundError: 20 | logger.error('subscriber `{}` does not exist.'.format(subscriber_name)) 21 | return abort(404) 22 | except ValueError: 23 | # json loading error 24 | # TODO: better handling 25 | return abort(400) 26 | else: 27 | return ('', 204) 28 | -------------------------------------------------------------------------------- /pyhermes/apps/flask/command.py: -------------------------------------------------------------------------------- 1 | import click 2 | from flask.cli import AppGroup 3 | from pyhermes.management import integrations_command_handler, TOPICS_ALL 4 | 5 | 6 | hermes_command = AppGroup('hermes') 7 | 8 | 9 | @hermes_command.command('test') 10 | @click.option( 11 | '-t', '--topic', 12 | help='topic', 13 | default=TOPICS_ALL, 14 | ) 15 | @click.option( 16 | '-m', '--message', 17 | help='test message', 18 | default='From pyhermes With Love' 19 | ) 20 | def test_hermes_command(topic, message): 21 | integrations_command_handler(topic, message) 22 | -------------------------------------------------------------------------------- /pyhermes/apps/flask/config.py: -------------------------------------------------------------------------------- 1 | from pyhermes.exceptions import PyhermesImproperlyConfiguredError 2 | from pyhermes.settings import HERMES_SETTINGS 3 | 4 | 5 | def _load_flask_config(app): 6 | hermes_settings = app.config.get('HERMES') 7 | if not hermes_settings: 8 | raise PyhermesImproperlyConfiguredError('Hermes settings not provided') 9 | HERMES_SETTINGS.update(**hermes_settings) 10 | 11 | 12 | def configure_pyhermes(app, url_prefix): 13 | from pyhermes.apps.flask.blueprints import subscriber_handler # noqa 14 | from pyhermes.apps.flask.command import hermes_command # noqa 15 | 16 | app.cli.add_command(hermes_command) 17 | app.register_blueprint(subscriber_handler, url_prefix=url_prefix) 18 | _load_flask_config(app) 19 | -------------------------------------------------------------------------------- /pyhermes/decorators.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from functools import wraps 3 | 4 | from pyhermes.registry import ( 5 | PublishersHandlersRegistry, 6 | SubscribersHandlersRegistry 7 | ) 8 | from pyhermes.publishing import _strip_topic_group, publish 9 | 10 | 11 | class subscriber(object): 12 | """ 13 | Mark function as subscription handler. Functions decorated with 14 | `subscriber` decorator will be called automatically by subscription handler 15 | 16 | Usage: 17 | @subscriber(topic='pl.allegro.pyhermes.topic1') 18 | def my_subscriber(data): 19 | ... 20 | """ 21 | def __init__(self, topic): 22 | self.topic = topic 23 | 24 | def _get_wrapper(self, func): 25 | return func 26 | 27 | def __call__(self, func): 28 | wrapper = self._get_wrapper(func) 29 | SubscribersHandlersRegistry.add_handler(self.topic, wrapper) 30 | return wrapper 31 | 32 | 33 | class publisher(object): 34 | """ 35 | Mark function as topic publisher. 36 | 37 | Usage: 38 | @publisher(topic='pl.allegro.pyhermes.topic1') 39 | def my_publisher(): 40 | ... 41 | 42 | Args: 43 | * topic - name of Hermes topic (could be with or without group name) 44 | * auto_publish_result - set to True if result of the function should be 45 | automatically published to Hermes. 46 | """ 47 | def __init__(self, topic, auto_publish_result=False): 48 | self.topic = _strip_topic_group(topic) 49 | self.auto_publish_result = auto_publish_result 50 | 51 | def __call__(self, func): 52 | @wraps(func) 53 | def wrapper(*args, **kwargs): 54 | result = func(*args, **kwargs) 55 | if self.auto_publish_result: 56 | publish(self.topic, result) 57 | return result 58 | PublishersHandlersRegistry.add_handler(self.topic, wrapper) 59 | wrapper._topic = self.topic 60 | return wrapper 61 | -------------------------------------------------------------------------------- /pyhermes/exceptions.py: -------------------------------------------------------------------------------- 1 | class PyhermesException(Exception): 2 | pass 3 | 4 | 5 | class HermesPublishException(PyhermesException): 6 | pass 7 | 8 | 9 | class TopicHandlersNotFoundError(PyhermesException): 10 | pass 11 | 12 | 13 | class PyhermesImproperlyConfiguredError(PyhermesException): 14 | pass 15 | -------------------------------------------------------------------------------- /pyhermes/management.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from pyhermes.exceptions import HermesPublishException 4 | 5 | TOPICS_ALL = 'all' 6 | 7 | 8 | def integrations_command_handler(topic, message): 9 | from pyhermes.publishing import publish 10 | from pyhermes.settings import HERMES_SETTINGS 11 | if not HERMES_SETTINGS.ENABLED: 12 | sys.stderr.write('Hermes integration is disabled. ' 13 | 'Check HERMES.ENABLED variable ' 14 | 'in your settings or environment.') 15 | return 16 | if topic == TOPICS_ALL: 17 | topics = HERMES_SETTINGS.PUBLISHING_TOPICS.keys() 18 | else: 19 | topics = [topic] 20 | 21 | if not topics: 22 | sys.stderr.write('Topics list is empty') 23 | 24 | for topic in topics: 25 | try: 26 | sys.stdout.write('Sending message to {}'.format(topic)) 27 | publish(topic, {'result': message}) 28 | except HermesPublishException as e: 29 | sys.stderr.write(str(e)) 30 | else: 31 | sys.stdout.write( 32 | 'Message was sent successfully to {}!'.format(topic) 33 | ) 34 | -------------------------------------------------------------------------------- /pyhermes/publishing.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | import requests 5 | from requests.exceptions import ConnectionError, HTTPError, Timeout 6 | 7 | from pyhermes.exceptions import HermesPublishException 8 | from pyhermes.settings import HERMES_SETTINGS 9 | from pyhermes.utils import retry 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | HERMES_VALID_RESPONSE_CODES = {201, 202} 14 | 15 | 16 | def _strip_topic_group(topic): 17 | """ 18 | Standardize topic name (remove group name from the beginning) 19 | """ 20 | group_name = HERMES_SETTINGS.PUBLISHING_GROUP['groupName'] 21 | if topic.startswith(group_name): 22 | topic = topic[len(group_name):].lstrip('.') 23 | return topic 24 | 25 | 26 | def _get_full_topic_name(topic): 27 | """ 28 | 29 | """ 30 | if not topic.startswith(HERMES_SETTINGS.PUBLISHING_GROUP['groupName']): 31 | topic = '{}.{}'.format( 32 | HERMES_SETTINGS.PUBLISHING_GROUP['groupName'], topic 33 | ) 34 | return topic 35 | 36 | 37 | def _handle_request_adapter(request_session): 38 | """ 39 | Handle custom rout-mapping 40 | See http://docs.python-requests.org/en/master/user/advanced/#transport-adapters # noqa 41 | for details 42 | """ 43 | if HERMES_SETTINGS.URL_ADAPTER: 44 | request_session.mount(*HERMES_SETTINGS.URL_ADAPTER()) 45 | 46 | 47 | @retry( 48 | max_attempts=HERMES_SETTINGS.RETRY_MAX_ATTEMTPS, 49 | retry_exceptions=(HermesPublishException,), 50 | logger=logger, 51 | ) 52 | def _send_message_to_hermes(url, headers, json_data): 53 | """ 54 | Send message to hermes with retrying. 55 | """ 56 | timeout = (HERMES_SETTINGS.CONNECT_TIMEOUT, HERMES_SETTINGS.READ_TIMEOUT) 57 | with requests.Session() as session: 58 | _handle_request_adapter(session) 59 | try: 60 | resp = session.post(url, headers=headers, data=json_data, timeout=timeout) 61 | except (ConnectionError, HTTPError, Timeout) as e: 62 | message = 'Error pushing event to Hermes: {}.'.format(e) 63 | raise HermesPublishException(message) 64 | 65 | if resp.status_code not in HERMES_VALID_RESPONSE_CODES: 66 | response_body = '' 67 | try: 68 | response_body = ' with reason: {}'.format(resp.json()) 69 | except ValueError: 70 | logger.debug( 71 | 'Unable to decode Hermes response as json', exc_info=True 72 | ) 73 | message = 'Bad response code during Hermes push: {}{}.'.format( 74 | resp.status_code, response_body 75 | ) 76 | raise HermesPublishException(message) 77 | return resp 78 | 79 | 80 | def publish(topic, data): 81 | """ 82 | Push an event to the Hermes. 83 | 84 | Args: 85 | topic: name of the topic 86 | data: data to push 87 | 88 | Returns: 89 | message id from Hermes 90 | """ 91 | # TODO: try-except 92 | if not HERMES_SETTINGS.ENABLED: 93 | logger.debug('Hermes integration is disabled') 94 | return 95 | json_data = json.dumps(data) 96 | headers = {'Content-Type': 'application/json'} 97 | url = "{}/topics/{}".format( 98 | HERMES_SETTINGS.BASE_URL, _get_full_topic_name(topic) 99 | ) 100 | logger.debug( 101 | 'Pushing message to topic "{}" (url: "{}") with data: {}'.format( 102 | topic, url, json_data 103 | ) 104 | ) 105 | try: 106 | resp = _send_message_to_hermes(url, headers, json_data) 107 | except HermesPublishException as e: 108 | message = 'Error pushing event to Hermes: {}.'.format(str(e)) 109 | logger.exception(message) 110 | raise 111 | 112 | hermes_event_id = resp.headers.get('Hermes-Message-Id') 113 | logger.info( 114 | 'Event with topic "{}"" sent to Hermes with event_id={}'.format( 115 | topic, hermes_event_id 116 | ) 117 | ) 118 | return hermes_event_id 119 | 120 | -------------------------------------------------------------------------------- /pyhermes/registry.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | 3 | import six 4 | 5 | from pyhermes.utils import Singleton 6 | from pyhermes.exceptions import TopicHandlersNotFoundError 7 | 8 | 9 | class HandlerRegistryMixin(object): 10 | def __init__(self): 11 | self.__registry = defaultdict(list) 12 | 13 | def add_handler(self, topic, handler): 14 | self.__registry[topic].append(handler) 15 | 16 | def get_handlers(self, topic): 17 | if topic in self.__registry: 18 | return self.__registry[topic] 19 | raise TopicHandlersNotFoundError( 20 | 'Handlers for topic {} not found'.format(topic) 21 | ) 22 | 23 | def get_all_handlers(self): 24 | return self.__registry.copy() 25 | 26 | 27 | class SubscribersHandlersRegistry( 28 | six.with_metaclass(Singleton, HandlerRegistryMixin) 29 | ): 30 | """ 31 | Registry of subscription handlers. 32 | """ 33 | pass 34 | 35 | 36 | class PublishersHandlersRegistry( 37 | six.with_metaclass(Singleton, HandlerRegistryMixin) 38 | ): 39 | """ 40 | Registry of publishers. 41 | """ 42 | pass 43 | 44 | 45 | SubscribersHandlersRegistry = SubscribersHandlersRegistry() 46 | PublishersHandlersRegistry = PublishersHandlersRegistry() 47 | -------------------------------------------------------------------------------- /pyhermes/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Proxy for all settings of pyhermes 3 | 4 | Handles: 5 | * django settings 6 | * custom calling of `update` method 7 | 8 | Use `HERMES_SETTINGS` for all of pyhermes settings. 9 | """ 10 | import six 11 | 12 | from pyhermes.exceptions import PyhermesImproperlyConfiguredError 13 | from pyhermes.utils import AttributeDict, Singleton 14 | 15 | _DEFAULT_GROUP_NAME = '__default__' 16 | DEFAULTS = { 17 | 'ENABLED': True, 18 | 'BASE_URL': '', 19 | 'URL_ADAPTER': None, 20 | 'PUBLISHING_GROUP': { 21 | 'groupName': _DEFAULT_GROUP_NAME, 22 | }, 23 | 'PUBLISHING_TOPICS': {}, 24 | 'SUBSCRIBERS_MAPPING': {}, 25 | 'RETRY_MAX_ATTEMTPS': 3, 26 | 'READ_TIMEOUT': 0.5, 27 | 'CONNECT_TIMEOUT': 0.3 28 | } 29 | 30 | 31 | class HermesSettings(six.with_metaclass(Singleton, object)): 32 | """ 33 | 34 | """ 35 | def __init__(self): 36 | self._wrapper = AttributeDict() 37 | 38 | def __getattr__(self, attr): 39 | try: 40 | from django.core.exceptions import ImproperlyConfigured # noqa: E501 41 | from django.conf import settings as django_settings 42 | except ImportError: 43 | pass 44 | else: 45 | try: 46 | self._wrapper = AttributeDict( 47 | {'HERMES': django_settings.HERMES} 48 | ) 49 | except (AttributeError, ImproperlyConfigured): 50 | pass 51 | 52 | try: 53 | return self._wrapper['HERMES'][attr] 54 | except KeyError: 55 | return DEFAULTS[attr] 56 | 57 | def update(self, **settings): 58 | if self._wrapper.get('HERMES'): 59 | self._wrapper['HERMES'].update(settings) 60 | else: 61 | self._wrapper['HERMES'] = settings 62 | 63 | 64 | HERMES_SETTINGS = HermesSettings() 65 | 66 | 67 | def _validate_hermes_settings(hermes_settings): 68 | if not hermes_settings.BASE_URL: 69 | raise PyhermesImproperlyConfiguredError('Hermes BASE_URL not provided') 70 | if ( 71 | not hermes_settings.PUBLISHING_GROUP or 72 | hermes_settings.PUBLISHING_GROUP['groupName'] == _DEFAULT_GROUP_NAME 73 | ): 74 | raise PyhermesImproperlyConfiguredError( 75 | 'Hermes GROUP info not provided' 76 | ) 77 | 78 | 79 | # TODO: validate 80 | # _validate_hermes_settings(HERMES_SETTINGS) 81 | -------------------------------------------------------------------------------- /pyhermes/subscription.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import json 3 | import logging 4 | 5 | from pyhermes.registry import SubscribersHandlersRegistry 6 | from pyhermes.settings import HERMES_SETTINGS 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | def handle_subscription(topic, raw_data, event_id, retry_count): 12 | """ 13 | Handler for topic subscription. Should be exposed (and possibly wrapped) 14 | through HTTP endpoint in chosen framework. 15 | 16 | Args: 17 | * topic: name of Hermes topic 18 | * raw_data: string with raw data for event 19 | * event_id: id of Hermes event 20 | * retry_count: number of retries to deliver this event 21 | (counting from 0) 22 | """ 23 | if not HERMES_SETTINGS.ENABLED: 24 | logger.debug('Hermes integration is disabled') 25 | return 26 | data = json.loads(raw_data) 27 | subscribers = SubscribersHandlersRegistry.get_handlers( 28 | HERMES_SETTINGS.SUBSCRIBERS_MAPPING.get(topic, topic) 29 | ) 30 | logger.info(( 31 | 'Received message for topic "{}" (eventID: {}, retry count: {})' 32 | ).format(topic, event_id, retry_count)) 33 | logger.debug('Message data {}'.format(str(data))) 34 | for subscriber in subscribers: 35 | subscriber(data) 36 | -------------------------------------------------------------------------------- /pyhermes/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import logging 3 | from functools import wraps 4 | 5 | 6 | class AttributeDict(dict): 7 | """ 8 | Dict which can handle access to keys using attributes. 9 | 10 | Example: 11 | >>> ad = AttributeDict({'a': 'b'}) 12 | >>> ad.a 13 | ... b 14 | """ 15 | def __init__(self, *args, **kwargs): 16 | super(AttributeDict, self).__init__(*args, **kwargs) 17 | self.__dict__ = self 18 | 19 | 20 | class Singleton(type): 21 | def __init__(cls, name, bases, dict): 22 | super(Singleton, cls).__init__(name, bases, dict) 23 | cls.instance = None 24 | 25 | def __call__(cls, *args, **kw): 26 | if cls.instance is None: 27 | cls.instance = super(Singleton, cls).__call__(*args, **kw) 28 | return cls.instance 29 | 30 | 31 | class override_hermes_settings(object): 32 | """ 33 | Utility context manager for overriding pyhermes settings in some context 34 | (ex. during single test). Could be used as a decorator as well: 35 | 36 | @override_hermes_settings(HERMES={'BASE_URL': 'http://hermes.local'}) 37 | def my_test(): 38 | ... 39 | """ 40 | def __init__(self, **kwargs): 41 | self.options = kwargs 42 | 43 | def __enter__(self): 44 | from pyhermes.settings import HERMES_SETTINGS 45 | HERMES_SETTINGS._wrapper = self.options 46 | 47 | def __exit__(self, exc_type, exc_value, traceback): 48 | from pyhermes.settings import HERMES_SETTINGS 49 | HERMES_SETTINGS._wrapper = None 50 | 51 | def __call__(self, func): 52 | @wraps(func) 53 | def wrapper(*args, **kwargs): 54 | with self: 55 | return func(*args, **kwargs) 56 | return wrapper 57 | 58 | 59 | # TODO(mkurek): add delay between consecutive retries 60 | class retry(object): 61 | """ 62 | Decorator providing retrying in case of error in wrapped function. 63 | 64 | Args: 65 | * max_attempts (int) - maximum number of retries 66 | * retry_exceptions (iterable) - exceptions, on which retry should 67 | happen 68 | * logger (Logger) - instance of python Logger 69 | 70 | Usage: 71 | @retry(max_attempts=4, retryExceptions=[ValueError]) 72 | def send_data(url, data): 73 | ... 74 | """ 75 | def __init__(self, max_attempts=1, retry_exceptions=None, logger=None): 76 | self.max_attempts = max_attempts 77 | assert self.max_attempts > 0 78 | self.retry_exceptions = retry_exceptions or (Exception,) 79 | self.logger = logger or logging.getLogger(__name__) 80 | 81 | def __call__(self, func): 82 | @wraps(func) 83 | def wrapper(*args, **kwargs): 84 | tries_left = self.max_attempts 85 | while tries_left > 1: 86 | tries_left -= 1 87 | try: 88 | return func(*args, **kwargs) 89 | except self.retry_exceptions as e: 90 | msg = 'Retrying because of {}'.format(str(e)) 91 | self.logger.warning(msg) 92 | return func(*args, **kwargs) 93 | return wrapper 94 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # pyproject.toml 2 | 3 | [build-system] 4 | requires = ["setuptools>=61.0"] 5 | build-backend = "setuptools.build_meta" 6 | 7 | [project] 8 | name = "pyhermes" 9 | description = "The Python interface to the Hermes message broker." 10 | readme = "README.md" 11 | requires-python = ">=3.9" 12 | license = "Apache-2.0" 13 | keywords = ["pyhermes"] 14 | dynamic = ["version"] 15 | 16 | authors = [ 17 | { name="Allegrogroup", email="pylabs@allegro.pl" }, 18 | ] 19 | 20 | classifiers = [ 21 | 'Development Status :: 4 - Beta', 22 | 'Framework :: Django', 23 | 'Framework :: Django :: 1.7', 24 | 'Framework :: Django :: 1.8', 25 | 'Framework :: Django :: 1.9', 26 | 'Framework :: Django :: 3.1', 27 | 'Intended Audience :: Developers', 28 | 'Natural Language :: English', 29 | 'Programming Language :: Python :: 2', 30 | 'Programming Language :: Python :: 2.7', 31 | 'Programming Language :: Python :: 3', 32 | 'Programming Language :: Python :: 3.9', 33 | ] 34 | 35 | dependencies = [ 36 | "six", 37 | "requests", 38 | ] 39 | 40 | [project.urls] 41 | "Homepage" = "https://github.com/allegro/pyhermes" 42 | 43 | [tool.setuptools] 44 | zip-safe = false 45 | include-package-data = true 46 | 47 | [tool.setuptools.packages.find] 48 | where = ["."] 49 | include = ["pyhermes*"] 50 | 51 | [tool.setuptools.package-data] 52 | "pyhermes" = ["py.typed"] 53 | 54 | [tool.setuptools.dynamic] 55 | version = {attr = "pyhermes.__version__"} 56 | -------------------------------------------------------------------------------- /requirements/base.txt: -------------------------------------------------------------------------------- 1 | # Additional requirements go here 2 | requests 3 | six 4 | # TODO: versions 5 | -------------------------------------------------------------------------------- /requirements/dev.txt: -------------------------------------------------------------------------------- 1 | bumpversion==0.5.3 2 | wheel==0.24.0 3 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | mock>=1.0.1 3 | flake8>=2.1.0 4 | tox>=1.7.0 5 | ddt 6 | responses 7 | 8 | # Additional test requirements go here 9 | -r base.txt 10 | -------------------------------------------------------------------------------- /runtests_django.py: -------------------------------------------------------------------------------- 1 | # TODO: separate django (apps) and non-django (non-apps) tests 2 | import sys 3 | 4 | try: 5 | from django.conf import settings 6 | from django.test.utils import get_runner 7 | 8 | settings.configure( 9 | DEBUG=True, 10 | USE_TZ=True, 11 | DATABASES={ 12 | "default": { 13 | "ENGINE": "django.db.backends.sqlite3", 14 | } 15 | }, 16 | ROOT_URLCONF="pyhermes.apps.django.urls", 17 | INSTALLED_APPS=[ 18 | "django.contrib.auth", 19 | "django.contrib.contenttypes", 20 | "django.contrib.sites", 21 | "pyhermes.apps.django", 22 | ], 23 | SITE_ID=1, 24 | MIDDLEWARE_CLASSES=(), 25 | ) 26 | 27 | try: 28 | import django 29 | setup = django.setup 30 | except AttributeError: 31 | pass 32 | else: 33 | setup() 34 | 35 | except ImportError: 36 | import traceback 37 | traceback.print_exc() 38 | raise ImportError("To fix this error, run: pip install -r requirements/test.txt") 39 | 40 | 41 | def run_tests(*test_args): 42 | if not test_args: 43 | test_args = ['tests'] 44 | 45 | # Run tests 46 | TestRunner = get_runner(settings) 47 | test_runner = TestRunner() 48 | failures = test_runner.run_tests(test_args, exclude_tags='tests/test_apps/test_flask/') 49 | 50 | if failures: 51 | sys.exit(bool(failures)) 52 | 53 | 54 | if __name__ == '__main__': 55 | run_tests(*sys.argv[1:]) 56 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allegro/pyhermes/17561042794ff930e0302bddf8fef3ab11f5071a/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_apps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allegro/pyhermes/17561042794ff930e0302bddf8fef3ab11f5071a/tests/test_apps/__init__.py -------------------------------------------------------------------------------- /tests/test_apps/test_django/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allegro/pyhermes/17561042794ff930e0302bddf8fef3ab11f5071a/tests/test_apps/test_django/__init__.py -------------------------------------------------------------------------------- /tests/test_apps/test_django/test_subscriber.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import json 3 | 4 | from ddt import ddt, data as ddt_data, unpack 5 | try: 6 | from django.urls import reverse 7 | except ImportError: 8 | from django.core.urlresolvers import reverse 9 | from django.test import Client, TestCase 10 | 11 | from pyhermes.decorators import subscriber 12 | 13 | 14 | @ddt 15 | class SubscriberTestCase(TestCase): 16 | def setUp(self): 17 | self.client = Client() 18 | 19 | def test_subscription_with_single_handler(self): 20 | topic = 'pl.allegro.pyhermes.test-subscriber-topic1' 21 | called = [False] 22 | data = {'a': 'b', 'c': 2} 23 | 24 | @subscriber(topic=topic) 25 | def subscriber_1(d): 26 | called[0] = True 27 | self.assertEqual(d, data) 28 | 29 | response = self.client.post( 30 | reverse('hermes-event-subscriber', args=(topic,)), 31 | data=json.dumps(data), 32 | content_type='application/json', 33 | ) 34 | self.assertEqual(response.status_code, 204) 35 | self.assertEqual(called, [True]) 36 | 37 | def test_subscription_with_multiple_handlers(self): 38 | topic = 'pl.allegro.pyhermes.test-subscriber-topic2' 39 | called = [0] 40 | data = {'a': 'b', 'c': 2} 41 | 42 | @subscriber(topic=topic) 43 | def subscriber_1(d): 44 | called[0] = called[0] + 1 45 | 46 | @subscriber(topic=topic) 47 | def subscriber_2(d): 48 | called[0] = called[0] + 1 49 | 50 | response = self.client.post( 51 | reverse('hermes-event-subscriber', args=(topic,)), 52 | data=json.dumps(data), 53 | content_type='application/json', 54 | ) 55 | self.assertEqual(response.status_code, 204) 56 | self.assertEqual(called, [2]) 57 | 58 | def test_subscription_handler_not_found(self): 59 | topic = 'pl.allegro.pyhermes.test-subscriber-handler-not-found' 60 | data = {'a': 'b', 'c': 2} 61 | response = self.client.post( 62 | reverse('hermes-event-subscriber', args=(topic,)), 63 | data=json.dumps(data), 64 | content_type='application/json', 65 | ) 66 | self.assertEqual(response.status_code, 404) 67 | 68 | @unpack 69 | @ddt_data( 70 | ('invalid_json',), 71 | ) 72 | def test_subscription_bad_request(self, data): 73 | topic = 'pl.allegro.pyhermes.test-subscriber-topic3' 74 | 75 | @subscriber(topic=topic) 76 | def subscriber_1(d): 77 | pass 78 | 79 | response = self.client.post( 80 | reverse('hermes-event-subscriber', args=(topic,)), 81 | data=data, 82 | content_type='application/json', 83 | ) 84 | self.assertEqual(response.status_code, 400) 85 | -------------------------------------------------------------------------------- /tests/test_apps/test_flask/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allegro/pyhermes/17561042794ff930e0302bddf8fef3ab11f5071a/tests/test_apps/test_flask/__init__.py -------------------------------------------------------------------------------- /tests/test_apps/test_flask/test_subscriber.py: -------------------------------------------------------------------------------- 1 | import json 2 | import unittest 3 | try: 4 | from flask import Flask 5 | except ImportError: 6 | pass 7 | from ddt import ddt, data as ddt_data, unpack 8 | 9 | from pyhermes.apps.flask import configure_pyhermes 10 | from pyhermes.decorators import subscriber 11 | 12 | 13 | @ddt 14 | class SubscriberTestCase(unittest.TestCase): 15 | 16 | def setUp(self): 17 | app = Flask(__name__) 18 | app.debug = True 19 | app.config['HERMES'] = { 20 | 'BASE_URL': 'http://hermes.local:8090', 21 | 'SUBSCRIBERS_MAPPING': {'pl.hermes.testTopic': 'new_message'}, 22 | 'PUBLISHING_TOPICS': { 23 | 'test1': { 24 | 'description': "test topic", 25 | 'ack': 'LEADER', 26 | 'retentionTime': 1, 27 | 'trackingEnabled': False, 28 | 'contentType': 'JSON', 29 | 'validationEnabled': False, 30 | } 31 | } 32 | } 33 | self.app_client = app.test_client() 34 | configure_pyhermes(app, url_prefix='/hermes') 35 | 36 | def test_subscription_with_single_handler(self): 37 | topic = 'new_message' 38 | called = [False] 39 | data = {'a': 'b', 'c': 2} 40 | 41 | @subscriber(topic=topic) 42 | def subscriber_1(d): 43 | called[0] = True 44 | self.assertEqual(d, data) 45 | 46 | response = self.app_client.post( 47 | '/hermes/events/pl.hermes.testTopic/', 48 | data=json.dumps(data), 49 | ) 50 | self.assertEqual(response.status_code, 204) 51 | self.assertEqual(called, [True]) 52 | 53 | def test_subscription_with_multiple_handlers(self): 54 | topic = 'new_message' 55 | called = [0] 56 | data = {'a': 'b', 'c': 2} 57 | 58 | @subscriber(topic=topic) 59 | def subscriber_1(d): 60 | called[0] = called[0] + 1 61 | 62 | @subscriber(topic=topic) 63 | def subscriber_2(d): 64 | called[0] = called[0] + 1 65 | 66 | response = self.app_client.post( 67 | '/hermes/events/pl.hermes.testTopic/', 68 | data=json.dumps(data), 69 | ) 70 | self.assertEqual(response.status_code, 204) 71 | self.assertEqual(called, [2]) 72 | 73 | def test_subscription_handler_not_found(self): 74 | data = {'a': 'b', 'c': 2} 75 | response = self.app_client.post( 76 | '/hermes/events/pl.hermes.topicNotFound/', 77 | data=json.dumps(data), 78 | ) 79 | self.assertEqual(response.status_code, 404) 80 | 81 | @unpack 82 | @ddt_data( 83 | ('invalid_json',), 84 | ) 85 | def test_subscription_bad_request(self, data): 86 | topic = 'new_message' 87 | 88 | @subscriber(topic=topic) 89 | def subscriber_1(d): 90 | pass 91 | 92 | response = self.app_client.post( 93 | '/hermes/events/pl.hermes.testTopic/', 94 | data=data, 95 | ) 96 | self.assertEqual(response.status_code, 400) 97 | 98 | 99 | if __name__ == '__main__': 100 | unittest.main() 101 | -------------------------------------------------------------------------------- /tests/test_internal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/allegro/pyhermes/17561042794ff930e0302bddf8fef3ab11f5071a/tests/test_internal/__init__.py -------------------------------------------------------------------------------- /tests/test_internal/test_decorators.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import unittest 4 | try: 5 | from unittest import mock 6 | except ImportError: 7 | import mock 8 | 9 | from pyhermes.decorators import publisher, subscriber 10 | from pyhermes.registry import ( 11 | PublishersHandlersRegistry, 12 | SubscribersHandlersRegistry 13 | ) 14 | 15 | 16 | @publisher('pl.allegro.pyhermes.topic1') 17 | def publisher1(a, b): 18 | return a + b 19 | 20 | 21 | @publisher('pl.allegro.pyhermes.topic1', auto_publish_result=True) 22 | def publisher11(a, b, c): 23 | return {'result': a + b + c} 24 | 25 | 26 | @subscriber('pl.allegro.pyhermes.topic1') 27 | def subscriber1(a, b): 28 | return a + b 29 | 30 | 31 | @subscriber('pl.allegro.pyhermes.topic1') 32 | def subscriber11(a, b): 33 | return a + b 34 | 35 | 36 | class PublisherDecoratorTestCase(unittest.TestCase): 37 | def test_publishers_registry(self): 38 | self.assertEqual( 39 | PublishersHandlersRegistry.get_handlers( 40 | 'pl.allegro.pyhermes.topic1' 41 | ), 42 | [publisher1, publisher11] 43 | ) 44 | 45 | def test_subscribers_registry(self): 46 | self.assertEqual( 47 | SubscribersHandlersRegistry.get_handlers( 48 | 'pl.allegro.pyhermes.topic1' 49 | ), 50 | [subscriber1, subscriber11] 51 | ) 52 | 53 | def test_publisher_set_topic(self): 54 | self.assertEqual(publisher1._topic, 'pl.allegro.pyhermes.topic1') 55 | 56 | @mock.patch('pyhermes.decorators.publish') 57 | def test_auto_publish_result(self, publish_mock): 58 | result = publisher11(2, 3, 4) 59 | # check if result is still properly returned 60 | self.assertEqual(result, {'result': 9}) 61 | publish_mock.assert_called_once_with( 62 | 'pl.allegro.pyhermes.topic1', {'result': 9} 63 | ) 64 | 65 | @mock.patch('pyhermes.decorators.publish') 66 | def test_auto_publish_result_turned_off(self, publish_mock): 67 | result = publisher1(2, 3) 68 | # check if result is still properly returned 69 | self.assertEqual(result, 5) 70 | self.assertEqual(publish_mock.call_count, 0) 71 | -------------------------------------------------------------------------------- /tests/test_internal/test_publishing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from unittest import TestCase 3 | 4 | import responses 5 | from ddt import ddt, data, unpack 6 | from requests.exceptions import ConnectionError, HTTPError, Timeout 7 | 8 | from pyhermes.exceptions import HermesPublishException 9 | from pyhermes.publishing import _strip_topic_group, publish 10 | from pyhermes.settings import HERMES_SETTINGS 11 | from pyhermes.utils import override_hermes_settings 12 | 13 | 14 | def fake_connection_error(request): 15 | raise ConnectionError('connection error') 16 | 17 | 18 | def fake_http_error(request): 19 | raise HTTPError('http error') 20 | 21 | 22 | def fake_timeout(request): 23 | raise Timeout('timeout') 24 | 25 | 26 | TEST_GROUP_NAME = 'pl.allegro.pyhermes' 27 | TEST_TOPIC = 'test-publisher-topic1' 28 | TEST_HERMES_SETTINGS = { 29 | 'BASE_URL': 'http://hermes.local', 30 | 'PUBLISHING_GROUP': { 31 | 'groupName': TEST_GROUP_NAME, 32 | } 33 | } 34 | 35 | 36 | @ddt 37 | class PublisherTestCase(TestCase): 38 | @override_hermes_settings(HERMES=TEST_HERMES_SETTINGS) 39 | @responses.activate 40 | @unpack 41 | @data( 42 | (201,), 43 | (202,), 44 | ) 45 | def test_publish_ok(self, status_code): 46 | hermes_event_id = 'hermes_ok' 47 | data = {'test': 'data'} 48 | responses.add( 49 | method=responses.POST, 50 | url="{}/topics/{}.{}".format( 51 | HERMES_SETTINGS.BASE_URL, TEST_GROUP_NAME, TEST_TOPIC 52 | ), 53 | match_querystring=True, 54 | body=None, 55 | status=status_code, 56 | content_type='application/json', 57 | adding_headers={ 58 | 'Hermes-Message-Id': hermes_event_id 59 | } 60 | ) 61 | # TODO: check data 62 | response = publish(TEST_TOPIC, data) 63 | self.assertEqual(response, hermes_event_id) 64 | 65 | @override_hermes_settings(HERMES=TEST_HERMES_SETTINGS) 66 | @responses.activate 67 | def test_publish_full_topic_name(self): 68 | hermes_event_id = 'hermes_ok' 69 | data = {'test': 'data'} 70 | responses.add( 71 | method=responses.POST, 72 | url="{}/topics/{}.{}".format( 73 | HERMES_SETTINGS.BASE_URL, TEST_GROUP_NAME, TEST_TOPIC 74 | ), 75 | match_querystring=True, 76 | body=None, 77 | status=201, 78 | content_type='application/json', 79 | adding_headers={ 80 | 'Hermes-Message-Id': hermes_event_id 81 | } 82 | ) 83 | # TODO: check data 84 | response = publish('{}.{}'.format(TEST_GROUP_NAME, TEST_TOPIC), data) 85 | self.assertEqual(response, hermes_event_id) 86 | 87 | @override_hermes_settings(HERMES=TEST_HERMES_SETTINGS) 88 | @responses.activate 89 | @unpack 90 | @data( 91 | (200,), 92 | (400,), 93 | (403,), 94 | (404,), 95 | (500,), 96 | ) 97 | def test_publish_bad_status_code(self, status_code): 98 | data = {'test': 'data'} 99 | responses.add( 100 | method=responses.POST, 101 | url="{}/topics/{}.{}".format( 102 | HERMES_SETTINGS.BASE_URL, TEST_GROUP_NAME, TEST_TOPIC 103 | ), 104 | match_querystring=True, 105 | body=None, 106 | status=status_code, 107 | content_type='application/json', 108 | ) 109 | with self.assertRaises(HermesPublishException) as cm: 110 | publish(TEST_TOPIC, data) 111 | self.assertEqual( 112 | str(cm.exception), 113 | 'Bad response code during Hermes push: {}.'.format(status_code) 114 | ) 115 | 116 | @override_hermes_settings(HERMES=TEST_HERMES_SETTINGS) 117 | @responses.activate 118 | @unpack 119 | @data( 120 | (fake_connection_error, 'connection error'), 121 | (fake_http_error, 'http error'), 122 | (fake_timeout, 'timeout') 123 | ) 124 | def test_publish_request_error(self, fake_handler, msg): 125 | data = {'test': 'data'} 126 | responses.add_callback( 127 | method=responses.POST, 128 | url="{}/topics/{}.{}".format( 129 | HERMES_SETTINGS.BASE_URL, TEST_GROUP_NAME, TEST_TOPIC 130 | ), 131 | match_querystring=True, 132 | content_type='application/json', 133 | callback=fake_handler, 134 | ) 135 | with self.assertRaises(HermesPublishException) as cm: 136 | publish(TEST_TOPIC, data) 137 | self.assertEqual( 138 | str(cm.exception), 139 | 'Error pushing event to Hermes: {}.'.format(msg) 140 | ) 141 | 142 | @override_hermes_settings(HERMES=TEST_HERMES_SETTINGS) 143 | @responses.activate 144 | def test_publish_request_error_with_retry(self): 145 | data = {'test': 'data'} 146 | hermes_event_id = 'hermes_ok' 147 | tries = [0] 148 | 149 | def callback(request): 150 | tries[0] += 1 151 | print(tries) 152 | if tries[0] <= 2: 153 | raise ConnectionError('connection error') 154 | print('Returning normal') 155 | return (201, {'Hermes-Message-Id': hermes_event_id}, "") 156 | 157 | responses.add_callback( 158 | method=responses.POST, 159 | url="{}/topics/{}.{}".format( 160 | HERMES_SETTINGS.BASE_URL, TEST_GROUP_NAME, TEST_TOPIC 161 | ), 162 | match_querystring=True, 163 | content_type='application/json', 164 | callback=callback, 165 | ) 166 | response = publish('{}.{}'.format(TEST_GROUP_NAME, TEST_TOPIC), data) 167 | self.assertEqual(response, hermes_event_id) 168 | self.assertEqual(tries[0], 3) 169 | 170 | @override_hermes_settings(HERMES=TEST_HERMES_SETTINGS) 171 | @responses.activate 172 | def test_publish_request_wrong_response_code_with_retry(self): 173 | data = {'test': 'data'} 174 | hermes_event_id = 'hermes_ok' 175 | tries = [0] 176 | 177 | def callback(request): 178 | tries[0] += 1 179 | if tries[0] <= 2: 180 | return (408, {}, "") 181 | return (202, {'Hermes-Message-Id': hermes_event_id}, "") 182 | 183 | responses.add_callback( 184 | method=responses.POST, 185 | url="{}/topics/{}.{}".format( 186 | HERMES_SETTINGS.BASE_URL, TEST_GROUP_NAME, TEST_TOPIC 187 | ), 188 | match_querystring=True, 189 | content_type='application/json', 190 | callback=callback, 191 | ) 192 | response = publish('{}.{}'.format(TEST_GROUP_NAME, TEST_TOPIC), data) 193 | self.assertEqual(response, hermes_event_id) 194 | self.assertEqual(tries[0], 3) 195 | 196 | 197 | class TestStripTopicGroupName(TestCase): 198 | @override_hermes_settings(HERMES=TEST_HERMES_SETTINGS) 199 | def test_stripping_group_name_when_topic_startswith_group_name(self): 200 | self.assertEqual( 201 | _strip_topic_group('pl.allegro.pyhermes.my-topic'), 202 | 'my-topic' 203 | ) 204 | 205 | @override_hermes_settings(HERMES=TEST_HERMES_SETTINGS) 206 | def test_stripping_group_name_when_topic_not_startswith_group_name(self): 207 | self.assertEqual( 208 | _strip_topic_group('my-topic'), 209 | 'my-topic' 210 | ) 211 | -------------------------------------------------------------------------------- /tests/test_internal/test_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import unittest 4 | try: 5 | from unittest import mock 6 | except ImportError: 7 | import mock 8 | 9 | from pyhermes.utils import retry 10 | 11 | 12 | class PublisherDecoratorTestCase(unittest.TestCase): 13 | def test_retry_without_any_exceptions(self): 14 | logger_mock = mock.MagicMock() 15 | 16 | @retry(max_attempts=3, logger=logger_mock) 17 | def test_func(a): 18 | return a 19 | 20 | result = test_func('ok') 21 | self.assertEqual(result, 'ok') 22 | self.assertEqual(logger_mock.warning.call_count, 0) 23 | 24 | def test_retry_with_exception_in_the_middle(self): 25 | tries = [0] 26 | logger_mock = mock.MagicMock() 27 | 28 | @retry(max_attempts=3, logger=logger_mock) 29 | def test_func(a): 30 | tries[0] += 1 31 | if tries[0] <= 2: 32 | raise ValueError() 33 | return a 34 | 35 | result = test_func('ok') 36 | self.assertEqual(result, 'ok') 37 | self.assertEqual(logger_mock.warning.call_count, 2) 38 | 39 | def test_retry_with_exception_after_tries(self): 40 | logger_mock = mock.MagicMock() 41 | 42 | @retry(max_attempts=3, logger=logger_mock) 43 | def test_func(a): 44 | raise ValueError() 45 | 46 | with self.assertRaises(ValueError): 47 | test_func('ok') 48 | 49 | self.assertEqual(logger_mock.warning.call_count, 2) 50 | 51 | def test_retry_with_custom_exception_no_retry_on_wrong_exception(self): 52 | logger_mock = mock.MagicMock() 53 | 54 | @retry(logger=logger_mock, retry_exceptions=(IndexError,)) 55 | def test_func(a): 56 | raise ValueError() 57 | 58 | with self.assertRaises(ValueError): 59 | test_func('ok') 60 | 61 | self.assertEqual(logger_mock.warning.call_count, 0) 62 | 63 | def test_retry_with_custom_exception_retry_on_proper_exception(self): 64 | logger_mock = mock.MagicMock() 65 | 66 | @retry(logger=logger_mock, retry_exceptions=(IndexError,)) 67 | def test_func(a): 68 | raise IndexError() 69 | 70 | with self.assertRaises(IndexError): 71 | test_func('ok') 72 | 73 | self.assertEqual(logger_mock.warning.call_count, 0) 74 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | {py27,py34,py35,py36,py39} 4 | {py27,py34,py35,py36,py39}-django{18,19,110,111,315,3215,dev}, 5 | {py27,py34,py39}-django{17}, 6 | {py27,py34,py35,py36,py39}-flask{012,10,112,dev}, 7 | 8 | [testenv] 9 | setenv = 10 | PYTHONPATH = {toxinidir}:{toxinidir}/pyhermes 11 | commands = python -m unittest discover tests/test_internal/ 12 | deps = 13 | -r{toxinidir}/requirements/test.txt 14 | 15 | 16 | [testenv:django] 17 | setenv = 18 | PYTHONPATH = {toxinidir}:{toxinidir}/pyhermes 19 | commands = python runtests_django.py tests/test_apps/test_django/ 20 | deps = 21 | django17: Django==1.7.11 22 | django18: Django==1.8.17 23 | django19: Django==1.9.12 24 | django110: Django==1.10.4 25 | django111: Django==1.11.a1 26 | dajngo315: Django==3.1.5 27 | dajngo3215: Django==3.2.15 28 | djangodev: git+git://github.com/django/django.git 29 | -r{toxinidir}/requirements/test.txt 30 | 31 | 32 | [testenv:flask] 33 | setenv = 34 | PYTHONPATH = {toxinidir}:{toxinidir}/pyhermes 35 | commands = python -m unittest discover tests/test_apps/test_flask/ 36 | deps = 37 | flask112: Flask==1.1.2 38 | flask122: Flask==0.12.2 39 | flask10: Flask==1.0.2 40 | flaskdev: git+git://github.com/pallets/flask.git 41 | -r{toxinidir}/requirements/test.txt 42 | --------------------------------------------------------------------------------