├── .circleci └── config.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .scrutinizer.yml ├── CHANGES.rst ├── CONTRIBUTORS.rst ├── Dockerfile.local ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── docker-compose.yml ├── docs ├── Makefile └── source │ ├── _static │ ├── sequence_loafer.diag │ └── sequence_loafer.png │ ├── changelog.rst │ ├── conf.py │ ├── contributors.rst │ ├── development │ ├── installation.rst │ └── release.rst │ ├── error_handlers.rst │ ├── exceptions.rst │ ├── faq.rst │ ├── generic_handlers.rst │ ├── handlers.rst │ ├── index.rst │ ├── managers.rst │ ├── message_translators.rst │ ├── overview.rst │ ├── providers.rst │ ├── quickstart │ └── installation.rst │ ├── routes.rst │ ├── settings.rst │ └── tutorial.rst ├── env.local ├── examples ├── echo │ ├── __init__.py │ ├── __main__.py │ └── sentry.py ├── entrypoint.sh └── sample.json ├── loafer ├── __init__.py ├── dispatchers.py ├── exceptions.py ├── ext │ ├── __init__.py │ ├── aws │ │ ├── __init__.py │ │ ├── bases.py │ │ ├── handlers.py │ │ ├── message_translators.py │ │ ├── providers.py │ │ └── routes.py │ └── sentry.py ├── managers.py ├── message_translators.py ├── providers.py ├── routes.py ├── runners.py └── utils.py ├── pytest.ini ├── requirements ├── local.txt └── test.txt ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── conftest.py ├── ext ├── __init__.py ├── aws │ ├── __init__.py │ ├── conftest.py │ ├── test_bases.py │ ├── test_handlers.py │ ├── test_message_translators.py │ ├── test_providers.py │ └── test_routes.py └── test_sentry.py ├── test_dispatchers.py ├── test_managers.py ├── test_message_translator.py ├── test_routes.py ├── test_runners.py └── test_utils.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build_py36: &test-template 4 | working_directory: ~/repo 5 | docker: 6 | - image: circleci/python:3.6.9-stretch 7 | environment: 8 | PRE_COMMIT_SKIP=no-commit-to-branch 9 | 10 | steps: 11 | - checkout 12 | 13 | - restore_cache: 14 | key: deps-{{ .Branch }}-{{ checksum "setup.py" }} 15 | 16 | - run: 17 | name: install 18 | command: | 19 | python3 -m venv venv 20 | . venv/bin/activate 21 | pip install -r requirements/test.txt 22 | pip install -e . 23 | 24 | - save_cache: 25 | key: deps-{{ .Branch }}-{{ checksum "setup.py" }} 26 | paths: 27 | - "venv" 28 | 29 | - run: 30 | name: linters 31 | command: | 32 | . venv/bin/activate 33 | SKIP=$PRE_COMMIT_SKIP pre-commit run -a -v 34 | 35 | - run: 36 | name: tests 37 | command: | 38 | . venv/bin/activate 39 | make test-cov 40 | 41 | - run: 42 | name: check-fixtures 43 | command: | 44 | . venv/bin/activate 45 | make check-fixtures 46 | 47 | build_py37: 48 | <<: *test-template 49 | docker: 50 | - image: circleci/python:3.7.7-stretch 51 | environment: 52 | PRE_COMMIT_SKIP=no-commit-to-branch 53 | 54 | build_py38: 55 | <<: *test-template 56 | docker: 57 | - image: circleci/python:3.8.2-buster 58 | environment: 59 | PRE_COMMIT_SKIP=no-commit-to-branch 60 | 61 | 62 | workflows: 63 | version: 2 64 | commit: 65 | jobs: 66 | - build_py36 67 | - build_py37 68 | - build_py38 69 | 70 | scheduled: 71 | triggers: 72 | - schedule: 73 | cron: "0 0 * * *" 74 | filters: 75 | branches: 76 | only: master 77 | 78 | jobs: 79 | - build_py36 80 | - build_py37 81 | - build_py38 82 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .pytest_cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | docs/build/ 58 | 59 | # PyBuilder 60 | target/ 61 | 62 | #Ipython Notebook 63 | .ipynb_checkpoints 64 | 65 | # environment 66 | .env 67 | .venv 68 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: git@github.com:pre-commit/pre-commit-hooks 3 | rev: v3.1.0 4 | hooks: 5 | - id: debug-statements 6 | - id: trailing-whitespace 7 | - id: check-merge-conflict 8 | - id: check-executables-have-shebangs 9 | - id: check-ast 10 | - id: check-byte-order-marker 11 | - id: check-json 12 | - id: check-symlinks 13 | - id: check-vcs-permalinks 14 | - id: check-xml 15 | - id: check-yaml 16 | - id: detect-private-key 17 | - id: forbid-new-submodules 18 | - id: no-commit-to-branch 19 | args: ['-b master'] 20 | 21 | - repo: https://gitlab.com/pycqa/flake8 22 | rev: 3.8.3 23 | hooks: 24 | - id: flake8 25 | args: ['--exclude=docs/*', '--ignore=E501'] 26 | 27 | - repo: https://github.com/timothycrosley/isort 28 | rev: 5.2.0 29 | hooks: 30 | - id: isort 31 | 32 | - repo: local 33 | hooks: 34 | 35 | - id: check-datetime-now 36 | name: check_datetime_now 37 | description: Prefer datetime.utcnow() 38 | language: pygrep 39 | entry: 'datetime\.now\(\)' 40 | types: [python] 41 | 42 | - repo: https://github.com/pre-commit/pygrep-hooks 43 | rev: v1.5.1 44 | hooks: 45 | - id: python-check-mock-methods 46 | 47 | - repo: git@github.com:buteco/hulks.git 48 | rev: 0.4.0 49 | hooks: 50 | - id: check-logger 51 | - id: check-mutable-defaults 52 | -------------------------------------------------------------------------------- /.scrutinizer.yml: -------------------------------------------------------------------------------- 1 | checks: 2 | python: 3 | code_rating: true 4 | duplicate_code: true 5 | 6 | build: 7 | environment: 8 | python: 3.6.3 9 | tests: 10 | before: 11 | - "python setup.py install" 12 | 13 | override: 14 | - py.test -vv --cov loafer --cov-report=term-missing 15 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | 2.0.1 (2020-07-28) 2 | ------------------ 3 | 4 | * Fix setry integration (# by @hartungstenio) 5 | * Minor improvements 6 | 7 | 2.0.0 (2020-05-16) 8 | ------------------ 9 | 10 | * Dropped support for Python 3.5 11 | * Added support for Python 3.8 12 | * Update aiobotocore dependency version (#61 by @GuilhermeVBeira) 13 | * Improvements due to changes in asyncio (#48, #52 by @lamenezes) 14 | * Sentry wrapper/helper updated to support new sdk (wip) 15 | * Minor documentation improvements 16 | 17 | 1.3.2 (2019-04-27) 18 | ------------------ 19 | 20 | * Improve message processing (#48 by @lamenezes) 21 | * Improve error logging (#39 by @wiliamsouza) 22 | * Refactor in message dispatcher and event-loop shutdown 23 | * Minor fixes and improvements 24 | 25 | 1.3.1 (2017-10-22) 26 | ------------------ 27 | 28 | * Improve performance (#35 by @allisson) 29 | * Fix requirement versions resolution 30 | * Minor fixes and improvements 31 | 32 | 1.3.0 (2017-09-26) 33 | ------------------ 34 | 35 | * Refactor tasks dispatching, it should improve performance 36 | * Refactor SQSProvider to ignore HTTP 404 errors when deleting messages 37 | * Minor fixes and improvements 38 | 39 | 1.2.1 (2017-09-11) 40 | ------------------ 41 | 42 | * Bump boto3 version (by @daneoshiga) 43 | 44 | 1.2.0 (2017-08-15) 45 | ------------------ 46 | 47 | * Enable provider parameters (boto client options) 48 | 49 | 1.1.1 (2017-06-14) 50 | ------------------ 51 | 52 | * Bugfix: fix SNS prefix value in use for topic name wildcard (by @lamenezes) 53 | 54 | 1.1.0 (2017-05-01) 55 | ------------------ 56 | 57 | * Added initial contracsts for class-based handlers 58 | * Added generic handlers: SQSHandler/SNSHander 59 | * Improve internal error handling 60 | * Improve docs 61 | 62 | 1.0.2 (2017-04-13) 63 | ------------------ 64 | 65 | * Fix sentry error handler integration 66 | 67 | 1.0.1 (2017-04-09) 68 | ------------------ 69 | 70 | * Add tox and execute tests for py36 71 | * Update aiohttp/aiobotocore versions 72 | * Minor fixes and enhancements 73 | 74 | 75 | 1.0.0 (2017-03-27) 76 | ------------------ 77 | 78 | * Major code rewrite 79 | * Remove CLI 80 | * Add better support for error handlers, including sentry/raven 81 | * Refactor exceptions 82 | * Add message metadata information 83 | * Update message lifecycle with handler/error handler return value 84 | * Enable execution of one service iteration (by default, it still runs "forever") 85 | 86 | 87 | 0.0.3 (2016-04-24) 88 | ------------------ 89 | 90 | * Improve documentation 91 | * Improve package metadata and dependencies 92 | * Add loafer.aws.message_translator.SNSMessageTranslator class 93 | * Fix ImportError exceptions for configuration that uses loafer.utils.import_callable 94 | 95 | 96 | 0.0.2 (2016-04-18) 97 | ------------------ 98 | 99 | * Fix build hardcoding tests dependencies 100 | 101 | 102 | 0.0.1 (2016-04-18) 103 | ------------------ 104 | 105 | * Initial release 106 | -------------------------------------------------------------------------------- /CONTRIBUTORS.rst: -------------------------------------------------------------------------------- 1 | Thanks to: 2 | ---------- 3 | 4 | * allisson 5 | * cleberzavadniak 6 | * danilo shiga 7 | * lamenezes 8 | * luizdepra 9 | * wiliamsouza 10 | * GuilhermeVBeira 11 | * hartungstenio 12 | -------------------------------------------------------------------------------- /Dockerfile.local: -------------------------------------------------------------------------------- 1 | FROM python:3.7-slim-buster 2 | 3 | WORKDIR /loafer 4 | COPY . /loafer 5 | 6 | RUN pip install awscli==1.18.32 7 | RUN pip install -e . 8 | 9 | ENTRYPOINT ["examples/entrypoint.sh"] 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 George Kussumoto 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include CHANGES.rst 4 | include CONTRIBUTORS.rst 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean-pyc 2 | 3 | default: test 4 | 5 | clean-pyc: 6 | @find . -iname '*.py[co]' -delete 7 | @find . -iname '__pycache__' -delete 8 | @find . -iname '.coverage' -delete 9 | @rm -rf htmlcov/ 10 | 11 | clean-dist: 12 | @rm -rf dist/ 13 | @rm -rf build/ 14 | @rm -rf *.egg-info 15 | 16 | clean: clean-pyc clean-dist 17 | 18 | test: 19 | pytest -vv tests 20 | 21 | test-cov: 22 | pytest -vv --cov=loafer tests 23 | 24 | cov: 25 | coverage report -m 26 | 27 | cov-report: 28 | pytest -vv --cov=loafer --cov-report=html tests 29 | 30 | check-fixtures: 31 | pytest --dead-fixtures 32 | 33 | dist: clean 34 | python setup.py sdist 35 | python setup.py bdist_wheel 36 | 37 | release: dist 38 | git tag `python setup.py -q version` 39 | git push origin `python setup.py -q version` 40 | twine upload dist/* 41 | 42 | changelog-preview: 43 | @echo "\nmaster ("$$(date '+%Y-%m-%d')")" 44 | @echo "-------------------\n" 45 | @git log $$(python setup.py -q version)...master --oneline --reverse 46 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Loafer 2 | ====== 3 | 4 | |PyPI latest| |PyPI Version| |PyPI License| |Docs| 5 | 6 | |CI Build Status| |Coverage Status| |Requirements Status| 7 | |Scrutinizer Code Quality| |Code Climate| 8 | 9 | ---- 10 | 11 | Loafer is an asynchronous message dispatcher for concurrent tasks processing. 12 | 13 | **Currently, only AWS SQS is supported** 14 | 15 | 16 | Features: 17 | 18 | * Encourages decoupling from message providers and consumers 19 | * Easy to extend and customize 20 | * Easy error handling, including integration with sentry 21 | * Easy to create one or multiple services 22 | * Generic Handlers 23 | * Amazon SQS integration 24 | 25 | 26 | It requires Python 3.6+ and is very experimental at the moment, expect a lot 27 | of changes until the first major version. 28 | 29 | 30 | Example 31 | ~~~~~~~ 32 | 33 | A simple message forwader, from ``source-queue`` to ``destination-queue``: 34 | 35 | .. code:: python 36 | 37 | from loafer.ext.aws.handlers import SQSHandler 38 | from loafer.ext.aws.routes import SQSRoute 39 | from loafer.managers import LoaferManager 40 | 41 | 42 | routes = [ 43 | SQSRoute('source-queue', handler=SQSHandler('destination-queue')), 44 | ] 45 | 46 | 47 | if __name__ == '__main__': 48 | manager = LoaferManager(routes) 49 | manager.run() 50 | 51 | 52 | Documentation 53 | ~~~~~~~~~~~~~ 54 | 55 | Check out the latest **Loafer** full documentation at `Read the Docs`_ website. 56 | 57 | 58 | .. _`Read the Docs`: http://loafer.readthedocs.org/ 59 | 60 | 61 | 62 | .. |Docs| image:: https://readthedocs.org/projects/loafer/badge/?version=latest 63 | :target: http://loafer.readthedocs.org/en/latest/?badge=latest 64 | .. |CI Build Status| image:: https://circleci.com/gh/georgeyk/loafer.svg?style=svg 65 | :target: https://circleci.com/gh/georgeyk/loafer 66 | .. |Coverage Status| image:: https://codecov.io/gh/georgeyk/loafer/branch/master/graph/badge.svg 67 | :target: https://codecov.io/gh/georgeyk/loafer 68 | .. |Requirements Status| image:: https://requires.io/github/georgeyk/loafer/requirements.svg?branch=master 69 | :target: https://requires.io/github/georgeyk/loafer/requirements/?branch=master 70 | .. |Scrutinizer Code Quality| image:: https://scrutinizer-ci.com/g/georgeyk/loafer/badges/quality-score.png?b=master 71 | :target: https://scrutinizer-ci.com/g/georgeyk/loafer/?branch=master 72 | .. |Code Climate| image:: https://codeclimate.com/github/georgeyk/loafer/badges/gpa.svg 73 | :target: https://codeclimate.com/github/georgeyk/loafer 74 | .. |PyPI Version| image:: https://img.shields.io/pypi/pyversions/loafer.svg?maxAge=2592000 75 | :target: https://pypi.python.org/pypi/loafer 76 | .. |PyPI License| image:: https://img.shields.io/pypi/l/loafer.svg?maxAge=2592000 77 | :target: https://pypi.python.org/pypi/loafer 78 | .. |PyPI latest| image:: https://img.shields.io/pypi/v/loafer.svg?maxAge=2592000 79 | :target: https://pypi.python.org/pypi/loafer 80 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2.0" 2 | services: 3 | echo: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile.local 7 | environment: 8 | AWS_DEFAULT_REGION: us-east-1 9 | AWS_ACCESS_KEY_ID: foobar 10 | AWS_SECRET_ACCESS_KEY: foobar 11 | AWS_ENDPOINT_URL: http://goaws:4100 12 | GOAWS_URL: http://goaws:4100 13 | depends_on: 14 | - goaws 15 | 16 | goaws: 17 | image: pafortin/goaws 18 | ports: 19 | - "4100:4100" 20 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source 21 | 22 | .PHONY: help 23 | help: 24 | @echo "Please use \`make ' where is one of" 25 | @echo " html to make standalone HTML files" 26 | @echo " dirhtml to make HTML files named index.html in directories" 27 | @echo " singlehtml to make a single large HTML file" 28 | @echo " pickle to make pickle files" 29 | @echo " json to make JSON files" 30 | @echo " htmlhelp to make HTML files and a HTML help project" 31 | @echo " qthelp to make HTML files and a qthelp project" 32 | @echo " applehelp to make an Apple Help Book" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " epub3 to make an epub3" 36 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 37 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 38 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 39 | @echo " text to make text files" 40 | @echo " man to make manual pages" 41 | @echo " texinfo to make Texinfo files" 42 | @echo " info to make Texinfo files and run them through makeinfo" 43 | @echo " gettext to make PO message catalogs" 44 | @echo " changes to make an overview of all changed/added/deprecated items" 45 | @echo " xml to make Docutils-native XML files" 46 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 47 | @echo " linkcheck to check all external links for integrity" 48 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 49 | @echo " coverage to run coverage check of the documentation (if enabled)" 50 | 51 | .PHONY: clean 52 | clean: 53 | rm -rf $(BUILDDIR)/* 54 | 55 | .PHONY: html 56 | html: 57 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 58 | @echo 59 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 60 | 61 | .PHONY: dirhtml 62 | dirhtml: 63 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 64 | @echo 65 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 66 | 67 | .PHONY: singlehtml 68 | singlehtml: 69 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 70 | @echo 71 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 72 | 73 | .PHONY: pickle 74 | pickle: 75 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 76 | @echo 77 | @echo "Build finished; now you can process the pickle files." 78 | 79 | .PHONY: json 80 | json: 81 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 82 | @echo 83 | @echo "Build finished; now you can process the JSON files." 84 | 85 | .PHONY: htmlhelp 86 | htmlhelp: 87 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 88 | @echo 89 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 90 | ".hhp project file in $(BUILDDIR)/htmlhelp." 91 | 92 | .PHONY: qthelp 93 | qthelp: 94 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 95 | @echo 96 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 97 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 98 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Loafer.qhcp" 99 | @echo "To view the help file:" 100 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Loafer.qhc" 101 | 102 | .PHONY: applehelp 103 | applehelp: 104 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 105 | @echo 106 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 107 | @echo "N.B. You won't be able to view it unless you put it in" \ 108 | "~/Library/Documentation/Help or install it in your application" \ 109 | "bundle." 110 | 111 | .PHONY: devhelp 112 | devhelp: 113 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 114 | @echo 115 | @echo "Build finished." 116 | @echo "To view the help file:" 117 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Loafer" 118 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Loafer" 119 | @echo "# devhelp" 120 | 121 | .PHONY: epub 122 | epub: 123 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 124 | @echo 125 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 126 | 127 | .PHONY: epub3 128 | epub3: 129 | $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 130 | @echo 131 | @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." 132 | 133 | .PHONY: latex 134 | latex: 135 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 136 | @echo 137 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 138 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 139 | "(use \`make latexpdf' here to do that automatically)." 140 | 141 | .PHONY: latexpdf 142 | latexpdf: 143 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 144 | @echo "Running LaTeX files through pdflatex..." 145 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 146 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 147 | 148 | .PHONY: latexpdfja 149 | latexpdfja: 150 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 151 | @echo "Running LaTeX files through platex and dvipdfmx..." 152 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 153 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 154 | 155 | .PHONY: text 156 | text: 157 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 158 | @echo 159 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 160 | 161 | .PHONY: man 162 | man: 163 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 164 | @echo 165 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 166 | 167 | .PHONY: texinfo 168 | texinfo: 169 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 170 | @echo 171 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 172 | @echo "Run \`make' in that directory to run these through makeinfo" \ 173 | "(use \`make info' here to do that automatically)." 174 | 175 | .PHONY: info 176 | info: 177 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 178 | @echo "Running Texinfo files through makeinfo..." 179 | make -C $(BUILDDIR)/texinfo info 180 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 181 | 182 | .PHONY: gettext 183 | gettext: 184 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 185 | @echo 186 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 187 | 188 | .PHONY: changes 189 | changes: 190 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 191 | @echo 192 | @echo "The overview file is in $(BUILDDIR)/changes." 193 | 194 | .PHONY: linkcheck 195 | linkcheck: 196 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 197 | @echo 198 | @echo "Link check complete; look for any errors in the above output " \ 199 | "or in $(BUILDDIR)/linkcheck/output.txt." 200 | 201 | .PHONY: doctest 202 | doctest: 203 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 204 | @echo "Testing of doctests in the sources finished, look at the " \ 205 | "results in $(BUILDDIR)/doctest/output.txt." 206 | 207 | .PHONY: coverage 208 | coverage: 209 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 210 | @echo "Testing of coverage in the sources finished, look at the " \ 211 | "results in $(BUILDDIR)/coverage/python.txt." 212 | 213 | .PHONY: xml 214 | xml: 215 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 216 | @echo 217 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 218 | 219 | .PHONY: pseudoxml 220 | pseudoxml: 221 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 222 | @echo 223 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 224 | -------------------------------------------------------------------------------- /docs/source/_static/sequence_loafer.diag: -------------------------------------------------------------------------------- 1 | seqdiag { 2 | Provider; Dispatcher; Route; Handler; "Error Handler"; 3 | default_fontsize = 12; 4 | 5 | Dispatcher -> Provider [label = "ask messages"]; 6 | Dispatcher <-- Provider [label = "return messages"]; 7 | Dispatcher -> Route [label = "ask for message delivery"]; 8 | Route -> Route [label = "apply message translation", rightnote = "in case of error, follow 'error processing'"]; 9 | Route <-- Route; 10 | Route -> Handler [label = "deliver message"]; 11 | Route <-- Handler [label = "confirm message"]; 12 | Route <-- Handler [label = "error", color = "red"]; 13 | Route -> "Error Handler" [label = "error processing", color = "red"]; 14 | Route <-- "Error Handler" [label = "message confirmation from error", color = "red"]; 15 | Dispatcher <-- Route [label = "message confirmation"]; 16 | Dispatcher -> Provider [label = "confirm message (deletion)"]; 17 | Dispatcher <-- Provider 18 | } 19 | -------------------------------------------------------------------------------- /docs/source/_static/sequence_loafer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/georgeyk/loafer/e878f27897776362f6661d31845f756ced7be711/docs/source/_static/sequence_loafer.png -------------------------------------------------------------------------------- /docs/source/changelog.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | .. include:: ../../CHANGES.rst 5 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Loafer documentation build configuration file, created by 5 | # sphinx-quickstart on Tue Apr 12 03:26:33 2016. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import os 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ------------------------------------------------ 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be 29 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 30 | # ones. 31 | extensions = [] 32 | 33 | # Add any paths that contain templates here, relative to this directory. 34 | templates_path = ['_templates'] 35 | 36 | # The suffix(es) of source filenames. 37 | # You can specify multiple suffix as a list of string: 38 | # source_suffix = ['.rst', '.md'] 39 | source_suffix = '.rst' 40 | 41 | # The encoding of source files. 42 | #source_encoding = 'utf-8-sig' 43 | 44 | # The master toctree document. 45 | master_doc = 'index' 46 | 47 | # General information about the project. 48 | project = 'Loafer' 49 | copyright = '2016-2020, George Y. Kussumoto' 50 | author = 'George Y. Kussumoto' 51 | 52 | # The version info for the project you're documenting, acts as replacement for 53 | # |version| and |release|, also used in various other places throughout the 54 | # built documents. 55 | # 56 | 57 | # The short X.Y version. 58 | version = '2.0.1' 59 | # The full version, including alpha/beta/rc tags. 60 | release = '2.0.1' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = None 68 | 69 | # There are two options for replacing |today|: either, you set today to some 70 | # non-false value, then it is used: 71 | #today = '' 72 | # Else, today_fmt is used as the format for a strftime call. 73 | #today_fmt = '%B %d, %Y' 74 | 75 | # List of patterns, relative to source directory, that match files and 76 | # directories to ignore when looking for source files. 77 | # This patterns also effect to html_static_path and html_extra_path 78 | exclude_patterns = [] 79 | 80 | # The reST default role (used for this markup: `text`) to use for all 81 | # documents. 82 | #default_role = None 83 | 84 | # If true, '()' will be appended to :func: etc. cross-reference text. 85 | #add_function_parentheses = True 86 | 87 | # If true, the current module name will be prepended to all description 88 | # unit titles (such as .. function::). 89 | #add_module_names = True 90 | 91 | # If true, sectionauthor and moduleauthor directives will be shown in the 92 | # output. They are ignored by default. 93 | #show_authors = False 94 | 95 | # The name of the Pygments (syntax highlighting) style to use. 96 | pygments_style = 'sphinx' 97 | 98 | # A list of ignored prefixes for module index sorting. 99 | #modindex_common_prefix = [] 100 | 101 | # If true, keep warnings as "system message" paragraphs in the built documents. 102 | #keep_warnings = False 103 | 104 | # If true, `todo` and `todoList` produce output, else they produce nothing. 105 | todo_include_todos = False 106 | 107 | 108 | # -- Options for HTML output ---------------------------------------------- 109 | 110 | # The theme to use for HTML and HTML Help pages. See the documentation for 111 | # a list of builtin themes. 112 | on_rtd = os.environ.get('READTHEDOCS', None) == 'True' 113 | if not on_rtd: 114 | import sphinx_rtd_theme 115 | html_theme = 'sphinx_rtd_theme' 116 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 117 | 118 | # Theme options are theme-specific and customize the look and feel of a theme 119 | # further. For a list of options available for each theme, see the 120 | # documentation. 121 | #html_theme_options = {} 122 | 123 | # Add any paths that contain custom themes here, relative to this directory. 124 | #html_theme_path = [] 125 | 126 | # The name for this set of Sphinx documents. 127 | # " v documentation" by default. 128 | #html_title = 'Loafer v0.0.1' 129 | 130 | # A shorter title for the navigation bar. Default is the same as html_title. 131 | #html_short_title = None 132 | 133 | # The name of an image file (relative to this directory) to place at the top 134 | # of the sidebar. 135 | #html_logo = None 136 | 137 | # The name of an image file (relative to this directory) to use as a favicon of 138 | # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 139 | # pixels large. 140 | #html_favicon = None 141 | 142 | # Add any paths that contain custom static files (such as style sheets) here, 143 | # relative to this directory. They are copied after the builtin static files, 144 | # so a file named "default.css" will overwrite the builtin "default.css". 145 | html_static_path = ['_static'] 146 | 147 | # Add any extra paths that contain custom files (such as robots.txt or 148 | # .htaccess) here, relative to this directory. These files are copied 149 | # directly to the root of the documentation. 150 | #html_extra_path = [] 151 | 152 | # If not None, a 'Last updated on:' timestamp is inserted at every page 153 | # bottom, using the given strftime format. 154 | # The empty string is equivalent to '%b %d, %Y'. 155 | #html_last_updated_fmt = None 156 | 157 | # If true, SmartyPants will be used to convert quotes and dashes to 158 | # typographically correct entities. 159 | #html_use_smartypants = True 160 | 161 | # Custom sidebar templates, maps document names to template names. 162 | #html_sidebars = {} 163 | 164 | # Additional templates that should be rendered to pages, maps page names to 165 | # template names. 166 | #html_additional_pages = {} 167 | 168 | # If false, no module index is generated. 169 | #html_domain_indices = True 170 | 171 | # If false, no index is generated. 172 | #html_use_index = True 173 | 174 | # If true, the index is split into individual pages for each letter. 175 | #html_split_index = False 176 | 177 | # If true, links to the reST sources are added to the pages. 178 | #html_show_sourcelink = True 179 | 180 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 181 | #html_show_sphinx = True 182 | 183 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 184 | #html_show_copyright = True 185 | 186 | # If true, an OpenSearch description file will be output, and all pages will 187 | # contain a tag referring to it. The value of this option must be the 188 | # base URL from which the finished HTML is served. 189 | #html_use_opensearch = '' 190 | 191 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 192 | #html_file_suffix = None 193 | 194 | # Language to be used for generating the HTML full-text search index. 195 | # Sphinx supports the following languages: 196 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' 197 | # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' 198 | #html_search_language = 'en' 199 | 200 | # A dictionary with options for the search language support, empty by default. 201 | # 'ja' uses this config value. 202 | # 'zh' user can custom change `jieba` dictionary path. 203 | #html_search_options = {'type': 'default'} 204 | 205 | # The name of a javascript file (relative to the configuration directory) that 206 | # implements a search results scorer. If empty, the default will be used. 207 | #html_search_scorer = 'scorer.js' 208 | 209 | # Output file base name for HTML help builder. 210 | htmlhelp_basename = 'Loaferdoc' 211 | 212 | # -- Options for LaTeX output --------------------------------------------- 213 | 214 | latex_elements = { 215 | # The paper size ('letterpaper' or 'a4paper'). 216 | #'papersize': 'letterpaper', 217 | 218 | # The font size ('10pt', '11pt' or '12pt'). 219 | #'pointsize': '10pt', 220 | 221 | # Additional stuff for the LaTeX preamble. 222 | #'preamble': '', 223 | 224 | # Latex figure (float) alignment 225 | #'figure_align': 'htbp', 226 | } 227 | 228 | # Grouping the document tree into LaTeX files. List of tuples 229 | # (source start file, target name, title, 230 | # author, documentclass [howto, manual, or own class]). 231 | latex_documents = [ 232 | (master_doc, 'Loafer.tex', 'Loafer Documentation', 233 | 'George Y. Kussumoto', 'manual'), 234 | ] 235 | 236 | # The name of an image file (relative to this directory) to place at the top of 237 | # the title page. 238 | #latex_logo = None 239 | 240 | # For "manual" documents, if this is true, then toplevel headings are parts, 241 | # not chapters. 242 | #latex_use_parts = False 243 | 244 | # If true, show page references after internal links. 245 | #latex_show_pagerefs = False 246 | 247 | # If true, show URL addresses after external links. 248 | #latex_show_urls = False 249 | 250 | # Documents to append as an appendix to all manuals. 251 | #latex_appendices = [] 252 | 253 | # If false, no module index is generated. 254 | #latex_domain_indices = True 255 | 256 | 257 | # -- Options for manual page output --------------------------------------- 258 | 259 | # One entry per manual page. List of tuples 260 | # (source start file, name, description, authors, manual section). 261 | man_pages = [ 262 | (master_doc, 'loafer', 'Loafer Documentation', 263 | [author], 1) 264 | ] 265 | 266 | # If true, show URL addresses after external links. 267 | #man_show_urls = False 268 | 269 | 270 | # -- Options for Texinfo output ------------------------------------------- 271 | 272 | # Grouping the document tree into Texinfo files. List of tuples 273 | # (source start file, target name, title, author, 274 | # dir menu entry, description, category) 275 | texinfo_documents = [ 276 | (master_doc, 'Loafer', 'Loafer Documentation', 277 | author, 'Loafer', 'One line description of project.', 278 | 'Miscellaneous'), 279 | ] 280 | 281 | # Documents to append as an appendix to all manuals. 282 | #texinfo_appendices = [] 283 | 284 | # If false, no module index is generated. 285 | #texinfo_domain_indices = True 286 | 287 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 288 | #texinfo_show_urls = 'footnote' 289 | 290 | # If true, do not generate a @detailmenu in the "Top" node's menu. 291 | #texinfo_no_detailmenu = False 292 | -------------------------------------------------------------------------------- /docs/source/contributors.rst: -------------------------------------------------------------------------------- 1 | Contributors 2 | ============ 3 | 4 | .. include:: ../../CONTRIBUTORS.rst 5 | -------------------------------------------------------------------------------- /docs/source/development/installation.rst: -------------------------------------------------------------------------------- 1 | Development Installation 2 | ======================== 3 | 4 | Requirements 5 | ------------ 6 | 7 | Python 3.6+ 8 | 9 | Note:: 10 | 11 | Some packages also needs python3.5-dev package (ubuntu) or similar 12 | 13 | 14 | Development install 15 | ------------------- 16 | 17 | After forking or checking out:: 18 | 19 | $ cd loafer/ 20 | $ pip install -r requirements/local.txt 21 | $ pre-commit install 22 | $ pip install -e . 23 | 24 | 25 | The requirements folder are only used for development, so we can easily 26 | install/track dependencies required to run the tests using continuous 27 | integration platforms. 28 | 29 | The official entrypoint for distritubution is the ``setup.py`` which also 30 | contains the minimum requirements to execute the tests. 31 | 32 | It's important to execute ``pip install -e .`` not only to install the main 33 | dependencies, but also to include ``loafer`` in our environment. 34 | 35 | 36 | Running tests:: 37 | 38 | $ make test 39 | 40 | Generating documentation:: 41 | 42 | $ cd docs/ 43 | $ make html 44 | 45 | 46 | To configure AWS access, check `boto3 configuration`_ or export (see `boto3 envvars`_):: 47 | 48 | $ export AWS_ACCESS_KEY_ID= 49 | $ export AWS_SECRET_ACCESS_KEY= 50 | $ export AWS_DEFAULT_REGION=sa-east-1 # for example 51 | 52 | 53 | .. _boto3 configuration: https://boto3.readthedocs.org/en/latest/guide/quickstart.html#configuration 54 | .. _boto3 envvars: http://boto3.readthedocs.org/en/latest/guide/configuration.html#environment-variable-configuration 55 | 56 | Check the :doc:`../settings` section to see specific configurations. 57 | -------------------------------------------------------------------------------- /docs/source/development/release.rst: -------------------------------------------------------------------------------- 1 | Release 2 | ------- 3 | 4 | To release a new version, a few steps are required: 5 | 6 | * Update version/release number in ``docs/source/conf.py`` 7 | 8 | * Add entry to ``CHANGES.rst`` and documentation 9 | 10 | * Review changes in test requirements (``requirements/test.txt`` and ``setup.py``) 11 | 12 | * Test with ``python setup.py test`` and ``make test-cov`` 13 | 14 | * Test build with ``make dist`` 15 | 16 | * Commit changes 17 | 18 | * Release with ``make release`` 19 | -------------------------------------------------------------------------------- /docs/source/error_handlers.rst: -------------------------------------------------------------------------------- 1 | Error Handlers 2 | -------------- 3 | 4 | Every ``Route`` implements a coroutine ``error_handler`` that can be used as an error hook. 5 | This hook are called when unhandled exceptions happen and by default it will only log the 6 | error and **not** acknowledge the message. 7 | 8 | ``Route`` accepts a callable parameter, ``error_handler``, so you can pass a custom function or 9 | coroutine to replace the default error handler behavior. 10 | 11 | The callable should be similar to:: 12 | 13 | async def custom_error_handler(exc_info, message): 14 | ... custom code here ... 15 | return True 16 | 17 | # Route(..., error_handler=custom_error_handler) 18 | 19 | 20 | The return value determines if the message that originated the error will be acknowledged or not. 21 | ``True`` means acknowledge it, ``False`` will only ignore the message (default behavior). 22 | 23 | 24 | Sentry 25 | ~~~~~~ 26 | 27 | 28 | To integrate with `sentry`_ you will need the `sdk`_ client and your account DSN. 29 | 30 | Then you can automatically create an ``error_handler`` with the following code:: 31 | 32 | import sentry_sdk 33 | from loafer.ext.sentry import sentry_handler 34 | 35 | sentry_sdk.init(...) 36 | error_handler = sentry_handler(sentry_sdk, delete_message=True) 37 | 38 | 39 | The optional ``delete_message`` parameter controls the message acknowledgement 40 | after the error report. By default, ``delete_message`` is ``False``. 41 | 42 | The ``error_handler`` defined can be set on any ``Route`` instance. 43 | 44 | .. _sentry: https://sentry.io/ 45 | .. _sdk: https://github.com/getsentry/sentry-python 46 | -------------------------------------------------------------------------------- /docs/source/exceptions.rst: -------------------------------------------------------------------------------- 1 | Exceptions 2 | ---------- 3 | 4 | All exceptions are defined at ``loafer.exceptions`` module. 5 | 6 | A description of all the exceptions: 7 | 8 | 9 | * ``ProviderError``: a fatal error from a provider instance. 10 | Loafer will stop all operations and shutdown. 11 | 12 | * ``ConfigurationError``: a configuration error. Loafer will 13 | stop all operations and shutdown. 14 | 15 | * ``DeleteMessage``: if any :doc:`handlers` raises this exception, the message will 16 | be rejected and acknowledged (the message will be deleted). 17 | 18 | * ``LoaferException``: the base exception for ``DeleteMessage``. 19 | -------------------------------------------------------------------------------- /docs/source/faq.rst: -------------------------------------------------------------------------------- 1 | FAQ 2 | --- 3 | 4 | 5 | **1. How do I run I/O blocking code that's not a coroutine ?** 6 | 7 | Any code that is blocking and not a coroutine could run in a separate thread. 8 | 9 | It's not recommended, but it looks like this:: 10 | 11 | import asyncio 12 | loop = asyncio.get_event_loop() 13 | loop.run_in_executor(None, your_callable, your_callable_args) 14 | # Important: do not close/stop the loop 15 | 16 | 17 | **2. How to integrate with newrelic ?** 18 | 19 | The `newrelic`_ should be the primary source of information. 20 | One alternative is to use environment variables ``NEW_RELIC_LICENSE_KEY`` and 21 | ``NEW_RELIC_APP_NAME`` and for every handler:: 22 | 23 | import newrelic.agent 24 | 25 | @newrelic.agent.background_task() 26 | def some_code(...): 27 | ... 28 | 29 | 30 | **3. Using different regions/credentials with SQSRoute/SNSRoute ?** 31 | 32 | ``SQSRoute``/``SNSRoute`` instantiates ``loafer.ext.aws.providers.SQSProvider``, 33 | therefore you can dinamically set these options to any of :doc:`providers` available. 34 | 35 | An example with explicity AWS credentials and provider options would look like:: 36 | 37 | from loafer.ext.aws.routes import SQSRoute 38 | 39 | route = SQSRoute( 40 | 'test-queue-name', name='my-route', handler=some_handler, 41 | provider_options={ 42 | 'aws_access_key_id': my_aws_access_key, 43 | 'aws_secret_access_key': my_secret_key, 44 | 'region_name': 'sa-east-1', 45 | 'options': {'WaitTimeSeconds': 3}, 46 | }, 47 | ) 48 | 49 | .. _newrelic: https://docs.newrelic.com/docs/agents/python-agent/getting-started/introduction-new-relic-python 50 | -------------------------------------------------------------------------------- /docs/source/generic_handlers.rst: -------------------------------------------------------------------------------- 1 | Generic Handlers 2 | ----------------- 3 | 4 | Here are some handlers that are easy to extend or use directly. 5 | 6 | 7 | loafer.ext.aws.handlers.SQSHandler 8 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 9 | 10 | A handler that publishes messages to a SQS queue. 11 | 12 | For instance, if we just want just redirect a message to a queue named "my-queue":: 13 | 14 | # in your route definition 15 | from loafer.ext.aws.handlers import SQSHandler 16 | 17 | Route(handler=SQSHandler('my-queue'), ...) 18 | 19 | The handler assumes a message that could be json encoded (usually a python ``dict`` instance). 20 | 21 | You can customize this handler by subclassing it:: 22 | 23 | class MyHandler(SQSHandler): 24 | queue_name = 'my-queue' 25 | 26 | async def handle(self, message, *args): 27 | text = message['text'] 28 | return await self.publish(text, encoder=None) 29 | 30 | In the example above, we are disabling the message encoding by passing ``None`` 31 | to the ``publish`` coroutine. 32 | 33 | The ``encoder`` parameter should be a callable that receives the message to be encoded. 34 | By default, it assumes ``json.dumps``. 35 | 36 | Take a note how **queue_name** was set. You can configure it when instantiate 37 | the handler or set the class attribute **queue_name**, both are valid and the 38 | attribute is mandatory. You can also use the queue URL directly, if you prefer. 39 | 40 | 41 | loafer.ext.aws.handlers.SNSHandler 42 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 43 | 44 | A handler that publishes messages to a SNS topic. 45 | 46 | This handler is very similar to **SQSHandler**, so going to a similar example:: 47 | 48 | from loafer.ext.aws.handlers import SNSHandler 49 | 50 | class MyHandler(SNSHandler): 51 | topic = 'my-topic' 52 | 53 | async def handle(self, message, *args): 54 | text = message['text'] 55 | return await self.publish(text, encoder=None) 56 | 57 | The handler also provides a ``publish`` coroutine with an ``encoder`` parameter 58 | that works in the same way as with **SQSHandler**, except it will publish in a 59 | SNS topic instead a queue. 60 | 61 | The **SNSHandler** also assumes a message that could be json encoded and the 62 | encoder default to ``json.dumps``. 63 | 64 | The **topic** is also mandatory and should be configured in the class 65 | definition or when creating the handler instance. 66 | 67 | You can set either the topic name or the topic arn (recommended), but when 68 | using the topic name the handler will use ``wildcards`` to match the topic arn. 69 | 70 | More details about SNS wildcards are available in their `documentation`_. 71 | 72 | .. _documentation: http://docs.aws.amazon.com/sns/latest/dg/UsingIAMwithSNS.html#SNS_ARN_Format 73 | -------------------------------------------------------------------------------- /docs/source/handlers.rst: -------------------------------------------------------------------------------- 1 | Handlers 2 | -------- 3 | 4 | Handlers are callables that receives the message. 5 | 6 | The ``handler`` signature should be similar to:: 7 | 8 | async def my_handler(message, metadata): 9 | ... code ... 10 | return True # or False 11 | 12 | Where ``message`` is the message to be processed and ``metadata`` is a ``dict`` 13 | with metadata information. 14 | 15 | The ``async def`` is the python coroutine syntax, but regular functions 16 | can also be used, but will run in a thread, outside the event loop. 17 | 18 | The return value indicates if the ``handler`` successfully processed the 19 | message or not. 20 | By returning ``True`` the message will be acknowledged (deleted). 21 | 22 | Another way to acknowledge messages inside a handler is to raise 23 | ``DeleteMessage`` exception. 24 | 25 | Any other exception will be redirected to an ``error_handler``, see more 26 | :doc:`error_handlers`. 27 | 28 | The default ``error_handler`` will log the error and **not** acknowledge the message. 29 | 30 | For some generic handlers that can give you a starting point, take a look at 31 | :doc:`generic_handlers` section. 32 | 33 | 34 | Class-based handlers 35 | ~~~~~~~~~~~~~~~~~~~~ 36 | 37 | 38 | You can also write handlers using classes. The class should implement a 39 | ``handle`` coroutine/method:: 40 | 41 | class MyHandler: 42 | 43 | async def handle(self, message, *args): 44 | ... code ... 45 | return True 46 | 47 | def stop(self): 48 | ... clean-up code ... 49 | 50 | 51 | The method ``stop`` is optional and will be called before loafer shutdown it's 52 | execution. Note that ``stop`` is not a coroutine. 53 | 54 | When configuring your :doc:`routes`, you can set ``handler`` to an instance of 55 | ``MyHandler`` instead of the ``handle`` (the callable) method (but both ways work):: 56 | 57 | Route(handler=MyHandler(), ...) 58 | # or 59 | Route(handler=MyHandler().handle) 60 | 61 | 62 | Message dependency 63 | ~~~~~~~~~~~~~~~~~~ 64 | 65 | Handlers are supposed to be stateless or have limited dependency on message values. 66 | Since the same handler instance object are used to process the incoming messages, 67 | we can't guarantee that an attached value will be kept among several concurrent 68 | calls to the same handler. 69 | 70 | This might be hard to detect in production and probably is an undesired side-effect:: 71 | 72 | class Handler: 73 | 74 | async def foo(self): 75 | # do something with `self.some_value` 76 | print(self.some_value) 77 | ... code ... 78 | 79 | async def handle(self, message, *args): 80 | self.some_value = message['foo'] 81 | await self.foo() 82 | return True 83 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. Loafer documentation master file, created by 2 | sphinx-quickstart on Tue Apr 12 03:26:33 2016. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to Loafer's documentation! 7 | ================================== 8 | 9 | Loafer is an asynchronous message dispatcher for concurrent tasks processing. 10 | 11 | 12 | Quickstart 13 | ---------- 14 | 15 | .. toctree:: 16 | :maxdepth: 1 17 | 18 | quickstart/installation.rst 19 | tutorial.rst 20 | 21 | 22 | User Guide 23 | ---------- 24 | 25 | .. toctree:: 26 | :maxdepth: 1 27 | 28 | overview.rst 29 | settings.rst 30 | exceptions.rst 31 | handlers.rst 32 | error_handlers.rst 33 | generic_handlers.rst 34 | message_translators.rst 35 | providers.rst 36 | routes.rst 37 | managers.rst 38 | 39 | 40 | Development 41 | ----------- 42 | 43 | .. toctree:: 44 | :maxdepth: 1 45 | 46 | development/installation.rst 47 | development/release.rst 48 | 49 | 50 | Other 51 | ----- 52 | 53 | .. toctree:: 54 | :maxdepth: 2 55 | 56 | faq.rst 57 | changelog.rst 58 | contributors.rst 59 | -------------------------------------------------------------------------------- /docs/source/managers.rst: -------------------------------------------------------------------------------- 1 | Managers 2 | -------- 3 | 4 | Managers are responsible for the execution of all the components. 5 | 6 | The ``LoaferManager`` (at ``loafer.managers``) receives a ``list`` of :doc:`routes`. 7 | 8 | Every service/application using ``loafer`` should instantiate a manager:: 9 | 10 | from loafer.managers import LoaferManager 11 | from .routes import routes # the list of routes 12 | 13 | manager = LoaferManager(routes=routes) 14 | manager.run() 15 | 16 | 17 | The default execution mode will run indefinitely. 18 | To run only one iteration of your services, the last line in the code above 19 | should be replaced with:: 20 | 21 | manager.run(forever=False) 22 | 23 | 24 | The "one iteration" could be a little tricky. For example, if you have one 25 | provider that fetches two messages at time, it means your handler will be called 26 | twice (one for each message) and then stop. 27 | -------------------------------------------------------------------------------- /docs/source/message_translators.rst: -------------------------------------------------------------------------------- 1 | Message Translators 2 | ------------------- 3 | 4 | The message translator receives a "raw message" and process it to a suitable 5 | format expected by the ``handler``. 6 | 7 | The "raw message" is the message received by the ``provider`` "as-is" and 8 | it might be delivered without any processing if the message translator was 9 | not set. 10 | 11 | In some cases, you should explicitly set ``message_translator=None`` to disable 12 | any configured translators. 13 | 14 | 15 | Implementation 16 | ~~~~~~~~~~~~~~ 17 | 18 | The message translator class should subclass ``AbstractMessageTranslator`` and 19 | implement the ``translate`` method like:: 20 | 21 | 22 | from loafer.message_translators import AbstractMessageTranslator 23 | 24 | 25 | class MyMessageTranslator(AbstractMessageTranslator): 26 | 27 | def translate(self, message): 28 | return {'content': int(message), 'metadata': {}} 29 | 30 | 31 | And it should return a dictionary in the format:: 32 | 33 | {'content': processed_message, 'metadata': {}} 34 | 35 | 36 | The ``processed_message`` and ``metadata`` (optional) will be delivered to 37 | ``handler``. 38 | 39 | If ``processed_message`` is ``None`` (or empty) the message will cause 40 | ``ValueError`` exception. 41 | 42 | All the exceptions in message translation will be caught by the configured 43 | :doc:`error_handlers`. 44 | 45 | The existing message translators are described below. 46 | 47 | 48 | loafer.message_translators.StringMessageTranslator 49 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 50 | 51 | A message translator that translates the given message to a string (python `str`). 52 | 53 | 54 | loafer.ext.aws.message_translators.SQSMessageTranslator 55 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 56 | 57 | A message translator that translates SQS messages. The expected message body 58 | is a **json** payload that will be decoded with ``json.loads``. 59 | 60 | All the keys will be kept in ``metadata`` key ``dict`` (except ``Body`` 61 | that was previously translated). 62 | 63 | 64 | loafer.ext.aws.message_translators.SNSMessageTranslator 65 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 66 | 67 | A message translator that translates SQS messages that came from SNS topic. 68 | The expected notification message is a **json** payload that will be decoded 69 | with ``json.loads``. 70 | 71 | SNS notifications wraps (and encodes) the message inside the body of a SQS 72 | message, so the ``SQSMessageTranslator`` will fail to properly 73 | translate those messages (or at least, fail to translate to the expected format). 74 | 75 | 76 | All the keys will be kept in ``metadata`` key ``dict`` (except ``Body``). 77 | 78 | 79 | For more details about message translators usage, check the :doc:`routes` examples. 80 | -------------------------------------------------------------------------------- /docs/source/overview.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | -------- 3 | 4 | Loafer is an asynchronous message dispatcher for concurrent tasks processing. 5 | 6 | To take full advantage, your tasks: 7 | 8 | 1. Should use asyncio 9 | 2. Should be I/O bounded 10 | 3. Should be decoupled from the message producer 11 | 12 | If your task are CPU bounded, you should look for projects that use 13 | ``multiprocessing`` python module or similar. 14 | 15 | We don't require the use of asyncio, but any code that's not a coroutine 16 | will run in thread, outside the event loop. Performance and error handling 17 | might be compromised in this scenarios, you might want to look for other 18 | alternatives. 19 | 20 | If your code are too tied to the message producer, you might end up writing 21 | too much boilerplate code in order to use ``Loafer``. 22 | 23 | 24 | Components 25 | ~~~~~~~~~~ 26 | 27 | 28 | The main components inside Loafer are: 29 | 30 | * **Manager** 31 | 32 | The manager is responsible to setup the event event loop and handle system errors. 33 | 34 | It prepares everything needed to run and starts the dispatcher. 35 | 36 | 37 | * **Dispatcher** 38 | 39 | The dispatcher starts the providers, schedules the message routing and message acknowledgment. 40 | 41 | 42 | * **Provider** 43 | 44 | The provider is responsible for retrieving messages and delete it when requested. 45 | 46 | The act is deleting a message is also known as message acknowledgment. 47 | 48 | At the moment, we only have provider for AWS SQS service. 49 | 50 | 51 | * **Message Translator** 52 | 53 | The message translator is the contract between provider and handler. 54 | 55 | At the moment, the message translator receives the "raw message" and 56 | transform it to an appropriate format that is expected by the handler. 57 | 58 | It may also add metadata information, if available. 59 | 60 | 61 | * **Route** 62 | 63 | The route is the link between the provider and handler. It is responsible 64 | to deliver the message to handler and receive its confirmation. 65 | 66 | 67 | * **Handler** 68 | 69 | Handler or task/job, the callable that will receive the message. 70 | 71 | 72 | **Error Handler** 73 | 74 | The optional callback to handle any errors in message translation or in 75 | the handler processing. 76 | 77 | 78 | The message lifecycle 79 | ~~~~~~~~~~~~~~~~~~~~~ 80 | 81 | A simplified view of a message lifecycle is illustrated below: 82 | 83 | .. image:: _static/sequence_loafer.png 84 | -------------------------------------------------------------------------------- /docs/source/providers.rst: -------------------------------------------------------------------------------- 1 | Providers 2 | --------- 3 | 4 | Providers are responsible to retrieve messages and acknowledge it 5 | (delete from source). 6 | 7 | 8 | SQSProvider 9 | ~~~~~~~~~~~ 10 | 11 | 12 | ``SQSProvider`` (located at ``loafer.ext.aws.providers``) receives the following options: 13 | 14 | * ``queue_name``: the queue name 15 | * ``options``: (optional): a ``dict`` with SQS options to retrieve messages. 16 | Example: ``{'WaitTimeSeconds: 5, 'MaxNumberOfMessages': 5}`` 17 | 18 | Also, you might override any of the parameters below from boto library (all optional): 19 | 20 | * ``api_version`` 21 | * ``aws_access_key_id`` 22 | * ``aws_secret_access_key`` 23 | * ``aws_session_token`` 24 | * ``endpoint_url`` 25 | * ``region_name`` 26 | * ``use_ssl`` 27 | * ``verify`` 28 | 29 | Check `boto3 client`_ documentation for detailed information. 30 | 31 | Usually, the provider are not configured manually, but set by :doc:`routes` and 32 | it's helper classes. 33 | 34 | .. _boto3 client: http://boto3.readthedocs.io/en/latest/reference/core/session.html#boto3.session.Session.client 35 | -------------------------------------------------------------------------------- /docs/source/quickstart/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | Requirements 5 | ------------ 6 | 7 | Python 3.6+ 8 | 9 | Note:: 10 | 11 | Some packages also needs python3.5-dev package (ubuntu) or similar 12 | 13 | 14 | To install via pip:: 15 | 16 | $ pip install loafer 17 | 18 | 19 | Basic configuration 20 | ------------------- 21 | 22 | 23 | To configure AWS access, check `boto3 configuration`_ or export (see `boto3 envvars`_):: 24 | 25 | $ export AWS_ACCESS_KEY_ID= 26 | $ export AWS_SECRET_ACCESS_KEY= 27 | $ export AWS_DEFAULT_REGION=sa-east-1 # for example 28 | 29 | You might find some configuration tips in our :doc:`../faq` as well. 30 | 31 | 32 | .. _boto3 configuration: https://boto3.readthedocs.org/en/latest/guide/quickstart.html#configuration 33 | .. _boto3 envvars: http://boto3.readthedocs.org/en/latest/guide/configuration.html#environment-variable-configuration 34 | -------------------------------------------------------------------------------- /docs/source/routes.rst: -------------------------------------------------------------------------------- 1 | Routes 2 | ------ 3 | 4 | A ``Route`` aggregate all the main entities previously described, the generic parameters are: 5 | 6 | * ``provider``: a provider instance 7 | * ``handler``: a handler instance 8 | * ``error_handler`` (optional): an error handler instance 9 | * ``message_translator`` (optional): a message translator instance 10 | * ``name`` (optional): a name for this route 11 | 12 | 13 | We provide some helper routes, so you don't need to setup all this boilerplate code: 14 | 15 | * ``loafer.ext.aws.routes.SQSRoute``: a route that configures a 16 | ``loafer.ext.aws.providers.SQSProvider`` and 17 | ``loafer.ext.aws.message_translators.SQSMessageTranslator``. 18 | 19 | A route for handlers that consume messages from SQS queue (expects json format messages). 20 | 21 | * ``loafer.ext.aws.routes.SNSQueueRoute``: a route that configures a 22 | ``loafer.ext.aws.providers.SQSProvider`` and 23 | ``loafer.ext.aws.message_translators.SNSMessageTranslator``. 24 | 25 | A route for handlers that consume messages from a SQS queue subscribed to 26 | a SNS topic (expects json format messages). 27 | 28 | 29 | Examples 30 | ~~~~~~~~ 31 | 32 | Some examples of route creation:: 33 | 34 | from loafer.ext.aws.routes import SQSRoute, SNSQueueRoute 35 | from loafer.message_translators import StringMessageTranslator 36 | 37 | 38 | # regular route 39 | route1 = SQSRoute('my-queue1', handler=..., name='route1') 40 | 41 | # route with custom SQSProvider parameters 42 | route2 = SNSQueueRoute('my-queue2', {'use_ssl': False, 'options': {'WaitTimeSeconds': 4}}, handler=..., name='route2') 43 | 44 | # route with custom message translator 45 | route3 = SQSRoute('my-queue3', message_translator=StringMessageTranslator(), handler=...) 46 | 47 | # route disabling message translation 48 | route4 = SNSQueueRoute('my-queue4', message_translator=None, handler=...) 49 | 50 | # route with custom error handler 51 | route5 = SQSRoute('my-queue5', handler=..., error_handler=custom_error_handler) 52 | -------------------------------------------------------------------------------- /docs/source/settings.rst: -------------------------------------------------------------------------------- 1 | Settings 2 | -------- 3 | 4 | 5 | AWS 6 | ~~~ 7 | 8 | To configure AWS access, check `boto3 configuration`_ or export (see `boto3 envvars`_):: 9 | 10 | $ export AWS_ACCESS_KEY_ID= 11 | $ export AWS_SECRET_ACCESS_KEY= 12 | $ export AWS_DEFAULT_REGION=sa-east-1 # for example 13 | 14 | 15 | You might find some configuration tips in our :doc:`faq` as well. 16 | 17 | 18 | .. _boto3 configuration: https://boto3.readthedocs.org/en/latest/guide/quickstart.html#configuration 19 | .. _boto3 envvars: http://boto3.readthedocs.org/en/latest/guide/configuration.html#environment-variable-configuration 20 | -------------------------------------------------------------------------------- /docs/source/tutorial.rst: -------------------------------------------------------------------------------- 1 | Tutorial 2 | -------- 3 | 4 | For this tutorial we assume you already have ``loafer`` installed and your 5 | aws credentials configured. 6 | 7 | Let's create a repository named ``foobar`` with the following structure:: 8 | 9 | foobar/ 10 | handlers.py 11 | routes.py 12 | __main__.py 13 | __init__.py # empty file 14 | 15 | 16 | The ``handlers.py``:: 17 | 18 | import asyncio 19 | 20 | async def print_handler(message, *args): 21 | print('message is {}'.format(message)) 22 | print('args is {}'.format(args)) 23 | 24 | # mimic IO processing 25 | await asyncio.sleep(0.1) 26 | return True 27 | 28 | 29 | async def error_handler(exc_info, message): 30 | print('exception {} received'.format(exc_info)) 31 | # do not delete the message that originated the error 32 | return False 33 | 34 | 35 | The ``routes.py``:: 36 | 37 | from loafer.ext.aws.routes import SQSRoute 38 | from .handlers import print_handler, error_handler 39 | 40 | # assuming a queue named "loafer-test" 41 | routes = ( 42 | SQSRoute('loafer-test', {'options': {'WaitTimeSeconds': 3}}, 43 | handler=print_handler, 44 | error_handler=error_handler), 45 | ) 46 | 47 | 48 | The ``__main__.py``:: 49 | 50 | from loafer.managers import LoaferManager 51 | from .routes import routes 52 | 53 | manager = LoaferManager(routes=routes) 54 | manager.run() 55 | 56 | 57 | To execute:: 58 | 59 | $ python -m foobar 60 | 61 | 62 | To see any output, publish some messages using AWS dashboard or utilities like `awscli`_. 63 | 64 | For example:: 65 | 66 | $ aws sqs send-message --queue-url http:///loafer-test --message-body '{"key": true}' 67 | 68 | .. _awscli: https://github.com/aws/aws-cli 69 | -------------------------------------------------------------------------------- /env.local: -------------------------------------------------------------------------------- 1 | PYTHONASYNCIODEBUG=1 2 | -------------------------------------------------------------------------------- /examples/echo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/georgeyk/loafer/e878f27897776362f6661d31845f756ced7be711/examples/echo/__init__.py -------------------------------------------------------------------------------- /examples/echo/__main__.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | 4 | from loafer.ext.aws.routes import SNSQueueRoute 5 | from loafer.managers import LoaferManager 6 | 7 | 8 | async def echo_message_handler(message, *args): 9 | print(f"message is {message}") 10 | print(f"args is {args}") 11 | 12 | await asyncio.sleep(0.5) 13 | return True 14 | 15 | 16 | async def error_handler(exc_type, message): 17 | print(f"exception {exc_type} received") 18 | return False 19 | 20 | 21 | endpoint_url = os.environ.get("AWS_ENDPOINT_URL", "http://localhost:4100") 22 | 23 | routes = ( 24 | SNSQueueRoute( 25 | "echo__loafer__notification", 26 | provider_options={"endpoint_url": endpoint_url}, 27 | handler=echo_message_handler, 28 | error_handler=error_handler, 29 | ), 30 | ) 31 | 32 | manager = LoaferManager(routes=routes) 33 | manager.run(debug=True) 34 | -------------------------------------------------------------------------------- /examples/echo/sentry.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import sentry_sdk 5 | 6 | from loafer.ext.sentry import sentry_handler 7 | 8 | sentry_sdk.init(os.environ.get("SENTRY_SDK_URL", None)) 9 | handler = sentry_handler(sentry_sdk) 10 | 11 | try: 12 | raise ValueError("test") 13 | except: # noqa 14 | handler(sys.exc_info(), "ping-message") 15 | -------------------------------------------------------------------------------- /examples/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -eux 3 | 4 | aws sns create-topic --endpoint-url=${GOAWS_URL} --name=loafer__notification 5 | topic=$(aws sns list-topics --endpoint-url=${GOAWS_URL} | grep loafer | cut -d\" -f4) 6 | 7 | aws sqs create-queue --endpoint-url=${GOAWS_URL} --queue-name=echo__loafer__notification 8 | queue_url=$(aws sqs list-queues --endpoint-url=${GOAWS_URL} | grep loafer | cut -d\" -f2) 9 | 10 | aws sns subscribe --endpoint-url=${GOAWS_URL} --topic-arn ${topic} --protocol sqs --notification-endpoint ${queue_url} 11 | 12 | aws sns publish --endpoint-url=${GOAWS_URL} --topic-arn ${topic} --message file://examples/sample.json 13 | exec python -m examples.echo 14 | -------------------------------------------------------------------------------- /examples/sample.json: -------------------------------------------------------------------------------- 1 | {"hello": "world"} 2 | -------------------------------------------------------------------------------- /loafer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/georgeyk/loafer/e878f27897776362f6661d31845f756ced7be711/loafer/__init__.py -------------------------------------------------------------------------------- /loafer/dispatchers.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import sys 4 | 5 | from .exceptions import DeleteMessage 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class LoaferDispatcher: 11 | 12 | def __init__(self, routes, max_jobs=None): 13 | self.routes = routes 14 | jobs = max_jobs or len(routes) * 10 15 | self._semaphore = asyncio.Semaphore(jobs) 16 | 17 | async def dispatch_message(self, message, route): 18 | logger.debug('dispatching message to route={}'.format(route)) 19 | confirm_message = False 20 | if not message: 21 | logger.warning('message will be ignored:\n{!r}\n'.format(message)) 22 | return confirm_message 23 | 24 | with await self._semaphore: 25 | try: 26 | confirm_message = await route.deliver(message) 27 | except DeleteMessage: 28 | logger.info('explicit message deletion\n{}\n'.format(message)) 29 | confirm_message = True 30 | except asyncio.CancelledError: 31 | msg = '"{!r}" was cancelled, the message will not be acknowledged:\n{}\n' 32 | logger.warning(msg.format(route.handler, message)) 33 | except Exception as exc: 34 | logger.exception('{!r}'.format(exc)) 35 | exc_info = sys.exc_info() 36 | confirm_message = await route.error_handler(exc_info, message) 37 | 38 | return confirm_message 39 | 40 | async def _process_message(self, message, route): 41 | confirmation = await self.dispatch_message(message, route) 42 | if confirmation: 43 | provider = route.provider 44 | await provider.confirm_message(message) 45 | return confirmation 46 | 47 | async def _get_route_messages(self, route): 48 | messages = await route.provider.fetch_messages() 49 | return messages, route 50 | 51 | async def _dispatch_tasks(self): 52 | provider_messages_tasks = [ 53 | self._get_route_messages(route) for route in self.routes 54 | ] 55 | 56 | process_messages_tasks = [] 57 | for provider_task in asyncio.as_completed(provider_messages_tasks): 58 | messages, route = await provider_task 59 | 60 | process_messages_tasks += [ 61 | self._process_message(message, route) for message in messages 62 | ] 63 | 64 | if not process_messages_tasks: 65 | return 66 | 67 | await asyncio.gather(*process_messages_tasks) 68 | 69 | async def dispatch_providers(self, forever=True): 70 | while True: 71 | await self._dispatch_tasks() 72 | 73 | if not forever: 74 | break 75 | 76 | def stop(self): 77 | for route in self.routes: 78 | route.stop() 79 | -------------------------------------------------------------------------------- /loafer/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | class ProviderError(Exception): 3 | pass 4 | 5 | 6 | class ConfigurationError(Exception): 7 | pass 8 | 9 | 10 | class LoaferException(Exception): 11 | pass 12 | 13 | 14 | class DeleteMessage(LoaferException): 15 | pass 16 | -------------------------------------------------------------------------------- /loafer/ext/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/georgeyk/loafer/e878f27897776362f6661d31845f756ced7be711/loafer/ext/__init__.py -------------------------------------------------------------------------------- /loafer/ext/aws/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/georgeyk/loafer/e878f27897776362f6661d31845f756ced7be711/loafer/ext/aws/__init__.py -------------------------------------------------------------------------------- /loafer/ext/aws/bases.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import aiobotocore 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | 8 | class _BotoClient: 9 | boto_service_name = None 10 | 11 | def __init__(self, **client_options): 12 | self._client_options = { 13 | 'api_version': client_options.get('api_version', None), 14 | 'aws_access_key_id': client_options.get('aws_access_key_id', None), 15 | 'aws_secret_access_key': client_options.get('aws_secret_access_key', None), 16 | 'aws_session_token': client_options.get('aws_session_token', None), 17 | 'endpoint_url': client_options.get('endpoint_url', None), 18 | 'region_name': client_options.get('region_name', None), 19 | 'use_ssl': client_options.get('use_ssl', True), 20 | 'verify': client_options.get('verify', None), 21 | } 22 | 23 | def get_client(self): 24 | session = aiobotocore.get_session() 25 | return session.create_client(self.boto_service_name, **self._client_options) 26 | 27 | async def stop(self): 28 | async with self.get_client() as client: 29 | logger.info('closing client{}'.format(client)) 30 | await client.close() 31 | 32 | 33 | class BaseSQSClient(_BotoClient): 34 | boto_service_name = 'sqs' 35 | 36 | def __init__(self, *args, **kwargs): 37 | super().__init__(*args, **kwargs) 38 | self._cached_queue_urls = {} 39 | 40 | async def get_queue_url(self, queue): 41 | if queue and (queue.startswith('http://') or queue.startswith('https://')): 42 | name = queue.split('/')[-1] 43 | self._cached_queue_urls[name] = queue 44 | queue = name 45 | 46 | if queue not in self._cached_queue_urls: 47 | async with self.get_client() as client: 48 | response = await client.get_queue_url(QueueName=queue) 49 | self._cached_queue_urls[queue] = response['QueueUrl'] 50 | 51 | return self._cached_queue_urls[queue] 52 | 53 | 54 | class BaseSNSClient(_BotoClient): 55 | boto_service_name = 'sns' 56 | 57 | async def get_topic_arn(self, topic): 58 | arn_prefix = 'arn:aws:sns:' 59 | if topic.startswith(arn_prefix): 60 | return topic 61 | return '{}*:{}'.format(arn_prefix, topic) 62 | -------------------------------------------------------------------------------- /loafer/ext/aws/handlers.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | from .bases import BaseSNSClient, BaseSQSClient 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class SQSHandler(BaseSQSClient): 10 | queue_name = None 11 | 12 | def __init__(self, queue_name=None, **kwargs): 13 | self.queue_name = queue_name or self.queue_name 14 | super().__init__(**kwargs) 15 | 16 | def __str__(self): 17 | return '<{}: {}>'.format(type(self).__name__, self.queue_name) 18 | 19 | async def publish(self, message, encoder=json.dumps): 20 | if not self.queue_name: 21 | raise ValueError('{}: missing queue_name attribute'.format(type(self).__name__)) 22 | 23 | if encoder: 24 | message = encoder(message) 25 | 26 | logger.debug('publishing, queue={}, message={}'.format(self.queue_name, message)) 27 | 28 | queue_url = await self.get_queue_url(self.queue_name) 29 | async with self.get_client() as client: 30 | return await client.send_message(QueueUrl=queue_url, MessageBody=message) 31 | 32 | async def handle(self, message, *args): 33 | return await self.publish(message) 34 | 35 | 36 | class SNSHandler(BaseSNSClient): 37 | topic = None 38 | 39 | def __init__(self, topic=None, **kwargs): 40 | self.topic = topic or self.topic 41 | super().__init__(**kwargs) 42 | 43 | def __str__(self): 44 | return '<{}: {}>'.format(type(self).__name__, self.topic) 45 | 46 | async def publish(self, message, encoder=json.dumps): 47 | if not self.topic: 48 | raise ValueError('{}: missing topic attribute'.format(type(self).__name__)) 49 | 50 | if encoder: 51 | message = encoder(message) 52 | 53 | topic_arn = await self.get_topic_arn(self.topic) 54 | logger.debug('publishing, topic={}, message={}'.format(topic_arn, message)) 55 | 56 | msg = json.dumps({'default': message}) 57 | async with self.get_client() as client: 58 | return await client.publish(TopicArn=topic_arn, MessageStructure='json', Message=msg) 59 | 60 | async def handle(self, message, *args): 61 | return await self.publish(message) 62 | -------------------------------------------------------------------------------- /loafer/ext/aws/message_translators.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | 4 | from loafer.message_translators import AbstractMessageTranslator 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class SQSMessageTranslator(AbstractMessageTranslator): 10 | 11 | def translate(self, message): 12 | translated = {'content': None, 'metadata': {}} 13 | try: 14 | body = message['Body'] 15 | except (KeyError, TypeError): 16 | logger.error('missing Body key in SQS message. It really came from SQS ?' 17 | '\nmessage={!r}'.format(message)) 18 | return translated 19 | 20 | try: 21 | translated['content'] = json.loads(body) 22 | except json.decoder.JSONDecodeError as exc: 23 | logger.error('error={!r}, message={!r}'.format(exc, message)) 24 | return translated 25 | 26 | message.pop('Body') 27 | translated['metadata'].update(message) 28 | return translated 29 | 30 | 31 | class SNSMessageTranslator(AbstractMessageTranslator): 32 | 33 | def translate(self, message): 34 | translated = {'content': None, 'metadata': {}} 35 | try: 36 | body = json.loads(message['Body']) 37 | message_body = body.pop('Message') 38 | except (KeyError, TypeError): 39 | logger.error( 40 | 'Missing Body or Message key in SQS message. It really came from SNS ?' 41 | '\nmessage={!r}'.format(message)) 42 | return translated 43 | 44 | translated['metadata'].update(message) 45 | translated['metadata'].pop('Body') 46 | 47 | try: 48 | translated['content'] = json.loads(message_body) 49 | except (json.decoder.JSONDecodeError, TypeError) as exc: 50 | logger.error('error={!r}, message={!r}'.format(exc, message)) 51 | return translated 52 | 53 | translated['metadata'].update(body) 54 | return translated 55 | -------------------------------------------------------------------------------- /loafer/ext/aws/providers.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | 4 | import botocore.exceptions 5 | 6 | from .bases import BaseSQSClient 7 | from loafer.exceptions import ProviderError 8 | from loafer.providers import AbstractProvider 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class SQSProvider(AbstractProvider, BaseSQSClient): 14 | 15 | def __init__(self, queue_name, options=None, **kwargs): 16 | self.queue_name = queue_name 17 | self._options = options or {} 18 | super().__init__(**kwargs) 19 | 20 | def __str__(self): 21 | return '<{}: {}>'.format(type(self).__name__, self.queue_name) 22 | 23 | async def confirm_message(self, message): 24 | receipt = message['ReceiptHandle'] 25 | logger.info('confirm message (ack/deletion), receipt={!r}'.format(receipt)) 26 | 27 | queue_url = await self.get_queue_url(self.queue_name) 28 | try: 29 | async with self.get_client() as client: 30 | return await client.delete_message(QueueUrl=queue_url, ReceiptHandle=receipt) 31 | except botocore.exceptions.ClientError as exc: 32 | if exc.response['ResponseMetadata']['HTTPStatusCode'] == 404: 33 | return True 34 | 35 | raise 36 | 37 | async def fetch_messages(self): 38 | logger.debug('fetching messages on {}'.format(self.queue_name)) 39 | try: 40 | queue_url = await self.get_queue_url(self.queue_name) 41 | async with self.get_client() as client: 42 | response = await client.receive_message(QueueUrl=queue_url, **self._options) 43 | except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as exc: 44 | raise ProviderError('error fetching messages from queue={}: {}'.format(self.queue_name, str(exc))) from exc 45 | 46 | return response.get('Messages', []) 47 | 48 | async def _client_stop(self): 49 | async with self.get_client() as client: 50 | await client.close() 51 | 52 | def stop(self): 53 | logger.info('stopping {}'.format(self)) 54 | loop = asyncio.get_event_loop() 55 | loop.run_until_complete(self._client_stop()) 56 | return super().stop() 57 | -------------------------------------------------------------------------------- /loafer/ext/aws/routes.py: -------------------------------------------------------------------------------- 1 | from ...routes import Route 2 | from .message_translators import SNSMessageTranslator, SQSMessageTranslator 3 | from .providers import SQSProvider 4 | 5 | 6 | class SQSRoute(Route): 7 | def __init__(self, provider_queue, provider_options=None, *args, **kwargs): 8 | provider_options = provider_options or {} 9 | provider = SQSProvider(provider_queue, **provider_options) 10 | kwargs['provider'] = provider 11 | if 'message_translator' not in kwargs: 12 | kwargs['message_translator'] = SQSMessageTranslator() 13 | if 'name' not in kwargs: 14 | kwargs['name'] = provider_queue 15 | 16 | super().__init__(*args, **kwargs) 17 | 18 | 19 | class SNSQueueRoute(Route): 20 | def __init__(self, provider_queue, provider_options=None, *args, **kwargs): 21 | provider_options = provider_options or {} 22 | provider = SQSProvider(provider_queue, **provider_options) 23 | kwargs['provider'] = provider 24 | if 'message_translator' not in kwargs: 25 | kwargs['message_translator'] = SNSMessageTranslator() 26 | if 'name' not in kwargs: 27 | kwargs['name'] = provider_queue 28 | 29 | super().__init__(*args, **kwargs) 30 | -------------------------------------------------------------------------------- /loafer/ext/sentry.py: -------------------------------------------------------------------------------- 1 | # TODO: it should be async 2 | 3 | 4 | def sentry_handler(sdk_or_hub, delete_message=False): 5 | 6 | def send_to_sentry(exc_info, message): 7 | with sdk_or_hub.push_scope() as scope: 8 | scope.set_extra("message", message) 9 | sdk_or_hub.capture_exception(exc_info) 10 | return delete_message 11 | 12 | return send_to_sentry 13 | -------------------------------------------------------------------------------- /loafer/managers.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import os 4 | 5 | from cached_property import cached_property 6 | 7 | from .dispatchers import LoaferDispatcher 8 | from .exceptions import ConfigurationError 9 | from .routes import Route 10 | from .runners import LoaferRunner 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class LoaferManager: 16 | 17 | def __init__(self, routes, runner=None, _concurrency_limit=None, _max_threads=None): 18 | self._concurrency_limit = _concurrency_limit 19 | if runner is None: 20 | self.runner = LoaferRunner(on_stop_callback=self.on_loop__stop, max_workers=_max_threads) 21 | else: 22 | self.runner = runner 23 | 24 | self.routes = routes 25 | 26 | @cached_property 27 | def dispatcher(self): 28 | if not (self.routes and all(isinstance(r, Route) for r in self.routes)): 29 | raise ConfigurationError('invalid routes to dispatch, routes={}'.format(self.routes)) 30 | 31 | return LoaferDispatcher(self.routes, max_jobs=self._concurrency_limit) 32 | 33 | def run(self, forever=True, debug=False): 34 | loop = self.runner.loop 35 | self._future = asyncio.ensure_future( 36 | self.dispatcher.dispatch_providers(forever=forever), 37 | loop=loop, 38 | ) 39 | 40 | self._future.add_done_callback(self.on_future__errors) 41 | if not forever: 42 | self._future.add_done_callback(self.runner.prepare_stop) 43 | 44 | start = 'starting loafer, pid={}, forever={}' 45 | logger.info(start.format(os.getpid(), forever)) 46 | self.runner.start(debug=debug) 47 | 48 | # 49 | # Callbacks 50 | # 51 | 52 | def on_future__errors(self, future): 53 | if future.cancelled(): 54 | return self.runner.prepare_stop() 55 | 56 | exc = future.exception() 57 | # Unhandled errors crashes the event loop execution 58 | if isinstance(exc, BaseException): 59 | logger.critical('fatal error caught: {!r}'.format(exc)) 60 | self.runner.prepare_stop() 61 | 62 | def on_loop__stop(self, *args, **kwargs): 63 | logger.info('cancel dispatcher operations ...') 64 | 65 | if hasattr(self, '_future'): 66 | self._future.cancel() 67 | 68 | self.dispatcher.stop() 69 | -------------------------------------------------------------------------------- /loafer/message_translators.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import logging 3 | 4 | logger = logging.getLogger(__name__) 5 | 6 | 7 | class AbstractMessageTranslator(abc.ABC): 8 | 9 | @abc.abstractmethod 10 | def translate(self, message): 11 | '''Translates a given message to an appropriate format to message processing. 12 | This method should return a `dict` instance with two keys: `content` 13 | and `metadata`. 14 | The `content` should contain the translated message and, `metadata` a 15 | dictionary with translation metadata or an empty `dict`. 16 | ''' 17 | 18 | 19 | class StringMessageTranslator(AbstractMessageTranslator): 20 | 21 | def translate(self, message): 22 | logger.debug('{!r} will translate {!r}'.format(type(self).__name__, message)) 23 | return {'content': str(message), 'metadata': {}} 24 | -------------------------------------------------------------------------------- /loafer/providers.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class AbstractProvider(abc.ABC): 5 | 6 | @abc.abstractmethod 7 | async def fetch_messages(self): 8 | '''A coroutine that returns a sequence of messages to be processed. 9 | If no messages are available, this coroutine should return an empty list. 10 | ''' 11 | 12 | @abc.abstractmethod 13 | async def confirm_message(self, message): 14 | '''A coroutine to confirm the message processing. 15 | After the message confirmation we should not receive the same message again. 16 | This usually means we need to delete the message in the provider. 17 | ''' 18 | 19 | def stop(self): 20 | '''Stops the provider. 21 | If needed, the provider should perform clean-up actions. 22 | This method is called whenever we need to shutdown the provider. 23 | ''' 24 | pass 25 | -------------------------------------------------------------------------------- /loafer/routes.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from .message_translators import AbstractMessageTranslator 4 | from .providers import AbstractProvider 5 | from .utils import run_in_loop_or_executor 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class Route: 11 | 12 | def __init__(self, provider, handler, name='default', 13 | message_translator=None, error_handler=None): 14 | self.name = name 15 | 16 | if not isinstance(provider, AbstractProvider): 17 | raise TypeError('invalid provider instance: {!r}'.format(provider)) 18 | 19 | self.provider = provider 20 | 21 | if message_translator: 22 | if not isinstance(message_translator, AbstractMessageTranslator): 23 | raise TypeError( 24 | 'invalid message translator instance: {!r}'.format(message_translator) 25 | ) 26 | 27 | self.message_translator = message_translator 28 | 29 | if error_handler: 30 | if not callable(error_handler): 31 | raise TypeError( 32 | 'error_handler must be a callable object: {!r}'.format(error_handler) 33 | ) 34 | 35 | self._error_handler = error_handler 36 | 37 | if callable(handler): 38 | self.handler = handler 39 | self._handler_instance = None 40 | else: 41 | self.handler = getattr(handler, 'handle', None) 42 | self._handler_instance = handler 43 | 44 | if not self.handler: 45 | raise ValueError( 46 | 'handler must be a callable object or implement `handle` method: {!r}'.format(self.handler) 47 | ) 48 | 49 | def __str__(self): 50 | return '<{}(name={} provider={!r} handler={!r})>'.format( 51 | type(self).__name__, self.name, self.provider, self.handler) 52 | 53 | def apply_message_translator(self, message): 54 | processed_message = {'content': message, 55 | 'metadata': {}} 56 | if not self.message_translator: 57 | return processed_message 58 | 59 | translated = self.message_translator.translate(processed_message['content']) 60 | processed_message['metadata'].update(translated.get('metadata', {})) 61 | processed_message['content'] = translated['content'] 62 | if not processed_message['content']: 63 | raise ValueError('{} failed to translate message={}'.format(self.message_translator, message)) 64 | 65 | return processed_message 66 | 67 | async def deliver(self, raw_message): 68 | message = self.apply_message_translator(raw_message) 69 | logger.info('delivering message route={}, message={!r}'.format(self, message)) 70 | return await run_in_loop_or_executor(self.handler, message['content'], message['metadata']) 71 | 72 | async def error_handler(self, exc_info, message): 73 | logger.info('error handler process originated by message={}'.format(message)) 74 | 75 | if self._error_handler is not None: 76 | return await run_in_loop_or_executor(self._error_handler, exc_info, message) 77 | 78 | return False 79 | 80 | def stop(self): 81 | logger.info('stopping route {}'.format(self)) 82 | self.provider.stop() 83 | # only for class-based handlers 84 | if hasattr(self._handler_instance, 'stop'): 85 | self._handler_instance.stop() 86 | -------------------------------------------------------------------------------- /loafer/runners.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | import signal 4 | from concurrent.futures import CancelledError, ThreadPoolExecutor 5 | from contextlib import suppress 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class LoaferRunner: 11 | 12 | def __init__(self, max_workers=None, on_stop_callback=None): 13 | self._on_stop_callback = on_stop_callback 14 | 15 | # XXX: See https://github.com/python/asyncio/issues/258 16 | # The minimum value depends on the number of cores in the machine 17 | # See https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor 18 | self._executor = ThreadPoolExecutor(max_workers) 19 | self.loop.set_default_executor(self._executor) 20 | 21 | @property 22 | def loop(self): 23 | return asyncio.get_event_loop() 24 | 25 | def start(self, debug=False): 26 | if debug: 27 | self.loop.set_debug(enabled=debug) 28 | 29 | self.loop.add_signal_handler(signal.SIGINT, self.prepare_stop) 30 | self.loop.add_signal_handler(signal.SIGTERM, self.prepare_stop) 31 | 32 | try: 33 | self.loop.run_forever() 34 | finally: 35 | self.stop() 36 | self.loop.close() 37 | logger.debug('loop.is_running={}'.format(self.loop.is_running())) 38 | logger.debug('loop.is_closed={}'.format(self.loop.is_closed())) 39 | 40 | def prepare_stop(self, *args): 41 | if self.loop.is_running(): 42 | # signals loop.run_forever to exit in the next iteration 43 | self.loop.stop() 44 | 45 | def stop(self, *args, **kwargs): 46 | logger.info('stopping Loafer ...') 47 | if callable(self._on_stop_callback): 48 | self._on_stop_callback() 49 | 50 | logger.info('cancel schedulled operations ...') 51 | for task in asyncio.Task.all_tasks(self.loop): 52 | task.cancel() 53 | if task.cancelled() or task.done(): 54 | continue 55 | 56 | with suppress(CancelledError): 57 | self.loop.run_until_complete(task) 58 | 59 | self._executor.shutdown(wait=True) 60 | -------------------------------------------------------------------------------- /loafer/utils.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import importlib 3 | import logging 4 | import os 5 | import sys 6 | from functools import wraps 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | def add_current_dir_to_syspath(f): 12 | 13 | @wraps(f) 14 | def wrapper(*args, **kwargs): 15 | current = os.getcwd() 16 | changed = False 17 | if current not in sys.path: 18 | sys.path.append(current) 19 | changed = True 20 | 21 | try: 22 | return f(*args, **kwargs) 23 | finally: 24 | # restore sys.path 25 | if changed is True: 26 | sys.path.remove(current) 27 | 28 | return wrapper 29 | 30 | 31 | @add_current_dir_to_syspath 32 | def import_callable(full_name): 33 | package, *name = full_name.rsplit('.', 1) 34 | try: 35 | module = importlib.import_module(package) 36 | except ValueError as exc: 37 | raise ImportError('Error trying to import {!r}'.format(full_name)) from exc 38 | 39 | if name: 40 | handler = getattr(module, name[0]) 41 | else: 42 | handler = module 43 | 44 | if not callable(handler): 45 | raise ImportError('{!r} should be callable'.format(full_name)) 46 | 47 | return handler 48 | 49 | 50 | async def run_in_loop_or_executor(func, *args): 51 | if asyncio.iscoroutinefunction(func): 52 | logger.debug('handler is coroutine! {!r}'.format(func)) 53 | return await func(*args) 54 | 55 | loop = asyncio.get_event_loop() 56 | logger.debug('handler will run in a separate thread: {!r}'.format(func)) 57 | return await loop.run_in_executor(None, func, *args) 58 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | testpaths = tests 3 | addopts = -vv --cov=loafer --cov-report=term-missing 4 | -------------------------------------------------------------------------------- /requirements/local.txt: -------------------------------------------------------------------------------- 1 | -r test.txt 2 | Sphinx 3 | sphinx-autobuild 4 | twine 5 | wheel 6 | sphinx-rtd-theme 7 | seqdiag 8 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | pytest-asyncio 3 | pytest-cov 4 | pytest-deadfixtures 5 | codecov 6 | asynctest 7 | pre-commit 8 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest --addopts '-vv tests' 3 | 4 | [bdist_wheel] 5 | python-tag = py3 6 | 7 | [isort] 8 | line_length=100 9 | multi_line_output=3 10 | known_local_folder=loafer,tests 11 | sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER 12 | default_section=THIRDPARTY 13 | include_trailing_comma=true 14 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | import os.path 3 | import re 4 | 5 | from setuptools import Command, find_packages, setup 6 | 7 | # metadata 8 | 9 | here = os.path.abspath(os.path.dirname(__file__)) 10 | version = "0.0.0" 11 | changes = os.path.join(here, "CHANGES.rst") 12 | pattern = r'^(?P[0-9]+.[0-9]+(.[0-9]+)?)' 13 | with codecs.open(changes, encoding='utf-8') as changes: 14 | for line in changes: 15 | match = re.match(pattern, line) 16 | if match: 17 | version = match.group("version") 18 | break 19 | 20 | 21 | class VersionCommand(Command): 22 | description = 'Show library version' 23 | user_options = [] 24 | 25 | def initialize_options(self): 26 | pass 27 | 28 | def finalize_options(self): 29 | pass 30 | 31 | def run(self): 32 | print(version) 33 | 34 | 35 | with codecs.open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: 36 | long_description = '\n{}'.format(f.read()) 37 | 38 | with codecs.open(os.path.join(here, 'CHANGES.rst'), encoding='utf-8') as f: 39 | changes = f.read() 40 | long_description += '\n\nChangelog:\n----------\n\n{}'.format(changes) 41 | 42 | 43 | # Requirements 44 | 45 | # Unduplicated tests_requirements and requirements/test.txt 46 | tests_requirements = [ 47 | 'pytest', 48 | 'pytest-asyncio', 49 | 'pytest-cov', 50 | 'pytest-deadfixtures', 51 | 'codecov', 52 | 'asynctest', 53 | 'pre-commit', 54 | ] 55 | 56 | # We depend on `aiohttp` and `boto3` and since `aiobotocore` works with a range 57 | # version of them, we will leave to aiobotocore setup the version requirements 58 | install_requirements = [ 59 | 'aiobotocore[boto3]>=1.0.4,<2', 60 | 'cached-property>=1.3.0,<2', 61 | ] 62 | 63 | 64 | # setup 65 | 66 | setup( 67 | name='loafer', 68 | version=version, 69 | description='Asynchronous message dispatcher for concurrent tasks processing', 70 | long_description=long_description, 71 | url='https://github.com/georgeyk/loafer/', 72 | download_url='https://github.com/georgeyk/loafer/releases', 73 | license='MIT', 74 | author='George Y. Kussumoto', 75 | author_email='contato@georgeyk.com.br', 76 | packages=find_packages(exclude=['docs', 'examples', 'tests', 'tests.*', 'requirements']), 77 | classifiers=[ 78 | 'Development Status :: 4 - Beta', 79 | 'Environment :: Console', 80 | 'Intended Audience :: Developers', 81 | 'License :: OSI Approved :: MIT License', 82 | 'Natural Language :: English', 83 | 'Operating System :: OS Independent', 84 | 'Programming Language :: Python :: 3.6', 85 | 'Programming Language :: Python :: 3.7', 86 | 'Programming Language :: Python :: 3.8', 87 | 'Topic :: System :: Distributed Computing', 88 | ], 89 | keywords='asynchronous asyncio message dispatcher tasks microservices', 90 | python_requires='>=3.6', 91 | setup_requires=['pytest-runner'], 92 | install_requires=install_requirements, 93 | tests_require=tests_requirements, 94 | cmdclass={'version': VersionCommand}, 95 | ) 96 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/georgeyk/loafer/e878f27897776362f6661d31845f756ced7be711/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from loafer.providers import AbstractProvider 4 | 5 | 6 | @pytest.fixture 7 | def dummy_handler(): 8 | def handler(message, *args): 9 | raise AssertionError('I should not be called') 10 | 11 | return handler 12 | 13 | 14 | @pytest.fixture 15 | def dummy_provider(): 16 | 17 | class Dummy(AbstractProvider): 18 | async def fetch_messages(self): 19 | raise AssertionError('I should not be called') 20 | 21 | async def confirm_message(self): 22 | raise AssertionError('I should not be called') 23 | 24 | def stop(self): 25 | raise AssertionError('I should not be called') 26 | 27 | return Dummy() 28 | -------------------------------------------------------------------------------- /tests/ext/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/georgeyk/loafer/e878f27897776362f6661d31845f756ced7be711/tests/ext/__init__.py -------------------------------------------------------------------------------- /tests/ext/aws/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/georgeyk/loafer/e878f27897776362f6661d31845f756ced7be711/tests/ext/aws/__init__.py -------------------------------------------------------------------------------- /tests/ext/aws/conftest.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import pytest 4 | from asynctest import CoroutineMock 5 | 6 | # boto client methods mock 7 | 8 | 9 | @pytest.fixture 10 | def queue_url(): 11 | queue_url = 'https://sqs.us-east-1.amazonaws.com/123456789012/queue-name' 12 | return {'QueueUrl': queue_url} 13 | 14 | 15 | @pytest.fixture 16 | def sqs_message(): 17 | message = {'Body': 'test'} 18 | return {'Messages': [message]} 19 | 20 | 21 | def sqs_send_message(): 22 | return {'MessageId': 'uuid', 'MD5OfMessageBody': 'md5', 23 | 'ResponseMetada': {'RequestId': 'uuid', 'HTTPStatusCode': 200}} 24 | 25 | 26 | @pytest.fixture 27 | def sns_list_topics(): 28 | return {'Topics': [{'TopicArn': 'arn:aws:sns:region:id:topic-name'}]} 29 | 30 | 31 | @pytest.fixture 32 | def sns_publish(): 33 | return {'ResponseMetadata': {'HTTPStatusCode': 200, 'RequestId': 'uuid'}, 34 | 'MessageId': 'uuid'} 35 | 36 | 37 | # boto client mock 38 | 39 | 40 | class ClientContextCreator: 41 | def __init__(self, client): 42 | self._client = client 43 | 44 | async def __aenter__(self): 45 | return self._client 46 | 47 | async def __aexit__(self, exc_type, exc_val, exc_tb): 48 | pass 49 | 50 | 51 | @pytest.fixture 52 | def boto_client_sqs(queue_url, sqs_message): 53 | mock_client = mock.Mock() 54 | mock_client.get_queue_url = CoroutineMock(return_value=queue_url) 55 | mock_client.delete_message = CoroutineMock() 56 | mock_client.receive_message = CoroutineMock(return_value=sqs_message) 57 | mock_client.send_message = CoroutineMock(return_value=sqs_send_message) 58 | mock_client.close = CoroutineMock() 59 | return mock_client 60 | 61 | 62 | @pytest.fixture 63 | def mock_boto_session_sqs(boto_client_sqs): 64 | mock_session = mock.Mock() 65 | mock_session.create_client.return_value = ClientContextCreator(boto_client_sqs) 66 | return mock.patch('aiobotocore.get_session', return_value=mock_session) 67 | 68 | 69 | @pytest.fixture 70 | def boto_client_sns(sns_publish, sns_list_topics): 71 | mock_client = mock.Mock() 72 | mock_client.publish = CoroutineMock(return_value=sns_publish) 73 | mock_client.close = CoroutineMock() 74 | return mock_client 75 | 76 | 77 | @pytest.fixture 78 | def mock_boto_session_sns(boto_client_sns): 79 | mock_session = mock.Mock() 80 | mock_session.create_client.return_value = ClientContextCreator(boto_client_sns) 81 | return mock.patch('aiobotocore.get_session', return_value=mock_session) 82 | -------------------------------------------------------------------------------- /tests/ext/aws/test_bases.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import pytest 4 | 5 | from loafer.ext.aws.bases import BaseSNSClient, BaseSQSClient 6 | 7 | 8 | @pytest.fixture 9 | def base_sqs_client(): 10 | return BaseSQSClient() 11 | 12 | 13 | @pytest.mark.asyncio 14 | async def test_get_queue_url(mock_boto_session_sqs, boto_client_sqs, base_sqs_client): 15 | with mock_boto_session_sqs as mock_sqs: 16 | queue_url = await base_sqs_client.get_queue_url('queue-name') 17 | assert queue_url.startswith('https://') 18 | assert queue_url.endswith('queue-name') 19 | 20 | assert mock_sqs.called 21 | assert boto_client_sqs.get_queue_url.called 22 | assert boto_client_sqs.get_queue_url.call_args == mock.call(QueueName='queue-name') 23 | 24 | 25 | @pytest.mark.asyncio 26 | async def test_cache_get_queue_url(mock_boto_session_sqs, boto_client_sqs, base_sqs_client): 27 | with mock_boto_session_sqs: 28 | await base_sqs_client.get_queue_url('queue-name') 29 | queue_url = await base_sqs_client.get_queue_url('queue-name') 30 | assert queue_url.startswith('https://') 31 | assert queue_url.endswith('queue-name') 32 | assert boto_client_sqs.get_queue_url.call_count == 1 33 | 34 | 35 | @pytest.mark.asyncio 36 | async def test_get_queue_url_when_queue_name_is_url(mock_boto_session_sqs, boto_client_sqs, base_sqs_client): 37 | with mock_boto_session_sqs: 38 | queue_url = await base_sqs_client.get_queue_url('https://aws-whatever/queue-name') 39 | assert queue_url.startswith('https://') 40 | assert queue_url.endswith('queue-name') 41 | assert boto_client_sqs.get_queue_url.call_count == 0 42 | 43 | 44 | @pytest.mark.asyncio 45 | async def test_sqs_close(mock_boto_session_sqs, base_sqs_client, boto_client_sqs): 46 | with mock_boto_session_sqs: 47 | await base_sqs_client.stop() 48 | boto_client_sqs.close.assert_awaited_once() 49 | 50 | 51 | @pytest.mark.asyncio 52 | async def test_sqs_get_client(mock_boto_session_sqs, base_sqs_client, boto_client_sqs): 53 | with mock_boto_session_sqs as mock_session: 54 | client_generator = base_sqs_client.get_client() 55 | assert mock_session.called 56 | async with client_generator as client: 57 | assert boto_client_sqs is client 58 | 59 | 60 | @pytest.fixture 61 | def base_sns_client(): 62 | return BaseSNSClient() 63 | 64 | 65 | @pytest.mark.asyncio 66 | async def test_get_topic_arn_using_topic_name(base_sns_client): 67 | arn = await base_sns_client.get_topic_arn('topic-name') 68 | assert arn == 'arn:aws:sns:*:topic-name' 69 | 70 | 71 | @pytest.mark.asyncio 72 | async def test_cache_get_topic_arn_with_arn(base_sns_client): 73 | arn = await base_sns_client.get_topic_arn('arn:aws:sns:whatever:topic-name') 74 | assert arn == 'arn:aws:sns:whatever:topic-name' 75 | 76 | 77 | @pytest.mark.asyncio 78 | async def test_sns_close(mock_boto_session_sns, base_sns_client, boto_client_sns): 79 | with mock_boto_session_sns: 80 | await base_sns_client.stop() 81 | boto_client_sns.close.assert_awaited_once() 82 | 83 | 84 | @pytest.mark.asyncio 85 | async def test_sns_get_client(mock_boto_session_sns, base_sns_client, boto_client_sns): 86 | with mock_boto_session_sns as mock_session: 87 | client_generator = base_sns_client.get_client() 88 | assert mock_session.called 89 | async with client_generator as client: 90 | assert boto_client_sns is client 91 | -------------------------------------------------------------------------------- /tests/ext/aws/test_handlers.py: -------------------------------------------------------------------------------- 1 | import json 2 | from unittest import mock 3 | 4 | import pytest 5 | from asynctest import CoroutineMock 6 | 7 | from loafer.ext.aws.handlers import SNSHandler, SQSHandler 8 | 9 | # SQSHandler 10 | 11 | 12 | @pytest.mark.asyncio 13 | @pytest.mark.parametrize('encoder', [json.dumps, str]) 14 | async def test_sqs_handler_publish(mock_boto_session_sqs, boto_client_sqs, encoder): 15 | handler = SQSHandler('queue-name') 16 | with mock_boto_session_sqs as mock_sqs: 17 | retval = await handler.publish('message', encoder=encoder) 18 | assert retval 19 | 20 | assert mock_sqs.called 21 | assert boto_client_sqs.send_message.called 22 | assert boto_client_sqs.send_message.call_args == mock.call( 23 | QueueUrl=await handler.get_queue_url('queue-name'), 24 | MessageBody=encoder('message')) 25 | 26 | 27 | @pytest.mark.asyncio 28 | async def test_sqs_handler_publish_without_encoder(mock_boto_session_sqs, boto_client_sqs): 29 | handler = SQSHandler('queue-name') 30 | with mock_boto_session_sqs as mock_sqs: 31 | retval = await handler.publish('message', encoder=None) 32 | assert retval 33 | 34 | assert mock_sqs.called 35 | assert boto_client_sqs.send_message.called 36 | assert boto_client_sqs.send_message.call_args == mock.call( 37 | QueueUrl=await handler.get_queue_url('queue-name'), 38 | MessageBody='message') 39 | 40 | 41 | @pytest.mark.asyncio 42 | async def test_sqs_handler_publish_without_queue_name(): 43 | handler = SQSHandler() 44 | with pytest.raises(ValueError): 45 | await handler.publish('wrong') 46 | 47 | 48 | @pytest.mark.asyncio 49 | async def test_sqs_handler_hadle(): 50 | handler = SQSHandler('foobar') 51 | handler.publish = CoroutineMock() 52 | await handler.handle('message', 'metadata') 53 | assert handler.publish.called 54 | handler.publish.assert_called_once_with('message') 55 | 56 | 57 | # SNSHandler 58 | 59 | @pytest.mark.asyncio 60 | @pytest.mark.parametrize('encoder', [json.dumps, str]) 61 | async def test_sns_handler_publisher(mock_boto_session_sns, boto_client_sns, encoder): 62 | handler = SNSHandler('arn:aws:sns:whatever:topic-name') 63 | with mock_boto_session_sns as mock_sns: 64 | retval = await handler.publish('message', encoder=encoder) 65 | assert retval 66 | 67 | assert mock_sns.called 68 | assert boto_client_sns.publish.called 69 | assert boto_client_sns.publish.call_args == mock.call( 70 | TopicArn='arn:aws:sns:whatever:topic-name', 71 | MessageStructure='json', 72 | Message=json.dumps({'default': encoder('message')})) 73 | 74 | 75 | @pytest.mark.asyncio 76 | async def test_sns_handler_publisher_without_encoder(mock_boto_session_sns, boto_client_sns): 77 | handler = SNSHandler('arn:aws:sns:whatever:topic-name') 78 | with mock_boto_session_sns as mock_sns: 79 | retval = await handler.publish('message', encoder=None) 80 | assert retval 81 | 82 | assert mock_sns.called 83 | assert boto_client_sns.publish.called 84 | assert boto_client_sns.publish.call_args == mock.call( 85 | TopicArn='arn:aws:sns:whatever:topic-name', 86 | MessageStructure='json', 87 | Message=json.dumps({'default': 'message'})) 88 | 89 | 90 | @pytest.mark.asyncio 91 | async def test_sns_handler_publish_without_topic(): 92 | handler = SNSHandler() 93 | with pytest.raises(ValueError): 94 | await handler.publish('wrong') 95 | 96 | 97 | @pytest.mark.asyncio 98 | async def test_sns_handler_hadle(): 99 | handler = SNSHandler('foobar') 100 | handler.publish = CoroutineMock() 101 | await handler.handle('message', 'metadata') 102 | assert handler.publish.called 103 | handler.publish.assert_called_once_with('message') 104 | -------------------------------------------------------------------------------- /tests/ext/aws/test_message_translators.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import pytest 4 | 5 | from loafer.ext.aws.message_translators import SNSMessageTranslator, SQSMessageTranslator 6 | 7 | # sqs 8 | 9 | 10 | @pytest.fixture 11 | def sqs_translator(): 12 | return SQSMessageTranslator() 13 | 14 | 15 | def test_translate_sqs(sqs_translator): 16 | original = {'Body': json.dumps('some-content')} 17 | content = sqs_translator.translate(original) 18 | assert 'content' in content 19 | assert content['content'] == 'some-content' 20 | 21 | original = {'Body': json.dumps({'key': 'value'})} 22 | content = sqs_translator.translate(original) 23 | assert content['content'] == {'key': 'value'} 24 | 25 | 26 | def test_sqs_metadata_extract(sqs_translator): 27 | original = {'Body': json.dumps('some-content'), 'whatever': 'whatever'} 28 | content = sqs_translator.translate(original) 29 | metadata = content['metadata'] 30 | assert metadata 31 | assert 'whatever' in metadata 32 | assert metadata['whatever'] == 'whatever' 33 | 34 | 35 | @pytest.fixture(params=[{'invalid': 'format'}, 'invalid format', 36 | 42, {}, [], (), '']) 37 | def parametrize_invalid_messages(request): 38 | return request.param 39 | 40 | 41 | def test_translate_sqs_handles_invalid_format(sqs_translator, parametrize_invalid_messages): 42 | content = sqs_translator.translate(parametrize_invalid_messages) 43 | assert content['content'] is None 44 | 45 | 46 | def test_translate_sqs_handles_json_error(sqs_translator): 47 | original = {'Body': 'invalid: json'} 48 | content = sqs_translator.translate(original) 49 | assert content['content'] is None 50 | 51 | # sns 52 | 53 | 54 | @pytest.fixture 55 | def sns_translator(): 56 | return SNSMessageTranslator() 57 | 58 | 59 | def test_translate_sns(sns_translator): 60 | message_content = 'here I am' 61 | message = json.dumps({'Message': json.dumps(message_content)}) 62 | original = {'Body': message} 63 | content = sns_translator.translate(original) 64 | assert content['content'] == message_content 65 | 66 | message_content = {'here': 'I am'} 67 | message = json.dumps({'Message': json.dumps(message_content)}) 68 | original = {'Body': message} 69 | content = sns_translator.translate(original) 70 | assert content['content'] == message_content 71 | 72 | 73 | def test_sns_metadata_extract(sns_translator): 74 | message_content = 'here I am' 75 | message = json.dumps({'Message': json.dumps(message_content), 'foo': 'nested'}) 76 | original = {'Body': message, 'bar': 'not nested'} 77 | content = sns_translator.translate(original) 78 | metadata = content['metadata'] 79 | assert metadata 80 | assert 'foo' in metadata 81 | assert metadata['foo'] == 'nested' 82 | assert 'bar' in metadata 83 | assert metadata['bar'] == 'not nested' 84 | 85 | 86 | def test_translate_sns_handles_invalid_content(sns_translator, parametrize_invalid_messages): 87 | message = json.dumps({'Message': parametrize_invalid_messages}) 88 | original = {'Body': message} 89 | content = sns_translator.translate(original) 90 | assert content['content'] is None 91 | 92 | 93 | def test_translate_sns_handles_invalid_format(sns_translator, parametrize_invalid_messages): 94 | content = sns_translator.translate(parametrize_invalid_messages) 95 | assert content['content'] is None 96 | -------------------------------------------------------------------------------- /tests/ext/aws/test_providers.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import pytest 4 | from asynctest import CoroutineMock 5 | from botocore.exceptions import BotoCoreError, ClientError 6 | 7 | from loafer.exceptions import ProviderError 8 | from loafer.ext.aws.providers import SQSProvider 9 | 10 | 11 | @pytest.mark.asyncio 12 | async def test_confirm_message(mock_boto_session_sqs, boto_client_sqs): 13 | with mock_boto_session_sqs: 14 | provider = SQSProvider('queue-name') 15 | message = {'ReceiptHandle': 'message-receipt-handle'} 16 | await provider.confirm_message(message) 17 | 18 | assert boto_client_sqs.delete_message.call_args == mock.call( 19 | QueueUrl=await provider.get_queue_url('queue-name'), 20 | ReceiptHandle='message-receipt-handle') 21 | 22 | 23 | @pytest.mark.asyncio 24 | async def test_confirm_message_not_found(mock_boto_session_sqs, boto_client_sqs): 25 | error = ClientError(error_response={'ResponseMetadata': {'HTTPStatusCode': 404}}, 26 | operation_name='whatever') 27 | boto_client_sqs.delete_message.side_effect = error 28 | with mock_boto_session_sqs: 29 | provider = SQSProvider('queue-name') 30 | message = {'ReceiptHandle': 'message-receipt-handle-not-found'} 31 | await provider.confirm_message(message) 32 | 33 | assert boto_client_sqs.delete_message.call_args == mock.call( 34 | QueueUrl=await provider.get_queue_url('queue-name'), 35 | ReceiptHandle='message-receipt-handle-not-found') 36 | 37 | 38 | @pytest.mark.asyncio 39 | async def test_confirm_message_unknown_error(mock_boto_session_sqs, boto_client_sqs): 40 | error = ClientError(error_response={'ResponseMetadata': {'HTTPStatusCode': 400}}, 41 | operation_name='whatever') 42 | boto_client_sqs.delete_message.side_effect = error 43 | with mock_boto_session_sqs: 44 | provider = SQSProvider('queue-name') 45 | message = {'ReceiptHandle': 'message-receipt-handle-not-found'} 46 | with pytest.raises(ClientError): 47 | await provider.confirm_message(message) 48 | 49 | 50 | @pytest.mark.asyncio 51 | async def test_fetch_messages(mock_boto_session_sqs, boto_client_sqs): 52 | options = {'WaitTimeSeconds': 5, 'MaxNumberOfMessages': 10} 53 | with mock_boto_session_sqs: 54 | provider = SQSProvider('queue-name', options=options) 55 | messages = await provider.fetch_messages() 56 | 57 | assert len(messages) == 1 58 | assert messages[0]['Body'] == 'test' 59 | 60 | assert boto_client_sqs.receive_message.call_args == mock.call( 61 | QueueUrl=await provider.get_queue_url('queue-name'), 62 | WaitTimeSeconds=options.get('WaitTimeSeconds'), 63 | MaxNumberOfMessages=options.get('MaxNumberOfMessages')) 64 | 65 | 66 | @pytest.mark.asyncio 67 | async def test_fetch_messages_returns_empty(mock_boto_session_sqs, boto_client_sqs): 68 | options = {'WaitTimeSeconds': 5, 'MaxNumberOfMessages': 10} 69 | boto_client_sqs.receive_message.return_value = {'Messages': []} 70 | with mock_boto_session_sqs: 71 | provider = SQSProvider('queue-name', options=options) 72 | messages = await provider.fetch_messages() 73 | 74 | assert messages == [] 75 | 76 | assert boto_client_sqs.receive_message.call_args == mock.call( 77 | QueueUrl=await provider.get_queue_url('queue-name'), 78 | WaitTimeSeconds=options.get('WaitTimeSeconds'), 79 | MaxNumberOfMessages=options.get('MaxNumberOfMessages')) 80 | 81 | 82 | @pytest.mark.asyncio 83 | async def test_fetch_messages_with_client_error(mock_boto_session_sqs, boto_client_sqs): 84 | with mock_boto_session_sqs: 85 | error = ClientError(error_response={'Error': {'Message': 'unknown'}}, 86 | operation_name='whatever') 87 | boto_client_sqs.receive_message.side_effect = error 88 | 89 | provider = SQSProvider('queue-name') 90 | with pytest.raises(ProviderError): 91 | await provider.fetch_messages() 92 | 93 | 94 | @pytest.mark.asyncio 95 | async def test_fetch_messages_with_botocoreerror(mock_boto_session_sqs, boto_client_sqs): 96 | with mock_boto_session_sqs: 97 | error = BotoCoreError() 98 | boto_client_sqs.receive_message.side_effect = error 99 | 100 | provider = SQSProvider('queue-name') 101 | with pytest.raises(ProviderError): 102 | await provider.fetch_messages() 103 | 104 | 105 | def test_stop(): 106 | provider = SQSProvider('queue-name') 107 | provider._client_stop = CoroutineMock() 108 | provider.stop() 109 | provider._client_stop.assert_awaited() 110 | -------------------------------------------------------------------------------- /tests/ext/aws/test_routes.py: -------------------------------------------------------------------------------- 1 | from loafer.ext.aws.message_translators import SNSMessageTranslator, SQSMessageTranslator 2 | from loafer.ext.aws.providers import SQSProvider 3 | from loafer.ext.aws.routes import SNSQueueRoute, SQSRoute 4 | 5 | 6 | def test_sqs_route(dummy_handler): 7 | route = SQSRoute('what', handler=dummy_handler) 8 | assert isinstance(route.message_translator, SQSMessageTranslator) 9 | assert isinstance(route.provider, SQSProvider) 10 | assert route.name == 'what' 11 | 12 | 13 | def test_sqs_route_keep_message_translator(dummy_handler): 14 | route = SQSRoute('what', handler=dummy_handler, message_translator=SNSMessageTranslator()) 15 | assert isinstance(route.message_translator, SNSMessageTranslator) 16 | route = SQSRoute('what', handler=dummy_handler, message_translator=None) 17 | assert route.message_translator is None 18 | 19 | 20 | def test_sqs_route_keep_name(dummy_handler): 21 | route = SQSRoute('what', handler=dummy_handler, name='foobar') 22 | assert route.name == 'foobar' 23 | 24 | 25 | def test_sqs_route_provider_options(dummy_handler): 26 | route = SQSRoute('what', {'use_ssl': False}, handler=dummy_handler, name='foobar') 27 | assert 'use_ssl' in route.provider._client_options 28 | assert route.provider._client_options['use_ssl'] is False 29 | 30 | 31 | def test_sns_queue_route(dummy_handler): 32 | route = SNSQueueRoute('what', handler=dummy_handler) 33 | assert isinstance(route.message_translator, SNSMessageTranslator) 34 | assert isinstance(route.provider, SQSProvider) 35 | assert route.name == 'what' 36 | 37 | 38 | def test_sns_queue_route_keep_message_translator(dummy_handler): 39 | route = SNSQueueRoute('what', handler=dummy_handler, message_translator=SQSMessageTranslator()) 40 | assert isinstance(route.message_translator, SQSMessageTranslator) 41 | route = SNSQueueRoute('what', handler=dummy_handler, message_translator=None) 42 | assert route.message_translator is None 43 | 44 | 45 | def test_sns_queue_route_keep_name(dummy_handler): 46 | route = SNSQueueRoute('what', handler=dummy_handler, name='foobar') 47 | assert route.name == 'foobar' 48 | 49 | 50 | def test_sns_queue_route_provider_options(dummy_handler): 51 | route = SNSQueueRoute('what', provider_options={'region_name': 'sa-east-1'}, handler=dummy_handler, name='foobar') 52 | assert 'region_name' in route.provider._client_options 53 | assert route.provider._client_options['region_name'] == 'sa-east-1' 54 | -------------------------------------------------------------------------------- /tests/ext/test_sentry.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | from loafer.ext.sentry import sentry_handler 4 | 5 | 6 | def test_sentry_handler(): 7 | mock_scope = mock.MagicMock() 8 | sdk_mocked = mock.Mock() 9 | sdk_mocked.push_scope.return_value = mock_scope 10 | 11 | handler = sentry_handler(sdk_mocked) 12 | exc = ValueError("test") 13 | exc_info = (type(exc), exc, None) 14 | delete_message = handler(exc_info, "test") 15 | 16 | assert delete_message is False 17 | assert sdk_mocked.push_scope.called 18 | mock_scope.__enter__.return_value.set_extra.assert_called_once_with( 19 | "message", "test" 20 | ) 21 | sdk_mocked.capture_exception.assert_called_once_with(exc_info) 22 | 23 | 24 | def test_sentry_handler_delete_message(): 25 | mock_scope = mock.MagicMock() 26 | sdk_mocked = mock.Mock() 27 | sdk_mocked.push_scope.return_value = mock_scope 28 | 29 | handler = sentry_handler(sdk_mocked, delete_message=True) 30 | exc = ValueError("test") 31 | exc_info = (type(exc), exc, None) 32 | delete_message = handler(exc_info, "test") 33 | 34 | assert delete_message is True 35 | assert sdk_mocked.push_scope.called 36 | mock_scope.__enter__.return_value.set_extra.assert_called_once_with( 37 | "message", "test" 38 | ) 39 | sdk_mocked.capture_exception.assert_called_once_with(exc_info) 40 | -------------------------------------------------------------------------------- /tests/test_dispatchers.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from unittest.mock import Mock 3 | 4 | import pytest 5 | from asynctest import CoroutineMock 6 | from asynctest import Mock as AsyncMock # flake8: NOQA 7 | 8 | from loafer.dispatchers import LoaferDispatcher 9 | from loafer.exceptions import DeleteMessage 10 | from loafer.routes import Route 11 | 12 | 13 | @pytest.fixture 14 | def provider(): 15 | return CoroutineMock(fetch_messages=CoroutineMock(return_value=['message']), 16 | confirm_message=CoroutineMock()) 17 | 18 | 19 | @pytest.fixture 20 | def route(provider): 21 | message_translator = Mock(translate=Mock(return_value={'content': 'message'})) 22 | route = AsyncMock(provider=provider, handler=Mock(), 23 | message_translator=message_translator, 24 | spec=Route) 25 | return route 26 | 27 | 28 | @pytest.mark.asyncio 29 | async def test_dispatch_message(route): 30 | route.deliver = CoroutineMock(return_value='confirmation') 31 | dispatcher = LoaferDispatcher([route]) 32 | 33 | message = 'foobar' 34 | confirmation = await dispatcher.dispatch_message(message, route) 35 | assert confirmation == 'confirmation' 36 | 37 | assert route.deliver.called 38 | route.deliver.assert_called_once_with(message) 39 | 40 | 41 | @pytest.mark.asyncio 42 | @pytest.mark.parametrize('message', [None, '']) 43 | async def test_dispatch_invalid_message(route, message): 44 | route.deliver = CoroutineMock() 45 | dispatcher = LoaferDispatcher([route]) 46 | 47 | confirmation = await dispatcher.dispatch_message(message, route) 48 | assert confirmation is False 49 | assert route.deliver.called is False 50 | 51 | 52 | @pytest.mark.asyncio 53 | async def test_dispatch_message_task_delete_message(route): 54 | route.deliver = CoroutineMock(side_effect=DeleteMessage) 55 | dispatcher = LoaferDispatcher([route]) 56 | 57 | message = 'rejected-message' 58 | confirmation = await dispatcher.dispatch_message(message, route) 59 | assert confirmation is True 60 | 61 | assert route.deliver.called 62 | route.deliver.assert_called_once_with(message) 63 | 64 | 65 | @pytest.mark.asyncio 66 | async def test_dispatch_message_task_error(route): 67 | exc = Exception() 68 | route.deliver = CoroutineMock(side_effect=exc) 69 | route.error_handler = CoroutineMock(return_value='confirmation') 70 | dispatcher = LoaferDispatcher([route]) 71 | 72 | message = 'message' 73 | confirmation = await dispatcher.dispatch_message(message, route) 74 | assert confirmation == 'confirmation' 75 | 76 | assert route.deliver.called is True 77 | route.deliver.assert_called_once_with(message) 78 | assert route.error_handler.called is True 79 | 80 | 81 | @pytest.mark.asyncio 82 | async def test_dispatch_message_task_cancel(route): 83 | route.deliver = CoroutineMock(side_effect=asyncio.CancelledError) 84 | dispatcher = LoaferDispatcher([route]) 85 | 86 | message = 'message' 87 | confirmation = await dispatcher.dispatch_message(message, route) 88 | assert confirmation is False 89 | 90 | assert route.deliver.called 91 | route.deliver.assert_called_once_with(message) 92 | 93 | 94 | @pytest.mark.asyncio 95 | async def test_message_processing(route): 96 | dispatcher = LoaferDispatcher([route]) 97 | dispatcher.dispatch_message = CoroutineMock() 98 | await dispatcher._process_message('message', route) 99 | 100 | assert dispatcher.dispatch_message.called 101 | dispatcher.dispatch_message.assert_called_once_with('message', route) 102 | assert route.provider.confirm_message.called 103 | route.provider.confirm_message.assert_called_once_with('message') 104 | 105 | 106 | @pytest.mark.asyncio 107 | async def test_dispatch_tasks(route): 108 | route.provider.fetch_messages = CoroutineMock(return_value=['message']) 109 | dispatcher = LoaferDispatcher([route]) 110 | await dispatcher._dispatch_tasks() 111 | 112 | assert route.provider.fetch_messages.called 113 | assert route.provider.confirm_message.called 114 | 115 | 116 | @pytest.mark.asyncio 117 | async def test_dispatch_without_tasks(route, event_loop): 118 | route.provider.fetch_messages = CoroutineMock(return_value=[]) 119 | dispatcher = LoaferDispatcher([route]) 120 | await dispatcher._dispatch_tasks() 121 | 122 | assert route.provider.fetch_messages.called 123 | assert route.provider.confirm_message.called is False 124 | 125 | 126 | @pytest.mark.asyncio 127 | async def test_dispatch_providers(route, event_loop): 128 | dispatcher = LoaferDispatcher([route]) 129 | dispatcher._dispatch_tasks = CoroutineMock() 130 | dispatcher.stop_providers = Mock() 131 | await dispatcher.dispatch_providers(forever=False) 132 | 133 | assert dispatcher._dispatch_tasks.called 134 | dispatcher._dispatch_tasks.assert_called_once_with() 135 | 136 | 137 | @pytest.mark.asyncio 138 | async def test_dispatch_providers_with_error(route, event_loop): 139 | route.provider.fetch_messages.side_effect = ValueError 140 | dispatcher = LoaferDispatcher([route]) 141 | with pytest.raises(ValueError): 142 | await dispatcher.dispatch_providers(forever=False) 143 | 144 | 145 | def test_dispatcher_stop(route): 146 | route.stop = Mock() 147 | dispatcher = LoaferDispatcher([route]) 148 | dispatcher.stop() 149 | assert route.stop.called 150 | -------------------------------------------------------------------------------- /tests/test_managers.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from unittest import mock 3 | 4 | import pytest 5 | 6 | from loafer.dispatchers import LoaferDispatcher 7 | from loafer.exceptions import ConfigurationError, ProviderError 8 | from loafer.managers import LoaferManager 9 | from loafer.routes import Route 10 | from loafer.runners import LoaferRunner 11 | 12 | 13 | @pytest.fixture 14 | def dummy_route(dummy_provider): 15 | return Route(dummy_provider, handler=mock.Mock()) 16 | 17 | 18 | def test_dispatcher_invalid_routes(): 19 | manager = LoaferManager(routes=[]) 20 | with pytest.raises(ConfigurationError): 21 | manager.dispatcher 22 | 23 | 24 | def test_dispatcher_invalid_route_instance(): 25 | manager = LoaferManager(routes=[mock.Mock()]) 26 | with pytest.raises(ConfigurationError): 27 | manager.dispatcher 28 | 29 | 30 | def test_dispatcher(dummy_route): 31 | manager = LoaferManager(routes=[dummy_route]) 32 | assert manager.dispatcher 33 | assert isinstance(manager.dispatcher, LoaferDispatcher) 34 | 35 | 36 | def test_default_runner(): 37 | manager = LoaferManager(routes=[]) 38 | assert manager.runner 39 | assert isinstance(manager.runner, LoaferRunner) 40 | 41 | 42 | def test_custom_runner(): 43 | runner = mock.Mock() 44 | manager = LoaferManager(routes=[], runner=runner) 45 | assert manager.runner 46 | assert isinstance(manager.runner, mock.Mock) 47 | 48 | 49 | def test_on_future_errors(): 50 | manager = LoaferManager(routes=[]) 51 | manager.runner = mock.Mock() 52 | future = asyncio.Future() 53 | future.set_exception(ProviderError) 54 | manager.on_future__errors(future) 55 | 56 | assert manager.runner.prepare_stop.called 57 | manager.runner.prepare_stop.assert_called_once_with() 58 | 59 | 60 | def test_on_future_errors_cancelled(): 61 | manager = LoaferManager(routes=[]) 62 | manager.runner = mock.Mock() 63 | future = asyncio.Future() 64 | future.cancel() 65 | manager.on_future__errors(future) 66 | 67 | assert manager.runner.prepare_stop.called 68 | manager.runner.prepare_stop.assert_called_once_with() 69 | 70 | 71 | def test_on_loop__stop(): 72 | manager = LoaferManager(routes=[]) 73 | manager.dispatcher = mock.Mock() 74 | manager._future = mock.Mock() 75 | manager.on_loop__stop() 76 | 77 | assert manager.dispatcher.stop.called 78 | assert manager._future.cancel.called 79 | -------------------------------------------------------------------------------- /tests/test_message_translator.py: -------------------------------------------------------------------------------- 1 | from loafer.message_translators import StringMessageTranslator 2 | 3 | 4 | def test_translate(): 5 | translator = StringMessageTranslator() 6 | message = translator.translate(1) 7 | assert message == {'content': '1', 'metadata': {}} 8 | 9 | message = translator.translate('test') 10 | assert message == {'content': 'test', 'metadata': {}} 11 | 12 | message = translator.translate(None) 13 | assert message == {'content': 'None', 'metadata': {}} 14 | -------------------------------------------------------------------------------- /tests/test_routes.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | import pytest 4 | from asynctest import CoroutineMock 5 | 6 | from loafer.message_translators import StringMessageTranslator 7 | from loafer.routes import Route 8 | 9 | 10 | def test_provider(dummy_provider): 11 | route = Route(dummy_provider, handler=mock.Mock()) 12 | assert route.provider is dummy_provider 13 | 14 | 15 | def test_provider_invalid(): 16 | with pytest.raises(TypeError): 17 | Route('invalid-provider', handler=mock.Mock()) 18 | 19 | 20 | def test_name(dummy_provider): 21 | route = Route(dummy_provider, handler=mock.Mock(), name='foo') 22 | assert route.name == 'foo' 23 | 24 | 25 | def test_message_translator(dummy_provider): 26 | translator = StringMessageTranslator() 27 | route = Route(dummy_provider, handler=mock.Mock(), message_translator=translator) 28 | assert isinstance(route.message_translator, StringMessageTranslator) 29 | 30 | 31 | def test_default_message_translator(dummy_provider): 32 | route = Route(dummy_provider, handler=mock.Mock()) 33 | assert route.message_translator is None 34 | 35 | 36 | def test_message_translator_invalid(dummy_provider): 37 | with pytest.raises(TypeError): 38 | Route(dummy_provider, handler=mock.Mock(), message_translator='invalid') 39 | 40 | 41 | def test_apply_message_translator(dummy_provider): 42 | translator = StringMessageTranslator() 43 | translator.translate = mock.Mock(return_value={'content': 'foobar', 'metadata': {}}) 44 | route = Route(dummy_provider, mock.Mock(), message_translator=translator) 45 | translated = route.apply_message_translator('message') 46 | assert translated['content'] == 'foobar' 47 | assert translated['metadata'] == {} 48 | assert translator.translate.called 49 | translator.translate.assert_called_once_with('message') 50 | 51 | 52 | def test_apply_message_translator_error(dummy_provider): 53 | translator = StringMessageTranslator() 54 | translator.translate = mock.Mock(return_value={'content': '', 'metadata': {}}) 55 | route = Route(dummy_provider, mock.Mock(), message_translator=translator) 56 | with pytest.raises(ValueError): 57 | route.apply_message_translator('message') 58 | assert translator.translate.called 59 | translator.translate.assert_called_once_with('message') 60 | 61 | 62 | @pytest.mark.asyncio 63 | async def test_error_handler_unset(dummy_provider): 64 | route = Route(dummy_provider, mock.Mock()) 65 | exc = TypeError() 66 | exc_info = (type(exc), exc, None) 67 | result = await route.error_handler(exc_info, 'whatever') 68 | assert result is False 69 | 70 | 71 | def test_error_handler_invalid(dummy_provider): 72 | with pytest.raises(TypeError): 73 | Route(dummy_provider, handler=mock.Mock(), error_handler='invalid') 74 | 75 | 76 | @pytest.mark.asyncio 77 | async def test_error_handler(dummy_provider): 78 | attrs = {} 79 | 80 | def error_handler(exc_info, message): 81 | attrs['exc_info'] = exc_info 82 | attrs['message'] = message 83 | return True 84 | 85 | # we cant mock regular functions in error handlers, because it will 86 | # be checked with asyncio.iscoroutinefunction() and pass as coro 87 | route = Route(dummy_provider, mock.Mock(), error_handler=error_handler) 88 | exc = TypeError() 89 | exc_info = (type(exc), exc, 'traceback') 90 | result = await route.error_handler(exc_info, 'whatever') 91 | assert result is True 92 | assert attrs['exc_info'] == exc_info 93 | assert attrs['message'] == 'whatever' 94 | 95 | 96 | @pytest.mark.asyncio 97 | async def test_error_handler_coroutine(dummy_provider): 98 | error_handler = CoroutineMock(return_value=True) 99 | route = Route(dummy_provider, mock.Mock(), error_handler=error_handler) 100 | exc = TypeError() 101 | exc_info = (type(exc), exc, 'traceback') 102 | result = await route.error_handler(exc_info, 'whatever') 103 | assert result is True 104 | assert error_handler.called 105 | error_handler.assert_called_once_with(exc_info, 'whatever') 106 | 107 | 108 | @pytest.mark.asyncio 109 | async def test_handler_class_based(dummy_provider): 110 | class handler: 111 | async def handle(self, *args, **kwargs): 112 | pass 113 | 114 | handler = handler() 115 | route = Route(dummy_provider, handler=handler) 116 | assert route.handler == handler.handle 117 | 118 | 119 | @pytest.mark.asyncio 120 | async def test_handler_class_based_invalid(dummy_provider): 121 | class handler: 122 | pass 123 | 124 | handler = handler() 125 | with pytest.raises(ValueError): 126 | Route(dummy_provider, handler=handler) 127 | 128 | 129 | @pytest.mark.asyncio 130 | async def test_handler_invalid(dummy_provider): 131 | with pytest.raises(ValueError): 132 | Route(dummy_provider, 'invalid-handler') 133 | 134 | 135 | def test_route_stop(dummy_provider): 136 | dummy_provider.stop = mock.Mock() 137 | route = Route(dummy_provider, handler=mock.Mock()) 138 | route.stop() 139 | 140 | assert dummy_provider.stop.called 141 | 142 | 143 | def test_route_stop_with_handler_stop(dummy_provider): 144 | class handler: 145 | def handle(self, *args): 146 | pass 147 | 148 | dummy_provider.stop = mock.Mock() 149 | handler = handler() 150 | handler.stop = mock.Mock() 151 | route = Route(dummy_provider, handler) 152 | route.stop() 153 | 154 | assert dummy_provider.stop.called 155 | assert handler.stop.called 156 | 157 | 158 | # FIXME: Improve all test_deliver* tests 159 | 160 | @pytest.mark.asyncio 161 | async def test_deliver(dummy_provider): 162 | attrs = {} 163 | 164 | def test_handler(*args, **kwargs): 165 | attrs['args'] = args 166 | attrs['kwargs'] = kwargs 167 | return True 168 | 169 | route = Route(dummy_provider, handler=test_handler) 170 | message = 'test' 171 | result = await route.deliver(message) 172 | 173 | assert result is True 174 | assert message in attrs['args'] 175 | 176 | 177 | @pytest.mark.asyncio 178 | async def test_deliver_with_coroutine(dummy_provider): 179 | mock_handler = CoroutineMock(return_value=False) 180 | route = Route(dummy_provider, mock_handler) 181 | message = 'test' 182 | result = await route.deliver(message) 183 | assert result is False 184 | assert mock_handler.called 185 | assert message in mock_handler.call_args[0] 186 | 187 | 188 | @pytest.mark.asyncio 189 | async def test_deliver_with_message_translator(dummy_provider): 190 | mock_handler = CoroutineMock(return_value=True) 191 | route = Route(dummy_provider, mock_handler) 192 | route.apply_message_translator = mock.Mock(return_value={'content': 'whatever', 'metadata': {}}) 193 | result = await route.deliver('test') 194 | assert result is True 195 | assert route.apply_message_translator.called 196 | assert mock_handler.called 197 | mock_handler.assert_called_once_with('whatever', {}) 198 | -------------------------------------------------------------------------------- /tests/test_runners.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | 3 | from loafer.runners import LoaferRunner 4 | 5 | 6 | @mock.patch('loafer.runners.LoaferRunner.loop', new_callable=mock.PropertyMock) 7 | def test_runner_start(loop_mock): 8 | runner = LoaferRunner() 9 | 10 | runner.start() 11 | 12 | assert loop_mock.return_value.run_forever.called 13 | 14 | 15 | @mock.patch('loafer.runners.LoaferRunner.loop', new_callable=mock.PropertyMock) 16 | def test_runner_start_with_debug(loop_mock): 17 | runner = LoaferRunner() 18 | 19 | runner.start(debug=True) 20 | 21 | loop_mock.return_value.set_debug.assert_called_once_with(enabled=True) 22 | 23 | 24 | @mock.patch('loafer.runners.LoaferRunner.loop', new_callable=mock.PropertyMock) 25 | def test_runner_start_and_stop(loop_mock): 26 | runner = LoaferRunner() 27 | runner.stop = mock.Mock() 28 | 29 | runner.start() 30 | 31 | assert runner.stop.called 32 | assert loop_mock.return_value.run_forever.called 33 | assert loop_mock.return_value.close.called 34 | 35 | 36 | @mock.patch('loafer.runners.LoaferRunner.loop', new_callable=mock.PropertyMock) 37 | def test_runner_prepare_stop(loop_mock): 38 | loop_mock.return_value.is_running.return_value = True 39 | runner = LoaferRunner() 40 | 41 | runner.prepare_stop() 42 | 43 | loop_mock.return_value.stop.assert_called_once_with() 44 | 45 | 46 | @mock.patch('loafer.runners.asyncio.get_event_loop') 47 | def test_runner_prepare_stop_already_stopped(get_loop_mock): 48 | loop = mock.Mock(is_running=mock.Mock(return_value=False)) 49 | get_loop_mock.return_value = loop 50 | runner = LoaferRunner() 51 | 52 | runner.prepare_stop() 53 | 54 | loop.is_running.assert_called_once_with() 55 | assert loop.stop.called is False 56 | 57 | 58 | @mock.patch('loafer.runners.asyncio.get_event_loop') 59 | def test_runner_stop_with_callback(loop_mock): 60 | callback = mock.Mock() 61 | runner = LoaferRunner(on_stop_callback=callback) 62 | 63 | runner.stop() 64 | 65 | assert callback.called 66 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import os 3 | import sys 4 | 5 | import pytest 6 | 7 | from loafer.utils import add_current_dir_to_syspath, import_callable 8 | 9 | 10 | def test_import_function(): 11 | func = import_callable('loafer.utils.import_callable') 12 | assert callable(func) 13 | assert inspect.isfunction(func) 14 | 15 | 16 | def test_import_class(): 17 | klass = import_callable('loafer.exceptions.ProviderError') 18 | assert klass.__name__ == 'ProviderError' 19 | assert inspect.isclass(klass) 20 | 21 | 22 | def test_error_on_method_name(): 23 | with pytest.raises(ImportError): 24 | import_callable('unittest.mock.Mock.call_count') 25 | 26 | 27 | def test_error_on_invalid_name(): 28 | with pytest.raises(ImportError): 29 | import_callable('invalid-1234') 30 | 31 | with pytest.raises(ImportError): 32 | import_callable('') 33 | 34 | 35 | def test_error_on_module(): 36 | with pytest.raises(ImportError): 37 | import_callable('examples') 38 | 39 | 40 | def test_error_on_non_callable(): 41 | with pytest.raises(ImportError): 42 | import_callable('loafer') 43 | 44 | 45 | @pytest.mark.xfail(os.getcwd() == '/tmp', run=False, 46 | reason='This test is invalid if you are at /tmp') 47 | def test_current_dir_in_syspath(): 48 | old_current = os.getcwd() 49 | os.chdir('/tmp') 50 | current = os.getcwd() 51 | if current not in sys.path: 52 | sys.path.append(current) 53 | 54 | @add_current_dir_to_syspath 55 | def inner_test(): 56 | assert current in sys.path 57 | 58 | inner_test() 59 | assert current in sys.path 60 | 61 | sys.path.remove(current) 62 | os.chdir(old_current) 63 | 64 | 65 | @pytest.mark.xfail(os.getcwd() == '/tmp', run=False, 66 | reason='This test is invalid if you are at /tmp') 67 | def test_current_dir_not_in_syspath(): 68 | old_current = os.getcwd() 69 | os.chdir('/tmp') 70 | current = os.getcwd() 71 | 72 | @add_current_dir_to_syspath 73 | def inner_test(): 74 | assert current in sys.path 75 | 76 | inner_test() 77 | assert current not in sys.path 78 | 79 | os.chdir(old_current) 80 | --------------------------------------------------------------------------------