├── test ├── __init__.py ├── conftest.py ├── util.py ├── test_configuration_async.py ├── test_listener.py ├── test_circuit.py ├── test_circuit_async.py ├── tests-old.py └── test_configuration.py ├── runtime.txt ├── docs ├── source │ ├── changelog.rst │ ├── storage.rst │ ├── aiobreaker.rst │ ├── index.rst │ ├── usage.rst │ └── conf.py ├── Makefile └── make.bat ├── aiobreaker ├── storage │ ├── __init__.py │ ├── memory.py │ ├── base.py │ └── redis.py ├── version.py ├── __init__.py ├── listener.py ├── state.py └── circuitbreaker.py ├── netlify.toml ├── .github └── workflows │ ├── publish.yml │ └── ci.yml ├── contributing.rst ├── setup.py ├── license.md ├── .gitignore ├── readme.rst └── changelog.rst /test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /runtime.txt: -------------------------------------------------------------------------------- 1 | 3.8 2 | -------------------------------------------------------------------------------- /docs/source/changelog.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../changelog.rst -------------------------------------------------------------------------------- /aiobreaker/storage/__init__.py: -------------------------------------------------------------------------------- 1 | from .memory import CircuitMemoryStorage 2 | from .redis import CircuitRedisStorage 3 | -------------------------------------------------------------------------------- /netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | publish = "docs/build" 3 | command = "pip install -e .[docs] && sphinx-build docs/source docs/build" -------------------------------------------------------------------------------- /aiobreaker/version.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.1.0" 2 | """The current version.""" 3 | 4 | short_version = ".".join(__version__.split(".")[:2]) 5 | """A short version.""" 6 | -------------------------------------------------------------------------------- /docs/source/storage.rst: -------------------------------------------------------------------------------- 1 | Storage Backends 2 | ================ 3 | 4 | .. automodule:: aiobreaker.storage 5 | 6 | Base Class 7 | ---------- 8 | 9 | .. automodule:: aiobreaker.storage.base 10 | 11 | Memory Storage 12 | -------------- 13 | 14 | .. automodule:: aiobreaker.storage.memory 15 | 16 | Redis Storage 17 | ------------- 18 | 19 | .. automodule:: aiobreaker.storage.redis 20 | -------------------------------------------------------------------------------- /docs/source/aiobreaker.rst: -------------------------------------------------------------------------------- 1 | Circuit Breaker 2 | =============== 3 | 4 | .. automodule:: aiobreaker 5 | 6 | CircuitBreaker 7 | -------------- 8 | 9 | .. automodule:: aiobreaker.circuitbreaker 10 | 11 | Listener 12 | -------- 13 | 14 | .. automodule:: aiobreaker.listener 15 | 16 | State 17 | ----- 18 | 19 | .. automodule:: aiobreaker.state 20 | 21 | Version 22 | ------- 23 | 24 | .. automodule:: aiobreaker.version 25 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. aiobreaker documentation master file, created by 2 | sphinx-quickstart on Sun Jan 13 23:49:31 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. include:: ../../readme.rst 7 | 8 | .. toctree:: 9 | :maxdepth: 2 10 | :caption: General 11 | 12 | usage 13 | changelog 14 | 15 | .. toctree:: 16 | :maxdepth: 2 17 | :caption: Reference 18 | 19 | aiobreaker 20 | storage -------------------------------------------------------------------------------- /aiobreaker/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | 3 | """ 4 | Threadsafe pure-Python implementation of the Circuit Breaker pattern, described 5 | by Michael T. Nygard in his book 'Release It!'. 6 | 7 | For more information on this and other patterns and best practices, buy the 8 | book at https://pragprog.com/titles/mnee2/release-it-second-edition/ 9 | """ 10 | 11 | from .circuitbreaker import CircuitBreaker 12 | from .listener import CircuitBreakerListener 13 | from .state import CircuitBreakerError, CircuitBreakerState 14 | from . import storage 15 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = source 8 | BUILDDIR = build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish to PyPi 2 | on: push 3 | jobs: 4 | build-n-publish: 5 | runs-on: ubuntu-latest 6 | strategy: 7 | matrix: 8 | python-version: [3.7] 9 | 10 | if: startsWith(github.ref, 'refs/tags') 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Set up Python ${{ matrix.python-version }} 14 | uses: actions/setup-python@v2 15 | with: 16 | python-version: ${{ matrix.python-version }} 17 | - name: Generate Artifacts 18 | run: >- 19 | pip install wheel 20 | python setup.py sdist bdist_wheel 21 | - name: Publish to PyPI 22 | 23 | uses: pypa/gh-action-pypi-publish@master 24 | with: 25 | password: ${{ secrets.PYPI_PASSWORD }} 26 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /test/conftest.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | 3 | from _pytest.fixtures import fixture 4 | from fakeredis import FakeStrictRedis 5 | 6 | from aiobreaker.state import CircuitBreakerState 7 | from aiobreaker.storage.memory import CircuitMemoryStorage 8 | from aiobreaker.storage.redis import CircuitRedisStorage 9 | 10 | __all__ = ('redis_storage', 'storage', 'memory_storage', 'delta') 11 | 12 | 13 | @fixture() 14 | def redis_storage(): 15 | redis = FakeStrictRedis() 16 | yield CircuitRedisStorage(CircuitBreakerState.CLOSED, redis) 17 | redis.flushall() 18 | 19 | 20 | @fixture() 21 | def memory_storage(): 22 | return CircuitMemoryStorage(CircuitBreakerState.CLOSED) 23 | 24 | 25 | @fixture(params=['memory_storage', 'redis_storage']) 26 | def storage(request): 27 | return request.getfixturevalue(request.param) 28 | 29 | 30 | @fixture() 31 | def delta(): 32 | return timedelta(seconds=1) 33 | -------------------------------------------------------------------------------- /contributing.rst: -------------------------------------------------------------------------------- 1 | Contributing to `aiobreaker` 2 | ============================ 3 | 4 | We love your input! We want to make contributing to this project as easy and transparent as possible, whether it's: 5 | 6 | - Reporting a bug 7 | - Discussing the current state of the code 8 | - Submitting a fix 9 | - Proposing new features 10 | - Becoming a maintainer 11 | 12 | Getting Started 13 | --------------- 14 | 15 | This library has no direct dependencies, so all you need is to create a venv of your choice 16 | and install the extra test dependencies via pip: 17 | 18 | .. code:: bash 19 | 20 | pyenv virtualenv aiobreaker 21 | pip install -e '.[test]' 22 | pytest test 23 | mypy test 24 | 25 | # if you'd like to build the docs 26 | pip install -e '.[docs]' 27 | sphinx-build docs/source docs/build 28 | 29 | Release 30 | ------- 31 | 32 | Releases are currently done manually using twine: 33 | 34 | .. code:: bash 35 | 36 | pip install twine wheel 37 | python setup.py sdist bdist_wheel 38 | twine upload -r pypi dist/* 39 | 40 | License 41 | ------- 42 | 43 | By contributing, you agree that your contributions will be licensed under its MIT License. -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Continuous Integration 5 | on: push 6 | jobs: 7 | test: 8 | runs-on: ubuntu-latest 9 | strategy: 10 | matrix: 11 | python-version: [3.6, 3.7, 3.8, 3.9] 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Set up Python ${{ matrix.python-version }} 16 | uses: actions/setup-python@v2 17 | with: 18 | python-version: ${{ matrix.python-version }} 19 | - name: Install dependencies 20 | run: pip install -e '.[test]' 21 | - name: Pytest 22 | run: pytest test --cov=aiobreaker --cov-report=xml 23 | - name: Upload coverage to Codecov 24 | uses: codecov/codecov-action@v1 25 | - name: PyLint 26 | run: pylint aiobreaker 27 | continue-on-error: true 28 | - name: Type Analysis 29 | run: mypy server; true 30 | continue-on-error: true 31 | - name: Dependency Warnings 32 | run: safety check 33 | - name: Security Warnings 34 | run: bandit -r aiobreaker 35 | -------------------------------------------------------------------------------- /aiobreaker/listener.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | 4 | class CircuitBreakerListener: 5 | """ 6 | Listener class used to plug code to a CircuitBreaker instance when certain events happen. 7 | 8 | todo async listener handlers 9 | """ 10 | 11 | def before_call(self, breaker: 'CircuitBreaker', func: Callable, *args, **kwargs) -> None: 12 | """ 13 | Called before a function is executed over a breaker. 14 | 15 | :param breaker: The breaker that is used. 16 | :param func: The function that is called. 17 | :param args: The args to the function. 18 | :param kwargs: The kwargs to the function. 19 | """ 20 | 21 | def failure(self, breaker: 'CircuitBreaker', exception: Exception) -> None: 22 | """ 23 | Called when a function executed over the circuit breaker 'breaker' fails. 24 | """ 25 | 26 | def success(self, breaker: 'CircuitBreaker') -> None: 27 | """ 28 | Called when a function executed over the circuit breaker 'breaker' succeeds. 29 | """ 30 | 31 | def state_change(self, breaker: 'CircuitBreaker', old: 'CircuitBreakerState', new: 'CircuitBreakerState') -> None: 32 | """ 33 | Called when the state of the circuit breaker 'breaker' changes. 34 | """ 35 | -------------------------------------------------------------------------------- /test/util.py: -------------------------------------------------------------------------------- 1 | from threading import Thread 2 | 3 | 4 | class DummyException(Exception): 5 | """ 6 | A more specific error to call during the tests. 7 | """ 8 | 9 | def __init__(self, val=0): 10 | self.val = val 11 | 12 | 13 | def func_exception(): 14 | raise DummyException() 15 | 16 | 17 | def func_succeed(): 18 | return True 19 | 20 | 21 | async def func_succeed_async(): 22 | return True 23 | 24 | 25 | async def func_exception_async(): 26 | raise DummyException() 27 | 28 | 29 | def func_succeed_counted(): 30 | def func_succeed(): 31 | func_succeed.call_count += 1 32 | return True 33 | 34 | func_succeed.call_count = 0 35 | 36 | return func_succeed 37 | 38 | 39 | def func_succeed_counted_async(): 40 | async def func_succeed_async(): 41 | func_succeed_async.call_count += 1 42 | return True 43 | 44 | func_succeed_async.call_count = 0 45 | 46 | return func_succeed_async 47 | 48 | 49 | def start_threads(target_function, n): 50 | """ 51 | Starts `n` threads that calls the target function and waits for them to finish. 52 | """ 53 | threads = [Thread(target=target_function) for _ in range(n)] 54 | 55 | for t in threads: 56 | t.start() 57 | 58 | for t in threads: 59 | t.join() 60 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:utf-8 -*- 3 | 4 | from setuptools import setup, find_packages 5 | 6 | from aiobreaker import version 7 | 8 | with open("readme.rst", "r") as fh: 9 | long_description = fh.read() 10 | 11 | test_dependencies = ['fakeredis', 'pytest>4', 'pytest-asyncio', 12 | 'mypy', 'pylint', 'safety', 'bandit', 'codecov', 'pytest-cov'] 13 | redis_dependencies = ['redis'] 14 | documentation_dependencies = [ 15 | 'sphinx', 'sphinx_rtd_theme', 'sphinx-autobuild', 'sphinx-autodoc-typehints'] 16 | 17 | setup( 18 | name='aiobreaker', 19 | version=version.__version__, 20 | url='https://github.com/arlyon/aiobreaker', 21 | license='BSD', 22 | author='Alexander Lyon', 23 | author_email='arlyon@me.com', 24 | description='Python implementation of the Circuit Breaker pattern.', 25 | long_description=long_description, 26 | long_description_content_type="text/markdown", 27 | packages=find_packages(), 28 | py_modules=['aiobreaker'], 29 | python_requires='>=3.6', 30 | install_requires=[], 31 | tests_require=test_dependencies, 32 | extras_require={ 33 | 'test': test_dependencies, 34 | 'docs': documentation_dependencies, 35 | 'redis': redis_dependencies, 36 | }, 37 | classifiers=[ 38 | 'Intended Audience :: Developers', 39 | 'License :: OSI Approved :: BSD License', 40 | 'Programming Language :: Python', 41 | 'Programming Language :: Python :: 3.6', 42 | 'Topic :: Software Development :: Libraries', 43 | ], 44 | ) 45 | -------------------------------------------------------------------------------- /license.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010-2014, Daniel Fernandes Martins 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of PyBreaker nor the names of its contributors may 15 | be used to endorse or promote products derived from this software 16 | software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /test/test_configuration_async.py: -------------------------------------------------------------------------------- 1 | from pytest import raises, mark 2 | 3 | from aiobreaker import CircuitBreaker 4 | from test.util import DummyException, func_succeed_async 5 | 6 | pytestmark = mark.asyncio 7 | 8 | async def test_call_with_no_args_async(): 9 | """ 10 | It should be able to invoke functions with no-args. 11 | """ 12 | breaker = CircuitBreaker() 13 | assert await breaker.call_async(func_succeed_async) 14 | 15 | 16 | async def test_call_with_args_async(): 17 | """ 18 | It should be able to invoke functions with args. 19 | """ 20 | 21 | async def func(arg1, arg2): 22 | return arg1, arg2 23 | 24 | breaker = CircuitBreaker() 25 | assert (42, 'abc') == await breaker.call_async(func, 42, 'abc') 26 | 27 | 28 | async def test_call_with_kwargs_async(): 29 | """ 30 | It should be able to invoke functions with kwargs. 31 | """ 32 | 33 | async def func(**kwargs): 34 | return kwargs 35 | 36 | breaker = CircuitBreaker() 37 | kwargs = {'a': 1, 'b': 2} 38 | assert kwargs == await breaker.call_async(func, **kwargs) 39 | 40 | 41 | async def test_decorator_async(): 42 | """ 43 | It should also be an async decorator. 44 | """ 45 | breaker = CircuitBreaker() 46 | 47 | @breaker 48 | async def suc(): 49 | """Docstring""" 50 | pass 51 | 52 | @breaker 53 | async def err(): 54 | """Docstring""" 55 | raise DummyException() 56 | 57 | assert 'Docstring' == suc.__doc__ 58 | assert 'Docstring' == err.__doc__ 59 | assert 'suc' == suc.__name__ 60 | assert 'err' == err.__name__ 61 | 62 | assert 0 == breaker.fail_counter 63 | 64 | with raises(DummyException): 65 | await err() 66 | 67 | assert 1 == breaker.fail_counter 68 | 69 | await suc() 70 | assert 0 == breaker.fail_counter 71 | -------------------------------------------------------------------------------- /aiobreaker/storage/memory.py: -------------------------------------------------------------------------------- 1 | from aiobreaker.state import CircuitBreakerState 2 | from .base import CircuitBreakerStorage 3 | 4 | 5 | class CircuitMemoryStorage(CircuitBreakerStorage): 6 | """ 7 | Implements a `CircuitBreakerStorage` in local memory. 8 | """ 9 | 10 | def __init__(self, state: CircuitBreakerState): 11 | """ 12 | Creates a new instance with the given `state`. 13 | """ 14 | super().__init__('memory') 15 | self._fail_counter = 0 16 | self._opened_at = None 17 | self._state = state 18 | 19 | @property 20 | def state(self) -> CircuitBreakerState: 21 | """ 22 | Returns the current circuit breaker state. 23 | """ 24 | return self._state 25 | 26 | @state.setter 27 | def state(self, state: CircuitBreakerState): 28 | """ 29 | Set the current circuit breaker state to `state`. 30 | """ 31 | self._state = state 32 | 33 | def increment_counter(self): 34 | """ 35 | Increases the failure counter by one. 36 | """ 37 | self._fail_counter += 1 38 | 39 | def reset_counter(self): 40 | """ 41 | Sets the failure counter to zero. 42 | """ 43 | self._fail_counter = 0 44 | 45 | @property 46 | def counter(self): 47 | """ 48 | Returns the current value of the failure counter. 49 | """ 50 | return self._fail_counter 51 | 52 | @property 53 | def opened_at(self): 54 | """ 55 | Returns the most recent value of when the circuit was opened. 56 | """ 57 | return self._opened_at 58 | 59 | @opened_at.setter 60 | def opened_at(self, date_time): 61 | """ 62 | Sets the most recent value of when the circuit was opened to 63 | `datetime`. 64 | """ 65 | self._opened_at = date_time 66 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Python template 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | db.sqlite3 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # Environments 87 | .env 88 | .venv 89 | env/ 90 | venv/ 91 | ENV/ 92 | env.bak/ 93 | venv.bak/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | 108 | # Compiled python code 109 | *.pyc 110 | 111 | # Pycharm directories 112 | .idea 113 | .pytest_cache 114 | -------------------------------------------------------------------------------- /readme.rst: -------------------------------------------------------------------------------- 1 | aiobreaker 2 | ========== 3 | 4 | aiobreaker is a Python implementation of the Circuit Breaker pattern, 5 | described in Michael T. Nygard's book `Release It!`_. 6 | 7 | Circuit breakers exist to allow one subsystem to fail without destroying 8 | the entire system. This is done by wrapping dangerous operations 9 | (typically integration points) with a component that can circumvent 10 | calls when the system is not healthy. 11 | 12 | This project is a fork of pybreaker_ by Daniel Fernandes Martins that 13 | replaces tornado with native asyncio, originally so I could practice 14 | packaging and learn about that shiny new ``typing`` package. 15 | 16 | .. _`Release It!`: https://pragprog.com/titles/mnee2/release-it-second-edition/ 17 | .. _pybreaker: https://github.com/danielfm/pybreaker 18 | 19 | Features 20 | -------- 21 | 22 | - Configurable list of excluded exceptions (e.g. business exceptions) 23 | - Configurable failure threshold and reset timeout 24 | - Support for several event listeners per circuit breaker 25 | - Can guard generator functions 26 | - Functions and properties for easy monitoring and management 27 | - ``asyncio`` support 28 | - Optional redis backing 29 | - Synchronous and asynchronous event listeners 30 | 31 | Requirements 32 | ------------ 33 | 34 | All you need is ``python 3.6`` or higher. 35 | 36 | Installation 37 | ------------ 38 | 39 | To install, simply download from pypi: 40 | 41 | .. code:: bash 42 | 43 | pip install aiobreaker 44 | 45 | Usage 46 | ----- 47 | 48 | The first step is to create an instance of ``CircuitBreaker`` for each 49 | integration point you want to protect against. 50 | 51 | .. code:: python 52 | 53 | from aiobreaker import CircuitBreaker 54 | 55 | # Used in database integration points 56 | db_breaker = CircuitBreaker(fail_max=5, reset_timeout=timedelta(seconds=60)) 57 | 58 | @db_breaker 59 | async def outside_integration(): 60 | """Hits the api""" 61 | ... 62 | 63 | At that point, go ahead and get familiar with the documentation. 64 | -------------------------------------------------------------------------------- /aiobreaker/storage/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from datetime import datetime 3 | 4 | from aiobreaker.state import CircuitBreakerState 5 | 6 | 7 | class CircuitBreakerStorage(ABC): 8 | """ 9 | Defines the underlying storage for a circuit breaker - the underlying 10 | implementation should be in a subclass that overrides the method this 11 | class defines. 12 | """ 13 | 14 | def __init__(self, name: str): 15 | """ 16 | Creates a new instance identified by `name`. 17 | """ 18 | self._name = name 19 | 20 | @property 21 | def name(self) -> str: 22 | """ 23 | Returns a human friendly name that identifies this state. 24 | """ 25 | return self._name 26 | 27 | @property 28 | @abstractmethod 29 | def state(self) -> CircuitBreakerState: 30 | """ 31 | Override this method to retrieve the current circuit breaker state. 32 | """ 33 | 34 | @state.setter 35 | @abstractmethod 36 | def state(self, state: CircuitBreakerState): 37 | """ 38 | Override this method to set the current circuit breaker state. 39 | """ 40 | 41 | @abstractmethod 42 | def increment_counter(self): 43 | """ 44 | Override this method to increase the failure counter by one. 45 | """ 46 | 47 | @abstractmethod 48 | def reset_counter(self): 49 | """ 50 | Override this method to set the failure counter to zero. 51 | """ 52 | 53 | @property 54 | @abstractmethod 55 | def counter(self) -> int: 56 | """ 57 | Override this method to retrieve the current value of the failure counter. 58 | """ 59 | 60 | @property 61 | @abstractmethod 62 | def opened_at(self) -> datetime: 63 | """ 64 | Override this method to retrieve the most recent value of when the 65 | circuit was opened. 66 | """ 67 | 68 | @opened_at.setter 69 | @abstractmethod 70 | def opened_at(self, date_time: datetime): 71 | """ 72 | Override this method to set the most recent value of when the circuit 73 | was opened. 74 | """ 75 | -------------------------------------------------------------------------------- /test/test_listener.py: -------------------------------------------------------------------------------- 1 | from _pytest.python_api import raises 2 | 3 | from aiobreaker import CircuitBreaker 4 | from aiobreaker.listener import CircuitBreakerListener 5 | from aiobreaker.state import CircuitBreakerState 6 | from test.util import DummyException, func_succeed, func_exception 7 | 8 | 9 | def test_transition_events(storage): 10 | """It should call the appropriate functions on every state transition.""" 11 | 12 | class Listener(CircuitBreakerListener): 13 | def __init__(self): 14 | self.out = [] 15 | 16 | def state_change(self, breaker, old, new): 17 | assert breaker 18 | self.out.append((old.state, new.state)) 19 | 20 | listener = Listener() 21 | breaker = CircuitBreaker(listeners=(listener,), state_storage=storage) 22 | assert CircuitBreakerState.CLOSED == breaker.current_state 23 | 24 | breaker.open() 25 | assert CircuitBreakerState.OPEN == breaker.current_state 26 | 27 | breaker.half_open() 28 | assert CircuitBreakerState.HALF_OPEN == breaker.current_state 29 | 30 | breaker.close() 31 | assert CircuitBreakerState.CLOSED == breaker.current_state 32 | 33 | assert [ 34 | (CircuitBreakerState.CLOSED, CircuitBreakerState.OPEN), 35 | (CircuitBreakerState.OPEN, CircuitBreakerState.HALF_OPEN), 36 | (CircuitBreakerState.HALF_OPEN, CircuitBreakerState.CLOSED) 37 | ] == listener.out 38 | 39 | 40 | def test_call_events(storage): 41 | """It should call the appropriate functions on every successful/failed call. 42 | """ 43 | 44 | class Listener(CircuitBreakerListener): 45 | def __init__(self): 46 | self.out = [] 47 | 48 | def before_call(self, breaker, func, *args, **kwargs): 49 | assert breaker 50 | self.out.append("CALL") 51 | 52 | def success(self, breaker): 53 | assert breaker 54 | self.out.append("SUCCESS") 55 | 56 | def failure(self, breaker, exception): 57 | assert breaker 58 | assert isinstance(exception, DummyException) 59 | self.out.append("FAILURE") 60 | 61 | listener = Listener() 62 | breaker = CircuitBreaker(listeners=(listener,), state_storage=storage) 63 | 64 | assert breaker.call(func_succeed) 65 | 66 | with raises(DummyException): 67 | breaker.call(func_exception) 68 | 69 | assert ["CALL", "SUCCESS", "CALL", "FAILURE"] == listener.out 70 | -------------------------------------------------------------------------------- /changelog.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | FORK 2.0.0 (TBD) 5 | 6 | * Move breaker state to enum 7 | 8 | FORK 1.1.0 (Jan 14, 2019) 9 | 10 | * Add logic to stop calling decorator trigger twice 11 | * Fix bug with timeout window growing with additional breakers defined (Thanks @shawndrape) 12 | * Remove threading support (unneeded with asyncio) 13 | 14 | FORK 1.0.0 (Aug 12, 2018) 15 | 16 | * Move over to asyncio 17 | * Drop < 3.4 support 18 | * Drop tornado support 19 | * Async call support 20 | 21 | Version 0.4.4 (May 21, 2018) 22 | 23 | * Fix PyPI release 24 | 25 | Version 0.4.3 (May 21, 2018) 26 | 27 | * Re-initialize state on Redis if missing (Thanks @scascketta!) 28 | * Add trigger exception into the CircuitBreakerError (Thanks @tczhaodachuan!) 29 | 30 | Version 0.4.2 (November 9, 2017) 31 | 32 | * Add optional name to CircuitBreaker (Thanks @chrisvaughn!) 33 | 34 | Version 0.4.1 (October 2, 2017) 35 | 36 | * Get initial CircuitBreaker state from state_storage (Thanks @alukach!) 37 | 38 | Version 0.4.0 (June 23, 2017) 39 | 40 | * Added optional support for asynchronous Tornado calls (Thanks @jeffrand!) 41 | * Fixed typo (issue #19) (Thanks @jeffrand!) 42 | 43 | 44 | Version 0.3.3 (June 2, 2017) 45 | 46 | * Fixed bug that caused pybreaker to break (!) if redis package was not 47 | present (Thanks @phillbaker!) 48 | 49 | 50 | Version 0.3.2 (June 1, 2017) 51 | 52 | * Added support for optional Redis backing (Thanks @phillbaker!) 53 | * Fixed: Should listener.failure be called when the circuit is closed 54 | and a call fails? (Thanks @sj175 for the report!) 55 | * Fixed: Wrapped function is called twice during successful call in open 56 | state (Thanks @jstordeur!) 57 | 58 | 59 | Version 0.3.1 (January 25, 2017) 60 | 61 | * Added support for optional Redis backing 62 | 63 | 64 | Version 0.3.0 (September 1, 2016) 65 | 66 | * Fixed generator issue. (Thanks @dpdornseifer!) 67 | 68 | 69 | Version 0.2.3 (July 25, 2014) 70 | 71 | * Added support to generator functions. (Thanks @mauriciosl!) 72 | 73 | 74 | Version 0.2.1 (October 23, 2010) 75 | 76 | * Fixed a few concurrency bugs. 77 | 78 | 79 | Version 0.2 (October 20, 2010) 80 | 81 | * Several API changes, breaks backwards compatibility. 82 | * New CircuitBreakerListener class that allows the user to listen to events in 83 | a circuit breaker without the need to subclass CircuitBreaker. 84 | * Decorator now uses 'functools.wraps' to avoid loss of information on decorated 85 | functions. 86 | 87 | 88 | Version 0.1.1 (October 17, 2010) 89 | 90 | * Instance of CircuitBreaker can now be used as a decorator. 91 | * Python 2.6+ is now required in order to make the same code base compatible 92 | with Python 3+. 93 | 94 | 95 | Version 0.1 (October 16, 2010) 96 | 97 | * First public release. 98 | -------------------------------------------------------------------------------- /docs/source/usage.rst: -------------------------------------------------------------------------------- 1 | Usage 2 | ----- 3 | 4 | .. include:: ../../readme.rst 5 | :start-after: 6 | Usage 7 | ----- 8 | 9 | Calling Functions With The Breaker 10 | ================================== 11 | 12 | There are two primary ways of calling functions using the 13 | breaker. The first is to wrap the functions you want in a 14 | decorator, as seen above. 15 | 16 | Using a :class:`~aiobreaker.circuitbreaker.CircuitBreaker` 17 | to decorate the functions you wish to use it with is a 18 | "set it and forget it" approach and works quite well 19 | however it requires you to `always` use the circuit breaker. 20 | Another option is the :func:`~aiobreaker.circuitbreaker.CircuitBreaker.call` 21 | or :func:`~aiobreaker.circuitbreaker.CircuitBreaker.call_async` 22 | functions which allow you to route functions through a breaker at will. 23 | 24 | .. code:: python 25 | 26 | from aiobreaker import CircuitBreaker 27 | 28 | breaker = CircuitBreaker() 29 | bar = breaker.call(foo) 30 | async_bar = await breaker.call_async(async_foo) 31 | 32 | By default there is a check to make sure if you were to pass 33 | a decorated function into a breaker's `call` method that the 34 | validation logic isn't performed twice. To disable this, set 35 | the ``ignore_on_call`` parameter to ``False`` on the decorator 36 | :func:`~aiobreaker.circuitbreaker.CircuitBreaker.__call__`. 37 | 38 | Manually Setting Or Resetting The Circuit 39 | ========================================= 40 | 41 | The circuit can be manually opened or closed if needed. 42 | :func:`~aiobreaker.circuitbreaker.CircuitBreaker.open` will open 43 | the circuit causing subsequent calls to fail until the 44 | :attr:`~aiobreaker.circuitbreaker.CircuitBreaker.timeout_duration` 45 | elapses. Similarly, :func:`~aiobreaker.circuitbreaker.CircuitBreaker.close` 46 | will override the checks and immediately close the breaker, causing 47 | further calls to succeed again. 48 | 49 | Listening For Events On The Breaker 50 | =================================== 51 | 52 | To listen for events, you must implement a listener. A listener 53 | is anything that subclasses and overrides the functions on the 54 | :class:`~aiobreaker.listener.CircuitBreakerListener` class. 55 | 56 | .. code:: python 57 | 58 | from aiobreaker import CircuitBreaker, CircuitBreakerListener 59 | 60 | class LogListener(CircuitBreakerListener): 61 | 62 | def state_change(self, breaker, old, new): 63 | logger.info(f"{old.state} -> {new.state}") 64 | 65 | 66 | breaker = CircuitBreaker(listeners=[LogListener()]) 67 | 68 | Listeners can be added and removed on the fly with the 69 | :func:`~aiobreaker.circuitbreaker.CircuitBreaker.add_listener` and 70 | :func:`~aiobreaker.circuitbreaker.CircuitBreaker.remove_listener` 71 | functions. 72 | 73 | 74 | Ignoring Specific Exceptions 75 | ============================ 76 | 77 | If there are specific exceptions that shouldn't be considered 78 | by the breaker, they can be specified, added, and removed in 79 | a similar way to the way Listeners are handled. 80 | 81 | .. code:: python 82 | 83 | from aiobreaker import CircuitBreaker 84 | import sqlite3 85 | 86 | breaker = CircuitBreaker(exclude=[sqlite3.Error]) 87 | 88 | Exceptions can be ignored on the fly with the 89 | :func:`~aiobreaker.circuitbreaker.CircuitBreaker.add_excluded_exception` and 90 | :func:`~aiobreaker.circuitbreaker.CircuitBreaker.remove_excluded_exception` 91 | functions. 92 | 93 | So as to cover cases where the exception class alone is not enough to determine whether it represents a system error, you may also pass a callable rather than a type: 94 | 95 | .. code:: python 96 | 97 | db_breaker = CircuitBreaker(exclude=[lambda e: type(e) == HTTPError and e.status_code < 500]) 98 | 99 | You may mix types and filter callables freely. 100 | -------------------------------------------------------------------------------- /aiobreaker/storage/redis.py: -------------------------------------------------------------------------------- 1 | import calendar 2 | import logging 3 | import time 4 | from datetime import datetime 5 | 6 | from aiobreaker.state import CircuitBreakerState 7 | from .base import CircuitBreakerStorage 8 | 9 | try: 10 | from redis.exceptions import RedisError 11 | except ImportError: 12 | HAS_REDIS_SUPPORT = False 13 | RedisError = None 14 | else: 15 | HAS_REDIS_SUPPORT = True 16 | 17 | 18 | class CircuitRedisStorage(CircuitBreakerStorage): 19 | """ 20 | Implements a `CircuitBreakerStorage` using redis. 21 | """ 22 | 23 | BASE_NAMESPACE = 'aiobreaker' 24 | 25 | logger = logging.getLogger(__name__) 26 | 27 | def __init__(self, state: CircuitBreakerState, redis_object, namespace=None, 28 | fallback_circuit_state=CircuitBreakerState.CLOSED): 29 | """ 30 | Creates a new instance with the given `state` and `redis` object. The 31 | redis object should be similar to pyredis' StrictRedis class. If there 32 | are any connection issues with redis, the `fallback_circuit_state` is 33 | used to determine the state of the circuit. 34 | """ 35 | 36 | # Module does not exist, so this feature is not available 37 | if not HAS_REDIS_SUPPORT: 38 | raise ImportError("CircuitRedisStorage can only be used if the required dependencies exist") 39 | 40 | super(CircuitRedisStorage, self).__init__('redis') 41 | 42 | try: 43 | self.RedisError = __import__('redis').exceptions.RedisError 44 | except ImportError: 45 | # Module does not exist, so this feature is not available 46 | raise ImportError("CircuitRedisStorage can only be used if 'redis' is available") 47 | 48 | self._redis = redis_object 49 | self._namespace_name = namespace 50 | self._fallback_circuit_state = fallback_circuit_state 51 | self._initial_state = state 52 | 53 | self._initialize_redis_state(self._initial_state) 54 | 55 | def _initialize_redis_state(self, state: CircuitBreakerState): 56 | self._redis.setnx(self._namespace('fail_counter'), 0) 57 | self._redis.setnx(self._namespace('state'), state.name) 58 | 59 | @property 60 | def state(self): 61 | """ 62 | Returns the current circuit breaker state. 63 | 64 | If the circuit breaker state on Redis is missing, re-initialize it 65 | with the fallback circuit state and reset the fail counter. 66 | """ 67 | try: 68 | state_bytes = self._redis.get(self._namespace('state')) 69 | except self.RedisError: 70 | self.logger.error('RedisError: falling back to default circuit state', exc_info=True) 71 | return self._fallback_circuit_state 72 | 73 | state = self._fallback_circuit_state 74 | if state_bytes is not None: 75 | state = state_bytes.decode('utf-8') 76 | else: 77 | # state retrieved from redis was missing, so we re-initialize 78 | # the circuit breaker state on redis 79 | self._initialize_redis_state(self._fallback_circuit_state) 80 | 81 | return getattr(CircuitBreakerState, state) 82 | 83 | @state.setter 84 | def state(self, state): 85 | """ 86 | Set the current circuit breaker state to `state`. 87 | """ 88 | try: 89 | self._redis.set(self._namespace('state'), state.name) 90 | except self.RedisError: 91 | self.logger.error('RedisError', exc_info=True) 92 | 93 | def increment_counter(self): 94 | """ 95 | Increases the failure counter by one. 96 | """ 97 | try: 98 | self._redis.incr(self._namespace('fail_counter')) 99 | except self.RedisError: 100 | self.logger.error('RedisError', exc_info=True) 101 | 102 | def reset_counter(self): 103 | """ 104 | Sets the failure counter to zero. 105 | """ 106 | try: 107 | self._redis.set(self._namespace('fail_counter'), 0) 108 | except self.RedisError: 109 | self.logger.error('RedisError', exc_info=True) 110 | 111 | @property 112 | def counter(self): 113 | """ 114 | Returns the current value of the failure counter. 115 | """ 116 | try: 117 | value = self._redis.get(self._namespace('fail_counter')) 118 | if value: 119 | return int(value) 120 | else: 121 | return 0 122 | except self.RedisError: 123 | self.logger.error('RedisError: Assuming no errors', exc_info=True) 124 | return 0 125 | 126 | @property 127 | def opened_at(self): 128 | """ 129 | Returns a datetime object of the most recent value of when the circuit 130 | was opened. 131 | """ 132 | try: 133 | timestamp = self._redis.get(self._namespace('opened_at')) 134 | if timestamp: 135 | return datetime(*time.gmtime(int(timestamp))[:6]) 136 | except self.RedisError: 137 | self.logger.error('RedisError', exc_info=True) 138 | return None 139 | 140 | @opened_at.setter 141 | def opened_at(self, now): 142 | """ 143 | Atomically sets the most recent value of when the circuit was opened 144 | to `now`. Stored in redis as a simple integer of unix epoch time. 145 | To avoid timezone issues between different systems, the passed in 146 | datetime should be in UTC. 147 | """ 148 | try: 149 | key = self._namespace('opened_at') 150 | 151 | def set_if_greater(pipe): 152 | current_value = pipe.get(key) 153 | next_value = int(calendar.timegm(now.timetuple())) 154 | pipe.multi() 155 | if not current_value or next_value > int(current_value): 156 | pipe.set(key, next_value) 157 | 158 | self._redis.transaction(set_if_greater, key) 159 | except self.RedisError: 160 | self.logger.error('RedisError', exc_info=True) 161 | 162 | def _namespace(self, key): 163 | name_parts = [self.BASE_NAMESPACE, key] 164 | if self._namespace_name: 165 | name_parts.insert(0, self._namespace_name) 166 | 167 | return ':'.join(name_parts) 168 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/master/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | import os 16 | import sys 17 | 18 | sys.path.insert(0, os.path.abspath('../..')) 19 | 20 | from aiobreaker.version import short_version, __version__ 21 | 22 | # -- Project information ----------------------------------------------------- 23 | 24 | project = 'aiobreaker' 25 | copyright = '2019, Daniel Martins, Alexander Lyon' 26 | author = 'Daniel Martins, Alexander Lyon' 27 | 28 | # The short X.Y version 29 | version = short_version 30 | # The full version, including alpha/beta/rc tags 31 | release = __version__ 32 | 33 | # -- General configuration --------------------------------------------------- 34 | 35 | # If your documentation needs a minimal Sphinx version, state it here. 36 | # 37 | # needs_sphinx = '1.0' 38 | 39 | # Add any Sphinx extension module names here, as strings. They can be 40 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 41 | # ones. 42 | extensions = [ 43 | 'sphinx.ext.autodoc', 44 | 'sphinx.ext.doctest', 45 | 'sphinx.ext.intersphinx', 46 | 'sphinx.ext.viewcode', 47 | 'sphinx_autodoc_typehints', 48 | ] 49 | 50 | # Add any paths that contain templates here, relative to this directory. 51 | templates_path = ['_templates'] 52 | 53 | # The suffix(es) of source filenames. 54 | # You can specify multiple suffix as a list of string: 55 | # 56 | # source_suffix = ['.rst', '.md'] 57 | source_suffix = '.rst' 58 | 59 | # The master toctree document. 60 | master_doc = 'index' 61 | 62 | # The language for content autogenerated by Sphinx. Refer to documentation 63 | # for a list of supported languages. 64 | # 65 | # This is also used if you do content translation via gettext catalogs. 66 | # Usually you set "language" from the command line for these cases. 67 | language = None 68 | 69 | # List of patterns, relative to source directory, that match files and 70 | # directories to ignore when looking for source files. 71 | # This pattern also affects html_static_path and html_extra_path. 72 | exclude_patterns = [] 73 | 74 | # The name of the Pygments (syntax highlighting) style to use. 75 | pygments_style = None 76 | 77 | # -- Options for HTML output ------------------------------------------------- 78 | 79 | # The theme to use for HTML and HTML Help pages. See the documentation for 80 | # a list of builtin themes. 81 | # 82 | html_theme = 'sphinx_rtd_theme' 83 | 84 | # Theme options are theme-specific and customize the look and feel of a theme 85 | # further. For a list of options available for each theme, see the 86 | # documentation. 87 | # 88 | # html_theme_options = {} 89 | 90 | # Add any paths that contain custom static files (such as style sheets) here, 91 | # relative to this directory. They are copied after the builtin static files, 92 | # so a file named "default.css" will overwrite the builtin "default.css". 93 | html_static_path = ['_static'] 94 | 95 | # Custom sidebar templates, must be a dictionary that maps document names 96 | # to template names. 97 | # 98 | # The default sidebars (for documents that don't match any pattern) are 99 | # defined by theme itself. Builtin themes are using these templates by 100 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 101 | # 'searchbox.html']``. 102 | # 103 | # html_sidebars = {} 104 | 105 | 106 | # -- Options for HTMLHelp output --------------------------------------------- 107 | 108 | # Output file base name for HTML help builder. 109 | htmlhelp_basename = 'aiobreakerdoc' 110 | 111 | # -- Options for LaTeX output ------------------------------------------------ 112 | 113 | latex_elements = { 114 | # The paper size ('letterpaper' or 'a4paper'). 115 | # 116 | # 'papersize': 'letterpaper', 117 | 118 | # The font size ('10pt', '11pt' or '12pt'). 119 | # 120 | # 'pointsize': '10pt', 121 | 122 | # Additional stuff for the LaTeX preamble. 123 | # 124 | # 'preamble': '', 125 | 126 | # Latex figure (float) alignment 127 | # 128 | # 'figure_align': 'htbp', 129 | } 130 | 131 | # Grouping the document tree into LaTeX files. List of tuples 132 | # (source start file, target name, title, 133 | # author, documentclass [howto, manual, or own class]). 134 | latex_documents = [ 135 | (master_doc, 'aiobreaker.tex', 'aiobreaker Documentation', 136 | 'Daniel Martins, Alexander Lyon', 'manual'), 137 | ] 138 | 139 | # -- Options for manual page output ------------------------------------------ 140 | 141 | # One entry per manual page. List of tuples 142 | # (source start file, name, description, authors, manual section). 143 | man_pages = [ 144 | (master_doc, 'aiobreaker', 'aiobreaker Documentation', 145 | [author], 1) 146 | ] 147 | 148 | # -- Options for Texinfo output ---------------------------------------------- 149 | 150 | # Grouping the document tree into Texinfo files. List of tuples 151 | # (source start file, target name, title, author, 152 | # dir menu entry, description, category) 153 | texinfo_documents = [ 154 | (master_doc, 'aiobreaker', 'aiobreaker Documentation', 155 | author, 'aiobreaker', 'One line description of project.', 156 | 'Miscellaneous'), 157 | ] 158 | 159 | # -- Options for Epub output ------------------------------------------------- 160 | 161 | # Bibliographic Dublin Core info. 162 | epub_title = project 163 | 164 | # The unique identifier of the text. This can be a ISBN number 165 | # or the project homepage. 166 | # 167 | # epub_identifier = '' 168 | 169 | # A unique identification for the text. 170 | # 171 | # epub_uid = '' 172 | 173 | # A list of files that should not be packed into the epub file. 174 | epub_exclude_files = ['search.html'] 175 | 176 | # -- Extension configuration ------------------------------------------------- 177 | 178 | autodoc_default_options = { 179 | 'members': None, 180 | 'member-order': 'bysource', 181 | 'special-members': '__init__,__call__,__await__', 182 | 'undoc-members': None, 183 | 'exclude-members': '__weakref__', 184 | 'show-inheritance': None 185 | } 186 | 187 | # -- Options for intersphinx extension --------------------------------------- 188 | 189 | # Example configuration for intersphinx: refer to the Python standard library. 190 | intersphinx_mapping = { 191 | 'python': ('https://docs.python.org/3', None), 192 | } 193 | -------------------------------------------------------------------------------- /test/test_circuit.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from time import sleep 3 | 4 | from pytest import raises 5 | 6 | from aiobreaker import CircuitBreaker, CircuitBreakerError 7 | from aiobreaker.state import CircuitBreakerState 8 | from aiobreaker.storage.memory import CircuitMemoryStorage 9 | from test.util import func_exception, func_succeed, DummyException, func_succeed_counted 10 | 11 | 12 | def test_successful_call(storage): 13 | """It should keep the circuit closed after a successful call.""" 14 | 15 | breaker = CircuitBreaker(state_storage=storage) 16 | assert breaker.call(func_succeed) 17 | assert 0 == breaker.fail_counter 18 | assert CircuitBreakerState.CLOSED == breaker.current_state 19 | 20 | 21 | def test_one_failed_call(storage): 22 | """It should keep the circuit closed after a few failures.""" 23 | 24 | breaker = CircuitBreaker(state_storage=storage) 25 | 26 | with raises(DummyException): 27 | breaker.call(func_exception) 28 | 29 | assert 1 == breaker.fail_counter 30 | assert CircuitBreakerState.CLOSED == breaker.current_state 31 | 32 | 33 | def test_one_successful_call_after_failed_call(storage): 34 | """It should keep the circuit closed after few mixed outcomes.""" 35 | 36 | breaker = CircuitBreaker(state_storage=storage) 37 | 38 | with raises(DummyException): 39 | breaker.call(func_exception) 40 | 41 | assert 1 == breaker.fail_counter 42 | 43 | assert breaker.call(func_succeed) 44 | assert 0 == breaker.fail_counter 45 | assert CircuitBreakerState.CLOSED == breaker.current_state 46 | 47 | 48 | def test_several_failed_calls(storage): 49 | """It should open the circuit after multiple failures.""" 50 | 51 | breaker = CircuitBreaker(state_storage=storage, fail_max=3) 52 | 53 | with raises(DummyException): 54 | breaker.call(func_exception) 55 | 56 | with raises(DummyException): 57 | breaker.call(func_exception) 58 | 59 | # Circuit should open 60 | with raises(CircuitBreakerError): 61 | breaker.call(func_exception) 62 | 63 | assert breaker.fail_counter == 3 64 | assert breaker.current_state == CircuitBreakerState.OPEN 65 | 66 | 67 | def test_failed_call_after_timeout(storage, delta): 68 | """It should half-open the circuit after timeout and close immediately on fail.""" 69 | 70 | breaker = CircuitBreaker(fail_max=3, timeout_duration=delta, state_storage=storage) 71 | 72 | with raises(DummyException): 73 | breaker.call(func_exception) 74 | 75 | with raises(DummyException): 76 | breaker.call(func_exception) 77 | 78 | assert CircuitBreakerState.CLOSED == breaker.current_state 79 | 80 | # Circuit should open 81 | with raises(CircuitBreakerError): 82 | breaker.call(func_exception) 83 | 84 | assert 3 == breaker.fail_counter 85 | 86 | sleep(delta.total_seconds() * 2) 87 | 88 | # Circuit should open again 89 | with raises(CircuitBreakerError): 90 | breaker.call(func_exception) 91 | 92 | assert 4 == breaker.fail_counter 93 | assert CircuitBreakerState.OPEN == breaker.current_state 94 | 95 | 96 | def test_successful_after_timeout(storage, delta): 97 | """It should close the circuit when a call succeeds after timeout.""" 98 | 99 | breaker = CircuitBreaker(fail_max=3, timeout_duration=delta, state_storage=storage) 100 | func_succeed = func_succeed_counted() 101 | 102 | with raises(DummyException): 103 | breaker.call(func_exception) 104 | 105 | with raises(DummyException): 106 | breaker.call(func_exception) 107 | 108 | assert CircuitBreakerState.CLOSED == breaker.current_state 109 | 110 | # Circuit should open 111 | with raises(CircuitBreakerError): 112 | breaker.call(func_exception) 113 | 114 | assert CircuitBreakerState.OPEN == breaker.current_state 115 | 116 | with raises(CircuitBreakerError): 117 | breaker.call(func_succeed) 118 | 119 | assert 3 == breaker.fail_counter 120 | 121 | # Wait for timeout, at least a second since redis rounds to a second 122 | sleep(delta.total_seconds() * 2) 123 | 124 | # Circuit should close again 125 | assert breaker.call(func_succeed) 126 | assert 0 == breaker.fail_counter 127 | assert CircuitBreakerState.CLOSED == breaker.current_state 128 | assert 1 == func_succeed.call_count 129 | 130 | 131 | def test_failed_call_when_half_open(storage): 132 | """It should open the circuit when a call fails in half-open state.""" 133 | 134 | breaker = CircuitBreaker(state_storage=storage) 135 | 136 | breaker.half_open() 137 | assert 0 == breaker.fail_counter 138 | assert CircuitBreakerState.HALF_OPEN == breaker.current_state 139 | 140 | with raises(CircuitBreakerError): 141 | breaker.call(func_exception) 142 | 143 | assert 1 == breaker.fail_counter 144 | assert CircuitBreakerState.OPEN == breaker.current_state 145 | 146 | 147 | def test_successful_call_when_half_open(storage): 148 | """It should close the circuit when a call succeeds in half-open state.""" 149 | 150 | breaker = CircuitBreaker(state_storage=storage) 151 | 152 | breaker.half_open() 153 | assert 0 == breaker.fail_counter 154 | assert CircuitBreakerState.HALF_OPEN == breaker.current_state 155 | 156 | # Circuit should open 157 | assert breaker.call(func_succeed) 158 | assert 0 == breaker.fail_counter 159 | assert CircuitBreakerState.CLOSED == breaker.current_state 160 | 161 | 162 | def test_state_opened_at_not_reset_during_creation(): 163 | for state in CircuitBreakerState: 164 | storage = CircuitMemoryStorage(state) 165 | now = datetime.now() 166 | storage.opened_at = now 167 | 168 | breaker = CircuitBreaker(state_storage=storage) 169 | assert breaker.state.state == state 170 | assert storage.opened_at == now 171 | 172 | 173 | def test_close(storage): 174 | """It should allow the circuit to be closed manually.""" 175 | 176 | breaker = CircuitBreaker(fail_max=3, state_storage=storage) 177 | 178 | breaker.open() 179 | assert 0 == breaker.fail_counter 180 | assert CircuitBreakerState.OPEN == breaker.current_state 181 | 182 | breaker.close() 183 | assert 0 == breaker.fail_counter 184 | assert CircuitBreakerState.CLOSED == breaker.current_state 185 | 186 | 187 | def test_generator(storage): 188 | """It should support generator values.""" 189 | 190 | breaker = CircuitBreaker(state_storage=storage) 191 | 192 | @breaker 193 | def func_yield_succeed(): 194 | """Docstring""" 195 | yield True 196 | 197 | @breaker 198 | def func_yield_exception(): 199 | """Docstring""" 200 | x = yield True 201 | raise DummyException(x) 202 | 203 | s = func_yield_succeed() 204 | e = func_yield_exception() 205 | next(e) 206 | 207 | with raises(DummyException): 208 | e.send(True) 209 | 210 | assert 1 == breaker.fail_counter 211 | assert next(s) 212 | 213 | with raises(StopIteration): 214 | next(s) 215 | 216 | assert 0 == breaker.fail_counter 217 | -------------------------------------------------------------------------------- /test/test_circuit_async.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from asyncio import sleep 3 | 4 | from pytest import mark, raises 5 | 6 | from aiobreaker import CircuitBreaker, CircuitBreakerError 7 | from aiobreaker.state import CircuitBreakerState 8 | from test.util import func_succeed_async, DummyException, func_exception_async, func_succeed_counted_async 9 | 10 | pytestmark = mark.asyncio 11 | 12 | 13 | async def test_successful_call_async(storage): 14 | """ 15 | It should keep the circuit closed after a successful call. 16 | """ 17 | breaker = CircuitBreaker(state_storage=storage) 18 | assert await breaker.call_async(func_succeed_async) 19 | assert 0 == breaker.fail_counter 20 | assert CircuitBreakerState.CLOSED == breaker.current_state 21 | 22 | 23 | async def test_one_failed_call(storage): 24 | """It should keep the circuit closed after a few failures.""" 25 | 26 | breaker = CircuitBreaker(state_storage=storage) 27 | 28 | with raises(DummyException): 29 | await breaker.call_async(func_exception_async) 30 | 31 | assert 1 == breaker.fail_counter 32 | assert CircuitBreakerState.CLOSED == breaker.current_state 33 | 34 | 35 | async def test_one_successful_call_after_failed_call(storage): 36 | """It should keep the circuit closed after few mixed outcomes.""" 37 | 38 | breaker = CircuitBreaker(state_storage=storage) 39 | 40 | with raises(DummyException): 41 | await breaker.call_async(func_exception_async) 42 | 43 | assert 1 == breaker.fail_counter 44 | 45 | assert await breaker.call_async(func_succeed_async) 46 | assert 0 == breaker.fail_counter 47 | assert CircuitBreakerState.CLOSED == breaker.current_state 48 | 49 | 50 | async def test_several_failed_calls(storage): 51 | """It should open the circuit after multiple failures.""" 52 | 53 | breaker = CircuitBreaker(state_storage=storage, fail_max=3) 54 | 55 | with raises(DummyException): 56 | await breaker.call_async(func_exception_async) 57 | 58 | with raises(DummyException): 59 | await breaker.call_async(func_exception_async) 60 | 61 | # Circuit should open 62 | with raises(CircuitBreakerError): 63 | await breaker.call_async(func_exception_async) 64 | 65 | assert breaker.fail_counter == 3 66 | assert breaker.current_state == CircuitBreakerState.OPEN 67 | 68 | 69 | async def test_failed_call_after_timeout(storage, delta): 70 | """It should half-open the circuit after timeout and close immediately on fail.""" 71 | 72 | breaker = CircuitBreaker(fail_max=3, timeout_duration=delta, state_storage=storage) 73 | 74 | with raises(DummyException): 75 | await breaker.call_async(func_exception_async) 76 | 77 | with raises(DummyException): 78 | await breaker.call_async(func_exception_async) 79 | 80 | assert CircuitBreakerState.CLOSED == breaker.current_state 81 | 82 | # Circuit should open 83 | with raises(CircuitBreakerError): 84 | await breaker.call_async(func_exception_async) 85 | 86 | assert 3 == breaker.fail_counter 87 | 88 | await sleep(delta.total_seconds() * 2) 89 | 90 | # Circuit should open again 91 | with raises(CircuitBreakerError): 92 | await breaker.call_async(func_exception_async) 93 | 94 | assert 4 == breaker.fail_counter 95 | assert CircuitBreakerState.OPEN == breaker.current_state 96 | 97 | 98 | async def test_successful_after_timeout(storage, delta): 99 | """It should close the circuit when a call succeeds after timeout.""" 100 | 101 | breaker = CircuitBreaker(fail_max=3, timeout_duration=delta, state_storage=storage) 102 | func_succeed_async = func_succeed_counted_async() 103 | 104 | with raises(DummyException): 105 | await breaker.call_async(func_exception_async) 106 | 107 | with raises(DummyException): 108 | await breaker.call_async(func_exception_async) 109 | 110 | assert CircuitBreakerState.CLOSED == breaker.current_state 111 | 112 | # Circuit should open 113 | with raises(CircuitBreakerError): 114 | await breaker.call_async(func_exception_async) 115 | 116 | assert CircuitBreakerState.OPEN == breaker.current_state 117 | 118 | with raises(CircuitBreakerError): 119 | await breaker.call_async(func_succeed_async) 120 | 121 | assert 3 == breaker.fail_counter 122 | 123 | # Wait for timeout, at least a second since redis rounds to a second 124 | await sleep(delta.total_seconds() * 2) 125 | 126 | # Circuit should close again 127 | assert await breaker.call_async(func_succeed_async) 128 | assert 0 == breaker.fail_counter 129 | assert CircuitBreakerState.CLOSED == breaker.current_state 130 | assert 1 == func_succeed_async.call_count 131 | 132 | 133 | async def test_successful_after_wait(storage, delta): 134 | """It should accurately report the time needed to wait.""" 135 | 136 | breaker = CircuitBreaker(fail_max=1, timeout_duration=delta, state_storage=storage) 137 | func_succeed_async = func_succeed_counted_async() 138 | 139 | try: 140 | await breaker.call_async(func_exception_async) 141 | except CircuitBreakerError as e: 142 | await asyncio.sleep(delta.total_seconds()) 143 | 144 | await breaker.call_async(func_succeed_async) 145 | assert func_succeed_async.call_count == 1 146 | 147 | 148 | async def test_failed_call_when_half_open(storage): 149 | """It should open the circuit when a call fails in half-open state.""" 150 | 151 | breaker = CircuitBreaker(state_storage=storage) 152 | 153 | breaker.half_open() 154 | assert 0 == breaker.fail_counter 155 | assert CircuitBreakerState.HALF_OPEN == breaker.current_state 156 | 157 | with raises(CircuitBreakerError): 158 | await breaker.call_async(func_exception_async) 159 | 160 | assert 1 == breaker.fail_counter 161 | assert CircuitBreakerState.OPEN == breaker.current_state 162 | 163 | 164 | async def test_successful_call_when_half_open(storage): 165 | """It should close the circuit when a call succeeds in half-open state.""" 166 | 167 | breaker = CircuitBreaker(state_storage=storage) 168 | 169 | breaker.half_open() 170 | assert 0 == breaker.fail_counter 171 | assert CircuitBreakerState.HALF_OPEN == breaker.current_state 172 | 173 | # Circuit should open 174 | assert await breaker.call_async(func_succeed_async) 175 | assert 0 == breaker.fail_counter 176 | assert CircuitBreakerState.CLOSED == breaker.current_state 177 | 178 | 179 | async def test_close(storage): 180 | """It should allow the circuit to be closed manually.""" 181 | 182 | breaker = CircuitBreaker(fail_max=3, state_storage=storage) 183 | 184 | breaker.open() 185 | assert 0 == breaker.fail_counter 186 | assert CircuitBreakerState.OPEN == breaker.current_state 187 | 188 | breaker.close() 189 | assert 0 == breaker.fail_counter 190 | assert CircuitBreakerState.CLOSED == breaker.current_state 191 | 192 | 193 | async def test_generator(storage): 194 | """It should support generator values.""" 195 | 196 | breaker = CircuitBreaker(state_storage=storage) 197 | 198 | @breaker 199 | def func_succeed(): 200 | """Docstring""" 201 | yield True 202 | 203 | @breaker 204 | def func_exception(): 205 | """Docstring""" 206 | x = yield True 207 | raise DummyException(x) 208 | 209 | s = func_succeed() 210 | e = func_exception() 211 | next(e) 212 | 213 | with raises(DummyException): 214 | e.send(True) 215 | 216 | assert 1 == breaker.fail_counter 217 | assert next(s) 218 | 219 | with raises(StopIteration): 220 | next(s) 221 | 222 | assert 0 == breaker.fail_counter 223 | -------------------------------------------------------------------------------- /test/tests-old.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | import logging 3 | import threading 4 | from datetime import timedelta 5 | from time import sleep 6 | from unittest import TestCase, main 7 | from unittest.mock import patch 8 | 9 | import fakeredis 10 | from pytest import raises 11 | from redis.exceptions import RedisError 12 | 13 | from aiobreaker import CircuitBreaker, CircuitBreakerError 14 | from listener import CircuitBreakerListener 15 | from state import STATE_OPEN, STATE_CLOSED, STATE_HALF_OPEN 16 | from storage.redis import CircuitRedisStorage 17 | from storage.memory import CircuitMemoryStorage 18 | 19 | 20 | class CircuitBreakerRedisTestCase(CircuitBreakerStorageBasedTestCase, 21 | CircuitBreakerStorageBasedTestCaseAsync): 22 | """ 23 | Tests for the CircuitBreaker class. 24 | """ 25 | 26 | def setUp(self): 27 | self.redis = fakeredis.FakeStrictRedis() 28 | self._breaker_kwargs = {'state_storage': CircuitRedisStorage('closed', self.redis)} 29 | 30 | @property 31 | def breaker_kwargs(self): 32 | return self._breaker_kwargs 33 | 34 | def tearDown(self): 35 | self.redis.flushall() 36 | 37 | def test_namespace(self): 38 | self.redis.flushall() 39 | self._breaker_kwargs = {'state_storage': CircuitRedisStorage('closed', self.redis, namespace='my_app')} 40 | breaker = CircuitBreaker(**self.breaker_kwargs) 41 | 42 | def func(): raise NotImplementedError() 43 | 44 | self.assertRaises(NotImplementedError, breaker.call, func) 45 | keys = self.redis.keys() 46 | self.assertEqual(2, len(keys)) 47 | self.assertTrue(keys[0].decode('utf-8').startswith('my_app')) 48 | self.assertTrue(keys[1].decode('utf-8').startswith('my_app')) 49 | 50 | def test_fallback_state(self): 51 | def func(_): 52 | raise RedisError() 53 | 54 | logger = logging.getLogger('aiobreaker') 55 | logger.setLevel(logging.FATAL) 56 | self._breaker_kwargs = { 57 | 'state_storage': CircuitRedisStorage('closed', self.redis, fallback_circuit_state='open') 58 | } 59 | 60 | breaker = CircuitBreaker(**self.breaker_kwargs) 61 | 62 | with patch.object(self.redis, 'get', new=func): 63 | state = breaker.state 64 | self.assertEqual('open', state.name) 65 | 66 | def test_missing_state(self): 67 | """CircuitBreakerRedis: If state on Redis is missing, it should set the 68 | fallback circuit state and reset the fail counter to 0. 69 | """ 70 | 71 | def func(): 72 | raise DummyException() 73 | 74 | self._breaker_kwargs = { 75 | 'state_storage': CircuitRedisStorage('closed', self.redis, fallback_circuit_state='open') 76 | } 77 | 78 | breaker = CircuitBreaker(**self.breaker_kwargs) 79 | self.assertRaises(DummyException, breaker.call, func) 80 | self.assertEqual(1, breaker.fail_counter) 81 | 82 | with patch.object(self.redis, 'get', new=lambda k: None): 83 | state = breaker.state 84 | self.assertEqual('open', state.name) 85 | self.assertEqual(0, breaker.fail_counter) 86 | 87 | class CircuitBreakerRedisConcurrencyTestCase(TestCase): 88 | """ 89 | Tests to reproduce common concurrency between different machines 90 | connecting to redis. This is simulated locally using threads. 91 | """ 92 | 93 | def setUp(self): 94 | self.redis = fakeredis.FakeStrictRedis() 95 | self.storage = CircuitRedisStorage('closed', self.redis) 96 | 97 | def tearDown(self): 98 | self.redis.flushall() 99 | 100 | @staticmethod 101 | def _start_threads(target, n): 102 | """ 103 | Starts `n` threads that calls `target` and waits for them to finish. 104 | """ 105 | threads = [threading.Thread(target=target) for _ in range(n)] 106 | 107 | for t in threads: 108 | t.start() 109 | 110 | for t in threads: 111 | t.join() 112 | 113 | def test_fail_thread_safety(self): 114 | """It should compute a failed call atomically to avoid race conditions.""" 115 | 116 | breaker = CircuitBreaker(fail_max=3000, timeout_duration=timedelta(seconds=1), state_storage=self.storage) 117 | 118 | @breaker 119 | def err(): 120 | raise DummyException() 121 | 122 | def trigger_error(): 123 | for n in range(500): 124 | try: 125 | err() 126 | except DummyException: 127 | pass 128 | 129 | def _inc_counter(): 130 | sleep(0.00005) 131 | breaker._state_storage.increment_counter() 132 | 133 | breaker._inc_counter = _inc_counter 134 | self._start_threads(trigger_error, 3) 135 | self.assertEqual(1500, breaker.fail_counter) 136 | 137 | def test_success_thread_safety(self): 138 | """It should compute a successful call atomically to avoid race conditions.""" 139 | 140 | breaker = CircuitBreaker(fail_max=3000, timeout_duration=timedelta(seconds=1), state_storage=self.storage) 141 | 142 | @breaker 143 | def suc(): 144 | return True 145 | 146 | def trigger_success(): 147 | for n in range(500): 148 | suc() 149 | 150 | class SuccessListener(CircuitBreakerListener): 151 | def success(self, breaker): 152 | c = 0 153 | if hasattr(breaker, '_success_counter'): 154 | c = breaker._success_counter 155 | sleep(0.00005) 156 | breaker._success_counter = c + 1 157 | 158 | breaker.add_listener(SuccessListener()) 159 | self._start_threads(trigger_success, 3) 160 | self.assertEqual(1500, breaker._success_counter) 161 | 162 | def test_half_open_thread_safety(self): 163 | """ 164 | it should allow only one trial call when the circuit is half-open. 165 | """ 166 | breaker = CircuitBreaker(fail_max=1, timeout_duration=timedelta(seconds=0.01)) 167 | 168 | breaker.open() 169 | sleep(0.01) 170 | 171 | @breaker 172 | def err(): 173 | raise DummyException() 174 | 175 | def trigger_failure(): 176 | try: 177 | err() 178 | except DummyException: 179 | pass 180 | except CircuitBreakerError: 181 | pass 182 | 183 | class StateListener(CircuitBreakerListener): 184 | def __init__(self): 185 | self._count = 0 186 | 187 | def before_call(self, breaker, func, *args, **kwargs): 188 | sleep(0.00005) 189 | 190 | def state_change(self, breaker, old, new): 191 | if new.name == STATE_HALF_OPEN: 192 | self._count += 1 193 | 194 | state_listener = StateListener() 195 | breaker.add_listener(state_listener) 196 | 197 | self._start_threads(trigger_failure, 5) 198 | self.assertEqual(1, state_listener._count) 199 | 200 | def test_fail_max_thread_safety(self): 201 | """CircuitBreaker: it should not allow more failed calls than 'fail_max' 202 | setting. Note that with Redis, where we have separate systems 203 | incrementing the counter, we can get concurrent updates such that the 204 | counter is greater than the 'fail_max' by the number of systems. To 205 | prevent this, we'd need to take out a lock amongst all systems before 206 | trying the call. 207 | """ 208 | 209 | breaker = CircuitBreaker() 210 | 211 | @breaker 212 | def err(): 213 | raise DummyException() 214 | 215 | def trigger_error(): 216 | for i in range(2000): 217 | try: 218 | err() 219 | except DummyException: 220 | pass 221 | except CircuitBreakerError: 222 | pass 223 | 224 | class SleepListener(CircuitBreakerListener): 225 | def before_call(self, breaker, func, *args, **kwargs): 226 | sleep(0.00005) 227 | 228 | breaker.add_listener(SleepListener()) 229 | num_threads = 3 230 | self._start_threads(trigger_error, num_threads) 231 | self.assertTrue(breaker.fail_counter < breaker.fail_max + num_threads) 232 | 233 | 234 | if __name__ == "__main__": 235 | main() 236 | -------------------------------------------------------------------------------- /test/test_configuration.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | 3 | from pytest import raises 4 | 5 | from aiobreaker import CircuitBreaker 6 | from aiobreaker.listener import CircuitBreakerListener 7 | from aiobreaker.state import CircuitBreakerState 8 | from aiobreaker.storage.memory import CircuitMemoryStorage 9 | from test.util import func_succeed, DummyException 10 | 11 | 12 | def test_default_state(): 13 | """It should get initial state from state_storage.""" 14 | for state in CircuitBreakerState: 15 | storage = CircuitMemoryStorage(state) 16 | breaker = CircuitBreaker(state_storage=storage) 17 | assert isinstance(breaker.state, state.value) 18 | assert breaker.state.state == state 19 | 20 | 21 | def test_default_params(): 22 | """It should define smart defaults.""" 23 | breaker = CircuitBreaker() 24 | 25 | assert 0 == breaker.fail_counter 26 | assert timedelta(seconds=60) == breaker.timeout_duration 27 | assert 5 == breaker.fail_max 28 | assert CircuitBreakerState.CLOSED == breaker.current_state 29 | assert () == breaker.excluded_exceptions 30 | assert () == breaker.listeners 31 | assert 'memory' == breaker._state_storage.name 32 | 33 | 34 | def test_new_with_custom_reset_timeout(): 35 | """It should support a custom reset timeout value.""" 36 | breaker = CircuitBreaker(timeout_duration=timedelta(seconds=30)) 37 | 38 | assert 0 == breaker.fail_counter 39 | assert timedelta(seconds=30) == breaker.timeout_duration 40 | assert 5 == breaker.fail_max 41 | assert () == breaker.excluded_exceptions 42 | assert () == breaker.listeners 43 | assert 'memory' == breaker._state_storage.name 44 | 45 | 46 | def test_new_with_custom_fail_max(): 47 | """It should support a custom maximum number of failures.""" 48 | breaker = CircuitBreaker(fail_max=10) 49 | assert 0 == breaker.fail_counter 50 | assert timedelta(seconds=60) == breaker.timeout_duration 51 | assert 10 == breaker.fail_max 52 | assert () == breaker.excluded_exceptions 53 | assert () == breaker.listeners 54 | assert 'memory' == breaker._state_storage.name 55 | 56 | 57 | def test_new_with_custom_excluded_exceptions(): 58 | """CircuitBreaker: it should support a custom list of excluded 59 | exceptions. 60 | """ 61 | breaker = CircuitBreaker(exclude=[Exception]) 62 | assert 0 == breaker.fail_counter 63 | assert timedelta(seconds=60) == breaker.timeout_duration 64 | assert 5 == breaker.fail_max 65 | assert (Exception,) == breaker.excluded_exceptions 66 | assert () == breaker.listeners 67 | assert 'memory' == breaker._state_storage.name 68 | 69 | 70 | def test_fail_max_setter(): 71 | """CircuitBreaker: it should allow the user to set a new value for 72 | 'fail_max'. 73 | """ 74 | breaker = CircuitBreaker() 75 | 76 | assert 5 == breaker.fail_max 77 | breaker.fail_max = 10 78 | assert 10 == breaker.fail_max 79 | 80 | 81 | def test_reset_timeout_setter(): 82 | """CircuitBreaker: it should allow the user to set a new value for 83 | 'reset_timeout'. 84 | """ 85 | breaker = CircuitBreaker() 86 | 87 | assert timedelta(seconds=60) == breaker.timeout_duration 88 | breaker.timeout_duration = timedelta(seconds=30) 89 | assert timedelta(seconds=30) == breaker.timeout_duration 90 | 91 | 92 | def test_call_with_no_args(): 93 | """ It should be able to invoke functions with no-args.""" 94 | breaker = CircuitBreaker() 95 | assert breaker.call(func_succeed) 96 | 97 | 98 | def test_call_with_args(): 99 | """ It should be able to invoke functions with args.""" 100 | 101 | def func(arg1, arg2): 102 | return arg1, arg2 103 | 104 | breaker = CircuitBreaker() 105 | 106 | assert (42, 'abc') == breaker.call(func, 42, 'abc') 107 | 108 | 109 | def test_call_with_kwargs(): 110 | """ It should be able to invoke functions with kwargs.""" 111 | 112 | def func(**kwargs): 113 | return kwargs 114 | 115 | breaker = CircuitBreaker() 116 | 117 | kwargs = {'a': 1, 'b': 2} 118 | 119 | assert kwargs == breaker.call(func, **kwargs) 120 | 121 | 122 | def test_add_listener(): 123 | """ It should allow the user to add a listener at a later time.""" 124 | breaker = CircuitBreaker() 125 | 126 | assert () == breaker.listeners 127 | 128 | first = CircuitBreakerListener() 129 | breaker.add_listener(first) 130 | assert (first,) == breaker.listeners 131 | 132 | second = CircuitBreakerListener() 133 | breaker.add_listener(second) 134 | assert (first, second) == breaker.listeners 135 | 136 | 137 | def test_add_listeners(): 138 | """ It should allow the user to add listeners at a later time.""" 139 | breaker = CircuitBreaker() 140 | 141 | first, second = CircuitBreakerListener(), CircuitBreakerListener() 142 | breaker.add_listeners(first, second) 143 | assert (first, second) == breaker.listeners 144 | 145 | 146 | def test_remove_listener(): 147 | """ it should allow the user to remove a listener.""" 148 | breaker = CircuitBreaker() 149 | 150 | first = CircuitBreakerListener() 151 | breaker.add_listener(first) 152 | assert (first,) == breaker.listeners 153 | 154 | breaker.remove_listener(first) 155 | assert () == breaker.listeners 156 | 157 | 158 | def test_excluded_exceptions(): 159 | """CircuitBreaker: it should ignore specific exceptions. 160 | """ 161 | breaker = CircuitBreaker( 162 | exclude=[LookupError, lambda e: type(e) == DummyException and e.val == 3]) 163 | 164 | def err_1(): raise LookupError() 165 | 166 | def err_2(): raise DummyException() 167 | 168 | def err_3(): raise KeyError() 169 | 170 | def err_4(): raise DummyException(val=3) 171 | 172 | # LookupError is not considered a system error 173 | with raises(LookupError): 174 | breaker.call(err_1) 175 | assert 0 == breaker.fail_counter 176 | 177 | with raises(DummyException): 178 | breaker.call(err_2) 179 | assert 1 == breaker.fail_counter 180 | 181 | # Should consider subclasses as well (KeyError is a subclass of 182 | # LookupError) 183 | with raises(KeyError): 184 | breaker.call(err_3) 185 | assert 0 == breaker.fail_counter 186 | 187 | # should filter based on functions as well 188 | with raises(DummyException): 189 | breaker.call(err_4) 190 | assert 0 == breaker.fail_counter 191 | 192 | 193 | def test_add_excluded_exception(): 194 | """ it should allow the user to exclude an exception at a later time.""" 195 | breaker = CircuitBreaker() 196 | 197 | assert () == breaker.excluded_exceptions 198 | 199 | breaker.add_excluded_exception(NotImplementedError) 200 | assert (NotImplementedError,) == breaker.excluded_exceptions 201 | 202 | breaker.add_excluded_exception(Exception) 203 | assert (NotImplementedError, Exception) == breaker.excluded_exceptions 204 | 205 | 206 | def test_add_excluded_exceptions(): 207 | """ it should allow the user to exclude exceptions at a later time.""" 208 | breaker = CircuitBreaker() 209 | 210 | breaker.add_excluded_exceptions(NotImplementedError, Exception) 211 | assert (NotImplementedError, Exception) == breaker.excluded_exceptions 212 | 213 | 214 | def test_remove_excluded_exception(): 215 | """It should allow the user to remove an excluded exception.""" 216 | breaker = CircuitBreaker() 217 | 218 | breaker.add_excluded_exception(NotImplementedError) 219 | assert (NotImplementedError,) == breaker.excluded_exceptions 220 | 221 | breaker.remove_excluded_exception(NotImplementedError) 222 | assert () == breaker.excluded_exceptions 223 | 224 | 225 | def test_decorator(): 226 | """It should be a decorator.""" 227 | 228 | breaker = CircuitBreaker() 229 | 230 | @breaker 231 | def suc(): 232 | """Docstring""" 233 | pass 234 | 235 | @breaker 236 | def err(): 237 | """Docstring""" 238 | raise DummyException() 239 | 240 | assert 'Docstring' == suc.__doc__ 241 | assert 'Docstring' == err.__doc__ 242 | assert 'suc' == suc.__name__ 243 | assert 'err' == err.__name__ 244 | 245 | assert 0 == breaker.fail_counter 246 | 247 | with raises(DummyException): 248 | err() 249 | 250 | assert 1 == breaker.fail_counter 251 | 252 | suc() 253 | assert 0 == breaker.fail_counter 254 | 255 | 256 | def test_decorator_arguments(): 257 | """It should accept arguments to the decorator.""" 258 | 259 | breaker = CircuitBreaker() 260 | 261 | @breaker(ignore_on_call=True) 262 | def suc(): 263 | """Docstring""" 264 | pass 265 | 266 | @breaker() 267 | def err(): 268 | """Docstring""" 269 | raise DummyException() 270 | 271 | assert 0 == breaker.fail_counter 272 | 273 | with raises(DummyException): 274 | err() 275 | 276 | assert 1 == breaker.fail_counter 277 | 278 | suc() 279 | assert 0 == breaker.fail_counter 280 | 281 | 282 | def test_decorator_positional_arguments(): 283 | """It should throw an error when positional arguments are supplied to the decorator.""" 284 | breaker = CircuitBreaker() 285 | 286 | with raises(TypeError): 287 | @breaker(True) 288 | def suc(): 289 | """Docstring""" 290 | pass 291 | 292 | 293 | def test_double_count(): 294 | """It should not trigger twice if you call CircuitBreaker#call on a decorated function.""" 295 | 296 | breaker = CircuitBreaker() 297 | 298 | @breaker 299 | def err(): 300 | """Docstring""" 301 | raise DummyException() 302 | 303 | assert 0 == breaker.fail_counter 304 | 305 | with raises(DummyException): 306 | breaker.call(err) 307 | 308 | assert 1 == breaker.fail_counter 309 | 310 | 311 | def test_name(): 312 | """It should allow an optional name to be set and retrieved.""" 313 | name = "test_breaker" 314 | breaker = CircuitBreaker(name=name) 315 | assert breaker.name == name 316 | 317 | name = "breaker_test" 318 | breaker.name = name 319 | assert breaker.name == name 320 | -------------------------------------------------------------------------------- /aiobreaker/state.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import types 3 | from abc import ABC 4 | from datetime import datetime, timedelta 5 | from enum import Enum 6 | from typing import Callable, Union, Optional, TypeVar, Awaitable, Generator 7 | 8 | 9 | class CircuitBreakerError(Exception): 10 | """ 11 | Raised when the function fails due to the breaker being open. 12 | """ 13 | 14 | def __init__(self, message: str, reopen_time: datetime): 15 | """ 16 | :param message: The reasoning. 17 | :param reopen_time: When the breaker re-opens. 18 | """ 19 | self.message = message 20 | self.reopen_time = reopen_time 21 | 22 | @property 23 | def time_remaining(self) -> timedelta: 24 | return self.reopen_time - datetime.now() 25 | 26 | async def sleep_until_open(self): 27 | await asyncio.sleep(self.time_remaining.total_seconds()) 28 | 29 | 30 | T = TypeVar('T') 31 | 32 | 33 | class CircuitBreakerBaseState(ABC): 34 | """ 35 | Implements the behavior needed by all circuit breaker states. 36 | """ 37 | 38 | def __init__(self, breaker: 'CircuitBreaker', state: 'CircuitBreakerState'): 39 | """ 40 | Creates a new instance associated with the circuit breaker `cb` and 41 | identified by `name`. 42 | """ 43 | self._breaker = breaker 44 | self._state = state 45 | 46 | @property 47 | def state(self) -> 'CircuitBreakerState': 48 | """ 49 | Returns a human friendly name that identifies this state. 50 | """ 51 | return self._state 52 | 53 | def _handle_error(self, func: Callable, exception: Exception): 54 | """ 55 | Handles a failed call to the guarded operation. 56 | 57 | :raises: The given exception, after calling all the handlers. 58 | """ 59 | if self._breaker.is_system_error(exception): 60 | self._breaker._inc_counter() 61 | for listener in self._breaker.listeners: 62 | listener.failure(self._breaker, exception) 63 | self.on_failure(exception) 64 | else: 65 | self._handle_success() 66 | raise exception 67 | 68 | def _handle_success(self): 69 | """ 70 | Handles a successful call to the guarded operation. 71 | """ 72 | self._breaker._state_storage.reset_counter() 73 | self.on_success() 74 | for listener in self._breaker.listeners: 75 | listener.success(self._breaker) 76 | 77 | def call(self, func: Callable[..., T], *args, **kwargs) -> T: 78 | """ 79 | Calls `func` with the given `args` and `kwargs`, and updates the 80 | circuit breaker state according to the result. 81 | """ 82 | ret = None 83 | 84 | self.before_call(func, *args, **kwargs) 85 | for listener in self._breaker.listeners: 86 | listener.before_call(self._breaker, func, *args, **kwargs) 87 | 88 | try: 89 | ret = func(*args, **kwargs) 90 | if isinstance(ret, types.GeneratorType): 91 | return self.generator_call(ret) 92 | except Exception as e: 93 | self._handle_error(func, e) 94 | else: 95 | self._handle_success() 96 | return ret 97 | 98 | async def call_async(self, func: Callable[..., Awaitable[T]], *args, **kwargs) -> Awaitable[T]: 99 | 100 | ret = None 101 | self.before_call(func, *args, **kwargs) 102 | for listener in self._breaker.listeners: 103 | listener.before_call(self._breaker, func, *args, **kwargs) 104 | 105 | try: 106 | ret = await func(*args, **kwargs) 107 | except Exception as e: 108 | self._handle_error(func, e) 109 | else: 110 | self._handle_success() 111 | return ret 112 | 113 | def generator_call(self, wrapped_generator: Generator): 114 | try: 115 | value = yield next(wrapped_generator) 116 | while True: 117 | value = yield wrapped_generator.send(value) 118 | except StopIteration: 119 | self._handle_success() 120 | return 121 | except Exception as e: 122 | self._handle_error(None, e) 123 | 124 | def before_call(self, func: Union[Callable[..., any], Callable[..., Awaitable]], *args, **kwargs): 125 | """ 126 | Override this method to be notified before a call to the guarded 127 | operation is attempted. 128 | """ 129 | pass 130 | 131 | def on_success(self): 132 | """ 133 | Override this method to be notified when a call to the guarded 134 | operation succeeds. 135 | """ 136 | pass 137 | 138 | def on_failure(self, exception: Exception): 139 | """ 140 | Override this method to be notified when a call to the guarded 141 | operation fails. 142 | """ 143 | pass 144 | 145 | 146 | class CircuitClosedState(CircuitBreakerBaseState): 147 | """ 148 | In the normal "closed" state, the circuit breaker executes operations as 149 | usual. If the call succeeds, nothing happens. If it fails, however, the 150 | circuit breaker makes a note of the failure. 151 | 152 | Once the number of failures exceeds a threshold, the circuit breaker trips 153 | and "opens" the circuit. 154 | """ 155 | 156 | def __init__(self, breaker, prev_state: Optional[CircuitBreakerBaseState] = None, notify=False): 157 | """ 158 | Moves the given circuit breaker to the "closed" state. 159 | """ 160 | super().__init__(breaker, CircuitBreakerState.CLOSED) 161 | if notify: 162 | # We only reset the counter if notify is True, otherwise the CircuitBreaker 163 | # will lose it's failure count due to a second CircuitBreaker being created 164 | # using the same _state_storage object, or if the _state_storage objects 165 | # share a central source of truth (as would be the case with the redis 166 | # storage). 167 | self._breaker._state_storage.reset_counter() 168 | for listener in self._breaker.listeners: 169 | listener.state_change(self._breaker, prev_state, self) 170 | 171 | def on_failure(self, exception: Exception): 172 | """ 173 | Moves the circuit breaker to the "open" state once the failures 174 | threshold is reached. 175 | """ 176 | if self._breaker._state_storage.counter >= self._breaker.fail_max: 177 | self._breaker.open() 178 | raise CircuitBreakerError('Failures threshold reached, circuit breaker opened.', self._breaker.opens_at) from exception 179 | 180 | 181 | class CircuitOpenState(CircuitBreakerBaseState): 182 | """ 183 | When the circuit is "open", calls to the circuit breaker fail immediately, 184 | without any attempt to execute the real operation. This is indicated by the 185 | ``CircuitBreakerError`` exception. 186 | 187 | After a suitable amount of time, the circuit breaker decides that the 188 | operation has a chance of succeeding, so it goes into the "half-open" state. 189 | """ 190 | 191 | def __init__(self, breaker, prev_state=None, notify=False): 192 | """ 193 | Moves the given circuit breaker to the "open" state. 194 | """ 195 | super().__init__(breaker, CircuitBreakerState.OPEN) 196 | if notify: 197 | for listener in self._breaker.listeners: 198 | listener.state_change(self._breaker, prev_state, self) 199 | 200 | def before_call(self, func, *args, **kwargs): 201 | """ 202 | After the timeout elapses, move the circuit breaker to the "half-open" state. 203 | :raises CircuitBreakerError: if the timeout has still to be elapsed. 204 | """ 205 | timeout = self._breaker.timeout_duration 206 | opened_at = self._breaker._state_storage.opened_at 207 | if opened_at and datetime.utcnow() < opened_at + timeout: 208 | raise CircuitBreakerError('Timeout not elapsed yet, circuit breaker still open', self._breaker.opens_at) 209 | 210 | def call(self, func, *args, **kwargs): 211 | """ 212 | Call before_call to check if the breaker should close and open it if it passes. 213 | """ 214 | self.before_call(func, *args, **kwargs) 215 | self._breaker.half_open() 216 | return self._breaker.call(func, *args, **kwargs) 217 | 218 | async def call_async(self, func, *args, **kwargs): 219 | """ 220 | Call before_call to check if the breaker should close and open it if it passes. 221 | """ 222 | self.before_call(func, *args, **kwargs) 223 | self._breaker.half_open() 224 | return await self._breaker.call_async(func, *args, **kwargs) 225 | 226 | 227 | class CircuitHalfOpenState(CircuitBreakerBaseState): 228 | """ 229 | In the "half-open" state, the next call to the circuit breaker is allowed 230 | to execute the dangerous operation. Should the call succeed, the circuit 231 | breaker resets and returns to the "closed" state. If this trial call fails, 232 | however, the circuit breaker returns to the "open" state until another 233 | timeout elapses. 234 | """ 235 | 236 | def __init__(self, breaker, prev_state=None, notify=False): 237 | """ 238 | Moves the given circuit breaker to the "half-open" state. 239 | """ 240 | super().__init__(breaker, CircuitBreakerState.HALF_OPEN) 241 | if notify: 242 | for listener in self._breaker._listeners: 243 | listener.state_change(self._breaker, prev_state, self) 244 | 245 | def on_failure(self, exception): 246 | """ 247 | Opens the circuit breaker. 248 | """ 249 | self._breaker.open() 250 | raise CircuitBreakerError('Trial call failed, circuit breaker opened.', 251 | self._breaker.opens_at) from exception 252 | 253 | def on_success(self): 254 | """ 255 | Closes the circuit breaker. 256 | """ 257 | self._breaker.close() 258 | 259 | 260 | class CircuitBreakerState(Enum): 261 | 262 | OPEN = CircuitOpenState 263 | CLOSED = CircuitClosedState 264 | HALF_OPEN = CircuitHalfOpenState 265 | -------------------------------------------------------------------------------- /aiobreaker/circuitbreaker.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import inspect 3 | from datetime import timedelta, datetime 4 | from functools import wraps 5 | from typing import Optional, Iterable, Callable, Coroutine, Type, List, Union, Tuple 6 | 7 | from .listener import CircuitBreakerListener 8 | from .state import CircuitBreakerState, CircuitBreakerBaseState 9 | from .storage.base import CircuitBreakerStorage 10 | from .storage.memory import CircuitMemoryStorage 11 | 12 | 13 | class CircuitBreaker: 14 | """ 15 | A circuit breaker is a route through which functions are executed. 16 | When a function is executed via a circuit breaker, the breaker is notified. 17 | Multiple failed attempts will open the breaker and block additional calls. 18 | """ 19 | 20 | def __init__(self, fail_max=5, 21 | timeout_duration: Optional[timedelta] = None, 22 | exclude: Optional[Iterable[Union[Callable, Type[Exception]]]] = None, 23 | listeners: Optional[Iterable[CircuitBreakerListener]] = None, 24 | state_storage: Optional[CircuitBreakerStorage] = None, 25 | name: Optional[str] = None): 26 | """ 27 | Creates a new circuit breaker with the given parameters. 28 | 29 | :param fail_max: The maximum number of failures for the breaker. 30 | :param timeout_duration: The timeout to elapse for a breaker to close again. 31 | :param exclude: A list of excluded :class:`Exception` types to ignore. 32 | :param listeners: A list of :class:`CircuitBreakerListener` 33 | :param state_storage: A type of storage. Defaults to :class:`~aiobreaker.storage.memory.CircuitMemoryStorage` 34 | """ 35 | self._state_storage = state_storage or CircuitMemoryStorage(CircuitBreakerState.CLOSED) 36 | self._state = self._create_new_state(self.current_state) 37 | 38 | self._fail_max = fail_max 39 | self._timeout_duration = timeout_duration if timeout_duration else timedelta(seconds=60) 40 | 41 | self._excluded_exceptions: List[Union[Callable, Type[Exception]]] = list(exclude or []) 42 | self._listeners = list(listeners or []) 43 | self._name = name 44 | 45 | @property 46 | def fail_counter(self): 47 | """ 48 | Returns the current number of consecutive failures. 49 | """ 50 | return self._state_storage.counter 51 | 52 | @property 53 | def fail_max(self): 54 | """ 55 | Returns the maximum number of failures tolerated before the circuit is opened. 56 | """ 57 | return self._fail_max 58 | 59 | @fail_max.setter 60 | def fail_max(self, number): 61 | """ 62 | Sets the maximum `number` of failures tolerated before the circuit is opened. 63 | """ 64 | self._fail_max = number 65 | 66 | @property 67 | def timeout_duration(self): 68 | """ 69 | Once this circuit breaker is opened, it should remain opened until the timeout period elapses. 70 | """ 71 | return self._timeout_duration 72 | 73 | @timeout_duration.setter 74 | def timeout_duration(self, timeout: datetime): 75 | """ 76 | Sets the timeout period this circuit breaker should be kept open. 77 | """ 78 | self._timeout_duration = timeout 79 | 80 | @property 81 | def opens_at(self) -> Optional[datetime]: 82 | """Gets the remaining timeout for a breaker.""" 83 | open_at = self._state_storage.opened_at + self.timeout_duration 84 | if open_at < datetime.now(): 85 | return None 86 | else: 87 | return open_at 88 | 89 | @property 90 | def time_until_open(self) -> Optional[timedelta]: 91 | """ 92 | Returns a timedelta representing the difference between the current 93 | time and when the breaker closes again, or None if it has elapsed. 94 | """ 95 | opens_in = self.opens_at - datetime.now() 96 | if opens_in < timedelta(0): 97 | return None 98 | return opens_in 99 | 100 | async def sleep_until_open(self): 101 | await asyncio.sleep(self.time_until_open.total_seconds()) 102 | 103 | def _create_new_state(self, new_state: CircuitBreakerState, prev_state=None, 104 | notify=False) -> 'CircuitBreakerBaseState': 105 | """ 106 | Return state object from state string, i.e., 107 | 'closed' -> 108 | """ 109 | 110 | try: 111 | return new_state.value(self, prev_state=prev_state, notify=notify) 112 | except KeyError: 113 | msg = "Unknown state {!r}, valid states: {}" 114 | raise ValueError(msg.format(new_state, ', '.join(state.value for state in CircuitBreakerState))) 115 | 116 | @property 117 | def state(self): 118 | """ 119 | Update (if needed) and returns the cached state object. 120 | """ 121 | # Ensure cached state is up-to-date 122 | if self.current_state != self._state.state: 123 | # If cached state is out-of-date, that means that it was likely 124 | # changed elsewhere (e.g. another process instance). We still send 125 | # out a notification, informing others that this particular circuit 126 | # breaker instance noticed the changed circuit. 127 | self.state = self.current_state 128 | return self._state 129 | 130 | @state.setter 131 | def state(self, state_str): 132 | """ 133 | Set cached state and notify listeners of newly cached state. 134 | """ 135 | self._state = self._create_new_state( 136 | state_str, prev_state=self._state, notify=True) 137 | 138 | @property 139 | def current_state(self) -> CircuitBreakerState: 140 | """ 141 | Returns a CircuitBreakerState that identifies the state of the circuit breaker. 142 | """ 143 | return self._state_storage.state 144 | 145 | @property 146 | def excluded_exceptions(self) -> tuple: 147 | """ 148 | Returns a tuple of the excluded exceptions, e.g., exceptions that should 149 | not be considered system errors by this circuit breaker. 150 | """ 151 | return tuple(self._excluded_exceptions) 152 | 153 | def add_excluded_exception(self, exception: Type[Exception]): 154 | """ 155 | Adds an exception to the list of excluded exceptions. 156 | """ 157 | self._excluded_exceptions.append(exception) 158 | 159 | def add_excluded_exceptions(self, *exceptions): 160 | """ 161 | Adds exceptions to the list of excluded exceptions. 162 | 163 | :param exceptions: Any Exception types you wish to ignore. 164 | """ 165 | for exc in exceptions: 166 | self.add_excluded_exception(exc) 167 | 168 | def remove_excluded_exception(self, exception: Type[Exception]): 169 | """ 170 | Removes an exception from the list of excluded exceptions. 171 | """ 172 | self._excluded_exceptions.remove(exception) 173 | 174 | def _inc_counter(self): 175 | """ 176 | Increments the counter of failed calls. 177 | """ 178 | self._state_storage.increment_counter() 179 | 180 | def is_system_error(self, exception: Exception): 181 | """ 182 | Returns whether the exception `exception` is considered a signal of 183 | system malfunction. Business exceptions should not cause this circuit 184 | breaker to open. 185 | """ 186 | exception_type = type(exception) 187 | for exclusion in self._excluded_exceptions: 188 | if type(exclusion) is type: 189 | if issubclass(exception_type, exclusion): 190 | return False 191 | elif callable(exclusion): 192 | if exclusion(exception): 193 | return False 194 | 195 | return True 196 | 197 | def call(self, func: Callable, *args, **kwargs): 198 | """ 199 | Calls `func` with the given `args` and `kwargs` according to the rules 200 | implemented by the current state of this circuit breaker. 201 | """ 202 | if getattr(func, "_ignore_on_call", False): 203 | # if the function has set `_ignore_on_call` to True, 204 | # it is a decorator that needs to avoid triggering 205 | # the circuit breaker twice 206 | return func(*args, **kwargs) 207 | 208 | return self.state.call(func, *args, **kwargs) 209 | 210 | async def call_async(self, func: Callable[..., Coroutine], *args, **kwargs): 211 | """ 212 | Calls `func` with the given `args` and `kwargs` according to the rules 213 | implemented by the current state of this circuit breaker. 214 | """ 215 | if getattr(func, "_ignore_on_call", False): 216 | # if the function has set `_ignore_on_call` to True, 217 | # it is a decorator that needs to avoid triggering 218 | # the circuit breaker twice 219 | return await func(*args, **kwargs) 220 | 221 | return await self.state.call_async(func, *args, **kwargs) 222 | 223 | def open(self): 224 | """ 225 | Opens the circuit, e.g., the following calls will immediately fail 226 | until timeout elapses. 227 | """ 228 | self._state_storage.opened_at = datetime.utcnow() 229 | self.state = self._state_storage.state = CircuitBreakerState.OPEN 230 | 231 | def half_open(self): 232 | """ 233 | Half-opens the circuit, e.g. lets the following call pass through and 234 | opens the circuit if the call fails (or closes the circuit if the call 235 | succeeds). 236 | """ 237 | self.state = self._state_storage.state = CircuitBreakerState.HALF_OPEN 238 | 239 | def close(self): 240 | """ 241 | Closes the circuit, e.g. lets the following calls execute as usual. 242 | """ 243 | self.state = self._state_storage.state = CircuitBreakerState.CLOSED 244 | 245 | def __call__(self, *call_args, ignore_on_call=True): 246 | """ 247 | Decorates the function such that calls are handled according to the rules 248 | implemented by the current state of this circuit breaker. 249 | 250 | :param ignore_on_call: Whether the decorated function should be ignored when using 251 | :func:`~CircuitBreaker.call`, preventing the breaker being triggered twice. 252 | """ 253 | 254 | def _outer_wrapper(func): 255 | @wraps(func) 256 | def _inner_wrapper(*args, **kwargs): 257 | return self.call(func, *args, **kwargs) 258 | 259 | @wraps(func) 260 | async def _inner_wrapper_async(*args, **kwargs): 261 | return await self.call_async(func, *args, **kwargs) 262 | 263 | return_func = _inner_wrapper_async if inspect.iscoroutinefunction(func) else _inner_wrapper 264 | return_func._ignore_on_call = ignore_on_call 265 | return return_func 266 | 267 | if len(call_args) == 1 and inspect.isfunction(call_args[0]): 268 | # if decorator called without arguments, pass the function on 269 | return _outer_wrapper(*call_args) 270 | elif len(call_args) == 0: 271 | # if decorator called with arguments, _outer_wrapper will receive the function 272 | return _outer_wrapper 273 | else: 274 | raise TypeError("Decorator does not accept positional arguments.") 275 | 276 | @property 277 | def listeners(self): 278 | """ 279 | Returns the registered listeners as a tuple. 280 | """ 281 | return tuple(self._listeners) 282 | 283 | def add_listener(self, listener): 284 | """ 285 | Registers a listener for this circuit breaker. 286 | """ 287 | self._listeners.append(listener) 288 | 289 | def add_listeners(self, *listeners): 290 | """ 291 | Registers listeners for this circuit breaker. 292 | """ 293 | for listener in listeners: 294 | self.add_listener(listener) 295 | 296 | def remove_listener(self, listener): 297 | """ 298 | Unregisters a listener of this circuit breaker. 299 | """ 300 | self._listeners.remove(listener) 301 | 302 | @property 303 | def name(self) -> str: 304 | """ 305 | Returns the name of this circuit breaker. Useful for logging. 306 | """ 307 | return self._name 308 | 309 | @name.setter 310 | def name(self, name: str): 311 | """ 312 | Set the name of this circuit breaker. 313 | """ 314 | self._name = name 315 | --------------------------------------------------------------------------------