├── setup.cfg ├── .gitignore ├── MANIFEST.in ├── requirements.txt ├── tox.ini ├── setup.py ├── tests └── test_awsretry.py ├── README.rst └── awsretry └── __init__.py /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=1 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.cache 3 | *.tox 4 | *.Python 5 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.rst 2 | include requirements.txt 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | boto 2 | boto3 3 | botocore 4 | 5 | [dev] 6 | check-manifest 7 | 8 | [test] 9 | coverage 10 | nose 11 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py{27,33} 3 | 4 | [testenv] 5 | basepython = 6 | py27: python2.7 7 | py33: python3 8 | deps = 9 | check-manifest 10 | {py27,py33}: readme_renderer 11 | flake8 12 | nose 13 | boto 14 | botocore 15 | boto3 16 | commands = 17 | check-manifest --ignore tox.ini,tests* 18 | {py27,py33}: python setup.py check -m -r -s 19 | flake8 --ignore=E501,F401 . 20 | nosetests 21 | [flake8] 22 | exclude = .tox,*.egg,build,data,lib,bin 23 | select = E,W,F 24 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | from os import path 3 | 4 | here = path.abspath(path.dirname(__file__)) 5 | 6 | setup( 7 | name='awsretry', 8 | 9 | version='1.0.2', 10 | 11 | description='Decorate your AWS Boto3 Calls with AWSRetry.backoff(). This will allows your calls to get around the AWS Eventual Consistency Errors.', 12 | long_description=open('README.rst').read(), 13 | 14 | url='https://github.com/linuxdynasty/awsretry', 15 | packages=find_packages(exclude=['tests*']), 16 | 17 | author='Allen Sanabria', 18 | author_email='asanabria@linuxdynasty.org', 19 | 20 | license='MIT', 21 | 22 | classifiers=[ 23 | 'Development Status :: 5 - Production/Stable', 24 | 25 | 'Intended Audience :: Developers', 26 | 'Topic :: Software Development :: Build Tools', 27 | 28 | 'License :: OSI Approved :: MIT License', 29 | 30 | 'Programming Language :: Python :: 2', 31 | 'Programming Language :: Python :: 2.7', 32 | 'Programming Language :: Python :: 3', 33 | 'Programming Language :: Python :: 3.6', 34 | ], 35 | 36 | keywords='boto3 aws retry awsretry backoff', 37 | 38 | install_requires=['boto', 'boto3', 'botocore'], 39 | 40 | extras_require={ 41 | 'dev': ['check-manifest'], 42 | 'test': ['coverage', 'nose'], 43 | }, 44 | ) 45 | -------------------------------------------------------------------------------- /tests/test_awsretry.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import botocore 4 | 5 | from awsretry import AWSRetry 6 | 7 | 8 | class RetryTestCase(unittest.TestCase): 9 | 10 | def test_no_failures(self): 11 | self.counter = 0 12 | 13 | @AWSRetry.backoff(tries=2, delay=0.1) 14 | def no_failures(): 15 | self.counter += 1 16 | 17 | no_failures() 18 | self.assertEqual(self.counter, 1) 19 | 20 | def test_retry_once(self): 21 | self.counter = 0 22 | err_msg = {'Error': {'Code': 'InstanceId.NotFound'}} 23 | 24 | @AWSRetry.backoff(tries=2, delay=0.1) 25 | def retry_once(): 26 | self.counter += 1 27 | if self.counter < 2: 28 | raise botocore.exceptions.ClientError( 29 | err_msg, 'Could not find you' 30 | ) 31 | else: 32 | return 'success' 33 | 34 | r = retry_once() 35 | self.assertEqual(r, 'success') 36 | self.assertEqual(self.counter, 2) 37 | 38 | def test_reached_limit(self): 39 | self.counter = 0 40 | err_msg = {'Error': {'Code': 'RequestLimitExceeded'}} 41 | 42 | @AWSRetry.backoff(tries=4, delay=0.1) 43 | def fail(): 44 | self.counter += 1 45 | raise botocore.exceptions.ClientError(err_msg, 'toooo fast!!') 46 | 47 | with self.assertRaises(botocore.exceptions.ClientError): 48 | fail() 49 | self.assertEqual(self.counter, 4) 50 | 51 | def test_unexpected_exception_does_not_retry(self): 52 | self.counter = 0 53 | err_msg = {'Error': {'Code': 'AuthFailure'}} 54 | 55 | @AWSRetry.backoff(tries=4, delay=0.1) 56 | def raise_unexpected_error(): 57 | self.counter += 1 58 | raise botocore.exceptions.ClientError(err_msg, 'unexpected error') 59 | 60 | with self.assertRaises(botocore.exceptions.ClientError): 61 | raise_unexpected_error() 62 | 63 | self.assertEqual(self.counter, 1) 64 | 65 | 66 | if __name__ == '__main__': 67 | unittest.main() 68 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | AWSRetry - Boto3 Retry/Backoff Decorator 3 | ======================================== 4 | 5 | AWSRetry is a Python Decorator that can be used to wrap boto3 function calls. 6 | This function was built out of the need to get around a couple of common issues 7 | when working with AWS API's. 8 | 9 | * Query API Request Rate 10 | * Eventual Consistency Model. 11 | 12 | 13 | Exceptions that will get retried when encountered 14 | ------------------------------------------------- 15 | * RequestLimitExceeded 16 | * Unavailable 17 | * ServiceUnavailable 18 | * InternalFailure 19 | * InternalError 20 | * ^\w+.NotFound 21 | 22 | This list can be extended. (http://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html) 23 | 24 | Quick Start 25 | ----------- 26 | Install awsretry. 27 | 28 | .. code-block:: sh 29 | 30 | $ pip install awsretry 31 | 32 | I will assume you know about setting up Boto3 Credentials, if not you can read 33 | the instructions here http://boto3.readthedocs.io/en/latest/guide/configuration.html 34 | 35 | 36 | Keyword Arguments that AWSRetry.backoff accepts 37 | ----------------------------------------------- 38 | 39 | * tries = The number of times to try before giving up. Default = 10 40 | * delay = The initial delay between retries in seconds. Default = 3 41 | * backoff = backoff multiplier e.g. value of 2 will double the delay each retry. Default = 1.1 42 | * added_exceptions = Other exceptions to retry on, beyond the defaults. Default = list() 43 | 44 | Examples 45 | -------- 46 | Write a quick function that implements AWSRetry.backoff() 47 | 48 | .. code-block:: python 49 | 50 | #!/usr/bin/env python 51 | 52 | import botocore 53 | import boto3 54 | from awsretry import AWSRetry 55 | 56 | 57 | @AWSRetry.backoff() 58 | def get_instances(): 59 | client = boto3.client('ec2') 60 | try: 61 | instances = client.describe_instances() 62 | return instances 63 | except botocore.exceptions.ClientError as e: 64 | raise e 65 | 66 | instances = get_instances() 67 | 68 | Write a quick function that will overwrite the default arguments. 69 | 70 | .. code-block:: python 71 | 72 | #!/usr/bin/env python 73 | 74 | import botocore 75 | import boto3 76 | from awsretry import AWSRetry 77 | 78 | 79 | @AWSRetry.backoff(tries=20, delay=2, backoff=1.5, added_exceptions=['ConcurrentTagAccess']) 80 | def create_tags(): 81 | client = boto3.client('ec2') 82 | try: 83 | resources = ['1-12345678891234'] 84 | tags = [{'Key': 'service', 'Value': 'web-app'}] 85 | instances = client.create_tags(Resources=resources, Tags=tags) 86 | except botocore.exceptions.ClientError as e: 87 | raise e 88 | 89 | create_tags() 90 | 91 | Development 92 | ----------- 93 | Assuming that you have Python and ``virtualenv`` installed, set up your 94 | environment and install the required dependencies like this instead of 95 | the ``pip install awsretry`` defined above: 96 | 97 | .. code-block:: sh 98 | 99 | $ git clone https://github.com/linuxdynasty/awsretry.git 100 | $ cd awsretry 101 | $ virtualenv venv 102 | ... 103 | $ . venv/bin/activate 104 | $ pip install -r requirements.txt 105 | $ pip install -e . 106 | 107 | Running Tests 108 | ------------- 109 | 110 | You can run the tests by using tox which implements nosetest or run them 111 | directly using nosetest. 112 | 113 | .. code-block:: sh 114 | 115 | $ tox 116 | $ tox tests/test_awsretry.py 117 | $ tox -e py27,py36 tests/ 118 | $ nosetest 119 | -------------------------------------------------------------------------------- /awsretry/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import absolute_import, division, print_function 4 | from __future__ import unicode_literals 5 | from functools import wraps 6 | import re 7 | import logging 8 | import time 9 | 10 | import botocore 11 | import boto 12 | import boto3 13 | 14 | __author__ = 'Allen Sanabria' 15 | __version__ = '1.0.2' 16 | 17 | 18 | class CloudRetry(object): 19 | """ CloudRetry can be used by any cloud provider, in order to implement a 20 | backoff algorithm/retry effect based on Status Code from Exceptions. 21 | """ 22 | # This is the base class of the exception. 23 | # AWS Example botocore.exceptions.ClientError 24 | @staticmethod 25 | def base_class(error): 26 | """ Return the base class of the error you are matching against. 27 | Args: 28 | error (object): The exception itself. 29 | """ 30 | pass 31 | 32 | @staticmethod 33 | def status_code_from_exception(error): 34 | """ Return the status code from the exception object. 35 | Args: 36 | error (object): The exception itself. 37 | """ 38 | pass 39 | 40 | @staticmethod 41 | def found(response_code): 42 | """ Return True if the Response Code to retry on was found. 43 | Args: 44 | response_code (str): This is the Response Code that is being 45 | matched against. 46 | """ 47 | pass 48 | 49 | @classmethod 50 | def backoff(cls, tries=10, delay=3, backoff=1.1, added_exceptions=list()): 51 | """ Retry calling the Cloud decorated function using an exponential 52 | backoff. 53 | Kwargs: 54 | tries (int): Number of times to try (not retry) before giving up. 55 | default=10 56 | delay (int): Initial delay between retries in seconds. 57 | default=3 58 | backoff (int): backoff multiplier e.g. value of 2 will double the 59 | delay each retry. 60 | default=2 61 | added_exceptions (list): Other exceptions to retry on. 62 | default=[] 63 | 64 | """ 65 | def deco(f): 66 | @wraps(f) 67 | def retry_func(*args, **kwargs): 68 | max_tries, max_delay = tries, delay 69 | while max_tries > 1: 70 | try: 71 | return f(*args, **kwargs) 72 | except Exception as e: 73 | base_exception_class = cls.base_class(e) 74 | if isinstance(e, base_exception_class): 75 | response_code = cls.status_code_from_exception(e) 76 | if cls.found(response_code, added_exceptions): 77 | logging.info("{0}: Retrying in {1} seconds...".format(str(e), max_delay)) 78 | time.sleep(max_delay) 79 | max_tries -= 1 80 | max_delay *= backoff 81 | else: 82 | # Return original exception if exception is not 83 | # a ClientError. 84 | raise e 85 | else: 86 | # Return original exception if exception is not a 87 | # ClientError 88 | raise e 89 | return f(*args, **kwargs) 90 | 91 | return retry_func # true decorator 92 | 93 | return deco 94 | 95 | 96 | class AWSRetry(CloudRetry): 97 | @staticmethod 98 | def base_class(error): 99 | if isinstance(error, botocore.exceptions.ClientError): 100 | return botocore.exceptions.ClientError 101 | 102 | elif isinstance(error, boto.compat.StandardError): 103 | return boto.compat.StandardError 104 | 105 | elif isinstance(error, botocore.exceptions.WaiterError): 106 | return botocore.exceptions.WaiterError 107 | 108 | else: 109 | return type(None) 110 | 111 | @staticmethod 112 | def status_code_from_exception(error): 113 | if isinstance(error, botocore.exceptions.ClientError): 114 | return error.response['Error']['Code'] 115 | if isinstance(error, botocore.exceptions.WaiterError): 116 | return error.last_response['Error']['Code'] 117 | elif hasattr(error, 'error_code'): 118 | return error.error_code 119 | return error.__class__.__name__ 120 | 121 | @staticmethod 122 | def found(response_code, added_exceptions): 123 | # This list of failures is based on this API Reference 124 | # http://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html 125 | retry_on = [ 126 | 'RequestLimitExceeded', 'Unavailable', 'ServiceUnavailable', 127 | 'InternalFailure', 'InternalError', 'LimitExceededException', 128 | 'TooManyRequestsException', 'ThrottlingException' 129 | ] 130 | retry_on.extend(added_exceptions) 131 | 132 | not_found = re.compile(r'^\w+.NotFound') 133 | if response_code in retry_on or not_found.search(response_code): 134 | return True 135 | else: 136 | return False 137 | --------------------------------------------------------------------------------