├── .gitignore ├── MANIFEST ├── CHANGELOG ├── setup.py ├── README.md ├── asyncdynamo ├── __init__.py ├── async_aws_sts.py └── asyncdynamo.py └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | *.pyc 3 | -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | setup.py 2 | LICENSE 3 | README.md 4 | asyncdynamo/__init__.py 5 | asyncdynamo/async_aws_sts.py 6 | asyncdynamo/asyncdynamo.py 7 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | Version 0.2.8 - 2013-02-28 2 | * Fix compatibility issues wth boto>=2.7.0 3 | 4 | Version 0.2.6 - 2013-01-10 5 | * Allow user-defined IOLoop 6 | * Change error construction to comply with boto 2.3.0 7 | * Handle edge case where no json response is received 8 | 9 | Version 0.2.5 - 2012-03-05 10 | * Improve error handling from STS 11 | 12 | Version 0.2.4 - 2012-02-23 13 | * Fix RangeKeyAttribute bug in queries 14 | * Add put_item method 15 | 16 | Version 0.2.3 - 2012-02-06 17 | * Always pass error argument 18 | 19 | Version 0.2.2 - 2012-02-06 20 | * Include response body to callbacks w/ errors 21 | 22 | Version 0.2.1 - 2012-02-06 23 | * Pass errors to callback instead of raising internally 24 | 25 | Version 0.2 - 2012-02-02 26 | * Configurable auth 27 | 28 | Version 0.1 - 2012-02-01 29 | * Initial release 30 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from distutils.core import setup 3 | 4 | # also update version in __init__.py 5 | version = '0.2.8' 6 | 7 | setup( 8 | name="asyncdynamo", 9 | version=version, 10 | keywords=["dynamo", "dynamodb", "amazon", "async", "tornado"], 11 | long_description=open(os.path.join(os.path.dirname(__file__), "README.md"), "r").read(), 12 | description="async Amazon DynamoDB library for Tornado", 13 | author="Dan Frank", 14 | author_email="df@bit.ly", 15 | url="http://github.com/bitly/asyncdynamo", 16 | license="Apache Software License", 17 | classifiers=[ 18 | "License :: OSI Approved :: Apache Software License", 19 | ], 20 | packages=['asyncdynamo'], 21 | install_requires=['tornado', 'boto>=2.3.0'], 22 | requires=['tornado'], 23 | download_url="https://s3.amazonaws.com/bitly-downloads/asyncdynamo/asyncdynamo-%s.tar.gz" % version, 24 | ) 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Asyncdynamo 2 | =========== 3 | 4 | Asynchronous Amazon DynamoDB library for Tornado 5 | 6 | Requires boto>=2.3 and python 2.7 7 | 8 | Tested with Tornado 1.2.1 9 | 10 | Installation 11 | ------------ 12 | 13 | Installing from github: `pip install git+https://github.com/bitly/asyncdynamo.git` 14 | 15 | Installing from source: `git clone git://github.com/bitly/asyncdynamo.git; cd asyncdynamo; python setup.py install` 16 | 17 | Usage 18 | ----- 19 | Asyncdynamo syntax seeks to mirror that of [Boto](http://github.com/boto/boto). 20 | 21 | ```python 22 | from asyncdynamo import asyncdynamo 23 | db = asyncdynamo.AsyncDynamoDB("YOUR_ACCESS_KEY", "YOUR_SECRET_KEY") 24 | 25 | def item_cb(item): 26 | print item 27 | 28 | db.get_item('YOUR_TABLE_NAME', 'ITEM_KEY', item_cb) 29 | ``` 30 | 31 | Requirements 32 | ------------ 33 | The following two python libraries are required 34 | 35 | * [boto](http://github.com/boto/boto) 36 | * [tornado](http://github.com/facebook/tornado) 37 | 38 | Issues 39 | ------ 40 | 41 | Please report any issues via [github issues](https://github.com/bitly/asyncdynamo/issues) 42 | -------------------------------------------------------------------------------- /asyncdynamo/__init__.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | # 3 | # Copyright 2010 bit.ly 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | 17 | """ 18 | async Amazon DynamoDB library for Tornado 19 | 20 | http://github.com/bitly/asyncdynamo 21 | """ 22 | try: 23 | import tornado 24 | except ImportError: 25 | raise ImportError("tornado library not installed. Install tornado. https://github.com/facebook/tornado") 26 | try: 27 | import boto 28 | assert tuple(map(int,boto.Version.split('.'))) >= (2,3,0), "Boto >= 2.3.0 required." 29 | except ImportError: 30 | raise ImportError("boto library not installed. Install boto. https://github.com/boto/boto") 31 | 32 | version = "0.2.8" 33 | version_info = (0, 2, 8) 34 | -------------------------------------------------------------------------------- /asyncdynamo/async_aws_sts.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | # 3 | # Copyright 2012 bit.ly 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | """ 17 | Created by Dan Frank on 2012-01-25. 18 | Copyright (c) 2012 bit.ly. All rights reserved. 19 | """ 20 | 21 | import functools 22 | from tornado.httpclient import HTTPRequest 23 | from tornado.httpclient import AsyncHTTPClient 24 | import xml.sax 25 | 26 | import boto 27 | from boto.sts.connection import STSConnection 28 | from boto.sts.credentials import Credentials 29 | from boto.exception import BotoServerError 30 | 31 | class InvalidClientTokenIdError(BotoServerError): 32 | ''' 33 | Error subclass to indicate that the client's token(s) is/are invalid 34 | ''' 35 | pass 36 | 37 | class AsyncAwsSts(STSConnection): 38 | ''' 39 | Class that manages session tokens. Users of AsyncDynamoDB should not 40 | need to worry about what goes on here. 41 | 42 | Usage: Keep an instance of this class (though it should be cheap to 43 | re instantiate) and periodically call get_session_token to get a new 44 | Credentials object when, say, your session token expires 45 | ''' 46 | 47 | def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, 48 | is_secure=True, port=None, proxy=None, proxy_port=None, 49 | proxy_user=None, proxy_pass=None, debug=0, 50 | https_connection_factory=None, region=None, path='/', 51 | converter=None, ioloop=None): 52 | STSConnection.__init__(self, aws_access_key_id, 53 | aws_secret_access_key, 54 | is_secure, port, proxy, proxy_port, 55 | proxy_user, proxy_pass, debug, 56 | https_connection_factory, region, path, converter) 57 | self.http_client = AsyncHTTPClient(io_loop=ioloop) 58 | 59 | def get_session_token(self, callback): 60 | ''' 61 | Gets a new Credentials object with a session token, using this 62 | instance's aws keys. Callback should operate on the new Credentials obj, 63 | or else a boto.exception.BotoServerError 64 | ''' 65 | return self.get_object('GetSessionToken', {}, Credentials, verb='POST', callback=callback) 66 | 67 | def get_object(self, action, params, cls, path="/", parent=None, verb="GET", callback=None): 68 | ''' 69 | Get an instance of `cls` using `action` 70 | ''' 71 | if not parent: 72 | parent = self 73 | self.make_request(action, params, path, verb, 74 | functools.partial(self._finish_get_object, callback=callback, parent=parent, cls=cls)) 75 | 76 | def _finish_get_object(self, response_body, callback, cls=None, parent=None, error=None): 77 | ''' 78 | Process the body returned by STS. If an error is present, convert from a tornado error 79 | to a boto error 80 | ''' 81 | if error: 82 | if error.code == 403: 83 | error_class = InvalidClientTokenIdError 84 | else: 85 | error_class = BotoServerError 86 | return callback(None, error=error_class(error.code, error.message, response_body)) 87 | obj = cls(parent) 88 | h = boto.handler.XmlHandler(obj, parent) 89 | xml.sax.parseString(response_body, h) 90 | return callback(obj) 91 | 92 | def make_request(self, action, params={}, path='/', verb='GET', callback=None): 93 | ''' 94 | Make an async request. This handles the logic of translating from boto params 95 | to a tornado request obj, issuing the request, and passing back the body. 96 | 97 | The callback should operate on the body of the response, and take an optional 98 | error argument that will be a tornado error 99 | ''' 100 | request = HTTPRequest('https://%s' % self.host, 101 | method=verb) 102 | request.params = params 103 | request.auth_path = '/' # need this for auth 104 | request.host = self.host # need this for auth 105 | if action: 106 | request.params['Action'] = action 107 | if self.APIVersion: 108 | request.params['Version'] = self.APIVersion 109 | self._auth_handler.add_auth(request) # add signature 110 | self.http_client.fetch(request, functools.partial(self._finish_make_request, callback=callback)) 111 | 112 | def _finish_make_request(self, response, callback): 113 | if response.error: 114 | return callback(response.body, error=response.error) 115 | return callback(response.body) 116 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /asyncdynamo/asyncdynamo.py: -------------------------------------------------------------------------------- 1 | #!/bin/env python 2 | # 3 | # Copyright 2012 bit.ly 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); you may 6 | # not use this file except in compliance with the License. You may obtain 7 | # a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | # License for the specific language governing permissions and limitations 15 | # under the License. 16 | """ 17 | Created by Dan Frank on 2012-01-23. 18 | Copyright (c) 2012 bit.ly. All rights reserved. 19 | """ 20 | import sys 21 | assert sys.version_info >= (2, 7), "run this with python2.7" 22 | 23 | import simplejson as json 24 | from tornado.httpclient import HTTPRequest 25 | from tornado.httpclient import AsyncHTTPClient 26 | from tornado.ioloop import IOLoop 27 | import functools 28 | from collections import deque 29 | import time 30 | import logging 31 | 32 | from boto.connection import AWSAuthConnection 33 | from boto.exception import DynamoDBResponseError 34 | from boto.auth import HmacAuthV3HTTPHandler 35 | from boto.provider import Provider 36 | 37 | from async_aws_sts import AsyncAwsSts, InvalidClientTokenIdError 38 | 39 | PENDING_SESSION_TOKEN_UPDATE = "this is not your session token" 40 | 41 | class AsyncDynamoDB(AWSAuthConnection): 42 | """ 43 | The main class for asynchronous connections to DynamoDB. 44 | 45 | The user should maintain one instance of this class (though more than one is ok), 46 | parametrized with the user's access key and secret key. Make calls with make_request 47 | or the helper methods, and AsyncDynamoDB will maintain session tokens in the background. 48 | 49 | 50 | As in Boto Layer1: 51 | "This is the lowest-level interface to DynamoDB. Methods at this 52 | layer map directly to API requests and parameters to the methods 53 | are either simple, scalar values or they are the Python equivalent 54 | of the JSON input as defined in the DynamoDB Developer's Guide. 55 | All responses are direct decoding of the JSON response bodies to 56 | Python data structures via the json or simplejson modules." 57 | """ 58 | 59 | DefaultHost = 'dynamodb.us-east-1.amazonaws.com' 60 | """The default DynamoDB API endpoint to connect to.""" 61 | 62 | ServiceName = 'DynamoDB' 63 | """The name of the Service""" 64 | 65 | Version = '20111205' 66 | """DynamoDB API version.""" 67 | 68 | ThruputError = "ProvisionedThroughputExceededException" 69 | """The error response returned when provisioned throughput is exceeded""" 70 | 71 | ExpiredSessionError = 'com.amazon.coral.service#ExpiredTokenException' 72 | """The error response returned when session token has expired""" 73 | 74 | UnrecognizedClientException = 'com.amazon.coral.service#UnrecognizedClientException' 75 | '''Another error response that is possible with a bad session token''' 76 | 77 | def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, 78 | is_secure=True, port=None, proxy=None, proxy_port=None, 79 | host=None, debug=0, session_token=None, 80 | authenticate_requests=True, validate_cert=True, max_sts_attempts=3, ioloop=None): 81 | if not host: 82 | host = self.DefaultHost 83 | self.validate_cert = validate_cert 84 | self.authenticate_requests = authenticate_requests 85 | AWSAuthConnection.__init__(self, host, 86 | aws_access_key_id, 87 | aws_secret_access_key, 88 | is_secure, port, proxy, proxy_port, 89 | debug=debug, security_token=session_token) 90 | self.ioloop = ioloop or IOLoop.instance() 91 | self.http_client = AsyncHTTPClient(io_loop=self.ioloop) 92 | self.pending_requests = deque() 93 | self.sts = AsyncAwsSts(aws_access_key_id, aws_secret_access_key, ioloop=self.ioloop) 94 | assert (isinstance(max_sts_attempts, int) and max_sts_attempts >= 0) 95 | self.max_sts_attempts = max_sts_attempts 96 | 97 | def _init_session_token_cb(self, error=None): 98 | if error: 99 | logging.warn("Unable to get session token: %s" % error) 100 | 101 | def _required_auth_capability(self): 102 | return ['hmac-v3-http'] 103 | 104 | def _update_session_token(self, callback, attempts=0, bypass_lock=False): 105 | ''' 106 | Begins the logic to get a new session token. Performs checks to ensure 107 | that only one request goes out at a time and that backoff is respected, so 108 | it can be called repeatedly with no ill effects. Set bypass_lock to True to 109 | override this behavior. 110 | ''' 111 | if self.provider.security_token == PENDING_SESSION_TOKEN_UPDATE and not bypass_lock: 112 | return 113 | self.provider.security_token = PENDING_SESSION_TOKEN_UPDATE # invalidate the current security token 114 | return self.sts.get_session_token( 115 | functools.partial(self._update_session_token_cb, callback=callback, attempts=attempts)) 116 | 117 | def _update_session_token_cb(self, creds, provider='aws', callback=None, error=None, attempts=0): 118 | ''' 119 | Callback to use with `async_aws_sts`. The 'provider' arg is a bit misleading, 120 | it is a relic from boto and should probably be left to its default. This will 121 | take the new Credentials obj from `async_aws_sts.get_session_token()` and use 122 | it to update self.provider, and then will clear the deque of pending requests. 123 | 124 | A callback is optional. If provided, it must be callable without any arguments, 125 | but also accept an optional error argument that will be an instance of BotoServerError. 126 | ''' 127 | def raise_error(): 128 | # get out of locked state 129 | self.provider.security_token = None 130 | if callable(callback): 131 | return callback(error=error) 132 | else: 133 | logging.error(error) 134 | raise error 135 | if error: 136 | if isinstance(error, InvalidClientTokenIdError): 137 | # no need to retry if error is due to bad tokens 138 | raise_error() 139 | else: 140 | if attempts > self.max_sts_attempts: 141 | raise_error() 142 | else: 143 | seconds_to_wait = (0.1*(2**attempts)) 144 | logging.warning("Got error[ %s ] getting session token, retrying in %.02f seconds" % (error, seconds_to_wait)) 145 | self.ioloop.add_timeout(time.time() + seconds_to_wait, 146 | functools.partial(self._update_session_token, attempts=attempts+1, callback=callback, bypass_lock=True)) 147 | return 148 | else: 149 | self.provider = Provider(provider, 150 | creds.access_key, 151 | creds.secret_key, 152 | creds.session_token) 153 | # force the correct auth, with the new provider 154 | self._auth_handler = HmacAuthV3HTTPHandler(self.host, None, self.provider) 155 | while self.pending_requests: 156 | request = self.pending_requests.pop() 157 | request() 158 | if callable(callback): 159 | return callback() 160 | 161 | def make_request(self, action, body='', callback=None, object_hook=None): 162 | ''' 163 | Make an asynchronous HTTP request to DynamoDB. Callback should operate on 164 | the decoded json response (with object hook applied, of course). It should also 165 | accept an error argument, which will be a boto.exception.DynamoDBResponseError. 166 | 167 | If there is not a valid session token, this method will ensure that a new one is fetched 168 | and cache the request when it is retrieved. 169 | ''' 170 | this_request = functools.partial(self.make_request, action=action, 171 | body=body, callback=callback,object_hook=object_hook) 172 | if self.authenticate_requests and self.provider.security_token in [None, PENDING_SESSION_TOKEN_UPDATE]: 173 | # we will not be able to complete this request because we do not have a valid session token. 174 | # queue it and try to get a new one. _update_session_token will ensure that only one request 175 | # for a session token goes out at a time 176 | self.pending_requests.appendleft(this_request) 177 | def cb_for_update(error=None): 178 | # create a callback to handle errors getting session token 179 | # callback here is assumed to take a json response, and an instance of DynamoDBResponseError 180 | if error: 181 | return callback({}, error=DynamoDBResponseError(error.status, error.reason, error.body)) 182 | else: 183 | return 184 | self._update_session_token(cb_for_update) 185 | return 186 | headers = {'X-Amz-Target' : '%s_%s.%s' % (self.ServiceName, 187 | self.Version, action), 188 | 'Content-Type' : 'application/x-amz-json-1.0', 189 | 'Content-Length' : str(len(body))} 190 | request = HTTPRequest('https://%s' % self.host, 191 | method='POST', 192 | headers=headers, 193 | body=body, 194 | validate_cert=self.validate_cert) 195 | request.path = '/' # Important! set the path variable for signing by boto (<2.7). '/' is the path for all dynamodb requests 196 | request.auth_path = '/' # Important! set the auth_path variable for signing by boto(>2.7). '/' is the path for all dynamodb requests 197 | if self.authenticate_requests: 198 | self._auth_handler.add_auth(request) # add signature to headers of the request 199 | self.http_client.fetch(request, functools.partial(self._finish_make_request, 200 | callback=callback, orig_request=this_request, token_used=self.provider.security_token, object_hook=object_hook)) # bam! 201 | 202 | def _finish_make_request(self, response, callback, orig_request, token_used, object_hook=None): 203 | ''' 204 | Check for errors and decode the json response (in the tornado response body), then pass on to orig callback. 205 | This method also contains some of the logic to handle reacquiring session tokens. 206 | ''' 207 | try: 208 | json_response = json.loads(response.body, object_hook=object_hook) 209 | except TypeError: 210 | json_response = None 211 | 212 | if json_response and response.error: 213 | # Normal error handling where we have a JSON response from AWS. 214 | if any((token_error in json_response.get('__type', []) \ 215 | for token_error in (self.ExpiredSessionError, self.UnrecognizedClientException))): 216 | if self.provider.security_token == token_used: 217 | # the token that we used has expired. wipe it out 218 | self.provider.security_token = None 219 | return orig_request() # make_request will handle logic to get a new token if needed, and queue until it is fetched 220 | else: 221 | # because some errors are benign, include the response when an error is passed 222 | return callback(json_response, error=DynamoDBResponseError(response.error.code, 223 | response.error.message, json_response)) 224 | 225 | if json_response is None: 226 | # We didn't get any JSON back, but we also didn't receive an error response. This can't be right. 227 | return callback(None, error=DynamoDBResponseError(response.code, response.body)) 228 | else: 229 | return callback(json_response, error=None) 230 | 231 | def get_item(self, table_name, key, callback, attributes_to_get=None, 232 | consistent_read=False, object_hook=None): 233 | ''' 234 | Return a set of attributes for an item that matches 235 | the supplied key. 236 | 237 | The callback should operate on a dict representing the decoded 238 | response from DynamoDB (using the object_hook, if supplied) 239 | 240 | :type table_name: str 241 | :param table_name: The name of the table to delete. 242 | 243 | :type key: dict 244 | :param key: A Python version of the Key data structure 245 | defined by DynamoDB. 246 | 247 | :type attributes_to_get: list 248 | :param attributes_to_get: A list of attribute names. 249 | If supplied, only the specified attribute names will 250 | be returned. Otherwise, all attributes will be returned. 251 | 252 | :type consistent_read: bool 253 | :param consistent_read: If True, a consistent read 254 | request is issued. Otherwise, an eventually consistent 255 | request is issued. ''' 256 | data = {'TableName': table_name, 257 | 'Key': key} 258 | if attributes_to_get: 259 | data['AttributesToGet'] = attributes_to_get 260 | if consistent_read: 261 | data['ConsistentRead'] = True 262 | return self.make_request('GetItem', body=json.dumps(data), 263 | callback=callback, object_hook=object_hook) 264 | 265 | def batch_get_item(self, request_items, callback): 266 | """ 267 | Return a set of attributes for a multiple items in 268 | multiple tables using their primary keys. 269 | 270 | The callback should operate on a dict representing the decoded 271 | response from DynamoDB (using the object_hook, if supplied) 272 | 273 | :type request_items: dict 274 | :param request_items: A Python version of the RequestItems 275 | data structure defined by DynamoDB. 276 | """ 277 | data = {'RequestItems' : request_items} 278 | json_input = json.dumps(data) 279 | self.make_request('BatchGetItem', json_input, callback) 280 | 281 | def put_item(self, table_name, item, callback, expected=None, return_values=None, object_hook=None): 282 | ''' 283 | Create a new item or replace an old item with a new 284 | item (including all attributes). If an item already 285 | exists in the specified table with the same primary 286 | key, the new item will completely replace the old item. 287 | You can perform a conditional put by specifying an 288 | expected rule. 289 | 290 | The callback should operate on a dict representing the decoded 291 | response from DynamoDB (using the object_hook, if supplied) 292 | 293 | :type table_name: str 294 | :param table_name: The name of the table to delete. 295 | 296 | :type item: dict 297 | :param item: A Python version of the Item data structure 298 | defined by DynamoDB. 299 | 300 | :type expected: dict 301 | :param expected: A Python version of the Expected 302 | data structure defined by DynamoDB. 303 | 304 | :type return_values: str 305 | :param return_values: Controls the return of attribute 306 | name-value pairs before then were changed. Possible 307 | values are: None or 'ALL_OLD'. If 'ALL_OLD' is 308 | specified and the item is overwritten, the content 309 | of the old item is returned. 310 | ''' 311 | data = {'TableName' : table_name, 312 | 'Item' : item} 313 | if expected: 314 | data['Expected'] = expected 315 | if return_values: 316 | data['ReturnValues'] = return_values 317 | json_input = json.dumps(data) 318 | return self.make_request('PutItem', json_input, callback=callback, 319 | object_hook=object_hook) 320 | 321 | def query(self, table_name, hash_key_value, callback, range_key_conditions=None, 322 | attributes_to_get=None, limit=None, consistent_read=False, 323 | scan_index_forward=True, exclusive_start_key=None, 324 | object_hook=None): 325 | ''' 326 | Perform a query of DynamoDB. This version is currently punting 327 | and expecting you to provide a full and correct JSON body 328 | which is passed as is to DynamoDB. 329 | 330 | The callback should operate on a dict representing the decoded 331 | response from DynamoDB (using the object_hook, if supplied) 332 | 333 | :type table_name: str 334 | :param table_name: The name of the table to delete. 335 | 336 | :type hash_key_value: dict 337 | :param key: A DynamoDB-style HashKeyValue. 338 | 339 | :type range_key_conditions: dict 340 | :param range_key_conditions: A Python version of the 341 | RangeKeyConditions data structure. 342 | 343 | :type attributes_to_get: list 344 | :param attributes_to_get: A list of attribute names. 345 | If supplied, only the specified attribute names will 346 | be returned. Otherwise, all attributes will be returned. 347 | 348 | :type limit: int 349 | :param limit: The maximum number of items to return. 350 | 351 | :type consistent_read: bool 352 | :param consistent_read: If True, a consistent read 353 | request is issued. Otherwise, an eventually consistent 354 | request is issued. 355 | 356 | :type scan_index_forward: bool 357 | :param scan_index_forward: Specified forward or backward 358 | traversal of the index. Default is forward (True). 359 | 360 | :type exclusive_start_key: list or tuple 361 | :param exclusive_start_key: Primary key of the item from 362 | which to continue an earlier query. This would be 363 | provided as the LastEvaluatedKey in that query. 364 | ''' 365 | data = {'TableName': table_name, 366 | 'HashKeyValue': hash_key_value} 367 | if range_key_conditions: 368 | data['RangeKeyCondition'] = range_key_conditions 369 | if attributes_to_get: 370 | data['AttributesToGet'] = attributes_to_get 371 | if limit: 372 | data['Limit'] = limit 373 | if consistent_read: 374 | data['ConsistentRead'] = True 375 | if scan_index_forward: 376 | data['ScanIndexForward'] = True 377 | else: 378 | data['ScanIndexForward'] = False 379 | if exclusive_start_key: 380 | data['ExclusiveStartKey'] = exclusive_start_key 381 | json_input = json.dumps(data) 382 | return self.make_request('Query', body=json_input, 383 | callback=callback, object_hook=object_hook) 384 | --------------------------------------------------------------------------------