├── .gitignore ├── LICENSE ├── LICENSE.txt ├── README.rst ├── asyncaws ├── __init__.py ├── core.py ├── sns.py └── sqs.py ├── docs ├── Makefile ├── conf.py ├── index.rst ├── sns.rst └── sqs.rst ├── examples ├── __init__.py ├── minimal.py ├── sns │ ├── __init__.py │ └── create_and_publish.py └── sqs │ ├── __init__.py │ ├── create_and_send.py │ └── on_message_callback.py ├── setup.cfg ├── setup.py └── tests ├── __init__.py └── integration ├── __init__.py ├── test_sns.py └── test_sqs.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Virtual Env 9 | venv 10 | 11 | # Distribution / packaging 12 | .Python 13 | env/ 14 | bin/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # Installer logs 29 | pip-log.txt 30 | pip-delete-this-directory.txt 31 | 32 | # Unit test / coverage reports 33 | htmlcov/ 34 | .tox/ 35 | .coverage 36 | .cache 37 | nosetests.xml 38 | coverage.xml 39 | 40 | # Translations 41 | *.mo 42 | 43 | # Mr Developer 44 | .mr.developer.cfg 45 | .project 46 | .pydevproject 47 | 48 | # Rope 49 | .ropeproject 50 | 51 | # Idea 52 | .idea 53 | 54 | # Django stuff: 55 | *.log 56 | *.pot 57 | 58 | # Sphinx documentation 59 | docs/_build/ 60 | 61 | # Private files 62 | config.json 63 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Anton Caceres 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Anton C 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | docs/index.rst -------------------------------------------------------------------------------- /asyncaws/__init__.py: -------------------------------------------------------------------------------- 1 | """AsyncAWS root, implements Facade for AWS APIs""" 2 | from asyncaws.sqs import SQS 3 | from asyncaws.sns import SNS 4 | -------------------------------------------------------------------------------- /asyncaws/core.py: -------------------------------------------------------------------------------- 1 | from tornado.httpclient import HTTPRequest, HTTPClient, AsyncHTTPClient 2 | from tornado.ioloop import IOLoop 3 | from tornado.httputil import url_concat 4 | from concurrent.futures import Future 5 | from urlparse import urlparse 6 | from lxml import objectify 7 | import datetime 8 | import hashlib 9 | import hmac 10 | 11 | 12 | def sign(key, msg): 13 | """Make sha256 signature""" 14 | return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest() 15 | 16 | 17 | def get_signature_key(key, date_stamp, region_name, service_name): 18 | """Sign all params sequentially""" 19 | k_date = sign(('AWS4' + key).encode('utf-8'), date_stamp) 20 | k_region = sign(k_date, region_name) 21 | k_service = sign(k_region, service_name) 22 | k_signing = sign(k_service, 'aws4_request') 23 | return k_signing 24 | 25 | 26 | class AWSRequest(HTTPRequest): 27 | """ 28 | Generic AWS Adapter for Tornado HTTP request 29 | Generates v4 signature and sets all required headers 30 | """ 31 | def __init__(self, *args, **kwargs): 32 | service = kwargs['service'] 33 | region = kwargs['region'] 34 | method = kwargs.get('method', 'GET') 35 | url = kwargs.get('url') or args[0] 36 | # tornado url_concat encodes spaces as '+', but AWS expects '%20' 37 | url = url.replace('+', '%20') 38 | parsed_url = urlparse(url) 39 | host = parsed_url.netloc 40 | canonical_uri = parsed_url.path 41 | # sort params alphabetically 42 | params = sorted(parsed_url.query.split('&')) 43 | canonical_querystring = '&'.join(params) 44 | kwargs['url'] = url.replace(parsed_url.query, canonical_querystring) 45 | # reset args, everything is passed with kwargs 46 | args = tuple() 47 | # prepare timestamps 48 | utc_time = datetime.datetime.utcnow() 49 | amz_date = utc_time.strftime('%Y%m%dT%H%M%SZ') 50 | date_stamp = utc_time.strftime('%Y%m%d') 51 | # prepare aws-specific headers 52 | canonical_headers = 'host:{host}\nx-amz-date:{amz_date}\n'.format( 53 | host=host, amz_date=amz_date) 54 | signed_headers = 'host;x-amz-date' 55 | # for GET requests payload is empty 56 | payload_hash = hashlib.sha256('').hexdigest() 57 | 58 | canonical_request = ( 59 | '{method}\n{canonical_uri}\n{canonical_querystring}' 60 | '\n{canonical_headers}\n{signed_headers}\n{payload_hash}' 61 | ).format( 62 | method=method, canonical_uri=canonical_uri, 63 | canonical_querystring=canonical_querystring, 64 | canonical_headers=canonical_headers, signed_headers=signed_headers, 65 | payload_hash=payload_hash 66 | ) 67 | # creating signature 68 | algorithm = 'AWS4-HMAC-SHA256' 69 | scope = '{date_stamp}/{region}/{service}/aws4_request'.format( 70 | date_stamp=date_stamp, region=region, service=service) 71 | string_to_sign = '{algorithm}\n{amz_date}\n{scope}\n{hash}'.format( 72 | algorithm=algorithm, amz_date=amz_date, scope=scope, 73 | hash=hashlib.sha256(canonical_request).hexdigest()) 74 | sign_key = get_signature_key(kwargs['secret_key'], 75 | date_stamp, region, service) 76 | hash_tuple = (sign_key, string_to_sign.encode('utf-8'), hashlib.sha256) 77 | signature = hmac.new(*hash_tuple).hexdigest() 78 | authorization_header = ( 79 | '{algorithm} Credential={access_key}/{scope}, ' 80 | 'SignedHeaders={signed_headers}, Signature={signature}' 81 | ).format( 82 | algorithm=algorithm, access_key=kwargs['access_key'], scope=scope, 83 | signed_headers=signed_headers, signature=signature 84 | ) 85 | # clean-up kwargs 86 | del kwargs['access_key'] 87 | del kwargs['secret_key'] 88 | del kwargs['service'] 89 | del kwargs['region'] 90 | # update headers 91 | headers = kwargs.get('headers', {}) 92 | headers.update({'x-amz-date': amz_date, 93 | 'Authorization': authorization_header}) 94 | kwargs['headers'] = headers 95 | # init Tornado HTTPRequest 96 | super(AWSRequest, self).__init__(*args, **kwargs) 97 | 98 | 99 | class AWS(object): 100 | """ 101 | Generic class for AWS API implementations: SQS, SNS, etc 102 | """ 103 | def __init__(self, access_key, secret_key, region, async=True): 104 | self.region = region 105 | self.__access_key = access_key 106 | self.__secret_key = secret_key 107 | self._http = AsyncHTTPClient() if async else HTTPClient() 108 | self._async = async 109 | 110 | def _process(self, url, params, service, parse_function): 111 | """Prepare request and result parsing callback""" 112 | full_url = url_concat(url, params) 113 | request = AWSRequest(full_url, service=service, region=self.region, 114 | access_key=self.__access_key, 115 | secret_key=self.__secret_key) 116 | if not self._async: 117 | http_response = self._http.fetch(request) 118 | xml_root = objectify.fromstring(http_response.body) 119 | response = parse_function(xml_root) 120 | return response 121 | 122 | ioloop = IOLoop.current() 123 | final_result = Future() 124 | 125 | def inject_result(future): 126 | """callback to connect AsyncHTTPClient future with parse function""" 127 | raw_response = future.result().body 128 | xml_root = objectify.fromstring(raw_response) 129 | final_result.set_result(parse_function(xml_root)) 130 | 131 | ioloop.add_future(self._http.fetch(request), inject_result) 132 | return final_result 133 | -------------------------------------------------------------------------------- /asyncaws/sns.py: -------------------------------------------------------------------------------- 1 | """Module that covers SNS API""" 2 | from asyncaws.core import AWS 3 | import json 4 | 5 | 6 | class SNS(AWS): 7 | """ 8 | :param access_key: AWS_ACCESS_KEY_ID 9 | :param secret_key: AWS_SECRET_ACCESS_KEY 10 | :param region: region name as string 11 | :param async: True by default, indicates that AsyncHTTPClient should 12 | be used. Otherwise HTTPClient (synchronous). Useful for debugging. 13 | """ 14 | common_params = { 15 | "Version": "2010-03-31", 16 | } 17 | service = 'sns' 18 | 19 | def create_topic(self, name): 20 | """ 21 | Creates a topic to which notifications can be published. 22 | This action is idempotent, so if the requester already owns 23 | a topic with the specified name, that topic's ARN 24 | is returned without creating a new topic. 25 | AWS API: CreateTopic_ 26 | 27 | :param name: The name of the topic to create. 28 | :return: TopicArn - The Amazon Resource Name assigned to the topic. 29 | """ 30 | params = { 31 | "Name": name, 32 | "Action": "CreateTopic" 33 | } 34 | params.update(self.common_params) 35 | url = "http://{service}.{region}.amazonaws.com/".format( 36 | service=self.service, region=self.region) 37 | parse_function = lambda root: root.CreateTopicResult.TopicArn.text 38 | return self._process(url, params, self.service, parse_function) 39 | 40 | def delete_topic(self, topic_arn): 41 | """ 42 | Deletes a topic and all its subscriptions. 43 | Deleting a topic might prevent some messages previously sent to the 44 | topic from being delivered to subscribers. This action is idempotent, 45 | so deleting a topic that does not exist does not result in an error. 46 | AWS API: DeleteTopic_ 47 | 48 | :param topic_arn: The ARN of the topic to delete 49 | :return: RequestId 50 | """ 51 | params = { 52 | "TopicArn": topic_arn, 53 | "Action": "DeleteTopic" 54 | } 55 | params.update(self.common_params) 56 | url = "http://{service}.{region}.amazonaws.com/".format( 57 | service=self.service, region=self.region) 58 | parse_function = lambda root: root.ResponseMetadata.RequestId.text 59 | return self._process(url, params, self.service, parse_function) 60 | 61 | def subscribe(self, endpoint, topic_arn, protocol): 62 | """ 63 | Prepares to subscribe an endpoint by sending the endpoint 64 | a confirmation message. To actually create a subscription, 65 | the endpoint owner must call the ConfirmSubscription action 66 | with the token from the confirmation 67 | AWS API: Subscribe_ 68 | 69 | :param endpoint: The endpoint to receive notifications. 70 | :param topic_arn: The ARN of the topic to subscribe to. 71 | :param protocol: The protocol to use (http, email, sms, sqs etc) 72 | :return: SubscriptionARN, if the service was able to create it 73 | immediately (without requiring endpoint owner confirmation). 74 | 75 | """ 76 | params = { 77 | "Endpoint": endpoint, 78 | "Protocol": protocol, 79 | "TopicArn": topic_arn, 80 | "Action": "Subscribe" 81 | } 82 | params.update(self.common_params) 83 | url = "http://{service}.{region}.amazonaws.com/".format( 84 | service=self.service, region=self.region) 85 | parse_function = lambda root: root.SubscribeResult.SubscriptionArn.text 86 | return self._process(url, params, self.service, parse_function) 87 | 88 | def confirm_subscription(self, topic_arn, token, auth_unsubscribe=False): 89 | """ 90 | Verifies an endpoint owner's intent to receive messages by validating 91 | the token sent to the endpoint by an earlier Subscribe action. 92 | If the token is valid, the action creates a new subscription 93 | and returns its Amazon Resource Name (ARN). 94 | AWS API: ConfirmSubscription_ 95 | 96 | :param topic_arn: The ARN of the topic for subscription. 97 | :param token: Short-lived token returned during the Subscribe action. 98 | :param auth_unsubscribe: Boolean, disallows unauthenticated 99 | unsubscribes of the subscription. 100 | :return: SubscriptionArn - The ARN of the created subscription. 101 | """ 102 | params = { 103 | "TopicArn": topic_arn, 104 | "Token": token, 105 | "Action": "ConfirmSubscription", 106 | "AuthenticateOnUnsubscribe": str(auth_unsubscribe).lower() 107 | } 108 | params.update(self.common_params) 109 | url = "http://{service}.{region}.amazonaws.com/".format( 110 | service=self.service, region=self.region) 111 | parse_func = lambda r: r.ConfirmSubscriptionResult.SubscriptionArn.text 112 | return self._process(url, params, self.service, parse_func) 113 | 114 | def publish(self, message, subject, topic_arn, target_arn=None, 115 | message_structure=None): 116 | """ 117 | Sends a message to all of a topic's subscribed endpoints. 118 | When a messageId is returned, the message has been saved and SNS 119 | will attempt to deliver it to the topic's subscribers shortly. 120 | The format of the outgoing message to each subscribed endpoint depends 121 | on the notification protocol selected. 122 | AWS API: Publish_ 123 | 124 | :param message: The message to send to the topic. To send the same 125 | message to all transport protocols, include the text of the message 126 | as a String value. To send different messages for each transport 127 | protocol, set the value of the MessageStructure parameter to json 128 | and use a JSON object for the Message parameter. If Python list or 129 | dict is passed, it will be converted to json automatically. 130 | :param subject: Optional parameter to be used as the "Subject" line 131 | when the message is delivered to email endpoints. 132 | :param topic_arn: The topic to publish to. 133 | :param target_arn: Either TopicArn or EndpointArn, but not both. 134 | :param message_structure: Should be empty to send the same message to 135 | all protocols, or "json" to send a different messages. 136 | :return: MessageId - Unique identifier assigned to the published message 137 | """ 138 | assert message_structure in (None, 'json') 139 | assert topic_arn or target_arn 140 | params = { 141 | "Message": message, 142 | "Subject": subject, 143 | "Action": "Publish" 144 | } 145 | # convert message to json if needed 146 | if message_structure == 'json': 147 | if not isinstance(message, (str, unicode)): 148 | params['Message'] = json.dumps(message) 149 | params['MessageStructure'] = message_structure 150 | # set topic_arn or target_arn 151 | if topic_arn: 152 | params["TopicArn"] = topic_arn 153 | else: 154 | params["TargetArn"] = target_arn 155 | params.update(self.common_params) 156 | url = "http://{service}.{region}.amazonaws.com/".format( 157 | service=self.service, region=self.region) 158 | parse_function = lambda root: root.PublishResult.MessageId.text 159 | return self._process(url, params, self.service, parse_function) 160 | -------------------------------------------------------------------------------- /asyncaws/sqs.py: -------------------------------------------------------------------------------- 1 | """Module that covers SQS API""" 2 | import json 3 | import hashlib 4 | from asyncaws.core import AWS 5 | 6 | 7 | class SQS(AWS): 8 | """ 9 | :param access_key: AWS_ACCESS_KEY_ID 10 | :param secret_key: AWS_SECRET_ACCESS_KEY 11 | :param region: region name as string 12 | :param async: True by default, indicates that AsyncHTTPClient should 13 | be used. Otherwise HTTPClient (synchronous). Useful for debugging. 14 | """ 15 | service = 'sqs' 16 | common_params = {"Version": "2012-11-05"} 17 | 18 | def listen_queue(self, queue_url, wait_time=15, max_messages=1, 19 | visibility_timeout=300): 20 | """ 21 | Retrieves one or more messages from the specified queue. 22 | Long poll support is enabled by using the WaitTimeSeconds parameter. 23 | AWS API: ReceiveMessage_ 24 | 25 | :param queue_url: The URL of the Amazon SQS queue to take action on. 26 | :param wait_time: The duration (in seconds) for which the call will 27 | wait for a message to arrive in the queue before returning. 28 | If a message is available, the call will return sooner. 29 | :param max_messages: The maximum number of messages to return. 30 | :param visibility_timeout: The duration (in seconds) that the received 31 | messages are hidden from subsequent retrieve requests after being 32 | retrieved by a ReceiveMessage request. 33 | :return: A message or a list of messages. 34 | """ 35 | def parse_function(root): 36 | if root.ReceiveMessageResult == '': 37 | return None 38 | message = root.ReceiveMessageResult.Message 39 | result = { 40 | 'Body': message.Body.text, 41 | 'MD5OfBody': message.MD5OfBody.text, 42 | 'ReceiptHandle': message.ReceiptHandle.text, 43 | 'Attributes': {} 44 | } 45 | for attr in message.Attribute: 46 | result['Attributes'][attr.Name.text] = attr.Value.text 47 | return result 48 | 49 | params = { 50 | "Action": "ReceiveMessage", 51 | "WaitTimeSeconds": wait_time, 52 | "MaxNumberOfMessages": max_messages, 53 | "VisibilityTimeout": visibility_timeout, 54 | "AttributeName": "All", 55 | } 56 | params.update(self.common_params) 57 | return self._process(queue_url, params, self.service, parse_function) 58 | 59 | def send_message(self, queue_url, message_body): 60 | """ 61 | Delivers a message to the specified queue. 62 | AWS API: SendMessage_ 63 | 64 | :param queue_url: The URL of the Amazon SQS queue to take action on. 65 | :param message_body: The message to send. String maximum 256 KB in size. 66 | :return: MD5OfMessageAttributes, MD5OfMessageBody, MessageId 67 | """ 68 | params = { 69 | "Action": "SendMessage", 70 | "MessageBody": message_body, 71 | } 72 | params.update(self.common_params) 73 | parse_function = lambda root: root.SendMessageResult.MessageId.text 74 | return self._process(queue_url, params, self.service, parse_function) 75 | 76 | def delete_message(self, queue_url, receipt_handle): 77 | """ 78 | Deletes the specified message from the specified queue. 79 | Specify the message by using the message's receipt handle, not ID. 80 | AWS API: DeleteMessage_ 81 | 82 | :param queue_url: The URL of the Amazon SQS queue to take action on. 83 | :param receipt_handle: The receipt handle associated with the message. 84 | :return: Request ID 85 | """ 86 | params = { 87 | "Action": "DeleteMessage", 88 | "ReceiptHandle": receipt_handle, 89 | } 90 | params.update(self.common_params) 91 | parse_function = lambda res: res.ResponseMetadata.RequestId.text 92 | return self._process(queue_url, params, self.service, parse_function) 93 | 94 | def create_queue(self, queue_name, attributes=None): 95 | """ 96 | Creates a new queue, or returns the URL of an existing one. 97 | To successfully create a new queue, a name that is unique within 98 | the scope of own queues should be provided. 99 | Beware: 60 seconds should pass between deleting a queue and creating 100 | another with the same name. 101 | AWS API: CreateQueue_ 102 | 103 | :param queue_name: The name for the queue to be created. 104 | :return: QueueUrl - the URL for the created Amazon SQS queue. 105 | """ 106 | if attributes is None: 107 | attributes = {} 108 | assert isinstance(attributes, dict) 109 | params = { 110 | "Action": "CreateQueue", 111 | "QueueName": queue_name, 112 | } 113 | for i, (key, value) in enumerate(attributes.items()): 114 | params['Attribute.%s.Name' % (i+1)] = key 115 | params['Attribute.%s.Value' % (i+1)] = value 116 | 117 | url = "http://{service}.{region}.amazonaws.com/".format( 118 | service=self.service, region=self.region) 119 | params.update(self.common_params) 120 | parse_function = lambda res: res.CreateQueueResult.QueueUrl.text 121 | return self._process(url, params, self.service, parse_function) 122 | 123 | def delete_queue(self, queue_url): 124 | """ 125 | Deletes the queue specified by the queue URL, regardless of whether 126 | the queue is empty. If the specified queue does not exist, SQS 127 | returns a successful response. 128 | Beware: 60 seconds should pass between deleting a queue and creating 129 | another with the same name. 130 | AWS API: DeleteQueue_ 131 | 132 | :param queue_url: The URL of the Amazon SQS queue to take action on. 133 | :return: Request ID 134 | """ 135 | params = { 136 | "Action": "DeleteQueue", 137 | } 138 | params.update(self.common_params) 139 | 140 | parse_function = lambda root: root.ResponseMetadata.RequestId.text 141 | return self._process(queue_url, params, self.service, parse_function) 142 | 143 | def get_queue_attributes(self, queue_url, attributes=('all',)): 144 | """ 145 | Gets attributes for the specified queue. 146 | The following attributes are supported: 147 | 148 | All (returns all values), ApproximateNumberOfMessages, 149 | ApproximateNumberOfMessagesNotVisible, 150 | VisibilityTimeout, CreatedTimestamp, LastModifiedTimestamp, Policy, 151 | MaximumMessageSize, MessageRetentionPeriod, QueueArn, 152 | ApproximateNumberOfMessagesDelayed, DelaySeconds, 153 | ReceiveMessageWaitTimeSeconds, RedrivePolicy. 154 | 155 | AWS API: GetQueueAttributes_ 156 | 157 | :param queue_url: The URL of the Amazon SQS queue to take action on. 158 | :param attributes: A list of attributes to retrieve, ['all'] by default. 159 | :return: A map of attributes to the respective values. 160 | """ 161 | assert isinstance(attributes, (list, set, tuple)) 162 | params = { 163 | "Action": "GetQueueAttributes", 164 | } 165 | for i, attr in enumerate(attributes): 166 | params['AttributeName.%s' % (i + 1)] = attr 167 | params.update(self.common_params) 168 | 169 | def parse_function(root): 170 | result = {} 171 | for attr in root.GetQueueAttributesResult.Attribute: 172 | result[attr.Name.text] = attr.Value.text 173 | return result 174 | return self._process(queue_url, params, self.service, parse_function) 175 | 176 | def set_queue_attributes(self, queue_url, attributes=None): 177 | """ 178 | Sets the value of one or more queue attributes. 179 | AWS API: SetQueueAttributes_ 180 | 181 | :param attributes: dict of attribute names and values. 182 | :param queue_url: The URL of the Amazon SQS queue to take action on. 183 | :return: Request ID 184 | """ 185 | if attributes is None: 186 | attributes = {} 187 | assert isinstance(attributes, dict) 188 | params = { 189 | "Action": "SetQueueAttributes" 190 | } 191 | for i, (key, value) in enumerate(attributes.items()): 192 | params['Attribute.%s.Name' % (i+1)] = key 193 | params['Attribute.%s.Value' % (i+1)] = value 194 | params.update(self.common_params) 195 | 196 | parse_function = lambda root: root.ResponseMetadata.RequestId.text 197 | return self._process(queue_url, params, self.service, parse_function) 198 | 199 | def add_permission(self, queue_url, account_ids, action_names, label): 200 | """ 201 | Adds a permission to a queue for a specific principal. 202 | This allows for sharing access to the queue. 203 | AWS API: AddPermission_ 204 | 205 | :param queue_url: The URL of the Amazon SQS queue to take action on. 206 | :param account_ids: List of AWS account numbers to grant a permission. 207 | :param action_names: List of actions the client wants to allow for the 208 | specified principal. The following are valid values: *, SendMessage, 209 | ReceiveMessage, DeleteMessage, ChangeMessageVisibility, 210 | GetQueueAttributes, GetQueueUrl. 211 | :param label: The unique identification of the permission. 212 | :return: Request ID 213 | """ 214 | assert isinstance(account_ids, (list, set, tuple)) 215 | assert isinstance(action_names, (list, set, tuple)) 216 | params = { 217 | "Action": "AddPermission", 218 | "Label": label, 219 | } 220 | for i, acc_id in enumerate(account_ids): 221 | params['AWSAccountId.%s' % (i + 1)] = acc_id 222 | for i, name in enumerate(action_names): 223 | params['ActionName.%s' % (i + 1)] = name 224 | params.update(self.common_params) 225 | 226 | parse_function = lambda root: root.ResponseMetadata.RequestId.text 227 | return self._process(queue_url, params, self.service, parse_function) 228 | 229 | # Helpers 230 | def allow_sns_topic(self, queue_url, queue_arn, topic_arn): 231 | """ 232 | Helper method to grant the sns topic a permission for publishing 233 | messages to queue. Calls set_queue_attributes under the hood. 234 | 235 | :param queue_url: url of queue to get messages 236 | :param queue_arn: arn of queue to get messages 237 | :param topic_arn: arn of topic to publish messages 238 | :return: None 239 | """ 240 | sid = hashlib.md5((topic_arn + queue_arn).encode('utf-8')).hexdigest() 241 | statement = {'Action': 'SQS:SendMessage', 242 | 'Effect': 'Allow', 243 | 'Principal': {'AWS': '*'}, 244 | 'Resource': queue_arn, 245 | 'Sid': sid, 246 | 'Condition': {'StringLike': {'aws:SourceArn': topic_arn}}} 247 | policy = {'Version': '2008-10-17', 248 | 'Id': queue_arn + '/SQSDefaultPolicy', 249 | 'Statement': [statement]} 250 | return self.set_queue_attributes(queue_url, 251 | {"Policy": json.dumps(policy)}) 252 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # User-friendly check for sphinx-build 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) 13 | endif 14 | 15 | # Internal variables. 16 | PAPEROPT_a4 = -D latex_paper_size=a4 17 | PAPEROPT_letter = -D latex_paper_size=letter 18 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 19 | # the i18n builder cannot share the environment and doctrees with the others 20 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 21 | 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 23 | 24 | help: 25 | @echo "Please use \`make ' where is one of" 26 | @echo " html to make standalone HTML files" 27 | @echo " dirhtml to make HTML files named index.html in directories" 28 | @echo " singlehtml to make a single large HTML file" 29 | @echo " pickle to make pickle files" 30 | @echo " json to make JSON files" 31 | @echo " htmlhelp to make HTML files and a HTML help project" 32 | @echo " qthelp to make HTML files and a qthelp project" 33 | @echo " devhelp to make HTML files and a Devhelp project" 34 | @echo " epub to make an epub" 35 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 36 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 37 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 38 | @echo " text to make text files" 39 | @echo " man to make manual pages" 40 | @echo " texinfo to make Texinfo files" 41 | @echo " info to make Texinfo files and run them through makeinfo" 42 | @echo " gettext to make PO message catalogs" 43 | @echo " changes to make an overview of all changed/added/deprecated items" 44 | @echo " xml to make Docutils-native XML files" 45 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 46 | @echo " linkcheck to check all external links for integrity" 47 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 48 | 49 | clean: 50 | rm -rf $(BUILDDIR)/* 51 | 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/AsyncAWS.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/AsyncAWS.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/AsyncAWS" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/AsyncAWS" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | latexpdfja: 120 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 121 | @echo "Running LaTeX files through platex and dvipdfmx..." 122 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 123 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 124 | 125 | text: 126 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 127 | @echo 128 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 129 | 130 | man: 131 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 132 | @echo 133 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 134 | 135 | texinfo: 136 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 137 | @echo 138 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 139 | @echo "Run \`make' in that directory to run these through makeinfo" \ 140 | "(use \`make info' here to do that automatically)." 141 | 142 | info: 143 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 144 | @echo "Running Texinfo files through makeinfo..." 145 | make -C $(BUILDDIR)/texinfo info 146 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 147 | 148 | gettext: 149 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 150 | @echo 151 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 152 | 153 | changes: 154 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 155 | @echo 156 | @echo "The overview file is in $(BUILDDIR)/changes." 157 | 158 | linkcheck: 159 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 160 | @echo 161 | @echo "Link check complete; look for any errors in the above output " \ 162 | "or in $(BUILDDIR)/linkcheck/output.txt." 163 | 164 | doctest: 165 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 166 | @echo "Testing of doctests in the sources finished, look at the " \ 167 | "results in $(BUILDDIR)/doctest/output.txt." 168 | 169 | xml: 170 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 171 | @echo 172 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 173 | 174 | pseudoxml: 175 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 176 | @echo 177 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 178 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import sys 4 | import os 5 | import sphinx_rtd_theme 6 | 7 | sys.path.insert(0, os.path.abspath('../')) 8 | 9 | # -- General configuration ------------------------------------------------ 10 | 11 | # If your documentation needs a minimal Sphinx version, state it here. 12 | #needs_sphinx = '1.0' 13 | 14 | # Add any Sphinx extension module names here, as strings. They can be 15 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 16 | # ones. 17 | extensions = [ 18 | 'sphinx.ext.autodoc', 19 | 'sphinx.ext.doctest', 20 | 'sphinx.ext.coverage', 21 | 'sphinx.ext.viewcode', 22 | ] 23 | 24 | # Add any paths that contain templates here, relative to this directory. 25 | templates_path = ['_templates'] 26 | 27 | # The suffix of source filenames. 28 | source_suffix = '.rst' 29 | 30 | # The encoding of source files. 31 | #source_encoding = 'utf-8-sig' 32 | 33 | # The master toctree document. 34 | master_doc = 'index' 35 | 36 | # General information about the project. 37 | project = u'AsyncAWS' 38 | copyright = u'2015, caceres.me' 39 | 40 | # The version info for the project you're documenting, acts as replacement for 41 | # |version| and |release|, also used in various other places throughout the 42 | # built documents. 43 | # 44 | # The short X.Y version. 45 | version = '0.1' 46 | # The full version, including alpha/beta/rc tags. 47 | release = '0.1' 48 | 49 | # The language for content autogenerated by Sphinx. Refer to documentation 50 | # for a list of supported languages. 51 | #language = None 52 | 53 | # There are two options for replacing |today|: either, you set today to some 54 | # non-false value, then it is used: 55 | #today = '' 56 | # Else, today_fmt is used as the format for a strftime call. 57 | #today_fmt = '%B %d, %Y' 58 | 59 | # List of patterns, relative to source directory, that match files and 60 | # directories to ignore when looking for source files. 61 | exclude_patterns = ['_build'] 62 | 63 | # The reST default role (used for this markup: `text`) to use for all 64 | # documents. 65 | #default_role = None 66 | 67 | # If true, '()' will be appended to :func: etc. cross-reference text. 68 | #add_function_parentheses = True 69 | 70 | # If true, the current module name will be prepended to all description 71 | # unit titles (such as .. function::). 72 | #add_module_names = True 73 | 74 | # If true, sectionauthor and moduleauthor directives will be shown in the 75 | # output. They are ignored by default. 76 | #show_authors = False 77 | 78 | # The name of the Pygments (syntax highlighting) style to use. 79 | pygments_style = 'sphinx' 80 | 81 | # A list of ignored prefixes for module index sorting. 82 | #modindex_common_prefix = [] 83 | 84 | # If true, keep warnings as "system message" paragraphs in the built documents. 85 | #keep_warnings = False 86 | 87 | 88 | # -- Options for HTML output ---------------------------------------------- 89 | 90 | # The theme to use for HTML and HTML Help pages. See the documentation for 91 | # a list of builtin themes. 92 | html_theme = 'sphinx_rtd_theme' 93 | 94 | # Theme options are theme-specific and customize the look and feel of a theme 95 | # further. For a list of options available for each theme, see the 96 | # documentation. 97 | #html_theme_options = {} 98 | 99 | # Add any paths that contain custom themes here, relative to this directory. 100 | html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] 101 | 102 | # The name for this set of Sphinx documents. If None, it defaults to 103 | # " v documentation". 104 | #html_title = None 105 | 106 | # A shorter title for the navigation bar. Default is the same as html_title. 107 | #html_short_title = None 108 | 109 | # The name of an image file (relative to this directory) to place at the top 110 | # of the sidebar. 111 | #html_logo = None 112 | 113 | # The name of an image file (within the static path) to use as favicon of the 114 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 115 | # pixels large. 116 | #html_favicon = None 117 | 118 | # Add any paths that contain custom static files (such as style sheets) here, 119 | # relative to this directory. They are copied after the builtin static files, 120 | # so a file named "default.css" will overwrite the builtin "default.css". 121 | html_static_path = ['_static'] 122 | 123 | # Add any extra paths that contain custom files (such as robots.txt or 124 | # .htaccess) here, relative to this directory. These files are copied 125 | # directly to the root of the documentation. 126 | #html_extra_path = [] 127 | 128 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 129 | # using the given strftime format. 130 | #html_last_updated_fmt = '%b %d, %Y' 131 | 132 | # If true, SmartyPants will be used to convert quotes and dashes to 133 | # typographically correct entities. 134 | #html_use_smartypants = True 135 | 136 | # Custom sidebar templates, maps document names to template names. 137 | #html_sidebars = {} 138 | 139 | # Additional templates that should be rendered to pages, maps page names to 140 | # template names. 141 | #html_additional_pages = {} 142 | 143 | # If false, no module index is generated. 144 | #html_domain_indices = True 145 | 146 | # If false, no index is generated. 147 | #html_use_index = True 148 | 149 | # If true, the index is split into individual pages for each letter. 150 | #html_split_index = False 151 | 152 | # If true, links to the reST sources are added to the pages. 153 | #html_show_sourcelink = True 154 | 155 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 156 | #html_show_sphinx = True 157 | 158 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 159 | #html_show_copyright = True 160 | 161 | # If true, an OpenSearch description file will be output, and all pages will 162 | # contain a tag referring to it. The value of this option must be the 163 | # base URL from which the finished HTML is served. 164 | #html_use_opensearch = '' 165 | 166 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 167 | #html_file_suffix = None 168 | 169 | # Output file base name for HTML help builder. 170 | htmlhelp_basename = 'AsyncAWSdoc' 171 | 172 | 173 | # -- Options for LaTeX output --------------------------------------------- 174 | 175 | latex_elements = { 176 | # The paper size ('letterpaper' or 'a4paper'). 177 | #'papersize': 'letterpaper', 178 | 179 | # The font size ('10pt', '11pt' or '12pt'). 180 | #'pointsize': '10pt', 181 | 182 | # Additional stuff for the LaTeX preamble. 183 | #'preamble': '', 184 | } 185 | 186 | # Grouping the document tree into LaTeX files. List of tuples 187 | # (source start file, target name, title, 188 | # author, documentclass [howto, manual, or own class]). 189 | latex_documents = [ 190 | ('index', 'AsyncAWS.tex', u'AsyncAWS Documentation', 191 | u'Ma.e5t.ro', 'manual'), 192 | ] 193 | 194 | # The name of an image file (relative to this directory) to place at the top of 195 | # the title page. 196 | #latex_logo = None 197 | 198 | # For "manual" documents, if this is true, then toplevel headings are parts, 199 | # not chapters. 200 | #latex_use_parts = False 201 | 202 | # If true, show page references after internal links. 203 | #latex_show_pagerefs = False 204 | 205 | # If true, show URL addresses after external links. 206 | #latex_show_urls = False 207 | 208 | # Documents to append as an appendix to all manuals. 209 | #latex_appendices = [] 210 | 211 | # If false, no module index is generated. 212 | #latex_domain_indices = True 213 | 214 | 215 | # -- Options for manual page output --------------------------------------- 216 | 217 | # One entry per manual page. List of tuples 218 | # (source start file, name, description, authors, manual section). 219 | man_pages = [ 220 | ('index', 'asyncaws', u'AsyncAWS Documentation', 221 | [u'Ma.e5t.ro'], 1) 222 | ] 223 | 224 | # If true, show URL addresses after external links. 225 | #man_show_urls = False 226 | 227 | 228 | # -- Options for Texinfo output ------------------------------------------- 229 | 230 | # Grouping the document tree into Texinfo files. List of tuples 231 | # (source start file, target name, title, author, 232 | # dir menu entry, description, category) 233 | texinfo_documents = [ 234 | ('index', 'AsyncAWS', u'AsyncAWS Documentation', 235 | u'caceres.me', 'AsyncAWS', 'Asynchronous AWS library for Python', 236 | 'Miscellaneous'), 237 | ] 238 | 239 | # Documents to append as an appendix to all manuals. 240 | #texinfo_appendices = [] 241 | 242 | # If false, no module index is generated. 243 | #texinfo_domain_indices = True 244 | 245 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 246 | #texinfo_show_urls = 'footnote' 247 | 248 | # If true, do not generate a @detailmenu in the "Top" node's menu. 249 | #texinfo_no_detailmenu = False 250 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. _Home: 2 | 3 | AsyncAWS - Asynchronous AWS library in Python 4 | ============================================= 5 | 6 | .. contents:: 7 | :depth: 2 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | SQS Reference 13 | SNS Reference 14 | 15 | About 16 | ----- 17 | AsyncAWS is a collection of convenient classes that provide abstract access 18 | to AWS API in Python. It's killer-feature is efficient asynchronous behaviour 19 | achieved with simple, sequential code. No callback spaghetti thanks to Python 20 | "yield" and coroutines. Just look: 21 | :: 22 | 23 | queue_url = yield sqs.create_queue("test-queue") 24 | message_id = yield sqs.send_message(queue_url, "Hello, World!") 25 | 26 | Used within a coroutine, this code will 'pause' on each yield keyword, letting 27 | IOLoop to run other stuff meanwhile. As soon as AWS will return some response, 28 | IOLoop will switch back to the "yield" point, and just continue as if "yield" was never there. 29 | This way can keep the usual sequential coding style, but run the code asynchronously. 30 | 31 | Installation 32 | ------------ 33 | I'm preparing a stable package for PyPI, meanwhile you can install it 34 | with pip directly from github: 35 | :: 36 | 37 | pip install git+git://github.com/MA3STR0/AsyncAWS.git 38 | 39 | Still not convinced? 40 | -------------------- 41 | Wondering what is the benefit? 42 | 43 | First of all, it's performance. Most of the time our code is waiting for IO, especially 44 | if it has to deal with remote connections: database, web-APIs, etc. Calling such resources 45 | asynchronously allows main Python thread to do other stuff in the meanwhile. 46 | 47 | Finally, running things like a message queue in a blocking while-True loop is not only 48 | embarrassing, but also expensive. You pay for every SQS API request, which means 49 | every time your code asks for new messages in a loop. Instead, AsyncAWS would 50 | make a long-polling request and wait. It will only re-connect when a message comes, 51 | or when SQS drops the connection to force you to pay at least something :) 52 | 53 | 54 | Minimal working example 55 | ----------------------- 56 | You need to define main function as a coroutine and schedule it in IOLoop. 57 | Here is code that creates an SQS queue and sends a message to it: 58 | 59 | .. literalinclude:: /../examples/minimal.py 60 | 61 | Development and contributions 62 | ----------------------------- 63 | AsyncAWS is developed on Github: https://github.com/MA3STR0/AsyncAWS 64 | 65 | Code is maximally PEP8-compliant, well-documented and easy to read, welcoming 66 | everyone to contribute and send pull requests. 67 | 68 | AsyncAWS is extremely easy to extend, there are just 2 points I would kindly ask to follow: 69 | * Project currently scores 9.6 with Pylint, the goal is to keep it above 8. 70 | * Most of this documentation is auto-generated, so every public method should have a nice docstring. 71 | 72 | FAQ 73 | --- 74 | Why yet another Python AWS library, if there is Boto? 75 | Because Boto is blocking, and asynchronous IO rocks. 76 | 77 | What's the difference? 78 | For example, you can set asynchronous callbacks for incoming SQS messages, and forget about polling in a while-True loop. 79 | 80 | But this lib is so small, mostly SQS-related stuff... 81 | Currently AsyncAWS implements only SQS and SNS, because those are most critical parts that HAVE to be async. 82 | For the rest, there still is Boto. They live together perfectly well. 83 | 84 | Any plans to implement other AWS APIs? 85 | I do my best to add new methods regularly, but it's quite fast and simple so you can contribute as well. 86 | The most unpleasant part was to create a base `AWSRequest` class 87 | that implements all crazy hashing & signing stuff required by AWS, and it's done. 88 | Implementing new APIs is mostly copy-and-paste of parameters. 89 | 90 | Why does this library rely on Tornado? 91 | Because Tornado has most mature async tools and ioloop for Python 2 and 3. Asyncio support is also planned. 92 | 93 | Can't we just use AWS HTTP API directly using requests/urllib/etc? 94 | We can, but the overhead of building, hashing and signing canonical HTTP requests will be huge. 95 | Even GET params should be sorted alphabetically. And this lib will do it all for you. 96 | 97 | Credits 98 | ------- 99 | AsyncAWS is created and maintained by `Anton Caceres `_. 100 | 101 | Special thanks to 102 | 103 | * `Stefan Behnel `_ for architectural advices 104 | * `Skoobe `_ for providing tools for development 105 | 106 | License 107 | ------- 108 | AsyncAWS is released under the terms of the `MIT license `_. 109 | 110 | .. literalinclude:: /../LICENSE 111 | 112 | The End. 113 | ++++++++ 114 | * Home_ 115 | * :ref:`genindex` 116 | 117 | -------------------------------------------------------------------------------- /docs/sns.rst: -------------------------------------------------------------------------------- 1 | SNS: Async Python API 2 | ===================== 3 | 4 | About 5 | ----- 6 | Amazon Simple Notification Service (SNS) is a web service that helps building distributed applications. 7 | It can broadcast real-time notification messages to interested subscribers over multiple delivery protocols, 8 | it particular Amazon SQS. 9 | 10 | AWS documentation: http://docs.aws.amazon.com/sns/latest/api/Welcome.html 11 | 12 | This module implements fully asynchronous access to SNS API in Python. 13 | 14 | Examples 15 | -------- 16 | Don't forget to set your AWS keys before running examples. You can set environment variables in your .profile, or just run 17 | :: 18 | 19 | AWS_ACCESS_KEY_ID=id AWS_SECRET_ACCESS_KEY=secret python example.py 20 | 21 | **Example 1.** Create a queue and send "Hello, World!" message to it. 22 | 23 | .. literalinclude:: /../examples/sqs/create_and_send.py 24 | 25 | **Example 2.** Listen to the queue and trigger `on_message` callback 26 | 27 | .. literalinclude:: /../examples/sqs/on_message_callback.py 28 | 29 | API documentation 30 | ----------------- 31 | 32 | .. autoclass:: asyncaws.SNS 33 | :members: 34 | 35 | .. _CreateTopic: http://docs.aws.amazon.com/sns/latest/APIReference/API_CreateTopic.html 36 | .. _Subscribe: http://docs.aws.amazon.com/sns/latest/APIReference/API_Subscribe.html 37 | .. _ConfirmSubscription: http://docs.aws.amazon.com/sns/latest/APIReference/API_ConfirmSubscription.html 38 | .. _Publish: http://docs.aws.amazon.com/sns/latest/APIReference/API_ConfirmSubscription.html 39 | .. _DeleteTopic: http://docs.aws.amazon.com/sns/latest/APIReference/API_DeleteTopic.html 40 | -------------------------------------------------------------------------------- /docs/sqs.rst: -------------------------------------------------------------------------------- 1 | SQS: Async Python API 2 | ===================== 3 | 4 | About 5 | ----- 6 | Amazon Simple Queue Service (SQS) is a hosted messaging queue service that handles text data transfer between components in a distributed system. 7 | 8 | SQS helps to move data between distributed application modules without directly connecting them. 9 | 10 | AWS documentation: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/Welcome.html 11 | 12 | This module implements fully asynchronous access to SQS API in Python. 13 | 14 | Examples 15 | -------- 16 | Don't forget to set your AWS keys before running examples. You can set environment variables in your .profile, or just run 17 | :: 18 | 19 | AWS_ACCESS_KEY_ID=id AWS_SECRET_ACCESS_KEY=secret python example.py 20 | 21 | **Example 1.** Create a queue and send "Hello, World!" message to it. 22 | 23 | .. literalinclude:: /../examples/sqs/create_and_send.py 24 | 25 | **Example 2.** Listen to the queue and trigger `on_message` callback 26 | 27 | .. literalinclude:: /../examples/sqs/on_message_callback.py 28 | 29 | API documentation 30 | ----------------- 31 | 32 | .. autoclass:: asyncaws.SQS 33 | :members: 34 | 35 | 36 | .. _ReceiveMessage: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ReceiveMessage.html 37 | .. _SendMessage: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html 38 | .. _DeleteMessage: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_DeleteMessage.html 39 | .. _CreateQueue: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_CreateQueue.html 40 | .. _DeleteQueue: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_DeleteQueue.html 41 | .. _GetQueueAttributes: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_GetQueueAttributes.html 42 | .. _SetQueueAttributes: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SetQueueAttributes.html 43 | .. _AddPermission: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_AddPermission.html 44 | -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MA3STR0/AsyncAWS/c7f795023b96331425cd2156e91f6205a4492096/examples/__init__.py -------------------------------------------------------------------------------- /examples/minimal.py: -------------------------------------------------------------------------------- 1 | from tornado.ioloop import IOLoop 2 | from tornado.gen import coroutine 3 | from asyncaws import SQS 4 | 5 | sqs = SQS('aws-key-id', 'sqs-key-secret', 'eu-west-1') 6 | ioloop = IOLoop.current() 7 | 8 | @coroutine 9 | def main(): 10 | queue_url = yield sqs.create_queue("test-queue") 11 | message_id = yield sqs.send_message(queue_url, "Hello, World!") 12 | print queue_url, message_id 13 | 14 | if __name__ == '__main__': 15 | ioloop.add_callback(main) 16 | ioloop.start() 17 | -------------------------------------------------------------------------------- /examples/sns/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MA3STR0/AsyncAWS/c7f795023b96331425cd2156e91f6205a4492096/examples/sns/__init__.py -------------------------------------------------------------------------------- /examples/sns/create_and_publish.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from tornado.ioloop import IOLoop 4 | from tornado.gen import coroutine 5 | from asyncaws import SNS 6 | 7 | ioloop = IOLoop.current() 8 | aws_key_id = os.environ['AWS_ACCESS_KEY_ID'] 9 | aws_key_secret = os.environ['AWS_SECRET_ACCESS_KEY'] 10 | 11 | sns = SNS(aws_key_id, aws_key_secret, "eu-west-1") 12 | 13 | 14 | @coroutine 15 | def create_and_publish(): 16 | """Create an SQS queue and send a message""" 17 | topic_arn = yield sns.create_topic("test-topic") 18 | yield sns.publish("Hello, World!", "Some subject", topic_arn) 19 | sys.exit(0) 20 | 21 | 22 | if __name__ == '__main__': 23 | ioloop.add_callback(create_and_publish) 24 | ioloop.start() -------------------------------------------------------------------------------- /examples/sqs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MA3STR0/AsyncAWS/c7f795023b96331425cd2156e91f6205a4492096/examples/sqs/__init__.py -------------------------------------------------------------------------------- /examples/sqs/create_and_send.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from tornado.ioloop import IOLoop 4 | from tornado.gen import coroutine 5 | from asyncaws import SQS 6 | 7 | ioloop = IOLoop.current() 8 | aws_key_id = os.environ['AWS_ACCESS_KEY_ID'] 9 | aws_key_secret = os.environ['AWS_SECRET_ACCESS_KEY'] 10 | 11 | sqs = SQS(aws_key_id, aws_key_secret, "eu-west-1") 12 | 13 | @coroutine 14 | def create_and_send(): 15 | """Create an SQS queue and send a message""" 16 | queue_url = yield sqs.create_queue("test-queue") 17 | yield sqs.send_message(queue_url, "Hello, World!") 18 | sys.exit(0) 19 | 20 | 21 | if __name__ == '__main__': 22 | ioloop.add_callback(create_and_send) 23 | ioloop.start() -------------------------------------------------------------------------------- /examples/sqs/on_message_callback.py: -------------------------------------------------------------------------------- 1 | import os 2 | from functools import partial 3 | from tornado.ioloop import IOLoop 4 | from tornado.gen import coroutine 5 | from asyncaws import SQS 6 | 7 | ioloop = IOLoop.current() 8 | aws_key_id = os.environ['AWS_ACCESS_KEY_ID'] 9 | aws_key_secret = os.environ['AWS_SECRET_ACCESS_KEY'] 10 | 11 | sqs = SQS(aws_key_id, aws_key_secret, "eu-west-1") 12 | queue_url = "https://sqs.eu-west-1.amazonaws.com/637085312181/test-queue" 13 | 14 | @coroutine 15 | def listen_queue(): 16 | """Wait for SQS messages using async long polling""" 17 | # tornado will "pause" the coroutine here until SQS returns something 18 | message = yield sqs.listen_queue(queue_url) 19 | if message: 20 | # schedule the callback 21 | ioloop.add_callback(partial(on_message, message)) 22 | # reconnect to SQS and repeat everything 23 | ioloop.add_callback(listen_queue) 24 | 25 | 26 | @coroutine 27 | def on_message(message): 28 | """This function will be called when new message arrives""" 29 | print "New message received:", message['Body'] 30 | yield sqs.delete_message(queue_url, message['ReceiptHandle']) 31 | 32 | 33 | if __name__ == '__main__': 34 | ioloop.add_callback(listen_queue) 35 | ioloop.start() -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal = 1 3 | 4 | [metadata] 5 | description-file = README.md -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | setup( 3 | name = 'asyncaws', 4 | packages = ['asyncaws'], 5 | version = '0.1.0', 6 | description = 'Asynchronous AWS library in Python', 7 | author = 'Anton Caceres', 8 | author_email = 'm@e5t.ro', 9 | url = 'http://caceres.me/asyncaws', 10 | download_url = 'https://github.com/MA3STR0/asyncaws/tarball/v0.1', 11 | keywords = ['aws', 'sqs', 'sns', 'tornado', 'boto'], 12 | classifiers = [], 13 | install_requires=[ 14 | "tornado", 15 | "lxml", 16 | "futures" 17 | ], 18 | ) 19 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MA3STR0/AsyncAWS/c7f795023b96331425cd2156e91f6205a4492096/tests/__init__.py -------------------------------------------------------------------------------- /tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MA3STR0/AsyncAWS/c7f795023b96331425cd2156e91f6205a4492096/tests/integration/__init__.py -------------------------------------------------------------------------------- /tests/integration/test_sns.py: -------------------------------------------------------------------------------- 1 | import os 2 | from asyncaws import SNS 3 | from tornado.testing import AsyncTestCase, gen_test 4 | from random import randint 5 | 6 | aws_key_id = os.environ['AWS_ACCESS_KEY_ID'] 7 | aws_key_secret = os.environ['AWS_SECRET_ACCESS_KEY'] 8 | aws_region = os.environ['AWS_REGION'] 9 | 10 | 11 | class TestSQS(AsyncTestCase): 12 | @classmethod 13 | def setUpClass(cls): 14 | cls.sns = SNS(aws_key_id, aws_key_secret, aws_region, async=False) 15 | cls.topic_name = "test-topic-%s" % randint(1000, 9999) 16 | cls.topic_arn = cls.sns.create_topic(cls.topic_name) 17 | 18 | @classmethod 19 | def tearDownClass(cls): 20 | cls.sns.delete_topic(cls.topic_arn) 21 | 22 | @gen_test 23 | def test_topic_actions(self): 24 | self.assertTrue(self.topic_arn.startswith('arn:')) 25 | mid = self.sns.publish("Hello, World!", "Test Subject", self.topic_arn) 26 | self.assertIsInstance(mid, str) 27 | -------------------------------------------------------------------------------- /tests/integration/test_sqs.py: -------------------------------------------------------------------------------- 1 | import os 2 | from asyncaws import SQS 3 | from tornado.testing import AsyncTestCase, gen_test 4 | from random import randint 5 | 6 | aws_key_id = os.environ['AWS_ACCESS_KEY_ID'] 7 | aws_key_secret = os.environ['AWS_SECRET_ACCESS_KEY'] 8 | aws_region = os.environ['AWS_REGION'] 9 | aws_test_account_id = "637085312181" 10 | 11 | 12 | class TestSQS(AsyncTestCase): 13 | 14 | @classmethod 15 | def setUpClass(cls): 16 | cls.sqs = SQS(aws_key_id, aws_key_secret, aws_region, async=False) 17 | cls.queue_name = "test-queue-%s" % randint(1000, 9999) 18 | cls.queue_url = cls.sqs.create_queue( 19 | cls.queue_name, {"MessageRetentionPeriod": 60}) 20 | 21 | @classmethod 22 | def tearDownClass(cls): 23 | cls.sqs.delete_queue(cls.queue_url) 24 | 25 | @gen_test 26 | def test_queue_actions(self): 27 | self.assertTrue(self.queue_url.startswith('http')) 28 | get_attr_result = self.sqs.get_queue_attributes( 29 | self.queue_url, ['MessageRetentionPeriod']) 30 | self.assertIsInstance(get_attr_result, dict) 31 | self.assertEqual(get_attr_result['MessageRetentionPeriod'], '60') 32 | add_perm_result = self.sqs.add_permission( 33 | self.queue_url, [aws_test_account_id], ["SendMessage"], "test-permission-id") 34 | self.assertIsInstance(add_perm_result, str) 35 | --------------------------------------------------------------------------------