├── .gitignore ├── LICENSE ├── sample_daemon.py ├── setup.py ├── sqs_launcher └── __init__.py ├── sqs_listener ├── daemon.py └── __init__.py └── README.rst /.gitignore: -------------------------------------------------------------------------------- 1 | venv 2 | .mypy_cache 3 | .vscode 4 | .idea 5 | *.pyc 6 | .DS_Store 7 | build 8 | dist 9 | pySqsListener.egg-info 10 | test_listen.py -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 Yaakov Gesher 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /sample_daemon.py: -------------------------------------------------------------------------------- 1 | """ 2 | a sample daemonization script for the sqs listener 3 | """ 4 | 5 | import sys 6 | from sqs_listener.daemon import Daemon 7 | from sqs_listener import SqsListener 8 | 9 | 10 | class MyListener(SqsListener): 11 | def handle_message(self, body, attributes, messages_attributes): 12 | pass 13 | # run your code here 14 | 15 | 16 | class MyDaemon(Daemon): 17 | def run(self): 18 | print("Initializing listener") 19 | listener = MyListener('main-queue', 'error-queue') 20 | listener.listen() 21 | 22 | 23 | if __name__ == "__main__": 24 | daemon = MyDaemon('/var/run/sqs_daemon.pid') 25 | if len(sys.argv) == 2: 26 | if 'start' == sys.argv[1]: 27 | print("Starting listener daemon") 28 | daemon.start() 29 | elif 'stop' == sys.argv[1]: 30 | print("Attempting to stop the daemon") 31 | daemon.stop() 32 | elif 'restart' == sys.argv[1]: 33 | daemon.restart() 34 | else: 35 | print("Unknown command") 36 | sys.exit(2) 37 | sys.exit(0) 38 | else: 39 | print("usage: %s start|stop|restart" % sys.argv[0]) 40 | sys.exit(2) 41 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | # To use a consistent encoding 3 | from codecs import open 4 | from os import path 5 | 6 | here = path.abspath(path.dirname(__file__)) 7 | 8 | # Get the long description from the README file 9 | with open(path.join(here, 'README.rst'), encoding='utf-8') as f: 10 | long_description = f.read() 11 | 12 | setup( 13 | name='pySqsListener', 14 | 15 | # Versions should comply with PEP440. For a discussion on single-sourcing 16 | # the version across setup.py and the project code, see 17 | # https://packaging.python.org/en/latest/single_source_version.html 18 | version='0.8.8', 19 | 20 | description='A simple Python SQS utility package', 21 | long_description=long_description, 22 | 23 | # The project's main homepage. 24 | url='https://github.com/jegesh/python-sqs-listener', 25 | 26 | # Author details 27 | author='Yaakov Gesher', 28 | author_email='yaakov@gesher.net', 29 | 30 | # Choose your license 31 | license='Apache Software License', 32 | 33 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 34 | classifiers=[ 35 | # How mature is this project? Common values are 36 | # 3 - Alpha 37 | # 4 - Beta 38 | # 5 - Production/Stable 39 | 'Development Status :: 4 - Beta', 40 | 41 | # Indicate who your project is intended for 42 | 'Intended Audience :: Developers', 43 | 'Topic :: Software Development :: Libraries', 44 | 45 | # Pick your license as you wish (should match "license" above) 46 | 'License :: OSI Approved :: Apache Software License', 47 | 48 | # Specify the Python versions you support here. In particular, ensure 49 | # that you indicate whether you support Python 2, Python 3 or both. 50 | 'Programming Language :: Python :: 2.7', 51 | 'Programming Language :: Python :: 3.6', 52 | ], 53 | 54 | # What does your project relate to? 55 | keywords='aws sqs listener and message launcher', 56 | 57 | # You can just specify the packages manually here if your project is 58 | # simple. Or you can use find_packages(). 59 | packages=find_packages(), 60 | 61 | # Alternatively, if you want to distribute just a my_module.py, uncomment 62 | # this: 63 | # py_modules=["my_module"], 64 | 65 | # List run-time dependencies here. These will be installed by pip when 66 | # your project is installed. For an analysis of "install_requires" vs pip's 67 | # requirements files see: 68 | # https://packaging.python.org/en/latest/requirements.html 69 | install_requires=['boto3'] 70 | 71 | ) 72 | -------------------------------------------------------------------------------- /sqs_launcher/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | class for running sqs message launcher 3 | 4 | Created December 22nd, 2016 5 | @author: Yaakov Gesher 6 | @version: 0.1.0 7 | @license: Apache 8 | """ 9 | 10 | # ================ 11 | # start imports 12 | # ================ 13 | 14 | import json 15 | import logging 16 | import os 17 | 18 | import boto3 19 | import boto3.session 20 | 21 | # ================ 22 | # start class 23 | # ================ 24 | 25 | sqs_logger = logging.getLogger('sqs_listener') 26 | 27 | 28 | class SqsLauncher(object): 29 | 30 | def __init__(self, queue=None, queue_url=None, create_queue=False, visibility_timeout='600'): 31 | """ 32 | :param queue: (str) name of queue to listen to 33 | :param queue_url: (str) url of queue to listen to 34 | :param create_queue (boolean) determines whether to create the queue if it doesn't exist. If False, an 35 | Exception will be raised if the queue doesn't already exist 36 | :param visibility_timeout: (str) Relevant to queue creation. Indicates the number of seconds for which the SQS will hide the message. 37 | Typically this should reflect the maximum amount of time your handler method will take 38 | to finish execution. See http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-visibility-timeout.html 39 | for more information 40 | """ 41 | if not any(queue, queue_url): 42 | raise ValueError('Either `queue` or `queue_url` should be provided.') 43 | if (not os.environ.get('AWS_ACCOUNT_ID', None) and 44 | not (boto3.Session().get_credentials().method in ['iam-role', 'assume-role'])): 45 | raise EnvironmentError('Environment variable `AWS_ACCOUNT_ID` not set and no role found.') 46 | # new session for each instantiation 47 | self._session = boto3.session.Session() 48 | self._client = self._session.client('sqs') 49 | 50 | self._queue_name = queue 51 | self._queue_url = queue_url 52 | if not queue_url: 53 | queues = self._client.list_queues(QueueNamePrefix=self._queue_name) 54 | exists = False 55 | for q in queues.get('QueueUrls', []): 56 | qname = q.split('/')[-1] 57 | if qname == self._queue_name: 58 | exists = True 59 | self._queue_url = q 60 | 61 | if not exists: 62 | if create_queue: 63 | q = self._client.create_queue( 64 | QueueName=self._queue_name, 65 | Attributes={ 66 | 'VisibilityTimeout': visibility_timeout # 10 minutes 67 | } 68 | ) 69 | self._queue_url = q['QueueUrl'] 70 | else: 71 | raise ValueError('No queue found with name ' + self._queue_name) 72 | else: 73 | self._queue_name = self._get_queue_name_from_url(queue_url) 74 | 75 | def launch_message(self, message, **kwargs): 76 | """ 77 | sends a message to the queue specified in the constructor 78 | :param message: (dict) 79 | :param kwargs: additional optional keyword arguments (DelaySeconds, MessageAttributes, MessageDeduplicationId, or MessageGroupId) 80 | See http://boto3.readthedocs.io/en/latest/reference/services/sqs.html#SQS.Client.send_message for more information 81 | :return: (dict) the message response from SQS 82 | """ 83 | sqs_logger.info("Sending message to queue " + self._queue_name) 84 | return self._client.send_message( 85 | QueueUrl=self._queue_url, 86 | MessageBody=json.dumps(message), 87 | **kwargs, 88 | ) 89 | 90 | def _get_queue_name_from_url(self, url): 91 | return url.split('/')[-1] 92 | -------------------------------------------------------------------------------- /sqs_listener/daemon.py: -------------------------------------------------------------------------------- 1 | """ 2 | taken from http://web.archive.org/web/20131017130434/http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/ 3 | """ 4 | 5 | import sys 6 | import os 7 | import time 8 | import atexit 9 | from signal import SIGTERM 10 | 11 | 12 | class Daemon: 13 | """ 14 | A generic daemon class. 15 | 16 | Usage: subclass the Daemon class and override the run() method 17 | """ 18 | 19 | def __init__(self, pidfile, overwrite=False, stdout='/dev/stdout', stderr='/dev/stderr', stdin='/dev/null'): 20 | self.stdin = stdin 21 | self.stdout = stdout 22 | self.stderr = stderr 23 | self.pidfile = pidfile 24 | self.overwrite_output = overwrite 25 | 26 | def daemonize(self): 27 | """ 28 | do the UNIX double-fork magic, see Stevens' "Advanced 29 | Programming in the UNIX Environment" for details (ISBN 0201563177) 30 | http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 31 | """ 32 | try: 33 | pid = os.fork() 34 | if pid > 0: 35 | # exit first parent 36 | sys.exit(0) 37 | except OSError as e: 38 | sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror)) 39 | sys.exit(1) 40 | 41 | # decouple from parent environment 42 | os.chdir("/") 43 | os.setsid() 44 | os.umask(0) 45 | 46 | # do second fork 47 | try: 48 | pid = os.fork() 49 | if pid > 0: 50 | # exit from second parent 51 | sys.exit(0) 52 | except OSError as e: 53 | sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror)) 54 | sys.exit(1) 55 | 56 | # redirect standard file descriptors 57 | sys.stdout.flush() 58 | sys.stderr.flush() 59 | si = open(self.stdin, 'r') 60 | if sys.stdout.isatty(): 61 | so = open(self.stdout, 'a+') 62 | se = open(self.stderr, 'a+') 63 | else: 64 | so = open(self.stdout, 'w') 65 | se = open(self.stderr, 'w') 66 | os.dup2(si.fileno(), sys.stdin.fileno()) 67 | os.dup2(so.fileno(), sys.stdout.fileno()) 68 | os.dup2(se.fileno(), sys.stderr.fileno()) 69 | 70 | # write pidfile 71 | atexit.register(self.delpid) 72 | pid = str(os.getpid()) 73 | open(self.pidfile, 'w+').write("%s\n" % pid) 74 | 75 | def delpid(self): 76 | os.remove(self.pidfile) 77 | 78 | def start(self): 79 | """ 80 | Start the daemon 81 | """ 82 | # Check for a pidfile to see if the daemon already runs 83 | try: 84 | pf = open(self.pidfile, 'r') 85 | pid = int(pf.read().strip()) 86 | pf.close() 87 | except IOError: 88 | pid = None 89 | 90 | if pid: 91 | message = "pidfile %s already exist. Daemon already running?\n" 92 | sys.stderr.write(message % self.pidfile) 93 | sys.exit(1) 94 | 95 | # wipe logs if necessary 96 | if self.overwrite_output: 97 | out_file = open(self.stdout, 'w') 98 | out_file.close() 99 | err_file = open(self.stderr, 'w') 100 | err_file.close() 101 | # Start the daemon 102 | self.daemonize() 103 | self.run() 104 | 105 | def stop(self): 106 | """ 107 | Stop the daemon 108 | """ 109 | # Get the pid from the pidfile 110 | try: 111 | pf = open(self.pidfile, 'r') 112 | pid = int(pf.read().strip()) 113 | pf.close() 114 | except IOError: 115 | pid = None 116 | 117 | if not pid: 118 | message = "pidfile %s does not exist. Daemon not running?\n" 119 | sys.stderr.write(message % self.pidfile) 120 | return # not an error in a restart 121 | 122 | # Try killing the daemon process 123 | try: 124 | while 1: 125 | os.kill(pid, SIGTERM) 126 | time.sleep(0.1) 127 | except OSError as err: 128 | err = str(err) 129 | if err.find("No such process") > 0: 130 | if os.path.exists(self.pidfile): 131 | os.remove(self.pidfile) 132 | else: 133 | print(str(err)) 134 | sys.exit(1) 135 | 136 | def restart(self): 137 | """ 138 | Restart the daemon 139 | """ 140 | self.stop() 141 | self.start() 142 | 143 | def run(self): 144 | """ 145 | You should override this method when you subclass Daemon. It will be called after the process has been 146 | daemonized by start() or restart(). 147 | """ 148 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | AWS SQS Listener 2 | ---------------- 3 | 4 | .. image:: https://img.shields.io/pypi/v/pySqsListener.svg?style=popout 5 | :alt: PyPI 6 | :target: https://github.com/jegesh/python-sqs-listener 7 | .. image:: https://img.shields.io/pypi/pyversions/pySqsListener.svg?style=popout 8 | :alt: PyPI - Python Version 9 | :target: https://pypi.org/project/pySqsListener/ 10 | 11 | 12 | 13 | 14 | This package takes care of the boilerplate involved in listening to an SQS 15 | queue, as well as sending messages to a queue. Works with python 2.7 & 3.6+. 16 | 17 | Installation 18 | ~~~~~~~~~~~~ 19 | 20 | ``pip install pySqsListener`` 21 | 22 | Listening to a queue 23 | ~~~~~~~~~~~~~~~~~~~~ 24 | 25 | | Using the listener is very straightforward - just inherit from the 26 | ``SqsListener`` class and implement the ``handle_message()`` method. 27 | The queue will be created at runtime if it doesn't already exist. 28 | You can also specify an error queue to automatically push any errors to. 29 | 30 | Here is a basic code sample: 31 | 32 | **Standard Listener** 33 | 34 | :: 35 | 36 | from sqs_listener import SqsListener 37 | 38 | class MyListener(SqsListener): 39 | def handle_message(self, body, attributes, messages_attributes): 40 | run_my_function(body['param1'], body['param2']) 41 | 42 | listener = MyListener('my-message-queue', error_queue='my-error-queue', region_name='us-east-1') 43 | listener.listen() 44 | 45 | **Error Listener** 46 | 47 | :: 48 | 49 | from sqs_listener import SqsListener 50 | class MyErrorListener(SqsListener): 51 | def handle_message(self, body, attributes, messages_attributes): 52 | save_to_log(body['exception_type'], body['error_message'] 53 | 54 | error_listener = MyErrorListener('my-error-queue') 55 | error_listener.listen() 56 | 57 | 58 | | The options available as ``kwargs`` are as follows: 59 | 60 | - error_queue (str) - name of queue to push errors. 61 | - force_delete (boolean) - delete the message received from the queue, whether or not the handler function is successful. By default the message is deleted only if the handler function returns with no exceptions 62 | - interval (int) - number of seconds in between polls. Set to 60 by default 63 | - visibility_timeout (str) - Number of seconds the message will be invisible ('in flight') after being read. After this time interval it reappear in the queue if it wasn't deleted in the meantime. Set to '600' (10 minutes) by default 64 | - error_visibility_timeout (str) - Same as previous argument, for the error queue. Applicable only if the ``error_queue`` argument is set, and the queue doesn't already exist. 65 | - wait_time (int) - number of seconds to wait for a message to arrive (for long polling). Set to 0 by default to provide short polling. 66 | - max_number_of_messages (int) - Max number of messages to receive from the queue. Set to 1 by default, max is 10 67 | - message_attribute_names (list) - message attributes by which to filter messages 68 | - attribute_names (list) - attributes by which to filter messages (see boto docs for difference between these two) 69 | - region_name (str) - AWS region name (defaults to ``us-east-1``) 70 | - queue_url (str) - overrides ``queue`` parameter. Mostly useful for getting around `this bug `_ in the boto library 71 | 72 | 73 | Running as a Daemon 74 | ~~~~~~~~~~~~~~~~~~~ 75 | 76 | | Typically, in a production environment, you'll want to listen to an SQS queue with a daemonized process. 77 | The simplest way to do this is by running the listener in a detached process. On a typical Linux distribution it might look like this: 78 | | 79 | ``nohup python my_listener.py > listener.log &`` 80 | | And saving the resulting process id for later (for stopping the listener via the ``kill`` command). 81 | | 82 | A more complete implementation can be achieved easily by inheriting from the package's ``Daemon`` class and overriding the ``run()`` method. 83 | | 84 | | The sample_daemon.py file in the source root folder provides a clear example for achieving this. Using this example, 85 | you can run the listener as a daemon with the command ``python sample_daemon.py start``. Similarly, the command 86 | ``python sample_daemon.py stop`` will stop the process. You'll most likely need to run the start script using ``sudo``. 87 | | 88 | 89 | Logging 90 | ~~~~~~~ 91 | 92 | | The listener and launcher instances push all their messages to a ``logger`` instance, called 'sqs_listener'. 93 | In order to view the messages, the logger needs to be redirected to ``stdout`` or to a log file. 94 | | 95 | | For instance: 96 | 97 | :: 98 | 99 | logger = logging.getLogger('sqs_listener') 100 | logger.setLevel(logging.INFO) 101 | 102 | sh = logging.StreamHandler(sys.stdout) 103 | sh.setLevel(logging.INFO) 104 | 105 | formatstr = '[%(asctime)s - %(name)s - %(levelname)s] %(message)s' 106 | formatter = logging.Formatter(formatstr) 107 | 108 | sh.setFormatter(formatter) 109 | logger.addHandler(sh) 110 | 111 | | 112 | | Or to a log file: 113 | 114 | :: 115 | 116 | logger = logging.getLogger('sqs_listener') 117 | logger.setLevel(logging.INFO) 118 | 119 | sh = logging.FileHandler('mylog.log') 120 | sh.setLevel(logging.INFO) 121 | 122 | formatstr = '[%(asctime)s - %(name)s - %(levelname)s] %(message)s' 123 | formatter = logging.Formatter(formatstr) 124 | 125 | sh.setFormatter(formatter) 126 | logger.addHandler(sh) 127 | 128 | Sending messages 129 | ~~~~~~~~~~~~~~~~ 130 | 131 | | In order to send a message, instantiate an ``SqsLauncher`` with the name of the queue. By default an exception will 132 | be raised if the queue doesn't exist, but it can be created automatically if the ``create_queue`` parameter is 133 | set to true. In such a case, there's also an option to set the newly created queue's ``VisibilityTimeout`` via the 134 | third parameter. 135 | | 136 | | After instantiation, use the ``launch_message()`` method to send the message. The message body should be a ``dict``, 137 | and additional kwargs can be specified as stated in the `SQS docs 138 | `_. 139 | The method returns the response from SQS. 140 | 141 | **Launcher Example** 142 | 143 | :: 144 | 145 | from sqs_launcher import SqsLauncher 146 | 147 | launcher = SqsLauncher('my-queue') 148 | response = launcher.launch_message({'param1': 'hello', 'param2': 'world'}) 149 | 150 | Important Notes 151 | ~~~~~~~~~~~~~~~ 152 | 153 | - The environment variable ``AWS_ACCOUNT_ID`` must be set, in addition 154 | to the environment having valid AWS credentials (via environment variables 155 | or a credentials file) or if running in an aws ec2 instance a role attached 156 | with the required permissions. 157 | - For both the main queue and the error queue, if the queue doesn’t 158 | exist (in the specified region), it will be created at runtime. 159 | - The error queue receives only two values in the message body: ``exception_type`` and ``error_message``. Both are of type ``str`` 160 | - If the function that the listener executes involves connecting to a database, you should explicitly close the connection at the end of the function. Otherwise, you're likely to get an error like this: ``OperationalError(2006, 'MySQL server has gone away')`` 161 | - Either the queue name or the queue url should be provided. When both are provided the queue url is used and the queue name is ignored. 162 | 163 | Contributing 164 | ~~~~~~~~~~~~ 165 | 166 | Fork the repo and make a pull request. 167 | -------------------------------------------------------------------------------- /sqs_listener/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | script for running sqs listener 3 | 4 | Created December 21st, 2016 5 | @author: Yaakov Gesher 6 | @version: 0.2.3 7 | @license: Apache 8 | """ 9 | 10 | # ================ 11 | # start imports 12 | # ================ 13 | 14 | import boto3 15 | import boto3.session 16 | import json 17 | import time 18 | import logging 19 | import os 20 | import sys 21 | from sqs_launcher import SqsLauncher 22 | from abc import ABCMeta, abstractmethod 23 | 24 | # ================ 25 | # start class 26 | # ================ 27 | 28 | sqs_logger = logging.getLogger('sqs_listener') 29 | 30 | class SqsListener(object): 31 | __metaclass__ = ABCMeta 32 | 33 | def __init__(self, queue, **kwargs): 34 | """ 35 | :param queue: (str) name of queue to listen to 36 | :param kwargs: options for fine tuning. see below 37 | """ 38 | aws_access_key = kwargs.get('aws_access_key', '') 39 | aws_secret_key = kwargs.get('aws_secret_key', '') 40 | 41 | if len(aws_access_key) != 0 and len(aws_secret_key) != 0: 42 | boto3_session = boto3.Session( 43 | aws_access_key_id=aws_access_key, 44 | aws_secret_access_key=aws_secret_key 45 | ) 46 | else: 47 | if (not os.environ.get('AWS_ACCOUNT_ID', None) and 48 | not ('iam-role' == boto3.Session().get_credentials().method)): 49 | raise EnvironmentError('Environment variable `AWS_ACCOUNT_ID` not set and no role found.') 50 | 51 | self._queue_name = queue 52 | self._poll_interval = kwargs.get("interval", 60) 53 | self._queue_visibility_timeout = kwargs.get('visibility_timeout', '600') 54 | self._error_queue_name = kwargs.get('error_queue', None) 55 | self._error_queue_visibility_timeout = kwargs.get('error_visibility_timeout', '600') 56 | self._queue_url = kwargs.get('queue_url', None) 57 | self._message_attribute_names = kwargs.get('message_attribute_names', []) 58 | self._attribute_names = kwargs.get('attribute_names', []) 59 | self._force_delete = kwargs.get('force_delete', False) 60 | self._endpoint_name = kwargs.get('endpoint_name', None) 61 | self._wait_time = kwargs.get('wait_time', 0) 62 | self._max_number_of_messages = kwargs.get('max_number_of_messages', 1) 63 | 64 | # must come last 65 | if boto3_session: 66 | self._session = boto3_session 67 | else: 68 | self._session = boto3.session.Session() 69 | self._region_name = kwargs.get('region_name', self._session.region_name) 70 | self._client = self._initialize_client() 71 | 72 | 73 | def _initialize_client(self): 74 | # new session for each instantiation 75 | ssl = True 76 | if self._region_name == 'elasticmq': 77 | ssl = False 78 | 79 | sqs = self._session.client('sqs', region_name=self._region_name, endpoint_url=self._endpoint_name, use_ssl=ssl) 80 | queues = sqs.list_queues(QueueNamePrefix=self._queue_name) 81 | mainQueueExists = False 82 | errorQueueExists = False 83 | if 'QueueUrls' in queues: 84 | for q in queues['QueueUrls']: 85 | qname = q.split('/')[-1] 86 | if qname == self._queue_name: 87 | mainQueueExists = True 88 | if self._error_queue_name and qname == self._error_queue_name: 89 | errorQueueExists = True 90 | 91 | 92 | # create queue if necessary. 93 | # creation is idempotent, no harm in calling on a queue if it already exists. 94 | if self._queue_url is None: 95 | if not mainQueueExists: 96 | sqs_logger.warning("main queue not found, creating now") 97 | 98 | # is this a fifo queue? 99 | if self._queue_name.endswith(".fifo"): 100 | fifoQueue="true" 101 | q = sqs.create_queue( 102 | QueueName=self._queue_name, 103 | Attributes={ 104 | 'VisibilityTimeout': self._queue_visibility_timeout, # 10 minutes 105 | 'FifoQueue':fifoQueue 106 | } 107 | ) 108 | else: 109 | # need to avoid FifoQueue property for normal non-fifo queues 110 | q = sqs.create_queue( 111 | QueueName=self._queue_name, 112 | Attributes={ 113 | 'VisibilityTimeout': self._queue_visibility_timeout, # 10 minutes 114 | } 115 | ) 116 | self._queue_url = q['QueueUrl'] 117 | 118 | if self._error_queue_name and not errorQueueExists: 119 | sqs_logger.warning("error queue not found, creating now") 120 | q = sqs.create_queue( 121 | QueueName=self._error_queue_name, 122 | Attributes={ 123 | 'VisibilityTimeout': self._queue_visibility_timeout # 10 minutes 124 | } 125 | ) 126 | 127 | if self._queue_url is None: 128 | if os.environ.get('AWS_ACCOUNT_ID', None): 129 | qs = sqs.get_queue_url(QueueName=self._queue_name, 130 | QueueOwnerAWSAccountId=os.environ.get('AWS_ACCOUNT_ID', None)) 131 | else: 132 | qs = sqs.get_queue_url(QueueName=self._queue_name) 133 | self._queue_url = qs['QueueUrl'] 134 | return sqs 135 | 136 | def _start_listening(self): 137 | # TODO consider incorporating output processing from here: https://github.com/debrouwere/sqs-antenna/blob/master/antenna/__init__.py 138 | while True: 139 | # calling with WaitTimeSecconds of zero show the same behavior as 140 | # not specifiying a wait time, ie: short polling 141 | messages = self._client.receive_message( 142 | QueueUrl=self._queue_url, 143 | MessageAttributeNames=self._message_attribute_names, 144 | AttributeNames=self._attribute_names, 145 | WaitTimeSeconds=self._wait_time, 146 | MaxNumberOfMessages=self._max_number_of_messages 147 | ) 148 | if 'Messages' in messages: 149 | 150 | sqs_logger.debug(messages) 151 | sqs_logger.info("{} messages received".format(len(messages['Messages']))) 152 | for m in messages['Messages']: 153 | receipt_handle = m['ReceiptHandle'] 154 | m_body = m['Body'] 155 | message_attribs = None 156 | attribs = None 157 | 158 | # catch problems with malformed JSON, usually a result of someone writing poor JSON directly in the AWS console 159 | try: 160 | params_dict = json.loads(m_body) 161 | except: 162 | sqs_logger.warning("Unable to parse message - JSON is not formatted properly") 163 | continue 164 | if 'MessageAttributes' in m: 165 | message_attribs = m['MessageAttributes'] 166 | if 'Attributes' in m: 167 | attribs = m['Attributes'] 168 | try: 169 | if self._force_delete: 170 | self._client.delete_message( 171 | QueueUrl=self._queue_url, 172 | ReceiptHandle=receipt_handle 173 | ) 174 | self.handle_message(params_dict, message_attribs, attribs) 175 | else: 176 | self.handle_message(params_dict, message_attribs, attribs) 177 | self._client.delete_message( 178 | QueueUrl=self._queue_url, 179 | ReceiptHandle=receipt_handle 180 | ) 181 | except Exception as ex: 182 | # need exception logtype to log stack trace 183 | sqs_logger.exception(ex) 184 | if self._error_queue_name: 185 | exc_type, exc_obj, exc_tb = sys.exc_info() 186 | 187 | sqs_logger.info( "Pushing exception to error queue") 188 | error_launcher = SqsLauncher(queue=self._error_queue_name, create_queue=True) 189 | error_launcher.launch_message( 190 | { 191 | 'exception_type': str(exc_type), 192 | 'error_message': str(ex.args) 193 | } 194 | ) 195 | 196 | else: 197 | time.sleep(self._poll_interval) 198 | 199 | def listen(self): 200 | sqs_logger.info( "Listening to queue " + self._queue_name) 201 | if self._error_queue_name: 202 | sqs_logger.info( "Using error queue " + self._error_queue_name) 203 | 204 | self._start_listening() 205 | 206 | def _prepare_logger(self): 207 | logger = logging.getLogger('eg_daemon') 208 | logger.setLevel(logging.INFO) 209 | 210 | sh = logging.StreamHandler(sys.stdout) 211 | sh.setLevel(logging.INFO) 212 | 213 | formatstr = '[%(asctime)s - %(name)s - %(levelname)s] %(message)s' 214 | formatter = logging.Formatter(formatstr) 215 | 216 | sh.setFormatter(formatter) 217 | logger.addHandler(sh) 218 | 219 | @abstractmethod 220 | def handle_message(self, body, attributes, messages_attributes): 221 | """ 222 | Implement this method to do something with the SQS message contents 223 | :param body: dict 224 | :param attributes: dict 225 | :param messages_attributes: dict 226 | :return: 227 | """ 228 | return 229 | --------------------------------------------------------------------------------