├── .buildkite └── pipeline.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── requirements-dev.txt ├── requirements.txt ├── setup.py ├── shudder ├── __init__.py ├── __main__.py ├── config.py ├── metadata.py └── queue.py └── tests └── shudder └── .gitkeep /.buildkite/pipeline.yml: -------------------------------------------------------------------------------- 1 | steps: 2 | - label: ":debian: Create Debian File" 3 | command: "buildkite-scripts/scripts/fpm.sh" 4 | env: 5 | DEB_NAME: virtru-devops-shudder 6 | 7 | - wait 8 | 9 | - label: ":debian: Upload Debian File" 10 | command: "buildkite-scripts/scripts/deb-s3.sh" 11 | env: 12 | APT_REPO_NAME: shared 13 | ENVIRONMENT_NAME: "0" 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.pyc 3 | env/ 4 | dist 5 | shudder.toml 6 | *.iml 7 | .idea/ 8 | .vscode/ 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include requirements.txt -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # shudder 2 | 3 | Shudder is a service for facilitating graceful shutdowns in AWS autoscaling 4 | groups. 5 | 6 | It works by making use of 7 | [Lifecycle Hooks](http://docs.aws.amazon.com/cli/latest/reference/autoscaling/put-lifecycle-hook.html). You 8 | give your autoscaling group a lifecycle hook that publishes to an SNS topic that 9 | you configure in shudder. When shudder starts up, it will create an SQS queue 10 | for the instance it is running on and subscribe it to the SNS topic. It polls 11 | for new messages and waits for one that is a termination command for this 12 | instance. It can then send a GET request to a configured endpoint telling it to 13 | shut down gracefully, or execute commands. 14 | 15 | It can also detect when a spot instance has been scheduled for termination, 16 | using the [instance termination notice](https://aws.amazon.com/blogs/aws/new-ec2-spot-instance-termination-notices/) 17 | available in instance metadata. The same configured endpoint will be hit if 18 | a scheduled termination of a spot instance is detected. 19 | 20 | ## Usage 21 | 22 | Install it! 23 | 24 | ``` 25 | pip install . 26 | ``` 27 | 28 | You need a toml file looking like this: 29 | 30 | ```toml 31 | sqs_prefix = "myapp" 32 | region = "us-east-1" 33 | sns_topic = "arn:aws:sns:us-east-1:723456455537:myapp-shutdowns" 34 | endpoints = ["http://127.0.0.1:5000/youaregoingtodiesoon", "http://127.0.0.1:5001/shutdown"] 35 | commands = [["//etc/init.d/nginx", "stop"], ["/etc/init.d/filebeats", "stop"]] 36 | queue_tags = { tag1 = "value1", tag2 = "value2"} 37 | ``` 38 | 39 | **sqs_prefix:** specifies the prefix of the sqs queue that will be created for the instance. 40 | Queues are named by concatenating the prefix to the instance id. 41 | 42 | **region:** specifies the AWS region 43 | 44 | **sns_topic:** specifies the arn of the SNS Topic that publishes lifecycle events for this 45 | instance's auto scaling group 46 | 47 | **endpoints:** specifies a list of http endpoints that shudder will execute a GET request on once 48 | shudder has received the shutdown lifecycle message 49 | 50 | **commands:** specifies a list of commands that shudder will execute once shudder has received the 51 | shutdown lifecycle message 52 | 53 | **queue_tags:** the aws resource tags to assign to the SQS queue that is created by Shudder for the 54 | instance its running on 55 | 56 | You can specify the config file path as an environment variable: 57 | 58 | ```bash 59 | CONFIG_FILE=/home/ubuntu/shudder.toml python -m shudder 60 | ``` 61 | 62 | Shudder expects you to have credentials *somehow*. Ideally you have an IAM role 63 | on your server and it can pick it up that way, otherwise it'll look for a 64 | `~/.boto` config or environment variables for `AWS_ACCESS_KEY_ID` and 65 | `AWS_SECRET_ACCESS_KEY`. 66 | 67 | *This project has to be run on an EC2 instance because it looks up the instance 68 | ID in the instance metadata. It'll break anywhere but on EC2.* 69 | 70 | ## Permissions 71 | 72 | Your credentials need to be able to subscribe to your SNS 73 | topic, unsubscribe from your subscription ARN, 74 | as well as create and read from SQS queues under the prefix configured. 75 | 76 | ### Example IAM Role for Instance running Shudder 77 | 78 | ```json 79 | { 80 | "Version": "2012-10-17", 81 | "Statement": [ 82 | { 83 | "Action": [ 84 | "autoscaling:RecordLifecycleActionHeartbeat", 85 | "autoscaling:CompleteLifecycleAction" 86 | ], 87 | "Resource": "arn:aws:autoscaling:*:*:*", 88 | "Effect": "Allow" 89 | }, 90 | { 91 | "Action": [ 92 | "sqs:*" 93 | ], 94 | "Resource": [ 95 | "arn:aws:sqs:*:0123456789:*:*" 96 | ], 97 | "Effect": "Allow" 98 | }, 99 | { 100 | "Action": [ 101 | "sns:Unsubscribe" 102 | ], 103 | "Resource": [ 104 | "*" 105 | ], 106 | "Effect": "Allow" 107 | }, 108 | { 109 | "Action": [ 110 | "sns:Subscribe" 111 | ], 112 | "Resource": [ 113 | "arn:aws:sns:us-east-1:0123456789:*" 114 | ], 115 | "Effect": "Allow" 116 | } 117 | ] 118 | } 119 | ``` 120 | 121 | ## Enabling lifecycle hooks 122 | Unfortunately, lifecycle hooks cannot be managed from CloudFormation or from the web console. To set up a hook, you may need to use the CLI as follows: 123 | 124 | ```bash 125 | aws autoscaling put-lifecycle-hook 126 | --lifecycle-hook-name really-cool-hook-name 127 | --auto-scaling-group-name my-asg-name 128 | --lifecycle-transition autoscaling:EC2_INSTANCE_TERMINATING 129 | --role-arn arn:aws:iam::0123456789:role/autoscaling-lifecycle-sqs 130 | --notification-target-arn arn:aws:sns:us-east-1:0123456789:instance-shutdowns 131 | --heartbeat-timeout 300 132 | --default-result CONTINUE 133 | ``` 134 | 135 | The specified role must have the right to publish to the specified topic: 136 | 137 | ```json 138 | { 139 | "Version": "2012-10-17", 140 | "Statement": [ 141 | { 142 | "Effect": "Allow", 143 | "Action": [ 144 | "sns:Publish" 145 | ], 146 | "Resource": "arn:aws:sns:us-east-1:0123456789:instance-shutdowns" 147 | } 148 | ] 149 | } 150 | ``` 151 | 152 | This role must be assumable by the autoscaling service, with a trust relationship policy like this: 153 | 154 | ```json 155 | { 156 | "Version": "2012-10-17", 157 | "Statement": [ 158 | { 159 | "Sid": "", 160 | "Effect": "Allow", 161 | "Principal": { 162 | "Service": "autoscaling.amazonaws.com" 163 | }, 164 | "Action": "sts:AssumeRole" 165 | } 166 | ] 167 | } 168 | ``` 169 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | nose 3 | elpy 4 | rope 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | boto3==1.7.58 2 | toml==0.9.4 3 | requests==2.21.0 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """A simple service for capturing autoscaling lifecycle hook actions 2 | and notifying another service that it needs a graceful shutdown. 3 | 4 | """ 5 | from setuptools import setup, find_packages 6 | 7 | with open('requirements.txt') as f: 8 | requirements = f.readlines() 9 | 10 | setup( 11 | name='shudder', 12 | description="Graceful shutdowns using autoscaling lifecycle hooks.", 13 | version='0.2.0', 14 | long_description=__doc__, 15 | packages=find_packages(), 16 | include_package_data=True, 17 | author='Anthony Grimes', 18 | author_email='anthony@scopely.com', 19 | url='https://github.com/scopely/shudder', 20 | license='Apache 2.0', 21 | install_requires=requirements 22 | ) 23 | -------------------------------------------------------------------------------- /shudder/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Scopely, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /shudder/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Scopely, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Start polling of SQS and metadata.""" 16 | import shudder.queue as queue 17 | import shudder.metadata as metadata 18 | from shudder.config import CONFIG, LOG_FILE 19 | import time 20 | import os 21 | import requests 22 | import signal 23 | import subprocess 24 | import sys 25 | import logging 26 | from requests.exceptions import ConnectionError 27 | 28 | logging.basicConfig(filename=LOG_FILE,format='%(asctime)s %(levelname)s:%(message)s',level=logging.INFO) 29 | 30 | 31 | def receive_signal(signum, stack): 32 | if signum in [1,2,3,15]: 33 | print 'Caught signal %s, exiting.' %(str(signum)) 34 | sys.exit() 35 | else: print 'Caught signal %s, ignoring.' %(str(signum)) 36 | 37 | if __name__ == '__main__': 38 | uncatchable = ['SIG_DFL','SIGSTOP','SIGKILL'] 39 | for i in [x for x in dir(signal) if x.startswith("SIG")]: 40 | if not i in uncatchable: 41 | signum = getattr(signal,i) 42 | signal.signal(signum,receive_signal) 43 | 44 | sqs_connection, sqs_queue = queue.create_queue() 45 | sns_connection, subscription_arn = queue.subscribe_sns(sqs_queue) 46 | 47 | running = True 48 | while running: 49 | try: 50 | message = queue.poll_queue(sqs_connection, sqs_queue) 51 | if message or metadata.poll_instance_metadata(): 52 | queue.clean_up_sns(sns_connection, subscription_arn, sqs_queue) 53 | if 'endpoint' in CONFIG: 54 | requests.get(CONFIG["endpoint"]) 55 | if 'endpoints' in CONFIG: 56 | for endpoint in CONFIG["endpoints"]: 57 | requests.get(endpoint) 58 | if 'commands' in CONFIG: 59 | for command in CONFIG["commands"]: 60 | print 'Running command: %s' % command 61 | process = subprocess.Popen(command) 62 | while process.poll() is None: 63 | time.sleep(30) 64 | """Send a heart beat to aws""" 65 | try: 66 | queue.record_lifecycle_action_heartbeat(message) 67 | except: 68 | logging.exception('Error sending hearbeat for') 69 | logging.info(message) 70 | 71 | """Send a complete lifecycle action""" 72 | queue.complete_lifecycle_action(message) 73 | running = False 74 | time.sleep(5) 75 | except ConnectionError: 76 | logging.exception('Connection issue') 77 | except: 78 | logging.exception('Something went wrong') 79 | logging.info('Success') 80 | sys.exit(0) 81 | -------------------------------------------------------------------------------- /shudder/config.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Scopely, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Configuration for shudder""" 16 | import os 17 | import toml 18 | 19 | 20 | CONFIG_FILE = os.environ.get('CONFIG_FILE', "shudder.toml") 21 | CONFIG = {} 22 | 23 | with open(CONFIG_FILE, 'r') as f: 24 | CONFIG = toml.loads(f.read()) 25 | if 'logfile' in CONFIG.values(): 26 | LOG_FILE = CONFIG['logfile'] 27 | else: 28 | LOG_FILE = os.environ.get('LOG_FILE', '/var/log/shudder.log') 29 | -------------------------------------------------------------------------------- /shudder/metadata.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Scopely, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Module to set up polling of instance metadata for the termination of a spot instance 16 | 17 | """ 18 | import requests 19 | import logging 20 | from shudder.config import CONFIG, LOG_FILE 21 | 22 | 23 | logging.basicConfig(filename=LOG_FILE,format='%(asctime)s %(levelname)s:%(message)s',level=logging.INFO) 24 | termination_time = "http://169.254.169.254/latest/meta-data/spot/termination-time" 25 | instance_id = "http://169.254.169.254/latest/meta-data/instance-id" 26 | 27 | def poll_instance_metadata(): 28 | """Check instance metadata for a scheduled termination""" 29 | try: 30 | r = requests.get(termination_time) 31 | return r.status_code < 400 32 | except: 33 | logging.exception('Request to ' + termination_time + ' failed.') 34 | 35 | def get_instance_id(): 36 | """Check instance metadata for an instance id""" 37 | try: 38 | r = requests.get(instance_id) 39 | return r.text 40 | except: 41 | logging.exception('Request to ' + instance_id + ' failed.') 42 | -------------------------------------------------------------------------------- /shudder/queue.py: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Scopely, Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Module for setting up an sqs queue subscribed to 16 | an sns topic polling for messages pertaining to our 17 | impending doom. 18 | 19 | """ 20 | import json 21 | import boto3 22 | import hashlib 23 | 24 | from shudder.config import CONFIG 25 | import shudder.metadata as metadata 26 | 27 | 28 | INSTANCE_ID = metadata.get_instance_id() 29 | QUEUE_NAME = "{prefix}-{id}".format(prefix=CONFIG['sqs_prefix'], 30 | id=INSTANCE_ID) 31 | 32 | 33 | def create_queue(): 34 | """Creates the SQS queue and returns the queue url and metadata""" 35 | conn = boto3.client('sqs', region_name=CONFIG['region']) 36 | queue_metadata = conn.create_queue(QueueName=QUEUE_NAME, Attributes={'VisibilityTimeout':'3600'}) 37 | 38 | if 'queue_tags' in CONFIG: 39 | conn.tag_queue(QueueUrl=queue_metadata['QueueUrl'], Tags=CONFIG['queue_tags']) 40 | 41 | """Get the SQS queue object from the queue URL""" 42 | sqs = boto3.resource('sqs', region_name=CONFIG['region']) 43 | queue = sqs.Queue(queue_metadata['QueueUrl']) 44 | return conn, queue 45 | 46 | 47 | def subscribe_sns(queue): 48 | """Attach a policy to allow incoming connections from SNS""" 49 | statement_id = hashlib.md5((CONFIG['sns_topic'] + queue.attributes.get('QueueArn')).encode('utf-8')).hexdigest() 50 | statement_id_exists = False 51 | existing_policy = queue.attributes.get('Policy') 52 | if existing_policy: 53 | policy = json.loads(existing_policy) 54 | else: 55 | policy = {} 56 | if 'Version' not in policy: 57 | policy['Version'] = '2008-10-17' 58 | if 'Statement' not in policy: 59 | policy['Statement'] = [] 60 | # See if a Statement with the Sid exists already. 61 | for statement in policy['Statement']: 62 | if statement['Sid'] == statement_id: 63 | statement_id_exists = True 64 | if not statement_id_exists: 65 | statement = {'Action': 'SQS:SendMessage', 66 | 'Effect': 'Allow', 67 | 'Principal': {'AWS': '*'}, 68 | 'Resource': queue.attributes.get('QueueArn'), 69 | 'Sid': statement_id, 70 | 'Condition': {"ForAllValues:ArnEquals":{"aws:SourceArn":CONFIG['sns_topic']}}} 71 | policy['Statement'].append(statement) 72 | queue.set_attributes(Attributes={'Policy':json.dumps(policy)}) 73 | """Subscribes the SNS topic to the queue.""" 74 | conn = boto3.client('sns', region_name=CONFIG['region']) 75 | sub = conn.subscribe(TopicArn=CONFIG['sns_topic'], Protocol='sqs', Endpoint=queue.attributes.get('QueueArn')) 76 | sns_arn = sub['SubscriptionArn'] 77 | return conn, sns_arn 78 | 79 | 80 | def should_terminate(msg): 81 | """Check if the termination message is about our instance""" 82 | first_box = json.loads(msg.body) 83 | message = json.loads(first_box['Message']) 84 | termination_msg = 'autoscaling:EC2_INSTANCE_TERMINATING' 85 | 86 | if 'LifecycleTransition' in message and message['LifecycleTransition'] == termination_msg and INSTANCE_ID == message['EC2InstanceId']: 87 | return message 88 | else: 89 | return None 90 | 91 | def clean_up_sns(sns_conn, sns_arn, queue): 92 | """Clean up SNS subscription and SQS queue""" 93 | queue.delete() 94 | sns_conn.unsubscribe(SubscriptionArn=sns_arn) 95 | 96 | 97 | def record_lifecycle_action_heartbeat(message): 98 | """Let AWS know we're still in the process of shutting down""" 99 | conn = boto3.client('autoscaling', region_name=CONFIG['region']) 100 | conn.record_lifecycle_action_heartbeat( 101 | LifecycleHookName=message['LifecycleHookName'], 102 | AutoScalingGroupName=message['AutoScalingGroupName'], 103 | LifecycleActionToken=message['LifecycleActionToken'], 104 | InstanceId=message['EC2InstanceId']) 105 | 106 | 107 | def complete_lifecycle_action(message): 108 | """Let AWS know it's safe to terminate the instance now""" 109 | conn = boto3.client('autoscaling', region_name=CONFIG['region']) 110 | conn.complete_lifecycle_action( 111 | LifecycleHookName=message['LifecycleHookName'], 112 | AutoScalingGroupName=message['AutoScalingGroupName'], 113 | LifecycleActionToken=message['LifecycleActionToken'], 114 | LifecycleActionResult='CONTINUE', 115 | InstanceId=message['EC2InstanceId']) 116 | 117 | 118 | def poll_queue(conn, queue): 119 | """Poll SQS until we get a termination message.""" 120 | messages = queue.receive_messages() 121 | for message in messages: 122 | message.delete() 123 | return should_terminate(message) 124 | return False 125 | -------------------------------------------------------------------------------- /tests/shudder/.gitkeep: -------------------------------------------------------------------------------- 1 | This is necessary because git won't track empty directories. --------------------------------------------------------------------------------