├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── pension ├── __init__.py ├── cli.py └── notify.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | .venv/ 59 | pension.toml 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Erik Price 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md LICENSE 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pension 2 | 3 | > Plan for your instances' retirement. 4 | 5 | Pension checks for EC2 instance retirement (e.g. due to hardware failures) and 6 | other maintenance events that could cause (or be the source of) issues down the 7 | line. It is meant to be run periodically (in a cron or some such) and provide 8 | some warning before bad things happen. 9 | 10 | Currently, pension knows how to ping a slack channel or send an email when 11 | something's broken. Additional notification methods are welcome contributions. 12 | Otherwise, parsing the JSON output format should be flexible enough for any 13 | internal notification tooling. 14 | 15 | ### usage 16 | 17 | `pip install pension` 18 | 19 | `pension [--dry-run] [--quiet] [--config path/to/pension.toml]` 20 | 21 | ### configuration 22 | 23 | pension uses [toml](https://github.com/mojombo/toml) as its configuration 24 | language. The config file can be specified on the command line via `--config`. 25 | If not provided, pension will try `./pension.toml` and `~/.pension.toml`. 26 | 27 | If no config file is available, pension will try to proceed using only AWS 28 | environment variables and the JSON console output. 29 | 30 | For more information on AWS credentials, refer to the 31 | [boto3 documentation](http://boto3.readthedocs.org/en/latest/guide/configuration.html). 32 | 33 | #### example `pension.toml` 34 | 35 | ```toml 36 | # (optional) profile names as configured in ~/.aws/credentials, defaults to 37 | # using boto's AWS environment variables. 38 | aws_profiles = ["dev", "prod"] 39 | 40 | # Enable notification via a slack message using webhooks. 41 | # Hooks can be setup here: https://my.slack.com/services/new/incoming-webhook/ 42 | [notify.slack] 43 | # (required) 44 | hook_url = "https://hooks.slack.com/..." 45 | # (optional) defaults to channel configured with webhook 46 | channel = "#general" 47 | # (optional) defaults to name configured with webhook 48 | user_name = "Bad News Bot" 49 | 50 | # Sends an email using SMTP 51 | [notify.email] 52 | # (required) 53 | server = "smtp.gmail.com" 54 | # (required) login user name 55 | user_name = "foobar@gmail.com" 56 | # (required) 57 | password = "hunter2" 58 | # (optional) defaults to the given user_name 59 | sender = "foo@bar.baz" 60 | # (required) 61 | recipients = ["oh@no.com", "alerts@myco.pagerduty.com"] 62 | # (required) 63 | subject = "Maintenance events detected for ec2 instances!" 64 | 65 | [notify.json] 66 | # (optional) file to write to. if not provided, will dump to stdout 67 | file = "output.json" 68 | ``` 69 | -------------------------------------------------------------------------------- /pension/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/erik/pension/d49a9890a385d2f6cda12f833f647d5c7a0af1e6/pension/__init__.py -------------------------------------------------------------------------------- /pension/cli.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | 4 | from boto3.session import Session 5 | import click 6 | import toml 7 | 8 | import notify 9 | 10 | 11 | def get_profiles(config): 12 | if 'aws_profiles' not in config: 13 | return { 14 | 'default': Session() 15 | } 16 | 17 | return { 18 | prof: Session(profile_name=prof) 19 | for prof in config['aws_profiles'] 20 | } 21 | 22 | 23 | # TODO: truly support instance-status, system-status. 24 | def get_instance_statuses(ec2_client, config): 25 | def _filter(filters): 26 | _statuses = [] 27 | next_token = '' 28 | 29 | while True: 30 | res = ec2_client.describe_instance_status( 31 | Filters=[ 32 | {'Name': k, 'Values': v} 33 | for k, v in filters.iteritems() 34 | ], 35 | NextToken=next_token, 36 | MaxResults=1000 37 | ) 38 | 39 | # We don't care about completed events. Thanks amazon for this 40 | # wonderful API. 41 | active_events = [ 42 | status 43 | for status in res['InstanceStatuses'] 44 | for event in status.get('Events', []) 45 | if not event['Description'].startswith('[Completed]') 46 | ] 47 | 48 | # Don't alert if we don't have any active events 49 | if len(active_events): 50 | _statuses.extend(active_events) 51 | next_token = res.get('NextToken') 52 | 53 | if not next_token: 54 | break 55 | 56 | return _statuses 57 | 58 | # describe_instance_status is an AND of filters, we want an OR 59 | seen = set() 60 | results = [ 61 | _filter({'event.code': ['*']}), 62 | _filter({'instance-status.status': ['impaired']}), 63 | _filter({'instance-status.reachability': ['failed']}), 64 | _filter({'system-status.status': ['impaired']}), 65 | _filter({'system-status.reachability': ['failed']}), 66 | ] 67 | 68 | statuses = [] 69 | for instances in results: 70 | statuses.extend([i for i in instances if i['InstanceId'] not in seen]) 71 | seen.update([i['InstanceId'] for i in instances]) 72 | 73 | return statuses 74 | 75 | 76 | def get_config(config_locations): 77 | for loc in config_locations: 78 | file_name = os.path.expanduser(loc) 79 | 80 | if os.path.exists(file_name): 81 | click.echo('using config %s' % file_name, err=True) 82 | 83 | with open(file_name) as fp: 84 | return toml.loads(fp.read()) 85 | 86 | 87 | @click.command() 88 | @click.option('--dry-run', is_flag=True, help='Disables sending alerts') 89 | @click.option('--config', required=False, help='Configuration file location') 90 | @click.option('--quiet', '-q', is_flag=True, help='Disables default JSON output') 91 | def main(dry_run, config, quiet): 92 | config_names = [config] if config else ['pension.toml', '~/.pension.toml'] 93 | 94 | try: 95 | config = get_config(config_names) 96 | except: 97 | click.echo('Failed to parse config', err=True) 98 | return -1 99 | 100 | if config is None: 101 | click.echo('No usable config file, trying environment vars', err=True) 102 | config = {'notify': {'json': {}}} 103 | 104 | data = {'instances': [], 'profiles': {}} 105 | instance_map = {} 106 | 107 | for prof, session in get_profiles(config).iteritems(): 108 | ec2_client = session.client('ec2') 109 | ec2 = session.resource('ec2') 110 | 111 | statuses = get_instance_statuses(ec2_client, config) 112 | 113 | data['profiles'][prof] = { 114 | 'region': session._session.get_config_variable('region'), 115 | 'instances': [s['InstanceId'] for s in statuses] 116 | } 117 | data['instances'].extend(statuses) 118 | 119 | if statuses: 120 | # Keep track of boto ec2 instances 121 | instance_list = ec2.instances.filter(InstanceIds=[ 122 | s['InstanceId'] for s in statuses 123 | ]) 124 | 125 | instance_map.update({ 126 | i.instance_id: i 127 | for i in instance_list 128 | }) 129 | 130 | click.echo('%d instance(s) have reported issues' % len(data['instances']), 131 | err=True) 132 | 133 | if dry_run: 134 | config['notify'] = {} if quiet else {'json': {}} 135 | 136 | notify.send(data, instance_map, config['notify']) 137 | 138 | 139 | if __name__ == '__main__': 140 | main() 141 | -------------------------------------------------------------------------------- /pension/notify.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8; -*- 2 | 3 | import email 4 | import json 5 | import smtplib 6 | 7 | from email.MIMEText import MIMEText 8 | 9 | import requests 10 | 11 | 12 | def json_serialize(obj): 13 | """handle datetime objects""" 14 | 15 | if hasattr(obj, 'isoformat'): 16 | return obj.isoformat() 17 | else: 18 | raise TypeError, 'Object of type %s with value of %s is not JSON \ 19 | serializable' % (type(obj), repr(obj)) 20 | 21 | 22 | def cleanup_statuses(statuses): 23 | """Downcase keys and only include relevant items""" 24 | 25 | def _format_item(item): 26 | status = {'status': item['Status']} 27 | status.update({ 28 | e['Name']: e['Status'] for e in item['Details'] 29 | }) 30 | 31 | return status 32 | 33 | info = {} 34 | 35 | for status in statuses: 36 | instance = status['InstanceId'] 37 | status = { 38 | 'events': status.get('Events', []), 39 | 'instance': _format_item(status['InstanceStatus']), 40 | 'system': _format_item(status['SystemStatus']), 41 | } 42 | 43 | info[instance] = status 44 | 45 | return info 46 | 47 | 48 | def send(data, inst_map, config): 49 | notify_fcns = { 50 | 'json': notify_json, 51 | 'slack': notify_slack, 52 | 'email': notify_email, 53 | } 54 | 55 | # Don't notify unless there's actually something wrong 56 | if not len(data['instances']): 57 | return 58 | 59 | data['instances'] = cleanup_statuses(data['instances']) 60 | 61 | for key, fcn in notify_fcns.iteritems(): 62 | if key in config: 63 | fcn(data, inst_map, config[key]) 64 | 65 | 66 | def notify_json(data, inst_map, config): 67 | if 'file' in config: 68 | with open(config['file'], 'w') as fp: 69 | json.dump(data, fp, indent=4, sort_keys=True, default=json_serialize) 70 | else: 71 | print json.dumps(data, indent=4, sort_keys=True, default=json_serialize) 72 | 73 | 74 | def notify_slack(data, inst_map, config): 75 | console_url = ': `{inst_name}` {events}' 77 | 78 | # This is kind of a gross API. 79 | tags = { 80 | inst_id: {t['Key']: t['Value'] for t in (inst.tags or [])} 81 | for inst_id, inst in inst_map.iteritems() 82 | } 83 | 84 | event_descriptions = {} 85 | 86 | for inst_id, status in data['instances'].iteritems(): 87 | events = [] 88 | 89 | for event in status.get('events', []): 90 | desc = event.get('Description') 91 | start = event.get('NotBefore') 92 | 93 | events.append('%s%s' % (desc, start.strftime('%m/%d %H:%M'))) 94 | 95 | if events: 96 | event_descriptions[inst_id] = ';'.join(events) 97 | 98 | instance_links = [ 99 | console_url.format( 100 | region=profile['region'], 101 | instance=inst_id, 102 | inst_name=tags[inst_id].get('Name', 'unnamed'), 103 | events=event_descriptions.get(inst_id, '') 104 | ) 105 | for profile in data['profiles'].values() 106 | for inst_id in profile['instances'] 107 | ] 108 | 109 | post_data = { 110 | 'text': '%d instance(s) have an issue: \n • %s' % ( 111 | len(data['instances']), '\n • '.join(instance_links)) 112 | } 113 | 114 | # Add in the optional keys 115 | if 'user_name' in config: 116 | post_data['username'] = config['user_name'] 117 | 118 | if 'channel' in config: 119 | post_data['channel'] = config['channel'] 120 | 121 | res = requests.post(config['hook_url'], json=post_data) 122 | res.raise_for_status() 123 | 124 | 125 | def notify_email(data, inst_map, config): 126 | 127 | msg_content = ''' 128 | {num_instances} EC2 instances currently have issues! 129 | ======== 130 | 131 | By AWS profile: 132 | -------- 133 | 134 | {profile_counts} 135 | 136 | Full details: 137 | -------- 138 | 139 | {details} 140 | '''.format( 141 | num_instances=len(data['instances']), 142 | profile_counts='\n'.join([ 143 | '- [%s] %s: %s' % (data['region'], name, ', '.join(data['instances'])) 144 | for name, data in data['profiles'].iteritems() 145 | ]), 146 | details=json.dumps(data, indent=4, sort_keys=True, default=json_serialize) 147 | ) 148 | 149 | msg = MIMEText(msg_content, 'plain') 150 | msg['Subject'] = config['subject'] 151 | msg['From'] = config.get('sender', config['user_name']) 152 | msg['To'] = ', '.join(config['recipients']) 153 | 154 | smtp = smtplib.SMTP_SSL(config['server']) 155 | smtp.login(config['user_name'], config['password']) 156 | smtp.sendmail(config['sender'], config['recipients'], msg.as_string()) 157 | smtp.close() 158 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | setup( 5 | name='pension', 6 | version='0.1.2', 7 | description='Alert when ec2 instances are scheduled for retirement', 8 | author='Erik Price', 9 | url='https://github.com/erik/pension', 10 | packages=['pension'], 11 | entry_points={ 12 | 'console_scripts': [ 13 | 'pension = pension.cli:main', 14 | ], 15 | }, 16 | license='MIT', 17 | install_requires=[ 18 | 'boto3==1.2.3', 19 | 'click==6.2', 20 | 'toml==0.9.1', 21 | 'requests==2.20.0' 22 | ] 23 | ) 24 | --------------------------------------------------------------------------------