├── .gitignore ├── .travis.yml ├── LICENSE.txt ├── MANIFEST.in ├── README.rst ├── backup_monkey ├── __init__.py ├── cli.py ├── core.py └── exceptions.py ├── bin └── backup-monkey ├── dev_requirements.txt ├── post_install.sh ├── requirements.txt ├── setup.cfg ├── setup.py └── tests ├── __init__.py └── unit ├── __init__.py ├── test_exceptions.py └── test_tags.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | 3 | # C extensions 4 | *.so 5 | 6 | # Packages 7 | *.egg 8 | *.egg-info 9 | dist 10 | build 11 | eggs 12 | parts 13 | #bin 14 | var 15 | sdist 16 | develop-eggs 17 | .installed.cfg 18 | #lib 19 | lib64 20 | 21 | # Installer logs 22 | pip-log.txt 23 | 24 | # Unit test / coverage reports 25 | .coverage 26 | .tox 27 | nosetests.xml 28 | 29 | # Translations 30 | *.mo 31 | 32 | # Mr Developer 33 | .mr.developer.cfg 34 | .project 35 | .pydevproject 36 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | install: 5 | - pip install --use-mirrors -r requirements.txt 6 | - pip install --use-mirrors -r dev_requirements.txt 7 | - python setup.py develop 8 | script: nosetests tests/unit 9 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and 10 | distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by the copyright 13 | owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other entities 16 | that control, are controlled by, or are under common control with that entity. 17 | For the purposes of this definition, "control" means (i) the power, direct or 18 | indirect, to cause the direction or management of such entity, whether by 19 | contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the 20 | outstanding shares, or (iii) beneficial ownership of such entity. 21 | 22 | "You" (or "Your") shall mean an individual or Legal Entity exercising 23 | permissions granted by this License. 24 | 25 | "Source" form shall mean the preferred form for making modifications, including 26 | but not limited to software source code, documentation source, and configuration 27 | files. 28 | 29 | "Object" form shall mean any form resulting from mechanical transformation or 30 | translation of a Source form, including but not limited to compiled object code, 31 | generated documentation, and conversions to other media types. 32 | 33 | "Work" shall mean the work of authorship, whether in Source or Object form, made 34 | available under the License, as indicated by a copyright notice that is included 35 | in or attached to the work (an example is provided in the Appendix below). 36 | 37 | "Derivative Works" shall mean any work, whether in Source or Object form, that 38 | is based on (or derived from) the Work and for which the editorial revisions, 39 | annotations, elaborations, or other modifications represent, as a whole, an 40 | original work of authorship. For the purposes of this License, Derivative Works 41 | shall not include works that remain separable from, or merely link (or bind by 42 | name) to the interfaces of, the Work and Derivative Works thereof. 43 | 44 | "Contribution" shall mean any work of authorship, including the original version 45 | of the Work and any modifications or additions to that Work or Derivative Works 46 | thereof, that is intentionally submitted to Licensor for inclusion in the Work 47 | by the copyright owner or by an individual or Legal Entity authorized to submit 48 | on behalf of the copyright owner. For the purposes of this definition, 49 | "submitted" means any form of electronic, verbal, or written communication sent 50 | to the Licensor or its representatives, including but not limited to 51 | communication on electronic mailing lists, source code control systems, and 52 | issue tracking systems that are managed by, or on behalf of, the Licensor for 53 | the purpose of discussing and improving the Work, but excluding communication 54 | that is conspicuously marked or otherwise designated in writing by the copyright 55 | owner as "Not a Contribution." 56 | 57 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf 58 | of whom a Contribution has been received by Licensor and subsequently 59 | incorporated within the Work. 60 | 61 | 2. Grant of Copyright License. 62 | 63 | Subject to the terms and conditions of this License, each Contributor hereby 64 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 65 | irrevocable copyright license to reproduce, prepare Derivative Works of, 66 | publicly display, publicly perform, sublicense, and distribute the Work and such 67 | Derivative Works in Source or Object form. 68 | 69 | 3. Grant of Patent License. 70 | 71 | Subject to the terms and conditions of this License, each Contributor hereby 72 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 73 | irrevocable (except as stated in this section) patent license to make, have 74 | made, use, offer to sell, sell, import, and otherwise transfer the Work, where 75 | such license applies only to those patent claims licensable by such Contributor 76 | that are necessarily infringed by their Contribution(s) alone or by combination 77 | of their Contribution(s) with the Work to which such Contribution(s) was 78 | submitted. If You institute patent litigation against any entity (including a 79 | cross-claim or counterclaim in a lawsuit) alleging that the Work or a 80 | Contribution incorporated within the Work constitutes direct or contributory 81 | patent infringement, then any patent licenses granted to You under this License 82 | for that Work shall terminate as of the date such litigation is filed. 83 | 84 | 4. Redistribution. 85 | 86 | You may reproduce and distribute copies of the Work or Derivative Works thereof 87 | in any medium, with or without modifications, and in Source or Object form, 88 | provided that You meet the following conditions: 89 | 90 | You must give any other recipients of the Work or Derivative Works a copy of 91 | this License; and 92 | You must cause any modified files to carry prominent notices stating that You 93 | changed the files; and 94 | You must retain, in the Source form of any Derivative Works that You distribute, 95 | all copyright, patent, trademark, and attribution notices from the Source form 96 | of the Work, excluding those notices that do not pertain to any part of the 97 | Derivative Works; and 98 | If the Work includes a "NOTICE" text file as part of its distribution, then any 99 | Derivative Works that You distribute must include a readable copy of the 100 | attribution notices contained within such NOTICE file, excluding those notices 101 | that do not pertain to any part of the Derivative Works, in at least one of the 102 | following places: within a NOTICE text file distributed as part of the 103 | Derivative Works; within the Source form or documentation, if provided along 104 | with the Derivative Works; or, within a display generated by the Derivative 105 | Works, if and wherever such third-party notices normally appear. The contents of 106 | the NOTICE file are for informational purposes only and do not modify the 107 | License. You may add Your own attribution notices within Derivative Works that 108 | You distribute, alongside or as an addendum to the NOTICE text from the Work, 109 | provided that such additional attribution notices cannot be construed as 110 | modifying the License. 111 | You may add Your own copyright statement to Your modifications and may provide 112 | additional or different license terms and conditions for use, reproduction, or 113 | distribution of Your modifications, or for any such Derivative Works as a whole, 114 | provided Your use, reproduction, and distribution of the Work otherwise complies 115 | with the conditions stated in this License. 116 | 117 | 5. Submission of Contributions. 118 | 119 | Unless You explicitly state otherwise, any Contribution intentionally submitted 120 | for inclusion in the Work by You to the Licensor shall be under the terms and 121 | conditions of this License, without any additional terms or conditions. 122 | Notwithstanding the above, nothing herein shall supersede or modify the terms of 123 | any separate license agreement you may have executed with Licensor regarding 124 | such Contributions. 125 | 126 | 6. Trademarks. 127 | 128 | This License does not grant permission to use the trade names, trademarks, 129 | service marks, or product names of the Licensor, except as required for 130 | reasonable and customary use in describing the origin of the Work and 131 | reproducing the content of the NOTICE file. 132 | 133 | 7. Disclaimer of Warranty. 134 | 135 | Unless required by applicable law or agreed to in writing, Licensor provides the 136 | Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, 137 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, 138 | including, without limitation, any warranties or conditions of TITLE, 139 | NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are 140 | solely responsible for determining the appropriateness of using or 141 | redistributing the Work and assume any risks associated with Your exercise of 142 | permissions under this License. 143 | 144 | 8. Limitation of Liability. 145 | 146 | In no event and under no legal theory, whether in tort (including negligence), 147 | contract, or otherwise, unless required by applicable law (such as deliberate 148 | and grossly negligent acts) or agreed to in writing, shall any Contributor be 149 | liable to You for damages, including any direct, indirect, special, incidental, 150 | or consequential damages of any character arising as a result of this License or 151 | out of the use or inability to use the Work (including but not limited to 152 | damages for loss of goodwill, work stoppage, computer failure or malfunction, or 153 | any and all other commercial damages or losses), even if such Contributor has 154 | been advised of the possibility of such damages. 155 | 156 | 9. Accepting Warranty or Additional Liability. 157 | 158 | While redistributing the Work or Derivative Works thereof, You may choose to 159 | offer, and charge a fee for, acceptance of support, warranty, indemnity, or 160 | other liability obligations and/or rights consistent with this License. However, 161 | in accepting such obligations, You may act only on Your own behalf and on Your 162 | sole responsibility, not on behalf of any other Contributor, and only if You 163 | agree to indemnify, defend, and hold each Contributor harmless for any liability 164 | incurred by, or claims asserted against, such Contributor by reason of your 165 | accepting any such warranty or additional liability. 166 | 167 | END OF TERMS AND CONDITIONS 168 | 169 | APPENDIX: How to apply the Apache License to your work 170 | 171 | To apply the Apache License to your work, attach the following boilerplate 172 | notice, with the fields enclosed by brackets "[]" replaced with your own 173 | identifying information. (Don't include the brackets!) The text should be 174 | enclosed in the appropriate comment syntax for the file format. We also 175 | recommend that a file or class name and description of purpose be included on 176 | the same "printed page" as the copyright notice for easier identification within 177 | third-party archives. 178 | 179 | Copyright 2013 Answers for AWS LLC 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | http://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. 192 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.txt README.rst requirements.txt dev_requirements.txt 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Backup Monkey 2 | ============= 3 | 4 | .. image:: https://travis-ci.org/Answers4AWS/backup-monkey.png?branch=master 5 | :target: https://travis-ci.org/Answers4AWS/backup-monkey 6 | :alt: Build Status 7 | 8 | A monkey that makes sure you have a backup of your EBS volumes in case something goes wrong. 9 | 10 | It is designed specifically for Amazon Web Services (AWS), and uses Python and Boto. 11 | 12 | This script is designed to be run on a schedule, probably by CRON. 13 | 14 | Usage 15 | ----- 16 | 17 | :: 18 | 19 | usage: backup-monkey [-h] [--region REGION] 20 | [--max-snapshots-per-volume SNAPSHOTS] [--snapshot-only] 21 | [--remove-only] [--verbose] [--version] 22 | [--tags TAGS [TAGS ...]] [--reverse-tags] 23 | [--label LABEL] 24 | [--cross-account-number CROSS_ACCOUNT_NUMBER] 25 | [--cross-account-role CROSS_ACCOUNT_ROLE] 26 | 27 | Loops through all EBS volumes, and snapshots them, then loops through all 28 | snapshots, and removes the oldest ones. 29 | 30 | optional arguments: 31 | -h, --help show this help message and exit 32 | --region REGION the region to loop through and snapshot (default is 33 | current region of EC2 instance this is running on). 34 | E.g. us-east-1 35 | --max-snapshots-per-volume SNAPSHOTS 36 | the maximum number of snapshots to keep per EBS 37 | volume. The oldest snapshots will be deleted. Default: 38 | 3 39 | --snapshot-only Only snapshot EBS volumes, do not remove old snapshots 40 | --remove-only Only remove old snapshots, do not create new snapshots 41 | --verbose, -v enable verbose output (-vvv for more) 42 | --version display version number and exit 43 | --tags TAGS [TAGS ...] 44 | Only snapshot instances that match passed in tags. 45 | E.g. --tag Name:foo will snapshot all instances with a 46 | tag `Name` and value is `foo` 47 | --reverse-tags Do a reverse match on the passed in tags. E.g. --tag 48 | Name:foo --reverse-tags will snapshot all instances 49 | that do not have a `Name` tag with the value `foo` 50 | --label LABEL 51 | Only snapshot instances that match passed in label 52 | are created or deleted. Default: None. Selected all 53 | snapshot. You have the posibility of create a different 54 | strategies for daily, weekly and monthly for example. 55 | Label daily won't deleted label weekly. E.g. 56 | backup-monkey --max-snapshots-per-volume 6 --label daily 57 | backup-monkey --max-snapshots-per-volume 4 --label weekly 58 | You save 6 + 4 snapshots max. instead 4 or 6 59 | --cross-account-number CROSS_ACCOUNT_NUMBER 60 | Do a cross-account snapshot (this is the account 61 | number to do snapshots on). NOTE: This requires that 62 | you pass in the --cross-account-role parameter. E.g. 63 | --cross-account-number 111111111111 --cross-account- 64 | role Snapshot 65 | --cross-account-role CROSS_ACCOUNT_ROLE 66 | The name of the role that backup-monkey will assume 67 | when doing a cross-account snapshot. E.g. --cross- 68 | account-role Snapshot 69 | 70 | Examples 71 | -------- 72 | 73 | Create snapshots of all EBS volumes in us-east-1: 74 | 75 | :: 76 | 77 | backup-monkey --region us-east-1 78 | 79 | Delete snapshots of EBS volumes in us-west-1 where a volume has more than 5 snapshots: 80 | 81 | :: 82 | 83 | backup-monkey --region us-west-1 --max-snapshots-per-volume 5 --remove-only 84 | 85 | 86 | Installation 87 | ------------ 88 | 89 | You can install Backup Monkey using the usual PyPI channels. Example: 90 | 91 | :: 92 | 93 | sudo pip install backup_monkey 94 | 95 | You can find the package details here: https://pypi.python.org/pypi/backup_monkey 96 | 97 | Alternatively, if you prefer to install from source: 98 | 99 | :: 100 | 101 | git clone git@github.com:Answers4AWS/backup-monkey.git 102 | cd backup-monkey 103 | python setup.py install 104 | 105 | 106 | Configuration 107 | ------------- 108 | 109 | This project uses `Boto `__ to 110 | call the AWS APIs. You can pass your AWS credentials to Boto can by using a 111 | :code:`.boto` file, IAM Roles or environment variables. Full information can be 112 | found here: 113 | 114 | http://boto.readthedocs.org/en/latest/boto_config_tut.html 115 | 116 | 117 | Run test 118 | -------- 119 | 120 | :: 121 | 122 | python -m unittest -v tests.unit.test_exceptions 123 | python -m unittest -v tests.unit.test_tags 124 | 125 | 126 | Warning 127 | ------- 128 | 129 | Make no mistake. This script WILL delete snapshots. This script WILL create 130 | snapshots, which can cost you money. There really are no warranties or 131 | guarantees. For costs, refer to http://aws.amazon.com/ec2/pricing/ 132 | 133 | 134 | Source Code 135 | ----------- 136 | 137 | The Python source code for Backup Monkey is available on GitHub: 138 | 139 | https://github.com/Answers4AWS/backup-monkey 140 | 141 | 142 | About Answers for AWS 143 | --------------------- 144 | 145 | This code was written by `Peter 146 | Sankauskas `__, founder of `Answers for 147 | AWS `__ - a company focused on helping businesses 148 | learn how to use AWS, without doing it the hard way. If you are looking for help 149 | with AWS, please `contact us `__. 150 | 151 | 152 | License 153 | ------- 154 | 155 | Copyright 2013 Answers for AWS LLC 156 | 157 | Licensed under the Apache License, Version 2.0 (the "License"); you may 158 | not use this file except in compliance with the License. You may obtain 159 | a copy of the License at 160 | 161 | http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable 162 | law or agreed to in writing, software distributed under the License is 163 | distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 164 | KIND, either express or implied. See the License for the specific 165 | language governing permissions and limitations under the License. 166 | 167 | -------------------------------------------------------------------------------- /backup_monkey/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Answers for AWS LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | __author__ = 'Peter Sankauskas' 16 | __version__ = '1.0.0' 17 | -------------------------------------------------------------------------------- /backup_monkey/cli.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Answers for AWS LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import argparse 16 | import logging 17 | import sys 18 | 19 | from backup_monkey.core import BackupMonkey, Logging 20 | from backup_monkey import __version__ 21 | from backup_monkey.exceptions import BackupMonkeyException 22 | 23 | from boto.utils import get_instance_metadata 24 | 25 | __all__ = ('run', ) 26 | log = logging.getLogger(__name__) 27 | LIMIT_LABEL = 32 # Label is added to description when created snapshot. 28 | # The description limit in aws is 255 29 | 30 | def _fail(message="Unknown failure", code=1): 31 | log.error(message) 32 | sys.exit(code) 33 | 34 | def run(): 35 | parser = argparse.ArgumentParser(description='Loops through all EBS volumes, and snapshots them, then loops through all snapshots, and removes the oldest ones.') 36 | parser.add_argument('--region', metavar='REGION', 37 | help='the region to loop through and snapshot (default is current region of EC2 instance this is running on). E.g. us-east-1') 38 | parser.add_argument('--max-snapshots-per-volume', metavar='SNAPSHOTS', default=3, type=int, 39 | help='the maximum number of snapshots to keep per EBS volume. The oldest snapshots will be deleted. Default: 3') 40 | parser.add_argument('--snapshot-only', action='store_true', default=False, 41 | help='Only snapshot EBS volumes, do not remove old snapshots') 42 | parser.add_argument('--remove-only', action='store_true', default=False, 43 | help='Only remove old snapshots, do not create new snapshots') 44 | parser.add_argument('--verbose', '-v', action='count', 45 | help='enable verbose output (-vvv for more)') 46 | parser.add_argument('--version', action='version', version='%(prog)s ' + __version__, 47 | help='display version number and exit') 48 | parser.add_argument('--tags', nargs="+", 49 | help='Only snapshot instances that match passed in tags. E.g. --tag Name:foo will snapshot all instances with a tag `Name` and value is `foo`') 50 | parser.add_argument('--reverse-tags', action='store_true', default=False, 51 | help='Do a reverse match on the passed in tags. E.g. --tag Name:foo --reverse-tags will snapshot all instances that do not have a `Name` tag with the value `foo`') 52 | parser.add_argument('--label', action='store', 53 | help='Only snapshot instances that match passed in label are created or deleted. Default: None. Selected all snapshot. You have the posibility of create a different strategies for daily, weekly and monthly for example. Label daily won\'t deleted label weekly') 54 | parser.add_argument('--cross-account-number', action='store', 55 | help='Do a cross-account snapshot (this is the account number to do snapshots on). NOTE: This requires that you pass in the --cross-account-role parameter. E.g. --cross-account-number 111111111111 --cross-account-role Snapshot') 56 | parser.add_argument('--cross-account-role', action='store', 57 | help='The name of the role that backup-monkey will assume when doing a cross-account snapshot. E.g. --cross-account-role Snapshot') 58 | 59 | args = parser.parse_args() 60 | 61 | if args.cross_account_number and not args.cross_account_role: 62 | parser.error('The --cross-account-role parameter is required if you specify --cross-account-number (doing a cross-account snapshot)') 63 | 64 | if args.cross_account_role and not args.cross_account_number: 65 | parser.error('The --cross-account-number parameter is required if you specify --cross-account-role (doing a cross-account snapshot)') 66 | 67 | if args.reverse_tags and not args.tags: 68 | parser.error('The --tags parameter is required if you specify --reverse-tags (doing a blacklist filter)') 69 | 70 | if args.label and len(args.label) > LIMIT_LABEL: 71 | parser.error('The --label parameter lenght should be less than 32') 72 | 73 | Logging().configure(args.verbose) 74 | 75 | log.debug("CLI parse args: %s", args) 76 | 77 | if args.region: 78 | region = args.region 79 | else: 80 | # If no region was specified, assume this is running on an EC2 instance 81 | # and work out what region it is in 82 | log.debug("Figure out which region I am running in...") 83 | instance_metadata = get_instance_metadata(timeout=5) 84 | log.debug('Instance meta-data: %s', instance_metadata) 85 | if not instance_metadata: 86 | _fail('Could not determine region. This script is either not running on an EC2 instance (in which case you should use the --region option), or the meta-data service is down') 87 | 88 | region = instance_metadata['placement']['availability-zone'][:-1] 89 | log.debug("Running in region: %s", region) 90 | 91 | try: 92 | monkey = BackupMonkey(region, 93 | args.max_snapshots_per_volume, 94 | args.tags, 95 | args.reverse_tags, 96 | args.label, 97 | args.cross_account_number, 98 | args.cross_account_role) 99 | 100 | if not args.remove_only: 101 | monkey.snapshot_volumes() 102 | if not args.snapshot_only: 103 | monkey.remove_old_snapshots() 104 | 105 | except BackupMonkeyException as e: 106 | _fail(e.message) 107 | 108 | log.info('Backup Monkey completed successfully!') 109 | sys.exit(0) 110 | -------------------------------------------------------------------------------- /backup_monkey/core.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Answers for AWS LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | import logging 15 | 16 | from boto.exception import NoAuthHandlerFound 17 | from boto import ec2 18 | 19 | from backup_monkey.exceptions import BackupMonkeyException 20 | 21 | __all__ = ('BackupMonkey', 'Logging') 22 | log = logging.getLogger(__name__) 23 | 24 | class BackupMonkey(object): 25 | def __init__(self, region, max_snapshots_per_volume, tags, reverse_tags, label, cross_account_number, cross_account_role): 26 | self._region = region 27 | self._prefix = 'BACKUP_MONKEY' 28 | if label: 29 | self._prefix += ' ' + label 30 | self._snapshots_per_volume = max_snapshots_per_volume 31 | self._tags = tags 32 | self._reverse_tags = reverse_tags 33 | self._cross_account_number = cross_account_number 34 | self._cross_account_role = cross_account_role 35 | self._conn = self.get_connection() 36 | 37 | def get_connection(self): 38 | ret = None 39 | if self._cross_account_number and self._cross_account_role: 40 | from boto.sts import STSConnection 41 | import boto 42 | try: 43 | role_arn = 'arn:aws:iam::%s:role/%s' % (self._cross_account_number, self._cross_account_role) 44 | sts = STSConnection() 45 | assumed_role = sts.assume_role(role_arn=role_arn, role_session_name='AssumeRoleSession') 46 | ret = ec2.connect_to_region( 47 | self._region, 48 | aws_access_key_id=assumed_role.credentials.access_key, 49 | aws_secret_access_key=assumed_role.credentials.secret_key, 50 | security_token=assumed_role.credentials.session_token 51 | ) 52 | except Exception,e: 53 | print e 54 | raise BackupMonkeyException('Cannot complete cross account access') 55 | else: 56 | log.info("Connecting to region %s", self._region) 57 | try: 58 | ret = ec2.connect_to_region(self._region) 59 | except NoAuthHandlerFound: 60 | log.error('Could not connect to region %s' % self._region) 61 | log.critical('No AWS credentials found. To configure Boto, please read: http://boto.readthedocs.org/en/latest/boto_config_tut.html') 62 | raise BackupMonkeyException('No AWS credentials found') 63 | if not ret: 64 | raise BackupMonkeyException('Could not connect to region `%s`. Check to make sure you are connecting to a valid region' % self._region) 65 | return ret 66 | 67 | def get_filters(self): 68 | filters = dict([t.split(':') for t in self._tags]) 69 | try: 70 | for f in filters.keys(): 71 | try: 72 | filters[f] = eval(filters[f]) 73 | except Exception: 74 | pass 75 | except ValueError: 76 | log.error('Invalid tag parameter') 77 | raise BackupMonkeyException('Invalid tag parameter') 78 | if not self._reverse_tags: 79 | for f in filters.keys(): 80 | filters['tag:%s' % f] = filters.pop(f) 81 | return filters 82 | 83 | def get_volumes_to_snapshot(self): 84 | volumes = [] 85 | if self._reverse_tags: 86 | filters = self.get_filters() 87 | black_list = [] 88 | for f in filters.keys(): 89 | if isinstance(filters[f], list): 90 | black_list = black_list + [(f, i) for i in filters[f]] 91 | else: 92 | black_list.append((f, filters[f])) 93 | for v in self._conn.get_all_volumes(): 94 | if len(set(v.tags.items()) - set(black_list)) == len(set(v.tags.items())): 95 | volumes.append(v) 96 | else: 97 | if self._tags: 98 | return self._conn.get_all_volumes(filters=self.get_filters()) 99 | else: 100 | volumes = self._conn.get_all_volumes() 101 | return volumes 102 | 103 | def snapshot_volumes(self): 104 | ''' Loops through all EBS volumes and creates snapshots of them ''' 105 | 106 | log.info('Getting list of EBS volumes') 107 | volumes = self.get_volumes_to_snapshot() 108 | log.info('Found %d volumes', len(volumes)) 109 | for volume in volumes: 110 | description_parts = [self._prefix] 111 | description_parts.append(volume.id) 112 | if volume.attach_data.instance_id: 113 | description_parts.append(volume.attach_data.instance_id) 114 | if volume.attach_data.device: 115 | description_parts.append(volume.attach_data.device) 116 | description = ' '.join(description_parts) 117 | log.info('Creating snapshot of %s: %s', volume.id, description) 118 | volume.create_snapshot(description) 119 | return True 120 | 121 | 122 | def remove_old_snapshots(self): 123 | ''' Loop through this account's snapshots, and remove the oldest ones 124 | where there are more snapshots per volume than required ''' 125 | 126 | log.info('Configured to keep %d snapshots per volume', self._snapshots_per_volume) 127 | log.info('Getting list of EBS snapshots') 128 | snapshots = self._conn.get_all_snapshots(owner='self') 129 | log.info('Found %d snapshots', len(snapshots)) 130 | vol_snap_map = {} 131 | for snapshot in snapshots: 132 | if not snapshot.description.startswith(self._prefix): 133 | log.debug('Skipping %s as prefix does not match', snapshot.id) 134 | continue 135 | if not snapshot.status == 'completed': 136 | log.debug('Skipping %s as it is not a complete snapshot', snapshot.id) 137 | continue 138 | 139 | log.debug('Found %s: %s', snapshot.id, snapshot.description) 140 | vol_snap_map.setdefault(snapshot.volume_id, []).append(snapshot) 141 | 142 | for volume_id, most_recent_snapshots in vol_snap_map.iteritems(): 143 | most_recent_snapshots.sort(key=lambda s: s.start_time, reverse=True) 144 | num_snapshots = len(most_recent_snapshots) 145 | log.info('Found %d snapshots for %s', num_snapshots, volume_id) 146 | 147 | for i in range(self._snapshots_per_volume, num_snapshots): 148 | snapshot = most_recent_snapshots[i] 149 | log.info(' Deleting %s: %s', snapshot.id, snapshot.description) 150 | snapshot.delete() 151 | return True 152 | 153 | 154 | 155 | class Logging(object): 156 | # Logging formats 157 | _log_simple_format = '%(asctime)s [%(levelname)s] %(message)s' 158 | _log_detailed_format = '%(asctime)s [%(levelname)s] [%(name)s(%(lineno)s):%(funcName)s] %(message)s' 159 | 160 | def configure(self, verbosity = None): 161 | ''' Configure the logging format and verbosity ''' 162 | 163 | # Configure our logging output 164 | if verbosity >= 2: 165 | logging.basicConfig(level=logging.DEBUG, format=self._log_detailed_format, datefmt='%F %T') 166 | elif verbosity >= 1: 167 | logging.basicConfig(level=logging.INFO, format=self._log_detailed_format, datefmt='%F %T') 168 | else: 169 | logging.basicConfig(level=logging.INFO, format=self._log_simple_format, datefmt='%F %T') 170 | 171 | # Configure Boto's logging output 172 | if verbosity >= 4: 173 | logging.getLogger('boto').setLevel(logging.DEBUG) 174 | elif verbosity >= 3: 175 | logging.getLogger('boto').setLevel(logging.INFO) 176 | else: 177 | logging.getLogger('boto').setLevel(logging.CRITICAL) 178 | 179 | -------------------------------------------------------------------------------- /backup_monkey/exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Answers for AWS LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | class BackupMonkeyException(Exception): 16 | ''' Base Backup Monkey Exception ''' 17 | pass 18 | -------------------------------------------------------------------------------- /bin/backup-monkey: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright 2013 Answers for AWS LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """ 17 | Backup Monkey 18 | ============= 19 | This script runs the Backup Monkey service 20 | """ 21 | 22 | import sys, os 23 | sys.path.insert(0, os.path.abspath('..')) 24 | 25 | from backup_monkey import cli 26 | 27 | if __name__ == '__main__': 28 | cli.run() 29 | -------------------------------------------------------------------------------- /dev_requirements.txt: -------------------------------------------------------------------------------- 1 | nose==1.3.7 2 | mock==1.3.0 3 | -------------------------------------------------------------------------------- /post_install.sh: -------------------------------------------------------------------------------- 1 | curl https://bootstrap.pypa.io/get-pip.py | python 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | boto==2.38.0 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_rpm] 2 | build_requires = rpm-build 3 | requires = python-boto = 2.38.0 4 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright 2013 Answers for AWS LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """ 17 | setuptools install script for Backup Monkey 18 | """ 19 | 20 | import sys 21 | major, minor = sys.version_info[0:2] 22 | if major != 2 or minor < 6: 23 | print 'Backup Monkey requires Python 2.6.x' 24 | sys.exit(1) 25 | 26 | from setuptools import setup, find_packages 27 | 28 | import backup_monkey 29 | 30 | with open('requirements.txt') as fh: 31 | requires = [requirement.strip() for requirement in fh] 32 | 33 | entry_points = { 34 | 'console_scripts': [ 35 | 'backup-monkey = backup_monkey.cli:run', 36 | ] 37 | } 38 | 39 | exclude_packages = [ 40 | 'tests', 41 | 'tests.*', 42 | ] 43 | 44 | def readme(): 45 | with open("README.rst") as f: 46 | return f.read() 47 | 48 | setup( 49 | name='backup_monkey', 50 | version=backup_monkey.__version__, 51 | description='A service that makes sure you have snapshots of all your EBS volumes', 52 | long_description = readme(), 53 | author=backup_monkey.__author__, 54 | author_email='info@answersforaws.com', 55 | url='https://github.com/Answers4AWS/backup-monkey', 56 | packages=find_packages(exclude=exclude_packages), 57 | package_dir={'backup_monkey': 'backup_monkey'}, 58 | include_package_data=True, 59 | zip_safe=False, 60 | install_requires=requires, 61 | entry_points=entry_points, 62 | license='Apache Software License', 63 | classifiers=( 64 | 'Development Status :: 4 - Beta', 65 | 'Environment :: Console', 66 | 'Intended Audience :: Developers', 67 | 'Intended Audience :: Information Technology', 68 | 'Intended Audience :: System Administrators', 69 | 'License :: OSI Approved :: Apache Software License', 70 | 'Natural Language :: English', 71 | 'Programming Language :: Python :: 2.6', 72 | 'Programming Language :: Python :: 2.7', 73 | 'Topic :: System :: Installation/Setup', 74 | 'Topic :: Utilities', 75 | ), 76 | options = { 77 | 'bdist_rpm': { 78 | 'post_install' : 'post_install.sh', 79 | } 80 | } 81 | ) 82 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Answers for AWS LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Answers for AWS LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /tests/unit/test_exceptions.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013 Answers for AWS LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from unittest import TestCase 16 | 17 | from backup_monkey.exceptions import * 18 | 19 | class ExceptionTests(TestCase): 20 | def test_new_exception(self): 21 | e = BackupMonkeyException() 22 | self.assertTrue(isinstance(e, BackupMonkeyException)) 23 | #self.assertIsInstance(e, BackupMonkeyException) 24 | 25 | def raise_BackupMonkeyException(self): 26 | raise BackupMonkeyException('msg') 27 | 28 | def test_raise_BackupMonkeyException(self): 29 | self.assertRaises(BackupMonkeyException, self.raise_BackupMonkeyException) 30 | -------------------------------------------------------------------------------- /tests/unit/test_tags.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | import mock 3 | from backup_monkey.core import BackupMonkey 4 | 5 | class MockVolume(object): 6 | def __init__(self, id, tags={}): 7 | self.id = id 8 | self.tags = tags 9 | 10 | class MockSnapshot(object): 11 | def __init__(self, id, description, start_time): 12 | self.id = id 13 | self.volume_id = 'vol-fb07ec3a' 14 | self.description = description 15 | self.start_time = start_time 16 | self.status = 'completed' 17 | def delete(self): 18 | self.status = 'deleted' 19 | 20 | tag = ['name:foo'] 21 | tag_or = ["name:['bar','baz']"] 22 | tag_multiple = ['name:foo', 'customer:bar'] 23 | 24 | def params_to_dict(params): 25 | ret = [] 26 | for p in params: 27 | v = p.split(':') 28 | try: 29 | v[1] = eval(v[1]) 30 | except Exception: 31 | pass 32 | if isinstance(v[1], list): 33 | for x in v[1]: 34 | ret = ret + [v[0], x] 35 | else: 36 | ret = ret + v 37 | return dict(zip(ret[0::2], ret[1::2])) 38 | 39 | a = MockVolume('vol-fb07ec3a') 40 | b = MockVolume('vol-089322fc') 41 | # matches ['name:foo'] 42 | match_tag = MockVolume('vol-d815a12c', params_to_dict(tag)) 43 | # matches ['name:bar'] 44 | match_tag_or_1 = MockVolume('vol-c2922336', dict([(tag_or[0].split(':')[0], eval(tag_or[0].split(':')[1])[0])])) 45 | # matches ['name:baz'] 46 | match_tag_or_2 = MockVolume('vol-99bf3b6d', dict([(tag_or[0].split(':')[0], eval(tag_or[0].split(':')[1])[1])])) 47 | # matches ['name:foo', 'customer:bar'] 48 | match_tag_multiple = MockVolume('vol-d59b7114', params_to_dict(tag_multiple)) 49 | 50 | volumes = [a, match_tag, match_tag_or_1, b, match_tag_or_2, match_tag_multiple] 51 | 52 | def create_snapshots(): 53 | snap_manual = MockSnapshot('snap-1a2b3c4d', 'no backup monkey', '2016-01-01T10:00:00.000Z') 54 | snap_daily = MockSnapshot('snap-2a2b3c4d', 'BACKUP_MONKEY daily vol-fb07ec3a', '2016-01-01T11:00:00.000Z') 55 | snap_weekly = MockSnapshot('snap-3a2b3c4d', 'BACKUP_MONKEY weekly vol-fb07ec3a', '2016-01-01T12:00:00.000Z') 56 | snap_weekly2 = MockSnapshot('snap-4a2b3c4d', 'BACKUP_MONKEY weekly vol-fb07ec3a', '2016-01-01T13:00:00.000Z') 57 | return [snap_manual, snap_daily, snap_weekly, snap_weekly2] 58 | 59 | 60 | class MockEC2Connection(object): 61 | def __init__(self): 62 | self.snapshots = create_snapshots() 63 | 64 | def get_all_volumes(self, filters=[]): 65 | if filters: 66 | if isinstance(filters['tag:name'], list): 67 | return [v for v in volumes if 'name' in v.tags and v.tags['name'] in filters['tag:name']] 68 | else: 69 | return [v for v in volumes if 'name' in v.tags and v.tags['name'] == filters['tag:name']] 70 | return volumes 71 | 72 | def get_all_snapshots(self, owner='self'): 73 | return self.snapshots 74 | 75 | def mock_get_connection(): 76 | return MockEC2Connection() 77 | 78 | class TagsTest(TestCase): 79 | 80 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 81 | def setUp(self, mock): 82 | self.backup_monkey = BackupMonkey('us-west-2', 3, [], None, None, None, None) 83 | 84 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 85 | def test_instance(self, mock): 86 | assert isinstance(self.backup_monkey.get_connection(), MockEC2Connection) == True 87 | assert self.backup_monkey.get_volumes_to_snapshot() == volumes 88 | 89 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 90 | def test_no_tag(self, mock): 91 | ret = self.backup_monkey.get_volumes_to_snapshot() 92 | assert len(ret) == 6 93 | assert ret == volumes 94 | 95 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 96 | def test_tag(self, mock): 97 | self.backup_monkey._tags = tag 98 | ret = self.backup_monkey.get_volumes_to_snapshot() 99 | assert len(ret) == 2 100 | assert ret == [match_tag, match_tag_multiple] 101 | 102 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 103 | def test_tag_or(self, mock): 104 | self.backup_monkey._tags = tag_or 105 | ret = self.backup_monkey.get_volumes_to_snapshot() 106 | assert len(ret) == 2 107 | assert ret == [match_tag_or_1, match_tag_or_2] 108 | 109 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 110 | def test_tag_multiple(self, mock): 111 | self.backup_monkey._tags = tag_multiple 112 | ret = self.backup_monkey.get_volumes_to_snapshot() 113 | assert len(ret) == 2 114 | assert ret == [match_tag, match_tag_multiple] 115 | 116 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 117 | def test_tag_reverse_tags(self, mock): 118 | self.backup_monkey._tags = tag 119 | self.backup_monkey._reverse_tags = True 120 | ret = self.backup_monkey.get_volumes_to_snapshot() 121 | assert len(ret) == 4 122 | assert ret == [a, match_tag_or_1, b, match_tag_or_2] 123 | 124 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 125 | def test_tag_or_reverse_tags(self, mock): 126 | self.backup_monkey._tags = tag_or 127 | self.backup_monkey._reverse_tags = True 128 | ret = self.backup_monkey.get_volumes_to_snapshot() 129 | assert len(ret) == 4 130 | assert ret == [a, match_tag, b, match_tag_multiple] 131 | 132 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 133 | def test_tag_multiple_reverse_tags(self, mock): 134 | self.backup_monkey._tags = tag_multiple 135 | self.backup_monkey._reverse_tags = True 136 | ret = self.backup_monkey.get_volumes_to_snapshot() 137 | assert len(ret) == 4 138 | assert ret == [a, match_tag_or_1, b, match_tag_or_2] 139 | 140 | class LabelTest(TestCase): 141 | 142 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 143 | def setUp(self, mock): 144 | self.bm = BackupMonkey('us-west-2', 1, [], None, None, None, None) 145 | self.bm_with_label_daily = BackupMonkey('us-west-2', 1, [], None, 'daily', None, None) 146 | self.bm_with_label_weekly = BackupMonkey('us-west-2', 1, [], None, 'weekly', None, None) 147 | 148 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 149 | def test_without_label(self, mock): 150 | snaps = self.bm._conn.get_all_snapshots() 151 | init_snapshot = filter(lambda x: x.status == 'completed', snaps) 152 | self.bm.remove_old_snapshots() 153 | snaps = self.bm._conn.get_all_snapshots() 154 | end_snapshot = filter(lambda x: x.status == 'completed', snaps) 155 | assert len(init_snapshot) == 4 156 | assert len(end_snapshot) == 2 157 | 158 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 159 | def test_with_label_daily(self, mock): 160 | snaps = self.bm_with_label_daily._conn.get_all_snapshots() 161 | init_snapshot = filter(lambda x: x.status == 'completed', snaps) 162 | self.bm_with_label_daily.remove_old_snapshots() 163 | snaps = self.bm_with_label_daily._conn.get_all_snapshots() 164 | end_snapshot = filter(lambda x: x.status == 'completed', snaps) 165 | assert len(init_snapshot) == 4 166 | assert len(end_snapshot) == 4 167 | 168 | @mock.patch('backup_monkey.core.BackupMonkey.get_connection', side_effect=mock_get_connection) 169 | def test_with_label_weekly(self, mock): 170 | snaps = self.bm_with_label_weekly._conn.get_all_snapshots() 171 | init_snapshot = filter(lambda x: x.status == 'completed', snaps) 172 | self.bm_with_label_weekly.remove_old_snapshots() 173 | snaps = self.bm_with_label_weekly._conn.get_all_snapshots() 174 | end_snapshot = filter(lambda x: x.status == 'completed', snaps) 175 | assert len(init_snapshot) == 4 176 | assert len(end_snapshot) == 3 177 | --------------------------------------------------------------------------------