├── __init__.py
├── requirements.txt
├── ut.py
├── .gitignore
├── manifests
    ├── secrets.yaml
    └── job.yaml
├── .travis.yml
├── CHANGELOG.md
├── Dockerfile
├── License
├── README.md
└── ebs_snapshooter.py


/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | boto
2 | 


--------------------------------------------------------------------------------
/ut.py:
--------------------------------------------------------------------------------
 1 | import unittest
 2 | 
 3 | class TestDateCompare(unittest.TestCase):
 4 | 
 5 |     def test_date_compare(self):
 6 |         self.assertEqual(1, 1)
 7 | 
 8 | 
 9 | if __name__ == '__main__':
10 |     unittest.main()


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Distribution / packaging
 2 | .Python
 3 | env/
 4 | build/
 5 | develop-eggs/
 6 | dist/
 7 | downloads/
 8 | eggs/
 9 | .eggs/
10 | lib/
11 | lib64/
12 | parts/
13 | sdist/
14 | var/
15 | *.egg-info/
16 | .installed.cfg
17 | *.egg
18 | 
19 | 


--------------------------------------------------------------------------------
/manifests/secrets.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | type: Opaque
3 | kind: Secret
4 | metadata:
5 |   name: ebs-snapshooter-seretes
6 | data:
7 |   aws-acces-key-id: <base64 encoded aws-access-key-id>
8 |   aws-secret-acces-key: <base64 encoded aws-secret-acces-key>
9 |   aws-sns-arn: <base64 encoded aws-sns-arn>


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: python
 2 | python:
 3 |   - "2.6"
 4 |   - "2.7"
 5 |   - "3.2"
 6 |   - "3.3"
 7 |   - "3.4"
 8 |   - "3.5"
 9 |   - "3.5-dev" # 3.5 development branch
10 |   - "3.6-dev" # 3.6 development branch
11 |   - "nightly" # currently points to 3.7-dev
12 | # command to install dependencies
13 | install: "pip install -r requirements.txt"
14 | # command to run tests
15 | script: python ut.py


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # Change Log
2 | 
3 | ## [1.0.0](https://github.com/smileisak/ebs-snapshooter/tree/1.0.0) (2016-12-09)
4 | [Full Changelog](https://github.com/smileisak/ebs-snapshooter/compare/1.0.1...1.0.0)
5 | 
6 | ## [1.0.1](https://github.com/smileisak/ebs-snapshooter/tree/1.0.1) (2016-12-09)
7 | 
8 | 
9 | \* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | # Note that ebs_snapshooter depends on aws creadentials that should be provided via ENV VARS
 2 | # Basing the Requirements Image 
 3 | FROM python:2.7-alpine
 4 | 
 5 | # Creating the code directory
 6 | RUN mkdir /code
 7 | WORKDIR /code
 8 | 
 9 | # Coping ebs_snapshooter and requirements to code dir
10 | COPY ebs_snapshooter.py /code
11 | COPY requirements.txt /code
12 | 
13 | # Install requirements
14 | RUN pip install -r requirements.txt
15 | 
16 | # RUN ebs_snapshooter as an ENTRYPOINT in docker container
17 | ENTRYPOINT ["python", "ebs_snapshooter.py"]


--------------------------------------------------------------------------------
/manifests/job.yaml:
--------------------------------------------------------------------------------
 1 | apiVersion: batch/v1
 2 | kind: Job
 3 | metadata:
 4 |   name: ebs-snapshots-job
 5 | spec:
 6 |   template:
 7 |     metadata:
 8 |       name: ebs-snapshots-job
 9 |     spec:
10 |       containers:
11 |       - name: ebs-snapshots-job
12 |         image: quay.io/smile/ebs-snapshooter
13 |         env:
14 |           - name: AWS_ACCESS_KEY
15 |             valueFrom:
16 |               secretKeyRef:
17 |                 name: ebs-snapshooter-seretes
18 |                 key: aws-acces-key-id
19 |           - name: AWS_SECRET_KEY
20 |             valueFrom:
21 |               secretKeyRef:
22 |                 name: ebs-snapshooter-seretes
23 |                 key: aws-secret-acces-key
24 |           - name: AWS_REGION_NAME
25 |             value: "eu-west-1"
26 |           - name: AWS_SNS_ARN
27 |             valueFrom:
28 |               secretKeyRef:
29 |                 name: ebs-snapshooter-seretes
30 |                 key: aws-sns-arn
31 |           - name: KEEP_WEEK
32 |             value: "2"
33 |           - name: KEEP_DAY
34 |             value: "3"
35 |           - name: KEEP_MONTH
36 |             value: "1"
37 |       restartPolicy: Never
38 | 


--------------------------------------------------------------------------------
/License:
--------------------------------------------------------------------------------
 1 | BSD 2-Clause License
 2 | 
 3 | Copyright (c) [2016], [Ismail KABOUBI]
 4 | All rights reserved.
 5 | 
 6 | Redistribution and use in source and binary forms, with or without
 7 | modification, are permitted provided that the following conditions are met:
 8 | 
 9 | * Redistributions of source code must retain the above copyright notice, this
10 |   list of conditions and the following disclaimer.
11 | 
12 | * Redistributions in binary form must reproduce the above copyright notice,
13 |   this list of conditions and the following disclaimer in the documentation
14 |   and/or other materials provided with the distribution.
15 | 
16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
20 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | 
 2 | # EBS-SnapShooter [![Build Status](https://travis-ci.org/smileisak/ebs-snapshooter.svg?branch=master)](https://travis-ci.org/smileisak/ebs-snapshooter) [![Docker Repository on Quay](https://quay.io/repository/smile/ebs-snapshooter/status "Docker Repository on Quay")](https://quay.io/repository/smile/ebs-snapshooter)
 3 | 
 4 | 
 5 | 
 6 | EBS-SnapShooter is a python script based on boto2, that creates daily, weekly or monthly snapshots for all your aws ebs volumes.
 7 | 
 8 | ### Requirements:
 9 | 
10 | * boto2 - Python package that provides interfaces to Amazon Web Services
11 | 
12 | To install requirements :
13 | ```
14 | (env)$ pip install -r requirements.txt
15 | ```
16 | 
17 | And of course EBS-Snapshooter itself is open source on GitHub.
18 | 
19 | ### How to run
20 | * Edit ebs-snpshooter.py file with your aws api creadentials 
21 | 
22 | * And just run the script ebs-snpshooter.py
23 | 
24 | ```
25 | export AWS_ACCESS_KEY=<aws-access-key> -e AWS_SECRET_KEY=<aws-secret-key> & python ebs-snapshooter.py
26 | ```
27 | 
28 | You can also run it within docker container. Go to --> https://quay.io/repository/smile/ebs-snapshooter
29 | 
30 | ---
31 | ### Run it as Kubernetes Job:
32 | 
33 | To run it as a Kubernetes Job all you need is to base64 your secrets within `manifests/secrets.yaml`.
34 |  Note that `aws-sns-arn` is optional if you want to create AWS SNS Notifications.
35 | 
36 | ```yml
37 | apiVersion: v1
38 | type: Opaque
39 | kind: Secret
40 | metadata:
41 |   name: ebs-snapshooter-seretes
42 | data:
43 |   aws-acces-key-id: <base64 encoded aws-access-key-id>
44 |   aws-secret-acces-key: <base64 encoded aws-secret-acces-key>
45 |   aws-sns-arn: <base64 encoded aws-sns-arn>
46 | ```
47 | 
48 | Create secrets and job: 
49 | 
50 | ```shell
51 | kubectl create -f manifests/secrets.yaml
52 | kubectl create -f manifests/job.yaml
53 | ```
54 | 
55 | ## License
56 | 
57 | EBS-SnapShooter is BSD-licensed
58 | 


--------------------------------------------------------------------------------
/ebs_snapshooter.py:
--------------------------------------------------------------------------------
  1 | from boto.ec2.connection import EC2Connection
  2 | from boto.ec2.regioninfo import RegionInfo
  3 | import boto.sns
  4 | from datetime import datetime
  5 | import time
  6 | import sys
  7 | import os
  8 | import logging
  9 | # Message to return result via SNS
 10 | from boto.exception import EC2ResponseError
 11 | 
 12 | 
 13 | def get_k8s_env_var(var_name, default_value=None):
 14 |     result = os.environ.get(var_name, default_value)
 15 |     if result is None:
 16 |         return None
 17 |     else:
 18 |         result = result.replace('\n', '')
 19 | 
 20 |     return result
 21 | 
 22 | def get_config(name, default=None):
 23 |     value = get_k8s_env_var(name)
 24 |     if value is not None:
 25 |         print "[INFO.get_config] - %s : %s " % (name, value)
 26 |         return value
 27 |     else:
 28 |         print "[INFO.get_config] - %s : %s " % (name, default)
 29 |         return default
 30 | 
 31 | 
 32 | message = ""
 33 | errmsg = ""
 34 | 
 35 | # Counters
 36 | total_creates = 0
 37 | total_deletes = 0
 38 | count_errors = 0
 39 | 
 40 | # List with snapshots to delete
 41 | deletelist = []
 42 | 
 43 | # define period
 44 | period = get_config("PERIOD", "day")
 45 | date_suffix = datetime.today().strftime('%a')
 46 | 
 47 | # period = 'week'
 48 | # date_suffix = datetime.today().strftime('%U')
 49 | 
 50 | # period = 'month'
 51 | # date_suffix = datetime.today().strftime('%b')
 52 | 
 53 | # Setup logging
 54 | log_file = get_config('LOG_FILE', "./EBS-Snapshot.log")
 55 | logging.basicConfig(filename="./EBS-Snapshot.log", level=logging.INFO)
 56 | start_message = 'Started taking %(period)s snapshots at %(date)s' % {
 57 |     'period': period,
 58 |     'date': datetime.today().strftime('%d-%m-%Y %H:%M:%S')
 59 | }
 60 | message += start_message + "\n\n"
 61 | logging.info(start_message)
 62 | 
 63 | # Get settings from config.py
 64 | aws_access_key = get_config('AWS_ACCESS_KEY')
 65 | aws_secret_key = get_config('AWS_SECRET_KEY')
 66 | ec2_region_name = get_config('AWS_REGION_NAME', "eu-west-1")
 67 | ec2_region_endpoint = "ec2." + ec2_region_name + ".amazonaws.com"
 68 | sns_arn =  get_config('AWS_SNS_ARN')
 69 | 
 70 | region = RegionInfo(name=ec2_region_name, endpoint=ec2_region_endpoint)
 71 | 
 72 | # Number of snapshots to keep
 73 | keep_week = int(get_config('KEEP_WEEK', 3))
 74 | keep_day = int(get_config('KEEP_DAY', 3))
 75 | keep_month = int(get_config('KEEP_MONTH', 1))
 76 | count_success = 0
 77 | count_total = 0
 78 | 
 79 | # Connect to AWS using the credentials provided above or in Environment vars or using IAM role.
 80 | print 'Connecting to AWS'
 81 | if aws_access_key:
 82 |     conn = EC2Connection(aws_access_key, aws_secret_key, region=region)
 83 | else:
 84 |     conn = EC2Connection(region=region)
 85 | 
 86 | if sns_arn:
 87 |     print 'Connecting to SNS'
 88 |     if aws_access_key:
 89 |         sns = boto.sns.connect_to_region(ec2_region_name, aws_access_key_id=aws_access_key,
 90 |                                          aws_secret_access_key=aws_secret_key)
 91 |     else:
 92 |         sns = boto.sns.connect_to_region(ec2_region_name)
 93 | 
 94 | 
 95 | def get_resource_tags(resource_id):
 96 |     resource_tags = {}
 97 |     if resource_id:
 98 |         tags = conn.get_all_tags({'resource-id': resource_id})
 99 |         for tag in tags:
100 |             # Tags starting with 'aws:' are reserved for internal use
101 |             if not tag.name.startswith('aws:'):
102 |                 resource_tags[tag.name] = tag.value
103 |     return resource_tags
104 | 
105 | 
106 | def set_resource_tags(resource, tags):
107 |     for tag_key, tag_value in tags.iteritems():
108 |         if tag_key not in resource.tags or resource.tags[tag_key] != tag_value:
109 |             print 'Tagging %(resource_id)s with [%(tag_key)s: %(tag_value)s]' % {
110 |                 'resource_id': resource.id,
111 |                 'tag_key': tag_key,
112 |                 'tag_value': tag_value
113 |             }
114 |             resource.add_tag(tag_key, tag_value)
115 | 
116 | 
117 | def date_compare(snap1, snap2):
118 |     if snap1.start_time < snap2.start_time:
119 |         return -1
120 |     elif snap1.start_time == snap2.start_time:
121 |         return 0
122 |     return 1
123 | 
124 | 
125 | print 'Finding all volumes ...'
126 | vols = conn.get_all_volumes()
127 | 
128 | ids_to_snapshots = {}
129 | for vol in vols:
130 |     ids_to_snapshots.setdefault(vol.id, []).append(vol.snapshots())
131 | 
132 | 
133 | for vol in vols:
134 |     try:
135 |         count_total += 1
136 |         logging.info(vol)
137 |         tags_volume = get_resource_tags(vol.id)
138 |         description = '%(period)s_snapshot %(vol_id)s_%(period)s_%(date_suffix)s by snapshot script at %(date)s' % {
139 |             'period': period,
140 |             'vol_id': vol.id,
141 |             'date_suffix': date_suffix,
142 |             'date': datetime.today().strftime('%d-%m-%Y %H:%M:%S')
143 |         }
144 |         # Creating snapshots
145 |         try:
146 |             current_snap = vol.create_snapshot(description)
147 |             set_resource_tags(current_snap, tags_volume)
148 |             suc_message = 'Snapshot created with description: %s and tags: %s' % (description, str(tags_volume))
149 |             print '     ' + suc_message
150 |             logging.info(suc_message)
151 |             total_creates += 1
152 |         except Exception, e:
153 |             print "Unexpected error:", sys.exc_info()[0]
154 |             logging.error(e)
155 |             pass
156 | 
157 |         # Deleting old snapshots
158 | 
159 |         snapshots = vol.snapshots()
160 |         deletelist = []
161 |         for snap in snapshots:
162 |             sndesc = snap.description
163 |             if (sndesc.startswith('week_snapshot') and period == 'week'):
164 |                 deletelist.append(snap)
165 |             elif (sndesc.startswith('day_snapshot') and period == 'day'):
166 |                 deletelist.append(snap)
167 |             elif (sndesc.startswith('month_snapshot') and period == 'month'):
168 |                 deletelist.append(snap)
169 |             else:
170 |                 logging.info('     Skipping, not added to deletelist: ' + sndesc)
171 | 
172 |         for snap in deletelist:
173 |             logging.info(snap)
174 |             logging.info(snap.start_time)
175 | 
176 |         deletelist.sort(date_compare)
177 |         if period == 'day':
178 |             keep = keep_day
179 |         elif period == 'week':
180 |             keep = keep_week
181 |         elif period == 'month':
182 |             keep = keep_month
183 |         delta = len(deletelist) - keep
184 |         for i in range(delta):
185 |             del_message = '     Deleting snapshot ' + deletelist[i].description
186 |             logging.info(del_message)
187 |             try:
188 |                 logging.info('Deleting: ' + vol.id)
189 |                 deletelist[i].delete()
190 |             except EC2ResponseError, e:
191 |                 pass
192 |                 logging.info(e.errors)
193 |                 total_deletes -= 2
194 |             total_deletes += 1
195 |         time.sleep(3)
196 | 
197 | 
198 |     except:
199 |         print "Unexpected error:", sys.exc_info()[0]
200 |         logging.error('Error in processing volume with id: ' + vol.id)
201 |         errmsg += 'Error in processing volume with id: ' + vol.id
202 |         count_errors += 1
203 |     else:
204 |         count_success += 1
205 | 
206 | result = '\nFinished making snapshots at %(date)s with %(count_success)s snapshots of %(count_total)s possible.\n\n' % {
207 |     'date': datetime.today().strftime('%d-%m-%Y %H:%M:%S'),
208 |     'count_success': count_success,
209 |     'count_total': count_total
210 | }
211 | 
212 | message += result
213 | message += "\nTotal snapshots created: " + str(total_creates)
214 | message += "\nTotal snapshots errors: " + str(count_errors)
215 | message += "\nTotal snapshots deleted: " + str(total_deletes) + "\n"
216 | 
217 | for key, value in ids_to_snapshots.items():
218 |     message += "\n" + str(key) + " have " + str(len(value[0])) + " snapshots ==> " + "\n" \
219 |                                                                              "Snapshot ids:  " + str(value[0]) + "\n"
220 | print '\n' + message + '\n'
221 | print result
222 | 
223 | # SNS reporting
224 | if sns_arn:
225 |     if errmsg:
226 |         sns.publish(sns_arn, 'Error in processing volumes: ' + errmsg, 'Error with AWS Snapshot')
227 |     sns.publish(sns_arn, message, 'Finished AWS snapshotting')
228 | 
229 | logging.info(result)
230 | 


--------------------------------------------------------------------------------