├── jenkins_backup_s3 ├── __init__.py └── backup.py ├── .gitignore ├── distribute.sh ├── setup.py ├── LICENSE └── README.md /jenkins_backup_s3/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | env 2 | .idea 3 | *.egg-info/ 4 | *.pyc 5 | build/ 6 | dist/ 7 | -------------------------------------------------------------------------------- /distribute.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -e 4 | 5 | python setup.py sdist bdist_wheel 6 | twine upload dist/* 7 | rm -rf build/ dist/ jenkins_backup_s3.egg-info/ 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | with open("README.md", "r") as fh: 4 | long_description = fh.read() 5 | 6 | setup( 7 | name='jenkins-backup-s3', 8 | version='0.1.9', 9 | description="Backup Jenkins to S3", 10 | long_description=long_description, 11 | long_description_content_type="text/markdown", 12 | url="http://github.com/artsy/jenkins-backup-s3", 13 | author='Isac Petruzzi', 14 | author_email='isac@artsymail.com', 15 | license='MIT', 16 | packages=find_packages(), 17 | entry_points={ 18 | 'console_scripts': [ 19 | 'backup-jenkins=jenkins_backup_s3.backup:main' 20 | ] 21 | }, 22 | install_requires=( 23 | 'boto3~=1.9', 24 | 'click~=7.0', 25 | 'colorama~=0.4', 26 | 'python-dateutil~=2.8', 27 | 'termcolor~=1.1' 28 | ) 29 | ) 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016-2017 Art.sy, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 9 | of the Software, and to permit persons to whom the Software is furnished to do 10 | so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # jenkins-backup-s3 2 | 3 | A collection of scripts to backup Jenkins configuration to S3, as well as manage and restore those backups. By default 4 | runs silently (no output) with proper exit codes. Log Level option enables output. 5 | 6 | ## Setup 7 | 8 | `pip install jenkins-backup-s3` 9 | 10 | ### Configure S3 and IAM 11 | 12 | - Create an S3 bucket to store backups. 13 | 14 | - Create an IAM role with STS:AssumeRole and a trust Service ec2.amazonaws.com. The IAM role must have the `GetObject`, `DeleteObject`, `PutObject` and `ListBucket` S3 permissions for that bucket. 15 | 16 | ## Usage 17 | 18 | Setup with cron for ideal usage. 19 | 20 | `backup-jenkins {OPTIONS} {COMMAND} {COMMAND_OPTIONS}` 21 | 22 | Options can be set directly or via and environment variable. 23 | 24 | The only required option is your S3 bucket: 25 | - `backup-jenkins --bucket={BUCKET_NAME}` 26 | 27 | Other available options are: 28 | 29 | Bucket prefix (defaults to "jenkins-backups"): 30 | - `backup-jenkins --bucket-prefix={BUCKET_PREFIX}` 31 | 32 | Bucket region (defaults to "us-east-1"): 33 | - `backup-jenkins --bucket-region={BUCKET_REGION}` 34 | 35 | Available commands: 36 | - `create` 37 | - `restore` 38 | - `list` 39 | - `delete` 40 | - `prune` 41 | 42 | Run `backup-jenkins {COMMAND} --help` for command-specific options. 43 | 44 | ## Running a daily backup on Jenkins 45 | 46 | Create a new item in Jenkins and configure a build of this repository. 47 | 48 | Set the shell / virtualenv builder (if you have it installed) to run `backup-jenkins create`. 49 | 50 | Set the build on a daily CRON schedule. 51 | -------------------------------------------------------------------------------- /jenkins_backup_s3/backup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import os 4 | import sys 5 | import datetime 6 | import boto3 7 | import click 8 | import logging 9 | from boto3.exceptions import S3UploadFailedError 10 | from subprocess import call 11 | from colorama import init 12 | from termcolor import colored 13 | 14 | init() 15 | logger = logging.getLogger(__name__) 16 | logger.setLevel(logging.DEBUG) 17 | 18 | DEFAULT_REGION='us-east-1' 19 | 20 | ch = logging.StreamHandler() 21 | 22 | 23 | class S3Backups(object): 24 | KEY_SUFFIX = '__jenkins-backup.tar.gz' 25 | 26 | def __init__(self, bucket, prefix, region): 27 | self.s3 = boto3.resource('s3') 28 | self.__bucket = bucket 29 | self.__bucket_prefix = prefix 30 | self.__bucket_region = region 31 | logger.debug(colored('Instantiated S3Backups class', 'white')) 32 | 33 | def __list_backups(self): 34 | logger.debug(colored("Fetching S3 objects from %s..." % self.__bucket, 'white')) 35 | 36 | objects = [] 37 | bucket = self.s3.Bucket(self.__bucket) 38 | for obj in bucket.objects.all(): 39 | objects.append(obj.key) 40 | logger.info(colored("Successfully fetched objects from %s" % self.__bucket, 'green')) 41 | return sorted(objects, reverse=True) 42 | 43 | def backups(self): 44 | backups = [] 45 | for key in self.__list_backups(): 46 | if self.KEY_SUFFIX not in key: 47 | continue 48 | backups.append(key.replace("%s/" % self.__bucket_prefix, '').replace(self.KEY_SUFFIX, '')) 49 | return backups 50 | 51 | def backup(self, file_path, backup_name): 52 | key = "%s/%s%s" % (self.__bucket_prefix, backup_name, self.KEY_SUFFIX) 53 | logger.debug(colored('Attempting to upload object to S3', 'white')) 54 | try: 55 | s3_object = self.s3.Object(self.__bucket, key).upload_file(file_path, Callback=logger.info(colored('File uploaded to S3 successfully', 'blue'))) 56 | except S3UploadFailedError as e: 57 | logger.critical(colored("Error uploading file to S3: %s" % e, 'red')) 58 | 59 | def delete(self, backup_name): 60 | key = "%s/%s%s" % (self.__bucket_prefix, backup_name, self.KEY_SUFFIX) 61 | logger.debug(colored('Attempting delete S3 object', 'white')) 62 | s3_object = self.s3.Object(self.__bucket, key).delete() 63 | 64 | def latest(self): 65 | backups = self.backups() 66 | if len(backups): 67 | return self.backups()[0] 68 | return None 69 | 70 | def restore(self, backup_name, target): 71 | key = "%s/%s%s" % (self.__bucket_prefix, backup_name, self.KEY_SUFFIX) 72 | 73 | logger.debug(colored('Attempting to fetch file from s3: %s' % key, 'white')) 74 | 75 | s3_object = self.s3.Object(self.__bucket, key).download_file(target) 76 | 77 | return target 78 | 79 | 80 | @click.group() 81 | @click.pass_context 82 | @click.option('--bucket', required=True, type=click.STRING, help='S3 bucket to store backups in') 83 | @click.option('--bucket-prefix', type=click.STRING, default='jenkins-backups', help='S3 bucket prefix : defaults to "jenkins-backups"') 84 | @click.option('--bucket-region', type=click.STRING, default=DEFAULT_REGION, help='S3 bucket region : defaults to "%s"' % DEFAULT_REGION) 85 | @click.option('--log-level', type=click.STRING, default='INFO', help='Display colorful status messages') 86 | def cli(ctx, bucket, bucket_prefix, bucket_region, log_level): 87 | """Manage Jenkins backups to S3""" 88 | ctx.obj['BUCKET'] = bucket 89 | ctx.obj['BUCKET_PREFIX'] = bucket_prefix 90 | ctx.obj['BUCKET_REGION'] = bucket_region 91 | 92 | ch.setLevel(log_level) 93 | formatter = logging.Formatter('[%(levelname)s]: %(message)s') 94 | ch.setFormatter(formatter) 95 | logger.addHandler(ch) 96 | 97 | 98 | @cli.command() 99 | @click.pass_context 100 | @click.option('--jenkins-home', type=click.STRING, default='/var/lib/jenkins', help='Jenkins home directory : defaults to "/var/lib/jenkins"') 101 | @click.option('--tmp', type=click.STRING, default='/tmp/jenkins-backup.tar.gz', help='Jenkins archive name : defaults to "/tmp/jenkins-backup.tar.gz"') 102 | @click.option('--tar', type=click.STRING, default='/bin/tar', help='tar executable : defaults to "/bin/tar"') 103 | @click.option('--tar-opts', type=click.STRING, default='cvfz', help='tar options : defaults to "cvfz"') 104 | @click.option('--exclude-jenkins-war/--include-jenkins-war', default=True, help='Exclude jenkins.war from the backup : defaults to true') 105 | @click.option('--exclude-vcs/--include-vcs', default=True, help='Exclude VCS from the backup : defaults to true') 106 | @click.option('--ignore-fail/--dont-ignore-fail', default=True, help='Tar should ignore failed reads : defaults to true') 107 | @click.option('--exclude-archive/--include-archive', default=True, help='Exclude archive directory from the backup : defaults to true') 108 | @click.option('--exclude-target/--include-target', default=True, help='Exclude target directory from the backup : defaults to true') 109 | @click.option('--exclude-builds/--include-builds', default=True, help='Exclude job builds directories from the backup : defaults to true') 110 | @click.option('--exclude-workspace/--include-workspace', default=True, help='Exclude job workspace directories from the backup : defaults to true') 111 | @click.option('--exclude-maven/--include-maven', default=True, help='Exclude maven repository from the backup : defaults to true') 112 | @click.option('--exclude-logs/--include-logs', default=True, help='Exclude logs from the backup : defaults to true') 113 | @click.option('--exclude', '-e', type=click.STRING, multiple=True, help='Additional directories to exclude from the backup') 114 | @click.option('--dry-run', type=click.BOOL, is_flag=True, help='Create tar archive as "tmp" but do not upload it to S3 : defaults to false') 115 | def create(ctx, jenkins_home, tmp, tar, tar_opts, exclude_jenkins_war, exclude_vcs, ignore_fail, exclude_archive, exclude_target, 116 | exclude_builds, exclude_workspace, exclude_maven, exclude_logs, exclude, dry_run): 117 | """Create a backup""" 118 | logger.info(colored("Backing up %s to %s/%s" % (jenkins_home, ctx.obj['BUCKET'], ctx.obj['BUCKET_PREFIX']), 'blue')) 119 | 120 | command = [tar, tar_opts, tmp, '-C', jenkins_home] 121 | 122 | if exclude_jenkins_war: 123 | command.append('--exclude=jenkins.war') 124 | if exclude_vcs: 125 | command.append('--exclude-vcs') 126 | if ignore_fail: 127 | command.append('--ignore-failed-read') 128 | if exclude_archive: 129 | command.append('--exclude=archive') 130 | if exclude_target: 131 | command.append('--exclude=target') 132 | if exclude_builds: 133 | command.append('--exclude=jobs/*/builds/*') 134 | if exclude_workspace: 135 | command.append('--exclude=jobs/*/workspace/*') 136 | if exclude_maven: 137 | command.append('--exclude=.m2/repository') 138 | if exclude_logs: 139 | command.append('--exclude=*.log') 140 | for e in exclude: 141 | command.append("--exclude=%s" % e) 142 | 143 | command.append('.') 144 | 145 | logger.info(colored("Executing command \"%s\"" % ' '.join(command), 'blue')) 146 | 147 | retval = call(command) 148 | if retval >= 2: 149 | logger.critical(colored("Creating tar archive failed with error code %s." % retval, 'red')) 150 | 151 | os.remove(tmp) 152 | sys.exit(retval) 153 | 154 | logger.debug(colored("Successfully created tar archive", 'white')) 155 | 156 | s3 = S3Backups(ctx.obj['BUCKET'], ctx.obj['BUCKET_PREFIX'], ctx.obj['BUCKET_REGION']) 157 | backup_id = str(datetime.datetime.now()).replace(' ', '_') 158 | 159 | if dry_run: 160 | logger.info(colored("Would have created backup id %s from %s" % (backup_id, tmp), 'green')) 161 | else: 162 | s3.backup(tmp, backup_id) 163 | os.remove(tmp) 164 | 165 | logger.info(colored("Created backup id %s" % backup_id, 'green')) 166 | 167 | 168 | @cli.command() 169 | @click.pass_context 170 | def list(ctx): 171 | """List available backups""" 172 | logger.info(colored("All backups for %s/%s..." % (ctx.obj['BUCKET'], ctx.obj['BUCKET_PREFIX']), 'blue')) 173 | logger.info(colored("------------------------", 'blue')) 174 | 175 | s3 = S3Backups(ctx.obj['BUCKET'], ctx.obj['BUCKET_PREFIX'], ctx.obj['BUCKET_REGION']) 176 | 177 | for backup in s3.backups(): 178 | logger.info(colored(backup, 'blue')) 179 | 180 | 181 | def _delete_command(backup_id, bucket, bucket_prefix, bucket_region, dry_run): 182 | logger.info(colored("Deleting backup %s in %s/%s..." % (backup_id, bucket, bucket_prefix), 'blue')) 183 | 184 | s3 = S3Backups(bucket, bucket_prefix, bucket_region) 185 | 186 | if dry_run: 187 | logger.info(colored("Would have deleted %s" % backup_id, 'blue')) 188 | else: 189 | s3.delete(backup_id) 190 | logger.info(colored("Deleted %s" % backup_id, 'green')) 191 | 192 | 193 | @cli.command() 194 | @click.pass_context 195 | @click.argument('backup-id', required=True, type=click.STRING) 196 | @click.option('--dry-run', type=click.BOOL, is_flag=True, help='List delete candidate, do not delete it') 197 | def delete(ctx, backup_id, dry_run): 198 | """Delete a backup by {backup-id}""" 199 | _delete_command(backup_id, ctx.obj['BUCKET'], ctx.obj['BUCKET_PREFIX'], ctx.obj['BUCKET_REGION'], dry_run) 200 | 201 | 202 | @cli.command() 203 | @click.pass_context 204 | @click.argument('keep', required=True, type=click.INT) 205 | @click.option('--dry-run', type=click.BOOL, is_flag=True, help='List delete candidates, do not delete them') 206 | def prune(ctx, keep, dry_run): 207 | """Delete any backups older than the latest {keep} number of backups""" 208 | logger.info(colored("Pruning backups in %s/%s..." % (ctx.obj['BUCKET'], ctx.obj['BUCKET_PREFIX']), 'blue')) 209 | 210 | s3 = S3Backups(ctx.obj['BUCKET'], ctx.obj['BUCKET_PREFIX'], ctx.obj['BUCKET_REGION']) 211 | for backup_id in s3.backups()[keep:]: 212 | _delete_command(backup_id, ctx.obj['BUCKET'], ctx.obj['BUCKET_PREFIX'], ctx.obj['BUCKET_REGION'], dry_run) 213 | 214 | 215 | @cli.command() 216 | @click.pass_context 217 | @click.argument('backup-id', required=True, type=click.STRING) 218 | @click.option('--jenkins-home', type=click.STRING, default='/var/lib/jenkins', help='Jenkins home directory : defaults to "/var/lib/jenkins"') 219 | @click.option('--tmp', type=click.STRING, default='/tmp/jenkins-backup.tar.gz', help='Temporary tar archive file : defaults to "/tmp/jenkins-backup.tar.gz"') 220 | @click.option('--tar', type=click.STRING, default='tar', help='tar executable : defaults to "tar"') 221 | @click.option('--tar-opts', type=click.STRING, default='xzf', help='tar options : defaults to "xzf"') 222 | @click.option('--dry-run', type=click.BOOL, is_flag=True, help='Download tar archive to "tmp" directory but do not decomress it to "jenkins-home"') 223 | def restore(ctx, backup_id, jenkins_home, tmp, tar, tar_opts, dry_run): 224 | """Restore a backup by {backup-id} or 'latest'""" 225 | logger.info(colored("Attempting to restore backup by criteria '%s'..." % backup_id, 'blue')) 226 | s3 = S3Backups(ctx.obj['BUCKET'], ctx.obj['BUCKET_PREFIX'], ctx.obj['BUCKET_REGION']) 227 | if backup_id == 'latest': 228 | backup_id = s3.latest() 229 | if backup_id is None: 230 | logger.info(colored("No backups found.", 'blue')) 231 | return 232 | 233 | logger.info(colored("Restoring %s from %s/%s/%s..." % (jenkins_home, ctx.obj['BUCKET'], ctx.obj['BUCKET_PREFIX'], backup_id), 'blue')) 234 | 235 | s3.restore(backup_id, tmp) 236 | 237 | if dry_run: 238 | logger.info(colored("Would have restored %s from %s" % (jenkins_home, tmp), 'blue')) 239 | else: 240 | command = [tar, tar_opts, tmp, '-C', jenkins_home] 241 | 242 | logger.info(colored("Executing %s" % ' '.join(command), 'blue')) 243 | 244 | retval = call(command) 245 | if retval >= 2: 246 | logger.critical(colored("Restoring tar archive failed with error code %s." % retval, 'red')) 247 | os.remove(tmp) 248 | sys.exit(retval) 249 | 250 | 251 | def main(): 252 | cli(obj={}) 253 | 254 | 255 | if __name__ == '__main__': 256 | main() 257 | --------------------------------------------------------------------------------