├── feather.cron.d.example ├── feather.yaml.dist ├── README.md └── feather /feather.cron.d.example: -------------------------------------------------------------------------------- 1 | */5 * * * * root /usr/bin/python /root/scripts/feather/feather -vvvvv /etc/feather.yaml >> /var/log/feather.log 2 | -------------------------------------------------------------------------------- /feather.yaml.dist: -------------------------------------------------------------------------------- 1 | # Global paths, if different from default 2 | cachedir: /usr/home/drue/tarsnap/cachedir 3 | keyfile: /usr/home/drue/tarsnap/tarsnap.key 4 | binpath: /usr/local/bin/ 5 | 6 | # perform a checkpoint every checkpoint_bytes, don't cross filesystems 7 | backup_args: "--one-file-system --checkpoint-bytes 104857600" 8 | 9 | # Kill the script after N seconds. 10 | max_runtime: 3600 11 | 12 | # Define the schedule 13 | # 14 | # period: Seconds. A backup is taken every period. 15 | # always_keep: Number of backups to keep of a particular period, before 16 | # pruning old backups. Backups younger than now()-period are 17 | # never removed. 18 | # implies: Include another defined schedule. i.e. if WEEKLY implies 19 | # MONTHLY, and you ask for WEEKLY backups, you will get WEEKLY 20 | # and MONTHLY. 21 | # before/after: Restrict running to a certain time of day (UTC) 22 | # 23 | schedule: 24 | - monthly: 25 | - period: 2592000 # 30 days 26 | - always_keep: 12 27 | - before: "0600" 28 | - weekly: 29 | - period: 604800 # 7 days 30 | - always_keep: 6 31 | - after: "0200" 32 | - before: "0600" 33 | - implies: monthly 34 | - daily: 35 | - period: 86400 # 1 day 36 | - always_keep: 14 37 | - after: "0200" 38 | - before: "0600" 39 | - implies: weekly 40 | - hourly: 41 | - period: 3600 42 | - always_keep: 24 43 | - implies: daily 44 | - realtime: 45 | - period: 900 46 | - always_keep: 10 47 | - implies: hourly 48 | 49 | # Define individual backups 50 | # Path can be a directory or a file. 51 | # Path can be a list or a single item. 52 | backups: 53 | - _usr_local: 54 | - schedule: daily 55 | - path: /usr/local 56 | - exclude: /usr/local/bin 57 | - _etc: 58 | - schedule: realtime 59 | - path: /etc 60 | - music: 61 | - schedule: monthly 62 | - workingpath: /home 63 | - path: 64 | - joe/music 65 | - bob/music 66 | - exclude: 67 | - joe/music/bieber # Not paying to back this up 68 | - bob/music/backstreet_boys # not paying to store this either 69 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Feather is a tarsnap backup scheduler that performs and maintains a set of 2 | backups as defined by a yaml configuration file. 3 | 4 | ### Features 5 | - Dynamic scheduling 6 | - Keep an arbitrary number of backups of each schedule type 7 | - Restrict schedules based on time of day 8 | - Restrict feather run to a certain amount of wall time (max_runtime) 9 | - Multiple backup paths per tarsnap 10 | - Multiple exclude list per tarsnap 11 | 12 | ### Requirements 13 | - Python 2 14 | - pyyaml 15 | 16 | ### Usage 17 | Feather is designed to be run from cron like this: 18 | 19 | */5 * * * * /usr/local/bin/feather /usr/local/etc/feather.yaml 20 | 21 | ### Configuration 22 | The best way to understand feather is to read an example configuration file: 23 | 24 | ```yaml 25 | # Global paths, if different from default 26 | cachedir: /usr/home/drue/tarsnap/cachedir 27 | keyfile: /usr/home/drue/tarsnap/tarsnap.key 28 | binpath: /usr/local/bin/ 29 | 30 | # perform a checkpoint every checkpoint_bytes, don't cross filesystems 31 | backup_args: "--one-file-system --checkpoint-bytes 104857600" 32 | 33 | # Kill the script after N seconds. 34 | max_runtime: 3600 35 | 36 | # Define the schedule 37 | # 38 | # period: Seconds. A backup is taken every period. 39 | # always_keep: Number of backups to keep of a particular period, before 40 | # pruning old backups. Backups younger than now()-period are 41 | # never removed. 42 | # implies: Include another defined schedule. i.e. if WEEKLY implies 43 | # MONTHLY, and you ask for WEEKLY backups, you will get WEEKLY 44 | # and MONTHLY. 45 | # before/after: Restrict running to a certain time of day (UTC) 46 | # 47 | schedule: 48 | - monthly: 49 | - period: 2592000 # 30 days 50 | - always_keep: 12 51 | - before: "0600" 52 | - weekly: 53 | - period: 604800 # 7 days 54 | - always_keep: 6 55 | - after: "0200" 56 | - before: "0600" 57 | - implies: monthly 58 | - daily: 59 | - period: 86400 # 1 day 60 | - always_keep: 14 61 | - after: "0200" 62 | - before: "0600" 63 | - implies: weekly 64 | - hourly: 65 | - period: 3600 66 | - always_keep: 24 67 | - implies: daily 68 | - realtime: 69 | - period: 900 70 | - always_keep: 10 71 | - implies: hourly 72 | 73 | # Define individual backups 74 | # Path can be a directory or a file. 75 | # Path can be a list or a single item. 76 | backups: 77 | - _usr_local: 78 | - schedule: daily 79 | - path: /usr/local 80 | - exclude: /usr/local/bin 81 | - _etc: 82 | - schedule: realtime 83 | - path: /etc 84 | - music: 85 | - schedule: monthly 86 | - workingpath: /home 87 | - path: 88 | - joe/music 89 | - bob/music 90 | - exclude: 91 | - joe/music/bieber # Not paying to back this up 92 | - bob/music/backstreet_boys # not paying to store this either 93 | 94 | ``` 95 | -------------------------------------------------------------------------------- /feather: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | ''' 4 | 5 | feather is a tarsnap script that performs and maintains a set of backups 6 | as defined in a yaml configuration file. 7 | 8 | TODO: 9 | - Order of backups (!!omap) 10 | - use localtime instead of hardcoding UTC 11 | 12 | ---------------------------------------------------------------------------- 13 | "THE BEER-WARE LICENSE" (Revision 43): 14 | 15 | wrote this file. As long as you retain this notice and 16 | never sue us for anything, you can do whatever you want with this stuff. 17 | If we meet some day, and you think this stuff is worth it, you can buy me 18 | a beer in return. -Dan Rue 19 | ---------------------------------------------------------------------------- 20 | 21 | Generous support for feather was provided by Prometheus Research, LLC 22 | 23 | ''' 24 | 25 | import datetime 26 | import hashlib 27 | import optparse 28 | import os 29 | import re 30 | import signal 31 | import subprocess 32 | import sys 33 | import time 34 | import yaml 35 | 36 | class backup_schedule(set): 37 | 38 | def __init__(self, schedule_from_yaml): 39 | # Reformat schedule into something a little more useable. 40 | self.schedule = {} 41 | for item in schedule_from_yaml: 42 | #{'WEEKLY': [{'period': 604800}, {'always_keep': 6}, 43 | # {'implies': 'MONTHLY'}]}, 44 | key = item.keys()[0] 45 | self.schedule[key] = {} 46 | for param in item[key]: 47 | param_key = param.keys()[0] 48 | param_key = param.keys()[0] 49 | if param_key == 'period': 50 | self.schedule[key][param_key] = \ 51 | datetime.timedelta(seconds=param[param_key]) 52 | else: 53 | self.schedule[key][param_key] = param[param_key] 54 | 55 | def __contains__(self, elem): 56 | return elem in self.schedule 57 | 58 | def __str__(self): 59 | return str(self.schedule) 60 | 61 | def get_schedule(self, snapshot): 62 | if 'implies' in self.schedule[snapshot]: 63 | return [snapshot] + \ 64 | self.get_schedule(self.schedule[snapshot]['implies']) 65 | else: 66 | return [snapshot] 67 | 68 | def schedule_timedelta(self, schedule): 69 | return self.schedule[schedule]['period'] 70 | 71 | def rotate(self, schedule, quantity): 72 | return quantity > self.schedule[schedule]['always_keep'] 73 | 74 | def timeok(self, schedule): 75 | ts = time.strftime("%H:%M", time.gmtime()) 76 | if 'after' in self.schedule[schedule]: 77 | if self.schedule[schedule]['after'] > ts: 78 | return False 79 | if 'before' in self.schedule[schedule]: 80 | if self.schedule[schedule]['before'] < ts: 81 | return False 82 | return True 83 | 84 | class ConcurrencyError(Exception): 85 | def __init__(self, value = "Concurrency Error"): 86 | self.value = value 87 | def __str__(self): 88 | return repr(self.value) 89 | 90 | class RecursionError(Exception): 91 | def __init__(self, value = "Recursion Loop Error"): 92 | self.value = value 93 | def __str__(self): 94 | return repr(self.value) 95 | 96 | class MaxRuntime(Exception): 97 | def __init__(self, value = "Maximum Runtime Exceeded"): 98 | self.value = value 99 | def __str__(self): 100 | return repr(self.value) 101 | 102 | class ConfigError(Exception): 103 | def __init__(self, value = "Configuration Error"): 104 | self.value = value 105 | def __str__(self): 106 | return repr(self.value) 107 | 108 | class lock(object): 109 | def __init__(self, pidfile): 110 | 111 | if (os.path.isfile(pidfile)): 112 | f = open(pidfile, "r") 113 | pid = f.readline() 114 | f.close() 115 | 116 | try: 117 | os.kill(int(pid), 0) 118 | # pid is running.. 119 | raise ConcurrencyError, \ 120 | "tarsnap already running %s" % pid 121 | 122 | except OSError, err: 123 | # pid isn't running, so remove the stale pidfile 124 | os.remove(pidfile) 125 | 126 | # write our pid 127 | file(pidfile,'w+').write("%s" % str(os.getpid())) 128 | 129 | 130 | class pr_tarsnap(object): 131 | 132 | archive_list = [] 133 | 134 | def __init__(self, yaml_file, verbosity=None, dry_run=False): 135 | if verbosity: 136 | self.verbosity = verbosity 137 | else: 138 | self.verbosity = 0 139 | 140 | self.dry_run = dry_run 141 | 142 | f = open(yaml_file) 143 | config = yaml.load(f.read()) 144 | f.close() 145 | 146 | self.max_runtime = config.get('max_runtime', None) 147 | if self.max_runtime: 148 | signal.signal(signal.SIGALRM, self.timeout) 149 | signal.alarm(self.max_runtime) 150 | 151 | self.handle = None 152 | self.cachedir = config.get('cachedir', None) 153 | self.keyfile = config.get('keyfile', None) 154 | self.binpath = config.get('binpath', None) 155 | 156 | self.backup_args = config.get('backup_args', None) 157 | if self.backup_args: 158 | self.backup_args = self.backup_args.split() 159 | 160 | self.schedule = backup_schedule(config['schedule']) 161 | 162 | # Reformat schedule into something a little more useable. 163 | self.backups = {} 164 | for item in config['backups']: 165 | #{'_usr_home_drue_irclogs': 166 | # [{'schedule': 'realtime'}, {'path': '/usr/home/drue/irclogs'}]} 167 | # OR 168 | # {'path': ['/path1', '/path2', '/file']} 169 | key = item.keys()[0] 170 | self.backups[key] = {} 171 | for param in item[key]: 172 | param_key = param.keys()[0] 173 | self.backups[key][param_key] = param[param_key] 174 | if 'schedule' not in self.backups[key]: 175 | raise ConfigError, "'schedule' not defined for backup %s" % key 176 | if 'path' not in self.backups[key]: 177 | raise ConfigError, "'path' not defined for backup %s" % key 178 | if 'exclude'in self.backups[key]: 179 | exclude = self.backups[key]['exclude'] 180 | else: 181 | exclude = [] 182 | 183 | path = self.backups[key]['path'] 184 | if isinstance(path, list): 185 | for item in path: 186 | path_to_check = item 187 | if 'workingpath' in self.backups[key]: 188 | wp = self.backups[key]['workingpath'] 189 | path_to_check = os.path.join(wp, item) 190 | self.check_valid_path(path_to_check) 191 | else: 192 | path_to_check = path 193 | if 'workingpath' in self.backups[key]: 194 | wp = self.backups[key]['workingpath'] 195 | path_to_check = os.path.join(wp, path_to_check) 196 | self.check_valid_path(path_to_check) 197 | 198 | self.populate_archive_list() 199 | 200 | def timeout(self, signum, frame): 201 | if self.handle: 202 | self.handle.kill() 203 | raise MaxRuntime, "Timeout of %ss exceeded; aborting" % self.max_runtime 204 | 205 | def tarsnap_cmd(self): 206 | cmd = [] 207 | if self.binpath: 208 | cmd.append("%s/tarsnap" % self.binpath) 209 | else: 210 | cmd.append("tarsnap") 211 | if self.cachedir: 212 | cmd += ["--cachedir", self.cachedir] 213 | if self.keyfile: 214 | cmd += ["--keyfile", self.keyfile] 215 | if self.verbosity > 0: 216 | cmd += ["--print-stats"] 217 | cmd += ["--humanize-numbers"] 218 | else: 219 | cmd += ["--no-print-stats"] 220 | return cmd 221 | 222 | def populate_archive_list(self): 223 | cmd = self.tarsnap_cmd() + ["--list-archives"] 224 | 225 | if self.verbosity > 1: 226 | print "Listing archives using tarsnap --list-archives" 227 | if self.verbosity > 2: print cmd 228 | output = self.execute(cmd) 229 | if self.verbosity > 2: print output 230 | self.archive_list = output.splitlines() 231 | self.archive_list.sort() 232 | 233 | def exists(self, base, schedule): 234 | if self.verbosity > 1: 235 | print "Checking if the following exists: %s %s" % (base, schedule) 236 | pattern = re.compile("^(.*)-(\d+|\d+-\d+-\d+_\d+:\d+)_?UTC-(\w+)$") 237 | 238 | # Loop through the list of archives to see if we can find one that 239 | # is within the schedule for the given path. 240 | # for instance, if path is /foo/bar and schedule is "DAILY", 241 | # look for an archive of /foo/bar made within 24h. 242 | for archive in self.archive_list: 243 | f = pattern.match(archive) 244 | if not f: 245 | sys.stderr.write("Archive label format unrecognized: %s\n" % 246 | archive) 247 | continue # unrecognized archive 248 | 249 | (archivepath, ts, type) = f.groups() 250 | if archivepath == base and type == schedule: 251 | try: 252 | ts = datetime.datetime.strptime(ts, "%Y%m%d%H%M") 253 | except: 254 | try: 255 | ts = datetime.datetime.strptime(ts,"%Y-%m-%d_%H:%M") 256 | except: 257 | sys.stderr.write("Unknown timestamp: %s\n" % archive) 258 | continue 259 | if ((datetime.datetime.utcnow()-ts) < 260 | self.schedule.schedule_timedelta(schedule)): 261 | return True 262 | 263 | return False 264 | 265 | def execute(self, cmd): 266 | self.handle = subprocess.Popen(cmd, stdout=subprocess.PIPE, 267 | stderr=subprocess.PIPE) 268 | (stdout, stderr) = self.handle.communicate() 269 | if (self.verbosity > 0 and self.dry_run) or self.handle.returncode > 0: 270 | print stderr 271 | return stdout 272 | 273 | def run_backups(self): 274 | for backup in self.backups: 275 | backup_path = self.backups[backup]['path'] 276 | if 'exclude' in self.backups[backup]: 277 | excl_path = self.backups[backup]['exclude'] 278 | else: 279 | excl_path = None 280 | if 'workingpath' in self.backups[backup]: 281 | working_path = self.backups[backup]['workingpath'] 282 | else: 283 | working_path = None 284 | backup_schedule = self.backups[backup]['schedule'] 285 | try: 286 | snapshots = self.schedule.get_schedule(backup_schedule) 287 | except RuntimeError: 288 | raise RecursionError, "Loop detected in backup configuration" 289 | 290 | for snapshot in snapshots: 291 | if self.verbosity > 1: 292 | print "Processing", backup, snapshot 293 | if self.exists(backup, snapshot): 294 | # backup already existing within schedule window 295 | continue 296 | if not self.schedule.timeok(snapshot): 297 | # Skip the snapshot if current time is not within 298 | # before: and after:, if set. 299 | if self.verbosity > 1: 300 | print "Skipping due to time of day:", backup, snapshot 301 | continue 302 | ts = datetime.datetime.utcnow().strftime("%Y-%m-%d_%H:%M_UTC") 303 | archive_name = backup+"-"+ts+"-"+snapshot 304 | if self.verbosity > 0: 305 | print "Taking backup", archive_name 306 | cmd = self.tarsnap_cmd() 307 | if self.dry_run: 308 | cmd += ["--dry-run"] 309 | if self.backup_args: 310 | cmd += self.backup_args 311 | cmd += ["-c", "-f", archive_name] 312 | if excl_path: 313 | if isinstance(excl_path, list): 314 | for pth in excl_path: 315 | cmd += ['--exclude', pth] 316 | else: 317 | cmd += ['--exclude', excl_path] 318 | if working_path: 319 | cmd += ['-C', working_path] 320 | if isinstance(backup_path, list): 321 | cmd.extend(backup_path) 322 | else: 323 | cmd += [backup_path] 324 | if self.verbosity > 2: 325 | print cmd 326 | output = self.execute(cmd) 327 | if self.verbosity > 2: 328 | print output 329 | 330 | 331 | def prune_parts(self, days=7): 332 | ''' 333 | Tarsnap saves .part files when used with checkpoint_bytes, and 334 | if a backup does not complete for some reason. 335 | 336 | This function looks .part files older than 'days' and removes them. 337 | 338 | We want to delete multiple backups in one go to optimize tarsnap bandwidth cost 339 | See: http://mail.tarsnap.com/tarsnap-users/msg00590.html 340 | Requires tarsnap version >= 1.0.27 341 | ''' 342 | pattern = re.compile("^(.*)-(\d+|\d+-\d+-\d+_\d+:\d+)_?UTC-(\w+)\.part$") 343 | cmd = self.tarsnap_cmd() + ["-d"] 344 | archives_to_delete = False 345 | for archive in self.archive_list: 346 | f = pattern.match(archive) 347 | if not f: 348 | continue # not a .part file 349 | 350 | (path, ts, type) = f.groups() 351 | 352 | try: 353 | ts = datetime.datetime.strptime(ts, "%Y%m%d%H%M") 354 | except: 355 | try: 356 | ts = datetime.datetime.strptime(ts,"%Y-%m-%d_%H:%M") 357 | except: 358 | sys.stderr.write("Unknown timestamp: %s\n" % archive) 359 | continue 360 | 361 | if ((datetime.datetime.utcnow()-ts) > 362 | datetime.timedelta(days=days)): 363 | if self.verbosity > 0: print "Adding archive", archive, "to delete list" 364 | cmd += ["-f", archive] 365 | archives_to_delete = True 366 | 367 | if archives_to_delete: 368 | if self.verbosity > 2: print cmd 369 | if not self.dry_run: 370 | output = self.execute(cmd) 371 | if self.verbosity > 2: print output 372 | 373 | def prune_backups(self): 374 | ''' 375 | prune_backups checks two conditions before removing an archive. 376 | 1) That there are 'always_keep' number of backups, AND 377 | 2) That the archive to remove is older than the predefined period 378 | of time. 379 | 380 | We want to delete multiple backups in one go to optimize tarsnap bandwidth cost 381 | See: http://mail.tarsnap.com/tarsnap-users/msg00590.html 382 | Requires tarsnap version >= 1.0.27 383 | 384 | ''' 385 | self.populate_archive_list() 386 | self.archive_list.sort() 387 | pattern = re.compile("^(.*)-(\d+|\d+-\d+-\d+_\d+:\d+)_?UTC-(\w+)$") 388 | 389 | quantity = {} 390 | # Count up how many of each type of archive exist 391 | for archive in self.archive_list: 392 | f = pattern.match(archive) 393 | if not f: 394 | sys.stderr.write("Unrecognizable archive: %s\n" % archive) 395 | continue # unrecognized archive 396 | 397 | (path, ts, type) = f.groups() 398 | if path+type in quantity: 399 | quantity[path+type] += 1 400 | else: 401 | quantity[path+type] = 1 402 | 403 | cmd = self.tarsnap_cmd() + ["-d"] 404 | archives_to_delete = False 405 | for archive in self.archive_list: 406 | f = pattern.match(archive) 407 | if not f: 408 | sys.stderr.write("Unrecognizable archive: %s\n" % archive) 409 | continue # unrecognized archive 410 | 411 | (path, ts, type) = f.groups() 412 | 413 | try: 414 | ts = datetime.datetime.strptime(ts, "%Y%m%d%H%M") 415 | except: 416 | try: 417 | ts = datetime.datetime.strptime(ts,"%Y-%m-%d_%H:%M") 418 | except: 419 | sys.stderr.write("Unknown timestamp: %s\n" % archive) 420 | continue 421 | 422 | if ((datetime.datetime.utcnow()-ts) > 423 | self.schedule.schedule_timedelta(type)): 424 | if self.schedule.rotate(type, quantity[path+type]): 425 | if self.verbosity > 0: print "Adding archive", archive, "to delete list" 426 | quantity[path+type] -= 1 # remove an item from quantity dict 427 | cmd += ["-f", archive] 428 | archives_to_delete = True 429 | 430 | if archives_to_delete: 431 | if self.verbosity > 2: print cmd 432 | if not self.dry_run: 433 | output = self.execute(cmd) 434 | if self.verbosity > 2: print output 435 | 436 | def check_valid_path(self, path): 437 | if not (os.path.isdir(path) or os.path.isfile(path)): 438 | raise ConfigError, "path does not exist: %s" % path 439 | 440 | 441 | 442 | def main(): 443 | 444 | usage = "usage: %prog [options] config_file" 445 | parser = optparse.OptionParser(usage) 446 | parser.add_option("-v", action="count", dest="verbosity", 447 | help="Verbosity; Additional -v options will provide " 448 | "additional detail.") 449 | parser.add_option("--dry-run", action="store_true", dest="dry_run", 450 | default=False, help="Do not create new archives or " 451 | "delete old archives.") 452 | 453 | (options, args) = parser.parse_args() 454 | if len(args) != 1: 455 | parser.error("YAML config file not specified") 456 | 457 | # Generate a unique pid lock file per config location 458 | config_path = os.path.realpath(args[0]) 459 | PIDFILE = "/tmp/feather-%s.pid" % hashlib.md5(config_path).hexdigest()[:8] 460 | 461 | lock(PIDFILE) 462 | os.nice(20) 463 | b = pr_tarsnap(args[0], options.verbosity, options.dry_run) 464 | b.run_backups() 465 | b.prune_backups() 466 | b.prune_parts() 467 | 468 | if __name__ == '__main__': 469 | main() 470 | --------------------------------------------------------------------------------