├── .rspec ├── .document ├── lib ├── astrails │ ├── safe │ │ ├── version.rb │ │ ├── svndump.rb │ │ ├── gzip.rb │ │ ├── pipe.rb │ │ ├── backup.rb │ │ ├── archive.rb │ │ ├── mongodump.rb │ │ ├── stream.rb │ │ ├── pgdump.rb │ │ ├── mysqldump.rb │ │ ├── sink.rb │ │ ├── gpg.rb │ │ ├── source.rb │ │ ├── tmp_file.rb │ │ ├── local.rb │ │ ├── config │ │ │ ├── node.rb │ │ │ └── builder.rb │ │ ├── ftp.rb │ │ ├── s3.rb │ │ ├── sftp.rb │ │ └── cloudfiles.rb │ └── safe.rb └── extensions │ └── mktmpdir.rb ├── .autotest ├── Rakefile ├── Gemfile ├── spec ├── spec_helper.rb ├── astrails │ └── safe │ │ ├── svndump_spec.rb │ │ ├── pgdump_spec.rb │ │ ├── mongodump_spec.rb │ │ ├── gzip_spec.rb │ │ ├── archive_spec.rb │ │ ├── mysqldump_spec.rb │ │ ├── local_spec.rb │ │ ├── gpg_spec.rb │ │ ├── s3_spec.rb │ │ ├── cloudfiles_spec.rb │ │ └── config_spec.rb └── integration │ ├── cleanup_spec.rb │ └── archive_integration_spec.rb ├── .gitignore ├── CHANGELOG ├── LICENSE.txt ├── TODO ├── astrails-safe.gemspec ├── bin └── astrails-safe ├── templates └── script.rb └── README.markdown /.rspec: -------------------------------------------------------------------------------- 1 | --colour 2 | --format d 3 | --profile 4 | -------------------------------------------------------------------------------- /.document: -------------------------------------------------------------------------------- 1 | README.rdoc 2 | lib/**/*.rb 3 | bin/* 4 | features/**/*.feature 5 | LICENSE 6 | -------------------------------------------------------------------------------- /lib/astrails/safe/version.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | VERSION = "0.3.1" 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /.autotest: -------------------------------------------------------------------------------- 1 | Autotest.add_hook(:initialize) do |at| 2 | at.add_mapping(%r{lib/astrails/safe/config/.*\.rb$}, true) {'spec/astrails/safe/config_spec.rb'} 3 | end 4 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | require 'rspec/core/rake_task' 4 | 5 | desc "run specs" 6 | RSpec::Core::RakeTask.new 7 | 8 | task default: :spec 9 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in astrails-safe.gemspec 4 | gemspec 5 | 6 | gem 'debugger' 7 | gem 'awesome_print' 8 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler/setup' 3 | require 'astrails/safe' 4 | require 'debugger' 5 | 6 | RSpec.configure do |config| 7 | config.mock_with :rr 8 | end 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | Gemfile.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | test/tmp 16 | test/version_tmp 17 | tmp 18 | tags 19 | -------------------------------------------------------------------------------- /lib/astrails/safe/svndump.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Svndump < Source 4 | 5 | def command 6 | "svnadmin dump #{config[:options]} #{config[:repo_path]}" 7 | end 8 | 9 | def extension; '.svn'; end 10 | 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/astrails/safe/gzip.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Gzip < Pipe 4 | 5 | protected 6 | 7 | def post_process 8 | @backup.compressed = true 9 | end 10 | 11 | def pipe 12 | "|gzip" 13 | end 14 | 15 | def extension 16 | ".gz" 17 | end 18 | 19 | def active? 20 | !@backup.compressed 21 | end 22 | 23 | end 24 | end 25 | end -------------------------------------------------------------------------------- /lib/astrails/safe/pipe.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Pipe < Stream 4 | # process adds required commands to the current 5 | # shell command string 6 | # :active?, :pipe, :extension and :post_process are 7 | # defined in inheriting pipe classes 8 | def process 9 | return unless active? 10 | 11 | @backup.command << pipe 12 | @backup.extension << extension 13 | post_process 14 | end 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/astrails/safe/backup.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Backup 4 | attr_accessor :id, :kind, :filename, :extension, :command, :compressed, :timestamp, :path 5 | def initialize(opts = {}) 6 | opts.each do |k, v| 7 | self.send("#{k}=", v) 8 | end 9 | end 10 | 11 | def run(config, *mods) 12 | mods.each do |mod| 13 | mod = mod.to_s 14 | mod[0] = mod[0..0].upcase 15 | Astrails::Safe.const_get(mod).new(config, self).process 16 | end 17 | end 18 | end 19 | end 20 | end -------------------------------------------------------------------------------- /lib/astrails/safe/archive.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Archive < Source 4 | 5 | def command 6 | "tar -cf - #{config[:options]} #{tar_exclude_files} #{tar_files}" 7 | end 8 | 9 | def extension; '.tar'; end 10 | 11 | protected 12 | 13 | def tar_exclude_files 14 | [*config[:exclude]].compact.map{|x| "--exclude=#{x}"}.join(" ") 15 | end 16 | 17 | def tar_files 18 | raise RuntimeError, "missing files for tar" unless config[:files] 19 | [*config[:files]].map{|s| s.strip}.join(" ") 20 | end 21 | 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/astrails/safe/mongodump.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Mongodump < Source 4 | 5 | def command 6 | opts = [] 7 | opts << "--host #{config[:host]}" if config[:host] 8 | opts << "-u #{config[:user]}" if config[:user] 9 | opts << "-p #{config[:password]}" if config[:password] 10 | opts << "--out #{output_directory}" 11 | 12 | "mongodump -q \"{xxxx : { \\$ne : 0 } }\" --db #{@id} #{opts.join(" ")} && cd #{output_directory} && tar cf - ." 13 | end 14 | 15 | def extension; '.tar'; end 16 | 17 | protected 18 | def output_directory 19 | File.join(TmpFile.tmproot, "mongodump") 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/astrails/safe/stream.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Stream 4 | 5 | attr_accessor :config, :backup 6 | def initialize(config, backup) 7 | @config, @backup = config, backup 8 | end 9 | # FIXME: move to Backup 10 | def expand(path) 11 | path . 12 | gsub(/:kind\b/, @backup.kind.to_s) . 13 | gsub(/:id\b/, @backup.id.to_s) . 14 | gsub(/:timestamp\b/, @backup.timestamp) 15 | end 16 | 17 | private 18 | 19 | def verbose? 20 | config[:verbose] 21 | end 22 | 23 | def local_only? 24 | config[:local_only] 25 | end 26 | 27 | def dry_run? 28 | config[:dry_run] 29 | end 30 | end 31 | end 32 | end -------------------------------------------------------------------------------- /lib/astrails/safe/pgdump.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Pgdump < Source 4 | 5 | def command 6 | if config["password"] 7 | ENV['PGPASSWORD'] = config["password"] 8 | else 9 | ENV['PGPASSWORD'] = nil 10 | end 11 | "pg_dump #{postgres_options} #{postgres_username} #{postgres_host} #{postgres_port} #{@id}" 12 | end 13 | 14 | def extension; '.sql'; end 15 | 16 | protected 17 | 18 | def postgres_options 19 | config[:options] 20 | end 21 | 22 | def postgres_host 23 | config["host"] && "--host='#{config["host"]}'" 24 | end 25 | 26 | def postgres_port 27 | config["port"] && "--port='#{config["port"]}'" 28 | end 29 | 30 | def postgres_username 31 | config["user"] && "--username='#{config["user"]}'" 32 | end 33 | 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/astrails/safe/mysqldump.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Mysqldump < Source 4 | 5 | def command 6 | "mysqldump --defaults-extra-file=#{mysql_password_file} #{config[:options]} #{mysql_skip_tables} #{@id}" 7 | end 8 | 9 | def extension; '.sql'; end 10 | 11 | protected 12 | 13 | def mysql_password_file 14 | Astrails::Safe::TmpFile.create("mysqldump") do |file| 15 | file.puts "[mysqldump]" 16 | %w/user password socket host port/.each do |k| 17 | v = config[k] 18 | # values are quoted if needed 19 | file.puts "#{k} = #{v.inspect}" if v 20 | end 21 | end 22 | end 23 | 24 | def mysql_skip_tables 25 | if skip_tables = config[:skip_tables] 26 | [*skip_tables].map{ |t| "--ignore-table=#{@id}.#{t}" }.join(" ") 27 | end 28 | end 29 | 30 | end 31 | end 32 | end -------------------------------------------------------------------------------- /spec/astrails/safe/svndump_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Astrails::Safe::Svndump do 4 | def def_config 5 | { 6 | :options => "OPTS", 7 | :repo_path => "bar/baz" 8 | } 9 | end 10 | 11 | def svndump(id = :foo, config = def_config) 12 | Astrails::Safe::Svndump.new(id, Astrails::Safe::Config::Node.new(nil, config)) 13 | end 14 | 15 | before(:each) do 16 | stub(Time).now.stub!.strftime {"NOW"} 17 | end 18 | 19 | after(:each) { Astrails::Safe::TmpFile.cleanup } 20 | 21 | describe :backup do 22 | before(:each) do 23 | @svn = svndump 24 | end 25 | 26 | { 27 | :id => "foo", 28 | :kind => "svndump", 29 | :extension => ".svn", 30 | :filename => "svndump-foo.NOW", 31 | :command => "svnadmin dump OPTS bar/baz", 32 | }.each do |k, v| 33 | it "should set #{k} to #{v}" do 34 | @svn.backup.send(k).should == v 35 | end 36 | end 37 | 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | 0.3.1 2 | 3 | * plain ftp support from seroy 4 | * mongodump support from Matt Berther 5 | 6 | 0.3.0 7 | 8 | * switch to bundler 9 | * fixed the rspec 10 | 11 | 0.2.8 12 | 13 | * ruby 1.9.2 compatibility (tests mostly) 14 | * code review, and tons of small fixes 15 | * check file size before attempting to upload to cloudfiles 16 | * testing framework changed from micronaut to rspec 17 | 18 | 0.2.7 19 | 20 | * default options for gpg now include '--no-use-agent' 21 | * support for 'command' option for gpg 22 | * quote values in mysql password file 23 | * add 'lib' to $: 24 | * [EXPERIMENTAL] Rackspace Cloud Files support 25 | 26 | 0.2.6 27 | 28 | * fix typo in the template config file. (change option to options in pgdump) 29 | * add example 'options' for tar in the template config file. 30 | * do not try to upload more then 5G of data to S3. print error instead 31 | 32 | 0.2.5 33 | 34 | * Safety mesure: Disable overwrite of existing configuration keys except for multi-value keys 35 | supported multi-value keys: skip_tables, exclude, files -------------------------------------------------------------------------------- /lib/astrails/safe/sink.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Sink < Stream 4 | 5 | def process 6 | return unless active? 7 | 8 | save 9 | cleanup 10 | end 11 | 12 | protected 13 | 14 | # path is defined in subclass 15 | # base is used in 'cleanup' to find all files that begin with base. the '.' 16 | # at the end is essential to distinguish b/w foo.* and foobar.* archives for example 17 | def base 18 | @base ||= File.join(path, File.basename(@backup.filename).split(".").first + '.') 19 | end 20 | 21 | def full_path 22 | @full_path ||= File.join(path, @backup.filename) + @backup.extension 23 | end 24 | 25 | # call block on files to be removed (all except for the LAST 'limit' files 26 | def cleanup_with_limit(files, limit, &block) 27 | return unless files.size > limit 28 | 29 | to_remove = files[0..(files.size - limit - 1)] 30 | # TODO: validate here 31 | to_remove.each(&block) 32 | end 33 | end 34 | end 35 | end -------------------------------------------------------------------------------- /spec/astrails/safe/pgdump_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Astrails::Safe::Pgdump do 4 | 5 | def def_config 6 | { 7 | :options => "OPTS", 8 | :user => "User", 9 | :password => "pwd", 10 | :host => "localhost", 11 | :port => 7777, 12 | :skip_tables => [:bar, :baz] 13 | } 14 | end 15 | 16 | def pgdump(id = :foo, config = def_config) 17 | Astrails::Safe::Pgdump.new(id, Astrails::Safe::Config::Node.new(nil, config)) 18 | end 19 | 20 | before(:each) do 21 | stub(Time).now.stub!.strftime {"NOW"} 22 | end 23 | 24 | after(:each) { Astrails::Safe::TmpFile.cleanup } 25 | 26 | describe :backup do 27 | before(:each) do 28 | @pg = pgdump 29 | end 30 | 31 | { 32 | :id => "foo", 33 | :kind => "pgdump", 34 | :extension => ".sql", 35 | :filename => "pgdump-foo.NOW", 36 | :command => "pg_dump OPTS --username='User' --host='localhost' --port='7777' foo", 37 | }.each do |k, v| 38 | it "should set #{k} to #{v}" do 39 | @pg.backup.send(k).should == v 40 | end 41 | end 42 | 43 | end 44 | 45 | end 46 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010-2013 Astrails Ltd. 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /lib/astrails/safe/gpg.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Gpg < Pipe 4 | 5 | protected 6 | 7 | def post_process 8 | @backup.compressed = true 9 | end 10 | 11 | def pipe 12 | command = config[:gpg, :command] || 'gpg' 13 | if key 14 | "|#{command} #{config[:gpg, :options]} -e -r #{key}" 15 | elsif password 16 | "|#{command} #{config[:gpg, :options]} -c --passphrase-file #{gpg_password_file(password)}" 17 | end 18 | end 19 | 20 | def extension 21 | ".gpg" 22 | end 23 | 24 | def active? 25 | raise RuntimeError, "can't use both gpg password and pubkey" if key && password 26 | 27 | !!(password || key) 28 | end 29 | 30 | private 31 | 32 | def password 33 | @password ||= config[:gpg, :password] 34 | end 35 | 36 | def key 37 | @key ||= config[:gpg, :key] 38 | end 39 | 40 | def gpg_password_file(pass) 41 | return "TEMP_GENERATED_FILENAME" if dry_run? 42 | Astrails::Safe::TmpFile.create("gpg-pass") { |file| file.write(pass) } 43 | end 44 | end 45 | end 46 | end -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | - refactor 2 | - refactor out global variables. pass a config object around instead 3 | - common logging 4 | - remove 1.8.6 support 5 | - module registry 6 | - base => prefix ? 7 | - move requires into specific modules 8 | - config.foo instead of config[:foo] 9 | 10 | - features 11 | - remote-only s3 support 12 | - generic notifier support 13 | - email notifier 14 | - hipchat 15 | - generic error notifier support 16 | - email 17 | - hipchat 18 | 19 | 20 | 21 | - add 'silent' 22 | - handle errors from mysqldump 23 | - check that gpg is installed 24 | - support percona XtraBackup as an option instead of mysqldump [patches anyone :) ?] 25 | - backup validation: 26 | - support for 'minsize' opition in backup that will check that produced backup is at least the expected size 27 | this should catch many backup failure scenarious (like broken mysql connection, insufficient disk space etc. 28 | - support differencial backups 29 | - it should be fairly easy for filesystem backups using tar's built in incremental functionality. 30 | - for mysql need to use XtraBackup 31 | - or we can keep the previous dump locally and store only diff with the latest dump 32 | -------------------------------------------------------------------------------- /lib/extensions/mktmpdir.rb: -------------------------------------------------------------------------------- 1 | require 'tmpdir' 2 | 3 | unless Dir.respond_to?(:mktmpdir) 4 | # backward compat for 1.8.6 5 | class Dir 6 | def Dir.mktmpdir(prefix_suffix=nil, tmpdir=nil) 7 | case prefix_suffix 8 | when nil 9 | prefix = "d" 10 | suffix = "" 11 | when String 12 | prefix = prefix_suffix 13 | suffix = "" 14 | when Array 15 | prefix = prefix_suffix[0] 16 | suffix = prefix_suffix[1] 17 | else 18 | raise ArgumentError, "unexpected prefix_suffix: #{prefix_suffix.inspect}" 19 | end 20 | tmpdir ||= Dir.tmpdir 21 | t = Time.now.strftime("%Y%m%d") 22 | n = nil 23 | begin 24 | path = "#{tmpdir}/#{prefix}#{t}-#{$$}-#{rand(0x100000000).to_s(36)}" 25 | path << "-#{n}" if n 26 | path << suffix 27 | Dir.mkdir(path, 0700) 28 | rescue Errno::EEXIST 29 | n ||= 0 30 | n += 1 31 | retry 32 | end 33 | 34 | if block_given? 35 | begin 36 | yield path 37 | ensure 38 | FileUtils.remove_entry_secure path 39 | end 40 | else 41 | path 42 | end 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /lib/astrails/safe/source.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Source < Stream 4 | 5 | attr_accessor :id 6 | def initialize(id, config) 7 | @id, @config = id.to_s, config 8 | end 9 | 10 | def timestamp 11 | Time.now.strftime("%y%m%d-%H%M") 12 | end 13 | 14 | def kind 15 | self.class.human_name 16 | end 17 | 18 | def filename 19 | @filename ||= expand(":kind-:id.:timestamp") 20 | end 21 | 22 | def backup 23 | return @backup if @backup 24 | @backup = Backup.new( 25 | :id => @id, 26 | :kind => kind, 27 | :extension => extension, 28 | :command => command, 29 | :timestamp => timestamp 30 | ) 31 | # can't do this in the initializer hash above since 32 | # filename() calls expand() which requires @backup 33 | # FIXME: move expansion to the backup (last step in ctor) assign :tags here 34 | @backup.filename = filename 35 | @backup 36 | end 37 | 38 | protected 39 | 40 | def self.human_name 41 | name.split('::').last.downcase 42 | end 43 | 44 | end 45 | end 46 | end 47 | 48 | -------------------------------------------------------------------------------- /lib/astrails/safe/tmp_file.rb: -------------------------------------------------------------------------------- 1 | require 'tmpdir' 2 | module Astrails 3 | module Safe 4 | module TmpFile 5 | @keep_files = [] 6 | 7 | def self.tmproot 8 | @tmproot ||= Dir.mktmpdir 9 | end 10 | 11 | def self.cleanup 12 | begin 13 | FileUtils.remove_entry_secure tmproot 14 | rescue ArgumentError => e 15 | if e.message =~ /parent directory is world writable/ 16 | puts <<-ERR 17 | 18 | 19 | ******************************************************************************** 20 | It looks like you have wrong permissions on your TEMP directory. The usual 21 | case is when you have world writable TEMP directory withOUT the sticky bit. 22 | 23 | Try "chmod +t" on it. 24 | 25 | ******************************************************************************** 26 | 27 | ERR 28 | else 29 | raise 30 | end 31 | end 32 | @tmproot = nil 33 | end 34 | 35 | def self.create(name) 36 | # create temp directory 37 | 38 | file = Tempfile.new(name, tmproot) 39 | 40 | yield file 41 | 42 | file.close 43 | @keep_files << file # so that it will not get gcollected and removed from filesystem until the end 44 | file.path 45 | end 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /astrails-safe.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'astrails/safe/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "astrails-safe" 8 | spec.version = Astrails::Safe::VERSION 9 | spec.authors = ["Vitaly Kushner"] 10 | spec.email = ["we@astrails.com"] 11 | spec.description = <<-DESC 12 | Astrails-Safe is a simple tool to backup databases (MySQL and PostgreSQL), Subversion repositories (with svndump) and just files. 13 | Backups can be stored locally or remotely and can be enctypted. 14 | Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain FTP/SFTP. 15 | DESC 16 | spec.summary = %Q{Backup filesystem and databases (MySQL and PostgreSQL) locally or to a remote server/service (with encryption)} 17 | spec.homepage = "http://astrails.com/blog/astrails-safe" 18 | spec.license = "MIT" 19 | 20 | spec.default_executable = %q{astrails-safe} 21 | 22 | spec.files = `git ls-files`.split($/) 23 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 24 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 25 | spec.require_paths = ["lib"] 26 | 27 | spec.add_dependency "aws-s3" 28 | spec.add_dependency "cloudfiles" 29 | spec.add_dependency "net-sftp" 30 | 31 | spec.add_development_dependency "bundler", "~> 1.3" 32 | spec.add_development_dependency "rake" 33 | spec.add_development_dependency "rspec" 34 | spec.add_development_dependency "rr", "~> 1.0.4" 35 | end 36 | -------------------------------------------------------------------------------- /lib/astrails/safe/local.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Local < Sink 4 | 5 | protected 6 | 7 | def active? 8 | # S3 can't upload from pipe. it needs to know file size, so we must pass through :local 9 | # will change once we add SSH/FTP sink 10 | true 11 | end 12 | 13 | def path 14 | @path ||= File.expand_path(expand(config[:local, :path] || raise(RuntimeError, "missing :local/:path"))) 15 | end 16 | 17 | def save 18 | puts "command: #{@backup.command}" if verbose? 19 | 20 | # FIXME: probably need to change this to smth like @backup.finalize! 21 | @backup.path = full_path # need to do it outside DRY_RUN so that it will be avialable for S3 DRY_RUN 22 | 23 | unless dry_run? 24 | FileUtils.mkdir_p(path) unless File.directory?(path) 25 | benchmark = Benchmark.realtime do 26 | system "#{@backup.command}>#{@backup.path}" 27 | end 28 | puts("command took " + sprintf("%.2f", benchmark) + " second(s).") if verbose? 29 | end 30 | 31 | end 32 | 33 | def cleanup 34 | return unless keep = config[:keep, :local] 35 | 36 | puts "listing files #{base}" if verbose? 37 | 38 | # TODO: cleanup ALL zero-length files 39 | 40 | files = Dir["#{base}*"] . 41 | select{|f| File.file?(f) && File.size(f) > 0} . 42 | sort 43 | 44 | cleanup_with_limit(files, keep) do |f| 45 | puts "removing local file #{f}" if dry_run? || verbose? 46 | File.unlink(f) unless dry_run? 47 | end 48 | end 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /spec/astrails/safe/mongodump_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Astrails::Safe::Mongodump do 4 | def def_config 5 | { 6 | :host => 'prod.example.com', 7 | :user => 'testuser', 8 | :password => 'p4ssw0rd', 9 | } 10 | end 11 | 12 | def mongodump(id = :foo, config = def_config) 13 | Astrails::Safe::Mongodump.new(id, Astrails::Safe::Config::Node.new(nil, config)) 14 | end 15 | 16 | before(:each) do 17 | stub(Time).now.stub!.strftime {"NOW"} 18 | @output_folder = File.join(Astrails::Safe::TmpFile.tmproot, 'mongodump') 19 | end 20 | 21 | after(:each) { Astrails::Safe::TmpFile.cleanup } 22 | 23 | describe :backup do 24 | before(:each) do 25 | @mongo = mongodump 26 | end 27 | 28 | { 29 | :id => "foo", 30 | :kind => "mongodump", 31 | :extension => ".tar", 32 | :filename => "mongodump-foo.NOW" 33 | }.each do |k, v| 34 | it "should set #{k} to #{v}" do 35 | @mongo.backup.send(k).should == v 36 | end 37 | end 38 | 39 | it "should set the command" do 40 | @mongo.backup.send(:command).should == "mongodump -q \"{xxxx : { \\$ne : 0 } }\" --db foo --host prod.example.com -u testuser -p p4ssw0rd --out #{@output_folder} && cd #{@output_folder} && tar cf - ." 41 | end 42 | 43 | { 44 | :host => "--host ", 45 | :user => "-u ", 46 | :password => "-p " 47 | }.each do |key, v| 48 | it "should not add #{key} to command if it is not present" do 49 | @mongo = mongodump(:foo, def_config.reject! {|k,v| k == key}) 50 | @mongo.backup.send(:command).should_not =~ /#{v}/ 51 | end 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /spec/astrails/safe/gzip_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Astrails::Safe::Gzip do 4 | 5 | def def_backup 6 | { 7 | :compressed => false, 8 | :command => "command", 9 | :extension => ".foo", 10 | :filename => "qweqwe" 11 | } 12 | end 13 | 14 | after(:each) { Astrails::Safe::TmpFile.cleanup } 15 | 16 | def gzip(config = {}, backup = def_backup) 17 | Astrails::Safe::Gzip.new( 18 | @config = Astrails::Safe::Config::Node.new(nil, config), 19 | @backup = Astrails::Safe::Backup.new(backup) 20 | ) 21 | end 22 | 23 | describe :preocess do 24 | 25 | describe "when not yet compressed" do 26 | before(:each) { @gzip = gzip } 27 | 28 | it "should add .gz extension" do 29 | mock(@backup.extension) << '.gz' 30 | @gzip.process 31 | end 32 | 33 | it "should add |gzip pipe" do 34 | mock(@backup.command) << '|gzip' 35 | @gzip.process 36 | end 37 | 38 | it "should set compressed" do 39 | mock(@backup).compressed = true 40 | @gzip.process 41 | end 42 | end 43 | 44 | describe "when already compressed" do 45 | 46 | before(:each) { @gzip = gzip({}, :extension => ".foo", :command => "foobar", :compressed => true) } 47 | 48 | it "should not touch extension" do 49 | @gzip.process 50 | @backup.extension.should == ".foo" 51 | end 52 | 53 | it "should not touch command" do 54 | @gzip.process 55 | @backup.command.should == "foobar" 56 | end 57 | 58 | it "should not touch compressed" do 59 | @gzip.process 60 | @backup.compressed.should == true 61 | end 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /bin/astrails-safe: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'rubygems' 4 | 5 | require 'astrails/safe' 6 | 7 | include Astrails::Safe 8 | 9 | def die(msg) 10 | puts "ERROR: #{msg}" 11 | exit 1 12 | end 13 | 14 | def usage 15 | puts <<-END 16 | Usage: astrails-safe [OPTIONS] CONFIG_FILE 17 | Options: 18 | -h, --help This help screen 19 | -v, --verbose be verbose, duh! 20 | -n, --dry-run just pretend, don't do anything. 21 | -L, --local skip S3 and Cloud Files 22 | 23 | Note: config file will be created from template if missing 24 | END 25 | exit 1 26 | end 27 | 28 | OPTS = [ 29 | '-h', '--help', 30 | '-v', '--verbose', '--not-verbose', 31 | '-n', '--dry-run', '--not-dry-run', 32 | '-L', '--local', '--not-local' 33 | ] 34 | def main 35 | opts = ARGV & OPTS 36 | args = ARGV - OPTS 37 | 38 | usage unless args.first 39 | usage if opts.delete("-h") || opts.delete("--help") 40 | 41 | config_file = File.expand_path(args.first) 42 | 43 | is_dry = (opts.delete('-n') || opts.delete('--dry-run')) && ! opts.delete('--not-dry-run') 44 | is_verbose = (opts.delete('-v') || opts.delete('--verbose')) && !opts.delete('--not-verbose') 45 | is_local_only = (opts.delete('-L') || opts.delete('--local')) && !opts.delete('--not-local') 46 | 47 | unless File.exists?(config_file) 48 | die "Missing configuration file. NOT CREATED! Rerun w/o the -n argument to create a template configuration file." if is_dry 49 | 50 | FileUtils.cp File.join(Astrails::Safe::ROOT, "templates", "script.rb"), config_file 51 | 52 | die "Created default #{config_file}. Please edit and run again." 53 | end 54 | 55 | config = eval(File.read(config_file)) 56 | 57 | config[:verbose] = is_verbose 58 | config[:dry_run] = is_dry 59 | config[:local_only] = is_local_only 60 | 61 | process config 62 | end 63 | 64 | main -------------------------------------------------------------------------------- /lib/astrails/safe.rb: -------------------------------------------------------------------------------- 1 | require "astrails/safe/version" 2 | 3 | require "aws/s3" 4 | require "cloudfiles" 5 | require 'net/sftp' 6 | require 'net/ftp' 7 | require 'fileutils' 8 | require 'benchmark' 9 | 10 | require 'tempfile' 11 | require 'extensions/mktmpdir' 12 | 13 | require 'astrails/safe/tmp_file' 14 | 15 | require 'astrails/safe/config/node' 16 | require 'astrails/safe/config/builder' 17 | 18 | require 'astrails/safe/stream' 19 | 20 | require 'astrails/safe/backup' 21 | 22 | require 'astrails/safe/source' 23 | require 'astrails/safe/mysqldump' 24 | require 'astrails/safe/pgdump' 25 | require 'astrails/safe/archive' 26 | require 'astrails/safe/svndump' 27 | require 'astrails/safe/mongodump' 28 | 29 | require 'astrails/safe/pipe' 30 | require 'astrails/safe/gpg' 31 | require 'astrails/safe/gzip' 32 | 33 | require 'astrails/safe/sink' 34 | require 'astrails/safe/local' 35 | require 'astrails/safe/s3' 36 | require 'astrails/safe/cloudfiles' 37 | require 'astrails/safe/sftp' 38 | require 'astrails/safe/ftp' 39 | 40 | module Astrails 41 | module Safe 42 | ROOT = File.join(File.dirname(__FILE__), "..", "..") 43 | 44 | def safe(&block) 45 | Config::Node.new(&block) 46 | end 47 | 48 | def process(config) 49 | 50 | [[Mysqldump, [:mysqldump, :databases]], 51 | [Pgdump, [:pgdump, :databases]], 52 | [Mongodump, [:mongodump, :databases]], 53 | [Archive, [:tar, :archives]], 54 | [Svndump, [:svndump, :repos]] 55 | ].each do |klass, path| 56 | if collection = config[*path] 57 | collection.each do |name, c| 58 | klass.new(name, c).backup.run(c, :gpg, :gzip, :local, :s3, :cloudfiles, :sftp, :ftp) 59 | end 60 | end 61 | end 62 | 63 | Astrails::Safe::TmpFile.cleanup 64 | end 65 | module_function :safe 66 | module_function :process 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /spec/integration/cleanup_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | require "fileutils" 4 | include FileUtils 5 | 6 | describe "tar backup" do 7 | before(:all) do 8 | # need both local and instance vars 9 | # instance variables are used in tests 10 | # local variables are used in the backup definition (instance vars can't be seen) 11 | @root = root = "tmp/cleanup_example" 12 | 13 | # clean state 14 | rm_rf @root 15 | mkdir_p @root 16 | 17 | # create source tree 18 | @src = src = "#{@root}/src" 19 | mkdir_p src 20 | 21 | File.open(qwe = "#{@src}/qwe", "w") {|f| f.write("qwe") } 22 | 23 | @dst = dst = "#{@root}/backup" 24 | mkdir_p "#{@dst}/archive" 25 | 26 | @now = Time.now 27 | @timestamp = @now.strftime("%y%m%d-%H%M") 28 | 29 | stub(Time).now {@now} # Freeze 30 | 31 | cp qwe, "#{dst}/archive/archive-foo.000001.tar.gz" 32 | cp qwe, "#{dst}/archive/archive-foo.000002.tar.gz" 33 | cp qwe, "#{dst}/archive/archive-foobar.000001.tar.gz" 34 | cp qwe, "#{dst}/archive/archive-foobar.000002.tar.gz" 35 | 36 | config = Astrails::Safe.safe do 37 | local :path => "#{dst}/:kind" 38 | tar do 39 | keep :local => 1 # only leave the latest 40 | archive :foo do 41 | files src 42 | end 43 | end 44 | end 45 | Astrails::Safe.process config 46 | 47 | @backup = "#{dst}/archive/archive-foo.#{@timestamp}.tar.gz" 48 | end 49 | 50 | it "should create backup file" do 51 | File.exists?(@backup).should be_true 52 | end 53 | 54 | it "should remove old backups" do 55 | Dir["#{@dst}/archive/archive-foo.*"].should == [@backup] 56 | end 57 | 58 | it "should NOT remove backups with base having same prefix" do 59 | Dir["#{@dst}/archive/archive-foobar.*"].sort.should == ["#{@dst}/archive/archive-foobar.000001.tar.gz", "#{@dst}/archive/archive-foobar.000002.tar.gz"] 60 | end 61 | 62 | end 63 | -------------------------------------------------------------------------------- /spec/astrails/safe/archive_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Astrails::Safe::Archive do 4 | 5 | def def_config 6 | { 7 | :options => "OPTS", 8 | :files => "apples", 9 | :exclude => "oranges" 10 | } 11 | end 12 | 13 | def archive(id = :foo, config = def_config) 14 | Astrails::Safe::Archive.new(id, Astrails::Safe::Config::Node.new(nil, config)) 15 | end 16 | 17 | after(:each) { Astrails::Safe::TmpFile.cleanup } 18 | 19 | describe :backup do 20 | before(:each) do 21 | @archive = archive 22 | stub(@archive).timestamp {"NOW"} 23 | end 24 | 25 | { 26 | :id => "foo", 27 | :kind => "archive", 28 | :extension => ".tar", 29 | :filename => "archive-foo.NOW", 30 | :command => "tar -cf - OPTS --exclude=oranges apples", 31 | }.each do |k, v| 32 | it "should set #{k} to #{v}" do 33 | @archive.backup.send(k).should == v 34 | end 35 | end 36 | end 37 | 38 | describe :tar_exclude_files do 39 | it "should return '' when no excludes" do 40 | archive(:foo, {}).send(:tar_exclude_files).should == '' 41 | end 42 | 43 | it "should accept single exclude as string" do 44 | archive(:foo, {:exclude => "bar"}).send(:tar_exclude_files).should == '--exclude=bar' 45 | end 46 | 47 | it "should accept multiple exclude as array" do 48 | archive(:foo, {:exclude => ["foo", "bar"]}).send(:tar_exclude_files).should == '--exclude=foo --exclude=bar' 49 | end 50 | end 51 | 52 | describe :tar_files do 53 | it "should raise RuntimeError when no files" do 54 | lambda { 55 | archive(:foo, {}).send(:tar_files) 56 | }.should raise_error(RuntimeError, "missing files for tar") 57 | end 58 | 59 | it "should accept single file as string" do 60 | archive(:foo, {:files => "foo"}).send(:tar_files).should == "foo" 61 | end 62 | 63 | it "should accept multiple files as array" do 64 | archive(:foo, {:files => ["foo", "bar"]}).send(:tar_files).should == "foo bar" 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /lib/astrails/safe/config/node.rb: -------------------------------------------------------------------------------- 1 | require 'astrails/safe/config/builder' 2 | module Astrails 3 | module Safe 4 | module Config 5 | class Node 6 | attr_reader :parent, :data 7 | 8 | def initialize(parent = nil, data = {}, &block) 9 | @parent = parent 10 | @data = {} 11 | merge data, &block 12 | end 13 | 14 | def merge data = {}, &block 15 | builder = Builder.new(self, data) 16 | builder.instance_eval(&block) if block 17 | self 18 | end 19 | 20 | # looks for the path from this node DOWN. will not delegate to parent 21 | def get(*path) 22 | key = path.shift 23 | value = @data[key.to_s] 24 | return value if (nil != value) && path.empty? 25 | 26 | value && value.get(*path) 27 | end 28 | 29 | # recursive find 30 | # starts at the node and continues to the parent 31 | def find(*path) 32 | get(*path) || @parent && @parent.find(*path) 33 | end 34 | alias :[] :find 35 | 36 | def set_multi(key, value) 37 | @data[key.to_s] ||= [] 38 | @data[key.to_s].concat [*value] 39 | end 40 | 41 | def set(key, value) 42 | @data[key.to_s] = value 43 | end 44 | alias :[]= :set 45 | 46 | def each(&block) 47 | @data.each(&block) 48 | end 49 | include Enumerable 50 | 51 | def to_hash 52 | @data.keys.inject({}) do |res, key| 53 | value = @data[key] 54 | res[key] = value.is_a?(Node) ? value.to_hash : value 55 | res 56 | end 57 | end 58 | 59 | def dump(indent = "") 60 | @data.each do |key, value| 61 | if value.is_a?(Node) 62 | puts "#{indent}#{key}:" 63 | value.dump(indent + " ") 64 | else 65 | puts "#{indent}#{key}: #{value.inspect}" 66 | end 67 | end 68 | end 69 | end 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/astrails/safe/ftp.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Ftp < Sink 4 | 5 | protected 6 | 7 | def active? 8 | host && user 9 | end 10 | 11 | def path 12 | @path ||= expand(config[:ftp, :path] || config[:local, :path] || ":kind/:id") 13 | end 14 | 15 | def save 16 | raise RuntimeError, "pipe-streaming not supported for FTP." unless @backup.path 17 | 18 | puts "Uploading #{host}:#{full_path} via FTP" if verbose? || dry_run? 19 | 20 | unless dry_run? || local_only? 21 | if !port 22 | port = 21 23 | end 24 | Net::FTP.open(host) do |ftp| 25 | ftp.connect(host, port) 26 | ftp.login(user, password) 27 | puts "Sending #{@backup.path} to #{full_path}" if verbose? 28 | begin 29 | ftp.put(@backup.path, full_path) 30 | rescue Net::FTPPermError 31 | puts "Ensuring remote path (#{path}) exists" if verbose? 32 | end 33 | end 34 | puts "...done" if verbose? 35 | end 36 | end 37 | 38 | def cleanup 39 | return if local_only? || dry_run? 40 | 41 | return unless keep = config[:keep, :ftp] 42 | 43 | puts "listing files: #{host}:#{base}*" if verbose? 44 | if !port 45 | port = 21 46 | end 47 | Net::FTP.open(host) do |ftp| 48 | ftp.connect(host, port) 49 | ftp.login(user, password) 50 | files = ftp.nlst(path) 51 | pattern = File.basename("#{base}") 52 | files = files.reject{ |x| !x.start_with?(pattern)} 53 | puts files.collect {|x| x} if verbose? 54 | 55 | files = files. 56 | collect {|x| x }. 57 | sort 58 | 59 | cleanup_with_limit(files, keep) do |f| 60 | file = File.join(path, f) 61 | puts "removing ftp file #{host}:#{file}" if dry_run? || verbose? 62 | ftp.delete(file) unless dry_run? || local_only? 63 | end 64 | end 65 | end 66 | 67 | def host 68 | config[:ftp, :host] 69 | end 70 | 71 | def user 72 | config[:ftp, :user] 73 | end 74 | 75 | def password 76 | config[:ftp, :password] 77 | end 78 | 79 | def port 80 | config[:ftp, :port] 81 | end 82 | 83 | end 84 | end 85 | end -------------------------------------------------------------------------------- /spec/astrails/safe/mysqldump_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Astrails::Safe::Mysqldump do 4 | 5 | def def_config(extra = {}) 6 | { 7 | :options => "OPTS", 8 | :user => "User", 9 | :password => "pwd", 10 | :host => "localhost", 11 | :port => 7777, 12 | :socket => "socket", 13 | :skip_tables => [:bar, :baz] 14 | }.merge(extra) 15 | end 16 | 17 | def mysqldump(id = :foo, config = def_config) 18 | Astrails::Safe::Mysqldump.new(id, Astrails::Safe::Config::Node.new(nil, config)) 19 | end 20 | 21 | before(:each) do 22 | stub(Time).now.stub!.strftime {"NOW"} 23 | end 24 | 25 | after(:each) { Astrails::Safe::TmpFile.cleanup } 26 | 27 | describe :backup do 28 | before(:each) do 29 | @mysql = mysqldump 30 | stub(@mysql).mysql_password_file {"/tmp/pwd"} 31 | end 32 | 33 | { 34 | :id => "foo", 35 | :kind => "mysqldump", 36 | :extension => ".sql", 37 | :filename => "mysqldump-foo.NOW", 38 | :command => "mysqldump --defaults-extra-file=/tmp/pwd OPTS --ignore-table=foo.bar --ignore-table=foo.baz foo", 39 | }.each do |k, v| 40 | it "should set #{k} to #{v}" do 41 | @mysql.backup.send(k).should == v 42 | end 43 | end 44 | 45 | end 46 | 47 | describe :mysql_skip_tables do 48 | it "should return nil if no skip_tables" do 49 | config = def_config.dup 50 | config.delete(:skip_tables) 51 | m = mysqldump(:foo, Astrails::Safe::Config::Node.new(nil, config)) 52 | stub(m).timestamp {"NOW"} 53 | m.send(:mysql_skip_tables).should be_nil 54 | m.backup.command.should_not match(/ignore-table/) 55 | end 56 | 57 | it "should return '' if skip_tables empty" do 58 | config = def_config.dup 59 | config[:skip_tables] = [] 60 | m = mysqldump(:foo, Astrails::Safe::Config::Node.new(nil, config)) 61 | stub(m).timestamp {"NOW"} 62 | m.send(:mysql_skip_tables).should == "" 63 | m.backup.command.should_not match(/ignore-table/) 64 | end 65 | 66 | end 67 | 68 | describe :mysql_password_file do 69 | it "should create passwords file with quoted values" do 70 | m = mysqldump(:foo, def_config(:password => '#qwe"asd\'zxc')) 71 | file = m.send(:mysql_password_file) 72 | File.exists?(file).should == true 73 | File.read(file).should == <<-PWD 74 | [mysqldump] 75 | user = "User" 76 | password = "#qwe\\"asd'zxc" 77 | socket = "socket" 78 | host = "localhost" 79 | port = 7777 80 | PWD 81 | end 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /lib/astrails/safe/s3.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class S3 < Sink 4 | MAX_S3_FILE_SIZE = 5368709120 5 | 6 | protected 7 | 8 | def active? 9 | bucket && key && secret 10 | end 11 | 12 | def path 13 | @path ||= expand(config[:s3, :path] || config[:local, :path] || ":kind/:id") 14 | end 15 | 16 | def save 17 | # FIXME: user friendly error here :) 18 | raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path 19 | 20 | # needed in cleanup even on dry run 21 | AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless local_only? 22 | 23 | puts "Uploading #{bucket}:#{full_path}" if verbose? || dry_run? 24 | unless dry_run? || local_only? 25 | if File.stat(@backup.path).size > MAX_S3_FILE_SIZE 26 | STDERR.puts "ERROR: File size exceeds maximum allowed for upload to S3 (#{MAX_S3_FILE_SIZE}): #{@backup.path}" 27 | return 28 | end 29 | benchmark = Benchmark.realtime do 30 | AWS::S3::Bucket.create(bucket) unless bucket_exists?(bucket) 31 | File.open(@backup.path) do |file| 32 | AWS::S3::S3Object.store(full_path, file, bucket) 33 | end 34 | end 35 | puts "...done" if verbose? 36 | puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if verbose? 37 | end 38 | end 39 | 40 | def cleanup 41 | return if local_only? 42 | 43 | return unless keep = config[:keep, :s3] 44 | 45 | puts "listing files: #{bucket}:#{base}*" if verbose? 46 | files = AWS::S3::Bucket.objects(bucket, :prefix => base, :max_keys => keep * 2) 47 | puts files.collect {|x| x.key} if verbose? 48 | 49 | files = files. 50 | collect {|x| x.key}. 51 | sort 52 | 53 | cleanup_with_limit(files, keep) do |f| 54 | puts "removing s3 file #{bucket}:#{f}" if dry_run? || verbose? 55 | AWS::S3::Bucket.objects(bucket, :prefix => f)[0].delete unless dry_run? || local_only? 56 | end 57 | end 58 | 59 | def bucket 60 | config[:s3, :bucket] 61 | end 62 | 63 | def key 64 | config[:s3, :key] 65 | end 66 | 67 | def secret 68 | config[:s3, :secret] 69 | end 70 | 71 | private 72 | 73 | def bucket_exists?(bucket) 74 | true if AWS::S3::Bucket.find(bucket) 75 | rescue AWS::S3::NoSuchBucket 76 | false 77 | end 78 | end 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /lib/astrails/safe/sftp.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Sftp < Sink 4 | 5 | protected 6 | 7 | def active? 8 | host && user 9 | end 10 | 11 | def path 12 | @path ||= expand(config[:sftp, :path] || config[:local, :path] || ":kind/:id") 13 | end 14 | 15 | def save 16 | raise RuntimeError, "pipe-streaming not supported for SFTP." unless @backup.path 17 | 18 | puts "Uploading #{host}:#{full_path} via SFTP" if verbose? || dry_run? 19 | 20 | unless dry_run? || local_only? 21 | opts = {} 22 | opts[:password] = password if password 23 | opts[:port] = port if port 24 | Net::SFTP.start(host, user, opts) do |sftp| 25 | puts "Sending #{@backup.path} to #{full_path}" if verbose? 26 | begin 27 | sftp.upload! @backup.path, full_path 28 | rescue Net::SFTP::StatusException 29 | puts "Ensuring remote path (#{path}) exists" if verbose? 30 | # mkdir -p 31 | folders = path.split('/') 32 | folders.each_index do |i| 33 | folder = folders[0..i].join('/') 34 | puts "Creating #{folder} on remote" if verbose? 35 | sftp.mkdir!(folder) rescue Net::SFTP::StatusException 36 | end 37 | retry 38 | end 39 | end 40 | puts "...done" if verbose? 41 | end 42 | end 43 | 44 | def cleanup 45 | return if local_only? || dry_run? 46 | 47 | return unless keep = config[:keep, :sftp] 48 | 49 | puts "listing files: #{host}:#{base}*" if verbose? 50 | opts = {} 51 | opts[:password] = password if password 52 | opts[:port] = port if port 53 | Net::SFTP.start(host, user, opts) do |sftp| 54 | files = sftp.dir.glob(path, File.basename("#{base}*")) 55 | 56 | puts files.collect {|x| x.name } if verbose? 57 | 58 | files = files. 59 | collect {|x| x.name }. 60 | sort 61 | 62 | cleanup_with_limit(files, keep) do |f| 63 | file = File.join(path, f) 64 | puts "removing sftp file #{host}:#{file}" if dry_run? || verbose? 65 | sftp.remove!(file) unless dry_run? || local_only? 66 | end 67 | end 68 | end 69 | 70 | def host 71 | config[:sftp, :host] 72 | end 73 | 74 | def user 75 | config[:sftp, :user] 76 | end 77 | 78 | def password 79 | config[:sftp, :password] 80 | end 81 | 82 | def port 83 | config[:sftp, :port] 84 | end 85 | 86 | end 87 | end 88 | end -------------------------------------------------------------------------------- /lib/astrails/safe/cloudfiles.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | class Cloudfiles < Sink 4 | MAX_CLOUDFILES_FILE_SIZE = 5368709120 5 | 6 | protected 7 | 8 | def active? 9 | container && user && api_key 10 | end 11 | 12 | def path 13 | @path ||= expand(config[:cloudfiles, :path] || config[:local, :path] || ":kind/:id") 14 | end 15 | 16 | # UGLY: we need this function for the reason that 17 | # we can't double mock on ruby 1.9.2, duh! 18 | # so we created this func to mock it all together 19 | def get_file_size(path) 20 | File.stat(path).size 21 | end 22 | 23 | def save 24 | raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path 25 | 26 | # needed in cleanup even on dry run 27 | cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless local_only? 28 | puts "Uploading #{container}:#{full_path} from #{@backup.path}" if verbose? || dry_run? 29 | unless dry_run? || local_only? 30 | if get_file_size(@backup.path) > MAX_CLOUDFILES_FILE_SIZE 31 | STDERR.puts "ERROR: File size exceeds maximum allowed for upload to Cloud Files (#{MAX_CLOUDFILES_FILE_SIZE}): #{@backup.path}" 32 | return 33 | end 34 | benchmark = Benchmark.realtime do 35 | cf_container = cf.create_container(container) 36 | o = cf_container.create_object(full_path,true) 37 | o.write(File.open(@backup.path)) 38 | end 39 | puts "...done" if verbose? 40 | puts("Upload took " + sprintf("%.2f", benchmark) + " second(s).") if verbose? 41 | end 42 | end 43 | 44 | def cleanup 45 | return if local_only? 46 | 47 | return unless keep = config[:keep, :cloudfiles] 48 | 49 | puts "listing files: #{container}:#{base}*" if verbose? 50 | cf = CloudFiles::Connection.new(user, api_key, true, service_net) unless local_only? 51 | cf_container = cf.container(container) 52 | files = cf_container.objects(:prefix => base).sort 53 | 54 | cleanup_with_limit(files, keep) do |f| 55 | puts "removing Cloud File #{container}:#{f}" if dry_run? || verbose? 56 | cf_container.delete_object(f) unless dry_run? || local_only? 57 | end 58 | end 59 | 60 | def container 61 | config[:cloudfiles, :container] 62 | end 63 | 64 | def user 65 | config[:cloudfiles, :user] 66 | end 67 | 68 | def api_key 69 | config[:cloudfiles, :api_key] 70 | end 71 | 72 | def service_net 73 | config[:cloudfiles, :service_net] || false 74 | end 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /spec/integration/archive_integration_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | require "fileutils" 4 | include FileUtils 5 | 6 | describe "tar backup" do 7 | before(:all) do 8 | # need both local and instance vars 9 | # instance variables are used in tests 10 | # local variables are used in the backup definition (instance vars can't be seen) 11 | @root = root = "tmp/archive_backup_example" 12 | 13 | # clean state 14 | rm_rf @root 15 | mkdir_p @root 16 | 17 | # create source tree 18 | @src = src = "#{@root}/src" 19 | mkdir_p "#{@src}/q/w/e" 20 | mkdir_p "#{@src}/a/s/d" 21 | 22 | File.open("#{@src}/qwe1", "w") {|f| f.write("qwe") } 23 | File.open("#{@src}/q/qwe2", "w") {|f| f.write("qwe"*2) } 24 | File.open("#{@src}/q/w/qwe3", "w") {|f| f.write("qwe"*3) } 25 | File.open("#{@src}/q/w/e/qwe4", "w") {|f| f.write("qwe"*4) } 26 | 27 | File.open("#{@src}/asd1", "w") {|f| f.write("asd") } 28 | File.open("#{@src}/a/asd2", "w") {|f| f.write("asd" * 2) } 29 | File.open("#{@src}/a/s/asd3", "w") {|f| f.write("asd" * 3) } 30 | 31 | @dst = dst = "#{@root}/backup" 32 | mkdir_p @dst 33 | 34 | @now = Time.now 35 | @timestamp = @now.strftime("%y%m%d-%H%M") 36 | 37 | stub(Time).now {@now} # Freeze 38 | 39 | config = Astrails::Safe.safe do 40 | local :path => "#{dst}/:kind" 41 | tar do 42 | archive :test1 do 43 | files src 44 | exclude "#{src}/q/w" 45 | exclude "#{src}/q/w/e" 46 | end 47 | end 48 | end 49 | Astrails::Safe.process config 50 | 51 | @backup = "#{dst}/archive/archive-test1.#{@timestamp}.tar.gz" 52 | end 53 | 54 | it "should create backup file" do 55 | File.exists?(@backup).should be_true 56 | end 57 | 58 | describe "after extracting" do 59 | before(:all) do 60 | # prepare target dir 61 | @target = "#{@root}/test" 62 | mkdir_p @target 63 | system "tar -zxvf #{@backup} -C #{@target}" 64 | 65 | @test = "#{@target}/#{@root}/src" 66 | puts @test 67 | end 68 | 69 | it "should include asd1/2/3" do 70 | File.exists?("#{@test}/asd1").should be_true 71 | File.exists?("#{@test}/a/asd2").should be_true 72 | File.exists?("#{@test}/a/s/asd3").should be_true 73 | end 74 | 75 | it "should only include qwe 1 and 2 (no 3)" do 76 | File.exists?("#{@test}/qwe1").should be_true 77 | File.exists?("#{@test}/q/qwe2").should be_true 78 | File.exists?("#{@test}/q/w/qwe3").should be_false 79 | File.exists?("#{@test}/q/w/e/qwe4").should be_false 80 | end 81 | 82 | it "should preserve file content" do 83 | File.read("#{@test}/qwe1").should == "qwe" 84 | File.read("#{@test}/q/qwe2").should == "qweqwe" 85 | File.read("#{@test}/a/s/asd3").should == "asdasdasd" 86 | end 87 | end 88 | 89 | end 90 | -------------------------------------------------------------------------------- /lib/astrails/safe/config/builder.rb: -------------------------------------------------------------------------------- 1 | module Astrails 2 | module Safe 3 | module Config 4 | class Builder 5 | 6 | def initialize(node, data = {}) 7 | @node = node 8 | data.each { |k, v| self.send k, v } 9 | end 10 | 11 | 12 | class << self 13 | def simple_value(*names) 14 | names.each do |m| 15 | define_method(m) do |value| 16 | ensure_uniq(m) 17 | @node.set m, value 18 | end 19 | end 20 | end 21 | 22 | def multi_value(*names) 23 | names.each do |m| 24 | define_method(m) do |value| 25 | value = value.map(&:to_s) if value.is_a?(Array) 26 | @node.set_multi m, value 27 | end 28 | end 29 | end 30 | 31 | def hash_value(*names) 32 | names.each do |m| 33 | define_method(m) do |data = {}, &block| 34 | ensure_uniq(m) 35 | ensure_hash(m, data) 36 | @node.set m, Node.new(@node, data || {}, &block) 37 | end 38 | end 39 | end 40 | 41 | def mixed_value(*names) 42 | names.each do |m| 43 | define_method(m) do |data={}, &block| 44 | ensure_uniq(m) 45 | if data.is_a?(Hash) || block 46 | ensure_hash(m, data) if block 47 | @node.set m, Node.new(@node, data, &block) 48 | else 49 | @node.set m, data 50 | end 51 | end 52 | end 53 | end 54 | 55 | def collection(*names) 56 | names.each do |m| 57 | define_method(m) do |id, data={}, &block| 58 | raise "bad collection id: #{id.inspect}" unless id 59 | ensure_hash(m, data) 60 | 61 | name = "#{m}s" 62 | collection = @node.get(name) || @node.set(name, Node.new(@node, {})) 63 | collection.set id, Node.new(collection, data, &block) 64 | end 65 | end 66 | end 67 | end 68 | 69 | simple_value :verbose, :dry_run, :local_only, :path, :command, 70 | :options, :user, :host, :port, :password, :key, :secret, :bucket, 71 | :api_key, :container, :socket, :service_net, :repo_path 72 | multi_value :skip_tables, :exclude, :files 73 | hash_value :mysqldump, :tar, :gpg, :keep, :pgdump, :tar, :svndump, 74 | :sftp, :ftp, :mongodump 75 | mixed_value :s3, :local, :cloudfiles 76 | collection :database, :archive, :repo 77 | 78 | private 79 | 80 | def ensure_uniq(m) 81 | raise(ArgumentError, "duplicate value for '#{m}'") if @node.get(m) 82 | end 83 | 84 | def ensure_hash(k, v) 85 | raise "#{k}: hash expected: #{v.inspect}" unless v.is_a?(Hash) 86 | end 87 | end 88 | end 89 | end 90 | end 91 | -------------------------------------------------------------------------------- /spec/astrails/safe/local_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Astrails::Safe::Local do 4 | def def_config 5 | { 6 | :local => { 7 | :path => "/:kind~:id~:timestamp" 8 | }, 9 | :keep => { 10 | :local => 2 11 | } 12 | } 13 | end 14 | 15 | def def_backup 16 | { 17 | :kind => "mysqldump", 18 | :id => "blog", 19 | :timestamp => "NoW", 20 | :compressed => true, 21 | :command => "command", 22 | :extension => ".foo.gz", 23 | :filename => "qweqwe" 24 | } 25 | end 26 | 27 | def local(config = def_config, backup = def_backup) 28 | Astrails::Safe::Local.new( 29 | @config = Astrails::Safe::Config::Node.new(nil, config), 30 | @backup = Astrails::Safe::Backup.new(backup) 31 | ) 32 | end 33 | 34 | describe :active? do 35 | it "should be true" do 36 | local.should be_active 37 | end 38 | end 39 | 40 | describe :path do 41 | it "should raise RuntimeError when no path" do 42 | lambda { 43 | local({}).send :path 44 | }.should raise_error(RuntimeError, "missing :local/:path") 45 | end 46 | 47 | it "should use local/path" do 48 | local.send(:path).should == "/mysqldump~blog~NoW" 49 | end 50 | end 51 | 52 | describe :save do 53 | before(:each) do 54 | @local = local 55 | stub(@local).system 56 | stub(@local).full_path {"file-path"} 57 | stub(FileUtils).mkdir_p 58 | end 59 | 60 | it "should call system to save the file" do 61 | mock(@local).system("command>file-path") 62 | @local.send(:save) 63 | end 64 | 65 | it "should create directory" do 66 | mock(FileUtils).mkdir_p("/mysqldump~blog~NoW") 67 | @local.send(:save) 68 | end 69 | 70 | it "should set backup.path" do 71 | mock(@backup).path = "file-path" 72 | @local.send(:save) 73 | end 74 | 75 | describe "dry run" do 76 | before(:each) { @local.config[:dry_run] = true } 77 | 78 | it "should not create directory" 79 | it "should not call system" 80 | it "should set backup.path" do 81 | mock(@backup).path = "file-path" 82 | @local.send(:save) 83 | end 84 | end 85 | end 86 | 87 | describe :cleanup do 88 | before(:each) do 89 | @files = [4,1,3,2].map { |i| "/mysqldump~blog~NoW/qweqwe.#{i}" } 90 | stub(File).file?(anything) {true} 91 | stub(File).size(anything) {1} 92 | stub(File).unlink 93 | end 94 | 95 | it "should check [:keep, :local]" do 96 | @local = local(def_config.merge(:keep => {})) 97 | dont_allow(Dir).[] 98 | @local.send :cleanup 99 | end 100 | 101 | it "should delete extra files" do 102 | @local = local 103 | mock(Dir).[]("/mysqldump~blog~NoW/qweqwe.*") {@files} 104 | mock(File).unlink("/mysqldump~blog~NoW/qweqwe.1") 105 | mock(File).unlink("/mysqldump~blog~NoW/qweqwe.2") 106 | @local.send :cleanup 107 | end 108 | end 109 | end 110 | -------------------------------------------------------------------------------- /spec/astrails/safe/gpg_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Astrails::Safe::Gpg do 4 | def def_backup 5 | { 6 | :compressed => false, 7 | :command => "command", 8 | :extension => ".foo", 9 | :filename => "qweqwe" 10 | } 11 | end 12 | 13 | def gpg(config = {}, backup = def_backup) 14 | Astrails::Safe::Gpg.new( 15 | Astrails::Safe::Config::Node.new(nil, config), 16 | Astrails::Safe::Backup.new(backup) 17 | ) 18 | end 19 | 20 | after(:each) { Astrails::Safe::TmpFile.cleanup } 21 | 22 | describe :process do 23 | 24 | before(:each) do 25 | @gpg = gpg() 26 | stub(@gpg).gpg_password_file {"pwd-file"} 27 | stub(@gpg).pipe {"|gpg -BLAH"} 28 | end 29 | 30 | describe "when active" do 31 | before(:each) do 32 | stub(@gpg).active? {true} 33 | end 34 | 35 | it "should add .gpg extension" do 36 | mock(@gpg.backup.extension) << '.gpg' 37 | @gpg.process 38 | end 39 | 40 | it "should add command pipe" do 41 | mock(@gpg.backup.command) << (/\|gpg -BLAH/) 42 | @gpg.process 43 | end 44 | 45 | it "should set compressed" do 46 | mock(@gpg.backup).compressed = true 47 | @gpg.process 48 | end 49 | end 50 | 51 | describe "when inactive" do 52 | before(:each) do 53 | stub(@gpg).active? {false} 54 | end 55 | 56 | it "should not touch extension" do 57 | dont_allow(@gpg.backup.extension) << anything 58 | @gpg.process 59 | end 60 | 61 | it "should not touch command" do 62 | dont_allow(@gpg.backup.command) << anything 63 | @gpg.process 64 | end 65 | 66 | it "should not touch compressed" do 67 | dont_allow(@gpg.backup).compressed = anything 68 | @gpg.process 69 | end 70 | end 71 | end 72 | 73 | describe :active? do 74 | 75 | describe "with key" do 76 | it "should be true" do 77 | gpg(:gpg => {:key => :foo}).should be_active 78 | end 79 | end 80 | 81 | describe "with password" do 82 | it "should be true" do 83 | gpg(:gpg => {:password => :foo}).should be_active 84 | end 85 | end 86 | 87 | describe "without key & password" do 88 | it "should be false" do 89 | gpg.should_not be_active 90 | end 91 | end 92 | 93 | describe "with key & password" do 94 | it "should raise RuntimeError" do 95 | lambda { 96 | gpg(:gpg => {:key => "foo", :password => "bar"}).send :active? 97 | }.should raise_error(RuntimeError, "can't use both gpg password and pubkey") 98 | end 99 | end 100 | end 101 | 102 | describe :pipe do 103 | 104 | describe "with key" do 105 | def kgpg(extra={}) 106 | gpg({:gpg => {:key => "foo", :options => "GPG-OPT"}.merge(extra), :options => "OPT"}) 107 | end 108 | 109 | it "should not call gpg_password_file" do 110 | g = kgpg 111 | dont_allow(g).gpg_password_file(anything) 112 | g.send(:pipe) 113 | end 114 | 115 | it "should use '-r' and :options" do 116 | kgpg.send(:pipe).should == "|gpg GPG-OPT -e -r foo" 117 | end 118 | 119 | it "should use the 'command' options" do 120 | kgpg(:command => 'other-gpg').send(:pipe).should == "|other-gpg GPG-OPT -e -r foo" 121 | end 122 | end 123 | 124 | describe "with password" do 125 | def pgpg(extra = {}) 126 | returning(gpg({:gpg => {:password => "bar", :options => "GPG-OPT"}.merge(extra), :options => "OPT"})) do |g| 127 | stub(g).gpg_password_file(anything) {"pass-file"} 128 | end 129 | end 130 | 131 | it "should use '--passphrase-file' and :options" do 132 | pgpg.send(:pipe).should == "|gpg GPG-OPT -c --passphrase-file pass-file" 133 | end 134 | 135 | it "should use the 'command' options" do 136 | pgpg(:command => 'other-gpg').send(:pipe).should == "|other-gpg GPG-OPT -c --passphrase-file pass-file" 137 | end 138 | end 139 | end 140 | 141 | describe :gpg_password_file do 142 | it "should create password file" do 143 | file = gpg.send(:gpg_password_file, "foo") 144 | File.exists?(file).should be_true 145 | File.read(file).should == "foo" 146 | end 147 | end 148 | end 149 | -------------------------------------------------------------------------------- /spec/astrails/safe/s3_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Astrails::Safe::S3 do 4 | 5 | def def_config 6 | { 7 | :s3 => { 8 | :bucket => "_bucket", 9 | :key => "_key", 10 | :secret => "_secret", 11 | }, 12 | :keep => { 13 | :s3 => 2 14 | } 15 | } 16 | end 17 | 18 | def def_backup(extra = {}) 19 | { 20 | :kind => "_kind", 21 | :filename => "/backup/somewhere/_kind-_id.NOW.bar", 22 | :extension => ".bar", 23 | :id => "_id", 24 | :timestamp => "NOW" 25 | }.merge(extra) 26 | end 27 | 28 | def s3(config = def_config, backup = def_backup) 29 | Astrails::Safe::S3.new( 30 | Astrails::Safe::Config::Node.new.merge(config), 31 | Astrails::Safe::Backup.new(backup) 32 | ) 33 | end 34 | 35 | describe :cleanup do 36 | 37 | before(:each) do 38 | @s3 = s3 39 | 40 | @files = [4,1,3,2].map { |i| stub(o = {}).key {"aaaaa#{i}"}; o } 41 | 42 | stub(AWS::S3::Bucket).objects("_bucket", :prefix => "_kind/_id/_kind-_id.", :max_keys => 4) {@files} 43 | stub(AWS::S3::Bucket).objects("_bucket", :prefix => anything).stub![0].stub!.delete 44 | end 45 | 46 | it "should check [:keep, :s3]" do 47 | @s3.config[:keep].data["s3"] = nil 48 | dont_allow(@s3.backup).filename 49 | @s3.send :cleanup 50 | end 51 | 52 | it "should delete extra files" do 53 | mock(AWS::S3::Bucket).objects("_bucket", :prefix => "aaaaa1").mock![0].mock!.delete 54 | mock(AWS::S3::Bucket).objects("_bucket", :prefix => "aaaaa2").mock![0].mock!.delete 55 | @s3.send :cleanup 56 | end 57 | 58 | end 59 | 60 | describe :active do 61 | before(:each) do 62 | @s3 = s3 63 | end 64 | 65 | it "should be true when all params are set" do 66 | @s3.should be_active 67 | end 68 | 69 | it "should be false if bucket is missing" do 70 | @s3.config[:s3].data["bucket"] = nil 71 | @s3.should_not be_active 72 | end 73 | 74 | it "should be false if key is missing" do 75 | @s3.config[:s3].data["key"] = nil 76 | @s3.should_not be_active 77 | end 78 | 79 | it "should be false if secret is missing" do 80 | @s3.config[:s3].data["secret"] = nil 81 | @s3.should_not be_active 82 | end 83 | end 84 | 85 | describe :path do 86 | before(:each) do 87 | @s3 = s3 88 | end 89 | it "should use s3/path 1st" do 90 | @s3.config[:s3].data["path"] = "s3_path" 91 | @s3.config[:local] = {:path => "local_path"} 92 | @s3.send(:path).should == "s3_path" 93 | end 94 | 95 | it "should use local/path 2nd" do 96 | @s3.config.merge local: {path: "local_path"} 97 | @s3.send(:path).should == "local_path" 98 | end 99 | 100 | it "should use constant 3rd" do 101 | @s3.send(:path).should == "_kind/_id" 102 | end 103 | 104 | end 105 | 106 | describe :save do 107 | def add_stubs(*stubs) 108 | stubs.each do |s| 109 | case s 110 | when :connection 111 | stub(AWS::S3::Base).establish_connection!(:access_key_id => "_key", :secret_access_key => "_secret", :use_ssl => true) 112 | when :stat 113 | stub(File).stat("foo").stub!.size {123} 114 | when :create_bucket 115 | stub(AWS::S3::Bucket).find('_bucket') { raise_error AWS::S3::NoSuchBucket } 116 | stub(AWS::S3::Bucket).create 117 | when :file_open 118 | stub(File).open("foo") {|f, block| block.call(:opened_file)} 119 | when :s3_store 120 | stub(AWS::S3::S3Object).store(@full_path, :opened_file, "_bucket") 121 | end 122 | end 123 | end 124 | 125 | before(:each) do 126 | @s3 = s3(def_config, def_backup(:path => "foo")) 127 | @full_path = "_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar" 128 | end 129 | 130 | it "should fail if no backup.file is set" do 131 | @s3.backup.path = nil 132 | proc {@s3.send(:save)}.should raise_error(RuntimeError) 133 | end 134 | 135 | it "should establish s3 connection" do 136 | mock(AWS::S3::Base).establish_connection!(:access_key_id => "_key", :secret_access_key => "_secret", :use_ssl => true) 137 | add_stubs(:stat, :create_bucket, :file_open, :s3_store) 138 | @s3.send(:save) 139 | end 140 | 141 | it "should open local file" do 142 | add_stubs(:connection, :stat, :create_bucket) 143 | mock(File).open("foo") 144 | @s3.send(:save) 145 | end 146 | 147 | it "should upload file" do 148 | add_stubs(:connection, :stat, :create_bucket, :file_open) 149 | mock(AWS::S3::S3Object).store(@full_path, :opened_file, "_bucket") 150 | @s3.send(:save) 151 | end 152 | 153 | it "should fail on files bigger then 5G" do 154 | add_stubs(:connection) 155 | mock(File).stat("foo").stub!.size {5*1024*1024*1024+1} 156 | mock(STDERR).puts(anything) 157 | dont_allow(Benchmark).realtime 158 | @s3.send(:save) 159 | end 160 | 161 | it 'should not create a bucket that already exists' do 162 | add_stubs(:connection, :stat, :file_open, :s3_store) 163 | stub(AWS::S3::Bucket).find('_bucket') { true } 164 | dont_allow(AWS::S3::Bucket).create 165 | @s3.send(:save) 166 | end 167 | end 168 | end 169 | -------------------------------------------------------------------------------- /spec/astrails/safe/cloudfiles_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Astrails::Safe::Cloudfiles do 4 | 5 | def def_config 6 | { 7 | cloudfiles: { 8 | container: '_container', 9 | user: '_user', 10 | api_key: '_api_key', 11 | }, 12 | keep: { cloudfiles: 2 } 13 | } 14 | end 15 | 16 | def def_backup(extra = {}) 17 | { 18 | kind: '_kind', 19 | filename: '/backup/somewhere/_kind-_id.NOW.bar', 20 | extension: '.bar', 21 | id: '_id', 22 | timestamp: 'NOW' 23 | }.merge(extra) 24 | end 25 | 26 | def cloudfiles(config = def_config, backup = def_backup) 27 | Astrails::Safe::Cloudfiles.new( 28 | Astrails::Safe::Config::Node.new.merge(config), 29 | Astrails::Safe::Backup.new(backup) 30 | ) 31 | end 32 | 33 | describe :cleanup do 34 | 35 | before(:each) do 36 | @cloudfiles = cloudfiles 37 | 38 | @files = [4,1,3,2].map { |i| "aaaaa#{i}" } 39 | 40 | @container = "container" 41 | 42 | stub(@container).objects(prefix: "_kind/_id/_kind-_id.") { @files } 43 | stub(@container).delete_object(anything) 44 | 45 | stub(CloudFiles::Connection). 46 | new('_user', '_api_key', true, false).stub!. 47 | container('_container') {@container} 48 | end 49 | 50 | it "should check [:keep, :cloudfiles]" do 51 | @cloudfiles.config[:keep].data["cloudfiles"] = nil 52 | dont_allow(@cloudfiles.backup).filename 53 | @cloudfiles.send :cleanup 54 | end 55 | 56 | it "should delete extra files" do 57 | mock(@container).delete_object('aaaaa1') 58 | mock(@container).delete_object('aaaaa2') 59 | @cloudfiles.send :cleanup 60 | end 61 | 62 | end 63 | 64 | describe :active do 65 | before(:each) do 66 | @cloudfiles = cloudfiles 67 | end 68 | 69 | it "should be true when all params are set" do 70 | @cloudfiles.should be_active 71 | end 72 | 73 | it "should be false if container is missing" do 74 | @cloudfiles.config[:cloudfiles].data["container"] = nil 75 | @cloudfiles.should_not be_active 76 | end 77 | 78 | it "should be false if user is missing" do 79 | @cloudfiles.config[:cloudfiles].data["user"] = nil 80 | @cloudfiles.should_not be_active 81 | end 82 | 83 | it "should be false if api_key is missing" do 84 | @cloudfiles.config[:cloudfiles].data["api_key"] = nil 85 | @cloudfiles.should_not be_active 86 | end 87 | end 88 | 89 | describe :path do 90 | before(:each) do 91 | @cloudfiles = cloudfiles 92 | end 93 | it "should use cloudfiles/path 1st" do 94 | @cloudfiles.config[:cloudfiles].data["path"] = "cloudfiles_path" 95 | @cloudfiles.config[:local] = {path: "local_path"} 96 | @cloudfiles.send(:path).should == "cloudfiles_path" 97 | end 98 | 99 | it "should use local/path 2nd" do 100 | @cloudfiles.config.merge local: {path: 'local_path'} 101 | @cloudfiles.send(:path).should == 'local_path' 102 | end 103 | 104 | it "should use constant 3rd" do 105 | @cloudfiles.send(:path).should == "_kind/_id" 106 | end 107 | 108 | end 109 | 110 | describe :save do 111 | def add_stubs(*stubs) 112 | stubs.each do |s| 113 | case s 114 | when :connection 115 | @connection = "connection" 116 | stub(CloudFiles::Authentication).new 117 | stub(CloudFiles::Connection). 118 | new('_user', '_api_key', true, false) {@connection} 119 | when :file_size 120 | stub(@cloudfiles).get_file_size("foo") {123} 121 | when :create_container 122 | @container = "container" 123 | stub(@container).create_object("_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar", true) {@object} 124 | stub(@connection).create_container {@container} 125 | when :file_open 126 | stub(File).open("foo") 127 | when :cloudfiles_store 128 | @object = "object" 129 | stub(@object).write(nil) {true} 130 | end 131 | end 132 | end 133 | 134 | before(:each) do 135 | @cloudfiles = cloudfiles(def_config, def_backup(path: 'foo')) 136 | @full_path = "_kind/_id/backup/somewhere/_kind-_id.NOW.bar.bar" 137 | end 138 | 139 | it "should fail if no backup.file is set" do 140 | @cloudfiles.backup.path = nil 141 | proc {@cloudfiles.send(:save)}.should raise_error(RuntimeError) 142 | end 143 | 144 | it "should establish Cloud Files connection" do 145 | add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store) 146 | @cloudfiles.send(:save) 147 | end 148 | 149 | it "should open local file" do 150 | add_stubs(:connection, :file_size, :create_container, :cloudfiles_store) 151 | mock(File).open("foo") 152 | @cloudfiles.send(:save) 153 | end 154 | 155 | it "should call write on the cloudfile object with files' descriptor" do 156 | add_stubs(:connection, :file_size, :create_container, :cloudfiles_store) 157 | stub(File).open("foo") {"qqq"} 158 | mock(@object).write("qqq") {true} 159 | @cloudfiles.send(:save) 160 | end 161 | 162 | it "should upload file" do 163 | add_stubs(:connection, :file_size, :create_container, :file_open, :cloudfiles_store) 164 | @cloudfiles.send(:save) 165 | end 166 | 167 | it "should fail on files bigger then 5G" do 168 | add_stubs(:connection) 169 | mock(File).stat("foo").stub!.size {5*1024*1024*1024+1} 170 | mock(STDERR).puts(anything) 171 | dont_allow(Benchmark).realtime 172 | @cloudfiles.send(:save) 173 | end 174 | end 175 | end 176 | -------------------------------------------------------------------------------- /templates/script.rb: -------------------------------------------------------------------------------- 1 | safe do 2 | # same as --verbose on the command line 3 | # verbose true 4 | 5 | # same as --local on the command line 6 | # local_only true 7 | 8 | # same as --dry-run on the command line 9 | # dry_run true 10 | 11 | # backup file path (not including filename) 12 | # supported substitutions: 13 | # :kind -> backup 'engine' kind, e.g. "mysqldump" or "archive" 14 | # :id -> backup 'id', e.g. "blog", "production", etc. 15 | # :timestamp -> current run timestamp (same for all the backups in the same 'run') 16 | # you can set separate :path for all backups (or once globally here) 17 | local do 18 | path "/backup/:kind" 19 | end 20 | 21 | ## uncomment to enable uploads to Amazon S3 22 | ## Amazon S3 auth (optional) 23 | # s3 do 24 | # key YOUR_S3_KEY 25 | # secret YOUR_S3_SECRET 26 | # bucket S3_BUCKET 27 | # # path for uploads to S3. supports same substitution like :local/:path 28 | # path ":kind/" # this is default 29 | # end 30 | 31 | ## alternative style: 32 | # s3 :key => YOUR_S3_KEY, :secret => YOUR_S3_SECRET, :bucket => S3_BUCKET, :path => ":kind/" 33 | 34 | ## uncomment to enable uploads to Rackspace Cloud Files 35 | ## http://www.rackspacecloud.com/cloud_hosting_products/files 36 | ## Rackspace auth (optional) 37 | # cloudfiles do 38 | # user "YOUR_RACKSPACE_CLOUD_USERNAME" 39 | # api_key "YOUR_RACKSPACE_API_KEY" 40 | # container "YOUR_CONTAINER_NAME" 41 | # # path for uploads to Cloud Files, supports same substitution like :local/:path 42 | # path ":kind/" # this is default 43 | # # If you are running the backup from a system within the Rackspace/Slicehost network and would like 44 | # # to back up over the private (unbilled) service net, set this value to true. 45 | # # service_net true 46 | # end 47 | 48 | ## uncomment to enable uploads via SFTP 49 | # sftp do 50 | # host "YOUR_REMOTE_HOSTNAME" 51 | # user "YOUR_REMOTE_USERNAME" 52 | # # port "NON STANDARD SSH PORT" 53 | # password "YOUR_REMOTE_PASSWORD" 54 | # path ":kind/:id" # this is the default 55 | # end 56 | 57 | ## uncomment to enable uploads via FTP 58 | # ftp do 59 | # host "YOUR_REMOTE_HOSTNAME" 60 | # user "YOUR_REMOTE_USERNAME" 61 | # # port "NON STANDARD FTP PORT" 62 | # password "YOUR_REMOTE_PASSWORD" 63 | # path ":kind/:id" # this is the default 64 | # end 65 | 66 | ## uncomment to enable GPG encryption. 67 | ## Note: you can use public 'key' or symmetric password but not both! 68 | # gpg do 69 | # # you can specify your own gpg executable with the 'command' options 70 | # # this can be useful for example to choose b/w gpg and gpg2 if both are installed 71 | # # some gpg installations will automatically set 'use-agent' option in the 72 | # # config file on the 1st run. see README for more details 73 | # options "--no-use-agent" 74 | # # command "/usr/local/bin/gpg" 75 | # # key "backup@astrails.com" 76 | # password "astrails" 77 | # end 78 | 79 | ## uncomment to enable backup rotation. keep only given number of latest 80 | ## backups. remove the rest 81 | # keep do 82 | # local 4 # keep 4 local backups 83 | # s3 20 # keep 20 S3 backups 84 | # end 85 | 86 | # backup mysql databases with mysqldump 87 | mysqldump do 88 | # you can override any setting from parent in a child: 89 | options "-ceKq --single-transaction --create-options" 90 | 91 | user "astrails" 92 | password "" 93 | # host "localhost" 94 | # port 3306 95 | socket "/var/run/mysqld/mysqld.sock" 96 | 97 | # database is a 'collection' element. it must have a hash or block parameter 98 | # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key 99 | # the following code will create mysqldump/databases/blog and mysqldump/databases/mysql configuration 'nodes' 100 | 101 | # backup database with default values 102 | # database :blog 103 | 104 | # backup overriding some values 105 | # database :production do 106 | # # you can override 'partially' 107 | # keep :local => 3 108 | # # keep/local is 3, and keep/s3 is 20 (from parent) 109 | 110 | # # local override for gpg password 111 | # gpg do 112 | # password "custom-production-pass" 113 | # end 114 | # # skip those tables during backup 115 | # # you can pass an array 116 | # skip_tables [:logger_exceptions, :request_logs] 117 | # # or pass them all separately 118 | # skip_tables :test1 119 | # skip_tables :test2 120 | # end 121 | 122 | end 123 | 124 | # # uncomment to enable 125 | # # backup PostgreSQL databases with pg_dump 126 | # pgdump do 127 | # options "-i -x -O" 128 | # 129 | # user "markmansour" 130 | # # password "" - leave this out if you have ident setup 131 | # 132 | # # database is a 'collection' element. it must have a hash or block parameter 133 | # # it will be 'collected' in a 'databases', with database id (1st arg) used as hash key 134 | # database :blog 135 | # database :production 136 | # end 137 | 138 | tar do 139 | # options "-h" # uncomment this to dereference symbolic links 140 | 141 | # 'archive' is a collection item, just like 'database' 142 | # archive "git-repositories" do 143 | # # files and directories to backup 144 | # files "/home/git/repositories" 145 | # # can have more then one 'files' lines or/and use an array 146 | # files ["/home/dev/work/foo", "/home/dev/work/bar"] 147 | # end 148 | 149 | # archive "etc-files" do 150 | # files "/etc" 151 | # # exlude those files/directories 152 | # exclude "/etc/puppet/other" 153 | # # can have multiple 'exclude' lines or/and use an array 154 | # exclude ["/etc/tmp/a", "/etc/tmp/b"] 155 | # end 156 | 157 | # archive "dot-configs" do 158 | # files "/home/*/.[^.]*" 159 | # end 160 | 161 | # archive "blog" do 162 | # files "/var/www/blog.astrails.com/" 163 | # # specify multiple files/directories as array 164 | # exclude ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"] 165 | # end 166 | 167 | # archive "site" do 168 | # files "/var/www/astrails.com/" 169 | # exclude ["/var/www/astrails.com/log", "/var/www/astrails.com/tmp"] 170 | # end 171 | 172 | # archive :misc do 173 | # files [ "/backup/*.rb" ] 174 | # end 175 | end 176 | 177 | # svndump do 178 | # repo :my_repo do 179 | # repo_path "/home/svn/my_repo" 180 | # end 181 | # end 182 | 183 | end 184 | -------------------------------------------------------------------------------- /spec/astrails/safe/config_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Astrails::Safe::Config do 4 | it "should parse example config" do 5 | config = Astrails::Safe::Config::Node.new do 6 | 7 | dry_run false 8 | local_only true 9 | verbose true 10 | 11 | local do 12 | path "path" 13 | end 14 | 15 | s3 do 16 | key "s3 key" 17 | secret "secret" 18 | bucket "bucket" 19 | path "path1" 20 | end 21 | 22 | sftp do 23 | user "sftp user" 24 | password "sftp password" 25 | host "sftp host" 26 | end 27 | 28 | gpg do 29 | password "astrails" 30 | key "gpg-key" 31 | end 32 | 33 | keep do 34 | s3 20 35 | local 4 36 | end 37 | 38 | mysqldump do 39 | options "-ceKq --single-transaction --create-options" 40 | 41 | user "astrails" 42 | password "" 43 | host "localhost" 44 | port 3306 45 | socket "/var/run/mysqld/mysqld.sock" 46 | 47 | database :blog 48 | 49 | database :production do 50 | keep :local => 3 51 | 52 | gpg do 53 | password "custom-production-pass" 54 | end 55 | 56 | skip_tables [:logger_exceptions, :request_logs] 57 | end 58 | 59 | end 60 | 61 | pgdump do 62 | options "-i -x -O" 63 | 64 | user "astrails" 65 | password "" 66 | host "localhost" 67 | port 5432 68 | 69 | database :blog 70 | 71 | database :production do 72 | keep :local => 3 73 | 74 | skip_tables [:logger_exceptions, :request_logs] 75 | end 76 | 77 | end 78 | 79 | svndump do 80 | repo :my_repo do 81 | repo_path "/home/svn/my_repo" 82 | end 83 | end 84 | 85 | tar do 86 | archive "git-repositories" do 87 | files "/home/git/repositories" 88 | end 89 | 90 | archive "etc-files" do 91 | files "/etc" 92 | exclude "/etc/puppet/other" 93 | end 94 | 95 | archive "dot-configs" do 96 | files "/home/*/.[^.]*" 97 | end 98 | 99 | archive "blog" do 100 | files "/var/www/blog.astrails.com/" 101 | exclude ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"] 102 | end 103 | 104 | archive :misc do 105 | files [ "/backup/*.rb" ] 106 | end 107 | end 108 | 109 | mongodump do 110 | host "host" 111 | database "database" 112 | user "user" 113 | password "password" 114 | end 115 | end 116 | 117 | expected = { 118 | "dry_run" => false, 119 | "local_only" => true, 120 | "verbose" => true, 121 | 122 | "local" => {"path" => "path"}, 123 | 124 | "s3" => { 125 | "key" => "s3 key", 126 | "secret" => "secret", 127 | "bucket" => "bucket", 128 | "path" => "path1", 129 | }, 130 | 131 | "sftp" => { 132 | "user" => "sftp user", 133 | "password" => "sftp password", 134 | "host" => "sftp host", 135 | }, 136 | 137 | "gpg" => {"password" => "astrails", "key" => "gpg-key"}, 138 | 139 | "keep" => {"s3" => 20, "local" => 4}, 140 | 141 | "mysqldump" => { 142 | "options" => "-ceKq --single-transaction --create-options", 143 | "user" => "astrails", 144 | "password" => "", 145 | "host" => "localhost", 146 | "port" => 3306, 147 | "socket" => "/var/run/mysqld/mysqld.sock", 148 | 149 | "databases" => { 150 | "blog" => {}, 151 | "production" => { 152 | "keep" => {"local" => 3}, 153 | "gpg" => {"password" => "custom-production-pass"}, 154 | "skip_tables" => ["logger_exceptions", "request_logs"], 155 | }, 156 | }, 157 | }, 158 | 159 | "pgdump" => { 160 | "options" => "-i -x -O", 161 | "user" => "astrails", 162 | "password" => "", 163 | "host" => "localhost", 164 | "port" => 5432, 165 | 166 | "databases" => { 167 | "blog" => {}, 168 | "production" => { 169 | "keep" => {"local" => 3}, 170 | "skip_tables" => ["logger_exceptions", "request_logs"], 171 | }, 172 | }, 173 | }, 174 | 175 | "svndump" => { 176 | "repos" => { 177 | "my_repo"=> { 178 | "repo_path" => "/home/svn/my_repo" 179 | } 180 | } 181 | }, 182 | 183 | "tar" => { 184 | "archives" => { 185 | "git-repositories" => {"files" => ["/home/git/repositories"]}, 186 | "etc-files" => {"files" => ["/etc"], "exclude" => ["/etc/puppet/other"]}, 187 | "dot-configs" => {"files" => ["/home/*/.[^.]*"]}, 188 | "blog" => { 189 | "files" => ["/var/www/blog.astrails.com/"], 190 | "exclude" => ["/var/www/blog.astrails.com/log", "/var/www/blog.astrails.com/tmp"], 191 | }, 192 | "misc" => { "files" => ["/backup/*.rb"] }, 193 | }, 194 | }, 195 | 196 | "mongodump" => { 197 | "host" => "host", 198 | "databases" => { 199 | "database" => {} 200 | }, 201 | "user" => "user", 202 | "password" => "password" 203 | } 204 | } 205 | 206 | config.to_hash.should == expected 207 | end 208 | 209 | it "should make an array from multivalues" do 210 | config = Astrails::Safe::Config::Node.new do 211 | skip_tables "a" 212 | skip_tables "b" 213 | files "/foo" 214 | files "/bar" 215 | exclude "/foo/bar" 216 | exclude "/foo/bar/baz" 217 | end 218 | 219 | expected = { 220 | "skip_tables" => ["a", "b"], 221 | "files" => ["/foo", "/bar"], 222 | "exclude" => ["/foo/bar", "/foo/bar/baz"], 223 | } 224 | 225 | config.to_hash.should == expected 226 | end 227 | 228 | it "should raise error on key duplication" do 229 | proc do 230 | Astrails::Safe::Config::Node.new do 231 | path "foo" 232 | path "bar" 233 | end 234 | end.should raise_error(ArgumentError, "duplicate value for 'path'") 235 | end 236 | 237 | it "should accept hash as data" do 238 | Astrails::Safe::Config::Node.new do 239 | tar do 240 | archive 'blog', files: 'foo', exclude: ['aaa', 'bbb'] 241 | end 242 | end.to_hash.should == { 243 | 'tar' => { 244 | 'archives' => { 245 | 'blog' => { 246 | 'files' => ['foo'], 247 | 'exclude' => ['aaa', 'bbb'] 248 | } 249 | } 250 | } 251 | } 252 | end 253 | 254 | it "should accept hash as data and a block" do 255 | Astrails::Safe::Config::Node.new do 256 | tar do 257 | archive 'blog', files: 'foo' do 258 | exclude ['aaa', 'bbb'] 259 | end 260 | end 261 | end.to_hash.should == { 262 | 'tar' => { 263 | 'archives' => { 264 | 'blog' => { 265 | 'files' => ['foo'], 266 | 'exclude' => ['aaa', 'bbb'] 267 | } 268 | } 269 | } 270 | } 271 | end 272 | 273 | it 'should accept multiple levels of data hash' do 274 | config = Astrails::Safe::Config::Node.new nil, tar: { 275 | s3: { bucket: '_bucket', key: '_key', secret: '_secret', }, 276 | keep: { s3: 2 } 277 | } 278 | 279 | config.to_hash.should == { 280 | 'tar' => { 281 | 's3' => { 'bucket' => '_bucket', 'key' => '_key', 'secret' => '_secret', }, 282 | 'keep' => { 's3' => 2 } 283 | } 284 | } 285 | end 286 | 287 | it 'should set multi value as array' do 288 | config = Astrails::Safe::Config::Node.new do 289 | tar do 290 | archive 'foo' do 291 | files 'bar' 292 | end 293 | end 294 | end 295 | 296 | config.to_hash.should == { 297 | 'tar' => { 298 | 'archives' => { 299 | 'foo' => { 300 | 'files' => ['bar'] 301 | } 302 | } 303 | } 304 | } 305 | end 306 | 307 | end 308 | -------------------------------------------------------------------------------- /README.markdown: -------------------------------------------------------------------------------- 1 | # astrails-safe 2 | 3 | Simple database and filesystem backups with S3 and Rackspace Cloud Files support (with optional encryption) 4 | 5 | * Home: [http://astrails.com/opensource/astrails-safe](http://astrails.com/opensource/astrails-safe) 6 | * Code: [http://github.com/astrails/safe](http://github.com/astrails/safe) 7 | * Blog: [http://astrails.com/blog/astrails-safe](http://astrails.com/blog/astrails-safe) 8 | 9 | [![Build Status](https://travis-ci.org/astrails/safe.png)](https://travis-ci.org/astrails/safe) 10 | [![Code Climate](https://codeclimate.com/github/astrails/safe.png)](https://codeclimate.com/github/astrails/safe) 11 | 12 | ## Motivation 13 | 14 | We needed a backup solution that will satisfy the following requirements: 15 | 16 | * opensource 17 | * simple to install and configure 18 | * support for simple ‘tar’ backups of directories (with includes/excludes) 19 | * support for simple mysqldump of mysql databases 20 | * support for symmetric or public key encryption 21 | * support for local filesystem, Amazon S3, and Rackspace Cloud Files for storage 22 | * support for backup rotation. we don’t want backups filling all the diskspace or cost a fortune on S3 or Cloud Files 23 | 24 | And since we didn't find any, we wrote our own :) 25 | 26 | ## Contributions 27 | 28 | The following functionality was contributed by astrails-safe users: 29 | 30 | * PostgreSQL dump using `pg_dump` (by Mark Mansour ) 31 | * Subversion dump using svndump (by Richard Luther ) 32 | * SFTP remote storage (by Adam ) 33 | * benchmarking output (By Neer) 34 | * README fixes (by Bobby Wilson) 35 | * improved config file parsing (by Fedor Kocherga ) 36 | * mysql password file quoting (by Jonathan Sutherland ) 37 | * Rackspace Cloud Files support (by H. Wade Minter ) 38 | * Plan FTP support (by seroy ) 39 | * mongodump support (by Matt Berther ) 40 | 41 | Thanks to all :) 42 | 43 | ## Installation 44 | 45 | sudo gem install astrails-safe --source http://gemcutter.org 46 | 47 | ## Reporting problems 48 | 49 | Please report problems at the [Issues tracker](http://github.com/astrails/safe/issues) 50 | 51 | ## Usage 52 | 53 | Usage: 54 | astrails-safe [OPTIONS] CONFIG_FILE 55 | Options: 56 | -h, --help This help screen 57 | -v, --verbose be verbose, duh! 58 | -n, --dry-run just pretend, don't do anything. 59 | -L, --local skip remote storage, only do local backups 60 | 61 | Note: CONFIG\_FILE will be created from template if missing 62 | 63 | ## Encryption 64 | 65 | If you want to encrypt your backups you have 2 options: 66 | * use simple password encryption 67 | * use GPG public key encryption 68 | 69 | > IMPORTANT: some gpg installations automatically set 'use-agent' option in the default 70 | > configuration file that is created when you run gpg for the first time. This will cause 71 | > gpg to fail on the 2nd run if you don't have the agent running. The result is that 72 | > 'astrails-safe' will work ONCE when you manually test it and then fail on any subsequent run. 73 | > The solution is to remove the 'use-agent' from the config file (usually /root/.gnupg/gpg.conf) 74 | > To mitigate this problem for the gpg 1.x series '--no-use-agent' option is added by defaults 75 | > to the autogenerated config file, but for gpg2 is doesn't work. as the manpage says it: 76 | > "This is dummy option. gpg2 always requires the agent." :( 77 | 78 | For simple password, just add password entry in gpg section. 79 | For public key encryption you will need to create a public/secret keypair. 80 | 81 | We recommend to create your GPG keys only on your local machine and then 82 | transfer your public key to the server that will do the backups. 83 | 84 | This way the server will only know how to encrypt the backups but only you 85 | will be able to decrypt them using the secret key you have locally. Of course 86 | you MUST backup your backup encryption key :) 87 | We recommend also pringing the hard paper copy of your GPG key 'just in case'. 88 | 89 | The procedure to create and transfer the key is as follows: 90 | 91 | 1. run 'gpg --gen-key' on your local machine and follow onscreen instructions to create the key 92 | (you can accept all the defaults). 93 | 94 | 2. extract your public key into a file (assuming you used test@example.com as your key email): 95 | `gpg -a --export test@example.com > test@example.com.pub` 96 | 97 | 3. transfer public key to the server 98 | `scp test@example.com.pub root@example.com:` 99 | 100 | 4. import public key on the remote system: 101 | 102 | $ gpg --import test@example.com.pub 103 | gpg: key 45CA9403: public key "Test Backup " imported 104 | gpg: Total number processed: 1 105 | gpg: imported: 1 106 | 107 | 5. since we don't keep the secret part of the key on the remote server, gpg has 108 | no way to know its yours and can be trusted. 109 | To fix that we can sign it with other trusted key, or just directly modify its 110 | trust level in gpg (use level 5): 111 | 112 | $ gpg --edit-key test@example.com 113 | ... 114 | Command> trust 115 | ... 116 | 1 = I don't know or won't say 117 | 2 = I do NOT trust 118 | 3 = I trust marginally 119 | 4 = I trust fully 120 | 5 = I trust ultimately 121 | m = back to the main menu 122 | 123 | Your decision? 5 124 | ... 125 | Command> quit 126 | 127 | 6. export your secret key for backup 128 | (we recommend to print it on paper and burn to a CD/DVD and store in a safe place): 129 | 130 | $ gpg -a --export-secret-key test@example.com > test@example.com.key 131 | 132 | 133 | 134 | ## Example configuration 135 | 136 | safe do 137 | verbose true 138 | 139 | local :path => "/backup/:kind/:id" 140 | 141 | s3 do 142 | key "...................." 143 | secret "........................................" 144 | bucket "backup.astrails.com" 145 | path "servers/alpha/:kind/:id" 146 | end 147 | 148 | cloudfiles do 149 | user "..........." 150 | api_key "................................." 151 | container "safe_backup" 152 | path ":kind/" # this is default 153 | service_net false 154 | end 155 | 156 | sftp do 157 | host "sftp.astrails.com" 158 | user "astrails" 159 | # port 8023 160 | password "ssh password for sftp" 161 | end 162 | 163 | gpg do 164 | command "/usr/local/bin/gpg" 165 | options "--no-use-agent" 166 | # symmetric encryption key 167 | # password "qwe" 168 | 169 | # public GPG key (must be known to GPG, i.e. be on the keyring) 170 | key "backup@astrails.com" 171 | end 172 | 173 | keep do 174 | local 20 175 | s3 100 176 | cloudfiles 100 177 | sftp 100 178 | end 179 | 180 | mysqldump do 181 | options "-ceKq --single-transaction --create-options" 182 | 183 | user "root" 184 | password "............" 185 | socket "/var/run/mysqld/mysqld.sock" 186 | 187 | database :blog 188 | database :servershape 189 | database :astrails_com 190 | database :secret_project_com do 191 | skip_tables "foo" 192 | skip_tables ["bar", "baz"] 193 | end 194 | 195 | end 196 | 197 | svndump do 198 | repo :my_repo do 199 | repo_path "/home/svn/my_repo" 200 | end 201 | end 202 | 203 | pgdump do 204 | options "-i -x -O" # -i => ignore version, -x => do not dump privileges (grant/revoke), -O => skip restoration of object ownership in plain text format 205 | 206 | user "username" 207 | password "............" # shouldn't be used, instead setup ident. Current functionality exports a password env to the shell which pg_dump uses - untested! 208 | 209 | database :blog 210 | database :stateofflux_com 211 | end 212 | 213 | tar do 214 | options "-h" # dereference symlinks 215 | archive "git-repositories", :files => "/home/git/repositories" 216 | archive "dot-configs", :files => "/home/*/.[^.]*" 217 | archive "etc", :files => "/etc", :exclude => "/etc/puppet/other" 218 | 219 | archive "blog-astrails-com" do 220 | files "/var/www/blog.astrails.com/" 221 | exclude "/var/www/blog.astrails.com/log" 222 | exclude "/var/www/blog.astrails.com/tmp" 223 | end 224 | 225 | archive "astrails-com" do 226 | files "/var/www/astrails.com/" 227 | exclude ["/var/www/astrails.com/log", "/var/www/astrails.com/tmp"] 228 | end 229 | end 230 | end 231 | 232 | ## Contributing 233 | 234 | 1. Fork it 235 | 2. Create your feature branch (`git checkout -b my-new-feature`) 236 | 3. Commit your changes (`git commit -am 'Add some feature'`) 237 | 4. Push to the branch (`git push origin my-new-feature`) 238 | 5. Create new Pull Request 239 | 240 | ## Copyright 241 | 242 | Copyright (c) 2010-2013 Astrails Ltd. See LICENSE.txt for details. 243 | --------------------------------------------------------------------------------