├── .gitignore ├── Gemfile ├── README.md ├── Rakefile ├── bin └── mys3ql ├── lib ├── mys3ql.rb └── mys3ql │ ├── conductor.rb │ ├── config.rb │ ├── mysql.rb │ ├── s3.rb │ ├── shell.rb │ └── version.rb └── mys3ql.gemspec /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | .bundle 3 | Gemfile.lock 4 | pkg/* 5 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | # Specify your gem's dependencies in mys3ql.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mys3ql = mysql + s3 2 | 3 | Simple backup of your MySQL database onto Amazon S3. 4 | 5 | See [Example: mysqldump + mysqlbinlog for Backup and Restore](https://dev.mysql.com/doc/refman/5.7/en/mysqlbinlog-backup.html#mysqlbinlog-backup-example). 6 | 7 | 8 | ## Quick start 9 | 10 | Install and configure as below. 11 | 12 | To perform a full backup: 13 | 14 | $ mys3ql full 15 | 16 | If you are using MySql's binary logging (see below), back up the binary logs like this: 17 | 18 | $ mys3ql incremental 19 | 20 | To restore from the latest backup (plus binlogs if present): 21 | 22 | $ mys3ql restore 23 | 24 | To restore a recent subset of binlogs: 25 | 26 | $ mys3ql restore --after NUMBER 27 | 28 | – where NUMBER is a 6-digit binlog file number. 29 | 30 | By default mys3ql looks for a configuration file at `~/.mys3ql`. You can override this like so: 31 | 32 | $ mys3ql [command] -c FILE 33 | $ mys3ql [command] --config=FILE 34 | 35 | 36 | ## Installation 37 | 38 | First install the gem: 39 | 40 | $ gem install mys3ql 41 | 42 | Second, create your config file: 43 | 44 | mysql: 45 | # Database to back up 46 | database: 47 | # MySql credentials 48 | user: 49 | password: 50 | # Path (with trailing slash) to mysql commands e.g. mysqldump 51 | bin_path: /usr/local/mysql/bin/ 52 | # If you are using MySql binary logging: 53 | # Path to the binary logs, should match the log_bin option in your my.cnf. 54 | # Comment out if you are not using mysql binary logging 55 | bin_log: /var/lib/mysql/binlog/mysql-bin 56 | 57 | s3: 58 | # S3 credentials 59 | access_key_id: XXXXXXXXXXXXXXXXXXXX 60 | secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 61 | # Bucket in which to store your backups 62 | bucket: db_backups 63 | # AWS region your bucket lives in. 64 | # (I suspect you only need to specify this when your 'location' is in a different region.) 65 | #region: eu-west-1 66 | 67 | If you only have one database to back up on your server, you can put the config file at `~/.mys3ql`. Otherwise, tell the `mys3ql` command where the config file is with the `--config=FILE` switch. 68 | 69 | ## Binary logging 70 | 71 | To use incremental backups you need to enable binary logging by making sure that the MySQL config file (`/etc/my.cnf`) has the following line in it: 72 | 73 | log_bin = /var/db/mysql/binlog/mysql-bin 74 | 75 | The MySQL user needs to have the RELOAD and the SUPER privileges. These can be granted with the following SQL commands (which need to be executed as the MySQL root user): 76 | 77 | GRANT RELOAD ON *.* TO 'user_name'@'%' IDENTIFIED BY 'password'; 78 | GRANT SUPER ON *.* TO 'user_name'@'%' IDENTIFIED BY 'password'; 79 | 80 | You may need to run mys3ql's incremental backup with special permissions (sudo) depending on the ownership of the binlogs directory. 81 | 82 | N.B. the binary logs contain updates to all the databases on the server. This means you can only switch on incremental backups for one database per server, because the logs will be purged each time a database is dumped. 83 | 84 | 85 | ## Inspiration 86 | 87 | Marc-André Cournoyer's [mysql_s3_backup](https://github.com/macournoyer/mysql_s3_backup). 88 | 89 | 90 | ## Intellectual property 91 | 92 | Copyright 2011-2021 Andy Stewart (boss@airbladesoftware.com). 93 | 94 | Released under the MIT licence. 95 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | -------------------------------------------------------------------------------- /bin/mys3ql: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) 4 | require 'mys3ql' 5 | require 'optparse' 6 | 7 | params = {} 8 | op = OptionParser.new do |opts| 9 | opts.banner = 'Usage: mys3ql []' 10 | 11 | opts.separator '' 12 | opts.separator 'Common options:' 13 | 14 | opts.on('-c', '--config CONFIG', 'Load configuration from YAML file (default ~/.mys3ql)') { |v| params[:config] = v } 15 | opts.on('-d', '--debug', 'Be verbose') { |v| params[:debug] = v } 16 | opts.on '-v', '--version', 'Print version' do 17 | puts "mys3ql v#{Mys3ql::VERSION}" 18 | exit 19 | end 20 | 21 | opts.separator '' 22 | opts.separator 'restore options:' 23 | opts.on('-a', '--after NUMBER', 'Use only the subset of binary logs after NUMBER') { |v| params[:after] = v } 24 | end 25 | op.parse! ARGV 26 | 27 | params[:command] = ARGV[0] 28 | unless %w[full incremental restore].include? params[:command] 29 | puts op.help 30 | exit 1 31 | end 32 | 33 | Mys3ql::Conductor.run params 34 | -------------------------------------------------------------------------------- /lib/mys3ql.rb: -------------------------------------------------------------------------------- 1 | require 'mys3ql/conductor' 2 | require 'mys3ql/version' 3 | -------------------------------------------------------------------------------- /lib/mys3ql/conductor.rb: -------------------------------------------------------------------------------- 1 | require 'tempfile' 2 | require 'mys3ql/config' 3 | require 'mys3ql/mysql' 4 | require 'mys3ql/s3' 5 | 6 | module Mys3ql 7 | class Conductor 8 | 9 | def self.run(args) 10 | conductor = Conductor.new args.fetch(:config, nil) 11 | conductor.debug = args.fetch(:debug, false) 12 | 13 | command = args.fetch(:command) 14 | if command == 'restore' 15 | conductor.restore args.fetch(:after, nil) 16 | else 17 | conductor.send command 18 | end 19 | end 20 | 21 | def initialize(config_file = nil) 22 | @config = Config.new(config_file) 23 | @mysql = Mysql.new @config 24 | @s3 = S3.new @config 25 | end 26 | 27 | # Dumps the database and uploads it to S3. 28 | # Copies the uploaded file to the key :latest. 29 | # Deletes binary logs from the file system. 30 | # Deletes binary logs from S3. 31 | def full 32 | @mysql.dump 33 | @s3.store @mysql.dump_file 34 | @mysql.delete_dump 35 | @s3.delete_bin_logs 36 | end 37 | 38 | # Uploads mysql's binary logs to S3. 39 | # The binary logs are left on the file system. 40 | # Log files already on S3 are not re-uploaded. 41 | def incremental 42 | @mysql.each_bin_log do |log| 43 | @s3.store log, false 44 | end 45 | end 46 | 47 | # When 'after' is nil: 48 | # 49 | # - downloads the latest dump from S3 and loads it into the database; 50 | # - downloads each binary log from S3 and loads it into the database. 51 | # 52 | # When 'after' is given: 53 | # 54 | # - downloads each binary log following 'after' from S3 and loads it into the database. 55 | # 56 | # Downloaded files are removed from the file system. 57 | def restore(after = nil) 58 | unless after 59 | # get latest dump 60 | with_temp_file do |file| 61 | @s3.retrieve :latest, file 62 | @mysql.restore file 63 | end 64 | end 65 | 66 | # apply bin logs 67 | begin 68 | tmpfiles = [] 69 | @s3.each_bin_log(after) do |log| 70 | file = Tempfile.new 'mys3ql' 71 | tmpfiles << file 72 | @s3.retrieve log, file.path 73 | end 74 | @mysql.apply_bin_logs tmpfiles.map(&:path) 75 | ensure 76 | tmpfiles.each &:close! 77 | end 78 | 79 | # NOTE: not sure about this: 80 | # puts "You might want to flush mysql's logs..." 81 | end 82 | 83 | def debug=(val) 84 | @config.debug = val 85 | end 86 | 87 | private 88 | 89 | def with_temp_file(&block) 90 | file = Tempfile.new 'mys3ql-sql' 91 | yield file.path 92 | nil 93 | ensure 94 | file.close! 95 | end 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /lib/mys3ql/config.rb: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | 3 | module Mys3ql 4 | class Config 5 | 6 | def initialize(config_file = nil) 7 | config_file = config_file || default_config_file 8 | @config = YAML.load_file File.expand_path(config_file) 9 | rescue Errno::ENOENT 10 | $stderr.puts "missing config file #{config_file}" 11 | exit 1 12 | end 13 | 14 | # 15 | # General 16 | # 17 | 18 | def debug=(val) 19 | @debug = val 20 | end 21 | 22 | def debugging? 23 | @debug 24 | end 25 | 26 | # 27 | # MySQL 28 | # 29 | 30 | def user 31 | mysql['user'] 32 | end 33 | 34 | def password 35 | mysql['password'] 36 | end 37 | 38 | def database 39 | mysql['database'] 40 | end 41 | 42 | def bin_path 43 | mysql['bin_path'] 44 | end 45 | 46 | def bin_log 47 | mysql['bin_log'] 48 | end 49 | 50 | # 51 | # S3 52 | # 53 | 54 | def access_key_id 55 | s3['access_key_id'] 56 | end 57 | 58 | def secret_access_key 59 | s3['secret_access_key'] 60 | end 61 | 62 | def bucket 63 | s3['bucket'] 64 | end 65 | 66 | def region 67 | s3['region'] 68 | end 69 | 70 | private 71 | 72 | def mysql 73 | @config['mysql'] 74 | end 75 | 76 | def s3 77 | @config['s3'] 78 | end 79 | 80 | def default_config_file 81 | File.join "#{ENV['HOME']}", '.mys3ql' 82 | end 83 | end 84 | end 85 | -------------------------------------------------------------------------------- /lib/mys3ql/mysql.rb: -------------------------------------------------------------------------------- 1 | require 'mys3ql/shell' 2 | 3 | module Mys3ql 4 | class Mysql 5 | include Shell 6 | 7 | def initialize(config) 8 | @config = config 9 | end 10 | 11 | # 12 | # dump 13 | # 14 | 15 | def dump 16 | # --master-data=2 include the current binary log coordinates in the log file 17 | # --delete-master-logs delete binary log files 18 | cmd = "#{@config.bin_path}mysqldump" 19 | cmd += ' --quick --single-transaction --create-options --no-tablespaces' 20 | cmd += ' --flush-logs --master-data=2 --delete-master-logs' if binary_logging? 21 | cmd += cli_options 22 | cmd += " | gzip > #{dump_file}" 23 | run cmd 24 | end 25 | 26 | def dump_file 27 | @dump_file ||= "#{timestamp}.sql.gz" 28 | end 29 | 30 | def delete_dump 31 | File.delete dump_file 32 | log "mysql: deleted #{dump_file}" 33 | end 34 | 35 | # 36 | # bin_logs 37 | # 38 | 39 | # flushes logs, yields each bar the last to the block 40 | def each_bin_log(&block) 41 | # FLUSH LOGS Closes and reopens any log file, including binary logs, 42 | # to which the server is writing. For binary logs, the sequence 43 | # number of the binary log file is incremented by one relative to 44 | # the previous file. 45 | # https://dev.mysql.com/doc/refman/5.7/en/flush.html#flush-logs 46 | # https://dev.mysql.com/doc/refman/5.7/en/flush.html#flush-binary-logs 47 | execute 'flush logs' 48 | Dir.glob("#{@config.bin_log}.[0-9]*") 49 | .sort_by { |f| f[/\d+/].to_i } 50 | .slice(0..-2) # all logs except the last, which is newly created 51 | .each do |log_file| 52 | yield log_file 53 | end 54 | end 55 | 56 | # 57 | # restore 58 | # 59 | 60 | def restore(file) 61 | run "gunzip -c #{file} | #{@config.bin_path}mysql #{cli_options}" 62 | end 63 | 64 | def apply_bin_logs(*files) 65 | cmd = "#{@config.bin_path}mysqlbinlog --database=#{@config.database} #{files.join ' '}" 66 | cmd += " | #{@config.bin_path}mysql -u'#{@config.user}'" 67 | cmd += " -p'#{@config.password}'" if @config.password 68 | run cmd 69 | end 70 | 71 | private 72 | 73 | def timestamp 74 | Time.now.utc.strftime "%Y%m%d%H%M" 75 | end 76 | 77 | def binary_logging? 78 | @config.bin_log && @config.bin_log.length > 0 79 | end 80 | 81 | def cli_options 82 | cmd = " -u'#{@config.user}'" 83 | cmd += " -p'#{@config.password}'" if @config.password 84 | cmd += " #{@config.database}" 85 | end 86 | 87 | def execute(sql) 88 | run %Q(#{@config.bin_path}mysql -e "#{sql}" #{cli_options}) 89 | end 90 | 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /lib/mys3ql/s3.rb: -------------------------------------------------------------------------------- 1 | require 'mys3ql/shell' 2 | require 'aws-sdk-s3' 3 | 4 | module Mys3ql 5 | class S3 6 | include Shell 7 | 8 | def initialize(config) 9 | @config = config 10 | end 11 | 12 | def store(file, dump = true) 13 | key = key_for(dump ? :dump : :bin_log, file) 14 | s3_file = save file, key 15 | if dump && s3_file 16 | copy_key = key_for :latest 17 | s3_file.copy_to bucket: bucket_name, key: copy_key 18 | log "s3: copied #{key} to #{copy_key}" 19 | end 20 | end 21 | 22 | def delete_bin_logs 23 | each_bin_log do |file| 24 | file.delete 25 | log "s3: deleted #{file.key}" 26 | end 27 | end 28 | 29 | def each_bin_log(after = nil, &block) 30 | if after && after !~ /^\d{6}$/ 31 | puts 'Binary log file number must be 6 digits.' 32 | exit 1 33 | end 34 | 35 | bucket.objects(prefix: bin_logs_prefix) 36 | .sort_by { |file| file.key[/\d+/].to_i } 37 | .select { |file| after.nil? || (file.key[/\d+/].to_i > after.to_i) } 38 | .each do |file| 39 | yield file 40 | end 41 | end 42 | 43 | def retrieve(s3_file, local_file) 44 | key = (s3_file == :latest) ? key_for(:latest) : s3_file.key 45 | get key, local_file 46 | end 47 | 48 | private 49 | 50 | def get(s3_key, local_file_name) 51 | s3.get_object( 52 | response_target: local_file_name, 53 | bucket: bucket_name, 54 | key: s3_key 55 | ) 56 | log "s3: pulled #{s3_key} to #{local_file_name}" 57 | end 58 | 59 | def save(local_file_name, s3_key) 60 | if bucket.object(s3_key).exists? 61 | log "s3: skipped #{local_file_name} - already exists" 62 | return 63 | end 64 | 65 | s3_file = bucket.put_object( 66 | key: s3_key, 67 | body: File.open(local_file_name), 68 | storage_class: 'STANDARD_IA', 69 | acl: 'private' 70 | ) 71 | log "s3: pushed #{local_file_name} to #{s3_key}" 72 | s3_file 73 | end 74 | 75 | def key_for(kind, file = nil) 76 | name = File.basename file if file 77 | case kind 78 | when :dump; "#{dumps_prefix}/#{name}" 79 | when :bin_log; "#{bin_logs_prefix}/#{name}" 80 | when :latest; "#{dumps_prefix}/latest.sql.gz" 81 | end 82 | end 83 | 84 | def s3 85 | @s3 ||= begin 86 | client = Aws::S3::Client.new( 87 | secret_access_key: @config.secret_access_key, 88 | access_key_id: @config.access_key_id, 89 | region: @config.region 90 | ) 91 | client 92 | end 93 | end 94 | 95 | def bucket 96 | @bucket ||= begin 97 | b = Aws::S3::Bucket.new bucket_name, client: s3 98 | raise "S3 bucket #{bucket_name} not found" unless b.exists? 99 | b 100 | end 101 | end 102 | 103 | def bucket_name 104 | @config.bucket 105 | end 106 | 107 | def dumps_prefix 108 | "#{@config.database}/dumps" 109 | end 110 | 111 | def bin_logs_prefix 112 | "#{@config.database}/bin_logs" 113 | end 114 | end 115 | end 116 | -------------------------------------------------------------------------------- /lib/mys3ql/shell.rb: -------------------------------------------------------------------------------- 1 | module Mys3ql 2 | ShellCommandError = Class.new RuntimeError 3 | 4 | module Shell 5 | def run(command) 6 | log command 7 | result = `#{command}` 8 | log "==> #{result}" unless result.empty? 9 | raise ShellCommandError, "error (exit status #{$?.exitstatus}): #{command} ==> #{result}: #{$?}" unless $?.success? 10 | result 11 | end 12 | 13 | def log(message) 14 | puts message if @config.debugging? 15 | end 16 | 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/mys3ql/version.rb: -------------------------------------------------------------------------------- 1 | module Mys3ql 2 | VERSION = '1.3.1' 3 | end 4 | -------------------------------------------------------------------------------- /mys3ql.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "mys3ql/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "mys3ql" 7 | s.version = Mys3ql::VERSION 8 | s.authors = ["Andy Stewart"] 9 | s.email = ["boss@airbladesoftware.com"] 10 | s.homepage = 'https://github.com/airblade/mys3ql' 11 | s.summary = 'Simple backup of your MySql database onto Amazon S3.' 12 | s.description = s.summary 13 | 14 | s.files = `git ls-files`.split("\n") 15 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 16 | s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 17 | s.require_paths = ["lib"] 18 | 19 | s.add_dependency 'aws-sdk-s3', '~> 1' 20 | s.add_development_dependency 'rake' 21 | end 22 | --------------------------------------------------------------------------------