├── .gitignore ├── Gemfile ├── lib ├── lvm │ ├── pv_config.rb │ ├── helpers.rb │ ├── lv_config.rb │ ├── vg_config.rb │ ├── logical_volume.rb │ ├── snapshot.rb │ └── thin_snapshot.rb ├── lvm.rb ├── vgcfgbackup.treetop └── vgcfgbackup.rb ├── manifests └── init.pp ├── spec ├── fixtures │ ├── trivial │ ├── vgcfgbackup │ ├── vgmetadata │ ├── physicalvolume │ ├── striped │ └── fullconfig ├── spec_helper.rb └── vg_cfg_spec.rb ├── Guardfile ├── Rakefile ├── test ├── 00simple_local_sync ├── 05sync_via_snapshot_file ├── run └── 10snapback ├── lvmsync.gemspec ├── bin └── lvmsync ├── README.md └── LICENCE /.gitignore: -------------------------------------------------------------------------------- 1 | Gemfile.lock 2 | pkg/ 3 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org/' 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /lib/lvm/pv_config.rb: -------------------------------------------------------------------------------- 1 | module LVM; end 2 | 3 | class LVM::PVConfig 4 | def initialize(tree) 5 | @root = tree 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /manifests/init.pp: -------------------------------------------------------------------------------- 1 | class lvmsync { 2 | package { "lvmsync": 3 | provider => "gem", 4 | ensure => "present" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /lib/lvm.rb: -------------------------------------------------------------------------------- 1 | require 'lvm/helpers' 2 | require 'lvm/thin_snapshot' 3 | require 'lvm/snapshot' 4 | require 'lvm/logical_volume' 5 | require 'lvm/lv_config' 6 | require 'lvm/vg_config' 7 | -------------------------------------------------------------------------------- /spec/fixtures/trivial: -------------------------------------------------------------------------------- 1 | # Generated by LVM2 version 2.02.104(2) (2013-11-13): Wed May 21 07:53:43 2 | # 2014 3 | 4 | contents = "Text Format Volume Group" 5 | version = 1 6 | 7 | description = "vgcfgbackup -f /tmp/faffen2" 8 | -------------------------------------------------------------------------------- /Guardfile: -------------------------------------------------------------------------------- 1 | guard 'spork' do 2 | watch('Gemfile') { :rspec } 3 | watch('Gemfile.lock') { :rspec } 4 | watch('spec/spec_helper.rb') { :rspec } 5 | end 6 | 7 | guard 'rspec', 8 | :cmd => "rspec --drb", 9 | :all_on_start => true do 10 | watch(%r{^spec/.+_spec\.rb$}) 11 | watch(%r{^lib/}) { "spec" } 12 | end 13 | 14 | -------------------------------------------------------------------------------- /spec/fixtures/vgcfgbackup: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Dumb test script to copy a dummy vgcfgbackup output file to where it 4 | # should be. Must be called as "$0 -f ", and 5 | # .../spec/fixtures/ will be copied to . 6 | 7 | set -e 8 | 9 | HERE="$(dirname $(readlink -f $0))" 10 | 11 | if [ "$#" != "3" ] || [ "$1" != "-f" ]; then 12 | echo "Called incorrectly" 13 | exit 1 14 | fi 15 | 16 | cp "$HERE/$3" "$2" 17 | -------------------------------------------------------------------------------- /spec/fixtures/vgmetadata: -------------------------------------------------------------------------------- 1 | # Generated by LVM2 version 2.02.104(2) (2013-11-13): Wed May 21 07:53:43 2014 2 | 3 | contents = "Text Format Volume Group" 4 | version = 1 5 | 6 | description = "vgcfgbackup -f /tmp/faffen2" 7 | 8 | vgmetadata { 9 | id = "zAPMOi-5QlD-fp1M-FMgz-biha-TQIP-X9TfqX" 10 | seqno = 436 11 | format = "lvm2" # informational 12 | status = ["RESIZEABLE", "READ", "WRITE"] 13 | flags = [] 14 | extent_size = 8192 # 4 Megabytes 15 | max_lv = 0 16 | max_pv = 0 17 | metadata_copies = 0 18 | } 19 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'spork' 2 | 3 | Spork.prefork do 4 | require 'bundler' 5 | Bundler.setup(:default, :test) 6 | require 'rspec/core' 7 | 8 | require 'rspec/mocks' 9 | 10 | require 'pry' 11 | require 'plymouth' 12 | 13 | RSpec.configure do |config| 14 | config.fail_fast = true 15 | # config.full_backtrace = true 16 | 17 | config.expect_with :rspec do |c| 18 | c.syntax = :expect 19 | end 20 | end 21 | end 22 | 23 | Spork.each_run do 24 | # Nothing to do here, specs will load the files they need 25 | end 26 | -------------------------------------------------------------------------------- /lib/lvm/helpers.rb: -------------------------------------------------------------------------------- 1 | module LVM; end 2 | 3 | module LVM::Helpers 4 | # Are we on a big-endian system? Needed for our htonq/ntohq methods 5 | def big_endian? 6 | @big_endian ||= [1].pack("s") == [1].pack("n") 7 | end 8 | 9 | def htonq val 10 | # This won't work on a nUxi byte-order machine, but if you have one of 11 | # those, I'm guessing you've got bigger problems 12 | big_endian? ? val : swap_longs(val) 13 | end 14 | 15 | def ntohq val 16 | big_endian? ? val : swap_longs(val) 17 | end 18 | 19 | # On-disk (LVM) format (which is little-endian) to host byte order 20 | def dtohq val 21 | big_endian? ? swap_longs(val) : val 22 | end 23 | 24 | def swap_longs val 25 | [val].pack("Q").reverse.unpack("Q").first 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/vgcfgbackup.treetop: -------------------------------------------------------------------------------- 1 | grammar VgCfgBackup 2 | rule config 3 | (variable / group / space / comment)+ 4 | end 5 | 6 | rule variable 7 | variable_name " = " (integer / string / list) 8 | end 9 | 10 | rule variable_name 11 | [a-zA-Z0-9_.+-]+ 12 | end 13 | 14 | rule group 15 | variable_name space "{" (variable / group / space / comment)+ "}" 16 | end 17 | 18 | rule integer 19 | [1-9] [0-9]* / "0" 20 | end 21 | 22 | rule string 23 | '"' [^"]* '"' 24 | end 25 | 26 | rule list 27 | "[" space? ((string / integer) "," space? / (string / integer))* space? "]" 28 | end 29 | 30 | rule space 31 | [\s]+ 32 | end 33 | 34 | rule comment 35 | "#" [^\n]* 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /spec/fixtures/physicalvolume: -------------------------------------------------------------------------------- 1 | # Generated by LVM2 version 2.02.104(2) (2013-11-13): Wed May 21 07:53:43 2014 2 | 3 | contents = "Text Format Volume Group" 4 | version = 1 5 | 6 | description = "vgcfgbackup -f /tmp/faffen2" 7 | 8 | physicalvolume { 9 | id = "zAPMOi-5QlD-fp1M-FMgz-biha-TQIP-X9TfqX" 10 | seqno = 436 11 | format = "lvm2" # informational 12 | status = ["RESIZEABLE", "READ", "WRITE"] 13 | flags = [] 14 | extent_size = 8192 # 4 Megabytes 15 | max_lv = 0 16 | max_pv = 0 17 | metadata_copies = 0 18 | 19 | physical_volumes { 20 | 21 | pv0 { 22 | id = "j7ZiWs-1fHB-aAwq-yL29-pBOk-H4hR-AOIe0P" 23 | device = "/dev/dm-0" # Hint only 24 | 25 | status = ["ALLOCATABLE"] 26 | flags = [] 27 | dev_size = 1952545832 # 931.046 Gigabytes 28 | pe_start = 384 29 | pe_count = 238347 # 931.043 Gigabytes 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler' 3 | 4 | begin 5 | Bundler.setup(:default, :development) 6 | rescue Bundler::BundlerError => e 7 | $stderr.puts e.message 8 | $stderr.puts "Run `bundle install` to install missing gems" 9 | exit e.status_code 10 | end 11 | 12 | require 'git-version-bump/rake-tasks' 13 | 14 | Bundler::GemHelper.install_tasks 15 | 16 | task :release do 17 | sh "git release" 18 | end 19 | 20 | require 'rdoc/task' 21 | 22 | Rake::RDocTask.new do |rd| 23 | rd.main = "README.md" 24 | rd.title = 'lvmsync' 25 | rd.rdoc_files.include("README.md", "lib/**/*.rb") 26 | end 27 | 28 | desc "Run guard" 29 | task :guard do 30 | require 'guard' 31 | ::Guard.start(:clear => true) 32 | while ::Guard.running do 33 | sleep 0.5 34 | end 35 | end 36 | 37 | require 'rspec/core/rake_task' 38 | RSpec::Core::RakeTask.new :test do |t| 39 | t.pattern = "spec/**/*_spec.rb" 40 | end 41 | -------------------------------------------------------------------------------- /lib/lvm/lv_config.rb: -------------------------------------------------------------------------------- 1 | module LVM; end 2 | 3 | class LVM::LVConfig 4 | attr_reader :name 5 | 6 | def initialize(tree, name, vgcfg) 7 | @root = tree 8 | @name = name 9 | @vgcfg = vgcfg 10 | end 11 | 12 | def thin? 13 | @root.groups['segment1'].variable_value('type') == 'thin' 14 | end 15 | 16 | def snapshot? 17 | thin? ? !origin.nil? : !@vgcfg.logical_volumes.values.find { |lv| lv.cow_store == name }.nil? 18 | end 19 | 20 | def thin_pool 21 | @root.groups['segment1'].variable_value('thin_pool') 22 | end 23 | 24 | def device_id 25 | @root.groups['segment1'].variable_value('device_id') 26 | end 27 | 28 | def origin 29 | @root.groups['segment1'].variable_value('origin') 30 | end 31 | 32 | def cow_store 33 | @root.groups['segment1'].variable_value('cow_store') 34 | end 35 | 36 | def chunk_size 37 | @root.groups['segment1'].variable_value('chunk_size') * 512 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/vgcfgbackup.rb: -------------------------------------------------------------------------------- 1 | module VgCfgBackup 2 | class Node < Treetop::Runtime::SyntaxNode 3 | end 4 | 5 | class Group < Node 6 | def name 7 | self.elements[0].text_value 8 | end 9 | 10 | def variables 11 | self.elements[3].elements.select { |e| e.is_a? Variable } 12 | end 13 | 14 | def variable_value(name) 15 | self.variables.find { |v| v.name == name }.value rescue nil 16 | end 17 | 18 | def groups 19 | self.elements[3].elements.select { |e| e.is_a? Group }.inject({}) { |h,v| h[v.name] = v; h } 20 | end 21 | end 22 | 23 | class Config < Group 24 | def name 25 | nil 26 | end 27 | 28 | def variables 29 | self.elements.select { |e| e.is_a? Variable } 30 | end 31 | 32 | def groups 33 | self.elements.select { |e| e.is_a? Group }.inject({}) { |h,v| h[v.name] = v; h } 34 | end 35 | end 36 | 37 | class Variable < Node 38 | def name 39 | self.elements[0].text_value 40 | end 41 | 42 | def value 43 | self.elements[2].value 44 | end 45 | end 46 | 47 | class VariableName < Node 48 | end 49 | 50 | class Integer < Node 51 | def value 52 | self.text_value.to_i 53 | end 54 | end 55 | 56 | class String < Node 57 | def value 58 | self.elements[1].text_value 59 | end 60 | end 61 | 62 | class List < Node 63 | def value 64 | self.elements.find { |e| e.is_a?(String) }.map { |e| e.value } 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /test/00simple_local_sync: -------------------------------------------------------------------------------- 1 | trap "lvremove -f $VG/__lvmsynctest_src || true; 2 | lvremove -f $VG/__lvmsynctest_dest || true; 3 | lvremove -f $VG/__lvmsynctest_snap || true" EXIT 4 | 5 | lvcreate -l 5 -n __lvmsynctest_src $VG >/dev/null 6 | lvcreate -l 5 -n __lvmsynctest_dest $VG >/dev/null 7 | 8 | # Fill src with gibberish 9 | dd if=/dev/urandom of=/dev/$VG/__lvmsynctest_src bs=1M >/dev/null 2>&1 || true 10 | 11 | # Snapshot 12 | lvcreate --snapshot $VG/__lvmsynctest_src -l 5 -n __lvmsynctest_snap >/dev/null 13 | 14 | # Initial copy 15 | dd if=/dev/$VG/__lvmsynctest_src of=/dev/$VG/__lvmsynctest_dest bs=1M >/dev/null 2>&1 16 | 17 | # Write in some gibberish at intervals across the LV 18 | i=0 19 | while dd if=/dev/urandom of=/dev/$VG/__lvmsynctest_src bs=512 count=1 seek=$(($i*2048)) >/dev/null 2>&1; do 20 | i=$(($i+1)) 21 | done 22 | 23 | # Now lvmsync! 24 | $LVMSYNC /dev/$VG/__lvmsynctest_snap /dev/$VG/__lvmsynctest_dest 2>/dev/null || true 25 | 26 | # Verify that our source and dest are now equal 27 | SRCSUM="$(md5sum /dev/null 32 | lvremove -f $VG/__lvmsynctest_dest >/dev/null 33 | lvremove -f $VG/__lvmsynctest_src >/dev/null 34 | 35 | trap "" EXIT 36 | 37 | # Make sure test succeeded 38 | if [ "$SRCSUM" != "$DESTSUM" ]; then 39 | echo "FAIL: 00simple_local_sync" 40 | exit 1 41 | else 42 | echo "OK: 00simple_local_sync" 43 | fi 44 | -------------------------------------------------------------------------------- /lvmsync.gemspec: -------------------------------------------------------------------------------- 1 | require 'git-version-bump' 2 | 3 | Gem::Specification.new do |s| 4 | s.name = "lvmsync" 5 | 6 | s.version = GVB.version 7 | s.date = GVB.date 8 | 9 | s.platform = Gem::Platform::RUBY 10 | 11 | s.homepage = "http://theshed.hezmatt.org/lvmsync" 12 | s.summary = "Efficiently transfer changes in LVM snapshots" 13 | s.authors = ["Matt Palmer"] 14 | 15 | s.extra_rdoc_files = ["README.md"] 16 | s.files = %w{ 17 | README.md 18 | LICENCE 19 | bin/lvmsync 20 | lib/lvm.rb 21 | lib/lvm/lv_config.rb 22 | lib/lvm/logical_volume.rb 23 | lib/lvm/thin_snapshot.rb 24 | lib/lvm/snapshot.rb 25 | lib/lvm/vg_config.rb 26 | lib/lvm/helpers.rb 27 | lib/lvm/pv_config.rb 28 | lib/vgcfgbackup.treetop 29 | lib/vgcfgbackup.rb 30 | } 31 | s.executables = ["lvmsync"] 32 | 33 | s.add_runtime_dependency "git-version-bump", "~> 0.10" 34 | s.add_runtime_dependency "treetop" 35 | 36 | s.add_development_dependency 'bundler' 37 | s.add_development_dependency 'github-release' 38 | s.add_development_dependency 'guard-spork' 39 | s.add_development_dependency 'guard-rspec' 40 | s.add_development_dependency 'plymouth' 41 | if RUBY_VERSION =~ /^1\./ 42 | s.add_development_dependency 'pry-debugger' 43 | else 44 | s.add_development_dependency 'pry-byebug' 45 | end 46 | s.add_development_dependency 'rake' 47 | # Needed for guard 48 | s.add_development_dependency 'rb-inotify', '~> 0.9' 49 | s.add_development_dependency 'rdoc' 50 | s.add_development_dependency 'rspec' 51 | end 52 | -------------------------------------------------------------------------------- /test/05sync_via_snapshot_file: -------------------------------------------------------------------------------- 1 | trap "lvremove -f $VG/__lvmsynctest_src || true; 2 | lvremove -f $VG/__lvmsynctest_dest || true; 3 | lvremove -f $VG/__lvmsynctest_snap || true; 4 | rm -f $HERE/,,snapshot_data_file || true" EXIT 5 | 6 | lvcreate -l 5 -n __lvmsynctest_src $VG >/dev/null 7 | lvcreate -l 5 -n __lvmsynctest_dest $VG >/dev/null 8 | 9 | # Fill src with gibberish 10 | dd if=/dev/urandom of=/dev/$VG/__lvmsynctest_src bs=1M >/dev/null 2>&1 || true 11 | 12 | # Snapshot 13 | lvcreate --snapshot $VG/__lvmsynctest_src -l 5 -n __lvmsynctest_snap >/dev/null 14 | 15 | # Initial copy 16 | dd if=/dev/$VG/__lvmsynctest_src of=/dev/$VG/__lvmsynctest_dest bs=1M >/dev/null 2>&1 17 | 18 | # Write in some gibberish at intervals across the LV 19 | i=0 20 | while dd if=/dev/urandom of=/dev/$VG/__lvmsynctest_src bs=512 count=1 seek=$(($i*2048)) >/dev/null 2>&1; do 21 | i=$(($i+1)) 22 | done 23 | 24 | # Now lvmsync -- dump to a file 25 | $LVMSYNC /dev/$VG/__lvmsynctest_snap /dev/$VG/__lvmsynctest_dest --stdout >$HERE/,,snapshot_data_file 2>/dev/null || true 26 | 27 | # Apply from the file 28 | $LVMSYNC --apply $HERE/,,snapshot_data_file /dev/$VG/__lvmsynctest_dest 2>/dev/null || true 29 | 30 | # Verify that our source and dest are now equal 31 | SRCSUM="$(md5sum /dev/null >/dev/null 36 | lvremove -f $VG/__lvmsynctest_dest >/dev/null >/dev/null 37 | lvremove -f $VG/__lvmsynctest_src >/dev/null >/dev/null 38 | rm -f $HERE/,,snapshot_data_file 39 | 40 | trap "" EXIT 41 | 42 | # Make sure test succeeded 43 | if [ "$SRCSUM" != "$DESTSUM" ]; then 44 | echo "FAIL: 05sync_via_snapshot_file" 45 | exit 1 46 | else 47 | echo "OK: 05sync_via_snapshot_file" 48 | fi 49 | -------------------------------------------------------------------------------- /spec/fixtures/striped: -------------------------------------------------------------------------------- 1 | contents = "Text Format Volume Group" 2 | version = 1 3 | 4 | description = "vgcfgbackup -f /tmp/faffen2" 5 | 6 | striped { 7 | id = "zAPMOi-5QlD-fp1M-FMgz-biha-TQIP-X9TfqX" 8 | seqno = 436 9 | format = "lvm2" # informational 10 | status = ["RESIZEABLE", "READ", "WRITE"] 11 | flags = [] 12 | extent_size = 8192 # 4 Megabytes 13 | max_lv = 0 14 | max_pv = 0 15 | metadata_copies = 0 16 | 17 | physical_volumes { 18 | 19 | pv0 { 20 | id = "j7ZiWs-1fHB-aAwq-yL29-pBOk-H4hR-AOIe0P" 21 | device = "/dev/dm-0" # Hint only 22 | 23 | status = ["ALLOCATABLE"] 24 | flags = [] 25 | dev_size = 1952545832 # 931.046 Gigabytes 26 | pe_start = 384 27 | pe_count = 238347 # 931.043 Gigabytes 28 | } 29 | 30 | pv1 { 31 | id = "afl7D7-Rpy1-Is3H-SFxo-d03x-s8dE-lEgZWU" 32 | device = "/dev/dm-1" # Hint only 33 | 34 | status = ["ALLOCATABLE"] 35 | flags = [] 36 | dev_size = 1952545832 # 931.046 Gigabytes 37 | pe_start = 384 38 | pe_count = 238347 # 931.043 Gigabytes 39 | } 40 | } 41 | 42 | logical_volumes { 43 | 44 | swap { 45 | id = "BOfqkb-V4jw-7cBX-UR3D-SN3f-wWvd-NwysSj" 46 | status = ["READ", "WRITE", "VISIBLE"] 47 | flags = [] 48 | segment_count = 1 49 | 50 | segment1 { 51 | start_extent = 0 52 | extent_count = 2048 # 8 Gigabytes 53 | 54 | type = "striped" 55 | stripe_count = 1 # linear 56 | 57 | stripes = [ 58 | "pv0", 0 59 | ] 60 | } 61 | } 62 | 63 | root { 64 | id = "8sbW3s-qB1u-6IlS-ABSj-PAMU-UxRk-I18ECh" 65 | status = ["READ", "WRITE", "VISIBLE"] 66 | flags = [] 67 | segment_count = 1 68 | 69 | segment1 { 70 | start_extent = 0 71 | extent_count = 3840 # 15 Gigabytes 72 | 73 | type = "striped" 74 | stripe_count = 2 75 | stripe_size = 128 # 64 Kilobytes 76 | 77 | stripes = [ 78 | "pv0", 21504, 79 | "pv1", 21504 80 | ] 81 | } 82 | } 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /test/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | # Must run as root, because we're doing LVM manipulation 6 | if [ "$EUID" != "0" ]; then 7 | echo "Must run this testsuite as root" >&2 8 | exit 1 9 | fi 10 | 11 | HERE="$(dirname $(dirname $(readlink -f $0)))" 12 | export RUBYLIB="$HERE/lib" 13 | LVMSYNC="$HERE/bin/lvmsync" 14 | 15 | # Make sure there aren't any remnant lvmsync test LVs from a previous test 16 | # run that bombed unceremoniously. Since we don't want to nuke *anything* 17 | # without being sure, we'll just exit and ask the user to cleanup manually. 18 | if lvs --noheadings -o lv_name | grep -q '__lvmsynctest'; then 19 | cat <&2 20 | ERROR: There appear to be some test LVs that this test suite didn't clean up 21 | previously. I'm not going to delete them myself, in case you created them 22 | deliberately. Check that the following LVs aren't something you want to keep, 23 | and then delete them: 24 | 25 | $(lvs --noheadings -o lv_name | grep '__lvmsynctest') 26 | EOF 27 | exit 1 28 | fi 29 | 30 | # We'll be nice and pick the VG with the largest free space. 31 | VG="$(vgs --noheadings -o vg_name --sort -vg_free | sed 's/ //g')" 32 | 33 | if [ -z "$VG" ]; then 34 | cat <&2 35 | No Volume Groups found. You must run this test suite on a machine that is 36 | running LVM. 37 | EOF 38 | exit 1 39 | fi 40 | 41 | # Make sure we've got enough space -- we'll be creating up to four LVs, each 42 | # up to five extents in size. So we'll need at least 20 free extents. 43 | VGFREE="$(vgs --noheadings -o vg_free_count $VG)" 44 | 45 | if (($VGFREE < 20)); then 46 | cat <&2 47 | Insufficient free space in VG $VG. Please ensure at least one volume group 48 | on your system has 20 free extents. 49 | EOF 50 | exit 1 51 | fi 52 | 53 | # Now we run our tests 54 | HERE="$(dirname $(readlink -f $0))" 55 | 56 | for i in $HERE/[0-9][0-9]*; do 57 | . $i 58 | done 59 | 60 | # Since our tests will exit immediately if there was a problem, if we've 61 | # gotten here then we've won! 62 | echo "ALL TESTS PASSED" 63 | -------------------------------------------------------------------------------- /lib/lvm/vg_config.rb: -------------------------------------------------------------------------------- 1 | require 'tempfile' 2 | require 'open3' 3 | require 'treetop' 4 | require File.expand_path('../../vgcfgbackup', __FILE__) 5 | 6 | Treetop.load(File.expand_path('../../vgcfgbackup.treetop', __FILE__)) 7 | 8 | require 'lvm/lv_config' 9 | require 'lvm/pv_config' 10 | require 'lvm/snapshot' 11 | require 'lvm/thin_snapshot' 12 | 13 | module LVM; end 14 | 15 | class LVM::VGConfig 16 | def initialize(vg_name, opts = {}) 17 | @vgcfgbackup_cmd = opts[:vgcfgbackup_command] || 'vgcfgbackup' 18 | @vg_name = vg_name 19 | @parser = VgCfgBackupParser.new 20 | @root = @parser.parse(vgcfgbackup_output) 21 | if @root.nil? 22 | raise RuntimeError, 23 | "Cannot parse vgcfgbackup output: #{@parser.failure_reason}" 24 | end 25 | end 26 | 27 | def version 28 | @version ||= @root.variable_value('version') 29 | end 30 | 31 | def description 32 | @description ||= @root.variable_value('description') 33 | end 34 | 35 | def uuid 36 | @uuid ||= volume_group.variable_value('id') 37 | end 38 | 39 | def volume_group 40 | @volume_group ||= @root.groups[@vg_name] 41 | end 42 | 43 | def physical_volumes 44 | @physical_volumes ||= volume_group.groups['physical_volumes'].groups.to_a.inject({}) { |h,v| h[v[0]] = LVM::PVConfig.new(v[1]); h } 45 | end 46 | 47 | def logical_volumes 48 | @logical_volumes ||= volume_group.groups['logical_volumes'].groups.to_a.inject({}) { |h,v| h[v[0]] = LVM::LVConfig.new(v[1], v[0], self); h } 49 | end 50 | 51 | private 52 | def vgcfgbackup_output 53 | @vgcfgbackup_output ||= begin 54 | out = nil 55 | 56 | Tempfile.open('vg_config') do |tmpf| 57 | cmd = "#{@vgcfgbackup_cmd} -f #{tmpf.path} #{@vg_name}" 58 | stdout = nil 59 | stderr = nil 60 | exit_status = nil 61 | 62 | Open3.popen3(cmd) do |stdin_fd, stdout_fd, stderr_fd, wait_thr| 63 | stdin_fd.close 64 | stdout = stdout_fd.read 65 | stderr = stderr_fd.read 66 | exit_status = wait_thr.value if wait_thr 67 | end 68 | 69 | if (exit_status or $?).exitstatus != 0 70 | raise RuntimeError, 71 | "Failed to run vgcfgbackup: #{stdout}\n#{stderr}" 72 | end 73 | 74 | out = File.read(tmpf.path) 75 | end 76 | 77 | out 78 | end 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /test/10snapback: -------------------------------------------------------------------------------- 1 | trap "lvremove -f $VG/__lvmsynctest_src || true; 2 | lvremove -f $VG/__lvmsynctest_dest || true; 3 | lvremove -f $VG/__lvmsynctest_snap || true; 4 | rm -f $HERE/,,snapback_file || true" EXIT 5 | 6 | lvcreate -l 5 -n __lvmsynctest_src $VG >/dev/null 7 | lvcreate -l 5 -n __lvmsynctest_dest $VG >/dev/null 8 | 9 | # Fill src with gibberish 10 | dd if=/dev/urandom of=/dev/$VG/__lvmsynctest_src bs=1M >/dev/null 2>&1 || true 11 | 12 | # Snapshot 13 | lvcreate --snapshot $VG/__lvmsynctest_src -l 5 -n __lvmsynctest_snap >/dev/null 14 | 15 | # Initial copy 16 | dd if=/dev/$VG/__lvmsynctest_src of=/dev/$VG/__lvmsynctest_dest bs=1M >/dev/null 2>&1 17 | 18 | # Write in some gibberish at intervals across the LV 19 | i=0 20 | while dd if=/dev/urandom of=/dev/$VG/__lvmsynctest_src bs=512 count=1 seek=$(($i*2048)) >/dev/null 2>&1; do 21 | i=$(($i+1)) 22 | done 23 | 24 | # We need to know what we had before the lvmsync, so we can make sure the 25 | # snapback file worked 26 | ORIGSUM="$(md5sum /dev/null || true 32 | 33 | # Verify that our source and dest are now equal 34 | SRCSUM="$(md5sum /dev/null || true 40 | 41 | # And get a sum of what the dest looks like now 42 | POSTSNAPBACKSUM="$(md5sum /dev/null 46 | lvremove -f $VG/__lvmsynctest_dest >/dev/null 47 | lvremove -f $VG/__lvmsynctest_src >/dev/null 48 | rm -f $HERE/,,snapback_file 49 | 50 | trap "" EXIT 51 | 52 | # Make sure test succeeded 53 | if [ "$SRCSUM" != "$DESTSUM" ]; then 54 | echo "FAIL: 10snapback: Initial lvmsync failed to apply correctly" 55 | exit 1 56 | fi 57 | 58 | if [ "$ORIGSUM" != "$POSTSNAPBACKSUM" ]; then 59 | echo "FAIL: 10snapback: snapback file did not restore original 60 | device contents" 61 | exit 1 62 | fi 63 | 64 | echo "OK: 10snapback" 65 | -------------------------------------------------------------------------------- /lib/lvm/logical_volume.rb: -------------------------------------------------------------------------------- 1 | module LVM; end 2 | 3 | # This class represents an LVM logical volume, in all its glory. You can 4 | # perform various operations on it. 5 | class LVM::LogicalVolume 6 | # Create a new instance of LVM::LogicalVolume. 7 | # 8 | # New instances can be created in one of two ways: 9 | # 10 | # * Pass a single argument, containing any path which LVM can resolve to 11 | # a logical volume. Typically, this will either be `/dev//` 12 | # or `/dev/mapper/-`, but we don't try to parse it ourselves, 13 | # relying instead on `lvs` to do the heavy lifting. 14 | # 15 | # * Pass two arguments, which are the volume group name and logical 16 | # volume name, respectively. 17 | # 18 | # This method will raise `RuntimeError` if the path specified can't be 19 | # resolved to an LV, or if the specified VG name or LV name don't resolve 20 | # to an active logical volume. 21 | # 22 | def initialize(path_or_vg_name, lv_name=nil) 23 | if lv_name.nil? 24 | path = path_or_vg_name 25 | @vg_name, @lv_name = `lvs --noheadings -o vg_name,lv_name #{path} 2>/dev/null`.strip.split(/\s+/, 2) 26 | if $?.exitstatus != 0 27 | raise RuntimeError, 28 | "Failed to interrogate LVM about '#{path}'. Perhaps you misspelt it?" 29 | end 30 | else 31 | @vg_name = path_or_vg_name 32 | @lv_name = lv_name 33 | end 34 | 35 | @vgcfg = LVM::VGConfig.new(@vg_name) 36 | @lvcfg = @vgcfg.logical_volumes[@lv_name] 37 | 38 | if @lvcfg.nil? 39 | raise RuntimeError, 40 | "Logical volume #{@lv_name} does not exist in volume group #{@vg_name}" 41 | end 42 | end 43 | 44 | # Return a string containing a canonical path to the block device 45 | # representing this LV. 46 | def path 47 | "/dev/mapper/#{@vg_name.gsub('-', '--')}-#{@lv_name.gsub('-', '--')}" 48 | end 49 | 50 | # Is this LV a snapshot? 51 | def snapshot? 52 | @lvcfg.snapshot? 53 | end 54 | 55 | # Return an LVM::LogicalVolume object which is the origin volume of 56 | # this one (if this LV is a snapshot), or `nil` otherwise. 57 | def origin 58 | return nil unless snapshot? 59 | 60 | if @lvcfg.origin 61 | LVM::LogicalVolume.new(@vg_name, @lvcfg.origin) 62 | else 63 | origin_lv_name = @vgcfg.logical_volumes.values.find { |lv| lv.cow_store == @lv_name }.origin 64 | LVM::LogicalVolume.new(@vg_name, origin_lv_name) 65 | end 66 | end 67 | 68 | # Return an array of ranges, each of which represents an inclusive range 69 | # of bytes which are different between this logical volume and its 70 | # origin. 71 | # 72 | # If this LV is not a snapshot, this method returns an empty array. 73 | # 74 | def changes 75 | return [] unless snapshot? 76 | 77 | if @lvcfg.thin? 78 | LVM::ThinSnapshot.new(@vg_name, @lv_name) 79 | else 80 | LVM::Snapshot.new(@vg_name, @lv_name) 81 | end.differences 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /lib/lvm/snapshot.rb: -------------------------------------------------------------------------------- 1 | require 'rexml/document' 2 | require 'lvm/helpers' 3 | 4 | module LVM; end 5 | 6 | class LVM::Snapshot 7 | include LVM::Helpers 8 | 9 | def initialize(vg, lv) 10 | @vg = vg 11 | @lv = lv 12 | end 13 | 14 | # Return an array of ranges which are the bytes which are different 15 | # between the origin and the snapshot. 16 | def differences 17 | @differences ||= begin 18 | # For a regular, old-skool snapshot, getting the differences is 19 | # pretty trivial -- just read through the snapshot metadata, and 20 | # the list of changed blocks is right there. 21 | # 22 | diff_block_list = [] 23 | 24 | File.open(metadata_device, 'r') do |metafd| 25 | in_progress = true 26 | 27 | # The first chunk of the metadata LV is the header, which we 28 | # don't care for at all 29 | metafd.seek chunk_size, IO::SEEK_SET 30 | 31 | while in_progress 32 | # The snapshot on-disk format is a stream of , 33 | # sets; within each , it's little-endian 64-bit block 34 | # IDs -- the first is the location (chunk_size * offset) in the origin 35 | # LV that the data has been changed, the second is the location (again, 36 | # chunk_size * offset) in the metadata LV where the changed data is 37 | # being stored. 38 | (chunk_size / 16).times do 39 | origin_offset, snap_offset = metafd.read(16).unpack("QQ") 40 | origin_offset = dtohq(origin_offset) 41 | snap_offset = dtohq(snap_offset) 42 | 43 | # A snapshot offset of 0 would point back to the metadata 44 | # device header, so that's clearly invalid -- hence it's the 45 | # "no more blocks" indicator. 46 | if snap_offset == 0 47 | in_progress = false 48 | break 49 | end 50 | 51 | diff_block_list << origin_offset 52 | end 53 | 54 | # We've read through a set of origin => data mappings; now we need 55 | # to take a giant leap over the data blocks that follow it. 56 | metafd.seek chunk_size * chunk_size / 16, IO::SEEK_CUR 57 | end 58 | end 59 | 60 | # Block-to-byte-range is pretty trivial, and we're done! 61 | diff_block_list.map do |b| 62 | ((b*chunk_size)..(((b+1)*chunk_size)-1)) 63 | end 64 | 65 | # There is one optimisation we could make here that we haven't -- 66 | # coalescing adjacent byte ranges into single larger ranges. I haven't 67 | # done it for two reasons: Firstly, I don't have any idea how much of a 68 | # real-world benefit it would be, and secondly, I couldn't work out how 69 | # to do it elegantly. So I punted. 70 | end 71 | end 72 | 73 | def origin 74 | # Man old-skool snapshots are weird 75 | vgcfg.logical_volumes.values.find { |lv| lv.cow_store == @lv }.origin 76 | end 77 | 78 | private 79 | def vgcfg 80 | @vgcfg ||= LVM::VGConfig.new(@vg) 81 | end 82 | 83 | def chunk_size 84 | @chunk_size ||= metadata_header[:chunk_size] 85 | end 86 | 87 | def metadata_header 88 | @metadata_header ||= begin 89 | magic, valid, version, chunk_size = File.read(metadata_device, 16).unpack("VVVV") 90 | 91 | unless magic == 0x70416e53 92 | raise RuntimeError, 93 | "#{@vg}/#{@lv}: Invalid snapshot magic number" 94 | end 95 | 96 | unless valid == 1 97 | raise RuntimeError, 98 | "#{@vg}/#{@lv}: Snapshot is marked as invalid" 99 | end 100 | 101 | unless version == 1 102 | raise RuntimeError, 103 | "#{@vg}/#{@lv}: Incompatible snapshot metadata version" 104 | end 105 | 106 | { :chunk_size => chunk_size * 512 } 107 | end 108 | end 109 | 110 | def metadata_device 111 | "/dev/mapper/#{@vg.gsub('-', '--')}-#{@lv.gsub('-', '--')}-cow" 112 | end 113 | end 114 | -------------------------------------------------------------------------------- /spec/vg_cfg_spec.rb: -------------------------------------------------------------------------------- 1 | require 'lvm/vg_config' 2 | 3 | describe LVM::VGConfig do 4 | let(:vgcfg) do 5 | LVM::VGConfig.new( 6 | vg_name, 7 | :vgcfgbackup_command => File.expand_path( 8 | '../fixtures/vgcfgbackup', 9 | __FILE__ 10 | ) 11 | ) 12 | end 13 | 14 | context "trivial config" do 15 | let(:vg_name) { "trivial" } 16 | 17 | it "parses successfully" do 18 | expect { vgcfg }.to_not raise_error 19 | end 20 | 21 | it "Gives us back a VgCfg" do 22 | expect(vgcfg).to be_an(LVM::VGConfig) 23 | end 24 | 25 | it "has a version" do 26 | expect(vgcfg.version).to eq(1) 27 | end 28 | 29 | it "has a description" do 30 | expect(vgcfg.description).to eq("vgcfgbackup -f /tmp/faffen2") 31 | end 32 | end 33 | 34 | context "volume group metadata" do 35 | let(:vg_name) { "vgmetadata" } 36 | 37 | it "parses successfully" do 38 | expect { vgcfg }.to_not raise_error 39 | end 40 | 41 | it "has a UUID" do 42 | expect(vgcfg.uuid).to match(/^[A-Za-z0-9-]+$/) 43 | end 44 | end 45 | 46 | context "physical volume" do 47 | let(:vg_name) { "physicalvolume" } 48 | 49 | it "parses successfully" do 50 | expect { vgcfg }.to_not raise_error 51 | end 52 | 53 | it "is its own class" do 54 | expect(vgcfg.physical_volumes["pv0"]).to be_an(LVM::PVConfig) 55 | end 56 | end 57 | 58 | context "complete config" do 59 | let(:vg_name) { "fullconfig" } 60 | 61 | it "parses successfully" do 62 | expect { vgcfg }.to_not raise_error 63 | end 64 | 65 | it "contains logical volumes" do 66 | expect(vgcfg.logical_volumes).to be_a(Hash) 67 | 68 | vgcfg.logical_volumes.values.each { |lv| expect(lv).to be_an(LVM::LVConfig) } 69 | end 70 | 71 | it "has an LV named thintest" do 72 | expect(vgcfg.logical_volumes['thintest']).to_not be(nil) 73 | end 74 | 75 | context "thintest LV" do 76 | let(:lv) { vgcfg.logical_volumes['thintest'] } 77 | 78 | it "is thin" do 79 | expect(lv.thin?).to be(true) 80 | end 81 | 82 | it "is not a snapshot" do 83 | expect(lv.snapshot?).to be(false) 84 | end 85 | 86 | it "belongs to 'thinpool'" do 87 | expect(lv.thin_pool).to eq("thinpool") 88 | end 89 | 90 | it "has device_id of 1" do 91 | expect(lv.device_id).to eq(1) 92 | end 93 | end 94 | 95 | it "has an LV named thinsnap2" do 96 | expect(vgcfg.logical_volumes['thinsnap2']).to_not be(nil) 97 | end 98 | 99 | context "thinsnap2 LV" do 100 | let(:lv) { vgcfg.logical_volumes['thinsnap2'] } 101 | 102 | it "is thin" do 103 | expect(lv.thin?).to be(true) 104 | end 105 | 106 | it "belongs to 'thinpool'" do 107 | expect(lv.thin_pool).to eq("thinpool") 108 | end 109 | 110 | it "has device_id of 3" do 111 | expect(lv.device_id).to eq(3) 112 | end 113 | 114 | it "is a snapshot" do 115 | expect(lv.snapshot?).to be(true) 116 | end 117 | 118 | it "is a snapshot of 'thintest'" do 119 | expect(lv.origin).to eq('thintest') 120 | end 121 | end 122 | 123 | context "snapshot0 LV" do 124 | let(:lv) { vgcfg.logical_volumes['snapshot0'] } 125 | 126 | it "has a CoW store" do 127 | expect(lv.cow_store).to eq('rootsnap') 128 | end 129 | 130 | it "is not thin" do 131 | expect(lv.thin?).to be(false) 132 | end 133 | 134 | it "is not a snapshot" do 135 | expect(lv.snapshot?).to be(false) 136 | end 137 | end 138 | 139 | context "rootsnap LV" do 140 | let(:lv) { vgcfg.logical_volumes['rootsnap'] } 141 | 142 | it "is not thin" do 143 | expect(lv.thin?).to be(false) 144 | end 145 | 146 | it "is a snapshot" do 147 | expect(lv.snapshot?).to be(true) 148 | end 149 | end 150 | 151 | context "thinpool" do 152 | let(:lv) { vgcfg.logical_volumes['thinpool'] } 153 | 154 | it "has a chunk size" do 155 | expect(lv.chunk_size).to eq(65536) 156 | end 157 | 158 | it "is not thin" do 159 | expect(lv.thin?).to be(false) 160 | end 161 | 162 | it "is not a snapshot" do 163 | expect(lv.snapshot?).to be(false) 164 | end 165 | end 166 | end 167 | 168 | context "striped config" do 169 | let(:vg_name) { "striped" } 170 | 171 | it "parses successfully" do 172 | expect { vgcfg }.to_not raise_error 173 | end 174 | 175 | it "contains logical volumes" do 176 | expect(vgcfg.logical_volumes).to be_a(Hash) 177 | 178 | vgcfg.logical_volumes.values.each { |lv| expect(lv).to be_an(LVM::LVConfig) } 179 | end 180 | end 181 | end 182 | -------------------------------------------------------------------------------- /spec/fixtures/fullconfig: -------------------------------------------------------------------------------- 1 | contents = "Text Format Volume Group" 2 | version = 1 3 | 4 | description = "vgcfgbackup -f /tmp/faffen2" 5 | 6 | fullconfig { 7 | id = "zAPMOi-5QlD-fp1M-FMgz-biha-TQIP-X9TfqX" 8 | seqno = 436 9 | format = "lvm2" # informational 10 | status = ["RESIZEABLE", "READ", "WRITE"] 11 | flags = [] 12 | extent_size = 8192 # 4 Megabytes 13 | max_lv = 0 14 | max_pv = 0 15 | metadata_copies = 0 16 | 17 | physical_volumes { 18 | 19 | pv0 { 20 | id = "j7ZiWs-1fHB-aAwq-yL29-pBOk-H4hR-AOIe0P" 21 | device = "/dev/dm-0" # Hint only 22 | 23 | status = ["ALLOCATABLE"] 24 | flags = [] 25 | dev_size = 1952545832 # 931.046 Gigabytes 26 | pe_start = 384 27 | pe_count = 238347 # 931.043 Gigabytes 28 | } 29 | } 30 | 31 | logical_volumes { 32 | 33 | swap { 34 | id = "BOfqkb-V4jw-7cBX-UR3D-SN3f-wWvd-NwysSj" 35 | status = ["READ", "WRITE", "VISIBLE"] 36 | flags = [] 37 | segment_count = 1 38 | 39 | segment1 { 40 | start_extent = 0 41 | extent_count = 2048 # 8 Gigabytes 42 | 43 | type = "striped" 44 | stripe_count = 1 # linear 45 | 46 | stripes = [ 47 | "pv0", 0 48 | ] 49 | } 50 | } 51 | 52 | root { 53 | id = "8sbW3s-qB1u-6IlS-ABSj-PAMU-UxRk-I18ECh" 54 | status = ["READ", "WRITE", "VISIBLE"] 55 | flags = [] 56 | segment_count = 1 57 | 58 | segment1 { 59 | start_extent = 0 60 | extent_count = 3840 # 15 Gigabytes 61 | 62 | type = "striped" 63 | stripe_count = 1 # linear 64 | 65 | stripes = [ 66 | "pv0", 21504 67 | ] 68 | } 69 | } 70 | 71 | thinpool { 72 | id = "YzvLKN-ly3F-gNgL-2YUQ-7J4v-nPw6-aEH0Q4" 73 | status = ["READ", "WRITE", "VISIBLE"] 74 | flags = [] 75 | segment_count = 1 76 | 77 | segment1 { 78 | start_extent = 0 79 | extent_count = 25600 # 100 Gigabytes 80 | 81 | type = "thin-pool" 82 | metadata = "thinpool_tmeta" 83 | pool = "thinpool_tdata" 84 | transaction_id = 13 85 | chunk_size = 128 # 64 Kilobytes 86 | discards = "passdown" 87 | zero_new_blocks = 1 88 | } 89 | } 90 | 91 | thintest { 92 | id = "KTu6v0-XGcp-dJnd-5IFv-t1rH-fD6t-gqijyN" 93 | status = ["READ", "WRITE", "VISIBLE"] 94 | flags = [] 95 | segment_count = 1 96 | 97 | segment1 { 98 | start_extent = 0 99 | extent_count = 2560 # 10 Gigabytes 100 | 101 | type = "thin" 102 | thin_pool = "thinpool" 103 | transaction_id = 10 104 | device_id = 1 105 | } 106 | } 107 | 108 | thinsnap { 109 | id = "ltQMX8-SaQV-S5fP-zF2m-wb78-bveG-JiCMnv" 110 | status = ["READ", "WRITE", "VISIBLE"] 111 | flags = ["ACTIVATION_SKIP"] 112 | segment_count = 1 113 | 114 | segment1 { 115 | start_extent = 0 116 | extent_count = 2560 # 10 Gigabytes 117 | 118 | type = "thin" 119 | thin_pool = "thinpool" 120 | transaction_id = 11 121 | device_id = 2 122 | origin = "thintest" 123 | } 124 | } 125 | 126 | thinsnap2 { 127 | id = "E6Pa80-8PIh-GPAM-Qe6G-CRLc-rgeN-rno8pZ" 128 | status = ["READ", "WRITE", "VISIBLE"] 129 | flags = ["ACTIVATION_SKIP"] 130 | segment_count = 1 131 | 132 | segment1 { 133 | start_extent = 0 134 | extent_count = 2560 # 10 Gigabytes 135 | 136 | type = "thin" 137 | thin_pool = "thinpool" 138 | transaction_id = 12 139 | device_id = 3 140 | origin = "thintest" 141 | } 142 | } 143 | 144 | rootsnap { 145 | id = "nzBjUt-ydUU-zytj-Crkm-5sCk-16aJ-fufe1C" 146 | status = ["READ", "WRITE"] 147 | flags = [] 148 | segment_count = 1 149 | 150 | segment1 { 151 | start_extent = 0 152 | extent_count = 2560 # 10 Gigabytes 153 | 154 | type = "striped" 155 | stripe_count = 1 # linear 156 | 157 | stripes = [ 158 | "pv0", 145433 159 | ] 160 | } 161 | } 162 | 163 | lvol0_pmspare { 164 | id = "81KshE-Cjpm-Ec2w-bHg9-imDu-dAD7-SzCULf" 165 | status = ["READ", "WRITE"] 166 | flags = [] 167 | segment_count = 1 168 | 169 | segment1 { 170 | start_extent = 0 171 | extent_count = 25 # 100 Megabytes 172 | 173 | type = "striped" 174 | stripe_count = 1 # linear 175 | 176 | stripes = [ 177 | "pv0", 2048 178 | ] 179 | } 180 | } 181 | 182 | thinpool_tmeta { 183 | id = "B8z5ds-mF2h-JfFb-pg4w-dNKA-6DUr-3sG58d" 184 | status = ["READ", "WRITE"] 185 | flags = [] 186 | segment_count = 1 187 | 188 | segment1 { 189 | start_extent = 0 190 | extent_count = 25 # 100 Megabytes 191 | 192 | type = "striped" 193 | stripe_count = 1 # linear 194 | 195 | stripes = [ 196 | "pv0", 145408 197 | ] 198 | } 199 | } 200 | 201 | thinpool_tdata { 202 | id = "gN0VJh-1k4l-DvcD-o3b2-dbSW-YL2P-Yq4sF0" 203 | status = ["READ", "WRITE"] 204 | flags = [] 205 | segment_count = 1 206 | 207 | segment1 { 208 | start_extent = 0 209 | extent_count = 25600 # 100 Gigabytes 210 | 211 | type = "striped" 212 | stripe_count = 1 # linear 213 | 214 | stripes = [ 215 | "pv0", 119808 216 | ] 217 | } 218 | } 219 | 220 | snapshot0 { 221 | id = "c1b4sb-GxW1-8P1A-pRO5-WgnT-6JtO-dnXIr1" 222 | status = ["READ", "WRITE", "VISIBLE"] 223 | flags = [] 224 | segment_count = 1 225 | 226 | segment1 { 227 | start_extent = 0 228 | extent_count = 3840 # 15 Gigabytes 229 | 230 | type = "snapshot" 231 | chunk_size = 8 232 | origin = "root" 233 | cow_store = "rootsnap" 234 | } 235 | } 236 | } 237 | } 238 | -------------------------------------------------------------------------------- /lib/lvm/thin_snapshot.rb: -------------------------------------------------------------------------------- 1 | require 'rexml/document' 2 | 3 | module LVM; end 4 | 5 | class LVM::ThinSnapshot 6 | def initialize(vg, lv) 7 | @vg = vg 8 | @lv = lv 9 | end 10 | 11 | # Return an array of ranges which are the bytes which are different 12 | # between the origin and the snapshot. 13 | def differences 14 | # This is a relatively complicated multi-step process. We have two 15 | # piles of => mappings, one for the "origin" 16 | # (the LV that's changing) and one for the "snapshot" (the LV that 17 | # represents some past point-in-time). What we need to get out at the 18 | # end is an array of (..) ranges which cover 19 | # the parts of the volumes which are different (or that at least point 20 | # to different blocks within the data pool). 21 | # 22 | # This is going to take a few steps to accomplish. 23 | # 24 | # First, we translate each of the hashes into a list of two-element 25 | # arrays, expanding out ranges, because it means we don't have to 26 | # handle ranges differently in later steps (a worthwhile optimisation, 27 | # in my opinion -- if you think differently, I'd *really* welcome a 28 | # patch that handles ranges in-place without turning into a complete 29 | # mind-fuck, because I couldn't manage it). 30 | # 31 | # Next, we work out which mappings are "different" in all the possible 32 | # ways. There's four cases we might come across: 33 | # 34 | # 1. Both origin and snapshot map the same LV block to the same data 35 | # block. This is a mapping we can discard from the set of 36 | # differences, because, well, it isn't a difference. 37 | # 38 | # 2. Both origin and snapshot map the same LV block, but they point 39 | # to different data blocks. That's the easiest sort of difference 40 | # to understand, and we *could* catch that just by comparing all 41 | # of the mappings in the origin with the mappings in the snapshot, 42 | # and listing those whose value differs. But that wouldn't catch 43 | # these next two cases... 44 | # 45 | # 3. The origin maps a particular LV block to a data block, but the 46 | # snapshot doesn't have any mapping for that LV block. This would 47 | # occur quite commonly -- whenever a location in the origin LV was 48 | # written to for the first time after the snapshot is taken. You 49 | # would catch all these (as well as the previous case) by taking 50 | # the origin block map and removing any mappings which were 51 | # identical in the snapshot block map. However, that would fail to 52 | # identify... 53 | # 54 | # 4. A block in the snapshot is mapped, when the corresponding origin 55 | # block is *not* mapped. Given the assumption that the snapshot 56 | # was never written to, how could this possibly happen? One word: 57 | # "discard". Mappings in the origin block list are removed if 58 | # the block to which they refer is discarded. Finding *these* (and also 59 | # all mappings of type 2) by the reverse process to that in case 60 | # 3 -- simply remove from the snapshot block list all mappings which 61 | # appear identically in the origin block list. 62 | # 63 | # In order to get all of 2, 3, and 4 together, we can simply do the 64 | # operations described in steps 3 & 4 and add the results together. Sure, 65 | # we'll get two copies of all "type 2" block maps, but #uniq is good at 66 | # fixing that. 67 | # 68 | @differences ||= begin 69 | diff_maps = ((flat_origin_blocklist - flat_snapshot_blocklist) + 70 | (flat_snapshot_blocklist - flat_origin_blocklist) 71 | ).uniq 72 | 73 | # At this point, we're off to a good start -- we've got the mappings 74 | # that are different. But we're not actually interested in the 75 | # mappings themselves -- all we want is "the list of LV blocks which 76 | # are different" (we'll translate LV blocks into byte ranges next). 77 | # 78 | changed_blocks = diff_maps.map { |m| m[0] }.uniq 79 | 80 | # Block-to-byte-range is pretty trivial, and we're done! 81 | changed_blocks.map do |b| 82 | ((b*chunk_size)..(((b+1)*chunk_size)-1)) 83 | end 84 | 85 | # There is one optimisation we could make here that we haven't -- 86 | # coalescing adjacent byte ranges into single larger ranges. I haven't 87 | # done it for two reasons: Firstly, I don't have any idea how much of a 88 | # real-world benefit it would be, and secondly, I couldn't work out how 89 | # to do it elegantly. So I punted. 90 | end 91 | end 92 | 93 | def origin 94 | @origin ||= vgcfg.logical_volumes[@lv].origin 95 | end 96 | 97 | private 98 | def vgcfg 99 | @vgcfg ||= LVM::VGConfig.new(@vg) 100 | end 101 | 102 | def flat_origin_blocklist 103 | @flat_origin_blocklist ||= flatten_blocklist(origin_blocklist) 104 | end 105 | 106 | def flat_snapshot_blocklist 107 | @flat_snapshot_blocklist ||= flatten_blocklist(snapshot_blocklist) 108 | end 109 | 110 | def origin_blocklist 111 | @origin_blocklist ||= vg_block_dump[@vgcfg.logical_volumes[origin].device_id] 112 | end 113 | 114 | def snapshot_blocklist 115 | @snapshot_blocklist ||= vg_block_dump[@vgcfg.logical_volumes[@lv].device_id] 116 | end 117 | 118 | def thin_pool_name 119 | @thin_pool_name ||= vgcfg.logical_volumes[@lv].thin_pool 120 | end 121 | 122 | def thin_pool 123 | @thin_pool ||= vgcfg.logical_volumes[thin_pool_name] 124 | end 125 | 126 | def chunk_size 127 | @chunk_size ||= thin_pool.chunk_size 128 | end 129 | 130 | # Take a hash of => elements and turn 131 | # it into an array of [block, block] pairs -- any => 132 | # elements get expanded out into their constituent => 133 | # parts. 134 | # 135 | def flatten_blocklist(bl) 136 | bl.to_a.map do |elem| 137 | # Ranges are *hard*, let's go shopping 138 | if elem[0].is_a? Range 139 | lv_blocks = elem[0].to_a 140 | data_blocks = elem[1].to_a 141 | 142 | # This will now produce an array of two-element arrays, which 143 | # will itself be inside the top-level array that we're mapping. 144 | # A flatten(1) at the end will take care of that problem, 145 | # though. 146 | lv_blocks.inject([]) { |a, v| a << [v, data_blocks[a.length]] } 147 | elsif elem[0].is_a? Fixnum 148 | # We wrap the [lv, data] pair that is `elem` into another array, 149 | # so that the coming #flatten call doesn't de-array our matched 150 | # pair 151 | [elem] 152 | else 153 | raise ArgumentError, 154 | "CAN'T HAPPEN: Unknown key type (#{elem.class}) found in blocklist" 155 | end 156 | end.flatten(1) 157 | end 158 | 159 | def vg_block_dump 160 | @vg_block_dump ||= begin 161 | dev_name = "/dev/mapper/#{@vg.gsub('-', '--')}-#{thin_pool_name.gsub('-','--')}" 162 | cmd = <&2; dmsetup message #{dev_name}-tpool 0 release_metadata_snap; exit 1; } 166 | thin_dump --format xml #{dev_name}_tmeta --metadata-snap "$block" || { dmsetup message #{dev_name}-tpool 0 release_metadata_snap; exit 1; } 167 | dmsetup message #{dev_name}-tpool 0 release_metadata_snap 168 | EOC 169 | result = %x[ #{cmd} ] 170 | unless $?.exitstatus == 0 171 | raise RuntimeError, 172 | "#{@vg}/#{@lv}: Failed to get metadata for #{thin_pool_name}" 173 | end 174 | 175 | doc = REXML::Document.new(result) 176 | unless doc != nil 177 | raise RuntimeError, 178 | "#{@vg}/#{@lv}: Failed to parse metadata for #{thin_pool_name}" 179 | end 180 | 181 | doc.elements['superblock'].inject({}) do |h, dev| 182 | next h unless dev.node_type == :element 183 | 184 | maps = dev.elements[''].inject({}) do |h2, r| 185 | next h2 unless r.node_type == :element 186 | 187 | if r.name == 'single_mapping' 188 | h2[r.attribute('origin_block').value.to_i] = r.attribute('data_block').value.to_i 189 | else 190 | len = r.attribute('length').value.to_i 191 | ori = r.attribute('origin_begin').value.to_i 192 | dat = r.attribute('data_begin').value.to_i 193 | h2[(ori..ori+len-1)] = (dat..dat+len-1) 194 | end 195 | 196 | h2 197 | end 198 | 199 | h[dev.attribute('dev_id').value.to_i] = maps 200 | h 201 | end 202 | end 203 | end 204 | end 205 | -------------------------------------------------------------------------------- /bin/lvmsync: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # Transfer a set of changes made to the origin of a snapshot LV to another 4 | # block device, possibly using SSH to send to a remote system. 5 | # 6 | # Usage: Start with lvmsync --help, or read the README for all the gory 7 | # details. 8 | # 9 | # Copyright (C) 2011-2014 Matt Palmer 10 | # 11 | # This program is free software: you can redistribute it and/or modify it 12 | # under the terms of the GNU General Public License version 3, as published 13 | # by the Free Software Foundation. 14 | # 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # `LICENCE` file for more details. 19 | # 20 | require 'optparse' 21 | require 'lvm' 22 | require 'git-version-bump' 23 | require 'open3' 24 | 25 | PROTOCOL_VERSION = "lvmsync PROTO[3]" 26 | 27 | include LVM::Helpers 28 | 29 | def main() 30 | # Parse me some options 31 | options = { :rsh => "ssh" } 32 | 33 | OptionParser.new do |opts| 34 | opts.banner = "Usage: lvmsync [options]" 35 | opts.separator "" 36 | opts.separator " lvmsync [-v|--verbose] [--snapback ] [--stdout | [:]]" 37 | opts.separator " lvmsync [-v|--verbose] [--snapback ] --apply " 38 | opts.separator " lvmsync <-V|--version>" 39 | opts.separator "" 40 | 41 | opts.on("--server", "Run in server mode (deprecated; use '--apply -' instead)") do |v| 42 | options[:server] = true 43 | end 44 | 45 | opts.on("-v", "--[no-]verbose", 46 | "Run verbosely") { |v| $verbose = v } 47 | 48 | opts.on("-d", "--[no-]debug", 49 | "Print debugging information") { |v| $debug = v } 50 | 51 | opts.on("-q", "--[no-]quiet", 52 | "Run quietly") { |v| $quiet = v } 53 | 54 | opts.on("-b ", "--snapback ", 55 | "Make a backup snapshot file on the destination") do |v| 56 | options[:snapback] = v 57 | end 58 | 59 | opts.on("-a", "--apply ", 60 | "Apply mode: write the contents of a snapback file to a device") do |v| 61 | options[:apply] = v 62 | end 63 | 64 | opts.on("-s", "--stdout", "Write output data to stdout rather than another lvmsync process") do |v| 65 | options[:stdout] = true 66 | end 67 | 68 | opts.on("-r", "--data-source ", "Read data blocks from a block device other than the snapshot origin") do |v| 69 | options[:source] = v 70 | end 71 | 72 | opts.on("-e", "--rsh ", "Use specified command when invoking SSH") do |v| 73 | options[:rsh] = v 74 | end 75 | 76 | opts.on("-V", "--version", "Print version of lvmsync") do |v| 77 | begin 78 | puts "lvmsync #{GVB.version}" 79 | exit 0 80 | rescue GVB::VersionUnobtainable 81 | fatal "Unable to determine lvmsync version.\n" + 82 | "Install lvmsync as a gem, or run it from within a git checkout" 83 | end 84 | end 85 | end.parse! 86 | 87 | if $quiet and ($verbose or $debug) 88 | fatal "I can't run quietly *and* verbosely at the same time!" 89 | end 90 | 91 | if options[:apply] 92 | if ARGV[0].nil? 93 | fatal "No destination device specified." 94 | end 95 | options[:device] = ARGV[0] 96 | run_apply(options) 97 | elsif options[:server] 98 | info "--server is deprecated; please use '--apply -' instead" 99 | if (ARGV[0].nil?) 100 | fatal "No destination block device specified. WTF?" 101 | end 102 | options[:apply] = '-' 103 | options[:device] = ARGV[0] 104 | run_apply(options) 105 | else 106 | if ARGV[0].nil? 107 | fatal "No snapshot specified. Exiting. Do you need --help?" 108 | end 109 | options[:snapdev] = ARGV[0] 110 | 111 | if options[:stdout] and options[:snapback] 112 | fatal "--snapback cannot be used with --stdout" 113 | end 114 | 115 | if (options[:stdout].nil? and ARGV[1].nil?) 116 | fatal "No destination specified." 117 | end 118 | if options[:stdout].nil? 119 | dev, host = ARGV[1].split(':', 2).reverse 120 | options[:desthost] = host 121 | options[:destdev] = dev 122 | end 123 | 124 | run_client(options) 125 | end 126 | end 127 | 128 | def run_apply(opts) 129 | snapfile = opts[:snapback] ? File.open(opts[:snapback], 'w') : nil 130 | infile = opts[:apply] == '-' ? $stdin : File.open(opts[:apply], 'r') 131 | destdev = opts[:device] 132 | 133 | process_dumpdata(infile, destdev, snapfile, opts) 134 | ensure 135 | snapfile.close unless snapfile.nil? 136 | infile.close unless infile.nil? or infile == $stdin 137 | end 138 | 139 | def process_dumpdata(instream, destdev, snapback = nil, opts = {}) 140 | handshake = instream.readline.chomp 141 | unless handshake == PROTOCOL_VERSION 142 | fatal "Handshake failed; protocol mismatch? (saw '#{handshake}' expected '#{PROTOCOL_VERSION}'" 143 | end 144 | 145 | snapback.puts handshake if snapback 146 | 147 | verbose "Writing changed data to #{destdev.inspect}" 148 | File.open(destdev, 'r+') do |dest| 149 | while header = instream.read(12) 150 | offset, chunksize = header.unpack("QN") 151 | offset = ntohq(offset) 152 | 153 | begin 154 | debug "Seeking to #{offset}" 155 | dest.seek offset 156 | rescue Errno::EINVAL 157 | # In certain rare circumstances, we want to transfer a block 158 | # device where the destination is smaller than the source (DRBD 159 | # volumes is the canonical use case). So, we ignore attempts to 160 | # seek past the end of the device. Yes, this may lose data, but 161 | # if you didn't notice that your dd shit itself, it's unlikely 162 | # you're going to notice now. 163 | 164 | info "Write occured past end of device" 165 | 166 | # Skip the chunk of data 167 | instream.read(chunksize) 168 | # Go to the next chunk 169 | next 170 | end 171 | 172 | if snapback 173 | snapback.write(header) 174 | snapback.write dest.read(chunksize) 175 | # Got to back to where we were before, since the read from dest 176 | # has advanced the file pointer by `chunksize` 177 | dest.seek offset 178 | end 179 | dest.write instream.read(chunksize) 180 | debug "Wrote #{chunksize} bytes at #{offset}" 181 | end 182 | end 183 | end 184 | 185 | def run_client(opts) 186 | snapshot = opts[:snapdev] 187 | desthost = opts[:desthost] 188 | destdev = opts[:destdev] 189 | outfd = nil 190 | 191 | lv = begin 192 | LVM::LogicalVolume.new(snapshot) 193 | rescue RuntimeError => e 194 | fatal "#{snapshot}: could not find logical volume (#{e.message})" 195 | end 196 | 197 | unless lv.snapshot? 198 | fatal "#{snapshot}: Not a snapshot device" 199 | end 200 | 201 | # Since, in principle, we're not supposed to be reading from snapshot 202 | # devices directly, the kernel makes no attempt to make the device's read 203 | # cache stay in sync with the actual state of the device. As a result, 204 | # we have to manually drop all caches before the data looks consistent. 205 | # PERFORMANCE WIN! 206 | File.open("/proc/sys/vm/drop_caches", 'w') { |fd| fd.print "3" } 207 | 208 | snapback = opts[:snapback] ? "--snapback #{opts[:snapback]}" : '' 209 | 210 | source = opts[:source] || lv.origin.path 211 | verbose "Data source: #{source}" 212 | 213 | if opts[:stdout] 214 | dump_changes(lv, source, $stdout, opts) 215 | else 216 | verbose = $verbose ? '-v' : '' 217 | debug = $debug ? '-d' : '' 218 | 219 | server_cmd = if desthost 220 | "#{opts[:rsh]} #{desthost} lvmsync --apply - #{snapback} #{verbose} #{debug} #{destdev}" 221 | else 222 | "#{$0} --apply - #{snapback} #{verbose} #{destdev}" 223 | end 224 | 225 | exit_status = nil 226 | errors = nil 227 | 228 | Open3.popen3(server_cmd) do |stdin_fd, stdout_fd, stderr_fd, wait_thr| 229 | fds = [stdout_fd, stderr_fd] 230 | 231 | dump_changes(lv, source, stdin_fd, opts) do 232 | # Remember that this fires between *every* block sent to the 233 | # receiver, so don't do anything particularly slow in here! 234 | until (active_fds = IO.select(fds, [], [], 0)).nil? 235 | active_fds[0].each do |fd| 236 | begin 237 | info "\e[2K\rremote:#{fd.readline}" 238 | rescue EOFError, Errno::EPIPE 239 | fd.close 240 | fds.delete(fd) 241 | end 242 | end 243 | end 244 | end 245 | 246 | stdin_fd.close 247 | 248 | # Read any residual data that might be left in the stdout/stderr of 249 | # the remote; we've got to do this with a timeout because of 250 | # OpenSSH, which, when used in ControlMaster ("multiplexing") mode, 251 | # holds open stderr, meaning that IO.select will never indicate 252 | # that stderr is finished. 253 | until (active_fds = IO.select(fds, [], [], 0.1)).nil? 254 | active_fds[0].each do |fd| 255 | begin 256 | info "\e[2K\rremote:#{fd.readline}" 257 | rescue EOFError, Errno::EPIPE 258 | fd.close 259 | fds.delete(fd) 260 | end 261 | end 262 | end 263 | exit_status = wait_thr.value if wait_thr 264 | end 265 | 266 | if (exit_status or $?).exitstatus != 0 267 | fatal "APPLY FAILED." 268 | end 269 | end 270 | end 271 | 272 | def dump_changes(snapshot, source, outfd, opts) 273 | outfd.puts PROTOCOL_VERSION 274 | 275 | start_time = Time.now 276 | xfer_count = 0 277 | xfer_size = 0 278 | total_size = 0 279 | change_count = snapshot.changes.length 280 | 281 | File.open(source, 'r') do |origindev| 282 | snapshot.changes.each do |r| 283 | xfer_count += 1 284 | chunk_size = r.last - r.first + 1 285 | xfer_size += chunk_size 286 | 287 | debug "Sending chunk #{r.to_s}..." 288 | 289 | origindev.seek(r.first, IO::SEEK_SET) 290 | 291 | begin 292 | outfd.print [htonq(r.first), chunk_size].pack("QN") 293 | outfd.print origindev.read(chunk_size) 294 | rescue Errno::EPIPE 295 | $stderr.puts "Remote prematurely closed the connection" 296 | yield if block_given? 297 | return 298 | end 299 | 300 | # Progress bar! 301 | if xfer_count % 100 == 50 and !$quiet 302 | $stderr.printf "\e[2K\rSending chunk %i of %i, %.2fMB/s", 303 | xfer_count, 304 | change_count, 305 | xfer_size / (Time.now - start_time) / 1048576 306 | $stderr.flush 307 | end 308 | yield if block_given? 309 | end 310 | 311 | origindev.seek(0, IO::SEEK_END) 312 | total_size = origindev.tell 313 | end 314 | 315 | unless $quiet 316 | $stderr.printf "\rTransferred %i bytes in %.2f seconds\n", 317 | xfer_size, Time.now - start_time 318 | 319 | $stderr.printf "You transferred your changes %.2fx faster than a full dd!\n", 320 | total_size.to_f / xfer_size 321 | end 322 | end 323 | 324 | # Take a device name in any number of different formats and return a [VG, LV] pair. 325 | # Raises ArgumentError if the name couldn't be parsed. 326 | def parse_snapshot_name(origname) 327 | case origname 328 | when %r{^/dev/mapper/(.*[^-])-([^-].*)$} then 329 | [$1, $2] 330 | when %r{^/dev/([^/]+)/(.+)$} then 331 | [$1, $2] 332 | when %r{^([^/]+)/(.*)$} then 333 | [$1, $2] 334 | else 335 | raise ArgumentError, 336 | "Could not determine snapshot name and VG from #{origname.inspect}" 337 | end 338 | end 339 | 340 | def debug(s) 341 | $stderr.puts s if $debug 342 | end 343 | 344 | def verbose(s) 345 | $stderr.puts s if $verbose or $debug 346 | end 347 | 348 | def info(s) 349 | $stderr.puts s unless $quiet 350 | end 351 | 352 | def fatal(s, status=1) 353 | $stderr.puts "FATAL ERROR: #{s}" 354 | exit status 355 | end 356 | 357 | main 358 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lvmsync 2 | 3 | Have you ever wanted to do a partial sync on a block device, possibly over a 4 | network, but were stymied by the fact that rsync just didn't work? 5 | 6 | Well, fret no longer. As long as you use LVM for your block devices, you 7 | too can have efficient delta-transfer of changed blocks. 8 | 9 | 10 | ## What is it good for? 11 | 12 | Mostly, transferring entire block devices from one machine to another, with 13 | minimal downtime. Until now, you had to shutdown your service/VM/whatever, 14 | do a big cross-network dd (using netcat or something), and wait while all 15 | that transferred. 16 | 17 | `lvmsync` allows you to use the following workflow to transfer a block 18 | device "mostly live" to another machine: 19 | 20 | 1. Take a snapshot of an existing LV. 21 | 1. Transfer the entire snapshot over the network, while whatever uses the 22 | block device itself keeps running. 23 | 1. When the initial transfer is finished, you shutdown/unmount/whatever the 24 | initial block device. 25 | 1. Run lvmsync on the snapshot to transfer the changed blocks 26 | * The only thing transferred over the network is the blocks that have 27 | changed (which, hopefully, will be minimal) 28 | 1. If you're paranoid, you can md5sum the content of the source and 29 | destination block devices, to make sure everything's OK (although this will 30 | destroy any performance benefit you got by running lvmsync in the first 31 | lace) 32 | 1. Bring the service/VM/whatever back up in it's new home in a *much* 33 | shorter (as in, "orders of magnitude") time than was previously possible. 34 | 35 | `lvmsync` also has a basic "snapshot-and-rollback" feature, where it can 36 | save a copy of the data in the LV that you're overwriting to a file for 37 | later application if you need to rollback. See "Snapback support" under 38 | "How do I use it?" for more details. 39 | 40 | 41 | ## How does it work? 42 | 43 | By the magic of LVM snapshots. `lvmsync` is able to read the metadata that 44 | device-mapper uses to keep track of what parts of the block device have 45 | changed, and use that information to only send those modified blocks over 46 | the network. 47 | 48 | If you're really interested in the gory details, there's a brief "Theory of 49 | Operation" section at the bottom of this README, or else you can just head 50 | straight for the source code. 51 | 52 | 53 | ## Installation 54 | 55 | To run `lvmsync`, you'll need to have a working installation of Ruby 1.8 (or 56 | later) on both the machine you're transferring from, and the machine you're 57 | transferring to. On the source, you'll need `vgcfgbackup` (which is part of 58 | the core LVM2 toolset), and if you want to deal with thin snapshots, you'll 59 | also need `thin_dump` (which is part of the "thin provisioning tools" which 60 | are highly recommended for anyone working with thin-provisioned LVs). For 61 | transferring dumps between machines, you'll need SSH installed and working 62 | between the two machines. 63 | 64 | Installing `lvmsync` itself is easiest using Rubygems: `gem install 65 | lvmsync`. This will install all the dependencies and (presumably) put the 66 | `lvmsync` command itself in root's PATH. If for some reason you want to 67 | install it all by hand, you'll need to copy the contents of `lib/` into a 68 | directory in your Ruby library path, copy `bin/lvmsync` to somewhere on your 69 | PATH, and install the `treetop` and `git-version-bump` gems. 70 | 71 | 72 | ## How do I use it? 73 | 74 | For an overview of all available options, run `lvmsync -h`. 75 | 76 | 77 | ### Efficient block device transfer 78 | 79 | At present, the only part of the block device syncing process that is 80 | automated is the actual transfer of the snapshot changes -- the rest (making 81 | the snapshot, doing the initial transfer, and stopping all writes to the LV) 82 | you'll have to do yourself. Those other steps aren't difficult, though, and 83 | are trivial to script to suit your local environment (see the example, 84 | below). 85 | 86 | Once you've got the snapshot installed, done the initial sync, and stopped 87 | I/O, you just call `lvmsync` like this: 88 | 89 | lvmsync : 90 | 91 | This requires that `lvmsync` is installed on ``, and that you 92 | have the ability to SSH into `` as root. All data transfer 93 | takes place over SSH, because we don't trust any network, and it simplifies 94 | so many things (such as link-level compression, if you want it). If CPU is 95 | an issue, you shouldn't be running LVM on your phone to begin with. 96 | 97 | The reason why `lvmsync` needs you to specify the snapshot you want to sync, 98 | and not the base LV, is that you might have more than one snapshot of a 99 | given LV, and while we can determine the base LV given a snapshot, you can't 100 | work out which snapshot to sync given a base LV. Remember to always specify 101 | the full device path, not just the LV name. 102 | 103 | 104 | #### Example 105 | 106 | Let's say you've got an LV, named `vmsrv1/somevm`, and you'd like to 107 | synchronise it to a new VM server, named `vmsrv2`. Assuming that `lvmsync` is 108 | installed on `vmsrv2` and `vmsrv2` has an LV named `vmsrv2/somevm` large 109 | enough to take the data, the following will do the trick rather nicely (all 110 | commands should be run on `vmsrv1`): 111 | 112 | # Take a snapshot before we do anything, so LVM will record all changes 113 | # made while we're doing the initial sync 114 | lvcreate --snapshot -L10G -n somevm-lvmsync vmsrv1/somevm 115 | 116 | # Pre-sync all data across -- this will take some time, but while it's 117 | # happening the VM is still serving traffic. pv is a great tool for 118 | # showing you how fast your data's moving, but you can leave it out of 119 | # the pipeline if you don't have it installed. 120 | dd if=/dev/vmsrv1/somevm-lvmsync bs=1M | pv -ptrb | ssh root@vmsrv2 dd of=/dev/vmsrv2/somevm bs=1M 121 | 122 | # Shutdown the VM -- the command you use will probably vary 123 | virsh shutdown somevm 124 | 125 | # Once it's shutdown and the block device isn't going to be written to 126 | # any more, then you can run lvmsync 127 | lvmsync /dev/vmsrv1/somevm-lvmsync vmsrv2:/dev/vmsrv2/somevm 128 | 129 | # You can now start up the VM on vmsrv2, after a fairly small period of 130 | # downtime. Once you're done, you can remove the snapshot and, 131 | # presumably, the LV itself, from `vmsrv1` 132 | 133 | 134 | ### Snapback support 135 | 136 | In addition to being able to efficiently transfer the changes to an LV 137 | across a network, `lvmsync` now supports a simple form of point-in-time 138 | recovery, which I've called 'snapback'. 139 | 140 | The way this works is startlingly simple: as `lvmsync` writes the changed 141 | blocks out to the destination block device, it reads the data that is being 142 | overwritten, and stores it to a file (specified with the `--snapback` 143 | option). The format of this file is the same as the wire protocol that 144 | `lvmsync` uses to transfer changed blocks over the network. This means 145 | that, in the event that you need to rollback a block device to an earlier 146 | state, you can do so by simply applying the saved snapback files created 147 | previously, until you get to the desired state. 148 | 149 | 150 | #### Example 151 | 152 | To setup a snapback process, you need to have a local LV, with a snapshot, 153 | whose contents have been sent to a remote server, perhaps something like 154 | this: 155 | 156 | lvcreate --snapshot -L10G -n somevm-snapback vmsrv1/somevm 157 | dd if=/dev/vmsrv1/somevm-snapback bs=1M | pv -ptrb | \ 158 | ssh root@vmsrv2 dd of=/dev/vmsrv2/somevm 159 | 160 | Now, you can run something like the following periodically (say, out of cron 161 | each hour): 162 | 163 | lvcreate --snapshot -L10G -n somevm-snapback-new vmsrv1/somevm 164 | lvmsync /dev/vmsrv1/somevm-snapback vmsrv2:/dev/vmsrv2/somevm --snapback \ 165 | /var/snapbacks/somevm.$(date +%Y%m%d-%H%M) 166 | lvremove -f vmsrv1/somevm-snapback 167 | lvrename vmsrv1/somevm-snapback-new somevm-snapback 168 | 169 | This will produce files in /var/snapbacks named `somevm.`. You 170 | need to create the `somevm-snapback-new` snapshot before you start 171 | `lvmsync`, so that you can guarantee no changes will go unnoticed. 172 | 173 | There are some fairly large caveats to this method -- the LV will still be 174 | collecting writes while you're transferring the snapshots, so you won't get 175 | a consistent snapshot (in the event you have to rollback, it's almost 176 | certain you'll need to fsck). You'll almost certainly want to incorporate 177 | some sort of I/O freezing into the process, but the exact execution of that 178 | is system-specific, and left as an exercise for the reader. 179 | 180 | Restoring data from a snapback setup is straightforward -- just take each 181 | snapback **in reverse order** and run it through `lvmsync --apply` on the 182 | destination machine (`vmsrv2` in our example). Say at 1145 `vmsrv1` 183 | crashed, and it was determined that you needed to rollback to the state of 184 | the system at 8am. You could do this: 185 | 186 | lvmsync --apply /var/snapbacks/somevm.20120119-1100 /dev/vmsrv2/somevm 187 | lvmsync --apply /var/snapbacks/somevm.20120119-1000 /dev/vmsrv2/somevm 188 | lvmsync --apply /var/snapbacks/somevm.20120119-0900 /dev/vmsrv2/somevm 189 | 190 | And you're done -- `/dev/vmsrv2/somevm` is now at the state it was at at 191 | 8am. A whole pile of fsck will no doubt be required, but hopefully you'll 192 | still be able to salvage *something*. 193 | 194 | If you're wondering why I only restored the 0900 snapback, and not the 0800 195 | one, it's because the snapback made at 0900 copied the changes that were sent 196 | at 0800 (and about to be overwritten at 0900) and wrote them to the 0900 197 | snapback file. Confused much? Good. 198 | 199 | 200 | ### Transferring snapshots on the same machine 201 | 202 | If you need to transfer an LV between different VGs on the same machine, 203 | then running everything through SSH is just an unnecessary overhead. If you 204 | instead just run `lvmsync` without the `:` in the destination 205 | specification, everything runs locally, like this: 206 | 207 | lvmsync /dev/vg0/srclv-snapshot /dev/vg1/destlv 208 | 209 | All other parts of the process (creating the snapshot, doing the initial 210 | data move with `dd`, and so on) are unchanged. 211 | 212 | As an aside, if you're trying to move LVs between PVs in the same VG, then 213 | you don't need `lvmsync`, you need `pvmove`. 214 | 215 | 216 | ### Taking a space- and IO-efficient snapshot of an LV 217 | 218 | But wait, there's more! `lvmsync` also has the ability to dump out the 219 | snapshot data to disk, rather than immediately applying it to another block 220 | device. 221 | 222 | To do this, use the `--stdout` option when you're running `lvmsync`, and 223 | instead of writing the changes to another block device, it'll instead dump 224 | the "change stream" to stdout (so redirect somewhere useful). This allows 225 | you to dump the changes to a file, or do some sort of fancy footwork to 226 | transfer the data to another lvmsync process to apply the changes to a block 227 | device. 228 | 229 | For example, if you just wanted to take a copy of the contents of a 230 | snapshot, you could do something like this: 231 | 232 | lvmsync --stdout /dev/somevg/somelv-snapshot >~/somechanges 233 | 234 | At a later date, if you wanted to apply those writes to a block device, 235 | you'd do it like this: 236 | 237 | lvmsync --apply ~/somechanges /dev/somevg/someotherlv 238 | 239 | You can also do things like do an lvmsync *from* the destination -- this is 240 | useful if (for example) you can SSH from the destination to the source 241 | machine, but not the other way around (fkkn firewalls, how do they work?). 242 | You could do this by running something like the following on the destination 243 | machine: 244 | 245 | ssh srcmachine lvmsync --stdout /dev/srcvg/srclv-snap | lvmsync --apply - /dev/destvg/destlv 246 | 247 | 248 | ## Theory of Operation 249 | 250 | This section is for those people who can't sleep well at night without 251 | knowing the magic behind the curtain (and to remind myself occasionally how 252 | this stuff works). It is completely unnecessary to read this section in 253 | order to work lvmsync. 254 | 255 | First, a little bit of background about how snapshot LVs work, before I 256 | describe how lvmsync makes use of them. 257 | 258 | An LVM snapshot "device" is actually not a block device in the usual sense. 259 | It isn't just a big area of disk space where you write things. Instead, it 260 | is a "meta" device, which points to both an "origin" LV, which is the LV 261 | from which the snapshot was made, and a "metadata" LV, which is where the 262 | magic happens. 263 | 264 | The "metadata" LV is a list of "chunks" of the origin LV which have been 265 | modified, along with the original contents of those chunks. In a way, you 266 | can think of it as a sort of "binary diff", which says "these are the ways 267 | in which this snapshot LV differs from the origin LV". When a write happens 268 | to the origin LV, this "diff" is potentially modified to maintain the 269 | original "view" from the time the snapshot was taken. 270 | 271 | (Sidenote: this is why you can write to snapshots -- if you write to a 272 | snapshot, the "diff" is written to some more, to say "here are some more 273 | differences between the origin and the snapshot"). 274 | 275 | From here, it shouldn't be hard to work out how LVM uses the combination of 276 | the origin and metadata LVs to give you a consistent snapshot view -- when 277 | you ask to read a chunk, LVM looks in the metadata LV to see if it has the 278 | chunk in there, and if not it can be sure that the chunk hasn't changed, so 279 | it just reads it from the origin LV. Miiiiighty clever! 280 | 281 | In lvmsync, we only make use of a tiny fraction of the data stored in the 282 | metadata LV for the snapshot. We don't care what the original contents were 283 | (they're what we're trying to get *away* from). What we want is the list of 284 | which chunks have been modified, because that's what we use to work out 285 | which blocks on the original LV we need to copy across. lvmsync never 286 | *actually* reads any disk data from the snapshot block device itself -- all 287 | it reads is the list of changed blocks, then it reads the changed data from 288 | the original LV (which is where the modified blocks are stored). 289 | 290 | By specifying a snapshot to lvmsync, you're telling it "this is the list of 291 | changes I want you to copy" -- it already knows which original LV it needs 292 | to copy from (the snapshot metadata has that info available). 293 | 294 | 295 | ## See Also 296 | 297 | Whilst I think `lvmsync` is awesome (and I hope you will too), here are some 298 | other tools that might be of use to you if `lvmsync` doesn't float your 299 | mustard: 300 | 301 | * [`blocksync.py`](http://www.bouncybouncy.net/programs/blocksync.py) -- 302 | Implements the "hash the chunks and send the ones that don't match" 303 | strategy of block device syncing. It needs to read the entire block 304 | device at each end to work out what to send, so it's not as efficient, 305 | but on the other hand it doesn't require LVM. 306 | 307 | * [`bdsync`](http://bdsync.rolf-fokkens.nl/) -- Another "hash the chunks" 308 | implementation, with the same limitations and advantages as 309 | `blocksync.py`. 310 | 311 | * [`ddsnap`](http://zumastor.org/man/ddsnap.8.html) -- Part of the 312 | "Zumastor" project, appears to provide some sort of network-aware block 313 | device snapshotting (I'm not sure, the Zumastor homepage includes the word 314 | "Enterprise", so I fell asleep before finishing reading). Seems to 315 | require kernel patches, so there's a non-trivial barrier to entry, but 316 | probably not such a big deal if you're after network-aware snapshots as 317 | part of your core infrastructure. 318 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | --------------------------------------------------------------------------------