├── README.md └── zincrsend /README.md: -------------------------------------------------------------------------------- 1 | zincrsend - ZFS Incremental Send 2 | ================================ 3 | 4 | Incremental ZFS send/recv backup script 5 | 6 | Configuration 7 | ------------- 8 | 9 | As of right now this script does not take any arguments or environmental variables, 10 | all configuration must be done in the script itself. 11 | 12 | Note: this script *should* take arguments or a config file - I was just lazy 13 | 14 | At the top of the script you'll see a `config` section... modify it 15 | to fit your environment 16 | 17 | ``` bash 18 | ######### 19 | # config 20 | ######### 21 | 22 | # datasets to send 23 | datasets=( 24 | ) 25 | # datasets to send recursively 26 | # even if a dataset does not have any descendant datasets, using a 27 | # recursive zfs send (-R) is preferable because it will result 28 | # in locally removed snapshots being removed on the remote end as well 29 | datasets_recursive=( 30 | goliath/public 31 | goliath/minecraft 32 | goliath/backups 33 | ) 34 | 35 | # information about the server on the receiving end 36 | remote_server='my-backup-server.example.com' 37 | remote_user='dave' 38 | remote_port='22' 39 | remote_dataset='paper' # zpool name most likely 40 | remote_command_prefix='sudo' # leave blank for nothing 41 | remote_ssh_opts=(-i /root/backup/backup.key) # additional opts to give to ssh 42 | 43 | # prefix to use for snapshots created by this script 44 | snapshot_prefix='zincrsend_' 45 | 46 | # higher = more output 47 | verbosity_level=0 48 | 49 | # function to execute at the end - can be anything 50 | # $1 - exit code - the code that will be used when this script exits 51 | # returns - nothing 52 | end() { 53 | local exitcode=$1 54 | local msg= 55 | case "$exitcode" in 56 | 0) msg='ok';; 57 | *) msg='failed';; 58 | esac 59 | /opt/custom/bin/pushover zincrsend "$msg - took $((SECONDS / 60)) minutes" 60 | } 61 | 62 | ############# 63 | # end config 64 | ############# 65 | ``` 66 | 67 | Example 68 | ------- 69 | 70 | ``` 71 | # ./zincrsend 72 | starting on Fri Dec 4 11:16:00 UTC 2015 73 | 74 | processing dataset: goliath/public 75 | 76 | creating snapshot locally: goliath/public@zincrsend_1449227760 77 | latest remote snapshot: paper/public@zincrsend_1449173284 78 | zfs sending (incremental) @zincrsend_1449173284 -> goliath/public@zincrsend_1449227760 to paper/public 79 | receiving incremental stream of goliath/public@zincrsend_1449227760 into paper/public@zincrsend_1449227760 80 | received 312B stream in 1 seconds (312B/sec) 81 | 82 | processing dataset: goliath/minecraft 83 | 84 | creating snapshot locally: goliath/minecraft@zincrsend_1449227763 85 | latest remote snapshot: paper/minecraft@zincrsend_1449173286 86 | zfs sending (incremental) @zincrsend_1449173286 -> goliath/minecraft@zincrsend_1449227763 to paper/minecraft 87 | receiving incremental stream of goliath/minecraft@zincrsend_1449227763 into paper/minecraft@zincrsend_1449227763 88 | received 312B stream in 1 seconds (312B/sec) 89 | 90 | processing dataset: goliath/backups 91 | 92 | creating snapshot locally: goliath/backups@zincrsend_1449227766 93 | latest remote snapshot: paper/backups@zincrsend_1449173288 94 | zfs sending (incremental) @zincrsend_1449173288 -> goliath/backups@zincrsend_1449227766 to paper/backups 95 | receiving incremental stream of goliath/backups@zincrsend_1449227766 into paper/backups@zincrsend_1449227766 96 | received 312B stream in 1 seconds (312B/sec) 97 | receiving incremental stream of goliath/backups/dave@zincrsend_1449227766 into paper/backups/dave@zincrsend_1449227766 98 | received 217MB stream in 367 seconds (607KB/sec) 99 | receiving incremental stream of goliath/backups/skye@zincrsend_1449227766 into paper/backups/skye@zincrsend_1449227766 100 | received 312B stream in 1 seconds (312B/sec) 101 | receiving incremental stream of goliath/backups/dad@zincrsend_1449227766 into paper/backups/dad@zincrsend_1449227766 102 | received 312B stream in 1 seconds (312B/sec) 103 | receiving incremental stream of goliath/backups/web@zincrsend_1449227766 into paper/backups/web@zincrsend_1449227766 104 | received 3.16MB stream in 5 seconds (647KB/sec) 105 | 106 | script ran for ~6 minutes (384 seconds) 107 | 108 | pushover sent! 109 | 110 | --------------------------------- 111 | ``` 112 | 113 | Notes 114 | ----- 115 | 116 | Consider using [ZFS Prune Snapshots](https://github.com/bahamas10/zfs-prune-snapshots) to remove 117 | old `zincrsend` snapshots 118 | 119 | License 120 | ------- 121 | 122 | MIT License 123 | -------------------------------------------------------------------------------- /zincrsend: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Incremental ZFS send/recv backup script 4 | # 5 | # Author: Dave Eddy 6 | # Date: November 20, 2015 7 | # License: MIT 8 | 9 | ######### 10 | # config 11 | ######### 12 | 13 | # datasets to send 14 | datasets=( 15 | ) 16 | # datasets to send recursively 17 | # even if a dataset does not have any descendant datasets, using a 18 | # recursive zfs send (-R) is preferable because it will result 19 | # in locally removed snapshots being removed on the remote end as well 20 | datasets_recursive=( 21 | goliath/public 22 | goliath/minecraft 23 | goliath/backups 24 | ) 25 | 26 | # information about the server on the receiving end 27 | remote_server='my-backup-server.example.com' 28 | remote_user='dave' 29 | remote_port='22' 30 | remote_dataset='paper' # zpool name most likely 31 | remote_command_prefix='sudo' # leave blank for nothing 32 | remote_ssh_opts=(-i /root/backup/backup.key) # additional opts to give to ssh 33 | 34 | # prefix to use for snapshots created by this script 35 | snapshot_prefix='zincrsend_' 36 | 37 | # higher = more output 38 | verbosity_level=0 39 | 40 | # function to execute at the end - can be anything 41 | # $1 - exit code - the code that will be used when this script exits 42 | # returns - nothing 43 | end() { 44 | local exitcode=$1 45 | local msg= 46 | case "$exitcode" in 47 | 0) msg='ok';; 48 | *) msg='failed';; 49 | esac 50 | /opt/custom/bin/pushover zincrsend "$msg - took $((SECONDS / 60)) minutes" 51 | } 52 | 53 | ############# 54 | # end config 55 | ############# 56 | 57 | debug() { 58 | ((verbosity_level >= 1)) && echo "[DEBUG] $*" >&2 59 | return 0 60 | } 61 | trace() { 62 | ((verbosity_level >= 2)) && echo "[TRACE] $*" >&2 63 | return 0 64 | } 65 | 66 | SSH() { 67 | trace "ssh ${remote_ssh_opts[*]} $remote_server $remote_command_prefix $*" 68 | ssh \ 69 | "${remote_ssh_opts[@]}" \ 70 | -l "$remote_user" \ 71 | -p "$remote_port" \ 72 | "$remote_server" \ 73 | "$remote_command_prefix" \ 74 | "$@" 75 | } 76 | 77 | process() { 78 | local ds=$1 79 | 80 | local snapshot_opts=() 81 | local send_opts=() 82 | if [[ -n $2 ]]; then 83 | # recursive 84 | snapshot_opts+=(-r) 85 | send_opts+=(-R) 86 | fi 87 | 88 | echo '' 89 | echo "processing dataset: $ds" 90 | echo '' 91 | 92 | # Step 1 - snapshot locally 93 | local now=$(date +%s) 94 | local snap=$ds@${snapshot_prefix}${now} 95 | echo "creating snapshot locally: $snap" 96 | if ! zfs snapshot "${snapshot_opts[@]}" "$snap"; then 97 | echo "[ERROR] failed to snapshot $ds" >&2 98 | return 1 99 | fi 100 | 101 | # Step 2 - find the latest remote snapshot 102 | local rds=$remote_dataset/${ds#*/} 103 | local inc_snap= 104 | local inc_opts=() 105 | debug "fetching latest remote snapshot for dataset: $rds" 106 | local rsnap=$(SSH zfs list -H -o name,creation -p -t snapshot -r "$rds" | \ 107 | grep "^$rds@" | \ 108 | sort -n -k 2 | \ 109 | tail -1 | \ 110 | awk '{ print $1 }') 111 | 112 | if [[ -n $rsnap ]]; then 113 | echo "latest remote snapshot: $rsnap" 114 | inc_snap=${rsnap#*@} 115 | # assert that $inc_snap exists locally 116 | if ! zfs list -t snapshot "$ds@$inc_snap" &>/dev/null; then 117 | echo "[ERROR] could not find $rsnap locally ($ds@$inc_snap not found)" >&2 118 | return 1 119 | fi 120 | inc_opts+=(-i "@$inc_snap") 121 | else 122 | echo "no snapshot found for $ds - doing full send/recv" 123 | fi 124 | 125 | # Step 3: send from latest remote to newly created 126 | # or do a full send 127 | if [[ -n $inc_snap ]]; then 128 | echo "zfs sending (incremental) @$inc_snap -> $snap to $rds" 129 | else 130 | echo "zfs sending $snap to $rds" 131 | fi 132 | if ! zfs send "${send_opts[@]}" "${inc_opts[@]}" "$snap" | SSH zfs recv -Fuv "$rds"; then 133 | echo "[ERROR] failed to send $snap to $remote_server $rds" >&2 134 | return 1 135 | fi 136 | 137 | return 0 138 | } 139 | 140 | echo "starting on $(date)" 141 | 142 | code=0 143 | for ds in "${datasets[@]}"; do 144 | process "$ds" || code=1 145 | done 146 | for ds in "${datasets_recursive[@]}"; do 147 | process "$ds" recursive || code=1 148 | done 149 | echo 150 | echo "script ran for ~$((SECONDS / 60)) minutes ($SECONDS seconds)" 151 | echo 152 | end "$code" 153 | echo 154 | echo '---------------------------------' 155 | 156 | exit "$code" 157 | --------------------------------------------------------------------------------