├── .gitignore ├── README.md ├── _testdata ├── bar └── foo ├── build.sh ├── datasets.go ├── doc └── zsnapper.rst ├── etc ├── site-zsnapper.xml └── zsnapper.yml.sample ├── expand.go ├── expand_test.go ├── go.mod ├── go.sum ├── job.go ├── job_test.go ├── main.go ├── man └── man1 │ └── zsnapper.1 └── zfs ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── Vagrantfile ├── error.go ├── error_test.go ├── utils.go ├── zfs.go ├── zfs_test.go └── zpool.go /.gitignore: -------------------------------------------------------------------------------- 1 | bin 2 | pkg 3 | *.tar.gz 4 | build 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | zsnapper 2 | ======== 3 | 4 | Zsnapper automatically creates ZFS snapshots on a specified schedule, while 5 | also removing old snapshots as required. 6 | 7 | Builds 8 | ------ 9 | 10 | https://github.com/calmh/zsnapper/releases 11 | 12 | Installation 13 | ------------ 14 | 15 | - Untar the distribution. 16 | - Copy and modify `etc/zsnapper.yml.sample`. 17 | - Run `zsnapper -c path-to-config-file -v` and observe output. 18 | 19 | For Solaris installations, a sample SMF manifest is in etc/site-zsnapper.xml. It 20 | assumes an installation under /opt/local/zsnapper but is easily modified for 21 | other locations. 22 | 23 | Documentation 24 | ------------- 25 | 26 | https://github.com/calmh/zsnapper/blob/master/doc/zsnapper.rst 27 | 28 | Building 29 | -------- 30 | 31 | - Install Go 32 | - `go get github.com/constabulary/gb/...` 33 | - `gb build` 34 | 35 | The bundled script `build.sh` creates distribution packages, assuming the above 36 | requirements are met. 37 | 38 | License 39 | ------- 40 | 41 | MIT 42 | -------------------------------------------------------------------------------- /_testdata/bar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/calmh/zsnapper/b7f23a5c7b750bffac714b1dab7d136dbdde2509/_testdata/bar -------------------------------------------------------------------------------- /_testdata/foo: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/calmh/zsnapper/b7f23a5c7b750bffac714b1dab7d136dbdde2509/_testdata/foo -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | hash pandoc 2>/dev/null && pandoc -s -f rst -t man < doc/zsnapper.rst > man/man1/zsnapper.1 5 | 6 | rm -rf build bin 7 | mkdir -p build/zsnapper 8 | 9 | for os in linux freebsd solaris ; do 10 | export GOARCH=amd64 11 | export GOOS="$os" 12 | go build -ldflags -w 13 | 14 | rm -rf build/zsnapper/bin 15 | mkdir build/zsnapper/bin 16 | mv zsnapper build/zsnapper/bin 17 | cp -r etc man build/zsnapper 18 | cp README.md build/zsnapper/README.txt 19 | tar -C build -zcf "zsnapper-$os-amd64.tar.gz" zsnapper 20 | done 21 | -------------------------------------------------------------------------------- /datasets.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/calmh/zsnapper/zfs" 9 | ) 10 | 11 | var ( 12 | datasetsMutex sync.Mutex 13 | datasetsLoaded time.Time 14 | datasetsCache []string 15 | ) 16 | 17 | func datasets() []string { 18 | datasetsMutex.Lock() 19 | defer datasetsMutex.Unlock() 20 | 21 | if time.Since(datasetsLoaded) > time.Minute { 22 | datasetsCache = make([]string, 0, len(datasetsCache)) 23 | 24 | dss, err := zfs.Filesystems("", 0) 25 | if err == nil { 26 | for _, ds := range dss { 27 | datasetsCache = append(datasetsCache, ds.Name) 28 | } 29 | } else if verbose { 30 | fmt.Println("Filesystems:", err) 31 | } 32 | 33 | dss, err = zfs.Volumes("", 0) 34 | if err == nil { 35 | for _, ds := range dss { 36 | datasetsCache = append(datasetsCache, ds.Name) 37 | } 38 | } else if verbose { 39 | fmt.Println("Volumes:", err) 40 | } 41 | } 42 | 43 | return datasetsCache 44 | } 45 | 46 | var ( 47 | datasetMutexes = make(map[string]*sync.Mutex) 48 | datasetMutexesMutex sync.Mutex 49 | ) 50 | 51 | func mutexFor(dataset string) *sync.Mutex { 52 | datasetMutexesMutex.Lock() 53 | defer datasetMutexesMutex.Unlock() 54 | if m, ok := datasetMutexes[dataset]; ok { 55 | return m 56 | } 57 | m := new(sync.Mutex) 58 | datasetMutexes[dataset] = m 59 | return m 60 | } 61 | -------------------------------------------------------------------------------- /doc/zsnapper.rst: -------------------------------------------------------------------------------- 1 | :Title: ZSNAPPER(1) 2 | :Author: Jakob Borg 3 | :Date: September 2015 4 | 5 | NAME 6 | ==== 7 | 8 | zsnapper - automatically manage ZFS snapshots 9 | 10 | SYNOSIS 11 | ======= 12 | 13 | | zsnapper [OPTIONS] 14 | 15 | OPTIONS 16 | ======= 17 | 18 | -c 19 | Path to configuration file (default **/opt/local/zsnapper/etc/zsnapper.yml**) 20 | 21 | -v 22 | Enable verbose output 23 | 24 | DESCRIPTION 25 | =========== 26 | 27 | Zsnapper automatically creates ZFS snapshots on a specified schedule, while 28 | also removing old snapshots as required. The behavior is specified by the 29 | **CONFIGURATION**. Snapshots are grouped into *families*, each having a name, 30 | containing a set of *datasets*, a *schedule*, a number of snapshots to *keep* 31 | and an option for *recursiveness*. 32 | 33 | Snapshots are named according to a "family-timestamp" convention. For example, 34 | a snapshot belonging to the "test" family may be called **test- 35 | 20150928T112300Z**. Time stamps are always in the UTC timezone and formatted 36 | in the ISO 8601 basic format. 37 | 38 | CONFIGURATION 39 | ============= 40 | 41 | The following is an example of a valid snapper configuration file:: 42 | 43 | - family: quick 44 | datasets: 45 | - zones/var 46 | - data/delegated/* 47 | schedule: "0 */5 * * * *" 48 | keep: 12 49 | recursive: false 50 | 51 | - family: hourly 52 | datasets: 53 | - zones/$(vmadm list -Ho uuid)* 54 | schedule: "@hourly" 55 | keep: 12 56 | recursive: true 57 | 58 | Two families are declared, **quick** and **hourly**. Each has a list of 59 | datasets, containing wildcards and shell expansions - see the **Dataset 60 | Expansions** section for the meaning and syntax of these. The **schedule** is 61 | a cron format string with a seconds field - see the **Schedule Format** 62 | section for the meaning of the individual fields. 63 | 64 | Dataset Expansions 65 | ------------------ 66 | 67 | The list of datasets configured per family is interpreted as a *filter* on the 68 | list of actually existing datasets at the time of job execution. This means 69 | that it's valid and not an error to mention a dataset that does not exist. Two 70 | forms of expansions are interpreted: wildcards, and shell invocations. 71 | Wildcards follow the following syntax, which is essentially standard shell 72 | glob patterns:: 73 | 74 | pattern: 75 | { term } 76 | 77 | term: 78 | '*' matches any sequence of non-/ characters 79 | '?' matches any single non-/ character 80 | '[' [ '^' ] { character-range } ']' 81 | character class (must be non-empty) 82 | c matches character c (c != '*', '?', '\\', '[') 83 | '\\' c matches character c 84 | 85 | character-range: 86 | c matches character c (c != '\\', '-', ']') 87 | '\\' c matches character c 88 | lo '-' hi matches character c for lo <= c <= hi 89 | 90 | Shell invocations take the form of ``$(command)``, where the command is 91 | executed using ``/bin/sh -c``. Each line of output from the command results in 92 | one pattern being added to the dataset list. If multiple shell invocations are 93 | present in the same pattern, they are expanded and all combinations are added. 94 | As an example, consider the pattern:: 95 | 96 | a-$(echo f1; echo f2)-b-$(echo f3; echo f4;)-* 97 | 98 | The shell invocations are expanded, resulting the following list of patterns:: 99 | 100 | a-f1-b-f3-* 101 | a-f1-b-f4-* 102 | a-f2-b-f3-* 103 | a-f2-b-f4-* 104 | 105 | This list of patterns is then compared against the list of existing datasets 106 | to form the list of datasets to snapshot. For a concrete use case, consider 107 | the pattern: 108 | 109 | zones/$(vmadm list -Ho uuid)* 110 | 111 | On SmartOS, the shell invocation expands into a list of virtual machine IDs. 112 | The result is a list of dataset patterns that match virtual machine zones, and 113 | virtual machine disk volumes, but not template images as these are not 114 | returned by the ``vmadm list`` command. 115 | 116 | Schedule Format 117 | --------------- 118 | 119 | > This section is copied from https://github.com/robfig/cron/blob/master/doc.go, formatted for this man page. 120 | 121 | A cron expression represents a set of times, using 6 space-separated fields:: 122 | 123 | Field name | Allowed values | Allowed special characters 124 | ---------- | -------------- | -------------------------- 125 | Seconds | 0-59 | * / , - 126 | Minutes | 0-59 | * / , - 127 | Hours | 0-23 | * / , - 128 | Day of month | 1-31 | * / , - ? 129 | Month | 1-12 or JAN-DEC | * / , - 130 | Day of week | 0-6 or SUN-SAT | * / , - ? 131 | 132 | Note: Month and Day-of-week field values are case insensitive. "SUN", "Sun", 133 | and "sun" are equally accepted. 134 | 135 | Special Characters 136 | ~~~~~~~~~~~~~~~~~~ 137 | 138 | Asterisk (``*``) 139 | The asterisk indicates that the cron expression will match for all values of the 140 | field; e.g., using an asterisk in the 5th field (month) would indicate every 141 | month. 142 | 143 | Slash (``/``) 144 | Slashes are used to describe increments of ranges. For example 3-59/15 in the 145 | 1st field (minutes) would indicate the 3rd minute of the hour and every 15 146 | minutes thereafter. The form "*\/..." is equivalent to the form "first-last/...", 147 | that is, an increment over the largest possible range of the field. The form 148 | "N/..." is accepted as meaning "N-MAX/...", that is, starting at N, use the 149 | increment until the end of that specific range. It does not wrap around. 150 | 151 | Comma (``,``) 152 | Commas are used to separate items of a list. For example, using "MON,WED,FRI" in 153 | the 5th field (day of week) would mean Mondays, Wednesdays and Fridays. 154 | 155 | Hyphen (``-``) 156 | Hyphens are used to define ranges. For example, 9-17 would indicate every 157 | hour between 9am and 5pm inclusive. 158 | 159 | Question mark (``?``) 160 | Question mark may be used instead of '*' for leaving either day-of-month or 161 | day-of-week blank. 162 | 163 | Predefined schedules 164 | ~~~~~~~~~~~~~~~~~~~~ 165 | 166 | You may use one of several pre-defined schedules in place of a cron expression: 167 | 168 | @yearly (or @annually) 169 | Run once a year, midnight, Jan. 1st (``0 0 0 1 1 *``) 170 | @monthly 171 | Run once a month, midnight, first of month (``0 0 0 1 * *``) 172 | @weekly 173 | Run once a week, midnight on Sunday (``0 0 0 * * 0``) 174 | @daily (or @midnight) 175 | Run once a day, midnight (``0 0 0 * * *``) 176 | @hourly 177 | Run once an hour, beginning of hour (``0 0 * * * *``) 178 | 179 | Intervals 180 | ~~~~~~~~~ 181 | 182 | You may also schedule a job to execute at fixed intervals. This is supported by 183 | formatting the cron spec like this:: 184 | 185 | @every 186 | 187 | where "duration" is a string accepted by time.ParseDuration 188 | (http://golang.org/pkg/time/#ParseDuration). 189 | 190 | For example, ``"@every 1h30m10s"`` would indicate a schedule that activates every 191 | 1 hour, 30 minutes, 10 seconds. 192 | 193 | Note: The interval does not take the job runtime into account. For example, 194 | if a job takes 3 minutes to run, and it is scheduled to run every 5 minutes, 195 | it will have only 2 minutes of idle time between each run. 196 | 197 | Time zones 198 | ~~~~~~~~~~ 199 | 200 | All interpretation and scheduling is done in the machine's local time zone (as 201 | provided by the Go time package (http://www.golang.org/pkg/time). 202 | 203 | Be aware that jobs scheduled during daylight-savings leap-ahead transitions will 204 | not be run! 205 | -------------------------------------------------------------------------------- /etc/site-zsnapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /etc/zsnapper.yml.sample: -------------------------------------------------------------------------------- 1 | - family: quick 2 | datasets: 3 | - zones/var 4 | - data/delegated/* 5 | schedule: "0 */5 * * * *" 6 | keep: 12 7 | recursive: false 8 | 9 | - family: hourly 10 | datasets: 11 | - zones/$(vmadm list -Ho uuid)* 12 | schedule: "@hourly" 13 | keep: 12 14 | recursive: true 15 | -------------------------------------------------------------------------------- /expand.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os/exec" 7 | "path" 8 | "regexp" 9 | "strings" 10 | ) 11 | 12 | var parensExp = regexp.MustCompile(`\$\(([^)]+)\)`) 13 | 14 | func expandLines(lines []string) []string { 15 | var res []string 16 | for _, line := range lines { 17 | res = append(res, expand(line)...) 18 | } 19 | return res 20 | } 21 | 22 | func expand(line string) []string { 23 | m := parensExp.FindStringSubmatch(line) 24 | if len(m) < 2 { 25 | return []string{line} 26 | } 27 | 28 | bs, err := exec.Command("/bin/sh", "-c", m[1]).Output() 29 | if err != nil { 30 | if verbose { 31 | fmt.Printf("Expanding %q: %v\n", m[1], err) 32 | } 33 | return []string{line} 34 | } 35 | 36 | var res []string 37 | repl := "$(" + m[1] + ")" 38 | for _, s := range bytes.Split(bs, []byte("\n")) { 39 | if len(s) == 0 { 40 | continue 41 | } 42 | res = append(res, expand(strings.Replace(line, repl, string(s), 1))...) 43 | } 44 | 45 | return res 46 | } 47 | 48 | func selectLines(pats, lines []string) []string { 49 | var res []string 50 | for _, line := range lines { 51 | if lineMatches(pats, line) { 52 | res = append(res, line) 53 | } 54 | } 55 | return res 56 | } 57 | 58 | func lineMatches(pats []string, line string) bool { 59 | for _, pat := range pats { 60 | if matched, err := path.Match(pat, line); err == nil && matched { 61 | return true 62 | } 63 | } 64 | return false 65 | } 66 | -------------------------------------------------------------------------------- /expand_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestExpandLines(t *testing.T) { 9 | in := []string{ 10 | "a-$(echo t1; echo t2)-b", 11 | "a-$(echo t1; echo t2)-b-$(ls _testdata)-c", 12 | } 13 | 14 | expected := []string{ 15 | "a-t1-b", 16 | "a-t2-b", 17 | "a-t1-b-bar-c", 18 | "a-t1-b-foo-c", 19 | "a-t2-b-bar-c", 20 | "a-t2-b-foo-c", 21 | } 22 | 23 | out := expandLines(in) 24 | if !reflect.DeepEqual(out, expected) { 25 | t.Errorf("Lines differ:\n%#v !=\n%#v", out, expected) 26 | } 27 | } 28 | 29 | func TestSelectLines(t *testing.T) { 30 | in := []string{ 31 | "data/test", 32 | "data/test/child", 33 | "data/foo", 34 | "data/foo1", 35 | "data/foo2", 36 | "data/foo2/child", 37 | "data/test21/foo", 38 | "data/test21/bar", 39 | "data/test31/foo", 40 | "data/test31/bar", 41 | } 42 | 43 | pats := []string{ 44 | "data/test", 45 | "data/foo*", 46 | "data/test2?/*", 47 | } 48 | 49 | expected := []string{ 50 | "data/test", 51 | "data/foo", 52 | "data/foo1", 53 | "data/foo2", 54 | "data/test21/foo", 55 | "data/test21/bar", 56 | } 57 | 58 | out := selectLines(pats, in) 59 | if !reflect.DeepEqual(out, expected) { 60 | t.Errorf("Selections differ:\n%#v !=\n%#v", out, expected) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/calmh/zsnapper 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/google/uuid v1.1.1 7 | github.com/kr/text v0.2.0 // indirect 8 | github.com/mistifyio/go-zfs v2.1.1+incompatible 9 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect 10 | github.com/robfig/cron v1.2.0 11 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect 12 | gopkg.in/yaml.v2 v2.2.8 13 | ) 14 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/calmh/zfs v0.0.0-20130813113238-80beb0b48dc1 h1:xpqRN8dgkJqEmDU9OVXWPnW61MCQyiMoeJuEwDb7k1A= 2 | github.com/calmh/zfs v0.0.0-20130813113238-80beb0b48dc1/go.mod h1:sdVO3aXg2EV06qszXHYLF8N8G8rmMzJDisTHfie8dp0= 3 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 4 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 5 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 6 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 7 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 8 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 9 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 10 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 11 | github.com/mistifyio/go-zfs v1.0.0 h1:F7aMMlUou4K65tJvBjoTNkcL2RdCScCrkuSjMO2Do4U= 12 | github.com/mistifyio/go-zfs v2.1.1+incompatible h1:gAMO1HM9xBRONLHHYnu5iFsOJUiJdNZo6oqSENd4eW8= 13 | github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= 14 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= 15 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= 16 | github.com/robfig/cron v1.0.1-0.20141203011048-67823cd24dec h1:mhyY8LLpu91YkLbPg2Zu8BnvmnP48A/Q27GlUw8+Jyc= 17 | github.com/robfig/cron v1.0.1-0.20141203011048-67823cd24dec/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= 18 | github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= 19 | github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= 20 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 21 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= 22 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 23 | gopkg.in/yaml.v2 v2.0.0-20150924142314-53feefa2559f h1:sOheF02XWNGQor9t3gRZtN/HlgP6sv3NozSahoTEmiM= 24 | gopkg.in/yaml.v2 v2.0.0-20150924142314-53feefa2559f/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 25 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 26 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 27 | -------------------------------------------------------------------------------- /job.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "sort" 8 | "strings" 9 | 10 | "github.com/calmh/zsnapper/zfs" 11 | "gopkg.in/yaml.v2" 12 | ) 13 | 14 | type Job struct { 15 | Family string // "test" 16 | Datasets []string // "data/foto" 17 | Schedule string // "0 */5 * * * *" 18 | Keep int // 12 19 | Recursive bool 20 | } 21 | 22 | func LoadJobs(r io.Reader) ([]Job, error) { 23 | bs, err := ioutil.ReadAll(r) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | var configs []Job 29 | if err := yaml.Unmarshal(bs, &configs); err != nil { 30 | return nil, err 31 | } 32 | 33 | return configs, nil 34 | } 35 | 36 | func (j Job) Run() { 37 | name := j.Family + "-" + timestamp() 38 | 39 | expandedSets := expandLines(j.Datasets) 40 | all := datasets() 41 | filteredDatasets := selectLines(expandedSets, all) 42 | 43 | for _, dataset := range filteredDatasets { 44 | if err := j.snapshot(dataset, name); err != nil { 45 | fmt.Println("Create snapshot:", err) 46 | } 47 | if err := j.clean(dataset); err != nil { 48 | fmt.Println("Clean snapshots:", err) 49 | } 50 | } 51 | } 52 | 53 | func (j Job) String() string { 54 | return fmt.Sprintf(`%v@%s, at "%s" (keep %d, recursive %v)`, j.Datasets, j.Family, j.Schedule, j.Keep, j.Recursive) 55 | } 56 | 57 | func (j Job) snapshot(dataset, name string) error { 58 | mutexFor(dataset).Lock() 59 | defer mutexFor(dataset).Unlock() 60 | 61 | ds, err := zfs.GetDataset(dataset) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | ss, err := ds.Snapshot(name, j.Recursive) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | if verbose { 72 | fmt.Println("Created", ss.Name) 73 | } 74 | 75 | return nil 76 | } 77 | 78 | func (j Job) clean(dataset string) error { 79 | mutexFor(dataset).Lock() 80 | defer mutexFor(dataset).Unlock() 81 | 82 | ds, err := zfs.GetDataset(dataset) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | snaps, err := ds.Snapshots() 88 | if err != nil { 89 | return err 90 | } 91 | 92 | prefix := dataset + "@" + j.Family + "-" 93 | var matching []*zfs.Dataset 94 | for _, snap := range snaps { 95 | if strings.HasPrefix(snap.Name, prefix) { 96 | matching = append(matching, snap) 97 | } 98 | } 99 | 100 | sort.Sort(datasetList(matching)) 101 | 102 | if len(matching) <= j.Keep { 103 | return nil 104 | } 105 | 106 | var flags zfs.DestroyFlag 107 | if j.Recursive { 108 | flags |= zfs.DestroyRecursive 109 | } 110 | for _, snap := range matching[:len(matching)-j.Keep] { 111 | if err := snap.Destroy(flags); err != nil { 112 | fmt.Printf("Destroy %s: %v\n", snap.Name, err) 113 | } else if verbose { 114 | fmt.Println("Destroyed", snap.Name) 115 | } 116 | } 117 | 118 | return nil 119 | } 120 | -------------------------------------------------------------------------------- /job_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | "testing" 7 | ) 8 | 9 | func TestLoadJobs(t *testing.T) { 10 | yaml := ` 11 | - family: test 12 | # comment 13 | datasets: 14 | - data/test 15 | schedule: "*/20 * * * * *" 16 | keep: 5 17 | recursive: true 18 | # comment 19 | - family: minute 20 | datasets: 21 | - data/test 22 | - data/foo/* 23 | schedule: "0 * * * * *" 24 | keep: 12 25 | recursive: false` 26 | 27 | expected := []Job{ 28 | Job{ 29 | Family: "test", 30 | Datasets: []string{"data/test"}, 31 | Schedule: "*/20 * * * * *", 32 | Keep: 5, 33 | Recursive: true, 34 | }, 35 | Job{ 36 | Family: "minute", 37 | Datasets: []string{"data/test", "data/foo/*"}, 38 | Schedule: "0 * * * * *", 39 | Keep: 12, 40 | Recursive: false, 41 | }, 42 | } 43 | 44 | r := strings.NewReader(yaml) 45 | jobs, err := LoadJobs(r) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | if len(jobs) != len(expected) { 50 | t.Fatalf("Number of jobs %d != %d", len(jobs), len(expected)) 51 | } 52 | for i := range expected { 53 | if !reflect.DeepEqual(jobs[i], expected[i]) { 54 | t.Errorf("%d: %+v != %+v", i, jobs[i], expected[i]) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | "github.com/calmh/zsnapper/zfs" 10 | "github.com/robfig/cron" 11 | ) 12 | 13 | var ( 14 | cfgFile = "/opt/local/zsnapper/etc/zsnapper.yml" 15 | verbose = false 16 | ) 17 | 18 | func main() { 19 | flag.StringVar(&cfgFile, "c", cfgFile, "Path to configuration file") 20 | flag.BoolVar(&verbose, "v", verbose, "Enables verbose output") 21 | flag.Parse() 22 | 23 | fd, err := os.Open(cfgFile) 24 | if err != nil { 25 | fatalln(err) 26 | } 27 | 28 | jobs, err := LoadJobs(fd) 29 | fd.Close() 30 | if err != nil { 31 | fatalln(err) 32 | } 33 | 34 | if len(jobs) == 0 { 35 | fatalln("No jobs to run") 36 | } 37 | 38 | j := cron.New() 39 | for _, job := range jobs { 40 | j.AddJob(job.Schedule, job) 41 | if verbose { 42 | fmt.Println("Added", job) 43 | } 44 | } 45 | j.Start() 46 | 47 | select {} 48 | } 49 | 50 | func timestamp() string { 51 | return time.Now().UTC().Format("20060102T150405Z") 52 | } 53 | 54 | func fatalln(args ...interface{}) { 55 | fmt.Println(args...) 56 | os.Exit(1) 57 | } 58 | 59 | type datasetList []*zfs.Dataset 60 | 61 | func (l datasetList) Len() int { 62 | return len(l) 63 | } 64 | func (l datasetList) Swap(a, b int) { 65 | l[a], l[b] = l[b], l[a] 66 | } 67 | func (l datasetList) Less(a, b int) bool { 68 | return l[a].Name < l[b].Name 69 | } 70 | -------------------------------------------------------------------------------- /man/man1/zsnapper.1: -------------------------------------------------------------------------------- 1 | .\" Automatically generated by Pandoc 2.9.2.1 2 | .\" 3 | .TH "ZSNAPPER" "1" "September 2015" "" "" 4 | .hy 5 | .SH NAME 6 | .PP 7 | zsnapper - automatically manage ZFS snapshots 8 | .SH SYNOSIS 9 | .PP 10 | zsnapper [OPTIONS] 11 | .SH OPTIONS 12 | .TP 13 | -c 14 | Path to configuration file (default 15 | \f[B]/opt/local/zsnapper/etc/zsnapper.yml\f[R]) 16 | .TP 17 | -v 18 | Enable verbose output 19 | .SH DESCRIPTION 20 | .PP 21 | Zsnapper automatically creates ZFS snapshots on a specified schedule, 22 | while also removing old snapshots as required. 23 | The behavior is specified by the \f[B]CONFIGURATION\f[R]. 24 | Snapshots are grouped into \f[I]families\f[R], each having a name, 25 | containing a set of \f[I]datasets\f[R], a \f[I]schedule\f[R], a number 26 | of snapshots to \f[I]keep\f[R] and an option for 27 | \f[I]recursiveness\f[R]. 28 | .PP 29 | Snapshots are named according to a \[dq]family-timestamp\[dq] 30 | convention. 31 | For example, a snapshot belonging to the \[dq]test\[dq] family may be 32 | called \f[B]test-20150928T112300Z\f[R]. 33 | Time stamps are always in the UTC timezone and formatted in the ISO 8601 34 | basic format. 35 | .SH CONFIGURATION 36 | .PP 37 | The following is an example of a valid snapper configuration file: 38 | .IP 39 | .nf 40 | \f[C] 41 | - family: quick 42 | datasets: 43 | - zones/var 44 | - data/delegated/* 45 | schedule: \[dq]0 */5 * * * *\[dq] 46 | keep: 12 47 | recursive: false 48 | 49 | - family: hourly 50 | datasets: 51 | - zones/$(vmadm list -Ho uuid)* 52 | schedule: \[dq]\[at]hourly\[dq] 53 | keep: 12 54 | recursive: true 55 | \f[R] 56 | .fi 57 | .PP 58 | Two families are declared, \f[B]quick\f[R] and \f[B]hourly\f[R]. 59 | Each has a list of datasets, containing wildcards and shell expansions - 60 | see the \f[B]Dataset Expansions\f[R] section for the meaning and syntax 61 | of these. 62 | The \f[B]schedule\f[R] is a cron format string with a seconds field - 63 | see the \f[B]Schedule Format\f[R] section for the meaning of the 64 | individual fields. 65 | .SS Dataset Expansions 66 | .PP 67 | The list of datasets configured per family is interpreted as a 68 | \f[I]filter\f[R] on the list of actually existing datasets at the time 69 | of job execution. 70 | This means that it\[aq]s valid and not an error to mention a dataset 71 | that does not exist. 72 | Two forms of expansions are interpreted: wildcards, and shell 73 | invocations. 74 | Wildcards follow the following syntax, which is essentially standard 75 | shell glob patterns: 76 | .IP 77 | .nf 78 | \f[C] 79 | pattern: 80 | { term } 81 | 82 | term: 83 | \[aq]*\[aq] matches any sequence of non-/ characters 84 | \[aq]?\[aq] matches any single non-/ character 85 | \[aq][\[aq] [ \[aq]\[ha]\[aq] ] { character-range } \[aq]]\[aq] 86 | character class (must be non-empty) 87 | c matches character c (c != \[aq]*\[aq], \[aq]?\[aq], \[aq]\[rs]\[rs]\[aq], \[aq][\[aq]) 88 | \[aq]\[rs]\[rs]\[aq] c matches character c 89 | 90 | character-range: 91 | c matches character c (c != \[aq]\[rs]\[rs]\[aq], \[aq]-\[aq], \[aq]]\[aq]) 92 | \[aq]\[rs]\[rs]\[aq] c matches character c 93 | lo \[aq]-\[aq] hi matches character c for lo <= c <= hi 94 | \f[R] 95 | .fi 96 | .PP 97 | Shell invocations take the form of \f[C]$(command)\f[R], where the 98 | command is executed using \f[C]/bin/sh -c\f[R]. 99 | Each line of output from the command results in one pattern being added 100 | to the dataset list. 101 | If multiple shell invocations are present in the same pattern, they are 102 | expanded and all combinations are added. 103 | As an example, consider the pattern: 104 | .IP 105 | .nf 106 | \f[C] 107 | a-$(echo f1; echo f2)-b-$(echo f3; echo f4;)-* 108 | \f[R] 109 | .fi 110 | .PP 111 | The shell invocations are expanded, resulting the following list of 112 | patterns: 113 | .IP 114 | .nf 115 | \f[C] 116 | a-f1-b-f3-* 117 | a-f1-b-f4-* 118 | a-f2-b-f3-* 119 | a-f2-b-f4-* 120 | \f[R] 121 | .fi 122 | .PP 123 | This list of patterns is then compared against the list of existing 124 | datasets to form the list of datasets to snapshot. 125 | For a concrete use case, consider the pattern: 126 | .RS 127 | .PP 128 | zones/$(vmadm list -Ho uuid)* 129 | .RE 130 | .PP 131 | On SmartOS, the shell invocation expands into a list of virtual machine 132 | IDs. 133 | The result is a list of dataset patterns that match virtual machine 134 | zones, and virtual machine disk volumes, but not template images as 135 | these are not returned by the \f[C]vmadm list\f[R] command. 136 | .SS Schedule Format 137 | .PP 138 | > This section is copied from 139 | , formatted for this 140 | man page. 141 | .PP 142 | A cron expression represents a set of times, using 6 space-separated 143 | fields: 144 | .IP 145 | .nf 146 | \f[C] 147 | Field name | Allowed values | Allowed special characters 148 | ---------- | -------------- | -------------------------- 149 | Seconds | 0-59 | * / , - 150 | Minutes | 0-59 | * / , - 151 | Hours | 0-23 | * / , - 152 | Day of month | 1-31 | * / , - ? 153 | Month | 1-12 or JAN-DEC | * / , - 154 | Day of week | 0-6 or SUN-SAT | * / , - ? 155 | \f[R] 156 | .fi 157 | .PP 158 | Note: Month and Day-of-week field values are case insensitive. 159 | \[dq]SUN\[dq], \[dq]Sun\[dq], and \[dq]sun\[dq] are equally accepted. 160 | .SS Special Characters 161 | .TP 162 | Asterisk (\f[B]\f[CB]*\f[B]\f[R]) 163 | The asterisk indicates that the cron expression will match for all 164 | values of the field; e.g., using an asterisk in the 5th field (month) 165 | would indicate every month. 166 | .TP 167 | Slash (\f[B]\f[CB]/\f[B]\f[R]) 168 | Slashes are used to describe increments of ranges. 169 | For example 3-59/15 in the 1st field (minutes) would indicate the 3rd 170 | minute of the hour and every 15 minutes thereafter. 171 | The form \[dq]*/...\[dq] is equivalent to the form 172 | \[dq]first-last/...\[dq], that is, an increment over the largest 173 | possible range of the field. 174 | The form \[dq]N/...\[dq] is accepted as meaning \[dq]N-MAX/...\[dq], 175 | that is, starting at N, use the increment until the end of that specific 176 | range. 177 | It does not wrap around. 178 | .TP 179 | Comma (\f[B]\f[CB],\f[B]\f[R]) 180 | Commas are used to separate items of a list. 181 | For example, using \[dq]MON,WED,FRI\[dq] in the 5th field (day of week) 182 | would mean Mondays, Wednesdays and Fridays. 183 | .TP 184 | Hyphen (\f[B]\f[CB]-\f[B]\f[R]) 185 | Hyphens are used to define ranges. 186 | For example, 9-17 would indicate every hour between 9am and 5pm 187 | inclusive. 188 | .TP 189 | Question mark (\f[B]\f[CB]?\f[B]\f[R]) 190 | Question mark may be used instead of \[aq]*\[aq] for leaving either 191 | day-of-month or day-of-week blank. 192 | .SS Predefined schedules 193 | .PP 194 | You may use one of several pre-defined schedules in place of a cron 195 | expression: 196 | .TP 197 | \[at]yearly (or \[at]annually) 198 | Run once a year, midnight, Jan. 199 | 1st (\f[C]0 0 0 1 1 *\f[R]) 200 | .TP 201 | \[at]monthly 202 | Run once a month, midnight, first of month (\f[C]0 0 0 1 * *\f[R]) 203 | .TP 204 | \[at]weekly 205 | Run once a week, midnight on Sunday (\f[C]0 0 0 * * 0\f[R]) 206 | .TP 207 | \[at]daily (or \[at]midnight) 208 | Run once a day, midnight (\f[C]0 0 0 * * *\f[R]) 209 | .TP 210 | \[at]hourly 211 | Run once an hour, beginning of hour (\f[C]0 0 * * * *\f[R]) 212 | .SS Intervals 213 | .PP 214 | You may also schedule a job to execute at fixed intervals. 215 | This is supported by formatting the cron spec like this: 216 | .IP 217 | .nf 218 | \f[C] 219 | \[at]every 220 | \f[R] 221 | .fi 222 | .PP 223 | where \[dq]duration\[dq] is a string accepted by time.ParseDuration 224 | (). 225 | .PP 226 | For example, \f[C]\[dq]\[at]every 1h30m10s\[dq]\f[R] would indicate a 227 | schedule that activates every 1 hour, 30 minutes, 10 seconds. 228 | .PP 229 | Note: The interval does not take the job runtime into account. 230 | For example, if a job takes 3 minutes to run, and it is scheduled to run 231 | every 5 minutes, it will have only 2 minutes of idle time between each 232 | run. 233 | .SS Time zones 234 | .PP 235 | All interpretation and scheduling is done in the machine\[aq]s local 236 | time zone (as provided by the Go time package 237 | (). 238 | .PP 239 | Be aware that jobs scheduled during daylight-savings leap-ahead 240 | transitions will not be run! 241 | .SH AUTHORS 242 | Jakob Borg. 243 | -------------------------------------------------------------------------------- /zfs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## How to Contribute ## 2 | 3 | We always welcome contributions to help make `go-zfs` better. Please take a moment to read this document if you would like to contribute. 4 | 5 | ### Reporting issues ### 6 | 7 | We use [Github issues](https://github.com/mistifyio/go-zfs/issues) to track bug reports, feature requests, and submitting pull requests. 8 | 9 | If you find a bug: 10 | 11 | * Use the GitHub issue search to check whether the bug has already been reported. 12 | * If the issue has been fixed, try to reproduce the issue using the latest `master` branch of the repository. 13 | * If the issue still reproduces or has not yet been reported, try to isolate the problem before opening an issue, if possible. Also provide the steps taken to reproduce the bug. 14 | 15 | ### Pull requests ### 16 | 17 | We welcome bug fixes, improvements, and new features. Before embarking on making significant changes, please open an issue and ask first so that you do not risk duplicating efforts or spending time working on something that may be out of scope. For minor items, just open a pull request. 18 | 19 | [Fork the project](https://help.github.com/articles/fork-a-repo), clone your fork, and add the upstream to your remote: 20 | 21 | $ git clone git@github.com:/go-zfs.git 22 | $ cd go-zfs 23 | $ git remote add upstream https://github.com/mistifyio/go-zfs.git 24 | 25 | If you need to pull new changes committed upstream: 26 | 27 | $ git checkout master 28 | $ git fetch upstream 29 | $ git merge upstream/master 30 | 31 | Don' work directly on master as this makes it harder to merge later. Create a feature branch for your fix or new feature: 32 | 33 | $ git checkout -b 34 | 35 | Please try to commit your changes in logical chunks. Ideally, you should include the issue number in the commit message. 36 | 37 | $ git commit -m "Issue # - " 38 | 39 | Push your feature branch to your fork. 40 | 41 | $ git push origin 42 | 43 | [Open a Pull Request](https://help.github.com/articles/using-pull-requests) against the upstream master branch. Please give your pull request a clear title and description and note which issue(s) your pull request fixes. 44 | 45 | * All Go code should be formatted using [gofmt](http://golang.org/cmd/gofmt/). 46 | * Every exported function should have [documentation](http://blog.golang.org/godoc-documenting-go-code) and corresponding [tests](http://golang.org/doc/code.html#Testing). 47 | 48 | **Important:** By submitting a patch, you agree to allow the project owners to license your work under the [Apache 2.0 License](./LICENSE). 49 | 50 | ### Go Tools ### 51 | For consistency and to catch minor issues for all of go code, please run the following: 52 | * goimports 53 | * go vet 54 | * golint 55 | * errcheck 56 | 57 | Many editors can execute the above on save. 58 | 59 | ---- 60 | Guidelines based on http://azkaban.github.io/contributing.html 61 | -------------------------------------------------------------------------------- /zfs/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright (c) 2014, OmniTI Computer Consulting, Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /zfs/README.md: -------------------------------------------------------------------------------- 1 | # Go Wrapper for ZFS # 2 | 3 | Simple wrappers for ZFS command line tools. 4 | 5 | [![GoDoc](https://godoc.org/github.com/mistifyio/go-zfs?status.svg)](https://godoc.org/github.com/mistifyio/go-zfs) 6 | 7 | ## Requirements ## 8 | 9 | You need a working ZFS setup. To use on Ubuntu 14.04, setup ZFS: 10 | 11 | sudo apt-get install python-software-properties 12 | sudo apt-add-repository ppa:zfs-native/stable 13 | sudo apt-get update 14 | sudo apt-get install ubuntu-zfs libzfs-dev 15 | 16 | Developed using Go 1.3, but currently there isn't anything 1.3 specific. Don't use Ubuntu packages for Go, use http://golang.org/doc/install 17 | 18 | Generally you need root privileges to use anything zfs related. 19 | 20 | ## Status ## 21 | 22 | This has been only been tested on Ubuntu 14.04 23 | 24 | In the future, we hope to work directly with libzfs. 25 | 26 | # Hacking # 27 | 28 | The tests have decent examples for most functions. 29 | 30 | ```go 31 | //assuming a zpool named test 32 | //error handling ommitted 33 | 34 | 35 | f, err := zfs.CreateFilesystem("test/snapshot-test", nil) 36 | ok(t, err) 37 | 38 | s, err := f.Snapshot("test", nil) 39 | ok(t, err) 40 | 41 | // snapshot is named "test/snapshot-test@test" 42 | 43 | c, err := s.Clone("test/clone-test", nil) 44 | 45 | err := c.Destroy() 46 | err := s.Destroy() 47 | err := f.Destroy() 48 | 49 | ``` 50 | 51 | # Contributing # 52 | 53 | See the [contributing guidelines](./CONTRIBUTING.md) 54 | 55 | -------------------------------------------------------------------------------- /zfs/Vagrantfile: -------------------------------------------------------------------------------- 1 | 2 | VAGRANTFILE_API_VERSION = "2" 3 | 4 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 5 | config.vm.box = "ubuntu/trusty64" 6 | config.ssh.forward_agent = true 7 | 8 | config.vm.synced_folder ".", "/home/vagrant/go/src/github.com/mistifyio/go-zfs", create: true 9 | 10 | config.vm.provision "shell", inline: < /etc/profile.d/go.sh 12 | export GOPATH=\\$HOME/go 13 | export PATH=\\$GOPATH/bin:/usr/local/go/bin:\\$PATH 14 | END 15 | 16 | chown -R vagrant /home/vagrant/go 17 | 18 | apt-get update 19 | apt-get install -y software-properties-common curl 20 | apt-add-repository --yes ppa:zfs-native/stable 21 | apt-get update 22 | apt-get install -y ubuntu-zfs 23 | 24 | cd /home/vagrant 25 | curl -z go1.3.3.linux-amd64.tar.gz -L -O https://storage.googleapis.com/golang/go1.3.3.linux-amd64.tar.gz 26 | tar -C /usr/local -zxf /home/vagrant/go1.3.3.linux-amd64.tar.gz 27 | 28 | cat << END > /etc/sudoers.d/go 29 | Defaults env_keep += "GOPATH" 30 | END 31 | 32 | EOF 33 | 34 | end 35 | -------------------------------------------------------------------------------- /zfs/error.go: -------------------------------------------------------------------------------- 1 | package zfs 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | // Error is an error which is returned when the `zfs` or `zpool` shell 8 | // commands return with a non-zero exit code. 9 | type Error struct { 10 | Err error 11 | Debug string 12 | Stderr string 13 | } 14 | 15 | // Error returns the string representation of an Error. 16 | func (e Error) Error() string { 17 | return fmt.Sprintf("%s: %q => %s", e.Err, e.Debug, e.Stderr) 18 | } 19 | -------------------------------------------------------------------------------- /zfs/error_test.go: -------------------------------------------------------------------------------- 1 | package zfs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "testing" 7 | ) 8 | 9 | func TestError(t *testing.T) { 10 | var tests = []struct { 11 | err error 12 | debug string 13 | stderr string 14 | }{ 15 | // Empty error 16 | {nil, "", ""}, 17 | // Typical error 18 | {errors.New("exit status foo"), "/sbin/foo bar qux", "command not found"}, 19 | // Quoted error 20 | {errors.New("exit status quoted"), "\"/sbin/foo\" bar qux", "\"some\" 'random' `quotes`"}, 21 | } 22 | 23 | for _, test := range tests { 24 | // Generate error from tests 25 | zErr := Error{ 26 | Err: test.err, 27 | Debug: test.debug, 28 | Stderr: test.stderr, 29 | } 30 | 31 | // Verify output format is consistent, so that any changes to the 32 | // Error method must be reflected by the test 33 | if str := zErr.Error(); str != fmt.Sprintf("%s: %q => %s", test.err, test.debug, test.stderr) { 34 | t.Fatalf("unexpected Error string: %v", str) 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /zfs/utils.go: -------------------------------------------------------------------------------- 1 | package zfs 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os/exec" 8 | "regexp" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/google/uuid" 13 | ) 14 | 15 | type command struct { 16 | Command string 17 | Stdin io.Reader 18 | Stdout io.Writer 19 | } 20 | 21 | func (c *command) Run(arg ...string) ([][]string, error) { 22 | 23 | cmd := exec.Command(c.Command, arg...) 24 | 25 | var stdout, stderr bytes.Buffer 26 | 27 | if c.Stdout == nil { 28 | cmd.Stdout = &stdout 29 | } else { 30 | cmd.Stdout = c.Stdout 31 | } 32 | 33 | if c.Stdin != nil { 34 | cmd.Stdin = c.Stdin 35 | 36 | } 37 | cmd.Stderr = &stderr 38 | 39 | id := uuid.New() 40 | joinedArgs := strings.Join(cmd.Args, " ") 41 | 42 | logger.Log([]string{"ID:" + id.String(), "START", joinedArgs}) 43 | err := cmd.Run() 44 | logger.Log([]string{"ID:" + id.String(), "FINISH"}) 45 | 46 | if err != nil { 47 | return nil, &Error{ 48 | Err: err, 49 | Debug: strings.Join([]string{cmd.Path, joinedArgs}, " "), 50 | Stderr: stderr.String(), 51 | } 52 | } 53 | 54 | // assume if you passed in something for stdout, that you know what to do with it 55 | if c.Stdout != nil { 56 | return nil, nil 57 | } 58 | 59 | lines := strings.Split(stdout.String(), "\n") 60 | 61 | //last line is always blank 62 | lines = lines[0 : len(lines)-1] 63 | output := make([][]string, len(lines)) 64 | 65 | for i, l := range lines { 66 | output[i] = strings.Fields(l) 67 | } 68 | 69 | return output, nil 70 | } 71 | 72 | func setString(field *string, value string) { 73 | v := "" 74 | if value != "-" { 75 | v = value 76 | } 77 | *field = v 78 | } 79 | 80 | func setUint(field *uint64, value string) error { 81 | var v uint64 82 | if value != "-" { 83 | var err error 84 | v, err = strconv.ParseUint(value, 10, 64) 85 | if err != nil { 86 | return err 87 | } 88 | } 89 | *field = v 90 | return nil 91 | } 92 | 93 | func (ds *Dataset) parseLine(line []string) error { 94 | prop := line[1] 95 | val := line[2] 96 | 97 | var err error 98 | 99 | switch prop { 100 | case "available": 101 | err = setUint(&ds.Avail, val) 102 | case "compression": 103 | setString(&ds.Compression, val) 104 | case "mountpoint": 105 | setString(&ds.Mountpoint, val) 106 | case "quota": 107 | err = setUint(&ds.Quota, val) 108 | case "type": 109 | setString(&ds.Type, val) 110 | case "origin": 111 | setString(&ds.Origin, val) 112 | case "used": 113 | err = setUint(&ds.Used, val) 114 | case "volsize": 115 | err = setUint(&ds.Volsize, val) 116 | case "written": 117 | err = setUint(&ds.Written, val) 118 | case "logicalused": 119 | err = setUint(&ds.Logicalused, val) 120 | } 121 | return err 122 | } 123 | 124 | /* 125 | * from zfs diff`s escape function: 126 | * 127 | * Prints a file name out a character at a time. If the character is 128 | * not in the range of what we consider "printable" ASCII, display it 129 | * as an escaped 3-digit octal value. ASCII values less than a space 130 | * are all control characters and we declare the upper end as the 131 | * DELete character. This also is the last 7-bit ASCII character. 132 | * We choose to treat all 8-bit ASCII as not printable for this 133 | * application. 134 | */ 135 | func unescapeFilepath(path string) (string, error) { 136 | buf := make([]byte, 0, len(path)) 137 | llen := len(path) 138 | for i := 0; i < llen; { 139 | if path[i] == '\\' { 140 | if llen < i+4 { 141 | return "", fmt.Errorf("Invalid octal code: too short") 142 | } 143 | octalCode := path[(i + 1):(i + 4)] 144 | val, err := strconv.ParseUint(octalCode, 8, 8) 145 | if err != nil { 146 | return "", fmt.Errorf("Invalid octal code: %v", err) 147 | } 148 | buf = append(buf, byte(val)) 149 | i += 4 150 | } else { 151 | buf = append(buf, path[i]) 152 | i++ 153 | } 154 | } 155 | return string(buf), nil 156 | } 157 | 158 | var changeTypeMap = map[string]ChangeType{ 159 | "-": Removed, 160 | "+": Created, 161 | "M": Modified, 162 | "R": Renamed, 163 | } 164 | var inodeTypeMap = map[string]InodeType{ 165 | "B": BlockDevice, 166 | "C": CharacterDevice, 167 | "/": Directory, 168 | ">": Door, 169 | "|": NamedPipe, 170 | "@": SymbolicLink, 171 | "P": EventPort, 172 | "=": Socket, 173 | "F": File, 174 | } 175 | 176 | // matches (+1) or (-1) 177 | var referenceCountRegex = regexp.MustCompile("\\(([+-]\\d+?)\\)") 178 | 179 | func parseReferenceCount(field string) (int, error) { 180 | matches := referenceCountRegex.FindStringSubmatch(field) 181 | if matches == nil { 182 | return 0, fmt.Errorf("Regexp does not match") 183 | } 184 | return strconv.Atoi(matches[1]) 185 | } 186 | 187 | func parseInodeChange(line []string) (*InodeChange, error) { 188 | llen := len(line) 189 | if llen < 1 { 190 | return nil, fmt.Errorf("Empty line passed") 191 | } 192 | 193 | changeType := changeTypeMap[line[0]] 194 | if changeType == 0 { 195 | return nil, fmt.Errorf("Unknown change type '%s'", line[0]) 196 | } 197 | 198 | switch changeType { 199 | case Renamed: 200 | if llen != 4 { 201 | return nil, fmt.Errorf("Mismatching number of fields: expect 4, got: %d", llen) 202 | } 203 | case Modified: 204 | if llen != 4 && llen != 3 { 205 | return nil, fmt.Errorf("Mismatching number of fields: expect 3..4, got: %d", llen) 206 | } 207 | default: 208 | if llen != 3 { 209 | return nil, fmt.Errorf("Mismatching number of fields: expect 3, got: %d", llen) 210 | } 211 | } 212 | 213 | inodeType := inodeTypeMap[line[1]] 214 | if inodeType == 0 { 215 | return nil, fmt.Errorf("Unknown inode type '%s'", line[1]) 216 | } 217 | 218 | path, err := unescapeFilepath(line[2]) 219 | if err != nil { 220 | return nil, fmt.Errorf("Failed to parse filename: %v", err) 221 | } 222 | 223 | var newPath string 224 | var referenceCount int 225 | switch changeType { 226 | case Renamed: 227 | newPath, err = unescapeFilepath(line[3]) 228 | if err != nil { 229 | return nil, fmt.Errorf("Failed to parse filename: %v", err) 230 | } 231 | case Modified: 232 | if llen == 4 { 233 | referenceCount, err = parseReferenceCount(line[3]) 234 | if err != nil { 235 | return nil, fmt.Errorf("Failed to parse reference count: %v", err) 236 | } 237 | } 238 | default: 239 | newPath = "" 240 | } 241 | 242 | return &InodeChange{ 243 | Change: changeType, 244 | Type: inodeType, 245 | Path: path, 246 | NewPath: newPath, 247 | ReferenceCountChange: referenceCount, 248 | }, nil 249 | } 250 | 251 | // example input 252 | //M / /testpool/bar/ 253 | //+ F /testpool/bar/hello.txt 254 | //M / /testpool/bar/hello.txt (+1) 255 | //M / /testpool/bar/hello-hardlink 256 | func parseInodeChanges(lines [][]string) ([]*InodeChange, error) { 257 | changes := make([]*InodeChange, len(lines)) 258 | 259 | for i, line := range lines { 260 | c, err := parseInodeChange(line) 261 | if err != nil { 262 | return nil, fmt.Errorf("Failed to parse line %d of zfs diff: %v, got: '%s'", i, err, line) 263 | } 264 | changes[i] = c 265 | } 266 | return changes, nil 267 | } 268 | 269 | func listByType(t, filter string, depth int) ([]*Dataset, error) { 270 | var args []string 271 | if depth > 0 { 272 | args = []string{"get", fmt.Sprintf("-d%d", depth), "-rHp", "-t", t, "all"} 273 | } else { 274 | args = []string{"get", "-rHp", "-t", t, "all"} 275 | } 276 | if filter != "" { 277 | args = append(args, filter) 278 | } 279 | out, err := zfs(args...) 280 | if err != nil { 281 | return nil, err 282 | } 283 | 284 | var datasets []*Dataset 285 | 286 | name := "" 287 | var ds *Dataset 288 | for _, line := range out { 289 | if name != line[0] { 290 | name = line[0] 291 | ds = &Dataset{Name: name} 292 | datasets = append(datasets, ds) 293 | } 294 | if err := ds.parseLine(line); err != nil { 295 | return nil, err 296 | } 297 | } 298 | 299 | return datasets, nil 300 | } 301 | 302 | func propsSlice(properties map[string]string) []string { 303 | args := make([]string, 0, len(properties)*3) 304 | for k, v := range properties { 305 | args = append(args, "-o") 306 | args = append(args, fmt.Sprintf("%s=%s", k, v)) 307 | } 308 | return args 309 | } 310 | 311 | func (z *Zpool) parseLine(line []string) error { 312 | prop := line[1] 313 | val := line[2] 314 | 315 | var err error 316 | 317 | switch prop { 318 | case "health": 319 | setString(&z.Health, val) 320 | case "allocated": 321 | err = setUint(&z.Allocated, val) 322 | case "size": 323 | err = setUint(&z.Size, val) 324 | case "free": 325 | err = setUint(&z.Free, val) 326 | } 327 | return err 328 | } 329 | -------------------------------------------------------------------------------- /zfs/zfs.go: -------------------------------------------------------------------------------- 1 | // Package zfs provides wrappers around the ZFS command line tools. 2 | package zfs 3 | 4 | import ( 5 | "errors" 6 | "fmt" 7 | "io" 8 | "strconv" 9 | "strings" 10 | ) 11 | 12 | // ZFS dataset types, which can indicate if a dataset is a filesystem, 13 | // snapshot, or volume. 14 | const ( 15 | DatasetFilesystem = "filesystem" 16 | DatasetSnapshot = "snapshot" 17 | DatasetVolume = "volume" 18 | ) 19 | 20 | // Dataset is a ZFS dataset. A dataset could be a clone, filesystem, snapshot, 21 | // or volume. The Type struct member can be used to determine a dataset's type. 22 | // 23 | // The field definitions can be found in the ZFS manual: 24 | // http://www.freebsd.org/cgi/man.cgi?zfs(8). 25 | type Dataset struct { 26 | Name string 27 | Origin string 28 | Used uint64 29 | Avail uint64 30 | Mountpoint string 31 | Compression string 32 | Type string 33 | Written uint64 34 | Volsize uint64 35 | Usedbydataset uint64 36 | Logicalused uint64 37 | Quota uint64 38 | } 39 | 40 | // InodeType is the type of inode as reported by Diff 41 | type InodeType int 42 | 43 | // Types of Inodes 44 | const ( 45 | _ = iota // 0 == unknown type 46 | BlockDevice InodeType = iota 47 | CharacterDevice 48 | Directory 49 | Door 50 | NamedPipe 51 | SymbolicLink 52 | EventPort 53 | Socket 54 | File 55 | ) 56 | 57 | // ChangeType is the type of inode change as reported by Diff 58 | type ChangeType int 59 | 60 | // Types of Changes 61 | const ( 62 | _ = iota // 0 == unknown type 63 | Removed ChangeType = iota 64 | Created 65 | Modified 66 | Renamed 67 | ) 68 | 69 | // DestroyFlag is the options flag passed to Destroy 70 | type DestroyFlag int 71 | 72 | // Valid destroy options 73 | const ( 74 | DestroyDefault DestroyFlag = 1 << iota 75 | DestroyRecursive = 1 << iota 76 | DestroyRecursiveClones = 1 << iota 77 | DestroyDeferDeletion = 1 << iota 78 | DestroyForceUmount = 1 << iota 79 | ) 80 | 81 | // InodeChange represents a change as reported by Diff 82 | type InodeChange struct { 83 | Change ChangeType 84 | Type InodeType 85 | Path string 86 | NewPath string 87 | ReferenceCountChange int 88 | } 89 | 90 | // Logger can be used to log commands/actions 91 | type Logger interface { 92 | Log(cmd []string) 93 | } 94 | 95 | type defaultLogger struct{} 96 | 97 | func (*defaultLogger) Log(cmd []string) { 98 | return 99 | } 100 | 101 | var logger Logger = &defaultLogger{} 102 | 103 | // SetLogger set a log handler to log all commands including arguments before 104 | // they are executed 105 | func SetLogger(l Logger) { 106 | if l != nil { 107 | logger = l 108 | } 109 | } 110 | 111 | // zfs is a helper function to wrap typical calls to zfs. 112 | func zfs(arg ...string) ([][]string, error) { 113 | c := command{Command: "zfs"} 114 | return c.Run(arg...) 115 | } 116 | 117 | // Datasets returns a slice of ZFS datasets, regardless of type. 118 | // A filter argument may be passed to select a dataset with the matching name, 119 | // or empty string ("") may be used to select all datasets. 120 | func Datasets(filter string, depth int) ([]*Dataset, error) { 121 | return listByType("all", filter, depth) 122 | } 123 | 124 | // Snapshots returns a slice of ZFS snapshots. 125 | // A filter argument may be passed to select a snapshot with the matching name, 126 | // or empty string ("") may be used to select all snapshots. 127 | func Snapshots(filter string, depth int) ([]*Dataset, error) { 128 | return listByType(DatasetSnapshot, filter, depth) 129 | } 130 | 131 | // Filesystems returns a slice of ZFS filesystems. 132 | // A filter argument may be passed to select a filesystem with the matching name, 133 | // or empty string ("") may be used to select all filesystems. 134 | func Filesystems(filter string, depth int) ([]*Dataset, error) { 135 | return listByType(DatasetFilesystem, filter, depth) 136 | } 137 | 138 | // Volumes returns a slice of ZFS volumes. 139 | // A filter argument may be passed to select a volume with the matching name, 140 | // or empty string ("") may be used to select all volumes. 141 | func Volumes(filter string, depth int) ([]*Dataset, error) { 142 | return listByType(DatasetVolume, filter, depth) 143 | } 144 | 145 | // GetDataset retrieves a single ZFS dataset by name. This dataset could be 146 | // any valid ZFS dataset type, such as a clone, filesystem, snapshot, or volume. 147 | func GetDataset(name string) (*Dataset, error) { 148 | out, err := zfs("get", "-Hp", "all", name) 149 | if err != nil { 150 | return nil, err 151 | } 152 | 153 | ds := &Dataset{Name: name} 154 | for _, line := range out { 155 | if err := ds.parseLine(line); err != nil { 156 | return nil, err 157 | } 158 | } 159 | 160 | return ds, nil 161 | } 162 | 163 | // Clone clones a ZFS snapshot and returns a clone dataset. 164 | // An error will be returned if the input dataset is not of snapshot type. 165 | func (d *Dataset) Clone(dest string, properties map[string]string) (*Dataset, error) { 166 | if d.Type != DatasetSnapshot { 167 | return nil, errors.New("can only clone snapshots") 168 | } 169 | args := make([]string, 2, 4) 170 | args[0] = "clone" 171 | args[1] = "-p" 172 | if properties != nil { 173 | args = append(args, propsSlice(properties)...) 174 | } 175 | args = append(args, []string{d.Name, dest}...) 176 | _, err := zfs(args...) 177 | if err != nil { 178 | return nil, err 179 | } 180 | return GetDataset(dest) 181 | } 182 | 183 | // ReceiveSnapshot receives a ZFS stream from the input io.Reader, creates a 184 | // new snapshot with the specified name, and streams the input data into the 185 | // newly-created snapshot. 186 | func ReceiveSnapshot(input io.Reader, name string) (*Dataset, error) { 187 | c := command{Command: "zfs", Stdin: input} 188 | _, err := c.Run("receive", name) 189 | if err != nil { 190 | return nil, err 191 | } 192 | return GetDataset(name) 193 | } 194 | 195 | // SendSnapshot sends a ZFS stream of a snapshot to the input io.Writer. 196 | // An error will be returned if the input dataset is not of snapshot type. 197 | func (d *Dataset) SendSnapshot(output io.Writer) error { 198 | if d.Type != DatasetSnapshot { 199 | return errors.New("can only send snapshots") 200 | } 201 | 202 | c := command{Command: "zfs", Stdout: output} 203 | _, err := c.Run("send", d.Name) 204 | return err 205 | } 206 | 207 | // CreateVolume creates a new ZFS volume with the specified name, size, and 208 | // properties. 209 | // A full list of available ZFS properties may be found here: 210 | // https://www.freebsd.org/cgi/man.cgi?zfs(8). 211 | func CreateVolume(name string, size uint64, properties map[string]string) (*Dataset, error) { 212 | args := make([]string, 4, 5) 213 | args[0] = "create" 214 | args[1] = "-p" 215 | args[2] = "-V" 216 | args[3] = strconv.FormatUint(size, 10) 217 | if properties != nil { 218 | args = append(args, propsSlice(properties)...) 219 | } 220 | args = append(args, name) 221 | _, err := zfs(args...) 222 | if err != nil { 223 | return nil, err 224 | } 225 | return GetDataset(name) 226 | } 227 | 228 | // Destroy destroys a ZFS dataset. If the destroy bit flag is set, any 229 | // descendents of the dataset will be recursively destroyed, including snapshots. 230 | // If the deferred bit flag is set, the snapshot is marked for deferred 231 | // deletion. 232 | func (d *Dataset) Destroy(flags DestroyFlag) error { 233 | args := make([]string, 1, 3) 234 | args[0] = "destroy" 235 | if flags&DestroyRecursive != 0 { 236 | args = append(args, "-r") 237 | } 238 | 239 | if flags&DestroyRecursiveClones != 0 { 240 | args = append(args, "-R") 241 | } 242 | 243 | if flags&DestroyDeferDeletion != 0 { 244 | args = append(args, "-d") 245 | } 246 | 247 | if flags&DestroyForceUmount != 0 { 248 | args = append(args, "-f") 249 | } 250 | 251 | args = append(args, d.Name) 252 | _, err := zfs(args...) 253 | return err 254 | } 255 | 256 | // SetProperty sets a ZFS property on the receiving dataset. 257 | // A full list of available ZFS properties may be found here: 258 | // https://www.freebsd.org/cgi/man.cgi?zfs(8). 259 | func (d *Dataset) SetProperty(key, val string) error { 260 | prop := strings.Join([]string{key, val}, "=") 261 | _, err := zfs("set", prop, d.Name) 262 | return err 263 | } 264 | 265 | // GetProperty returns the current value of a ZFS property from the 266 | // receiving dataset. 267 | // A full list of available ZFS properties may be found here: 268 | // https://www.freebsd.org/cgi/man.cgi?zfs(8). 269 | func (d *Dataset) GetProperty(key string) (string, error) { 270 | out, err := zfs("get", key, d.Name) 271 | if err != nil { 272 | return "", err 273 | } 274 | 275 | return out[0][2], nil 276 | } 277 | 278 | // Snapshots returns a slice of all ZFS snapshots of a given dataset. 279 | func (d *Dataset) Snapshots() ([]*Dataset, error) { 280 | return Snapshots(d.Name, 1) 281 | } 282 | 283 | // CreateFilesystem creates a new ZFS filesystem with the specified name and 284 | // properties. 285 | // A full list of available ZFS properties may be found here: 286 | // https://www.freebsd.org/cgi/man.cgi?zfs(8). 287 | func CreateFilesystem(name string, properties map[string]string) (*Dataset, error) { 288 | args := make([]string, 1, 4) 289 | args[0] = "create" 290 | 291 | if properties != nil { 292 | args = append(args, propsSlice(properties)...) 293 | } 294 | 295 | args = append(args, name) 296 | _, err := zfs(args...) 297 | if err != nil { 298 | return nil, err 299 | } 300 | return GetDataset(name) 301 | } 302 | 303 | // Snapshot creates a new ZFS snapshot of the receiving dataset, using the 304 | // specified name. Optionally, the snapshot can be taken recursively, creating 305 | // snapshots of all descendent filesystems in a single, atomic operation. 306 | func (d *Dataset) Snapshot(name string, recursive bool) (*Dataset, error) { 307 | args := make([]string, 1, 4) 308 | args[0] = "snapshot" 309 | if recursive { 310 | args = append(args, "-r") 311 | } 312 | snapName := fmt.Sprintf("%s@%s", d.Name, name) 313 | args = append(args, snapName) 314 | _, err := zfs(args...) 315 | if err != nil { 316 | return nil, err 317 | } 318 | return GetDataset(snapName) 319 | } 320 | 321 | // Rollback rolls back the receiving ZFS dataset to a previous snapshot. 322 | // Optionally, intermediate snapshots can be destroyed. A ZFS snapshot 323 | // rollback cannot be completed without this option, if more recent 324 | // snapshots exist. 325 | // An error will be returned if the input dataset is not of snapshot type. 326 | func (d *Dataset) Rollback(destroyMoreRecent bool) error { 327 | if d.Type != DatasetSnapshot { 328 | return errors.New("can only rollback snapshots") 329 | } 330 | 331 | args := make([]string, 1, 3) 332 | args[0] = "rollback" 333 | if destroyMoreRecent { 334 | args = append(args, "-r") 335 | } 336 | args = append(args, d.Name) 337 | 338 | _, err := zfs(args...) 339 | return err 340 | } 341 | 342 | // Children returns a slice of children of the receiving ZFS dataset. 343 | // A recursion depth may be specified, or a depth of 0 allows unlimited 344 | // recursion. 345 | func (d *Dataset) Children(depth uint64) ([]*Dataset, error) { 346 | args := []string{"get", "-t", "all", "-Hp", "all"} 347 | if depth > 0 { 348 | args = append(args, "-d") 349 | args = append(args, strconv.FormatUint(depth, 10)) 350 | } else { 351 | args = append(args, "-r") 352 | } 353 | args = append(args, d.Name) 354 | 355 | out, err := zfs(args...) 356 | if err != nil { 357 | return nil, err 358 | } 359 | 360 | var datasets []*Dataset 361 | name := "" 362 | var ds *Dataset 363 | for _, line := range out { 364 | if name != line[0] { 365 | name = line[0] 366 | ds = &Dataset{Name: name} 367 | datasets = append(datasets, ds) 368 | } 369 | if err := ds.parseLine(line); err != nil { 370 | return nil, err 371 | } 372 | } 373 | return datasets[1:], nil 374 | } 375 | 376 | // Diff returns changes between a snapshot and the given ZFS dataset. 377 | // The snapshot name must include the filesystem part as it is possible to 378 | // compare clones with their origin snapshots. 379 | func (d *Dataset) Diff(snapshot string) ([]*InodeChange, error) { 380 | args := []string{"diff", "-FH", snapshot, d.Name}[:] 381 | out, err := zfs(args...) 382 | if err != nil { 383 | return nil, err 384 | } 385 | inodeChanges, err := parseInodeChanges(out) 386 | if err != nil { 387 | return nil, err 388 | } 389 | return inodeChanges, nil 390 | } 391 | -------------------------------------------------------------------------------- /zfs/zfs_test.go: -------------------------------------------------------------------------------- 1 | package zfs_test 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "math" 7 | "os" 8 | "path/filepath" 9 | "reflect" 10 | "runtime" 11 | "testing" 12 | "time" 13 | 14 | "github.com/mistifyio/go-zfs" 15 | ) 16 | 17 | func sleep(delay int) { 18 | time.Sleep(time.Duration(delay) * time.Second) 19 | } 20 | 21 | func pow2(x int) int64 { 22 | return int64(math.Pow(2, float64(x))) 23 | } 24 | 25 | //https://github.com/benbjohnson/testing 26 | // assert fails the test if the condition is false. 27 | func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { 28 | if !condition { 29 | _, file, line, _ := runtime.Caller(1) 30 | fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) 31 | tb.FailNow() 32 | } 33 | } 34 | 35 | // ok fails the test if an err is not nil. 36 | func ok(tb testing.TB, err error) { 37 | if err != nil { 38 | _, file, line, _ := runtime.Caller(1) 39 | fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error()) 40 | tb.FailNow() 41 | } 42 | } 43 | 44 | // equals fails the test if exp is not equal to act. 45 | func equals(tb testing.TB, exp, act interface{}) { 46 | if !reflect.DeepEqual(exp, act) { 47 | _, file, line, _ := runtime.Caller(1) 48 | fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) 49 | tb.FailNow() 50 | } 51 | } 52 | 53 | func zpoolTest(t *testing.T, fn func()) { 54 | tempfiles := make([]string, 3) 55 | for i := range tempfiles { 56 | f, _ := ioutil.TempFile("/tmp/", "zfs-") 57 | defer f.Close() 58 | err := f.Truncate(pow2(30)) 59 | ok(t, err) 60 | tempfiles[i] = f.Name() 61 | defer os.Remove(f.Name()) 62 | } 63 | 64 | pool, err := zfs.CreateZpool("test", nil, tempfiles...) 65 | ok(t, err) 66 | defer pool.Destroy() 67 | ok(t, err) 68 | fn() 69 | 70 | } 71 | 72 | func TestDatasets(t *testing.T) { 73 | zpoolTest(t, func() { 74 | _, err := zfs.Datasets("") 75 | ok(t, err) 76 | 77 | ds, err := zfs.GetDataset("test") 78 | ok(t, err) 79 | equals(t, zfs.DatasetFilesystem, ds.Type) 80 | equals(t, "", ds.Origin) 81 | assert(t, ds.Logicalused > 0, "Logicalused is not greater than 0") 82 | }) 83 | } 84 | 85 | func TestSnapshots(t *testing.T) { 86 | 87 | zpoolTest(t, func() { 88 | snapshots, err := zfs.Snapshots("") 89 | ok(t, err) 90 | 91 | for _, snapshot := range snapshots { 92 | equals(t, zfs.DatasetSnapshot, snapshot.Type) 93 | } 94 | }) 95 | } 96 | 97 | func TestFilesystems(t *testing.T) { 98 | zpoolTest(t, func() { 99 | f, err := zfs.CreateFilesystem("test/filesystem-test", nil) 100 | ok(t, err) 101 | 102 | filesystems, err := zfs.Filesystems("") 103 | ok(t, err) 104 | 105 | for _, filesystem := range filesystems { 106 | equals(t, zfs.DatasetFilesystem, filesystem.Type) 107 | } 108 | 109 | ok(t, f.Destroy(zfs.DestroyDefault)) 110 | }) 111 | } 112 | 113 | func TestCreateFilesystemWithProperties(t *testing.T) { 114 | zpoolTest(t, func() { 115 | props := map[string]string{ 116 | "compression": "lz4", 117 | } 118 | 119 | f, err := zfs.CreateFilesystem("test/filesystem-test", props) 120 | ok(t, err) 121 | 122 | equals(t, "lz4", f.Compression) 123 | 124 | filesystems, err := zfs.Filesystems("") 125 | ok(t, err) 126 | 127 | for _, filesystem := range filesystems { 128 | equals(t, zfs.DatasetFilesystem, filesystem.Type) 129 | } 130 | 131 | ok(t, f.Destroy(zfs.DestroyDefault)) 132 | }) 133 | } 134 | 135 | func TestVolumes(t *testing.T) { 136 | zpoolTest(t, func() { 137 | v, err := zfs.CreateVolume("test/volume-test", uint64(pow2(23)), nil) 138 | ok(t, err) 139 | 140 | // volumes are sometimes "busy" if you try to manipulate them right away 141 | sleep(1) 142 | 143 | equals(t, zfs.DatasetVolume, v.Type) 144 | volumes, err := zfs.Volumes("") 145 | ok(t, err) 146 | 147 | for _, volume := range volumes { 148 | equals(t, zfs.DatasetVolume, volume.Type) 149 | } 150 | 151 | ok(t, v.Destroy(zfs.DestroyDefault)) 152 | }) 153 | } 154 | 155 | func TestSnapshot(t *testing.T) { 156 | zpoolTest(t, func() { 157 | f, err := zfs.CreateFilesystem("test/snapshot-test", nil) 158 | ok(t, err) 159 | 160 | filesystems, err := zfs.Filesystems("") 161 | ok(t, err) 162 | 163 | for _, filesystem := range filesystems { 164 | equals(t, zfs.DatasetFilesystem, filesystem.Type) 165 | } 166 | 167 | s, err := f.Snapshot("test", false) 168 | ok(t, err) 169 | 170 | equals(t, zfs.DatasetSnapshot, s.Type) 171 | 172 | equals(t, "test/snapshot-test@test", s.Name) 173 | 174 | ok(t, s.Destroy(zfs.DestroyDefault)) 175 | 176 | ok(t, f.Destroy(zfs.DestroyDefault)) 177 | }) 178 | } 179 | 180 | func TestClone(t *testing.T) { 181 | zpoolTest(t, func() { 182 | f, err := zfs.CreateFilesystem("test/snapshot-test", nil) 183 | ok(t, err) 184 | 185 | filesystems, err := zfs.Filesystems("") 186 | ok(t, err) 187 | 188 | for _, filesystem := range filesystems { 189 | equals(t, zfs.DatasetFilesystem, filesystem.Type) 190 | } 191 | 192 | s, err := f.Snapshot("test", false) 193 | ok(t, err) 194 | 195 | equals(t, zfs.DatasetSnapshot, s.Type) 196 | equals(t, "test/snapshot-test@test", s.Name) 197 | 198 | c, err := s.Clone("test/clone-test", nil) 199 | ok(t, err) 200 | 201 | equals(t, zfs.DatasetFilesystem, c.Type) 202 | 203 | ok(t, c.Destroy(zfs.DestroyDefault)) 204 | 205 | ok(t, s.Destroy(zfs.DestroyDefault)) 206 | 207 | ok(t, f.Destroy(zfs.DestroyDefault)) 208 | }) 209 | } 210 | 211 | func TestSendSnapshot(t *testing.T) { 212 | zpoolTest(t, func() { 213 | f, err := zfs.CreateFilesystem("test/snapshot-test", nil) 214 | ok(t, err) 215 | 216 | filesystems, err := zfs.Filesystems("") 217 | ok(t, err) 218 | 219 | for _, filesystem := range filesystems { 220 | equals(t, zfs.DatasetFilesystem, filesystem.Type) 221 | } 222 | 223 | s, err := f.Snapshot("test", false) 224 | ok(t, err) 225 | 226 | file, _ := ioutil.TempFile("/tmp/", "zfs-") 227 | defer file.Close() 228 | err = file.Truncate(pow2(30)) 229 | ok(t, err) 230 | defer os.Remove(file.Name()) 231 | 232 | err = s.SendSnapshot(file) 233 | ok(t, err) 234 | 235 | ok(t, s.Destroy(zfs.DestroyDefault)) 236 | 237 | ok(t, f.Destroy(zfs.DestroyDefault)) 238 | }) 239 | } 240 | 241 | func TestChildren(t *testing.T) { 242 | zpoolTest(t, func() { 243 | f, err := zfs.CreateFilesystem("test/snapshot-test", nil) 244 | ok(t, err) 245 | 246 | s, err := f.Snapshot("test", false) 247 | ok(t, err) 248 | 249 | equals(t, zfs.DatasetSnapshot, s.Type) 250 | equals(t, "test/snapshot-test@test", s.Name) 251 | 252 | children, err := f.Children(0) 253 | ok(t, err) 254 | 255 | equals(t, 1, len(children)) 256 | equals(t, "test/snapshot-test@test", children[0].Name) 257 | 258 | ok(t, s.Destroy(zfs.DestroyDefault)) 259 | ok(t, f.Destroy(zfs.DestroyDefault)) 260 | }) 261 | } 262 | 263 | func TestListZpool(t *testing.T) { 264 | zpoolTest(t, func() { 265 | pools, err := zfs.ListZpools() 266 | ok(t, err) 267 | equals(t, "test", pools[0].Name) 268 | 269 | }) 270 | } 271 | 272 | func TestRollback(t *testing.T) { 273 | zpoolTest(t, func() { 274 | f, err := zfs.CreateFilesystem("test/snapshot-test", nil) 275 | ok(t, err) 276 | 277 | filesystems, err := zfs.Filesystems("") 278 | ok(t, err) 279 | 280 | for _, filesystem := range filesystems { 281 | equals(t, zfs.DatasetFilesystem, filesystem.Type) 282 | } 283 | 284 | s1, err := f.Snapshot("test", false) 285 | ok(t, err) 286 | 287 | _, err = f.Snapshot("test2", false) 288 | ok(t, err) 289 | 290 | s3, err := f.Snapshot("test3", false) 291 | ok(t, err) 292 | 293 | err = s3.Rollback(false) 294 | ok(t, err) 295 | 296 | err = s1.Rollback(false) 297 | assert(t, err != nil, "should error when rolling back beyond most recent without destroyMoreRecent = true") 298 | 299 | err = s1.Rollback(true) 300 | ok(t, err) 301 | 302 | ok(t, s1.Destroy(zfs.DestroyDefault)) 303 | 304 | ok(t, f.Destroy(zfs.DestroyDefault)) 305 | }) 306 | } 307 | 308 | func TestDiff(t *testing.T) { 309 | zpoolTest(t, func() { 310 | fs, err := zfs.CreateFilesystem("test/origin", nil) 311 | ok(t, err) 312 | 313 | linkedFile, err := os.Create(filepath.Join(fs.Mountpoint, "linked")) 314 | ok(t, err) 315 | 316 | movedFile, err := os.Create(filepath.Join(fs.Mountpoint, "file")) 317 | ok(t, err) 318 | 319 | snapshot, err := fs.Snapshot("snapshot", false) 320 | ok(t, err) 321 | 322 | unicodeFile, err := os.Create(filepath.Join(fs.Mountpoint, "i ❤ unicode")) 323 | ok(t, err) 324 | 325 | err = os.Rename(movedFile.Name(), movedFile.Name()+"-new") 326 | ok(t, err) 327 | 328 | err = os.Link(linkedFile.Name(), linkedFile.Name()+"_hard") 329 | ok(t, err) 330 | 331 | inodeChanges, err := fs.Diff(snapshot.Name) 332 | ok(t, err) 333 | equals(t, 4, len(inodeChanges)) 334 | 335 | equals(t, "/test/origin/", inodeChanges[0].Path) 336 | equals(t, zfs.Directory, inodeChanges[0].Type) 337 | equals(t, zfs.Modified, inodeChanges[0].Change) 338 | 339 | equals(t, "/test/origin/linked", inodeChanges[1].Path) 340 | equals(t, zfs.File, inodeChanges[1].Type) 341 | equals(t, zfs.Modified, inodeChanges[1].Change) 342 | equals(t, 1, inodeChanges[1].ReferenceCountChange) 343 | 344 | equals(t, "/test/origin/file", inodeChanges[2].Path) 345 | equals(t, "/test/origin/file-new", inodeChanges[2].NewPath) 346 | equals(t, zfs.File, inodeChanges[2].Type) 347 | equals(t, zfs.Renamed, inodeChanges[2].Change) 348 | 349 | equals(t, "/test/origin/i ❤ unicode", inodeChanges[3].Path) 350 | equals(t, zfs.File, inodeChanges[3].Type) 351 | equals(t, zfs.Created, inodeChanges[3].Change) 352 | 353 | ok(t, movedFile.Close()) 354 | ok(t, unicodeFile.Close()) 355 | ok(t, linkedFile.Close()) 356 | ok(t, snapshot.Destroy(zfs.DestroyForceUmount)) 357 | ok(t, fs.Destroy(zfs.DestroyForceUmount)) 358 | }) 359 | } 360 | -------------------------------------------------------------------------------- /zfs/zpool.go: -------------------------------------------------------------------------------- 1 | package zfs 2 | 3 | // ZFS zpool states, which can indicate if a pool is online, offline, 4 | // degraded, etc. More information regarding zpool states can be found here: 5 | // https://docs.oracle.com/cd/E19253-01/819-5461/gamno/index.html. 6 | const ( 7 | ZpoolOnline = "ONLINE" 8 | ZpoolDegraded = "DEGRADED" 9 | ZpoolFaulted = "FAULTED" 10 | ZpoolOffline = "OFFLINE" 11 | ZpoolUnavail = "UNAVAIL" 12 | ZpoolRemoved = "REMOVED" 13 | ) 14 | 15 | // Zpool is a ZFS zpool. A pool is a top-level structure in ZFS, and can 16 | // contain many descendent datasets. 17 | type Zpool struct { 18 | Name string 19 | Health string 20 | Allocated uint64 21 | Size uint64 22 | Free uint64 23 | } 24 | 25 | // zpool is a helper function to wrap typical calls to zpool. 26 | func zpool(arg ...string) ([][]string, error) { 27 | c := command{Command: "zpool"} 28 | return c.Run(arg...) 29 | } 30 | 31 | // GetZpool retrieves a single ZFS zpool by name. 32 | func GetZpool(name string) (*Zpool, error) { 33 | out, err := zpool("get", "all", "-p", name) 34 | if err != nil { 35 | return nil, err 36 | } 37 | 38 | // there is no -H 39 | out = out[1:] 40 | 41 | z := &Zpool{Name: name} 42 | for _, line := range out { 43 | if err := z.parseLine(line); err != nil { 44 | return nil, err 45 | } 46 | } 47 | 48 | return z, nil 49 | } 50 | 51 | // Datasets returns a slice of all ZFS datasets in a zpool. 52 | func (z *Zpool) Datasets(depth int) ([]*Dataset, error) { 53 | return Datasets(z.Name, depth) 54 | } 55 | 56 | // Snapshots returns a slice of all ZFS snapshots in a zpool. 57 | func (z *Zpool) Snapshots(depth int) ([]*Dataset, error) { 58 | return Snapshots(z.Name, depth) 59 | } 60 | 61 | // CreateZpool creates a new ZFS zpool with the specified name, properties, 62 | // and optional arguments. 63 | // A full list of available ZFS properties and command-line arguments may be 64 | // found here: https://www.freebsd.org/cgi/man.cgi?zfs(8). 65 | func CreateZpool(name string, properties map[string]string, args ...string) (*Zpool, error) { 66 | cli := make([]string, 1, 4) 67 | cli[0] = "create" 68 | if properties != nil { 69 | cli = append(cli, propsSlice(properties)...) 70 | } 71 | cli = append(cli, name) 72 | cli = append(cli, args...) 73 | _, err := zpool(cli...) 74 | if err != nil { 75 | return nil, err 76 | } 77 | 78 | return &Zpool{Name: name}, nil 79 | } 80 | 81 | // Destroy destroys a ZFS zpool by name. 82 | func (z *Zpool) Destroy() error { 83 | _, err := zpool("destroy", z.Name) 84 | return err 85 | } 86 | 87 | // ListZpools list all ZFS zpools accessible on the current system. 88 | func ListZpools() ([]*Zpool, error) { 89 | args := []string{"list", "-Ho", "name"} 90 | out, err := zpool(args...) 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | var pools []*Zpool 96 | 97 | for _, line := range out { 98 | z, err := GetZpool(line[0]) 99 | if err != nil { 100 | return nil, err 101 | } 102 | pools = append(pools, z) 103 | } 104 | return pools, nil 105 | } 106 | --------------------------------------------------------------------------------