├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── blake2.go ├── cmd ├── libzipfs-combiner │ └── combinercmd.go └── mountzip │ └── mountzip.go ├── combiner.go ├── combiner_test.go ├── debug.go ├── doc.md ├── exists.go ├── gitcommit.go ├── libzipfs.go ├── libzipfs_test.go ├── offset_test.go ├── splitter.go ├── splitter_test.go ├── testfiles ├── api.go ├── expected.hello ├── expectedCombined ├── hi.zip ├── padded8hi ├── tester └── tester.go ├── umount.go ├── util.go ├── util_test.go └── vprint.go /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | cmd/libzipfs-combiner/libzipfs-combiner 3 | api-demo-combo 4 | testfiles/api-demo 5 | api-demo 6 | 7 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 8 | *.o 9 | *.a 10 | *.so 11 | 12 | # Folders 13 | _obj 14 | _test 15 | 16 | # Architecture specific extensions/prefixes 17 | *.[568vq] 18 | [568vq].out 19 | 20 | *.cgo1.go 21 | *.cgo2.c 22 | _cgo_defun.c 23 | _cgo_gotypes.go 24 | _cgo_export.* 25 | 26 | _testmain.go 27 | 28 | *.exe 29 | *.test 30 | *.prof 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT license. 2 | 3 | Portions Copyright (c) 2015 Jason E. Aten, Ph.D. 4 | Portions Copyright (c) 2014 Tommi Virtanen. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: all demo demo2 install clean 2 | 3 | curdir = $(shell pwd) 4 | 5 | # determine appropriate umount command, which depends on OS. 6 | UMOUNT := fusermount -u 7 | 8 | UNAME_S := $(shell uname -s) 9 | ifeq ($(UNAME_S),Linux) 10 | UMOUNT = fusermount -u 11 | endif 12 | ifeq ($(UNAME_S),Darwin) 13 | UMOUNT = umount 14 | endif 15 | 16 | all: 17 | /bin/echo "package libzipfs" > gitcommit.go 18 | /bin/echo "func init() { GITLASTTAG = \"$(shell git describe --abbrev=0 --tags)\"; GITLASTCOMMIT = \"$(shell git rev-parse HEAD)\" }" >> gitcommit.go 19 | cd cmd/libzipfs-combiner && go install 20 | cd cmd/mountzip && go install 21 | 22 | install: all 23 | 24 | clean: 25 | rm -f api-demo-combo api-demo cmd/libzipfs-combiner/libzipfs-combiner *~ 26 | 27 | demo: 28 | cd cmd/libzipfs-combiner && go install 29 | go build -o api-demo testfiles/api.go 30 | rm -f ./api-demo-combo 31 | libzipfs-combiner -exe ./api-demo -zip testfiles/hi.zip -o ./api-demo-combo 32 | ./api-demo-combo 33 | 34 | 35 | # This is not a great demo under a makefile, but still demonstrates 36 | # steps you would do manually during the process of inspecting your combo file's 37 | # Zip contents by mounting it with mountzip. 38 | # 39 | # Possible bad side effect: if you are running other mountzip, this will pkill them too. 40 | demo2: 41 | cd cmd/mountzip && go install 42 | rmdir testfiles/mnt || true # it is okay if this complains. We are just trying to cleanup any previous attempt. 43 | mkdir testfiles/mnt 44 | ${GOPATH}/bin/mountzip -zip testfiles/expectedCombined -mnt testfiles/mnt & 45 | sleep 1 46 | # next line should output 'salutations', reading from inside the expectedCombined combo file. 47 | cat testfiles/mnt/dirA/dirB/hello 48 | diff testfiles/mnt/dirA/dirB/hello testfiles/expected.hello 49 | pkill mountzip 50 | sleep 1 51 | # mountzip will try to umount, but it can't always succeed, depending on what the kernel thinks. 52 | # Hence we do an additional umount attempt following the sleep 1 above. This generally succeeds. 53 | $(UMOUNT) ${curdir}/testfiles/mnt 54 | rmdir testfiles/mnt 55 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | libzipfs 2 | =========== 3 | 4 | ~~~ 5 | ------------------- --------------- 6 | | go executable | | zip file | 7 | ------------------- --------------- 8 | \ / 9 | --> libzipfs-combiner <- 10 | | 11 | v 12 | ---------------------------------------------------- 13 | | go executable | zip file | 256-byte footer | = a new single executable with all media inside 14 | ---------------------------------------------------- 15 | ~~~ 16 | 17 | 18 | Libzipfs lets you ship a filesystem of media resources inside your 19 | golang application. This is done by attaching a Zip file containing 20 | the directories of your choice to the end of your application, and 21 | following it with a short footer to allow the go executable to 22 | locate the files and serve them via a fuse mount point. The client 23 | will typically be either be Go or C embedded in the go executable 24 | ([see for example testfiles/api.go](https://github.com/glycerine/libzipfs/blob/master/testfiles/api.go)), but can be another application 25 | altogether. For the later use, 26 | see for example the [libzipfs/cmd/mountzip/mountzip.go example source code.](https://github.com/glycerine/libzipfs/blob/master/cmd/mountzip/mountzip.go) 27 | 28 | ## Use cases 29 | 30 | 1. You have a bunch of images/scripts/files to be served from a webserving application, 31 | and you want to bundle those resources alongside your Go webapp. libzipfs lets 32 | you easily create a single executable that contains all your resources in one 33 | place. 34 | 35 | 1. If you are using CGO to call C code, and that C code expects to be able to 36 | read files from somewhere on the filesystem, you can package up all those 37 | files, ship them with the executable, and they can be read from the 38 | fuse mountpoint -- where the C code can find and use them. For example, 39 | my https://github.com/glycerine/rmq project embeds R inside a Go binary, 40 | and libzipfs allows R libraries to be easily shipped all together in a 41 | single binary. 42 | 43 | ## status 44 | 45 | Excellent. Works well and is very useful. I only use it on OSX and Linux. On OSX 46 | you need to have [OSX Fuse](https://osxfuse.github.io/) installed first. On Linux you'll need to either `sudo yum install fuse` or `sudo apt-get install fuse-utils` to obtain the `/bin/fusermount` utility. 47 | 48 | ## installation 49 | 50 | ~~~ 51 | $ go get -t -u -v github.com/glycerine/libzipfs 52 | $ cd $GOPATH/src/github.com/glycerine/libzipfs && make 53 | $ ## note 1: the libzipfs-combiner and mountzip utilities are now in your $GOPATH/bin 54 | $ ## note 2: be sure that $GOPATH/bin is added to your PATH env variable 55 | $ ## e.g. in your ~/.bashrc you have: export PATH=$GOPATH/bin:$PATH 56 | $ go test -v # to run the test suite 57 | $ make demo # to see the demo code run 58 | ~~~ 59 | 60 | ## origins 61 | 62 | The libzipfs library is derived from Tommi Virtanen's work https://github.com/bazil/zipfs, 63 | which is fantastic and provides a fuse-mounted read-only filesystem from a Zip file. 64 | The zipfs library and https://github.com/bazil/fuse are doing the heavy lifting 65 | behind the scenes. 66 | 67 | The project was inspired by https://github.com/bazil/zipfs and https://github.com/shurcooL/vfsgen 68 | 69 | In particular, vfsgen is a similar approach, but I needed the ability to serve files to legacy code that 70 | expects to read from a file system. 71 | 72 | ## libzipfs: a fully integrated solution 73 | 74 | Libzipfs builds on top of zipfs to allow developers the ability to assemble a "combined" file from an executable 75 | and a Zip file containing the directories you wish to have available to your application at runtime. 76 | The structure of the combined file looks like this: 77 | 78 | ~~~ 79 | ---------------------------------------------------- 80 | | executable | zip file | 256-byte footer | 81 | ---------------------------------------------------- 82 | ^ ^ 83 | byte 0 byte N 84 | ~~~ 85 | 86 | The embedded Zip file (the middle part in the diagram above) can 87 | then be made available via a fuse mountpoint. 88 | The Go executable will contain Go code to accomplish this. The 256-bite 89 | footer at the end of the file describes the location of the 90 | embedded zip file. The combined file is still an executable, 91 | and can be run directly. 92 | 93 | ### creating a combined executable and Zip file 94 | 95 | the `libzipfs-combiner` utility does this for you. 96 | 97 | For example: assuming that `my.go.binary` and `hi.zip` already exist, 98 | and you wish to create a new combo executable called `my.go.binary.combo`, 99 | you would do: 100 | 101 | ~~~ 102 | $ libzipfs-combiner --help 103 | libzipfs-combiner --help 104 | Usage of libzipfs-combiner: 105 | -exe string 106 | path to the executable file 107 | -o string 108 | path to the combined output file to be written (or split if -split given) 109 | -split 110 | split the output file back apart (instead of combine which is the default) 111 | -zip string 112 | path to the Zip file to embed 113 | 114 | $ libzipfs-combiner --exe my.go.binary -o my.go.binary.combo -zip hi.zip 115 | ~~~ 116 | 117 | ### api/code code inside your `my.go.binary.combo` binary: 118 | 119 | type `make demo` and see testfiles/api.go for a full demo: 120 | 121 | Our demo zip file `testfiles/hi.zip` is a simple zip file with one file `hello` that resides inside two nested directories: 122 | 123 | ~~~ 124 | $ unzip -Z -z testfiles/hi.zip 125 | Archive: testfiles/hi.zip 478 bytes 3 files 126 | drwxr-xr-x 3.0 unx 0 bx stor 19-Dec-15 17:27 dirA/ 127 | drwxr-xr-x 3.0 unx 0 bx stor 19-Dec-15 17:27 dirA/dirB/ 128 | -rw-r--r-- 3.0 unx 12 tx stor 19-Dec-15 17:27 dirA/dirB/hello 129 | 3 files, 12 bytes uncompressed, 12 bytes compressed: 0.0% 130 | ~~~ 131 | 132 | the go code: 133 | 134 | ~~~ 135 | package main 136 | 137 | import ( 138 | "bytes" 139 | "fmt" 140 | "io/ioutil" 141 | "path" 142 | 143 | "github.com/glycerine/libzipfs" 144 | ) 145 | 146 | // build instructions: 147 | // 148 | // cd libzipfs && make 149 | // cd testfiles; go build -o api-demo api.go 150 | // libzipfs-combiner -exe api-demo -zip hi.zip -o api-demo-combo 151 | // ./api-demo-combo 152 | 153 | func main() { 154 | z, mountpoint, err := libzipfs.MountComboZip() 155 | if err != nil { 156 | panic(err) 157 | } 158 | defer z.Stop() // if you want to stop serving files 159 | 160 | // access the files from `hi.zip` at mountpoint 161 | 162 | by, err := ioutil.ReadFile(path.Join(mountpoint, "dirA", "dirB", "hello")) 163 | if err != nil { 164 | panic(err) 165 | } 166 | by = bytes.TrimRight(by, "\n") 167 | fmt.Printf("we should see our file dirA/dirB/hello from inside hi.zip, containing 'salutations'.... '%s'\n", string(by)) 168 | 169 | if string(by) != "salutations" { 170 | panic("problem detected") 171 | } 172 | fmt.Printf("Excellent: all looks good.\n") 173 | } 174 | ~~~ 175 | 176 | ## example number 2: mountzip 177 | 178 | The `mountzip` utility (see the source code in `libzipfs/cmd/mountzip/mountzip.go`) mounts a zip file of your choice on a directory of your choice. 179 | 180 | ~~~ 181 | $ cd $GOPATH/src/github.com/glycerine/libzipfs 182 | $ make # installs the mountzip utility into $GOPATH/bin 183 | $ mountzip -help 184 | Usage of mountzip: 185 | -mnt string 186 | directory to fuse-mount the Zip file on 187 | -zip string 188 | path to the Zip file to mount 189 | $ 190 | $ mkdir /tmp/hi 191 | $ mountzip -zip testfiles/hi.zip -mnt /tmp/hi 192 | Zip file 'hi.zip' mounted at directory '/tmp/hi'. [press ctrl-c to exit and unmount] 193 | 194 | ~~~ 195 | 196 | license 197 | ------- 198 | 199 | [MIT license](http://opensource.org/licenses/mit-license.php). See enclosed LICENSE file. 200 | -------------------------------------------------------------------------------- /blake2.go: -------------------------------------------------------------------------------- 1 | package libzipfs 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "io" 8 | "os" 9 | 10 | "github.com/codahale/blake2" 11 | ) 12 | 13 | func (foot *Footer) FillHashes(cfg *CombinerConfig) error { 14 | 15 | copy(foot.MagicFooterNumber1[:], MAGIC1[:]) 16 | copy(foot.MagicFooterNumber2[:], MAGIC2[:]) 17 | 18 | var hash []byte 19 | var sz int64 20 | var err error 21 | 22 | hash, sz, err = Blake2HashFile(cfg.ExecutablePath) 23 | if err != nil { 24 | return err 25 | } 26 | copy(foot.ExecutableBlake2Checksum[:], hash) 27 | foot.ExecutableLengthBytes = sz 28 | 29 | hash, sz, err = Blake2HashFile(cfg.ZipfilePath) 30 | if err != nil { 31 | return err 32 | } 33 | copy(foot.ZipfileBlake2Checksum[:], hash) 34 | foot.ZipfileLengthBytes = sz 35 | 36 | // fill FooterChecksum 37 | foot.FooterLengthBytes = 256 38 | 39 | hash = foot.GetFooterChecksum() 40 | 41 | copy(foot.FooterBlake2Checksum[:], hash) 42 | VPrintf("debug: foot.FooterBlake2Checksum = '%x'\n", foot.FooterBlake2Checksum) 43 | 44 | return nil 45 | } 46 | 47 | func (foot *Footer) GetFooterChecksum() []byte { 48 | // preserve any checksum, so we can zero it for the hashing 49 | var footerCheck [64]byte 50 | copy(footerCheck[:], foot.FooterBlake2Checksum[:]) 51 | 52 | for i := 0; i < 64; i++ { 53 | foot.FooterBlake2Checksum[i] = 0 54 | } 55 | 56 | h := blake2.New(nil) 57 | h.Write(foot.ToBytes()) 58 | 59 | // restore any checksum already there 60 | copy(foot.FooterBlake2Checksum[:], footerCheck[:]) 61 | 62 | return []byte(h.Sum(nil)) 63 | } 64 | 65 | func Blake2HashFile(path string) (hash []byte, length int64, err error) { 66 | if !FileExists(path) { 67 | return nil, 0, fmt.Errorf("no such file: '%s'", path) 68 | } 69 | 70 | of, err := os.Open(path) 71 | if err != nil { 72 | return nil, 0, fmt.Errorf("Blake2HashFile() error during opening file '%s': '%s'", path, err) 73 | } 74 | defer of.Close() 75 | 76 | h := blake2.New(nil) 77 | length, err = io.Copy(h, of) 78 | if err != nil { 79 | return nil, 0, fmt.Errorf("Blake2HashFile() error during reading from file '%s': '%s'", path, err) 80 | } 81 | hash = h.Sum(nil) 82 | VPrintf("hash = '%x' for file '%s'\n", hash, path) 83 | return hash, length, nil 84 | } 85 | 86 | func (f *Footer) ToBytes() []byte { 87 | // Create a struct and write it. 88 | buf := &bytes.Buffer{} 89 | err := binary.Write(buf, binary.BigEndian, f) 90 | if err != nil { 91 | panic(err) 92 | } 93 | VPrintf("ToBytes() debug: wrote %#v to string of bytes '%x'\n", *f, string(buf.Bytes())) 94 | return buf.Bytes() 95 | } 96 | 97 | func (f *Footer) FromBytes(by []byte) { 98 | // Read into an empty struct. 99 | *f = Footer{} 100 | err := binary.Read(bytes.NewBuffer(by), binary.BigEndian, f) 101 | if err != nil { 102 | panic(err) 103 | } 104 | VPrintf("FromBytes() debug: read f = '%#v' from bytes '%x'\n", *f, string(by)) 105 | } 106 | 107 | func compareByteSlices(a, b []byte, sz int) (diffpos int, err error) { 108 | for i := 0; i < sz; i++ { 109 | if a[i] != b[i] { 110 | return i, fmt.Errorf("first difference at position %d (out of %d)", i, sz) 111 | } 112 | } 113 | return -1, nil 114 | } 115 | -------------------------------------------------------------------------------- /cmd/libzipfs-combiner/combinercmd.go: -------------------------------------------------------------------------------- 1 | /* 2 | combiner appends a zip file to an executable and further appends a footer 3 | in the last 256 bytes that describes the combination. libzipfs will look 4 | for this footer and use it to determine where the internalized zipfile 5 | filesystem starts. 6 | */ 7 | package main 8 | 9 | import ( 10 | "flag" 11 | "fmt" 12 | "log" 13 | "os" 14 | "path" 15 | 16 | lzf "github.com/glycerine/libzipfs" 17 | ) 18 | 19 | var progName string = path.Base(os.Args[0]) 20 | 21 | func usage() { 22 | fmt.Fprintf(os.Stderr, "") 23 | } 24 | 25 | func main() { 26 | lzf.DisplayVersionAndExitIfRequested() 27 | 28 | myflags := flag.NewFlagSet(progName, flag.ExitOnError) 29 | cfg := &lzf.CombinerConfig{} 30 | cfg.DefineFlags(myflags) 31 | 32 | err := myflags.Parse(os.Args[1:]) 33 | err = cfg.ValidateConfig() 34 | if err != nil { 35 | myflags.PrintDefaults() 36 | log.Fatalf("%s command line flag error: '%s'", progName, err) 37 | } 38 | 39 | if cfg.Split { 40 | _, err = lzf.DoSplitOutExeAndZip(cfg) 41 | } else { 42 | err = lzf.DoCombineExeAndZip(cfg) 43 | } 44 | panicOn(err) 45 | } 46 | 47 | func panicOn(err error) { 48 | if err != nil { 49 | panic(err) 50 | } 51 | } 52 | 53 | func exitOn(err error) { 54 | if err != nil { 55 | fmt.Fprintf(os.Stderr, "fatal error: '%s'\n", err) 56 | os.Exit(1) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /cmd/mountzip/mountzip.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/signal" 9 | "path" 10 | 11 | "github.com/glycerine/libzipfs" 12 | ) 13 | 14 | var progName string = path.Base(os.Args[0]) 15 | 16 | type MntzipConfig struct { 17 | ZipfilePath string 18 | MountPath string 19 | } 20 | 21 | // call DefineFlags before myflags.Parse() 22 | func (c *MntzipConfig) DefineFlags(fs *flag.FlagSet) { 23 | fs.StringVar(&c.ZipfilePath, "zip", "", "path to the Zip file (or combo exe+Zip+footer file) to mount") 24 | fs.StringVar(&c.MountPath, "mnt", "", "directory to fuse-mount the Zip file on") 25 | } 26 | 27 | // call c.ValidateConfig() after myflags.Parse() 28 | func (c *MntzipConfig) ValidateConfig() error { 29 | if c.ZipfilePath == "" { 30 | return fmt.Errorf("-zip flag required and missing") 31 | } 32 | if c.MountPath == "" { 33 | return fmt.Errorf("-mnt file required and missing") 34 | } 35 | 36 | if !libzipfs.FileExists(c.ZipfilePath) { 37 | return fmt.Errorf("-zip path '%s' not found.", c.ZipfilePath) 38 | } 39 | 40 | if !libzipfs.DirExists(c.MountPath) { 41 | return fmt.Errorf("-mnt mount path '%s' not found.", c.MountPath) 42 | } 43 | 44 | return nil 45 | } 46 | 47 | func main() { 48 | libzipfs.DisplayVersionAndExitIfRequested() 49 | 50 | // grab ctrl-c 51 | ctrl_C_chan := make(chan os.Signal, 1) 52 | signal.Notify(ctrl_C_chan, os.Interrupt) 53 | 54 | // process command line args 55 | myflags := flag.NewFlagSet(progName, flag.ExitOnError) 56 | cfg := &MntzipConfig{} 57 | cfg.DefineFlags(myflags) 58 | 59 | err := myflags.Parse(os.Args[1:]) 60 | err = cfg.ValidateConfig() 61 | if err != nil { 62 | fmt.Fprintf(os.Stderr, "mountzip: mount a regular Zip file or a libzipfs combo "+ 63 | "(exe+Zip+footer) file's Zip content at the requested mount point. Combo "+ 64 | "files are automatically detected.\n") 65 | myflags.PrintDefaults() 66 | log.Fatalf("%s command line flag error: '%s'", progName, err) 67 | } 68 | 69 | byteOffsetToZipFileStart := int64(0) 70 | bytesAvail := int64(0) 71 | footerBytes := int64(0) 72 | 73 | // detect if this is a combo file 74 | _, foot, comb, err := libzipfs.ReadFooter(cfg.ZipfilePath) 75 | if err != nil { 76 | // assume it is a regular zip file, not a combo file. 77 | } else { 78 | comb.Close() 79 | byteOffsetToZipFileStart = foot.ExecutableLengthBytes 80 | bytesAvail = foot.ZipfileLengthBytes 81 | footerBytes = foot.FooterLengthBytes 82 | } 83 | 84 | z := libzipfs.NewFuseZipFs(cfg.ZipfilePath, 85 | cfg.MountPath, byteOffsetToZipFileStart, bytesAvail, footerBytes) 86 | 87 | err = z.Start() 88 | if err != nil { 89 | log.Fatalf("%s error calling z.Start() to start serving fuse requests: '%s'", progName, err) 90 | } 91 | 92 | fmt.Printf("\nZip file '%s' mounted at directory '%s'. [press ctrl-c to exit and unmount]\n", 93 | cfg.ZipfilePath, cfg.MountPath) 94 | 95 | select { 96 | case <-ctrl_C_chan: 97 | case <-z.Done: 98 | // can happen if someone force unmounts the mount from under us. 99 | } 100 | 101 | err = z.Stop() // stop serving files and unmount at end 102 | if err != nil { 103 | log.Fatalf("%s error while shutting down: '%s'", progName, err) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /combiner.go: -------------------------------------------------------------------------------- 1 | /* 2 | combiner appends a zip file to an executable and further appends a footer 3 | in the last 256 bytes that describes the combination. libzipfs will look 4 | for this footer and use it to determine where the internalized zipfile 5 | filesystem starts. 6 | */ 7 | package libzipfs 8 | 9 | import ( 10 | "bytes" 11 | "flag" 12 | "fmt" 13 | "io" 14 | "os" 15 | ) 16 | 17 | var MAGIC1 = []byte("\nLibZipFs00\n") 18 | var MAGIC2 = []byte("\nLibZipFsEnd\n") 19 | 20 | const LIBZIPFS_FOOTER_LEN = 256 21 | const BLAKE2_HASH_LEN = 64 22 | const MAGIC_NUM_LEN = 16 23 | 24 | type FooterArray [LIBZIPFS_FOOTER_LEN]byte 25 | 26 | type Footer struct { 27 | Reserved1 int64 28 | MagicFooterNumber1 [MAGIC_NUM_LEN]byte 29 | 30 | ExecutableLengthBytes int64 31 | ZipfileLengthBytes int64 32 | FooterLengthBytes int64 33 | 34 | ExecutableBlake2Checksum [BLAKE2_HASH_LEN]byte 35 | ZipfileBlake2Checksum [BLAKE2_HASH_LEN]byte 36 | FooterBlake2Checksum [BLAKE2_HASH_LEN]byte // has itself set to zero when taking the hash. 37 | 38 | MagicFooterNumber2 [MAGIC_NUM_LEN]byte 39 | } 40 | 41 | type CombinerConfig struct { 42 | ExecutablePath string 43 | ZipfilePath string 44 | OutputPath string 45 | Split bool 46 | } 47 | 48 | // call DefineFlags before myflags.Parse() 49 | func (c *CombinerConfig) DefineFlags(fs *flag.FlagSet) { 50 | fs.StringVar(&c.ExecutablePath, "exe", "", "path to the executable file") 51 | fs.StringVar(&c.ZipfilePath, "zip", "", "path to the zip file to embed") 52 | fs.StringVar(&c.OutputPath, "o", "", "path to the combined output file to be written (or split if -split given)") 53 | fs.BoolVar(&c.Split, "split", false, "split the output file back apart (instead of combine which is the default)") 54 | } 55 | 56 | // call c.ValidateConfig() after myflags.Parse() 57 | func (c *CombinerConfig) ValidateConfig() error { 58 | if c.ExecutablePath == "" { 59 | return fmt.Errorf("-exe flag required and missing") 60 | } 61 | if c.ZipfilePath == "" { 62 | return fmt.Errorf("-zip flag required and missing") 63 | } 64 | if c.OutputPath == "" { 65 | return fmt.Errorf("-o file required and missing") 66 | } 67 | 68 | if c.Split { 69 | 70 | if FileExists(c.ExecutablePath) { 71 | return fmt.Errorf("-exe path '%s' found but should not exist yet, as during -split we will write to it", c.ExecutablePath) 72 | } 73 | 74 | if FileExists(c.ZipfilePath) { 75 | return fmt.Errorf("-zip path '%s' found but should not exist yet, as during -split we will write to it", c.ZipfilePath) 76 | } 77 | 78 | if !FileExists(c.OutputPath) { 79 | return fmt.Errorf("-o path '%s' not found for splitting", c.OutputPath) 80 | } 81 | 82 | } else { 83 | 84 | if !FileExists(c.ExecutablePath) { 85 | return fmt.Errorf("-exe path '%s' not found", c.ExecutablePath) 86 | } 87 | 88 | if !FileExists(c.ZipfilePath) { 89 | return fmt.Errorf("-zip path '%s' not found", c.ZipfilePath) 90 | } 91 | 92 | if FileExists(c.OutputPath) { 93 | return fmt.Errorf("-o path '%s' already exists but should not", c.OutputPath) 94 | } 95 | } 96 | 97 | return nil 98 | } 99 | 100 | func DoCombineExeAndZip(cfg *CombinerConfig) error { 101 | 102 | xi, err := os.Stat(cfg.ExecutablePath) 103 | if err != nil { 104 | return fmt.Errorf("DoCombinedExeAndZip() error: could not stat exe path '%s': '%s'", cfg.ExecutablePath, err) 105 | } 106 | VPrintf("xi = '%#v'", xi) 107 | 108 | zi, err := os.Stat(cfg.ZipfilePath) 109 | if err != nil { 110 | return fmt.Errorf("DoCombinedExeAndZip() error: could not stat zipfile path '%s': '%s'", cfg.ZipfilePath, err) 111 | } 112 | VPrintf("zi = '%#v'", zi) 113 | 114 | // create the footer metadata 115 | var foot Footer 116 | err = foot.FillHashes(cfg) 117 | if err != nil { 118 | return fmt.Errorf("DoCombinedExeAndZip() error in FillHashes() for cfg '%#v': '%s'", cfg, err) 119 | } 120 | footBuf := bytes.NewBuffer(foot.ToBytes()) 121 | 122 | // sanity check against the stat info 123 | if xi.Size() != foot.ExecutableLengthBytes { 124 | panic(fmt.Errorf("%d == xi.Size() != foot.ExecutableLengthBytes == %d", xi.Size(), foot.ExecutableLengthBytes)) 125 | } 126 | if zi.Size() != foot.ZipfileLengthBytes { 127 | panic(fmt.Errorf("%d == zi.Size() != foot.ZipfileLengthBytes == %d", zi.Size(), foot.ZipfileLengthBytes)) 128 | } 129 | 130 | // create the output file, o 131 | o, err := os.Create(cfg.OutputPath) 132 | panicOn(err) 133 | defer o.Close() 134 | 135 | // write to the output file from exe, zip, then footer: 136 | 137 | // open exe 138 | exeFd, err := os.Open(cfg.ExecutablePath) 139 | panicOn(err) 140 | defer exeFd.Close() 141 | 142 | // open zip 143 | zipFd, err := os.Open(cfg.ZipfilePath) 144 | panicOn(err) 145 | defer zipFd.Close() 146 | 147 | // copy exe to o 148 | exeSz, err := io.Copy(o, exeFd) 149 | panicOn(err) 150 | if exeSz != foot.ExecutableLengthBytes { 151 | panic("wrong exeSz!") 152 | } 153 | 154 | // copy zip to o 155 | zipSz, err := io.Copy(o, zipFd) 156 | panicOn(err) 157 | if zipSz != foot.ZipfileLengthBytes { 158 | panic("wrong zipSz!") 159 | } 160 | 161 | // copy footer to o 162 | footSz, err := io.Copy(o, footBuf) 163 | panicOn(err) 164 | if footSz != foot.FooterLengthBytes { 165 | panic("wrong footSz!") 166 | } 167 | 168 | o.Close() 169 | var executable = os.FileMode(0755) 170 | err = os.Chmod(cfg.OutputPath, executable) 171 | panicOn(err) 172 | 173 | return nil 174 | } 175 | 176 | func panicOn(err error) { 177 | if err != nil { 178 | panic(err) 179 | } 180 | } 181 | 182 | func exitOn(err error) { 183 | if err != nil { 184 | fmt.Fprintf(os.Stderr, "fatal error: '%s'\n", err) 185 | os.Exit(1) 186 | } 187 | } 188 | 189 | func (foot *Footer) VerifyExeZipChecksums(cfg *CombinerConfig) (err error) { 190 | 191 | hash, _, err := Blake2HashFile(cfg.ExecutablePath) 192 | if err != nil { 193 | return err 194 | } 195 | _, err = compareByteSlices(foot.ExecutableBlake2Checksum[:], hash, BLAKE2_HASH_LEN) 196 | if err != nil { 197 | return fmt.Errorf("executable blake2 checksum mismatch: '%s'", err) 198 | } 199 | 200 | hash, _, err = Blake2HashFile(cfg.ZipfilePath) 201 | if err != nil { 202 | return err 203 | } 204 | _, err = compareByteSlices(foot.ZipfileBlake2Checksum[:], hash, BLAKE2_HASH_LEN) 205 | if err != nil { 206 | return fmt.Errorf("zipfile blake2 checksum mismatch: '%s'", err) 207 | } 208 | return nil 209 | } 210 | -------------------------------------------------------------------------------- /combiner_test.go: -------------------------------------------------------------------------------- 1 | package libzipfs 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "os/exec" 9 | "testing" 10 | 11 | cv "github.com/glycerine/goconvey/convey" 12 | ) 13 | 14 | func Test002CombinerAndSplitterAreInverses(t *testing.T) { 15 | 16 | /* expected sizes: 17 | -rw-r--r-- 1 jaten staff 478 Dec 19 17:27 hi.zip 18 | -rwxr-xr-x 1 jaten staff 2315808 Dec 19 22:17 tester 19 | */ 20 | cv.Convey("out footer should report the proper sizes for the input exe and .zip files", t, func() { 21 | testOutputPath, err := ioutil.TempFile("", "libzipfs.test.") 22 | panicOn(err) 23 | defer os.Remove(testOutputPath.Name()) 24 | 25 | var cfg CombinerConfig 26 | cfg.OutputPath = testOutputPath.Name() // should resemble "testfiles/expectedCombined" at the end. 27 | cfg.ExecutablePath = "testfiles/tester" 28 | cfg.ZipfilePath = "testfiles/hi.zip" 29 | 30 | var foot Footer 31 | err = foot.FillHashes(&cfg) 32 | panicOn(err) 33 | footBuf := bytes.NewBuffer(foot.ToBytes()) 34 | VPrintf("footBuf = '%x'\n", footBuf) 35 | 36 | cv.So(foot.ExecutableLengthBytes, cv.ShouldEqual, 2315808) 37 | cv.So(foot.ZipfileLengthBytes, cv.ShouldEqual, 478) 38 | cv.So(foot.FooterLengthBytes, cv.ShouldEqual, LIBZIPFS_FOOTER_LEN) 39 | 40 | cv.So(len(footBuf.Bytes()), cv.ShouldEqual, LIBZIPFS_FOOTER_LEN) 41 | 42 | VPrintf("exe checksum = '%x'\n", foot.ExecutableBlake2Checksum) 43 | VPrintf("zip checksum = '%x'\n", foot.ZipfileBlake2Checksum) 44 | VPrintf("foot checksum = '%x'\n", foot.FooterBlake2Checksum) 45 | 46 | cv.So(foot.MagicFooterNumber1[:len(MAGIC1)], cv.ShouldResemble, MAGIC1) 47 | cv.So(foot.MagicFooterNumber2[:len(MAGIC2)], cv.ShouldResemble, MAGIC2) 48 | 49 | cv.So(fmt.Sprintf("%x", foot.ExecutableBlake2Checksum), cv.ShouldResemble, `61af446f097d3b6c80a910dc295c1aef98f760a61ba3d324d98f134193a79d86ee7db4c46ca33a55879bc561638d0eaed774124d73d2776b21d8b697b98cc04a`) 50 | cv.So(fmt.Sprintf("%x", foot.ZipfileBlake2Checksum), cv.ShouldResemble, `13dad78f512d559c9661e23fe77040f6b08134ab7a29f90ac94c4280454e0973dc95ea034586621392dc8d02b8166326ffa812de9dbc9e1b471f977d8907d719`) 51 | cv.So(fmt.Sprintf("%x", foot.FooterBlake2Checksum), cv.ShouldResemble, `a24fb6f047d66d431e166abc8d008755d3cdfe2b07f4c06b256912151feb114c7cea606b35726e1ae2d2c133b50a7360fe2fce7ca950086f97aa58479e057a22`) 52 | 53 | // first combine files 54 | err = DoCombineExeAndZip(&cfg) 55 | panicOn(err) 56 | 57 | expectedOutPath := "testfiles/expectedCombined" 58 | data, err := exec.Command("diff", "-u", cfg.OutputPath, expectedOutPath).CombinedOutput() 59 | if len(data) > 0 { 60 | panic(fmt.Errorf("combiner error: generated output in '%s' did not match expected output in '%s'. diff output: '%s'", cfg.OutputPath, expectedOutPath, string(data))) 61 | } 62 | panicOn(err) 63 | 64 | // now split it back apart and check it 65 | splitCfg := cfg 66 | 67 | testSplitToExePath, err := ioutil.TempFile("", "libzipfs.test.") 68 | panicOn(err) 69 | defer testSplitToExePath.Close() 70 | defer os.Remove(testSplitToExePath.Name()) 71 | 72 | testSplitToZipPath, err := ioutil.TempFile("", "libzipfs.test.") 73 | panicOn(err) 74 | defer testSplitToZipPath.Close() 75 | defer os.Remove(testSplitToZipPath.Name()) 76 | 77 | splitCfg.ExecutablePath = testSplitToExePath.Name() 78 | splitCfg.ZipfilePath = testSplitToZipPath.Name() 79 | splitCfg.Split = true 80 | 81 | VPrintf("splitCfg = %#v\n", splitCfg) 82 | recoveredFoot, err := DoSplitOutExeAndZip(&splitCfg) 83 | panicOn(err) 84 | cv.So(recoveredFoot, cv.ShouldResemble, &foot) 85 | }) 86 | } 87 | -------------------------------------------------------------------------------- /debug.go: -------------------------------------------------------------------------------- 1 | // +build debug 2 | 3 | // To get FUSE debug log activated, build with `go build -tags debug`. 4 | 5 | package libzipfs 6 | 7 | import ( 8 | "log" 9 | 10 | "bazil.org/fuse" 11 | ) 12 | 13 | func init() { 14 | fuse.Debug = func(msg interface{}) { 15 | log.Print(msg) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /doc.md: -------------------------------------------------------------------------------- 1 | # `zipfs` -- example FUSE filesystem 2 | 3 | `zipfs` is an example of a [`bazil.org/fuse`](http://bazil.org/fuse/) 4 | filesystem that serves a Zip archive: 5 | 6 | ``` console 7 | $ unzip -v archive.zip 8 | Archive: archive.zip 9 | Length Method Size Cmpr Date Time CRC-32 Name 10 | -------- ------ ------- ---- ---------- ----- -------- ---- 11 | 0 Stored 0 0% 2014-12-11 04:03 00000000 buried/ 12 | 0 Stored 0 0% 2014-12-11 04:03 00000000 buried/deep/ 13 | 5 Stored 5 0% 2014-12-11 04:03 2efcceec buried/deep/loot 14 | 13 Stored 13 0% 2014-12-11 04:03 f4247453 greeting 15 | -------- ------- --- ------- 16 | 18 18 0% 4 files 17 | $ zipfs archive.zip mnt & 18 | $ tree mnt 19 | mnt 20 | ├── buried 21 | │   └── deep 22 | │   └── loot 23 | └── greeting 24 | 25 | 2 directories, 2 files 26 | $ cat mnt/greeting 27 | hello, world 28 | ``` 29 | 30 | # FUSE 31 | 32 | [FUSE](https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fuse.txt) 33 | (Filesystem In Userpace) is a Linux kernel filesystem that sends the 34 | incoming requests over a file descriptor to userspace. Historically, 35 | these have been served with a 36 | [C library of the same name](http://fuse.sourceforge.net/), but 37 | ultimately FUSE is just a protocol. Since then, the protocol has been 38 | implemented for other platforms such as OS X, FreeBSD and OpenBSD. 39 | 40 | [bazil.org/fuse](http://bazil.org/fuse) is a reimplementation of that 41 | protocol in pure Go. 42 | 43 | 44 | # Structure of Unix filesystems 45 | 46 | Unix filesystems consist of *inodes* ("index nodes"). These nodes are 47 | files, directories, etc. *Directories* contain *directory entries* 48 | (*dirent*) that point to child *inodes*. A directory entry is 49 | identified by its name, and carries very little metadata. The *inode* 50 | manages both the metadata (including things like access control) and 51 | the content of the file. 52 | 53 | Open files are identified in userspace with *file descriptors*, which 54 | are just safe references to kernel objects known as *handles*. 55 | 56 | 57 | # Go API 58 | 59 | Our FUSE library is split into two parts. The low-level protocol is in 60 | [`bazil.org/fuse`](http://godoc.org/bazil.org/fuse) while the 61 | higher-level, optional, state machine keeping track of object 62 | lifetimes is 63 | [`bazil.org/fuse/fs`](http://godoc.org/bazil.org/fuse/fs). 64 | 65 | Each file system has a *root entry*. The interface 66 | [`fs.FS`](http://godoc.org/bazil.org/fuse/fs#FS) has a method 67 | [`Root`](http://godoc.org/bazil.org/fuse/fs#FS.Root) that returns an 68 | [`fs.Node`](http://godoc.org/bazil.org/fuse/fs#Node). 69 | 70 | To access a file (see its metadata, open it, etc), the kernel looks it 71 | up by name by sending a 72 | [`fuse.LookupRequest`](http://godoc.org/bazil.org/fuse#LookupRequest) 73 | to the FUSE server, stating the parent directory and basename. This 74 | request is served by a 75 | [`Lookup`](http://godoc.org/bazil.org/fuse/fs#NodeRequestLookuper) 76 | method on the parent 77 | [`fs.Node`](http://godoc.org/bazil.org/fuse/fs#Node). The method 78 | returns an [`fs.Node`](http://godoc.org/bazil.org/fuse/fs#Node), and 79 | the result is cached in the kernel and reference counted. Dropping a 80 | cache entry sends a 81 | [`ForgetRequest`](http://godoc.org/bazil.org/fuse#ForgetRequest), and 82 | when the reference count reaches zero, 83 | [`Forget`](http://godoc.org/bazil.org/fuse/fs#NodeForgetter) gets 84 | called. 85 | 86 | Files are renamed with 87 | [`Rename`](http://godoc.org/bazil.org/fuse/fs#NodeRenamer), deleted 88 | with [`Remove`](http://godoc.org/bazil.org/fuse/fs#NodeRemover), and 89 | so on. 90 | 91 | Kernel file *handles* are created for example by opening a file. 92 | Opening an existing file sends an 93 | [`OpenRequest`](http://godoc.org/bazil.org/fuse#OpenRequest), you 94 | guessed it, served by 95 | [`Open`](http://godoc.org/bazil.org/fuse/fs#NodeOpener). All methods 96 | creating new handles return a 97 | [`Handle`](http://godoc.org/bazil.org/fuse/fs#Handle). Handles are 98 | closed by a combination of 99 | [`Flush`](http://godoc.org/bazil.org/fuse/fs#HandleFlusher) and 100 | [`Release`](http://godoc.org/bazil.org/fuse/fs#HandleReleaser). 101 | 102 | The default [`Open`](http://godoc.org/bazil.org/fuse/fs#NodeOpener) 103 | action, if the method is not implemented, is to use the 104 | [`fs.Node`](http://godoc.org/bazil.org/fuse/fs#Node) also as a 105 | [`Handle`](http://godoc.org/bazil.org/fuse/fs#Handle); this tends to 106 | work well for stateless read-only files. 107 | 108 | Reads from a [`Handle`](http://godoc.org/bazil.org/fuse/fs#Handle) are 109 | served by [`Read`](http://godoc.org/bazil.org/fuse/fs#HandleReader), 110 | writes with 111 | [`Write`](http://godoc.org/bazil.org/fuse/fs#HandleWriter), and apart 112 | from all the extra data available these look similar to 113 | [`io.ReaderAt`](http://golang.org/pkg/io/#ReaderAt) and 114 | [`io.WriterAt`](http://golang.org/pkg/io/#WriterAt). Note that file 115 | size changes via 116 | [`Setattr`](http://godoc.org/bazil.org/fuse/fs#NodeSetattrer), not 117 | based on [`Write`](http://godoc.org/bazil.org/fuse/fs#HandleWriter), 118 | and [`Attr`](http://godoc.org/bazil.org/fuse/fs#Node) needs to return 119 | the correct [`Size`](http://godoc.org/bazil.org/fuse#Attr.Size). 120 | 121 | Listing a directory happens by reading an open file handle that is a 122 | directory. Instead of file contents, the read returns marshaled 123 | directory entries. The 124 | [`ReadDir`](http://godoc.org/bazil.org/fuse/fs#HandleReadDirer) method 125 | implements a slightly higher-level API, where you return a slice of 126 | directory entries. 127 | 128 | And so on. Learning to write a file system requires a decent 129 | understanding of the kernel data structures and their state changes 130 | on an abstract level, but the actual Go parts of it are quite simple. 131 | So let's dive into the code. 132 | 133 | 134 | # `zipfs` 135 | 136 | As our example project, we'll write a filesystem that shows a 137 | read-only view of the contents of a 138 | [Zip archive](http://golang.org/pkg/archive/zip/). 139 | 140 | The full source code is available at 141 | https://github.com/bazil/zipfs 142 | 143 | ## Skeleton 144 | 145 | Let's start with a skeleton with argument parsing: 146 | 147 | ``` go 148 | package main 149 | 150 | import ( 151 | "archive/zip" 152 | "flag" 153 | "fmt" 154 | "io" 155 | "log" 156 | "os" 157 | "path/filepath" 158 | "strings" 159 | 160 | "bazil.org/fuse" 161 | "bazil.org/fuse/fs" 162 | ) 163 | 164 | // We assume the zip file contains entries for directories too. 165 | 166 | var progName = filepath.Base(os.Args[0]) 167 | 168 | func usage() { 169 | fmt.Fprintf(os.Stderr, "Usage of %s:\n", progName) 170 | fmt.Fprintf(os.Stderr, " %s ZIP MOUNTPOINT\n", progName) 171 | flag.PrintDefaults() 172 | } 173 | 174 | func main() { 175 | log.SetFlags(0) 176 | log.SetPrefix(progName + ": ") 177 | 178 | flag.Usage = usage 179 | flag.Parse() 180 | 181 | if flag.NArg() != 2 { 182 | usage() 183 | os.Exit(2) 184 | } 185 | path := flag.Arg(0) 186 | mountpoint := flag.Arg(1) 187 | if err := mount(path, mountpoint); err != nil { 188 | log.Fatal(err) 189 | } 190 | } 191 | ``` 192 | 193 | Mounting is a bit cumbersome due to OSXFUSE behaving very differently 194 | from Linux; there are several stages where errors may show up. 195 | 196 | ``` go 197 | func mount(path, mountpoint string) error { 198 | archive, err := zip.OpenReader(path) 199 | if err != nil { 200 | return err 201 | } 202 | defer archive.Close() 203 | 204 | c, err := fuse.Mount(mountpoint) 205 | if err != nil { 206 | return err 207 | } 208 | defer c.Close() 209 | 210 | filesys := &FS{ 211 | archive: &archive.Reader, 212 | } 213 | if err := fs.Serve(c, filesys); err != nil { 214 | return err 215 | } 216 | 217 | // check if the mount process has an error to report 218 | <-c.Ready 219 | if err := c.MountError; err != nil { 220 | return err 221 | } 222 | 223 | return nil 224 | } 225 | ``` 226 | 227 | ## Filesystem 228 | 229 | On to the actual file system. We just hold a pointer to the zip 230 | archive: 231 | 232 | ``` go 233 | type FS struct { 234 | archive *zip.Reader 235 | } 236 | ``` 237 | 238 | And we need to provide the `Root` method: 239 | 240 | ``` go 241 | var _ fs.FS = (*FS)(nil) 242 | 243 | func (f *FS) Root() (fs.Node, fuse.Error) { 244 | n := &Dir{ 245 | archive: f.archive, 246 | } 247 | return n, nil 248 | } 249 | ``` 250 | 251 | ## Directories 252 | 253 | Zip files contain a list of files, but typical zip archivers include 254 | entries for the directories, with a name ending in a slash. We rely on 255 | this behavior later. 256 | 257 | Let's define our `Dir` type, and implement the mandatory `Attr` 258 | method. We use the `*zip.File` to serve directory metadata. 259 | 260 | ``` go 261 | type Dir struct { 262 | archive *zip.Reader 263 | // nil for the root directory, which has no entry in the zip 264 | file *zip.File 265 | } 266 | 267 | var _ fs.Node = (*Dir)(nil) 268 | 269 | func zipAttr(f *zip.File) fuse.Attr { 270 | return fuse.Attr{ 271 | Size: f.UncompressedSize64, 272 | Mode: f.Mode(), 273 | Mtime: f.ModTime(), 274 | Ctime: f.ModTime(), 275 | Crtime: f.ModTime(), 276 | } 277 | } 278 | 279 | func (d *Dir) Attr() fuse.Attr { 280 | if d.file == nil { 281 | // root directory 282 | return fuse.Attr{Mode: os.ModeDir | 0755} 283 | } 284 | return zipAttr(d.file) 285 | } 286 | ``` 287 | 288 | ## Directory entry lookup 289 | 290 | For our filesystem to contain anything useful, we need to be able to 291 | find entries by name. We just iterate over the zip entries, matching 292 | paths: 293 | 294 | ``` go 295 | var _ = fs.NodeRequestLookuper(&Dir{}) 296 | 297 | func (d *Dir) Lookup(req *fuse.LookupRequest, resp *fuse.LookupResponse, intr fs.Intr) (fs.Node, fuse.Error) { 298 | path := req.Name 299 | if d.file != nil { 300 | path = d.file.Name + path 301 | } 302 | for _, f := range d.archive.File { 303 | switch { 304 | case f.Name == path: 305 | child := &File{ 306 | file: f, 307 | } 308 | return child, nil 309 | case f.Name[:len(f.Name)-1] == path && f.Name[len(f.Name)-1] == '/': 310 | child := &Dir{ 311 | archive: d.archive, 312 | file: f, 313 | } 314 | return child, nil 315 | } 316 | } 317 | return nil, fuse.ENOENT 318 | } 319 | ``` 320 | 321 | ## Files 322 | 323 | Our `Lookup` above returned `File` types when the matched entry did 324 | not end in a slash. Let's define type `File`, using the same `zipAttr` 325 | helper as for directories: 326 | 327 | ``` go 328 | type File struct { 329 | file *zip.File 330 | } 331 | 332 | var _ fs.Node = (*File)(nil) 333 | 334 | func (f *File) Attr() fuse.Attr { 335 | return zipAttr(f.file) 336 | } 337 | ``` 338 | 339 | Files are not very useful unless you can open them: 340 | 341 | ``` go 342 | var _ = fs.NodeOpener(&File{}) 343 | 344 | func (f *File) Open(req *fuse.OpenRequest, resp *fuse.OpenResponse, intr fs.Intr) (fs.Handle, fuse.Error) { 345 | r, err := f.file.Open() 346 | if err != nil { 347 | return nil, err 348 | } 349 | // individual entries inside a zip file are not seekable 350 | resp.Flags |= fuse.OpenNonSeekable 351 | return &FileHandle{r: r}, nil 352 | } 353 | ``` 354 | 355 | ## Handles 356 | 357 | 358 | ``` go 359 | type FileHandle struct { 360 | r io.ReadCloser 361 | } 362 | 363 | var _ fs.Handle = (*FileHandle)(nil) 364 | ``` 365 | 366 | We hold an "open file" inside our handle. In this case, it's just a 367 | helper type in `archive/zip`, but in another filesystem this might be 368 | a `*os.File`, a network connection, or such. We should be careful to 369 | close them: 370 | 371 | ``` go 372 | var _ fs.HandleReleaser = (*FileHandle)(nil) 373 | 374 | func (fh *FileHandle) Release(req *fuse.ReleaseRequest, intr fs.Intr) fuse.Error { 375 | return fh.r.Close() 376 | } 377 | ``` 378 | 379 | And then let's handle actual `Read` operations: 380 | 381 | ``` go 382 | var _ = fs.HandleReader(&FileHandle{}) 383 | 384 | func (fh *FileHandle) Read(req *fuse.ReadRequest, resp *fuse.ReadResponse, intr fs.Intr) fuse.Error { 385 | // We don't actually enforce Offset to match where previous read 386 | // ended. Maybe we should, but that would mean'd we need to track 387 | // it. The kernel *should* do it for us, based on the 388 | // fuse.OpenNonSeekable flag. 389 | buf := make([]byte, req.Size) 390 | n, err := fh.r.Read(buf) 391 | resp.Data = buf[:n] 392 | return err 393 | } 394 | ``` 395 | 396 | ## Readdir 397 | 398 | At this point, our files are accessible by `cat` and such, but you 399 | need to know their names. Let's add support for `ReadDir`: 400 | 401 | ``` go 402 | var _ = fs.HandleReadDirer(&Dir{}) 403 | 404 | func (d *Dir) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) { 405 | prefix := "" 406 | if d.file != nil { 407 | prefix = d.file.Name 408 | } 409 | 410 | var res []fuse.Dirent 411 | for _, f := range d.archive.File { 412 | if !strings.HasPrefix(f.Name, prefix) { 413 | continue 414 | } 415 | name := f.Name[len(prefix):] 416 | if name == "" { 417 | // the dir itself, not a child 418 | continue 419 | } 420 | if strings.ContainsRune(name[:len(name)-1], '/') { 421 | // contains slash in the middle -> is in a deeper subdir 422 | continue 423 | } 424 | var de fuse.Dirent 425 | if name[len(name)-1] == '/' { 426 | // directory 427 | name = name[:len(name)-1] 428 | de.Type = fuse.DT_Dir 429 | } 430 | de.Name = name 431 | res = append(res, de) 432 | } 433 | return res, nil 434 | } 435 | ``` 436 | 437 | # Testing zipfs 438 | 439 | Prepare a zip file: 440 | 441 | ``` console 442 | $ mkdir -p data/buried/deep 443 | $ echo hello, world >data/greeting 444 | $ echo gold >data/buried/deep/loot 445 | $ ( cd data && zip -r -q ../archive.zip . ) 446 | ``` 447 | 448 | Mount it: 449 | 450 | ``` console 451 | $ mkdir mnt 452 | $ zipfs archive.zip mnt & 453 | ``` 454 | 455 | Lookup directory entries: 456 | 457 | ``` console 458 | $ ls -ld mnt/greeting 459 | -rw-r--r-- 1 root root 13 Dec 11 2014 mnt/greeting 460 | $ ls -ld mnt/buried 461 | drwxr-xr-x 1 root root 0 Dec 11 2014 mnt/buried 462 | ``` 463 | 464 | Read file contents: 465 | 466 | ``` console 467 | $ cat mnt/greeting 468 | hello, world 469 | $ cat mnt/buried/deep/loot 470 | gold 471 | ``` 472 | 473 | Readdir (the "total 0" is not correct, but that doesn't matter): 474 | 475 | ``` console 476 | $ ls -l mnt 477 | total 0 478 | drwxr-xr-x 1 root root 0 Dec 11 2014 buried 479 | -rw-r--r-- 1 root root 13 Dec 11 2014 greeting 480 | $ ls -l mnt/buried 481 | total 0 482 | drwxr-xr-x 1 root root 0 Dec 11 2014 deep 483 | ``` 484 | 485 | Unmount (for OS X, use `umount mnt`): 486 | 487 | ``` console 488 | $ fusermount -u mnt 489 | ``` 490 | 491 | That's it! For a longer and more featureful examples to read, see 492 | https://github.com/bazil/bolt-mount 493 | ([screencast of a code walkthrough](http://eagain.net/talks/bolt-mount/)) 494 | and all of the 495 | [projects importing fuse](http://godoc.org/bazil.org/fuse?importers). 496 | 497 | # Resources 498 | 499 | - [Bazil](http://bazil.org/) is a distributed file system designed for 500 | single-person disconnected operation. It lets you share your files 501 | across all your computers, with or without cloud services. 502 | 503 | - [FUSE](https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fuse.txt) 504 | is a Linux kernel filesystem that makes calls to userspace to serve 505 | filesystem content. 506 | 507 | - Confusingly also known as [FUSE](http://fuse.sourceforge.net/) is 508 | the C library for implementing userspace FUSE filesystems. 509 | 510 | - [bazil.org/fuse](http://bazil.org/fuse) is a Go library for writing 511 | filesystems. See also GoDoc for 512 | [`fuse`](http://godoc.org/bazil.org/fuse) and 513 | [`fuse/fs`](http://godoc.org/bazil.org/fuse/fs) 514 | 515 | - [OSXFUSE](https://osxfuse.github.io/) is a FUSE kernel 516 | implementation for OS X. 517 | 518 | - [`bolt-mount`](https://github.com/bazil/bolt-mount) is a more 519 | comprehensive example filesystem, including write operations. See 520 | also a 521 | [screencast of a code walkthrough](http://eagain.net/talks/bolt-mount/). 522 | 523 | - [*Writing a file system in Go*](http://bazil.org/talks/2013-06-10-la-gophers/) 524 | is an earlier talk that explains FUSE a bit more. 525 | 526 | - FUSE questions are welcome on the 527 | [bazil-dev Google Group](https://groups.google.com/forum/#!forum/bazil-dev) 528 | or on IRC channel 529 | [#go-nuts on irc.freenode.net](irc:irc.freenode.net/go-nuts). 530 | -------------------------------------------------------------------------------- /exists.go: -------------------------------------------------------------------------------- 1 | package libzipfs 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | func FileExists(name string) bool { 8 | fi, err := os.Stat(name) 9 | if err != nil { 10 | return false 11 | } 12 | if fi.IsDir() { 13 | return false 14 | } 15 | return true 16 | } 17 | 18 | func DirExists(name string) bool { 19 | fi, err := os.Stat(name) 20 | if err != nil { 21 | return false 22 | } 23 | if fi.IsDir() { 24 | return true 25 | } 26 | return false 27 | } 28 | -------------------------------------------------------------------------------- /gitcommit.go: -------------------------------------------------------------------------------- 1 | package libzipfs 2 | func init() { GITLASTTAG = "v1.2"; GITLASTCOMMIT = "df3e318f521647869a7d438c9c3b65bcf91a3078" } 3 | -------------------------------------------------------------------------------- /libzipfs.go: -------------------------------------------------------------------------------- 1 | package libzipfs 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | "sync" 11 | 12 | "archive/zip" 13 | 14 | "bazil.org/fuse" 15 | "bazil.org/fuse/fs" 16 | "golang.org/x/net/context" 17 | ) 18 | 19 | // track git version of this lib 20 | var GITLASTTAG string // git describe --abbrev=0 --tags 21 | var GITLASTCOMMIT string // git rev-parse HEAD 22 | 23 | func VersionString() string { 24 | return fmt.Sprintf("%s/%s", GITLASTTAG, GITLASTCOMMIT) 25 | } 26 | 27 | func DisplayVersionAndExitIfRequested() { 28 | for i := range os.Args { 29 | if os.Args[i] == "-version" || os.Args[i] == "--version" { 30 | fmt.Printf("%s\n", VersionString()) 31 | os.Exit(0) 32 | } 33 | } 34 | } 35 | 36 | // We assume the zip file contains entries for directories too. 37 | 38 | var progName = filepath.Base(os.Args[0]) 39 | 40 | type FuseZipFs struct { 41 | ZipfilePath string 42 | MountPoint string 43 | 44 | Ready chan bool 45 | ReqStop chan bool 46 | Done chan bool 47 | 48 | mut sync.Mutex 49 | stopped bool 50 | serveErr error 51 | connErr error 52 | conn *fuse.Conn 53 | 54 | filesys *FS 55 | //archive *zip.ReadCloser 56 | archive *zip.Reader 57 | 58 | offset int64 59 | bytesAvail int64 // -1 => unknown 60 | footerBytes int64 61 | 62 | fd *os.File 63 | } 64 | 65 | // Mount a possibly combined/zipfile at mountpiont. Call Start() to start servicing fuse reads. 66 | // 67 | // If the file has a libzipfs footer on it, set footerBytes == LIBZIPFS_FOOTER_LEN. 68 | // The bytesAvail value should describe how long the zipfile is in bytes, and byteOffsetToZipFileStart 69 | // should describe how far into the (possibly combined) zipFilePath the actual zipfile starts. 70 | func NewFuseZipFs(zipFilePath, mountpoint string, byteOffsetToZipFileStart int64, bytesAvail int64, footerBytes int64) *FuseZipFs { 71 | 72 | // must trim any trailing slash from the mountpoint, or else mount can fail 73 | mountpoint = TrimTrailingSlashes(mountpoint) 74 | 75 | p := &FuseZipFs{ 76 | ZipfilePath: zipFilePath, 77 | MountPoint: mountpoint, 78 | Ready: make(chan bool), 79 | ReqStop: make(chan bool), 80 | Done: make(chan bool), 81 | offset: byteOffsetToZipFileStart, 82 | bytesAvail: bytesAvail, 83 | footerBytes: footerBytes, 84 | } 85 | 86 | return p 87 | } 88 | 89 | // The Main API entry point for mounting a combo file vis FUSE to make 90 | // the embedded Zip file directory available. Users should call 91 | // fzfs.Stop() when/if they wish to stop serving files at mountpoint. 92 | // 93 | func MountComboZip() (fzfs *FuseZipFs, mountpoint string, err error) { 94 | comboFilePath := os.Args[0] 95 | fzfs, mountpoint, err = NewFuzeZipFsFromCombo(comboFilePath) 96 | if err != nil { 97 | return nil, "", err 98 | } 99 | err = fzfs.Start() 100 | if err != nil { 101 | return nil, "", err 102 | } 103 | return fzfs, mountpoint, nil 104 | } 105 | 106 | // mount the comboFilePath file in a temp directory mountpoint created 107 | // just for this purpose, and return the mountpoint and a handle to the 108 | // fuse fileserver in fzfs. 109 | func NewFuzeZipFsFromCombo(comboFilePath string) (fzfs *FuseZipFs, mountpoint string, err error) { 110 | dir := "" // => use system tmp dir 111 | mountPoint, err := ioutil.TempDir(dir, "libzipfs.auto-combo.") 112 | if err != nil { 113 | return nil, "", fmt.Errorf("NewFuzeZipFsFromCombo() error, could not create mountpoint: '%s'", err) 114 | } 115 | VPrintf("\n\n mountPoint = '%s'\n", mountPoint) 116 | 117 | _, foot, comb, err := ReadFooter(comboFilePath) 118 | if err != nil { 119 | return nil, "", fmt.Errorf("NewFuzeZipFsFromCombo() error, could not reader "+ 120 | "Footer from comboFilePath '%s': '%s'", 121 | comboFilePath, err) 122 | } 123 | defer comb.Close() 124 | byteOffsetToZipFileStart := foot.ExecutableLengthBytes 125 | 126 | z := NewFuseZipFs(comboFilePath, mountPoint, byteOffsetToZipFileStart, foot.ZipfileLengthBytes, LIBZIPFS_FOOTER_LEN) 127 | return z, mountPoint, nil 128 | } 129 | 130 | func (p *FuseZipFs) Stop() error { 131 | p.mut.Lock() 132 | defer p.mut.Unlock() 133 | if p.stopped { 134 | return nil 135 | } 136 | err := p.unmount() 137 | if err != nil { 138 | VPrintf("unmount() of p.MountPoint='%s' failed with error: '%s'\n", p.MountPoint, err) 139 | return err 140 | } 141 | VPrintf("unmount() of p.MountPoint='%s' succeeded.\n", p.MountPoint) 142 | p.stopped = true 143 | <-p.Done 144 | 145 | p.fd.Close() 146 | p.conn.Close() 147 | 148 | // we don't do the following anymore since forcing the unmount 149 | // always results in 'bad file descriptor'. 150 | // if p.serveErr != nil { 151 | // return p.serveErr // always 'bad file descriptor', so skip 152 | // } 153 | 154 | // check if the mount process has an error to report: 155 | <-p.conn.Ready 156 | p.connErr = p.conn.MountError 157 | return p.connErr 158 | } 159 | 160 | func (p *FuseZipFs) Start() error { 161 | var err error 162 | 163 | if p.bytesAvail <= 0 { 164 | statinfo, err := os.Stat(p.ZipfilePath) 165 | if err != nil { 166 | return err 167 | } 168 | p.bytesAvail = statinfo.Size() - (p.offset + p.footerBytes) 169 | if p.bytesAvail <= 0 { 170 | return fmt.Errorf("FuseZipFs.Start() error: no bytes available to read from ZipfilePath '%s' (of size %d bytes) after subtracting offset %d", p.ZipfilePath, statinfo.Size(), p.offset) 171 | } 172 | } 173 | 174 | fd, err := os.Open(p.ZipfilePath) 175 | if err != nil { 176 | return err 177 | } 178 | p.fd = fd 179 | rat := io.NewSectionReader(p.fd, p.offset, p.bytesAvail) 180 | 181 | p.archive, err = zip.NewReader(rat, p.bytesAvail) 182 | if err != nil { 183 | return err 184 | } 185 | 186 | c, err := fuse.Mount(p.MountPoint) 187 | if err != nil { 188 | return err 189 | } 190 | p.conn = c 191 | 192 | p.filesys = &FS{ 193 | archive: p.archive, 194 | } 195 | 196 | go func() { 197 | select { 198 | case <-p.ReqStop: 199 | case <-p.Done: 200 | } 201 | p.Stop() // be sure we cleanup 202 | }() 203 | 204 | go func() { 205 | p.serveErr = fs.Serve(c, p.filesys) 206 | 207 | // shutdown sequence: possibly requested, possibly an error. 208 | close(p.Done) 209 | }() 210 | 211 | err = WaitUntilMounted(p.MountPoint) 212 | if err != nil { 213 | return fmt.Errorf("FuseZipFs.Start() error: could not detect mounted filesystem at mount point %s: '%s'", p.MountPoint, err) 214 | } 215 | close(p.Ready) 216 | 217 | return nil 218 | } 219 | 220 | type FS struct { 221 | archive *zip.Reader 222 | } 223 | 224 | var _ fs.FS = (*FS)(nil) 225 | 226 | func (f *FS) Root() (fs.Node, error) { 227 | n := &Dir{ 228 | archive: f.archive, 229 | } 230 | return n, nil 231 | } 232 | 233 | type Dir struct { 234 | archive *zip.Reader 235 | // nil for the root directory, which has no entry in the zip 236 | file *zip.File 237 | } 238 | 239 | var _ fs.Node = (*Dir)(nil) 240 | 241 | func zipAttr(f *zip.File, a *fuse.Attr) { 242 | a.Size = f.UncompressedSize64 243 | a.Mode = f.Mode() 244 | a.Mtime = f.ModTime() 245 | a.Ctime = f.ModTime() 246 | a.Crtime = f.ModTime() 247 | } 248 | 249 | func (d *Dir) Attr(ctx context.Context, a *fuse.Attr) error { 250 | if d.file == nil { 251 | // root directory 252 | a.Mode = os.ModeDir | 0755 253 | return nil 254 | } 255 | zipAttr(d.file, a) 256 | return nil 257 | } 258 | 259 | var _ = fs.NodeRequestLookuper(&Dir{}) 260 | 261 | func (d *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) { 262 | path := req.Name 263 | if d.file != nil { 264 | path = d.file.Name + path 265 | } 266 | for _, f := range d.archive.File { 267 | switch { 268 | case f.Name == path: 269 | child := &File{ 270 | file: f, 271 | } 272 | return child, nil 273 | case f.Name[:len(f.Name)-1] == path && f.Name[len(f.Name)-1] == '/': 274 | child := &Dir{ 275 | archive: d.archive, 276 | file: f, 277 | } 278 | return child, nil 279 | } 280 | } 281 | return nil, fuse.ENOENT 282 | } 283 | 284 | var _ = fs.HandleReadDirAller(&Dir{}) 285 | 286 | func (d *Dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { 287 | prefix := "" 288 | if d.file != nil { 289 | prefix = d.file.Name 290 | } 291 | 292 | var res []fuse.Dirent 293 | for _, f := range d.archive.File { 294 | if !strings.HasPrefix(f.Name, prefix) { 295 | continue 296 | } 297 | name := f.Name[len(prefix):] 298 | if name == "" { 299 | // the dir itself, not a child 300 | continue 301 | } 302 | if strings.ContainsRune(name[:len(name)-1], '/') { 303 | // contains slash in the middle -> is in a deeper subdir 304 | continue 305 | } 306 | var de fuse.Dirent 307 | if name[len(name)-1] == '/' { 308 | // directory 309 | name = name[:len(name)-1] 310 | de.Type = fuse.DT_Dir 311 | } 312 | de.Name = name 313 | res = append(res, de) 314 | } 315 | return res, nil 316 | } 317 | 318 | type File struct { 319 | file *zip.File 320 | } 321 | 322 | var _ fs.Node = (*File)(nil) 323 | 324 | func (f *File) Attr(ctx context.Context, a *fuse.Attr) error { 325 | zipAttr(f.file, a) 326 | return nil 327 | } 328 | 329 | var _ = fs.NodeOpener(&File{}) 330 | 331 | func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { 332 | r, err := f.file.Open() 333 | if err != nil { 334 | return nil, err 335 | } 336 | // individual entries inside a zip file are not seekable 337 | resp.Flags |= fuse.OpenNonSeekable 338 | return &FileHandle{r: r}, nil 339 | } 340 | 341 | type FileHandle struct { 342 | r io.ReadCloser 343 | } 344 | 345 | var _ fs.Handle = (*FileHandle)(nil) 346 | 347 | var _ fs.HandleReleaser = (*FileHandle)(nil) 348 | 349 | func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error { 350 | return fh.r.Close() 351 | } 352 | 353 | var _ = fs.HandleReader(&FileHandle{}) 354 | 355 | func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { 356 | // We don't actually enforce Offset to match where previous read 357 | // ended. Maybe we should, but that would mean'd we need to track 358 | // it. The kernel *should* do it for us, based on the 359 | // fuse.OpenNonSeekable flag. 360 | // 361 | // One exception to the above is if we fail to fully populate a 362 | // page cache page; a read into page cache is always page aligned. 363 | // Make sure we never serve a partial read, to avoid that. 364 | buf := make([]byte, req.Size) 365 | n, err := io.ReadFull(fh.r, buf) 366 | if err == io.ErrUnexpectedEOF || err == io.EOF { 367 | err = nil 368 | } 369 | resp.Data = buf[:n] 370 | return err 371 | } 372 | 373 | // helper for reading in a loop. will panic on unknown error. 374 | func ShouldRetry(err error) bool { 375 | if err == nil { 376 | return false 377 | } 378 | switch e := err.(type) { 379 | case *os.PathError: 380 | if strings.HasSuffix(e.Error(), "interrupted system call") { 381 | return true // EINTR, must simply retry. 382 | } 383 | panic(fmt.Errorf("got unknown os.PathError, e = '%#v'. e.Error()='%#v'\n", e, e.Error())) 384 | default: 385 | fmt.Printf("unknown err was '%#v' / '%s'\n", err, err.Error()) 386 | panic(err) 387 | } 388 | } 389 | -------------------------------------------------------------------------------- /libzipfs_test.go: -------------------------------------------------------------------------------- 1 | package libzipfs 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "testing" 9 | 10 | cv "github.com/glycerine/goconvey/convey" 11 | ) 12 | 13 | func Test001WeCanMountInTheTmpDir(t *testing.T) { 14 | 15 | cv.Convey("we should be able to mount a zipfile image in the tmp dir", t, func() { 16 | //dir := "/tmp" // => /tmp easier to debug/shorter to type. 17 | dir := "" // => use system tmp dir 18 | mountPoint, err := ioutil.TempDir(dir, "libzipfs") 19 | VPrintf("\n\n mountPoint = '%s'\n", mountPoint) 20 | cv.So(err, cv.ShouldEqual, nil) 21 | 22 | zipFile := "testfiles/hi.zip" 23 | z := NewFuseZipFs(zipFile, mountPoint, 0, -1, 0) 24 | 25 | err = z.Start() 26 | if err != nil { 27 | panic(fmt.Sprintf("error during starting FuseZipFs "+ 28 | "for file '%s' at mount point %s: '%s'", zipFile, mountPoint, err)) 29 | } 30 | 31 | VPrintf("\n\n z.Start() succeeded, with mountPoint = '%s'\n", mountPoint) 32 | expectedFile := path.Join(mountPoint, "dirA", "dirB", "hello") 33 | expectedFileContent := []byte("salutations\n") 34 | 35 | fmt.Printf("\n we should be able to read back a file from the mounted filesystem without errors.\n") 36 | for { 37 | ef, err := os.Open(expectedFile) 38 | if ShouldRetry(err) { 39 | continue 40 | } 41 | cv.So(err, cv.ShouldBeNil) 42 | cv.So(ef, cv.ShouldNotBeNil) 43 | err = ef.Close() 44 | cv.So(err, cv.ShouldBeNil) 45 | break 46 | } 47 | 48 | for { 49 | by, err := ioutil.ReadFile(expectedFile) 50 | if ShouldRetry(err) { 51 | continue 52 | } 53 | cv.So(err, cv.ShouldBeNil) 54 | cv.So(len(expectedFileContent), cv.ShouldEqual, len(by)) 55 | diff, err := compareByteSlices(expectedFileContent, by, len(expectedFileContent)) 56 | cv.So(err, cv.ShouldBeNil) 57 | cv.So(diff, cv.ShouldEqual, -1) 58 | break 59 | } 60 | 61 | err = z.Stop() 62 | if err != nil { 63 | panic(fmt.Sprintf("error: could not z.Stop() FuseZipFs for file '%s' at %s: '%s'", zipFile, mountPoint, err)) 64 | } 65 | 66 | VPrintf("\n\n z.Stop() succeeded, with mountPoint = '%s'\n", mountPoint) 67 | }) 68 | 69 | } 70 | -------------------------------------------------------------------------------- /offset_test.go: -------------------------------------------------------------------------------- 1 | package libzipfs 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "testing" 9 | 10 | cv "github.com/glycerine/goconvey/convey" 11 | ) 12 | 13 | func Test004WeCanMountAnOffsetZipFile(t *testing.T) { 14 | 15 | cv.Convey("we should be able to mount a zipfile image from the second half of a file, i.e. given an offset into the file, mount from the middle of the file should work", t, func() { 16 | //dir := "/tmp" // => /tmp easier to debug/shorter to type. 17 | dir := "" // => use system tmp dir 18 | mountPoint, err := ioutil.TempDir(dir, "libzipfs") 19 | VPrintf("\n\n mountPoint = '%s'\n", mountPoint) 20 | cv.So(err, cv.ShouldEqual, nil) 21 | 22 | // padded8hi has 8 bytes of pre-padding, and no footer. 23 | comboFile := "testfiles/padded8hi" 24 | 25 | byteOffsetToZipFileStart := int64(8) 26 | z := NewFuseZipFs(comboFile, mountPoint, byteOffsetToZipFileStart, 478, 0) 27 | 28 | err = z.Start() 29 | if err != nil { 30 | panic(fmt.Sprintf("error during starting FuseZipFs "+ 31 | "for file '%s' (at offset %d) at mount point %s: '%s'", 32 | comboFile, byteOffsetToZipFileStart, mountPoint, err)) 33 | } 34 | 35 | VPrintf("\n\n z.Start() succeeded, with mountPoint = '%s'\n", mountPoint) 36 | expectedFile := path.Join(mountPoint, "dirA", "dirB", "hello") 37 | expectedFileContent := []byte("salutations\n") 38 | 39 | fmt.Printf("\n we should be able to read back a file from " + 40 | "the mounted filesystem without errors.\n") 41 | for { 42 | ef, err := os.Open(expectedFile) 43 | if ShouldRetry(err) { 44 | continue 45 | } 46 | cv.So(err, cv.ShouldBeNil) 47 | cv.So(ef, cv.ShouldNotBeNil) 48 | err = ef.Close() 49 | cv.So(err, cv.ShouldBeNil) 50 | break 51 | } 52 | 53 | for { 54 | by, err := ioutil.ReadFile(expectedFile) 55 | if ShouldRetry(err) { 56 | continue 57 | } 58 | cv.So(err, cv.ShouldBeNil) 59 | cv.So(len(expectedFileContent), cv.ShouldEqual, len(by)) 60 | diff, err := compareByteSlices(expectedFileContent, by, len(expectedFileContent)) 61 | cv.So(err, cv.ShouldBeNil) 62 | cv.So(diff, cv.ShouldEqual, -1) 63 | break 64 | } 65 | err = z.Stop() 66 | if err != nil { 67 | panic(fmt.Sprintf("error: could not z.Stop() FuseZipFs for file '%s' at %s: '%s'", comboFile, mountPoint, err)) 68 | } 69 | 70 | VPrintf("\n\n z.Stop() succeeded, with mountPoint = '%s'\n", mountPoint) 71 | }) 72 | 73 | } 74 | 75 | func Test004bWeCanMountARegularZipFile(t *testing.T) { 76 | 77 | cv.Convey("exact same test as 004 but with a regular zipfile (no padding before it) for diagnostics/comparison: testfiles/hi.zip", t, func() { 78 | //dir := "/tmp" // => /tmp easier to debug/shorter to type. 79 | dir := "" // => use system tmp dir 80 | mountPoint, err := ioutil.TempDir(dir, "libzipfs") 81 | VPrintf("\n\n mountPoint = '%s'\n", mountPoint) 82 | cv.So(err, cv.ShouldEqual, nil) 83 | 84 | // padded8hi has 8 bytes of pre-padding, and no footer. 85 | comboFile := "testfiles/hi.zip" 86 | 87 | byteOffsetToZipFileStart := int64(0) 88 | z := NewFuseZipFs(comboFile, mountPoint, byteOffsetToZipFileStart, 478, 0) 89 | 90 | err = z.Start() 91 | if err != nil { 92 | panic(fmt.Sprintf("error during starting FuseZipFs "+ 93 | "for file '%s' (at offset %d) at mount point %s: '%s'", 94 | comboFile, byteOffsetToZipFileStart, mountPoint, err)) 95 | } 96 | 97 | VPrintf("\n\n z.Start() succeeded, with mountPoint = '%s'\n", mountPoint) 98 | expectedFile := path.Join(mountPoint, "dirA", "dirB", "hello") 99 | expectedFileContent := []byte("salutations\n") 100 | 101 | fmt.Printf("\n we should be able to read back a file from the mounted filesystem without errors.\n") 102 | for { 103 | ef, err := os.Open(expectedFile) 104 | if ShouldRetry(err) { 105 | continue 106 | } 107 | cv.So(err, cv.ShouldBeNil) 108 | cv.So(ef, cv.ShouldNotBeNil) 109 | err = ef.Close() 110 | cv.So(err, cv.ShouldBeNil) 111 | break 112 | } 113 | 114 | for { 115 | by, err := ioutil.ReadFile(expectedFile) 116 | if ShouldRetry(err) { 117 | continue 118 | } 119 | cv.So(err, cv.ShouldBeNil) 120 | cv.So(len(expectedFileContent), cv.ShouldEqual, len(by)) 121 | diff, err := compareByteSlices(expectedFileContent, by, len(expectedFileContent)) 122 | cv.So(err, cv.ShouldBeNil) 123 | cv.So(diff, cv.ShouldEqual, -1) 124 | break 125 | } 126 | err = z.Stop() 127 | if err != nil { 128 | panic(fmt.Sprintf("error: could not z.Stop() FuseZipFs for file '%s' at %s: '%s'", comboFile, mountPoint, err)) 129 | } 130 | 131 | VPrintf("\n\n z.Stop() succeeded, with mountPoint = '%s'\n", mountPoint) 132 | }) 133 | 134 | } 135 | 136 | func Test006WeCanMountAnOffsetComboFile(t *testing.T) { 137 | 138 | cv.Convey("we should be able to mount an offset zipfile from the second half of a combo file, i.e. given an offset into the file, mount from the middle of the file should work", t, func() { 139 | //dir := "/tmp" // => /tmp easier to debug/shorter to type. 140 | dir := "" // => use system tmp dir 141 | mountPoint, err := ioutil.TempDir(dir, "libzipfs") 142 | VPrintf("\n\n mountPoint = '%s'\n", mountPoint) 143 | cv.So(err, cv.ShouldEqual, nil) 144 | 145 | comboFile := "testfiles/expectedCombined" 146 | 147 | _, foot, comb, err := ReadFooter(comboFile) 148 | panicOn(err) 149 | defer comb.Close() 150 | byteOffsetToZipFileStart := foot.ExecutableLengthBytes 151 | 152 | z := NewFuseZipFs(comboFile, mountPoint, byteOffsetToZipFileStart, foot.ZipfileLengthBytes, LIBZIPFS_FOOTER_LEN) 153 | 154 | err = z.Start() 155 | if err != nil { 156 | panic(fmt.Sprintf("error during starting FuseZipFs "+ 157 | "for file '%s' (at offset %d) at mount point %s: '%s'", 158 | comboFile, byteOffsetToZipFileStart, mountPoint, err)) 159 | } 160 | 161 | VPrintf("\n\n z.Start() succeeded, with mountPoint = '%s'\n", mountPoint) 162 | expectedFile := path.Join(mountPoint, "dirA", "dirB", "hello") 163 | expectedFileContent := []byte("salutations\n") 164 | 165 | fmt.Printf("\n we should be able to read back a file from the mounted filesystem without errors.\n") 166 | for { 167 | ef, err := os.Open(expectedFile) 168 | if ShouldRetry(err) { 169 | continue 170 | } 171 | cv.So(err, cv.ShouldBeNil) 172 | cv.So(ef, cv.ShouldNotBeNil) 173 | err = ef.Close() 174 | cv.So(err, cv.ShouldBeNil) 175 | break 176 | } 177 | 178 | for { 179 | by, err := ioutil.ReadFile(expectedFile) 180 | if ShouldRetry(err) { 181 | continue 182 | } 183 | cv.So(err, cv.ShouldBeNil) 184 | cv.So(len(expectedFileContent), cv.ShouldEqual, len(by)) 185 | diff, err := compareByteSlices(expectedFileContent, by, len(expectedFileContent)) 186 | cv.So(err, cv.ShouldBeNil) 187 | cv.So(diff, cv.ShouldEqual, -1) 188 | break 189 | } 190 | err = z.Stop() 191 | if err != nil { 192 | panic(fmt.Sprintf("error: could not z.Stop() FuseZipFs for file '%s' at %s: '%s'", comboFile, mountPoint, err)) 193 | } 194 | 195 | VPrintf("\n\n z.Stop() succeeded, with mountPoint = '%s'\n", mountPoint) 196 | }) 197 | 198 | } 199 | -------------------------------------------------------------------------------- /splitter.go: -------------------------------------------------------------------------------- 1 | /* 2 | combiner appends a zip file to an executable and further appends a footer 3 | in the last 256 bytes that describes the combination. libzipfs will look 4 | for this footer and use it to determine where the internalized zipfile 5 | filesystem starts. 6 | */ 7 | package libzipfs 8 | 9 | import ( 10 | "fmt" 11 | "io" 12 | "os" 13 | ) 14 | 15 | // client take responsibility for closing combFd when done with it; it is the open 16 | // file handled (if err == nil) for reading from the file at combinedPath. 17 | func ReadFooter(combinedPath string) (footerStartOffset int64, ft *Footer, comb *os.File, err error) { 18 | 19 | // read last 256 bytes of combined file and extract the footer 20 | // cfg.OutputPath is our input now. 21 | var combi os.FileInfo 22 | combi, err = os.Stat(combinedPath) 23 | if err != nil { 24 | return -1, nil, nil, fmt.Errorf("could not stat path '%s': '%s'", combinedPath, err) 25 | } 26 | VPrintf("\n combi = '%#v'\n", combi) 27 | 28 | if combi.Size() < LIBZIPFS_FOOTER_LEN { 29 | return -1, nil, nil, fmt.Errorf("path to split '%s' smaller (bytes=%d) than "+ 30 | "footer(bytes=%d), cannot be a combiner output file", 31 | combinedPath, combi.Size(), LIBZIPFS_FOOTER_LEN) 32 | } 33 | 34 | comb, err = os.Open(combinedPath) 35 | if err != nil { 36 | return -1, nil, nil, fmt.Errorf("could not open path '%s': '%s'", combinedPath, err) 37 | } 38 | defer func() { 39 | // don't leak the comb *os.File if returning an error 40 | if err != nil && comb != nil { 41 | comb.Close() 42 | } 43 | }() 44 | 45 | footerStartOffset, err = comb.Seek(-LIBZIPFS_FOOTER_LEN, 2) 46 | if err != nil { 47 | return -1, nil, nil, fmt.Errorf("could not seek to footer position inside file '%s': '%s'", 48 | combinedPath, err) 49 | } 50 | VPrintf("footerStartOffset = %d\n", footerStartOffset) 51 | 52 | by := make([]byte, LIBZIPFS_FOOTER_LEN) 53 | var n int 54 | n, err = comb.Read(by) 55 | if err != io.EOF && err != nil { 56 | return -1, nil, nil, fmt.Errorf("could not read at footer position inside file '%s': '%s'", 57 | combinedPath, err) 58 | } 59 | if n != LIBZIPFS_FOOTER_LEN { 60 | return -1, nil, nil, fmt.Errorf("could not read the full footer length from file '%s' "+ 61 | "starting at offset %d: %d == bytes_read_in != LIBZIPFS_FOOTER_LEN == %d", 62 | combinedPath, footerStartOffset, n, LIBZIPFS_FOOTER_LEN) 63 | } 64 | 65 | // must return err if foot is bad 66 | var foot *Footer 67 | foot, err = ReifyFooterAndDoInexpensiveChecks(by[:], combinedPath, footerStartOffset) 68 | if err != nil { 69 | return -1, nil, nil, err 70 | } 71 | return footerStartOffset, foot, comb, err 72 | } 73 | 74 | func DoSplitOutExeAndZip(cfg *CombinerConfig) (*Footer, error) { 75 | 76 | if cfg.Split != true { 77 | return nil, fmt.Errorf("DoSplitOutExeAndZip() error: cfg.Split flag "+ 78 | "must be set to true for splitting call. cfg = '%#v'", cfg) 79 | } 80 | 81 | _, foot, comb, err := ReadFooter(cfg.OutputPath) 82 | defer comb.Close() 83 | 84 | // create the split out exe and zip files 85 | exeFd, err := os.Create(cfg.ExecutablePath) 86 | panicOn(err) 87 | defer exeFd.Close() 88 | 89 | exeStartOffset, err := comb.Seek(0, 0) 90 | panicOn(err) 91 | if exeStartOffset != 0 { 92 | panic(fmt.Errorf("exeStartOffset was %d but should be 0", exeStartOffset)) 93 | } 94 | 95 | _, err = io.CopyN(exeFd, comb, foot.ExecutableLengthBytes) 96 | panicOn(err) 97 | exeFd.Close() 98 | 99 | zipFd, err := os.Create(cfg.ZipfilePath) 100 | panicOn(err) 101 | defer zipFd.Close() 102 | 103 | _, err = io.CopyN(zipFd, comb, foot.ZipfileLengthBytes) 104 | panicOn(err) 105 | zipFd.Close() 106 | 107 | err = foot.VerifyExeZipChecksums(cfg) 108 | 109 | return foot, err 110 | } 111 | 112 | // must return err if foot is bad 113 | func ReifyFooterAndDoInexpensiveChecks(by []byte, combinedPath string, footerStartOffset int64) (*Footer, error) { 114 | var err error 115 | var foot Footer 116 | foot.FromBytes(by[:]) 117 | 118 | // NB must use len(MAGIC1) instead of MAGIC_NUM_LEN since len(MAGIC1) is smaller 119 | _, err = compareByteSlices(foot.MagicFooterNumber1[:len(MAGIC1)], MAGIC1, len(MAGIC1)) 120 | if err != nil { 121 | return nil, fmt.Errorf("footer magic number1 not found") 122 | } 123 | 124 | _, err = compareByteSlices(foot.MagicFooterNumber2[:len(MAGIC2)], MAGIC2, len(MAGIC2)) 125 | if err != nil { 126 | return nil, fmt.Errorf("footer magic number2 not found") 127 | } 128 | 129 | // check the checksum over the footer itself 130 | chk := foot.GetFooterChecksum() 131 | for i := 0; i < 64; i++ { 132 | if chk[i] != foot.FooterBlake2Checksum[i] { 133 | return nil, fmt.Errorf("DoSplitOutexeAndZip() error: reified footer from file '%s' does not have the expected checksum, file corrupt or not a combined file? at i=%d, disk position footerStartOffset=%d, computed footer checksum='%x', versus read-from-disk footer checksum = '%x'", combinedPath, i, footerStartOffset, chk, foot.FooterBlake2Checksum) 134 | } 135 | } 136 | 137 | // validate that the component sizes add up 138 | sumFirstTwo := foot.ZipfileLengthBytes + foot.ExecutableLengthBytes 139 | if footerStartOffset != sumFirstTwo { 140 | return nil, fmt.Errorf("DoSplitOutExeAndZip() error: consistency check failed: footerStartOffset(%d) != foot.ZipfileLengthBytes(%d) + foot.ExecutableLengthBytes(%d) == %d", footerStartOffset, foot.ZipfileLengthBytes, foot.ExecutableLengthBytes, sumFirstTwo) 141 | } 142 | 143 | return &foot, nil 144 | } 145 | -------------------------------------------------------------------------------- /splitter_test.go: -------------------------------------------------------------------------------- 1 | package libzipfs 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | cv "github.com/glycerine/goconvey/convey" 10 | ) 11 | 12 | func Test003SplitterDetectsCorruptFooters(t *testing.T) { 13 | 14 | cv.Convey("corrupt or missing footers should be detected and cause an early exit", t, func() { 15 | testOutputPath, err := ioutil.TempFile("", "libzipfs.test.") 16 | panicOn(err) 17 | defer os.Remove(testOutputPath.Name()) 18 | 19 | var cfg CombinerConfig 20 | expectedOutPath := "testfiles/expectedCombined" 21 | cfg.OutputPath = expectedOutPath 22 | cfg.ExecutablePath = "testfiles/tester" 23 | cfg.ZipfilePath = "testfiles/hi.zip" 24 | 25 | var foot Footer 26 | err = foot.FillHashes(&cfg) 27 | panicOn(err) 28 | footBuf := bytes.NewBuffer(foot.ToBytes()) 29 | VPrintf("footBuf = '%x'\n", footBuf) 30 | 31 | // now split it back apart and check it 32 | splitCfg := cfg 33 | 34 | testSplitToExePath, err := ioutil.TempFile("", "libzipfs.test.") 35 | panicOn(err) 36 | defer testSplitToExePath.Close() 37 | defer os.Remove(testSplitToExePath.Name()) 38 | 39 | testSplitToZipPath, err := ioutil.TempFile("", "libzipfs.test.") 40 | panicOn(err) 41 | defer testSplitToZipPath.Close() 42 | defer os.Remove(testSplitToZipPath.Name()) 43 | 44 | splitCfg.ExecutablePath = testSplitToExePath.Name() 45 | splitCfg.ZipfilePath = testSplitToZipPath.Name() 46 | splitCfg.Split = true 47 | 48 | VPrintf("splitCfg = %#v\n", splitCfg) 49 | recoveredFoot, err := DoSplitOutExeAndZip(&splitCfg) 50 | cv.Convey("uncorrupt recoveredFoot should not raise an error, part 1", func() { 51 | cv.So(err, cv.ShouldBeNil) 52 | cv.So(recoveredFoot, cv.ShouldResemble, &foot) 53 | }) 54 | 55 | footerStartOffset := recoveredFoot.ZipfileLengthBytes + recoveredFoot.ExecutableLengthBytes 56 | uncorruptBytes := recoveredFoot.ToBytes() 57 | _, err = ReifyFooterAndDoInexpensiveChecks(uncorruptBytes, splitCfg.OutputPath, footerStartOffset) 58 | cv.Convey("uncorrupt recoveredFoot should not raise an error, part 2", func() { 59 | cv.So(err, cv.ShouldBeNil) 60 | }) 61 | cv.Convey("corrupted recoveredFoot should raise an error, at any byte position", func() { 62 | corruptBytes := make([]byte, len(uncorruptBytes)) 63 | copy(corruptBytes, uncorruptBytes) 64 | for i := range corruptBytes { 65 | // corrupt up 66 | corruptBytes[i]++ 67 | _, err = ReifyFooterAndDoInexpensiveChecks(corruptBytes, splitCfg.OutputPath, footerStartOffset) 68 | cv.So(err, cv.ShouldNotBeNil) 69 | corruptBytes[i] = uncorruptBytes[i] 70 | // or corrupt down 71 | corruptBytes[i]-- 72 | _, err = ReifyFooterAndDoInexpensiveChecks(corruptBytes, splitCfg.OutputPath, footerStartOffset) 73 | cv.So(err, cv.ShouldNotBeNil) 74 | corruptBytes[i] = uncorruptBytes[i] 75 | } 76 | }) 77 | }) 78 | } 79 | -------------------------------------------------------------------------------- /testfiles/api.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | "strings" 10 | 11 | "github.com/glycerine/libzipfs" 12 | ) 13 | 14 | // build instructions: 15 | // 16 | // cd libzipfs && make 17 | // cd testfiles; go build -o api-demo api.go 18 | // libzipfs-combiner -exe api-demo -zip hi.zip -o api-demo-combo 19 | // ./api-demo-combo 20 | 21 | func main() { 22 | // libzipfs.MountComboZip() call serves the 23 | // zipfile described by the footer in the current 24 | // program, as generated by the libzipfs-combiner 25 | // utility. 26 | z, mountpoint, err := libzipfs.MountComboZip() 27 | if err != nil { 28 | panic(err) 29 | } 30 | defer z.Stop() // if you want to stop serving files 31 | 32 | // access the file from `hi.zip` at mountpoint 33 | 34 | // since we may get EINTR and have to retry, we loop over ReadFile() 35 | var by []byte 36 | for { 37 | by, err = ioutil.ReadFile(path.Join(mountpoint, "dirA", "dirB", "hello")) 38 | if err == nil { 39 | break 40 | } 41 | // could use here instead: libzipfs.ShoouldRetry(err) { continue } 42 | switch e := err.(type) { 43 | case *os.PathError: 44 | if strings.HasSuffix(e.Error(), "interrupted system call") { 45 | continue // EINTR, must simply retry. 46 | } 47 | panic(fmt.Errorf("got unknown os.PathError, e = '%#v'. e.Error()='%#v'\n", e, e.Error())) 48 | default: 49 | fmt.Printf("unknown err was '%#v' / '%s'\n", err, err.Error()) 50 | panic(err) 51 | } 52 | break 53 | } 54 | 55 | by = bytes.TrimRight(by, "\n") 56 | fmt.Printf("we should see our file dirA/dirB/hello from inside hi.zip, containing 'salutations'.... '%s'\n", string(by)) 57 | 58 | if string(by) != "salutations" { 59 | panic("problem detected") 60 | } 61 | fmt.Printf("Excellent: all looks good.\n") 62 | } 63 | -------------------------------------------------------------------------------- /testfiles/expected.hello: -------------------------------------------------------------------------------- 1 | salutations 2 | -------------------------------------------------------------------------------- /testfiles/expectedCombined: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glycerine/libzipfs/13d7bfb1ede9ef500b91a24ce13c3066c78cd1de/testfiles/expectedCombined -------------------------------------------------------------------------------- /testfiles/hi.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glycerine/libzipfs/13d7bfb1ede9ef500b91a24ce13c3066c78cd1de/testfiles/hi.zip -------------------------------------------------------------------------------- /testfiles/padded8hi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glycerine/libzipfs/13d7bfb1ede9ef500b91a24ce13c3066c78cd1de/testfiles/padded8hi -------------------------------------------------------------------------------- /testfiles/tester: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glycerine/libzipfs/13d7bfb1ede9ef500b91a24ce13c3066c78cd1de/testfiles/tester -------------------------------------------------------------------------------- /testfiles/tester.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | func main() { 6 | fmt.Printf("hello from the libzipfs test program!\n") 7 | } 8 | -------------------------------------------------------------------------------- /umount.go: -------------------------------------------------------------------------------- 1 | package libzipfs 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "os/exec" 7 | "strings" 8 | "time" 9 | ) 10 | 11 | // locate the mount and umount commands in the filesystem 12 | 13 | type mountCmdLoc struct { 14 | MountPath string 15 | UmountPath string 16 | } 17 | 18 | var utilLoc mountCmdLoc 19 | 20 | func WaitUntilMounted(mountPoint string) error { 21 | 22 | mpBytes := []byte(mountPoint) 23 | dur := 3 * time.Millisecond 24 | tries := 40 25 | var found bool 26 | for i := 0; i < tries; i++ { 27 | out, err := exec.Command(utilLoc.MountPath).Output() 28 | if err != nil { 29 | return fmt.Errorf("could not query for mount points with %s: '%s'", utilLoc.MountPath, err) 30 | } 31 | VPrintf("\n out = '%s'\n", string(out)) 32 | found = bytes.Contains(out, mpBytes) 33 | if found { 34 | VPrintf("\n found mountPoint '%s' on try %d\n", mountPoint, i+1) 35 | return nil 36 | } 37 | time.Sleep(dur) 38 | } 39 | return fmt.Errorf("WaitUntilMounted() error: could not locate mount point '%s' in %s output, "+ 40 | "even after %d tries with %v sleep between.", mountPoint, utilLoc.MountPath, tries, dur) 41 | } 42 | 43 | // 44 | // linux when regular user umount attempts: 45 | // getting error: umount: /tmp/libzipfs694201669 is not in the fstab (and you are not root) 46 | // => need to do fusermount -u mnt instead of umount 47 | func (p *FuseZipFs) unmount() error { 48 | args := []string{p.MountPoint} 49 | if strings.HasSuffix(utilLoc.UmountPath, `fusermount`) { 50 | args = []string{"-u", p.MountPoint} 51 | } 52 | 53 | // exactly two attemps seems to be exactly what is needed 54 | pasted := strings.Join(args, " ") 55 | sleepDur := 20 * time.Millisecond 56 | tries := 2 57 | k := 0 58 | for { 59 | out, err := exec.Command(utilLoc.UmountPath, args...).CombinedOutput() 60 | if err != nil { 61 | VPrintf("\n *** Unmount() error: could not %s %s: '%s' / output: '%s'. That was attempt %d.\n", utilLoc.UmountPath, pasted, err, string(out), k+1) 62 | k++ 63 | if k >= tries { 64 | // generally we'll have been successful on the 2nd try even though the error still shows up. 65 | break 66 | } 67 | time.Sleep(sleepDur) 68 | } 69 | } 70 | 71 | err := WaitUntilUnmounted(p.MountPoint) 72 | if err != nil { 73 | return fmt.Errorf("Unmount() error: tried to wait for mount %s to become unmounted, but got error: '%s'", p.MountPoint, err) 74 | } 75 | return nil 76 | } 77 | 78 | func WaitUntilUnmounted(mountPoint string) error { 79 | 80 | mpBytes := []byte(mountPoint) 81 | dur := 3 * time.Millisecond 82 | tries := 40 83 | var found bool 84 | for i := 0; i < tries; i++ { 85 | out, err := exec.Command(utilLoc.MountPath).Output() 86 | if err != nil { 87 | return fmt.Errorf("could not query for mount points with %s: '%s'", utilLoc.MountPath, err) 88 | } 89 | VPrintf("\n out = '%s'\n", string(out)) 90 | found = bytes.Contains(out, mpBytes) 91 | if !found { 92 | VPrintf("\n mountPoint '%s' was not in mount output on try %d\n", mountPoint, i+1) 93 | return nil 94 | } 95 | time.Sleep(dur) 96 | } 97 | return fmt.Errorf("WaitUntilUnmounted() error: mount point '%s' in (output of %s invocation) was always present, "+ 98 | "even after %d waits with %v sleep between each.", mountPoint, utilLoc.MountPath, tries, dur) 99 | } 100 | 101 | func FindMountUmount() error { 102 | err := FindMount() 103 | if err != nil { 104 | return err 105 | } 106 | err = FindUmount() 107 | if err != nil { 108 | return err 109 | } 110 | return nil 111 | } 112 | 113 | func FindMount() error { 114 | candidates := []string{`/sbin/mount`, `/bin/mount`, `/usr/sbin/mount`, `/usr/bin/mount`} 115 | for _, f := range candidates { 116 | if FileExists(f) { 117 | utilLoc.MountPath = f 118 | return nil 119 | } 120 | } 121 | return fmt.Errorf("mount not found") 122 | } 123 | 124 | func FindUmount() error { 125 | // put the linux fusermount utils first 126 | candidates := []string{`/bin/fusermount`, `/sbin/fusermount`, `/sbin/umount`, `/bin/umount`, `/usr/sbin/umount`, `/usr/bin/umount`} 127 | for _, f := range candidates { 128 | if FileExists(f) { 129 | utilLoc.UmountPath = f 130 | return nil 131 | } 132 | } 133 | return fmt.Errorf("umount not found") 134 | } 135 | 136 | func init() { 137 | err := FindMountUmount() 138 | if err != nil { 139 | panic(err) 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package libzipfs 2 | 3 | // must trim any trailing slash from the mountpoint, or else mount can fail 4 | func TrimTrailingSlashes(mountpoint string) string { 5 | m := len(mountpoint) - 1 6 | for i := 0; i < m; i++ { 7 | if mountpoint[m-i] == '/' { 8 | mountpoint = mountpoint[:(m - i)] 9 | } else { 10 | break 11 | } 12 | } 13 | return mountpoint 14 | } 15 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | package libzipfs 2 | 3 | import ( 4 | "testing" 5 | 6 | cv "github.com/glycerine/goconvey/convey" 7 | ) 8 | 9 | func TestTrimTrailingSlashesWorks(t *testing.T) { 10 | 11 | cv.Convey("TrimTrailingSlashes(`hello///`) should return `hello` and similar correct trims", t, func() { 12 | cv.So(TrimTrailingSlashes(`hello///`), cv.ShouldEqual, `hello`) 13 | cv.So(TrimTrailingSlashes(`hello`), cv.ShouldEqual, `hello`) 14 | cv.So(TrimTrailingSlashes(``), cv.ShouldEqual, ``) 15 | cv.So(TrimTrailingSlashes(`abc`), cv.ShouldEqual, `abc`) 16 | cv.So(TrimTrailingSlashes(`/a/b/c/d/`), cv.ShouldEqual, `/a/b/c/d`) 17 | cv.So(TrimTrailingSlashes(`/a/b/c/d`), cv.ShouldEqual, `/a/b/c/d`) 18 | }) 19 | } 20 | -------------------------------------------------------------------------------- /vprint.go: -------------------------------------------------------------------------------- 1 | package libzipfs 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | var Verbose bool 9 | 10 | func VPrintf(format string, a ...interface{}) { 11 | if Verbose { 12 | TSPrintf(format, a...) 13 | } 14 | } 15 | 16 | func ts() string { 17 | return time.Now().Format("2006-01-02 15:04:05.999 -0700 MST") 18 | } 19 | 20 | // time-stamped printf 21 | func TSPrintf(format string, a ...interface{}) { 22 | fmt.Printf("%s ", ts()) 23 | fmt.Printf(format, a...) 24 | } 25 | --------------------------------------------------------------------------------