├── testdata ├── fs │ ├── empty │ └── a │ │ └── b │ │ └── c │ │ └── d ├── fs2 │ ├── f3.bin │ ├── f2.bin │ └── f1.bin ├── .gitignore ├── fs.zip ├── fs2.zip ├── fs.tar.bz2 ├── fs.tar.gz ├── update-fs.sh ├── download-data.sh └── fs.tar ├── .gitignore ├── README.md ├── write_test.go ├── bench_test.go ├── doc.go ├── open_test.go ├── map.go ├── rewriter.go ├── ro.go ├── chroot.go ├── write.go ├── vfs.go ├── open.go ├── file_util.go ├── fs.go ├── mounter.go ├── file.go ├── util.go ├── mem.go ├── vfs_test.go └── LICENSE /testdata/fs/empty: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testdata/fs/a/b/c/d: -------------------------------------------------------------------------------- 1 | go -------------------------------------------------------------------------------- /testdata/fs2/f3.bin: -------------------------------------------------------------------------------- 1 | f2.bin -------------------------------------------------------------------------------- /testdata/.gitignore: -------------------------------------------------------------------------------- 1 | go1.3.src.tar.gz 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.so 3 | *.o 4 | *.a 5 | .libs/ 6 | .*.swp 7 | -------------------------------------------------------------------------------- /testdata/fs.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rainycape/vfs/HEAD/testdata/fs.zip -------------------------------------------------------------------------------- /testdata/fs2.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rainycape/vfs/HEAD/testdata/fs2.zip -------------------------------------------------------------------------------- /testdata/fs.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rainycape/vfs/HEAD/testdata/fs.tar.bz2 -------------------------------------------------------------------------------- /testdata/fs.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rainycape/vfs/HEAD/testdata/fs.tar.gz -------------------------------------------------------------------------------- /testdata/fs2/f2.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rainycape/vfs/HEAD/testdata/fs2/f2.bin -------------------------------------------------------------------------------- /testdata/update-fs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | cd fs 5 | zip -r ../fs.zip * 6 | tar cvvf ../fs.tar * 7 | tar cvvzf ../fs.tar.gz * 8 | tar cvvjf ../fs.tar.bz2 * 9 | cd - 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # vfs 2 | 3 | vfs implements Virtual File Systems with read-write support in Go (golang) 4 | 5 | [![GoDoc](https://godoc.org/github.com/rainycape/vfs?status.svg)](https://godoc.org/github.com/rainycape/vfs) 6 | -------------------------------------------------------------------------------- /testdata/download-data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | SRC=https://storage.googleapis.com/golang/go1.3.src.tar.gz 4 | if which curl > /dev/null 2>&1; then 5 | curl -O ${SRC} 6 | elif which wget > /dev/null 2&1; then 7 | wget -O `basename ${SRC}` ${SRC} 8 | else 9 | echo "no curl nor wget found" 1>&2 10 | exit 1 11 | fi 12 | -------------------------------------------------------------------------------- /write_test.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "path/filepath" 7 | "testing" 8 | ) 9 | 10 | type writeTester struct { 11 | name string 12 | writer func(io.Writer, VFS) error 13 | reader func(io.Reader) (VFS, error) 14 | } 15 | 16 | func TestWrite(t *testing.T) { 17 | var ( 18 | writeTests = []writeTester{ 19 | {"zip", WriteZip, func(r io.Reader) (VFS, error) { return Zip(r, 0) }}, 20 | {"tar", WriteTar, Tar}, 21 | {"tar.gz", WriteTarGzip, TarGzip}, 22 | } 23 | ) 24 | p := filepath.Join("testdata", "fs.zip") 25 | fs, err := Open(p) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | var buf bytes.Buffer 30 | for _, v := range writeTests { 31 | buf.Reset() 32 | if err := v.writer(&buf, fs); err != nil { 33 | t.Fatalf("error writing %s: %s", v.name, err) 34 | } 35 | newFs, err := v.reader(&buf) 36 | if err != nil { 37 | t.Fatalf("error reading %s: %s", v.name, err) 38 | } 39 | testOpenedVFS(t, newFs) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /bench_test.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "io/ioutil" 7 | "os" 8 | "testing" 9 | ) 10 | 11 | func BenchmarkLoadGoSrc(b *testing.B) { 12 | f := openOptionalTestFile(b, goTestFile) 13 | defer f.Close() 14 | // Decompress to avoid measuring the time to gunzip 15 | zr, err := gzip.NewReader(f) 16 | if err != nil { 17 | b.Fatal(err) 18 | } 19 | defer zr.Close() 20 | data, err := ioutil.ReadAll(zr) 21 | if err != nil { 22 | b.Fatal(err) 23 | } 24 | b.ResetTimer() 25 | for ii := 0; ii < b.N; ii++ { 26 | if _, err := Tar(bytes.NewReader(data)); err != nil { 27 | b.Fatal(err) 28 | } 29 | } 30 | } 31 | 32 | func BenchmarkWalkGoSrc(b *testing.B) { 33 | f := openOptionalTestFile(b, goTestFile) 34 | defer f.Close() 35 | fs, err := TarGzip(f) 36 | if err != nil { 37 | b.Fatal(err) 38 | } 39 | b.ResetTimer() 40 | for ii := 0; ii < b.N; ii++ { 41 | Walk(fs, "/", func(_ VFS, _ string, _ os.FileInfo, _ error) error { return nil }) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package vfs implements Virtual File Systems with read-write support. 2 | // 3 | // All implementatations use slash ('/') separated paths, with / representing 4 | // the root directory. This means that to manipulate or construct paths, the 5 | // functions in path package should be used, like path.Join or path.Dir. 6 | // There's also no notion of the current directory nor relative paths. The paths 7 | // /a/b/c and a/b/c are considered to point to the same element. 8 | // 9 | // This package also implements some shorthand functions which might be used with 10 | // any VFS implementation, providing the same functionality than functions in the 11 | // io/ioutil, os and path/filepath packages: 12 | // 13 | // io/ioutil.ReadFile => ReadFile 14 | // io/ioutil.WriteFile => WriteFile 15 | // os.IsExist => IsExist 16 | // os.IsNotExist => IsNotExist 17 | // os.MkdirAll => MkdirAll 18 | // os.RemoveAll => RemoveAll 19 | // path/filepath.Walk => Walk 20 | // 21 | // All VFS implementations are thread safe, so multiple readers and writers might 22 | // operate on them at any time. 23 | package vfs 24 | -------------------------------------------------------------------------------- /open_test.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | ) 7 | 8 | func testOpenedVFS(t *testing.T, fs VFS) { 9 | data1, err := ReadFile(fs, "a/b/c/d") 10 | if err != nil { 11 | t.Fatal(err) 12 | } 13 | if string(data1) != "go" { 14 | t.Errorf("expecting a/b/c/d to contain \"go\", it contains %q instead", string(data1)) 15 | } 16 | data2, err := ReadFile(fs, "empty") 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | if len(data2) > 0 { 21 | t.Error("non-empty empty file") 22 | } 23 | } 24 | 25 | func testOpenFilename(t *testing.T, filename string) { 26 | p := filepath.Join("testdata", filename) 27 | fs, err := Open(p) 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | testOpenedVFS(t, fs) 32 | } 33 | 34 | func TestOpenZip(t *testing.T) { 35 | testOpenFilename(t, "fs.zip") 36 | } 37 | 38 | func TestOpenTar(t *testing.T) { 39 | testOpenFilename(t, "fs.tar") 40 | } 41 | 42 | func TestOpenTarGzip(t *testing.T) { 43 | testOpenFilename(t, "fs.tar.gz") 44 | } 45 | 46 | func TestOpenTarBzip2(t *testing.T) { 47 | testOpenFilename(t, "fs.tar.bz2") 48 | } 49 | -------------------------------------------------------------------------------- /map.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "path" 5 | "sort" 6 | ) 7 | 8 | // Map returns an in-memory file system using the given files argument to 9 | // populate it (which might be nil). Note that the files map does 10 | // not need to contain any directories, they will be created automatically. 11 | // If the files contain conflicting paths (e.g. files named a and a/b, thus 12 | // making "a" both a file and a directory), an error will be returned. 13 | func Map(files map[string]*File) (VFS, error) { 14 | fs := newMemory() 15 | keys := make([]string, 0, len(files)) 16 | for k := range files { 17 | keys = append(keys, k) 18 | } 19 | sort.Strings(keys) 20 | var dir *Dir 21 | var prevDir *Dir 22 | var prevDirPath string 23 | for _, k := range keys { 24 | file := files[k] 25 | if file.Mode == 0 { 26 | file.Mode = 0644 27 | } 28 | fileDir, fileBase := path.Split(k) 29 | if prevDir != nil && fileDir == prevDirPath { 30 | dir = prevDir 31 | } else { 32 | if err := MkdirAll(fs, fileDir, 0755); err != nil { 33 | return nil, err 34 | } 35 | var err error 36 | dir, err = fs.dirEntry(fileDir, true) 37 | if err != nil { 38 | return nil, err 39 | } 40 | prevDir = dir 41 | prevDirPath = fileDir 42 | } 43 | if err := dir.Add(fileBase, file); err != nil { 44 | return nil, err 45 | } 46 | } 47 | return fs, nil 48 | } 49 | -------------------------------------------------------------------------------- /rewriter.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | type rewriterFileSystem struct { 9 | fs VFS 10 | rewriter func(string) string 11 | } 12 | 13 | func (fs *rewriterFileSystem) VFS() VFS { 14 | return fs.fs 15 | } 16 | 17 | func (fs *rewriterFileSystem) Open(path string) (RFile, error) { 18 | return fs.fs.Open(fs.rewriter(path)) 19 | } 20 | 21 | func (fs *rewriterFileSystem) OpenFile(path string, flag int, perm os.FileMode) (WFile, error) { 22 | return fs.fs.OpenFile(fs.rewriter(path), flag, perm) 23 | } 24 | 25 | func (fs *rewriterFileSystem) Lstat(path string) (os.FileInfo, error) { 26 | return fs.fs.Lstat(fs.rewriter(path)) 27 | } 28 | 29 | func (fs *rewriterFileSystem) Stat(path string) (os.FileInfo, error) { 30 | return fs.fs.Stat(fs.rewriter(path)) 31 | } 32 | 33 | func (fs *rewriterFileSystem) ReadDir(path string) ([]os.FileInfo, error) { 34 | return fs.fs.ReadDir(fs.rewriter(path)) 35 | } 36 | 37 | func (fs *rewriterFileSystem) Mkdir(path string, perm os.FileMode) error { 38 | return fs.fs.Mkdir(fs.rewriter(path), perm) 39 | } 40 | 41 | func (fs *rewriterFileSystem) Remove(path string) error { 42 | return fs.fs.Remove(fs.rewriter(path)) 43 | } 44 | 45 | func (fs *rewriterFileSystem) String() string { 46 | return fmt.Sprintf("Rewriter %s", fs.fs.String()) 47 | } 48 | 49 | // Rewriter returns a file system which uses the provided function 50 | // to rewrite paths. 51 | func Rewriter(fs VFS, rewriter func(oldPath string) (newPath string)) VFS { 52 | if rewriter == nil { 53 | return fs 54 | } 55 | return &rewriterFileSystem{fs: fs, rewriter: rewriter} 56 | } 57 | -------------------------------------------------------------------------------- /ro.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | ) 8 | 9 | var ( 10 | // ErrReadOnlyFileSystem is the error returned by read only file systems 11 | // from calls which would result in a write operation. 12 | ErrReadOnlyFileSystem = errors.New("read-only filesystem") 13 | ) 14 | 15 | type readOnlyFileSystem struct { 16 | fs VFS 17 | } 18 | 19 | func (fs *readOnlyFileSystem) VFS() VFS { 20 | return fs.fs 21 | } 22 | 23 | func (fs *readOnlyFileSystem) Open(path string) (RFile, error) { 24 | return fs.fs.Open(path) 25 | } 26 | 27 | func (fs *readOnlyFileSystem) OpenFile(path string, flag int, perm os.FileMode) (WFile, error) { 28 | if flag&(os.O_CREATE|os.O_WRONLY|os.O_RDWR) != 0 { 29 | return nil, ErrReadOnlyFileSystem 30 | } 31 | return fs.fs.OpenFile(path, flag, perm) 32 | } 33 | 34 | func (fs *readOnlyFileSystem) Lstat(path string) (os.FileInfo, error) { 35 | return fs.fs.Lstat(path) 36 | } 37 | 38 | func (fs *readOnlyFileSystem) Stat(path string) (os.FileInfo, error) { 39 | return fs.fs.Stat(path) 40 | } 41 | 42 | func (fs *readOnlyFileSystem) ReadDir(path string) ([]os.FileInfo, error) { 43 | return fs.fs.ReadDir(path) 44 | } 45 | 46 | func (fs *readOnlyFileSystem) Mkdir(path string, perm os.FileMode) error { 47 | return ErrReadOnlyFileSystem 48 | } 49 | 50 | func (fs *readOnlyFileSystem) Remove(path string) error { 51 | return ErrReadOnlyFileSystem 52 | } 53 | 54 | func (fs *readOnlyFileSystem) String() string { 55 | return fmt.Sprintf("RO %s", fs.fs.String()) 56 | } 57 | 58 | // ReadOnly returns a read-only filesystem wrapping the given fs. 59 | func ReadOnly(fs VFS) VFS { 60 | return &readOnlyFileSystem{fs: fs} 61 | } 62 | -------------------------------------------------------------------------------- /chroot.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | ) 8 | 9 | type chrootFileSystem struct { 10 | root string 11 | fs VFS 12 | } 13 | 14 | func (fs *chrootFileSystem) path(p string) string { 15 | // root always ends with /, if there are double 16 | // slashes they will be fixed by the underlying 17 | // VFS 18 | return fs.root + p 19 | } 20 | 21 | func (fs *chrootFileSystem) VFS() VFS { 22 | return fs.fs 23 | } 24 | 25 | func (fs *chrootFileSystem) Open(path string) (RFile, error) { 26 | return fs.fs.Open(fs.path(path)) 27 | } 28 | 29 | func (fs *chrootFileSystem) OpenFile(path string, flag int, perm os.FileMode) (WFile, error) { 30 | return fs.fs.OpenFile(fs.path(path), flag, perm) 31 | } 32 | 33 | func (fs *chrootFileSystem) Lstat(path string) (os.FileInfo, error) { 34 | return fs.fs.Lstat(fs.path(path)) 35 | } 36 | 37 | func (fs *chrootFileSystem) Stat(path string) (os.FileInfo, error) { 38 | return fs.fs.Stat(fs.path(path)) 39 | } 40 | 41 | func (fs *chrootFileSystem) ReadDir(path string) ([]os.FileInfo, error) { 42 | return fs.fs.ReadDir(fs.path(path)) 43 | } 44 | 45 | func (fs *chrootFileSystem) Mkdir(path string, perm os.FileMode) error { 46 | return fs.fs.Mkdir(fs.path(path), perm) 47 | } 48 | 49 | func (fs *chrootFileSystem) Remove(path string) error { 50 | return fs.fs.Remove(fs.path(path)) 51 | } 52 | 53 | func (fs *chrootFileSystem) String() string { 54 | return fmt.Sprintf("Chroot %s %s", fs.root, fs.fs.String()) 55 | } 56 | 57 | // Chroot returns a new VFS wrapping the given VFS, making the given 58 | // directory the new root ("/"). Note that root must be an existing 59 | // directory in the given file system, otherwise an error is returned. 60 | func Chroot(root string, fs VFS) (VFS, error) { 61 | root = path.Clean("/" + root) 62 | st, err := fs.Stat(root) 63 | if err != nil { 64 | return nil, err 65 | } 66 | if !st.IsDir() { 67 | return nil, fmt.Errorf("%s is not a directory", root) 68 | } 69 | return &chrootFileSystem{root: root + "/", fs: fs}, nil 70 | } 71 | -------------------------------------------------------------------------------- /write.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "archive/tar" 5 | "archive/zip" 6 | "compress/gzip" 7 | "io" 8 | "os" 9 | ) 10 | 11 | func copyVFS(fs VFS, copier func(p string, info os.FileInfo, f io.Reader) error) error { 12 | return Walk(fs, "/", func(vfs VFS, p string, info os.FileInfo, err error) error { 13 | if err != nil { 14 | return err 15 | } 16 | if info.IsDir() { 17 | return nil 18 | } 19 | f, err := fs.Open(p) 20 | if err != nil { 21 | return err 22 | } 23 | defer f.Close() 24 | return copier(p[1:], info, f) 25 | }) 26 | } 27 | 28 | // WriteZip writes the given VFS as a zip file to the given io.Writer. 29 | func WriteZip(w io.Writer, fs VFS) error { 30 | zw := zip.NewWriter(w) 31 | err := copyVFS(fs, func(p string, info os.FileInfo, f io.Reader) error { 32 | hdr, err := zip.FileInfoHeader(info) 33 | if err != nil { 34 | return err 35 | } 36 | hdr.Name = p 37 | fw, err := zw.CreateHeader(hdr) 38 | if err != nil { 39 | return err 40 | } 41 | _, err = io.Copy(fw, f) 42 | return err 43 | }) 44 | if err != nil { 45 | return err 46 | } 47 | return zw.Close() 48 | } 49 | 50 | // WriteTar writes the given VFS as a tar file to the given io.Writer. 51 | func WriteTar(w io.Writer, fs VFS) error { 52 | tw := tar.NewWriter(w) 53 | err := copyVFS(fs, func(p string, info os.FileInfo, f io.Reader) error { 54 | hdr, err := tar.FileInfoHeader(info, "") 55 | if err != nil { 56 | return err 57 | } 58 | hdr.Name = p 59 | if err := tw.WriteHeader(hdr); err != nil { 60 | return err 61 | } 62 | _, err = io.Copy(tw, f) 63 | return err 64 | }) 65 | if err != nil { 66 | return err 67 | } 68 | return tw.Close() 69 | } 70 | 71 | // WriteTarGzip writes the given VFS as a tar.gz file to the given io.Writer. 72 | func WriteTarGzip(w io.Writer, fs VFS) error { 73 | gw, err := gzip.NewWriterLevel(w, gzip.BestCompression) 74 | if err != nil { 75 | return err 76 | } 77 | if err := WriteTar(gw, fs); err != nil { 78 | return err 79 | } 80 | return gw.Close() 81 | } 82 | -------------------------------------------------------------------------------- /vfs.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | // Opener is the interface which specifies the methods for 9 | // opening a file. All the VFS implementations implement 10 | // this interface. 11 | type Opener interface { 12 | // Open returns a readable file at the given path. See also 13 | // the shorthand function ReadFile. 14 | Open(path string) (RFile, error) 15 | // OpenFile returns a readable and writable file at the given 16 | // path. Note that, depending on the flags, the file might be 17 | // only readable or only writable. See also the shorthand 18 | // function WriteFile. 19 | OpenFile(path string, flag int, perm os.FileMode) (WFile, error) 20 | } 21 | 22 | // RFile is the interface implemented by the returned value from a VFS 23 | // Open method. It allows reading and seeking, and must be closed after use. 24 | type RFile interface { 25 | io.Reader 26 | io.Seeker 27 | io.Closer 28 | } 29 | 30 | // WFile is the interface implemented by the returned value from a VFS 31 | // OpenFile method. It allows reading, seeking and writing, and must 32 | // be closed after use. Note that, depending on the flags passed to 33 | // OpenFile, the Read or Write methods might always return an error (e.g. 34 | // if the file was opened in read-only or write-only mode). 35 | type WFile interface { 36 | io.Reader 37 | io.Writer 38 | io.Seeker 39 | io.Closer 40 | } 41 | 42 | // VFS is the interface implemented by all the Virtual File Systems. 43 | type VFS interface { 44 | Opener 45 | // Lstat returns the os.FileInfo for the given path, without 46 | // following symlinks. 47 | Lstat(path string) (os.FileInfo, error) 48 | // Stat returns the os.FileInfo for the given path, following 49 | // symlinks. 50 | Stat(path string) (os.FileInfo, error) 51 | // ReadDir returns the contents of the directory at path as an slice 52 | // of os.FileInfo, ordered alphabetically by name. If path is not a 53 | // directory or the permissions don't allow it, an error will be 54 | // returned. 55 | ReadDir(path string) ([]os.FileInfo, error) 56 | // Mkdir creates a directory at the given path. If the directory 57 | // already exists or its parent directory does not exist or 58 | // the permissions don't allow it, an error will be returned. See 59 | // also the shorthand function MkdirAll. 60 | Mkdir(path string, perm os.FileMode) error 61 | // Remove removes the item at the given path. If the path does 62 | // not exists or the permissions don't allow removing it or it's 63 | // a non-empty directory, an error will be returned. See also 64 | // the shorthand function RemoveAll. 65 | Remove(path string) error 66 | // String returns a human-readable description of the VFS. 67 | String() string 68 | } 69 | 70 | // TemporaryVFS represents a temporary on-disk file system which can be removed 71 | // by calling its Close method. 72 | type TemporaryVFS interface { 73 | VFS 74 | // Root returns the root directory for the temporary VFS. 75 | Root() string 76 | // Close removes all the files in temporary VFS. 77 | Close() error 78 | } 79 | 80 | // Container is implemented by some file systems which 81 | // contain another one. 82 | type Container interface { 83 | // VFS returns the underlying VFS. 84 | VFS() VFS 85 | } 86 | -------------------------------------------------------------------------------- /open.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "archive/tar" 5 | "archive/zip" 6 | "bytes" 7 | "compress/bzip2" 8 | "compress/gzip" 9 | "fmt" 10 | "io" 11 | "io/ioutil" 12 | "os" 13 | "path/filepath" 14 | "strings" 15 | ) 16 | 17 | // Zip returns an in-memory VFS initialized with the 18 | // contents of the .zip file read from the given io.Reader. 19 | // Since archive/zip requires an io.ReaderAt rather than an 20 | // io.Reader, and a known size, Zip will read the whole file 21 | // into memory and provide its own buffering if r does not 22 | // implement io.ReaderAt or size is <= 0. 23 | func Zip(r io.Reader, size int64) (VFS, error) { 24 | rat, _ := r.(io.ReaderAt) 25 | if rat == nil || size <= 0 { 26 | data, err := ioutil.ReadAll(r) 27 | if err != nil { 28 | return nil, err 29 | } 30 | rat = bytes.NewReader(data) 31 | size = int64(len(data)) 32 | } 33 | zr, err := zip.NewReader(rat, size) 34 | if err != nil { 35 | return nil, err 36 | } 37 | files := make(map[string]*File) 38 | for _, file := range zr.File { 39 | if file.Mode().IsDir() { 40 | continue 41 | } 42 | f, err := file.Open() 43 | if err != nil { 44 | return nil, err 45 | } 46 | data, err := ioutil.ReadAll(f) 47 | f.Close() 48 | if err != nil { 49 | return nil, err 50 | } 51 | files[file.Name] = &File{ 52 | Data: data, 53 | Mode: file.Mode(), 54 | ModTime: file.ModTime(), 55 | } 56 | } 57 | return Map(files) 58 | } 59 | 60 | // Tar returns an in-memory VFS initialized with the 61 | // contents of the .tar file read from the given io.Reader. 62 | func Tar(r io.Reader) (VFS, error) { 63 | files := make(map[string]*File) 64 | tr := tar.NewReader(r) 65 | for { 66 | hdr, err := tr.Next() 67 | if err != nil { 68 | if err == io.EOF { 69 | break 70 | } 71 | return nil, err 72 | } 73 | if hdr.FileInfo().IsDir() { 74 | continue 75 | } 76 | data, err := ioutil.ReadAll(tr) 77 | if err != nil { 78 | return nil, err 79 | } 80 | files[hdr.Name] = &File{ 81 | Data: data, 82 | Mode: hdr.FileInfo().Mode(), 83 | ModTime: hdr.ModTime, 84 | } 85 | } 86 | return Map(files) 87 | } 88 | 89 | // TarGzip returns an in-memory VFS initialized with the 90 | // contents of the .tar.gz file read from the given io.Reader. 91 | func TarGzip(r io.Reader) (VFS, error) { 92 | zr, err := gzip.NewReader(r) 93 | if err != nil { 94 | return nil, err 95 | } 96 | defer zr.Close() 97 | return Tar(zr) 98 | } 99 | 100 | // TarBzip2 returns an in-memory VFS initialized with the 101 | // contents of then .tar.bz2 file read from the given io.Reader. 102 | func TarBzip2(r io.Reader) (VFS, error) { 103 | bzr := bzip2.NewReader(r) 104 | return Tar(bzr) 105 | } 106 | 107 | // Open returns an in-memory VFS initialized with the contents 108 | // of the given filename, which must have one of the following 109 | // extensions: 110 | // 111 | // - .zip 112 | // - .tar 113 | // - .tar.gz 114 | // - .tar.bz2 115 | func Open(filename string) (VFS, error) { 116 | f, err := os.Open(filename) 117 | if err != nil { 118 | return nil, err 119 | } 120 | defer f.Close() 121 | base := filepath.Base(filename) 122 | ext := strings.ToLower(filepath.Ext(base)) 123 | nonExt := filename[:len(filename)-len(ext)] 124 | if strings.ToLower(filepath.Ext(nonExt)) == ".tar" { 125 | ext = ".tar" + ext 126 | } 127 | switch ext { 128 | case ".zip": 129 | st, err := f.Stat() 130 | if err != nil { 131 | return nil, err 132 | } 133 | return Zip(f, st.Size()) 134 | case ".tar": 135 | return Tar(f) 136 | case ".tar.gz": 137 | return TarGzip(f) 138 | case ".tar.bz2": 139 | return TarBzip2(f) 140 | } 141 | return nil, fmt.Errorf("can't open a VFS from a %s file", ext) 142 | } 143 | -------------------------------------------------------------------------------- /file_util.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "bytes" 5 | "compress/zlib" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "os" 10 | "runtime" 11 | "time" 12 | ) 13 | 14 | var ( 15 | errFileClosed = errors.New("file is closed") 16 | ) 17 | 18 | // NewRFile returns a RFile from a *File. 19 | func NewRFile(f *File) (RFile, error) { 20 | data, err := fileData(f) 21 | if err != nil { 22 | return nil, err 23 | } 24 | return &file{f: f, data: data, readable: true}, nil 25 | } 26 | 27 | // NewWFile returns a WFile from a *File. 28 | func NewWFile(f *File, read bool, write bool) (WFile, error) { 29 | data, err := fileData(f) 30 | if err != nil { 31 | return nil, err 32 | } 33 | w := &file{f: f, data: data, readable: read, writable: write} 34 | runtime.SetFinalizer(w, closeFile) 35 | return w, nil 36 | } 37 | 38 | func closeFile(f *file) { 39 | f.Close() 40 | } 41 | 42 | func fileData(f *File) ([]byte, error) { 43 | if len(f.Data) == 0 || f.Mode&ModeCompress == 0 { 44 | return f.Data, nil 45 | } 46 | zr, err := zlib.NewReader(bytes.NewReader(f.Data)) 47 | if err != nil { 48 | return nil, err 49 | } 50 | defer zr.Close() 51 | var out bytes.Buffer 52 | if _, err := io.Copy(&out, zr); err != nil { 53 | return nil, err 54 | } 55 | return out.Bytes(), nil 56 | } 57 | 58 | type file struct { 59 | f *File 60 | data []byte 61 | offset int 62 | readable bool 63 | writable bool 64 | closed bool 65 | } 66 | 67 | func (f *file) Read(p []byte) (int, error) { 68 | if !f.readable { 69 | return 0, ErrWriteOnly 70 | } 71 | f.f.RLock() 72 | defer f.f.RUnlock() 73 | if f.closed { 74 | return 0, errFileClosed 75 | } 76 | if f.offset > len(f.data) { 77 | return 0, io.EOF 78 | } 79 | n := copy(p, f.data[f.offset:]) 80 | f.offset += n 81 | if n < len(p) { 82 | return n, io.EOF 83 | } 84 | return n, nil 85 | } 86 | 87 | func (f *file) Seek(offset int64, whence int) (int64, error) { 88 | f.f.Lock() 89 | defer f.f.Unlock() 90 | if f.closed { 91 | return 0, errFileClosed 92 | } 93 | switch whence { 94 | case os.SEEK_SET: 95 | f.offset = int(offset) 96 | case os.SEEK_CUR: 97 | f.offset += int(offset) 98 | case os.SEEK_END: 99 | f.offset = len(f.data) + int(offset) 100 | default: 101 | panic(fmt.Errorf("Seek: invalid whence %d", whence)) 102 | } 103 | if f.offset > len(f.data) { 104 | f.offset = len(f.data) 105 | } else if f.offset < 0 { 106 | f.offset = 0 107 | } 108 | return int64(f.offset), nil 109 | } 110 | 111 | func (f *file) Write(p []byte) (int, error) { 112 | if !f.writable { 113 | return 0, ErrReadOnly 114 | } 115 | f.f.Lock() 116 | defer f.f.Unlock() 117 | if f.closed { 118 | return 0, errFileClosed 119 | } 120 | count := len(p) 121 | n := copy(f.data[f.offset:], p) 122 | if n < count { 123 | f.data = append(f.data, p[n:]...) 124 | } 125 | f.offset += count 126 | f.f.ModTime = time.Now() 127 | return count, nil 128 | } 129 | 130 | func (f *file) Close() error { 131 | if !f.closed { 132 | f.f.Lock() 133 | defer f.f.Unlock() 134 | if !f.closed { 135 | if f.f.Mode&ModeCompress != 0 { 136 | var buf bytes.Buffer 137 | zw := zlib.NewWriter(&buf) 138 | if _, err := zw.Write(f.data); err != nil { 139 | return err 140 | } 141 | if err := zw.Close(); err != nil { 142 | return err 143 | } 144 | if buf.Len() < len(f.data) { 145 | f.f.Data = buf.Bytes() 146 | } else { 147 | f.f.Mode &= ^ModeCompress 148 | f.f.Data = f.data 149 | } 150 | } else { 151 | f.f.Data = f.data 152 | } 153 | f.closed = true 154 | } 155 | } 156 | return nil 157 | } 158 | 159 | func (f *file) IsCompressed() bool { 160 | return f.f.Mode&ModeCompress != 0 161 | } 162 | 163 | func (f *file) SetCompressed(c bool) { 164 | f.f.Lock() 165 | defer f.f.Unlock() 166 | if c { 167 | f.f.Mode |= ModeCompress 168 | } else { 169 | f.f.Mode &= ^ModeCompress 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /fs.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "path/filepath" 9 | ) 10 | 11 | // IMPORTANT: Note about wrapping os. functions: os.Open, os.OpenFile etc... will return a non-nil 12 | // interface pointing to a nil instance in case of error (whoever decided this disctintion in Go 13 | // was a good idea deservers to be hung by his thumbs). This is highly undesirable, since users 14 | // can't rely on checking f != nil to know if a correct handle was returned. That's why the 15 | // methods in fileSystem do the error checking themselves and return a true nil in case of error. 16 | 17 | type fileSystem struct { 18 | root string 19 | temporary bool 20 | } 21 | 22 | func (fs *fileSystem) path(name string) string { 23 | name = path.Clean("/" + name) 24 | return filepath.Join(fs.root, filepath.FromSlash(name)) 25 | } 26 | 27 | // Root returns the root directory of the fileSystem, as an 28 | // absolute path native to the current operating system. 29 | func (fs *fileSystem) Root() string { 30 | return fs.root 31 | } 32 | 33 | // IsTemporary returns wheter the fileSystem is temporary. 34 | func (fs *fileSystem) IsTemporary() bool { 35 | return fs.temporary 36 | } 37 | 38 | func (fs *fileSystem) Open(path string) (RFile, error) { 39 | f, err := os.Open(fs.path(path)) 40 | if err != nil { 41 | return nil, err 42 | } 43 | return f, nil 44 | } 45 | 46 | func (fs *fileSystem) OpenFile(path string, flag int, mode os.FileMode) (WFile, error) { 47 | f, err := os.OpenFile(fs.path(path), flag, mode) 48 | if err != nil { 49 | return nil, err 50 | } 51 | return f, nil 52 | } 53 | 54 | func (fs *fileSystem) Lstat(path string) (os.FileInfo, error) { 55 | info, err := os.Lstat(fs.path(path)) 56 | if err != nil { 57 | return nil, err 58 | } 59 | return info, nil 60 | } 61 | 62 | func (fs *fileSystem) Stat(path string) (os.FileInfo, error) { 63 | info, err := os.Stat(fs.path(path)) 64 | if err != nil { 65 | return nil, err 66 | } 67 | return info, nil 68 | } 69 | 70 | func (fs *fileSystem) ReadDir(path string) ([]os.FileInfo, error) { 71 | files, err := ioutil.ReadDir(fs.path(path)) 72 | if err != nil { 73 | return nil, err 74 | } 75 | return files, nil 76 | } 77 | 78 | func (fs *fileSystem) Mkdir(path string, perm os.FileMode) error { 79 | return os.Mkdir(fs.path(path), perm) 80 | } 81 | 82 | func (fs *fileSystem) Remove(path string) error { 83 | return os.Remove(fs.path(path)) 84 | } 85 | 86 | func (fs *fileSystem) String() string { 87 | return fmt.Sprintf("fileSystem: %s", fs.root) 88 | } 89 | 90 | // Close is a no-op on non-temporary filesystems. On temporary 91 | // ones (as returned by TmpFS), it removes all the temporary files. 92 | func (f *fileSystem) Close() error { 93 | if f.temporary { 94 | return os.RemoveAll(f.root) 95 | } 96 | return nil 97 | } 98 | 99 | func newFS(root string) (*fileSystem, error) { 100 | abs, err := filepath.Abs(root) 101 | if err != nil { 102 | return nil, err 103 | } 104 | return &fileSystem{root: abs}, nil 105 | } 106 | 107 | // FS returns a VFS at the given path, which must be provided 108 | // as native path of the current operating system. The path might be 109 | // either absolute or relative, but the fileSystem will be anchored 110 | // at the absolute path represented by root at the time of the function 111 | // call. 112 | func FS(root string) (VFS, error) { 113 | return newFS(root) 114 | } 115 | 116 | // TmpFS returns a temporary file system with the given prefix and its root 117 | // directory name, which might be empty. The temporary file system is created 118 | // in the default temporary directory for the operating system. Once you're 119 | // done with the temporary filesystem, you might can all its files by calling 120 | // its Close method. 121 | func TmpFS(prefix string) (TemporaryVFS, error) { 122 | dir, err := ioutil.TempDir("", prefix) 123 | if err != nil { 124 | return nil, err 125 | } 126 | fs, err := newFS(dir) 127 | if err != nil { 128 | return nil, err 129 | } 130 | fs.temporary = true 131 | return fs, nil 132 | } 133 | -------------------------------------------------------------------------------- /mounter.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path" 7 | "strings" 8 | ) 9 | 10 | const ( 11 | separator = "/" 12 | ) 13 | 14 | func hasSubdir(root, dir string) (string, bool) { 15 | root = path.Clean(root) 16 | if !strings.HasSuffix(root, separator) { 17 | root += separator 18 | } 19 | dir = path.Clean(dir) 20 | if !strings.HasPrefix(dir, root) { 21 | return "", false 22 | } 23 | return dir[len(root):], true 24 | } 25 | 26 | type mountPoint struct { 27 | point string 28 | fs VFS 29 | } 30 | 31 | func (m *mountPoint) String() string { 32 | return fmt.Sprintf("%s at %s", m.fs, m.point) 33 | } 34 | 35 | // Mounter implements the VFS interface and allows mounting different virtual 36 | // file systems at arbitraty points, working much like a UNIX filesystem. 37 | // Note that the first mounted filesystem must be always at "/". 38 | type Mounter struct { 39 | points []*mountPoint 40 | } 41 | 42 | func (m *Mounter) fs(p string) (VFS, string, error) { 43 | for ii := len(m.points) - 1; ii >= 0; ii-- { 44 | if rel, ok := hasSubdir(m.points[ii].point, p); ok { 45 | return m.points[ii].fs, rel, nil 46 | } 47 | } 48 | return nil, "", os.ErrNotExist 49 | } 50 | 51 | // Mount mounts the given filesystem at the given mount point. Unless the 52 | // mount point is /, it must be an already existing directory. 53 | func (m *Mounter) Mount(fs VFS, point string) error { 54 | point = path.Clean(point) 55 | if point == "." || point == "" { 56 | point = "/" 57 | } 58 | if point == "/" { 59 | if len(m.points) > 0 { 60 | return fmt.Errorf("%s is already mounted at /", m.points[0]) 61 | } 62 | m.points = append(m.points, &mountPoint{point, fs}) 63 | return nil 64 | } 65 | stat, err := m.Stat(point) 66 | if err != nil { 67 | return err 68 | } 69 | if !stat.IsDir() { 70 | return fmt.Errorf("%s is not a directory", point) 71 | } 72 | m.points = append(m.points, &mountPoint{point, fs}) 73 | return nil 74 | } 75 | 76 | // Umount umounts the filesystem from the given mount point. If there are other filesystems 77 | // mounted below it or there's no filesystem mounted at that point, an error is returned. 78 | func (m *Mounter) Umount(point string) error { 79 | point = path.Clean(point) 80 | for ii, v := range m.points { 81 | if v.point == point { 82 | // Check if we have mount points below this one 83 | for _, vv := range m.points[ii:] { 84 | if _, ok := hasSubdir(v.point, vv.point); ok { 85 | return fmt.Errorf("can't umount %s because %s is mounted below it", point, vv) 86 | } 87 | } 88 | m.points = append(m.points[:ii], m.points[ii+1:]...) 89 | return nil 90 | } 91 | } 92 | return fmt.Errorf("no filesystem mounted at %s", point) 93 | } 94 | 95 | func (m *Mounter) Open(path string) (RFile, error) { 96 | fs, p, err := m.fs(path) 97 | if err != nil { 98 | return nil, err 99 | } 100 | return fs.Open(p) 101 | } 102 | 103 | func (m *Mounter) OpenFile(path string, flag int, perm os.FileMode) (WFile, error) { 104 | fs, p, err := m.fs(path) 105 | if err != nil { 106 | return nil, err 107 | } 108 | return fs.OpenFile(p, flag, perm) 109 | } 110 | 111 | func (m *Mounter) Lstat(path string) (os.FileInfo, error) { 112 | fs, p, err := m.fs(path) 113 | if err != nil { 114 | return nil, err 115 | } 116 | return fs.Lstat(p) 117 | } 118 | 119 | func (m *Mounter) Stat(path string) (os.FileInfo, error) { 120 | fs, p, err := m.fs(path) 121 | if err != nil { 122 | return nil, err 123 | } 124 | return fs.Stat(p) 125 | } 126 | 127 | func (m *Mounter) ReadDir(path string) ([]os.FileInfo, error) { 128 | fs, p, err := m.fs(path) 129 | if err != nil { 130 | return nil, err 131 | } 132 | return fs.ReadDir(p) 133 | } 134 | 135 | func (m *Mounter) Mkdir(path string, perm os.FileMode) error { 136 | fs, p, err := m.fs(path) 137 | if err != nil { 138 | return err 139 | } 140 | return fs.Mkdir(p, perm) 141 | } 142 | 143 | func (m *Mounter) Remove(path string) error { 144 | // TODO: Don't allow removing an empty directory 145 | // with a mount below it. 146 | fs, p, err := m.fs(path) 147 | if err != nil { 148 | return err 149 | } 150 | return fs.Remove(p) 151 | } 152 | 153 | func (m *Mounter) String() string { 154 | s := make([]string, len(m.points)) 155 | for ii, v := range m.points { 156 | s[ii] = v.String() 157 | } 158 | return fmt.Sprintf("Mounter: %s", strings.Join(s, ", ")) 159 | } 160 | 161 | func mounterCompileTimeCheck() VFS { 162 | return &Mounter{} 163 | } 164 | -------------------------------------------------------------------------------- /file.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "os" 5 | "path" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | // EntryType indicates the type of the entry. 11 | type EntryType uint8 12 | 13 | const ( 14 | // EntryTypeFile indicates the entry is a file. 15 | EntryTypeFile EntryType = iota + 1 16 | // EntryTypeDir indicates the entry is a directory. 17 | EntryTypeDir 18 | ) 19 | 20 | const ( 21 | ModeCompress os.FileMode = 1 << 16 22 | ) 23 | 24 | // Entry is the interface implemented by the in-memory representations 25 | // of files and directories. 26 | type Entry interface { 27 | // Type returns the entry type, either EntryTypeFile or 28 | // EntryTypeDir. 29 | Type() EntryType 30 | // Size returns the file size. For directories, it's always zero. 31 | Size() int64 32 | // FileMode returns the file mode as an os.FileMode. 33 | FileMode() os.FileMode 34 | // ModificationTime returns the last time the file or the directory 35 | // was modified. 36 | ModificationTime() time.Time 37 | } 38 | 39 | // Type File represents an in-memory file. Most in-memory VFS implementations 40 | // should use this structure to represent their files, in order to save work. 41 | type File struct { 42 | sync.RWMutex 43 | // Data contains the file data. 44 | Data []byte 45 | // Mode is the file or directory mode. Note that some filesystems 46 | // might ignore the permission bits. 47 | Mode os.FileMode 48 | // ModTime represents the last modification time to the file. 49 | ModTime time.Time 50 | } 51 | 52 | func (f *File) Type() EntryType { 53 | return EntryTypeFile 54 | } 55 | 56 | func (f *File) Size() int64 { 57 | f.RLock() 58 | defer f.RUnlock() 59 | return int64(len(f.Data)) 60 | } 61 | 62 | func (f *File) FileMode() os.FileMode { 63 | return f.Mode 64 | } 65 | 66 | func (f *File) ModificationTime() time.Time { 67 | f.RLock() 68 | defer f.RUnlock() 69 | return f.ModTime 70 | } 71 | 72 | // Type Dir represents an in-memory directory. Most in-memory VFS 73 | // implementations should use this structure to represent their 74 | // directories, in order to save work. 75 | type Dir struct { 76 | sync.RWMutex 77 | // Mode is the file or directory mode. Note that some filesystems 78 | // might ignore the permission bits. 79 | Mode os.FileMode 80 | // ModTime represents the last modification time to directory. 81 | ModTime time.Time 82 | // Entry names in this directory, in order. 83 | EntryNames []string 84 | // Entries in the same order as EntryNames. 85 | Entries []Entry 86 | } 87 | 88 | func (d *Dir) Type() EntryType { 89 | return EntryTypeDir 90 | } 91 | 92 | func (d *Dir) Size() int64 { 93 | return 0 94 | } 95 | 96 | func (d *Dir) FileMode() os.FileMode { 97 | return d.Mode 98 | } 99 | 100 | func (d *Dir) ModificationTime() time.Time { 101 | d.RLock() 102 | defer d.RUnlock() 103 | return d.ModTime 104 | } 105 | 106 | // Add ads a new entry to the directory. If there's already an 107 | // entry ith the same name, an error is returned. 108 | func (d *Dir) Add(name string, entry Entry) error { 109 | // TODO: Binary search 110 | for ii, v := range d.EntryNames { 111 | if v > name { 112 | names := make([]string, len(d.EntryNames)+1) 113 | copy(names, d.EntryNames[:ii]) 114 | names[ii] = name 115 | copy(names[ii+1:], d.EntryNames[ii:]) 116 | d.EntryNames = names 117 | 118 | entries := make([]Entry, len(d.Entries)+1) 119 | copy(entries, d.Entries[:ii]) 120 | entries[ii] = entry 121 | copy(entries[ii+1:], d.Entries[ii:]) 122 | 123 | d.Entries = entries 124 | return nil 125 | } 126 | if v == name { 127 | return os.ErrExist 128 | } 129 | } 130 | // Not added yet, put at the end 131 | d.EntryNames = append(d.EntryNames, name) 132 | d.Entries = append(d.Entries, entry) 133 | return nil 134 | } 135 | 136 | // Find returns the entry with the given name and its index, 137 | // or an error if an entry with that name does not exist in 138 | // the directory. 139 | func (d *Dir) Find(name string) (Entry, int, error) { 140 | for ii, v := range d.EntryNames { 141 | if v == name { 142 | return d.Entries[ii], ii, nil 143 | } 144 | } 145 | return nil, -1, os.ErrNotExist 146 | } 147 | 148 | // EntryInfo implements the os.FileInfo interface wrapping 149 | // a given File and its Path in its VFS. 150 | type EntryInfo struct { 151 | // Path is the full path to the entry in its VFS. 152 | Path string 153 | // Entry is the instance used by the VFS to represent 154 | // the in-memory entry. 155 | Entry Entry 156 | } 157 | 158 | func (info *EntryInfo) Name() string { 159 | return path.Base(info.Path) 160 | } 161 | 162 | func (info *EntryInfo) Size() int64 { 163 | return info.Entry.Size() 164 | } 165 | 166 | func (info *EntryInfo) Mode() os.FileMode { 167 | return info.Entry.FileMode() 168 | } 169 | 170 | func (info *EntryInfo) ModTime() time.Time { 171 | return info.Entry.ModificationTime() 172 | } 173 | 174 | func (info *EntryInfo) IsDir() bool { 175 | return info.Entry.Type() == EntryTypeDir 176 | } 177 | 178 | // Sys returns the underlying Entry. 179 | func (info *EntryInfo) Sys() interface{} { 180 | return info.Entry 181 | } 182 | 183 | // FileInfos represents an slice of os.FileInfo which 184 | // implements the sort.Interface. This type is only 185 | // exported for users who want to implement their own 186 | // filesystems, since VFS.ReadDir requires the returned 187 | // []os.FileInfo to be sorted by name. 188 | type FileInfos []os.FileInfo 189 | 190 | func (f FileInfos) Len() int { 191 | return len(f) 192 | } 193 | 194 | func (f FileInfos) Less(i, j int) bool { 195 | return f[i].Name() < f[j].Name() 196 | } 197 | 198 | func (f FileInfos) Swap(i, j int) { 199 | f[i], f[j] = f[j], f[i] 200 | } 201 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | pathpkg "path" 9 | "strings" 10 | ) 11 | 12 | var ( 13 | // SkipDir is used by a WalkFunc to signal Walk that 14 | // it wans to skip the given directory. 15 | SkipDir = errors.New("skip this directory") 16 | // ErrReadOnly is returned from Write() on a read-only file. 17 | ErrReadOnly = errors.New("can't write to read only file") 18 | // ErrWriteOnly is returned from Read() on a write-only file. 19 | ErrWriteOnly = errors.New("can't read from write only file") 20 | ) 21 | 22 | // WalkFunc is the function type used by Walk to iterate over a VFS. 23 | type WalkFunc func(fs VFS, path string, info os.FileInfo, err error) error 24 | 25 | func walk(fs VFS, p string, info os.FileInfo, fn WalkFunc) error { 26 | err := fn(fs, p, info, nil) 27 | if err != nil { 28 | if info.IsDir() && err == SkipDir { 29 | err = nil 30 | } 31 | return err 32 | } 33 | if !info.IsDir() { 34 | return nil 35 | } 36 | infos, err := fs.ReadDir(p) 37 | if err != nil { 38 | return fn(fs, p, info, err) 39 | } 40 | for _, v := range infos { 41 | name := pathpkg.Join(p, v.Name()) 42 | fileInfo, err := fs.Lstat(name) 43 | if err != nil { 44 | if err := fn(fs, name, fileInfo, err); err != nil && err != SkipDir { 45 | return err 46 | } 47 | continue 48 | } 49 | if err := walk(fs, name, fileInfo, fn); err != nil && (!fileInfo.IsDir() || err != SkipDir) { 50 | return err 51 | } 52 | } 53 | return nil 54 | } 55 | 56 | // Walk iterates over all the files in the VFS which descend from the given 57 | // root (including root itself), descending into any subdirectories. In each 58 | // directory, files are visited in alphabetical order. The given function might 59 | // chose to skip a directory by returning SkipDir. 60 | func Walk(fs VFS, root string, fn WalkFunc) error { 61 | info, err := fs.Lstat(root) 62 | if err != nil { 63 | return fn(fs, root, nil, err) 64 | } 65 | return walk(fs, root, info, fn) 66 | } 67 | 68 | func makeDir(fs VFS, path string, perm os.FileMode) error { 69 | stat, err := fs.Lstat(path) 70 | if err == nil { 71 | if !stat.IsDir() { 72 | return fmt.Errorf("%s exists and is not a directory", path) 73 | } 74 | } else { 75 | if err := fs.Mkdir(path, perm); err != nil { 76 | return err 77 | } 78 | } 79 | return nil 80 | } 81 | 82 | // MkdirAll makes all directories pointed by the given path, using the same 83 | // permissions for all of them. Note that MkdirAll skips directories which 84 | // already exists rather than returning an error. 85 | func MkdirAll(fs VFS, path string, perm os.FileMode) error { 86 | cur := "/" 87 | if err := makeDir(fs, cur, perm); err != nil { 88 | return err 89 | } 90 | parts := strings.Split(path, "/") 91 | for _, v := range parts { 92 | cur += v 93 | if err := makeDir(fs, cur, perm); err != nil { 94 | return err 95 | } 96 | cur += "/" 97 | } 98 | return nil 99 | } 100 | 101 | // RemoveAll removes all files from the given fs and path, including 102 | // directories (by removing its contents first). 103 | func RemoveAll(fs VFS, path string) error { 104 | stat, err := fs.Lstat(path) 105 | if err != nil { 106 | if err == os.ErrNotExist { 107 | return nil 108 | } 109 | return err 110 | } 111 | if stat.IsDir() { 112 | files, err := fs.ReadDir(path) 113 | if err != nil { 114 | return err 115 | } 116 | for _, v := range files { 117 | filePath := pathpkg.Join(path, v.Name()) 118 | if err := RemoveAll(fs, filePath); err != nil { 119 | return err 120 | } 121 | } 122 | } 123 | return fs.Remove(path) 124 | } 125 | 126 | // ReadFile reads the file at the given path from the given fs, returning 127 | // either its contents or an error if the file couldn't be read. 128 | func ReadFile(fs VFS, path string) ([]byte, error) { 129 | f, err := fs.Open(path) 130 | if err != nil { 131 | return nil, err 132 | } 133 | defer f.Close() 134 | return ioutil.ReadAll(f) 135 | } 136 | 137 | // WriteFile writes a file at the given path and fs with the given data and 138 | // permissions. If the file already exists, WriteFile truncates it before 139 | // writing. If the file can't be created, an error will be returned. 140 | func WriteFile(fs VFS, path string, data []byte, perm os.FileMode) error { 141 | f, err := fs.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm) 142 | if err != nil { 143 | return err 144 | } 145 | if _, err := f.Write(data); err != nil { 146 | f.Close() 147 | return err 148 | } 149 | return f.Close() 150 | } 151 | 152 | // Clone copies all the files from the src VFS to dst. Note that files or directories with 153 | // all permissions set to 0 will be set to 0755 for directories and 0644 for files. If you 154 | // need more granularity, use Walk directly to clone the file systems. 155 | func Clone(dst VFS, src VFS) error { 156 | err := Walk(src, "/", func(fs VFS, path string, info os.FileInfo, err error) error { 157 | if err != nil { 158 | return err 159 | } 160 | if info.IsDir() { 161 | perm := info.Mode() & os.ModePerm 162 | if perm == 0 { 163 | perm = 0755 164 | } 165 | err := dst.Mkdir(path, info.Mode()|perm) 166 | if err != nil && !IsExist(err) { 167 | return err 168 | } 169 | return nil 170 | } 171 | data, err := ReadFile(fs, path) 172 | if err != nil { 173 | return err 174 | } 175 | perm := info.Mode() & os.ModePerm 176 | if perm == 0 { 177 | perm = 0644 178 | } 179 | if err := WriteFile(dst, path, data, info.Mode()|perm); err != nil { 180 | return err 181 | } 182 | return nil 183 | }) 184 | return err 185 | } 186 | 187 | // IsExist returns wheter the error indicates that the file or directory 188 | // already exists. 189 | func IsExist(err error) bool { 190 | return os.IsExist(err) 191 | } 192 | 193 | // IsExist returns wheter the error indicates that the file or directory 194 | // does not exist. 195 | func IsNotExist(err error) bool { 196 | return os.IsNotExist(err) 197 | } 198 | 199 | // Compressor is the interface implemented by VFS files which can be 200 | // transparently compressed and decompressed. Currently, this is only 201 | // supported by the in-memory filesystems. 202 | type Compressor interface { 203 | IsCompressed() bool 204 | SetCompressed(c bool) 205 | } 206 | 207 | // Compress is a shorthand method for compressing all the files in a VFS. 208 | // Note that not all file systems support transparent compression/decompression. 209 | func Compress(fs VFS) error { 210 | return Walk(fs, "/", func(fs VFS, p string, info os.FileInfo, err error) error { 211 | if err != nil { 212 | return err 213 | } 214 | mode := info.Mode() 215 | if mode.IsDir() || mode&ModeCompress != 0 { 216 | return nil 217 | } 218 | f, err := fs.Open(p) 219 | if err != nil { 220 | return err 221 | } 222 | if c, ok := f.(Compressor); ok { 223 | c.SetCompressed(true) 224 | } 225 | return f.Close() 226 | }) 227 | } 228 | -------------------------------------------------------------------------------- /mem.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | pathpkg "path" 8 | "path/filepath" 9 | "strings" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | var ( 15 | errNoEmptyNameFile = errors.New("can't create file with empty name") 16 | errNoEmptyNameDir = errors.New("can't create directory with empty name") 17 | ) 18 | 19 | type memoryFileSystem struct { 20 | mu sync.RWMutex 21 | root *Dir 22 | } 23 | 24 | // entry must always be called with the lock held 25 | func (fs *memoryFileSystem) entry(path string, followSymlinks bool) (Entry, *Dir, int, error) { 26 | path = cleanPath(path) 27 | if path == "" || path == "/" || path == "." { 28 | return fs.root, nil, 0, nil 29 | } 30 | if path[0] == '/' { 31 | path = path[1:] 32 | } 33 | dir := fs.root 34 | cur := path 35 | for { 36 | p := strings.IndexByte(cur, '/') 37 | name := cur 38 | if p > 0 { 39 | name = cur[:p] 40 | cur = cur[p+1:] 41 | } else { 42 | cur = "" 43 | } 44 | dir.RLock() 45 | entry, pos, err := dir.Find(name) 46 | dir.RUnlock() 47 | if err != nil { 48 | return nil, nil, 0, err 49 | } 50 | if len(cur) == 0 { 51 | // We got the entry. Check if it's a symlink. 52 | if followSymlinks && entry.FileMode()&os.ModeSymlink != 0 { 53 | file := entry.(*File) 54 | newpath := filepath.Join(filepath.Dir(path), string(file.Data)) 55 | return fs.entry(newpath, true) 56 | } 57 | return entry, dir, pos, nil 58 | } 59 | if entry.Type() != EntryTypeDir { 60 | // Check if we found a symlink pointing to a directory 61 | if followSymlinks && entry.FileMode()&os.ModeSymlink != 0 { 62 | file := entry.(*File) 63 | rel := string(file.Data) 64 | from := filepath.Clean(path[:len(path)-len(cur)]) 65 | newdirpath := filepath.Join(filepath.Dir(from), rel) 66 | linkedEntry, _, _, err := fs.entry(newdirpath, true) 67 | if err == nil && linkedEntry.Type() == EntryTypeDir { 68 | dir = linkedEntry.(*Dir) 69 | continue 70 | } 71 | } 72 | break 73 | } 74 | dir = entry.(*Dir) 75 | } 76 | return nil, nil, 0, os.ErrNotExist 77 | } 78 | 79 | func (fs *memoryFileSystem) dirEntry(path string, followSymlinks bool) (*Dir, error) { 80 | entry, _, _, err := fs.entry(path, followSymlinks) 81 | if err != nil { 82 | return nil, err 83 | } 84 | if entry.Type() != EntryTypeDir { 85 | return nil, fmt.Errorf("%s it's not a directory", path) 86 | } 87 | return entry.(*Dir), nil 88 | } 89 | 90 | func (fs *memoryFileSystem) Open(path string) (RFile, error) { 91 | entry, _, _, err := fs.entry(path, true) 92 | if err != nil { 93 | return nil, err 94 | } 95 | if entry.Type() != EntryTypeFile { 96 | return nil, fmt.Errorf("%s is not a file", path) 97 | } 98 | return NewRFile(entry.(*File)) 99 | } 100 | 101 | func (fs *memoryFileSystem) OpenFile(path string, flag int, mode os.FileMode) (WFile, error) { 102 | if mode&os.ModeType != 0 { 103 | return nil, fmt.Errorf("%T does not support special files", fs) 104 | } 105 | path = cleanPath(path) 106 | dir, base := pathpkg.Split(path) 107 | if base == "" { 108 | return nil, errNoEmptyNameFile 109 | } 110 | fs.mu.RLock() 111 | d, err := fs.dirEntry(dir, true) 112 | fs.mu.RUnlock() 113 | if err != nil { 114 | return nil, err 115 | } 116 | 117 | d.Lock() 118 | defer d.Unlock() 119 | f, _, _ := d.Find(base) 120 | if f == nil && flag&os.O_CREATE == 0 { 121 | return nil, os.ErrNotExist 122 | } 123 | // Read only file? 124 | if flag&os.O_WRONLY == 0 && flag&os.O_RDWR == 0 { 125 | if f == nil { 126 | return nil, os.ErrNotExist 127 | } 128 | return NewWFile(f.(*File), true, false) 129 | } 130 | // Write file, either f != nil or flag&os.O_CREATE 131 | if f != nil { 132 | if f.Type() != EntryTypeFile { 133 | return nil, fmt.Errorf("%s is not a file", path) 134 | } 135 | if flag&os.O_EXCL != 0 { 136 | return nil, os.ErrExist 137 | } 138 | // Check if we should truncate 139 | if flag&os.O_TRUNC != 0 { 140 | file := f.(*File) 141 | file.Lock() 142 | file.ModTime = time.Now() 143 | file.Data = nil 144 | file.Unlock() 145 | } 146 | } else { 147 | f = &File{ModTime: time.Now()} 148 | d.Add(base, f) 149 | } 150 | return NewWFile(f.(*File), flag&os.O_RDWR != 0, true) 151 | } 152 | 153 | func (fs *memoryFileSystem) stat(path string, followSymlinks bool) (os.FileInfo, error) { 154 | entry, _, _, err := fs.entry(path, followSymlinks) 155 | if err != nil { 156 | return nil, err 157 | } 158 | return &EntryInfo{Path: path, Entry: entry}, nil 159 | } 160 | 161 | func (fs *memoryFileSystem) Lstat(path string) (os.FileInfo, error) { 162 | return fs.stat(path, false) 163 | } 164 | 165 | func (fs *memoryFileSystem) Stat(path string) (os.FileInfo, error) { 166 | return fs.stat(path, true) 167 | } 168 | 169 | func (fs *memoryFileSystem) ReadDir(path string) ([]os.FileInfo, error) { 170 | fs.mu.RLock() 171 | defer fs.mu.RUnlock() 172 | return fs.readDir(path) 173 | } 174 | 175 | func (fs *memoryFileSystem) readDir(path string) ([]os.FileInfo, error) { 176 | entry, _, _, err := fs.entry(path, true) 177 | if err != nil { 178 | return nil, err 179 | } 180 | if entry.Type() != EntryTypeDir { 181 | return nil, fmt.Errorf("%s is not a directory", path) 182 | } 183 | dir := entry.(*Dir) 184 | dir.RLock() 185 | infos := make([]os.FileInfo, len(dir.Entries)) 186 | for ii, v := range dir.EntryNames { 187 | infos[ii] = &EntryInfo{ 188 | Path: pathpkg.Join(path, v), 189 | Entry: dir.Entries[ii], 190 | } 191 | } 192 | dir.RUnlock() 193 | return infos, nil 194 | } 195 | 196 | func (fs *memoryFileSystem) Mkdir(path string, perm os.FileMode) error { 197 | path = cleanPath(path) 198 | dir, base := pathpkg.Split(path) 199 | if base == "" { 200 | if dir == "/" || dir == "" { 201 | return os.ErrExist 202 | } 203 | return errNoEmptyNameDir 204 | } 205 | fs.mu.RLock() 206 | d, err := fs.dirEntry(dir, true) 207 | fs.mu.RUnlock() 208 | if err != nil { 209 | return err 210 | } 211 | d.Lock() 212 | defer d.Unlock() 213 | if _, p, _ := d.Find(base); p >= 0 { 214 | return os.ErrExist 215 | } 216 | d.Add(base, &Dir{ 217 | Mode: os.ModeDir | perm, 218 | ModTime: time.Now(), 219 | }) 220 | return nil 221 | } 222 | 223 | func (fs *memoryFileSystem) Remove(path string) error { 224 | entry, dir, pos, err := fs.entry(path, true) 225 | if err != nil { 226 | return err 227 | } 228 | if entry.Type() == EntryTypeDir && len(entry.(*Dir).Entries) > 0 { 229 | return fmt.Errorf("directory %s not empty", path) 230 | } 231 | // Lock again, the position might have changed 232 | dir.Lock() 233 | _, pos, err = dir.Find(pathpkg.Base(path)) 234 | if err == nil { 235 | dir.EntryNames = append(dir.EntryNames[:pos], dir.EntryNames[pos+1:]...) 236 | dir.Entries = append(dir.Entries[:pos], dir.Entries[pos+1:]...) 237 | } 238 | dir.Unlock() 239 | return err 240 | } 241 | 242 | func (fs *memoryFileSystem) String() string { 243 | return "MemoryFileSystem" 244 | } 245 | 246 | func newMemory() *memoryFileSystem { 247 | fs := &memoryFileSystem{ 248 | root: &Dir{ 249 | Mode: os.ModeDir | 0755, 250 | ModTime: time.Now(), 251 | }, 252 | } 253 | return fs 254 | } 255 | 256 | // Memory returns an empty in memory VFS. 257 | func Memory() VFS { 258 | return newMemory() 259 | } 260 | 261 | func cleanPath(path string) string { 262 | return strings.Trim(pathpkg.Clean("/"+path), "/") 263 | } 264 | -------------------------------------------------------------------------------- /testdata/fs.tar: -------------------------------------------------------------------------------- 1 | a/0000775000175000017500000000000012354616524007722 5ustar fiamfiama/b/0000775000175000017500000000000012354616524010143 5ustar fiamfiama/b/c/0000775000175000017500000000000012354616540010363 5ustar fiamfiama/b/c/d0000664000175000017500000000000212354616540010521 0ustar fiamfiamgoempty0000664000175000017500000000000012354616544010553 0ustar fiamfiam -------------------------------------------------------------------------------- /vfs_test.go: -------------------------------------------------------------------------------- 1 | package vfs 2 | 3 | import ( 4 | "crypto/sha1" 5 | "crypto/sha256" 6 | "encoding/hex" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "os" 11 | "path/filepath" 12 | "reflect" 13 | "strings" 14 | "testing" 15 | ) 16 | 17 | const ( 18 | goTestFile = "go1.3.src.tar.gz" 19 | ) 20 | 21 | type errNoTestFile string 22 | 23 | func (e errNoTestFile) Error() string { 24 | return fmt.Sprintf("%s test file not found, use testdata/download-data.sh to fetch it", filepath.Base(string(e))) 25 | } 26 | 27 | func openOptionalTestFile(t testing.TB, name string) *os.File { 28 | filename := filepath.Join("testdata", name) 29 | f, err := os.Open(filename) 30 | if err != nil { 31 | t.Skip(errNoTestFile(filename)) 32 | } 33 | return f 34 | } 35 | 36 | func testVFS(t *testing.T, fs VFS) { 37 | if err := WriteFile(fs, "a", []byte("A"), 0644); err != nil { 38 | t.Fatal(err) 39 | } 40 | data, err := ReadFile(fs, "a") 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | if string(data) != "A" { 45 | t.Errorf("expecting file a to contain \"A\" got %q instead", string(data)) 46 | } 47 | if err := WriteFile(fs, "b", []byte("B"), 0755); err != nil { 48 | t.Fatal(err) 49 | } 50 | if _, err := fs.OpenFile("b", os.O_CREATE|os.O_TRUNC|os.O_EXCL|os.O_WRONLY, 0755); err == nil || !IsExist(err) { 51 | t.Errorf("error should be ErrExist, it's %v", err) 52 | } 53 | fb, err := fs.OpenFile("b", os.O_TRUNC|os.O_WRONLY, 0755) 54 | if err != nil { 55 | t.Fatalf("error opening b: %s", err) 56 | } 57 | if _, err := fb.Write([]byte("BB")); err != nil { 58 | t.Errorf("error writing to b: %s", err) 59 | } 60 | if _, err := fb.Seek(0, os.SEEK_SET); err != nil { 61 | t.Errorf("error seeking b: %s", err) 62 | } 63 | if _, err := fb.Read(make([]byte, 2)); err == nil { 64 | t.Error("allowed reading WRONLY file b") 65 | } 66 | if err := fb.Close(); err != nil { 67 | t.Errorf("error closing b: %s", err) 68 | } 69 | files, err := fs.ReadDir("/") 70 | if err != nil { 71 | t.Fatal(err) 72 | } 73 | if len(files) != 2 { 74 | t.Errorf("expecting 2 files, got %d", len(files)) 75 | } 76 | if n := files[0].Name(); n != "a" { 77 | t.Errorf("expecting first file named \"a\", got %q", n) 78 | } 79 | if n := files[1].Name(); n != "b" { 80 | t.Errorf("expecting first file named \"b\", got %q", n) 81 | } 82 | for ii, v := range files { 83 | es := int64(ii + 1) 84 | if s := v.Size(); es != s { 85 | t.Errorf("expecting file %s to have size %d, has %d", v.Name(), es, s) 86 | } 87 | } 88 | if err := MkdirAll(fs, "a/b/c/d", 0); err == nil { 89 | t.Error("should not allow dir over file") 90 | } 91 | if err := MkdirAll(fs, "c/d", 0755); err != nil { 92 | t.Fatal(err) 93 | } 94 | // Idempotent 95 | if err := MkdirAll(fs, "c/d", 0755); err != nil { 96 | t.Fatal(err) 97 | } 98 | if err := fs.Mkdir("c", 0755); err == nil || !IsExist(err) { 99 | t.Errorf("err should be ErrExist, it's %v", err) 100 | } 101 | // Should fail to remove, c is not empty 102 | if err := fs.Remove("c"); err == nil { 103 | t.Fatalf("removed non-empty directory") 104 | } 105 | var walked []os.FileInfo 106 | var walkedNames []string 107 | err = Walk(fs, "c", func(fs VFS, path string, info os.FileInfo, err error) error { 108 | if err != nil { 109 | return err 110 | } 111 | walked = append(walked, info) 112 | walkedNames = append(walkedNames, path) 113 | return nil 114 | }) 115 | if err != nil { 116 | t.Fatal(err) 117 | } 118 | if exp := []string{"c", "c/d"}; !reflect.DeepEqual(exp, walkedNames) { 119 | t.Error("expecting walked names %v, got %v", exp, walkedNames) 120 | } 121 | for _, v := range walked { 122 | if !v.IsDir() { 123 | t.Errorf("%s should be a dir", v.Name()) 124 | } 125 | } 126 | if err := RemoveAll(fs, "c"); err != nil { 127 | t.Fatal(err) 128 | } 129 | err = Walk(fs, "c", func(fs VFS, path string, info os.FileInfo, err error) error { 130 | return err 131 | }) 132 | if err == nil || !IsNotExist(err) { 133 | t.Errorf("error should be ErrNotExist, it's %v", err) 134 | } 135 | } 136 | 137 | func TestMapFS(t *testing.T) { 138 | fs, err := Map(nil) 139 | if err != nil { 140 | t.Fatal(err) 141 | } 142 | testVFS(t, fs) 143 | } 144 | 145 | func TestPopulatedMap(t *testing.T) { 146 | files := map[string]*File{ 147 | "a/1": &File{}, 148 | "a/2": &File{}, 149 | } 150 | fs, err := Map(files) 151 | if err != nil { 152 | t.Fatal(err) 153 | } 154 | infos, err := fs.ReadDir("a") 155 | if err != nil { 156 | t.Fatal(err) 157 | } 158 | if c := len(infos); c != 2 { 159 | t.Fatalf("expecting 2 files in a, got %d", c) 160 | } 161 | if infos[0].Name() != "1" || infos[1].Name() != "2" { 162 | t.Errorf("expecting names 1, 2 got %q, %q", infos[0].Name(), infos[1].Name()) 163 | } 164 | } 165 | 166 | func TestBadPopulatedMap(t *testing.T) { 167 | // 1 can't be file and directory 168 | files := map[string]*File{ 169 | "a/1": &File{}, 170 | "a/1/2": &File{}, 171 | } 172 | _, err := Map(files) 173 | if err == nil { 174 | t.Fatal("Map should not work with a path as both file and directory") 175 | } 176 | } 177 | 178 | func TestTmpFS(t *testing.T) { 179 | fs, err := TmpFS("vfs-test") 180 | if err != nil { 181 | t.Fatal(err) 182 | } 183 | defer fs.Close() 184 | testVFS(t, fs) 185 | } 186 | 187 | const ( 188 | go13FileCount = 4157 189 | // +1 because of the root, the real count is 407 190 | go13DirCount = 407 + 1 191 | ) 192 | 193 | func countFileSystem(fs VFS) (int, int, error) { 194 | files, dirs := 0, 0 195 | err := Walk(fs, "/", func(fs VFS, _ string, info os.FileInfo, err error) error { 196 | if err != nil { 197 | return err 198 | } 199 | if info.IsDir() { 200 | dirs++ 201 | } else { 202 | files++ 203 | } 204 | return nil 205 | }) 206 | return files, dirs, err 207 | } 208 | 209 | func testGoFileCount(t *testing.T, fs VFS) { 210 | files, dirs, err := countFileSystem(fs) 211 | if err != nil { 212 | t.Fatal(err) 213 | } 214 | if files != go13FileCount { 215 | t.Errorf("expecting %d files in go1.3, got %d instead", go13FileCount, files) 216 | } 217 | if dirs != go13DirCount { 218 | t.Errorf("expecting %d directories in go1.3, got %d instead", go13DirCount, dirs) 219 | } 220 | } 221 | 222 | func TestGo13Files(t *testing.T) { 223 | f := openOptionalTestFile(t, goTestFile) 224 | defer f.Close() 225 | fs, err := TarGzip(f) 226 | if err != nil { 227 | t.Fatal(err) 228 | } 229 | testGoFileCount(t, fs) 230 | } 231 | 232 | func TestMounter(t *testing.T) { 233 | m := &Mounter{} 234 | f := openOptionalTestFile(t, goTestFile) 235 | defer f.Close() 236 | fs, err := TarGzip(f) 237 | if err != nil { 238 | t.Fatal(err) 239 | } 240 | m.Mount(fs, "/") 241 | testGoFileCount(t, m) 242 | } 243 | 244 | func TestClone(t *testing.T) { 245 | fs, err := Open(filepath.Join("testdata", "fs.zip")) 246 | if err != nil { 247 | t.Fatal(err) 248 | } 249 | infos1, err := fs.ReadDir("/") 250 | if err != nil { 251 | t.Fatal(err) 252 | } 253 | mem1 := Memory() 254 | if err := Clone(mem1, fs); err != nil { 255 | t.Fatal(err) 256 | } 257 | infos2, err := mem1.ReadDir("/") 258 | if err != nil { 259 | t.Fatal(err) 260 | } 261 | if len(infos2) != len(infos1) { 262 | t.Fatalf("cloned fs has %d entries in / rather than %d", len(infos2), len(infos1)) 263 | } 264 | mem2 := Memory() 265 | if err := Clone(mem2, mem1); err != nil { 266 | t.Fatal(err) 267 | } 268 | infos3, err := mem2.ReadDir("/") 269 | if err != nil { 270 | t.Fatal(err) 271 | } 272 | if len(infos3) != len(infos2) { 273 | t.Fatalf("cloned fs has %d entries in / rather than %d", len(infos3), len(infos2)) 274 | } 275 | } 276 | 277 | func measureVFSMemorySize(t testing.TB, fs VFS) int { 278 | mem, ok := fs.(*memoryFileSystem) 279 | if !ok { 280 | t.Fatalf("%T is not a memory filesystem", fs) 281 | } 282 | var total int 283 | var f func(d *Dir) 284 | f = func(d *Dir) { 285 | for _, v := range d.Entries { 286 | total += int(v.Size()) 287 | if sd, ok := v.(*Dir); ok { 288 | f(sd) 289 | } 290 | } 291 | } 292 | f(mem.root) 293 | return total 294 | } 295 | 296 | func hashVFS(t testing.TB, fs VFS) string { 297 | sha := sha1.New() 298 | err := Walk(fs, "/", func(fs VFS, p string, info os.FileInfo, err error) error { 299 | if err != nil || info.IsDir() { 300 | return err 301 | } 302 | f, err := fs.Open(p) 303 | if err != nil { 304 | return err 305 | } 306 | defer f.Close() 307 | if _, err := io.Copy(sha, f); err != nil { 308 | return err 309 | } 310 | return nil 311 | }) 312 | if err != nil { 313 | t.Fatal(err) 314 | } 315 | return hex.EncodeToString(sha.Sum(nil)) 316 | } 317 | 318 | func TestCompress(t *testing.T) { 319 | f := openOptionalTestFile(t, goTestFile) 320 | defer f.Close() 321 | fs, err := TarGzip(f) 322 | if err != nil { 323 | t.Fatal(err) 324 | } 325 | size1 := measureVFSMemorySize(t, fs) 326 | hash1 := hashVFS(t, fs) 327 | if err := Compress(fs); err != nil { 328 | t.Fatalf("can't compress fs: %s", err) 329 | } 330 | testGoFileCount(t, fs) 331 | size2 := measureVFSMemorySize(t, fs) 332 | hash2 := hashVFS(t, fs) 333 | if size2 >= size1 { 334 | t.Fatalf("compressed fs takes more memory %d than bare fs %d", size2, size1) 335 | } 336 | if hash1 != hash2 { 337 | t.Fatalf("compressing fs changed hash from %s to %s", hash1, hash2) 338 | } 339 | } 340 | 341 | func testHashes(t *testing.T, fs VFS, hashes map[string]string) { 342 | for k, v := range hashes { 343 | f, err := fs.Open(k) 344 | if err != nil { 345 | t.Fatal(err) 346 | } 347 | data, err := ioutil.ReadAll(f) 348 | if err != nil { 349 | t.Fatal(err) 350 | } 351 | if err := f.Close(); err != nil { 352 | t.Fatal(err) 353 | } 354 | p := strings.Split(v, ":") 355 | var h []byte 356 | switch p[0] { 357 | case "sha1": 358 | sha1h := sha1.Sum(data) 359 | h = sha1h[:] 360 | case "sha256": 361 | sha256h := sha256.Sum256(data) 362 | h = sha256h[:] 363 | default: 364 | t.Fatalf("unknown hash algorithm %q", p[0]) 365 | } 366 | s := hex.EncodeToString(h[:]) 367 | if s != p[1] { 368 | t.Errorf("expected %s(%s) = %q, got %q instead", p[0], k, v, p[1]) 369 | } 370 | } 371 | } 372 | 373 | func TestZipSymlinks(t *testing.T) { 374 | fs, err := Open(filepath.Join("testdata", "fs2.zip")) 375 | if err != nil { 376 | t.Fatal(err) 377 | } 378 | var hashes = map[string]string{ 379 | "f1.bin": "sha1:b98c6a155dc7a778874dfc6023be2bacc2e495dd", 380 | "f2.bin": "sha1:989c1dda053300c5d2c101240acb2e6678f0319a", 381 | "f3.bin": "sha1:989c1dda053300c5d2c101240acb2e6678f0319a", 382 | } 383 | testHashes(t, fs, hashes) 384 | // Make sure f3.bin is a symlink 385 | st1, err := fs.Lstat("f2.bin") 386 | if err != nil { 387 | t.Fatal(err) 388 | } 389 | if st1.Mode()&os.ModeSymlink != 0 { 390 | t.Error("f2.bin is a symlink, it shouldn't be") 391 | } 392 | st2, err := fs.Lstat("f3.bin") 393 | if err != nil { 394 | t.Fatal(err) 395 | } 396 | if st2.Mode()&os.ModeSymlink == 0 { 397 | t.Error("f3.bin isn't a symlink, it should be") 398 | } 399 | 400 | } 401 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /testdata/fs2/f1.bin: -------------------------------------------------------------------------------- 1 | --------------------------------------------------------------------------------