├── LICENSE ├── Makefile ├── README.md ├── go.mod ├── go.sum ├── mergefs.go ├── mergefs_test.go └── testdata └── b └── z └── foo.cue /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Amir Laher 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | .DEFAULT_GOAL := help 3 | 4 | .PHONY: test 5 | test: ## run tests (using go1.16beta1 for now) 6 | go1.16beta1 test -v -race . 7 | 8 | .PHONY: help 9 | help: 10 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mergefs 2 | 3 | A tiny go package which combines together fs.FS filesystems. 4 | 5 | `mergefs.FS` looks through a slice of `fs.FS` filesytems in order to find a given file. It returns the first match, or `os.ErrNotExist`. 6 | 7 | [![Go Reference](https://pkg.go.dev/badge/github.com/laher/mergefs.svg)](https://pkg.go.dev/github.com/laher/mergefs) 8 | 9 | # Related work 10 | 11 | mergefs could be used to overlay multiple fs.FS filesystems on top of each other. 12 | 13 | * [marshalfs](https://pkg.go.dev/github.com/laher/marshalfs) for backing a fileystem with your objects. 14 | * Standard Library: 15 | * [dirfs](https://tip.golang.org/pkg/os/) contains `os.DirFS` - this 'default' implementation is backed by an actual filesystem. 16 | * [testfs](https://tip.golang.org/pkg/testing/fstest/) contains a memory-map implementation and a testing tool. The standard library contains a few other fs.FS implementations (like 'zip') 17 | * [embedfs](https://tip.golang.org/pkg/embed/) provides access to files embedded in the running Go program. 18 | * An earlier work, [afero](https://github.com/spf13/afero) is a filesystem abstraction for Go, which has been the standard for filesystem abstractions up until go1.15. It's read-write, and it's a mature project. The interfaces look very different (big with lots of methods), so it's not really compatible. 19 | * [s3fs](https://github.com/jszwec/s3fs) is a fs.FS backed by the AWS S3 client 20 | * [hashfs](https://pkg.go.dev/github.com/benbjohnson/hashfs) appends SHA256 hashes to filenames to allow for aggressive HTTP caching. 21 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/laher/mergefs 2 | 3 | go 1.16 4 | 5 | require github.com/matryer/is v1.4.0 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= 2 | github.com/matryer/is v1.4.0/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= 3 | -------------------------------------------------------------------------------- /mergefs.go: -------------------------------------------------------------------------------- 1 | package mergefs 2 | 3 | import ( 4 | "errors" 5 | "io/fs" 6 | "os" 7 | ) 8 | 9 | // Merge filesystems 10 | func Merge(filesystems ...fs.FS) fs.FS { 11 | return MergedFS{filesystems: filesystems} 12 | } 13 | 14 | // MergedFS combines filesystems. Each filesystem can serve different paths. The first FS takes precedence 15 | type MergedFS struct { 16 | filesystems []fs.FS 17 | } 18 | 19 | // Open opens the named file. 20 | func (mfs MergedFS) Open(name string) (fs.File, error) { 21 | for _, fs := range mfs.filesystems { 22 | file, err := fs.Open(name) 23 | if err == nil { // TODO should we return early when it's not an os.ErrNotExist? Should we offer options to decide this behaviour? 24 | return file, nil 25 | } 26 | } 27 | return nil, os.ErrNotExist 28 | } 29 | 30 | // ReadDir reads from the directory, and produces a DirEntry array of different 31 | // directories. 32 | // 33 | // It iterates through all different filesystems that exist in the mfs MergeFS 34 | // filesystem slice and it identifies overlapping directories that exist in different 35 | // filesystems 36 | func (mfs MergedFS) ReadDir(name string) ([]fs.DirEntry, error) { 37 | dirsMap := make(map[string]fs.DirEntry) 38 | notExistCount := 0 39 | for _, filesystem := range mfs.filesystems { 40 | dir, err := fs.ReadDir(filesystem, name) 41 | if err != nil { 42 | if errors.Is(err, fs.ErrNotExist) { 43 | notExistCount++ 44 | continue 45 | } 46 | return nil, err 47 | } 48 | for _, v := range dir { 49 | if _, ok := dirsMap[v.Name()]; !ok { 50 | dirsMap[v.Name()] = v 51 | } 52 | } 53 | continue 54 | } 55 | if len(mfs.filesystems) == notExistCount { 56 | return nil, fs.ErrNotExist 57 | } 58 | dirs := make([]fs.DirEntry, 0, len(dirsMap)) 59 | 60 | for _, value := range dirsMap { 61 | dirs = append(dirs, value) 62 | } 63 | 64 | return dirs, nil 65 | } 66 | -------------------------------------------------------------------------------- /mergefs_test.go: -------------------------------------------------------------------------------- 1 | package mergefs_test 2 | 3 | import ( 4 | "io/fs" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | "testing/fstest" 9 | 10 | "github.com/laher/mergefs" 11 | "github.com/matryer/is" 12 | ) 13 | 14 | func TestMergeFS(t *testing.T) { 15 | 16 | t.Run("testing different filesystems", func(t *testing.T) { 17 | a := fstest.MapFS{"a": &fstest.MapFile{Data: []byte("text")}} 18 | b := fstest.MapFS{"b": &fstest.MapFile{Data: []byte("text")}} 19 | filesystem := mergefs.Merge(a, b) 20 | 21 | if _, err := filesystem.Open("a"); err != nil { 22 | t.Fatalf("file should exist") 23 | } 24 | if _, err := filesystem.Open("b"); err != nil { 25 | t.Fatalf("file should exist") 26 | } 27 | }) 28 | 29 | var filePaths = []struct { 30 | path string 31 | dirArrayLength int 32 | child string 33 | }{ 34 | // MapFS takes in account the current directory in addition to all included directories and produces a "" dir 35 | {"a", 1, "z"}, 36 | {"a/z", 1, "bar.cue"}, 37 | {"b", 1, "z"}, 38 | {"b/z", 1, "foo.cue"}, 39 | } 40 | 41 | tempDir := os.DirFS(filepath.Join("testdata")) 42 | a := fstest.MapFS{ 43 | "a": &fstest.MapFile{Mode: fs.ModeDir}, 44 | "a/z": &fstest.MapFile{Mode: fs.ModeDir}, 45 | "a/z/bar.cue": &fstest.MapFile{Data: []byte("bar")}, 46 | } 47 | 48 | filesystem := mergefs.Merge(tempDir, a) 49 | 50 | t.Run("testing mergefs.ReadDir", func(t *testing.T) { 51 | for _, fp := range filePaths { 52 | t.Run("testing path: "+fp.path, func(t *testing.T) { 53 | is := is.New(t) 54 | dirs, err := fs.ReadDir(filesystem, fp.path) 55 | is.NoErr(err) 56 | is.Equal(len(dirs), fp.dirArrayLength) 57 | 58 | for i := 0; i < len(dirs); i++ { 59 | is.Equal(dirs[i].Name(), fp.child) 60 | } 61 | }) 62 | } 63 | }) 64 | 65 | t.Run("testing mergefs.Open", func(t *testing.T) { 66 | is := is.New(t) 67 | data := make([]byte, 3) 68 | file, err := filesystem.Open("a/z/bar.cue") 69 | is.NoErr(err) 70 | 71 | _, err = file.Read(data) 72 | is.NoErr(err) 73 | is.Equal("bar", string(data)) 74 | 75 | file, err = filesystem.Open("b/z/foo.cue") 76 | is.NoErr(err) 77 | 78 | _, err = file.Read(data) 79 | is.NoErr(err) 80 | is.Equal("foo", string(data)) 81 | 82 | err = file.Close() 83 | is.NoErr(err) 84 | }) 85 | } 86 | -------------------------------------------------------------------------------- /testdata/b/z/foo.cue: -------------------------------------------------------------------------------- 1 | foo 2 | --------------------------------------------------------------------------------