├── .gitignore ├── CHANGELOG.md ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── TODO.md ├── adapter └── http │ ├── file.go │ └── system.go ├── archive ├── rewrite.go ├── rewrite_test.go └── uploader.go ├── asset ├── create.go ├── doc.go ├── doc_test.go ├── load.go ├── modify.go ├── resource.go └── resource_test.go ├── base ├── doc.go ├── manager.go ├── manager_test.go ├── reader.go ├── reader_test.go ├── retry.go ├── storager.go └── uploader.go ├── cache ├── build.go ├── cache.go ├── compression.go ├── const.go ├── doc.go ├── entry.go ├── packer.go ├── service.go ├── service_test.go └── singleton.go ├── copy.go ├── copy_test.go ├── coverage.out ├── coverage_badge.png ├── doc.go ├── embed ├── f2 │ └── 1.txt ├── holder.go ├── holder_test.go ├── init.go ├── list.go ├── manager.go ├── manager_test.go ├── open.go ├── provider.go ├── scheme.go └── test │ ├── f1.tmpl │ ├── f2.tmpl │ └── folder │ └── f3.tmpl ├── example ├── app.war └── test │ └── app.war ├── example_test.go ├── faker.go ├── faker_test.go ├── file ├── README.md ├── const.go ├── create.go ├── create_test.go ├── delete.go ├── doc.go ├── info.go ├── info_test.go ├── list.go ├── list_test.go ├── manager.go ├── manager_test.go ├── mode.go ├── mode_test.go ├── move.go ├── move_test.go ├── open.go ├── path.go ├── path_test.go ├── provider.go ├── scheme.go ├── upload.go └── writer.go ├── go.mod ├── http ├── README.md ├── client.go ├── client_test.go ├── create.go ├── create_test.go ├── delete.go ├── delete_test.go ├── doc.go ├── exists.go ├── exists_test.go ├── header.go ├── header_test.go ├── list.go ├── list_test.go ├── manager.go ├── open.go ├── open_test.go ├── parrot_test.go ├── provider.go ├── response.go ├── run.go ├── run_test.go ├── scheme.go ├── server_test.go ├── status.go ├── upload.go └── upload_test.go ├── init.go ├── list.go ├── list_test.go ├── matcher ├── basic.go ├── basic_test.go ├── doc.go ├── example_test.go ├── filepath.go ├── filepath_test.go ├── helper.go ├── ignore.go ├── ignore_test.go ├── modification.go └── modification_test.go ├── mem ├── README.md ├── create.go ├── delete.go ├── delete_test.go ├── doc.go ├── example_test.go ├── file.go ├── folder.go ├── folder_test.go ├── list.go ├── list_test.go ├── manager.go ├── open.go ├── open_test.go ├── parent.go ├── provider.go ├── reader.go ├── root.go ├── scheme.go ├── singleton.go ├── singleton_test.go ├── split.go ├── split_test.go ├── storager.go ├── upload.go └── upload_test.go ├── modifier ├── replacer.go └── replacer_test.go ├── move.go ├── move_test.go ├── object ├── doc.go ├── link.go └── object.go ├── option ├── acl.go ├── append.go ├── append_test.go ├── assign.go ├── assign_test.go ├── auth.go ├── cache.go ├── checksum.go ├── content │ ├── const.go │ └── meta.go ├── crc.go ├── crc_test.go ├── cred.go ├── dest.go ├── doc.go ├── empty.go ├── encryption.go ├── error.go ├── generation.go ├── grant.go ├── key.go ├── key_test.go ├── list.go ├── list_test.go ├── location.go ├── logger.go ├── matcher.go ├── md5.go ├── md5_test.go ├── method.go ├── modifier.go ├── nocache.go ├── object.go ├── osflag.go ├── override.go ├── page.go ├── page_test.go ├── presign.go ├── proxy.go ├── recursive.go ├── refresh.go ├── region.go ├── size.go ├── source.go ├── status.go ├── stream.go ├── timeout.go └── walk.go ├── parrot ├── data.go ├── data_test.go ├── doc.go ├── mem.go ├── mem_test.go ├── static.go ├── static_test.go ├── test │ └── runner.go ├── test_data │ └── extract.txt └── util.go ├── registry.go ├── scp ├── README.md ├── auth.go ├── client_test.go ├── const.go ├── doc.go ├── example_test.go ├── info.go ├── info_test.go ├── manager.go ├── manager_test.go ├── path.go ├── provider.go ├── reader.go ├── scheme.go ├── session.go ├── session_test.go ├── storager.go └── storager_test.go ├── service.go ├── service_test.go ├── ssh └── scheme.go ├── storage ├── auth.go ├── checker.go ├── copier.go ├── doc.go ├── manager.go ├── mover.go ├── object.go ├── option.go ├── options.go ├── sizer.go ├── storager.go ├── uploader.go ├── walker.go └── writer.go ├── sync ├── counter.go ├── counter_test.go └── doc.go ├── tar ├── README.md ├── doc.go ├── doc_test.go ├── manager.go ├── manager_test.go ├── provider.go ├── scheme.go ├── storager.go ├── storager_test.go ├── test │ ├── app.war │ └── test.tar ├── uploader.go ├── uploader_test.go ├── walker.go ├── walker_test.go └── writer.go ├── uploader.go ├── uploader_test.go ├── url ├── base.go ├── base_test.go ├── dir.go ├── doc.go ├── equals.go ├── host.go ├── host_test.go ├── join.go ├── join_test.go ├── normalize.go ├── path.go ├── path_test.go ├── relative.go ├── scheme.go ├── scheme_test.go ├── split.go └── split_test.go ├── walker.go ├── walker ├── doc.go └── walker.go ├── walker_test.go ├── writer.go └── zip ├── README.md ├── doc.go ├── doc_test.go ├── manager.go ├── manager_test.go ├── provider.go ├── scheme.go ├── storager.go ├── storager_test.go ├── test ├── app.war └── test.zip ├── uploader.go ├── uploader_test.go ├── walker.go ├── walker_test.go └── writer.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | local_test.go 3 | *.iml -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Viant, inc. Abstract File Storage (afs) 2 | Copyright 2012-2016 Viant 3 | 4 | This product includes software developed at 5 | Viant (http://viantinc.com/) -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | ### TODO list 2 | 3 | - add git ignore example 4 | - add docker connector 5 | - add git connector 6 | - add generate content to in memory fs from arbitrary URL -------------------------------------------------------------------------------- /adapter/http/file.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/viant/afs" 7 | "github.com/viant/afs/storage" 8 | "io" 9 | "io/fs" 10 | "net/http" 11 | ) 12 | 13 | type file struct { 14 | fs afs.Service 15 | reader io.ReadCloser 16 | object storage.Object 17 | } 18 | 19 | func (f *file) Close() error { 20 | if f.reader == nil { 21 | return fmt.Errorf("reader was nil") 22 | } 23 | return f.reader.Close() 24 | } 25 | 26 | func (f *file) Read(p []byte) (n int, err error) { 27 | if f.reader == nil { 28 | return 0, fmt.Errorf("reader was nil") 29 | } 30 | return f.reader.Read(p) 31 | } 32 | 33 | func (f *file) Seek(offset int64, whence int) (int64, error) { 34 | seeker, ok := f.reader.(io.Seeker) 35 | if !ok { 36 | return 0, fmt.Errorf("invalid reader type: %T", seeker) 37 | } 38 | return seeker.Seek(offset, whence) 39 | } 40 | 41 | func (f *file) Readdir(count int) ([]fs.FileInfo, error) { 42 | if !f.object.IsDir() { 43 | return nil, fmt.Errorf("not directory: %v ", f.object.URL()) 44 | } 45 | objects, err := f.fs.List(context.Background(), f.object.URL()) 46 | if err != nil { 47 | return nil, err 48 | } 49 | var result = make([]fs.FileInfo, 0, len(objects)) 50 | for i := range objects { 51 | result = append(result, objects[i]) 52 | } 53 | if count > 0 { 54 | result = result[:count] 55 | } 56 | return result, nil 57 | } 58 | 59 | func (f *file) Stat() (fs.FileInfo, error) { 60 | return f.object, nil 61 | } 62 | 63 | // NewFile creates a http.File 64 | func NewFile(object storage.Object, fs afs.Service) (http.File, error) { 65 | ret := &file{object: object, fs: fs} 66 | if object.IsDir() { 67 | return ret, nil 68 | } 69 | reader, err := fs.Open(context.Background(), object) 70 | if err != nil { 71 | return nil, err 72 | } 73 | ret.reader = reader 74 | return ret, nil 75 | } 76 | -------------------------------------------------------------------------------- /adapter/http/system.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs" 6 | "github.com/viant/afs/storage" 7 | "github.com/viant/afs/url" 8 | "net/http" 9 | ) 10 | 11 | type Filesystem struct { 12 | fs afs.Service 13 | dir string 14 | options []storage.Option 15 | } 16 | 17 | func (f *Filesystem) Open(name string) (http.File, error) { 18 | object, err := f.fs.Object(context.Background(), url.Join(f.dir, name), f.options...) 19 | if err != nil { 20 | return nil, err 21 | } 22 | return NewFile(object, f.fs) 23 | } 24 | 25 | // New creates http filesystem 26 | func New(fs afs.Service, dir string, options ...storage.Option) http.FileSystem { 27 | return &Filesystem{fs: fs, dir: dir, options: options} 28 | } 29 | -------------------------------------------------------------------------------- /archive/uploader.go: -------------------------------------------------------------------------------- 1 | package archive 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/asset" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | "path" 10 | ) 11 | 12 | //RewriteUploader represents rewrite uploaderMe 13 | type RewriteUploader struct { 14 | resources []*asset.Resource 15 | listener func(resources []*asset.Resource) error 16 | } 17 | 18 | //Upload returns upload handler, and upload closer for batch upload or error 19 | func (r *RewriteUploader) Upload(ctx context.Context, parent string, info os.FileInfo, reader io.Reader) error { 20 | var data []byte 21 | var err error 22 | if !info.IsDir() { 23 | if data, err = ioutil.ReadAll(reader); err != nil { 24 | return err 25 | } 26 | } 27 | resource := asset.New(path.Join(parent, info.Name()), info.Mode(), info.IsDir(), "", data) 28 | resource.FileInfo = info 29 | r.resources = append(r.resources, resource) 30 | return nil 31 | } 32 | 33 | //Close notifies specified listener 34 | func (r *RewriteUploader) Close() error { 35 | return r.listener(r.resources) 36 | } 37 | 38 | //NewRewriteUploader returns new rewrite uploader 39 | func NewRewriteUploader(listener func(resources []*asset.Resource) error) *RewriteUploader { 40 | return &RewriteUploader{ 41 | resources: make([]*asset.Resource, 0), 42 | listener: listener, 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /asset/create.go: -------------------------------------------------------------------------------- 1 | package asset 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/storage" 6 | "github.com/viant/afs/url" 7 | ) 8 | 9 | //Create creates supplied assets, links or folders in provided location (for testing purpose) 10 | func Create(manager storage.Manager, URL string, resources []*Resource) error { 11 | return modify(manager, URL, resources, true) 12 | } 13 | 14 | //Cleanup removes supplied locations 15 | func Cleanup(manager storage.Manager, URL string) error { 16 | ctx := context.Background() 17 | URL = url.Normalize(URL, manager.Scheme()) 18 | return manager.Delete(ctx, URL) 19 | } 20 | -------------------------------------------------------------------------------- /asset/doc.go: -------------------------------------------------------------------------------- 1 | //Package asset define asset testing utility 2 | package asset 3 | -------------------------------------------------------------------------------- /asset/doc_test.go: -------------------------------------------------------------------------------- 1 | package asset_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/viant/afs" 7 | "github.com/viant/afs/asset" 8 | "github.com/viant/afs/storage" 9 | "log" 10 | ) 11 | 12 | //Example_Create crete test assets example 13 | func Example_Create() { 14 | 15 | var useCases = []struct { 16 | description string 17 | location string 18 | options []storage.Option 19 | assets []*asset.Resource 20 | }{} 21 | 22 | ctx := context.Background() 23 | for _, useCase := range useCases { 24 | service := afs.New() 25 | mgr, err := afs.Manager(useCase.location, useCase.options...) 26 | if err != nil { 27 | log.Fatal(err) 28 | } 29 | err = asset.Create(mgr, useCase.location, useCase.assets) 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | _, err = service.Exists(ctx, useCase.location) 34 | if err != nil { 35 | log.Fatal(err) 36 | } 37 | 38 | actuals, err := asset.Load(mgr, useCase.location) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | fmt.Printf("actuals: %v\n", actuals) 43 | _ = asset.Cleanup(mgr, useCase.location) 44 | 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /asset/load.go: -------------------------------------------------------------------------------- 1 | package asset 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/storage" 6 | "github.com/viant/afs/url" 7 | "github.com/viant/afs/walker" 8 | "io" 9 | "io/ioutil" 10 | "os" 11 | "path" 12 | ) 13 | 14 | //Load loads location resources for supplied manager 15 | func Load(manager storage.Manager, URL string) (map[string]*Resource, error) { 16 | URL = url.Normalize(URL, manager.Scheme()) 17 | managerWalker, ok := manager.(storage.Walker) 18 | if !ok { 19 | managerWalker = walker.New(manager) 20 | } 21 | var result = make(map[string]*Resource) 22 | err := managerWalker.Walk(context.Background(), URL, func(ctx context.Context, baseURL string, parent string, info os.FileInfo, reader io.Reader) (toContinue bool, err error) { 23 | key := path.Join(parent, info.Name()) 24 | var data []byte 25 | if !info.IsDir() { 26 | data, err = ioutil.ReadAll(reader) 27 | } 28 | result[key] = New(key, info.Mode(), info.IsDir(), "", data) 29 | return true, nil 30 | }) 31 | return result, err 32 | } 33 | -------------------------------------------------------------------------------- /asset/modify.go: -------------------------------------------------------------------------------- 1 | package asset 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "github.com/viant/afs/storage" 7 | "github.com/viant/afs/url" 8 | "os" 9 | "path" 10 | "path/filepath" 11 | "time" 12 | ) 13 | 14 | //Modify modify supplied assets, links or folders in provided location (for testing purpose) 15 | func Modify(manager storage.Manager, URL string, resources []*Resource) error { 16 | return modify(manager, URL, resources, false) 17 | } 18 | 19 | func modify(manager storage.Manager, URL string, resources []*Resource, recreatedURL bool) error { 20 | if len(resources) == 0 { 21 | return nil 22 | } 23 | URL = url.Normalize(URL, manager.Scheme()) 24 | ctx := context.Background() 25 | if recreatedURL { 26 | _ = manager.Delete(ctx, URL) 27 | } 28 | _ = manager.Create(ctx, URL, 0744, true) 29 | baseURL, URLPath := url.Base(URL, manager.Scheme()) 30 | 31 | for _, asset := range resources { 32 | if !asset.Dir { 33 | continue 34 | } 35 | baseURL, URLPath := url.Base(URL, manager.Scheme()) 36 | resourceURL := url.Join(baseURL, path.Join(URLPath, asset.Name)) 37 | 38 | if err := manager.Create(ctx, resourceURL, asset.Mode, true); err != nil { 39 | return err 40 | } 41 | } 42 | for _, asset := range resources { 43 | if asset.Dir || asset.Link != "" { 44 | continue 45 | } 46 | resourceURL := url.Join(baseURL, path.Join(URLPath, asset.Name)) 47 | modTime := time.Now() 48 | if asset.ModTime != nil { 49 | modTime = *asset.ModTime 50 | } 51 | if err := manager.Upload(ctx, resourceURL, asset.Mode, bytes.NewReader(asset.Data), modTime); err != nil { 52 | return err 53 | } 54 | } 55 | 56 | for _, asset := range resources { 57 | if asset.Link == "" { 58 | continue 59 | } 60 | symlink := filepath.Join(URLPath, asset.Name) 61 | source := path.Join(URLPath, asset.Link) 62 | if err := os.Symlink(source, symlink); err != nil { 63 | return err 64 | } 65 | } 66 | return nil 67 | } 68 | -------------------------------------------------------------------------------- /asset/resource_test.go: -------------------------------------------------------------------------------- 1 | package asset_test 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/viant/afs/asset" 6 | "github.com/viant/afs/file" 7 | "io/ioutil" 8 | "testing" 9 | ) 10 | 11 | func TestNew(t *testing.T) { 12 | mgr := file.New() 13 | baseURL := "file://localhost/tmp/assets" 14 | _ = asset.Cleanup(mgr, baseURL) 15 | 16 | err := asset.Create(mgr, baseURL, []*asset.Resource{ 17 | asset.NewFile("file1.txt", []byte("123"), 0644), 18 | asset.NewDir("dir1", 0755), 19 | asset.NewLink("file2.txt", "file1.txt", 0644), 20 | }) 21 | assert.Nil(t, err) 22 | resources, err := asset.Load(mgr, baseURL) 23 | assert.Nil(t, err) 24 | assert.NotNil(t, resources["file1.txt"]) 25 | 26 | resource := resources["file1.txt"] 27 | assert.EqualValues(t, resource.Info().Name(), "file1.txt") 28 | data, err := ioutil.ReadAll(resource.Reader()) 29 | assert.Nil(t, err) 30 | assert.EqualValues(t, "123", data) 31 | err = resource.MergeFrom(resource) 32 | assert.Nil(t, err) 33 | assert.NotNil(t, resources["file2.txt"]) 34 | assert.NotNil(t, resources["dir1"]) 35 | 36 | } 37 | -------------------------------------------------------------------------------- /base/doc.go: -------------------------------------------------------------------------------- 1 | //Package base define base manager 2 | package base 3 | -------------------------------------------------------------------------------- /base/retry.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | //Retry represents abstraction holding sleep duration between retries (back-off) 9 | type Retry struct { 10 | Count int 11 | Initial time.Duration 12 | Max time.Duration 13 | Multiplier float64 14 | duration time.Duration 15 | } 16 | 17 | // Pause returns the next time.Duration that the caller should use to backoff. 18 | func (b *Retry) Pause() time.Duration { 19 | if b.Initial == 0 { 20 | b.Initial = time.Second 21 | } 22 | if b.duration == 0 { 23 | b.duration = b.Initial 24 | } 25 | if b.Max == 0 { 26 | b.Max = 30 * time.Second 27 | } 28 | if b.Multiplier < 1 { 29 | b.Multiplier = 2 30 | } 31 | 32 | rnd := rand.New(rand.NewSource(time.Now().UnixNano())) 33 | result := time.Duration(1 + rnd.Int63n(int64(b.duration))) 34 | b.duration = time.Duration(float64(b.duration) * b.Multiplier) 35 | if b.duration > b.Max { 36 | b.duration = b.Max 37 | } 38 | return result 39 | } 40 | 41 | //NewRetry creates a retry 42 | func NewRetry() *Retry { 43 | return &Retry{} 44 | } 45 | -------------------------------------------------------------------------------- /base/storager.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import ( 4 | "context" 5 | "github.com/go-errors/errors" 6 | "github.com/viant/afs/option" 7 | "github.com/viant/afs/storage" 8 | "os" 9 | ) 10 | 11 | type List func(ctx context.Context, location string, options ...storage.Option) ([]os.FileInfo, error) 12 | 13 | //Storager represents a base storager 14 | type Storager struct { 15 | List func(ctx context.Context, location string, options ...storage.Option) ([]os.FileInfo, error) 16 | } 17 | 18 | //Get returns an object for supplied location 19 | func (s *Storager) Get(ctx context.Context, location string, options ...storage.Option) (os.FileInfo, error) { 20 | options = append(options, option.NewPage(0, 1)) 21 | objects, err := s.List(ctx, location, options) 22 | if err != nil { 23 | return nil, err 24 | } 25 | if len(objects) == 0 { 26 | return nil, errors.Errorf("failed to get object: %v", location) 27 | } 28 | return objects[0], nil 29 | } 30 | -------------------------------------------------------------------------------- /base/uploader.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/file" 6 | "github.com/viant/afs/storage" 7 | "github.com/viant/afs/url" 8 | "io" 9 | "os" 10 | "path" 11 | "strings" 12 | "sync/atomic" 13 | ) 14 | 15 | type uploader struct { 16 | storage.Manager 17 | } 18 | 19 | //Close implements closer 20 | func (u *uploader) Close() error { 21 | return nil 22 | } 23 | 24 | func (u *uploader) Uploader(ctx context.Context, URL string, options ...storage.Option) (storage.Upload, io.Closer, error) { 25 | index := int32(0) 26 | handler := func(ctx context.Context, parent string, info os.FileInfo, reader io.Reader) error { 27 | location := path.Join(parent, info.Name()) 28 | 29 | if atomic.AddInt32(&index, 1) == 1 { 30 | if strings.HasSuffix(URL, location) { 31 | URL = string(URL[:len(URL)-len(location)]) 32 | } 33 | } 34 | URL := url.Join(URL, location) 35 | if info.Mode()&os.ModeSymlink > 0 { 36 | if rawInfo, ok := info.(*file.Info); ok && rawInfo.Linkname != "" { 37 | options = append(options, rawInfo.Link) 38 | } 39 | } 40 | if info.IsDir() { 41 | return u.Manager.Create(ctx, URL, info.Mode(), info.IsDir(), options...) 42 | } 43 | return u.Manager.Upload(ctx, URL, info.Mode(), reader, options...) 44 | } 45 | return handler, u, nil 46 | } 47 | 48 | //NewUploader creates a new batch uploader 49 | func NewUploader(manager storage.Manager) storage.BatchUploader { 50 | return &uploader{manager} 51 | } 52 | -------------------------------------------------------------------------------- /cache/build.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "github.com/viant/afs" 8 | "github.com/viant/afs/file" 9 | "github.com/viant/afs/option" 10 | "github.com/viant/afs/storage" 11 | "io/ioutil" 12 | "strings" 13 | "sync" 14 | ) 15 | 16 | func uploadCacheFile(ctx context.Context, cache *Cache, cacheURL string, service afs.Service) error { 17 | data, err := json.Marshal(cache) 18 | if err != nil { 19 | return err 20 | } 21 | if strings.HasSuffix(cacheURL, ".gz") { 22 | data, _ = compressWithGzip(data) 23 | } 24 | err = service.Upload(ctx, cacheURL, file.DefaultFileOsMode, bytes.NewReader(data)) 25 | if isRateError(err) || isPreConditionError(err) { //ignore rate or generation errors 26 | err = nil 27 | } 28 | return err 29 | } 30 | 31 | func build(ctx context.Context, baseURL, cacheName string, service afs.Service, opts ...storage.Option) (*Cache, error) { 32 | opts = append(opts, option.NewRecursive(true)) 33 | objects, err := service.List(ctx, baseURL, opts...) 34 | if err != nil { 35 | return nil, err 36 | } 37 | var items = make([]*Entry, 0) 38 | entries := NewEntries(&items) 39 | wg := sync.WaitGroup{} 40 | for i, obj := range objects { 41 | if obj.IsDir() || obj.Name() == cacheName { 42 | continue 43 | } 44 | wg.Add(1) 45 | go func(object storage.Object) { 46 | defer wg.Done() 47 | reader, oErr := service.OpenURL(ctx, object.URL()) 48 | if err != nil { 49 | err = oErr 50 | return 51 | } 52 | data, oErr := ioutil.ReadAll(reader) 53 | _ = reader.Close() 54 | if oErr != nil { 55 | err = oErr 56 | return 57 | } 58 | entries.Append(&Entry{ 59 | URL: object.URL(), 60 | Data: data, 61 | Size: object.Size(), 62 | ModTime: object.ModTime(), 63 | }) 64 | 65 | }(objects[i]) 66 | } 67 | wg.Wait() 68 | cacheEntries := &Cache{ 69 | Items: items, 70 | } 71 | return cacheEntries, nil 72 | } 73 | -------------------------------------------------------------------------------- /cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "time" 4 | 5 | //Cache represent a cache 6 | type Cache struct { 7 | URL string 8 | Items []*Entry 9 | At time.Time 10 | } 11 | -------------------------------------------------------------------------------- /cache/compression.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "io" 7 | ) 8 | 9 | func compressWithGzip(data []byte) ([]byte, error) { 10 | buf := new(bytes.Buffer) 11 | writer := gzip.NewWriter(buf) 12 | if _, err := io.Copy(writer, bytes.NewReader(data)); err != nil { 13 | return nil, err 14 | } 15 | if err := writer.Flush(); err != nil { 16 | return nil, err 17 | } 18 | err := writer.Close() 19 | return buf.Bytes(), err 20 | } 21 | 22 | func uncompressWithGzip(data []byte) ([]byte, error) { 23 | reader, err := gzip.NewReader(bytes.NewReader(data)) 24 | if err != nil { 25 | return nil, err 26 | } 27 | reader.Close() 28 | buf := new(bytes.Buffer) 29 | _, err = io.Copy(buf, reader) 30 | data = buf.Bytes() 31 | return data, err 32 | } 33 | -------------------------------------------------------------------------------- /cache/const.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | //CacheFile cache file 4 | const CacheFile = "_.cache" 5 | -------------------------------------------------------------------------------- /cache/doc.go: -------------------------------------------------------------------------------- 1 | //Package cache define cache afs.Service to cache read operation for specified URL 2 | package cache 3 | -------------------------------------------------------------------------------- /cache/entry.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | //Entry represents cache entry 9 | type Entry struct { 10 | URL string 11 | ModTime time.Time 12 | Size int64 13 | Data []byte 14 | } 15 | 16 | type Entries struct { 17 | ptr *[]*Entry 18 | mux sync.Mutex 19 | } 20 | 21 | func (e *Entries) Append(entry *Entry) { 22 | e.mux.Lock() 23 | *e.ptr = append(*e.ptr, entry) 24 | e.mux.Unlock() 25 | } 26 | 27 | func NewEntries(ptr *[]*Entry) *Entries { 28 | return &Entries{ptr: ptr} 29 | } 30 | -------------------------------------------------------------------------------- /cache/packer.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs" 6 | "github.com/viant/afs/file" 7 | "github.com/viant/afs/option" 8 | "github.com/viant/afs/storage" 9 | "github.com/viant/afs/url" 10 | "strings" 11 | ) 12 | 13 | //Package creates cache file for source URL with rewrite 14 | func Package(ctx context.Context, sourceURL string, rewriteBaseURL string, options ...storage.Option) error { 15 | var cacheOption = &option.Cache{} 16 | option.Assign(options, &cacheOption) 17 | if cacheOption.Name == "" { 18 | cacheOption.Name = CacheFile 19 | } 20 | cacheOption.Init() 21 | cacheURL := url.Join(sourceURL, cacheOption.Name) 22 | fs := afs.New() 23 | cache, err := build(ctx, sourceURL, cacheOption.Name, fs, options...) 24 | if err != nil || len(cache.Items) == 0 { 25 | return err 26 | } 27 | sourceURL = url.Normalize(sourceURL, file.Scheme) 28 | for _, entry := range cache.Items { 29 | location := strings.Replace(entry.URL, sourceURL, "", 1) 30 | entry.URL = url.Join(rewriteBaseURL, location) 31 | } 32 | return uploadCacheFile(ctx, cache, cacheURL, fs) 33 | } 34 | -------------------------------------------------------------------------------- /cache/singleton.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "github.com/viant/afs" 5 | "github.com/viant/afs/storage" 6 | ) 7 | 8 | var singleton afs.Service 9 | 10 | //Singleton returns caching Service for specified URL 11 | func Singleton(URL string, opts ...storage.Option) afs.Service { 12 | if singleton != nil { 13 | return singleton 14 | } 15 | singleton = New(URL, afs.New(), opts...) 16 | return singleton 17 | } 18 | -------------------------------------------------------------------------------- /coverage_badge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viant/afs/f200dceb289ef98dfa3ebd148bdfb8a642ade924/coverage_badge.png -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package afs (abstract file storage) defines interface and abstraction for storage systems 2 | package afs 3 | -------------------------------------------------------------------------------- /embed/f2/1.txt: -------------------------------------------------------------------------------- 1 | 123 -------------------------------------------------------------------------------- /embed/holder_test.go: -------------------------------------------------------------------------------- 1 | package embed 2 | 3 | import ( 4 | "embed" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "io" 8 | "testing" 9 | ) 10 | 11 | //go:embed test/* 12 | var testEmbedFs embed.FS 13 | 14 | func TestHolder_Add(t *testing.T) { 15 | 16 | embedFs := NewHolder() 17 | embedFs.Add("zerba.txt", "this is foo context") 18 | embedFs.Add("foo/bar.txt", "this is context") 19 | embedFs.Add("foo/dummy.txt", "this is context") 20 | embedFs.Add("foo/sub/dummy.txt", "this is context") 21 | fs := embedFs.EmbedFs() 22 | entries, err := fs.ReadDir("foo") 23 | assert.Nil(t, err) 24 | assert.Equal(t, 5, len(entries)) 25 | 26 | { 27 | fh, err := fs.Open("foo/bar.txt") 28 | assert.Nil(t, err) 29 | data, err := io.ReadAll(fh) 30 | assert.Nil(t, err) 31 | assert.EqualValues(t, "this is context", string(data)) 32 | _ = fh.Close() 33 | } 34 | 35 | aHolder := NewHolder() 36 | aHolder.AddFs(&testEmbedFs, "embed:///test") 37 | merged := aHolder.EmbedFs() 38 | fmt.Println(merged) 39 | } 40 | -------------------------------------------------------------------------------- /embed/init.go: -------------------------------------------------------------------------------- 1 | package embed 2 | 3 | import ( 4 | "github.com/viant/afs" 5 | ) 6 | 7 | func init() { 8 | afs.GetRegistry().Register(Scheme, Provider) 9 | } 10 | 11 | -------------------------------------------------------------------------------- /embed/list.go: -------------------------------------------------------------------------------- 1 | package embed 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/viant/afs/file" 7 | "github.com/viant/afs/object" 8 | "github.com/viant/afs/option" 9 | "github.com/viant/afs/storage" 10 | "github.com/viant/afs/url" 11 | "strings" 12 | ) 13 | 14 | 15 | func (s *manager) List(ctx context.Context, URL string, options ...storage.Option) ([]storage.Object, error) { 16 | if s.err != nil { 17 | return nil, s.err 18 | } 19 | baseURL, filePath := url.Base(URL, Scheme) 20 | fPath := file.Path(filePath) 21 | fPath = strings.Trim(fPath, "/") 22 | fh, err := s.fs.Open(fPath) 23 | if err != nil { 24 | return nil, fmt.Errorf("unable to open '%v', %w", fPath, err) 25 | } 26 | match, page := option.GetListOptions(options) 27 | defer func() { _ = fh.Close() }() 28 | stat, err := fh.Stat() 29 | if err != nil { 30 | return nil, err 31 | } 32 | if !stat.IsDir() { 33 | return []storage.Object{ 34 | object.New(URL, stat, nil), 35 | }, nil 36 | } 37 | files, err := s.fs.ReadDir(fPath) 38 | if err != nil { 39 | return nil, err 40 | } 41 | var result = make([]storage.Object, 0) 42 | result = append(result, object.New(URL, stat, nil)) 43 | for _, fileInfo := range files { 44 | info, err := fileInfo.Info() 45 | if err != nil { 46 | return nil, fmt.Errorf("failed to get info for: %v, %w", fileInfo.Name(), err) 47 | } 48 | if !match(filePath, info) { 49 | continue 50 | } 51 | page.Increment() 52 | if page.ShallSkip() { 53 | continue 54 | } 55 | fileURL := url.Join(baseURL, filePath, fileInfo.Name()) 56 | result = append(result, object.New(fileURL, info, nil)) 57 | if page.HasReachedLimit() { 58 | break 59 | } 60 | } 61 | return result, nil 62 | } 63 | -------------------------------------------------------------------------------- /embed/manager.go: -------------------------------------------------------------------------------- 1 | package embed 2 | 3 | import ( 4 | "context" 5 | "embed" 6 | "fmt" 7 | "github.com/viant/afs/storage" 8 | "io" 9 | "os" 10 | ) 11 | 12 | type manager struct { 13 | fs *embed.FS 14 | err error 15 | } 16 | 17 | func (s *manager) Upload(ctx context.Context, URL string, mode os.FileMode, reader io.Reader, options ...storage.Option) error { 18 | return fmt.Errorf("unsupproted Upload operation for %v", URL) 19 | } 20 | 21 | func (s *manager) Open(ctx context.Context, object storage.Object, options ...storage.Option) (io.ReadCloser, error) { 22 | if s.err != nil { 23 | return nil, s.err 24 | } 25 | return s.OpenURL(ctx, object.URL(), options...) 26 | } 27 | 28 | // Delete unsupported 29 | func (s *manager) Delete(ctx context.Context, URL string, options ...storage.Option) error { 30 | return fmt.Errorf("unsupproted Delete operation for %v", URL) 31 | } 32 | 33 | // Create unsupported 34 | func (s *manager) Create(ctx context.Context, URL string, mode os.FileMode, isDir bool, options ...storage.Option) error { 35 | return fmt.Errorf("unsupproted Create operation for %v", URL) 36 | } 37 | 38 | // Close closes mananger 39 | func (s *manager) Close() error { 40 | return nil 41 | } 42 | 43 | // Scheme returns schmea 44 | func (s *manager) Scheme() string { 45 | return Scheme 46 | } 47 | 48 | func newManager(options ...storage.Option) *manager { 49 | var fs *embed.FS 50 | 51 | for _, option := range options { 52 | switch v := option.(type) { 53 | case *embed.FS: 54 | fs = v 55 | case embed.FS: 56 | fs = &v 57 | } 58 | } 59 | var err error 60 | if fs == nil { 61 | err = fmt.Errorf("expcted %T", fs) 62 | } 63 | return &manager{ 64 | fs: fs, 65 | err: err, 66 | } 67 | } 68 | 69 | // New creates HTTP manager 70 | func New(options ...storage.Option) storage.Manager { 71 | return newManager(options...) 72 | } 73 | -------------------------------------------------------------------------------- /embed/manager_test.go: -------------------------------------------------------------------------------- 1 | package embed_test 2 | 3 | import ( 4 | "context" 5 | "embed" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/afs" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | //go:embed test/* 13 | var eFs embed.FS 14 | 15 | func TestManager_List(t *testing.T) { 16 | fs := afs.New() 17 | objects, err := fs.List(context.Background(), "embed:///test", eFs) 18 | assert.Nil(t, err) 19 | assert.EqualValues(t, 4, len(objects)) 20 | for _, object := range objects { 21 | if object.IsDir() { 22 | continue 23 | } 24 | data, err := fs.Download(context.Background(), object) 25 | assert.Nil(t, err) 26 | assert.True(t, strings.Contains(string(data), ".")) 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /embed/open.go: -------------------------------------------------------------------------------- 1 | package embed 2 | 3 | import ( 4 | "context" 5 | "embed" 6 | "github.com/viant/afs/file" 7 | "github.com/viant/afs/option" 8 | "github.com/viant/afs/storage" 9 | "io" 10 | "os" 11 | "strings" 12 | ) 13 | 14 | //Open downloads asset for supplied object 15 | func (s *manager) OpenURL(ctx context.Context, URL string, options ...storage.Option) (io.ReadCloser, error) { 16 | filePath := file.Path(URL) 17 | filePath = strings.Trim(filePath, "/") 18 | var efs *embed.FS 19 | if _, ok := option.Assign(options, &efs); ok { 20 | return efs.Open(filePath) 21 | } 22 | return os.Open(filePath) 23 | } 24 | -------------------------------------------------------------------------------- /embed/provider.go: -------------------------------------------------------------------------------- 1 | package embed 2 | 3 | import ( 4 | "github.com/viant/afs/storage" 5 | ) 6 | 7 | //Provider provider function 8 | func Provider(options ...storage.Option) (storage.Manager, error) { 9 | return New(options...), nil 10 | } 11 | -------------------------------------------------------------------------------- /embed/scheme.go: -------------------------------------------------------------------------------- 1 | package embed 2 | 3 | //Scheme file URL scheme 4 | const Scheme = "embed" 5 | 6 | 7 | -------------------------------------------------------------------------------- /embed/test/f1.tmpl: -------------------------------------------------------------------------------- 1 | .. -------------------------------------------------------------------------------- /embed/test/f2.tmpl: -------------------------------------------------------------------------------- 1 | ... -------------------------------------------------------------------------------- /embed/test/folder/f3.tmpl: -------------------------------------------------------------------------------- 1 | . -------------------------------------------------------------------------------- /example/app.war: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viant/afs/f200dceb289ef98dfa3ebd148bdfb8a642ade924/example/app.war -------------------------------------------------------------------------------- /example/test/app.war: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viant/afs/f200dceb289ef98dfa3ebd148bdfb8a642ade924/example/test/app.war -------------------------------------------------------------------------------- /faker.go: -------------------------------------------------------------------------------- 1 | package afs 2 | 3 | //NewFaker returns new faker service. All operation uses in memory service 4 | func NewFaker() Service { 5 | return newService(true) 6 | } 7 | -------------------------------------------------------------------------------- /file/README.md: -------------------------------------------------------------------------------- 1 | # file - file storage 2 | 3 | This package provides a file system manager that wraps os operations. 4 | 5 | 6 | ### Usage 7 | 8 | - **[Service](../service.go)** 9 | ```go 10 | func main() { 11 | service := afs.New() 12 | ctx := context.Background() 13 | err := service.Upload(ctx, "file:///folder1/asset.txt", 0644, strings.NewReader("some content")) 14 | if err != nil { 15 | log.Fatal(err) 16 | } 17 | objects, err := service.List(ctx, "file:///folder1/") 18 | if err != nil { 19 | log.Fatal(err) 20 | } 21 | for _, object := range objects { 22 | fmt.Printf("%v %v\n", object.URL(), object.Name()) 23 | } 24 | } 25 | ``` 26 | 27 | 28 | -------------------------------------------------------------------------------- /file/const.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import "os" 4 | 5 | const ( 6 | //DefaultDirOsMode folder mode default 7 | DefaultDirOsMode = os.ModeDir | 0755 8 | //DefaultFileOsMode file mode default 9 | DefaultFileOsMode = os.FileMode(0644) 10 | ) 11 | -------------------------------------------------------------------------------- /file/create.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/storage" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | //Create creates a new file or directory 11 | func Create(ctx context.Context, URL string, mode os.FileMode, isDir bool, options ...storage.Option) error { 12 | filePath := Path(URL) 13 | if isDir { 14 | mode = mode | os.ModeDir 15 | if err := EnsureParentPathExists(filePath, mode); err != nil { 16 | return err 17 | } 18 | if stat, _ := os.Stat(filePath); stat != nil { 19 | return os.Chmod(filePath, mode) 20 | } 21 | return os.MkdirAll(filePath, mode) 22 | } 23 | return Upload(ctx, URL, mode, strings.NewReader(""), options...) 24 | } 25 | -------------------------------------------------------------------------------- /file/create_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "context" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/viant/afs/url" 7 | "os" 8 | "path" 9 | "testing" 10 | ) 11 | 12 | func TestCreate(t *testing.T) { 13 | 14 | ctx := context.Background() 15 | 16 | tempDir := os.TempDir() 17 | 18 | var useCases = []struct { 19 | description string 20 | URL string 21 | isDir bool 22 | override bool 23 | hasError bool 24 | }{ 25 | 26 | { 27 | description: "create file", 28 | 29 | URL: path.Join(tempDir, "afs", "create", "bar1.txt"), 30 | }, 31 | { 32 | description: "override", 33 | override: true, 34 | URL: path.Join(tempDir, "afs", "create", "error.txt"), 35 | }, 36 | { 37 | description: "create directory", 38 | isDir: true, 39 | URL: path.Join(tempDir, "afs", "create", "subdir"), 40 | }, 41 | { 42 | description: "override directory", 43 | override: true, 44 | URL: path.Join(tempDir, "afs", "create", "error"), 45 | }, 46 | } 47 | 48 | for _, useCase := range useCases { 49 | _ = Delete(ctx, useCase.URL) 50 | if useCase.override { 51 | isDir := useCase.isDir 52 | if useCase.hasError { 53 | isDir = !isDir 54 | } 55 | err := Create(ctx, useCase.URL, 0644, isDir) 56 | assert.Nil(t, err, useCase.description) 57 | } 58 | err := Create(ctx, useCase.URL, 0644, useCase.isDir) 59 | 60 | if useCase.hasError { 61 | assert.NotNil(t, err, useCase.description) 62 | continue 63 | } 64 | 65 | assert.Nil(t, err, useCase.description) 66 | filePath := Path(url.Path(useCase.URL)) 67 | stat, err := os.Stat(filePath) 68 | assert.Nil(t, err, useCase.description) 69 | assert.EqualValues(t, useCase.isDir, stat.IsDir()) 70 | 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /file/delete.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/storage" 6 | "os" 7 | ) 8 | 9 | //Delete removes file or directory 10 | func Delete(ctx context.Context, URL string, options ...storage.Option) error { 11 | filePath := Path(URL) 12 | return os.RemoveAll(filePath) 13 | } 14 | -------------------------------------------------------------------------------- /file/doc.go: -------------------------------------------------------------------------------- 1 | //Package file defines a file system storage 2 | package file 3 | -------------------------------------------------------------------------------- /file/info.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "github.com/viant/afs/object" 5 | "github.com/viant/afs/option" 6 | "github.com/viant/afs/storage" 7 | "os" 8 | "time" 9 | ) 10 | 11 | //Info represents a file info 12 | type Info struct { 13 | name string 14 | size int64 15 | mode os.FileMode 16 | modTime time.Time 17 | isDir bool 18 | *object.Link 19 | } 20 | 21 | //Name returns a name 22 | func (i *Info) Name() string { 23 | return i.name 24 | } 25 | 26 | //Size returns file size 27 | func (i *Info) Size() int64 { 28 | return i.size 29 | } 30 | 31 | //Mode returns file mode 32 | func (i *Info) Mode() os.FileMode { 33 | return i.mode 34 | } 35 | 36 | //ModTime returns modification time 37 | func (i *Info) ModTime() time.Time { 38 | return i.modTime 39 | } 40 | 41 | //IsDir returns true if resoruce is directory 42 | func (i *Info) IsDir() bool { 43 | return i.isDir 44 | } 45 | 46 | //Sys returns sys object 47 | func (i *Info) Sys() interface{} { 48 | return i.Source 49 | } 50 | 51 | //NewInfo returns a ew file Info 52 | func NewInfo(name string, size int64, mode os.FileMode, modificationTime time.Time, isDir bool, options ...storage.Option) os.FileInfo { 53 | link := &object.Link{} 54 | option.Assign(options, &link) 55 | if link.Source == nil && len(options) == 1 { 56 | link.Source = options[0] 57 | } 58 | return &Info{ 59 | name: name, 60 | size: size, 61 | mode: mode, 62 | modTime: modificationTime, 63 | isDir: isDir, 64 | Link: link, 65 | } 66 | } 67 | 68 | //AdjustInfoSize adjust file info size 69 | func AdjustInfoSize(info os.FileInfo, size int) os.FileInfo { 70 | if int(info.Size()) == size { 71 | return info 72 | } 73 | if fileInfo, ok := info.(*Info); ok { 74 | fileInfo.size = int64(size) 75 | } else { 76 | info = NewInfo(info.Name(), int64(size), info.Mode().Perm(), info.ModTime(), info.IsDir(), info.Sys()) 77 | } 78 | return info 79 | } 80 | -------------------------------------------------------------------------------- /file/info_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "os" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | func TestNewInfo(t *testing.T) { 11 | 12 | var useCases = []struct { 13 | description string 14 | name string 15 | size int64 16 | mode os.FileMode 17 | modificationTime time.Time 18 | isDir bool 19 | }{ 20 | { 21 | description: "folder test", 22 | name: "abc", 23 | size: 2, 24 | mode: 0777, 25 | modificationTime: time.Now(), 26 | isDir: false, 27 | }, 28 | { 29 | description: "dir test", 30 | name: "abc", 31 | size: 2, 32 | mode: 0777, 33 | modificationTime: time.Now(), 34 | isDir: true, 35 | }, 36 | } 37 | 38 | for _, useCase := range useCases { 39 | info := NewInfo(useCase.name, useCase.size, useCase.mode, useCase.modificationTime, useCase.isDir) 40 | assert.EqualValues(t, useCase.name, info.Name()) 41 | assert.EqualValues(t, useCase.size, info.Size()) 42 | assert.EqualValues(t, useCase.mode, info.Mode()) 43 | assert.EqualValues(t, useCase.modificationTime, info.ModTime()) 44 | assert.EqualValues(t, useCase.isDir, info.IsDir()) 45 | 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /file/list.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "context" 5 | "github.com/pkg/errors" 6 | "github.com/viant/afs/object" 7 | "github.com/viant/afs/option" 8 | "github.com/viant/afs/storage" 9 | "github.com/viant/afs/url" 10 | "os" 11 | "path" 12 | ) 13 | 14 | //List list directory or returns a file Info 15 | func List(ctx context.Context, URL string, options ...storage.Option) ([]storage.Object, error) { 16 | baseURL, filePath := url.Base(URL, Scheme) 17 | file, err := os.Open(Path(filePath)) 18 | if err != nil { 19 | return nil, errors.Wrap(err, "unable to open "+filePath) 20 | } 21 | match, page := option.GetListOptions(options) 22 | defer func() { _ = file.Close() }() 23 | stat, err := file.Stat() 24 | if err != nil { 25 | return nil, err 26 | } 27 | if !stat.IsDir() { 28 | return []storage.Object{ 29 | object.New(URL, stat, nil), 30 | }, nil 31 | } 32 | files, err := file.Readdir(0) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | var result = make([]storage.Object, 0) 38 | result = append(result, object.New(URL, stat, nil)) 39 | for _, fileInfo := range files { 40 | if !match(filePath, fileInfo) { 41 | continue 42 | } 43 | page.Increment() 44 | if page.ShallSkip() { 45 | continue 46 | } 47 | 48 | if fileInfo.Mode()&os.ModeSymlink > 0 { 49 | linkname, err := os.Readlink(path.Join(filePath, fileInfo.Name())) 50 | if err == nil { 51 | fileInfo = NewInfo(fileInfo.Name(), fileInfo.Size(), fileInfo.Mode(), fileInfo.ModTime(), fileInfo.IsDir(), object.NewLink(linkname, url.Join(baseURL, linkname), fileInfo)) 52 | } 53 | } 54 | fileURL := url.Join(baseURL, filePath, fileInfo.Name()) 55 | result = append(result, object.New(fileURL, fileInfo, nil)) 56 | if page.HasReachedLimit() { 57 | break 58 | } 59 | } 60 | return result, nil 61 | } 62 | -------------------------------------------------------------------------------- /file/manager.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/storage" 6 | "io" 7 | "os" 8 | ) 9 | 10 | type manager struct{} 11 | 12 | func (s *manager) List(ctx context.Context, URL string, options ...storage.Option) ([]storage.Object, error) { 13 | return List(ctx, URL, options...) 14 | } 15 | 16 | func (s *manager) Upload(ctx context.Context, URL string, mode os.FileMode, reader io.Reader, options ...storage.Option) error { 17 | return Upload(ctx, URL, mode, reader, options...) 18 | } 19 | 20 | func (s *manager) Open(ctx context.Context, object storage.Object, options ...storage.Option) (io.ReadCloser, error) { 21 | return Open(ctx, object, options...) 22 | } 23 | 24 | func (s *manager) OpenURL(ctx context.Context, URL string, options ...storage.Option) (io.ReadCloser, error) { 25 | return OpenURL(ctx, URL, options...) 26 | } 27 | 28 | func (s *manager) Delete(ctx context.Context, URL string, options ...storage.Option) error { 29 | return Delete(ctx, URL, options...) 30 | } 31 | 32 | func (s *manager) Create(ctx context.Context, URL string, mode os.FileMode, isDir bool, options ...storage.Option) error { 33 | return Create(ctx, URL, mode, isDir, options) 34 | } 35 | 36 | func (s *manager) Move(ctx context.Context, sourceURL, destURL string, options ...storage.Option) error { 37 | return Move(ctx, sourceURL, destURL, options...) 38 | } 39 | 40 | func (s *manager) NewWriter(_ context.Context, URL string, mode os.FileMode, options ...storage.Option) (io.WriteCloser, error) { 41 | return NewWriter(nil, URL, mode, options...) 42 | } 43 | 44 | func (s *manager) Close() error { 45 | return nil 46 | } 47 | 48 | func (s *manager) Scheme() string { 49 | return Scheme 50 | } 51 | 52 | //New returns a file manager 53 | func New() storage.Manager { 54 | return &manager{} 55 | } 56 | -------------------------------------------------------------------------------- /file/manager_test.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "context" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/viant/afs/storage" 7 | "io/ioutil" 8 | "os" 9 | "path" 10 | "strings" 11 | "testing" 12 | ) 13 | 14 | func TestManager_Copy(t *testing.T) { 15 | 16 | const expectedContent = "this is test" 17 | 18 | ctx := context.Background() 19 | storager := New() 20 | tempDir := os.TempDir() 21 | 22 | var useCases = []struct { 23 | description string 24 | source string 25 | dest string 26 | hasError bool 27 | }{ 28 | 29 | { 30 | description: "simple copy", 31 | source: path.Join(tempDir, "afs", "copy", "src", "bar1.txt"), 32 | dest: path.Join(tempDir, "afs", "copy", "URL", "bar1.txt"), 33 | }, 34 | { 35 | description: "simple copy - missing source", 36 | hasError: true, 37 | source: path.Join(tempDir, "afs", "copy", "src", "bar2.txt"), 38 | dest: path.Join(tempDir, "afs", "copy", "URL", "bar2.txt"), 39 | }, 40 | } 41 | 42 | for _, useCase := range useCases { 43 | 44 | _ = storager.Delete(ctx, useCase.source) 45 | _ = storager.Delete(ctx, useCase.dest) 46 | if !useCase.hasError { 47 | err := storager.Upload(ctx, useCase.source, 0644, strings.NewReader(expectedContent)) 48 | assert.Nil(t, err, useCase.description) 49 | } 50 | 51 | mover, _ := storager.(storage.Mover) 52 | 53 | err := mover.Move(ctx, useCase.source, useCase.dest) 54 | 55 | if useCase.hasError { 56 | assert.NotNil(t, err, useCase.description) 57 | continue 58 | } 59 | 60 | if !assert.Nil(t, err) { 61 | continue 62 | } 63 | 64 | list, err := storager.List(ctx, useCase.dest) 65 | if !assert.Nil(t, err) { 66 | continue 67 | } 68 | assert.EqualValues(t, 1, len(list), useCase.description) 69 | reader, err := storager.Open(ctx, list[0]) 70 | if !assert.Nil(t, err) { 71 | continue 72 | } 73 | actualContent, err := ioutil.ReadAll(reader) 74 | _ = reader.Close() 75 | if !assert.Nil(t, err) { 76 | continue 77 | } 78 | assert.EqualValues(t, expectedContent, string(actualContent)) 79 | err = storager.Delete(ctx, useCase.dest) 80 | assert.Nil(t, err) 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /file/mode.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | ) 8 | 9 | //NewMode returns a new file mode for supplied attributes 10 | func NewMode(attributes string) (os.FileMode, error) { 11 | var result os.FileMode 12 | if len(attributes) != 10 { 13 | return result, fmt.Errorf("invalid attribute length %v %v", attributes, len(attributes)) 14 | } 15 | 16 | const fileType = "dalTLDpSugct?" 17 | var fileModePosition = strings.Index(fileType, string(attributes[0])) 18 | 19 | if fileModePosition != -1 { 20 | result = 1 << uint(32-1-fileModePosition) 21 | } 22 | 23 | const filePermission = "rwxrwxrwx" 24 | for i, c := range filePermission { 25 | if c == rune(attributes[i+1]) { 26 | result = result | 1< 0 { 24 | flag |= int(flagOpt) 25 | } else { //by default append is file exists 26 | if exists { 27 | flag |= os.O_APPEND 28 | } 29 | } 30 | if !exists { 31 | parent, _ := path.Split(location) 32 | EnsureParentPathExists(parent, DefaultDirOsMode) 33 | } 34 | return os.OpenFile(location, flag, mode) 35 | } 36 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/viant/afs 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/go-errors/errors v1.4.2 7 | github.com/pkg/errors v0.9.1 8 | github.com/stretchr/testify v1.8.1 9 | github.com/viant/toolbox v0.34.6-0.20221112031702-3e7cdde7f888 10 | github.com/viant/xunsafe v0.9.2 11 | golang.org/x/crypto v0.3.0 12 | ) 13 | 14 | require ( 15 | cloud.google.com/go v0.65.0 // indirect 16 | github.com/davecgh/go-spew v1.1.1 // indirect 17 | github.com/kr/pretty v0.1.0 // indirect 18 | github.com/pmezard/go-difflib v1.0.0 // indirect 19 | github.com/viant/xreflect v0.0.0-20230303201326-f50afb0feb0d // indirect 20 | golang.org/x/sys v0.2.0 // indirect 21 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 22 | gopkg.in/yaml.v2 v2.4.0 // indirect 23 | gopkg.in/yaml.v3 v3.0.1 // indirect 24 | ) 25 | -------------------------------------------------------------------------------- /http/README.md: -------------------------------------------------------------------------------- 1 | # Http storage 2 | 3 | This package defines http base storage 4 | 5 | - [Usage](#usage) 6 | - [Options](#options) 7 | * [Http Client Provider](#http-client-provider) 8 | * [Basic Auth](#basic-auth) 9 | * [Custom Header](#custom-header) 10 | * [Response](#response) 11 | 12 | ### Usage 13 | 14 | ### afs.Service 15 | 16 | ```go 17 | 18 | ctx := context.Background() 19 | service := afs.New() 20 | service.Copy() 21 | 22 | reader, err := service.DownloadWithURL(ctx, URL) 23 | err := service.Create(ctx, useCase.URL, 0744, false, reader) 24 | err := service.Upload(ctx, useCase.URL, 0744, reader) 25 | err := service.Delete(ctx, URL) 26 | ``` 27 | 28 | 29 | 30 | 31 | 32 | ### Options 33 | 34 | ##### Http Client Provider 35 | 36 | ```go 37 | 38 | ctx := context.Background() 39 | var clientProvider = func(baseURL string, options ...storage.Option) (*http.Client, error) { 40 | return http.DefaultClient, nil 41 | } 42 | service := http.New(clientProvider) 43 | reader, err := service.DownloadWithURL(ctx, URL) 44 | err := service.Delete(ctx, URL, clientProvider) 45 | 46 | ``` 47 | 48 | ##### Basic Auth 49 | 50 | ```go 51 | 52 | ctx := context.Background() 53 | authProvider := option.NewBasicAuth("user", "password") 54 | service := http.New() 55 | reader, err := service.DownloadWithURL(ctx, URL, authProvider) 56 | ``` 57 | 58 | ##### Custom Header 59 | 60 | ```go 61 | 62 | ctx := context.Background() 63 | header := htttp.Header{} 64 | header.Set("Set-Cookie", "id=a3fWa; Expires=Wed, 21 Oct 2035 07:28:00 GMT; Secure; HttpOnly") 65 | reader, err := manager.DownloadWithURL(ctx, URL, header) 66 | 67 | 68 | ``` 69 | 70 | ##### Response 71 | 72 | ```go 73 | ctx := context.Background() 74 | response := &http.Response{} 75 | service := http.New() 76 | reader, err := service.DownloadWithURL(ctx, URL, response) 77 | err := service.Create(ctx, useCase.URL, 0744, false, reader, response) 78 | err := service.Upload(ctx, useCase.URL, 0744, reader, response) 79 | err := service.Delete(ctx, URL. response) 80 | 81 | ``` 82 | -------------------------------------------------------------------------------- /http/client.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "github.com/viant/afs/option" 5 | "github.com/viant/afs/storage" 6 | "github.com/viant/afs/url" 7 | "net/http" 8 | ) 9 | 10 | //ClientProvider represents clinet provider option 11 | type ClientProvider func(baseURL string, options ...storage.Option) (*http.Client, error) 12 | 13 | func (s *manager) getClient(baseURL string, options ...storage.Option) (*http.Client, error) { 14 | baseURL, _ = url.Base(baseURL, Scheme) 15 | s.mux.Lock() 16 | defer s.mux.Unlock() 17 | client, ok := s.baseURLClients[baseURL] 18 | if ok { 19 | return client, nil 20 | } 21 | if len(s.options) > 0 { 22 | options = append(s.options, options...) 23 | } 24 | var clientProvider ClientProvider 25 | option.Assign(options, &clientProvider) 26 | if clientProvider == nil { 27 | if s.client == nil { 28 | s.client = http.DefaultClient 29 | } 30 | return s.client, nil 31 | } 32 | var err error 33 | if clientProvider != nil { 34 | if client, err = clientProvider(baseURL, options); err != nil { 35 | return nil, err 36 | } 37 | s.baseURLClients[baseURL] = client 38 | } 39 | return client, nil 40 | } 41 | 42 | func (s *manager) authWithBasicCred(request *http.Request, authenticator option.BasicAuth) { 43 | if authenticator == nil { 44 | return 45 | } 46 | username, password := authenticator.Credentials() 47 | request.SetBasicAuth(username, password) 48 | } 49 | -------------------------------------------------------------------------------- /http/client_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/viant/afs/storage" 7 | "net" 8 | "net/http" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func TestClientProvider(t *testing.T) { 14 | const testTimeout = 40000 15 | testError := false 16 | 17 | var testProvider = func(baseURL string, options ...storage.Option) (*http.Client, error) { 18 | roundTripper := http.Transport{ 19 | Proxy: http.ProxyFromEnvironment, 20 | DialContext: (&net.Dialer{ 21 | Timeout: testTimeout, 22 | KeepAlive: 10000, 23 | }).DialContext} 24 | client := &http.Client{ 25 | Transport: &roundTripper, 26 | Timeout: testTimeout, 27 | } 28 | if testError { 29 | return nil, fmt.Errorf("test error") 30 | } 31 | return client, nil 32 | } 33 | 34 | var useCases = []struct { 35 | description string 36 | baseURL string 37 | clientTimeout time.Duration 38 | options []storage.Option 39 | hasError bool 40 | }{ 41 | { 42 | description: "custom client", 43 | baseURL: "", 44 | clientTimeout: testTimeout, 45 | options: []storage.Option{ 46 | testProvider, 47 | }, 48 | }, 49 | { 50 | description: "default HTTP client", 51 | baseURL: "/foo", 52 | clientTimeout: http.DefaultClient.Timeout, 53 | }, 54 | { 55 | description: "test error", 56 | baseURL: "/foo", 57 | hasError: true, 58 | options: []storage.Option{ 59 | testProvider, 60 | }, 61 | }, 62 | } 63 | 64 | for _, useCase := range useCases { 65 | manager := newManager(useCase.options...) 66 | testError = useCase.hasError 67 | client1, err := manager.getClient(useCase.baseURL) 68 | if testError { 69 | assert.NotNil(t, err, useCase.description) 70 | continue 71 | } 72 | assert.Nil(t, err, useCase.description) 73 | assert.EqualValues(t, useCase.clientTimeout, int(client1.Timeout), useCase.description) 74 | client2, _ := manager.getClient(useCase.baseURL) 75 | assert.Equal(t, client1, client2, useCase.description) 76 | _ = manager.Close() 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /http/create.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/viant/afs/option" 7 | "github.com/viant/afs/storage" 8 | "io" 9 | "net/http" 10 | "os" 11 | ) 12 | 13 | //Create send post request 14 | func (s *manager) Create(ctx context.Context, URL string, mode os.FileMode, isDir bool, options ...storage.Option) error { 15 | var reader io.Reader 16 | option.Assign(options, &reader) 17 | request, err := http.NewRequest(http.MethodPost, URL, reader) 18 | if err != nil { 19 | return err 20 | } 21 | response, err := s.run(ctx, URL, request, options...) 22 | if err != nil { 23 | return err 24 | } 25 | s.closeResponse(response) 26 | if IsStatusOK(response) { 27 | return nil 28 | } 29 | return fmt.Errorf("invalid status code: %v", response.StatusCode) 30 | 31 | } 32 | -------------------------------------------------------------------------------- /http/create_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/afs/url" 8 | "io/ioutil" 9 | "net/http" 10 | "strings" 11 | "testing" 12 | ) 13 | 14 | func TestManager_Create(t *testing.T) { 15 | 16 | testPort := 8876 17 | baseURL := fmt.Sprintf("http://localhost:%v", testPort) 18 | ctx := context.Background() 19 | var useCases = []struct { 20 | description string 21 | URL string 22 | expect string 23 | putParrot *http.Response 24 | hasError bool 25 | }{ 26 | { 27 | description: "asset create", 28 | URL: url.Join(baseURL, "/foo/bar.txt"), 29 | expect: "test is test", 30 | putParrot: &http.Response{ 31 | StatusCode: 200, 32 | Body: ioutil.NopCloser(strings.NewReader("test is test")), 33 | }, 34 | }, 35 | { 36 | description: "not found error download", 37 | URL: url.Join(baseURL, "/foo/error.txt"), 38 | hasError: true, 39 | }, 40 | } 41 | 42 | parrots := map[string]*http.Response{} 43 | for _, useCase := range useCases { 44 | addURLParrots(http.MethodPost, useCase.URL, useCase.putParrot, parrots) 45 | } 46 | go startServer(testPort, parrotHandler(parrots)) 47 | 48 | for _, useCase := range useCases { 49 | manager := newManager() 50 | err := manager.Create(ctx, useCase.URL, 0744, false, strings.NewReader(useCase.expect)) 51 | if useCase.hasError { 52 | assert.NotNil(t, err, useCase.description) 53 | continue 54 | } 55 | if !assert.Nil(t, err, useCase.description) { 56 | continue 57 | } 58 | 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /http/delete.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/viant/afs/storage" 7 | "net/http" 8 | ) 9 | 10 | //Delete sends delete method with supplied URL 11 | func (s *manager) Delete(ctx context.Context, URL string, options ...storage.Option) error { 12 | request, err := http.NewRequest(http.MethodDelete, URL, nil) 13 | if err != nil { 14 | return err 15 | } 16 | response, err := s.run(ctx, URL, request, options...) 17 | if err != nil { 18 | return err 19 | } 20 | defer s.closeResponse(response) 21 | if IsStatusOK(response) { 22 | return nil 23 | } 24 | return fmt.Errorf("invalid status code: %v", response.StatusCode) 25 | } 26 | -------------------------------------------------------------------------------- /http/delete_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/afs/url" 8 | "io/ioutil" 9 | "net/http" 10 | "strings" 11 | "testing" 12 | ) 13 | 14 | func TestManager_Delete(t *testing.T) { 15 | 16 | testPort := 8878 17 | baseURL := fmt.Sprintf("http://localhost:%v", testPort) 18 | ctx := context.Background() 19 | var useCases = []struct { 20 | description string 21 | URL string 22 | expect string 23 | putParrot *http.Response 24 | hasError bool 25 | }{ 26 | { 27 | description: "asset delete", 28 | URL: url.Join(baseURL, "/foo/bar.txt"), 29 | expect: "test is test", 30 | putParrot: &http.Response{ 31 | StatusCode: 200, 32 | Body: ioutil.NopCloser(strings.NewReader("test is test")), 33 | }, 34 | }, 35 | { 36 | description: "not found error delete", 37 | URL: url.Join(baseURL, "/foo/error.txt"), 38 | hasError: true, 39 | }, 40 | } 41 | 42 | parrots := map[string]*http.Response{} 43 | for _, useCase := range useCases { 44 | addURLParrots(http.MethodDelete, useCase.URL, useCase.putParrot, parrots) 45 | } 46 | go startServer(testPort, parrotHandler(parrots)) 47 | 48 | for _, useCase := range useCases { 49 | manager := newManager() 50 | err := manager.Delete(ctx, useCase.URL) 51 | if useCase.hasError { 52 | assert.NotNil(t, err, useCase.description) 53 | continue 54 | } 55 | if !assert.Nil(t, err, useCase.description) { 56 | continue 57 | } 58 | 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /http/doc.go: -------------------------------------------------------------------------------- 1 | //Package http defines simple http based storage operation 2 | package http 3 | -------------------------------------------------------------------------------- /http/exists.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/storage" 6 | "net/http" 7 | ) 8 | 9 | //Exists checks if asset exists 10 | func (s *manager) Exists(ctx context.Context, URL string, options ...storage.Option) (bool, error) { 11 | 12 | for _, method := range []string{http.MethodHead, http.MethodGet, http.MethodPost, http.MethodPut} { 13 | request, err := http.NewRequest(method, URL, nil) 14 | if err != nil { 15 | return false, err 16 | } 17 | response, err := s.run(ctx, URL, request, options...) 18 | if err != nil { 19 | return false, err 20 | } 21 | s.closeResponse(response) 22 | if IsStatusOK(response) { 23 | return true, nil 24 | } 25 | } 26 | return false, nil 27 | } 28 | -------------------------------------------------------------------------------- /http/exists_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/afs/url" 8 | "io/ioutil" 9 | "net/http" 10 | "strings" 11 | "testing" 12 | ) 13 | 14 | func TestManager_Exists(t *testing.T) { 15 | testPort := 8873 16 | baseURL := fmt.Sprintf("http://localhost:%v", testPort) 17 | 18 | var useCases = []struct { 19 | description string 20 | URL string 21 | getParrot *http.Response 22 | headParrot *http.Response 23 | exists bool 24 | hasError bool 25 | }{ 26 | { 27 | description: "head based exists", 28 | URL: url.Join(baseURL, "/foo/asset1.txt"), 29 | headParrot: &http.Response{ 30 | StatusCode: http.StatusOK, 31 | Body: ioutil.NopCloser(strings.NewReader("test")), 32 | }, 33 | exists: true, 34 | }, 35 | { 36 | description: "get based exists", 37 | URL: url.Join(baseURL, "/foo/asset2.txt"), 38 | getParrot: &http.Response{ 39 | StatusCode: http.StatusOK, 40 | Body: ioutil.NopCloser(strings.NewReader("test")), 41 | }, 42 | exists: true, 43 | }, 44 | { 45 | description: "not found", 46 | URL: url.Join(baseURL, "/foo/asset3.txt"), 47 | }, 48 | { 49 | description: "http error", 50 | URL: url.Join("http://localhost:2222", "/foo/asset4.txt"), 51 | hasError: true, 52 | }, 53 | } 54 | 55 | ctx := context.Background() 56 | parrots := map[string]*http.Response{} 57 | 58 | for _, useCase := range useCases { 59 | if useCase.getParrot != nil { 60 | addGetURLParrots(useCase.URL, useCase.getParrot, parrots) 61 | } 62 | if useCase.headParrot != nil { 63 | addHeadURLParrots(useCase.URL, useCase.headParrot, parrots) 64 | } 65 | } 66 | go startServer(testPort, parrotHandler(parrots)) 67 | 68 | for _, useCase := range useCases { 69 | manager := newManager() 70 | actual, err := manager.Exists(ctx, useCase.URL) 71 | if useCase.hasError { 72 | assert.NotNil(t, err, useCase.description) 73 | continue 74 | } 75 | if !assert.Nil(t, err, useCase.description) { 76 | continue 77 | } 78 | assert.EqualValues(t, useCase.exists, actual) 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /http/header.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | "time" 7 | ) 8 | 9 | func (s *manager) setHeader(request *http.Request, header http.Header) { 10 | if len(header) == 0 { 11 | return 12 | } 13 | if len(request.Header) == 0 { 14 | request.Header = header 15 | } 16 | for k, v := range header { 17 | request.Header[k] = v 18 | } 19 | } 20 | 21 | 22 | func (s *manager) setCookies(cookies []*http.Cookie, request *http.Request) { 23 | if len(cookies) > 0 { 24 | for _, cookie := range cookies { 25 | request.AddCookie(cookie) 26 | } 27 | } 28 | } 29 | 30 | 31 | var timeLayouts = []string{"Mon, 02 Jan 2006 15:04:05 GMT", time.RFC850, time.ANSIC} 32 | 33 | //HeaderTime returns time for header key 34 | func HeaderTime(header http.Header, key string, defaultValue time.Time) time.Time { 35 | if len(header) == 0 { 36 | return defaultValue 37 | } 38 | value, ok := header[key] 39 | if !ok { 40 | key = strings.ToLower(key) 41 | for k, v := range header { 42 | if strings.ToLower(k) == key { 43 | value = v 44 | } 45 | } 46 | } 47 | 48 | if len(value) == 0 { 49 | return defaultValue 50 | } 51 | if result, err := ParseHTTPDate(value[0]); err == nil { 52 | return result 53 | } 54 | return defaultValue 55 | } 56 | 57 | //ParseHTTPDate parses date assigned 58 | func ParseHTTPDate(value string) (result time.Time, err error) { 59 | for i := range timeLayouts { 60 | if result, err = time.Parse(timeLayouts[i], value); err == nil { 61 | return result, nil 62 | } 63 | } 64 | return result, err 65 | } 66 | -------------------------------------------------------------------------------- /http/list.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/viant/afs/file" 7 | "github.com/viant/afs/object" 8 | "github.com/viant/afs/storage" 9 | "github.com/viant/afs/url" 10 | "net/http" 11 | "path" 12 | "time" 13 | ) 14 | 15 | const lastModifiedHeader = "Last-Modified" 16 | 17 | var assetMode, _ = file.NewMode("-rw-r--r--") 18 | 19 | func (s *manager) List(ctx context.Context, URL string, options ...storage.Option) ([]storage.Object, error) { 20 | request, err := http.NewRequest(http.MethodGet, URL, nil) 21 | if err != nil { 22 | return nil, err 23 | } 24 | response, err := s.run(ctx, URL, request, options...) 25 | if err != nil { 26 | return nil, err 27 | } 28 | defer s.closeResponse(response) 29 | if !IsStatusOK(response) { 30 | return nil, fmt.Errorf("resource not found, statusCode: %v, url: %v", response.StatusCode, URL) 31 | } 32 | _, URLPath := url.Base(URL, Scheme) 33 | _, name := path.Split(URLPath) 34 | modified := HeaderTime(response.Header, lastModifiedHeader, time.Now()) 35 | info := file.NewInfo(name, response.ContentLength, assetMode, modified, false) 36 | asset := object.New(URL, info, response) 37 | return []storage.Object{ 38 | asset, 39 | }, nil 40 | } 41 | -------------------------------------------------------------------------------- /http/manager.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "github.com/viant/afs/storage" 5 | "net/http" 6 | "sync" 7 | ) 8 | 9 | type manager struct { 10 | client *http.Client 11 | mux sync.Mutex 12 | baseURLClients map[string]*http.Client 13 | options []storage.Option 14 | } 15 | 16 | //CloseIdleConnections closes iddle connections 17 | func CloseIdleConnections(client interface{}) { 18 | type closeIdler interface { 19 | CloseIdleConnections() 20 | } 21 | if closer, ok := client.(closeIdler); ok { 22 | closer.CloseIdleConnections() 23 | } 24 | } 25 | 26 | //Close closes mananger 27 | func (s *manager) Close() error { 28 | if s.client != nil { 29 | CloseIdleConnections(s.client) 30 | } 31 | for _, client := range s.baseURLClients { 32 | CloseIdleConnections(client) 33 | } 34 | return nil 35 | } 36 | 37 | //Scheme returns schmea 38 | func (s *manager) Scheme() string { 39 | return Scheme 40 | } 41 | 42 | func newManager(options ...storage.Option) *manager { 43 | return &manager{ 44 | options: options, 45 | baseURLClients: make(map[string]*http.Client), 46 | } 47 | } 48 | 49 | //New creates HTTP manager 50 | func New(options ...storage.Option) storage.Manager { 51 | return newManager(options...) 52 | } 53 | -------------------------------------------------------------------------------- /http/open.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/viant/afs/option" 7 | "github.com/viant/afs/storage" 8 | "io" 9 | "net/http" 10 | ) 11 | 12 | //Open downloads asset for supplied object 13 | func (s *manager) Open(ctx context.Context, object storage.Object, options ...storage.Option) (io.ReadCloser, error) { 14 | return s.OpenURL(ctx, object.URL(), options...) 15 | } 16 | 17 | //Open downloads asset for supplied object 18 | func (s *manager) OpenURL(ctx context.Context, URL string, options ...storage.Option) (io.ReadCloser, error) { 19 | var method option.HTTPMethod 20 | if _, ok := option.Assign(options, &method); !ok { 21 | method = http.MethodGet 22 | } 23 | var reader io.Reader 24 | option.Assign(options, &reader) 25 | request, err := http.NewRequest(string(method), URL, reader) 26 | if err != nil { 27 | return nil, err 28 | } 29 | response, err := s.run(ctx, URL, request, options...) 30 | if err != nil { 31 | return nil, err 32 | } 33 | var status = &option.Status{} 34 | option.Assign(options, &status) 35 | status.Code = response.StatusCode 36 | if response.Body != nil { 37 | return response.Body, nil 38 | } 39 | return nil, fmt.Errorf("invalid status code: %v", response.StatusCode) 40 | } 41 | -------------------------------------------------------------------------------- /http/open_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/afs/url" 8 | "io/ioutil" 9 | "net/http" 10 | "strings" 11 | 12 | "testing" 13 | ) 14 | 15 | func TestManager_Download(t *testing.T) { 16 | 17 | testPort := 8875 18 | baseURL := fmt.Sprintf("http://localhost:%v", testPort) 19 | ctx := context.Background() 20 | var useCases = []struct { 21 | description string 22 | URL string 23 | expect string 24 | putParrot *http.Response 25 | hasError bool 26 | }{ 27 | { 28 | description: "asset download", 29 | URL: url.Join(baseURL, "/foo/bar.txt"), 30 | expect: "test is test", 31 | putParrot: &http.Response{ 32 | StatusCode: 200, 33 | Body: ioutil.NopCloser(strings.NewReader("test is test")), 34 | }, 35 | }, 36 | { 37 | description: "not found error download", 38 | URL: url.Join(baseURL, "/foo/error.txt"), 39 | hasError: true, 40 | }, 41 | } 42 | 43 | parrots := map[string]*http.Response{} 44 | for _, useCase := range useCases { 45 | addURLParrots(http.MethodGet, useCase.URL, useCase.putParrot, parrots) 46 | } 47 | go startServer(testPort, parrotHandler(parrots)) 48 | 49 | for _, useCase := range useCases { 50 | manager := newManager() 51 | reader, err := manager.OpenURL(ctx, useCase.URL) 52 | if useCase.hasError { 53 | assert.NotNil(t, err, useCase.description) 54 | continue 55 | } 56 | if !assert.Nil(t, err, useCase.description) { 57 | continue 58 | } 59 | data, err := ioutil.ReadAll(reader) 60 | assert.Nil(t, err) 61 | _ = reader.Close() 62 | assert.EqualValues(t, useCase.expect, string(data), useCase.description) 63 | 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /http/parrot_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "fmt" 5 | "github.com/viant/afs/url" 6 | "io/ioutil" 7 | "net/http" 8 | ) 9 | 10 | func addGetURLParrots(URL string, response *http.Response, result map[string]*http.Response) { 11 | addURLParrots(http.MethodGet, URL, response, result) 12 | } 13 | 14 | func addHeadURLParrots(URL string, response *http.Response, result map[string]*http.Response) { 15 | addURLParrots(http.MethodHead, URL, response, result) 16 | } 17 | 18 | func addURLParrots(method, URL string, response *http.Response, result map[string]*http.Response) { 19 | if response == nil { 20 | return 21 | } 22 | _, URLPath := url.Base(URL, Scheme) 23 | key := method + ":" + URLPath 24 | result[key] = response 25 | } 26 | 27 | func parrotHandler(responses map[string]*http.Response) func(writer http.ResponseWriter, request *http.Request) { 28 | return func(writer http.ResponseWriter, request *http.Request) { 29 | key := fmt.Sprintf("%v:%v", request.Method, request.URL.Path) 30 | response, ok := responses[key] 31 | 32 | if !ok { 33 | http.NotFound(writer, request) 34 | return 35 | } 36 | 37 | if len(response.Header) > 0 { 38 | for k, v := range response.Header { 39 | writer.Header().Set(k, v[0]) 40 | } 41 | } 42 | if data, err := ioutil.ReadAll(response.Body); err == nil { 43 | _, _ = writer.Write(data) 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /http/provider.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "github.com/viant/afs/storage" 5 | ) 6 | 7 | //Provider returns a http manager 8 | func Provider(options ...storage.Option) (storage.Manager, error) { 9 | return New(options...), nil 10 | } 11 | -------------------------------------------------------------------------------- /http/response.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | func (s *manager) closeResponse(response *http.Response) { 8 | if response.Body != nil { 9 | _ = response.Body.Close() 10 | } 11 | 12 | } 13 | -------------------------------------------------------------------------------- /http/run.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/option" 6 | "github.com/viant/afs/storage" 7 | "net/http" 8 | ) 9 | 10 | func (s *manager) run(ctx context.Context, URL string, request *http.Request, options ...storage.Option) (*http.Response, error) { 11 | var clientProvider ClientProvider 12 | var basicAuthProvider option.BasicAuth 13 | resp := &http.Response{} 14 | header := http.Header{} 15 | cookies :=[]*http.Cookie{} 16 | option.Assign(options, &clientProvider, &basicAuthProvider, &header, &resp, &cookies) 17 | s.setHeader(request, header) 18 | s.setCookies(cookies, request) 19 | s.authWithBasicCred(request, basicAuthProvider) 20 | client, err := s.getClient(URL, options...) 21 | if err != nil { 22 | return nil, err 23 | } 24 | if ctx != nil { 25 | request.WithContext(ctx) 26 | } 27 | response, err := client.Do(request) 28 | if err == nil && resp != nil { 29 | *resp = *response 30 | } 31 | return response, err 32 | } 33 | -------------------------------------------------------------------------------- /http/run_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/afs/url" 8 | "io/ioutil" 9 | "net/http" 10 | "strings" 11 | "testing" 12 | ) 13 | 14 | func TestManager_Run(t *testing.T) { 15 | 16 | testPort := 8880 17 | baseURL := fmt.Sprintf("http://localhost:%v", testPort) 18 | ctx := context.Background() 19 | var useCases = []struct { 20 | description string 21 | URL string 22 | expect string 23 | putParrot *http.Response 24 | hasError bool 25 | }{ 26 | { 27 | description: "asset download", 28 | URL: url.Join(baseURL, "/foo/bar.txt"), 29 | expect: "test is test", 30 | 31 | putParrot: &http.Response{ 32 | StatusCode: 200, 33 | Body: ioutil.NopCloser(strings.NewReader("test is test")), 34 | }, 35 | }, 36 | { 37 | description: "not found error download", 38 | URL: url.Join(baseURL, "/foo/error.txt"), 39 | hasError: true, 40 | }, 41 | } 42 | 43 | parrots := map[string]*http.Response{} 44 | for _, useCase := range useCases { 45 | addURLParrots(http.MethodGet, useCase.URL, useCase.putParrot, parrots) 46 | } 47 | go startServer(testPort, parrotHandler(parrots)) 48 | 49 | for _, useCase := range useCases { 50 | manager := newManager() 51 | response := &http.Response{} 52 | reader, err := manager.OpenURL(ctx, useCase.URL, response) 53 | if useCase.hasError { 54 | assert.NotNil(t, err, useCase.description) 55 | continue 56 | } 57 | if !assert.Nil(t, err, useCase.description) { 58 | continue 59 | } 60 | data, err := ioutil.ReadAll(reader) 61 | assert.Nil(t, err) 62 | _ = reader.Close() 63 | assert.EqualValues(t, http.StatusOK, response.StatusCode) 64 | assert.EqualValues(t, useCase.expect, string(data), useCase.description) 65 | 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /http/scheme.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | //Scheme represents http url scheme 4 | const Scheme = "http" 5 | 6 | //SecureScheme represents secure http url scheme 7 | const SecureScheme = "https" 8 | -------------------------------------------------------------------------------- /http/server_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | func startServer(port int, handler func(writer http.ResponseWriter, request *http.Request)) { 9 | httpServer := &http.Server{Addr: fmt.Sprintf(":%v", port), Handler: newHandler(handler)} 10 | _ = httpServer.ListenAndServe() 11 | } 12 | 13 | type handler struct { 14 | onRequest func(writer http.ResponseWriter, request *http.Request) 15 | } 16 | 17 | func (h *handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) { 18 | h.onRequest(writer, request) 19 | } 20 | 21 | func newHandler(onRequest func(writer http.ResponseWriter, request *http.Request)) http.Handler { 22 | return &handler{onRequest: onRequest} 23 | } 24 | -------------------------------------------------------------------------------- /http/status.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import "net/http" 4 | 5 | //IsStatusOK returns true if status is 2xxx 6 | func IsStatusOK(response *http.Response) bool { 7 | return response.StatusCode >= 200 && response.StatusCode <= 299 8 | } 9 | -------------------------------------------------------------------------------- /http/upload.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/viant/afs/storage" 7 | "io" 8 | "net/http" 9 | "os" 10 | ) 11 | 12 | //Upload sends put request to supplied URL with provided reader 13 | func (s *manager) Upload(ctx context.Context, URL string, mode os.FileMode, reader io.Reader, options ...storage.Option) error { 14 | request, err := http.NewRequest(http.MethodPut, URL, reader) 15 | if err != nil { 16 | return err 17 | } 18 | response, err := s.run(ctx, URL, request, options...) 19 | if err != nil { 20 | return err 21 | } 22 | defer s.closeResponse(response) 23 | if IsStatusOK(response) { 24 | return nil 25 | } 26 | return fmt.Errorf("invalid status code: %v", response.StatusCode) 27 | } 28 | -------------------------------------------------------------------------------- /http/upload_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/afs/url" 8 | "io/ioutil" 9 | "net/http" 10 | "strings" 11 | "testing" 12 | ) 13 | 14 | func TestManager_Upload(t *testing.T) { 15 | 16 | testPort := 8877 17 | baseURL := fmt.Sprintf("http://localhost:%v", testPort) 18 | ctx := context.Background() 19 | var useCases = []struct { 20 | description string 21 | URL string 22 | expect string 23 | putParrot *http.Response 24 | hasError bool 25 | }{ 26 | { 27 | description: "asset create", 28 | URL: url.Join(baseURL, "/foo/bar.txt"), 29 | expect: "test is test", 30 | putParrot: &http.Response{ 31 | StatusCode: 200, 32 | Body: ioutil.NopCloser(strings.NewReader("test is test")), 33 | }, 34 | }, 35 | { 36 | description: "not found error download", 37 | URL: url.Join(baseURL, "/foo/error.txt"), 38 | hasError: true, 39 | }, 40 | } 41 | 42 | parrots := map[string]*http.Response{} 43 | for _, useCase := range useCases { 44 | addURLParrots(http.MethodPut, useCase.URL, useCase.putParrot, parrots) 45 | } 46 | go startServer(testPort, parrotHandler(parrots)) 47 | 48 | for _, useCase := range useCases { 49 | manager := newManager() 50 | err := manager.Upload(ctx, useCase.URL, 0744, strings.NewReader(useCase.expect)) 51 | if useCase.hasError { 52 | assert.NotNil(t, err, useCase.description) 53 | continue 54 | } 55 | if !assert.Nil(t, err, useCase.description) { 56 | continue 57 | } 58 | 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /init.go: -------------------------------------------------------------------------------- 1 | package afs 2 | 3 | import ( 4 | "github.com/viant/afs/file" 5 | "github.com/viant/afs/http" 6 | "github.com/viant/afs/mem" 7 | "github.com/viant/afs/scp" 8 | "github.com/viant/afs/ssh" 9 | "github.com/viant/afs/tar" 10 | "github.com/viant/afs/zip" 11 | ) 12 | 13 | func init() { 14 | registry := GetRegistry() 15 | registry.Register(file.Scheme, file.Provider) 16 | registry.Register(mem.Scheme, mem.Provider) 17 | registry.Register(http.Scheme, http.Provider) 18 | registry.Register(http.SecureScheme, http.Provider) 19 | registry.Register(scp.Scheme, scp.Provider) 20 | registry.Register(ssh.Scheme, scp.Provider) 21 | registry.Register(zip.Scheme, zip.Provider) 22 | registry.Register(tar.Scheme, tar.Provider) 23 | } 24 | -------------------------------------------------------------------------------- /matcher/basic.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | import ( 4 | "os" 5 | "path" 6 | "regexp" 7 | "strings" 8 | ) 9 | 10 | //Basic represents prefix, suffix or regexp matcher 11 | type Basic struct { 12 | Prefix string `json:",omitempty"` 13 | Suffix string `json:",omitempty"` 14 | Filter string `json:",omitempty"` 15 | Exclusion string `json:",omitempty"` 16 | 17 | Directory *bool `json:",omitempty"` 18 | compiledFilter *regexp.Regexp 19 | comiledExclusion *regexp.Regexp 20 | } 21 | 22 | //Match matcher parent and info with matcher rules 23 | func (r *Basic) Match(parent string, info os.FileInfo) bool { 24 | 25 | if r.Directory != nil { 26 | expectDir := *r.Directory 27 | if expectDir != info.IsDir() { 28 | return false 29 | } 30 | } 31 | if r.Filter != "" && r.compiledFilter == nil { 32 | r.compiledFilter, _ = regexp.Compile(r.Filter) 33 | } 34 | location := path.Join(parent, info.Name()) 35 | if r.compiledFilter != nil { 36 | if !r.compiledFilter.MatchString(location) { 37 | return false 38 | } 39 | } 40 | if r.Prefix != "" { 41 | if !strings.HasPrefix(location, r.Prefix) { 42 | return false 43 | } 44 | } 45 | if r.Suffix != "" { 46 | if !strings.HasSuffix(location, r.Suffix) { 47 | return false 48 | } 49 | } 50 | if r.Exclusion != "" && r.comiledExclusion == nil { 51 | r.comiledExclusion, _ = regexp.Compile(r.Exclusion) 52 | } 53 | if r.comiledExclusion != nil { 54 | if r.comiledExclusion.MatchString(location) { 55 | return false 56 | } 57 | } 58 | return true 59 | } 60 | 61 | //NewBasic creates basic matcher 62 | func NewBasic(prefix, suffix, filter string, dir *bool) (matcher *Basic, err error) { 63 | matcher = &Basic{ 64 | Prefix: prefix, 65 | Suffix: suffix, 66 | Filter: filter, 67 | Directory: dir, 68 | } 69 | if filter != "" { 70 | matcher.compiledFilter, err = regexp.Compile(filter) 71 | } 72 | return matcher, err 73 | } 74 | -------------------------------------------------------------------------------- /matcher/doc.go: -------------------------------------------------------------------------------- 1 | //Package matcher define common resource matcher 2 | package matcher 3 | -------------------------------------------------------------------------------- /matcher/example_test.go: -------------------------------------------------------------------------------- 1 | package matcher_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/viant/afs/file" 6 | "github.com/viant/afs/matcher" 7 | "log" 8 | "time" 9 | ) 10 | 11 | func ExampleBasic_Match() { 12 | basicMatcher, err := matcher.NewBasic("", "", "asset\\d+\\.txt", nil) 13 | if err != nil { 14 | log.Fatal(err) 15 | } 16 | matched := basicMatcher.Match("parent location", file.NewInfo("asset001.txt", 20, 0644, time.Now(), false)) 17 | fmt.Printf("matched: %v\n", matched) 18 | 19 | } 20 | -------------------------------------------------------------------------------- /matcher/filepath.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | import ( 4 | "os" 5 | "path" 6 | "path/filepath" 7 | ) 8 | 9 | /* 10 | Filepath returns filepath based filepath matcher 11 | 12 | // The pattern syntax is: 13 | // 14 | // pattern: 15 | // { term } 16 | // term: 17 | // '*' matches any sequence of non-Separator characters 18 | // '?' matches any single non-Separator character 19 | // '[' [ '^' ] { character-range } ']' 20 | // character class (must be non-empty) 21 | // c matches character c (c != '*', '?', '\\', '[') 22 | // '\\' c matches character c 23 | 24 | */ 25 | func Filepath(pattern string) func(parent string, info os.FileInfo) bool { 26 | return func(parent string, info os.FileInfo) bool { 27 | name := path.Join(parent, info.Name()) 28 | hasMatch, _ := filepath.Match(pattern, name) 29 | return hasMatch 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /matcher/filepath_test.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/viant/afs/file" 6 | "path" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestFilepath(t *testing.T) { 12 | 13 | var useCases = []struct { 14 | description string 15 | pattern string 16 | location string 17 | expect bool 18 | }{ 19 | { 20 | description: "ext match", 21 | pattern: "*.txt", 22 | location: "text.txt", 23 | expect: true, 24 | }, 25 | { 26 | description: "ext not match", 27 | pattern: "*.txt", 28 | location: "text.csv", 29 | expect: false, 30 | }, 31 | { 32 | description: "name match", 33 | pattern: "text*", 34 | location: "text.txt", 35 | expect: true, 36 | }, 37 | { 38 | description: "name not match", 39 | pattern: "bar*", 40 | location: "text.csv", 41 | expect: false, 42 | }, 43 | { 44 | description: "path match", 45 | pattern: "bar/text*", 46 | location: "bar/text.txt", 47 | expect: true, 48 | }, 49 | { 50 | description: "name not match", 51 | pattern: "foo/bar*", 52 | location: "foo/text.csv", 53 | expect: false, 54 | }, 55 | { 56 | description: "wildcard match", 57 | pattern: "bar/t*.txt", 58 | location: "bar/text.txt", 59 | expect: true, 60 | }, 61 | { 62 | description: "wildcard not match", 63 | pattern: "bar/t*.txt", 64 | location: "bar/abc.txt", 65 | expect: false, 66 | }, 67 | } 68 | 69 | for _, useCase := range useCases { 70 | 71 | matcher := Filepath(useCase.pattern) 72 | parent, name := path.Split(useCase.location) 73 | info := file.NewInfo(name, 0, 0644, time.Now(), false) 74 | actual := matcher(parent, info) 75 | assert.EqualValues(t, useCase.expect, actual, useCase.description) 76 | 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /matcher/helper.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | -------------------------------------------------------------------------------- /matcher/modification.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | import ( 4 | "github.com/viant/afs/option" 5 | "os" 6 | "time" 7 | ) 8 | 9 | //Modification represents modification matcher 10 | type Modification struct { 11 | After *time.Time 12 | Before *time.Time 13 | matchers []option.Match 14 | } 15 | 16 | //Match matcher parent and info with matcher rules 17 | func (r *Modification) Match(parent string, info os.FileInfo) bool { 18 | if r.After != nil { 19 | if !r.After.Before(info.ModTime()) { 20 | return false 21 | } 22 | } 23 | if r.Before != nil { 24 | if !r.Before.After(info.ModTime()) { 25 | return false 26 | } 27 | } 28 | for i := range r.matchers { 29 | if !r.matchers[i](parent, info) { 30 | return false 31 | } 32 | } 33 | return true 34 | } 35 | 36 | //NewModification creates a modification time matcher 37 | func NewModification(before, after *time.Time, matchers ...option.Match) *Modification { 38 | return &Modification{ 39 | Before: before, 40 | After: after, 41 | matchers: matchers, 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /mem/README.md: -------------------------------------------------------------------------------- 1 | # mem - in memory file storage 2 | 3 | This package defines in memory file storage. 4 | 5 | ### Usage 6 | 7 | - **[Service](../service.go)** 8 | ```go 9 | service := afs.New() 10 | ctx := context.Background() 11 | err := service.Upload(ctx, "mem://localhost/folder1/asset.txt", 0644, strings.NewReader("some content")) 12 | if err != nil { 13 | log.Fatal(err) 14 | } 15 | objects, err := service.List(ctx, "mem://localhost/folder1/") 16 | if err != nil { 17 | log.Fatal(err) 18 | } 19 | for _, object := range objects { 20 | fmt.Printf("%v %v\n", object.URL(), object.Name()) 21 | } 22 | ``` 23 | 24 | - **[Manager](../storage/manager.go)** 25 | 26 | ```go 27 | manager := mem.New() 28 | ctx := context.Background() 29 | err := manager.Upload(ctx, "mem://localhost/folder1/asset.txt", 0644, strings.NewReader("some content")) 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | objects, err := manager.List(ctx, "mem://localhost/folder1/") 34 | if err != nil { 35 | log.Fatal(err) 36 | } 37 | for _, object := range objects { 38 | fmt.Printf("%v %v\n", object.URL(), object.Name()) 39 | } 40 | ``` 41 | 42 | - **[Storager](../storage/storager.go)** 43 | 44 | ```go 45 | func main() { 46 | ctx := context.Background() 47 | storager := mem.NewStorager("mem://localhost/") 48 | err := storager.Upload(ctx, "folder1/asset1", 0644, []byte("some content")) 49 | if err != nil { 50 | log.Fatal(err) 51 | } 52 | err = storager.Upload(ctx, "folder1/asset2", 0644, []byte("some content")) 53 | if err != nil { 54 | log.Fatal(err) 55 | } 56 | 57 | fileInfos, err := storager.List(ctx, "folder1/", 0, 0) 58 | if err != nil { 59 | log.Fatal(err) 60 | } 61 | for _, info := range fileInfos { 62 | fmt.Printf("%v\n", info.Name()) 63 | } 64 | } 65 | ``` 66 | 67 | ### Options 68 | 69 | - [Errors](../option/error.go) -------------------------------------------------------------------------------- /mem/create.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/storage" 6 | "io" 7 | "os" 8 | ) 9 | 10 | //Create creates a new file or directory 11 | func (s *storager) Create(ctx context.Context, location string, mode os.FileMode, reader io.Reader, isDir bool, options ...storage.Option) error { 12 | root := s.Root 13 | if isDir { 14 | _, err := root.Folder(location, mode) 15 | return err 16 | } 17 | return s.Upload(ctx, location, mode, reader) 18 | } 19 | -------------------------------------------------------------------------------- /mem/delete.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/storage" 6 | "path" 7 | ) 8 | 9 | //Delete removes file or directory 10 | func (s *storager) Delete(ctx context.Context, location string, options ...storage.Option) error { 11 | parent, err := s.parent(location, 0) 12 | if err != nil { 13 | return err 14 | } 15 | _, name := path.Split(location) 16 | return parent.Delete(name) 17 | } 18 | -------------------------------------------------------------------------------- /mem/delete_test.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "context" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestDelete(t *testing.T) { 10 | 11 | const testContent = "this is test" 12 | ctx := context.Background() 13 | var useCases = []struct { 14 | description string 15 | URL string 16 | isDir bool 17 | hasError bool 18 | }{ 19 | 20 | { 21 | description: "file deletion", 22 | URL: "mem:///folder/file.txt", 23 | }, 24 | { 25 | description: "file deletion", 26 | URL: "mem:///folder", 27 | isDir: true, 28 | }, 29 | { 30 | description: "file deletion error", 31 | URL: "mem:///folder/file.txt", 32 | hasError: true, 33 | }, 34 | } 35 | 36 | storager := New() 37 | for _, useCase := range useCases { 38 | 39 | if !useCase.hasError { 40 | err := storager.Create(ctx, useCase.URL, 0744, useCase.isDir) 41 | assert.Nil(t, err, useCase.description) 42 | } 43 | err := storager.Delete(ctx, useCase.URL) 44 | if useCase.hasError { 45 | assert.NotNil(t, err, useCase.description) 46 | continue 47 | } 48 | 49 | assert.Nil(t, err, useCase.description) 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /mem/doc.go: -------------------------------------------------------------------------------- 1 | //Package mem implements in memory file system 2 | package mem 3 | -------------------------------------------------------------------------------- /mem/example_test.go: -------------------------------------------------------------------------------- 1 | package mem_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/viant/afs" 7 | "github.com/viant/afs/mem" 8 | "log" 9 | "strings" 10 | ) 11 | 12 | func ExampleService() { 13 | service := afs.New() 14 | ctx := context.Background() 15 | err := service.Upload(ctx, "mem://localhost/folder1/asset.txt", 0644, strings.NewReader("some content")) 16 | if err != nil { 17 | log.Fatal(err) 18 | } 19 | objects, err := service.List(ctx, "mem://localhost/folder1/") 20 | if err != nil { 21 | log.Fatal(err) 22 | } 23 | for _, object := range objects { 24 | fmt.Printf("%v %v\n", object.URL(), object.Name()) 25 | } 26 | } 27 | 28 | func ExampleNew() { 29 | manager := mem.New() 30 | ctx := context.Background() 31 | err := manager.Upload(ctx, "mem://localhost/folder1/asset.txt", 0644, strings.NewReader("some content")) 32 | if err != nil { 33 | log.Fatal(err) 34 | } 35 | objects, err := manager.List(ctx, "mem://localhost/folder1/") 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | for _, object := range objects { 40 | fmt.Printf("%v %v\n", object.URL(), object.Name()) 41 | } 42 | } 43 | 44 | func ExampleNewStorager() { 45 | ctx := context.Background() 46 | storager := mem.NewStorager("mem://localhost/") 47 | err := storager.Upload(ctx, "folder1/asset1", 0644, strings.NewReader("some content")) 48 | if err != nil { 49 | log.Fatal(err) 50 | } 51 | err = storager.Upload(ctx, "folder1/asset2", 0644, strings.NewReader("some content")) 52 | if err != nil { 53 | log.Fatal(err) 54 | } 55 | 56 | fileInfos, err := storager.List(ctx, "folder1/", 0, 0) 57 | if err != nil { 58 | log.Fatal(err) 59 | } 60 | for _, info := range fileInfos { 61 | fmt.Printf("%v\n", info.Name()) 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /mem/file.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "bytes" 5 | "github.com/viant/afs/file" 6 | "github.com/viant/afs/object" 7 | "github.com/viant/afs/option" 8 | "github.com/viant/afs/storage" 9 | "github.com/viant/afs/url" 10 | "io" 11 | "io/ioutil" 12 | "os" 13 | "path" 14 | "strings" 15 | "time" 16 | ) 17 | 18 | //File represents in memory file 19 | type File struct { 20 | storage.Object 21 | content []byte 22 | modTime time.Time 23 | downloadError error 24 | uploadError error 25 | readerError error 26 | generation int64 27 | } 28 | 29 | //NewReader return new Reader 30 | func (f *File) NewReader() io.ReadCloser { 31 | var reader io.Reader = bytes.NewReader(f.content) 32 | if f.readerError != nil { 33 | reader = &fakeReader{error: f.readerError} 34 | } 35 | return ioutil.NopCloser(reader) 36 | } 37 | 38 | //SetErrors sets test errors 39 | func (f *File) SetErrors(errors ...*option.Error) { 40 | if len(errors) > 0 { 41 | for i := range errors { 42 | switch strings.ToLower(errors[i].Type) { 43 | case option.ErrorTypeDownload: 44 | f.downloadError = errors[i].Error 45 | case option.ErrorTypeUpload: 46 | f.uploadError = errors[i].Error 47 | case option.ErrorTypeReader: 48 | f.readerError = errors[i].Error 49 | } 50 | } 51 | } 52 | } 53 | 54 | //NewFile create a file 55 | func NewFile(URL string, mode os.FileMode, content []byte, modTime time.Time) *File { 56 | baseURL, URLPath := Split(URL) 57 | URL = url.Join(baseURL, URLPath) 58 | _, name := path.Split(URLPath) 59 | info := file.NewInfo(name, 0, mode, modTime, false) 60 | result := &File{ 61 | content: content, 62 | } 63 | result.Object = object.New(URL, info, result) 64 | return result 65 | } 66 | -------------------------------------------------------------------------------- /mem/list.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/option" 6 | "github.com/viant/afs/storage" 7 | "os" 8 | ) 9 | 10 | //List list directory or returns a file info 11 | func (s *storager) List(ctx context.Context, location string, options ...storage.Option) ([]os.FileInfo, error) { 12 | root := s.Root 13 | object, err := root.Lookup(location, 0) 14 | if err != nil { 15 | return nil, err 16 | } 17 | match, page := option.GetListOptions(options) 18 | if object.IsDir() { 19 | folder := &Folder{} 20 | if err = object.Unwrap(&folder); err != nil { 21 | return nil, err 22 | } 23 | var objects = folder.Objects() 24 | var result = make([]os.FileInfo, 0) 25 | 26 | for i := range objects { 27 | if !match(location, objects[i]) { 28 | continue 29 | } 30 | page.Increment() 31 | if page.ShallSkip() { 32 | continue 33 | } 34 | result = append(result, objects[i]) 35 | if page.HasReachedLimit() { 36 | break 37 | } 38 | } 39 | return result, nil 40 | } 41 | 42 | if !match(location, object) { 43 | return []os.FileInfo{}, nil 44 | } 45 | generation := &option.Generation{} 46 | if _, ok := option.Assign(options, &generation); ok { 47 | if file, ok := object.(*File); ok { 48 | generation.Generation = file.generation 49 | } 50 | } 51 | return []os.FileInfo{object}, nil 52 | } 53 | 54 | //Exists checks if location exists 55 | func (s *storager) Exists(ctx context.Context, location string, options ...storage.Option) (bool, error) { 56 | root := s.Root 57 | object, err := root.Lookup(location, 0) 58 | if err != nil { 59 | return false, nil 60 | } 61 | generation := &option.Generation{} 62 | if _, ok := option.Assign(options, &generation); ok { 63 | if file, ok := object.(*File); ok { 64 | generation.Generation = file.generation 65 | } 66 | } 67 | return true, nil 68 | 69 | } 70 | -------------------------------------------------------------------------------- /mem/manager.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/base" 6 | "github.com/viant/afs/option" 7 | "github.com/viant/afs/storage" 8 | "io" 9 | "net/http" 10 | "os" 11 | "strings" 12 | ) 13 | 14 | type manager struct { 15 | *base.Manager 16 | } 17 | 18 | func (m *manager) provider(ctx context.Context, baseURL string, options ...storage.Option) (storage.Storager, error) { 19 | return NewStorager(baseURL), nil 20 | } 21 | 22 | func (m *manager) ErrorCode(err error) int { 23 | if err == nil { 24 | return 0 25 | } 26 | if strings.Contains(err.Error(), preconditionErrorMessage) { 27 | return http.StatusPreconditionFailed 28 | } 29 | if strings.Contains(err.Error(), noSuchFileOrDirectoryErrorMessage) { 30 | return http.StatusNotFound 31 | } 32 | 33 | return 0 34 | } 35 | 36 | func (m *manager) setErrors(ctx context.Context, URL string, mode os.FileMode, reader io.Reader, options []storage.Option) error { 37 | errors := option.Errors{} 38 | optError := &option.Error{} 39 | option.Assign(options, &errors, &optError) 40 | if optError.Type != "" && len(errors) == 0 { 41 | errors = append(errors, optError) 42 | } 43 | if len(errors) == 0 { 44 | return nil 45 | } 46 | if objects, err := m.List(ctx, URL); err == nil && len(objects) == 1 { 47 | file := &File{} 48 | if err = objects[0].Unwrap(&file); err != nil { 49 | return err 50 | } 51 | file.SetErrors(errors...) 52 | if file.uploadError != nil { 53 | return file.uploadError 54 | } 55 | } 56 | return nil 57 | } 58 | 59 | func (m *manager) Upload(ctx context.Context, URL string, mode os.FileMode, reader io.Reader, options ...storage.Option) error { 60 | err := m.Manager.Upload(ctx, URL, mode, reader, options...) 61 | if err == nil { 62 | err = m.setErrors(ctx, URL, mode, reader, options) 63 | } 64 | return err 65 | } 66 | 67 | //New create a in memory storage 68 | func New(options ...storage.Option) storage.Manager { 69 | return newManager(options...) 70 | } 71 | 72 | func newManager(options ...storage.Option) *manager { 73 | result := &manager{} 74 | baseMgr := base.New(result, Scheme, result.provider, options) 75 | result.Manager = baseMgr 76 | return result 77 | } 78 | -------------------------------------------------------------------------------- /mem/open.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/option" 6 | "github.com/viant/afs/storage" 7 | "io" 8 | ) 9 | 10 | //Open downloads content for the supplied object 11 | func (s *storager) Open(ctx context.Context, location string, options ...storage.Option) (io.ReadCloser, error) { 12 | root := s.Root 13 | file, err := root.File(location) 14 | if err != nil { 15 | return nil, err 16 | } 17 | generation := &option.Generation{} 18 | if _, ok := option.Assign(options, &generation); ok { 19 | generation.Generation = file.generation 20 | } 21 | return file.NewReader(), file.downloadError 22 | } 23 | -------------------------------------------------------------------------------- /mem/parent.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "os" 5 | "path" 6 | ) 7 | 8 | //parent return parent folder 9 | func (s *storager) parent(location string, dirMode os.FileMode) (*Folder, error) { 10 | root := s.Root 11 | parentPath, _ := path.Split(location) 12 | return root.Folder(parentPath, dirMode) 13 | } 14 | -------------------------------------------------------------------------------- /mem/provider.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "github.com/viant/afs/storage" 5 | ) 6 | 7 | //Provider manager provider function 8 | func Provider(options ...storage.Option) (storage.Manager, error) { 9 | return Singleton(options...), nil 10 | } 11 | -------------------------------------------------------------------------------- /mem/reader.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | type fakeReader struct { 4 | error error 5 | } 6 | 7 | //Read returns defined error 8 | func (r *fakeReader) Read(p []byte) (n int, err error) { 9 | return 0, r.error 10 | } 11 | -------------------------------------------------------------------------------- /mem/root.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/storage" 6 | "github.com/viant/afs/url" 7 | ) 8 | 9 | //Root returns memory system root folder for supplied base URL 10 | func (s *manager) Root(ctx context.Context, baseURL string) *Folder { 11 | baseURL, _ = url.Base(baseURL, Scheme) 12 | srv, err := s.Storager(ctx, baseURL, []storage.Option{}) 13 | if err != nil { 14 | return nil 15 | } 16 | memStorager, ok := srv.(*storager) 17 | if !ok { 18 | return nil 19 | } 20 | return memStorager.Root 21 | } 22 | -------------------------------------------------------------------------------- /mem/scheme.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | //Scheme memory URL scheme 4 | const Scheme = "mem" 5 | -------------------------------------------------------------------------------- /mem/singleton.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import "github.com/viant/afs/storage" 4 | 5 | var singleton *manager 6 | 7 | //Singleton returns singleton manager 8 | func Singleton(options ...storage.Option) storage.Manager { 9 | if singleton != nil { 10 | return singleton 11 | } 12 | singleton = newManager(options...) 13 | return singleton 14 | } 15 | 16 | //ResetSingleton rest singleton 17 | func ResetSingleton(options ...storage.Option) { 18 | singleton = newManager(options...) 19 | } 20 | -------------------------------------------------------------------------------- /mem/singleton_test.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestSingleton(t *testing.T) { 9 | 10 | s1 := Singleton() 11 | s2 := Singleton() 12 | assert.Equal(t, s1, s2) 13 | ResetSingleton() 14 | s3 := Singleton() 15 | assert.True(t, s3 != s1) 16 | } 17 | -------------------------------------------------------------------------------- /mem/split.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "github.com/viant/afs/url" 5 | "strings" 6 | ) 7 | 8 | //SplitPath splits path 9 | func SplitPath(URLPath string) []string { 10 | var result = make([]string, 0) 11 | var elements = strings.Split(URLPath, "/") 12 | if len(elements) == 0 { 13 | return result 14 | } 15 | for _, elem := range elements { 16 | if elem == "" { 17 | continue 18 | } 19 | result = append(result, elem) 20 | } 21 | return result 22 | } 23 | 24 | //Split split URL with the last URI element and its parent path 25 | func Split(URL string) (string, string) { 26 | return url.Split(URL, Scheme) 27 | } 28 | -------------------------------------------------------------------------------- /mem/split_test.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestSplitPath(t *testing.T) { 9 | 10 | var useCases = []struct { 11 | description string 12 | URLPath string 13 | expect []string 14 | }{ 15 | { 16 | description: "root path", 17 | URLPath: "/", 18 | expect: []string{}, 19 | }, 20 | { 21 | description: "folder path", 22 | URLPath: "/folder", 23 | expect: []string{"folder"}, 24 | }, 25 | { 26 | description: "folder path with slash", 27 | URLPath: "/folder/", 28 | expect: []string{"folder"}, 29 | }, 30 | { 31 | description: "subfolder", 32 | URLPath: "/folder/subfolder", 33 | expect: []string{"folder", "subfolder"}, 34 | }, 35 | { 36 | description: "file ", 37 | URLPath: "/folder/subfolder/file.txt", 38 | expect: []string{"folder", "subfolder", "file.txt"}, 39 | }, 40 | } 41 | 42 | for _, useCase := range useCases { 43 | 44 | actual := SplitPath(useCase.URLPath) 45 | assert.EqualValues(t, useCase.expect, actual, useCase.description) 46 | 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /mem/storager.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "github.com/viant/afs/base" 5 | "github.com/viant/afs/file" 6 | "github.com/viant/afs/storage" 7 | "github.com/viant/afs/url" 8 | "sync" 9 | ) 10 | 11 | type storager struct { 12 | base.Storager 13 | scheme string 14 | Root *Folder 15 | mux sync.Mutex 16 | } 17 | 18 | func (s *storager) Close() error { 19 | return nil 20 | } 21 | 22 | //NewStorager create a new in memeory storage service 23 | func NewStorager(baseURL string) storage.Storager { 24 | baseURL, _ = url.Base(baseURL, Scheme) 25 | result := &storager{ 26 | Root: NewFolder(baseURL, file.DefaultDirOsMode), 27 | scheme: url.Scheme(baseURL, Scheme), 28 | } 29 | result.Storager.List = result.List 30 | return result 31 | } 32 | -------------------------------------------------------------------------------- /mem/upload.go: -------------------------------------------------------------------------------- 1 | package mem 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/pkg/errors" 7 | "github.com/viant/afs/file" 8 | "github.com/viant/afs/option" 9 | "github.com/viant/afs/storage" 10 | "io" 11 | "io/ioutil" 12 | "net/http" 13 | "os" 14 | "time" 15 | ) 16 | 17 | var preconditionErrorMessage = fmt.Sprintf("precondition failed: %v ", http.StatusPreconditionFailed) 18 | 19 | //Upload writes fakeReader TestContent to supplied URL path. 20 | func (s *storager) Upload(ctx context.Context, location string, mode os.FileMode, reader io.Reader, options ...storage.Option) error { 21 | s.mux.Lock() 22 | parent, err := s.parent(location, file.DefaultDirOsMode) 23 | s.mux.Unlock() 24 | 25 | generation := &option.Generation{} 26 | _, ok := option.Assign(options, &generation) 27 | if !ok { 28 | generation = nil 29 | } 30 | parent, err = s.parent(location, file.DefaultDirOsMode) 31 | if err != nil { 32 | return err 33 | } 34 | var data []byte 35 | if reader != nil { 36 | if data, err = ioutil.ReadAll(reader); err != nil { 37 | return err 38 | } 39 | } 40 | modTime := time.Now() 41 | option.Assign(options, &modTime) 42 | memFile := NewFile(location, mode, data, modTime) 43 | parent.mutex.Lock() 44 | if prev, ok := parent.files[memFile.Name()]; ok { 45 | memFile.generation = prev.generation 46 | } 47 | parent.mutex.Unlock() 48 | if generation != nil { 49 | if generation.WhenMatch { 50 | if generation.Generation != memFile.generation { 51 | return errors.Errorf(preconditionErrorMessage+" expected: %v, but had: %v", generation.Generation, memFile.generation) 52 | } 53 | } else { 54 | if generation.Generation == memFile.generation { 55 | return errors.Errorf(preconditionErrorMessage+" unexpected: %v", generation.Generation) 56 | } 57 | } 58 | } 59 | memFile.generation++ 60 | return parent.Put(memFile.Object) 61 | } 62 | -------------------------------------------------------------------------------- /modifier/replacer.go: -------------------------------------------------------------------------------- 1 | package modifier 2 | 3 | import ( 4 | "github.com/viant/afs/file" 5 | "github.com/viant/afs/option" 6 | "io" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | // Replace return modification handler with the specified replacements map 12 | func Replace(replacements map[string]string) option.Modifier { 13 | return func(_ string, info os.FileInfo, reader io.ReadCloser) (os.FileInfo, io.ReadCloser, error) { 14 | data, err := io.ReadAll(reader) 15 | if err != nil { 16 | return nil, nil, err 17 | } 18 | _ = reader.Close() 19 | text := string(data) 20 | for k, v := range replacements { 21 | if count := strings.Count(text, k); count > 0 { 22 | text = strings.Replace(text, k, v, count) 23 | } 24 | } 25 | info = file.AdjustInfoSize(info, len(text)) 26 | return info, io.NopCloser(strings.NewReader(text)), nil 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /modifier/replacer_test.go: -------------------------------------------------------------------------------- 1 | package modifier 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/viant/afs/file" 6 | "io/ioutil" 7 | "strings" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func TestReplace(t *testing.T) { 13 | replaceer := Replace(map[string]string{ 14 | "test": "Test", 15 | }) 16 | 17 | info := file.NewInfo("blah", 0, 0644, time.Now(), false) 18 | _, reader, err := replaceer("", info, ioutil.NopCloser(strings.NewReader("test is test"))) 19 | assert.Nil(t, err) 20 | actual, _ := ioutil.ReadAll(reader) 21 | assert.EqualValues(t, "Test is Test", actual) 22 | } 23 | -------------------------------------------------------------------------------- /move.go: -------------------------------------------------------------------------------- 1 | package afs 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/file" 6 | "github.com/viant/afs/option" 7 | "github.com/viant/afs/storage" 8 | "github.com/viant/afs/url" 9 | ) 10 | 11 | func (s *service) Move(ctx context.Context, sourceURL, destURL string, options ...storage.Option) error { 12 | sourceURL = url.Normalize(sourceURL, file.Scheme) 13 | destURL = url.Normalize(destURL, file.Scheme) 14 | destURL = s.updateDestURL(sourceURL, destURL) 15 | sourceOptions := option.NewSource() 16 | destOptions := option.NewDest() 17 | option.Assign(options, &sourceOptions, &destOptions) 18 | if url.IsSchemeEquals(sourceURL, destURL) { 19 | if sourceManager, err := s.manager(ctx, sourceURL, *sourceOptions); err == nil { 20 | if mover, ok := sourceManager.(storage.Mover); ok { 21 | if !s.IsAuthChanged(ctx, sourceManager, sourceURL, *destOptions) { 22 | return mover.Move(ctx, sourceURL, destURL, *sourceOptions...) 23 | } 24 | } 25 | } 26 | } 27 | _ = s.Delete(ctx, destURL, *destOptions...) 28 | if err := s.Copy(ctx, sourceURL, destURL, sourceOptions, destOptions); err != nil { 29 | return err 30 | } 31 | return s.Delete(ctx, sourceURL, *sourceOptions...) 32 | } 33 | -------------------------------------------------------------------------------- /object/doc.go: -------------------------------------------------------------------------------- 1 | //Package object provide storage object/link implementation 2 | package object 3 | -------------------------------------------------------------------------------- /object/link.go: -------------------------------------------------------------------------------- 1 | package object 2 | 3 | //Link represents a link source wrapper 4 | type Link struct { 5 | Source interface{} 6 | Linkname string 7 | LinkURL string 8 | } 9 | 10 | //NewLink create a link 11 | func NewLink(linkname, linkURL string, source interface{}) *Link { 12 | return &Link{Linkname: linkname, LinkURL: linkURL, Source: source} 13 | } 14 | -------------------------------------------------------------------------------- /object/object.go: -------------------------------------------------------------------------------- 1 | package object 2 | 3 | import ( 4 | "fmt" 5 | "github.com/viant/afs/storage" 6 | "os" 7 | "reflect" 8 | ) 9 | 10 | //Object represents abstract storage object 11 | type Object struct { 12 | url string 13 | Source interface{} 14 | linkname string 15 | linkURL string 16 | os.FileInfo 17 | } 18 | 19 | //URL return storage url 20 | func (o *Object) URL() string { 21 | return o.url 22 | } 23 | 24 | //Linkname returns a link name 25 | func (o *Object) Linkname() string { 26 | return o.linkname 27 | } 28 | 29 | //LinkURL returns link URL (absolute path) 30 | func (o *Object) LinkURL() string { 31 | return o.linkURL 32 | } 33 | 34 | //Wrap wraps Source storage object 35 | func (o *Object) Wrap(source interface{}) { 36 | o.Source = source 37 | } 38 | 39 | //Unwrap unwrap source storage to target pointer 40 | func (o *Object) Unwrap(target interface{}) error { 41 | if o.Source == nil { 42 | return nil 43 | } 44 | targetValue := reflect.ValueOf(target) 45 | sourceValue := reflect.ValueOf(o.Source) 46 | 47 | if sourceValue.Type().AssignableTo(targetValue.Type()) { 48 | return fmt.Errorf("unable to assign %T to %T", o.Source, target) 49 | } 50 | targetValue.Elem().Set(sourceValue) 51 | return nil 52 | } 53 | 54 | //New creates a new storage object 55 | func New(URL string, info os.FileInfo, source interface{}) storage.Object { 56 | linkname := "" 57 | linkURL := "" 58 | link, ok := source.(*Link) 59 | if ok { 60 | linkname = link.Linkname 61 | linkURL = link.LinkURL 62 | source = link.Source 63 | } 64 | var result = &Object{ 65 | url: URL, 66 | Source: source, 67 | linkname: linkname, 68 | linkURL: linkURL, 69 | FileInfo: info, 70 | } 71 | return result 72 | } 73 | -------------------------------------------------------------------------------- /option/acl.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //ACL represents acl 4 | type ACL struct { 5 | ACL string 6 | } 7 | 8 | //NewACL creates an acl option 9 | func NewACL(acl string) *ACL { 10 | return &ACL{ACL: acl} 11 | } 12 | -------------------------------------------------------------------------------- /option/append.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import "github.com/viant/afs/storage" 4 | 5 | //Append storage options 6 | func Append(options []storage.Option, newOptions ...storage.Option) []storage.Option { 7 | if len(options) == 0 { 8 | return newOptions 9 | } 10 | return append(options, newOptions...) 11 | } 12 | -------------------------------------------------------------------------------- /option/append_test.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/viant/afs/storage" 6 | "testing" 7 | ) 8 | 9 | func TestAppend(t *testing.T) { 10 | options := Append([]storage.Option{ 11 | NewLocation("/tmp"), 12 | NewTimeout(100), 13 | NewSource(), 14 | NewDest(), 15 | }, NewBasicAuth("user", "pass")) 16 | assert.EqualValues(t, 5, len(options)) 17 | } 18 | -------------------------------------------------------------------------------- /option/assign.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/viant/afs/storage" 5 | "reflect" 6 | ) 7 | 8 | //Assign assign supplied option, if returns un assign options and true if assign at least one 9 | func Assign(options []storage.Option, supported ...interface{}) ([]storage.Option, bool) { 10 | return assign(options, supported) 11 | } 12 | 13 | //Assign assign supplied option 14 | func assign(options []storage.Option, supported []interface{}) ([]storage.Option, bool) { 15 | var unfiltered = make([]storage.Option, 0) 16 | if len(options) == 0 { 17 | return options, false 18 | } 19 | if len(supported) == 0 { 20 | return options, false 21 | } 22 | 23 | var index = make(map[reflect.Type]interface{}) 24 | for i := range supported { 25 | index[reflect.TypeOf(supported[i]).Elem()] = supported[i] 26 | } 27 | assigned := false 28 | for i := range options { 29 | option := options[i] 30 | if option == nil { 31 | continue 32 | } 33 | optionValue := reflect.ValueOf(option) 34 | target, ok := index[optionValue.Type()] 35 | if !ok { 36 | for k, v := range index { 37 | if optionValue.Type().AssignableTo(k) { 38 | target = v 39 | ok = true 40 | break 41 | } 42 | } 43 | } 44 | if !ok { 45 | unfiltered = append(unfiltered, options[i]) 46 | continue 47 | } 48 | assigned = true 49 | reflect.ValueOf(target).Elem().Set(optionValue) 50 | } 51 | return unfiltered, assigned 52 | } 53 | -------------------------------------------------------------------------------- /option/assign_test.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/afs/storage" 8 | "io" 9 | "reflect" 10 | "testing" 11 | ) 12 | 13 | type testFilter struct { 14 | message string 15 | } 16 | 17 | type Calc func(ops ...int) int 18 | 19 | func TestFilter(t *testing.T) { 20 | 21 | testOption := &testFilter{} 22 | var reader io.Reader 23 | var calc Calc 24 | 25 | var useCases = []struct { 26 | description string 27 | options []storage.Option 28 | target interface{} 29 | assigned bool 30 | }{ 31 | 32 | { 33 | description: "test interface option", 34 | options: []storage.Option{ 35 | io.Reader(new(bytes.Buffer)), 36 | }, 37 | assigned: true, 38 | target: &reader, 39 | }, 40 | { 41 | description: "test function option", 42 | assigned: true, 43 | options: []storage.Option{ 44 | Calc(func(ops ...int) int { 45 | return 0 46 | }), 47 | }, 48 | target: &calc, 49 | }, 50 | { 51 | description: "empty list", 52 | options: []storage.Option{}, 53 | }, 54 | 55 | { 56 | description: "test struct option", 57 | assigned: true, 58 | options: []storage.Option{ 59 | &testFilter{message: "abc"}, 60 | }, 61 | target: &testOption, 62 | }, 63 | } 64 | 65 | for _, useCase := range useCases { 66 | targets := make([]interface{}, 0) 67 | if useCase.target != nil { 68 | targets = append(targets, useCase.target) 69 | } 70 | 71 | _, assigned := Assign(useCase.options, targets...) 72 | if !assert.EqualValues(t, assigned, useCase.assigned, useCase.description) { 73 | continue 74 | } 75 | if !assigned { 76 | continue 77 | } 78 | if useCase.target != nil { 79 | assert.EqualValues(t, fmt.Sprintf("%v", reflect.ValueOf(useCase.target).Elem().Interface()), fmt.Sprintf("%v", useCase.options[0]), useCase.description) 80 | } 81 | } 82 | 83 | } 84 | -------------------------------------------------------------------------------- /option/auth.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //Auth auth options to force auth, instead of reusing previous auth session 4 | type Auth struct { 5 | Force bool 6 | } 7 | 8 | //NewAuth create an auth option 9 | func NewAuth(force bool) *Auth { 10 | return &Auth{Force: force} 11 | } 12 | -------------------------------------------------------------------------------- /option/cache.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import "strings" 4 | 5 | //Cache represents cache option 6 | type Cache struct { 7 | Name string 8 | Compression string 9 | } 10 | 11 | func (c *Cache) Init() { 12 | if strings.HasSuffix(c.Name, ".gz") { 13 | if c.Compression == "" { 14 | c.Compression = "gzip" 15 | } 16 | } else if c.Compression == "gzip" { 17 | c.Name += ".gz" 18 | } 19 | } 20 | 21 | //WithCache creates cache name option 22 | func WithCache(name, compression string) *Cache { 23 | return &Cache{Name: name, Compression: compression} 24 | } 25 | -------------------------------------------------------------------------------- /option/checksum.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //SkipChecksum represents checksum option 4 | type SkipChecksum struct { 5 | Skip bool 6 | } 7 | 8 | //NewSkipChecksum returns checksum options for supplied skip flag 9 | func NewSkipChecksum(skip bool) *SkipChecksum { 10 | return &SkipChecksum{Skip: skip} 11 | } 12 | -------------------------------------------------------------------------------- /option/content/const.go: -------------------------------------------------------------------------------- 1 | package content 2 | 3 | const ( 4 | //Type represents content meta type 5 | Type = "Content-Type" 6 | //Encoding content encoding 7 | Encoding = "Content-Encoding" 8 | //Language content languate 9 | Language = "Content-Language" 10 | ) 11 | -------------------------------------------------------------------------------- /option/content/meta.go: -------------------------------------------------------------------------------- 1 | package content 2 | 3 | //Meta represents content meta options 4 | type Meta struct { 5 | Values map[string]string 6 | } 7 | 8 | //NewMeta represents content meta 9 | func NewMeta(kvPairs ...string) *Meta { 10 | result := &Meta{Values: make(map[string]string)} 11 | for i := 0; i < len(kvPairs); i += 2 { 12 | value := "" 13 | if i+1 < len(kvPairs) { 14 | value = kvPairs[i+1] 15 | } 16 | result.Values[kvPairs[i]] = value 17 | } 18 | return result 19 | } 20 | -------------------------------------------------------------------------------- /option/crc.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "hash/crc32" 7 | ) 8 | 9 | //Crc represents crc hash 10 | type Crc struct { 11 | Hash uint32 12 | } 13 | 14 | //Encode encodes hash 15 | func (c *Crc) Encode() string { 16 | b := []byte{byte(c.Hash >> 24), byte(c.Hash >> 16), byte(c.Hash >> 8), byte(c.Hash)} 17 | return base64.StdEncoding.EncodeToString(b) 18 | } 19 | 20 | //Decode decodes base64 encoded hash 21 | func (c *Crc) Decode(encoded string) error { 22 | d, err := base64.StdEncoding.DecodeString(encoded) 23 | if err != nil { 24 | return err 25 | } 26 | if len(d) != 4 { 27 | return fmt.Errorf("storage: %q does not encode a 32-bit value", d) 28 | } 29 | c.Hash = uint32(d[0])<<24 + uint32(d[1])<<16 + uint32(d[2])<<8 + uint32(d[3]) 30 | return nil 31 | } 32 | 33 | //NewCrc returns a crc hash for supplied data 34 | func NewCrc(data []byte) *Crc { 35 | crc32Hash := crc32.New(crc32.MakeTable(crc32.Castagnoli)) 36 | _, _ = crc32Hash.Write(data) 37 | return &Crc{Hash: crc32Hash.Sum32()} 38 | } 39 | -------------------------------------------------------------------------------- /option/crc_test.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestNewCrc(t *testing.T) { 9 | { 10 | crcHash := NewCrc([]byte("test is test")) 11 | actual := crcHash.Encode() 12 | assert.EqualValues(t, 0x84cd7d5, crcHash.Hash) 13 | assert.EqualValues(t, "CEzX1Q==", actual) 14 | } 15 | { 16 | crcHash := &Crc{} 17 | err := crcHash.Decode("CEzX1Q==") 18 | assert.Nil(t, err) 19 | assert.EqualValues(t, 0x84cd7d5, crcHash.Hash) 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /option/cred.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //BasicAuth represents a basic auth 4 | type BasicAuth interface { 5 | Credentials() (user, password string) 6 | } 7 | 8 | type basicAuth struct { 9 | user string 10 | _password string 11 | } 12 | 13 | func (a *basicAuth) Credentials() (user, password string) { 14 | return a.user, a._password 15 | } 16 | 17 | //NewBasicAuth returns credential authenticator 18 | func NewBasicAuth(user, password string) BasicAuth { 19 | return &basicAuth{user: user, _password: password} 20 | } 21 | -------------------------------------------------------------------------------- /option/dest.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import "github.com/viant/afs/storage" 4 | 5 | //Dest represents dest options 6 | type Dest storage.Options 7 | 8 | //NewDest returns new source options 9 | func NewDest(options ...storage.Option) *Dest { 10 | result := Dest(options) 11 | return &result 12 | } 13 | -------------------------------------------------------------------------------- /option/doc.go: -------------------------------------------------------------------------------- 1 | //Package option define storage options 2 | package option 3 | -------------------------------------------------------------------------------- /option/empty.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //Empty represents empty pipeline writer option 4 | type Empty struct { 5 | Allowed bool 6 | } 7 | 8 | //NewEmpty creates a new empty option 9 | func NewEmpty(allowed bool) *Empty { 10 | return &Empty{Allowed: allowed} 11 | } 12 | -------------------------------------------------------------------------------- /option/encryption.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //ServerSideEncryption represents server side encryption 4 | type ServerSideEncryption struct { 5 | Algorithm string 6 | } 7 | 8 | //NewServerSideEncryption creates a server side encryption 9 | func NewServerSideEncryption(alg string) *ServerSideEncryption { 10 | return &ServerSideEncryption{Algorithm: alg} 11 | } 12 | -------------------------------------------------------------------------------- /option/error.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | const ( 4 | //ErrorTypeDownload download error type 5 | ErrorTypeDownload = "download" 6 | //ErrorTypeUpload upload error type 7 | ErrorTypeUpload = "upload" 8 | //ErrorTypeReader reader error type 9 | ErrorTypeReader = "reader" 10 | ) 11 | 12 | //Error represents a simulation error 13 | type Error struct { 14 | Type string 15 | Error error 16 | } 17 | 18 | //Errors represents simulation errors 19 | type Errors []*Error 20 | 21 | //NewUploadError creates an upload error 22 | func NewUploadError(err error) *Error { 23 | return &Error{ 24 | Type: ErrorTypeUpload, 25 | Error: err, 26 | } 27 | } 28 | 29 | //NewDownloadError creates a download error 30 | func NewDownloadError(err error) *Error { 31 | return &Error{ 32 | Type: ErrorTypeDownload, 33 | Error: err, 34 | } 35 | } 36 | 37 | //NewReaderError creates a reader error 38 | func NewReaderError(err error) *Error { 39 | return &Error{ 40 | Type: ErrorTypeReader, 41 | Error: err, 42 | } 43 | } 44 | 45 | //NewErrors creates an error slice for supplied errors 46 | func NewErrors(errors ...*Error) []*Error { 47 | return errors 48 | } 49 | -------------------------------------------------------------------------------- /option/generation.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //Generation represent generation option 4 | type Generation struct { 5 | WhenMatch bool 6 | Generation int64 7 | } 8 | 9 | //NewGeneration create a generation 10 | func NewGeneration(whenMatch bool, generation int64) *Generation { 11 | return &Generation{ 12 | WhenMatch: whenMatch, 13 | Generation: generation, 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /option/grant.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //Grant represents a grant option 4 | type Grant struct { 5 | FullControl string 6 | Read string 7 | ReadACP string 8 | WriteACP string 9 | } 10 | 11 | //NewGrant creates a grant option 12 | func NewGrant(fullControl, read, readACP, writeACP string) *Grant { 13 | return &Grant{ 14 | FullControl: fullControl, 15 | Read: read, 16 | ReadACP: readACP, 17 | WriteACP: writeACP, 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /option/key.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "crypto/md5" 5 | "crypto/sha256" 6 | "encoding/base64" 7 | "fmt" 8 | ) 9 | 10 | //AES256Key represents custom key 11 | type AES256Key struct { 12 | Key []byte 13 | Base64Key string 14 | Base64KeyMd5Hash string 15 | Base64KeySha256Hash string 16 | } 17 | 18 | //Init initialises key 19 | func (k *AES256Key) Init() (err error) { 20 | if k.Base64Key != "" && len(k.Key) == 0 { 21 | if k.Key, err = base64.StdEncoding.DecodeString(k.Base64Key); err != nil { 22 | return err 23 | } 24 | } else if k.Base64Key == "" && len(k.Key) > 0 { 25 | k.Base64Key = base64.StdEncoding.EncodeToString(k.Key) 26 | } 27 | if k.Base64KeyMd5Hash == "" { 28 | md5keyHash := md5.New() 29 | md5keyHash.Write(k.Key) 30 | k.Base64KeyMd5Hash = base64.StdEncoding.EncodeToString(md5keyHash.Sum(nil)) 31 | } 32 | if k.Base64KeySha256Hash == "" { 33 | sha256keyHash := sha256.Sum256(k.Key) 34 | k.Base64KeySha256Hash = base64.StdEncoding.EncodeToString(sha256keyHash[:]) 35 | } 36 | return err 37 | } 38 | 39 | //Validate checks if key is valid 40 | func (k *AES256Key) Validate() error { 41 | if len(k.Key) != 32 { 42 | return fmt.Errorf("%s: not a 32-byte AES-256 key", k.Key) 43 | } 44 | return nil 45 | } 46 | 47 | //NewAES256Key returns new key 48 | func NewAES256Key(key []byte) (result *AES256Key, err error) { 49 | result = &AES256Key{Key: key} 50 | if err = result.Init(); err == nil { 51 | err = result.Validate() 52 | } 53 | return result, err 54 | } 55 | 56 | //NewBase64AES256Key create a AES256Key from base64 encoded key 57 | func NewBase64AES256Key(base64Key string) (result *AES256Key, err error) { 58 | result = &AES256Key{Base64Key: base64Key} 59 | if err = result.Init(); err == nil { 60 | err = result.Validate() 61 | } 62 | return result, err 63 | } 64 | -------------------------------------------------------------------------------- /option/key_test.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestNewAES256Key(t *testing.T) { 9 | 10 | { 11 | data := []byte("this is test this is test key") 12 | key, err := NewAES256Key(data) 13 | assert.Nil(t, err) 14 | assert.Equal(t, key.Key, data) 15 | err = key.Init() 16 | assert.Nil(t, err) 17 | err = key.Validate() 18 | assert.Nil(t, err) 19 | } 20 | { 21 | //invalid lenth 22 | _, err := NewAES256Key([]byte("abc")) 23 | assert.NotNil(t, err) 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /option/list.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/viant/afs/storage" 5 | ) 6 | 7 | //GetListOptions returns list options 8 | func GetListOptions(options []storage.Option) (Match, *Page) { 9 | var matcher Matcher 10 | var match Match 11 | page := Page{} 12 | Assign(options, &match, &page, &matcher) 13 | if matcher != nil { 14 | match = matcher.Match 15 | } 16 | return GetMatchFunc(match), &page 17 | } 18 | -------------------------------------------------------------------------------- /option/list_test.go: -------------------------------------------------------------------------------- 1 | package option_test 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/viant/afs/matcher" 7 | "github.com/viant/afs/option" 8 | "github.com/viant/afs/storage" 9 | "testing" 10 | ) 11 | 12 | //GetListOptions returns list options 13 | func Test_GetListOptions(t *testing.T) { 14 | 15 | basic, _ := matcher.NewBasic("", "", "", nil) 16 | 17 | var useCases = []struct { 18 | description string 19 | options []storage.Option 20 | expectMatch bool 21 | expectPage bool 22 | }{ 23 | { 24 | description: "only page", 25 | options: []storage.Option{ 26 | &option.Page{}, 27 | }, 28 | expectPage: true, 29 | }, 30 | { 31 | description: "only matcher", 32 | options: []storage.Option{ 33 | basic, 34 | }, 35 | expectMatch: true, 36 | }, 37 | { 38 | description: "only matcher", 39 | options: []storage.Option{ 40 | basic.Match, 41 | }, 42 | expectMatch: true, 43 | }, 44 | } 45 | 46 | for _, useCase := range useCases { 47 | 48 | match, page := option.GetListOptions(useCase.options) 49 | var defaultMatch interface{} = option.DefaultMatch 50 | if useCase.expectMatch { 51 | assert.True(t, fmt.Sprintf("%v", defaultMatch) != fmt.Sprintf("%v", match)) 52 | assert.NotNil(t, match) 53 | } 54 | if useCase.expectPage { 55 | assert.NotNil(t, page) 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /option/location.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //Location represents a location 4 | type Location struct { 5 | Path string 6 | } 7 | 8 | //NewLocation create a location with supplied path 9 | func NewLocation(path string) *Location { 10 | return &Location{Path: path} 11 | } 12 | -------------------------------------------------------------------------------- /option/logger.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | type Logger struct { 4 | logf func(format string, args ...interface{}) 5 | } 6 | 7 | func (l *Logger) Logf(format string, args ...interface{}) { 8 | if l.logf == nil { 9 | return 10 | } 11 | l.logf(format, args...) 12 | } 13 | 14 | func WithLogger(logf func(format string, args ...interface{})) *Logger { 15 | return &Logger{logf: logf} 16 | } 17 | -------------------------------------------------------------------------------- /option/matcher.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import "os" 4 | 5 | //Match represents a matching function 6 | type Match func(parent string, info os.FileInfo) bool 7 | 8 | //Matcher represents a matcher 9 | type Matcher interface { 10 | Match(parent string, info os.FileInfo) bool 11 | } 12 | 13 | func DefaultMatch(parent string, info os.FileInfo) bool { 14 | return true 15 | } 16 | 17 | //GetMatchFunc returns supplied matcher or default matcher 18 | func GetMatchFunc(matcher Match) Match { 19 | if matcher != nil { 20 | return matcher 21 | } 22 | return DefaultMatch 23 | } 24 | -------------------------------------------------------------------------------- /option/md5.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/base64" 6 | ) 7 | 8 | //Md5 represents md5 value 9 | type Md5 struct { 10 | Hash []byte 11 | } 12 | 13 | //Encode encode base64 hash value 14 | func (m *Md5) Encode() string { 15 | return base64.StdEncoding.EncodeToString(m.Hash) 16 | } 17 | 18 | //Decode base64 decode 19 | func (m *Md5) Decode(encoded string) (err error) { 20 | m.Hash, err = base64.StdEncoding.DecodeString(encoded) 21 | return err 22 | } 23 | 24 | //NewMd5 returns a MD5 hash for supplied data 25 | func NewMd5(data []byte) *Md5 { 26 | hash := md5.New() 27 | _, _ = hash.Write(data) 28 | return &Md5{Hash: hash.Sum(nil)} 29 | } 30 | -------------------------------------------------------------------------------- /option/md5_test.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestNewMd5(t *testing.T) { 9 | md5Hash := NewMd5([]byte("test is test")) 10 | hash := []byte{0x97, 0x6f, 0x7e, 0xa4, 0x4c, 0xce, 0x92, 0x2e, 0x6e, 0x5a, 0x27, 0x57, 0xa7, 0x87, 0x25, 0xe8} 11 | 12 | { 13 | actual := md5Hash.Encode() 14 | assert.EqualValues(t, hash, md5Hash.Hash) 15 | assert.EqualValues(t, "l29+pEzOki5uWidXp4cl6A==", actual) 16 | } 17 | { 18 | md5Hash := &Md5{} 19 | err := md5Hash.Decode("l29+pEzOki5uWidXp4cl6A==") 20 | assert.Nil(t, err) 21 | assert.EqualValues(t, hash, md5Hash.Hash) 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /option/method.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //HTTPMethod represent HTTP meothd 4 | type HTTPMethod string 5 | -------------------------------------------------------------------------------- /option/modifier.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "io" 5 | "os" 6 | ) 7 | 8 | //Modifier option to modify content, 9 | type Modifier func(parent string, info os.FileInfo, reader io.ReadCloser) (os.FileInfo, io.ReadCloser, error) 10 | -------------------------------------------------------------------------------- /option/nocache.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | const ( 4 | //NoCacheBaseURL no cache base URL 5 | NoCacheBaseURL = iota + 1 6 | ) 7 | 8 | //NoCache represents nocache option 9 | type NoCache struct { 10 | Source int 11 | } 12 | -------------------------------------------------------------------------------- /option/object.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //ObjectKind represents an option to indicate operation object kind 4 | type ObjectKind struct { 5 | File bool 6 | } 7 | 8 | //NewObject creates a new object 9 | func NewObjectKind(file bool) *ObjectKind { 10 | return &ObjectKind{File: file} 11 | } 12 | -------------------------------------------------------------------------------- /option/osflag.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //OsFlag represents os flag 4 | type OsFlag int 5 | -------------------------------------------------------------------------------- /option/override.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //Override override flag option 4 | type Override struct { 5 | Override bool 6 | } 7 | -------------------------------------------------------------------------------- /option/page.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "sync/atomic" 5 | ) 6 | 7 | //Page represents a page 8 | type Page struct { 9 | counter uint32 10 | limit int 11 | offset int 12 | } 13 | 14 | //ShallSkip returns true if item needs to be skipped 15 | func (p *Page) ShallSkip() bool { 16 | if p.limit == 0 { 17 | return false 18 | } 19 | return int(atomic.LoadUint32(&p.counter)) < p.offset 20 | } 21 | 22 | //MaxResult returns max results or zero 23 | func (p *Page) MaxResult() int64 { 24 | if p.offset > 0 { 25 | return 0 26 | } 27 | return int64(p.limit) 28 | } 29 | 30 | //HasReachedLimit returns true if limit has been reaced 31 | func (p *Page) HasReachedLimit() bool { 32 | if p.limit == 0 { 33 | return false 34 | } 35 | return int(atomic.LoadUint32(&p.counter)) >= p.limit 36 | } 37 | 38 | //Increment increment counter 39 | func (p *Page) Increment() int { 40 | return int(atomic.AddUint32(&p.counter, 1)) 41 | } 42 | 43 | //NewPage returns a page 44 | func NewPage(offset, limit int) *Page { 45 | return &Page{ 46 | offset: offset, 47 | limit: limit, 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /option/page_test.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestNewPage(t *testing.T) { 9 | page := NewPage(2, 4) 10 | assert.True(t, page.ShallSkip()) 11 | page.Increment() 12 | page.Increment() 13 | assert.False(t, page.ShallSkip()) 14 | assert.False(t, page.HasReachedLimit()) 15 | page.Increment() 16 | assert.False(t, page.HasReachedLimit()) 17 | page.Increment() 18 | assert.True(t, page.HasReachedLimit()) 19 | 20 | } 21 | -------------------------------------------------------------------------------- /option/presign.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | ) 7 | 8 | //TimeToLive represents presign URL 9 | type PreSign struct { 10 | URL string 11 | Header http.Header 12 | TimeToLive time.Duration 13 | } 14 | 15 | //NewPreSign creates a presign option 16 | func NewPreSign(timeToLive time.Duration) *PreSign { 17 | return &PreSign{TimeToLive: timeToLive} 18 | } 19 | -------------------------------------------------------------------------------- /option/proxy.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //Proxy represents http proxy 4 | type Proxy struct { 5 | //URL proxy //URL 6 | URL string 7 | //TimeoutMs connection timeout 8 | TimeoutMs int 9 | //Fallback if proxy fails retry without proxy 10 | Fallback bool 11 | } 12 | 13 | //NewProxy creates a new proxy 14 | func NewProxy(URL string, timeoutMs int, fallback bool) *Proxy { 15 | return &Proxy{ 16 | URL: URL, 17 | TimeoutMs: timeoutMs, 18 | Fallback: fallback, 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /option/recursive.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //Recursive represents recursive option 4 | type Recursive struct { 5 | Flag bool 6 | } 7 | 8 | //NewRecursive returns a recursive option 9 | func NewRecursive(flag bool) *Recursive { 10 | return &Recursive{ 11 | Flag: flag, 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /option/refresh.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import "time" 4 | 5 | //RefreshInterval represents interval option 6 | type RefreshInterval struct { 7 | IntervalMs int 8 | } 9 | 10 | //Duration returns a duration 11 | func (i *RefreshInterval) Duration() time.Duration { 12 | return time.Duration(i.IntervalMs) * time.Millisecond 13 | } 14 | 15 | //NewRefreshInterval create refresh interval option 16 | func NewRefreshInterval(intervalMs int) *RefreshInterval { 17 | return &RefreshInterval{IntervalMs: intervalMs} 18 | } 19 | -------------------------------------------------------------------------------- /option/region.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //Region represents cloud region/location option 4 | type Region struct { 5 | Name string 6 | } 7 | 8 | //NewRegion creates a region for specified name 9 | func NewRegion(name string) *Region { 10 | return &Region{Name: name} 11 | } 12 | -------------------------------------------------------------------------------- /option/size.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //Size represents resource size 4 | type Size int 5 | -------------------------------------------------------------------------------- /option/source.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import "github.com/viant/afs/storage" 4 | 5 | //Source represents source options 6 | type Source storage.Options 7 | 8 | //NewSource returns new source options 9 | func NewSource(options ...storage.Option) *Source { 10 | result := Source(options) 11 | return &result 12 | } 13 | -------------------------------------------------------------------------------- /option/status.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //Status represents status code 4 | type Status struct { 5 | Code int 6 | } 7 | 8 | func NewStatus() *Status { 9 | return &Status{} 10 | } -------------------------------------------------------------------------------- /option/stream.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | //Stream represents stream option for download reader 4 | type Stream struct { 5 | PartSize int 6 | Size int 7 | } 8 | 9 | //NewStream returns a new stream 10 | func NewStream(partSize, size int) *Stream { 11 | return &Stream{ 12 | PartSize: partSize, 13 | Size: size, 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /option/timeout.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import "time" 4 | 5 | //Timeout represents timeout option 6 | type Timeout struct { 7 | time.Duration 8 | } 9 | 10 | //NewTimeout creates a new timeout option 11 | func NewTimeout(durationInMs int) Timeout { 12 | return Timeout{Duration: time.Millisecond * time.Duration(durationInMs)} 13 | } 14 | -------------------------------------------------------------------------------- /option/walk.go: -------------------------------------------------------------------------------- 1 | package option 2 | 3 | import ( 4 | "github.com/viant/afs/storage" 5 | ) 6 | 7 | //GetWalkOptions returns walk options 8 | func GetWalkOptions(options []storage.Option) (Match, Modifier) { 9 | var match Match 10 | var matcher Matcher 11 | var modifier Modifier 12 | Assign(options, &match, &modifier, &matcher) 13 | if matcher != nil { 14 | match = matcher.Match 15 | } 16 | match = GetMatchFunc(match) 17 | return match, modifier 18 | } 19 | -------------------------------------------------------------------------------- /parrot/data.go: -------------------------------------------------------------------------------- 1 | package parrot 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | //Data represents parrot data 9 | type Data []byte 10 | 11 | //AsBytesLiteral returns literal bytes representation 12 | func (d Data) AsBytesLiteral(ASCII bool) string { 13 | if ASCII { 14 | data := string(d) 15 | var count = strings.Count(data, "`") 16 | if count > 0 { 17 | data = strings.Replace(data, "`", "`+\"`\"+`", count) 18 | return fmt.Sprintf("[]byte(`%s`)", data) 19 | } 20 | return fmt.Sprintf("[]byte(`%s`)", data) 21 | 22 | } 23 | var parts = make([]string, 0) 24 | for i := 0; i < len(d); i += 16 { 25 | part := make([]string, 16) 26 | j := 0 27 | for j = 0; (j+i) < len(d) && j < 16; j++ { 28 | part[j] = fmt.Sprintf("0x%x", d[i+j]) 29 | } 30 | parts = append(parts, strings.Join(part[:j], ",")) 31 | } 32 | return fmt.Sprintf("[]byte{%s}", strings.Join(parts, ",\n")) 33 | } 34 | -------------------------------------------------------------------------------- /parrot/data_test.go: -------------------------------------------------------------------------------- 1 | package parrot 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestData_AsBytesLiteral(t *testing.T) { 9 | 10 | var useCases = []struct { 11 | description string 12 | data Data 13 | expect string 14 | ASCII bool 15 | }{ 16 | { 17 | description: "short binary data", 18 | data: []byte("this is test"), 19 | expect: `[]byte{0x74,0x68,0x69,0x73,0x20,0x69,0x73,0x20,0x74,0x65,0x73,0x74}`, 20 | }, 21 | { 22 | description: "long binary data", 23 | data: []byte("this is test with extra data Lorem Ipsum"), 24 | expect: `[]byte{0x74,0x68,0x69,0x73,0x20,0x69,0x73,0x20,0x74,0x65,0x73,0x74,0x20,0x77,0x69,0x74, 25 | 0x68,0x20,0x65,0x78,0x74,0x72,0x61,0x20,0x64,0x61,0x74,0x61,0x20,0x4c,0x6f,0x72, 26 | 0x65,0x6d,0x20,0x49,0x70,0x73,0x75,0x6d}`, 27 | }, 28 | { 29 | description: "literal data", 30 | data: []byte("this is test"), 31 | expect: "[]byte(`this is test`)", 32 | ASCII: true, 33 | }, 34 | } 35 | 36 | for _, useCase := range useCases { 37 | actual := useCase.data.AsBytesLiteral(useCase.ASCII) 38 | assert.EqualValues(t, useCase.expect, actual, useCase.description) 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /parrot/doc.go: -------------------------------------------------------------------------------- 1 | //Package parrot provide storage to go memory or static go file mapping code generation 2 | package parrot 3 | -------------------------------------------------------------------------------- /parrot/mem.go: -------------------------------------------------------------------------------- 1 | package parrot 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/viant/afs" 7 | "github.com/viant/afs/file" 8 | "github.com/viant/afs/storage" 9 | "github.com/viant/afs/url" 10 | "io" 11 | "io/ioutil" 12 | "os" 13 | "path" 14 | "strings" 15 | ) 16 | 17 | //GenerateWithMem generate code that pre loads src location assets into memory storage 18 | func GenerateWithMem(ctx context.Context, src, dest string, useASCII bool, opts ...storage.Option) (err error) { 19 | fs := afs.New() 20 | var uploads = make([]string, 0) 21 | err = fs.Walk(ctx, src, func(ctx context.Context, baseURL string, parent string, info os.FileInfo, reader io.Reader) (toContinue bool, err error) { 22 | if info.IsDir() { 23 | return true, nil 24 | } 25 | var data Data 26 | data, err = ioutil.ReadAll(reader) 27 | destURL := url.Join(dest, path.Join(parent, info.Name())) 28 | uploads = append(uploads, 29 | fmt.Sprintf(` 30 | err = fs.Upload(ctx, "%v", file.DefaultFileOsMode, bytes.NewReader(%v)) 31 | if err != nil { 32 | log.Printf("failed to upload: %v %v", err) 33 | } 34 | `, destURL, data.AsBytesLiteral(useASCII), destURL, `%v`)) 35 | return true, nil 36 | }, opts...) 37 | 38 | if err != nil { 39 | return err 40 | } 41 | if len(uploads) == 0 { 42 | return nil 43 | } 44 | payload := fmt.Sprintf(`package %v 45 | import ( 46 | "bytes" 47 | "log" 48 | "github.com/viant/afs" 49 | "github.com/viant/afs/file" 50 | "context" 51 | ) 52 | 53 | func init() { 54 | fs := afs.New() 55 | ctx := context.Background() 56 | var err error 57 | %v 58 | } 59 | 60 | `, Pkg(dest), strings.Join(uploads, "\n")) 61 | return fs.Upload(ctx, dest, file.DefaultFileOsMode, strings.NewReader(payload)) 62 | } 63 | -------------------------------------------------------------------------------- /parrot/mem_test.go: -------------------------------------------------------------------------------- 1 | package parrot 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/afs" 8 | "github.com/viant/toolbox" 9 | "path" 10 | "strings" 11 | "testing" 12 | ) 13 | 14 | func TestGenerateWithMem(t *testing.T) { 15 | 16 | parent := toolbox.CallerDirectory(3) 17 | 18 | var useCases = []struct { 19 | description string 20 | src string 21 | dest string 22 | useASCII bool 23 | expectFragment string 24 | }{ 25 | { 26 | description: "folder mapping", 27 | src: path.Join(parent, "test"), 28 | dest: "mem://localhost/gen/test1.go", 29 | useASCII: true, 30 | expectFragment: "func run() {}", 31 | }, 32 | { 33 | description: "file mapping", 34 | src: path.Join(parent, "test/runner.go"), 35 | dest: "mem://localhost/gen/test2.go", 36 | useASCII: true, 37 | expectFragment: "func run() {}", 38 | }, 39 | } 40 | 41 | fs := afs.New() 42 | for _, useCase := range useCases { 43 | ctx := context.Background() 44 | err := GenerateWithMem(ctx, useCase.src, useCase.dest, useCase.useASCII) 45 | if !assert.Nil(t, err, useCase.description) { 46 | continue 47 | } 48 | actual, err := fs.DownloadWithURL(ctx, useCase.dest) 49 | if !assert.Nil(t, err, useCase.description) { 50 | continue 51 | } 52 | if !assert.True(t, strings.Contains(string(actual), useCase.expectFragment), useCase.description) { 53 | fmt.Printf("%s\n", actual) 54 | } 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /parrot/static.go: -------------------------------------------------------------------------------- 1 | package parrot 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/viant/afs" 7 | "github.com/viant/afs/file" 8 | "github.com/viant/afs/storage" 9 | "github.com/viant/afs/url" 10 | "io" 11 | "io/ioutil" 12 | "os" 13 | "path" 14 | "strings" 15 | ) 16 | 17 | //Generate generate code that maps source file into destination go files 18 | func Generate(ctx context.Context, src, dest string, useASCII bool, opts ...storage.Option) (err error) { 19 | fs := afs.New() 20 | return fs.Walk(ctx, src, func(ctx context.Context, baseURL string, parent string, info os.FileInfo, reader io.Reader) (toContinue bool, err error) { 21 | if info.IsDir() { 22 | return true, nil 23 | } 24 | var data Data 25 | data, err = ioutil.ReadAll(reader) 26 | container := info.Name() 27 | ext := path.Ext(container) 28 | source := path.Join(parent, info.Name()) 29 | name := "gen" 30 | if ext != "" { 31 | container = strings.ToLower(container[:len(container)-len(ext)]) 32 | name = strings.ToLower(ext[1:]) 33 | } 34 | destURL := url.Join(dest, path.Join(parent, container, name+".go")) 35 | payload := fmt.Sprintf(`package %v 36 | //%v content from %v 37 | var %v = %v`, container, strings.ToUpper(name), source, strings.ToUpper(name), data.AsBytesLiteral(useASCII)) 38 | err = fs.Upload(ctx, destURL, file.DefaultFileOsMode, strings.NewReader(payload)) 39 | return err == nil, err 40 | }, opts...) 41 | 42 | } 43 | -------------------------------------------------------------------------------- /parrot/static_test.go: -------------------------------------------------------------------------------- 1 | package parrot 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/afs" 8 | "github.com/viant/toolbox" 9 | "path" 10 | "strings" 11 | "testing" 12 | ) 13 | 14 | func TestGenerate(t *testing.T) { 15 | 16 | parent := toolbox.CallerDirectory(3) 17 | 18 | var useCases = []struct { 19 | description string 20 | src string 21 | dest string 22 | useASCII bool 23 | expectURL string 24 | expectFragment string 25 | }{ 26 | { 27 | description: "folder mapping", 28 | src: path.Join(parent, "test_data"), 29 | dest: "mem://localhost/data", 30 | useASCII: true, 31 | expectURL: "mem://localhost/data/extract/txt.go", 32 | expectFragment: "var TXT = []byte(`Lorem ipsum dolor sit amet, consectetur adipiscing elit`)", 33 | }, 34 | } 35 | 36 | fs := afs.New() 37 | for _, useCase := range useCases { 38 | ctx := context.Background() 39 | err := Generate(ctx, useCase.src, useCase.dest, useCase.useASCII) 40 | if !assert.Nil(t, err, useCase.description) { 41 | continue 42 | } 43 | actual, err := fs.DownloadWithURL(ctx, useCase.expectURL) 44 | if !assert.Nil(t, err, useCase.description) { 45 | continue 46 | } 47 | if !assert.True(t, strings.Contains(string(actual), useCase.expectFragment), useCase.description) { 48 | fmt.Printf("%s\n", actual) 49 | } 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /parrot/test/runner.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | func run() {} 4 | -------------------------------------------------------------------------------- /parrot/test_data/extract.txt: -------------------------------------------------------------------------------- 1 | Lorem ipsum dolor sit amet, consectetur adipiscing elit -------------------------------------------------------------------------------- /parrot/util.go: -------------------------------------------------------------------------------- 1 | package parrot 2 | 3 | import ( 4 | "github.com/viant/afs/file" 5 | "github.com/viant/afs/url" 6 | ) 7 | 8 | //Pkg returns package name for location 9 | func Pkg(location string) string { 10 | parent, _ := url.Split(location, file.Scheme) 11 | _, pkg := url.Split(parent, file.Scheme) 12 | return pkg 13 | } 14 | -------------------------------------------------------------------------------- /registry.go: -------------------------------------------------------------------------------- 1 | package afs 2 | 3 | import ( 4 | "fmt" 5 | "github.com/viant/afs/file" 6 | "github.com/viant/afs/storage" 7 | "github.com/viant/afs/url" 8 | "sync" 9 | ) 10 | 11 | //Provider represents manager provider 12 | type Provider func(options ...storage.Option) (storage.Manager, error) 13 | 14 | //Registry represents abstract file system service provider registry 15 | type Registry interface { 16 | //Register register schemeURL with storage service 17 | Register(uRLScheme string, provider Provider) 18 | 19 | //Get returns service provider for supplied schemeURL 20 | Get(uRLScheme string) (Provider, error) 21 | } 22 | 23 | type registry struct { 24 | providers map[string]Provider 25 | *sync.RWMutex 26 | } 27 | 28 | func (r *registry) Register(URLScheme string, provider Provider) { 29 | r.Lock() 30 | defer r.Unlock() 31 | r.providers[URLScheme] = provider 32 | 33 | } 34 | 35 | func (r *registry) Get(uRLScheme string) (Provider, error) { 36 | r.RLock() 37 | defer r.RUnlock() 38 | provider, ok := r.providers[uRLScheme] 39 | if !ok { 40 | return nil, fmt.Errorf("failed to lookup storage provider %v", uRLScheme) 41 | } 42 | return provider, nil 43 | } 44 | 45 | var singleton Registry 46 | 47 | //GetRegistry return singleton registry 48 | func GetRegistry() Registry { 49 | if singleton != nil { 50 | return singleton 51 | } 52 | singleton = ®istry{ 53 | providers: make(map[string]Provider), 54 | RWMutex: &sync.RWMutex{}, 55 | } 56 | return singleton 57 | } 58 | 59 | //Manager returns a manager for supplied sourceURL 60 | func Manager(URL string, options ...storage.Option) (storage.Manager, error) { 61 | scheme := url.Scheme(URL, file.Scheme) 62 | provider, err := GetRegistry().Get(scheme) 63 | if err != nil { 64 | return nil, err 65 | } 66 | return provider(options) 67 | } 68 | -------------------------------------------------------------------------------- /scp/client_test.go: -------------------------------------------------------------------------------- 1 | package scp 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "golang.org/x/crypto/ssh" 6 | ) 7 | 8 | func newTestClient(address string) (*ssh.Client, error) { 9 | authenticator, err := LocalhostKeyAuth("") 10 | if err != nil { 11 | return nil, err 12 | } 13 | provider := NewAuthProvider(authenticator, nil) 14 | config, err := provider.ClientConfig() 15 | if err != nil { 16 | return nil, errors.Wrap(err, "failed to create *ssh.ClientConfig") 17 | } 18 | return ssh.Dial("tcp", address, config) 19 | } 20 | -------------------------------------------------------------------------------- /scp/const.go: -------------------------------------------------------------------------------- 1 | package scp 2 | 3 | const ( 4 | //FileToken crate file token 5 | FileToken = 'C' 6 | //DirToken create directory token 7 | DirToken = 'D' 8 | //TimestampToken timestamp token 9 | TimestampToken = 'T' 10 | //EndDirToken end of dir token 11 | EndDirToken = 'E' 12 | //WarningToken warning token 13 | WarningToken = 0x1 14 | //ErrorToken error token 15 | ErrorToken = 0x2 16 | //DefaultPort default SSH port 17 | DefaultPort = 22 18 | ) 19 | -------------------------------------------------------------------------------- /scp/doc.go: -------------------------------------------------------------------------------- 1 | //Package scp implements SSH scp storager and abstract file manager 2 | package scp 3 | -------------------------------------------------------------------------------- /scp/path.go: -------------------------------------------------------------------------------- 1 | package scp 2 | 3 | import ( 4 | "github.com/viant/afs/file" 5 | "os" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | //adjustPath tracks current and previous relative path to adjust accordingly 11 | func adjustPath(prev, current string, moveDown func(info os.FileInfo) error, moveUp func() error) error { 12 | if prev == current { 13 | return nil 14 | } 15 | var prevElements []string 16 | var currElements []string 17 | 18 | if prev != "" { 19 | prevElements = strings.Split(prev, "/") 20 | } 21 | if prev != "" { 22 | currElements = strings.Split(current, "/") 23 | } 24 | if len(prevElements) < len(currElements) { 25 | for i := len(prevElements); i < len(currElements); i++ { 26 | dirInfo := file.NewInfo(currElements[i], 0, file.DefaultDirOsMode, time.Now(), true) 27 | if err := moveDown(dirInfo); err != nil { 28 | return err 29 | } 30 | } 31 | } 32 | var downElements = make([]string, 0) 33 | for i := len(prevElements) - 1; i >= 0; i-- { 34 | prevElem := prevElements[i] 35 | currentElem := "" 36 | if i < len(currElements) { 37 | currentElem = currElements[i] 38 | } 39 | if currentElem == prevElem { 40 | break 41 | } 42 | if currentElem != "" { 43 | downElements = append(downElements, currentElem) 44 | } 45 | if err := moveUp(); err != nil { 46 | return err 47 | } 48 | } 49 | for _, element := range downElements { 50 | dirInfo := file.NewInfo(element, 0, file.DefaultDirOsMode, time.Now(), true) 51 | if err := moveDown(dirInfo); err != nil { 52 | return err 53 | } 54 | } 55 | return nil 56 | } 57 | -------------------------------------------------------------------------------- /scp/provider.go: -------------------------------------------------------------------------------- 1 | package scp 2 | 3 | import ( 4 | "github.com/viant/afs/storage" 5 | ) 6 | 7 | //Provider returns a http manager 8 | func Provider(options ...storage.Option) (storage.Manager, error) { 9 | return New(options...), nil 10 | } 11 | -------------------------------------------------------------------------------- /scp/reader.go: -------------------------------------------------------------------------------- 1 | package scp 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "sync/atomic" 7 | "time" 8 | ) 9 | 10 | type reader struct { 11 | processors int 12 | reader io.Reader 13 | outputChan chan []byte 14 | closedChan chan bool 15 | errorChan chan error 16 | closed uint32 17 | } 18 | 19 | func (r *reader) sendCloseNotification() { 20 | for i := 0; i < r.processors; i++ { 21 | select { 22 | case r.closedChan <- true: 23 | case <-time.After(time.Millisecond): 24 | } 25 | } 26 | 27 | } 28 | 29 | func (r *reader) isClosed() bool { 30 | return atomic.LoadUint32(&r.closed) == 1 31 | } 32 | 33 | func (r *reader) close() { 34 | if atomic.CompareAndSwapUint32(&r.closed, 0, 1) { 35 | close(r.errorChan) 36 | close(r.closedChan) 37 | close(r.outputChan) 38 | } 39 | } 40 | 41 | func (r *reader) read(timeout time.Duration) ([]byte, error) { 42 | if r.isClosed() { 43 | return nil, fmt.Errorf("closed") 44 | } 45 | select { 46 | case data := <-r.outputChan: 47 | return data, nil 48 | case err := <-r.errorChan: 49 | return nil, err 50 | case <-time.Tick(timeout): 51 | return nil, fmt.Errorf("exceeded timeout %s", timeout) 52 | case <-r.closedChan: 53 | return nil, fmt.Errorf("closed") 54 | } 55 | } 56 | 57 | func (r *reader) readInBackground() { 58 | for { 59 | var buffer = make([]byte, 4096) 60 | n, err := r.reader.Read(buffer) 61 | if err != nil { 62 | r.closeWithError(err) 63 | return 64 | } 65 | if r.isClosed() { 66 | return 67 | } 68 | if n == 0 { 69 | continue 70 | } 71 | select { 72 | case r.outputChan <- buffer[:n]: 73 | continue 74 | case <-r.closedChan: 75 | return 76 | } 77 | } 78 | } 79 | 80 | func (r *reader) closeWithError(err error) { 81 | if r.isClosed() { 82 | return 83 | } 84 | r.errorChan <- err 85 | r.sendCloseNotification() 86 | } 87 | 88 | func newReader(ioReader io.Reader) *reader { 89 | const processors = 3 90 | return &reader{ 91 | reader: ioReader, 92 | outputChan: make(chan []byte, processors), 93 | closedChan: make(chan bool, processors), 94 | errorChan: make(chan error, processors), 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /scp/scheme.go: -------------------------------------------------------------------------------- 1 | package scp 2 | 3 | //Scheme defines scp URL scheme 4 | const Scheme = "scp" 5 | -------------------------------------------------------------------------------- /ssh/scheme.go: -------------------------------------------------------------------------------- 1 | package ssh 2 | 3 | const Scheme = "ssh" 4 | -------------------------------------------------------------------------------- /storage/auth.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import "context" 4 | 5 | //Authenticator represents an authennticator 6 | type Authenticator interface { 7 | //Auth authenticate URL scheme with authentication option 8 | Auth(baseURL string, option ...Option) 9 | } 10 | 11 | //AuthTracker represents auth change tracker 12 | type AuthTracker interface { 13 | //IsAuthChanged return true if auth has changed 14 | IsAuthChanged(ctx context.Context, baseURL string, options []Option) bool 15 | } 16 | 17 | //StoragerAuthTracker represents auth manager 18 | type StoragerAuthTracker interface { 19 | 20 | //FilterAuthOptions filters auth options 21 | FilterAuthOptions(option []Option) []Option 22 | 23 | //IsAuthChanged return true if auth has changes 24 | IsAuthChanged(authOptions []Option) bool 25 | } 26 | -------------------------------------------------------------------------------- /storage/checker.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import "context" 4 | 5 | //Checker represents abstraction that check if resource exists 6 | type Checker interface { 7 | //Exists returns true if resource exists 8 | Exists(ctx context.Context, URL string, options ...Option) (bool, error) 9 | } 10 | -------------------------------------------------------------------------------- /storage/copier.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | //Copier represents an asset copier 8 | type Copier interface { 9 | //Copy copies source to dest 10 | Copy(ctx context.Context, sourceURL, destURL string, options ...Option) error 11 | } 12 | -------------------------------------------------------------------------------- /storage/doc.go: -------------------------------------------------------------------------------- 1 | //Package storage defines Storage API 2 | package storage 3 | -------------------------------------------------------------------------------- /storage/manager.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | ) 8 | 9 | //Manager represents storage manager 10 | type Manager interface { 11 | Lister 12 | Opener 13 | Uploader 14 | Deleter 15 | Creator 16 | io.Closer 17 | Scheme() string 18 | } 19 | 20 | //Lister represents asset lister 21 | type Lister interface { 22 | //List returns a list of object for supplied url 23 | List(ctx context.Context, URL string, options ...Option) ([]Object, error) 24 | } 25 | 26 | //Getter represents asset getter 27 | type Getter interface { 28 | //List returns a list of object for supplied url 29 | Object(ctx context.Context, URL string, options ...Option) (Object, error) 30 | } 31 | 32 | //Opener represents a downloader 33 | type Opener interface { 34 | //Open returns reader for downloaded storage object 35 | Open(ctx context.Context, object Object, options ...Option) (io.ReadCloser, error) 36 | 37 | //Open returns reader for downloaded storage object 38 | OpenURL(ctx context.Context, URL string, options ...Option) (io.ReadCloser, error) 39 | } 40 | 41 | //Deleter represents a deleter 42 | type Deleter interface { 43 | //Delete removes passed in storage object 44 | Delete(ctx context.Context, URL string, options ...Option) error 45 | } 46 | 47 | //Creator represents a creator 48 | type Creator interface { 49 | //CreateBucket creates a bucket 50 | Create(ctx context.Context, URL string, mode os.FileMode, isDir bool, options ...Option) error 51 | } 52 | 53 | //ErrorCoder represents error coder 54 | type ErrorCoder interface { 55 | //ErrorCode returns an error code 56 | ErrorCode(err error) int 57 | } 58 | -------------------------------------------------------------------------------- /storage/mover.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import "context" 4 | 5 | //Mover represents an asset mover 6 | type Mover interface { 7 | //Move moves source to dest 8 | Move(ctx context.Context, sourceURL, destURL string, options ...Option) error 9 | } 10 | -------------------------------------------------------------------------------- /storage/object.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "os" 5 | "sync" 6 | ) 7 | 8 | //Object represents a storage object 9 | type Object interface { 10 | os.FileInfo 11 | //URL return storage url 12 | URL() string 13 | 14 | //Wrap wraps source storage object 15 | Wrap(source interface{}) 16 | //Unwrap unwraps source storage object into provided target. 17 | Unwrap(target interface{}) error 18 | //FileInfo return file info 19 | } 20 | 21 | //Objects represents synchromized object collection wrapper 22 | type Objects struct { 23 | ptr *[]Object 24 | mux sync.Mutex 25 | } 26 | 27 | //Append appens object 28 | func (s *Objects) Append(object Object) { 29 | s.mux.Lock() 30 | *s.ptr = append(*s.ptr, object) 31 | s.mux.Unlock() 32 | } 33 | 34 | //Objects returns objects 35 | func (s *Objects) Objects() []Object { 36 | return *s.ptr 37 | } 38 | 39 | //NewObjects creates sycnronized objects 40 | func NewObjects(ptr *[]Object) *Objects { 41 | if ptr == nil { 42 | obj := make([]Object, 0) 43 | ptr = &obj 44 | } 45 | return &Objects{ptr: ptr} 46 | } 47 | -------------------------------------------------------------------------------- /storage/option.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | //Option represents generic service operation option 4 | type Option interface{} 5 | -------------------------------------------------------------------------------- /storage/options.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | //Options represents options 4 | type Options []Option 5 | 6 | //NewOptions returns new options 7 | func NewOptions(options []Option, extraOptions ...Option) []Option { 8 | return append(options, extraOptions...) 9 | } 10 | -------------------------------------------------------------------------------- /storage/sizer.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | //Sizer represents abstraction returing a size 4 | type Sizer interface { 5 | Size() int64 6 | } 7 | -------------------------------------------------------------------------------- /storage/storager.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | ) 8 | 9 | //Storager represents path oriented storage service 10 | type Storager interface { 11 | io.Closer 12 | 13 | //Exists returns true if location exists 14 | Exists(ctx context.Context, location string, options ...Option) (bool, error) 15 | 16 | //List lists location assets 17 | List(ctx context.Context, location string, options ...Option) ([]os.FileInfo, error) 18 | 19 | //Get returns a file info for supplied location 20 | Get(ctx context.Context, location string, options ...Option) (os.FileInfo, error) 21 | 22 | //Open returns a reader closer for supplied resources 23 | Open(ctx context.Context, location string, options ...Option) (io.ReadCloser, error) 24 | 25 | //Upload uploads 26 | Upload(ctx context.Context, destination string, mode os.FileMode, reader io.Reader, options ...Option) error 27 | 28 | //Create create file or directory 29 | Create(ctx context.Context, destination string, mode os.FileMode, reader io.Reader, isDir bool, options ...Option) error 30 | 31 | //Delete deletes locations 32 | Delete(ctx context.Context, location string, options ...Option) error 33 | } 34 | -------------------------------------------------------------------------------- /storage/uploader.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | ) 8 | 9 | //Upload uploads content 10 | type Upload func(ctx context.Context, parent string, info os.FileInfo, reader io.Reader) error 11 | 12 | //Uploader represents an uploader 13 | type Uploader interface { 14 | //Upload uploads provided reader content for supplied storage object. 15 | Upload(ctx context.Context, URL string, mode os.FileMode, reader io.Reader, options ...Option) error 16 | } 17 | 18 | //BatchUploader represents a batch uploader 19 | type BatchUploader interface { 20 | //Uploader returns upload handler, and upload closer for batch upload or error 21 | Uploader(ctx context.Context, URL string, options ...Option) (Upload, io.Closer, error) 22 | } 23 | -------------------------------------------------------------------------------- /storage/walker.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | ) 8 | 9 | //OnVisit represents on location visit handler 10 | type OnVisit func(ctx context.Context, baseURL string, parent string, info os.FileInfo, reader io.Reader) (toContinue bool, err error) 11 | 12 | //Walker represents abstract storage walker 13 | type Walker interface { 14 | //Walk traverses URL and calls handler on all file or folder 15 | Walk(ctx context.Context, URL string, handler OnVisit, options ...Option) error 16 | } 17 | -------------------------------------------------------------------------------- /storage/writer.go: -------------------------------------------------------------------------------- 1 | package storage 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | ) 8 | 9 | //WriterProvider represents writer provider 10 | type WriterProvider interface { 11 | NewWriter(ctx context.Context, URL string, mode os.FileMode, options ...Option) (io.WriteCloser, error) 12 | } 13 | -------------------------------------------------------------------------------- /sync/counter_test.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "context" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/viant/afs" 7 | "testing" 8 | ) 9 | 10 | func TestCounter_Increment(t *testing.T) { 11 | 12 | var useCases = []struct { 13 | description string 14 | URL string 15 | Data interface{} 16 | }{ 17 | { 18 | description: "memory sync counter", 19 | URL: "mem://localhost/counter/case001/data.cnt", 20 | }, 21 | { 22 | description: "counter with data", 23 | URL: "mem://localhost/counter/case001/data.cnt", 24 | Data: 123, 25 | }, 26 | } 27 | 28 | fs := afs.New() 29 | for nil, useCase := range useCases { 30 | ctx := context.Background() 31 | counter := NewCounter(useCase.URL, fs) 32 | counter.Data = useCase.Data 33 | for i := 0; i < 10; i++ { 34 | count, err := counter.Increment(ctx) 35 | if assert.Nil(t, err, useCase.description) { 36 | continue 37 | } 38 | if useCase.Data != nil { 39 | assert.EqualValues(t, useCase.Data, counter.Data, useCase.description) 40 | } 41 | counter.Data = struct{}{} 42 | assert.EqualValues(t, i+1, count) 43 | } 44 | for i := 10; i >= 0; i-- { 45 | count, err := counter.Decrement(ctx) 46 | if assert.Nil(t, err, useCase.description) { 47 | continue 48 | } 49 | assert.EqualValues(t, i-1, count) 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /sync/doc.go: -------------------------------------------------------------------------------- 1 | //Package sync define atomic file system base counter 2 | package sync 3 | -------------------------------------------------------------------------------- /tar/README.md: -------------------------------------------------------------------------------- 1 | # tar - tar archives 2 | 3 | ## Usage 4 | 5 | 6 | 7 | * **Service** 8 | 9 | ```go 10 | service := afs.New() 11 | ctx := context.Background() 12 | objects, err := service.List(ctx, "file:/tmp/app.war/tar://localhost/WEB-INF") 13 | if err != nil { 14 | log.Fatal(err) 15 | } 16 | for _, object := range objects { 17 | fmt.Printf("%v %v\n", object.Name(), object.URL()) 18 | if object.IsDir() { 19 | continue 20 | } 21 | reader, err := service.Download(ctx, object) 22 | if err != nil { 23 | log.Fatal(err) 24 | } 25 | data, err := ioutil.ReadAll(reader) 26 | if err != nil { 27 | log.Fatal(err) 28 | } 29 | fmt.Printf("%s\n", data) 30 | } 31 | ``` 32 | 33 | * **Walker** 34 | 35 | ```go 36 | ctx := context.Background() 37 | service := afs.New() 38 | walker := tar.NewWalker(file.New()) 39 | err := service.Copy(ctx, "/tmp/test.tar", "mem://dest/folder/test", walker) 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | ``` 44 | 45 | * **Uploader** 46 | 47 | ```go 48 | ctx := context.Background() 49 | service := afs.New() 50 | uploader := tar.NewBatchUploader(file.New()) 51 | err := service.Copy(ctx, "/tmp/test/data", "/tmp/data.tar", uploader) 52 | if err != nil { 53 | log.Fatal(err) 54 | } 55 | ``` 56 | -------------------------------------------------------------------------------- /tar/doc.go: -------------------------------------------------------------------------------- 1 | //Package tar provides support for operating on TAR archives 2 | package tar 3 | -------------------------------------------------------------------------------- /tar/doc_test.go: -------------------------------------------------------------------------------- 1 | package tar_test 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs" 6 | "github.com/viant/afs/file" 7 | "github.com/viant/afs/tar" 8 | "log" 9 | ) 10 | 11 | func ExampleNewWalker() { 12 | ctx := context.Background() 13 | service := afs.New() 14 | walker := tar.NewWalker(file.New()) 15 | err := service.Copy(ctx, "/tmp/test.tar", "mem://dest/folder/test", walker) 16 | if err != nil { 17 | log.Fatal(err) 18 | } 19 | 20 | } 21 | 22 | func ExampleNewBatchUploader() { 23 | ctx := context.Background() 24 | service := afs.New() 25 | uploader := tar.NewBatchUploader(file.New()) 26 | err := service.Copy(ctx, "/tmp/test/data", "/tmp/data.tar", uploader) 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /tar/manager_test.go: -------------------------------------------------------------------------------- 1 | package tar_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/afs" 8 | "path" 9 | "runtime" 10 | "testing" 11 | ) 12 | 13 | func TestNew(t *testing.T) { 14 | 15 | _, filename, _, _ := runtime.Caller(0) 16 | baseDir, _ := path.Split(filename) 17 | ctx := context.Background() 18 | 19 | var useCases = []struct { 20 | description string 21 | URL string 22 | expect map[string]bool 23 | }{ 24 | { 25 | description: "list war classes", 26 | URL: fmt.Sprintf("file:%v/test/app.war/tar://localhost/WEB-INF/classes", baseDir), 27 | expect: map[string]bool{ 28 | "classes": true, 29 | "HelloWorld.class": true, 30 | "config.properties": true, 31 | }, 32 | }, 33 | 34 | { 35 | description: "list war classes", 36 | URL: fmt.Sprintf("file:%v/test/app.war/tar://localhost/WEB-INF/classes/config.properties", baseDir), 37 | expect: map[string]bool{ 38 | "config.properties": true, 39 | }, 40 | }, 41 | } 42 | 43 | for _, useCase := range useCases { 44 | service := afs.New() 45 | objects, err := service.List(ctx, useCase.URL) 46 | assert.Nil(t, err, useCase.description) 47 | assert.EqualValues(t, len(useCase.expect), len(objects)) 48 | for _, obj := range objects { 49 | assert.True(t, useCase.expect[obj.Name()], useCase.description+" "+obj.URL()) 50 | } 51 | 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /tar/provider.go: -------------------------------------------------------------------------------- 1 | package tar 2 | 3 | import "github.com/viant/afs/storage" 4 | 5 | //Provider returns a http manager 6 | func Provider(options ...storage.Option) (storage.Manager, error) { 7 | return New(options...), nil 8 | } 9 | -------------------------------------------------------------------------------- /tar/scheme.go: -------------------------------------------------------------------------------- 1 | package tar 2 | 3 | //Scheme defines tar URL scheme 4 | const Scheme = "tar" 5 | -------------------------------------------------------------------------------- /tar/test/app.war: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viant/afs/f200dceb289ef98dfa3ebd148bdfb8a642ade924/tar/test/app.war -------------------------------------------------------------------------------- /tar/test/test.tar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viant/afs/f200dceb289ef98dfa3ebd148bdfb8a642ade924/tar/test/test.tar -------------------------------------------------------------------------------- /tar/uploader.go: -------------------------------------------------------------------------------- 1 | package tar 2 | 3 | import ( 4 | "archive/tar" 5 | "bytes" 6 | "context" 7 | "github.com/viant/afs/file" 8 | "github.com/viant/afs/option" 9 | "github.com/viant/afs/storage" 10 | "io" 11 | "os" 12 | "path" 13 | ) 14 | 15 | type uploader struct { 16 | uploader storage.Uploader 17 | buffer *bytes.Buffer 18 | } 19 | 20 | func (u *uploader) Uploader(ctx context.Context, URL string, options ...storage.Option) (storage.Upload, io.Closer, error) { 21 | var uploader storage.Uploader 22 | option.Assign(options, &u.buffer, &uploader) 23 | if uploader == nil { 24 | uploader = u.uploader 25 | } 26 | if u.buffer == nil { 27 | u.buffer = new(bytes.Buffer) 28 | } 29 | writer := newWriter(ctx, u.buffer, URL, uploader) 30 | return func(ctx context.Context, parent string, info os.FileInfo, reader io.Reader) error { 31 | link := "" 32 | var options []storage.Option 33 | if fileInfo, ok := info.(*file.Info); ok { 34 | link = fileInfo.Linkname 35 | options = make([]storage.Option, 0) 36 | options = append(options, fileInfo.Link) 37 | } 38 | filename := path.Join(parent, info.Name()) 39 | info = file.NewInfo(filename, info.Size(), info.Mode(), info.ModTime(), info.IsDir(), options...) 40 | header, err := tar.FileInfoHeader(info, link) 41 | if err != nil { 42 | return err 43 | } 44 | if info.IsDir() { 45 | header.Typeflag = tar.TypeDir 46 | } 47 | if err = writer.WriteHeader(header); err != nil { 48 | return err 49 | } 50 | if info.Mode().IsRegular() && reader != nil { 51 | _, err = io.Copy(writer, reader) 52 | } 53 | return err 54 | 55 | }, writer, nil 56 | } 57 | 58 | //newBatchUploader returns a batch uploader 59 | func newBatchUploader(dest storage.Uploader) *uploader { 60 | return &uploader{uploader: dest} 61 | } 62 | 63 | //NewBatchUploader returns a batch uploader 64 | func NewBatchUploader(dest storage.Uploader) storage.BatchUploader { 65 | return newBatchUploader(dest) 66 | } 67 | -------------------------------------------------------------------------------- /tar/uploader_test.go: -------------------------------------------------------------------------------- 1 | package tar 2 | 3 | import ( 4 | "context" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/viant/afs/asset" 7 | "github.com/viant/afs/file" 8 | "os" 9 | "path" 10 | "testing" 11 | ) 12 | 13 | func TestUploader_Uploader(t *testing.T) { 14 | 15 | baseDir := os.TempDir() 16 | fileManager := file.New() 17 | 18 | var useCases = []struct { 19 | description string 20 | destURL string 21 | createLocation bool 22 | assets []*asset.Resource 23 | }{ 24 | 25 | { 26 | description: "multi download - unordered", 27 | destURL: path.Join(baseDir, "tar_upload_01/test.tar"), 28 | assets: []*asset.Resource{ 29 | asset.NewDir("test", 0744), 30 | asset.NewDir("test/folder2/sub", 0744), 31 | asset.NewFile("test/asset1.txt", []byte("xyz"), 0644), 32 | asset.NewFile("test/asset2.txt", []byte("xyz"), 0644), 33 | asset.NewDir("test/folder1", 0744), 34 | asset.NewFile("test/folder1/res.txt", []byte("xyz"), 0644), 35 | asset.NewFile("test/folder2/res1.txt", []byte("xyz"), 0644), 36 | }, 37 | }, 38 | 39 | { 40 | description: "multi download - link", 41 | destURL: path.Join(baseDir, "tar_upload_02/test.tar"), 42 | assets: []*asset.Resource{ 43 | asset.NewFile("foo1.txt", []byte("abc"), 0644), 44 | asset.NewFile("foo2.txt", []byte("xyz"), 0644), 45 | asset.NewLink("sym.txt", "foo1.txt", 0644), 46 | }, 47 | }, 48 | } 49 | 50 | for _, useCase := range useCases { 51 | ctx := context.Background() 52 | uploader := NewBatchUploader(fileManager) 53 | upload, closer, err := uploader.Uploader(ctx, useCase.destURL) 54 | if !assert.Nil(t, err, useCase.description) { 55 | 56 | } 57 | for _, asset := range useCase.assets { 58 | relative, _ := path.Split(asset.Name) 59 | err = upload(ctx, relative, asset.Info(), asset.Reader()) 60 | assert.Nil(t, err, useCase.description+" "+asset.Name) 61 | } 62 | err = closer.Close() 63 | assert.Nil(t, err, useCase.description) 64 | _, err = os.Stat(useCase.destURL) 65 | assert.Nil(t, err, useCase.description) 66 | _ = asset.Cleanup(fileManager, useCase.destURL) 67 | 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /tar/walker_test.go: -------------------------------------------------------------------------------- 1 | package tar_test 2 | 3 | import ( 4 | "context" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/viant/afs" 7 | "github.com/viant/afs/asset" 8 | "github.com/viant/afs/file" 9 | "github.com/viant/afs/tar" 10 | "io" 11 | "io/ioutil" 12 | "os" 13 | "path" 14 | "runtime" 15 | "testing" 16 | ) 17 | 18 | func TestWalker_Walk(t *testing.T) { 19 | 20 | _, filename, _, _ := runtime.Caller(0) 21 | baseDir, _ := path.Split(filename) 22 | 23 | var useCases = []struct { 24 | description string 25 | location string 26 | expect []*asset.Resource 27 | }{ 28 | { 29 | description: "tar walking", 30 | location: path.Join(baseDir, "test/test.tar"), 31 | expect: []*asset.Resource{ 32 | asset.NewDir("test", file.DefaultDirOsMode), 33 | asset.NewFile("test/asset1.txt", []byte("test is test\n"), 0644), 34 | asset.NewFile("test/asset2.txt", []byte("test is second test\n"), 0644), 35 | asset.NewDir("test/folder1", file.DefaultDirOsMode), 36 | asset.NewFile("test/folder1/res.txt", []byte("resource 1\n"), 0644), 37 | asset.NewDir("test/folder2", file.DefaultDirOsMode), 38 | asset.NewFile("test/folder2/res1.txt", []byte("resource 1\n"), 0644), 39 | }, 40 | }, 41 | } 42 | 43 | for _, useCase := range useCases { 44 | walker := tar.NewWalker(afs.New()) 45 | ctx := context.Background() 46 | actuals := make(map[string]*asset.Resource) 47 | err := walker.Walk(ctx, useCase.location, func(ctx context.Context, baseURL string, parent string, info os.FileInfo, reader io.Reader) (toContinue bool, err error) { 48 | 49 | resourceLocation := path.Join(parent, info.Name()) 50 | linkName := "" 51 | if rawInfo, ok := info.(*file.Info); ok { 52 | linkName = rawInfo.Linkname 53 | } 54 | var data []byte 55 | if reader != nil { 56 | data, err = ioutil.ReadAll(reader) 57 | if err != nil { 58 | return false, err 59 | } 60 | } 61 | actuals[resourceLocation] = asset.New(parent, info.Mode(), info.IsDir(), linkName, data) 62 | return true, nil 63 | }) 64 | assert.Nil(t, err, useCase.description) 65 | 66 | for _, asset := range useCase.expect { 67 | _, ok := actuals[asset.Name] 68 | assert.True(t, ok, useCase.description+" "+asset.Name) 69 | } 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /tar/writer.go: -------------------------------------------------------------------------------- 1 | package tar 2 | 3 | import ( 4 | "archive/tar" 5 | "bytes" 6 | "context" 7 | "github.com/viant/afs/storage" 8 | ) 9 | 10 | type writer struct { 11 | ctx context.Context 12 | buffer *bytes.Buffer 13 | *tar.Writer 14 | destURL string 15 | uploader storage.Uploader 16 | } 17 | 18 | func newWriter(ctx context.Context, buffer *bytes.Buffer, URL string, uploader storage.Uploader) *writer { 19 | if buffer == nil { 20 | buffer = new(bytes.Buffer) 21 | } 22 | return &writer{ 23 | ctx: ctx, 24 | buffer: buffer, 25 | Writer: tar.NewWriter(buffer), 26 | destURL: URL, 27 | uploader: uploader, 28 | } 29 | } 30 | 31 | func (w *writer) Close() error { 32 | err := w.Writer.Flush() 33 | if err == nil { 34 | if err = w.Writer.Close(); err == nil { 35 | if w.uploader != nil { 36 | err = w.uploader.Upload(w.ctx, w.destURL, 0644, bytes.NewReader(w.buffer.Bytes())) 37 | } 38 | } 39 | } 40 | return err 41 | } 42 | -------------------------------------------------------------------------------- /uploader.go: -------------------------------------------------------------------------------- 1 | package afs 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs/base" 6 | "github.com/viant/afs/file" 7 | "github.com/viant/afs/storage" 8 | "github.com/viant/afs/url" 9 | "io" 10 | ) 11 | 12 | //UploadInBatch default implementation for UploadInBatch 13 | func (s *service) Uploader(ctx context.Context, URL string, options ...storage.Option) (storage.Upload, io.Closer, error) { 14 | URL = url.Normalize(URL, file.Scheme) 15 | manager, err := s.manager(ctx, URL, options) 16 | if err != nil { 17 | return nil, nil, err 18 | } 19 | batchUploader, ok := manager.(storage.BatchUploader) 20 | if !ok { 21 | batchUploader = base.NewUploader(manager) 22 | } 23 | return batchUploader.Uploader(ctx, URL, options...) 24 | } 25 | -------------------------------------------------------------------------------- /url/base.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | //Base returns base URL and URL path 8 | func Base(URL string, defaultSchema string) (string, string) { 9 | schema := Scheme(URL, defaultSchema) 10 | schemaExt := SchemeExtensionURL(URL) 11 | host := Host(URL) 12 | path := Path(URL) 13 | if schemaExt != "" { 14 | schemaExt = strings.Replace(schemaExt, "://", ":", 1) 15 | base := schemaExt + "/" + schema + "://" + host 16 | return base, path 17 | } 18 | return schema + "://" + host, path 19 | } 20 | -------------------------------------------------------------------------------- /url/base_test.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestBase(t *testing.T) { 9 | var useCases = []struct { 10 | description string 11 | URL string 12 | expectBaseURL string 13 | expectPath string 14 | }{ 15 | { 16 | description: "simple path", 17 | URL: "/tmp/abc.txt", 18 | expectBaseURL: "file://localhost", 19 | expectPath: "/tmp/abc.txt", 20 | }, 21 | { 22 | description: "scp path", 23 | URL: "scp://10.1.22.1/tmp/abc.txt", 24 | expectBaseURL: "scp://10.1.22.1", 25 | expectPath: "/tmp/abc.txt", 26 | }, 27 | 28 | { 29 | description: "ftp path", 30 | URL: "ftp://10.1.22.1:33/tmp/abc.txt", 31 | expectBaseURL: "ftp://10.1.22.1:33", 32 | expectPath: "/tmp/abc.txt", 33 | }, 34 | 35 | { 36 | description: "ftp root", 37 | URL: "ftp://10.1.22.1:33/", 38 | expectBaseURL: "ftp://10.1.22.1:33", 39 | expectPath: "/", 40 | }, 41 | { 42 | description: "mem", 43 | URL: "mem:///var/folders/gl/5550g3kj6tn1rbz8chqx1c61ycmmm1/", 44 | expectBaseURL: "mem://", 45 | expectPath: "/var/folders/gl/5550g3kj6tn1rbz8chqx1c61ycmmm1/", 46 | }, 47 | } 48 | 49 | for _, useCase := range useCases { 50 | actualBaseURL, actualPath := Base(useCase.URL, "file") 51 | assert.EqualValues(t, useCase.expectBaseURL, actualBaseURL, useCase.description) 52 | assert.EqualValues(t, useCase.expectPath, actualPath, useCase.description) 53 | 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /url/dir.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "path" 5 | "strings" 6 | ) 7 | 8 | //Dir returns director 9 | func Dir(URL string) string { 10 | _, URLPath := Base(URL, "file") 11 | if strings.HasSuffix(URLPath, "/") { 12 | URLPath = string(URLPath[:len(URLPath)-1]) 13 | } 14 | parent, _ := path.Split(URLPath) 15 | return parent 16 | } 17 | -------------------------------------------------------------------------------- /url/doc.go: -------------------------------------------------------------------------------- 1 | //Package url define URL utilites 2 | package url 3 | -------------------------------------------------------------------------------- /url/equals.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import "strings" 4 | 5 | //Equals checks if url are the same 6 | func Equals(URL1, URL2 string) bool { 7 | base1, path1 := Base(URL1, "file") 8 | base2, path2 := Base(URL2, "file") 9 | if base1 != base2 { 10 | return false 11 | } 12 | return strings.Trim(path1, "/") == strings.Trim(path2, "/") 13 | } 14 | -------------------------------------------------------------------------------- /url/host.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | const ( 8 | //Localhost default host 9 | Localhost = "localhost" 10 | ) 11 | 12 | //Host extract host from URL 13 | func Host(URL string) string { 14 | 15 | index := strings.Index(URL, "://") 16 | if index == -1 { 17 | return Localhost 18 | } 19 | fragment := string(URL[index+3:]) 20 | index = strings.Index(fragment, "/") 21 | if index != -1 { 22 | return string(fragment[0:index]) 23 | } 24 | return fragment 25 | } 26 | -------------------------------------------------------------------------------- /url/host_test.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestHost(t *testing.T) { 9 | 10 | var useCases = []struct { 11 | description string 12 | URL string 13 | expect string 14 | }{ 15 | 16 | { 17 | URL: "/tmp", 18 | expect: Localhost, 19 | }, 20 | { 21 | URL: "scp://myhost:22/dad", 22 | expect: "myhost:22", 23 | }, 24 | 25 | { 26 | description: "host only", 27 | URL: "scp://myhost:22", 28 | expect: "myhost:22", 29 | }, 30 | } 31 | 32 | for _, useCase := range useCases { 33 | actual := Host(useCase.URL) 34 | assert.EqualValues(t, useCase.expect, actual, useCase.description) 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /url/join.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | //Join joins base URL with path elements 8 | func Join(baseURL string, elements ...string) string { 9 | if strings.HasSuffix(baseURL, "://") { 10 | baseURL += Localhost 11 | } else if strings.HasSuffix(baseURL, "/") { 12 | index := strings.LastIndex(baseURL, "/") 13 | if index != -1 { 14 | baseURL = baseURL[:index] 15 | } 16 | } 17 | if len(elements) == 0 { 18 | return baseURL 19 | } 20 | 21 | for i := range elements { 22 | elements[i] = strings.Trim(elements[i], "/") 23 | } 24 | return baseURL + "/" + strings.Join(elements, "/") 25 | } 26 | 27 | //JoinUNC joins base URL with path elements, it support '.' or '..' elements 28 | func JoinUNC(baseURL string, fragments ...string) string { 29 | if strings.HasSuffix(baseURL, "://") { 30 | baseURL += Localhost 31 | } else if strings.HasSuffix(baseURL, "/") { 32 | index := strings.LastIndex(baseURL, "/") 33 | if index != -1 { 34 | baseURL = baseURL[:index] 35 | } 36 | } 37 | if len(fragments) == 0 { 38 | return baseURL 39 | } 40 | schema := Scheme(baseURL, "file") 41 | baseURL, basePath := Base(baseURL, schema) 42 | result := strings.Split(basePath, "/") 43 | for i := range fragments { 44 | fragment := strings.Trim(fragments[i], "/") 45 | 46 | elements := strings.Split(fragment, "/") 47 | 48 | for _, element := range elements { 49 | if element == "." || element == "" { 50 | continue 51 | } 52 | if element == ".." { 53 | if len(result) > 0 { 54 | result = result[:len(result)-1] 55 | } 56 | continue 57 | } 58 | result = append(result, element) 59 | } 60 | } 61 | location := strings.Join(result, "/") 62 | return baseURL + "/" + strings.Trim(location, "/") 63 | } 64 | -------------------------------------------------------------------------------- /url/join_test.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestJoin(t *testing.T) { 9 | 10 | var useCases = []struct { 11 | description string 12 | baseURL string 13 | elemenets []string 14 | expect string 15 | }{ 16 | { 17 | description: "relative elements", 18 | baseURL: "ftp://localhost", 19 | elemenets: []string{"foo", "bar.txt"}, 20 | expect: "ftp://localhost/foo/bar.txt", 21 | }, 22 | { 23 | description: "trimmed elements", 24 | baseURL: "ftp://localhost", 25 | elemenets: []string{"/foo/", "/bar.txt"}, 26 | expect: "ftp://localhost/foo/bar.txt", 27 | }, 28 | { 29 | description: "base path only", 30 | baseURL: "ftp://localhost", 31 | elemenets: []string{}, 32 | expect: "ftp://localhost", 33 | }, 34 | { 35 | description: "relative path", 36 | baseURL: "/tmp", 37 | elemenets: []string{"foo", "data.bar"}, 38 | expect: "/tmp/foo/data.bar", 39 | }, 40 | } 41 | 42 | for _, useCase := range useCases { 43 | actual := Join(useCase.baseURL, useCase.elemenets...) 44 | assert.EqualValues(t, useCase.expect, actual, useCase.description) 45 | } 46 | 47 | } 48 | 49 | func TestJoinUNC(t *testing.T) { 50 | 51 | var useCases = []struct { 52 | description string 53 | baseURL string 54 | elemenets []string 55 | expect string 56 | }{ 57 | { 58 | description: "relative elements", 59 | baseURL: "ftp://localhost/path/subpath", 60 | elemenets: []string{"foo/bar.txt"}, 61 | expect: "ftp://localhost/path/subpath/foo/bar.txt", 62 | }, 63 | { 64 | description: ".. elements", 65 | baseURL: "ftp://localhost/data/path/subpath", 66 | elemenets: []string{"../../foo/bar.txt"}, 67 | expect: "ftp://localhost/data/foo/bar.txt", 68 | }, 69 | } 70 | 71 | for _, useCase := range useCases { 72 | actual := JoinUNC(useCase.baseURL, useCase.elemenets...) 73 | assert.EqualValues(t, useCase.expect, actual, useCase.description) 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /url/normalize.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | ) 7 | 8 | //Normalize normalizes URL 9 | func Normalize(URL, scheme string) string { 10 | if strings.Index(URL, ":") == -1 { 11 | if URL != "" && URL[0] != '/' { 12 | if basePath, err := os.Getwd(); err == nil { 13 | URL = Join(basePath, URL) 14 | } 15 | } 16 | } 17 | schema := Scheme(URL, scheme) 18 | baseURL, URLPath := Base(URL, schema) 19 | return Join(baseURL, URLPath) 20 | } 21 | -------------------------------------------------------------------------------- /url/path.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "runtime" 5 | "strings" 6 | ) 7 | 8 | //Path returns path for an URL or path 9 | func Path(URL string) string { 10 | location := URL 11 | if location == "" { 12 | return "/" 13 | } 14 | 15 | if runtime.GOOS == "windows" { 16 | location = strings.ReplaceAll(location, "\\", "/") 17 | } 18 | if index := strings.Index(location, "://"); index != -1 { 19 | location = string(location[index+3:]) 20 | index := strings.Index(location, "/") 21 | if index == -1 { 22 | return "" 23 | } 24 | location = string(location[index:]) 25 | } 26 | return location 27 | } 28 | -------------------------------------------------------------------------------- /url/path_test.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "os" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func TestPath(t *testing.T) { 11 | tempDir := os.TempDir() 12 | 13 | var useCases = []struct { 14 | description string 15 | URL string 16 | expect string 17 | }{ 18 | { 19 | description: "raw path", 20 | URL: "/tmp/foo/bar.txt", 21 | expect: "/tmp/foo/bar.txt", 22 | }, 23 | { 24 | description: "ftp path", 25 | URL: "ftp://localhost/tmp/foo/bar.txt", 26 | expect: "/tmp/foo/bar.txt", 27 | }, 28 | { 29 | description: "ftp root path", 30 | URL: "ftp://localhost/", 31 | expect: "/", 32 | }, 33 | { 34 | description: "root path", 35 | URL: "/", 36 | expect: "/", 37 | }, 38 | 39 | { 40 | description: "empty path", 41 | URL: "", 42 | expect: "/", 43 | }, 44 | { 45 | description: "relative path", 46 | URL: "abc/too.bar", 47 | expect: "abc/too.bar", 48 | }, 49 | { 50 | description: "tmp folder path", 51 | URL: tempDir, 52 | expect: strings.ReplaceAll(tempDir, `\`, `/`), 53 | }, 54 | } 55 | 56 | for _, useCase := range useCases { 57 | actual := Path(useCase.URL) 58 | assert.EqualValues(t, useCase.expect, actual, useCase.description) 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /url/relative.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import "strings" 4 | 5 | //IsRelative returns true if location is relative path 6 | func IsRelative(location string) bool { 7 | return !(strings.HasPrefix(location, "/") || strings.Contains(location, "://")) 8 | } 9 | -------------------------------------------------------------------------------- /url/scheme.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import "strings" 4 | 5 | //Scheme extracts URL scheme 6 | func Scheme(URL, defaultSchema string) string { 7 | index := strings.Index(URL, "://") 8 | if index == -1 { 9 | return defaultSchema 10 | } 11 | schema := string(URL[:index]) 12 | 13 | if index := strings.LastIndex(schema, "/"); index != -1 { 14 | schema = string(schema[index+1:]) 15 | } 16 | return schema 17 | } 18 | 19 | //IsSchemeEquals returns true if scheme is equals 20 | func IsSchemeEquals(URL1, URL2 string) bool { 21 | scheme1 := Scheme(URL1, "file") 22 | scheme2 := Scheme(URL2, "file") 23 | return scheme1 == scheme2 24 | 25 | } 26 | 27 | //SchemeExtensionURL extract scheme extension or empty string 28 | func SchemeExtensionURL(URL string) string { 29 | index := strings.Index(URL, "://") 30 | if index == -1 { 31 | return "" 32 | } 33 | schema := string(URL[:index]) 34 | if index := strings.LastIndex(schema, "/"); index != -1 { 35 | extension := string(schema[:index]) 36 | return strings.Replace(extension, ":", "://", 1) 37 | } 38 | return "" 39 | } 40 | -------------------------------------------------------------------------------- /url/scheme_test.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestScheme(t *testing.T) { 9 | 10 | var useCases = []struct { 11 | description string 12 | URL string 13 | expect string 14 | extensionURL string 15 | }{ 16 | 17 | { 18 | description: "raw path", 19 | URL: "/foo/var.txt", 20 | expect: "file", 21 | }, 22 | { 23 | description: "fpt path", 24 | URL: "ftp://localhosy/foo/var.txt", 25 | expect: "ftp", 26 | }, 27 | { 28 | description: "zip extended path", 29 | URL: "s3:myBucket/root/path/app.zip/zip://localhost/foo/var.txt", 30 | expect: "zip", 31 | extensionURL: "s3://myBucket/root/path/app.zip", 32 | }, 33 | } 34 | 35 | for _, useCase := range useCases { 36 | actual := Scheme(useCase.URL, "file") 37 | assert.EqualValues(t, useCase.expect, actual, useCase.description) 38 | schmeExtensionURL := SchemeExtensionURL(useCase.URL) 39 | assert.EqualValues(t, useCase.extensionURL, schmeExtensionURL, useCase.description) 40 | 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /url/split.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "path" 5 | "strings" 6 | ) 7 | 8 | //Split splits URL with the last URI element and its parent path 9 | func Split(URL, defaultScheme string) (string, string) { 10 | baseURL, URLPath := Base(URL, defaultScheme) 11 | if strings.HasSuffix(URLPath, "/") { 12 | URLPath = string(URLPath[:len(URLPath)-1]) 13 | } 14 | parent, name := path.Split(URLPath) 15 | return Join(baseURL, parent), name 16 | } 17 | -------------------------------------------------------------------------------- /url/split_test.go: -------------------------------------------------------------------------------- 1 | package url 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestSplit(t *testing.T) { 9 | 10 | var useCases = []struct { 11 | description string 12 | URL string 13 | URLParent string 14 | URIName string 15 | }{ 16 | { 17 | description: "file uri", 18 | URL: "scp://localhost/foo/bar.txt", 19 | URLParent: "scp://localhost/foo", 20 | URIName: "bar.txt", 21 | }, 22 | { 23 | description: "path uri", 24 | URL: "scp://localhost/foo/bar", 25 | URLParent: "scp://localhost/foo", 26 | URIName: "bar", 27 | }, 28 | { 29 | description: "path uri", 30 | URL: "scp://localhost/foo/bar/", 31 | URLParent: "scp://localhost/foo", 32 | URIName: "bar", 33 | }, 34 | } 35 | 36 | for _, useCase := range useCases { 37 | parent, name := Split(useCase.URL, "file") 38 | assert.Equal(t, useCase.URLParent, parent, useCase.description) 39 | assert.Equal(t, useCase.URIName, name, useCase.description) 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /walker.go: -------------------------------------------------------------------------------- 1 | package afs 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "github.com/viant/afs/file" 7 | "github.com/viant/afs/storage" 8 | "github.com/viant/afs/url" 9 | "github.com/viant/afs/walker" 10 | ) 11 | 12 | //Walk visits all location recursively within provided sourceURL 13 | func (s *service) Walk(ctx context.Context, URL string, handler storage.OnVisit, options ...storage.Option) error { 14 | if URL == "" { 15 | return errors.New("URL was empty") 16 | } 17 | URL = url.Normalize(URL, file.Scheme) 18 | manager, err := s.manager(ctx, URL, options) 19 | if err != nil { 20 | return err 21 | } 22 | URL = url.Normalize(URL, file.Scheme) 23 | managerWalker, ok := manager.(storage.Walker) 24 | if ok { 25 | return managerWalker.Walk(ctx, URL, handler, options...) 26 | } 27 | managerWalker = walker.New(manager) 28 | return managerWalker.Walk(ctx, URL, handler, options...) 29 | } 30 | -------------------------------------------------------------------------------- /walker/doc.go: -------------------------------------------------------------------------------- 1 | //Package walker define storager based walker 2 | package walker 3 | -------------------------------------------------------------------------------- /zip/README.md: -------------------------------------------------------------------------------- 1 | # zip - zip archives 2 | 3 | ## Usage 4 | 5 | 6 | * **Service** 7 | 8 | ```go 9 | service := afs.New() 10 | ctx := context.Background() 11 | objects, err := service.List(ctx, "file:/tmp/app.war/zip://localhost/WEB-INF") 12 | if err != nil { 13 | log.Fatal(err) 14 | } 15 | for _, object := range objects { 16 | fmt.Printf("%v %v\n", object.Name(), object.URL()) 17 | if object.IsDir() { 18 | continue 19 | } 20 | reader, err := service.Download(ctx, object) 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | data, err := ioutil.ReadAll(reader) 25 | if err != nil { 26 | log.Fatal(err) 27 | } 28 | fmt.Printf("%s\n", data) 29 | } 30 | ``` 31 | 32 | * **Walker** 33 | 34 | ```go 35 | ctx := context.Background() 36 | service := afs.New() 37 | walker := zip.NewWalker(file.New()) 38 | err := service.Copy(ctx, "/tmp/test.zip", "mem://dest/folder/test", walker) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | ``` 43 | 44 | * **Uploader** 45 | 46 | ```go 47 | ctx := context.Background() 48 | service := afs.New() 49 | uploader := zip.NewBatchUploader(file.New()) 50 | err := service.Copy(ctx, "/tmp/test/data", "/tmp/data.zip", uploader) 51 | if err != nil { 52 | log.Fatal(err) 53 | } 54 | ``` 55 | -------------------------------------------------------------------------------- /zip/doc.go: -------------------------------------------------------------------------------- 1 | //Package zip provides support for operating on ZIP archives 2 | package zip 3 | -------------------------------------------------------------------------------- /zip/doc_test.go: -------------------------------------------------------------------------------- 1 | package zip_test 2 | 3 | import ( 4 | "context" 5 | "github.com/viant/afs" 6 | "github.com/viant/afs/file" 7 | "github.com/viant/afs/zip" 8 | "log" 9 | ) 10 | 11 | func ExampleNewWalker() { 12 | ctx := context.Background() 13 | service := afs.New() 14 | walker := zip.NewWalker(file.New()) 15 | err := service.Copy(ctx, "/tmp/test.zip", "mem://dest/folder/test", walker) 16 | if err != nil { 17 | log.Fatal(err) 18 | } 19 | 20 | } 21 | 22 | func ExampleNewBatchUploader() { 23 | ctx := context.Background() 24 | service := afs.New() 25 | uploader := zip.NewBatchUploader(file.New()) 26 | err := service.Copy(ctx, "/tmp/test/data", "/tmp/data.zip", uploader) 27 | if err != nil { 28 | log.Fatal(err) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /zip/manager.go: -------------------------------------------------------------------------------- 1 | package zip 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/viant/afs/base" 7 | "github.com/viant/afs/option" 8 | "github.com/viant/afs/storage" 9 | "github.com/viant/afs/url" 10 | "io" 11 | "os" 12 | ) 13 | 14 | type manager struct { 15 | *base.Manager 16 | } 17 | 18 | func (m *manager) provider(ctx context.Context, baseURL string, options ...storage.Option) (storage.Storager, error) { 19 | var manager storage.Manager 20 | option.Assign(options, &manager) 21 | URL := url.SchemeExtensionURL(baseURL) 22 | if URL == "" { 23 | return nil, fmt.Errorf("extneded URL was empty: %v", baseURL) 24 | } 25 | if manager == nil { 26 | return nil, fmt.Errorf("manager for URL was empty: %v", URL) 27 | } 28 | return newStorager(ctx, baseURL, manager) 29 | } 30 | 31 | func (m *manager) Walk(ctx context.Context, URL string, handler storage.OnVisit, options ...storage.Option) error { 32 | baseURL, URLPath := url.Base(URL, Scheme) 33 | srv, err := m.Storager(ctx, baseURL, options) 34 | if err != nil { 35 | return err 36 | } 37 | service, ok := srv.(*storager) 38 | if !ok { 39 | return fmt.Errorf("unsupported storager type: expected: %T, but had %T", service, srv) 40 | } 41 | return service.Walk(ctx, URLPath, func(parent string, info os.FileInfo, reader io.Reader) (shallContinue bool, err error) { 42 | shallContinue, err = handler(ctx, baseURL, parent, info, reader) 43 | return shallContinue, err 44 | }, options...) 45 | } 46 | 47 | func (m *manager) Uploader(ctx context.Context, URL string, options ...storage.Option) (storage.Upload, io.Closer, error) { 48 | _, URLPath := url.Base(URL, Scheme) 49 | srv, err := m.Storager(ctx, URL, options) 50 | if err != nil { 51 | return nil, nil, err 52 | } 53 | service, ok := srv.(*storager) 54 | if !ok { 55 | return nil, nil, fmt.Errorf("unsupported storager type: expected: %T, but had %T", service, srv) 56 | } 57 | return service.Uploader(ctx, URLPath) 58 | } 59 | 60 | func newManager(options ...storage.Option) *manager { 61 | result := &manager{} 62 | baseMgr := base.New(result, Scheme, result.provider, options) 63 | result.Manager = baseMgr 64 | return result 65 | } 66 | 67 | //New creates zip manager 68 | func New(options ...storage.Option) storage.Manager { 69 | return newManager(options...) 70 | } 71 | -------------------------------------------------------------------------------- /zip/manager_test.go: -------------------------------------------------------------------------------- 1 | package zip_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/viant/afs" 8 | "github.com/viant/afs/option" 9 | "github.com/viant/afs/storage" 10 | "path" 11 | "runtime" 12 | "testing" 13 | ) 14 | 15 | type useCaseFn func(s afs.Service, ctx context.Context, url string) ([]storage.Object, error) 16 | 17 | func TestNew(t *testing.T) { 18 | testCases(t, func(service afs.Service, ctx context.Context, url string) ([]storage.Object, error) { 19 | return service.List(ctx, url) 20 | }) 21 | } 22 | 23 | 24 | func testCases(t *testing.T, callList useCaseFn) { 25 | _, filename, _, _ := runtime.Caller(0) 26 | baseDir, _ := path.Split(filename) 27 | ctx := context.Background() 28 | 29 | var useCases = []struct { 30 | description string 31 | URL string 32 | expect map[string]bool 33 | }{ 34 | { 35 | description: "list war classes", 36 | URL: fmt.Sprintf("file:%v/test/app.war/zip://localhost/WEB-INF/classes", baseDir), 37 | expect: map[string]bool{ 38 | "classes": true, 39 | "HelloWorld.class": true, 40 | "config.properties": true, 41 | }, 42 | }, 43 | 44 | { 45 | description: "list war classes", 46 | URL: fmt.Sprintf("file:%v/test/app.war/zip://localhost/WEB-INF/classes/config.properties", baseDir), 47 | expect: map[string]bool{ 48 | "config.properties": true, 49 | }, 50 | }, 51 | } 52 | 53 | for _, useCase := range useCases { 54 | service := afs.New() 55 | objects, err := callList(service, ctx, useCase.URL) 56 | assert.Nil(t, err, useCase.description) 57 | assert.EqualValues(t, len(useCase.expect), len(objects)) 58 | for _, obj := range objects { 59 | assert.True(t, useCase.expect[obj.Name()], useCase.description+" "+obj.URL()) 60 | } 61 | 62 | } 63 | } 64 | 65 | func TestNoCache(t *testing.T) { 66 | testCases(t, func(service afs.Service, ctx context.Context, url string) ([]storage.Object, error) { 67 | return service.List(ctx, url, &option.NoCache{Source: option.NoCacheBaseURL}) 68 | }) 69 | } 70 | -------------------------------------------------------------------------------- /zip/provider.go: -------------------------------------------------------------------------------- 1 | package zip 2 | 3 | import "github.com/viant/afs/storage" 4 | 5 | //Provider returns a http manager 6 | func Provider(options ...storage.Option) (storage.Manager, error) { 7 | return New(options...), nil 8 | } 9 | -------------------------------------------------------------------------------- /zip/scheme.go: -------------------------------------------------------------------------------- 1 | package zip 2 | 3 | //Scheme defines zip URL scheme 4 | const Scheme = "zip" 5 | -------------------------------------------------------------------------------- /zip/test/app.war: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viant/afs/f200dceb289ef98dfa3ebd148bdfb8a642ade924/zip/test/app.war -------------------------------------------------------------------------------- /zip/test/test.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/viant/afs/f200dceb289ef98dfa3ebd148bdfb8a642ade924/zip/test/test.zip -------------------------------------------------------------------------------- /zip/uploader.go: -------------------------------------------------------------------------------- 1 | package zip 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "context" 7 | "github.com/viant/afs/file" 8 | "github.com/viant/afs/option" 9 | "github.com/viant/afs/storage" 10 | "io" 11 | "os" 12 | "path" 13 | "strings" 14 | ) 15 | 16 | type uploader struct { 17 | uploader storage.Uploader 18 | buffer *bytes.Buffer 19 | } 20 | 21 | func (u *uploader) Uploader(ctx context.Context, URL string, options ...storage.Option) (storage.Upload, io.Closer, error) { 22 | var uploader storage.Uploader 23 | option.Assign(options, &u.buffer, &uploader) 24 | if uploader == nil { 25 | uploader = u.uploader 26 | } 27 | if u.buffer == nil { 28 | u.buffer = new(bytes.Buffer) 29 | } 30 | writer := newWriter(ctx, u.buffer, URL, uploader) 31 | return func(ctx context.Context, parent string, info os.FileInfo, reader io.Reader) error { 32 | filename := path.Join(parent, info.Name()) 33 | mode := info.Mode().Perm() 34 | if info.IsDir() { 35 | mode |= os.ModeDir 36 | } 37 | info = file.NewInfo(filename, info.Size(), mode, info.ModTime(), info.IsDir()) 38 | header, err := zip.FileInfoHeader(info) 39 | if err != nil { 40 | return err 41 | } 42 | header.Method = zip.Deflate 43 | header.Name = filename 44 | if info.IsDir() && !strings.HasSuffix(filename, "/") { 45 | header.Name += "/" 46 | } 47 | writer, err := writer.CreateHeader(header) 48 | if reader != nil { 49 | _, err = io.Copy(writer, reader) 50 | if err != nil { 51 | return err 52 | } 53 | } 54 | return err 55 | }, writer, nil 56 | } 57 | 58 | //newBatchUploader returns a batch uploader 59 | func newBatchUploader(dest storage.Uploader) *uploader { 60 | return &uploader{uploader: dest} 61 | } 62 | 63 | //NewBatchUploader returns a batch uploader 64 | func NewBatchUploader(dest storage.Uploader) storage.BatchUploader { 65 | return newBatchUploader(dest) 66 | } 67 | -------------------------------------------------------------------------------- /zip/uploader_test.go: -------------------------------------------------------------------------------- 1 | package zip 2 | 3 | import ( 4 | "context" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/viant/afs/asset" 7 | "github.com/viant/afs/file" 8 | "os" 9 | "path" 10 | "testing" 11 | ) 12 | 13 | func TestUploader_Uploader(t *testing.T) { 14 | 15 | baseDir := os.TempDir() 16 | fileManager := file.New() 17 | 18 | var useCases = []struct { 19 | description string 20 | destURL string 21 | createLocation bool 22 | assets []*asset.Resource 23 | }{ 24 | 25 | { 26 | description: "multi download - unordered", 27 | destURL: path.Join(baseDir, "zip_upload_01/test.zip"), 28 | assets: []*asset.Resource{ 29 | asset.NewDir("test", 0744), 30 | asset.NewDir("test/folder2/sub", 0744), 31 | asset.NewFile("test/asset1.txt", []byte("xyz"), 0644), 32 | asset.NewFile("test/asset2.txt", []byte("xyz"), 0644), 33 | asset.NewDir("test/folder1", 0744), 34 | asset.NewFile("test/folder1/res.txt", []byte("xyz"), 0644), 35 | asset.NewFile("test/folder2/res1.txt", []byte("xyz"), 0644), 36 | }, 37 | }, 38 | 39 | { 40 | description: "multi download - link", 41 | destURL: path.Join(baseDir, "zip_upload_02/test.zip"), 42 | assets: []*asset.Resource{ 43 | asset.NewFile("foo1.txt", []byte("abc"), 0644), 44 | asset.NewFile("foo2.txt", []byte("xyz"), 0644), 45 | asset.NewLink("sym.txt", "foo1.txt", 0644), 46 | }, 47 | }, 48 | } 49 | 50 | for _, useCase := range useCases { 51 | ctx := context.Background() 52 | uploader := NewBatchUploader(fileManager) 53 | upload, closer, err := uploader.Uploader(ctx, useCase.destURL) 54 | if !assert.Nil(t, err, useCase.description) { 55 | 56 | } 57 | for _, asset := range useCase.assets { 58 | relative, _ := path.Split(asset.Name) 59 | err = upload(ctx, relative, asset.Info(), asset.Reader()) 60 | assert.Nil(t, err, useCase.description+" "+asset.Name) 61 | } 62 | err = closer.Close() 63 | assert.Nil(t, err, useCase.description) 64 | _, err = os.Stat(useCase.destURL) 65 | assert.Nil(t, err, useCase.description) 66 | _ = asset.Cleanup(fileManager, useCase.destURL) 67 | 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /zip/walker_test.go: -------------------------------------------------------------------------------- 1 | package zip_test 2 | 3 | import ( 4 | "context" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/viant/afs" 7 | "github.com/viant/afs/asset" 8 | "github.com/viant/afs/file" 9 | "github.com/viant/afs/zip" 10 | "io" 11 | "io/ioutil" 12 | "os" 13 | "path" 14 | "runtime" 15 | "testing" 16 | ) 17 | 18 | func TestWalker_Walk(t *testing.T) { 19 | 20 | _, filename, _, _ := runtime.Caller(0) 21 | baseDir, _ := path.Split(filename) 22 | 23 | var useCases = []struct { 24 | description string 25 | location string 26 | expect []*asset.Resource 27 | }{ 28 | { 29 | description: "zip walking", 30 | location: path.Join(baseDir, "test/test.zip"), 31 | expect: []*asset.Resource{ 32 | asset.NewDir("test", file.DefaultDirOsMode), 33 | asset.NewFile("test/asset1.txt", []byte("test is test\n"), 0644), 34 | asset.NewFile("test/asset2.txt", []byte("test is second test\n"), 0644), 35 | asset.NewDir("test/folder1", file.DefaultDirOsMode), 36 | asset.NewFile("test/folder1/res.txt", []byte("resource 1\n"), 0644), 37 | asset.NewDir("test/folder2", file.DefaultDirOsMode), 38 | asset.NewFile("test/folder2/res1.txt", []byte("resource 1\n"), 0644), 39 | }, 40 | }, 41 | } 42 | 43 | for _, useCase := range useCases { 44 | walker := zip.NewWalker(afs.New()) 45 | 46 | ctx := context.Background() 47 | actuals := make(map[string]*asset.Resource) 48 | err := walker.Walk(ctx, useCase.location, func(ctx context.Context, baseURL string, parent string, info os.FileInfo, reader io.Reader) (toContinue bool, err error) { 49 | 50 | resourceLocation := path.Join(parent, info.Name()) 51 | linkName := "" 52 | if rawInfo, ok := info.(*file.Info); ok { 53 | linkName = rawInfo.Linkname 54 | } 55 | var data []byte 56 | if reader != nil { 57 | data, err = ioutil.ReadAll(reader) 58 | if err != nil { 59 | return false, err 60 | } 61 | } 62 | actuals[resourceLocation] = asset.New(parent, info.Mode(), info.IsDir(), linkName, data) 63 | return true, nil 64 | }) 65 | assert.Nil(t, err, useCase.description) 66 | 67 | for _, asset := range useCase.expect { 68 | _, ok := actuals[asset.Name] 69 | assert.True(t, ok, useCase.description+" "+asset.Name) 70 | } 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /zip/writer.go: -------------------------------------------------------------------------------- 1 | package zip 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "context" 7 | "github.com/viant/afs/storage" 8 | ) 9 | 10 | type writer struct { 11 | ctx context.Context 12 | buffer *bytes.Buffer 13 | *zip.Writer 14 | destURL string 15 | uploader storage.Uploader 16 | } 17 | 18 | func newWriter(ctx context.Context, buffer *bytes.Buffer, URL string, uploader storage.Uploader) *writer { 19 | if buffer == nil { 20 | buffer = new(bytes.Buffer) 21 | } 22 | return &writer{ 23 | ctx: ctx, 24 | buffer: buffer, 25 | Writer: zip.NewWriter(buffer), 26 | destURL: URL, 27 | uploader: uploader, 28 | } 29 | } 30 | 31 | func (w *writer) Close() error { 32 | err := w.Writer.Flush() 33 | if err == nil { 34 | if err = w.Writer.Close(); err == nil { 35 | if w.uploader != nil { 36 | err = w.uploader.Upload(w.ctx, w.destURL, 0644, bytes.NewReader(w.buffer.Bytes())) 37 | } 38 | } 39 | } 40 | return err 41 | } 42 | --------------------------------------------------------------------------------