├── .gitignore ├── .travis.yml ├── BestPractices.md ├── LICENSE ├── Makefile ├── README.md ├── azure ├── config.go ├── container.go ├── doc.go ├── item.go ├── location.go ├── multipart.go ├── multipart_test.go └── stow_test.go ├── b2 ├── config.go ├── container.go ├── item.go ├── location.go └── stow_test.go ├── doc.go ├── go.mod ├── go.sum ├── google ├── README.md ├── config.go ├── container.go ├── doc.go ├── item.go ├── location.go └── stow_test.go ├── local ├── container.go ├── container_test.go ├── doc.go ├── filedata_darwin.go ├── filedata_linux.go ├── filedata_test.go ├── filedata_windows.go ├── item.go ├── item_test.go ├── local.go ├── location.go ├── location_test.go ├── stow_test.go ├── testdata │ ├── .dotfile │ └── .gitkeep └── util_test.go ├── location_test.go ├── oracle ├── config.go ├── container.go ├── doc.go ├── item.go ├── location.go └── stow_test.go ├── s3 ├── README.md ├── config.go ├── container.go ├── doc.go ├── item.go ├── location.go ├── stow_iam_test.go ├── stow_test.go └── v2signer.go ├── sftp ├── config.go ├── container.go ├── item.go ├── location.go └── stow_test.go ├── stow-aeroplane.png ├── stow-definition.png ├── stow.go ├── stow_test.go ├── swift ├── config.go ├── container.go ├── doc.go ├── item.go ├── location.go └── stow_test.go ├── test ├── Dockerfile └── test.go └── walk.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | tests.out 27 | tests.xml 28 | .idea 29 | 30 | vendor/ 31 | 32 | .vscode/ 33 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | sudo: required 4 | 5 | before_install: 6 | - sudo apt-get -qq update 7 | - sudo apt-get install -y lsof 8 | 9 | go: 10 | - tip 11 | 12 | install: go get -v ./... 13 | 14 | script: go test -v ./... 15 | -------------------------------------------------------------------------------- /BestPractices.md: -------------------------------------------------------------------------------- 1 | # Best practices 2 | 3 | ## Configuring Stow once 4 | 5 | It is recommended that you create a single file that imports Stow and all the implementations to save you from doing so in every code file where you might use Stow. You can also take the opportunity to abstract the top level Stow methods that your code uses to save other code files (including other packages) from importing Stow at all. 6 | 7 | Create a file called `storage.go` in your package and add the following code: 8 | 9 | ```go 10 | import ( 11 | "github.com/graymeta/stow" 12 | // support Azure storage 13 | _ "github.com/graymeta/stow/azure" 14 | // support Google storage 15 | _ "github.com/graymeta/stow/google" 16 | // support local storage 17 | _ "github.com/graymeta/stow/local" 18 | // support swift storage 19 | _ "github.com/graymeta/stow/swift" 20 | // support s3 storage 21 | _ "github.com/graymeta/stow/s3" 22 | // support oracle storage 23 | _ "github.com/graymeta/stow/oracle" 24 | ) 25 | 26 | // Dial dials stow storage. 27 | // See stow.Dial for more information. 28 | func Dial(kind string, config stow.Config) (stow.Location, error) { 29 | return stow.Dial(kind, config) 30 | } 31 | ``` 32 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test 2 | WORKSPACE = $(shell pwd) 3 | 4 | topdir = /tmp/$(pkg)-$(version) 5 | 6 | all: container runcontainer 7 | @true 8 | 9 | container: 10 | docker build --no-cache -t builder-stow test/ 11 | 12 | runcontainer: 13 | docker run -v $(WORKSPACE):/mnt/src/github.com/graymeta/stow builder-stow 14 | 15 | test: clean vet 16 | go test -v ./... 17 | 18 | vet: 19 | go vet ./... 20 | 21 | clean: 22 | @true 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Stow logo](stow-aeroplane.png) 2 | ![Stow definition](stow-definition.png) 3 | [![GoDoc](https://godoc.org/github.com/graymeta/stow?status.svg)](https://godoc.org/github.com/graymeta/stow) 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/graymeta/stow)](https://goreportcard.com/report/github.com/graymeta/stow) 5 | 6 | Cloud storage abstraction package for Go. 7 | 8 | * Version: 0.1.0 9 | * Project status: Stable. Approaching v1 release 10 | * It is recommended that you vendor this package as changes are possible before v1 11 | 12 | * Contributors: [Mat Ryer](https://github.com/matryer), [David Hernandez](https://github.com/dahernan), [Ernesto Jiménez](https://github.com/ernesto-jimenez), [Corey Prak](https://github.com/Xercoy), [Piotr Rojek](https://github.com/piotrrojek), [Jason Hancock](https://github.com/jasonhancock) 13 | * Artwork by Mel Jensen inspired by [Renee French](http://reneefrench.blogspot.co.uk) and and is licensed under the [Creative Commons Attribution 3.0 License](https://creativecommons.org/licenses/by/3.0/) 14 | 15 | ## How it works 16 | 17 | Stow provides implementations for storage services, blob stores, cloud storage etc. Read the [blog post announcing the project](https://medium.com/@matryer/introducing-stow-cloud-storage-abstraction-package-for-go-20cf2928d93c). 18 | 19 | ## Implementations 20 | 21 | * Local (folders are containers, files are items) 22 | * Amazon S3 23 | * Google Cloud Storage 24 | * Microsoft Azure Blob Storage 25 | * Openstack Swift (with auth v2) 26 | * Oracle Storage Cloud Service 27 | * SFTP 28 | 29 | ## Concepts 30 | 31 | The concepts of Stow are modeled around the most popular object storage services, and are made up of three main objects: 32 | 33 | * `Location` - a place where many `Container` objects are stored 34 | * `Container` - a named group of `Item` objects 35 | * `Item` - an individual file 36 | 37 | ``` 38 | location1 (e.g. Azure) 39 | ├── container1 40 | ├───── item1.1 41 | ├───── item1.2 42 | ├───── item1.3 43 | ├── container2 44 | ├───── item2.1 45 | ├───── item2.2 46 | location2 (e.g. local storage) 47 | ├── container1 48 | ├───── item1.1 49 | ├───── item1.2 50 | ├───── item1.3 51 | ├── container2 52 | ├───── item2.1 53 | ├───── item2.2 54 | ``` 55 | 56 | * A location contains many containers 57 | * A container contains many items 58 | * Containers do not contain other containers 59 | * Items must belong to a container 60 | * Item names may be a path 61 | 62 | ## Guides 63 | 64 | * [Using Stow](#using-stow) 65 | * [Connecting to locations](#connecting-to-locations) 66 | * [Walking containers](#walking-containers) 67 | * [Walking items](#walking-items) 68 | * [Downloading a file](#downloading-afile) 69 | * [Uploading a file](#uploading-a-file) 70 | * [Stow URLs](#stow-urls) 71 | * [Cursors](#cursors) 72 | 73 | ### Using Stow 74 | 75 | Import Stow plus any of the implementation packages that you wish to provide. For example, to support Google Cloud Storage and Amazon S3 you would write: 76 | 77 | ```go 78 | import ( 79 | "github.com/graymeta/stow" 80 | _ "github.com/graymeta/stow/google" 81 | _ "github.com/graymeta/stow/s3" 82 | ) 83 | ``` 84 | 85 | The underscore indicates that you do not intend to use the package in your code. Importing it is enough, as the implementation packages register themselves with Stow during initialization. 86 | 87 | * For more information about using Stow, see the [Best practices documentation](BestPractices.md). 88 | * Some implementation packages provide ways to access the underlying connection details for use-cases where more control over a specific service is needed. See the implementation package documentation for details. 89 | 90 | ### Connecting to locations 91 | 92 | To connect to a location, you need to know the `kind` string (available by accessing the `Kind` constant in the implementation package) and a `stow.Config` object that contains any required configuration information (such as account names, API keys, credentials, etc). Configuration is implementation specific, so you should consult each implementation to see what fields are required. 93 | 94 | ```go 95 | kind := "s3" 96 | config := stow.ConfigMap{ 97 | s3.ConfigAccessKeyID: "246810", 98 | s3.ConfigSecretKey: "abc123", 99 | s3.ConfigRegion: "eu-west-1", 100 | } 101 | location, err := stow.Dial(kind, config) 102 | if err != nil { 103 | return err 104 | } 105 | defer location.Close() 106 | 107 | // TODO: use location 108 | ``` 109 | 110 | ### Walking containers 111 | 112 | You can walk every Container using the `stow.WalkContainers` function: 113 | 114 | ```go 115 | func WalkContainers(location Location, prefix string, pageSize int, fn WalkContainersFunc) error 116 | ``` 117 | 118 | For example: 119 | 120 | ```go 121 | err = stow.WalkContainers(location, stow.NoPrefix, 100, func(c stow.Container, err error) error { 122 | if err != nil { 123 | return err 124 | } 125 | switch c.Name() { 126 | case c1.Name(), c2.Name(), c3.Name(): 127 | found++ 128 | } 129 | return nil 130 | }) 131 | if err != nil { 132 | return err 133 | } 134 | ``` 135 | 136 | ### Walking items 137 | 138 | Once you have a `Container`, you can walk every Item inside it using the `stow.Walk` function: 139 | 140 | ```go 141 | func Walk(container Container, prefix string, pageSize int, fn WalkFunc) error 142 | ``` 143 | 144 | For example: 145 | 146 | ```go 147 | err = stow.Walk(containers[0], stow.NoPrefix, 100, func(item stow.Item, err error) error { 148 | if err != nil { 149 | return err 150 | } 151 | log.Println(item.Name()) 152 | return nil 153 | }) 154 | if err != nil { 155 | return err 156 | } 157 | ``` 158 | 159 | ### Downloading a file 160 | 161 | Once you have found a `stow.Item` that you are interested in, you can stream its contents by first calling the `Open` method and reading from the returned `io.ReadCloser` (remembering to close the reader): 162 | 163 | ```go 164 | r, err := item.Open() 165 | if err != nil { 166 | return err 167 | } 168 | defer r.Close() 169 | 170 | // TODO: stream the contents by reading from r 171 | ``` 172 | 173 | ### Uploading a file 174 | 175 | If you want to write a new item into a Container, you can do so using the `container.Put` method passing in an `io.Reader` for the contents along with the size: 176 | 177 | ```go 178 | contents := "This is a new file stored in the cloud" 179 | r := strings.NewReader(contents) 180 | size := int64(len(contents)) 181 | 182 | item, err := container.Put(name, r, size, nil) 183 | if err != nil { 184 | return err 185 | } 186 | 187 | // item represents the newly created/updated item 188 | ``` 189 | 190 | ### Stow URLs 191 | 192 | An `Item` can return a URL via the `URL()` method. While a valid URL, they are useful only within the context of Stow. Within a Location, you can get items using these URLs via the `Location.ItemByURL` method. 193 | 194 | #### Getting an `Item` by URL 195 | 196 | If you have a Stow URL, you can use it to lookup the kind of location: 197 | 198 | ```go 199 | kind, err := stow.KindByURL(url) 200 | ``` 201 | 202 | `kind` will be a string describing the kind of storage. You can then pass `kind` along with a `Config` to `stow.New` to create a new `Location` where the item for the URL is: 203 | 204 | ```go 205 | location, err := stow.Dial(kind, config) 206 | ``` 207 | 208 | You can then get the `Item` for the specified URL from the location: 209 | 210 | ```go 211 | item, err := location.ItemByURL(url) 212 | ``` 213 | 214 | ### Cursors 215 | 216 | Cursors are strings that provide a pointer to items in sets allowing for paging over the entire set. 217 | 218 | Call such methods first passing in `stow.CursorStart` as the cursor, which indicates the first item/page. The method will, as one of its return arguments, provide a new cursor which you can pass into subsequent calls to the same method. 219 | 220 | When `stow.IsCursorEnd(cursor)` returns `true`, you have reached the end of the set. 221 | -------------------------------------------------------------------------------- /azure/config.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | 7 | az "github.com/Azure/azure-sdk-for-go/storage" 8 | "github.com/graymeta/stow" 9 | ) 10 | 11 | // ConfigAccount and ConfigKey are the supported configuration items for 12 | // Azure blob storage. 13 | const ( 14 | ConfigAccount = "account" 15 | ConfigKey = "key" 16 | ) 17 | 18 | // Kind is the kind of Location this package provides. 19 | const Kind = "azure" 20 | 21 | func init() { 22 | validatefn := func(config stow.Config) error { 23 | _, ok := config.Config(ConfigAccount) 24 | if !ok { 25 | return errors.New("missing account id") 26 | } 27 | _, ok = config.Config(ConfigKey) 28 | if !ok { 29 | return errors.New("missing auth key") 30 | } 31 | return nil 32 | } 33 | makefn := func(config stow.Config) (stow.Location, error) { 34 | _, ok := config.Config(ConfigAccount) 35 | if !ok { 36 | return nil, errors.New("missing account id") 37 | } 38 | _, ok = config.Config(ConfigKey) 39 | if !ok { 40 | return nil, errors.New("missing auth key") 41 | } 42 | l := &location{ 43 | config: config, 44 | } 45 | var err error 46 | l.client, err = newBlobStorageClient(l.config) 47 | if err != nil { 48 | return nil, err 49 | } 50 | // test the connection 51 | _, _, err = l.Containers("", stow.CursorStart, 1) 52 | if err != nil { 53 | return nil, err 54 | } 55 | return l, nil 56 | } 57 | kindfn := func(u *url.URL) bool { 58 | return u.Scheme == Kind 59 | } 60 | stow.Register(Kind, makefn, kindfn, validatefn) 61 | } 62 | 63 | func newBlobStorageClient(cfg stow.Config) (*az.BlobStorageClient, error) { 64 | acc, ok := cfg.Config(ConfigAccount) 65 | if !ok { 66 | return nil, errors.New("missing account id") 67 | } 68 | key, ok := cfg.Config(ConfigKey) 69 | if !ok { 70 | return nil, errors.New("missing auth key") 71 | } 72 | basicClient, err := az.NewBasicClient(acc, key) 73 | if err != nil { 74 | return nil, errors.New("bad credentials") 75 | } 76 | client := basicClient.GetBlobService() 77 | return &client, err 78 | } 79 | -------------------------------------------------------------------------------- /azure/container.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "io" 5 | "strings" 6 | "time" 7 | 8 | az "github.com/Azure/azure-sdk-for-go/storage" 9 | "github.com/graymeta/stow" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | // The maximum size of an object that can be Put in a single request 14 | const maxPutSize = 256 * 1024 * 1024 15 | 16 | // timeFormat is the time format for azure. 17 | var timeFormat = "Mon, 2 Jan 2006 15:04:05 MST" 18 | 19 | type container struct { 20 | id string 21 | properties az.ContainerProperties 22 | client *az.BlobStorageClient 23 | } 24 | 25 | var _ stow.Container = (*container)(nil) 26 | 27 | func (c *container) ID() string { 28 | return c.id 29 | } 30 | 31 | func (c *container) Name() string { 32 | return c.id 33 | } 34 | 35 | func (c *container) Item(id string) (stow.Item, error) { 36 | blob := c.client.GetContainerReference(c.id).GetBlobReference(id) 37 | err := blob.GetProperties(nil) 38 | if err != nil { 39 | if strings.Contains(err.Error(), "404") { 40 | return nil, stow.ErrNotFound 41 | } 42 | return nil, err 43 | } 44 | item := &item{ 45 | id: id, 46 | container: c, 47 | client: c.client, 48 | properties: blob.Properties, 49 | } 50 | 51 | etag := cleanEtag(item.properties.Etag) // Etags returned from this method include quotes. Strip them. 52 | item.properties.Etag = etag // Assign the corrected string value to the field. 53 | 54 | return item, nil 55 | } 56 | 57 | func (c *container) Items(prefix, cursor string, count int) ([]stow.Item, string, error) { 58 | params := az.ListBlobsParameters{ 59 | Prefix: prefix, 60 | MaxResults: uint(count), 61 | } 62 | if cursor != "" { 63 | params.Marker = cursor 64 | } 65 | listblobs, err := c.client.GetContainerReference(c.id).ListBlobs(params) 66 | if err != nil { 67 | return nil, "", err 68 | } 69 | items := make([]stow.Item, len(listblobs.Blobs)) 70 | for i, blob := range listblobs.Blobs { 71 | 72 | // Clean Etag just in case. 73 | blob.Properties.Etag = cleanEtag(blob.Properties.Etag) 74 | 75 | items[i] = &item{ 76 | id: blob.Name, 77 | container: c, 78 | client: c.client, 79 | properties: blob.Properties, 80 | } 81 | } 82 | return items, listblobs.NextMarker, nil 83 | } 84 | 85 | func (c *container) Put(name string, r io.Reader, size int64, metadata map[string]interface{}) (stow.Item, error) { 86 | mdParsed, err := prepMetadata(metadata) 87 | if err != nil { 88 | return nil, errors.Wrap(err, "unable to create or update Item, preparing metadata") 89 | } 90 | 91 | name = strings.Replace(name, " ", "+", -1) 92 | 93 | if size > maxPutSize { 94 | // Do a multipart upload 95 | err := c.multipartUpload(name, r, size) 96 | if err != nil { 97 | return nil, errors.Wrap(err, "multipart upload") 98 | } 99 | } else { 100 | err = c.client.GetContainerReference(c.id).GetBlobReference(name).CreateBlockBlobFromReader(r, nil) 101 | if err != nil { 102 | return nil, errors.Wrap(err, "unable to create or update Item") 103 | } 104 | } 105 | 106 | err = c.SetItemMetadata(name, mdParsed) 107 | if err != nil { 108 | return nil, errors.Wrap(err, "unable to create or update item, setting Item metadata") 109 | } 110 | 111 | item := &item{ 112 | id: name, 113 | container: c, 114 | client: c.client, 115 | properties: az.BlobProperties{ 116 | LastModified: az.TimeRFC1123(time.Now()), 117 | Etag: "", 118 | ContentLength: size, 119 | }, 120 | } 121 | return item, nil 122 | } 123 | 124 | func (c *container) SetItemMetadata(itemName string, md map[string]string) error { 125 | blob := c.client.GetContainerReference(c.id).GetBlobReference(itemName) 126 | blob.Metadata = md 127 | return blob.SetMetadata(nil) 128 | } 129 | 130 | func parseMetadata(md map[string]string) (map[string]interface{}, error) { 131 | rtnMap := make(map[string]interface{}, len(md)) 132 | for key, value := range md { 133 | rtnMap[key] = value 134 | } 135 | return rtnMap, nil 136 | } 137 | 138 | func prepMetadata(md map[string]interface{}) (map[string]string, error) { 139 | rtnMap := make(map[string]string, len(md)) 140 | for key, value := range md { 141 | str, ok := value.(string) 142 | if !ok { 143 | return nil, errors.Errorf(`value of key '%s' in metadata must be of type string`, key) 144 | } 145 | rtnMap[key] = str 146 | } 147 | return rtnMap, nil 148 | } 149 | 150 | func (c *container) RemoveItem(id string) error { 151 | return c.client.GetContainerReference(c.id).GetBlobReference(id).Delete(nil) 152 | } 153 | 154 | // Remove quotation marks from beginning and end. This includes quotations that 155 | // are escaped. Also removes leading `W/` from prefix for weak Etags. 156 | // 157 | // Based on the Etag spec, the full etag value () can include: 158 | // - W/"" 159 | // - "" 160 | // - "" 161 | // Source: https://tools.ietf.org/html/rfc7232#section-2.3 162 | // 163 | // Based on HTTP spec, forward slash is a separator and must be enclosed in 164 | // quotes to be used as a valid value. Hence, the returned value may include: 165 | // - "" 166 | // - \"\" 167 | // Source: https://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 168 | // 169 | // This function contains a loop to check for the presence of the three possible 170 | // filler characters and strips them, resulting in only the Etag value. 171 | func cleanEtag(etag string) string { 172 | for { 173 | // Check if the filler characters are present 174 | if strings.HasPrefix(etag, `\"`) { 175 | etag = strings.Trim(etag, `\"`) 176 | 177 | } else if strings.HasPrefix(etag, `"`) { 178 | etag = strings.Trim(etag, `"`) 179 | 180 | } else if strings.HasPrefix(etag, `W/`) { 181 | etag = strings.Replace(etag, `W/`, "", 1) 182 | 183 | } else { 184 | 185 | break 186 | } 187 | } 188 | 189 | return etag 190 | } 191 | -------------------------------------------------------------------------------- /azure/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package azure provides an abstraction for the Microsoft Azure Storage service. In this package, an Azure Resource of type "Storage account" is represented by a Stow Location and an Azure blob is represented by a Stow Item. 3 | 4 | Usage and Credentials 5 | 6 | Two peices of information are needed to access an Azure Resorce of type "Storage account": the Resource Name (found in the "All Resources" tab of the Azure Portal console) and the Access Key (found in the "Access Keys" tab in the "Settings" pane of a Resource's details page). 7 | 8 | stow.Dial requires both a string value of the particular Stow Location Kind ("azure") and a stow.Config instance. The stow.Config instance requires two entries with the specific key value attributes: 9 | 10 | - a key of azure.ConfigAccount with a value of the Azure Resource Name 11 | - a key of azure.ConfigKey with a value of the Azure Access Key 12 | 13 | Location 14 | 15 | There are azure.location methods which allow the retrieval of a single Azure Storage Service. A stow.Item representation of an Azure Storage Blob can also be retrieved based on the Object's URL (ItemByURL). 16 | 17 | Additional azure.location methods provide capabilities to create and remove Azure Storage Containers. 18 | 19 | Container 20 | 21 | Methods of an azure.container allow the retrieval of an Azure Storage Container's: 22 | 23 | - name (ID or Name) 24 | - blob or complete list of blobs (Item or Items) 25 | 26 | Additional azure.container methods allow Stow to : 27 | 28 | - remove a Blob (RemoveItem) 29 | - update or create a Blob (Put) 30 | 31 | Item 32 | 33 | Methods of azure.Item allow the retrieval of an Azure Storage Container's: 34 | - name (ID or name) 35 | - URL 36 | - size in bytes 37 | - Azure Storage blob specific metadata (information stored within the Azure Cloud Service) 38 | - last modified date 39 | - Etag 40 | - content 41 | 42 | Caveats 43 | 44 | At this point in time, the upload limit of a blob is about 60MB. This is an implementation restraint. 45 | */ 46 | package azure 47 | -------------------------------------------------------------------------------- /azure/item.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "io" 5 | "net/url" 6 | "sync" 7 | "time" 8 | 9 | az "github.com/Azure/azure-sdk-for-go/storage" 10 | "github.com/graymeta/stow" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | type item struct { 15 | id string 16 | container *container 17 | client *az.BlobStorageClient 18 | properties az.BlobProperties 19 | url url.URL 20 | metadata map[string]interface{} 21 | infoOnce sync.Once 22 | infoErr error 23 | } 24 | 25 | var ( 26 | _ stow.Item = (*item)(nil) 27 | _ stow.ItemRanger = (*item)(nil) 28 | ) 29 | 30 | func (i *item) ID() string { 31 | return i.id 32 | } 33 | 34 | func (i *item) Name() string { 35 | return i.id 36 | } 37 | 38 | func (i *item) URL() *url.URL { 39 | u := i.client.GetContainerReference(i.container.id).GetBlobReference(i.id).GetURL() 40 | url, _ := url.Parse(u) 41 | url.Scheme = "azure" 42 | return url 43 | } 44 | 45 | func (i *item) Size() (int64, error) { 46 | return i.properties.ContentLength, nil 47 | } 48 | 49 | func (i *item) Open() (io.ReadCloser, error) { 50 | return i.client.GetContainerReference(i.container.id).GetBlobReference(i.id).Get(nil) 51 | } 52 | 53 | func (i *item) ETag() (string, error) { 54 | return i.properties.Etag, nil 55 | } 56 | 57 | func (i *item) LastMod() (time.Time, error) { 58 | return time.Time(i.properties.LastModified), nil 59 | } 60 | 61 | func (i *item) Metadata() (map[string]interface{}, error) { 62 | err := i.ensureInfo() 63 | if err != nil { 64 | return nil, errors.Wrap(err, "retrieving metadata") 65 | } 66 | 67 | return i.metadata, nil 68 | } 69 | 70 | func (i *item) ensureInfo() error { 71 | if i.metadata == nil { 72 | i.infoOnce.Do(func() { 73 | blob := i.client.GetContainerReference(i.container.Name()).GetBlobReference(i.Name()) 74 | infoErr := blob.GetMetadata(nil) 75 | if infoErr != nil { 76 | i.infoErr = infoErr 77 | return 78 | } 79 | 80 | mdParsed, infoErr := parseMetadata(blob.Metadata) 81 | if infoErr != nil { 82 | i.infoErr = infoErr 83 | return 84 | } 85 | i.metadata = mdParsed 86 | }) 87 | } 88 | 89 | return i.infoErr 90 | } 91 | 92 | func (i *item) getInfo() (stow.Item, error) { 93 | itemInfo, err := i.container.Item(i.ID()) 94 | if err != nil { 95 | return nil, err 96 | } 97 | return itemInfo, nil 98 | } 99 | 100 | // OpenRange opens the item for reading starting at byte start and ending 101 | // at byte end. 102 | func (i *item) OpenRange(start, end uint64) (io.ReadCloser, error) { 103 | opts := &az.GetBlobRangeOptions{ 104 | Range: &az.BlobRange{ 105 | Start: start, 106 | End: end, 107 | }, 108 | } 109 | return i.client.GetContainerReference(i.container.id).GetBlobReference(i.id).GetRange(opts) 110 | } 111 | -------------------------------------------------------------------------------- /azure/location.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | "strings" 7 | "time" 8 | 9 | az "github.com/Azure/azure-sdk-for-go/storage" 10 | "github.com/graymeta/stow" 11 | ) 12 | 13 | type location struct { 14 | config stow.Config 15 | client *az.BlobStorageClient 16 | } 17 | 18 | func (l *location) Close() error { 19 | return nil // nothing to close 20 | } 21 | 22 | func (l *location) CreateContainer(name string) (stow.Container, error) { 23 | err := l.client.GetContainerReference(name).Create(&az.CreateContainerOptions{Access: az.ContainerAccessTypeBlob}) 24 | if err != nil { 25 | if strings.Contains(err.Error(), "ErrorCode=ContainerAlreadyExists") { 26 | return l.Container(name) 27 | } 28 | return nil, err 29 | } 30 | container := &container{ 31 | id: name, 32 | properties: az.ContainerProperties{ 33 | LastModified: time.Now().Format(timeFormat), 34 | }, 35 | client: l.client, 36 | } 37 | time.Sleep(time.Second * 3) 38 | return container, nil 39 | } 40 | 41 | func (l *location) Containers(prefix, cursor string, count int) ([]stow.Container, string, error) { 42 | params := az.ListContainersParameters{ 43 | MaxResults: uint(count), 44 | Prefix: prefix, 45 | } 46 | if cursor != stow.CursorStart { 47 | params.Marker = cursor 48 | } 49 | response, err := l.client.ListContainers(params) 50 | if err != nil { 51 | return nil, "", err 52 | } 53 | containers := make([]stow.Container, len(response.Containers)) 54 | for i, azureContainer := range response.Containers { 55 | containers[i] = &container{ 56 | id: azureContainer.Name, 57 | properties: azureContainer.Properties, 58 | client: l.client, 59 | } 60 | } 61 | return containers, response.NextMarker, nil 62 | } 63 | 64 | func (l *location) Container(id string) (stow.Container, error) { 65 | cursor := stow.CursorStart 66 | for { 67 | containers, crsr, err := l.Containers(id[:3], cursor, 100) 68 | if err != nil { 69 | return nil, stow.ErrNotFound 70 | } 71 | for _, i := range containers { 72 | if i.ID() == id { 73 | return i, nil 74 | } 75 | } 76 | 77 | cursor = crsr 78 | if cursor == "" { 79 | break 80 | } 81 | } 82 | 83 | return nil, stow.ErrNotFound 84 | } 85 | 86 | func (l *location) ItemByURL(url *url.URL) (stow.Item, error) { 87 | if url.Scheme != "azure" { 88 | return nil, errors.New("not valid azure URL") 89 | } 90 | location := strings.Split(url.Host, ".")[0] 91 | a, ok := l.config.Config(ConfigAccount) 92 | if !ok { 93 | // shouldn't really happen 94 | return nil, errors.New("missing " + ConfigAccount + " config") 95 | } 96 | if a != location { 97 | return nil, errors.New("wrong azure URL") 98 | } 99 | path := strings.TrimLeft(url.Path, "/") 100 | params := strings.SplitN(path, "/", 2) 101 | if len(params) != 2 { 102 | return nil, errors.New("wrong path") 103 | } 104 | c, err := l.Container(params[0]) 105 | if err != nil { 106 | return nil, err 107 | } 108 | return c.Item(params[1]) 109 | } 110 | 111 | func (l *location) RemoveContainer(id string) error { 112 | return l.client.GetContainerReference(id).Delete(nil) 113 | } 114 | -------------------------------------------------------------------------------- /azure/multipart.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/binary" 6 | "errors" 7 | "io" 8 | 9 | az "github.com/Azure/azure-sdk-for-go/storage" 10 | ) 11 | 12 | // constants related to multi-part uploads 13 | const ( 14 | startChunkSize = 4 * 1024 * 1024 15 | maxChunkSize = 100 * 1024 * 1024 16 | maxParts = 50000 17 | ) 18 | 19 | // errMultiPartUploadTooBig is the error returned when a file is just too big to upload 20 | var errMultiPartUploadTooBig = errors.New("size exceeds maximum capacity for a single multi-part upload") 21 | 22 | // encodedBlockID returns the base64 encoded block id as expected by azure 23 | func encodedBlockID(id uint64) string { 24 | bytesID := make([]byte, 8) 25 | binary.LittleEndian.PutUint64(bytesID, id) 26 | return base64.StdEncoding.EncodeToString(bytesID) 27 | } 28 | 29 | // determineChunkSize determines the chunk size for a multi-part upload. 30 | func determineChunkSize(size int64) (int64, error) { 31 | var chunkSize = int64(startChunkSize) 32 | 33 | for { 34 | parts := size / chunkSize 35 | rem := size % chunkSize 36 | 37 | if rem != 0 { 38 | parts++ 39 | } 40 | 41 | if parts <= maxParts { 42 | break 43 | } 44 | 45 | if chunkSize == maxChunkSize { 46 | return 0, errMultiPartUploadTooBig 47 | } 48 | 49 | chunkSize *= 2 50 | if chunkSize > maxChunkSize { 51 | chunkSize = maxChunkSize 52 | } 53 | } 54 | 55 | return chunkSize, nil 56 | } 57 | 58 | // multipartUpload performs a multi-part upload by chunking the data, putting each chunk, then 59 | // assembling the chunks into a blob 60 | func (c *container) multipartUpload(name string, r io.Reader, size int64) error { 61 | chunkSize, err := determineChunkSize(size) 62 | if err != nil { 63 | return err 64 | } 65 | var buf = make([]byte, chunkSize) 66 | 67 | var blocks []az.Block 68 | var rawID uint64 69 | blob := c.client.GetContainerReference(c.id).GetBlobReference(name) 70 | 71 | // TODO: upload the parts in parallel 72 | for { 73 | n, err := r.Read(buf) 74 | if err != nil { 75 | if err == io.EOF { 76 | break 77 | } 78 | return err 79 | } 80 | 81 | blockID := encodedBlockID(rawID) 82 | chunk := buf[:n] 83 | 84 | if err := blob.PutBlock(blockID, chunk, nil); err != nil { 85 | return err 86 | } 87 | 88 | blocks = append(blocks, az.Block{ 89 | ID: blockID, 90 | Status: az.BlockStatusLatest, 91 | }) 92 | rawID++ 93 | } 94 | 95 | return blob.PutBlockList(blocks, nil) 96 | } 97 | -------------------------------------------------------------------------------- /azure/multipart_test.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "crypto/md5" 5 | "fmt" 6 | "io" 7 | "os" 8 | "testing" 9 | 10 | "github.com/cheekybits/is" 11 | "github.com/graymeta/stow" 12 | ) 13 | 14 | func TestChunkSize(t *testing.T) { 15 | is := is.New(t) 16 | 17 | // 10 bytes, should fit in a single chunk 18 | sz, err := determineChunkSize(10) 19 | is.NoErr(err) 20 | is.Equal(sz, startChunkSize) 21 | 22 | // Scale up the chunk size 23 | sz, err = determineChunkSize(maxParts*startChunkSize + 1) 24 | is.NoErr(err) 25 | is.Equal(sz, startChunkSize*2) 26 | 27 | // Maximum size 28 | sz, err = determineChunkSize(maxParts * maxChunkSize) 29 | is.NoErr(err) 30 | is.Equal(sz, maxChunkSize) 31 | 32 | // Add one byte, we shouldn't be able to upload this 33 | sz, err = determineChunkSize(maxParts*maxChunkSize + 1) 34 | is.Err(err) 35 | is.Equal(err, errMultiPartUploadTooBig) 36 | } 37 | 38 | func TestEncodeBlockID(t *testing.T) { 39 | is := is.New(t) 40 | 41 | is.Equal(encodedBlockID(10), "CgAAAAAAAAA=") 42 | is.Equal(encodedBlockID(600), "WAIAAAAAAAA=") 43 | } 44 | 45 | func TestMultipartUpload(t *testing.T) { 46 | is := is.New(t) 47 | 48 | if azureaccount == "" || azurekey == "" { 49 | t.Skip("skipping test because missing either AZUREACCOUNT or AZUREKEY") 50 | } 51 | cfg := stow.ConfigMap{"account": azureaccount, "key": azurekey} 52 | 53 | bigfile := os.Getenv("BIG_FILE_TO_UPLOAD") 54 | if bigfile == "" { 55 | t.Skip("skipping test because BIG_FILE_TO_UPLOAD was not set") 56 | } 57 | 58 | location, err := stow.Dial("azure", cfg) 59 | is.NoErr(err) 60 | is.OK(location) 61 | 62 | defer location.Close() 63 | 64 | cont, err := location.CreateContainer("bigfiletest") 65 | is.NoErr(err) 66 | is.OK(cont) 67 | 68 | defer func() { 69 | is.NoErr(location.RemoveContainer(cont.ID())) 70 | }() 71 | 72 | f, err := os.Open(bigfile) 73 | is.NoErr(err) 74 | defer f.Close() 75 | 76 | fi, err := f.Stat() 77 | is.NoErr(err) 78 | 79 | name := "bigfile/thebigfile" 80 | azc, ok := cont.(*container) 81 | is.OK(ok) 82 | is.NoErr(azc.multipartUpload(name, f, fi.Size())) 83 | 84 | item, err := cont.Item(name) 85 | is.NoErr(err) 86 | 87 | defer cont.RemoveItem(name) 88 | 89 | r, err := item.Open() 90 | is.NoErr(err) 91 | defer r.Close() 92 | 93 | hashNew := md5.New() 94 | _, err = io.Copy(hashNew, r) 95 | is.NoErr(err) 96 | 97 | f.Seek(0, 0) 98 | hashOld := md5.New() 99 | _, err = io.Copy(hashOld, f) 100 | is.NoErr(err) 101 | 102 | is.Equal(fmt.Sprintf("%x", hashOld.Sum(nil)), fmt.Sprintf("%x", hashNew.Sum(nil))) 103 | } 104 | -------------------------------------------------------------------------------- /azure/stow_test.go: -------------------------------------------------------------------------------- 1 | package azure 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/cheekybits/is" 10 | "github.com/graymeta/stow" 11 | "github.com/graymeta/stow/test" 12 | ) 13 | 14 | var ( 15 | azureaccount = os.Getenv("AZUREACCOUNT") 16 | azurekey = os.Getenv("AZUREKEY") 17 | ) 18 | 19 | func TestStow(t *testing.T) { 20 | if azureaccount == "" || azurekey == "" { 21 | t.Skip("skipping test because missing either AZUREACCOUNT or AZUREKEY") 22 | } 23 | 24 | cfg := stow.ConfigMap{"account": azureaccount, "key": azurekey} 25 | test.All(t, "azure", cfg) 26 | } 27 | 28 | func TestEtagCleanup(t *testing.T) { 29 | etagValue := "9c51403a2255f766891a1382288dece4" 30 | permutations := []string{ 31 | `"%s"`, // Enclosing quotations 32 | `W/\"%s\"`, // Weak tag identifier with escapted quotes 33 | `W/"%s"`, // Weak tag identifier with quotes 34 | `"\"%s"\"`, // Double quotes, inner escaped 35 | `""%s""`, // Double quotes, 36 | `"W/"%s""`, // Double quotes with weak identifier 37 | `"W/\"%s\""`, // Double quotes with weak identifier, inner escaped 38 | } 39 | for index, p := range permutations { 40 | testStr := fmt.Sprintf(p, etagValue) 41 | cleanTestStr := cleanEtag(testStr) 42 | if etagValue != cleanTestStr { 43 | t.Errorf(`Failure at permutation #%d (%s), result: %s`, 44 | index, permutations[index], cleanTestStr) 45 | } 46 | } 47 | } 48 | 49 | func TestPrepMetadataSuccess(t *testing.T) { 50 | is := is.New(t) 51 | 52 | m := make(map[string]string) 53 | m["one"] = "two" 54 | m["3"] = "4" 55 | m["ninety-nine"] = "100" 56 | 57 | m2 := make(map[string]interface{}) 58 | for key, value := range m { 59 | m2[key] = value 60 | } 61 | 62 | //returns map[string]interface 63 | returnedMap, err := prepMetadata(m2) 64 | is.NoErr(err) 65 | 66 | if !reflect.DeepEqual(returnedMap, m) { 67 | t.Errorf("Expected map (%+v) and returned map (%+v) are not equal.", m, returnedMap) 68 | } 69 | } 70 | 71 | func TestPrepMetadataFailureWithNonStringValues(t *testing.T) { 72 | is := is.New(t) 73 | 74 | m := make(map[string]interface{}) 75 | m["float"] = 8.9 76 | m["number"] = 9 77 | 78 | _, err := prepMetadata(m) 79 | is.Err(err) 80 | } 81 | -------------------------------------------------------------------------------- /b2/config.go: -------------------------------------------------------------------------------- 1 | package b2 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | 7 | "github.com/graymeta/stow" 8 | "gopkg.in/kothar/go-backblaze.v0" 9 | ) 10 | 11 | // Config key constants. 12 | const ( 13 | ConfigAccountID = "account_id" 14 | ConfigApplicationKey = "application_key" 15 | ConfigKeyID = "application_key_id" 16 | ) 17 | 18 | // Kind is the kind of Location this package provides. 19 | const Kind = "b2" 20 | 21 | func init() { 22 | validatefn := func(config stow.Config) error { 23 | _, ok := config.Config(ConfigApplicationKey) 24 | if !ok { 25 | return errors.New("missing application key") 26 | } 27 | accountID, _ := config.Config(ConfigAccountID) 28 | keyID, _ := config.Config(ConfigKeyID) 29 | if accountID == "" && keyID == "" { 30 | return errors.New("account ID or applicaton key ID needs to be set") 31 | } 32 | return nil 33 | } 34 | makefn := func(config stow.Config) (stow.Location, error) { 35 | if err := validatefn(config); err != nil { 36 | return nil, err 37 | } 38 | l := &location{ 39 | config: config, 40 | } 41 | var err error 42 | l.client, err = newB2Client(l.config) 43 | if err != nil { 44 | return nil, err 45 | } 46 | return l, nil 47 | } 48 | kindfn := func(u *url.URL) bool { 49 | return u.Scheme == Kind 50 | } 51 | stow.Register(Kind, makefn, kindfn, validatefn) 52 | } 53 | 54 | func newB2Client(cfg stow.Config) (*backblaze.B2, error) { 55 | accountID, _ := cfg.Config(ConfigAccountID) 56 | applicationKey, _ := cfg.Config(ConfigApplicationKey) 57 | keyID, _ := cfg.Config(ConfigKeyID) 58 | 59 | client, err := backblaze.NewB2(backblaze.Credentials{ 60 | AccountID: accountID, 61 | ApplicationKey: applicationKey, 62 | KeyID: keyID, 63 | }) 64 | 65 | if err != nil { 66 | return nil, errors.New("Unable to create client") 67 | } 68 | 69 | err = client.AuthorizeAccount() 70 | if err != nil { 71 | return nil, errors.New("Unable to authenticate") 72 | } 73 | 74 | return client, nil 75 | } 76 | -------------------------------------------------------------------------------- /b2/container.go: -------------------------------------------------------------------------------- 1 | package b2 2 | 3 | import ( 4 | "io" 5 | "strings" 6 | "time" 7 | 8 | "github.com/graymeta/stow" 9 | "github.com/pkg/errors" 10 | "gopkg.in/kothar/go-backblaze.v0" 11 | ) 12 | 13 | type container struct { 14 | bucket *backblaze.Bucket 15 | } 16 | 17 | var _ stow.Container = (*container)(nil) 18 | 19 | // ID returns the name of a bucket 20 | func (c *container) ID() string { 21 | // Although backblaze does give an ID for buckets, some operations deal with bucket 22 | // names instead of the ID (specifically the B2.Bucket method). For that reason, 23 | // return name instead of ID. We can still use the id field internally when necessary 24 | return c.bucket.Name 25 | } 26 | 27 | // Name returns the name of the bucket 28 | func (c *container) Name() string { 29 | return c.bucket.Name 30 | } 31 | 32 | // Item returns a stow.Item given the item's ID 33 | func (c *container) Item(id string) (stow.Item, error) { 34 | return c.getItem(id) 35 | } 36 | 37 | // Items retreives a list of items from b2. Since the b2 ListFileNames operation 38 | // does not natively support a prefix, we fake it ourselves 39 | func (c *container) Items(prefix, cursor string, count int) ([]stow.Item, string, error) { 40 | items := make([]stow.Item, 0, count) 41 | for { 42 | response, err := c.bucket.ListFileNames(cursor, count) 43 | if err != nil { 44 | return nil, "", err 45 | } 46 | 47 | for _, obj := range response.Files { 48 | if prefix != stow.NoPrefix && !strings.HasPrefix(obj.Name, prefix) { 49 | continue 50 | } 51 | items = append(items, &item{ 52 | id: obj.ID, 53 | name: obj.Name, 54 | size: int64(obj.Size), 55 | lastModified: time.Unix(obj.UploadTimestamp/1000, 0), 56 | bucket: c.bucket, 57 | }) 58 | if len(items) == count { 59 | break 60 | } 61 | } 62 | 63 | cursor = response.NextFileName 64 | 65 | if prefix == "" || cursor == "" { 66 | return items, cursor, nil 67 | } 68 | 69 | if len(items) == count { 70 | break 71 | } 72 | 73 | if !strings.HasPrefix(cursor, prefix) { 74 | return items, "", nil 75 | } 76 | } 77 | 78 | if cursor != "" && cursor != items[len(items)-1].Name() { 79 | // append a space because that's a funny quirk of backblaze's implementation 80 | cursor = items[len(items)-1].Name() + " " 81 | } 82 | 83 | return items, cursor, nil 84 | } 85 | 86 | // Put uploads a file 87 | func (c *container) Put(name string, r io.Reader, size int64, metadata map[string]interface{}) (stow.Item, error) { 88 | // Convert map[string]interface{} to map[string]string 89 | mdPrepped, err := prepMetadata(metadata) 90 | if err != nil { 91 | return nil, errors.Wrap(err, "unable to create or update item, preparing metadata") 92 | } 93 | 94 | file, err := c.bucket.UploadFile(name, mdPrepped, r) 95 | if err != nil { 96 | return nil, err 97 | } 98 | 99 | return &item{ 100 | id: file.ID, 101 | name: file.Name, 102 | size: file.ContentLength, 103 | bucket: c.bucket, 104 | }, nil 105 | } 106 | 107 | // RemoveItem identifies the file by it's ID, then removes all versions of that file 108 | func (c *container) RemoveItem(id string) error { 109 | item, err := c.getItem(id) 110 | if err != nil { 111 | return err 112 | } 113 | 114 | // files can have multiple versions in backblaze. You have to delete 115 | // files one version at a time. 116 | for { 117 | response, err := item.bucket.ListFileNames(item.Name(), 1) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | var fileStatus *backblaze.FileStatus 123 | for i := range response.Files { 124 | if response.Files[i].Name == item.Name() { 125 | fileStatus = &response.Files[i] 126 | break 127 | } 128 | } 129 | if fileStatus == nil { 130 | // we've deleted all versions of the file 131 | return nil 132 | } 133 | 134 | if _, err := c.bucket.DeleteFileVersion(item.name, response.Files[0].ID); err != nil { 135 | return err 136 | } 137 | } 138 | } 139 | 140 | func (c *container) getItem(id string) (*item, error) { 141 | file, err := c.bucket.GetFileInfo(id) 142 | if err != nil { 143 | lowered := strings.ToLower(err.Error()) 144 | if (strings.Contains(lowered, "not") && strings.Contains(lowered, "found")) || (strings.Contains(lowered, "bad") && strings.Contains(lowered, "fileid")) { 145 | return nil, stow.ErrNotFound 146 | } 147 | return nil, err 148 | } 149 | 150 | return &item{ 151 | id: file.ID, 152 | name: file.Name, 153 | size: file.ContentLength, 154 | bucket: c.bucket, 155 | }, nil 156 | } 157 | 158 | // prepMetadata parses a raw map into the native type required by b2 to set metadata (map[string]string). 159 | // This function also assumes that the value of a key value pair is a string. 160 | func prepMetadata(md map[string]interface{}) (map[string]string, error) { 161 | m := make(map[string]string, len(md)) 162 | for key, value := range md { 163 | strValue, valid := value.(string) 164 | if !valid { 165 | return nil, errors.Errorf(`value of key '%s' in metadata must be of type string`, key) 166 | } 167 | m[key] = strValue 168 | } 169 | return m, nil 170 | } 171 | 172 | // parseMetadata transforms a map[string]string to a map[string]interface{} 173 | func parseMetadata(md map[string]string) map[string]interface{} { 174 | m := make(map[string]interface{}, len(md)) 175 | for key, value := range md { 176 | m[key] = value 177 | } 178 | return m 179 | } 180 | -------------------------------------------------------------------------------- /b2/item.go: -------------------------------------------------------------------------------- 1 | package b2 2 | 3 | import ( 4 | "io" 5 | "net/url" 6 | "sync" 7 | "time" 8 | 9 | "github.com/graymeta/stow" 10 | 11 | "github.com/pkg/errors" 12 | "gopkg.in/kothar/go-backblaze.v0" 13 | ) 14 | 15 | type item struct { 16 | id string 17 | name string 18 | size int64 19 | lastModified time.Time 20 | bucket *backblaze.Bucket 21 | 22 | metadata map[string]interface{} 23 | infoOnce sync.Once 24 | infoErr error 25 | } 26 | 27 | var ( 28 | _ stow.Item = (*item)(nil) 29 | _ stow.ItemRanger = (*item)(nil) 30 | ) 31 | 32 | // ID returns this item's ID 33 | func (i *item) ID() string { 34 | return i.id 35 | } 36 | 37 | // Name returns this item's name 38 | func (i *item) Name() string { 39 | return i.name 40 | } 41 | 42 | // URL returns the stow url for this item 43 | func (i *item) URL() *url.URL { 44 | str, err := i.bucket.FileURL(i.name) 45 | if err != nil { 46 | return nil 47 | } 48 | 49 | url, _ := url.Parse(str) 50 | url.Scheme = Kind 51 | 52 | return url 53 | } 54 | 55 | // Metadata returns additional item metadata fields that were set when the file was uploaded 56 | func (i *item) Metadata() (map[string]interface{}, error) { 57 | if err := i.ensureInfo(); err != nil { 58 | return nil, errors.Wrap(err, "retrieving item metadata") 59 | } 60 | return i.metadata, nil 61 | } 62 | 63 | // size returns the file's size, in bytes 64 | func (i *item) Size() (int64, error) { 65 | return i.size, nil 66 | } 67 | 68 | // Open downloads the item 69 | func (i *item) Open() (io.ReadCloser, error) { 70 | _, r, err := i.bucket.DownloadFileByName(i.name) 71 | return r, err 72 | } 73 | 74 | // OpenRange opens the item for reading starting at byte start and ending 75 | // at byte end. 76 | func (i *item) OpenRange(start, end uint64) (io.ReadCloser, error) { 77 | _, r, err := i.bucket.DownloadFileRangeByName( 78 | i.name, 79 | &backblaze.FileRange{Start: int64(start), End: int64(end)}, 80 | ) 81 | return r, err 82 | } 83 | 84 | // ETag returns an etag for an item. In this implementation we use the file's last modified timestamp 85 | func (i *item) ETag() (string, error) { 86 | if err := i.ensureInfo(); err != nil { 87 | return "", errors.Wrap(err, "retreiving etag") 88 | } 89 | return i.lastModified.String(), nil 90 | } 91 | 92 | // LastMod returns the file's last modified timestamp 93 | func (i *item) LastMod() (time.Time, error) { 94 | if err := i.ensureInfo(); err != nil { 95 | return time.Time{}, errors.Wrap(err, "retrieving Last Modified information of Item") 96 | } 97 | return i.lastModified, nil 98 | } 99 | 100 | func (i *item) ensureInfo() error { 101 | if i.metadata == nil || i.lastModified.IsZero() { 102 | i.infoOnce.Do(func() { 103 | f, err := i.bucket.GetFileInfo(i.id) 104 | if err != nil { 105 | i.infoErr = err 106 | return 107 | } 108 | 109 | i.lastModified = time.Unix(f.UploadTimestamp/1000, 0) 110 | i.metadata = parseMetadata(f.FileInfo) 111 | }) 112 | } 113 | return i.infoErr 114 | } 115 | -------------------------------------------------------------------------------- /b2/location.go: -------------------------------------------------------------------------------- 1 | package b2 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | "strings" 7 | 8 | "github.com/graymeta/stow" 9 | "gopkg.in/kothar/go-backblaze.v0" 10 | ) 11 | 12 | type location struct { 13 | config stow.Config 14 | client *backblaze.B2 15 | } 16 | 17 | // Close closes the interface. It's a Noop for this B2 implementation 18 | func (l *location) Close() error { 19 | return nil // nothing to close 20 | } 21 | 22 | // CreateContainer creates a new container (bucket) 23 | func (l *location) CreateContainer(name string) (stow.Container, error) { 24 | bucket, err := l.client.CreateBucket(name, backblaze.AllPrivate) 25 | if err != nil { 26 | return nil, err 27 | } 28 | return &container{ 29 | bucket: bucket, 30 | }, nil 31 | } 32 | 33 | // Containers lists all containers in the location 34 | func (l *location) Containers(prefix string, cursor string, count int) ([]stow.Container, string, error) { 35 | response, err := l.client.ListBuckets() 36 | if err != nil { 37 | return nil, "", err 38 | } 39 | 40 | containers := make([]stow.Container, 0, len(response)) 41 | for _, cont := range response { 42 | // api and/or library don't seem to support prefixes, so do it ourself 43 | if strings.HasPrefix(cont.Name, prefix) { 44 | containers = append(containers, &container{ 45 | bucket: cont, 46 | }) 47 | } 48 | } 49 | 50 | return containers, "", nil 51 | } 52 | 53 | // Container returns a stow.Contaner given a container id. In this case, the 'id' 54 | // is really the bucket name 55 | func (l *location) Container(id string) (stow.Container, error) { 56 | bucket, err := l.client.Bucket(id) 57 | if err != nil || bucket == nil { 58 | return nil, stow.ErrNotFound 59 | } 60 | 61 | return &container{ 62 | bucket: bucket, 63 | }, nil 64 | } 65 | 66 | // ItemByURL returns a stow.Item given a b2 stow url 67 | func (l *location) ItemByURL(u *url.URL) (stow.Item, error) { 68 | if u.Scheme != Kind { 69 | return nil, errors.New("not valid b2 URL") 70 | } 71 | 72 | // b2://f001.backblaze.com/file// 73 | pieces := strings.SplitN(u.Path, "/", 4) 74 | 75 | c, err := l.Container(pieces[2]) 76 | if err != nil { 77 | return nil, err 78 | } 79 | 80 | filename, err := url.QueryUnescape(pieces[3]) 81 | if err != nil { 82 | return nil, err 83 | } 84 | response, err := c.(*container).bucket.ListFileNames(filename, 1) 85 | if err != nil { 86 | return nil, stow.ErrNotFound 87 | } 88 | 89 | if len(response.Files) != 1 { 90 | return nil, errors.New("unexpected number of responses from ListFileNames") 91 | } 92 | 93 | return c.Item(response.Files[0].ID) 94 | } 95 | 96 | // RemoveContainer removes the specified bucket. In this case, the 'id' 97 | // is really the bucket name 98 | func (l *location) RemoveContainer(id string) error { 99 | stowCont, err := l.Container(id) 100 | if err != nil { 101 | return err 102 | } 103 | 104 | return stowCont.(*container).bucket.Delete() 105 | } 106 | -------------------------------------------------------------------------------- /b2/stow_test.go: -------------------------------------------------------------------------------- 1 | package b2 2 | 3 | import ( 4 | "math/rand" 5 | "os" 6 | "strings" 7 | "testing" 8 | "time" 9 | 10 | isi "github.com/cheekybits/is" 11 | "github.com/graymeta/stow" 12 | "github.com/graymeta/stow/test" 13 | ) 14 | 15 | func TestStow(t *testing.T) { 16 | is := isi.New(t) 17 | accountID := os.Getenv("B2_ACCOUNT_ID") 18 | applicationKey := os.Getenv("B2_APPLICATION_KEY") 19 | 20 | if accountID == "" || applicationKey == "" { 21 | t.Skip("Backblaze credentials missing from environment. Skipping tests") 22 | } 23 | 24 | cfg := stow.ConfigMap{ 25 | "account_id": accountID, 26 | "application_key": applicationKey, 27 | } 28 | 29 | location, err := stow.Dial("b2", cfg) 30 | is.NoErr(err) 31 | is.OK(location) 32 | 33 | t.Run("basic stow interface tests", func(t *testing.T) { 34 | test.All(t, "b2", cfg) 35 | }) 36 | 37 | // This test is designed to test the container.Items() function. B2 doesn't 38 | // support listing items in a bucket by prefix, so our implementation fakes this 39 | // functionality by requesting additional pages of files 40 | t.Run("Items with prefix", func(t *testing.T) { 41 | is := isi.New(t) 42 | container, err := location.CreateContainer("stowtest" + randName(10)) 43 | is.NoErr(err) 44 | is.OK(container) 45 | 46 | defer func() { 47 | is.NoErr(location.RemoveContainer(container.ID())) 48 | }() 49 | 50 | // add some items to the container 51 | content := "foo" 52 | item1, err := container.Put("b/a", strings.NewReader(content), int64(len(content)), nil) 53 | is.NoErr(err) 54 | item2, err := container.Put("b/bb", strings.NewReader(content), int64(len(content)), nil) 55 | is.NoErr(err) 56 | item3, err := container.Put("b/bc", strings.NewReader(content), int64(len(content)), nil) 57 | is.NoErr(err) 58 | item4, err := container.Put("b/bd", strings.NewReader(content), int64(len(content)), nil) 59 | is.NoErr(err) 60 | 61 | defer func() { 62 | is.NoErr(container.RemoveItem(item1.ID())) 63 | is.NoErr(container.RemoveItem(item2.ID())) 64 | is.NoErr(container.RemoveItem(item3.ID())) 65 | is.NoErr(container.RemoveItem(item4.ID())) 66 | }() 67 | 68 | items, cursor, err := container.Items("b/b", stow.CursorStart, 2) 69 | is.NoErr(err) 70 | is.Equal(len(items), 2) 71 | is.Equal(cursor, "b/bc ") 72 | 73 | items, cursor, err = container.Items("", stow.CursorStart, 2) 74 | is.NoErr(err) 75 | is.Equal(len(items), 2) 76 | is.Equal(cursor, "b/bb ") 77 | }) 78 | 79 | t.Run("Item Delete", func(t *testing.T) { 80 | is := isi.New(t) 81 | container, err := location.CreateContainer("stowtest" + randName(10)) 82 | is.NoErr(err) 83 | is.OK(container) 84 | 85 | defer func() { 86 | is.NoErr(location.RemoveContainer(container.ID())) 87 | }() 88 | 89 | // Put an item twice, creating two versions of the file 90 | content := "foo" 91 | i, err := container.Put("foo", strings.NewReader(content), int64(len(content)), nil) 92 | is.NoErr(err) 93 | content = "foo_v2" 94 | _, err = container.Put("foo", strings.NewReader(content), int64(len(content)), nil) 95 | is.NoErr(err) 96 | 97 | is.NoErr(container.RemoveItem(i.ID())) 98 | 99 | // verify item is gone 100 | _, err = container.Item(i.ID()) 101 | is.Equal(err, stow.ErrNotFound) 102 | }) 103 | } 104 | 105 | func randName(length int) string { 106 | b := make([]rune, length) 107 | for i := range b { 108 | b[i] = letters[rand.Intn(len(letters))] 109 | } 110 | return string(b) 111 | } 112 | 113 | var letters = []rune("abcdefghijklmnopqrstuvwxyz") 114 | 115 | func init() { 116 | rand.Seed(int64(time.Now().Nanosecond())) 117 | } 118 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package stow provides an abstraction on cloud storage capabilities. 2 | package stow 3 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/graymeta/stow 2 | 3 | go 1.14 4 | 5 | require ( 6 | cloud.google.com/go v0.38.0 7 | github.com/Azure/azure-sdk-for-go v32.5.0+incompatible 8 | github.com/Azure/go-autorest/autorest v0.9.0 // indirect 9 | github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect 10 | github.com/aws/aws-sdk-go v1.23.4 11 | github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 12 | github.com/dnaeon/go-vcr v1.1.0 // indirect 13 | github.com/google/readahead v0.0.0-20161222183148-eaceba169032 // indirect 14 | github.com/hashicorp/go-multierror v1.0.0 15 | github.com/kr/fs v0.1.0 // indirect 16 | github.com/ncw/swift v1.0.49 17 | github.com/pkg/errors v0.8.1 18 | github.com/pkg/sftp v1.10.0 19 | github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9 // indirect 20 | github.com/satori/go.uuid v1.2.0 // indirect 21 | github.com/stretchr/testify v1.4.0 22 | golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 23 | golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 // indirect 24 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 25 | google.golang.org/api v0.8.0 26 | gopkg.in/kothar/go-backblaze.v0 v0.0.0-20190520213052-702d4e7eb465 27 | ) 28 | -------------------------------------------------------------------------------- /google/README.md: -------------------------------------------------------------------------------- 1 | # Google Cloud Storage Stow Implementation 2 | 3 | Location = Google Cloud Storage 4 | 5 | Container = Bucket 6 | 7 | Item = File 8 | 9 | ## How to access underlying service types 10 | 11 | Use a type conversion to extract the underlying `Location`, `Container`, or `Item` implementations. Then use the Google-specific getters to access the internal Google Cloud Storage `Service`, `Bucket`, and `Object` values. 12 | 13 | ```go 14 | import ( 15 | "log" 16 | "github.com/graymeta/stow" 17 | stowgs "github.com/graymeta/stow/google" 18 | ) 19 | 20 | stowLoc, err := stow.Dial(stowgs.Kind, stow.ConfigMap{ 21 | stowgs.ConfigJSON: "", 22 | stowgs.ConfigProjectId: "", 23 | }) 24 | if err != nil { 25 | log.Fatal(err) 26 | } 27 | 28 | stowBucket, err = stowLoc.Container("mybucket") 29 | if err != nil { 30 | log.Fatal(err) 31 | } 32 | 33 | if gsBucket, ok := stowBucket.(*stowgs.Container); ok { 34 | if gsLoc, ok := stowLoc.(*stowgs.Location); ok { 35 | 36 | googleService := gsLoc.Service() 37 | googleBucket, err := gsBucket.Bucket() 38 | 39 | // < Send platform-specific commands here > 40 | 41 | } 42 | } 43 | ``` 44 | 45 | By default, Stow uses `https://www.googleapis.com/auth/devstorage.read_write` scope. Different scopes can be used by passing a comma separated list of scopes, like below: 46 | ```go 47 | stowLoc, err := stow.Dial(stowgs.Kind, stow.ConfigMap{ 48 | stowgs.ConfigJSON: "", 49 | stowgs.ConfigProjectId: "", 50 | stowgs.ConfigScopes: ",", 51 | }) 52 | ``` 53 | 54 | --- 55 | 56 | Configuration... You need to create a project in google, and then create a service account in google tied to that project. You will need to download a `.json` file with the configuration for the service account. To run the test suite, the service account will need edit privileges inside the project. 57 | 58 | To run the test suite, set the `GOOGLE_CREDENTIALS_FILE` environment variable to point to the location of the .json file containing the service account credentials and set `GOOGLE_PROJECT_ID` to the project ID, otherwise the test suite will not be run. 59 | 60 | --- 61 | 62 | Concerns: 63 | 64 | - Google's storage plaform is more _eventually consistent_ than other platforms. Sometimes, the tests appear to be flaky because of this. One example is when deleting files from a bucket, then immediately deleting the bucket...sometimes the bucket delete will fail saying that the bucket isn't empty simply because the file delete messages haven't propagated through Google's infrastructure. We may need to add some delay into the test suite to account for this. 65 | -------------------------------------------------------------------------------- /google/config.go: -------------------------------------------------------------------------------- 1 | package google 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net/url" 7 | "strings" 8 | 9 | "cloud.google.com/go/storage" 10 | "golang.org/x/oauth2/google" 11 | "google.golang.org/api/option" 12 | 13 | "github.com/graymeta/stow" 14 | ) 15 | 16 | // Kind represents the name of the location/storage type. 17 | const Kind = "google" 18 | 19 | const ( 20 | // The service account json blob 21 | ConfigJSON = "json" 22 | ConfigProjectId = "project_id" 23 | ConfigScopes = "scopes" 24 | ) 25 | 26 | func init() { 27 | validatefn := func(config stow.Config) error { 28 | _, ok := config.Config(ConfigJSON) 29 | if !ok { 30 | return errors.New("missing JSON configuration") 31 | } 32 | 33 | _, ok = config.Config(ConfigProjectId) 34 | if !ok { 35 | return errors.New("missing Project ID") 36 | } 37 | return nil 38 | } 39 | makefn := func(config stow.Config) (stow.Location, error) { 40 | _, ok := config.Config(ConfigJSON) 41 | if !ok { 42 | return nil, errors.New("missing JSON configuration") 43 | } 44 | 45 | _, ok = config.Config(ConfigProjectId) 46 | if !ok { 47 | return nil, errors.New("missing Project ID") 48 | } 49 | 50 | // Create a new client 51 | ctx, client, err := newGoogleStorageClient(config) 52 | if err != nil { 53 | return nil, err 54 | } 55 | 56 | // Create a location with given config and client 57 | loc := &Location{ 58 | config: config, 59 | client: client, 60 | ctx: ctx, 61 | } 62 | 63 | return loc, nil 64 | } 65 | 66 | kindfn := func(u *url.URL) bool { 67 | return u.Scheme == Kind 68 | } 69 | 70 | stow.Register(Kind, makefn, kindfn, validatefn) 71 | } 72 | 73 | // Attempts to create a session based on the information given. 74 | func newGoogleStorageClient(config stow.Config) (context.Context, *storage.Client, error) { 75 | json, _ := config.Config(ConfigJSON) 76 | 77 | scopes := []string{storage.ScopeFullControl} 78 | if s, ok := config.Config(ConfigScopes); ok && s != "" { 79 | scopes = strings.Split(s, ",") 80 | } 81 | 82 | ctx := context.Background() 83 | var creds *google.Credentials 84 | var err error 85 | if json != "" { 86 | creds, err = google.CredentialsFromJSON(ctx, []byte(json), scopes...) 87 | if err != nil { 88 | return nil, nil, err 89 | } 90 | } else { 91 | creds, err = google.FindDefaultCredentials(ctx, scopes...) 92 | if err != nil { 93 | return nil, nil, err 94 | } 95 | } 96 | 97 | client, err := storage.NewClient(ctx, option.WithCredentials(creds)) 98 | if err != nil { 99 | return nil, nil, err 100 | } 101 | return ctx, client, nil 102 | } 103 | -------------------------------------------------------------------------------- /google/container.go: -------------------------------------------------------------------------------- 1 | package google 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "cloud.google.com/go/storage" 8 | "github.com/pkg/errors" 9 | "google.golang.org/api/iterator" 10 | 11 | "github.com/graymeta/stow" 12 | ) 13 | 14 | type Container struct { 15 | // Name is needed to retrieve items. 16 | name string 17 | 18 | // Client is responsible for performing the requests. 19 | client *storage.Client 20 | 21 | // ctx is used on google storage API calls 22 | ctx context.Context 23 | } 24 | 25 | // ID returns a string value which represents the name of the container. 26 | func (c *Container) ID() string { 27 | return c.name 28 | } 29 | 30 | // Name returns a string value which represents the name of the container. 31 | func (c *Container) Name() string { 32 | return c.name 33 | } 34 | 35 | // Bucket returns the google bucket attributes 36 | func (c *Container) Bucket() *storage.BucketHandle{ 37 | return c.client.Bucket(c.name) 38 | } 39 | 40 | // Item returns a stow.Item instance of a container based on the 41 | // name of the container 42 | func (c *Container) Item(id string) (stow.Item, error) { 43 | item, err := c.Bucket().Object(id).Attrs(c.ctx) 44 | if err != nil { 45 | if err == storage.ErrObjectNotExist { 46 | return nil, stow.ErrNotFound 47 | } 48 | return nil, err 49 | } 50 | 51 | return c.convertToStowItem(item) 52 | } 53 | 54 | // Items retrieves a list of items that are prepended with 55 | // the prefix argument. The 'cursor' variable facilitates pagination. 56 | func (c *Container) Items(prefix string, cursor string, count int) ([]stow.Item, string, error) { 57 | query := &storage.Query{Prefix: prefix} 58 | call := c.Bucket().Objects(c.ctx, query) 59 | 60 | p := iterator.NewPager(call, count, cursor) 61 | var results []*storage.ObjectAttrs 62 | nextPageToken, err := p.NextPage(&results) 63 | if err != nil { 64 | return nil, "", err 65 | } 66 | 67 | var items []stow.Item 68 | for _, item := range results { 69 | i, err := c.convertToStowItem(item) 70 | if err != nil { 71 | return nil, "", err 72 | } 73 | 74 | items = append(items, i) 75 | } 76 | 77 | return items, nextPageToken, nil 78 | } 79 | 80 | // RemoveItem will delete a google storage Object 81 | func (c *Container) RemoveItem(id string) error { 82 | return c.Bucket().Object(id).Delete(c.ctx) 83 | } 84 | 85 | // Put sends a request to upload content to the container. The arguments 86 | // received are the name of the item, a reader representing the 87 | // content, and the size of the file. 88 | func (c *Container) Put(name string, r io.Reader, size int64, metadata map[string]interface{}) (stow.Item, error) { 89 | obj := c.Bucket().Object(name) 90 | 91 | mdPrepped, err := prepMetadata(metadata) 92 | if err != nil { 93 | return nil, err 94 | } 95 | 96 | w := obj.NewWriter(c.ctx) 97 | if _, err := io.Copy(w, r); err != nil { 98 | return nil, err 99 | } 100 | w.Close() 101 | 102 | attr, err := obj.Update(c.ctx, storage.ObjectAttrsToUpdate{Metadata: mdPrepped}) 103 | if err != nil { 104 | return nil, err 105 | } 106 | 107 | return c.convertToStowItem(attr) 108 | } 109 | 110 | func (c *Container) convertToStowItem(attr *storage.ObjectAttrs) (stow.Item, error) { 111 | u, err := prepUrl(attr.MediaLink) 112 | if err != nil { 113 | return nil, err 114 | } 115 | 116 | mdParsed, err := parseMetadata(attr.Metadata) 117 | if err != nil { 118 | return nil, err 119 | } 120 | 121 | return &Item{ 122 | name: attr.Name, 123 | container: c, 124 | client: c.client, 125 | size: attr.Size, 126 | etag: attr.Etag, 127 | hash: string(attr.MD5), 128 | lastModified: attr.Updated, 129 | url: u, 130 | metadata: mdParsed, 131 | object: attr, 132 | ctx: c.ctx, 133 | }, nil 134 | } 135 | 136 | func parseMetadata(metadataParsed map[string]string) (map[string]interface{}, error) { 137 | metadataParsedMap := make(map[string]interface{}, len(metadataParsed)) 138 | for key, value := range metadataParsed { 139 | metadataParsedMap[key] = value 140 | } 141 | return metadataParsedMap, nil 142 | } 143 | 144 | func prepMetadata(metadataParsed map[string]interface{}) (map[string]string, error) { 145 | returnMap := make(map[string]string, len(metadataParsed)) 146 | for key, value := range metadataParsed { 147 | str, ok := value.(string) 148 | if !ok { 149 | return nil, errors.Errorf(`value of key '%s' in metadata must be of type string`, key) 150 | } 151 | returnMap[key] = str 152 | } 153 | return returnMap, nil 154 | } 155 | -------------------------------------------------------------------------------- /google/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package google provides an abstraction of Google Cloud Storage. In this package, a Google Cloud Storage Bucket is represented by a Stow Container and a Google Cloud Storage Object is represented by a Stow Item. Note that directories may exist within a Bucket. 3 | 4 | Usage and Credentials 5 | 6 | A path to the JSON file representing configuration information for the service account is needed, as well as the Project ID that it is tied to. 7 | 8 | stow.Dial requires both a string value of the particular Stow Location Kind ("google") and a stow.Config instance. The stow.Config instance requires two entries with the specific key value attributes: 9 | 10 | - a key of google.ConfigJSON with a value of the path of the JSON configuration file 11 | - a key of google.ConfigProjectID with a value of the Project ID 12 | 13 | Location 14 | 15 | There are google.location methods which allow the retrieval of a Google Cloud Storage Object (Container or Containers). An Object can also be retrieved based on the its URL (ItemByURL). 16 | 17 | Additional google.location methods provide capabilities to create and remove Google Cloud Storage Buckets (CreateContainer or RemoveContainer). 18 | 19 | Container 20 | 21 | Methods of stow.container allow the retrieval of a Google Bucket's: 22 | 23 | - name(ID or Name) 24 | - object or complete list of objects (Item or Items, respectively) 25 | 26 | Additional methods of google.container allow Stow to: 27 | 28 | - remove an Object (RemoveItem) 29 | - update or create an Object (Put) 30 | 31 | Item 32 | 33 | Methods of google.Item allow the retrieval of a Google Cloud Storage Object's: 34 | - name (ID or name) 35 | - URL 36 | - size in bytes 37 | - Object specific metadata (information stored within the Google Cloud Service) 38 | - last modified date 39 | - Etag 40 | - content 41 | */ 42 | package google 43 | -------------------------------------------------------------------------------- /google/item.go: -------------------------------------------------------------------------------- 1 | package google 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net/url" 7 | "time" 8 | 9 | "cloud.google.com/go/storage" 10 | ) 11 | 12 | type Item struct { 13 | container *Container // Container information is required by a few methods. 14 | client *storage.Client // A client is needed to make requests. 15 | name string 16 | hash string 17 | etag string 18 | size int64 19 | url *url.URL 20 | lastModified time.Time 21 | metadata map[string]interface{} 22 | object *storage.ObjectAttrs 23 | ctx context.Context 24 | } 25 | 26 | // ID returns a string value that represents the name of a file. 27 | func (i *Item) ID() string { 28 | return i.name 29 | } 30 | 31 | // Name returns a string value that represents the name of the file. 32 | func (i *Item) Name() string { 33 | return i.name 34 | } 35 | 36 | // Size returns the size of an item in bytes. 37 | func (i *Item) Size() (int64, error) { 38 | return i.size, nil 39 | } 40 | 41 | // URL returns a url which follows the predefined format 42 | func (i *Item) URL() *url.URL { 43 | return i.url 44 | } 45 | 46 | // Open returns an io.ReadCloser to the object. Useful for downloading/streaming the object. 47 | func (i *Item) Open() (io.ReadCloser, error) { 48 | obj := i.container.Bucket().Object(i.name) 49 | return obj.NewReader(i.ctx) 50 | } 51 | 52 | // OpenRange returns an io.Reader to the object for a specific byte range 53 | func (i *Item) OpenRange(start, end uint64) (io.ReadCloser, error) { 54 | obj := i.container.Bucket().Object(i.name) 55 | return obj.NewRangeReader(i.ctx, int64(start), int64(end - start) + 1) 56 | } 57 | 58 | // LastMod returns the last modified date of the item. 59 | func (i *Item) LastMod() (time.Time, error) { 60 | return i.lastModified, nil 61 | } 62 | 63 | // Metadata returns a nil map and no error. 64 | func (i *Item) Metadata() (map[string]interface{}, error) { 65 | return i.metadata, nil 66 | } 67 | 68 | // ETag returns the ETag value 69 | func (i *Item) ETag() (string, error) { 70 | return i.etag, nil 71 | } 72 | 73 | // Object returns the Google Storage Object 74 | func (i *Item) StorageObject() *storage.ObjectAttrs { 75 | return i.object 76 | } 77 | 78 | // prepUrl takes a MediaLink string and returns a url 79 | func prepUrl(str string) (*url.URL, error) { 80 | u, err := url.Parse(str) 81 | if err != nil { 82 | return nil, err 83 | } 84 | u.Scheme = "google" 85 | 86 | // Discard the query string 87 | u.RawQuery = "" 88 | return u, nil 89 | } 90 | -------------------------------------------------------------------------------- /google/location.go: -------------------------------------------------------------------------------- 1 | package google 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "net/url" 7 | "strings" 8 | 9 | "cloud.google.com/go/storage" 10 | "google.golang.org/api/googleapi" 11 | "google.golang.org/api/iterator" 12 | 13 | "github.com/graymeta/stow" 14 | ) 15 | 16 | // A Location contains a client + the configurations used to create the client. 17 | type Location struct { 18 | config stow.Config 19 | client *storage.Client 20 | ctx context.Context 21 | } 22 | 23 | func (l *Location) Service() *storage.Client { 24 | return l.client 25 | } 26 | 27 | // Close simply satisfies the Location interface. There's nothing that 28 | // needs to be done in order to satisfy the interface. 29 | func (l *Location) Close() error { 30 | return nil // nothing to close 31 | } 32 | 33 | // CreateContainer creates a new container, in this case a bucket. 34 | func (l *Location) CreateContainer(containerName string) (stow.Container, error) { 35 | projId, _ := l.config.Config(ConfigProjectId) 36 | bucket := l.client.Bucket(containerName) 37 | if err := bucket.Create(l.ctx, projId, nil); err != nil { 38 | if e, ok := err.(*googleapi.Error); ok && e.Code == 409 { 39 | return &Container{ 40 | name: containerName, 41 | client: l.client, 42 | }, nil 43 | } 44 | return nil, err 45 | } 46 | 47 | return &Container{ 48 | name: containerName, 49 | client: l.client, 50 | ctx: l.ctx, 51 | }, nil 52 | } 53 | 54 | // Containers returns a slice of the Container interface, a cursor, and an error. 55 | func (l *Location) Containers(prefix string, cursor string, count int) ([]stow.Container, string, error) { 56 | projId, _ := l.config.Config(ConfigProjectId) 57 | call := l.client.Buckets(l.ctx, projId) 58 | if prefix != "" { 59 | call.Prefix = prefix 60 | } 61 | 62 | p := iterator.NewPager(call, count, cursor) 63 | var results []*storage.BucketAttrs 64 | nextPageToken, err := p.NextPage(&results) 65 | if err != nil { 66 | return nil, "", err 67 | } 68 | 69 | var containers []stow.Container 70 | for _, container := range results { 71 | containers = append(containers, &Container{ 72 | name: container.Name, 73 | client: l.client, 74 | ctx: l.ctx, 75 | }) 76 | } 77 | 78 | return containers, nextPageToken, nil 79 | } 80 | 81 | // Container retrieves a stow.Container based on its name which must be 82 | // exact. 83 | func (l *Location) Container(id string) (stow.Container, error) { 84 | attrs, err := l.client.Bucket(id).Attrs(l.ctx) 85 | if err != nil { 86 | if err == storage.ErrBucketNotExist { 87 | return nil, stow.ErrNotFound 88 | } 89 | return nil, err 90 | } 91 | 92 | c := &Container{ 93 | name: attrs.Name, 94 | client: l.client, 95 | ctx: l.ctx, 96 | } 97 | 98 | return c, nil 99 | } 100 | 101 | // RemoveContainer removes a container simply by name. 102 | func (l *Location) RemoveContainer(id string) error { 103 | if err := l.client.Bucket(id).Delete(l.ctx); err != nil { 104 | if e, ok := err.(*googleapi.Error); ok && e.Code == 404 { 105 | return stow.ErrNotFound 106 | } 107 | return err 108 | } 109 | 110 | return nil 111 | } 112 | 113 | // ItemByURL retrieves a stow.Item by parsing the URL, in this 114 | // case an item is an object. 115 | func (l *Location) ItemByURL(url *url.URL) (stow.Item, error) { 116 | if url.Scheme != Kind { 117 | return nil, errors.New("not valid google storage URL") 118 | } 119 | 120 | // /download/storage/v1/b/stowtesttoudhratik/o/a_first%2Fthe%20item 121 | pieces := strings.SplitN(url.Path, "/", 8) 122 | 123 | c, err := l.Container(pieces[5]) 124 | if err != nil { 125 | return nil, stow.ErrNotFound 126 | } 127 | 128 | i, err := c.Item(pieces[7]) 129 | if err != nil { 130 | return nil, stow.ErrNotFound 131 | } 132 | 133 | return i, nil 134 | } 135 | -------------------------------------------------------------------------------- /google/stow_test.go: -------------------------------------------------------------------------------- 1 | package google 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/cheekybits/is" 10 | 11 | "github.com/graymeta/stow" 12 | "github.com/graymeta/stow/test" 13 | ) 14 | 15 | func TestStow(t *testing.T) { 16 | 17 | credFile := os.Getenv("GOOGLE_CREDENTIALS_FILE") 18 | projectId := os.Getenv("GOOGLE_PROJECT_ID") 19 | 20 | if credFile == "" || projectId == "" { 21 | t.Skip("skipping test because GOOGLE_CREDENTIALS_FILE or GOOGLE_PROJECT_ID not set.") 22 | } 23 | 24 | b, err := ioutil.ReadFile(credFile) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | 29 | config := stow.ConfigMap{ 30 | "json": string(b), 31 | "project_id": projectId, 32 | } 33 | 34 | test.All(t, "google", config) 35 | } 36 | 37 | func TestPrepMetadataSuccess(t *testing.T) { 38 | is := is.New(t) 39 | 40 | m := make(map[string]string) 41 | m["one"] = "two" 42 | m["3"] = "4" 43 | m["ninety-nine"] = "100" 44 | 45 | m2 := make(map[string]interface{}) 46 | for key, value := range m { 47 | m2[key] = value 48 | } 49 | 50 | //returns map[string]interface 51 | returnedMap, err := prepMetadata(m2) 52 | is.NoErr(err) 53 | 54 | if !reflect.DeepEqual(returnedMap, m) { 55 | t.Errorf("Expected map (%+v) and returned map (%+v) are not equal.", m, returnedMap) 56 | } 57 | } 58 | 59 | func TestPrepMetadataFailureWithNonStringValues(t *testing.T) { 60 | is := is.New(t) 61 | 62 | m := make(map[string]interface{}) 63 | m["float"] = 8.9 64 | m["number"] = 9 65 | 66 | _, err := prepMetadata(m) 67 | is.Err(err) 68 | } 69 | -------------------------------------------------------------------------------- /local/container.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "net/url" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | "github.com/graymeta/stow" 12 | ) 13 | 14 | type container struct { 15 | name string 16 | path string 17 | } 18 | 19 | func (c *container) ID() string { 20 | return c.path 21 | } 22 | 23 | func (c *container) Name() string { 24 | return c.name 25 | } 26 | 27 | func (c *container) URL() *url.URL { 28 | return &url.URL{ 29 | Scheme: "file", 30 | Path: filepath.Clean(c.path), 31 | } 32 | } 33 | 34 | func (c *container) CreateItem(name string) (stow.Item, io.WriteCloser, error) { 35 | path := filepath.Join(c.path, filepath.FromSlash(name)) 36 | item := &item{ 37 | path: path, 38 | contPrefixLen: len(c.path) + 1, 39 | } 40 | f, err := os.Create(path) 41 | if err != nil { 42 | return nil, nil, err 43 | } 44 | return item, f, nil 45 | } 46 | 47 | func (c *container) RemoveItem(id string) error { 48 | return os.Remove(id) 49 | } 50 | 51 | func (c *container) Put(name string, r io.Reader, size int64, metadata map[string]interface{}) (stow.Item, error) { 52 | if len(metadata) > 0 { 53 | return nil, stow.NotSupported("metadata") 54 | } 55 | 56 | path := filepath.Join(c.path, filepath.FromSlash(name)) 57 | item := &item{ 58 | path: path, 59 | contPrefixLen: len(c.path) + 1, 60 | } 61 | err := os.MkdirAll(filepath.Dir(path), 0777) 62 | if err != nil { 63 | return nil, err 64 | } 65 | f, err := os.Create(path) 66 | if err != nil { 67 | return nil, err 68 | } 69 | defer f.Close() 70 | n, err := io.Copy(f, r) 71 | if err != nil { 72 | return nil, err 73 | } 74 | if n != size { 75 | return nil, errors.New("bad size") 76 | } 77 | return item, nil 78 | } 79 | 80 | func (c *container) Items(prefix, cursor string, count int) ([]stow.Item, string, error) { 81 | prefix = filepath.FromSlash(prefix) 82 | files, err := flatdirs(c.path) 83 | if err != nil { 84 | return nil, "", err 85 | } 86 | if cursor != stow.CursorStart { 87 | // seek to the cursor 88 | ok := false 89 | for i, file := range files { 90 | if file.Name() == cursor { 91 | files = files[i:] 92 | ok = true 93 | break 94 | } 95 | } 96 | if !ok { 97 | return nil, "", stow.ErrBadCursor 98 | } 99 | } 100 | if len(files) > count { 101 | cursor = files[count].Name() 102 | files = files[:count] 103 | } else if len(files) <= count { 104 | cursor = "" // end 105 | } 106 | var items []stow.Item 107 | for _, f := range files { 108 | if f.IsDir() { 109 | continue 110 | } 111 | path, err := filepath.Abs(filepath.Join(c.path, f.Name())) 112 | if err != nil { 113 | return nil, "", err 114 | } 115 | if !strings.HasPrefix(f.Name(), prefix) { 116 | continue 117 | } 118 | item := &item{ 119 | path: path, 120 | contPrefixLen: len(c.path) + 1, 121 | } 122 | items = append(items, item) 123 | } 124 | return items, cursor, nil 125 | } 126 | 127 | func (c *container) Item(id string) (stow.Item, error) { 128 | path := id 129 | if !filepath.IsAbs(id) { 130 | path = filepath.Join(c.path, filepath.FromSlash(id)) 131 | } 132 | info, err := os.Stat(path) 133 | if os.IsNotExist(err) { 134 | return nil, stow.ErrNotFound 135 | } 136 | if info.IsDir() { 137 | return nil, errors.New("unexpected directory") 138 | } 139 | _, err = filepath.Rel(c.path, path) 140 | if err != nil { 141 | return nil, err 142 | } 143 | item := &item{ 144 | path: path, 145 | contPrefixLen: len(c.path) + 1, 146 | } 147 | return item, nil 148 | } 149 | 150 | // flatdirs walks the entire tree returning a list of 151 | // os.FileInfo for all items encountered. 152 | func flatdirs(path string) ([]os.FileInfo, error) { 153 | var list []os.FileInfo 154 | err := filepath.Walk(path, func(p string, info os.FileInfo, err error) error { 155 | if err != nil { 156 | return err 157 | } 158 | if info.IsDir() { 159 | return nil 160 | } 161 | flatname, err := filepath.Rel(path, p) 162 | if err != nil { 163 | return err 164 | } 165 | list = append(list, fileinfo{ 166 | FileInfo: info, 167 | name: flatname, 168 | }) 169 | return nil 170 | }) 171 | if err != nil { 172 | return nil, err 173 | } 174 | return list, nil 175 | } 176 | 177 | type fileinfo struct { 178 | os.FileInfo 179 | name string 180 | } 181 | 182 | func (f fileinfo) Name() string { 183 | return f.name 184 | } 185 | -------------------------------------------------------------------------------- /local/container_test.go: -------------------------------------------------------------------------------- 1 | package local_test 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/cheekybits/is" 9 | "github.com/graymeta/stow" 10 | "github.com/graymeta/stow/local" 11 | ) 12 | 13 | func TestItemsPaging(t *testing.T) { 14 | is := is.New(t) 15 | testDir, teardown, err := setup() 16 | is.NoErr(err) 17 | defer teardown() 18 | cfg := stow.ConfigMap{"path": testDir} 19 | l, err := stow.Dial(local.Kind, cfg) 20 | is.NoErr(err) 21 | is.OK(l) 22 | 23 | // get the first actual container to work with (not "All" container) 24 | containers, _, err := l.Containers("", stow.CursorStart, 10) 25 | is.NoErr(err) 26 | is.True(len(containers) > 0) 27 | container := containers[1] 28 | 29 | // make 25 items 30 | for i := 0; i < 25; i++ { 31 | _, err := container.Put(fmt.Sprintf("item-%02d", i), strings.NewReader(`item`), 4, nil) 32 | is.NoErr(err) 33 | } 34 | 35 | // get the first page 36 | items, cursor, err := container.Items("item-", stow.CursorStart, 10) 37 | is.NoErr(err) 38 | is.OK(items) 39 | is.Equal(len(items), 10) 40 | is.OK(cursor) 41 | is.Equal(cursor, "item-10") 42 | 43 | // get the next page 44 | items, cursor, err = container.Items("item-", cursor, 10) 45 | is.NoErr(err) 46 | is.OK(items) 47 | is.Equal(len(items), 10) 48 | is.OK(cursor) 49 | is.Equal(cursor, "item-20") 50 | 51 | // get the last page 52 | items, cursor, err = container.Items("item-", cursor, 10) 53 | is.NoErr(err) 54 | is.OK(items) 55 | is.Equal(len(items), 5) 56 | is.True(stow.IsCursorEnd(cursor)) 57 | 58 | // bad cursor 59 | _, _, err = container.Items("item-", "made up cursor", 10) 60 | is.Equal(err, stow.ErrBadCursor) 61 | 62 | } 63 | -------------------------------------------------------------------------------- /local/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package local provides an abstraction of a general filesystem. A Stow Container is a directory, and a Stow Item is a file. 3 | 4 | Credentials 5 | 6 | The only information required in accessing a filesystem via Stow is the path of a directory. 7 | 8 | Usage 9 | 10 | Aside from providing stow.Dial with the correct Kind ("local"), a stow.Config instance is needed. This instance requires an entry with a key of stow.ConfigKeyPath and a value of the path of the directory. 11 | 12 | Location 13 | 14 | There are local.location methods which allow the retrieval of one or more directories (Container or Containers). A stow.Item representation of a file can also be achieved (ItemByURL). 15 | 16 | Additional methods provide capabilities to create and remove directories (CreateContainer, RemoveContainer). 17 | 18 | Container 19 | 20 | Of a directory, methods of local.container allow the retrieval of its name (ID or Name) as well as one or more files (Item or Items) that exist within. 21 | 22 | Additional local.container methods allow the removal of a file (RemoveItem) and the creation of one (Put). 23 | 24 | Item 25 | 26 | Methods of local.Item allow the retrieval of quite detailed information. They are: 27 | - full path (ID) 28 | - base file name (Name) 29 | - size in bytes (Size) 30 | - file metadata (path, inode, directory, permission bits, etc) 31 | - last modified date (ETag for string, LastMod for time.Time) 32 | - content (Open) 33 | */ 34 | package local 35 | -------------------------------------------------------------------------------- /local/filedata_darwin.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "syscall" 9 | "time" 10 | ) 11 | 12 | func getFileMetadata(path string, info os.FileInfo) map[string]interface{} { 13 | 14 | hardlink := false 15 | symlink := false 16 | var linkTarget string 17 | var inodedata interface{} 18 | if inode, err := getInodeinfo(info); err != nil { 19 | inodedata = map[string]interface{}{"error": err.Error()} 20 | } else { 21 | inodedata = inode 22 | if inode.NLink > 1 { 23 | hardlink = true 24 | } 25 | } 26 | if info.Mode()&os.ModeSymlink == os.ModeSymlink { 27 | symlink = true 28 | linkTarget, _ = os.Readlink(path) 29 | } 30 | m := map[string]interface{}{ 31 | MetadataPath: filepath.Clean(path), 32 | MetadataIsDir: info.IsDir(), 33 | MetadataDir: filepath.Dir(path), 34 | MetadataName: info.Name(), 35 | MetadataMode: fmt.Sprintf("%o", info.Mode()), 36 | MetadataModeD: fmt.Sprintf("%v", uint32(info.Mode())), 37 | MetadataPerm: info.Mode().String(), 38 | MetadataINode: inodedata, 39 | MetadataSize: info.Size(), 40 | MetadataIsHardlink: hardlink, 41 | MetadataIsSymlink: symlink, 42 | MetadataLink: linkTarget, 43 | } 44 | 45 | if stat := info.Sys().(*syscall.Stat_t); stat != nil { 46 | m["atime"] = time.Unix(int64(stat.Atimespec.Sec), int64(stat.Atimespec.Nsec)).Format(time.RFC3339Nano) 47 | m["mtime"] = time.Unix(int64(stat.Mtimespec.Sec), int64(stat.Mtimespec.Nsec)).Format(time.RFC3339Nano) 48 | m["uid"] = stat.Uid 49 | m["gid"] = stat.Gid 50 | } 51 | 52 | ext := filepath.Ext(info.Name()) 53 | if len(ext) > 0 { 54 | m["ext"] = ext 55 | } 56 | 57 | return m 58 | } 59 | 60 | type inodeinfo struct { 61 | // NLink is the number of times this file is linked to by 62 | // hardlinks. 63 | NLink uint16 64 | // Ino is the inode number for the file. 65 | Ino uint64 66 | } 67 | 68 | func getInodeinfo(fi os.FileInfo) (*inodeinfo, error) { 69 | var statT *syscall.Stat_t 70 | var ok bool 71 | if statT, ok = fi.Sys().(*syscall.Stat_t); !ok { 72 | return nil, errors.New("unable to determine if file is a hardlink (expected syscall.Stat_t)") 73 | } 74 | return &inodeinfo{ 75 | Ino: statT.Ino, 76 | NLink: uint16(statT.Nlink), 77 | }, nil 78 | } 79 | -------------------------------------------------------------------------------- /local/filedata_linux.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "syscall" 9 | "time" 10 | ) 11 | 12 | func getFileMetadata(path string, info os.FileInfo) map[string]interface{} { 13 | 14 | hardlink := false 15 | symlink := false 16 | var linkTarget string 17 | var inodedata interface{} 18 | if inode, err := getInodeinfo(info); err != nil { 19 | inodedata = map[string]interface{}{"error": err.Error()} 20 | } else { 21 | inodedata = inode 22 | if inode.NLink > 1 { 23 | hardlink = true 24 | } 25 | } 26 | if info.Mode()&os.ModeSymlink == os.ModeSymlink { 27 | symlink = true 28 | linkTarget, _ = os.Readlink(path) 29 | } 30 | m := map[string]interface{}{ 31 | MetadataPath: filepath.Clean(path), 32 | MetadataIsDir: info.IsDir(), 33 | MetadataDir: filepath.Dir(path), 34 | MetadataName: info.Name(), 35 | MetadataMode: fmt.Sprintf("%o", info.Mode()), 36 | MetadataModeD: fmt.Sprintf("%v", uint32(info.Mode())), 37 | MetadataPerm: info.Mode().String(), 38 | MetadataINode: inodedata, 39 | MetadataSize: info.Size(), 40 | MetadataIsHardlink: hardlink, 41 | MetadataIsSymlink: symlink, 42 | MetadataLink: linkTarget, 43 | } 44 | 45 | if stat := info.Sys().(*syscall.Stat_t); stat != nil { 46 | m["atime"] = time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec)).Format(time.RFC3339Nano) 47 | m["mtime"] = time.Unix(int64(stat.Mtim.Sec), int64(stat.Mtim.Nsec)).Format(time.RFC3339Nano) 48 | m["uid"] = stat.Uid 49 | m["gid"] = stat.Gid 50 | } 51 | 52 | ext := filepath.Ext(info.Name()) 53 | if len(ext) > 0 { 54 | m["ext"] = ext 55 | } 56 | 57 | return m 58 | } 59 | 60 | type inodeinfo struct { 61 | // NLink is the number of times this file is linked to by 62 | // hardlinks. 63 | NLink uint64 64 | // Ino is the inode number for the file. 65 | Ino uint64 66 | } 67 | 68 | func getInodeinfo(fi os.FileInfo) (*inodeinfo, error) { 69 | var statT *syscall.Stat_t 70 | var ok bool 71 | if statT, ok = fi.Sys().(*syscall.Stat_t); !ok { 72 | return nil, errors.New("unable to determine if file is a hardlink (expected syscall.Stat_t)") 73 | } 74 | return &inodeinfo{ 75 | Ino: statT.Ino, 76 | NLink: uint64(statT.Nlink), 77 | }, nil 78 | } 79 | -------------------------------------------------------------------------------- /local/filedata_test.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "os" 5 | "runtime" 6 | "testing" 7 | 8 | "github.com/cheekybits/is" 9 | ) 10 | 11 | func TestFileData(t *testing.T) { 12 | is := is.New(t) 13 | 14 | info, err := os.Stat("./filedata_test.go") 15 | is.NoErr(err) 16 | is.OK(info) 17 | 18 | data := getFileMetadata("./filedata_test.go", info) 19 | 20 | is.Equal(data["is_dir"], false) 21 | is.Equal(data["ext"], ".go") 22 | is.Equal(data["path"], "filedata_test.go") 23 | is.Equal(data["name"], "filedata_test.go") 24 | is.True(data["mode"] == "644" || data["mode"] == "664" || data["mode"] == "666") 25 | is.True(data["mode_d"] == "420" || data["mode_d"] == "436" || data["mode_d"] == "438") 26 | is.True(data["perm"] == "-rw-r--r--" || data["perm"] == "-rw-rw-rw-" || data["perm"] == "-rw-rw-r--") 27 | if runtime.GOOS != "windows" { 28 | is.OK(data["inode"]) 29 | } 30 | is.False(data["is_hardlink"]) 31 | is.False(data["is_symlink"]) 32 | is.OK(data["size"]) 33 | 34 | } 35 | 36 | func TestDotFile(t *testing.T) { 37 | is := is.New(t) 38 | 39 | info, err := os.Stat("./testdata/.dotfile") 40 | is.NoErr(err) 41 | is.OK(info) 42 | 43 | data := getFileMetadata("./testdata/.dotfile", info) 44 | 45 | is.Equal(data["ext"], ".dotfile") 46 | is.Equal(data["name"], ".dotfile") 47 | } 48 | -------------------------------------------------------------------------------- /local/filedata_windows.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "syscall" 8 | "time" 9 | ) 10 | 11 | func getFileMetadata(path string, info os.FileInfo) map[string]interface{} { 12 | hardlink, symlink, linkTarget := false, false, "" 13 | if info.Mode()&os.ModeSymlink == os.ModeSymlink { 14 | symlink = true 15 | linkTarget, _ = os.Readlink(path) 16 | } 17 | m := map[string]interface{}{ 18 | MetadataPath: filepath.Clean(path), 19 | MetadataIsDir: info.IsDir(), 20 | MetadataDir: filepath.Dir(path), 21 | MetadataName: info.Name(), 22 | MetadataMode: fmt.Sprintf("%o", info.Mode()), 23 | MetadataModeD: fmt.Sprintf("%v", uint32(info.Mode())), 24 | MetadataPerm: info.Mode().String(), 25 | MetadataSize: info.Size(), 26 | MetadataIsSymlink: symlink, 27 | MetadataIsHardlink: hardlink, 28 | MetadataLink: linkTarget, 29 | } 30 | 31 | if stat := info.Sys().(*syscall.Win32FileAttributeData); stat != nil { 32 | m["atime"] = time.Unix(0, stat.LastAccessTime.Nanoseconds()).Format(time.RFC3339Nano) 33 | m["mtime"] = time.Unix(0, stat.LastWriteTime.Nanoseconds()).Format(time.RFC3339Nano) 34 | } 35 | 36 | ext := filepath.Ext(info.Name()) 37 | if len(ext) > 0 { 38 | m["ext"] = ext 39 | } 40 | 41 | return m 42 | } 43 | -------------------------------------------------------------------------------- /local/item.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "io" 5 | "net/url" 6 | "os" 7 | "path/filepath" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | // Metadata constants describe the metadata available 13 | // for a local Item. 14 | const ( 15 | MetadataPath = "path" 16 | MetadataIsDir = "is_dir" 17 | MetadataDir = "dir" 18 | MetadataName = "name" 19 | MetadataMode = "mode" 20 | MetadataModeD = "mode_d" 21 | MetadataPerm = "perm" 22 | MetadataINode = "inode" 23 | MetadataSize = "size" 24 | MetadataIsHardlink = "is_hardlink" 25 | MetadataIsSymlink = "is_symlink" 26 | MetadataLink = "link" 27 | ) 28 | 29 | type item struct { 30 | path string 31 | contPrefixLen int 32 | infoOnce sync.Once // protects info 33 | info os.FileInfo 34 | infoErr error 35 | metadata map[string]interface{} 36 | } 37 | 38 | func (i *item) ID() string { 39 | return i.path 40 | } 41 | 42 | func (i *item) Name() string { 43 | return filepath.ToSlash(i.path[i.contPrefixLen:]) 44 | } 45 | 46 | func (i *item) Size() (int64, error) { 47 | err := i.ensureInfo() 48 | if err != nil { 49 | return 0, err 50 | } 51 | return i.info.Size(), nil 52 | } 53 | 54 | func (i *item) URL() *url.URL { 55 | return &url.URL{ 56 | Scheme: "file", 57 | Path: filepath.Clean(i.path), 58 | } 59 | } 60 | 61 | func (i *item) ETag() (string, error) { 62 | err := i.ensureInfo() 63 | if err != nil { 64 | return "", nil 65 | } 66 | return i.info.ModTime().String(), nil 67 | } 68 | 69 | // Open opens the file for reading. 70 | func (i *item) Open() (io.ReadCloser, error) { 71 | return os.Open(i.path) 72 | } 73 | 74 | func (i *item) LastMod() (time.Time, error) { 75 | err := i.ensureInfo() 76 | if err != nil { 77 | return time.Time{}, nil 78 | } 79 | 80 | return i.info.ModTime(), nil 81 | } 82 | 83 | func (i *item) ensureInfo() error { 84 | i.infoOnce.Do(func() { 85 | i.info, i.infoErr = os.Lstat(i.path) // retrieve item file info 86 | 87 | if i.infoErr != nil { 88 | return 89 | } 90 | i.setMetadata(i.info) // merge file and metadata maps 91 | }) 92 | return i.infoErr 93 | } 94 | 95 | func (i *item) setMetadata(info os.FileInfo) { 96 | fileMetadata := getFileMetadata(i.path, info) // retrieve file metadata 97 | i.metadata = fileMetadata 98 | } 99 | 100 | // Metadata gets stat information for the file. 101 | func (i *item) Metadata() (map[string]interface{}, error) { 102 | err := i.ensureInfo() 103 | if err != nil { 104 | return nil, err 105 | } 106 | return i.metadata, nil 107 | } 108 | -------------------------------------------------------------------------------- /local/item_test.go: -------------------------------------------------------------------------------- 1 | package local_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/url" 6 | "os" 7 | "path/filepath" 8 | "runtime" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/cheekybits/is" 13 | "github.com/graymeta/stow" 14 | "github.com/graymeta/stow/local" 15 | ) 16 | 17 | func TestItemReader(t *testing.T) { 18 | is := is.New(t) 19 | testDir, teardown, err := setup() 20 | is.NoErr(err) 21 | defer teardown() 22 | 23 | cfg := stow.ConfigMap{"path": testDir} 24 | l, err := stow.Dial(local.Kind, cfg) 25 | is.NoErr(err) 26 | is.OK(l) 27 | containers, cursor, err := l.Containers("t", stow.CursorStart, 10) 28 | is.NoErr(err) 29 | is.OK(containers) 30 | is.Equal(cursor, "") 31 | three, err := l.Container(containers[0].ID()) 32 | is.NoErr(err) 33 | 34 | items, cursor, err := three.Items("", stow.CursorStart, 10) 35 | is.NoErr(err) 36 | is.Equal(cursor, "") 37 | item1 := items[0] 38 | 39 | rc, err := item1.Open() 40 | is.NoErr(err) 41 | defer rc.Close() 42 | b, err := ioutil.ReadAll(rc) 43 | is.NoErr(err) 44 | is.Equal("3.1", string(b)) 45 | 46 | } 47 | 48 | func TestHardlink(t *testing.T) { 49 | if runtime.GOOS == "windows" { 50 | t.SkipNow() 51 | } 52 | is := is.New(t) 53 | testDir, teardown, err := setup() 54 | is.NoErr(err) 55 | defer teardown() 56 | 57 | cfg := stow.ConfigMap{"path": testDir} 58 | l, err := stow.Dial(local.Kind, cfg) 59 | is.NoErr(err) 60 | is.OK(l) 61 | 62 | containers, cursor, err := l.Containers("z", stow.CursorStart, 10) 63 | is.NoErr(err) 64 | is.OK(containers) 65 | is.Equal(cursor, "") 66 | 67 | links, err := l.Container(containers[0].ID()) 68 | is.NoErr(err) 69 | 70 | items, cursor, err := links.Items("", stow.CursorStart, 10) 71 | is.NoErr(err) 72 | is.Equal(cursor, "") 73 | 74 | for _, item := range items { 75 | if item.Name() == "hardlink" { 76 | meta, err := item.Metadata() 77 | is.NoErr(err) 78 | is.OK(meta) 79 | 80 | is.Equal(meta["is_dir"], false) 81 | is.True(meta["is_hardlink"]) 82 | is.False(meta["is_symlink"]) 83 | break 84 | } 85 | } 86 | } 87 | 88 | func TestSymLink(t *testing.T) { 89 | is := is.New(t) 90 | testDir, teardown, err := setup() 91 | is.NoErr(err) 92 | defer teardown() 93 | 94 | cfg := stow.ConfigMap{"path": testDir} 95 | l, err := stow.Dial(local.Kind, cfg) 96 | is.NoErr(err) 97 | is.OK(l) 98 | 99 | containers, cursor, err := l.Containers("z", stow.CursorStart, 10) 100 | is.NoErr(err) 101 | is.OK(containers) 102 | is.Equal(cursor, "") 103 | 104 | links, err := l.Container(containers[0].ID()) 105 | is.NoErr(err) 106 | 107 | items, cursor, err := links.Items("", stow.CursorStart, 10) 108 | is.NoErr(err) 109 | is.Equal(cursor, "") 110 | 111 | for _, item := range items { 112 | if item.Name() == "symlink" { 113 | meta, err := item.Metadata() 114 | is.NoErr(err) 115 | is.OK(meta) 116 | 117 | is.Equal(meta["is_dir"], false) 118 | is.False(meta["is_hardlink"]) 119 | is.True(meta["is_symlink"]) 120 | 121 | linkStr, ok := meta["link"].(string) 122 | is.True(ok) 123 | is.OK(linkStr) 124 | 125 | is.True(strings.Contains(linkStr, "symtarget")) 126 | break 127 | } 128 | } 129 | } 130 | 131 | func TestItemFromURL(t *testing.T) { 132 | //A file-system path /a/b/c can be broken into (container, name) 133 | //either as (/a, b/c) or as (/a/b,c). This test imposes 134 | //the convention that the name is the last path component. 135 | is := is.New(t) 136 | testDir, teardown, err := setup() 137 | is.NoErr(err) 138 | defer teardown() 139 | os.MkdirAll(filepath.Join(testDir, "a", "b"), 0777) 140 | ioutil.WriteFile(filepath.Join(testDir, "a", "f2"), []byte("abc"), 0666) 141 | ioutil.WriteFile(filepath.Join(testDir, "a", "b", "f3"), []byte("abc"), 0666) 142 | 143 | cfg := stow.ConfigMap{"path": testDir} 144 | l, err := stow.Dial(local.Kind, cfg) 145 | is.NoErr(err) 146 | 147 | item2, err := l.ItemByURL(&url.URL{ 148 | Scheme: "file", 149 | Path: filepath.Join(testDir, "a", "f2"), 150 | }) 151 | is.NoErr(err) 152 | is.Equal(item2.Name(), "f2") 153 | 154 | item3, err := l.ItemByURL(&url.URL{ 155 | Scheme: "file", 156 | Path: filepath.Join(testDir, "a", "b", "f3"), 157 | }) 158 | is.NoErr(err) 159 | is.Equal(item3.Name(), "f3") 160 | } 161 | -------------------------------------------------------------------------------- /local/local.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | "os" 7 | 8 | "github.com/graymeta/stow" 9 | ) 10 | 11 | // ConfigKeys are the supported configuration items for 12 | // local storage. 13 | const ( 14 | ConfigKeyPath = "path" 15 | ) 16 | 17 | // Kind is the kind of Location this package provides. 18 | const Kind = "local" 19 | 20 | const ( 21 | paramTypeValue = "item" 22 | ) 23 | 24 | func init() { 25 | validatefn := func(config stow.Config) error { 26 | _, ok := config.Config(ConfigKeyPath) 27 | if !ok { 28 | return errors.New("missing path config") 29 | } 30 | return nil 31 | } 32 | makefn := func(config stow.Config) (stow.Location, error) { 33 | path, ok := config.Config(ConfigKeyPath) 34 | if !ok { 35 | return nil, errors.New("missing path config") 36 | } 37 | info, err := os.Stat(path) 38 | if err != nil { 39 | return nil, err 40 | } 41 | if !info.IsDir() { 42 | return nil, errors.New("path must be directory") 43 | } 44 | return &location{ 45 | config: config, 46 | }, nil 47 | } 48 | kindfn := func(u *url.URL) bool { 49 | return u.Scheme == "file" 50 | } 51 | stow.Register(Kind, makefn, kindfn, validatefn) 52 | } 53 | -------------------------------------------------------------------------------- /local/location.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/graymeta/stow" 10 | ) 11 | 12 | type location struct { 13 | // config is the configuration for this location. 14 | config stow.Config 15 | } 16 | 17 | func (l *location) Close() error { 18 | return nil // nothing to close 19 | } 20 | 21 | func (l *location) ItemByURL(u *url.URL) (stow.Item, error) { 22 | dir, _ := filepath.Split(u.Path) 23 | return &item{ 24 | path: u.Path, 25 | contPrefixLen: len(dir), 26 | }, nil 27 | } 28 | 29 | func (l *location) RemoveContainer(id string) error { 30 | return os.RemoveAll(id) 31 | } 32 | 33 | func (l *location) CreateContainer(name string) (stow.Container, error) { 34 | path, ok := l.config.Config(ConfigKeyPath) 35 | if !ok { 36 | return nil, errors.New("missing " + ConfigKeyPath + " configuration") 37 | } 38 | fullpath := filepath.Join(path, name) 39 | if err := os.Mkdir(fullpath, 0777); err != nil { 40 | return nil, err 41 | } 42 | abspath, err := filepath.Abs(fullpath) 43 | if err != nil { 44 | return nil, err 45 | } 46 | return &container{ 47 | name: name, 48 | path: abspath, 49 | }, nil 50 | } 51 | 52 | func (l *location) Containers(prefix string, cursor string, count int) ([]stow.Container, string, error) { 53 | path, ok := l.config.Config(ConfigKeyPath) 54 | if !ok { 55 | return nil, "", errors.New("missing " + ConfigKeyPath + " configuration") 56 | } 57 | files, err := filepath.Glob(filepath.Join(path, prefix+"*")) 58 | if err != nil { 59 | return nil, "", err 60 | } 61 | 62 | var cs []stow.Container 63 | 64 | if prefix == stow.NoPrefix && cursor == stow.CursorStart { 65 | allContainer := container{ 66 | name: "All", 67 | path: path, 68 | } 69 | 70 | cs = append(cs, &allContainer) 71 | } 72 | 73 | cc, err := l.filesToContainers(path, files...) 74 | if err != nil { 75 | return nil, "", err 76 | } 77 | 78 | cs = append(cs, cc...) 79 | 80 | if cursor != stow.CursorStart { 81 | // seek to the cursor 82 | ok := false 83 | for i, c := range cs { 84 | if c.ID() == cursor { 85 | ok = true 86 | cs = cs[i:] 87 | break 88 | } 89 | } 90 | if !ok { 91 | return nil, "", stow.ErrBadCursor 92 | } 93 | } 94 | if len(cs) > count { 95 | cursor = cs[count].ID() 96 | cs = cs[:count] // limit cs to count 97 | } else if len(cs) <= count { 98 | cursor = "" 99 | } 100 | 101 | return cs, cursor, err 102 | } 103 | 104 | func (l *location) Container(id string) (stow.Container, error) { 105 | path, ok := l.config.Config(ConfigKeyPath) 106 | if !ok { 107 | return nil, errors.New("missing " + ConfigKeyPath + " configuration") 108 | } 109 | var fullPath string 110 | if filepath.IsAbs(id) { 111 | fullPath = id 112 | } else { 113 | fullPath = filepath.Join(path, id) 114 | } 115 | 116 | containers, err := l.filesToContainers(path, fullPath) 117 | if err != nil { 118 | if os.IsNotExist(err) { 119 | return nil, stow.ErrNotFound 120 | } 121 | return nil, err 122 | } 123 | if len(containers) == 0 { 124 | return nil, stow.ErrNotFound 125 | } 126 | return containers[0], nil 127 | } 128 | 129 | // filesToContainers takes a list of files and turns it into a 130 | // stow.ContainerList. 131 | func (l *location) filesToContainers(root string, files ...string) ([]stow.Container, error) { 132 | cs := make([]stow.Container, 0, len(files)) 133 | for _, f := range files { 134 | info, err := os.Stat(f) 135 | if err != nil { 136 | return nil, err 137 | } 138 | if !info.IsDir() { 139 | continue 140 | } 141 | absroot, err := filepath.Abs(root) 142 | if err != nil { 143 | return nil, err 144 | } 145 | path, err := filepath.Abs(f) 146 | if err != nil { 147 | return nil, err 148 | } 149 | name, err := filepath.Rel(absroot, path) 150 | if err != nil { 151 | return nil, err 152 | } 153 | cs = append(cs, &container{ 154 | name: name, 155 | path: path, 156 | }) 157 | } 158 | return cs, nil 159 | } 160 | -------------------------------------------------------------------------------- /local/location_test.go: -------------------------------------------------------------------------------- 1 | package local_test 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/cheekybits/is" 9 | "github.com/graymeta/stow" 10 | "github.com/graymeta/stow/local" 11 | ) 12 | 13 | func TestContainers(t *testing.T) { 14 | is := is.New(t) 15 | testDir, teardown, err := setup() 16 | is.NoErr(err) 17 | defer teardown() 18 | 19 | cfg := stow.ConfigMap{"path": testDir} 20 | 21 | l, err := stow.Dial(local.Kind, cfg) 22 | is.NoErr(err) 23 | is.OK(l) 24 | 25 | items, cursor, err := l.Containers("", stow.CursorStart, 10) 26 | is.NoErr(err) 27 | is.Equal(cursor, "") 28 | is.OK(items) 29 | 30 | is.Equal(len(items), 5) 31 | isDir(is, items[0].ID()) 32 | is.Equal(items[0].Name(), "All") 33 | isDir(is, items[1].ID()) 34 | is.Equal(items[1].Name(), "one") 35 | isDir(is, items[2].ID()) 36 | is.Equal(items[2].Name(), "three") 37 | isDir(is, items[3].ID()) 38 | is.Equal(items[3].Name(), "two") 39 | } 40 | 41 | func TestAllContainer(t *testing.T) { 42 | is := is.New(t) 43 | 44 | testDir, teardown, err := setup() 45 | is.NoErr(err) 46 | defer teardown() 47 | 48 | cfg := stow.ConfigMap{"path": testDir} 49 | 50 | l, err := stow.Dial(local.Kind, cfg) 51 | is.NoErr(err) 52 | is.OK(l) 53 | 54 | containers, cursor, err := l.Containers("", stow.CursorStart, 10) 55 | is.NoErr(err) 56 | is.Equal(cursor, "") 57 | is.OK(containers) 58 | 59 | is.Equal(containers[0].Name(), "All") 60 | 61 | items, cursor, err := containers[0].Items("root", stow.CursorStart, 10) 62 | is.Equal(cursor, "") 63 | is.OK(items) 64 | is.NoErr(err) 65 | is.Equal(len(items), 1) 66 | } 67 | 68 | func TestContainersPrefix(t *testing.T) { 69 | is := is.New(t) 70 | testDir, teardown, err := setup() 71 | is.NoErr(err) 72 | defer teardown() 73 | 74 | cfg := stow.ConfigMap{"path": testDir} 75 | 76 | l, err := stow.Dial(local.Kind, cfg) 77 | is.NoErr(err) 78 | is.OK(l) 79 | 80 | containers, cursor, err := l.Containers("t", stow.CursorStart, 10) 81 | is.NoErr(err) 82 | is.OK(containers) 83 | is.Equal(cursor, "") 84 | 85 | is.Equal(len(containers), 2) 86 | isDir(is, containers[0].ID()) 87 | is.Equal(containers[0].Name(), "three") 88 | isDir(is, containers[1].ID()) 89 | is.Equal(containers[1].Name(), "two") 90 | 91 | cthree, err := l.Container(containers[0].ID()) 92 | is.NoErr(err) 93 | is.Equal(cthree.Name(), "three") 94 | } 95 | 96 | func TestContainer(t *testing.T) { 97 | is := is.New(t) 98 | testDir, teardown, err := setup() 99 | is.NoErr(err) 100 | defer teardown() 101 | 102 | cfg := stow.ConfigMap{"path": testDir} 103 | 104 | l, err := stow.Dial(local.Kind, cfg) 105 | is.NoErr(err) 106 | is.OK(l) 107 | 108 | containers, cursor, err := l.Containers("t", stow.CursorStart, 10) 109 | is.NoErr(err) 110 | is.OK(containers) 111 | is.Equal(cursor, "") 112 | 113 | is.Equal(len(containers), 2) 114 | isDir(is, containers[0].ID()) 115 | 116 | cthree, err := l.Container(containers[0].ID()) 117 | is.NoErr(err) 118 | is.Equal(cthree.Name(), "three") 119 | 120 | } 121 | 122 | func TestCreateContainer(t *testing.T) { 123 | is := is.New(t) 124 | testDir, teardown, err := setup() 125 | is.NoErr(err) 126 | defer teardown() 127 | 128 | cfg := stow.ConfigMap{"path": testDir} 129 | 130 | l, err := stow.Dial(local.Kind, cfg) 131 | is.NoErr(err) 132 | is.OK(l) 133 | 134 | c, err := l.CreateContainer("new_test_container") 135 | is.NoErr(err) 136 | is.OK(c) 137 | is.Equal(c.ID(), filepath.Join(testDir, "new_test_container")) 138 | is.Equal(c.Name(), "new_test_container") 139 | 140 | containers, cursor, err := l.Containers("new", stow.CursorStart, 10) 141 | is.NoErr(err) 142 | is.OK(containers) 143 | is.Equal(cursor, "") 144 | 145 | is.Equal(len(containers), 1) 146 | isDir(is, containers[0].ID()) 147 | is.Equal(containers[0].Name(), "new_test_container") 148 | } 149 | 150 | func TestByURL(t *testing.T) { 151 | is := is.New(t) 152 | testDir, teardown, err := setup() 153 | is.NoErr(err) 154 | defer teardown() 155 | 156 | cfg := stow.ConfigMap{"path": testDir} 157 | 158 | l, err := stow.Dial(local.Kind, cfg) 159 | is.NoErr(err) 160 | is.OK(l) 161 | 162 | containers, cursor, err := l.Containers("t", stow.CursorStart, 10) 163 | is.NoErr(err) 164 | is.OK(containers) 165 | is.Equal(cursor, "") 166 | 167 | three, err := l.Container(containers[0].ID()) 168 | is.NoErr(err) 169 | items, cursor, err := three.Items("", stow.CursorStart, 10) 170 | is.NoErr(err) 171 | is.OK(items) 172 | is.Equal(cursor, "") 173 | is.Equal(len(items), 3) 174 | 175 | item1 := items[0] 176 | 177 | // make sure we know the kind by URL 178 | kind, err := stow.KindByURL(item1.URL()) 179 | is.NoErr(err) 180 | is.Equal(kind, local.Kind) 181 | 182 | i, err := l.ItemByURL(item1.URL()) 183 | is.NoErr(err) 184 | is.OK(i) 185 | is.Equal(i.ID(), item1.ID()) 186 | is.Equal(i.Name(), item1.Name()) 187 | is.Equal(i.URL().String(), item1.URL().String()) 188 | 189 | } 190 | 191 | func TestContainersPaging(t *testing.T) { 192 | is := is.New(t) 193 | testDir, teardown, err := setup() 194 | is.NoErr(err) 195 | defer teardown() 196 | cfg := stow.ConfigMap{"path": testDir} 197 | l, err := stow.Dial(local.Kind, cfg) 198 | is.NoErr(err) 199 | is.OK(l) 200 | 201 | for i := 0; i < 25; i++ { 202 | _, err := l.CreateContainer(fmt.Sprintf("container-%02d", i)) 203 | is.NoErr(err) 204 | } 205 | 206 | // get the first page 207 | containers, cursor, err := l.Containers("container-", stow.CursorStart, 10) 208 | is.NoErr(err) 209 | is.OK(containers) 210 | is.Equal(len(containers), 10) 211 | is.OK(cursor) 212 | is.Equal(filepath.Base(cursor), "container-10") 213 | 214 | // get next page 215 | containers, cursor, err = l.Containers("container-", cursor, 10) 216 | is.NoErr(err) 217 | is.OK(containers) 218 | is.Equal(len(containers), 10) 219 | is.OK(cursor) 220 | is.Equal(filepath.Base(cursor), "container-20") 221 | 222 | // get last page 223 | containers, cursor, err = l.Containers("container-", cursor, 10) 224 | is.NoErr(err) 225 | is.OK(containers) 226 | is.Equal(len(containers), 5) 227 | is.True(stow.IsCursorEnd(cursor)) 228 | 229 | // bad cursor 230 | _, _, err = l.Containers("container-", "made-up-cursor", 10) 231 | is.Equal(err, stow.ErrBadCursor) 232 | 233 | } 234 | -------------------------------------------------------------------------------- /local/stow_test.go: -------------------------------------------------------------------------------- 1 | package local_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "testing" 7 | 8 | "github.com/cheekybits/is" 9 | "github.com/graymeta/stow" 10 | "github.com/graymeta/stow/test" 11 | ) 12 | 13 | func TestStow(t *testing.T) { 14 | is := is.New(t) 15 | 16 | dir, err := ioutil.TempDir("testdata", "stow") 17 | is.NoErr(err) 18 | defer os.RemoveAll(dir) 19 | cfg := stow.ConfigMap{"path": dir} 20 | 21 | test.All(t, "local", cfg) 22 | } 23 | -------------------------------------------------------------------------------- /local/testdata/.dotfile: -------------------------------------------------------------------------------- 1 | This file is used to test dot files. Do not remove. -------------------------------------------------------------------------------- /local/testdata/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WasabiAiR/stow/973a61f346d598a566affb53c4698764a67df164/local/testdata/.gitkeep -------------------------------------------------------------------------------- /local/util_test.go: -------------------------------------------------------------------------------- 1 | package local_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/cheekybits/is" 11 | "github.com/graymeta/stow" 12 | "github.com/graymeta/stow/local" 13 | ) 14 | 15 | func setup() (string, func() error, error) { 16 | done := func() error { return nil } // noop 17 | dir, err := ioutil.TempDir("testdata", "stow") 18 | if err != nil { 19 | return dir, done, err 20 | } 21 | done = func() error { 22 | return os.RemoveAll(dir) 23 | } 24 | // add some "containers" 25 | err = os.Mkdir(filepath.Join(dir, "one"), 0777) 26 | if err != nil { 27 | return dir, done, err 28 | } 29 | err = os.Mkdir(filepath.Join(dir, "two"), 0777) 30 | if err != nil { 31 | return dir, done, err 32 | } 33 | err = os.Mkdir(filepath.Join(dir, "three"), 0777) 34 | if err != nil { 35 | return dir, done, err 36 | } 37 | 38 | // add three items 39 | err = ioutil.WriteFile(filepath.Join(dir, "three", "item1"), []byte("3.1"), 0777) 40 | if err != nil { 41 | return dir, done, err 42 | } 43 | err = ioutil.WriteFile(filepath.Join(dir, "three", "item2"), []byte("3.2"), 0777) 44 | if err != nil { 45 | return dir, done, err 46 | } 47 | err = ioutil.WriteFile(filepath.Join(dir, "three", "item3"), []byte("3.3"), 0777) 48 | if err != nil { 49 | return dir, done, err 50 | } 51 | 52 | // make symlinks and hardlinks 53 | 54 | // make separate "container" for links 55 | // naming it with "z-" prefix, so other tests that depend on container order do not fail 56 | err = os.Mkdir(filepath.Join(dir, "z-links"), 0777) 57 | if err != nil { 58 | return dir, done, err 59 | } 60 | // make sym- and hardlink targets 61 | err = ioutil.WriteFile(filepath.Join(dir, "z-links", "symtarget"), []byte("symlink target"), 0777) 62 | if err != nil { 63 | return dir, done, err 64 | } 65 | err = ioutil.WriteFile(filepath.Join(dir, "z-links", "hardtarget"), []byte("hardlink target"), 0777) 66 | if err != nil { 67 | return dir, done, err 68 | } 69 | 70 | // make hard- and softlinks themselves 71 | err = os.Symlink(filepath.Join(dir, "z-links", "symtarget"), filepath.Join(dir, "z-links", "symlink")) 72 | if err != nil { 73 | return dir, done, err 74 | } 75 | err = os.Link(filepath.Join(dir, "z-links", "hardtarget"), filepath.Join(dir, "z-links", "hardlink")) 76 | if err != nil { 77 | return dir, done, err 78 | } 79 | 80 | // make some root item 81 | err = ioutil.WriteFile(filepath.Join(dir, "rootitem"), []byte("root target"), 0777) 82 | if err != nil { 83 | return dir, done, err 84 | } 85 | 86 | // make testpath absolute 87 | absdir, err := filepath.Abs(dir) 88 | if err != nil { 89 | return dir, done, err 90 | } 91 | return absdir, done, nil 92 | } 93 | 94 | func TestCreateItem(t *testing.T) { 95 | is := is.New(t) 96 | testDir, teardown, err := setup() 97 | is.NoErr(err) 98 | defer teardown() 99 | 100 | cfg := stow.ConfigMap{"path": testDir} 101 | l, err := stow.Dial(local.Kind, cfg) 102 | is.NoErr(err) 103 | is.OK(l) 104 | 105 | containers, cursor, err := l.Containers("t", stow.CursorStart, 10) 106 | is.NoErr(err) 107 | is.OK(containers) 108 | c1 := containers[0] 109 | items, cursor, err := c1.Items("", stow.CursorStart, 10) 110 | is.NoErr(err) 111 | is.Equal(cursor, "") 112 | beforecount := len(items) 113 | 114 | content := "new item contents" 115 | newitem, err := c1.Put("new_item", strings.NewReader(content), int64(len(content)), nil) 116 | is.NoErr(err) 117 | is.OK(newitem) 118 | is.Equal(newitem.Name(), "new_item") 119 | 120 | // get the container again 121 | containers, cursor, err = l.Containers("t", stow.CursorStart, 10) 122 | is.NoErr(err) 123 | is.OK(containers) 124 | is.Equal(cursor, "") 125 | c1 = containers[0] 126 | items, cursor, err = c1.Items("", stow.CursorStart, 10) 127 | is.NoErr(err) 128 | is.Equal(cursor, "") 129 | aftercount := len(items) 130 | 131 | is.Equal(aftercount, beforecount+1) 132 | 133 | // get new item 134 | item := items[len(items)-1] 135 | etag, err := item.ETag() 136 | is.NoErr(err) 137 | is.OK(etag) 138 | r, err := item.Open() 139 | is.NoErr(err) 140 | defer r.Close() 141 | itemContents, err := ioutil.ReadAll(r) 142 | is.NoErr(err) 143 | is.Equal("new item contents", string(itemContents)) 144 | 145 | } 146 | 147 | func TestItems(t *testing.T) { 148 | is := is.New(t) 149 | testDir, teardown, err := setup() 150 | is.NoErr(err) 151 | defer teardown() 152 | 153 | cfg := stow.ConfigMap{"path": testDir} 154 | 155 | l, err := stow.Dial(local.Kind, cfg) 156 | is.NoErr(err) 157 | is.OK(l) 158 | 159 | containers, cursor, err := l.Containers("t", stow.CursorStart, 10) 160 | is.NoErr(err) 161 | is.OK(containers) 162 | is.Equal(cursor, "") 163 | three, err := l.Container(containers[0].ID()) 164 | is.NoErr(err) 165 | items, cursor, err := three.Items("", stow.CursorStart, 10) 166 | is.NoErr(err) 167 | is.OK(items) 168 | is.Equal(cursor, "") 169 | 170 | is.Equal(len(items), 3) 171 | is.Equal(items[0].ID(), filepath.Join(containers[0].ID(), "item1")) 172 | is.Equal(items[0].Name(), "item1") 173 | } 174 | 175 | func isDir(is is.I, path string) { 176 | info, err := os.Stat(path) 177 | is.NoErr(err) 178 | is.True(info.IsDir()) 179 | } 180 | -------------------------------------------------------------------------------- /location_test.go: -------------------------------------------------------------------------------- 1 | package stow_test 2 | 3 | import ( 4 | "net/url" 5 | 6 | "github.com/graymeta/stow" 7 | ) 8 | 9 | func init() { 10 | validatefn := func(config stow.Config) error { 11 | return nil 12 | } 13 | makefn := func(config stow.Config) (stow.Location, error) { 14 | return &testLocation{ 15 | config: config, 16 | }, nil 17 | } 18 | kindfn := func(u *url.URL) bool { 19 | return u.Scheme == testKind 20 | } 21 | stow.Register(testKind, makefn, kindfn, validatefn) 22 | } 23 | 24 | const testKind = "test" 25 | 26 | type testLocation struct { 27 | config stow.Config 28 | } 29 | 30 | func (l *testLocation) Close() error { 31 | return nil 32 | } 33 | 34 | func (l *testLocation) CreateContainer(name string) (stow.Container, error) { 35 | return nil, nil 36 | } 37 | func (l *testLocation) RemoveContainer(id string) error { 38 | return nil 39 | } 40 | 41 | func (l *testLocation) Container(id string) (stow.Container, error) { 42 | return nil, nil 43 | } 44 | func (l *testLocation) Containers(prefix string, cursor string, count int) ([]stow.Container, string, error) { 45 | return nil, "", nil 46 | } 47 | 48 | func (l *testLocation) ItemByURL(u *url.URL) (stow.Item, error) { 49 | return nil, nil 50 | } 51 | 52 | func (l *testLocation) ContainerByURL(u *url.URL) (stow.Container, error) { 53 | return nil, nil 54 | } 55 | -------------------------------------------------------------------------------- /oracle/config.go: -------------------------------------------------------------------------------- 1 | package oracle 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "strings" 9 | 10 | "github.com/graymeta/stow" 11 | "github.com/ncw/swift" 12 | ) 13 | 14 | const ( 15 | // ConfigUsername is the username associated with the account 16 | ConfigUsername = "username" 17 | 18 | // ConfigPassword is the user password associated with the account 19 | ConfigPassword = "password" 20 | 21 | // ConfigAuthEndpoint is the identity domain associated with the account 22 | ConfigAuthEndpoint = "authorization_endpoint" 23 | ) 24 | 25 | // Kind is the kind of Location this package provides. 26 | const Kind = "oracle" 27 | 28 | func init() { 29 | validatefn := func(config stow.Config) error { 30 | _, ok := config.Config(ConfigUsername) 31 | if !ok { 32 | return errors.New("missing account username") 33 | } 34 | 35 | _, ok = config.Config(ConfigPassword) 36 | if !ok { 37 | return errors.New("missing account password") 38 | } 39 | 40 | _, ok = config.Config(ConfigAuthEndpoint) 41 | if !ok { 42 | return errors.New("missing authorization endpoint") 43 | } 44 | return nil 45 | } 46 | makefn := func(config stow.Config) (stow.Location, error) { 47 | _, ok := config.Config(ConfigUsername) 48 | if !ok { 49 | return nil, errors.New("missing account username") 50 | } 51 | 52 | _, ok = config.Config(ConfigPassword) 53 | if !ok { 54 | return nil, errors.New("missing account password") 55 | } 56 | 57 | _, ok = config.Config(ConfigAuthEndpoint) 58 | if !ok { 59 | return nil, errors.New("missing authorization endpoint") 60 | } 61 | 62 | l := &location{ 63 | config: config, 64 | } 65 | 66 | var err error 67 | l.client, err = newSwiftClient(l.config) 68 | if err != nil { 69 | return nil, err 70 | } 71 | 72 | return l, nil 73 | } 74 | 75 | kindfn := func(u *url.URL) bool { 76 | return u.Scheme == Kind 77 | } 78 | 79 | stow.Register(Kind, makefn, kindfn, validatefn) 80 | } 81 | 82 | func newSwiftClient(cfg stow.Config) (*swift.Connection, error) { 83 | client, err := parseConfig(cfg) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | err = client.Authenticate() 89 | if err != nil { 90 | return nil, errors.New("unable to authenticate") 91 | } 92 | return client, nil 93 | } 94 | 95 | func parseConfig(cfg stow.Config) (*swift.Connection, error) { 96 | cfgUsername, _ := cfg.Config(ConfigUsername) 97 | cfgAuthEndpoint, _ := cfg.Config(ConfigAuthEndpoint) 98 | 99 | // Auth Endpoint contains most of the information needed to make a client, 100 | // find the indexes of the symbols that separate them. 101 | dotIndex := strings.Index(cfgAuthEndpoint, `.`) 102 | if dotIndex == -1 { 103 | return nil, errors.New("stow: oracle: bad format for " + ConfigAuthEndpoint) 104 | } 105 | dashIndex := strings.Index(cfgAuthEndpoint[:dotIndex], `-`) 106 | slashIndex := strings.Index(cfgAuthEndpoint, `//`) + 1 // Add 1 to move index to second slash 107 | 108 | var metered bool 109 | 110 | // metered storage endpoints should not have a dash before the dot index. 111 | if dashIndex == -1 { 112 | metered = true 113 | } 114 | 115 | var swiftTenantName, swiftUsername, instanceName string 116 | var startIndex int 117 | 118 | if metered { 119 | startIndex = slashIndex + 1 120 | instanceName = "Storage" 121 | } else { 122 | startIndex = dashIndex + 1 123 | instanceName = cfgAuthEndpoint[slashIndex+1 : dashIndex] 124 | } 125 | 126 | swiftTenantName = cfgAuthEndpoint[startIndex:dotIndex] 127 | swiftUsername = fmt.Sprintf("%s-%s:%s", instanceName, swiftTenantName, cfgUsername) 128 | 129 | // The client's key is the user account's Password 130 | swiftKey, _ := cfg.Config(ConfigPassword) 131 | 132 | client := swift.Connection{ 133 | UserName: swiftUsername, 134 | ApiKey: swiftKey, 135 | AuthUrl: cfgAuthEndpoint, 136 | Tenant: swiftTenantName, 137 | Transport: &fixLastModifiedTransport{http.DefaultTransport}, 138 | } 139 | 140 | return &client, nil 141 | } 142 | 143 | type fixLastModifiedTransport struct { 144 | http.RoundTripper 145 | } 146 | 147 | func (t *fixLastModifiedTransport) RoundTrip(r *http.Request) (*http.Response, error) { 148 | res, err := t.RoundTripper.RoundTrip(r) 149 | if err != nil { 150 | return res, err 151 | } 152 | if lastMod := res.Header.Get("Last-Modified"); strings.Contains(lastMod, "UTC") { 153 | res.Header.Set("Last-Modified", strings.Replace(lastMod, "UTC", "GMT", 1)) 154 | } 155 | return res, err 156 | } 157 | -------------------------------------------------------------------------------- /oracle/container.go: -------------------------------------------------------------------------------- 1 | package oracle 2 | 3 | import ( 4 | "io" 5 | "strings" 6 | 7 | "github.com/graymeta/stow" 8 | "github.com/ncw/swift" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | type container struct { 13 | id string 14 | client *swift.Connection 15 | } 16 | 17 | var _ stow.Container = (*container)(nil) 18 | 19 | // ID returns a string value representing a unique container, in this case it's 20 | // the Container's name. 21 | func (c *container) ID() string { 22 | return c.id 23 | } 24 | 25 | // Name returns a string value representing a unique container, in this case 26 | // it's the Container's name. 27 | func (c *container) Name() string { 28 | return c.id 29 | } 30 | 31 | func (c *container) Item(id string) (stow.Item, error) { 32 | return c.getItem(id) 33 | } 34 | 35 | // Items returns a collection of CloudStorage objects based on a matching 36 | // prefix string and cursor information. 37 | func (c *container) Items(prefix, cursor string, count int) ([]stow.Item, string, error) { 38 | params := &swift.ObjectsOpts{ 39 | Limit: count, 40 | Marker: cursor, 41 | Prefix: prefix, 42 | } 43 | objects, err := c.client.Objects(c.id, params) 44 | if err != nil { 45 | return nil, "", err 46 | } 47 | 48 | items := make([]stow.Item, len(objects)) 49 | for i, obj := range objects { 50 | 51 | items[i] = &item{ 52 | id: obj.Name, 53 | container: c, 54 | client: c.client, 55 | hash: obj.Hash, 56 | size: obj.Bytes, 57 | lastModified: obj.LastModified, 58 | } 59 | } 60 | marker := "" 61 | if len(objects) == count { 62 | marker = objects[len(objects)-1].Name 63 | } 64 | return items, marker, nil 65 | } 66 | 67 | // Put creates or updates a CloudStorage object within the given container. 68 | func (c *container) Put(name string, r io.Reader, size int64, metadata map[string]interface{}) (stow.Item, error) { 69 | mdPrepped, err := prepMetadata(metadata) 70 | if err != nil { 71 | return nil, errors.Wrap(err, "unable to create or update Item, preparing metadata") 72 | } 73 | 74 | _, err = c.client.ObjectPut(c.id, name, r, false, "", "", nil) 75 | if err != nil { 76 | return nil, errors.Wrap(err, "unable to create or update Item") 77 | } 78 | 79 | err = c.client.ObjectUpdate(c.id, name, mdPrepped) 80 | if err != nil { 81 | return nil, errors.Wrap(err, "unable to update Item metadata") 82 | } 83 | 84 | item := &item{ 85 | id: name, 86 | container: c, 87 | client: c.client, 88 | size: size, 89 | // not setting metadata here, the refined version isn't available 90 | // unless an explicit getItem() is done. Possible to write a func to facilitate 91 | // this. 92 | } 93 | return item, nil 94 | } 95 | 96 | // RemoveItem removes a CloudStorage object located within the given 97 | // container. 98 | func (c *container) RemoveItem(id string) error { 99 | return c.client.ObjectDelete(c.id, id) 100 | } 101 | 102 | func (c *container) getItem(id string) (*item, error) { 103 | info, headers, err := c.client.Object(c.id, id) 104 | if err != nil { 105 | if strings.Contains(err.Error(), "Object Not Found") { 106 | return nil, stow.ErrNotFound 107 | } 108 | return nil, err 109 | } 110 | 111 | md, err := parseMetadata(headers) 112 | if err != nil { 113 | return nil, errors.Wrap(err, "unable to retrieve Item information, parsing metadata") 114 | } 115 | 116 | item := &item{ 117 | id: id, 118 | container: c, 119 | client: c.client, 120 | hash: info.Hash, 121 | size: info.Bytes, 122 | lastModified: info.LastModified, 123 | metadata: md, 124 | } 125 | 126 | return item, nil 127 | } 128 | 129 | // Keys are returned as all lowercase 130 | func parseMetadata(md swift.Headers) (map[string]interface{}, error) { 131 | m := make(map[string]interface{}, len(md)) 132 | for key, value := range md.ObjectMetadata() { 133 | m[key] = value 134 | } 135 | return m, nil 136 | } 137 | 138 | func prepMetadata(md map[string]interface{}) (map[string]string, error) { 139 | m := make(map[string]string, len(md)) 140 | for key, value := range md { 141 | str, ok := value.(string) 142 | if !ok { 143 | return nil, errors.Errorf(`value of key '%s' in metadata must be of type string`, key) 144 | } 145 | m["X-Object-Meta-"+key] = str 146 | } 147 | return m, nil 148 | } 149 | -------------------------------------------------------------------------------- /oracle/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package oracle provides an absraction of the Oracle Storage Cloud Service. In this package, an Oracle Service Instance of type Storage is represented by a Stow Container, and an Oracle Storage Object is represented by a Stow Item. 3 | 4 | Oracle Storage Cloud Service is strictly a blob storage service, therefore nested directories do not exist. 5 | 6 | Usage and Credentials 7 | 8 | The most important detail in accessing the service is the authorization endpoint of the Service Instance. This URL can be found in the Overview page of the Instance, and is the value which corresponds to the "Service REST endpoint" field. 9 | 10 | The remaining two parts of information needed are the user name and password of the account which will be used to manipulate the service. Ensure that the AWS User whose credentials are used to manipulate service has permissions to do so. 11 | 12 | stow.Dial requires both a string value of the particular Stow Location Kind ("oracle") and a stow.Config instance. The stow.Config instance requires two entries with the specific key value attributes: 13 | 14 | - a key of oracle.ConfigUsername with a value of the account user name 15 | - a key of oracle.ConfigPassword with a value of the account pasword 16 | - a key of oracle.AuthEndpoint with a value of the authorization endpoint 17 | 18 | Location 19 | 20 | Methods of oracle.location allow the retrieval of an Oracle Cloud Service Storage Instance (Container or Containers). A stow.Item representation of an Oracle Object can also be retrieved based on the Object's URL (ItemByURL). 21 | 22 | Additional oracle.location methods provide capabilities to create and remove Storage Service Instances (CreateContainer or RemoveContainer, respectively). 23 | 24 | Container 25 | 26 | Methods of an oracle.container allow the retrieval of a Storage Service Instance's: 27 | 28 | - name(ID or Name) 29 | - item or complete list of items (Item or Items, respectively) 30 | 31 | Additional methods of an oracle.container allow Stow to: 32 | 33 | - remove an Object (RemoveItem) 34 | - update or create an Object (Put) 35 | 36 | Item 37 | 38 | Methods of oracle.Item allow the retrieval of a Storage Service Instance's: 39 | - name (ID or name) 40 | - URL 41 | - size in bytes 42 | - Oracle Storage Object specific metadata (information stored within Oracle Cloud Service) 43 | - last modified date 44 | - Etag 45 | - content 46 | */ 47 | package oracle 48 | -------------------------------------------------------------------------------- /oracle/item.go: -------------------------------------------------------------------------------- 1 | package oracle 2 | 3 | import ( 4 | "io" 5 | "net/url" 6 | "path" 7 | "sync" 8 | "time" 9 | 10 | "github.com/graymeta/stow" 11 | "github.com/ncw/swift" 12 | ) 13 | 14 | type item struct { 15 | id string 16 | container *container 17 | client *swift.Connection 18 | //properties az.BlobProperties 19 | hash string 20 | size int64 21 | url url.URL 22 | lastModified time.Time 23 | metadata map[string]interface{} 24 | infoOnce sync.Once 25 | infoErr error 26 | } 27 | 28 | var _ stow.Item = (*item)(nil) 29 | 30 | // ID returns a string value representing the Item, in this case it's the 31 | // name of the object. 32 | func (i *item) ID() string { 33 | return i.id 34 | } 35 | 36 | // Name returns a string value representing the Item, in this case it's the 37 | // name of the object. 38 | func (i *item) Name() string { 39 | return i.id 40 | } 41 | 42 | // URL returns a URL that for the given CloudStorage object. 43 | func (i *item) URL() *url.URL { 44 | url, _ := url.Parse(i.client.StorageUrl) 45 | url.Scheme = Kind 46 | url.Path = path.Join(url.Path, i.container.id, i.id) 47 | return url 48 | } 49 | 50 | // Size returns the size in bytes of the CloudStorage object. 51 | func (i *item) Size() (int64, error) { 52 | return i.size, nil 53 | } 54 | 55 | // Open is a method that returns an io.ReadCloser which represents the content 56 | // of the CloudStorage object. 57 | func (i *item) Open() (io.ReadCloser, error) { 58 | r, _, err := i.client.ObjectOpen(i.container.id, i.id, false, nil) 59 | var res io.ReadCloser = r 60 | // FIXME: this is a workaround to issue https://github.com/graymeta/stow/issues/120 61 | if s, ok := res.(readSeekCloser); ok { 62 | res = &fixReadSeekCloser{readSeekCloser: s, item: i} 63 | } 64 | return res, err 65 | } 66 | 67 | type readSeekCloser interface { 68 | io.ReadSeeker 69 | io.Closer 70 | } 71 | 72 | type fixReadSeekCloser struct { 73 | readSeekCloser 74 | item *item 75 | read bool 76 | } 77 | 78 | func (f *fixReadSeekCloser) Read(p []byte) (int, error) { 79 | f.read = true 80 | return f.readSeekCloser.Read(p) 81 | } 82 | 83 | func (f *fixReadSeekCloser) Seek(offset int64, whence int) (int64, error) { 84 | if offset == 0 && whence == io.SeekEnd && !f.read { 85 | return f.item.size, nil 86 | } 87 | return f.readSeekCloser.Seek(offset, whence) 88 | } 89 | 90 | // ETag returns a string value representing the CloudStorage Object 91 | func (i *item) ETag() (string, error) { 92 | return i.hash, nil 93 | } 94 | 95 | // LastMod returns a time.Time object representing information on the date 96 | // of the last time the CloudStorage object was modified. 97 | func (i *item) LastMod() (time.Time, error) { 98 | // If an object is PUT, certain information is missing. Detect 99 | // if the lastModified field is missing, send a request to retrieve 100 | // it, and save both this and other missing information so that a 101 | // request doesn't have to be sent again. 102 | err := i.ensureInfo() 103 | if err != nil { 104 | return time.Time{}, err 105 | } 106 | return i.lastModified, nil 107 | } 108 | 109 | // Metadata returns a map of key value pairs representing an Item's metadata 110 | func (i *item) Metadata() (map[string]interface{}, error) { 111 | err := i.ensureInfo() 112 | if err != nil { 113 | return nil, err 114 | } 115 | return i.metadata, nil 116 | } 117 | 118 | // ensureInfo checks the fields that may be empty when an item is PUT. 119 | // Verify if the fields are empty, get information on the item, fill in 120 | // the missing fields. 121 | func (i *item) ensureInfo() error { 122 | // If lastModified is empty, so is hash. get info on the Item and 123 | // update the necessary fields at the same time. 124 | if i.lastModified.IsZero() || i.hash == "" || i.metadata == nil { 125 | i.infoOnce.Do(func() { 126 | itemInfo, infoErr := i.getInfo() 127 | if infoErr != nil { 128 | i.infoErr = infoErr 129 | return 130 | } 131 | 132 | i.lastModified, i.infoErr = itemInfo.LastMod() 133 | if infoErr != nil { 134 | i.infoErr = infoErr 135 | return 136 | } 137 | 138 | i.metadata, i.infoErr = itemInfo.Metadata() 139 | if infoErr != nil { 140 | i.infoErr = infoErr 141 | return 142 | } 143 | }) 144 | } 145 | return i.infoErr 146 | } 147 | 148 | func (i *item) getInfo() (stow.Item, error) { 149 | itemInfo, err := i.container.getItem(i.ID()) 150 | if err != nil { 151 | return nil, err 152 | } 153 | return itemInfo, nil 154 | } 155 | -------------------------------------------------------------------------------- /oracle/location.go: -------------------------------------------------------------------------------- 1 | package oracle 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | "strings" 7 | 8 | "github.com/graymeta/stow" 9 | "github.com/ncw/swift" 10 | ) 11 | 12 | type location struct { 13 | config stow.Config 14 | client *swift.Connection 15 | } 16 | 17 | // Close fulfills the stow.Location interface since there's nothing to close. 18 | func (l *location) Close() error { 19 | return nil // nothing to close 20 | } 21 | 22 | // CreateContainer creates a new container with the given name while returning a 23 | // container instance with the given information. 24 | func (l *location) CreateContainer(name string) (stow.Container, error) { 25 | err := l.client.ContainerCreate(name, nil) 26 | if err != nil { 27 | return nil, err 28 | } 29 | container := &container{ 30 | id: name, 31 | client: l.client, 32 | } 33 | return container, nil 34 | } 35 | 36 | // Containers returns a collection of containers based on the given prefix and cursor. 37 | func (l *location) Containers(prefix, cursor string, count int) ([]stow.Container, string, error) { 38 | params := &swift.ContainersOpts{ 39 | Limit: count, 40 | Prefix: prefix, 41 | Marker: cursor, 42 | } 43 | response, err := l.client.Containers(params) 44 | if err != nil { 45 | return nil, "", err 46 | } 47 | containers := make([]stow.Container, len(response)) 48 | for i, cont := range response { 49 | containers[i] = &container{ 50 | id: cont.Name, 51 | client: l.client, 52 | // count: cont.Count, 53 | // bytes: cont.Bytes, 54 | } 55 | } 56 | marker := "" 57 | if len(response) == count { 58 | marker = response[len(response)-1].Name 59 | } 60 | return containers, marker, nil 61 | } 62 | 63 | // Container utilizes the client to retrieve container information based on its 64 | // name. 65 | func (l *location) Container(id string) (stow.Container, error) { 66 | _, _, err := l.client.Container(id) 67 | // TODO: grab info + headers 68 | if err != nil { 69 | return nil, stow.ErrNotFound 70 | } 71 | 72 | c := &container{ 73 | id: id, 74 | client: l.client, 75 | } 76 | 77 | return c, nil 78 | } 79 | 80 | // ItemByURL returns information on a CloudStorage object based on its name. 81 | func (l *location) ItemByURL(url *url.URL) (stow.Item, error) { 82 | 83 | if url.Scheme != Kind { 84 | return nil, errors.New("not valid URL") 85 | } 86 | 87 | path := strings.TrimLeft(url.Path, "/") 88 | pieces := strings.SplitN(path, "/", 4) 89 | 90 | c, err := l.Container(pieces[2]) 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | return c.Item(pieces[3]) 96 | } 97 | 98 | // RemoveContainer attempts to remove a container. Nonempty containers cannot 99 | // be removed. 100 | func (l *location) RemoveContainer(id string) error { 101 | return l.client.ContainerDelete(id) 102 | } 103 | -------------------------------------------------------------------------------- /oracle/stow_test.go: -------------------------------------------------------------------------------- 1 | package oracle 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | "os" 7 | "reflect" 8 | "testing" 9 | 10 | "github.com/cheekybits/is" 11 | "github.com/graymeta/stow" 12 | "github.com/graymeta/stow/test" 13 | ) 14 | 15 | var cfgUnmetered = stow.ConfigMap{ 16 | "username": os.Getenv("SWIFTUNMETEREDUSERNAME"), 17 | "password": os.Getenv("SWIFTUNMETEREDPASSWORD"), 18 | "authorization_endpoint": os.Getenv("SWIFTUNMETEREDAUTHENDPOINT"), 19 | } 20 | 21 | var cfgMetered = stow.ConfigMap{ 22 | "username": os.Getenv("SWIFTMETEREDUSERNAME"), 23 | "password": os.Getenv("SWIFTMETEREDPASSWORD"), 24 | "authorization_endpoint": os.Getenv("SWIFTMETEREDAUTHENDPOINT"), 25 | } 26 | 27 | func checkCredentials(config stow.Config) error { 28 | v, ok := config.Config(ConfigUsername) 29 | if !ok || v == "" { 30 | return errors.New("missing account username") 31 | } 32 | 33 | v, ok = config.Config(ConfigPassword) 34 | if !ok || v == "" { 35 | return errors.New("missing account password") 36 | } 37 | 38 | v, ok = config.Config(ConfigAuthEndpoint) 39 | if !ok || v == "" { 40 | return errors.New("missing authorization endpoint") 41 | } 42 | 43 | return nil 44 | } 45 | 46 | func TestStowMetered(t *testing.T) { 47 | err := checkCredentials(cfgMetered) 48 | if err != nil { 49 | t.Skip("skipping test because " + err.Error()) 50 | } 51 | test.All(t, "oracle", cfgMetered) 52 | } 53 | 54 | func TestStowUnMetered(t *testing.T) { 55 | err := checkCredentials(cfgUnmetered) 56 | if err != nil { 57 | t.Skip("skipping test because " + err.Error()) 58 | } 59 | test.All(t, "oracle", cfgUnmetered) 60 | } 61 | 62 | func TestGetItemUTCLastModified(t *testing.T) { 63 | err := checkCredentials(cfgMetered) 64 | if err != nil { 65 | t.Skip("skipping test because " + err.Error()) 66 | } 67 | 68 | tr := http.DefaultTransport 69 | http.DefaultTransport = &bogusLastModifiedTransport{tr} 70 | defer func() { 71 | http.DefaultTransport = tr 72 | }() 73 | 74 | test.All(t, "oracle", cfgMetered) 75 | } 76 | 77 | type bogusLastModifiedTransport struct { 78 | http.RoundTripper 79 | } 80 | 81 | func (t *bogusLastModifiedTransport) RoundTrip(r *http.Request) (*http.Response, error) { 82 | res, err := t.RoundTripper.RoundTrip(r) 83 | if err != nil { 84 | return res, err 85 | } 86 | res.Header.Set("Last-Modified", "Tue, 23 Aug 2016 15:12:44 UTC") 87 | return res, err 88 | } 89 | 90 | func (t *bogusLastModifiedTransport) CloseIdleConnections() { 91 | if tr, ok := t.RoundTripper.(interface { 92 | CloseIdleConnections() 93 | }); ok { 94 | tr.CloseIdleConnections() 95 | } 96 | } 97 | 98 | func TestPrepMetadataSuccess(t *testing.T) { 99 | is := is.New(t) 100 | 101 | m := make(map[string]string) 102 | m["one"] = "two" 103 | m["3"] = "4" 104 | m["ninety-nine"] = "100" 105 | 106 | m2 := make(map[string]interface{}) 107 | for key, value := range m { 108 | m2[key] = value 109 | } 110 | 111 | assertionM := make(map[string]string) 112 | assertionM["X-Object-Meta-one"] = "two" 113 | assertionM["X-Object-Meta-3"] = "4" 114 | assertionM["X-Object-Meta-ninety-nine"] = "100" 115 | 116 | //returns map[string]interface 117 | returnedMap, err := prepMetadata(m2) 118 | is.NoErr(err) 119 | 120 | if !reflect.DeepEqual(returnedMap, assertionM) { 121 | t.Errorf("Expected map (%+v) and returned map (%+v) are not equal.", assertionM, returnedMap) 122 | } 123 | } 124 | 125 | func TestPrepMetadataFailureWithNonStringValues(t *testing.T) { 126 | is := is.New(t) 127 | 128 | m := make(map[string]interface{}) 129 | m["float"] = 8.9 130 | m["number"] = 9 131 | 132 | _, err := prepMetadata(m) 133 | is.Err(err) 134 | } 135 | -------------------------------------------------------------------------------- /s3/README.md: -------------------------------------------------------------------------------- 1 | # S3 Stow Implementation 2 | 3 | Location = Amazon S3 4 | 5 | Container = Bucket 6 | 7 | Item = File 8 | 9 | Helpful Links: 10 | 11 | `http://docs.aws.amazon.com/sdk-for-go/api/service/s3/#example_S3_ListBuckets` 12 | 13 | --- 14 | 15 | SDK Notes: 16 | 17 | - Metadata of an S3 Object can only be set when the Object is created. 18 | 19 | --- 20 | 21 | Concerns: 22 | 23 | - An AWS account may have credentials which temporarily modifies permissions. This is specified by a token value. This feature is implemented but disabled and added as a TODO. 24 | 25 | --- 26 | 27 | Things to know: 28 | 29 | - Paging for the list of containers doesn't exist yet, this is because there's a hard limit of about 100 containers for every account. 30 | 31 | - A client is required to provide a region. Manipulating buckets that reside within other regions isn't possible. 32 | 33 | --- 34 | 35 | ###### Dev Notes 36 | 37 | The init function of every implementation of `stow` must call `stow.Register`. 38 | 39 | `stow.Register` accepts a few things: 40 | 41 | ### Kind, a string argument respresenting the name of the location. 42 | 43 | `makefn` a function that accepts any type that conforms to the stow.Config 44 | interface. It first validates the values of the `Config` argument, and then 45 | attempts to use the configuration to create a new client. If successful, An 46 | instance of a data type that conforms to the `stow.Location` interface is 47 | created. This Location should have fields that contain the client and 48 | configuration. 49 | 50 | Further calls in the hierarchy of a Location, Container, and Item depend 51 | on the values of the configuration + the client to send and receive information. 52 | 53 | - `kingmatchfn` a function that ensures that a given URL matches the `Kind` of the type of storage. 54 | 55 | --- 56 | 57 | **stow.Register(kind string, makefn func(Config) (Locaion, error), kindmatchfn func(*url.URL) bool)** 58 | 59 | - Adds `kind` and `makefn` into a map that contains a list of locations. 60 | 61 | - Adds `kind` to a slice that contains all of the different kinds. 62 | 63 | - Adds `kind` as part of an anonymous function which validates the scheme of the url.URL 64 | 65 | Once the `stow.Register` function is completed, a location of the given kind is returned. 66 | 67 | --- 68 | -------------------------------------------------------------------------------- /s3/config.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | "time" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/credentials" 10 | "github.com/aws/aws-sdk-go/aws/session" 11 | "github.com/aws/aws-sdk-go/service/s3" 12 | "github.com/graymeta/stow" 13 | "github.com/pkg/errors" 14 | ) 15 | 16 | // Kind represents the name of the location/storage type. 17 | const Kind = "s3" 18 | 19 | var ( 20 | authTypeAccessKey = "accesskey" 21 | authTypeIAM = "iam" 22 | ) 23 | 24 | const ( 25 | // ConfigAuthType is an optional argument that defines whether to use an IAM role or access key based auth 26 | ConfigAuthType = "auth_type" 27 | 28 | // ConfigAccessKeyID is one key of a pair of AWS credentials. 29 | ConfigAccessKeyID = "access_key_id" 30 | 31 | // ConfigSecretKey is one key of a pair of AWS credentials. 32 | ConfigSecretKey = "secret_key" 33 | 34 | // ConfigToken is an optional argument which is required when providing 35 | // credentials with temporary access. 36 | // ConfigToken = "token" 37 | 38 | // ConfigRegion represents the region/availability zone of the session. 39 | ConfigRegion = "region" 40 | 41 | // ConfigEndpoint is optional config value for changing s3 endpoint 42 | // used for e.g. minio.io 43 | ConfigEndpoint = "endpoint" 44 | 45 | // ConfigDisableSSL is optional config value for disabling SSL support on custom endpoints 46 | // Its default value is "false", to disable SSL set it to "true". 47 | ConfigDisableSSL = "disable_ssl" 48 | 49 | // ConfigV2Signing is an optional config value for signing requests with the v2 signature. 50 | // Its default value is "false", to enable set to "true". 51 | // This feature is useful for s3-compatible blob stores -- ie minio. 52 | ConfigV2Signing = "v2_signing" 53 | ) 54 | 55 | func init() { 56 | validatefn := func(config stow.Config) error { 57 | authType, ok := config.Config(ConfigAuthType) 58 | if !ok || authType == "" { 59 | authType = authTypeAccessKey 60 | } 61 | 62 | if !(authType == authTypeAccessKey || authType == authTypeIAM) { 63 | return errors.New("invalid auth_type") 64 | } 65 | 66 | if authType == authTypeAccessKey { 67 | _, ok := config.Config(ConfigAccessKeyID) 68 | if !ok { 69 | return errors.New("missing Access Key ID") 70 | } 71 | 72 | _, ok = config.Config(ConfigSecretKey) 73 | if !ok { 74 | return errors.New("missing Secret Key") 75 | } 76 | } 77 | return nil 78 | } 79 | makefn := func(config stow.Config) (stow.Location, error) { 80 | 81 | authType, ok := config.Config(ConfigAuthType) 82 | if !ok || authType == "" { 83 | authType = authTypeAccessKey 84 | } 85 | 86 | if !(authType == authTypeAccessKey || authType == authTypeIAM) { 87 | return nil, errors.New("invalid auth_type") 88 | } 89 | 90 | if authType == authTypeAccessKey { 91 | _, ok := config.Config(ConfigAccessKeyID) 92 | if !ok { 93 | return nil, errors.New("missing Access Key ID") 94 | } 95 | 96 | _, ok = config.Config(ConfigSecretKey) 97 | if !ok { 98 | return nil, errors.New("missing Secret Key") 99 | } 100 | } 101 | 102 | // Create a new client (s3 session) 103 | client, endpoint, err := newS3Client(config, "") 104 | if err != nil { 105 | return nil, err 106 | } 107 | 108 | // Create a location with given config and client (s3 session). 109 | loc := &location{ 110 | config: config, 111 | client: client, 112 | customEndpoint: endpoint, 113 | } 114 | 115 | return loc, nil 116 | } 117 | 118 | kindfn := func(u *url.URL) bool { 119 | return u.Scheme == Kind 120 | } 121 | 122 | stow.Register(Kind, makefn, kindfn, validatefn) 123 | } 124 | 125 | // Attempts to create a session based on the information given. 126 | func newS3Client(config stow.Config, region string) (client *s3.S3, endpoint string, err error) { 127 | authType, _ := config.Config(ConfigAuthType) 128 | accessKeyID, _ := config.Config(ConfigAccessKeyID) 129 | secretKey, _ := config.Config(ConfigSecretKey) 130 | // token, _ := config.Config(ConfigToken) 131 | 132 | if authType == "" { 133 | authType = authTypeAccessKey 134 | } 135 | 136 | awsConfig := aws.NewConfig(). 137 | WithHTTPClient(http.DefaultClient). 138 | WithMaxRetries(aws.UseServiceDefaultRetries). 139 | WithLogger(aws.NewDefaultLogger()). 140 | WithLogLevel(aws.LogOff). 141 | WithSleepDelay(time.Sleep) 142 | 143 | if region == "" { 144 | region, _ = config.Config(ConfigRegion) 145 | } 146 | if region != "" { 147 | awsConfig.WithRegion(region) 148 | } else { 149 | awsConfig.WithRegion("us-east-1") 150 | } 151 | 152 | if authType == authTypeAccessKey { 153 | awsConfig.WithCredentials(credentials.NewStaticCredentials(accessKeyID, secretKey, "")) 154 | } 155 | 156 | endpoint, ok := config.Config(ConfigEndpoint) 157 | if ok { 158 | awsConfig.WithEndpoint(endpoint). 159 | WithS3ForcePathStyle(true) 160 | } 161 | 162 | disableSSL, ok := config.Config(ConfigDisableSSL) 163 | if ok && disableSSL == "true" { 164 | awsConfig.WithDisableSSL(true) 165 | } 166 | 167 | sess, err := session.NewSession(awsConfig) 168 | if err != nil { 169 | return nil, "", err 170 | } 171 | if sess == nil { 172 | return nil, "", errors.New("creating the S3 session") 173 | } 174 | 175 | s3Client := s3.New(sess) 176 | 177 | usev2, ok := config.Config(ConfigV2Signing) 178 | if ok && usev2 == "true" { 179 | setv2Handlers(s3Client) 180 | } 181 | 182 | return s3Client, endpoint, nil 183 | } 184 | -------------------------------------------------------------------------------- /s3/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package s3 provides an abstraction of Amazon S3 (Simple Storage Service). An S3 Bucket is a Stow Container and an S3 Object is a Stow Item. Recall that nested directories exist within S3. 3 | 4 | Usage and Credentials 5 | 6 | There are three separate pieces of information required by Stow to have access to an S3 Stow Location: an AWS User's ACCESS_KEY_ID and SECRET_KEY fields, as well as the physical region of the S3 Endpoint. Ensure that the AWS User whose credentials are used to manipulate the S3 endpoint has permissions to do so. 7 | 8 | stow.Dial requires both a string value ("s3") of the particular Stow Location Kind and a stow.Config instance. The stow.Config instance requires three entries with the specific key value attributes: 9 | 10 | - a key of s3.ConfigAccessKeyID with a value of the AWS account's Access Key ID 11 | - a key of s3.ConfigSecretKey with a value of the AWS account's Secret Key 12 | - a key of s3.ConfigRegion with a value of the S3 endpoint's region (in all lowercase) 13 | 14 | Location 15 | 16 | The s3.location methods allow the retrieval of an S3 endpoint's Bucket or list of Buckets (Container or Containers). A stow.Item representation of an S3 Object can also be retrieved based on the Object's URL (ItemByURL). 17 | 18 | Additional s3.location methods provide capabilities to create and remove S3 Buckets (CreateContainer or RemoveContainer, respectively). 19 | 20 | Container 21 | 22 | There are s3.container methods which can retrieve an S3 Bucket's: 23 | 24 | - name (ID or Name) 25 | - Object or complete list of Objects (Item or Items) 26 | - region 27 | 28 | Additional s3.container methods give Stow the ability to: 29 | 30 | - remove an S3 Bucket (RemoveItem) 31 | - update or create an S3 Object (Put) 32 | 33 | Item 34 | 35 | Methods within an s3.item allow the retrieval of an S3 Object's: 36 | - name (ID or name) 37 | - URL (ItemByUrl) 38 | - size in bytes (Size) 39 | - S3 specific metadata (Metadata, key value pairs usually found within the console) 40 | - last modified date (LastMod) 41 | - Etag (Etag) 42 | - content (Open) 43 | */ 44 | package s3 45 | -------------------------------------------------------------------------------- /s3/item.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/url" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/service/s3" 13 | "github.com/graymeta/stow" 14 | "github.com/pkg/errors" 15 | ) 16 | 17 | // The item struct contains an id (also the name of the file/S3 Object/Item), 18 | // a container which it belongs to (s3 Bucket), a client, and a URL. The last 19 | // field, properties, contains information about the item, including the ETag, 20 | // file name/id, size, owner, last modified date, and storage class. 21 | // see Object type at http://docs.aws.amazon.com/sdk-for-go/api/service/s3/ 22 | // for more info. 23 | // All fields are unexported because methods exist to facilitate retrieval. 24 | type item struct { 25 | // Container information is required by a few methods. 26 | container *container 27 | // A client is needed to make requests. 28 | client *s3.S3 29 | // properties represent the characteristics of the file. Name, Etag, etc. 30 | properties properties 31 | infoOnce sync.Once 32 | infoErr error 33 | tags map[string]interface{} 34 | tagsOnce sync.Once 35 | tagsErr error 36 | } 37 | 38 | type properties struct { 39 | ETag *string `type:"string"` 40 | Key *string `min:"1" type:"string"` 41 | LastModified *time.Time `type:"timestamp" timestampFormat:"iso8601"` 42 | Owner *s3.Owner `type:"structure"` 43 | Size *int64 `type:"integer"` 44 | StorageClass *string `type:"string" enum:"ObjectStorageClass"` 45 | Metadata map[string]interface{} 46 | } 47 | 48 | // ID returns a string value that represents the name of a file. 49 | func (i *item) ID() string { 50 | return *i.properties.Key 51 | } 52 | 53 | // Name returns a string value that represents the name of the file. 54 | func (i *item) Name() string { 55 | return *i.properties.Key 56 | } 57 | 58 | // Size returns the size of an item in bytes. 59 | func (i *item) Size() (int64, error) { 60 | return *i.properties.Size, nil 61 | } 62 | 63 | // URL returns a formatted string which follows the predefined format 64 | // that every S3 asset is given. 65 | func (i *item) URL() *url.URL { 66 | if i.container.customEndpoint == "" { 67 | genericURL := fmt.Sprintf("https://s3-%s.amazonaws.com/%s/%s", i.container.Region(), i.container.Name(), i.Name()) 68 | 69 | return &url.URL{ 70 | Scheme: "s3", 71 | Path: genericURL, 72 | } 73 | } 74 | 75 | genericURL := fmt.Sprintf("%s/%s", i.container.Name(), i.Name()) 76 | return &url.URL{ 77 | Scheme: "s3", 78 | Path: genericURL, 79 | } 80 | } 81 | 82 | // Open retrieves specic information about an item based on the container name 83 | // and path of the file within the container. This response includes the body of 84 | // resource which is returned along with an error. 85 | func (i *item) Open() (io.ReadCloser, error) { 86 | params := &s3.GetObjectInput{ 87 | Bucket: aws.String(i.container.Name()), 88 | Key: aws.String(i.ID()), 89 | } 90 | 91 | response, err := i.client.GetObject(params) 92 | if err != nil { 93 | return nil, errors.Wrap(err, "Open, getting the object") 94 | } 95 | return response.Body, nil 96 | } 97 | 98 | // LastMod returns the last modified date of the item. The response of an item that is PUT 99 | // does not contain this field. Solution? Detect when the LastModified field (a *time.Time) 100 | // is nil, then do a manual request for it via the Item() method of the container which 101 | // does return the specified field. This more detailed information is kept so that we 102 | // won't have to do it again. 103 | func (i *item) LastMod() (time.Time, error) { 104 | err := i.ensureInfo() 105 | if err != nil { 106 | return time.Time{}, errors.Wrap(err, "retrieving Last Modified information of Item") 107 | } 108 | return *i.properties.LastModified, nil 109 | } 110 | 111 | // ETag returns the ETag value from the properies field of an item. 112 | func (i *item) ETag() (string, error) { 113 | return *(i.properties.ETag), nil 114 | } 115 | 116 | func (i *item) Metadata() (map[string]interface{}, error) { 117 | err := i.ensureInfo() 118 | if err != nil { 119 | return nil, errors.Wrap(err, "retrieving metadata") 120 | } 121 | return i.properties.Metadata, nil 122 | } 123 | 124 | func (i *item) ensureInfo() error { 125 | if i.properties.Metadata == nil || i.properties.LastModified == nil { 126 | i.infoOnce.Do(func() { 127 | // Retrieve Item information 128 | itemInfo, infoErr := i.getInfo() 129 | if infoErr != nil { 130 | i.infoErr = infoErr 131 | return 132 | } 133 | 134 | // Set metadata field 135 | i.properties.Metadata, infoErr = itemInfo.Metadata() 136 | if infoErr != nil { 137 | i.infoErr = infoErr 138 | return 139 | } 140 | 141 | // Set LastModified field 142 | lmValue, infoErr := itemInfo.LastMod() 143 | if infoErr != nil { 144 | i.infoErr = infoErr 145 | return 146 | } 147 | i.properties.LastModified = &lmValue 148 | }) 149 | } 150 | return i.infoErr 151 | } 152 | 153 | func (i *item) getInfo() (stow.Item, error) { 154 | itemInfo, err := i.container.getItem(i.ID()) 155 | if err != nil { 156 | return nil, err 157 | } 158 | return itemInfo, nil 159 | } 160 | 161 | // Tags returns a map of tags on an Item 162 | func (i *item) Tags() (map[string]interface{}, error) { 163 | i.tagsOnce.Do(func() { 164 | params := &s3.GetObjectTaggingInput{ 165 | Bucket: aws.String(i.container.name), 166 | Key: aws.String(i.ID()), 167 | } 168 | 169 | res, err := i.client.GetObjectTagging(params) 170 | if err != nil { 171 | if strings.Contains(err.Error(), "NoSuchKey") { 172 | i.tagsErr = stow.ErrNotFound 173 | return 174 | } 175 | i.tagsErr = errors.Wrap(err, "getObjectTagging") 176 | return 177 | } 178 | 179 | i.tags = make(map[string]interface{}) 180 | for _, t := range res.TagSet { 181 | i.tags[*t.Key] = *t.Value 182 | } 183 | }) 184 | 185 | return i.tags, i.tagsErr 186 | } 187 | 188 | // OpenRange opens the item for reading starting at byte start and ending 189 | // at byte end. 190 | func (i *item) OpenRange(start, end uint64) (io.ReadCloser, error) { 191 | params := &s3.GetObjectInput{ 192 | Bucket: aws.String(i.container.Name()), 193 | Key: aws.String(i.ID()), 194 | Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)), 195 | } 196 | 197 | response, err := i.client.GetObject(params) 198 | if err != nil { 199 | return nil, errors.Wrap(err, "Open, getting the object") 200 | } 201 | return response.Body, nil 202 | } 203 | -------------------------------------------------------------------------------- /s3/location.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "context" 5 | "net/url" 6 | "strings" 7 | "time" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/aws/awserr" 11 | "github.com/aws/aws-sdk-go/service/s3" 12 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 13 | "github.com/graymeta/stow" 14 | "github.com/pkg/errors" 15 | ) 16 | 17 | // A location contains a client + the configurations used to create the client. 18 | type location struct { 19 | config stow.Config 20 | customEndpoint string 21 | client *s3.S3 22 | } 23 | 24 | // CreateContainer creates a new container, in this case an S3 bucket. 25 | // The bare minimum needed is a container name, but there are many other 26 | // options that can be provided. 27 | func (l *location) CreateContainer(containerName string) (stow.Container, error) { 28 | createBucketParams := &s3.CreateBucketInput{ 29 | Bucket: aws.String(containerName), // required 30 | } 31 | 32 | _, err := l.client.CreateBucket(createBucketParams) 33 | if err != nil { 34 | return nil, errors.Wrap(err, "CreateContainer, creating the bucket") 35 | } 36 | 37 | region, _ := l.config.Config("region") 38 | 39 | newContainer := &container{ 40 | name: containerName, 41 | client: l.client, 42 | region: region, 43 | customEndpoint: l.customEndpoint, 44 | } 45 | 46 | return newContainer, nil 47 | } 48 | 49 | // Containers returns a slice of the Container interface, a cursor, and an error. 50 | // This doesn't seem to exist yet in the API without doing a ton of manual work. 51 | // Get the list of buckets, query every single one to retrieve region info, and finally 52 | // return the list of containers that have a matching region against the client. It's not 53 | // possible to manipulate a container within a region that doesn't match the clients'. 54 | // This is because AWS user credentials can be tied to regions. One solution would be 55 | // to start a new client for every single container where the region matches, this would 56 | // also check the credentials on every new instance... Tabled for later. 57 | func (l *location) Containers(prefix, cursor string, count int) ([]stow.Container, string, error) { 58 | // Response returns exported Owner(*s3.Owner) and Bucket(*s3.[]Bucket) 59 | var params *s3.ListBucketsInput 60 | bucketList, err := l.client.ListBuckets(params) 61 | if err != nil { 62 | return nil, "", errors.Wrap(err, "Containers, listing the buckets") 63 | } 64 | 65 | // Seek to the current bucket, according to cursor. 66 | if cursor != stow.CursorStart { 67 | ok := false 68 | for i, b := range bucketList.Buckets { 69 | if *b.Name == cursor { 70 | ok = true 71 | bucketList.Buckets = bucketList.Buckets[i:] 72 | break 73 | } 74 | } 75 | if !ok { 76 | return nil, "", stow.ErrBadCursor 77 | } 78 | } 79 | cursor = "" 80 | 81 | // Region is pulled from stow.Config. If Region is specified, only add 82 | // Bucket to Container list if it is located in configured Region. 83 | region, regionSet := l.config.Config(ConfigRegion) 84 | 85 | // Endpoint would indicate that we are using s3-compatible storage, which 86 | // does not support s3session.GetBucketRegion(). 87 | endpoint, endpointSet := l.config.Config(ConfigEndpoint) 88 | 89 | // Iterate through the slice of pointers to buckets 90 | var containers []stow.Container 91 | for _, bucket := range bucketList.Buckets { 92 | if len(containers) == count { 93 | cursor = *bucket.Name 94 | break 95 | } 96 | 97 | if !strings.HasPrefix(*bucket.Name, prefix) { 98 | continue 99 | } 100 | 101 | var err error 102 | client := l.client 103 | bucketRegion := region 104 | if !endpointSet && endpoint == "" { 105 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 106 | bucketRegion, err = s3manager.GetBucketRegionWithClient(ctx, l.client, *bucket.Name) 107 | cancel() 108 | if err != nil { 109 | if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" { 110 | // sometimes buckets will still show up int eh ListBuckets results after 111 | // being deleted, but will 404 when determining the region. Use this as a 112 | // strong signal that the bucket has been deleted. 113 | continue 114 | } 115 | return nil, "", errors.Wrapf(err, "Containers, getting bucket region for: %s", *bucket.Name) 116 | } 117 | if regionSet && region != "" && bucketRegion != region { 118 | continue 119 | } 120 | 121 | client, _, err = newS3Client(l.config, bucketRegion) 122 | if err != nil { 123 | return nil, "", errors.Wrapf(err, "Containers, creating new client for region: %s", bucketRegion) 124 | } 125 | } 126 | 127 | newContainer := &container{ 128 | name: *(bucket.Name), 129 | client: client, 130 | region: bucketRegion, 131 | customEndpoint: l.customEndpoint, 132 | } 133 | 134 | containers = append(containers, newContainer) 135 | } 136 | 137 | return containers, cursor, nil 138 | } 139 | 140 | // Close simply satisfies the Location interface. There's nothing that 141 | // needs to be done in order to satisfy the interface. 142 | func (l *location) Close() error { 143 | return nil // nothing to close 144 | } 145 | 146 | // Container retrieves a stow.Container based on its name which must be 147 | // exact. 148 | func (l *location) Container(id string) (stow.Container, error) { 149 | client := l.client 150 | bucketRegion, bucketRegionSet := l.config.Config(ConfigRegion) 151 | 152 | // Endpoint would indicate that we are using s3-compatible storage, which 153 | // does not support s3session.GetBucketRegion(). 154 | if endpoint, endpointSet := l.config.Config(ConfigEndpoint); !endpointSet && endpoint == "" { 155 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) 156 | bucketRegion, _ = s3manager.GetBucketRegionWithClient(ctx, l.client, id) 157 | cancel() 158 | 159 | var err error 160 | client, _, err = newS3Client(l.config, bucketRegion) 161 | if err != nil { 162 | return nil, errors.Wrapf(err, "Container, creating new client for region: %s", bucketRegion) 163 | } 164 | } 165 | 166 | c := &container{ 167 | name: id, 168 | client: client, 169 | region: bucketRegion, 170 | customEndpoint: l.customEndpoint, 171 | } 172 | 173 | if bucketRegionSet || bucketRegion != "" { 174 | return c, nil 175 | } 176 | 177 | params := &s3.GetBucketLocationInput{ 178 | Bucket: aws.String(id), 179 | } 180 | 181 | _, err := client.GetBucketLocation(params) 182 | if err != nil { 183 | if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NoSuchBucket" { 184 | return nil, stow.ErrNotFound 185 | } 186 | 187 | return nil, errors.Wrap(err, "GetBucketLocation") 188 | } 189 | 190 | return c, nil 191 | } 192 | 193 | // RemoveContainer removes a container simply by name. 194 | func (l *location) RemoveContainer(id string) error { 195 | params := &s3.DeleteBucketInput{ 196 | Bucket: aws.String(id), 197 | } 198 | 199 | _, err := l.client.DeleteBucket(params) 200 | if err != nil { 201 | return errors.Wrap(err, "RemoveContainer, deleting the bucket") 202 | } 203 | 204 | return nil 205 | } 206 | 207 | // ItemByURL retrieves a stow.Item by parsing the URL, in this 208 | // case an item is an object. 209 | func (l *location) ItemByURL(url *url.URL) (stow.Item, error) { 210 | if l.customEndpoint == "" { 211 | genericURL := []string{"https://s3-", ".amazonaws.com/"} 212 | 213 | // Remove genericURL[0] from URL: 214 | // url = 215 | firstCut := strings.Replace(url.Path, genericURL[0], "", 1) 216 | 217 | // find first dot so that we could extract region. 218 | dotIndex := strings.Index(firstCut, ".") 219 | 220 | // region of the s3 bucket. 221 | region := firstCut[0:dotIndex] 222 | 223 | // Remove from 224 | // 225 | secondCut := strings.Replace(firstCut, region+genericURL[1], "", 1) 226 | 227 | // Get the index of the first slash to get the end of the bucket name. 228 | firstSlash := strings.Index(secondCut, "/") 229 | 230 | // Grab bucket name 231 | bucketName := secondCut[:firstSlash] 232 | 233 | // Everything afterwards pertains to object. 234 | objectPath := secondCut[firstSlash+1:] 235 | 236 | // Get the container by bucket name. 237 | cont, err := l.Container(bucketName) 238 | if err != nil { 239 | return nil, errors.Wrapf(err, "ItemByURL, getting container by the bucketname %s", bucketName) 240 | } 241 | 242 | // Get the item by object name. 243 | it, err := cont.Item(objectPath) 244 | if err != nil { 245 | return nil, errors.Wrapf(err, "ItemByURL, getting item by object name %s", objectPath) 246 | } 247 | 248 | return it, err 249 | } 250 | 251 | // url looks like this: s3:/// 252 | // example: s3://graymeta-demo/DPtest.txt 253 | containerName := url.Host 254 | itemName := strings.TrimPrefix(url.Path, "/") 255 | 256 | c, err := l.Container(containerName) 257 | if err != nil { 258 | return nil, errors.Wrapf(err, "ItemByURL, getting container by the bucketname %s", containerName) 259 | } 260 | 261 | i, err := c.Item(itemName) 262 | if err != nil { 263 | return nil, errors.Wrapf(err, "ItemByURL, getting item by object name %s", itemName) 264 | } 265 | return i, nil 266 | } 267 | -------------------------------------------------------------------------------- /s3/stow_iam_test.go: -------------------------------------------------------------------------------- 1 | // +build iam 2 | 3 | package s3 4 | 5 | import ( 6 | "os" 7 | "testing" 8 | 9 | "github.com/graymeta/stow" 10 | "github.com/graymeta/stow/test" 11 | ) 12 | 13 | func TestStowIAM(t *testing.T) { 14 | region := os.Getenv("S3REGION") 15 | 16 | if region == "" { 17 | t.Skip("skipping test because missing S3REGION") 18 | } 19 | 20 | config := stow.ConfigMap{ 21 | "auth_type": "iam", 22 | "region": region, 23 | } 24 | 25 | test.All(t, "s3", config) 26 | } 27 | -------------------------------------------------------------------------------- /s3/stow_test.go: -------------------------------------------------------------------------------- 1 | // build +disabled 2 | package s3 3 | 4 | import ( 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "net/http/httptest" 9 | "net/url" 10 | "os" 11 | "reflect" 12 | "strings" 13 | "testing" 14 | 15 | "github.com/aws/aws-sdk-go/aws" 16 | "github.com/aws/aws-sdk-go/service/s3" 17 | "github.com/cheekybits/is" 18 | "github.com/graymeta/stow" 19 | "github.com/graymeta/stow/test" 20 | ) 21 | 22 | func TestStow(t *testing.T) { 23 | accessKeyId := os.Getenv("S3ACCESSKEYID") 24 | secretKey := os.Getenv("S3SECRETKEY") 25 | region := os.Getenv("S3REGION") 26 | 27 | if accessKeyId == "" || secretKey == "" || region == "" { 28 | t.Skip("skipping test because missing one or more of S3ACCESSKEYID S3SECRETKEY S3REGION") 29 | } 30 | 31 | config := stow.ConfigMap{ 32 | "access_key_id": accessKeyId, 33 | "secret_key": secretKey, 34 | "region": region, 35 | } 36 | 37 | test.All(t, "s3", config) 38 | } 39 | 40 | func TestEtagCleanup(t *testing.T) { 41 | etagValue := "9c51403a2255f766891a1382288dece4" 42 | permutations := []string{ 43 | `"%s"`, // Enclosing quotations 44 | `W/\"%s\"`, // Weak tag identifier with escapted quotes 45 | `W/"%s"`, // Weak tag identifier with quotes 46 | `"\"%s"\"`, // Double quotes, inner escaped 47 | `""%s""`, // Double quotes, 48 | `"W/"%s""`, // Double quotes with weak identifier 49 | `"W/\"%s\""`, // Double quotes with weak identifier, inner escaped 50 | } 51 | for index, p := range permutations { 52 | testStr := fmt.Sprintf(p, etagValue) 53 | cleanTestStr := cleanEtag(testStr) 54 | if etagValue != cleanTestStr { 55 | t.Errorf(`Failure at permutation #%d (%s), result: %s`, 56 | index, permutations[index], cleanTestStr) 57 | } 58 | } 59 | } 60 | 61 | func TestPrepMetadataSuccess(t *testing.T) { 62 | is := is.New(t) 63 | 64 | m := make(map[string]*string) 65 | m["one"] = aws.String("two") 66 | m["3"] = aws.String("4") 67 | m["ninety-nine"] = aws.String("100") 68 | 69 | m2 := make(map[string]interface{}) 70 | for key, value := range m { 71 | str := *value 72 | m2[key] = str 73 | } 74 | 75 | returnedMap, err := prepMetadata(m2) 76 | is.NoErr(err) 77 | 78 | if !reflect.DeepEqual(m, returnedMap) { 79 | t.Error("Expected and returned maps are not equal.") 80 | } 81 | } 82 | 83 | func TestPrepMetadataFailureWithNonStringValues(t *testing.T) { 84 | is := is.New(t) 85 | 86 | m := make(map[string]interface{}) 87 | m["float"] = 8.9 88 | m["number"] = 9 89 | 90 | _, err := prepMetadata(m) 91 | is.Err(err) 92 | } 93 | 94 | func TestInvalidAuthtype(t *testing.T) { 95 | is := is.New(t) 96 | 97 | config := stow.ConfigMap{ 98 | "auth_type": "foo", 99 | } 100 | _, err := stow.Dial("s3", config) 101 | is.Err(err) 102 | is.True(strings.Contains(err.Error(), "invalid auth_type")) 103 | } 104 | 105 | func TestV2SigningEnabled(t *testing.T) { 106 | is := is.New(t) 107 | 108 | //check v2 singing occurs 109 | v2Server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 110 | is.True(strings.HasPrefix(r.Header.Get("Authorization"), "AWS access-key:")) 111 | w.Header().Add("ETag", "something") 112 | w.WriteHeader(http.StatusOK) 113 | })) 114 | defer v2Server.Close() 115 | 116 | uri, err := url.Parse(v2Server.URL + "/testing") 117 | is.NoErr(err) 118 | 119 | config := stow.ConfigMap{ 120 | "access_key_id": "access-key", 121 | "secret_key": "secret-key", 122 | "region": "do-not-care", 123 | "v2_signing": "true", 124 | "endpoint": v2Server.URL, 125 | } 126 | 127 | location, err := stow.Dial("s3", config) 128 | is.NoErr(err) 129 | _, _ = location.ItemByURL(uri) 130 | 131 | //check v2 signing does not occur 132 | v4Server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 133 | is.False(strings.HasPrefix(r.Header.Get("Authorization"), "AWS access-key:")) 134 | w.Header().Add("ETag", "something") 135 | w.WriteHeader(http.StatusOK) 136 | })) 137 | defer v4Server.Close() 138 | 139 | uri, err = url.Parse(v4Server.URL + "/testing") 140 | is.NoErr(err) 141 | 142 | config = stow.ConfigMap{ 143 | "access_key_id": "access-key", 144 | "secret_key": "secret-key", 145 | "region": "do-not-care", 146 | "v2_signing": "false", 147 | "endpoint": v4Server.URL, 148 | } 149 | 150 | location, err = stow.Dial("s3", config) 151 | is.NoErr(err) 152 | _, _ = location.ItemByURL(uri) 153 | } 154 | 155 | func TestWillNotRequestRegionWhenConfigured(t *testing.T) { 156 | is := is.New(t) 157 | 158 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 159 | is.Fail("Request should not occur") 160 | w.WriteHeader(http.StatusBadRequest) 161 | })) 162 | defer server.Close() 163 | 164 | config := stow.ConfigMap{ 165 | "access_key_id": "access-key", 166 | "secret_key": "secret-key", 167 | "region": "do-not-care", 168 | "endpoint": server.URL, 169 | } 170 | 171 | location, err := stow.Dial("s3", config) 172 | is.NoErr(err) 173 | 174 | _, err = location.Container("Whatever") 175 | 176 | is.NoErr(err) 177 | } 178 | 179 | func TestWillRequestRegionWhenConfigured(t *testing.T) { 180 | is := is.New(t) 181 | 182 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 183 | awsLocationQuery, err := url.ParseQuery("location") 184 | is.NoErr(err) 185 | is.Equal(awsLocationQuery.Encode(), r.URL.RawQuery) 186 | b, _ := json.Marshal(s3.GetBucketLocationOutput{ 187 | LocationConstraint: aws.String("whatever"), 188 | }) 189 | w.Write(b) 190 | w.WriteHeader(http.StatusOK) 191 | })) 192 | defer server.Close() 193 | 194 | config := stow.ConfigMap{ 195 | "access_key_id": "access-key", 196 | "secret_key": "secret-key", 197 | "endpoint": server.URL, 198 | } 199 | 200 | location, err := stow.Dial("s3", config) 201 | is.NoErr(err) 202 | 203 | _, err = location.Container("Whatever") 204 | 205 | // Make sure that this is an error 206 | is.NoErr(err) 207 | } 208 | -------------------------------------------------------------------------------- /s3/v2signer.go: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013 Damien Le Berrigaud and Nick Wade 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy 5 | of this software and associated documentation files (the "Software"), to deal 6 | in the Software without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the Software is 9 | furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | */ 22 | 23 | package s3 24 | 25 | import ( 26 | "crypto/hmac" 27 | "crypto/sha1" 28 | "encoding/base64" 29 | "fmt" 30 | "log" 31 | "net/http" 32 | "net/url" 33 | "sort" 34 | "strings" 35 | "time" 36 | 37 | "github.com/aws/aws-sdk-go/aws" 38 | "github.com/aws/aws-sdk-go/aws/corehandlers" 39 | "github.com/aws/aws-sdk-go/aws/credentials" 40 | "github.com/aws/aws-sdk-go/aws/request" 41 | "github.com/aws/aws-sdk-go/service/s3" 42 | ) 43 | 44 | const ( 45 | signatureVersion = "2" 46 | signatureMethod = "HmacSHA1" 47 | timeFormat = "2006-01-02T15:04:05Z" 48 | ) 49 | 50 | type signer struct { 51 | // Values that must be populated from the request 52 | Request *http.Request 53 | Time time.Time 54 | Credentials *credentials.Credentials 55 | Debug aws.LogLevelType 56 | Logger aws.Logger 57 | 58 | Query url.Values 59 | stringToSign string 60 | signature string 61 | } 62 | 63 | var s3ParamsToSign = map[string]bool{ 64 | "acl": true, 65 | "location": true, 66 | "logging": true, 67 | "notification": true, 68 | "partNumber": true, 69 | "policy": true, 70 | "requestPayment": true, 71 | "torrent": true, 72 | "uploadId": true, 73 | "uploads": true, 74 | "versionId": true, 75 | "versioning": true, 76 | "versions": true, 77 | "response-content-type": true, 78 | "response-content-language": true, 79 | "response-expires": true, 80 | "response-cache-control": true, 81 | "response-content-disposition": true, 82 | "response-content-encoding": true, 83 | "website": true, 84 | "delete": true, 85 | } 86 | 87 | func setv2Handlers(svc *s3.S3) { 88 | svc.Handlers.Build.PushBack(func(r *request.Request) { 89 | parsedURL, err := url.Parse(r.HTTPRequest.URL.String()) 90 | if err != nil { 91 | log.Fatal("Failed to parse URL", err) 92 | } 93 | r.HTTPRequest.URL.Opaque = parsedURL.Path 94 | }) 95 | 96 | svc.Handlers.Sign.Clear() 97 | svc.Handlers.Sign.PushBack(Sign) 98 | svc.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler) 99 | } 100 | 101 | // Sign requests with signature version 2. 102 | // 103 | // Will sign the requests with the service config's Credentials object 104 | // Signing is skipped if the credentials is the credentials.AnonymousCredentials 105 | // object. 106 | func Sign(req *request.Request) { 107 | // If the request does not need to be signed ignore the signing of the 108 | // request if the AnonymousCredentials object is used. 109 | if req.Config.Credentials == credentials.AnonymousCredentials { 110 | return 111 | } 112 | 113 | v2 := signer{ 114 | Request: req.HTTPRequest, 115 | Time: req.Time, 116 | Credentials: req.Config.Credentials, 117 | Debug: req.Config.LogLevel.Value(), 118 | Logger: req.Config.Logger, 119 | } 120 | 121 | req.Error = v2.Sign() 122 | 123 | if req.Error != nil { 124 | return 125 | } 126 | } 127 | 128 | func (v2 *signer) Sign() error { 129 | credValue, err := v2.Credentials.Get() 130 | if err != nil { 131 | return err 132 | } 133 | accessKey := credValue.AccessKeyID 134 | var ( 135 | md5, ctype, date, xamz string 136 | xamzDate bool 137 | sarray []string 138 | ) 139 | 140 | headers := v2.Request.Header 141 | params := v2.Request.URL.Query() 142 | parsedURL, err := url.Parse(v2.Request.URL.String()) 143 | if err != nil { 144 | return err 145 | } 146 | host, canonicalPath := parsedURL.Host, parsedURL.Path 147 | v2.Request.Header["Host"] = []string{host} 148 | v2.Request.Header["x-amz-date"] = []string{v2.Time.In(time.UTC).Format(time.RFC1123)} 149 | 150 | for k, v := range headers { 151 | k = strings.ToLower(k) 152 | switch k { 153 | case "content-md5": 154 | md5 = v[0] 155 | case "content-type": 156 | ctype = v[0] 157 | case "date": 158 | if !xamzDate { 159 | date = v[0] 160 | } 161 | default: 162 | if strings.HasPrefix(k, "x-amz-") { 163 | vall := strings.Join(v, ",") 164 | sarray = append(sarray, k+":"+vall) 165 | if k == "x-amz-date" { 166 | xamzDate = true 167 | date = "" 168 | } 169 | } 170 | } 171 | } 172 | if len(sarray) > 0 { 173 | sort.StringSlice(sarray).Sort() 174 | xamz = strings.Join(sarray, "\n") + "\n" 175 | } 176 | 177 | expires := false 178 | if v, ok := params["Expires"]; ok { 179 | expires = true 180 | date = v[0] 181 | params["AWSAccessKeyId"] = []string{accessKey} 182 | } 183 | 184 | sarray = sarray[0:0] 185 | for k, v := range params { 186 | if s3ParamsToSign[k] { 187 | for _, vi := range v { 188 | if vi == "" { 189 | sarray = append(sarray, k) 190 | } else { 191 | sarray = append(sarray, k+"="+vi) 192 | } 193 | } 194 | } 195 | } 196 | if len(sarray) > 0 { 197 | sort.StringSlice(sarray).Sort() 198 | canonicalPath = canonicalPath + "?" + strings.Join(sarray, "&") 199 | } 200 | 201 | v2.stringToSign = strings.Join([]string{ 202 | v2.Request.Method, 203 | md5, 204 | ctype, 205 | date, 206 | xamz + canonicalPath, 207 | }, "\n") 208 | hash := hmac.New(sha1.New, []byte(credValue.SecretAccessKey)) 209 | hash.Write([]byte(v2.stringToSign)) 210 | v2.signature = base64.StdEncoding.EncodeToString(hash.Sum(nil)) 211 | 212 | if expires { 213 | params["Signature"] = []string{string(v2.signature)} 214 | } else { 215 | headers["Authorization"] = []string{"AWS " + accessKey + ":" + string(v2.signature)} 216 | } 217 | 218 | if v2.Debug.Matches(aws.LogDebugWithSigning) { 219 | v2.logSigningInfo() 220 | } 221 | return nil 222 | } 223 | 224 | const logSignInfoMsg = `DEBUG: Request Signature: 225 | ---[ STRING TO SIGN ]-------------------------------- 226 | %s 227 | ---[ SIGNATURE ]------------------------------------- 228 | %s 229 | -----------------------------------------------------` 230 | 231 | func (v2 *signer) logSigningInfo() { 232 | msg := fmt.Sprintf(logSignInfoMsg, v2.stringToSign, v2.signature) 233 | v2.Logger.Log(msg) 234 | } -------------------------------------------------------------------------------- /sftp/config.go: -------------------------------------------------------------------------------- 1 | package sftp 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "strconv" 7 | 8 | "github.com/graymeta/stow" 9 | "github.com/pkg/errors" 10 | "github.com/pkg/sftp" 11 | "golang.org/x/crypto/ssh" 12 | ) 13 | 14 | // Kind represents the name of the location/storage type. 15 | const Kind = "sftp" 16 | 17 | const ( 18 | // ConfigHost is the hostname or IP address to connect to. 19 | ConfigHost = "host" 20 | 21 | // ConfigPort is the numeric port number the ssh daemon is listening on. 22 | ConfigPort = "port" 23 | 24 | // ConfigUsername is the username of the user to connect as. 25 | ConfigUsername = "username" 26 | 27 | // ConfigPassword is the password use to authentiate with. Can be used instead 28 | // of a private key (if your server allows this). 29 | ConfigPassword = "password" 30 | 31 | // ConfigPrivateKey is a private ssh key to use to authenticate. These are the 32 | // bytes representing the key, not the path on disk to the key file. 33 | ConfigPrivateKey = "private_key" 34 | 35 | // ConfigPrivateKeyPassphrase is an optional passphrase to use with the private key. 36 | ConfigPrivateKeyPassphrase = "private_key_passphrase" 37 | 38 | // ConfigHostPublicKey is the public host key (in the same format as would be 39 | // found in the known_hosts file). If this is not specified, or is set to an 40 | // empty string, the host key validation will be disabled. 41 | ConfigHostPublicKey = "host_public_key" 42 | 43 | // ConfigBasePath is the path to the root folder on the remote server. It can be 44 | // relative to the user's home directory, or an absolute path. If not set, or 45 | // set to an empty string, the user's home directory will be used. 46 | ConfigBasePath = "base_path" 47 | ) 48 | 49 | type conf struct { 50 | host string 51 | port int 52 | basePath string 53 | sshConfig ssh.ClientConfig 54 | } 55 | 56 | func (c conf) Host() string { 57 | return fmt.Sprintf("%s:%d", c.host, c.port) 58 | } 59 | 60 | func parseConfig(config stow.Config) (*conf, error) { 61 | var c conf 62 | var ok bool 63 | 64 | c.host, ok = config.Config(ConfigHost) 65 | if !ok || c.host == "" { 66 | return nil, errors.New("invalid hostname") 67 | } 68 | 69 | port, ok := config.Config(ConfigPort) 70 | if !ok { 71 | return nil, errors.New("port not specified") 72 | 73 | } 74 | var err error 75 | c.port, err = strconv.Atoi(port) 76 | if err != nil || c.port < 1 { 77 | return nil, errors.New("invalid port configuration") 78 | } 79 | 80 | c.sshConfig.User, ok = config.Config(ConfigUsername) 81 | if !ok || c.sshConfig.User == "" { 82 | return nil, errors.New("invalid username") 83 | } 84 | 85 | // If a private key is specified, load it up and add it to the list of 86 | // authentication methods to attempt against the server. 87 | privKey, ok := config.Config(ConfigPrivateKey) 88 | if ok && privKey != "" { 89 | // If a passphrase for the key is specified, use it to unlock the key. 90 | passphrase, _ := config.Config(ConfigPrivateKeyPassphrase) 91 | var signer ssh.Signer 92 | if passphrase != "" { 93 | signer, err = ssh.ParsePrivateKeyWithPassphrase([]byte(privKey), []byte(passphrase)) 94 | if err != nil { 95 | return nil, errors.Wrap(err, "parsing key with passphrase") 96 | } 97 | } else { 98 | signer, err = ssh.ParsePrivateKey([]byte(privKey)) 99 | if err != nil { 100 | return nil, errors.Wrap(err, "parsing key") 101 | } 102 | } 103 | c.sshConfig.Auth = append(c.sshConfig.Auth, ssh.PublicKeys(signer)) 104 | } 105 | 106 | // If a password was specified, add password auth to the list of auth methods 107 | // to try. 108 | password, _ := config.Config(ConfigPassword) 109 | if password != "" { 110 | c.sshConfig.Auth = append(c.sshConfig.Auth, ssh.Password(password)) 111 | } 112 | 113 | // Require at least 1 authentication method. 114 | if len(c.sshConfig.Auth) == 0 { 115 | return nil, errors.New("no authentication methods specified") 116 | } 117 | 118 | // Start by ignoring host keys. If a host key is specified in the configuration, 119 | // use that to validate the remote server. 120 | c.sshConfig.HostKeyCallback = ssh.InsecureIgnoreHostKey() 121 | if hostKey, ok := config.Config(ConfigHostPublicKey); ok && hostKey != "" { 122 | _, _, parsedHostKey, _, _, err := ssh.ParseKnownHosts([]byte(hostKey)) 123 | if err != nil { 124 | return nil, errors.Wrap(err, "parsing host key") 125 | } 126 | 127 | c.sshConfig.HostKeyCallback = ssh.FixedHostKey(parsedHostKey) 128 | } 129 | 130 | c.basePath, ok = config.Config(ConfigBasePath) 131 | if !ok || c.basePath == "" { 132 | c.basePath = "." 133 | } 134 | 135 | return &c, nil 136 | } 137 | 138 | func init() { 139 | validatefn := func(config stow.Config) error { 140 | _, err := parseConfig(config) 141 | return err 142 | } 143 | makefn := func(config stow.Config) (stow.Location, error) { 144 | c, err := parseConfig(config) 145 | if err != nil { 146 | return nil, err 147 | } 148 | 149 | loc := &location{ 150 | config: c, 151 | } 152 | 153 | // Connect to the remote server and perform the SSH handshake. 154 | loc.sshClient, err = ssh.Dial("tcp", c.Host(), &c.sshConfig) 155 | if err != nil { 156 | return nil, errors.Wrap(err, "ssh connection") 157 | } 158 | 159 | // Open an SFTP session over an existing ssh connection. 160 | loc.sftpClient, err = sftp.NewClient(loc.sshClient) 161 | if err != nil { 162 | // close the ssh connection if the sftp connection fails. This avoids leaking 163 | // the ssh connection. 164 | loc.Close() 165 | return nil, errors.Wrap(err, "sftp connection") 166 | } 167 | 168 | return loc, nil 169 | } 170 | 171 | kindfn := func(u *url.URL) bool { 172 | return u.Scheme == Kind 173 | } 174 | 175 | stow.Register(Kind, makefn, kindfn, validatefn) 176 | } 177 | -------------------------------------------------------------------------------- /sftp/container.go: -------------------------------------------------------------------------------- 1 | package sftp 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | 10 | "github.com/graymeta/stow" 11 | ) 12 | 13 | type container struct { 14 | name string 15 | location *location 16 | } 17 | 18 | // ID returns a string value which represents the name of the container. 19 | func (c *container) ID() string { 20 | return c.name 21 | } 22 | 23 | // Name returns a string value which represents the name of the container. 24 | func (c *container) Name() string { 25 | return c.name 26 | } 27 | 28 | // Item returns a stow.Item instance of a container based on the name of the 29 | // container and the file. 30 | func (c *container) Item(id string) (stow.Item, error) { 31 | path := filepath.Join(c.location.config.basePath, c.name, filepath.FromSlash(id)) 32 | info, err := c.location.sftpClient.Stat(path) 33 | if err != nil { 34 | if os.IsNotExist(err) { 35 | return nil, stow.ErrNotFound 36 | } 37 | return nil, err 38 | } 39 | 40 | if info.IsDir() { 41 | return nil, errors.New("unexpected directory") 42 | } 43 | 44 | return &item{ 45 | container: c, 46 | path: id, 47 | size: info.Size(), 48 | modTime: info.ModTime(), 49 | md: getFileMetadata(info), 50 | }, nil 51 | } 52 | 53 | // Items sends a request to retrieve a list of items that are prepended with 54 | // the prefix argument. The 'cursor' variable facilitates pagination. 55 | func (c *container) Items(prefix, cursor string, count int) ([]stow.Item, string, error) { 56 | var entries []entry 57 | entries, cursor, err := c.getFolderItems([]entry{}, prefix, "", filepath.Join(c.location.config.basePath, c.name), cursor, count, false) 58 | if err != nil { 59 | return nil, "", err 60 | } 61 | 62 | // Convert entries to []stow.Item 63 | sItems := make([]stow.Item, len(entries)) 64 | for i := range entries { 65 | sItems[i] = entries[i].item 66 | } 67 | 68 | return sItems, cursor, nil 69 | } 70 | 71 | const separator = "/" 72 | 73 | // we use this struct to keep track of stuff when walking. 74 | type entry struct { 75 | Name string 76 | ID string 77 | RelPath string 78 | item stow.Item 79 | } 80 | 81 | func (c *container) getFolderItems(entries []entry, prefix, relPath, id, cursor string, limit int, initialStart bool) ([]entry, string, error) { 82 | relCursor := cursor 83 | if relPath != "" { 84 | relCursor = strings.TrimPrefix(cursor, relPath+separator) 85 | } 86 | cursorPieces := strings.Split(relCursor, separator) 87 | files, err := c.location.sftpClient.ReadDir(id) 88 | if err != nil { 89 | return nil, "", err 90 | } 91 | 92 | var start bool 93 | if cursorPieces[0] == "" { 94 | start = true 95 | } 96 | 97 | for i, file := range files { 98 | fileRelPath := filepath.Join(relPath, file.Name()) 99 | if !start && cursorPieces[0] != "" && (file.Name() >= cursorPieces[0] || fileRelPath >= cursor) { 100 | start = true 101 | if file.Name() == cursorPieces[0] && !file.IsDir() { 102 | continue 103 | } 104 | } 105 | 106 | if !start { 107 | continue 108 | } 109 | 110 | if file.IsDir() { 111 | var err error 112 | var retCursor string 113 | entries, retCursor, err = c.getFolderItems(entries, prefix, fileRelPath, filepath.Join(id, file.Name()), cursor, limit, true) 114 | if err != nil { 115 | return nil, "", err 116 | } 117 | if len(entries) != limit { 118 | continue 119 | } 120 | 121 | if retCursor == "" && i < len(files)-1 { 122 | retCursor = entries[len(entries)-1].RelPath 123 | } 124 | 125 | return entries, retCursor, nil 126 | } 127 | 128 | // TODO: prefix could be optimized to not look down paths that don't match, 129 | // but this is a quick/cheap first implementation. 130 | filePath := strings.TrimPrefix(filepath.Join(id, file.Name()), filepath.Join(c.location.config.basePath, c.name)+"/") 131 | if !strings.HasPrefix(filePath, prefix) { 132 | continue 133 | } 134 | 135 | entries = append( 136 | entries, 137 | entry{ 138 | Name: file.Name(), 139 | ID: filepath.Join(id, file.Name()), 140 | RelPath: fileRelPath, 141 | item: &item{ 142 | container: c, 143 | path: filePath, 144 | size: file.Size(), 145 | modTime: file.ModTime(), 146 | md: getFileMetadata(file), 147 | }, 148 | }, 149 | ) 150 | if len(entries) == limit { 151 | retCursor := entries[len(entries)-1].RelPath 152 | if i == len(files)-1 { 153 | retCursor = "" 154 | } 155 | return entries, retCursor, nil 156 | } 157 | } 158 | 159 | return entries, "", nil 160 | } 161 | 162 | // RemoveItem removes a file from the remote server. 163 | func (c *container) RemoveItem(id string) error { 164 | return c.location.sftpClient.Remove(filepath.Join(c.location.config.basePath, c.name, filepath.FromSlash(id))) 165 | } 166 | 167 | // Put sends a request to upload content to the container. 168 | func (c *container) Put(name string, r io.Reader, size int64, metadata map[string]interface{}) (stow.Item, error) { 169 | if len(metadata) > 0 { 170 | return nil, stow.NotSupported("metadata") 171 | } 172 | 173 | path := filepath.Join(c.location.config.basePath, c.name, filepath.FromSlash(name)) 174 | item := &item{ 175 | container: c, 176 | path: name, 177 | size: size, 178 | } 179 | err := c.location.sftpClient.MkdirAll(filepath.Dir(path)) 180 | if err != nil { 181 | return nil, err 182 | } 183 | f, err := c.location.sftpClient.Create(path) 184 | if err != nil { 185 | return nil, err 186 | } 187 | defer f.Close() 188 | n, err := io.Copy(f, r) 189 | if err != nil { 190 | return nil, err 191 | } 192 | if n != size { 193 | return nil, errors.New("bad size") 194 | } 195 | 196 | info, err := c.location.sftpClient.Stat(path) 197 | if err != nil { 198 | return nil, err 199 | } 200 | item.modTime = info.ModTime() 201 | item.md = getFileMetadata(info) 202 | 203 | return item, nil 204 | } 205 | -------------------------------------------------------------------------------- /sftp/item.go: -------------------------------------------------------------------------------- 1 | package sftp 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/url" 7 | "os" 8 | "path/filepath" 9 | "time" 10 | 11 | "github.com/graymeta/stow/local" 12 | ) 13 | 14 | type item struct { 15 | container *container 16 | path string 17 | size int64 18 | modTime time.Time 19 | md map[string]interface{} 20 | } 21 | 22 | // ID returns a string value that represents the name of a file. 23 | func (i *item) ID() string { 24 | return i.path 25 | } 26 | 27 | // Name returns a string value that represents the name of the file. 28 | func (i *item) Name() string { 29 | return i.path 30 | } 31 | 32 | // Size returns the size of an item in bytes. 33 | func (i *item) Size() (int64, error) { 34 | return i.size, nil 35 | } 36 | 37 | // URL returns a formatted string identifying this asset. 38 | // Format is: sftp://@:// 39 | func (i *item) URL() *url.URL { 40 | genericURL := fmt.Sprintf("/%s/%s", i.container.Name(), i.Name()) 41 | return &url.URL{ 42 | Scheme: Kind, 43 | User: url.User(i.container.location.config.sshConfig.User), 44 | Path: genericURL, 45 | Host: i.container.location.config.Host(), 46 | } 47 | } 48 | 49 | // Open retrieves specic information about an item based on the container name 50 | // and path of the file within the container. This response includes the body of 51 | // resource which is returned along with an error. 52 | func (i *item) Open() (io.ReadCloser, error) { 53 | return i.container.location.sftpClient.Open( 54 | filepath.Join( 55 | i.container.location.config.basePath, 56 | i.container.Name(), 57 | i.Name(), 58 | ), 59 | ) 60 | } 61 | 62 | // LastMod returns the last modified date of the item. 63 | func (i *item) LastMod() (time.Time, error) { 64 | return i.modTime, nil 65 | } 66 | 67 | // ETag returns the ETag value from the properies field of an item. 68 | func (i *item) ETag() (string, error) { 69 | return i.modTime.String(), nil 70 | } 71 | 72 | // Metadata returns some item level metadata about the item. 73 | func (i *item) Metadata() (map[string]interface{}, error) { 74 | return i.md, nil 75 | } 76 | 77 | func getFileMetadata(info os.FileInfo) map[string]interface{} { 78 | return map[string]interface{}{ 79 | // Reuse the constants from the local package for consistency. 80 | local.MetadataIsDir: info.IsDir(), 81 | local.MetadataName: info.Name(), 82 | local.MetadataMode: fmt.Sprintf("%o", info.Mode()), 83 | local.MetadataModeD: fmt.Sprintf("%v", uint32(info.Mode())), 84 | local.MetadataPerm: info.Mode().String(), 85 | local.MetadataSize: info.Size(), 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /sftp/location.go: -------------------------------------------------------------------------------- 1 | package sftp 2 | 3 | import ( 4 | "net/url" 5 | "os" 6 | "path/filepath" 7 | "sort" 8 | "strings" 9 | 10 | "github.com/graymeta/stow" 11 | "github.com/hashicorp/go-multierror" 12 | "github.com/pkg/errors" 13 | "github.com/pkg/sftp" 14 | "golang.org/x/crypto/ssh" 15 | ) 16 | 17 | type location struct { 18 | // we keep config here so that we can access the server information (username/host) 19 | // when constructing the item urls. 20 | config *conf 21 | sshClient *ssh.Client 22 | sftpClient *sftp.Client 23 | } 24 | 25 | // CreateContainer creates a new container, in this case a directory on the remote server. 26 | func (l *location) CreateContainer(containerName string) (stow.Container, error) { 27 | if err := l.sftpClient.Mkdir(filepath.Join(l.config.basePath, containerName)); err != nil { 28 | return nil, err 29 | } 30 | 31 | return &container{ 32 | location: l, 33 | name: containerName, 34 | }, nil 35 | } 36 | 37 | // Containers returns a slice of the Container interface, a cursor, and an error. 38 | func (l *location) Containers(prefix, cursor string, count int) ([]stow.Container, string, error) { 39 | infos, err := l.sftpClient.ReadDir(l.config.basePath) 40 | if err != nil { 41 | return nil, "", err 42 | } 43 | 44 | sort.Slice(infos, func(i, j int) bool { return infos[i].Name() < infos[j].Name() }) 45 | 46 | var cont []stow.Container 47 | for i, v := range infos { 48 | if !v.IsDir() { 49 | continue 50 | } 51 | 52 | if !strings.HasPrefix(v.Name(), prefix) { 53 | continue 54 | } 55 | 56 | if v.Name() <= cursor { 57 | continue 58 | } 59 | 60 | cont = append(cont, &container{ 61 | location: l, 62 | name: v.Name(), 63 | }) 64 | 65 | if len(cont) == count { 66 | // Check if any of the remaining entries are directories. If they are, then return 67 | // v.Name() as the cursor. This prevents us from serving up an empty page of 68 | // containers if all of the entries after v.Name() are files. 69 | for j := i + 1; j < len(infos); j++ { 70 | if infos[j].IsDir() { 71 | return cont, v.Name(), nil 72 | } 73 | } 74 | return cont, "", nil 75 | } 76 | } 77 | 78 | return cont, "", nil 79 | } 80 | 81 | // Close closes the underlying sftp/ssh connections. 82 | func (l *location) Close() error { 83 | var errs error 84 | 85 | if l.sftpClient != nil { 86 | if err := l.sftpClient.Close(); err != nil { 87 | errs = multierror.Append(errs, errors.Wrap(err, "closing sftp conn")) 88 | } 89 | } 90 | 91 | if l.sshClient != nil { 92 | if err := l.sshClient.Close(); err != nil { 93 | errs = multierror.Append(errs, errors.Wrap(err, "closing ssh conn")) 94 | } 95 | } 96 | 97 | return errs 98 | } 99 | 100 | // Container retrieves a stow.Container based on its name which must be exact. 101 | func (l *location) Container(id string) (stow.Container, error) { 102 | fi, err := l.sftpClient.Stat(filepath.Join(l.config.basePath, id)) 103 | if err != nil { 104 | if os.IsNotExist(err) { 105 | return nil, stow.ErrNotFound 106 | } 107 | return nil, err 108 | } 109 | if !fi.IsDir() { 110 | return nil, stow.ErrNotFound 111 | } 112 | return &container{ 113 | location: l, 114 | name: id, 115 | }, nil 116 | } 117 | 118 | // RemoveContainer removes a container by name. 119 | func (l *location) RemoveContainer(id string) error { 120 | return recurseRemove(l.sftpClient, filepath.Join(l.config.basePath, id)) 121 | } 122 | 123 | // recurseRemove recursively purges content from a path. 124 | func recurseRemove(client *sftp.Client, path string) error { 125 | infos, err := client.ReadDir(path) 126 | if err != nil { 127 | return err 128 | } 129 | 130 | for _, v := range infos { 131 | if !v.IsDir() { 132 | return errors.Errorf("directory not empty - %q", v.Name()) 133 | } 134 | if err := recurseRemove(client, filepath.Join(path, v.Name())); err != nil { 135 | return err 136 | } 137 | } 138 | 139 | return client.RemoveDirectory(path) 140 | } 141 | 142 | // ItemByURL retrieves a stow.Item by parsing the URL. 143 | func (l *location) ItemByURL(u *url.URL) (stow.Item, error) { 144 | // expect sftp://@:// 145 | // example: sftp://someuser@example.com:22/foo/blah/baz.txt 146 | 147 | urlParts := strings.Split(u.Path, "/") 148 | if len(urlParts) < 3 { 149 | return nil, errors.New("parsing ItemByURL unexpected length") 150 | } 151 | 152 | containerName := urlParts[1] 153 | itemName := strings.Join(urlParts[2:], "/") 154 | 155 | c, err := l.Container(containerName) 156 | if err != nil { 157 | return nil, errors.Wrapf(err, "ItemByURL, getting container %q", containerName) 158 | } 159 | 160 | i, err := c.Item(itemName) 161 | if err != nil { 162 | return nil, errors.Wrapf(err, "ItemByURL, getting item %q", itemName) 163 | } 164 | 165 | return i, nil 166 | } 167 | -------------------------------------------------------------------------------- /sftp/stow_test.go: -------------------------------------------------------------------------------- 1 | package sftp 2 | 3 | import ( 4 | "io/ioutil" 5 | "math/rand" 6 | "os" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/graymeta/stow" 12 | "github.com/graymeta/stow/test" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func TestStow(t *testing.T) { 17 | config := stow.ConfigMap{ 18 | ConfigHost: os.Getenv("SFTP_HOST"), 19 | ConfigPort: os.Getenv("SFTP_PORT"), 20 | ConfigUsername: os.Getenv("SFTP_USERNAME"), 21 | } 22 | if config[ConfigHost] == "" || 23 | config[ConfigPort] == "" || 24 | config[ConfigUsername] == "" { 25 | t.Skip("skipping tests because environment isn't configured") 26 | } 27 | 28 | b, err := ioutil.ReadFile(os.Getenv("SFTP_PRIVATE_KEY_FILE")) 29 | require.NoError(t, err) 30 | config[ConfigPrivateKey] = string(b) 31 | config[ConfigPrivateKeyPassphrase] = os.Getenv("SFTP_PRIVATE_KEY_PASSPHRASE") 32 | 33 | t.Run("stow_tests", func(t *testing.T) { 34 | test.All(t, Kind, config) 35 | }) 36 | 37 | t.Run("additional_tests", func(t *testing.T) { 38 | location, err := stow.Dial(Kind, config) 39 | require.NoError(t, err) 40 | defer location.Close() 41 | 42 | t.Run("set of files 1", func(t *testing.T) { 43 | cont, err := location.CreateContainer("stowtest" + randName(10)) 44 | require.NoError(t, err) 45 | defer location.RemoveContainer(cont.ID()) 46 | 47 | files := []string{ 48 | "a.jpg", 49 | "bar/a.jpg", 50 | "bar/b.jpg", 51 | "bar/baz/a.jpg", 52 | "bar/baz/b.jpg", 53 | "foo/a.jpg", 54 | "foo/b.jpg", 55 | "z.jpg", 56 | } 57 | 58 | setupFiles(t, cont, files) 59 | 60 | t.Run("no prefix, no cursor, len 4", func(t *testing.T) { 61 | items, cursor, err := cont.Items("", "", 4) 62 | require.NoError(t, err) 63 | require.Len(t, items, 4) 64 | require.Equal(t, files[0], items[0].ID()) 65 | require.Equal(t, files[1], items[1].ID()) 66 | require.Equal(t, files[2], items[2].ID()) 67 | require.Equal(t, files[3], items[3].ID()) 68 | require.Equal(t, cursor, "bar/baz/a.jpg") 69 | }) 70 | 71 | t.Run("no prefix, no cursor, len 5", func(t *testing.T) { 72 | items, cursor, err := cont.Items("", "", 5) 73 | require.NoError(t, err) 74 | require.Len(t, items, 5) 75 | require.Equal(t, files[0], items[0].ID()) 76 | require.Equal(t, files[1], items[1].ID()) 77 | require.Equal(t, files[2], items[2].ID()) 78 | require.Equal(t, files[3], items[3].ID()) 79 | require.Equal(t, files[4], items[4].ID()) 80 | require.Equal(t, cursor, "bar/baz/b.jpg") 81 | }) 82 | 83 | t.Run("no prefix, no cursor, len 100", func(t *testing.T) { 84 | items, cursor, err := cont.Items("", "", 100) 85 | require.NoError(t, err) 86 | require.Len(t, items, len(files)) 87 | require.Equal(t, "", cursor) 88 | }) 89 | 90 | t.Run("no prefix, with cursor, len 100", func(t *testing.T) { 91 | items, cursor, err := cont.Items("", "bar/baz/a.jpg", 100) 92 | require.NoError(t, err) 93 | require.Len(t, items, 4) 94 | require.Equal(t, "", cursor) 95 | }) 96 | }) 97 | 98 | t.Run("set of files 2", func(t *testing.T) { 99 | cont, err := location.CreateContainer("stowtest" + randName(10)) 100 | require.NoError(t, err) 101 | defer location.RemoveContainer(cont.ID()) 102 | 103 | files := []string{ 104 | "bar/baz/a.jpg", 105 | "bar/baz/b.jpg", 106 | "bar/baz/c.jpg", 107 | "bar/baz/d.jpg", 108 | "bar/baz/e.jpg", 109 | } 110 | 111 | setupFiles(t, cont, files) 112 | 113 | t.Run("no prefix, no cursor, len 3", func(t *testing.T) { 114 | items, cursor, err := cont.Items("", "", 3) 115 | require.NoError(t, err) 116 | require.Len(t, items, 3) 117 | require.Equal(t, files[2], cursor) 118 | }) 119 | 120 | t.Run("no prefix, no cursor, len 5", func(t *testing.T) { 121 | items, cursor, err := cont.Items("", "", 5) 122 | require.NoError(t, err) 123 | require.Len(t, items, 5) 124 | require.Equal(t, "", cursor) 125 | }) 126 | 127 | t.Run("no prefix, with cursor, len 5", func(t *testing.T) { 128 | items, cursor, err := cont.Items("", "bar/baz/b.jpg", 5) 129 | require.NoError(t, err) 130 | require.Len(t, items, 3) 131 | require.Equal(t, "", cursor) 132 | }) 133 | 134 | t.Run("no prefix, with cursor (a), len 5", func(t *testing.T) { 135 | items, cursor, err := cont.Items("", "a", 5) 136 | require.NoError(t, err) 137 | require.Len(t, items, 5) 138 | require.Equal(t, "", cursor) 139 | }) 140 | }) 141 | }) 142 | 143 | t.Run("stow_tests - with base path", func(t *testing.T) { 144 | basePath := os.Getenv("SFTP_BASEPATH") 145 | if basePath == "" { 146 | t.Skip("skipping base paths test due to SFTP_BASEPATH not being set") 147 | } 148 | config[ConfigBasePath] = basePath 149 | test.All(t, Kind, config) 150 | }) 151 | } 152 | 153 | func setupFiles(t *testing.T, c stow.Container, files []string) { 154 | for _, file := range files { 155 | _, err := c.Put(file, strings.NewReader(""), 0, nil) 156 | require.NoError(t, err) 157 | } 158 | } 159 | 160 | var letters = []rune("abcdefghijklmnopqrstuvwxyz") 161 | 162 | func init() { 163 | rand.Seed(int64(time.Now().Nanosecond())) 164 | } 165 | 166 | func randName(length int) string { 167 | b := make([]rune, length) 168 | for i := range b { 169 | b[i] = letters[rand.Intn(len(letters))] 170 | } 171 | return string(b) 172 | } 173 | -------------------------------------------------------------------------------- /stow-aeroplane.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WasabiAiR/stow/973a61f346d598a566affb53c4698764a67df164/stow-aeroplane.png -------------------------------------------------------------------------------- /stow-definition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WasabiAiR/stow/973a61f346d598a566affb53c4698764a67df164/stow-definition.png -------------------------------------------------------------------------------- /stow_test.go: -------------------------------------------------------------------------------- 1 | package stow_test 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | "testing" 7 | 8 | "github.com/cheekybits/is" 9 | "github.com/graymeta/stow" 10 | ) 11 | 12 | func TestKindByURL(t *testing.T) { 13 | is := is.New(t) 14 | u, err := url.Parse("test://container/item") 15 | is.NoErr(err) 16 | kind, err := stow.KindByURL(u) 17 | is.NoErr(err) 18 | is.Equal(kind, testKind) 19 | } 20 | 21 | func TestKinds(t *testing.T) { 22 | is := is.New(t) 23 | stow.Register("example", nil, nil, nil) 24 | is.Equal(stow.Kinds(), []string{"test", "example"}) 25 | } 26 | 27 | func TestIsCursorEnd(t *testing.T) { 28 | is := is.New(t) 29 | is.True(stow.IsCursorEnd("")) 30 | is.False(stow.IsCursorEnd("anything")) 31 | } 32 | 33 | func TestErrNotSupported(t *testing.T) { 34 | is := is.New(t) 35 | err := errors.New("something") 36 | is.False(stow.IsNotSupported(err)) 37 | err = stow.NotSupported("feature") 38 | is.True(stow.IsNotSupported(err)) 39 | } 40 | 41 | func TestDuplicateKinds(t *testing.T) { 42 | is := is.New(t) 43 | stow.Register("example", nil, nil, nil) 44 | is.Equal(stow.Kinds(), []string{"test", "example"}) 45 | stow.Register("example", nil, nil, nil) 46 | is.Equal(stow.Kinds(), []string{"test", "example"}) 47 | } 48 | -------------------------------------------------------------------------------- /swift/config.go: -------------------------------------------------------------------------------- 1 | package swift 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | "net/url" 7 | 8 | "github.com/graymeta/stow" 9 | "github.com/ncw/swift" 10 | ) 11 | 12 | // Config key constants. 13 | const ( 14 | ConfigUsername = "username" 15 | ConfigKey = "key" 16 | ConfigTenantName = "tenant_name" 17 | ConfigTenantAuthURL = "tenant_auth_url" 18 | ) 19 | 20 | // Kind is the kind of Location this package provides. 21 | const Kind = "swift" 22 | 23 | func init() { 24 | validatefn := func(config stow.Config) error { 25 | _, ok := config.Config(ConfigUsername) 26 | if !ok { 27 | return errors.New("missing account username") 28 | } 29 | _, ok = config.Config(ConfigKey) 30 | if !ok { 31 | return errors.New("missing api key") 32 | } 33 | _, ok = config.Config(ConfigTenantName) 34 | if !ok { 35 | return errors.New("missing tenant name") 36 | } 37 | _, ok = config.Config(ConfigTenantAuthURL) 38 | if !ok { 39 | return errors.New("missing tenant auth url") 40 | } 41 | return nil 42 | } 43 | makefn := func(config stow.Config) (stow.Location, error) { 44 | _, ok := config.Config(ConfigUsername) 45 | if !ok { 46 | return nil, errors.New("missing account username") 47 | } 48 | _, ok = config.Config(ConfigKey) 49 | if !ok { 50 | return nil, errors.New("missing api key") 51 | } 52 | _, ok = config.Config(ConfigTenantName) 53 | if !ok { 54 | return nil, errors.New("missing tenant name") 55 | } 56 | _, ok = config.Config(ConfigTenantAuthURL) 57 | if !ok { 58 | return nil, errors.New("missing tenant auth url") 59 | } 60 | l := &location{ 61 | config: config, 62 | } 63 | var err error 64 | l.client, err = newSwiftClient(l.config) 65 | if err != nil { 66 | return nil, err 67 | } 68 | return l, nil 69 | } 70 | kindfn := func(u *url.URL) bool { 71 | return u.Scheme == Kind 72 | } 73 | stow.Register(Kind, makefn, kindfn, validatefn) 74 | } 75 | 76 | func newSwiftClient(cfg stow.Config) (*swift.Connection, error) { 77 | username, _ := cfg.Config(ConfigUsername) 78 | key, _ := cfg.Config(ConfigKey) 79 | tenantName, _ := cfg.Config(ConfigTenantName) 80 | tenantAuthURL, _ := cfg.Config(ConfigTenantAuthURL) 81 | client := swift.Connection{ 82 | UserName: username, 83 | ApiKey: key, 84 | AuthUrl: tenantAuthURL, 85 | //Domain: "domain", // Name of the domain (v3 auth only) 86 | Tenant: tenantName, // Name of the tenant (v2 auth only) 87 | // Add Default transport 88 | Transport: http.DefaultTransport, 89 | } 90 | err := client.Authenticate() 91 | if err != nil { 92 | return nil, errors.New("Unable to authenticate") 93 | } 94 | return &client, nil 95 | } 96 | -------------------------------------------------------------------------------- /swift/container.go: -------------------------------------------------------------------------------- 1 | package swift 2 | 3 | import ( 4 | "io" 5 | "strings" 6 | 7 | "github.com/pkg/errors" 8 | 9 | "github.com/graymeta/stow" 10 | "github.com/ncw/swift" 11 | ) 12 | 13 | type container struct { 14 | id string 15 | client *swift.Connection 16 | } 17 | 18 | var _ stow.Container = (*container)(nil) 19 | 20 | func (c *container) ID() string { 21 | return c.id 22 | } 23 | 24 | func (c *container) Name() string { 25 | return c.id 26 | } 27 | 28 | func (c *container) Item(id string) (stow.Item, error) { 29 | return c.getItem(id) 30 | } 31 | 32 | func (c *container) Items(prefix, cursor string, count int) ([]stow.Item, string, error) { 33 | params := &swift.ObjectsOpts{ 34 | Limit: count, 35 | Marker: cursor, 36 | Prefix: prefix, 37 | } 38 | objects, err := c.client.Objects(c.id, params) 39 | if err != nil { 40 | return nil, "", err 41 | } 42 | items := make([]stow.Item, len(objects)) 43 | for i, obj := range objects { 44 | 45 | items[i] = &item{ 46 | id: obj.Name, 47 | container: c, 48 | client: c.client, 49 | hash: obj.Hash, 50 | size: obj.Bytes, 51 | lastModified: obj.LastModified, 52 | } 53 | } 54 | marker := "" 55 | if len(objects) == count { 56 | marker = objects[len(objects)-1].Name 57 | } 58 | return items, marker, nil 59 | } 60 | 61 | func (c *container) Put(name string, r io.Reader, size int64, metadata map[string]interface{}) (stow.Item, error) { 62 | mdPrepped, err := prepMetadata(metadata) 63 | if err != nil { 64 | return nil, errors.Wrap(err, "unable to create or update Item, preparing metadata") 65 | } 66 | 67 | headers, err := c.client.ObjectPut(c.id, name, r, false, "", "", mdPrepped) 68 | if err != nil { 69 | return nil, errors.Wrap(err, "unable to create or update Item") 70 | } 71 | 72 | mdParsed, err := parseMetadata(headers) 73 | if err != nil { 74 | return nil, errors.Wrap(err, "unable to create or update Item, parsing metadata") 75 | } 76 | 77 | item := &item{ 78 | id: name, 79 | container: c, 80 | client: c.client, 81 | size: size, 82 | metadata: mdParsed, 83 | } 84 | return item, nil 85 | } 86 | 87 | func (c *container) RemoveItem(id string) error { 88 | return c.client.ObjectDelete(c.id, id) 89 | } 90 | 91 | func (c *container) getItem(id string) (*item, error) { 92 | info, headers, err := c.client.Object(c.id, id) 93 | if err != nil { 94 | if strings.Contains(err.Error(), "Object Not Found") { 95 | return nil, stow.ErrNotFound 96 | } 97 | return nil, errors.Wrap(err, "error retrieving item") 98 | } 99 | 100 | md, err := parseMetadata(headers) 101 | if err != nil { 102 | return nil, errors.Wrap(err, "unable to retrieve Item information, parsing metadata") 103 | } 104 | 105 | item := &item{ 106 | id: id, 107 | container: c, 108 | client: c.client, 109 | hash: info.Hash, 110 | size: info.Bytes, 111 | lastModified: info.LastModified, 112 | metadata: md, 113 | } 114 | return item, nil 115 | } 116 | 117 | // Keys are returned as all lowercase, dashes are allowed 118 | func parseMetadata(md swift.Headers) (map[string]interface{}, error) { 119 | m := make(map[string]interface{}, len(md)) 120 | for key, value := range md.ObjectMetadata() { 121 | m[key] = value 122 | } 123 | return m, nil 124 | } 125 | 126 | // TODO determine invalid keys. 127 | func prepMetadata(md map[string]interface{}) (map[string]string, error) { 128 | m := make(map[string]string, len(md)) 129 | for key, value := range md { 130 | str, ok := value.(string) 131 | if !ok { 132 | return nil, errors.New("could not convert key value") // add a msg mentioning strings only? 133 | } 134 | m["X-Object-Meta-"+key] = str 135 | } 136 | return m, nil 137 | } 138 | -------------------------------------------------------------------------------- /swift/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package swift provides an absraction of the Openstack Swift storage technology. An Openstack Swift Container is represented by a Stow Container, and an Openstack Swift Object is represented by a Stow Item. Note that directories may exist within a Swift Container. 3 | 4 | Usage and Credentials 5 | 6 | Four pieces of information are needed: the user's name and password which will be used to access the Swift endpoint, the tenant name used to identify the storage endpoint, and the authentication URL. 7 | 8 | stow.Dial requires both a string value of the particular Stow Location Kind ("swift") and a stow.Config instance. The stow.Config instance requires two entries with the specific key value attributes: 9 | 10 | - a key of swift.ConfigUsername with a value of the user account name 11 | - a key of swift.ConfigKey with a value of the user account password 12 | - a key of swift.ConfigTenantName with a value of the Swift endpoint's tenant name 13 | - a key of swift.ConfigTenantAuthURL with a value of the Swift endpoint's authentication URL 14 | 15 | Location 16 | 17 | Methods of swift.location allow the retrieval of a Swift Container (Container or Containers). A stow.Item representation of a Swift Object can also be retrieved based on the Object's URL (ItemByURL). 18 | 19 | Additional swift.location methods provide capabilities to create and remove Swift Containers (CreateContainer or RemoveContainer). 20 | 21 | Container 22 | 23 | Methods of stow.container allow the retrieval of a Swift Container's: 24 | 25 | - name(ID or Name) 26 | - item or complete list of items (Item or Items, respectively) 27 | 28 | Additional methods of swift.container allow Stow to: 29 | 30 | - remove a stow.Item (RemoveItem) 31 | - update or create a stow.Item (Put) 32 | 33 | Item 34 | 35 | Methods of swift.Item allow the retrieval of a Swift Object's: 36 | - name (ID or name) 37 | - URL 38 | - size in bytes 39 | - Object specific metadata (information stored within the service) 40 | - last modified date 41 | - Etag 42 | - content 43 | */ 44 | package swift 45 | -------------------------------------------------------------------------------- /swift/item.go: -------------------------------------------------------------------------------- 1 | package swift 2 | 3 | import ( 4 | "io" 5 | "net/url" 6 | "path" 7 | "sync" 8 | "time" 9 | 10 | "github.com/graymeta/stow" 11 | "github.com/ncw/swift" 12 | ) 13 | 14 | type item struct { 15 | id string 16 | container *container 17 | client *swift.Connection 18 | //properties az.BlobProperties 19 | hash string 20 | size int64 21 | url url.URL 22 | lastModified time.Time 23 | metadata map[string]interface{} 24 | infoOnce sync.Once 25 | infoErr error 26 | } 27 | 28 | var _ stow.Item = (*item)(nil) 29 | 30 | func (i *item) ID() string { 31 | return i.id 32 | } 33 | 34 | func (i *item) Name() string { 35 | return i.id 36 | } 37 | 38 | func (i *item) URL() *url.URL { 39 | // StorageUrl looks like this: 40 | // https://lax-proxy-03.storagesvc.sohonet.com/v1/AUTH_b04239c7467548678b4822e9dad96030 41 | // We want something like this: 42 | // swift://lax-proxy-03.storagesvc.sohonet.com/v1/AUTH_b04239c7467548678b4822e9dad96030// 43 | url, _ := url.Parse(i.client.StorageUrl) 44 | url.Scheme = Kind 45 | url.Path = path.Join(url.Path, i.container.id, i.id) 46 | return url 47 | } 48 | 49 | func (i *item) Size() (int64, error) { 50 | return i.size, nil 51 | } 52 | 53 | func (i *item) Open() (io.ReadCloser, error) { 54 | r, _, err := i.client.ObjectOpen(i.container.id, i.id, false, nil) 55 | return r, err 56 | } 57 | 58 | func (i *item) ETag() (string, error) { 59 | err := i.ensureInfo() 60 | if err != nil { 61 | return "", err 62 | } 63 | return i.hash, nil 64 | } 65 | 66 | func (i *item) LastMod() (time.Time, error) { 67 | err := i.ensureInfo() 68 | if err != nil { 69 | return time.Time{}, err 70 | } 71 | return i.lastModified, nil 72 | } 73 | 74 | // Metadata returns a map of key value pairs representing an Item's metadata 75 | func (i *item) Metadata() (map[string]interface{}, error) { 76 | err := i.ensureInfo() 77 | if err != nil { 78 | return nil, err 79 | } 80 | return i.metadata, nil 81 | } 82 | 83 | // ensureInfo checks the fields that may be empty when an item is PUT. 84 | // Verify if the fields are empty, get information on the item, fill in 85 | // the missing fields. 86 | func (i *item) ensureInfo() error { 87 | // If lastModified is empty, so is hash. get info on the Item and 88 | // update the necessary fields at the same time. 89 | if i.lastModified.IsZero() || i.hash == "" || i.metadata == nil { 90 | i.infoOnce.Do(func() { 91 | itemInfo, infoErr := i.getInfo() 92 | if infoErr != nil { 93 | i.infoErr = infoErr 94 | return 95 | } 96 | i.hash, i.infoErr = itemInfo.ETag() 97 | if infoErr != nil { 98 | i.infoErr = infoErr 99 | return 100 | } 101 | i.lastModified, i.infoErr = itemInfo.LastMod() 102 | if infoErr != nil { 103 | i.infoErr = infoErr 104 | return 105 | } 106 | i.metadata, i.infoErr = itemInfo.Metadata() 107 | if infoErr != nil { 108 | i.infoErr = infoErr 109 | return 110 | } 111 | }) 112 | } 113 | return i.infoErr 114 | } 115 | 116 | func (i *item) getInfo() (stow.Item, error) { 117 | itemInfo, err := i.container.getItem(i.ID()) 118 | if err != nil { 119 | return nil, err 120 | } 121 | return itemInfo, nil 122 | } 123 | -------------------------------------------------------------------------------- /swift/location.go: -------------------------------------------------------------------------------- 1 | package swift 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | "strings" 7 | 8 | "github.com/graymeta/stow" 9 | "github.com/ncw/swift" 10 | ) 11 | 12 | type location struct { 13 | config stow.Config 14 | client *swift.Connection 15 | } 16 | 17 | func (l *location) Close() error { 18 | return nil // nothing to close 19 | } 20 | 21 | func (l *location) CreateContainer(name string) (stow.Container, error) { 22 | err := l.client.ContainerCreate(name, nil) 23 | if err != nil { 24 | return nil, err 25 | } 26 | container := &container{ 27 | id: name, 28 | client: l.client, 29 | } 30 | return container, nil 31 | } 32 | 33 | func (l *location) Containers(prefix, cursor string, count int) ([]stow.Container, string, error) { 34 | params := &swift.ContainersOpts{ 35 | Limit: count, 36 | Prefix: prefix, 37 | Marker: cursor, 38 | } 39 | response, err := l.client.Containers(params) 40 | if err != nil { 41 | return nil, "", err 42 | } 43 | containers := make([]stow.Container, len(response)) 44 | for i, cont := range response { 45 | containers[i] = &container{ 46 | id: cont.Name, 47 | client: l.client, 48 | // count: cont.Count, 49 | // bytes: cont.Bytes, 50 | } 51 | } 52 | marker := "" 53 | if len(response) == count { 54 | marker = response[len(response)-1].Name 55 | } 56 | return containers, marker, nil 57 | } 58 | 59 | func (l *location) Container(id string) (stow.Container, error) { 60 | _, _, err := l.client.Container(id) 61 | // TODO: grab info + headers 62 | if err != nil { 63 | return nil, stow.ErrNotFound 64 | } 65 | 66 | c := &container{ 67 | id: id, 68 | client: l.client, 69 | } 70 | 71 | return c, nil 72 | } 73 | 74 | func (l *location) ItemByURL(url *url.URL) (stow.Item, error) { 75 | 76 | if url.Scheme != Kind { 77 | return nil, errors.New("not valid swift URL") 78 | } 79 | 80 | path := strings.TrimLeft(url.Path, "/") 81 | pieces := strings.SplitN(path, "/", 4) 82 | 83 | // swift://lax-proxy-03.storagesvc.sohonet.com/v1/AUTH_b04239c7467548678b4822e9dad96030// 84 | 85 | c, err := l.Container(pieces[2]) 86 | if err != nil { 87 | return nil, err 88 | } 89 | 90 | return c.Item(pieces[3]) 91 | } 92 | 93 | func (l *location) RemoveContainer(id string) error { 94 | return l.client.ContainerDelete(id) 95 | } 96 | -------------------------------------------------------------------------------- /swift/stow_test.go: -------------------------------------------------------------------------------- 1 | package swift 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/cheekybits/is" 9 | "github.com/graymeta/stow" 10 | "github.com/graymeta/stow/test" 11 | ) 12 | 13 | func TestStow(t *testing.T) { 14 | username := os.Getenv("SWIFTUSERNAME") 15 | key := os.Getenv("SWIFTKEY") 16 | tenantName := os.Getenv("SWIFTTENANTNAME") 17 | tenantAuthUrl := os.Getenv("SWIFTTENANTAUTHURL") 18 | 19 | if username == "" || key == "" || tenantName == "" || tenantAuthUrl == "" { 20 | t.Skip("skipping test because missing one or more of SWIFTUSERNAME SWIFTKEY SWIFTTENANTNAME SWIFTTENANTAUTHURL") 21 | } 22 | 23 | cfg := stow.ConfigMap{ 24 | "username": username, 25 | "key": key, 26 | "tenant_name": tenantName, 27 | "tenant_auth_url": tenantAuthUrl, 28 | //"tenant_id": "b04239c7467548678b4822e9dad96030", 29 | } 30 | test.All(t, "swift", cfg) 31 | } 32 | 33 | func TestPrepMetadataSuccess(t *testing.T) { 34 | is := is.New(t) 35 | 36 | m := make(map[string]string) 37 | m["one"] = "two" 38 | m["3"] = "4" 39 | m["ninety-nine"] = "100" 40 | 41 | m2 := make(map[string]interface{}) 42 | for key, value := range m { 43 | m2[key] = value 44 | } 45 | 46 | assertionM := make(map[string]string) 47 | assertionM["X-Object-Meta-one"] = "two" 48 | assertionM["X-Object-Meta-3"] = "4" 49 | assertionM["X-Object-Meta-ninety-nine"] = "100" 50 | 51 | //returns map[string]interface 52 | returnedMap, err := prepMetadata(m2) 53 | is.NoErr(err) 54 | 55 | if !reflect.DeepEqual(returnedMap, assertionM) { 56 | t.Errorf("Expected map (%+v) and returned map (%+v) are not equal.", assertionM, returnedMap) 57 | } 58 | } 59 | 60 | func TestPrepMetadataFailureWithNonStringValues(t *testing.T) { 61 | is := is.New(t) 62 | 63 | m := make(map[string]interface{}) 64 | m["float"] = 8.9 65 | m["number"] = 9 66 | 67 | _, err := prepMetadata(m) 68 | is.Err(err) 69 | } 70 | -------------------------------------------------------------------------------- /test/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:7 2 | 3 | RUN yum -y install \ 4 | gcc \ 5 | git \ 6 | lsof \ 7 | make \ 8 | mercurial \ 9 | wget \ 10 | which \ 11 | && yum clean all 12 | 13 | ENV goversion 1.11.13 14 | ENV gofile go${goversion}.linux-amd64.tar.gz 15 | ENV gourl https://storage.googleapis.com/golang/${gofile} 16 | 17 | RUN wget -q -O /usr/local/${gofile} ${gourl} \ 18 | && mkdir /usr/local/go \ 19 | && tar -xzf /usr/local/${gofile} -C /usr/local/go --strip 1 20 | 21 | CMD cd /mnt/src/github.com/graymeta/stow && GO111MODULE=on GOPATH=/mnt PATH=/usr/local/go/bin:$GOPATH/bin:$PATH make test 22 | -------------------------------------------------------------------------------- /walk.go: -------------------------------------------------------------------------------- 1 | package stow 2 | 3 | // DEV NOTE: tests for this are in test/test.go 4 | 5 | // WalkFunc is a function called for each Item visited 6 | // by Walk. 7 | // If there was a problem, the incoming error will describe 8 | // the problem and the function can decide how to handle 9 | // that error. 10 | // If an error is returned, processing stops. 11 | type WalkFunc func(item Item, err error) error 12 | 13 | // Walk walks all Items in the Container. 14 | // Returns the first error returned by the WalkFunc or 15 | // nil if no errors were returned. 16 | // The pageSize is the number of Items to get per request. 17 | func Walk(container Container, prefix string, pageSize int, fn WalkFunc) error { 18 | var ( 19 | err error 20 | items []Item 21 | cursor = CursorStart 22 | ) 23 | for { 24 | items, cursor, err = container.Items(prefix, cursor, pageSize) 25 | if err != nil { 26 | err = fn(nil, err) 27 | if err != nil { 28 | return err 29 | } 30 | } 31 | for _, item := range items { 32 | err = fn(item, nil) 33 | if err != nil { 34 | return err 35 | } 36 | } 37 | if IsCursorEnd(cursor) { 38 | break 39 | } 40 | } 41 | return nil 42 | } 43 | 44 | // WalkContainersFunc is a function called for each Container visited 45 | // by WalkContainers. 46 | // If there was a problem, the incoming error will describe 47 | // the problem and the function can decide how to handle 48 | // that error. 49 | // If an error is returned, processing stops. 50 | type WalkContainersFunc func(container Container, err error) error 51 | 52 | // WalkContainers walks all Containers in the Location. 53 | // Returns the first error returned by the WalkContainersFunc or 54 | // nil if no errors were returned. 55 | // The pageSize is the number of Containers to get per request. 56 | func WalkContainers(location Location, prefix string, pageSize int, fn WalkContainersFunc) error { 57 | var ( 58 | err error 59 | containers []Container 60 | cursor = CursorStart 61 | ) 62 | for { 63 | containers, cursor, err = location.Containers(prefix, cursor, pageSize) 64 | if err != nil { 65 | err = fn(nil, err) 66 | if err != nil { 67 | return err 68 | } 69 | } 70 | for _, container := range containers { 71 | err = fn(container, nil) 72 | if err != nil { 73 | return err 74 | } 75 | } 76 | if IsCursorEnd(cursor) { 77 | break 78 | } 79 | } 80 | return nil 81 | } 82 | --------------------------------------------------------------------------------