├── storing.go ├── testdata └── testfile.txt ├── .codeclimate.yml ├── .gitignore ├── awss3 ├── awss3_test.go └── awss3.go ├── README.md └── mock └── storing.go /storing.go: -------------------------------------------------------------------------------- 1 | package storing 2 | 3 | // Storing interface 4 | type Storing interface { 5 | Upload(string, string, []byte) (string, error) 6 | Download(string) ([]byte, error) 7 | Provider() string 8 | Delete(string) error 9 | } 10 | -------------------------------------------------------------------------------- /testdata/testfile.txt: -------------------------------------------------------------------------------- 1 | This adventure is made possible by generations of searchers 2 | strictly adherent to a simple set of rules. Test ideas by 3 | experiments and observations. Build on those ideas that pass 4 | the test. Reject the ones that fail. Follow the evidence 5 | wherever it leads, and question everything. Accept these terms, 6 | and the cosmos is yours. 7 | -------------------------------------------------------------------------------- /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | --- 2 | engines: 3 | golint: 4 | enabled: true 5 | gofmt: 6 | enabled: true 7 | govet: 8 | enabled: true 9 | fixme: 10 | enabled: true 11 | duplication: 12 | enabled: true 13 | config: 14 | languages: 15 | - go 16 | ratings: 17 | paths: 18 | - "**.go" 19 | exclude_paths: 20 | - .vendor/ 21 | - vendor/ 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | -------------------------------------------------------------------------------- /awss3/awss3_test.go: -------------------------------------------------------------------------------- 1 | package awss3 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "testing" 7 | ) 8 | 9 | func TestS3(t *testing.T) { 10 | s, err := New() 11 | if err != nil { 12 | t.Fatal(err) 13 | } 14 | bUp, err := ioutil.ReadFile("../testdata/testfile.txt") 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | 19 | // Upload 20 | _, err = s.Upload("testfile.txt", "text/plain", bUp) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | 25 | // Download 26 | var bDown []byte 27 | bDown, err = s.Download("testfile.txt") 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | 32 | if !bytes.Equal(bUp, bDown) { 33 | t.Fatal("Uploaded data are different from the data received in the download.") 34 | } 35 | 36 | // Delete 37 | err = s.Delete("testfile.txt") 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | 42 | _, err = s.Download("testfile.txt") 43 | if err == nil { 44 | t.Fatal("An error was expected") 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # storing 2 | Take care of writing and reading on AWS S3 and others storage providers 3 | 4 | ## Environment setting 5 | 6 | To use with AWS it is necessary to set the following environment variables: 7 | ```shell 8 | export AWS_ACCESS_KEY_ID=XXXXXXXXX 9 | export AWS_SECRET_ACCESS_KEY=XXXXXXXXX 10 | export AWS_REGION=XXXXXXXXX 11 | export AWS_BUCKET=XXXXXXXXX 12 | export AWS_ACL=XXXXXXXXX 13 | ``` 14 | 15 | ## Installation 16 | 17 | ```shell 18 | go get github.com/nuveo/storing 19 | ``` 20 | 21 | ## Examples 22 | 23 | ### Upload 24 | ```go 25 | // read file to upload 26 | bUp, err := ioutil.ReadFile("../testdata/testfile.txt") 27 | if err != nil { 28 | panic(err) 29 | } 30 | 31 | // Upload 32 | var path string 33 | path, err = Upload("testfile.txt", "text/plain", bUp) 34 | if err != nil { 35 | panic(err) 36 | } 37 | println(path) 38 | ``` 39 | 40 | ### Download 41 | ```go 42 | bDown, err := Download("testfile.txt") 43 | if err != nil { 44 | panic(err) 45 | } 46 | println(string(bDown)) 47 | ``` 48 | 49 | ### Delete 50 | ```go 51 | err := Delete("testfile.txt") 52 | if err != nil { 53 | panic(err) 54 | } 55 | ``` 56 | -------------------------------------------------------------------------------- /mock/storing.go: -------------------------------------------------------------------------------- 1 | package mock 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | // Item mock storing 8 | type Item struct { 9 | ContentType string 10 | Content []byte 11 | } 12 | 13 | // Storing mock 14 | type Storing struct { 15 | mtx sync.RWMutex 16 | UploadErr error 17 | DownloadErr error 18 | DeleteErr error 19 | Items map[string]Item 20 | } 21 | 22 | // New mock storing 23 | func New() *Storing { 24 | return &Storing{ 25 | Items: make(map[string]Item), 26 | } 27 | } 28 | 29 | // Provider returns the name of the provider of the current adapter. 30 | func (s *Storing) Provider() string { 31 | return "Mock" 32 | } 33 | 34 | // Upload upload file to mock 35 | func (s *Storing) Upload(name string, contentType string, content []byte) (path string, err error) { 36 | err = s.UploadErr 37 | if err == nil { 38 | s.mtx.Lock() 39 | s.Items[name] = Item{ContentType: contentType, Content: content} 40 | s.mtx.Unlock() 41 | path = name 42 | } 43 | return 44 | } 45 | 46 | // Download file from mock 47 | func (s *Storing) Download(path string) (b []byte, err error) { 48 | err = s.DownloadErr 49 | if err == nil { 50 | s.mtx.Lock() 51 | b = s.Items[path].Content 52 | s.mtx.Unlock() 53 | } 54 | return 55 | } 56 | 57 | // Delete from mock 58 | func (s *Storing) Delete(key string) (err error) { 59 | err = s.DeleteErr 60 | if err == nil { 61 | s.mtx.Lock() 62 | delete(s.Items, key) 63 | s.mtx.Unlock() 64 | } 65 | return 66 | } 67 | -------------------------------------------------------------------------------- /awss3/awss3.go: -------------------------------------------------------------------------------- 1 | package awss3 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "os" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/s3" 11 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 12 | "github.com/nuveo/log" 13 | ) 14 | 15 | // Storing implementation for aws s3 16 | type Storing struct { 17 | Session *session.Session 18 | Bucket string 19 | ACL string 20 | } 21 | 22 | // New s3 storing 23 | func New(opts ...func(*Storing) error) (st *Storing, err error) { 24 | ss, err := session.NewSession() 25 | if err != nil { 26 | return 27 | } 28 | st = &Storing{ 29 | Session: ss, 30 | ACL: os.Getenv("AWS_ACL"), 31 | Bucket: os.Getenv("AWS_BUCKET"), 32 | } 33 | for _, opt := range opts { 34 | if err = opt(st); err != nil { 35 | st = nil 36 | return 37 | } 38 | } 39 | return 40 | } 41 | 42 | // ACL is an option to set when storing is created 43 | func ACL(acl string) func(*Storing) error { 44 | return func(st *Storing) error { 45 | st.ACL = acl 46 | return nil 47 | } 48 | } 49 | 50 | // Bucket is an option to set when storing is created 51 | func Bucket(bucket string) func(*Storing) error { 52 | return func(st *Storing) error { 53 | st.Bucket = bucket 54 | return nil 55 | } 56 | } 57 | 58 | // CustomSession is an option to set when storing is created 59 | func CustomSession(s *session.Session) func(*Storing) error { 60 | return func(st *Storing) error { 61 | st.Session = s 62 | return nil 63 | } 64 | } 65 | 66 | // Provider returns the name of the provider of the current adapter. 67 | func (s *Storing) Provider() string { 68 | return "s3" 69 | } 70 | 71 | // Upload upload file to S3 72 | func (s *Storing) Upload(name string, contentType string, content []byte) (path string, err error) { 73 | log.Printf("upload file to s3 %v\n", name) 74 | uploader := s3manager.NewUploader(s.Session) 75 | 76 | var file *s3manager.UploadOutput 77 | file, err = uploader.Upload(&s3manager.UploadInput{ 78 | Bucket: aws.String(s.Bucket), 79 | ACL: aws.String(s.ACL), 80 | Key: aws.String(name), 81 | ContentType: aws.String(contentType), 82 | Body: bytes.NewReader(content), 83 | }) 84 | if err != nil { 85 | return 86 | } 87 | path = file.Location 88 | return 89 | } 90 | 91 | // Download file from s3 92 | func (s *Storing) Download(path string) (b []byte, err error) { 93 | log.Printf("download file to s3 %v\n", path) 94 | downloader := s3manager.NewDownloader(s.Session) 95 | if err != nil { 96 | return 97 | } 98 | 99 | tmpfile, err := ioutil.TempFile("", "nuveo") 100 | if err != nil { 101 | return 102 | } 103 | defer func() { 104 | rmErr := os.Remove(tmpfile.Name()) 105 | if rmErr != nil { 106 | log.Errorln(rmErr) 107 | } 108 | }() 109 | 110 | _, err = downloader.Download(tmpfile, &s3.GetObjectInput{ 111 | Bucket: aws.String(s.Bucket), 112 | Key: aws.String(path), 113 | }) 114 | if err != nil { 115 | return 116 | } 117 | 118 | b, err = ioutil.ReadAll(tmpfile) 119 | return 120 | } 121 | 122 | // Delete from s3 123 | func (s *Storing) Delete(key string) (err error) { 124 | log.Printf("download file to s3 %v\n", key) 125 | svc := s3.New(s.Session) 126 | obj := &s3.DeleteObjectInput{ 127 | Bucket: aws.String(s.Bucket), 128 | Key: aws.String(key), 129 | } 130 | _, err = svc.DeleteObject(obj) 131 | return 132 | } 133 | --------------------------------------------------------------------------------