├── .gitignore
├── LICENSE
├── README.md
├── acl.go
├── bucket.go
├── bytes.go
├── client.go
├── delete.go
├── error.go
├── examples
    ├── bucket
    │   └── main.go
    ├── list
    │   └── main.go
    └── s3
    │   └── main.go
├── get.go
├── go.mod
├── go.sum
├── http.go
├── list.go
├── object.go
├── sign.go
├── trace.go
├── upload.go
└── util.go


/.gitignore:
--------------------------------------------------------------------------------
1 | /s3
2 | /bucket
3 | /list
4 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2017 James Hunt
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to
 7 | deal in the Software without restriction, including without limitation the
 8 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 9 | sell copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in
13 | all copies or substantial portions of the Software..
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 | IN THE SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | go-s3
 2 | =====
 3 | 
 4 | A simple library for interfacing with Amazon S3 from Go.
 5 | 
 6 | Example
 7 | -------
 8 | 
 9 | ```go
10 | package main
11 | 
12 | import (
13 | 	"fmt"
14 | 	"io"
15 | 	"os"
16 | 
17 | 	"github.com/jhunt/go-s3"
18 | )
19 | 
20 | func main() {
21 | 	/* ... some setup ... */
22 | 
23 | 	c, err := s3.NewClient(&s3.Client{
24 | 		AccessKeyID:     aki,
25 | 		SecretAccessKey: key,
26 | 		Region:          reg,
27 | 		Bucket:          bkt,
28 | 	})
29 | 	if err != nil {
30 | 		fmt.Fprintf(os.Stderr, "!!! unable to configure s3 client: %s\n", err)
31 | 		os.Exit(1)
32 | 	}
33 | 
34 | 	u, err := c.NewUpload(path)
35 | 	if err != nil {
36 | 		fmt.Fprintf(os.Stderr, "!!! unable to start multipart upload: %s\n", err)
37 | 		os.Exit(1)
38 | 	}
39 | 
40 | 	n, err := u.Stream(os.Stdin, 5*1024*1024*1024)
41 | 	if err != nil {
42 | 		fmt.Fprintf(os.Stderr, "!!! unable to stream <stdin> in 5m parts: %s\n", err)
43 | 		os.Exit(1)
44 | 	}
45 | 
46 | 	err = u.Done()
47 | 	if err != nil {
48 | 		fmt.Fprintf(os.Stderr, "!!! unable to complete multipart upload: %s\n", err)
49 | 		os.Exit(1)
50 | 	}
51 | }
52 | ```
53 | 
54 | Environment Variables
55 | ---------------------
56 | 
57 | The following environment variables affect the behavior of this
58 | library:
59 | 
60 |   - `$S3_TRACE` - If set to "yes", "y", or "1" (case-insensitive),
61 |     any and all HTTP(S) requests will be dumped to standard error.
62 |     If set to the value "header" or "headers", only the headers of
63 |     requests and responses will be dumped.
64 | 


--------------------------------------------------------------------------------
/acl.go:
--------------------------------------------------------------------------------
 1 | package s3
 2 | 
 3 | import (
 4 | 	"encoding/xml"
 5 | 	"io/ioutil"
 6 | 	"net/http"
 7 | )
 8 | 
 9 | type Grant struct {
10 | 	GranteeID   string
11 | 	GranteeName string
12 | 	Group       string
13 | 	Permission  string
14 | }
15 | 
16 | const EveryoneURI = "http://acs.amazonaws.com/groups/global/AllUsers"
17 | 
18 | type ACL []Grant
19 | 
20 | func (c *Client) GetACL(key string) (ACL, error) {
21 | 	res, err := c.get(key+"?acl", nil)
22 | 	if err != nil {
23 | 		return nil, err
24 | 	}
25 | 
26 | 	if res.StatusCode != 200 {
27 | 		return nil, ResponseError(res)
28 | 	}
29 | 
30 | 	var r struct {
31 | 		XMLName xml.Name `xml:"AccessControlPolicy"`
32 | 		List    struct {
33 | 			Grant []struct {
34 | 				Grantee struct {
35 | 					ID   string `xml:"ID"`
36 | 					Name string `xml:"DisplayName"`
37 | 					URI  string `xml:"URI"`
38 | 				} `xml:"Grantee"`
39 | 				Permission string `xml:"Permission"`
40 | 			} `xml:"Grant"`
41 | 		} `xml:"AccessControlList"`
42 | 	}
43 | 
44 | 	b, err := ioutil.ReadAll(res.Body)
45 | 	if err != nil {
46 | 		return nil, err
47 | 	}
48 | 	if err := xml.Unmarshal(b, &r); err != nil {
49 | 		return nil, err
50 | 	}
51 | 
52 | 	var acl ACL
53 | 	for _, g := range r.List.Grant {
54 | 		group := ""
55 | 		if g.Grantee.URI == EveryoneURI {
56 | 			group = "EVERYONE"
57 | 		}
58 | 		acl = append(acl, Grant{
59 | 			GranteeID:   g.Grantee.ID,
60 | 			GranteeName: g.Grantee.Name,
61 | 			Group:       group,
62 | 			Permission:  g.Permission,
63 | 		})
64 | 	}
65 | 	return acl, nil
66 | }
67 | 
68 | func (c *Client) ChangeACL(path, acl string) error {
69 | 	headers := make(http.Header)
70 | 	headers.Set("x-amz-acl", acl)
71 | 
72 | 	res, err := c.put(path+"?acl", nil, &headers)
73 | 	if err != nil {
74 | 		return err
75 | 	}
76 | 
77 | 	if res.StatusCode != 200 {
78 | 		return ResponseError(res)
79 | 	}
80 | 
81 | 	return nil
82 | }
83 | 


--------------------------------------------------------------------------------
/bucket.go:
--------------------------------------------------------------------------------
  1 | package s3
  2 | 
  3 | import (
  4 | 	"encoding/xml"
  5 | 	"fmt"
  6 | 	"io/ioutil"
  7 | 	"net/http"
  8 | 	"regexp"
  9 | 	"time"
 10 | )
 11 | 
 12 | const (
 13 | 	PrivateACL                = "private"
 14 | 	PublicReadACL             = "public-read"
 15 | 	PublicReadWriteACL        = "public-read-write"
 16 | 	AWSExecReadACL            = "aws-exec-read"
 17 | 	AuthenticatedReadACL      = "authenticated-read"
 18 | 	BucketOwnerReadACL        = "bucket-owner-read"
 19 | 	BucketOwnerFullControlACL = "bucket-owner-full-control"
 20 | 	LogDeliveryWriteACL       = "log-delivery-write"
 21 | )
 22 | 
 23 | func (c *Client) CreateBucket(name, region, acl string) error {
 24 | 	/* validate that the bucket name is:
 25 | 
 26 | 	   - between 3 and 63 characters long (inclusive)
 27 | 	   - not include periods (for TLS wildcard matching)
 28 | 	   - lower case
 29 | 	   - rfc952 compliant
 30 | 	*/
 31 | 	if ok, _ := regexp.MatchString(`^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$`, name); !ok {
 32 | 		return fmt.Errorf("invalid s3 bucket name")
 33 | 	}
 34 | 
 35 | 	was := c.Bucket
 36 | 	defer func() { c.Bucket = was }()
 37 | 	c.Bucket = name
 38 | 
 39 | 	b := []byte{}
 40 | 	if region != "" {
 41 | 		var payload struct {
 42 | 			XMLName xml.Name `xml:"CreateBucketConfiguration"`
 43 | 			Region  string   `xml:"LocationConstraint"`
 44 | 		}
 45 | 		payload.Region = region
 46 | 
 47 | 		var err error
 48 | 		b, err = xml.Marshal(payload)
 49 | 		if err != nil {
 50 | 			return err
 51 | 		}
 52 | 	}
 53 | 
 54 | 	headers := make(http.Header)
 55 | 	headers.Set("x-amz-acl", acl)
 56 | 
 57 | 	res, err := c.put("/", b, &headers)
 58 | 	if err != nil {
 59 | 		return err
 60 | 	}
 61 | 
 62 | 	if res.StatusCode != 200 {
 63 | 		return ResponseError(res)
 64 | 	}
 65 | 
 66 | 	return nil
 67 | }
 68 | 
 69 | func (c *Client) DeleteBucket(name string) error {
 70 | 	was := c.Bucket
 71 | 	defer func() { c.Bucket = was }()
 72 | 	c.Bucket = name
 73 | 
 74 | 	res, err := c.delete("/", nil)
 75 | 	if err != nil {
 76 | 		return err
 77 | 	}
 78 | 
 79 | 	if res.StatusCode != 204 {
 80 | 		return ResponseError(res)
 81 | 	}
 82 | 
 83 | 	return nil
 84 | }
 85 | 
 86 | type Bucket struct {
 87 | 	Name         string
 88 | 	CreationDate time.Time
 89 | 	OwnerID      string
 90 | 	OwnerName    string
 91 | }
 92 | 
 93 | func (c *Client) ListBuckets() ([]Bucket, error) {
 94 | 	prev := c.Bucket
 95 | 	c.Bucket = ""
 96 | 	res, err := c.get("/", nil)
 97 | 	c.Bucket = prev
 98 | 	if err != nil {
 99 | 		return nil, err
100 | 	}
101 | 
102 | 	var r struct {
103 | 		XMLName xml.Name `xml:"ListAllMyBucketsResult"`
104 | 		Owner   struct {
105 | 			ID          string `xml:"ID"`
106 | 			DisplayName string `xml:"DisplayName"`
107 | 		} `xml:"Owner"`
108 | 		Buckets struct {
109 | 			Bucket []struct {
110 | 				Name         string `xml:"Name"`
111 | 				CreationDate string `xml:"CreationDate"`
112 | 			} `xml:"Bucket"`
113 | 		} `xml:"Buckets"`
114 | 	}
115 | 	b, err := ioutil.ReadAll(res.Body)
116 | 	if err != nil {
117 | 		return nil, err
118 | 	}
119 | 
120 | 	if res.StatusCode != 200 {
121 | 		return nil, ResponseErrorFrom(b)
122 | 	}
123 | 
124 | 	err = xml.Unmarshal(b, &r)
125 | 	if err != nil {
126 | 		return nil, err
127 | 	}
128 | 
129 | 	s := make([]Bucket, len(r.Buckets.Bucket))
130 | 	for i, bkt := range r.Buckets.Bucket {
131 | 		s[i].OwnerID = r.Owner.ID
132 | 		s[i].OwnerName = r.Owner.DisplayName
133 | 		s[i].Name = bkt.Name
134 | 
135 | 		created, _ := time.Parse("2006-01-02T15:04:05.000Z", bkt.CreationDate)
136 | 		s[i].CreationDate = created
137 | 	}
138 | 	return s, nil
139 | }
140 | 


--------------------------------------------------------------------------------
/bytes.go:
--------------------------------------------------------------------------------
 1 | package s3
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | )
 6 | 
 7 | type Bytes int64
 8 | 
 9 | func (b Bytes) String() string {
10 | 	if b < 1<<10 {
11 | 		return b.Bytes()
12 | 	}
13 | 	if b < 1<<20 {
14 | 		return b.Kilobytes()
15 | 	}
16 | 	if b < 1<<30 {
17 | 		return b.Megabytes()
18 | 	}
19 | 	if b < 1<<40 {
20 | 		return b.Gigabytes()
21 | 	}
22 | 	if b < 1<<50 {
23 | 		return b.Terabytes()
24 | 	}
25 | 	if b < 1<<60 {
26 | 		return b.Petabytes()
27 | 	}
28 | 	return b.Exabytes()
29 | }
30 | 
31 | func (b Bytes) Bytes() string {
32 | 	return fmt.Sprintf("%db", b)
33 | }
34 | 
35 | func (b Bytes) Kilobytes() string {
36 | 	return fmt.Sprintf("%dk", b/(1<<10))
37 | }
38 | 
39 | func (b Bytes) Megabytes() string {
40 | 	return fmt.Sprintf("%dm", b/(1<<20))
41 | }
42 | 
43 | func (b Bytes) Gigabytes() string {
44 | 	return fmt.Sprintf("%dg", b/(1<<30))
45 | }
46 | 
47 | func (b Bytes) Terabytes() string {
48 | 	return fmt.Sprintf("%dt", b/(1<<40))
49 | }
50 | 
51 | func (b Bytes) Petabytes() string {
52 | 	return fmt.Sprintf("%dp", b/(1<<50))
53 | }
54 | 
55 | func (b Bytes) Exabytes() string {
56 | 	return fmt.Sprintf("%dx", b/(1<<60))
57 | }
58 | 


--------------------------------------------------------------------------------
/client.go:
--------------------------------------------------------------------------------
  1 | package s3
  2 | 
  3 | import (
  4 | 	"crypto/tls"
  5 | 	"crypto/x509"
  6 | 	"fmt"
  7 | 	"io"
  8 | 	"net/http"
  9 | 	"os"
 10 | 	"strings"
 11 | 
 12 | 	"golang.org/x/net/proxy"
 13 | )
 14 | 
 15 | type Client struct {
 16 | 	AccessKeyID     string
 17 | 	SecretAccessKey string
 18 | 	Token           string
 19 | 	Region          string
 20 | 	Bucket          string
 21 | 	Domain          string
 22 | 	Protocol        string
 23 | 	SOCKS5Proxy     string
 24 | 
 25 | 	SignatureVersion int
 26 | 
 27 | 	CACertificates     []string
 28 | 	SkipSystemCAs      bool
 29 | 	InsecureSkipVerify bool
 30 | 
 31 | 	UsePathBuckets bool
 32 | 
 33 | 	ua *http.Client
 34 | 
 35 | 	trace     bool
 36 | 	traceTo   io.Writer
 37 | 	traceBody bool
 38 | }
 39 | 
 40 | func NewClient(c *Client) (*Client, error) {
 41 | 	var (
 42 | 		roots *x509.CertPool
 43 | 		err   error
 44 | 	)
 45 | 
 46 | 	if c.SignatureVersion == 0 {
 47 | 		c.SignatureVersion = 4
 48 | 	}
 49 | 
 50 | 	if trace := os.Getenv("S3_TRACE"); trace != "" {
 51 | 		switch strings.ToLower(trace) {
 52 | 		case "yes", "y", "1":
 53 | 			c.Trace(os.Stderr, true, true)
 54 | 		case "headers", "header":
 55 | 			c.Trace(os.Stderr, true, false)
 56 | 		default:
 57 | 			c.Trace(os.Stderr, false, false)
 58 | 		}
 59 | 	}
 60 | 
 61 | 	if !c.SkipSystemCAs {
 62 | 		roots, err = x509.SystemCertPool()
 63 | 		if err != nil {
 64 | 			return nil, fmt.Errorf("unable to retrieve system root certificate authorities: %s", err)
 65 | 		}
 66 | 	} else {
 67 | 		roots = x509.NewCertPool()
 68 | 	}
 69 | 
 70 | 	for _, ca := range c.CACertificates {
 71 | 		if ok := roots.AppendCertsFromPEM([]byte(ca)); !ok {
 72 | 			return nil, fmt.Errorf("unable to append CA certificate")
 73 | 		}
 74 | 	}
 75 | 
 76 | 	dial := http.DefaultTransport.(*http.Transport).Dial
 77 | 	if c.SOCKS5Proxy != "" {
 78 | 		dialer, err := proxy.SOCKS5("tcp", c.SOCKS5Proxy, nil, proxy.Direct)
 79 | 		if err != nil {
 80 | 			return nil, err
 81 | 		}
 82 | 		dial = dialer.Dial
 83 | 	}
 84 | 
 85 | 	c.ua = &http.Client{
 86 | 		Transport: &http.Transport{
 87 | 			Dial:  dial,
 88 | 			Proxy: http.ProxyFromEnvironment,
 89 | 			TLSClientConfig: &tls.Config{
 90 | 				RootCAs:            roots,
 91 | 				InsecureSkipVerify: c.InsecureSkipVerify,
 92 | 			},
 93 | 		},
 94 | 	}
 95 | 
 96 | 	return c, nil
 97 | }
 98 | 
 99 | func (c *Client) domain() string {
100 | 	if c.Domain == "" {
101 | 		return "s3.amazonaws.com"
102 | 	}
103 | 	return c.Domain
104 | }
105 | 


--------------------------------------------------------------------------------
/delete.go:
--------------------------------------------------------------------------------
 1 | package s3
 2 | 
 3 | func (c *Client) Delete(path string) error {
 4 | 	res, err := c.delete(path, nil)
 5 | 	if err != nil {
 6 | 		return err
 7 | 	}
 8 | 
 9 | 	if res.StatusCode != 204 {
10 | 		return ResponseError(res)
11 | 	}
12 | 
13 | 	return nil
14 | }
15 | 


--------------------------------------------------------------------------------
/error.go:
--------------------------------------------------------------------------------
 1 | package s3
 2 | 
 3 | import (
 4 | 	"encoding/xml"
 5 | 	"fmt"
 6 | 	"io/ioutil"
 7 | 	"net/http"
 8 | )
 9 | 
10 | func ResponseError(res *http.Response) error {
11 | 	b, err := ioutil.ReadAll(res.Body)
12 | 	if err != nil {
13 | 		return err
14 | 	}
15 | 	return ResponseErrorFrom(b)
16 | }
17 | 
18 | func ResponseErrorFrom(b []byte) error {
19 | 	var payload struct {
20 | 		XMLName xml.Name `xml:"Error"`
21 | 		Code    string   `xml:"Code"`
22 | 		Message string   `xml:"Message"`
23 | 	}
24 | 	if err := xml.Unmarshal(b, &payload); err != nil {
25 | 		return fmt.Errorf("unable to parse response xml: %s", err)
26 | 	}
27 | 
28 | 	return fmt.Errorf("%s (%s) [raw %s]", payload.Message, payload.Code, string(b))
29 | }
30 | 


--------------------------------------------------------------------------------
/examples/bucket/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"os"
 6 | 
 7 | 	"github.com/jhunt/go-s3"
 8 | )
 9 | 
10 | func main() {
11 | 	aki := os.Getenv("S3_AKI")
12 | 	if aki == "" {
13 | 		fmt.Fprintf(os.Stderr, "!!! no S3_AKI env var set...\n")
14 | 		os.Exit(1)
15 | 	}
16 | 
17 | 	key := os.Getenv("S3_KEY")
18 | 	if key == "" {
19 | 		fmt.Fprintf(os.Stderr, "!!! no S3_KEY env var set...\n")
20 | 		os.Exit(1)
21 | 	}
22 | 
23 | 	reg := os.Getenv("S3_REGION")
24 | 	/* region can be blank - no location constraint */
25 | 
26 | 	sigv := 4
27 | 	if os.Getenv("S3_SIGV2") != "" {
28 | 		sigv = 2
29 | 	}
30 | 
31 | 	if len(os.Args) != 3 {
32 | 		fmt.Fprintf(os.Stderr, "USAGE: bucket NAME ACL\n")
33 | 		os.Exit(1)
34 | 	}
35 | 	bkt := os.Args[1]
36 | 	acl := os.Args[2]
37 | 
38 | 	c, err := s3.NewClient(&s3.Client{
39 | 		AccessKeyID:      aki,
40 | 		SecretAccessKey:  key,
41 | 		Region:           "us-east-1", /* AWS requires bucket creation to go to us-east-1 */
42 | 		SignatureVersion: sigv,
43 | 	})
44 | 	if err != nil {
45 | 		fmt.Fprintf(os.Stderr, "!!! unable to configure s3 client: %s\n", err)
46 | 		os.Exit(1)
47 | 	}
48 | 
49 | 	err = c.CreateBucket(bkt, reg, acl)
50 | 	if err != nil {
51 | 		fmt.Fprintf(os.Stderr, "!!! unable to create bucket '%s': %s\n", bkt, err)
52 | 		os.Exit(1)
53 | 	}
54 | 
55 | 	err = c.DeleteBucket(bkt)
56 | 	if err != nil {
57 | 		fmt.Fprintf(os.Stderr, "!!! unable to delete bucket '%s': %s\n", bkt, err)
58 | 		os.Exit(1)
59 | 	}
60 | 
61 | 	os.Exit(0)
62 | }
63 | 


--------------------------------------------------------------------------------
/examples/list/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"os"
 6 | 
 7 | 	"github.com/jhunt/go-s3"
 8 | )
 9 | 
10 | func main() {
11 | 	aki := os.Getenv("S3_AKI")
12 | 	if aki == "" {
13 | 		fmt.Fprintf(os.Stderr, "!!! no S3_AKI env var set...\n")
14 | 		os.Exit(1)
15 | 	}
16 | 
17 | 	key := os.Getenv("S3_KEY")
18 | 	if key == "" {
19 | 		fmt.Fprintf(os.Stderr, "!!! no S3_KEY env var set...\n")
20 | 		os.Exit(1)
21 | 	}
22 | 
23 | 	bkt := os.Getenv("S3_BUCKET")
24 | 	if bkt == "" {
25 | 		fmt.Fprintf(os.Stderr, "!!! no S3_BUCKET env var set...\n")
26 | 		os.Exit(1)
27 | 	}
28 | 
29 | 	reg := os.Getenv("S3_REGION")
30 | 	if reg == "" {
31 | 		reg = "us-east-1"
32 | 	}
33 | 
34 | 	sigv := 4
35 | 	if os.Getenv("S3_SIGV2") != "" {
36 | 		sigv = 2
37 | 	}
38 | 
39 | 	if len(os.Args) != 1 {
40 | 		fmt.Fprintf(os.Stderr, "USAGE: list\n")
41 | 		os.Exit(1)
42 | 	}
43 | 
44 | 	c, err := s3.NewClient(&s3.Client{
45 | 		AccessKeyID:      aki,
46 | 		SecretAccessKey:  key,
47 | 		Bucket:           bkt,
48 | 		Region:           reg,
49 | 		SignatureVersion: sigv,
50 | 	})
51 | 	if err != nil {
52 | 		fmt.Fprintf(os.Stderr, "!!! unable to configure s3 client: %s\n", err)
53 | 		os.Exit(1)
54 | 	}
55 | 
56 | 	files, err := c.List()
57 | 	if err != nil {
58 | 		fmt.Fprintf(os.Stderr, "!!! unable to list files in bucket '%s': %s\n", bkt, err)
59 | 		os.Exit(1)
60 | 	}
61 | 
62 | 	for _, f := range files {
63 | 		fmt.Printf("- %s\n", f.Key)
64 | 	}
65 | 	os.Exit(0)
66 | }
67 | 


--------------------------------------------------------------------------------
/examples/s3/main.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"io"
 6 | 	"os"
 7 | 
 8 | 	"github.com/jhunt/go-s3"
 9 | )
10 | 
11 | func main() {
12 | 	aki := os.Getenv("S3_AKI")
13 | 	if aki == "" {
14 | 		fmt.Fprintf(os.Stderr, "!!! no S3_AKI env var set...\n")
15 | 		os.Exit(1)
16 | 	}
17 | 
18 | 	key := os.Getenv("S3_KEY")
19 | 	if key == "" {
20 | 		fmt.Fprintf(os.Stderr, "!!! no S3_KEY env var set...\n")
21 | 		os.Exit(1)
22 | 	}
23 | 
24 | 	bkt := os.Getenv("S3_BUCKET")
25 | 	if bkt == "" {
26 | 		fmt.Fprintf(os.Stderr, "!!! no S3_BUCKET env var set...\n")
27 | 		os.Exit(1)
28 | 	}
29 | 
30 | 	reg := os.Getenv("S3_REGION")
31 | 	if reg == "" {
32 | 		reg = "us-east-1"
33 | 	}
34 | 
35 | 	sigv := 4
36 | 	if os.Getenv("S3_SIGV2") != "" {
37 | 		sigv = 2
38 | 	}
39 | 
40 | 	if len(os.Args) != 2 {
41 | 		fmt.Fprintf(os.Stderr, "USAGE: s3 path/in/bucket <file >out\n")
42 | 		os.Exit(1)
43 | 	}
44 | 	path := os.Args[1]
45 | 
46 | 	c, err := s3.NewClient(&s3.Client{
47 | 		AccessKeyID:      aki,
48 | 		SecretAccessKey:  key,
49 | 		Region:           reg,
50 | 		Bucket:           bkt,
51 | 		SignatureVersion: sigv,
52 | 		UsePathBuckets:   os.Getenv("S3_PATH_BUCKET") != "",
53 | 	})
54 | 	if err != nil {
55 | 		fmt.Fprintf(os.Stderr, "!!! unable to configure s3 client: %s\n", err)
56 | 		os.Exit(1)
57 | 	}
58 | 
59 | 	u, err := c.NewUpload(path)
60 | 	if err != nil {
61 | 		fmt.Fprintf(os.Stderr, "!!! unable to start multipart upload: %s\n", err)
62 | 		os.Exit(1)
63 | 	}
64 | 
65 | 	n, err := u.Stream(os.Stdin, 5*1024*1024*1024)
66 | 	if err != nil {
67 | 		fmt.Fprintf(os.Stderr, "!!! unable to stream <stdin> in 5m parts: %s\n", err)
68 | 		os.Exit(1)
69 | 	}
70 | 
71 | 	err = u.Done()
72 | 	if err != nil {
73 | 		fmt.Fprintf(os.Stderr, "!!! unable to complete multipart upload: %s\n", err)
74 | 		os.Exit(1)
75 | 	}
76 | 
77 | 	fmt.Fprintf(os.Stderr, "wrote %d bytes\n", n)
78 | 
79 | 	out, err := c.Get(path)
80 | 	if err != nil {
81 | 		fmt.Fprintf(os.Stderr, "!!! unable to retrieve our uploaded file: %s\n", err)
82 | 		os.Exit(1)
83 | 	}
84 | 	io.Copy(os.Stdout, out)
85 | 
86 | 	err = c.Delete(path)
87 | 	if err != nil {
88 | 		fmt.Fprintf(os.Stderr, "!!! unable to remove our uploaded file: %s\n", err)
89 | 		os.Exit(1)
90 | 	}
91 | 
92 | 	os.Exit(0)
93 | }
94 | 


--------------------------------------------------------------------------------
/get.go:
--------------------------------------------------------------------------------
 1 | package s3
 2 | 
 3 | import (
 4 | 	"io"
 5 | )
 6 | 
 7 | func (c *Client) Get(key string) (io.Reader, error) {
 8 | 	res, err := c.get(key, nil)
 9 | 	if err != nil {
10 | 		return nil, err
11 | 	}
12 | 
13 | 	if res.StatusCode != 200 {
14 | 		return nil, ResponseError(res)
15 | 	}
16 | 
17 | 	return res.Body, nil
18 | }
19 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/jhunt/go-s3
 2 | 
 3 | go 1.14
 4 | 
 5 | require (
 6 | 	github.com/jhunt/go-ansi v0.0.0-20171120214513-52b391f2c38f
 7 | 	github.com/mattn/go-isatty v0.0.12 // indirect
 8 | 	golang.org/x/net v0.0.0-20171128172551-6921abc35dff
 9 | )
10 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/jhunt/go-ansi v0.0.0-20171120214513-52b391f2c38f h1:PTMStXfVTLSPBm1MyQ9HslFVAfvTo7YZj1a4kqmd5Ug=
2 | github.com/jhunt/go-ansi v0.0.0-20171120214513-52b391f2c38f/go.mod h1:zx5sSmwzYAXhfPcRBU7SuiAwK+8vC/LTgAoHyJiclzI=
3 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
4 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
5 | golang.org/x/net v0.0.0-20171128172551-6921abc35dff h1:pdX52r+M5ygFqwLJvCKnFCLBqXqQ71jIYybR+BawQJ0=
6 | golang.org/x/net v0.0.0-20171128172551-6921abc35dff/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
7 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
8 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
9 | 


--------------------------------------------------------------------------------
/http.go:
--------------------------------------------------------------------------------
 1 | package s3
 2 | 
 3 | import (
 4 | 	"bytes"
 5 | 	"fmt"
 6 | 	"net/http"
 7 | 	"regexp"
 8 | )
 9 | 
10 | func (c *Client) url(path string) string {
11 | 	if path == "" || path[0:1] != "/" {
12 | 		path = "/" + path
13 | 	}
14 | 	scheme := c.Protocol
15 | 	if scheme == "" {
16 | 		scheme = "https"
17 | 	}
18 | 
19 | 	if c.Bucket == "" {
20 | 		return fmt.Sprintf("%s://%s%s", scheme, c.domain(), path)
21 | 	}
22 | 
23 | 	if c.UsePathBuckets {
24 | 		return fmt.Sprintf("%s://%s/%s%s", scheme, c.domain(), c.Bucket, path)
25 | 	} else {
26 | 		return fmt.Sprintf("%s://%s.%s%s", scheme, c.Bucket, c.domain(), path)
27 | 	}
28 | }
29 | 
30 | func (c *Client) request(method, path string, payload []byte, headers *http.Header) (*http.Response, error) {
31 | 	in := bytes.NewBuffer(payload)
32 | 	req, err := http.NewRequest(method, c.url(path), in)
33 | 	if err != nil {
34 | 		return nil, err
35 | 	}
36 | 
37 | 	/* copy in any headers */
38 | 	if headers != nil {
39 | 		for header, values := range *headers {
40 | 			for _, value := range values {
41 | 				req.Header.Add(header, value)
42 | 			}
43 | 		}
44 | 	}
45 | 
46 | 	/* sign the request */
47 | 	req.ContentLength = int64(len(payload))
48 | 	req.Header.Set("Authorization", c.signature(req, payload))
49 | 
50 | 	/* stupid continuation tokens sometimes have literal +'s in them */
51 | 	req.URL.RawQuery = regexp.MustCompile(`\+`).ReplaceAllString(req.URL.RawQuery, "%2B")
52 | 
53 | 	/* optional debugging */
54 | 	if err := c.traceRequest(req); err != nil {
55 | 		return nil, err
56 | 	}
57 | 
58 | 	/* submit the request */
59 | 	res, err := c.ua.Do(req)
60 | 	if err != nil {
61 | 		return nil, err
62 | 	}
63 | 
64 | 	/* optional debugging */
65 | 	if err := c.traceResponse(res); err != nil {
66 | 		return nil, err
67 | 	}
68 | 	return res, nil
69 | }
70 | 
71 | func (c *Client) post(path string, payload []byte, headers *http.Header) (*http.Response, error) {
72 | 	return c.request("POST", path, payload, headers)
73 | }
74 | 
75 | func (c *Client) put(path string, payload []byte, headers *http.Header) (*http.Response, error) {
76 | 	return c.request("PUT", path, payload, headers)
77 | }
78 | 
79 | func (c *Client) get(path string, headers *http.Header) (*http.Response, error) {
80 | 	return c.request("GET", path, nil, headers)
81 | }
82 | 
83 | func (c *Client) delete(path string, headers *http.Header) (*http.Response, error) {
84 | 	return c.request("DELETE", path, nil, headers)
85 | }
86 | 


--------------------------------------------------------------------------------
/list.go:
--------------------------------------------------------------------------------
 1 | package s3
 2 | 
 3 | import (
 4 | 	"encoding/xml"
 5 | 	"fmt"
 6 | 	"io/ioutil"
 7 | 	"time"
 8 | )
 9 | 
10 | func (c *Client) List() ([]Object, error) {
11 | 	objects := make([]Object, 0)
12 | 	ctok := ""
13 | 	for {
14 | 		res, err := c.get(fmt.Sprintf("/?list-type=2&fetch-owner=true%s", ctok), nil)
15 | 		if err != nil {
16 | 			return nil, err
17 | 		}
18 | 
19 | 		var r struct {
20 | 			XMLName  xml.Name `xml:"ListBucketResult"`
21 | 			Next     string   `xml:"NextContinuationToken"`
22 | 			Contents []struct {
23 | 				Key          string `xml:"Key"`
24 | 				LastModified string `xml:"LastModified"`
25 | 				ETag         string `xml:"ETag"`
26 | 				Size         int64  `xml:"Size"`
27 | 				StorageClass string `xml:"StorageClass"`
28 | 				Owner        struct {
29 | 					ID          string `xml:"ID"`
30 | 					DisplayName string `xml:"DisplayName"`
31 | 				} `xml:"Owner"`
32 | 			} `xml:"Contents"`
33 | 		}
34 | 		b, err := ioutil.ReadAll(res.Body)
35 | 		if err != nil {
36 | 			return nil, err
37 | 		}
38 | 
39 | 		if res.StatusCode != 200 {
40 | 			return nil, ResponseErrorFrom(b)
41 | 		}
42 | 
43 | 		err = xml.Unmarshal(b, &r)
44 | 		if err != nil {
45 | 			return nil, err
46 | 		}
47 | 
48 | 		for _, f := range r.Contents {
49 | 			mod, _ := time.Parse("2006-01-02T15:04:05.000Z", f.LastModified)
50 | 			objects = append(objects, Object{
51 | 				Key:          f.Key,
52 | 				LastModified: mod,
53 | 				ETag:         f.ETag[1 : len(f.ETag)-1],
54 | 				Size:         Bytes(f.Size),
55 | 				StorageClass: f.StorageClass,
56 | 				OwnerID:      f.Owner.ID,
57 | 				OwnerName:    f.Owner.DisplayName,
58 | 			})
59 | 		}
60 | 
61 | 		if r.Next == "" {
62 | 			return objects, nil
63 | 		}
64 | 
65 | 		ctok = fmt.Sprintf("&continuation-token=%s", r.Next)
66 | 	}
67 | }
68 | 


--------------------------------------------------------------------------------
/object.go:
--------------------------------------------------------------------------------
 1 | package s3
 2 | 
 3 | import (
 4 | 	"time"
 5 | )
 6 | 
 7 | type Object struct {
 8 | 	Key          string
 9 | 	LastModified time.Time
10 | 	ETag         string
11 | 	Size         Bytes
12 | 	StorageClass string
13 | 	OwnerID      string
14 | 	OwnerName    string
15 | }
16 | 


--------------------------------------------------------------------------------
/sign.go:
--------------------------------------------------------------------------------
  1 | package s3
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	"crypto/hmac"
  6 | 	"crypto/sha1"
  7 | 	"crypto/sha256"
  8 | 	"fmt"
  9 | 	"net/http"
 10 | 	"regexp"
 11 | 	"sort"
 12 | 	"strings"
 13 | 	"time"
 14 | )
 15 | 
 16 | func v2Resource(bucket string, req *http.Request) []byte {
 17 | 	r := []byte(fmt.Sprintf("/%s%s", bucket, req.URL.Path))
 18 | 	if req.URL.RawQuery == "" {
 19 | 		return r
 20 | 	}
 21 | 
 22 | 	qq := strings.Split(req.URL.RawQuery, "&")
 23 | 	sort.Strings(qq)
 24 | 
 25 | 	ll := make([][]byte, len(qq))
 26 | 	for i := range qq {
 27 | 		kv := strings.SplitN(qq[i], "=", 2)
 28 | 		k := uriencode(kv[0], true)
 29 | 		if len(kv) == 2 {
 30 | 			v := uriencode(kv[1], true)
 31 | 
 32 | 			ll[i] = make([]byte, len(k)+1+len(v))
 33 | 			copy(ll[i], k)
 34 | 			ll[i][len(k)] = 0x3d
 35 | 			copy(ll[i][1+len(k):], v)
 36 | 
 37 | 		} else {
 38 | 			ll[i] = k
 39 | 		}
 40 | 	}
 41 | 
 42 | 	return bytes.Join([][]byte{
 43 | 		r,
 44 | 		bytes.Join(ll, []byte{0x26}),
 45 | 	}, []byte{0x3f})
 46 | }
 47 | 
 48 | func mac256(key, msg []byte) []byte {
 49 | 	h := hmac.New(sha256.New, key)
 50 | 	h.Write(msg)
 51 | 	return h.Sum(nil)
 52 | }
 53 | 
 54 | func (c *Client) signature(req *http.Request, raw []byte) string {
 55 | 	if c.SignatureVersion == 2 {
 56 | 		return c.v2signature(req, raw)
 57 | 	}
 58 | 	if c.SignatureVersion == 4 {
 59 | 		return c.v4signature(req, raw)
 60 | 	}
 61 | 	panic(fmt.Sprintf("unrecognized aws/s3 signature version %d", c.SignatureVersion))
 62 | }
 63 | 
 64 | func v2Headers(req *http.Request) []byte {
 65 | 	subset := make(map[string]string)
 66 | 	names := make([]string, 0)
 67 | 
 68 | 	for header := range req.Header {
 69 | 		lc := strings.ToLower(header)
 70 | 		if strings.HasPrefix(lc, "x-amz-") {
 71 | 			names = append(names, lc)
 72 | 			subset[lc] = strings.Trim(req.Header.Get(header), " \t\r\n\f") + "\n"
 73 | 		}
 74 | 	}
 75 | 	sort.Strings(names)
 76 | 
 77 | 	ll := make([][]byte, len(names))
 78 | 	for i, header := range names {
 79 | 		ll[i] = bytes.Join([][]byte{[]byte(header), []byte(subset[header])}, []byte{0x3a})
 80 | 	}
 81 | 	return bytes.Join(ll, []byte{})
 82 | }
 83 | 
 84 | func (c *Client) v2signature(req *http.Request, raw []byte) string {
 85 | 	now := time.Now().UTC()
 86 | 
 87 | 	req.Header.Set("x-amz-date", now.Format("20060102T150405Z"))
 88 | 	req.Header.Set("host", regexp.MustCompile(`:.*`).ReplaceAllString(req.URL.Host, ""))
 89 | 	if c.Token != "" {
 90 | 		req.Header.Set("X-Amz-Security-Token", c.Token)
 91 | 	}
 92 | 	//req.Header.Set("host", "go-s3-bd6cf051-8023-4d2b-8bf2-7aaa477862ea.s3.amazonaws.com")
 93 | 
 94 | 	h := hmac.New(sha1.New, []byte(c.SecretAccessKey))
 95 | 	h.Write([]byte(req.Method + "\n"))
 96 | 	h.Write([]byte(req.Header.Get("Content-MD5") + "\n"))
 97 | 	h.Write([]byte(req.Header.Get("Content-Type") + "\n"))
 98 | 	h.Write([]byte(req.Header.Get("Date") + "\n"))
 99 | 	h.Write(v2Headers(req))
100 | 	h.Write(v2Resource(c.Bucket, req))
101 | 
102 | 	//fmt.Printf("CANONICAL:\n---\n%s\n%s\n%s\n%s\n%s%s%s]---\n",
103 | 	//	req.Method, req.Header.Get("Content-MD5"), req.Header.Get("Content-Type"), req.Header.Get("Date"), string(v2Headers(req)), v2Resource(c.Bucket, req))
104 | 
105 | 	//fmt.Printf("AWS %s:%s\n", c.AccessKeyID, base64(h.Sum(nil)))
106 | 	return fmt.Sprintf("AWS %s:%s", c.AccessKeyID, base64(h.Sum(nil)))
107 | }
108 | 
109 | func v4Headers(req *http.Request) ([]byte, []byte) {
110 | 	subset := make(map[string]string)
111 | 	names := make([]string, 0)
112 | 
113 | 	for header := range req.Header {
114 | 		lc := strings.ToLower(header)
115 | 		if lc == "host" || strings.HasPrefix(lc, "x-amz-") {
116 | 			names = append(names, lc)
117 | 			subset[lc] = strings.Trim(req.Header.Get(header), " \t\r\n\f")
118 | 		}
119 | 	}
120 | 	sort.Strings(names)
121 | 
122 | 	ll := make([][]byte, len(names))
123 | 	nn := make([][]byte, len(names))
124 | 	for i, header := range names {
125 | 		nn[i] = []byte(header)
126 | 		ll[i] = bytes.Join([][]byte{nn[i], []byte(subset[header])}, []byte{0x3a})
127 | 	}
128 | 
129 | 	signed := bytes.Join(nn, []byte{0x3b})
130 | 	return signed, bytes.Join([][]byte{
131 | 		bytes.Join(ll, []byte{0x0a}),
132 | 		nil, /* force an empty line */
133 | 		signed,
134 | 	}, []byte{0x0a})
135 | }
136 | 
137 | func v4QueryString(s string) []byte {
138 | 	if s == "" {
139 | 		return []byte{}
140 | 	}
141 | 
142 | 	qq := strings.Split(s, "&")
143 | 	sort.Strings(qq)
144 | 
145 | 	ll := make([][]byte, len(qq))
146 | 	for i := range qq {
147 | 		kv := strings.SplitN(qq[i], "=", 2)
148 | 		k := uriencode(kv[0], true)
149 | 		var v []byte
150 | 		if len(kv) == 2 {
151 | 			v = uriencode(kv[1], true)
152 | 		}
153 | 
154 | 		ll[i] = make([]byte, len(k)+1+len(v))
155 | 		copy(ll[i], k)
156 | 		ll[i][len(k)] = 0x3d
157 | 		copy(ll[i][1+len(k):], v)
158 | 	}
159 | 
160 | 	return bytes.Join(ll, []byte{0x26})
161 | }
162 | 
163 | func (c *Client) v4signature(req *http.Request, raw []byte) string {
164 | 	/* step 0: assemble some temporary values we will need */
165 | 	now := time.Now().UTC()
166 | 	yyyymmdd := now.Format("20060102")
167 | 	scope := fmt.Sprintf("%s/%s/s3/aws4_request", yyyymmdd, c.Region)
168 | 	req.Header.Set("x-amz-date", now.Format("20060102T150405Z"))
169 | 	req.Header.Set("host", req.URL.Host)
170 | 	if c.Token != "" {
171 | 		req.Header.Set("X-Amz-Security-Token", c.Token)
172 | 	}
173 | 	//req.Header.Set("host", "go-s3-bd6cf051-8023-4d2b-8bf2-7aaa477862ea.s3.amazonaws.com")
174 | 
175 | 	payload := sha256.New()
176 | 	payload.Write(raw)
177 | 	hashed := hex(payload.Sum(nil))
178 | 	req.Header.Set("x-amz-content-sha256", hashed)
179 | 
180 | 	/* step 1: generate the CanonicalRequest (+sha256() it)
181 | 
182 | 	   METHOD \n
183 | 	   uri() \n
184 | 	   querystring() \n
185 | 	   headers() \n
186 | 	   signed() \n
187 | 	   payload()
188 | 	*/
189 | 
190 | 	headers, hsig := v4Headers(req)
191 | 	canon := sha256.New()
192 | 	canon.Write([]byte(req.Method))
193 | 	canon.Write([]byte("\n"))
194 | 	canon.Write(uriencode(req.URL.Path, false))
195 | 	canon.Write([]byte("\n"))
196 | 	canon.Write(v4QueryString(req.URL.RawQuery))
197 | 	canon.Write([]byte("\n"))
198 | 	canon.Write(hsig)
199 | 	canon.Write([]byte("\n"))
200 | 	canon.Write([]byte(hashed))
201 | 
202 | 	//fmt.Printf("CANONICAL:\n---\n%s\n%s\n%s\n%s\n%s]---\n",
203 | 	//	req.Method, string(uriencode(req.URL.Path, false)), string(v4QueryString(req.URL.RawQuery)), string(hsig), hashed)
204 | 
205 | 	/* step 2: generate the StringToSign
206 | 
207 | 	   AWS4-HMAC-SHA256 \n
208 | 	   YYYYMMDDTHHMMSSZ \n
209 | 	   "yyyymmdd/region/s3/aws_request" \n
210 | 	   hex(sha256(canonical()))
211 | 	*/
212 | 	cleartext := "AWS4-HMAC-SHA256" +
213 | 		"\n" + now.Format("20060102T150405Z") +
214 | 		"\n" + scope +
215 | 		"\n" + hex(canon.Sum(nil))
216 | 
217 | 	//fmt.Printf("CLEARTEXT:\n---\n%s\n---\n", cleartext)
218 | 
219 | 	/* step 3: generate the Signature
220 | 
221 | 	   datekey = hmac-sha256("AWS4" + secret_key, YYYYMMDD)
222 | 	   datereg = hmac-sha256(datekey, region)
223 | 	   drsvc   = hmac-sha256(datereg, "s3")
224 | 	   sigkey  = hmac-sha256(drsvc, "aws4_request")
225 | 
226 | 	   hex(hmac-sha256(sigkey, cleartext))
227 | 
228 | 	*/
229 | 	k1 := mac256([]byte("AWS4"+c.SecretAccessKey), []byte(yyyymmdd))
230 | 	k2 := mac256(k1, []byte(c.Region))
231 | 	k3 := mac256(k2, []byte("s3"))
232 | 	k4 := mac256(k3, []byte("aws4_request"))
233 | 	sig := hex(mac256(k4, []byte(cleartext)))
234 | 
235 | 	/* step 4: assemble and return the Authorize: header */
236 | 	return "AWS4-HMAC-SHA256" +
237 | 		" " + fmt.Sprintf("Credential=%s/%s", c.AccessKeyID, scope) +
238 | 		"," + fmt.Sprintf("SignedHeaders=%s", string(headers)) +
239 | 		"," + fmt.Sprintf("Signature=%s", sig)
240 | }
241 | 


--------------------------------------------------------------------------------
/trace.go:
--------------------------------------------------------------------------------
 1 | package s3
 2 | 
 3 | import (
 4 | 	"io"
 5 | 	"net/http"
 6 | 	"net/http/httputil"
 7 | 
 8 | 	fmt "github.com/jhunt/go-ansi"
 9 | )
10 | 
11 | func (c *Client) Trace(out io.Writer, yes, body bool) {
12 | 	c.traceTo = out
13 | 	c.trace = yes || body
14 | 	c.traceBody = body
15 | }
16 | 
17 | func (c *Client) traceRequest(r *http.Request) error {
18 | 	if c.trace {
19 | 		what, err := httputil.DumpRequest(r, c.traceBody)
20 | 		if err != nil {
21 | 			return err
22 | 		}
23 | 		fmt.Fprintf(c.traceTo, "---[ request ]-----------------------------------\n@C{%s}\n\n", what)
24 | 	}
25 | 	return nil
26 | }
27 | 
28 | func (c *Client) traceResponse(r *http.Response) error {
29 | 	if c.trace {
30 | 		what, err := httputil.DumpResponse(r, c.traceBody)
31 | 		if err != nil {
32 | 			return err
33 | 		}
34 | 		fmt.Fprintf(c.traceTo, "---[ response ]----------------------------------\n@W{%s}\n\n", what)
35 | 	}
36 | 	return nil
37 | }
38 | 


--------------------------------------------------------------------------------
/upload.go:
--------------------------------------------------------------------------------
  1 | package s3
  2 | 
  3 | import (
  4 | 	"encoding/xml"
  5 | 	"fmt"
  6 | 	"io"
  7 | 	"io/ioutil"
  8 | 	"net/http"
  9 | 	"sync"
 10 | )
 11 | 
 12 | type xmlpart struct {
 13 | 	PartNumber int    `xml:"PartNumber"`
 14 | 	ETag       string `xml:"ETag"`
 15 | }
 16 | 
 17 | type Upload struct {
 18 | 	Key  string
 19 | 	c    *Client
 20 | 	n    int
 21 | 	id   string
 22 | 	sig  string
 23 | 	path string
 24 | 
 25 | 	parts []xmlpart
 26 | }
 27 | 
 28 | func (c *Client) NewUpload(path string, headers *http.Header) (*Upload, error) {
 29 | 	res, err := c.post(path+"?uploads", nil, headers)
 30 | 	if err != nil {
 31 | 		return nil, err
 32 | 	}
 33 | 
 34 | 	b, err := ioutil.ReadAll(res.Body)
 35 | 	if err != nil {
 36 | 		return nil, err
 37 | 	}
 38 | 
 39 | 	if res.StatusCode != 200 {
 40 | 		return nil, ResponseErrorFrom(b)
 41 | 	}
 42 | 
 43 | 	var payload struct {
 44 | 		Bucket   string `xml:"Bucket"`
 45 | 		Key      string `xml:"Key"`
 46 | 		UploadId string `xml:"UploadId"`
 47 | 	}
 48 | 	err = xml.Unmarshal(b, &payload)
 49 | 	if err != nil {
 50 | 		return nil, err
 51 | 	}
 52 | 
 53 | 	return &Upload{
 54 | 		Key: payload.Key,
 55 | 
 56 | 		c:    c,
 57 | 		id:   payload.UploadId,
 58 | 		path: path,
 59 | 		n:    0,
 60 | 	}, nil
 61 | }
 62 | 
 63 | func (u *Upload) nextPart() int {
 64 | 	u.parts = append(u.parts, xmlpart{})
 65 | 	u.n = u.n + 1
 66 | 	return u.n
 67 | }
 68 | 
 69 | func (u *Upload) writePart(b []byte, n int) error {
 70 | 	if n > 10000 {
 71 | 		return fmt.Errorf("S3 limits the number of multipart upload segments to 10k")
 72 | 	}
 73 | 
 74 | 	res, err := u.c.put(fmt.Sprintf("%s?partNumber=%d&uploadId=%s", u.path, n, u.id), b, nil)
 75 | 	if err != nil {
 76 | 		return err
 77 | 	}
 78 | 	defer res.Body.Close()
 79 | 
 80 | 	u.parts[n - 1] = xmlpart{
 81 | 		PartNumber: n,
 82 | 		ETag:       res.Header.Get("ETag"),
 83 | 	}
 84 | 	return nil
 85 | }
 86 | 
 87 | func (u *Upload) Write(b []byte) error {
 88 | 	return u.writePart(b, u.nextPart())
 89 | }
 90 | 
 91 | func (u *Upload) Done() error {
 92 | 	var payload struct {
 93 | 		XMLName xml.Name  `xml:"CompleteMultipartUpload"`
 94 | 		Parts   []xmlpart `xml:"Part"`
 95 | 	}
 96 | 	payload.Parts = u.parts
 97 | 
 98 | 	b, err := xml.Marshal(payload)
 99 | 	if err != nil {
100 | 		return err
101 | 	}
102 | 
103 | 	res, err := u.c.post(fmt.Sprintf("%s?uploadId=%s", u.path, u.id), b, nil)
104 | 	if err != nil {
105 | 		return err
106 | 	}
107 | 	defer res.Body.Close()
108 | 	if res.StatusCode != 200 {
109 | 		return ResponseError(res)
110 | 	}
111 | 	return nil
112 | }
113 | 
114 | func (u *Upload) ParallelStream(in io.Reader, block int, threads int) (int64, error) {
115 | 	if block < 5*1024*1024 {
116 | 		return 0, fmt.Errorf("S3 requires block sizes of 5MB or higher")
117 | 	}
118 | 
119 | 	type chunk struct {
120 | 		n int
121 | 		block []byte
122 | 	}
123 | 
124 | 	var wg sync.WaitGroup
125 | 	chunks := make(chan chunk, 0)
126 | 	errors := make(chan error, threads)
127 | 	for i := 0; i < threads; i++ {
128 | 		wg.Add(1)
129 | 		//idx := i
130 | 		go func() {
131 | 			defer wg.Done()
132 | 			//fmt.Printf("io/%d: waiting for first chunk...\n", idx)
133 | 			for chunk := range chunks {
134 | 				//fmt.Printf("io/%d: writing %d bytes in chunk %d via writePart...\n", idx, len(chunk.block), chunk.n)
135 | 				err := u.writePart(chunk.block, chunk.n)
136 | 				if err != nil {
137 | 					errors <- err
138 | 					return
139 | 				}
140 | 				//fmt.Printf("io/%d: waiting for next chunk...\n", idx)
141 | 			}
142 | 		}()
143 | 	}
144 | 
145 | 	var total int64
146 | 	defer func() {
147 | 		//fmt.Printf("... closing chunks channel\n")
148 | 		close(chunks)
149 | 		//fmt.Printf("... waiting for upload goroutines to exit")
150 | 		wg.Wait()
151 | 	}()
152 | 
153 | 	for {
154 | 		buf := make([]byte, block)
155 | 		nread, err := io.ReadAtLeast(in, buf, block)
156 | 		if err != nil && err != io.ErrUnexpectedEOF {
157 | 			if err == io.EOF {
158 | 				return total, nil
159 | 			}
160 | 			return total, err
161 | 		}
162 | 
163 | 		chunks <- chunk{
164 | 			n: u.nextPart(),
165 | 			block: buf[0:nread],
166 | 		}
167 | 
168 | 		total += int64(nread)
169 | 		if err == io.ErrUnexpectedEOF {
170 | 			return total, nil
171 | 		}
172 | 
173 | 		select {
174 | 		default:
175 | 		case err := <-errors:
176 | 			return total, err
177 | 		}
178 | 	}
179 | }
180 | 
181 | func (u *Upload) Stream(in io.Reader, block int) (int64, error) {
182 | 	return u.ParallelStream(in, block, 1)
183 | }
184 | 


--------------------------------------------------------------------------------
/util.go:
--------------------------------------------------------------------------------
  1 | package s3
  2 | 
  3 | func uriencode(s string, encodeSlash bool) []byte {
  4 | 	bb := []byte(s)
  5 | 
  6 | 	n := 0
  7 | 	for _, b := range bb {
  8 | 		/* percent-encode everything that isn't
  9 | 
 10 | 		   A-Z: 0x41 - 0x5a
 11 | 		   a-z: 0x61 - 0x7a
 12 | 		   0-9: 0x30 - 0x39
 13 | 		   -:   0x2d
 14 | 		   .:   0x2e
 15 | 		   _:   0x5f
 16 | 		   ~:   0x7e
 17 | 		   /:   0x2f (but only if encodeSlash == true)
 18 | 
 19 | 		*/
 20 | 
 21 | 		switch {
 22 | 		case b >= 0x41 && b <= 0x5a:
 23 | 			n++
 24 | 		case b >= 0x61 && b <= 0x7a:
 25 | 			n++
 26 | 		case b >= 0x30 && b <= 0x39:
 27 | 			n++
 28 | 		case b == 0x2d || b == 0x2e || b == 0x5f || b == 0x7e:
 29 | 			n++
 30 | 		case b == 0x2f && !encodeSlash:
 31 | 			n++
 32 | 		default: /* %xx-encoded */
 33 | 			n += 3
 34 | 		}
 35 | 	}
 36 | 
 37 | 	out := make([]byte, n)
 38 | 	n = 0
 39 | 
 40 | 	for _, b := range bb {
 41 | 		switch {
 42 | 		case b >= 0x41 && b <= 0x5a:
 43 | 			out[n] = b
 44 | 			n++
 45 | 		case b >= 0x61 && b <= 0x7a:
 46 | 			out[n] = b
 47 | 			n++
 48 | 		case b >= 0x30 && b <= 0x39:
 49 | 			out[n] = b
 50 | 			n++
 51 | 		case b == 0x2d || b == 0x2e || b == 0x5f || b == 0x7e:
 52 | 			out[n] = b
 53 | 			n++
 54 | 		case b == 0x2f && !encodeSlash:
 55 | 			out[n] = b
 56 | 			n++
 57 | 		default: /* %xx-encoded */
 58 | 			out[n] = 0x25 /* '%' */
 59 | 			out[n+1] = "0123456789ABCDEF"[b>>4]
 60 | 			out[n+2] = "0123456789ABCDEF"[b&0xf]
 61 | 			n += 3
 62 | 		}
 63 | 	}
 64 | 
 65 | 	return out
 66 | }
 67 | 
 68 | func hex(b []byte) string {
 69 | 	out := make([]byte, len(b)*2)
 70 | 	for i, c := range b {
 71 | 		out[i*2] = "0123456789abcdef"[c>>4]
 72 | 		out[i*2+1] = "0123456789abcdef"[c&0xf]
 73 | 	}
 74 | 	return string(out)
 75 | }
 76 | 
 77 | func base64(b []byte) []byte {
 78 | 	/* per RFC2054 */
 79 | 	alpha := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
 80 | 	out := make([]byte, ((len(b) + 2) / 3 * 4))
 81 | 
 82 | 	i, j := 0, 0
 83 | 	n := len(b) / 3 * 3
 84 | 
 85 | 	for i < n {
 86 | 		v := uint(b[i])<<16 | uint(b[i+1])<<8 | uint(b[i+2])
 87 | 		out[j+0] = alpha[v>>18&0x3f]
 88 | 		out[j+1] = alpha[v>>12&0x3f]
 89 | 		out[j+2] = alpha[v>>6&0x3f]
 90 | 		out[j+3] = alpha[v&0x3f]
 91 | 		i += 3
 92 | 		j += 4
 93 | 	}
 94 | 
 95 | 	left := len(b) - n
 96 | 	if left > 0 {
 97 | 		v := uint(b[i]) << 16
 98 | 		if left == 2 {
 99 | 			v |= uint(b[i+1]) << 8
100 | 		}
101 | 		out[j+0] = alpha[v>>18&0x3f]
102 | 		out[j+1] = alpha[v>>12&0x3f]
103 | 		if left == 2 {
104 | 			out[j+2] = alpha[v>>6&0x3f]
105 | 		} else {
106 | 			out[j+2] = alpha[64]
107 | 		}
108 | 		out[j+3] = alpha[64]
109 | 	}
110 | 	return out
111 | }
112 | 


--------------------------------------------------------------------------------