├── .gitignore ├── LICENSE.txt ├── README.md ├── aws ├── awserr │ ├── error.go │ └── types.go ├── awsutil │ ├── copy.go │ ├── path_value.go │ ├── string_value.go │ └── util.go ├── config.go ├── config_test.go ├── context.go ├── credentials │ ├── chain_provider.go │ ├── chain_provider_test.go │ ├── credentials.go │ ├── credentials_test.go │ ├── ec2_role_provider.go │ ├── ec2_role_provider_test.go │ ├── env_provider.go │ ├── env_provider_test.go │ ├── example.ini │ ├── ini.go │ ├── shared_credentials_provider.go │ ├── shared_credentials_provider_test.go │ ├── static_provider.go │ └── static_provider_test.go ├── default_redirect.go ├── handler_functions.go ├── handler_functions_test.go ├── handlers.go ├── handlers_test.go ├── param_validator.go ├── param_validator_test.go ├── progress.go ├── request.go ├── request_pagination_test.go ├── request_test.go ├── retry │ ├── exponential_retry.go │ ├── fixed_retry.go │ ├── linear_retry.go │ ├── no_delay_retry.go │ ├── random_retry.go │ ├── retry.go │ └── retryable_error.go ├── service.go ├── types.go └── version.go ├── go.mod ├── go.sum ├── internal ├── apierr │ └── error.go ├── context │ └── background_go1.5.go ├── crc │ └── crc64.go ├── endpoints │ ├── endpoints.go │ ├── endpoints_map.go │ └── endpoints_test.go ├── protocol │ ├── body │ │ └── body.go │ ├── ec2query │ │ ├── build.go │ │ ├── build_test.go │ │ └── unmarshal.go │ ├── json │ │ └── jsonutil │ │ │ ├── build.go │ │ │ ├── build_test.go │ │ │ └── unmarshal.go │ ├── jsonrpc │ │ ├── build_test.go │ │ ├── jsonrpc.go │ │ └── unmarshal_test.go │ ├── query │ │ ├── build.go │ │ ├── build_test.go │ │ ├── queryutil │ │ │ └── queryutil.go │ │ ├── unmarshal.go │ │ ├── unmarshal_error.go │ │ └── unmarshal_test.go │ ├── rest │ │ ├── build.go │ │ ├── payload.go │ │ └── unmarshal.go │ ├── restjson │ │ ├── restjson.go │ │ └── unmarshal_test.go │ ├── restxml │ │ └── restxml.go │ └── xml │ │ └── xmlutil │ │ ├── build.go │ │ ├── unmarshal.go │ │ └── xml_to_struct.go ├── signer │ ├── v2 │ │ └── v2.go │ └── v4 │ │ ├── v4.go │ │ └── v4_test.go └── util │ └── util.go ├── sdk.go ├── service ├── generate.go └── s3 │ ├── api.go │ ├── api_object.go │ ├── bucket_location.go │ ├── checkpoint.go │ ├── const.go │ ├── copier.go │ ├── cors.go │ ├── crc_check.go │ ├── data_redundancy_switch.go │ ├── decompresspolicy.go │ ├── downloader.go │ ├── errors.go │ ├── host_style_bucket.go │ ├── inventory.go │ ├── lifecycle.go │ ├── mirror.go │ ├── replication.go │ ├── retention.go │ ├── s3iface │ └── interface.go │ ├── s3manager │ ├── fileInfo.go │ ├── file_counter.go │ └── upload.go │ ├── service.go │ ├── try.go │ ├── uploader.go │ └── util.go └── test ├── bucketsample_test.go ├── bucketwithcontext_test.go ├── common_test.go ├── object_encryption_test.go ├── objectsample_test.go └── objectwithcontext_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | doc 3 | .idea/ 4 | .DS_Store 5 | go-sdk-test* -------------------------------------------------------------------------------- /aws/awserr/error.go: -------------------------------------------------------------------------------- 1 | // Package awserr represents API error interface accessors for the SDK. 2 | package awserr 3 | 4 | // An Error wraps lower level errors with code, message and an original error. 5 | // The underlying concrete error type may also satisfy other interfaces which 6 | // can be to used to obtain more specific information about the error. 7 | // 8 | // Calling Error() or String() will always include the full information about 9 | // an error based on its underlying type. 10 | // 11 | // Example: 12 | // 13 | // output, err := s3manage.Upload(svc, input, opts) 14 | // if err != nil { 15 | // if awsErr, ok := err.(awserr.Error); ok { 16 | // // Get error details 17 | // log.Println("Error:", err.Code(), err.Message()) 18 | // 19 | // Prints out full error message, including original error if there was one. 20 | // log.Println("Error:", err.Error()) 21 | // 22 | // // Get original error 23 | // if origErr := err.Err(); origErr != nil { 24 | // // operate on original error. 25 | // } 26 | // } else { 27 | // fmt.Println(err.Error()) 28 | // } 29 | // } 30 | // 31 | type Error interface { 32 | // Satisfy the generic error interface. 33 | error 34 | 35 | // Returns the short phrase depicting the classification of the error. 36 | Code() string 37 | 38 | // Returns the error details message. 39 | Message() string 40 | 41 | // Returns the original error if one was set. Nil is returned if not set. 42 | OrigErr() error 43 | } 44 | 45 | // A RequestFailure is an interface to extract request failure information from 46 | // an Error such as the request ID of the failed request returned by a service. 47 | // RequestFailures may not always have a requestID value if the request failed 48 | // prior to reaching the service such as a connection error. 49 | // 50 | // Example: 51 | // 52 | // output, err := s3manage.Upload(svc, input, opts) 53 | // if err != nil { 54 | // if reqerr, ok := err.(RequestFailure); ok { 55 | // log.Printf("Request failed", reqerr.Code(), reqerr.Message(), reqerr.RequestID()) 56 | // } else { 57 | // log.Printf("Error:", err.Error() 58 | // } 59 | // } 60 | // 61 | // Combined with awserr.Error: 62 | // 63 | // output, err := s3manage.Upload(svc, input, opts) 64 | // if err != nil { 65 | // if awsErr, ok := err.(awserr.Error); ok { 66 | // // Generic AWS Error with Code, Message, and original error (if any) 67 | // fmt.Println(awsErr.Code(), awsErr.Message(), awsErr.OrigErr()) 68 | // 69 | // if reqErr, ok := err.(awserr.RequestFailure); ok { 70 | // // A service error occurred 71 | // fmt.Println(reqErr.StatusCode(), reqErr.RequestID()) 72 | // } 73 | // } else { 74 | // fmt.Println(err.Error()) 75 | // } 76 | // } 77 | // 78 | type RequestFailure interface { 79 | Error 80 | 81 | // The status code of the HTTP response. 82 | StatusCode() int 83 | 84 | // The request ID returned by the service for a request failure. This will 85 | // be empty if no request ID is available such as the request failed due 86 | // to a connection error. 87 | RequestID() string 88 | } 89 | 90 | // New returns an Error object described by the code, message, and origErr. 91 | // 92 | // If origErr satisfies the Error interface it will not be wrapped within a new 93 | // Error object and will instead be returned. 94 | func New(code, message string, origErr error) Error { 95 | if e, ok := origErr.(Error); ok && e != nil { 96 | return e 97 | } 98 | return newBaseError(code, message, origErr) 99 | } 100 | 101 | // NewRequestFailure returns a new request error wrapper for the given Error 102 | // provided. 103 | func NewRequestFailure(err Error, statusCode int, reqID string) RequestFailure { 104 | return newRequestError(err, statusCode, reqID) 105 | } 106 | -------------------------------------------------------------------------------- /aws/awserr/types.go: -------------------------------------------------------------------------------- 1 | package awserr 2 | 3 | import "fmt" 4 | 5 | // SprintError returns a string of the formatted error code. 6 | // 7 | // Both extra and origErr are optional. If they are included their lines 8 | // will be added, but if they are not included their lines will be ignored. 9 | func SprintError(code, message, extra string, origErr error) string { 10 | msg := fmt.Sprintf("%s: %s", code, message) 11 | if extra != "" { 12 | msg = fmt.Sprintf("%s\n\t%s", msg, extra) 13 | } 14 | if origErr != nil { 15 | msg = fmt.Sprintf("%s\ncaused by: %s", msg, origErr.Error()) 16 | } 17 | return msg 18 | } 19 | 20 | // A baseError wraps the code and message which defines an error. It also 21 | // can be used to wrap an original error object. 22 | // 23 | // Should be used as the root for errors satisfying the awserr.Error. Also 24 | // for any error which does not fit into a specific error wrapper type. 25 | type baseError struct { 26 | // Classification of error 27 | code string 28 | 29 | // Detailed information about error 30 | message string 31 | 32 | // Optional original error this error is based off of. Allows building 33 | // chained errors. 34 | origErr error 35 | } 36 | 37 | // newBaseError returns an error object for the code, message, and err. 38 | // 39 | // code is a short no whitespace phrase depicting the classification of 40 | // the error that is being created. 41 | // 42 | // message is the free flow string containing detailed information about the error. 43 | // 44 | // origErr is the error object which will be nested under the new error to be returned. 45 | func newBaseError(code, message string, origErr error) *baseError { 46 | return &baseError{ 47 | code: code, 48 | message: message, 49 | origErr: origErr, 50 | } 51 | } 52 | 53 | // Error returns the string representation of the error. 54 | // 55 | // See ErrorWithExtra for formatting. 56 | // 57 | // Satisfies the error interface. 58 | func (b baseError) Error() string { 59 | return SprintError(b.code, b.message, "", b.origErr) 60 | } 61 | 62 | // String returns the string representation of the error. 63 | // Alias for Error to satisfy the stringer interface. 64 | func (b baseError) String() string { 65 | return b.Error() 66 | } 67 | 68 | // Code returns the short phrase depicting the classification of the error. 69 | func (b baseError) Code() string { 70 | return b.code 71 | } 72 | 73 | // Message returns the error details message. 74 | func (b baseError) Message() string { 75 | return b.message 76 | } 77 | 78 | // OrigErr returns the original error if one was set. Nil is returned if no error 79 | // was set. 80 | func (b baseError) OrigErr() error { 81 | return b.origErr 82 | } 83 | 84 | // So that the Error interface type can be included as an anonymous field 85 | // in the requestError struct and not conflict with the error.Error() method. 86 | type awsError Error 87 | 88 | // A requestError wraps a request or service error. 89 | // 90 | // Composed of baseError for code, message, and original error. 91 | type requestError struct { 92 | awsError 93 | statusCode int 94 | requestID string 95 | } 96 | 97 | // newRequestError returns a wrapped error with additional information for request 98 | // status code, and service requestID. 99 | // 100 | // Should be used to wrap all request which involve service requests. Even if 101 | // the request failed without a service response, but had an HTTP status code 102 | // that may be meaningful. 103 | // 104 | // Also wraps original errors via the baseError. 105 | func newRequestError(err Error, statusCode int, requestID string) *requestError { 106 | return &requestError{ 107 | awsError: err, 108 | statusCode: statusCode, 109 | requestID: requestID, 110 | } 111 | } 112 | 113 | // Error returns the string representation of the error. 114 | // Satisfies the error interface. 115 | func (r requestError) Error() string { 116 | extra := fmt.Sprintf("status code: %d, request id: [%s]", 117 | r.statusCode, r.requestID) 118 | return SprintError(r.Code(), r.Message(), extra, r.OrigErr()) 119 | } 120 | 121 | // String returns the string representation of the error. 122 | // Alias for Error to satisfy the stringer interface. 123 | func (r requestError) String() string { 124 | return r.Error() 125 | } 126 | 127 | // StatusCode returns the wrapped status code for the error 128 | func (r requestError) StatusCode() int { 129 | return r.statusCode 130 | } 131 | 132 | // RequestID returns the wrapped requestID 133 | func (r requestError) RequestID() string { 134 | return r.requestID 135 | } 136 | -------------------------------------------------------------------------------- /aws/awsutil/copy.go: -------------------------------------------------------------------------------- 1 | package awsutil 2 | 3 | import ( 4 | "io" 5 | "reflect" 6 | ) 7 | 8 | // Copy deeply copies a src structure to dst. Useful for copying request and 9 | // response structures. 10 | // 11 | // Can copy between structs of different type, but will only copy fields which 12 | // are assignable, and exist in both structs. Fields which are not assignable, 13 | // or do not exist in both structs are ignored. 14 | func Copy(dst, src interface{}) { 15 | dstval := reflect.ValueOf(dst) 16 | if !dstval.IsValid() { 17 | panic("Copy dst cannot be nil") 18 | } 19 | 20 | rcopy(dstval, reflect.ValueOf(src), true) 21 | } 22 | 23 | // CopyOf returns a copy of src while also allocating the memory for dst. 24 | // src must be a pointer type or this operation will fail. 25 | func CopyOf(src interface{}) (dst interface{}) { 26 | dsti := reflect.New(reflect.TypeOf(src).Elem()) 27 | dst = dsti.Interface() 28 | rcopy(dsti, reflect.ValueOf(src), true) 29 | return 30 | } 31 | 32 | // rcopy performs a recursive copy of values from the source to destination. 33 | // 34 | // root is used to skip certain aspects of the copy which are not valid 35 | // for the root node of a object. 36 | func rcopy(dst, src reflect.Value, root bool) { 37 | if !src.IsValid() { 38 | return 39 | } 40 | 41 | switch src.Kind() { 42 | case reflect.Ptr: 43 | if _, ok := src.Interface().(io.Reader); ok { 44 | if dst.Kind() == reflect.Ptr && dst.Elem().CanSet() { 45 | dst.Elem().Set(src) 46 | } else if dst.CanSet() { 47 | dst.Set(src) 48 | } 49 | } else { 50 | e := src.Type().Elem() 51 | if dst.CanSet() && !src.IsNil() { 52 | dst.Set(reflect.New(e)) 53 | } 54 | if src.Elem().IsValid() { 55 | // Keep the current root state since the depth hasn't changed 56 | rcopy(dst.Elem(), src.Elem(), root) 57 | } 58 | } 59 | case reflect.Struct: 60 | if !root { 61 | dst.Set(reflect.New(src.Type()).Elem()) 62 | } 63 | 64 | t := dst.Type() 65 | for i := 0; i < t.NumField(); i++ { 66 | name := t.Field(i).Name 67 | srcVal := src.FieldByName(name) 68 | dstVal := dst.FieldByName(name) 69 | if srcVal.IsValid() && dstVal.CanSet() { 70 | rcopy(dstVal, srcVal, false) 71 | } 72 | } 73 | case reflect.Slice: 74 | s := reflect.MakeSlice(src.Type(), src.Len(), src.Cap()) 75 | dst.Set(s) 76 | for i := 0; i < src.Len(); i++ { 77 | rcopy(dst.Index(i), src.Index(i), false) 78 | } 79 | case reflect.Map: 80 | s := reflect.MakeMap(src.Type()) 81 | dst.Set(s) 82 | for _, k := range src.MapKeys() { 83 | v := src.MapIndex(k) 84 | v2 := reflect.New(v.Type()).Elem() 85 | rcopy(v2, v, false) 86 | dst.SetMapIndex(k, v2) 87 | } 88 | default: 89 | // Assign the value if possible. If its not assignable, the value would 90 | // need to be converted and the impact of that may be unexpected, or is 91 | // not compatible with the dst type. 92 | if src.Type().AssignableTo(dst.Type()) { 93 | dst.Set(src) 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /aws/awsutil/path_value.go: -------------------------------------------------------------------------------- 1 | package awsutil 2 | 3 | import ( 4 | "reflect" 5 | "regexp" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | var indexRe = regexp.MustCompile(`(.+)\[(-?\d+)?\]$`) 11 | 12 | // rValuesAtPath returns a slice of values found in value v. The values 13 | // in v are explored recursively so all nested values are collected. 14 | func rValuesAtPath(v interface{}, path string, create bool, caseSensitive bool) []reflect.Value { 15 | pathparts := strings.Split(path, "||") 16 | if len(pathparts) > 1 { 17 | for _, pathpart := range pathparts { 18 | vals := rValuesAtPath(v, pathpart, create, caseSensitive) 19 | if vals != nil && len(vals) > 0 { 20 | return vals 21 | } 22 | } 23 | return nil 24 | } 25 | 26 | values := []reflect.Value{reflect.Indirect(reflect.ValueOf(v))} 27 | components := strings.Split(path, ".") 28 | for len(values) > 0 && len(components) > 0 { 29 | var index *int64 30 | var indexStar bool 31 | c := strings.TrimSpace(components[0]) 32 | if c == "" { // no actual component, illegal syntax 33 | return nil 34 | } else if caseSensitive && c != "*" && strings.ToLower(c[0:1]) == c[0:1] { 35 | // TODO normalize case for user 36 | return nil // don't support unexported fields 37 | } 38 | 39 | // parse this component 40 | if m := indexRe.FindStringSubmatch(c); m != nil { 41 | c = m[1] 42 | if m[2] == "" { 43 | index = nil 44 | indexStar = true 45 | } else { 46 | i, _ := strconv.ParseInt(m[2], 10, 32) 47 | index = &i 48 | indexStar = false 49 | } 50 | } 51 | 52 | nextvals := []reflect.Value{} 53 | for _, value := range values { 54 | // pull component name out of struct member 55 | if value.Kind() != reflect.Struct { 56 | continue 57 | } 58 | 59 | if c == "*" { // pull all members 60 | for i := 0; i < value.NumField(); i++ { 61 | if f := reflect.Indirect(value.Field(i)); f.IsValid() { 62 | nextvals = append(nextvals, f) 63 | } 64 | } 65 | continue 66 | } 67 | 68 | value = value.FieldByNameFunc(func(name string) bool { 69 | if c == name { 70 | return true 71 | } else if !caseSensitive && strings.ToLower(name) == strings.ToLower(c) { 72 | return true 73 | } 74 | return false 75 | }) 76 | 77 | if create && value.Kind() == reflect.Ptr && value.IsNil() { 78 | value.Set(reflect.New(value.Type().Elem())) 79 | value = value.Elem() 80 | } else { 81 | value = reflect.Indirect(value) 82 | } 83 | 84 | if value.IsValid() { 85 | nextvals = append(nextvals, value) 86 | } 87 | } 88 | values = nextvals 89 | 90 | if indexStar || index != nil { 91 | nextvals = []reflect.Value{} 92 | for _, value := range values { 93 | value := reflect.Indirect(value) 94 | if value.Kind() != reflect.Slice { 95 | continue 96 | } 97 | 98 | if indexStar { // grab all indices 99 | for i := 0; i < value.Len(); i++ { 100 | idx := reflect.Indirect(value.Index(i)) 101 | if idx.IsValid() { 102 | nextvals = append(nextvals, idx) 103 | } 104 | } 105 | continue 106 | } 107 | 108 | // pull out index 109 | i := int(*index) 110 | if i >= value.Len() { // check out of bounds 111 | if create { 112 | // TODO resize slice 113 | } else { 114 | continue 115 | } 116 | } else if i < 0 { // support negative indexing 117 | i = value.Len() + i 118 | } 119 | value = reflect.Indirect(value.Index(i)) 120 | 121 | if value.IsValid() { 122 | nextvals = append(nextvals, value) 123 | } 124 | } 125 | values = nextvals 126 | } 127 | 128 | components = components[1:] 129 | } 130 | return values 131 | } 132 | 133 | // ValuesAtPath returns a list of objects at the lexical path inside of a structure 134 | func ValuesAtPath(i interface{}, path string) []interface{} { 135 | if rvals := rValuesAtPath(i, path, false, true); rvals != nil { 136 | vals := make([]interface{}, len(rvals)) 137 | for i, rval := range rvals { 138 | vals[i] = rval.Interface() 139 | } 140 | return vals 141 | } 142 | return nil 143 | } 144 | 145 | // ValuesAtAnyPath returns a list of objects at the case-insensitive lexical 146 | // path inside of a structure 147 | func ValuesAtAnyPath(i interface{}, path string) []interface{} { 148 | if rvals := rValuesAtPath(i, path, false, false); rvals != nil { 149 | vals := make([]interface{}, len(rvals)) 150 | for i, rval := range rvals { 151 | vals[i] = rval.Interface() 152 | } 153 | return vals 154 | } 155 | return nil 156 | } 157 | 158 | // SetValueAtPath sets an object at the lexical path inside of a structure 159 | func SetValueAtPath(i interface{}, path string, v interface{}) { 160 | if rvals := rValuesAtPath(i, path, true, true); rvals != nil { 161 | for _, rval := range rvals { 162 | rval.Set(reflect.ValueOf(v)) 163 | } 164 | } 165 | } 166 | 167 | // SetValueAtAnyPath sets an object at the case insensitive lexical path inside 168 | // of a structure 169 | func SetValueAtAnyPath(i interface{}, path string, v interface{}) { 170 | if rvals := rValuesAtPath(i, path, true, false); rvals != nil { 171 | for _, rval := range rvals { 172 | rval.Set(reflect.ValueOf(v)) 173 | } 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /aws/awsutil/string_value.go: -------------------------------------------------------------------------------- 1 | package awsutil 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "reflect" 8 | "strings" 9 | ) 10 | 11 | // StringValue returns the string representation of a value. 12 | func StringValue(i interface{}) string { 13 | var buf bytes.Buffer 14 | stringValue(reflect.ValueOf(i), 0, &buf) 15 | return buf.String() 16 | } 17 | 18 | // stringValue will recursively walk value v to build a textual 19 | // representation of the value. 20 | func stringValue(v reflect.Value, indent int, buf *bytes.Buffer) { 21 | for v.Kind() == reflect.Ptr { 22 | v = v.Elem() 23 | } 24 | 25 | switch v.Kind() { 26 | case reflect.Struct: 27 | strtype := v.Type().String() 28 | if strtype == "time.Time" { 29 | fmt.Fprintf(buf, "%s", v.Interface()) 30 | break 31 | } else if strings.HasPrefix(strtype, "io.") { 32 | buf.WriteString("") 33 | break 34 | } 35 | 36 | buf.WriteString("{\n") 37 | 38 | names := []string{} 39 | for i := 0; i < v.Type().NumField(); i++ { 40 | name := v.Type().Field(i).Name 41 | f := v.Field(i) 42 | if name[0:1] == strings.ToLower(name[0:1]) { 43 | continue // ignore unexported fields 44 | } 45 | if (f.Kind() == reflect.Ptr || f.Kind() == reflect.Slice || f.Kind() == reflect.Map) && f.IsNil() { 46 | continue // ignore unset fields 47 | } 48 | names = append(names, name) 49 | } 50 | 51 | for i, n := range names { 52 | val := v.FieldByName(n) 53 | buf.WriteString(strings.Repeat(" ", indent+2)) 54 | buf.WriteString(n + ": ") 55 | stringValue(val, indent+2, buf) 56 | 57 | if i < len(names)-1 { 58 | buf.WriteString(",\n") 59 | } 60 | } 61 | 62 | buf.WriteString("\n" + strings.Repeat(" ", indent) + "}") 63 | case reflect.Slice: 64 | nl, id, id2 := "", "", "" 65 | if v.Len() > 3 { 66 | nl, id, id2 = "\n", strings.Repeat(" ", indent), strings.Repeat(" ", indent+2) 67 | } 68 | buf.WriteString("[" + nl) 69 | for i := 0; i < v.Len(); i++ { 70 | buf.WriteString(id2) 71 | stringValue(v.Index(i), indent+2, buf) 72 | 73 | if i < v.Len()-1 { 74 | buf.WriteString("," + nl) 75 | } 76 | } 77 | 78 | buf.WriteString(nl + id + "]") 79 | case reflect.Map: 80 | buf.WriteString("{\n") 81 | 82 | for i, k := range v.MapKeys() { 83 | buf.WriteString(strings.Repeat(" ", indent+2)) 84 | buf.WriteString(k.String() + ": ") 85 | stringValue(v.MapIndex(k), indent+2, buf) 86 | 87 | if i < v.Len()-1 { 88 | buf.WriteString(",\n") 89 | } 90 | } 91 | 92 | buf.WriteString("\n" + strings.Repeat(" ", indent) + "}") 93 | default: 94 | format := "%v" 95 | switch v.Interface().(type) { 96 | case string: 97 | format = "%q" 98 | case io.ReadSeeker, io.Reader: 99 | format = "buffer(%p)" 100 | } 101 | fmt.Fprintf(buf, format, v.Interface()) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /aws/awsutil/util.go: -------------------------------------------------------------------------------- 1 | package awsutil 2 | 3 | import ( 4 | "bytes" 5 | "crypto/md5" 6 | "encoding/base64" 7 | "encoding/xml" 8 | "fmt" 9 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil" 10 | "go/format" 11 | "io" 12 | "os" 13 | "path/filepath" 14 | "reflect" 15 | "regexp" 16 | "strings" 17 | ) 18 | 19 | // GoFmt returns the Go formated string of the input. 20 | // 21 | // Panics if the format fails. 22 | func GoFmt(buf string) string { 23 | formatted, err := format.Source([]byte(buf)) 24 | if err != nil { 25 | panic(fmt.Errorf("%s\nOriginal code:\n%s", err.Error(), buf)) 26 | } 27 | return string(formatted) 28 | } 29 | 30 | var reTrim = regexp.MustCompile(`\s{2,}`) 31 | 32 | // Trim removes all leading and trailing white space. 33 | // 34 | // All consecutive spaces will be reduced to a single space. 35 | func Trim(s string) string { 36 | return strings.TrimSpace(reTrim.ReplaceAllString(s, " ")) 37 | } 38 | 39 | // Capitalize capitalizes the first character of the string. 40 | func Capitalize(s string) string { 41 | if len(s) == 1 { 42 | return strings.ToUpper(s) 43 | } 44 | return strings.ToUpper(s[0:1]) + s[1:] 45 | } 46 | 47 | // SortXML sorts the reader's XML elements 48 | func SortXML(r io.Reader) string { 49 | var buf bytes.Buffer 50 | d := xml.NewDecoder(r) 51 | root, _ := xmlutil.XMLToStruct(d, nil) 52 | e := xml.NewEncoder(&buf) 53 | xmlutil.StructToXML(e, root, true) 54 | return buf.String() 55 | } 56 | 57 | // PrettyPrint generates a human readable representation of the value v. 58 | // All values of v are recursively found and pretty printed also. 59 | func PrettyPrint(v interface{}) string { 60 | value := reflect.ValueOf(v) 61 | switch value.Kind() { 62 | case reflect.Struct: 63 | str := fullName(value.Type()) + "{\n" 64 | for i := 0; i < value.NumField(); i++ { 65 | l := string(value.Type().Field(i).Name[0]) 66 | if strings.ToUpper(l) == l { 67 | str += value.Type().Field(i).Name + ": " 68 | str += PrettyPrint(value.Field(i).Interface()) 69 | str += ",\n" 70 | } 71 | } 72 | str += "}" 73 | return str 74 | case reflect.Map: 75 | str := "map[" + fullName(value.Type().Key()) + "]" + fullName(value.Type().Elem()) + "{\n" 76 | for _, k := range value.MapKeys() { 77 | str += "\"" + k.String() + "\": " 78 | str += PrettyPrint(value.MapIndex(k).Interface()) 79 | str += ",\n" 80 | } 81 | str += "}" 82 | return str 83 | case reflect.Ptr: 84 | if e := value.Elem(); e.IsValid() { 85 | return "&" + PrettyPrint(e.Interface()) 86 | } 87 | return "nil" 88 | case reflect.Slice: 89 | str := "[]" + fullName(value.Type().Elem()) + "{\n" 90 | for i := 0; i < value.Len(); i++ { 91 | str += PrettyPrint(value.Index(i).Interface()) 92 | str += ",\n" 93 | } 94 | str += "}" 95 | return str 96 | default: 97 | return fmt.Sprintf("%#v", v) 98 | } 99 | } 100 | 101 | func pkgName(t reflect.Type) string { 102 | pkg := t.PkgPath() 103 | c := strings.Split(pkg, "/") 104 | return c[len(c)-1] 105 | } 106 | 107 | func fullName(t reflect.Type) string { 108 | if pkg := pkgName(t); pkg != "" { 109 | return pkg + "." + t.Name() 110 | } 111 | return t.Name() 112 | } 113 | 114 | //获取指定目录及所有子目录下的所有文件，可以匹配后缀过滤。 115 | func WalkDir(dirPth, suffix string) (files []string, err error) { 116 | files = make([]string, 0, 30) 117 | suffix = strings.ToUpper(suffix) //忽略后缀匹配的大小写 118 | 119 | err = filepath.Walk(dirPth, func(filename string, fi os.FileInfo, err error) error { //遍历目录 120 | //if err != nil { //忽略错误 121 | // return err 122 | //} 123 | 124 | if fi.IsDir() { // 忽略目录 125 | return nil 126 | } 127 | 128 | if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) { 129 | files = append(files, filename) 130 | } 131 | 132 | return nil 133 | }) 134 | return files, err 135 | } 136 | func IsHidden(path string) bool { 137 | fileInfo, err := os.Stat(path) 138 | if err != nil { 139 | return false 140 | } 141 | return fileInfo.Name()[0] == '.' 142 | } 143 | func ComputeMD5Hash(input []byte) []byte { 144 | h := md5.New() 145 | h.Write(input) 146 | return h.Sum(nil) 147 | } 148 | 149 | func EncodeAsString(bytes []byte) string { 150 | return base64.StdEncoding.EncodeToString(bytes) 151 | } 152 | -------------------------------------------------------------------------------- /aws/config_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "reflect" 7 | "testing" 8 | "time" 9 | 10 | "github.com/ks3sdklib/aws-sdk-go/aws/credentials" 11 | ) 12 | 13 | var testCredentials = credentials.NewChainCredentials([]credentials.Provider{ 14 | &credentials.EnvProvider{}, 15 | &credentials.SharedCredentialsProvider{ 16 | Filename: "TestFilename", 17 | Profile: "TestProfile"}, 18 | &credentials.EC2RoleProvider{ExpiryWindow: 5 * time.Minute}, 19 | }) 20 | 21 | var copyTestConfig = Config{ 22 | Credentials: testCredentials, 23 | Endpoint: "CopyTestEndpoint", 24 | Region: "COPY_TEST_AWS_REGION", 25 | DisableSSL: true, 26 | ManualSend: true, 27 | HTTPClient: http.DefaultClient, 28 | LogHTTPBody: true, 29 | LogLevel: 2, 30 | Logger: os.Stdout, 31 | MaxRetries: DefaultMaxRetries, 32 | DisableParamValidation: true, 33 | DisableComputeChecksums: true, 34 | S3ForcePathStyle: true, 35 | } 36 | 37 | func TestCopy(t *testing.T) { 38 | want := copyTestConfig 39 | got := copyTestConfig.Copy() 40 | if !reflect.DeepEqual(got, want) { 41 | t.Errorf("Copy() = %+v", got) 42 | t.Errorf(" want %+v", want) 43 | } 44 | } 45 | 46 | func TestCopyReturnsNewInstance(t *testing.T) { 47 | want := copyTestConfig 48 | got := copyTestConfig.Copy() 49 | if &got == &want { 50 | t.Errorf("Copy() = %p; want different instance as source %p", &got, &want) 51 | } 52 | } 53 | 54 | var mergeTestZeroValueConfig = Config{MaxRetries: DefaultMaxRetries} 55 | 56 | var mergeTestConfig = Config{ 57 | Credentials: testCredentials, 58 | Endpoint: "MergeTestEndpoint", 59 | Region: "MERGE_TEST_AWS_REGION", 60 | DisableSSL: true, 61 | ManualSend: true, 62 | HTTPClient: http.DefaultClient, 63 | LogHTTPBody: true, 64 | LogLevel: 2, 65 | Logger: os.Stdout, 66 | MaxRetries: 10, 67 | DisableParamValidation: true, 68 | DisableComputeChecksums: true, 69 | S3ForcePathStyle: true, 70 | } 71 | 72 | var mergeTests = []struct { 73 | cfg *Config 74 | in *Config 75 | want *Config 76 | }{ 77 | {&Config{}, nil, &Config{}}, 78 | {&Config{}, &mergeTestZeroValueConfig, &Config{}}, 79 | {&Config{}, &mergeTestConfig, &mergeTestConfig}, 80 | } 81 | 82 | func TestMerge(t *testing.T) { 83 | for _, tt := range mergeTests { 84 | got := tt.cfg.Merge(tt.in) 85 | if !reflect.DeepEqual(got, tt.want) { 86 | t.Errorf("Config %+v", tt.cfg) 87 | t.Errorf(" Merge(%+v)", tt.in) 88 | t.Errorf(" got %+v", got) 89 | t.Errorf(" want %+v", tt.want) 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /aws/context.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "context" 5 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 6 | ) 7 | 8 | type Context = context.Context 9 | 10 | // SetContext adds a Context to the current request that can be used to cancel 11 | func (r *Request) SetContext(ctx Context) { 12 | if ctx == nil { 13 | r.Error = apierr.New("InvalidParameter", "context cannot be nil", nil) 14 | } 15 | r.context = ctx 16 | r.HTTPRequest = r.HTTPRequest.WithContext(ctx) 17 | } 18 | 19 | // BackgroundContext returns a context that will never be canceled 20 | func BackgroundContext() Context { 21 | return context.Background() 22 | } 23 | -------------------------------------------------------------------------------- /aws/credentials/chain_provider.go: -------------------------------------------------------------------------------- 1 | package credentials 2 | 3 | import ( 4 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 5 | ) 6 | 7 | var ( 8 | // ErrNoValidProvidersFoundInChain Is returned when there are no valid 9 | // providers in the ChainProvider. 10 | ErrNoValidProvidersFoundInChain = apierr.New("NoCredentialProviders", "no valid providers in chain", nil) 11 | ) 12 | 13 | // A ChainProvider will search for a provider which returns credentials 14 | // and cache that provider until Retrieve is called again. 15 | // 16 | // The ChainProvider provides a way of chaining multiple providers together 17 | // which will pick the first available using priority order of the Providers 18 | // in the list. 19 | // 20 | // If none of the Providers retrieve valid credentials Value, ChainProvider's 21 | // Retrieve() will return the error ErrNoValidProvidersFoundInChain. 22 | // 23 | // If a Provider is found which returns valid credentials Value ChainProvider 24 | // will cache that Provider for all calls to IsExpired(), until Retrieve is 25 | // called again. 26 | // 27 | // Example of ChainProvider to be used with an EnvProvider and EC2RoleProvider. 28 | // In this example EnvProvider will first check if any credentials are available 29 | // vai the environment variables. If there are none ChainProvider will check 30 | // the next Provider in the list, EC2RoleProvider in this case. If EC2RoleProvider 31 | // does not return any credentials ChainProvider will return the error 32 | // ErrNoValidProvidersFoundInChain 33 | // 34 | // creds := NewChainCredentials( 35 | // []Provider{ 36 | // &EnvProvider{}, 37 | // &EC2RoleProvider{}, 38 | // }) 39 | // creds.Retrieve() 40 | // 41 | type ChainProvider struct { 42 | Providers []Provider 43 | curr Provider 44 | } 45 | 46 | // NewChainCredentials returns a pointer to a new Credentials object 47 | // wrapping a chain of providers. 48 | func NewChainCredentials(providers []Provider) *Credentials { 49 | return NewCredentials(&ChainProvider{ 50 | Providers: append([]Provider{}, providers...), 51 | }) 52 | } 53 | 54 | // Retrieve returns the credentials value or error if no provider returned 55 | // without error. 56 | // 57 | // If a provider is found it will be cached and any calls to IsExpired() 58 | // will return the expired state of the cached provider. 59 | func (c *ChainProvider) Retrieve() (Value, error) { 60 | for _, p := range c.Providers { 61 | if creds, err := p.Retrieve(); err == nil { 62 | c.curr = p 63 | return creds, nil 64 | } 65 | } 66 | c.curr = nil 67 | 68 | // TODO better error reporting. maybe report error for each failed retrieve? 69 | 70 | return Value{}, ErrNoValidProvidersFoundInChain 71 | } 72 | 73 | // IsExpired will returned the expired state of the currently cached provider 74 | // if there is one. If there is no current provider, true will be returned. 75 | func (c *ChainProvider) IsExpired() bool { 76 | if c.curr != nil { 77 | return c.curr.IsExpired() 78 | } 79 | 80 | return true 81 | } 82 | -------------------------------------------------------------------------------- /aws/credentials/chain_provider_test.go: -------------------------------------------------------------------------------- 1 | package credentials 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestChainProviderGet(t *testing.T) { 11 | p := &ChainProvider{ 12 | Providers: []Provider{ 13 | &stubProvider{err: apierr.New("FirstError", "first provider error", nil)}, 14 | &stubProvider{err: apierr.New("SecondError", "second provider error", nil)}, 15 | &stubProvider{ 16 | creds: Value{ 17 | AccessKeyID: "AKID", 18 | SecretAccessKey: "SECRET", 19 | SessionToken: "", 20 | }, 21 | }, 22 | }, 23 | } 24 | 25 | creds, err := p.Retrieve() 26 | assert.Nil(t, err, "Expect no error") 27 | assert.Equal(t, "AKID", creds.AccessKeyID, "Expect access key ID to match") 28 | assert.Equal(t, "SECRET", creds.SecretAccessKey, "Expect secret access key to match") 29 | assert.Empty(t, creds.SessionToken, "Expect session token to be empty") 30 | } 31 | 32 | func TestChainProviderIsExpired(t *testing.T) { 33 | stubProvider := &stubProvider{expired: true} 34 | p := &ChainProvider{ 35 | Providers: []Provider{ 36 | stubProvider, 37 | }, 38 | } 39 | 40 | assert.True(t, p.IsExpired(), "Expect expired to be true before any Retrieve") 41 | _, err := p.Retrieve() 42 | assert.Nil(t, err, "Expect no error") 43 | assert.False(t, p.IsExpired(), "Expect not expired after retrieve") 44 | 45 | stubProvider.expired = true 46 | assert.True(t, p.IsExpired(), "Expect return of expired provider") 47 | 48 | _, err = p.Retrieve() 49 | assert.False(t, p.IsExpired(), "Expect not expired after retrieve") 50 | } 51 | 52 | func TestChainProviderWithNoProvider(t *testing.T) { 53 | p := &ChainProvider{ 54 | Providers: []Provider{}, 55 | } 56 | 57 | assert.True(t, p.IsExpired(), "Expect expired with no providers") 58 | _, err := p.Retrieve() 59 | assert.Equal(t, ErrNoValidProvidersFoundInChain, err, "Expect no providers error returned") 60 | } 61 | 62 | func TestChainProviderWithNoValidProvider(t *testing.T) { 63 | p := &ChainProvider{ 64 | Providers: []Provider{ 65 | &stubProvider{err: apierr.New("FirstError", "first provider error", nil)}, 66 | &stubProvider{err: apierr.New("SecondError", "second provider error", nil)}, 67 | }, 68 | } 69 | 70 | assert.True(t, p.IsExpired(), "Expect expired with no providers") 71 | _, err := p.Retrieve() 72 | assert.Equal(t, ErrNoValidProvidersFoundInChain, err, "Expect no providers error returned") 73 | } 74 | -------------------------------------------------------------------------------- /aws/credentials/credentials.go: -------------------------------------------------------------------------------- 1 | // Package credentials provides credential retrieval and management 2 | // 3 | // The Credentials is the primary method of getting access to and managing 4 | // credentials Values. Using dependency injection retrieval of the credential 5 | // values is handled by a object which satisfies the Provider interface. 6 | // 7 | // By default the Credentials.Get() will cache the successful result of a 8 | // Provider's Retrieve() until Provider.IsExpired() returns true. At which 9 | // point Credentials will call Provider's Retrieve() to get new credential Value. 10 | // 11 | // The Provider is responsible for determining when credentials Value have expired. 12 | // It is also important to note that Credentials will always call Retrieve the 13 | // first time Credentials.Get() is called. 14 | // 15 | // Example of using the environment variable credentials. 16 | // 17 | // creds := NewEnvCredentials() 18 | // 19 | // // Retrieve the credentials value 20 | // credValue, err := creds.Get() 21 | // if err != nil { 22 | // // handle error 23 | // } 24 | // 25 | // Example of forcing credentials to expire and be refreshed on the next Get(). 26 | // This may be helpful to proactively expire credentials and refresh them sooner 27 | // than they would naturally expire on their own. 28 | // 29 | // creds := NewCredentials(&EC2RoleProvider{}) 30 | // creds.Expire() 31 | // credsValue, err := creds.Get() 32 | // // New credentials will be retrieved instead of from cache. 33 | // 34 | // 35 | // Custom Provider 36 | // 37 | // Each Provider built into this package also provides a helper method to generate 38 | // a Credentials pointer setup with the provider. To use a custom Provider just 39 | // create a type which satisfies the Provider interface and pass it to the 40 | // NewCredentials method. 41 | // 42 | // type MyProvider struct{} 43 | // func (m *MyProvider) Retrieve() (Value, error) {...} 44 | // func (m *MyProvider) IsExpired() bool {...} 45 | // 46 | // creds := NewCredentials(&MyProvider{}) 47 | // credValue, err := creds.Get() 48 | // 49 | package credentials 50 | 51 | import ( 52 | "sync" 53 | "time" 54 | ) 55 | 56 | // Create an empty Credential object that can be used as dummy placeholder 57 | // credentials for requests that do not need signed. 58 | // 59 | // This Credentials can be used to configure a service to not sign requests 60 | // when making service API calls. For example, when accessing public 61 | // s3 buckets. 62 | // 63 | // svc := s3.New(&aws.Config{Credentials: AnonymousCredentials}) 64 | // // Access public S3 buckets. 65 | // 66 | var AnonymousCredentials = NewStaticCredentials("", "", "") 67 | 68 | // A Value is the AWS credentials value for individual credential fields. 69 | type Value struct { 70 | // AWS Access key ID 71 | AccessKeyID string 72 | 73 | // AWS Secret Access Key 74 | SecretAccessKey string 75 | 76 | // AWS Session Token 77 | SessionToken string 78 | } 79 | 80 | // A Provider is the interface for any component which will provide credentials 81 | // Value. A provider is required to manage its own Expired state, and what to 82 | // be expired means. 83 | // 84 | // The Provider should not need to implement its own mutexes, because 85 | // that will be managed by Credentials. 86 | type Provider interface { 87 | // Refresh returns nil if it successfully retrieved the value. 88 | // Error is returned if the value were not obtainable, or empty. 89 | Retrieve() (Value, error) 90 | 91 | // IsExpired returns if the credentials are no longer valid, and need 92 | // to be retrieved. 93 | IsExpired() bool 94 | } 95 | 96 | // A Credentials provides synchronous safe retrieval of AWS credentials Value. 97 | // Credentials will cache the credentials value until they expire. Once the value 98 | // expires the next Get will attempt to retrieve valid credentials. 99 | // 100 | // Credentials is safe to use across multiple goroutines and will manage the 101 | // synchronous state so the Providers do not need to implement their own 102 | // synchronization. 103 | // 104 | // The first Credentials.Get() will always call Provider.Retrieve() to get the 105 | // first instance of the credentials Value. All calls to Get() after that 106 | // will return the cached credentials Value until IsExpired() returns true. 107 | type Credentials struct { 108 | creds Value 109 | forceRefresh bool 110 | m sync.Mutex 111 | 112 | provider Provider 113 | } 114 | 115 | // NewCredentials returns a pointer to a new Credentials with the provider set. 116 | func NewCredentials(provider Provider) *Credentials { 117 | return &Credentials{ 118 | provider: provider, 119 | forceRefresh: true, 120 | } 121 | } 122 | 123 | // Get returns the credentials value, or error if the credentials Value failed 124 | // to be retrieved. 125 | // 126 | // Will return the cached credentials Value if it has not expired. If the 127 | // credentials Value has expired the Provider's Retrieve() will be called 128 | // to refresh the credentials. 129 | // 130 | // If Credentials.Expire() was called the credentials Value will be force 131 | // expired, and the next call to Get() will cause them to be refreshed. 132 | func (c *Credentials) Get() (Value, error) { 133 | c.m.Lock() 134 | defer c.m.Unlock() 135 | 136 | if c.isExpired() { 137 | creds, err := c.provider.Retrieve() 138 | if err != nil { 139 | return Value{}, err 140 | } 141 | c.creds = creds 142 | c.forceRefresh = false 143 | } 144 | 145 | return c.creds, nil 146 | } 147 | 148 | // Expire expires the credentials and forces them to be retrieved on the 149 | // next call to Get(). 150 | // 151 | // This will override the Provider's expired state, and force Credentials 152 | // to call the Provider's Retrieve(). 153 | func (c *Credentials) Expire() { 154 | c.m.Lock() 155 | defer c.m.Unlock() 156 | 157 | c.forceRefresh = true 158 | } 159 | 160 | // IsExpired returns if the credentials are no longer valid, and need 161 | // to be retrieved. 162 | // 163 | // If the Credentials were forced to be expired with Expire() this will 164 | // reflect that override. 165 | func (c *Credentials) IsExpired() bool { 166 | c.m.Lock() 167 | defer c.m.Unlock() 168 | 169 | return c.isExpired() 170 | } 171 | 172 | // isExpired helper method wrapping the definition of expired credentials. 173 | func (c *Credentials) isExpired() bool { 174 | return c.forceRefresh || c.provider.IsExpired() 175 | } 176 | 177 | // Provide a stub-able time.Now for unit tests so expiry can be tested. 178 | var currentTime = time.Now 179 | -------------------------------------------------------------------------------- /aws/credentials/credentials_test.go: -------------------------------------------------------------------------------- 1 | package credentials 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ks3sdklib/aws-sdk-go/aws/awserr" 7 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | type stubProvider struct { 12 | creds Value 13 | expired bool 14 | err error 15 | } 16 | 17 | func (s *stubProvider) Retrieve() (Value, error) { 18 | s.expired = false 19 | return s.creds, s.err 20 | } 21 | func (s *stubProvider) IsExpired() bool { 22 | return s.expired 23 | } 24 | 25 | func TestCredentialsGet(t *testing.T) { 26 | c := NewCredentials(&stubProvider{ 27 | creds: Value{ 28 | AccessKeyID: "AKID", 29 | SecretAccessKey: "SECRET", 30 | SessionToken: "", 31 | }, 32 | expired: true, 33 | }) 34 | 35 | creds, err := c.Get() 36 | assert.Nil(t, err, "Expected no error") 37 | assert.Equal(t, "AKID", creds.AccessKeyID, "Expect access key ID to match") 38 | assert.Equal(t, "SECRET", creds.SecretAccessKey, "Expect secret access key to match") 39 | assert.Empty(t, creds.SessionToken, "Expect session token to be empty") 40 | } 41 | 42 | func TestCredentialsGetWithError(t *testing.T) { 43 | c := NewCredentials(&stubProvider{err: apierr.New("provider error", "", nil), expired: true}) 44 | 45 | _, err := c.Get() 46 | assert.Equal(t, "provider error", err.(awserr.Error).Code(), "Expected provider error") 47 | } 48 | 49 | func TestCredentialsExpire(t *testing.T) { 50 | stub := &stubProvider{} 51 | c := NewCredentials(stub) 52 | 53 | stub.expired = false 54 | assert.True(t, c.IsExpired(), "Expected to start out expired") 55 | c.Expire() 56 | assert.True(t, c.IsExpired(), "Expected to be expired") 57 | 58 | c.forceRefresh = false 59 | assert.False(t, c.IsExpired(), "Expected not to be expired") 60 | 61 | stub.expired = true 62 | assert.True(t, c.IsExpired(), "Expected to be expired") 63 | } 64 | -------------------------------------------------------------------------------- /aws/credentials/ec2_role_provider.go: -------------------------------------------------------------------------------- 1 | package credentials 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "time" 9 | 10 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 11 | ) 12 | 13 | const metadataCredentialsEndpoint = "http://169.254.169.254/latest/meta-data/iam/security-credentials/" 14 | 15 | // A EC2RoleProvider retrieves credentials from the EC2 service, and keeps track if 16 | // those credentials are expired. 17 | // 18 | // Example how to configure the EC2RoleProvider with custom http Client, Endpoint 19 | // or ExpiryWindow 20 | // 21 | // p := &credentials.EC2RoleProvider{ 22 | // // Pass in a custom timeout to be used when requesting 23 | // // IAM EC2 Role credentials. 24 | // Client: &http.Client{ 25 | // Timeout: 10 * time.Second, 26 | // }, 27 | // // Use default EC2 Role metadata endpoint, Alternate endpoints can be 28 | // // specified setting Endpoint to something else. 29 | // Endpoint: "", 30 | // // Do not use early expiry of credentials. If a non zero value is 31 | // // specified the credentials will be expired early 32 | // ExpiryWindow: 0, 33 | // } 34 | // 35 | type EC2RoleProvider struct { 36 | // Endpoint must be fully quantified URL 37 | Endpoint string 38 | 39 | // HTTP client to use when connecting to EC2 service 40 | Client *http.Client 41 | 42 | // ExpiryWindow will allow the credentials to trigger refreshing prior to 43 | // the credentials actually expiring. This is beneficial so race conditions 44 | // with expiring credentials do not cause request to fail unexpectedly 45 | // due to ExpiredTokenException exceptions. 46 | // 47 | // So a ExpiryWindow of 10s would cause calls to IsExpired() to return true 48 | // 10 seconds before the credentials are actually expired. 49 | // 50 | // If ExpiryWindow is 0 or less it will be ignored. 51 | ExpiryWindow time.Duration 52 | 53 | // The date/time at which the credentials expire. 54 | expiresOn time.Time 55 | } 56 | 57 | // NewEC2RoleCredentials returns a pointer to a new Credentials object 58 | // wrapping the EC2RoleProvider. 59 | // 60 | // Takes a custom http.Client which can be configured for custom handling of 61 | // things such as timeout. 62 | // 63 | // Endpoint is the URL that the EC2RoleProvider will connect to when retrieving 64 | // role and credentials. 65 | // 66 | // Window is the expiry window that will be subtracted from the expiry returned 67 | // by the role credential request. This is done so that the credentials will 68 | // expire sooner than their actual lifespan. 69 | func NewEC2RoleCredentials(client *http.Client, endpoint string, window time.Duration) *Credentials { 70 | return NewCredentials(&EC2RoleProvider{ 71 | Endpoint: endpoint, 72 | Client: client, 73 | ExpiryWindow: window, 74 | }) 75 | } 76 | 77 | // Retrieve retrieves credentials from the EC2 service. 78 | // Error will be returned if the request fails, or unable to extract 79 | // the desired credentials. 80 | func (m *EC2RoleProvider) Retrieve() (Value, error) { 81 | if m.Client == nil { 82 | m.Client = http.DefaultClient 83 | } 84 | if m.Endpoint == "" { 85 | m.Endpoint = metadataCredentialsEndpoint 86 | } 87 | 88 | credsList, err := requestCredList(m.Client, m.Endpoint) 89 | if err != nil { 90 | return Value{}, err 91 | } 92 | 93 | if len(credsList) == 0 { 94 | return Value{}, apierr.New("EmptyEC2RoleList", "empty EC2 Role list", nil) 95 | } 96 | credsName := credsList[0] 97 | 98 | roleCreds, err := requestCred(m.Client, m.Endpoint, credsName) 99 | if err != nil { 100 | return Value{}, err 101 | } 102 | 103 | m.expiresOn = roleCreds.Expiration 104 | if m.ExpiryWindow > 0 { 105 | // Offset based on expiry window if set. 106 | m.expiresOn = m.expiresOn.Add(-m.ExpiryWindow) 107 | } 108 | 109 | return Value{ 110 | AccessKeyID: roleCreds.AccessKeyID, 111 | SecretAccessKey: roleCreds.SecretAccessKey, 112 | SessionToken: roleCreds.Token, 113 | }, nil 114 | } 115 | 116 | // IsExpired returns if the credentials are expired. 117 | func (m *EC2RoleProvider) IsExpired() bool { 118 | return m.expiresOn.Before(currentTime()) 119 | } 120 | 121 | // A ec2RoleCredRespBody provides the shape for deserializing credential 122 | // request responses. 123 | type ec2RoleCredRespBody struct { 124 | Expiration time.Time 125 | AccessKeyID string 126 | SecretAccessKey string 127 | Token string 128 | } 129 | 130 | // requestCredList requests a list of credentials from the EC2 service. 131 | // If there are no credentials, or there is an error making or receiving the request 132 | func requestCredList(client *http.Client, endpoint string) ([]string, error) { 133 | resp, err := client.Get(endpoint) 134 | if err != nil { 135 | return nil, apierr.New("ListEC2Role", "failed to list EC2 Roles", err) 136 | } 137 | defer resp.Body.Close() 138 | 139 | credsList := []string{} 140 | s := bufio.NewScanner(resp.Body) 141 | for s.Scan() { 142 | credsList = append(credsList, s.Text()) 143 | } 144 | 145 | if err := s.Err(); err != nil { 146 | return nil, apierr.New("ReadEC2Role", "failed to read list of EC2 Roles", err) 147 | } 148 | 149 | return credsList, nil 150 | } 151 | 152 | // requestCred requests the credentials for a specific credentials from the EC2 service. 153 | // 154 | // If the credentials cannot be found, or there is an error reading the response 155 | // and error will be returned. 156 | func requestCred(client *http.Client, endpoint, credsName string) (*ec2RoleCredRespBody, error) { 157 | resp, err := client.Get(endpoint + credsName) 158 | if err != nil { 159 | return nil, apierr.New("GetEC2RoleCredentials", 160 | fmt.Sprintf("failed to get %s EC2 Role credentials", credsName), 161 | err) 162 | } 163 | defer resp.Body.Close() 164 | 165 | respCreds := &ec2RoleCredRespBody{} 166 | if err := json.NewDecoder(resp.Body).Decode(respCreds); err != nil { 167 | return nil, apierr.New("DecodeEC2RoleCredentials", 168 | fmt.Sprintf("failed to decode %s EC2 Role credentials", credsName), 169 | err) 170 | } 171 | 172 | return respCreds, nil 173 | } 174 | -------------------------------------------------------------------------------- /aws/credentials/ec2_role_provider_test.go: -------------------------------------------------------------------------------- 1 | package credentials 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "net/http" 7 | "net/http/httptest" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func initTestServer(expireOn string) *httptest.Server { 13 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 14 | if r.RequestURI == "/" { 15 | fmt.Fprintln(w, "/creds") 16 | } else { 17 | fmt.Fprintf(w, `{ 18 | "AccessKeyId" : "accessKey", 19 | "SecretAccessKey" : "secret", 20 | "Token" : "token", 21 | "Expiration" : "%s" 22 | }`, expireOn) 23 | } 24 | })) 25 | 26 | return server 27 | } 28 | 29 | func TestEC2RoleProvider(t *testing.T) { 30 | server := initTestServer("2014-12-16T01:51:37Z") 31 | defer server.Close() 32 | 33 | p := &EC2RoleProvider{Client: http.DefaultClient, Endpoint: server.URL} 34 | 35 | creds, err := p.Retrieve() 36 | assert.Nil(t, err, "Expect no error") 37 | 38 | assert.Equal(t, "accessKey", creds.AccessKeyID, "Expect access key ID to match") 39 | assert.Equal(t, "secret", creds.SecretAccessKey, "Expect secret access key to match") 40 | assert.Equal(t, "token", creds.SessionToken, "Expect session token to match") 41 | } 42 | 43 | func TestEC2RoleProviderIsExpired(t *testing.T) { 44 | server := initTestServer("2014-12-16T01:51:37Z") 45 | defer server.Close() 46 | 47 | p := &EC2RoleProvider{Client: http.DefaultClient, Endpoint: server.URL} 48 | defer func() { 49 | currentTime = time.Now 50 | }() 51 | currentTime = func() time.Time { 52 | return time.Date(2014, 12, 15, 21, 26, 0, 0, time.UTC) 53 | } 54 | 55 | assert.True(t, p.IsExpired(), "Expect creds to be expired before retrieve.") 56 | 57 | _, err := p.Retrieve() 58 | assert.Nil(t, err, "Expect no error") 59 | 60 | assert.False(t, p.IsExpired(), "Expect creds to not be expired after retrieve.") 61 | 62 | currentTime = func() time.Time { 63 | return time.Date(3014, 12, 15, 21, 26, 0, 0, time.UTC) 64 | } 65 | 66 | assert.True(t, p.IsExpired(), "Expect creds to be expired.") 67 | } 68 | 69 | func TestEC2RoleProviderExpiryWindowIsExpired(t *testing.T) { 70 | server := initTestServer("2014-12-16T01:51:37Z") 71 | defer server.Close() 72 | 73 | p := &EC2RoleProvider{Client: http.DefaultClient, Endpoint: server.URL, ExpiryWindow: time.Hour * 1} 74 | defer func() { 75 | currentTime = time.Now 76 | }() 77 | currentTime = func() time.Time { 78 | return time.Date(2014, 12, 15, 0, 51, 37, 0, time.UTC) 79 | } 80 | 81 | assert.True(t, p.IsExpired(), "Expect creds to be expired before retrieve.") 82 | 83 | _, err := p.Retrieve() 84 | assert.Nil(t, err, "Expect no error") 85 | 86 | assert.False(t, p.IsExpired(), "Expect creds to not be expired after retrieve.") 87 | 88 | currentTime = func() time.Time { 89 | return time.Date(2014, 12, 16, 0, 55, 37, 0, time.UTC) 90 | } 91 | 92 | assert.True(t, p.IsExpired(), "Expect creds to be expired.") 93 | } 94 | 95 | func BenchmarkEC2RoleProvider(b *testing.B) { 96 | server := initTestServer("2014-12-16T01:51:37Z") 97 | defer server.Close() 98 | 99 | p := &EC2RoleProvider{Client: http.DefaultClient, Endpoint: server.URL} 100 | _, err := p.Retrieve() 101 | if err != nil { 102 | b.Fatal(err) 103 | } 104 | 105 | b.ResetTimer() 106 | b.RunParallel(func(pb *testing.PB) { 107 | for pb.Next() { 108 | _, err := p.Retrieve() 109 | if err != nil { 110 | b.Fatal(err) 111 | } 112 | } 113 | }) 114 | } 115 | -------------------------------------------------------------------------------- /aws/credentials/env_provider.go: -------------------------------------------------------------------------------- 1 | package credentials 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 7 | ) 8 | 9 | var ( 10 | // ErrAccessKeyIDNotFound is returned when the AWS Access Key ID can't be 11 | // found in the process's environment. 12 | ErrAccessKeyIDNotFound = apierr.New("EnvAccessKeyNotFound", "AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY not found in environment", nil) 13 | // ErrSecretAccessKeyNotFound is returned when the AWS Secret Access Key 14 | // can't be found in the process's environment. 15 | ErrSecretAccessKeyNotFound = apierr.New("EnvSecretNotFound", "AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY not found in environment", nil) 16 | ) 17 | 18 | // A EnvProvider retrieves credentials from the environment variables of the 19 | // running process. Environment credentials never expire. 20 | // 21 | // Environment variables used: 22 | // - Access Key ID: AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY 23 | // - Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY 24 | type EnvProvider struct { 25 | retrieved bool 26 | } 27 | 28 | // NewEnvCredentials returns a pointer to a new Credentials object 29 | // wrapping the environment variable provider. 30 | func NewEnvCredentials() *Credentials { 31 | return NewCredentials(&EnvProvider{}) 32 | } 33 | 34 | // Retrieve retrieves the keys from the environment. 35 | func (e *EnvProvider) Retrieve() (Value, error) { 36 | e.retrieved = false 37 | 38 | id := os.Getenv("AWS_ACCESS_KEY_ID") 39 | if id == "" { 40 | id = os.Getenv("AWS_ACCESS_KEY") 41 | } 42 | 43 | secret := os.Getenv("AWS_SECRET_ACCESS_KEY") 44 | if secret == "" { 45 | secret = os.Getenv("AWS_SECRET_KEY") 46 | } 47 | 48 | if id == "" { 49 | return Value{}, ErrAccessKeyIDNotFound 50 | } 51 | 52 | if secret == "" { 53 | return Value{}, ErrSecretAccessKeyNotFound 54 | } 55 | 56 | e.retrieved = true 57 | return Value{ 58 | AccessKeyID: id, 59 | SecretAccessKey: secret, 60 | SessionToken: os.Getenv("AWS_SESSION_TOKEN"), 61 | }, nil 62 | } 63 | 64 | // IsExpired returns if the credentials have been retrieved. 65 | func (e *EnvProvider) IsExpired() bool { 66 | return !e.retrieved 67 | } 68 | -------------------------------------------------------------------------------- /aws/credentials/env_provider_test.go: -------------------------------------------------------------------------------- 1 | package credentials 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestEnvProviderRetrieve(t *testing.T) { 10 | os.Clearenv() 11 | os.Setenv("AWS_ACCESS_KEY_ID", "access") 12 | os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") 13 | os.Setenv("AWS_SESSION_TOKEN", "token") 14 | 15 | e := EnvProvider{} 16 | creds, err := e.Retrieve() 17 | assert.Nil(t, err, "Expect no error") 18 | 19 | assert.Equal(t, "access", creds.AccessKeyID, "Expect access key ID to match") 20 | assert.Equal(t, "secret", creds.SecretAccessKey, "Expect secret access key to match") 21 | assert.Equal(t, "token", creds.SessionToken, "Expect session token to match") 22 | } 23 | 24 | func TestEnvProviderIsExpired(t *testing.T) { 25 | os.Clearenv() 26 | os.Setenv("AWS_ACCESS_KEY_ID", "access") 27 | os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") 28 | os.Setenv("AWS_SESSION_TOKEN", "token") 29 | 30 | e := EnvProvider{} 31 | 32 | assert.True(t, e.IsExpired(), "Expect creds to be expired before retrieve.") 33 | 34 | _, err := e.Retrieve() 35 | assert.Nil(t, err, "Expect no error") 36 | 37 | assert.False(t, e.IsExpired(), "Expect creds to not be expired after retrieve.") 38 | } 39 | 40 | func TestEnvProviderNoAccessKeyID(t *testing.T) { 41 | os.Clearenv() 42 | os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") 43 | 44 | e := EnvProvider{} 45 | creds, err := e.Retrieve() 46 | assert.Equal(t, ErrAccessKeyIDNotFound, err, "ErrAccessKeyIDNotFound expected, but was %#v error: %#v", creds, err) 47 | } 48 | 49 | func TestEnvProviderNoSecretAccessKey(t *testing.T) { 50 | os.Clearenv() 51 | os.Setenv("AWS_ACCESS_KEY_ID", "access") 52 | 53 | e := EnvProvider{} 54 | creds, err := e.Retrieve() 55 | assert.Equal(t, ErrSecretAccessKeyNotFound, err, "ErrSecretAccessKeyNotFound expected, but was %#v error: %#v", creds, err) 56 | } 57 | 58 | func TestEnvProviderAlternateNames(t *testing.T) { 59 | os.Clearenv() 60 | os.Setenv("AWS_ACCESS_KEY", "access") 61 | os.Setenv("AWS_SECRET_KEY", "secret") 62 | 63 | e := EnvProvider{} 64 | creds, err := e.Retrieve() 65 | assert.Nil(t, err, "Expect no error") 66 | 67 | assert.Equal(t, "access", creds.AccessKeyID, "Expected access key ID") 68 | assert.Equal(t, "secret", creds.SecretAccessKey, "Expected secret access key") 69 | assert.Empty(t, creds.SessionToken, "Expected no token") 70 | } 71 | -------------------------------------------------------------------------------- /aws/credentials/example.ini: -------------------------------------------------------------------------------- 1 | [default] 2 | aws_access_key_id = accessKey 3 | aws_secret_access_key = secret 4 | aws_session_token = token 5 | 6 | [no_token] 7 | aws_access_key_id = accessKey 8 | aws_secret_access_key = secret 9 | -------------------------------------------------------------------------------- /aws/credentials/ini.go: -------------------------------------------------------------------------------- 1 | // Package ini provides functions for parsing INI configuration files. 2 | package credentials 3 | 4 | import ( 5 | "bufio" 6 | "fmt" 7 | "io" 8 | "os" 9 | "regexp" 10 | "strings" 11 | ) 12 | 13 | var ( 14 | sectionRegex = regexp.MustCompile(`^\[(.*)\]$`) 15 | assignRegex = regexp.MustCompile(`^([^=]+)=(.*)$`) 16 | ) 17 | 18 | // ErrSyntax is returned when there is a syntax error in an INI file. 19 | type ErrSyntax struct { 20 | Line int 21 | Source string // The contents of the erroneous line, without leading or trailing whitespace 22 | } 23 | 24 | func (e ErrSyntax) Error() string { 25 | return fmt.Sprintf("invalid INI syntax on line %d: %s", e.Line, e.Source) 26 | } 27 | 28 | // A File represents a parsed INI file. 29 | type File map[string]Section 30 | 31 | // A Section represents a single section of an INI file. 32 | type Section map[string]string 33 | 34 | // Returns a named Section. A Section will be created if one does not already exist for the given name. 35 | func (f File) Section(name string) Section { 36 | section := f[name] 37 | if section == nil { 38 | section = make(Section) 39 | f[name] = section 40 | } 41 | return section 42 | } 43 | 44 | // Looks up a value for a key in a section and returns that value, along with a boolean result similar to a map lookup. 45 | func (f File) Get(section, key string) (value string, ok bool) { 46 | if s := f[section]; s != nil { 47 | value, ok = s[key] 48 | } 49 | return 50 | } 51 | 52 | // Loads INI data from a reader and stores the data in the File. 53 | func (f File) Load(in io.Reader) (err error) { 54 | bufin, ok := in.(*bufio.Reader) 55 | if !ok { 56 | bufin = bufio.NewReader(in) 57 | } 58 | return parseFile(bufin, f) 59 | } 60 | 61 | // Loads INI data from a named file and stores the data in the File. 62 | func (f File) LoadFile(file string) (err error) { 63 | in, err := os.Open(file) 64 | if err != nil { 65 | return 66 | } 67 | defer in.Close() 68 | return f.Load(in) 69 | } 70 | 71 | func parseFile(in *bufio.Reader, file File) (err error) { 72 | section := "" 73 | lineNum := 0 74 | for done := false; !done; { 75 | var line string 76 | if line, err = in.ReadString('\n'); err != nil { 77 | if err == io.EOF { 78 | done = true 79 | } else { 80 | return 81 | } 82 | } 83 | lineNum++ 84 | line = strings.TrimSpace(line) 85 | if len(line) == 0 { 86 | // Skip blank lines 87 | continue 88 | } 89 | if line[0] == ';' || line[0] == '#' { 90 | // Skip comments 91 | continue 92 | } 93 | 94 | if groups := assignRegex.FindStringSubmatch(line); groups != nil { 95 | key, val := groups[1], groups[2] 96 | key, val = strings.TrimSpace(key), strings.TrimSpace(val) 97 | file.Section(section)[key] = val 98 | } else if groups := sectionRegex.FindStringSubmatch(line); groups != nil { 99 | name := strings.TrimSpace(groups[1]) 100 | section = name 101 | // Create the section if it does not exist 102 | file.Section(section) 103 | } else { 104 | return ErrSyntax{lineNum, line} 105 | } 106 | 107 | } 108 | return nil 109 | } 110 | 111 | // Loads and returns a File from a reader. 112 | func Load(in io.Reader) (File, error) { 113 | file := make(File) 114 | err := file.Load(in) 115 | return file, err 116 | } 117 | 118 | // Loads and returns an INI File from a file on disk. 119 | func LoadFile(filename string) (File, error) { 120 | file := make(File) 121 | err := file.LoadFile(filename) 122 | return file, err 123 | } -------------------------------------------------------------------------------- /aws/credentials/shared_credentials_provider.go: -------------------------------------------------------------------------------- 1 | package credentials 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 9 | ) 10 | 11 | var ( 12 | // ErrSharedCredentialsHomeNotFound is emitted when the user directory cannot be found. 13 | ErrSharedCredentialsHomeNotFound = apierr.New("UserHomeNotFound", "user home directory not found.", nil) 14 | ) 15 | 16 | // A SharedCredentialsProvider retrieves credentials from the current user's home 17 | // directory, and keeps track if those credentials are expired. 18 | // 19 | // Profile ini file example: $HOME/.aws/credentials 20 | type SharedCredentialsProvider struct { 21 | // Path to the shared credentials file. If empty will default to current user's 22 | // home directory. 23 | Filename string 24 | 25 | // AWS Profile to extract credentials from the shared credentials file. If empty 26 | // will default to environment variable "AWS_PROFILE" or "default" if 27 | // environment variable is also not set. 28 | Profile string 29 | 30 | // retrieved states if the credentials have been successfully retrieved. 31 | retrieved bool 32 | } 33 | 34 | // NewSharedCredentials returns a pointer to a new Credentials object 35 | // wrapping the Profile file provider. 36 | func NewSharedCredentials(filename, profile string) *Credentials { 37 | return NewCredentials(&SharedCredentialsProvider{ 38 | Filename: filename, 39 | Profile: profile, 40 | }) 41 | } 42 | 43 | // Retrieve reads and extracts the shared credentials from the current 44 | // users home directory. 45 | func (p *SharedCredentialsProvider) Retrieve() (Value, error) { 46 | p.retrieved = false 47 | 48 | filename, err := p.filename() 49 | if err != nil { 50 | return Value{}, err 51 | } 52 | 53 | creds, err := loadProfile(filename, p.profile()) 54 | if err != nil { 55 | return Value{}, err 56 | } 57 | 58 | p.retrieved = true 59 | return creds, nil 60 | } 61 | 62 | // IsExpired returns if the shared credentials have expired. 63 | func (p *SharedCredentialsProvider) IsExpired() bool { 64 | return !p.retrieved 65 | } 66 | 67 | // loadProfiles loads from the file pointed to by shared credentials filename for profile. 68 | // The credentials retrieved from the profile will be returned or error. Error will be 69 | // returned if it fails to read from the file, or the data is invalid. 70 | func loadProfile(filename, profile string) (Value, error) { 71 | config, err := LoadFile(filename) 72 | if err != nil { 73 | return Value{}, apierr.New("SharedCredsLoad", "failed to load shared credentials file", err) 74 | } 75 | iniProfile := config.Section(profile) 76 | 77 | id, ok := iniProfile["aws_access_key_id"] 78 | if !ok { 79 | return Value{}, apierr.New("SharedCredsAccessKey", 80 | fmt.Sprintf("shared credentials %s in %s did not contain aws_access_key_id", profile, filename), 81 | nil) 82 | } 83 | 84 | secret, ok := iniProfile["aws_secret_access_key"] 85 | if !ok { 86 | return Value{}, apierr.New("SharedCredsSecret", 87 | fmt.Sprintf("shared credentials %s in %s did not contain aws_secret_access_key", profile, filename), 88 | nil) 89 | } 90 | 91 | token := iniProfile["aws_session_token"] 92 | 93 | return Value{ 94 | AccessKeyID: id, 95 | SecretAccessKey: secret, 96 | SessionToken: token, 97 | }, nil 98 | } 99 | 100 | // filename returns the filename to use to read AWS shared credentials. 101 | // 102 | // Will return an error if the user's home directory path cannot be found. 103 | func (p *SharedCredentialsProvider) filename() (string, error) { 104 | if p.Filename == "" { 105 | homeDir := os.Getenv("HOME") // *nix 106 | if homeDir == "" { // Windows 107 | homeDir = os.Getenv("USERPROFILE") 108 | } 109 | if homeDir == "" { 110 | return "", ErrSharedCredentialsHomeNotFound 111 | } 112 | 113 | p.Filename = filepath.Join(homeDir, ".aws", "credentials") 114 | } 115 | 116 | return p.Filename, nil 117 | } 118 | 119 | // profile returns the AWS shared credentials profile. If empty will read 120 | // environment variable "AWS_PROFILE". If that is not set profile will 121 | // return "default". 122 | func (p *SharedCredentialsProvider) profile() string { 123 | if p.Profile == "" { 124 | p.Profile = os.Getenv("AWS_PROFILE") 125 | } 126 | if p.Profile == "" { 127 | p.Profile = "default" 128 | } 129 | 130 | return p.Profile 131 | } 132 | -------------------------------------------------------------------------------- /aws/credentials/shared_credentials_provider_test.go: -------------------------------------------------------------------------------- 1 | package credentials 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestSharedCredentialsProvider(t *testing.T) { 10 | os.Clearenv() 11 | 12 | p := SharedCredentialsProvider{Filename: "example.ini", Profile: ""} 13 | creds, err := p.Retrieve() 14 | assert.Nil(t, err, "Expect no error") 15 | 16 | assert.Equal(t, "accessKey", creds.AccessKeyID, "Expect access key ID to match") 17 | assert.Equal(t, "secret", creds.SecretAccessKey, "Expect secret access key to match") 18 | assert.Equal(t, "token", creds.SessionToken, "Expect session token to match") 19 | } 20 | 21 | func TestSharedCredentialsProviderIsExpired(t *testing.T) { 22 | os.Clearenv() 23 | 24 | p := SharedCredentialsProvider{Filename: "example.ini", Profile: ""} 25 | 26 | assert.True(t, p.IsExpired(), "Expect creds to be expired before retrieve") 27 | 28 | _, err := p.Retrieve() 29 | assert.Nil(t, err, "Expect no error") 30 | 31 | assert.False(t, p.IsExpired(), "Expect creds to not be expired after retrieve") 32 | } 33 | 34 | func TestSharedCredentialsProviderWithAWS_PROFILE(t *testing.T) { 35 | os.Clearenv() 36 | os.Setenv("AWS_PROFILE", "no_token") 37 | 38 | p := SharedCredentialsProvider{Filename: "example.ini", Profile: ""} 39 | creds, err := p.Retrieve() 40 | assert.Nil(t, err, "Expect no error") 41 | 42 | assert.Equal(t, "accessKey", creds.AccessKeyID, "Expect access key ID to match") 43 | assert.Equal(t, "secret", creds.SecretAccessKey, "Expect secret access key to match") 44 | assert.Empty(t, creds.SessionToken, "Expect no token") 45 | } 46 | 47 | func TestSharedCredentialsProviderWithoutTokenFromProfile(t *testing.T) { 48 | os.Clearenv() 49 | 50 | p := SharedCredentialsProvider{Filename: "example.ini", Profile: "no_token"} 51 | creds, err := p.Retrieve() 52 | assert.Nil(t, err, "Expect no error") 53 | 54 | assert.Equal(t, "accessKey", creds.AccessKeyID, "Expect access key ID to match") 55 | assert.Equal(t, "secret", creds.SecretAccessKey, "Expect secret access key to match") 56 | assert.Empty(t, creds.SessionToken, "Expect no token") 57 | } 58 | 59 | func BenchmarkSharedCredentialsProvider(b *testing.B) { 60 | os.Clearenv() 61 | 62 | p := SharedCredentialsProvider{Filename: "example.ini", Profile: ""} 63 | _, err := p.Retrieve() 64 | if err != nil { 65 | b.Fatal(err) 66 | } 67 | 68 | b.ResetTimer() 69 | b.RunParallel(func(pb *testing.PB) { 70 | for pb.Next() { 71 | _, err := p.Retrieve() 72 | if err != nil { 73 | b.Fatal(err) 74 | } 75 | } 76 | }) 77 | } 78 | -------------------------------------------------------------------------------- /aws/credentials/static_provider.go: -------------------------------------------------------------------------------- 1 | package credentials 2 | 3 | import ( 4 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 5 | ) 6 | 7 | var ( 8 | // ErrStaticCredentialsEmpty is emitted when static credentials are empty. 9 | ErrStaticCredentialsEmpty = apierr.New("EmptyStaticCreds", "static credentials are empty", nil) 10 | ) 11 | 12 | // A StaticProvider is a set of credentials which are set pragmatically, 13 | // and will never expire. 14 | type StaticProvider struct { 15 | Value 16 | } 17 | 18 | // NewStaticCredentials returns a pointer to a new Credentials object 19 | // wrapping a static credentials value provider. 20 | func NewStaticCredentials(id, secret, token string) *Credentials { 21 | return NewCredentials(&StaticProvider{Value: Value{ 22 | AccessKeyID: id, 23 | SecretAccessKey: secret, 24 | SessionToken: token, 25 | }}) 26 | } 27 | 28 | // Retrieve returns the credentials or error if the credentials are invalid. 29 | func (s *StaticProvider) Retrieve() (Value, error) { 30 | if s.AccessKeyID == "" || s.SecretAccessKey == "" { 31 | return Value{}, ErrStaticCredentialsEmpty 32 | } 33 | 34 | return s.Value, nil 35 | } 36 | 37 | // IsExpired returns if the credentials are expired. 38 | // 39 | // For StaticProvider, the credentials never expired. 40 | func (s *StaticProvider) IsExpired() bool { 41 | return false 42 | } 43 | -------------------------------------------------------------------------------- /aws/credentials/static_provider_test.go: -------------------------------------------------------------------------------- 1 | package credentials 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestStaticProviderGet(t *testing.T) { 9 | s := StaticProvider{ 10 | Value: Value{ 11 | AccessKeyID: "AKID", 12 | SecretAccessKey: "SECRET", 13 | SessionToken: "", 14 | }, 15 | } 16 | 17 | creds, err := s.Retrieve() 18 | assert.Nil(t, err, "Expect no error") 19 | assert.Equal(t, "AKID", creds.AccessKeyID, "Expect access key ID to match") 20 | assert.Equal(t, "SECRET", creds.SecretAccessKey, "Expect secret access key to match") 21 | assert.Empty(t, creds.SessionToken, "Expect no session token") 22 | } 23 | 24 | func TestStaticProviderIsExpired(t *testing.T) { 25 | s := StaticProvider{ 26 | Value: Value{ 27 | AccessKeyID: "AKID", 28 | SecretAccessKey: "SECRET", 29 | SessionToken: "", 30 | }, 31 | } 32 | 33 | assert.False(t, s.IsExpired(), "Expect static credentials to never expire") 34 | } 35 | -------------------------------------------------------------------------------- /aws/default_redirect.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | ) 7 | 8 | func defaultHTTPRedirect(client *http.Client) { 9 | client.CheckRedirect = func(req *http.Request, via []*http.Request) error { 10 | if len(via) >= 10 { 11 | return errors.New("stopped after 10 redirects") 12 | } 13 | 14 | // use prev Authorization if request has no Authorization 15 | if req.Header.Get("Authorization") == "" { 16 | prevAuth := via[len(via)-1].Header.Get("Authorization") 17 | req.Header.Set("Authorization", prevAuth) 18 | } 19 | 20 | return nil 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /aws/handler_functions.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/ks3sdklib/aws-sdk-go/aws/retry" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "net/url" 11 | "regexp" 12 | "strconv" 13 | "time" 14 | 15 | "github.com/ks3sdklib/aws-sdk-go/aws/awserr" 16 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 17 | ) 18 | 19 | var sleepDelay = func(delay time.Duration) { 20 | time.Sleep(delay) 21 | } 22 | 23 | // Interface for matching types which also have a Len method. 24 | type lener interface { 25 | Len() int 26 | } 27 | 28 | // BuildContentLength builds the content length of a request based on the body, 29 | // or will use the HTTPRequest.Header's "Content-Length" if defined. If unable 30 | // to determine request body length and no "Content-Length" was specified it will panic. 31 | func BuildContentLength(r *Request) { 32 | if slength := r.HTTPRequest.Header.Get("Content-Length"); slength != "" { 33 | length, _ := strconv.ParseInt(slength, 10, 64) 34 | r.HTTPRequest.ContentLength = length 35 | return 36 | } 37 | 38 | var length int64 39 | switch body := r.Body.(type) { 40 | case nil: 41 | length = 0 42 | case lener: 43 | length = int64(body.Len()) 44 | case io.Seeker: 45 | r.bodyStart, _ = body.Seek(0, 1) 46 | end, _ := body.Seek(0, 2) 47 | body.Seek(r.bodyStart, 0) // make sure to seek back to original location 48 | length = end - r.bodyStart 49 | default: 50 | panic("Cannot get length of body, must provide `ContentLength`") 51 | } 52 | 53 | r.HTTPRequest.ContentLength = length 54 | r.HTTPRequest.Header.Set("Content-Length", fmt.Sprintf("%d", length)) 55 | } 56 | 57 | // UserAgentHandler is a request handler for injecting User agent into requests. 58 | func UserAgentHandler(r *Request) { 59 | r.HTTPRequest.Header.Set("User-Agent", SDKName+"/"+SDKVersion) 60 | } 61 | func ContentTypeHandler(r *Request) { 62 | if len(r.HTTPRequest.Header["Content-Type"]) == 0 { 63 | r.HTTPRequest.Header.Set("Content-Type", "application/xml") 64 | } 65 | } 66 | 67 | var reStatusCode = regexp.MustCompile(`^(\d+)`) 68 | 69 | // SendHandler is a request handler to send service request using HTTP client. 70 | func SendHandler(r *Request) { 71 | 72 | var err error 73 | if r.HTTPRequest.ContentLength <= 0 { 74 | r.HTTPRequest.Body = http.NoBody 75 | } 76 | r.HTTPResponse, err = r.Service.Config.HTTPClient.Do(r.HTTPRequest) 77 | if err != nil { 78 | // Capture the case where url.Error is returned for error processing 79 | // response. e.g. 301 without location header comes back as string 80 | // error and r.HTTPResponse is nil. Other url redirect errors will 81 | // comeback in a similar method. 82 | if e, ok := err.(*url.Error); ok { 83 | if s := reStatusCode.FindStringSubmatch(e.Error()); s != nil { 84 | code, _ := strconv.ParseInt(s[1], 10, 64) 85 | r.HTTPResponse = &http.Response{ 86 | StatusCode: int(code), 87 | Status: http.StatusText(int(code)), 88 | Body: ioutil.NopCloser(bytes.NewReader([]byte{})), 89 | } 90 | return 91 | } 92 | } 93 | // Catch all other request errors. 94 | r.Error = apierr.New("RequestError", "send request failed", err) 95 | r.Retryable.Set(true) // network errors are retryable 96 | } 97 | } 98 | 99 | // ValidateResponseHandler is a request handler to validate service response. 100 | func ValidateResponseHandler(r *Request) { 101 | if r.HTTPResponse.StatusCode == 0 || r.HTTPResponse.StatusCode >= 300 { 102 | // this may be replaced by an UnmarshalError handler 103 | r.Error = apierr.New("UnknownError", "unknown error", nil) 104 | } 105 | } 106 | 107 | // AfterRetryHandler performs final checks to determine if the request should 108 | // be retried and how long to delay. 109 | func AfterRetryHandler(r *Request) { 110 | // If one of the other handlers already set the retry state 111 | // we don't want to override it based on the service's state 112 | if !r.Retryable.IsSet() { 113 | r.Retryable.Set(r.Service.ShouldRetry(r.Error)) 114 | } 115 | 116 | if r.WillRetry() { 117 | r.RetryCount++ 118 | delay := r.Service.RetryRule.GetDelay(int(r.RetryCount)) 119 | if delay < 0 { 120 | delay = 0 121 | } 122 | r.RetryDelay = delay 123 | 124 | r.Config.LogWarn("Tried %d times, will retry in %d ms.", r.RetryCount, r.RetryDelay.Milliseconds()) 125 | sleepDelay(r.RetryDelay) 126 | 127 | // when the expired token exception occurs the credentials 128 | // need to be expired locally so that the next request to 129 | // get credentials will trigger a credentials refresh. 130 | if r.Error != nil { 131 | if err, ok := r.Error.(awserr.Error); ok { 132 | if retry.IsCodeExpiredCreds(err.Code()) { 133 | r.Config.Credentials.Expire() 134 | } 135 | } 136 | } 137 | 138 | r.Error = nil 139 | } 140 | } 141 | 142 | var ( 143 | // ErrMissingRegion is an error that is returned if region configuration is 144 | // not found. 145 | ErrMissingRegion error = apierr.New("MissingRegion", "could not find region configuration", nil) 146 | 147 | // ErrMissingEndpoint is an error that is returned if an endpoint cannot be 148 | // resolved for a service. 149 | ErrMissingEndpoint error = apierr.New("MissingEndpoint", "'Endpoint' configuration is required for this service", nil) 150 | ) 151 | 152 | // ValidateEndpointHandler is a request handler to validate a request had the 153 | // appropriate Region and Endpoint set. Will set r.Error if the endpoint or 154 | // region is not valid. 155 | func ValidateEndpointHandler(r *Request) { 156 | if r.Service.SigningRegion == "" && r.Service.Config.Region == "" { 157 | r.Error = ErrMissingRegion 158 | } else if r.Service.Endpoint == "" { 159 | r.Error = ErrMissingEndpoint 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /aws/handler_functions_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | "testing" 7 | 8 | "github.com/ks3sdklib/aws-sdk-go/aws/credentials" 9 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestValidateEndpointHandler(t *testing.T) { 14 | os.Clearenv() 15 | svc := NewService(&Config{Region: "us-west-2"}) 16 | svc.Handlers.Clear() 17 | svc.Handlers.Validate.PushBack(ValidateEndpointHandler) 18 | 19 | req := NewRequest(svc, &Operation{Name: "Operation"}, nil, nil) 20 | err := req.Build() 21 | 22 | assert.NoError(t, err) 23 | } 24 | 25 | func TestValidateEndpointHandlerErrorRegion(t *testing.T) { 26 | os.Clearenv() 27 | svc := NewService(nil) 28 | svc.Handlers.Clear() 29 | svc.Handlers.Validate.PushBack(ValidateEndpointHandler) 30 | 31 | req := NewRequest(svc, &Operation{Name: "Operation"}, nil, nil) 32 | err := req.Build() 33 | 34 | assert.Error(t, err) 35 | assert.Equal(t, ErrMissingRegion, err) 36 | } 37 | 38 | type mockCredsProvider struct { 39 | expired bool 40 | retreiveCalled bool 41 | } 42 | 43 | func (m *mockCredsProvider) Retrieve() (credentials.Value, error) { 44 | m.retreiveCalled = true 45 | return credentials.Value{}, nil 46 | } 47 | 48 | func (m *mockCredsProvider) IsExpired() bool { 49 | return m.expired 50 | } 51 | 52 | func TestAfterRetryRefreshCreds(t *testing.T) { 53 | os.Clearenv() 54 | credProvider := &mockCredsProvider{} 55 | svc := NewService(&Config{Credentials: credentials.NewCredentials(credProvider), MaxRetries: 1}) 56 | 57 | svc.Handlers.Clear() 58 | svc.Handlers.ValidateResponse.PushBack(func(r *Request) { 59 | r.Error = apierr.New("UnknownError", "", nil) 60 | r.HTTPResponse = &http.Response{StatusCode: 400} 61 | }) 62 | svc.Handlers.UnmarshalError.PushBack(func(r *Request) { 63 | r.Error = apierr.New("ExpiredTokenException", "", nil) 64 | }) 65 | svc.Handlers.AfterRetry.PushBack(func(r *Request) { 66 | AfterRetryHandler(r) 67 | }) 68 | 69 | assert.True(t, svc.Config.Credentials.IsExpired(), "Expect to start out expired") 70 | assert.False(t, credProvider.retreiveCalled) 71 | 72 | req := NewRequest(svc, &Operation{Name: "Operation"}, nil, nil) 73 | req.Send() 74 | 75 | assert.True(t, svc.Config.Credentials.IsExpired()) 76 | assert.False(t, credProvider.retreiveCalled) 77 | 78 | _, err := svc.Config.Credentials.Get() 79 | assert.NoError(t, err) 80 | assert.True(t, credProvider.retreiveCalled) 81 | } 82 | -------------------------------------------------------------------------------- /aws/handlers.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | // A Handlers provides a collection of request handlers for various 4 | // stages of handling requests. 5 | type Handlers struct { 6 | Validate HandlerList 7 | Build HandlerList 8 | Sign HandlerList 9 | Send HandlerList 10 | ValidateResponse HandlerList 11 | Unmarshal HandlerList 12 | UnmarshalMeta HandlerList 13 | UnmarshalError HandlerList 14 | Retry HandlerList 15 | AfterRetry HandlerList 16 | CheckCrc64 HandlerList 17 | } 18 | 19 | // copy returns of this handler's lists. 20 | func (h *Handlers) copy() Handlers { 21 | return Handlers{ 22 | Validate: h.Validate.copy(), 23 | Build: h.Build.copy(), 24 | Sign: h.Sign.copy(), 25 | Send: h.Send.copy(), 26 | ValidateResponse: h.ValidateResponse.copy(), 27 | Unmarshal: h.Unmarshal.copy(), 28 | UnmarshalError: h.UnmarshalError.copy(), 29 | UnmarshalMeta: h.UnmarshalMeta.copy(), 30 | Retry: h.Retry.copy(), 31 | AfterRetry: h.AfterRetry.copy(), 32 | CheckCrc64: h.CheckCrc64.copy(), 33 | } 34 | } 35 | 36 | // Clear removes callback functions for all handlers 37 | func (h *Handlers) Clear() { 38 | h.Validate.Clear() 39 | h.Build.Clear() 40 | h.Send.Clear() 41 | h.Sign.Clear() 42 | h.Unmarshal.Clear() 43 | h.UnmarshalMeta.Clear() 44 | h.UnmarshalError.Clear() 45 | h.ValidateResponse.Clear() 46 | h.Retry.Clear() 47 | h.AfterRetry.Clear() 48 | h.CheckCrc64.Clear() 49 | } 50 | 51 | // A HandlerList manages zero or more handlers in a list. 52 | type HandlerList struct { 53 | list []func(*Request) 54 | } 55 | 56 | // copy creates a copy of the handler list. 57 | func (l *HandlerList) copy() HandlerList { 58 | var n HandlerList 59 | n.list = append([]func(*Request){}, l.list...) 60 | return n 61 | } 62 | 63 | // Clear clears the handler list. 64 | func (l *HandlerList) Clear() { 65 | l.list = []func(*Request){} 66 | } 67 | 68 | // Len returns the number of handlers in the list. 69 | func (l *HandlerList) Len() int { 70 | return len(l.list) 71 | } 72 | 73 | // PushBack pushes handlers f to the back of the handler list. 74 | func (l *HandlerList) PushBack(f ...func(*Request)) { 75 | l.list = append(l.list, f...) 76 | } 77 | 78 | // PushFront pushes handlers f to the front of the handler list. 79 | func (l *HandlerList) PushFront(f ...func(*Request)) { 80 | l.list = append(f, l.list...) 81 | } 82 | 83 | // Run executes all handlers in the list with a given request object. 84 | func (l *HandlerList) Run(r *Request) { 85 | for _, f := range l.list { 86 | f(r) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /aws/handlers_test.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestHandlerList(t *testing.T) { 10 | s := "" 11 | r := &Request{} 12 | l := HandlerList{} 13 | l.PushBack(func(r *Request) { 14 | s += "a" 15 | r.Data = s 16 | }) 17 | l.Run(r) 18 | assert.Equal(t, "a", s) 19 | assert.Equal(t, "a", r.Data) 20 | } 21 | 22 | func TestMultipleHandlers(t *testing.T) { 23 | r := &Request{} 24 | l := HandlerList{} 25 | l.PushBack(func(r *Request) { r.Data = nil }) 26 | l.PushFront(func(r *Request) { r.Data = Boolean(true) }) 27 | l.Run(r) 28 | if r.Data != nil { 29 | t.Error("Expected handler to execute") 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /aws/param_validator.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strings" 7 | 8 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 9 | ) 10 | 11 | // ValidateParameters is a request handler to validate the input parameters. 12 | // Validating parameters only has meaning if done prior to the request being sent. 13 | func ValidateParameters(r *Request) { 14 | if r.ParamsFilled() { 15 | v := validator{errors: []string{}} 16 | v.validateAny(reflect.ValueOf(r.Params), "") 17 | 18 | if count := len(v.errors); count > 0 { 19 | format := "%d validation errors:\n- %s" 20 | msg := fmt.Sprintf(format, count, strings.Join(v.errors, "\n- ")) 21 | r.Error = apierr.New("InvalidParameter", msg, nil) 22 | } 23 | } 24 | } 25 | 26 | // A validator validates values. Collects validations errors which occurs. 27 | type validator struct { 28 | errors []string 29 | } 30 | 31 | // validateAny will validate any struct, slice or map type. All validations 32 | // are also performed recursively for nested types. 33 | func (v *validator) validateAny(value reflect.Value, path string) { 34 | value = reflect.Indirect(value) 35 | if !value.IsValid() { 36 | return 37 | } 38 | 39 | switch value.Kind() { 40 | case reflect.Struct: 41 | v.validateStruct(value, path) 42 | case reflect.Slice: 43 | for i := 0; i < value.Len(); i++ { 44 | v.validateAny(value.Index(i), path+fmt.Sprintf("[%d]", i)) 45 | } 46 | case reflect.Map: 47 | for _, n := range value.MapKeys() { 48 | v.validateAny(value.MapIndex(n), path+fmt.Sprintf("[%q]", n.String())) 49 | } 50 | } 51 | } 52 | 53 | // validateStruct will validate the struct value's fields. If the structure has 54 | // nested types those types will be validated also. 55 | func (v *validator) validateStruct(value reflect.Value, path string) { 56 | prefix := "." 57 | if path == "" { 58 | prefix = "" 59 | } 60 | 61 | for i := 0; i < value.Type().NumField(); i++ { 62 | f := value.Type().Field(i) 63 | if strings.ToLower(f.Name[0:1]) == f.Name[0:1] { 64 | continue 65 | } 66 | fvalue := value.FieldByName(f.Name) 67 | 68 | notset := false 69 | if f.Tag.Get("required") != "" { 70 | switch fvalue.Kind() { 71 | case reflect.Ptr, reflect.Slice, reflect.Map: 72 | if fvalue.IsNil() { 73 | notset = true 74 | } 75 | default: 76 | if !fvalue.IsValid() { 77 | notset = true 78 | } 79 | } 80 | } 81 | 82 | if notset { 83 | msg := "missing required parameter: " + path + prefix + f.Name 84 | v.errors = append(v.errors, msg) 85 | } else { 86 | v.validateAny(fvalue, path+prefix+f.Name) 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /aws/param_validator_test.go: -------------------------------------------------------------------------------- 1 | package aws_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ks3sdklib/aws-sdk-go/aws" 7 | "github.com/ks3sdklib/aws-sdk-go/aws/awserr" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | var service = func() *aws.Service { 12 | s := &aws.Service{ 13 | Config: &aws.Config{}, 14 | ServiceName: "mock-service", 15 | APIVersion: "2015-01-01", 16 | } 17 | return s 18 | }() 19 | 20 | type StructShape struct { 21 | RequiredList []*ConditionalStructShape `required:"true"` 22 | RequiredMap map[string]*ConditionalStructShape `required:"true"` 23 | RequiredBool *bool `required:"true"` 24 | OptionalStruct *ConditionalStructShape 25 | 26 | hiddenParameter *string 27 | 28 | metadataStructureShape 29 | } 30 | 31 | type metadataStructureShape struct { 32 | SDKShapeTraits bool 33 | } 34 | 35 | type ConditionalStructShape struct { 36 | Name *string `required:"true"` 37 | SDKShapeTraits bool 38 | } 39 | 40 | func TestNoErrors(t *testing.T) { 41 | input := &StructShape{ 42 | RequiredList: []*ConditionalStructShape{}, 43 | RequiredMap: map[string]*ConditionalStructShape{ 44 | "key1": {Name: aws.String("Name")}, 45 | "key2": {Name: aws.String("Name")}, 46 | }, 47 | RequiredBool: aws.Boolean(true), 48 | OptionalStruct: &ConditionalStructShape{Name: aws.String("Name")}, 49 | } 50 | 51 | req := aws.NewRequest(service, &aws.Operation{}, input, nil) 52 | aws.ValidateParameters(req) 53 | assert.NoError(t, req.Error) 54 | } 55 | 56 | func TestMissingRequiredParameters(t *testing.T) { 57 | input := &StructShape{} 58 | req := aws.NewRequest(service, &aws.Operation{}, input, nil) 59 | aws.ValidateParameters(req) 60 | 61 | assert.Error(t, req.Error) 62 | assert.Equal(t, "InvalidParameter", req.Error.(awserr.Error).Code()) 63 | assert.Equal(t, "3 validation errors:\n- missing required parameter: RequiredList\n- missing required parameter: RequiredMap\n- missing required parameter: RequiredBool", req.Error.(awserr.Error).Message()) 64 | } 65 | 66 | func TestNestedMissingRequiredParameters(t *testing.T) { 67 | input := &StructShape{ 68 | RequiredList: []*ConditionalStructShape{{}}, 69 | RequiredMap: map[string]*ConditionalStructShape{ 70 | "key1": {Name: aws.String("Name")}, 71 | "key2": {}, 72 | }, 73 | RequiredBool: aws.Boolean(true), 74 | OptionalStruct: &ConditionalStructShape{}, 75 | } 76 | 77 | req := aws.NewRequest(service, &aws.Operation{}, input, nil) 78 | aws.ValidateParameters(req) 79 | 80 | assert.Error(t, req.Error) 81 | assert.Equal(t, "InvalidParameter", req.Error.(awserr.Error).Code()) 82 | assert.Equal(t, "3 validation errors:\n- missing required parameter: RequiredList[0].Name\n- missing required parameter: RequiredMap[\"key2\"].Name\n- missing required parameter: OptionalStruct.Name", req.Error.(awserr.Error).Message()) 83 | 84 | } 85 | -------------------------------------------------------------------------------- /aws/progress.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | type ProgressFunc func(increment, completed, total int64) 11 | 12 | type teeReader struct { 13 | reader io.Reader 14 | writer io.Writer 15 | tracker *readerTracker 16 | } 17 | 18 | type readerTracker struct { 19 | completedBytes int64 20 | totalBytes int64 21 | progressFunc ProgressFunc 22 | } 23 | 24 | // TeeReader returns a Reader that writes to w what it reads from r. 25 | // All reads from r performed through it are matched with 26 | // corresponding writes to w. There is no internal buffering - 27 | // to write must complete before the read completes. 28 | // Any error encountered while writing is reported as a read error. 29 | func TeeReader(reader io.Reader, writer io.Writer, totalBytes int64, progressFunc ProgressFunc) io.ReadCloser { 30 | return &teeReader{ 31 | reader: reader, 32 | writer: writer, 33 | tracker: &readerTracker{ 34 | completedBytes: 0, 35 | totalBytes: totalBytes, 36 | progressFunc: progressFunc, 37 | }, 38 | } 39 | } 40 | 41 | func (t *teeReader) Read(p []byte) (n int, err error) { 42 | n, err = t.reader.Read(p) 43 | 44 | // Read encountered error 45 | if err != nil && err != io.EOF { 46 | return 47 | } 48 | 49 | if n > 0 { 50 | // update completedBytes 51 | t.tracker.completedBytes += int64(n) 52 | 53 | if t.tracker.progressFunc != nil { 54 | // report progress 55 | t.tracker.progressFunc(int64(n), t.tracker.completedBytes, t.tracker.totalBytes) 56 | } 57 | // CRC 58 | if t.writer != nil { 59 | if n, err := t.writer.Write(p[:n]); err != nil { 60 | return n, err 61 | } 62 | } 63 | } 64 | 65 | return 66 | } 67 | 68 | func (t *teeReader) Close() error { 69 | if rc, ok := t.reader.(io.ReadCloser); ok { 70 | return rc.Close() 71 | } 72 | return nil 73 | } 74 | 75 | // GetReaderLen returns the length of the reader 76 | func GetReaderLen(reader io.Reader) int64 { 77 | var contentLength int64 78 | switch v := reader.(type) { 79 | case *bytes.Buffer: 80 | contentLength = int64(v.Len()) 81 | case *bytes.Reader: 82 | contentLength = int64(v.Len()) 83 | case *strings.Reader: 84 | contentLength = int64(v.Len()) 85 | case *os.File: 86 | fileInfo, err := v.Stat() 87 | if err != nil { 88 | contentLength = 0 89 | } else { 90 | contentLength = fileInfo.Size() 91 | } 92 | default: 93 | contentLength = 0 94 | } 95 | return contentLength 96 | } 97 | -------------------------------------------------------------------------------- /aws/retry/exponential_retry.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "math" 5 | "time" 6 | ) 7 | 8 | // ExponentialRetryRule 指数级增长等待时间重试规则 9 | type ExponentialRetryRule struct { 10 | baseDelay time.Duration // 基础等待时间 11 | maxDelay time.Duration // 单次最大等待时间 12 | } 13 | 14 | var DefaultExponentialRetryRule = NewExponentialRetryRule(DefaultBaseDelay, DefaultMaxDelay) 15 | 16 | func NewExponentialRetryRule(baseDelay time.Duration, maxDelay time.Duration) ExponentialRetryRule { 17 | if baseDelay < 0 { 18 | baseDelay = DefaultBaseDelay 19 | } 20 | 21 | if maxDelay < 0 { 22 | maxDelay = DefaultMaxDelay 23 | } 24 | 25 | return ExponentialRetryRule{ 26 | baseDelay: baseDelay, 27 | maxDelay: maxDelay, 28 | } 29 | } 30 | 31 | func (r ExponentialRetryRule) GetDelay(attempts int) time.Duration { 32 | delay := r.baseDelay * time.Duration(math.Pow(2, float64(attempts-1))) 33 | if delay > r.maxDelay { 34 | return r.maxDelay 35 | } 36 | return delay 37 | } 38 | -------------------------------------------------------------------------------- /aws/retry/fixed_retry.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // FixedRetryRule 固定等待时间重试规则 8 | type FixedRetryRule struct { 9 | baseDelay time.Duration // 基础等待时间 10 | } 11 | 12 | var DefaultFixedRetryRule = NewFixedRetryRule(DefaultBaseDelay) 13 | 14 | func NewFixedRetryRule(baseDelay time.Duration) FixedRetryRule { 15 | if baseDelay < 0 { 16 | baseDelay = DefaultBaseDelay 17 | } 18 | 19 | return FixedRetryRule{ 20 | baseDelay: baseDelay, 21 | } 22 | } 23 | 24 | func (r FixedRetryRule) GetDelay(attempts int) time.Duration { 25 | return r.baseDelay 26 | } 27 | -------------------------------------------------------------------------------- /aws/retry/linear_retry.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // LinearRetryRule 线性增长等待时间重试规则 8 | type LinearRetryRule struct { 9 | baseDelay time.Duration // 基础等待时间 10 | maxDelay time.Duration // 单次最大等待时间 11 | } 12 | 13 | var DefaultLinearRetryRule = NewLinearRetryRule(DefaultBaseDelay, DefaultMaxDelay) 14 | 15 | func NewLinearRetryRule(baseDelay time.Duration, maxDelay time.Duration) LinearRetryRule { 16 | if baseDelay < 0 { 17 | baseDelay = DefaultBaseDelay 18 | } 19 | 20 | if maxDelay < 0 { 21 | maxDelay = DefaultMaxDelay 22 | } 23 | 24 | return LinearRetryRule{ 25 | baseDelay: baseDelay, 26 | maxDelay: maxDelay, 27 | } 28 | } 29 | 30 | func (r LinearRetryRule) GetDelay(attempts int) time.Duration { 31 | delay := r.baseDelay * time.Duration(attempts) 32 | if delay > r.maxDelay { 33 | return r.maxDelay 34 | } 35 | return delay 36 | } 37 | -------------------------------------------------------------------------------- /aws/retry/no_delay_retry.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // NoDelayRetryRule 不等待重试规则（立即重试） 8 | type NoDelayRetryRule struct{} 9 | 10 | var DefaultNoDelayRetryRule = NewNoDelayRetryRule() 11 | 12 | func NewNoDelayRetryRule() NoDelayRetryRule { 13 | return NoDelayRetryRule{} 14 | } 15 | 16 | func (r NoDelayRetryRule) GetDelay(attempts int) time.Duration { 17 | return 0 18 | } 19 | -------------------------------------------------------------------------------- /aws/retry/random_retry.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | // RandomRetryRule 随机等待时间重试规则 9 | type RandomRetryRule struct { 10 | minDelay time.Duration // 最小随机等待时间 11 | maxDelay time.Duration // 最大随机等待时间 12 | } 13 | 14 | var DefaultRandomRetryRule = NewRandomRetryRule(DefaultRandomMinDelay, DefaultRandomMaxDelay) 15 | 16 | func NewRandomRetryRule(minDelay time.Duration, maxDelay time.Duration) RandomRetryRule { 17 | if minDelay < 0 { 18 | minDelay = DefaultRandomMinDelay 19 | } 20 | 21 | if maxDelay < 0 { 22 | maxDelay = DefaultRandomMaxDelay 23 | } 24 | 25 | if maxDelay < minDelay { 26 | minDelay = DefaultRandomMinDelay 27 | maxDelay = DefaultRandomMaxDelay 28 | } 29 | 30 | return RandomRetryRule{ 31 | minDelay: minDelay, 32 | maxDelay: maxDelay, 33 | } 34 | } 35 | 36 | func (r RandomRetryRule) GetDelay(attempts int) time.Duration { 37 | delay := r.minDelay + time.Duration(rand.Int63n(int64(r.maxDelay-r.minDelay+1))) 38 | return delay 39 | } 40 | -------------------------------------------------------------------------------- /aws/retry/retry.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import "time" 4 | 5 | const ( 6 | // DefaultMaxDelay 默认最大等待时间 7 | DefaultMaxDelay = 20 * time.Second 8 | // DefaultBaseDelay 默认基础等待时间 9 | DefaultBaseDelay = 200 * time.Millisecond 10 | // DefaultRandomMinDelay 默认随机最小等待时间 11 | DefaultRandomMinDelay = 0 12 | // DefaultRandomMaxDelay 默认随机最大等待时间 13 | DefaultRandomMaxDelay = 200 * time.Millisecond 14 | ) 15 | 16 | // RetryRule 重试等待规则 17 | type RetryRule interface { 18 | GetDelay(attempts int) time.Duration 19 | } 20 | -------------------------------------------------------------------------------- /aws/retry/retryable_error.go: -------------------------------------------------------------------------------- 1 | package retry 2 | 3 | import ( 4 | "errors" 5 | "github.com/ks3sdklib/aws-sdk-go/aws/awserr" 6 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 7 | ) 8 | 9 | // ShouldRetry 判断是否需要重试 10 | // 重试条件： 11 | // 1.状态码为5xx 12 | // 2.状态码在retryErrorCodes中 13 | // 3.错误码在retryableCodes中 14 | func ShouldRetry(err error) bool { 15 | var requestError *apierr.RequestError 16 | if errors.As(err, &requestError) { 17 | if requestError.StatusCode() >= 500 { 18 | return true 19 | } 20 | 21 | for _, code := range retryErrorCodes { 22 | if requestError.StatusCode() == code { 23 | return true 24 | } 25 | } 26 | } 27 | 28 | if err, ok := err.(awserr.Error); ok { 29 | return IsCodeRetryable(err.Code()) 30 | } 31 | 32 | return false 33 | } 34 | 35 | // 重试错误码 36 | var retryErrorCodes = []int{ 37 | 408, // RequestTimeout 38 | 429, // TooManyRequests 39 | } 40 | 41 | // retryableCodes is a collection of service response codes which are retry-able 42 | // without any further action. 43 | var retryableCodes = map[string]struct{}{ 44 | "RequestError": {}, 45 | "ProvisionedThroughputExceededException": {}, 46 | "Throttling": {}, 47 | } 48 | 49 | // credsExpiredCodes is a collection of error codes which signify the credentials 50 | // need to be refreshed. Expired tokens require refreshing of credentials, and 51 | // resigning before the request can be retried. 52 | var credsExpiredCodes = map[string]struct{}{ 53 | "ExpiredToken": {}, 54 | "ExpiredTokenException": {}, 55 | "RequestExpired": {}, 56 | } 57 | 58 | func IsCodeRetryable(code string) bool { 59 | if _, ok := retryableCodes[code]; ok { 60 | return true 61 | } 62 | 63 | return IsCodeExpiredCreds(code) 64 | } 65 | 66 | func IsCodeExpiredCreds(code string) bool { 67 | _, ok := credsExpiredCodes[code] 68 | return ok 69 | } 70 | -------------------------------------------------------------------------------- /aws/service.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "github.com/ks3sdklib/aws-sdk-go/aws/retry" 5 | "github.com/ks3sdklib/aws-sdk-go/internal/endpoints" 6 | "net/http" 7 | "net/http/httputil" 8 | "regexp" 9 | ) 10 | 11 | // A Service implements the base service request and response handling 12 | // used by all services. 13 | type Service struct { 14 | Config *Config 15 | Handlers Handlers 16 | ManualSend bool 17 | ServiceName string 18 | APIVersion string 19 | Endpoint string 20 | SigningName string 21 | SigningRegion string 22 | JSONVersion string 23 | TargetPrefix string 24 | RetryRule retry.RetryRule 25 | ShouldRetry func(error) bool 26 | MaxRetries int 27 | } 28 | 29 | var schemeRE = regexp.MustCompile("^([^:]+)://") 30 | 31 | // NewService will return a pointer to a new Server object initialized. 32 | func NewService(config *Config) *Service { 33 | svc := &Service{Config: config} 34 | svc.Initialize() 35 | return svc 36 | } 37 | 38 | // Initialize initializes the service. 39 | func (s *Service) Initialize() { 40 | if s.Config == nil { 41 | s.Config = &Config{} 42 | } 43 | if s.Config.HTTPClient == nil { 44 | s.Config.HTTPClient = http.DefaultClient 45 | } 46 | 47 | if s.RetryRule == nil { 48 | s.RetryRule = s.Config.RetryRule 49 | } 50 | 51 | if s.ShouldRetry == nil { 52 | s.ShouldRetry = s.Config.ShouldRetry 53 | } 54 | 55 | s.MaxRetries = s.Config.MaxRetries 56 | s.Handlers.Validate.PushBack(ValidateEndpointHandler) 57 | s.Handlers.Build.PushBack(UserAgentHandler) 58 | s.Handlers.Sign.PushBack(BuildContentLength) 59 | s.Handlers.Send.PushBack(SendHandler) 60 | s.Handlers.AfterRetry.PushBack(AfterRetryHandler) 61 | s.Handlers.ValidateResponse.PushBack(ValidateResponseHandler) 62 | s.AddDebugHandlers() 63 | s.buildEndpoint() 64 | 65 | if !s.Config.DisableParamValidation { 66 | s.Handlers.Validate.PushBack(ValidateParameters) 67 | } 68 | } 69 | 70 | // buildEndpoint builds the endpoint values the service will use to make requests with. 71 | func (s *Service) buildEndpoint() { 72 | if s.Config.Endpoint != "" { 73 | s.Endpoint = s.Config.Endpoint 74 | } else { 75 | s.Endpoint, s.SigningRegion = 76 | endpoints.EndpointForRegion(s.ServiceName, s.Config.Region) 77 | } 78 | if s.Endpoint != "" && !schemeRE.MatchString(s.Endpoint) { 79 | scheme := "https" 80 | if s.Config.DisableSSL { 81 | scheme = "http" 82 | } 83 | s.Endpoint = scheme + "://" + s.Endpoint 84 | } 85 | } 86 | 87 | // AddDebugHandlers injects debug logging handlers into the service to log request 88 | // debug information. 89 | func (s *Service) AddDebugHandlers() { 90 | if s.Config.LogLevel < Debug { 91 | return 92 | } 93 | 94 | s.Handlers.Send.PushFront(func(r *Request) { 95 | logBody := r.Config.LogHTTPBody 96 | dumpedBody, _ := httputil.DumpRequestOut(r.HTTPRequest, logBody) 97 | r.Config.LogDebug("---[ REQUEST ]-----------------------------") 98 | r.Config.LogDebug("%s", string(dumpedBody)) 99 | r.Config.LogDebug("-----------------------------------------------------") 100 | }) 101 | s.Handlers.Send.PushBack(func(r *Request) { 102 | r.Config.LogDebug("---[ RESPONSE ]--------------------------------------") 103 | if r.HTTPResponse != nil { 104 | logBody := r.Config.LogHTTPBody 105 | dumpedBody, _ := httputil.DumpResponse(r.HTTPResponse, logBody) 106 | r.Config.LogDebug("%s", string(dumpedBody)) 107 | } else if r.Error != nil { 108 | r.Config.LogDebug("%s", r.Error.Error()) 109 | } 110 | r.Config.LogDebug("-----------------------------------------------------") 111 | }) 112 | } 113 | -------------------------------------------------------------------------------- /aws/types.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "time" 7 | ) 8 | 9 | // String converts a Go string into a string pointer. 10 | func String(v string) *string { 11 | return &v 12 | } 13 | 14 | // ToString converts a string pointer into a string. 15 | func ToString(p *string) (v string) { 16 | if p == nil { 17 | return v 18 | } 19 | 20 | return *p 21 | } 22 | 23 | // Boolean converts a Go bool into a boolean pointer. 24 | func Boolean(v bool) *bool { 25 | return &v 26 | } 27 | 28 | // ToBoolean converts a boolean pointer into a bool. 29 | func ToBoolean(p *bool) (v bool) { 30 | if p == nil { 31 | return v 32 | } 33 | 34 | return *p 35 | } 36 | 37 | // Long converts a Go int64 into a long pointer. 38 | func Long(v int64) *int64 { 39 | return &v 40 | } 41 | 42 | // ToLong converts a long pointer into a int64. 43 | func ToLong(p *int64) (v int64) { 44 | if p == nil { 45 | return v 46 | } 47 | 48 | return *p 49 | } 50 | 51 | // Double converts a Go float64 into a double pointer. 52 | func Double(v float64) *float64 { 53 | return &v 54 | } 55 | 56 | // ToDouble converts a double pointer into a float64. 57 | func ToDouble(p *float64) (v float64) { 58 | if p == nil { 59 | return v 60 | } 61 | 62 | return *p 63 | } 64 | 65 | // Time converts a Go Time into a Time pointer 66 | func Time(t time.Time) *time.Time { 67 | return &t 68 | } 69 | 70 | // ReadSeekCloser wraps a io.Reader returning a ReaderSeakerCloser 71 | func ReadSeekCloser(r io.Reader) ReaderSeekerCloser { 72 | return ReaderSeekerCloser{r} 73 | } 74 | 75 | // ReaderSeekerCloser represents a reader that can also delegate io.Seeker and 76 | // io.Closer interfaces to the underlying object if they are available. 77 | type ReaderSeekerCloser struct { 78 | r io.Reader 79 | } 80 | 81 | // Read reads from the reader up to size of p. The number of bytes read, and 82 | // error if it occurred will be returned. 83 | // 84 | // If the reader is not an io.Reader zero bytes read, and nil error will be returned. 85 | // 86 | // Performs the same functionality as io.Reader Read 87 | func (r ReaderSeekerCloser) Read(p []byte) (int, error) { 88 | switch t := r.r.(type) { 89 | case io.Reader: 90 | return t.Read(p) 91 | } 92 | return 0, nil 93 | } 94 | 95 | // Seek sets the offset for the next Read to offset, interpreted according to 96 | // whence: 0 means relative to the origin of the file, 1 means relative to the 97 | // current offset, and 2 means relative to the end. Seek returns the new offset 98 | // and an error, if any. 99 | // 100 | // If the ReaderSeekerCloser is not an io.Seeker nothing will be done. 101 | func (r ReaderSeekerCloser) Seek(offset int64, whence int) (int64, error) { 102 | switch t := r.r.(type) { 103 | case io.Seeker: 104 | return t.Seek(offset, whence) 105 | } 106 | return int64(0), nil 107 | } 108 | 109 | // Close closes the ReaderSeekerCloser. 110 | // 111 | // If the ReaderSeekerCloser is not an io.Closer nothing will be done. 112 | func (r ReaderSeekerCloser) Close() error { 113 | switch t := r.r.(type) { 114 | case io.Closer: 115 | return t.Close() 116 | } 117 | return nil 118 | } 119 | 120 | // A SettableBool provides a boolean value which includes the state if 121 | // the value was set or unset. The set state is in addition to the value's 122 | // value(true|false) 123 | type SettableBool struct { 124 | value bool 125 | set bool 126 | } 127 | 128 | // SetBool returns a SettableBool with a value set 129 | func SetBool(value bool) SettableBool { 130 | return SettableBool{value: value, set: true} 131 | } 132 | 133 | // Get returns the value. Will always be false if the SettableBool was not set. 134 | func (b *SettableBool) Get() bool { 135 | if !b.set { 136 | return false 137 | } 138 | return b.value 139 | } 140 | 141 | // Set sets the value and updates the state that the value has been set. 142 | func (b *SettableBool) Set(value bool) { 143 | b.value = value 144 | b.set = true 145 | } 146 | 147 | // IsSet returns if the value has been set 148 | func (b *SettableBool) IsSet() bool { 149 | return b.set 150 | } 151 | 152 | // Reset resets the state and value of the SettableBool to its initial default 153 | // state of not set and zero value. 154 | func (b *SettableBool) Reset() { 155 | b.value = false 156 | b.set = false 157 | } 158 | 159 | // String returns the string representation of the value if set. Zero if not set. 160 | func (b *SettableBool) String() string { 161 | return fmt.Sprintf("%t", b.Get()) 162 | } 163 | 164 | // GoString returns the string representation of the SettableBool value and state 165 | func (b *SettableBool) GoString() string { 166 | return fmt.Sprintf("Bool{value:%t, set:%t}", b.value, b.set) 167 | } 168 | -------------------------------------------------------------------------------- /aws/version.go: -------------------------------------------------------------------------------- 1 | package aws 2 | 3 | // SDKName is the name of this KS3 SDK 4 | const SDKName = "ks3-sdk-go" 5 | 6 | // SDKVersion is the version of this SDK 7 | const SDKVersion = "1.6.1" 8 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ks3sdklib/aws-sdk-go 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/stretchr/testify v1.8.1 7 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 5 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 6 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 7 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 8 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 9 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 10 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 11 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 12 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 13 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 14 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 15 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 16 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 17 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 18 | -------------------------------------------------------------------------------- /internal/apierr/error.go: -------------------------------------------------------------------------------- 1 | // Package apierr represents API error types. 2 | package apierr 3 | 4 | import "fmt" 5 | 6 | // A BaseError wraps the code and message which defines an error. It also 7 | // can be used to wrap an original error object. 8 | // 9 | // Should be used as the root for errors satisfying the awserr.Error. Also 10 | // for any error which does not fit into a specific error wrapper type. 11 | type BaseError struct { 12 | // Classification of error 13 | code string 14 | 15 | // Detailed information about error 16 | message string 17 | 18 | // Optional original error this error is based off of. Allows building 19 | // chained errors. 20 | origErr error 21 | } 22 | 23 | // New returns an error object for the code, message, and err. 24 | // 25 | // code is a short no whitespace phrase depicting the classification of 26 | // the error that is being created. 27 | // 28 | // message is the free flow string containing detailed information about the error. 29 | // 30 | // origErr is the error object which will be nested under the new error to be returned. 31 | func New(code, message string, origErr error) *BaseError { 32 | return &BaseError{ 33 | code: code, 34 | message: message, 35 | origErr: origErr, 36 | } 37 | } 38 | 39 | // Error returns the string representation of the error. 40 | // 41 | // See ErrorWithExtra for formatting. 42 | // 43 | // Satisfies the error interface. 44 | func (b *BaseError) Error() string { 45 | return b.ErrorWithExtra("") 46 | } 47 | 48 | // String returns the string representation of the error. 49 | // Alias for Error to satisfy the stringer interface. 50 | func (b *BaseError) String() string { 51 | return b.Error() 52 | } 53 | 54 | // Code returns the short phrase depicting the classification of the error. 55 | func (b *BaseError) Code() string { 56 | return b.code 57 | } 58 | 59 | // Message returns the error details message. 60 | func (b *BaseError) Message() string { 61 | return b.message 62 | } 63 | 64 | // OrigErr returns the original error if one was set. Nil is returned if no error 65 | // was set. 66 | func (b *BaseError) OrigErr() error { 67 | return b.origErr 68 | } 69 | 70 | // ErrorWithExtra is a helper method to add an extra string to the stratified 71 | // error message. The extra message will be added on the next line below the 72 | // error message like the following: 73 | // 74 | // : 75 | // 76 | // 77 | // If there is a original error the error will be included on a new line. 78 | // 79 | // : 80 | // 81 | // caused by: 82 | func (b *BaseError) ErrorWithExtra(extra string) string { 83 | msg := fmt.Sprintf("%s: %s", b.code, b.message) 84 | if extra != "" { 85 | msg = fmt.Sprintf("%s\n\t%s", msg, extra) 86 | } 87 | if b.origErr != nil { 88 | msg = fmt.Sprintf("%s\ncaused by: %s", msg, b.origErr.Error()) 89 | } 90 | return msg 91 | } 92 | 93 | // A RequestError wraps a request or service error. 94 | // 95 | // Composed of BaseError for code, message, and original error. 96 | type RequestError struct { 97 | *BaseError 98 | statusCode int 99 | requestID string 100 | } 101 | 102 | // NewRequestError returns a wrapped error with additional information for request 103 | // status code, and service requestID. 104 | // 105 | // Should be used to wrap all request which involve service requests. Even if 106 | // the request failed without a service response, but had an HTTP status code 107 | // that may be meaningful. 108 | // 109 | // Also wraps original errors via the BaseError. 110 | func NewRequestError(base *BaseError, statusCode int, requestID string) *RequestError { 111 | return &RequestError{ 112 | BaseError: base, 113 | statusCode: statusCode, 114 | requestID: requestID, 115 | } 116 | } 117 | 118 | // Error returns the string representation of the error. 119 | // Satisfies the error interface. 120 | func (r *RequestError) Error() string { 121 | return r.ErrorWithExtra(fmt.Sprintf("status code: %d, request id: [%s]", 122 | r.statusCode, r.requestID)) 123 | } 124 | 125 | // String returns the string representation of the error. 126 | // Alias for Error to satisfy the stringer interface. 127 | func (r *RequestError) String() string { 128 | return r.Error() 129 | } 130 | 131 | // StatusCode returns the wrapped status code for the error 132 | func (r *RequestError) StatusCode() int { 133 | return r.statusCode 134 | } 135 | 136 | // RequestID returns the wrapped requestID 137 | func (r *RequestError) RequestID() string { 138 | return r.requestID 139 | } 140 | -------------------------------------------------------------------------------- /internal/context/background_go1.5.go: -------------------------------------------------------------------------------- 1 | // +build !go1.7 2 | 3 | package context 4 | 5 | import "time" 6 | 7 | // An emptyCtx is a copy of the Go 1.7 context.emptyCtx type. This is copied to 8 | // provide a 1.6 and 1.5 safe version of context that is compatible with Go 9 | // 1.7's Context. 10 | // 11 | // An emptyCtx is never canceled, has no values, and has no deadline. It is not 12 | // struct{}, since vars of this type must have distinct addresses. 13 | type emptyCtx int 14 | 15 | func (*emptyCtx) Deadline() (deadline time.Time, ok bool) { 16 | return 17 | } 18 | 19 | func (*emptyCtx) Done() <-chan struct{} { 20 | return nil 21 | } 22 | 23 | func (*emptyCtx) Err() error { 24 | return nil 25 | } 26 | 27 | func (*emptyCtx) Value(key interface{}) interface{} { 28 | return nil 29 | } 30 | 31 | func (e *emptyCtx) String() string { 32 | switch e { 33 | case BackgroundCtx: 34 | return "aws.BackgroundContext" 35 | } 36 | return "unknown empty Context" 37 | } 38 | 39 | // BackgroundCtx is the common base context. 40 | var BackgroundCtx = new(emptyCtx) 41 | -------------------------------------------------------------------------------- /internal/crc/crc64.go: -------------------------------------------------------------------------------- 1 | package crc 2 | 3 | import ( 4 | "hash" 5 | "hash/crc64" 6 | ) 7 | 8 | // digest represents the partial evaluation of a checksum. 9 | type digest struct { 10 | crc uint64 11 | tab *crc64.Table 12 | } 13 | 14 | // NewCRC creates a new hash.Hash64 computing the CRC64 checksum 15 | // using the polynomial represented by the Table. 16 | func NewCRC(tab *crc64.Table, init uint64) hash.Hash64 { return &digest{init, tab} } 17 | 18 | var CrcTable = func() *crc64.Table { 19 | return crc64.MakeTable(crc64.ECMA) 20 | } 21 | 22 | // Size returns the number of bytes sum will return. 23 | func (d *digest) Size() int { return crc64.Size } 24 | 25 | // BlockSize returns the hash's underlying block size. 26 | // The Write method must be able to accept any amount 27 | // of data, but it may operate more efficiently if all writes 28 | // are a multiple of the block size. 29 | func (d *digest) BlockSize() int { return 1 } 30 | 31 | // Reset resets the hash to its initial state. 32 | func (d *digest) Reset() { d.crc = 0 } 33 | 34 | // Write (via the embedded io.Writer interface) adds more data to the running hash. 35 | // It never returns an error. 36 | func (d *digest) Write(p []byte) (n int, err error) { 37 | d.crc = crc64.Update(d.crc, d.tab, p) 38 | return len(p), nil 39 | } 40 | 41 | // Sum64 returns CRC64 value. 42 | func (d *digest) Sum64() uint64 { return d.crc } 43 | 44 | // Sum returns hash value. 45 | func (d *digest) Sum(in []byte) []byte { 46 | s := d.Sum64() 47 | return append(in, byte(s>>56), byte(s>>48), byte(s>>40), byte(s>>32), byte(s>>24), byte(s>>16), byte(s>>8), byte(s)) 48 | } 49 | 50 | // gf2Dim dimension of GF(2) vectors (length of CRC) 51 | const gf2Dim int = 64 52 | 53 | func gf2MatrixTimes(mat []uint64, vec uint64) uint64 { 54 | var sum uint64 55 | for i := 0; vec != 0; i++ { 56 | if vec&1 != 0 { 57 | sum ^= mat[i] 58 | } 59 | 60 | vec >>= 1 61 | } 62 | return sum 63 | } 64 | 65 | func gf2MatrixSquare(square []uint64, mat []uint64) { 66 | for n := 0; n < gf2Dim; n++ { 67 | square[n] = gf2MatrixTimes(mat, mat[n]) 68 | } 69 | } 70 | 71 | // CRC64Combine combines CRC64 72 | func CRC64Combine(crc1 uint64, crc2 uint64, len2 uint64) uint64 { 73 | var even [gf2Dim]uint64 // Even-power-of-two zeros operator 74 | var odd [gf2Dim]uint64 // Odd-power-of-two zeros operator 75 | 76 | // Degenerate case 77 | if len2 == 0 { 78 | return crc1 79 | } 80 | 81 | // Put operator for one zero bit in odd 82 | odd[0] = crc64.ECMA // CRC64 polynomial 83 | var row uint64 = 1 84 | for n := 1; n < gf2Dim; n++ { 85 | odd[n] = row 86 | row <<= 1 87 | } 88 | 89 | // Put operator for two zero bits in even 90 | gf2MatrixSquare(even[:], odd[:]) 91 | 92 | // Put operator for four zero bits in odd 93 | gf2MatrixSquare(odd[:], even[:]) 94 | 95 | // Apply len2 zeros to crc1, first square will put the operator for one zero byte, eight zero bits, in even 96 | for { 97 | // Apply zeros operator for this bit of len2 98 | gf2MatrixSquare(even[:], odd[:]) 99 | 100 | if len2&1 != 0 { 101 | crc1 = gf2MatrixTimes(even[:], crc1) 102 | } 103 | 104 | len2 >>= 1 105 | 106 | // If no more bits set, then done 107 | if len2 == 0 { 108 | break 109 | } 110 | 111 | // Another iteration of the loop with odd and even swapped 112 | gf2MatrixSquare(odd[:], even[:]) 113 | if len2&1 != 0 { 114 | crc1 = gf2MatrixTimes(odd[:], crc1) 115 | } 116 | len2 >>= 1 117 | 118 | // If no more bits set, then done 119 | if len2 == 0 { 120 | break 121 | } 122 | } 123 | 124 | // Return combined CRC 125 | crc1 ^= crc2 126 | return crc1 127 | } 128 | -------------------------------------------------------------------------------- /internal/endpoints/endpoints.go: -------------------------------------------------------------------------------- 1 | // Package endpoints validates regional endpoints for services. 2 | package endpoints 3 | 4 | //go:generate go run ../model/cli/gen-endpoints/main.go endpoints.json endpoints_map.go 5 | //go:generate gofmt -s -w endpoints_map.go 6 | 7 | import "strings" 8 | 9 | // EndpointForRegion returns an endpoint and its signing region for a service and region. 10 | // if the service and region pair are not found endpoint and signingRegion will be empty. 11 | func EndpointForRegion(svcName, region string) (endpoint, signingRegion string) { 12 | derivedKeys := []string{ 13 | region + "/" + svcName, 14 | region + "/*", 15 | "*/" + svcName, 16 | "*/*", 17 | } 18 | 19 | for _, key := range derivedKeys { 20 | if val, ok := endpointsMap.Endpoints[key]; ok { 21 | ep := val.Endpoint 22 | ep = strings.Replace(ep, "{region}", region, -1) 23 | ep = strings.Replace(ep, "{service}", svcName, -1) 24 | 25 | endpoint = ep 26 | signingRegion = val.SigningRegion 27 | return 28 | } 29 | } 30 | return 31 | } 32 | -------------------------------------------------------------------------------- /internal/endpoints/endpoints_map.go: -------------------------------------------------------------------------------- 1 | package endpoints 2 | 3 | // THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT. 4 | 5 | type endpointStruct struct { 6 | Version int 7 | Endpoints map[string]endpointEntry 8 | } 9 | 10 | type endpointEntry struct { 11 | Endpoint string 12 | SigningRegion string 13 | } 14 | 15 | var endpointsMap = endpointStruct{ 16 | Version: 2, 17 | Endpoints: map[string]endpointEntry{ 18 | "*/*": { 19 | Endpoint: "{service}.{region}.amazonaws.com", 20 | }, 21 | "*/cloudfront": { 22 | Endpoint: "cloudfront.amazonaws.com", 23 | SigningRegion: "us-east-1", 24 | }, 25 | "*/cloudsearchdomain": { 26 | Endpoint: "", 27 | SigningRegion: "us-east-1", 28 | }, 29 | "*/iam": { 30 | Endpoint: "iam.amazonaws.com", 31 | SigningRegion: "us-east-1", 32 | }, 33 | "*/importexport": { 34 | Endpoint: "importexport.amazonaws.com", 35 | SigningRegion: "us-east-1", 36 | }, 37 | "*/route53": { 38 | Endpoint: "route53.amazonaws.com", 39 | SigningRegion: "us-east-1", 40 | }, 41 | "*/sts": { 42 | Endpoint: "sts.amazonaws.com", 43 | SigningRegion: "us-east-1", 44 | }, 45 | "ap-northeast-1/s3": { 46 | Endpoint: "s3-{region}.amazonaws.com", 47 | }, 48 | "ap-southeast-1/s3": { 49 | Endpoint: "s3-{region}.amazonaws.com", 50 | }, 51 | "ap-southeast-2/s3": { 52 | Endpoint: "s3-{region}.amazonaws.com", 53 | }, 54 | "cn-north-1/*": { 55 | Endpoint: "{service}.{region}.amazonaws.com.cn", 56 | }, 57 | "eu-central-1/s3": { 58 | Endpoint: "{service}.{region}.amazonaws.com", 59 | }, 60 | "eu-west-1/s3": { 61 | Endpoint: "s3-{region}.amazonaws.com", 62 | }, 63 | "sa-east-1/s3": { 64 | Endpoint: "s3-{region}.amazonaws.com", 65 | }, 66 | "us-east-1/s3": { 67 | Endpoint: "s3.amazonaws.com", 68 | }, 69 | "us-east-1/sdb": { 70 | Endpoint: "sdb.amazonaws.com", 71 | SigningRegion: "us-east-1", 72 | }, 73 | "us-gov-west-1/iam": { 74 | Endpoint: "iam.us-gov.amazonaws.com", 75 | }, 76 | "us-gov-west-1/s3": { 77 | Endpoint: "s3-{region}.amazonaws.com", 78 | }, 79 | "us-gov-west-1/sts": { 80 | Endpoint: "sts.us-gov-west-1.amazonaws.com", 81 | }, 82 | "us-west-1/s3": { 83 | Endpoint: "s3-{region}.amazonaws.com", 84 | }, 85 | "us-west-2/s3": { 86 | Endpoint: "s3-{region}.amazonaws.com", 87 | }, 88 | }, 89 | } 90 | -------------------------------------------------------------------------------- /internal/endpoints/endpoints_test.go: -------------------------------------------------------------------------------- 1 | package endpoints 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestGlobalEndpoints(t *testing.T) { 10 | region := "mock-region-1" 11 | svcs := []string{"cloudfront", "iam", "importexport", "route53", "sts"} 12 | 13 | for _, name := range svcs { 14 | ep, sr := EndpointForRegion(name, region) 15 | assert.Equal(t, name+".amazonaws.com", ep) 16 | assert.Equal(t, "us-east-1", sr) 17 | } 18 | } 19 | 20 | func TestServicesInCN(t *testing.T) { 21 | region := "cn-north-1" 22 | svcs := []string{"cloudfront", "iam", "importexport", "route53", "sts", "s3"} 23 | 24 | for _, name := range svcs { 25 | ep, _ := EndpointForRegion(name, region) 26 | assert.Equal(t, name+"."+region+".amazonaws.com.cn", ep) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /internal/protocol/body/body.go: -------------------------------------------------------------------------------- 1 | package body 2 | 3 | import ( 4 | "github.com/ks3sdklib/aws-sdk-go/aws" 5 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest" 6 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/restjson" 7 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/restxml" 8 | ) 9 | 10 | // Build builds the REST component of a service request. 11 | func Build(r *aws.Request) { 12 | if r.ContentType == "application/json" { 13 | restjson.Build(r) 14 | } else { 15 | restxml.Build(r) 16 | } 17 | } 18 | 19 | // UnmarshalBody unmarshal a response body for the REST protocol. 20 | func UnmarshalBody(r *aws.Request) { 21 | rest.Unmarshal(r) 22 | if r.ContentType == "application/json" { 23 | restjson.Unmarshal(r) 24 | } else { 25 | restxml.Unmarshal(r) 26 | } 27 | } 28 | 29 | // UnmarshalMeta unmarshal response headers for the REST protocol. 30 | func UnmarshalMeta(r *aws.Request) { 31 | rest.UnmarshalMeta(r) 32 | } 33 | 34 | // UnmarshalError unmarshal a response error for the REST protocol. 35 | func UnmarshalError(r *aws.Request) { 36 | restxml.UnmarshalError(r) 37 | } 38 | -------------------------------------------------------------------------------- /internal/protocol/ec2query/build.go: -------------------------------------------------------------------------------- 1 | // Package ec2query provides serialisation of AWS EC2 requests and responses. 2 | package ec2query 3 | 4 | //go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/input/ec2.json build_test.go 5 | 6 | import ( 7 | "net/url" 8 | 9 | "github.com/ks3sdklib/aws-sdk-go/aws" 10 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 11 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/queryutil" 12 | ) 13 | 14 | // Build builds a request for the EC2 protocol. 15 | func Build(r *aws.Request) { 16 | body := url.Values{ 17 | "Action": {r.Operation.Name}, 18 | "Version": {r.Service.APIVersion}, 19 | } 20 | if err := queryutil.Parse(body, r.Params, true); err != nil { 21 | r.Error = apierr.New("Marshal", "failed encoding EC2 Query request", err) 22 | } 23 | 24 | if r.ExpireTime == 0 { 25 | r.HTTPRequest.Method = "POST" 26 | r.HTTPRequest.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8") 27 | r.SetBufferBody([]byte(body.Encode())) 28 | } else { // This is a pre-signed request 29 | r.HTTPRequest.Method = "GET" 30 | r.HTTPRequest.URL.RawQuery = body.Encode() 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /internal/protocol/ec2query/unmarshal.go: -------------------------------------------------------------------------------- 1 | package ec2query 2 | 3 | //go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/output/ec2.json unmarshal_test.go 4 | 5 | import ( 6 | "encoding/xml" 7 | "io" 8 | 9 | "github.com/ks3sdklib/aws-sdk-go/aws" 10 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 11 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil" 12 | ) 13 | 14 | // Unmarshal unmarshals a response body for the EC2 protocol. 15 | func Unmarshal(r *aws.Request) { 16 | defer r.HTTPResponse.Body.Close() 17 | if r.DataFilled() { 18 | decoder := xml.NewDecoder(r.HTTPResponse.Body) 19 | err := xmlutil.UnmarshalXML(r.Data, decoder, "") 20 | if err != nil { 21 | r.Error = apierr.New("Unmarshal", "failed decoding EC2 Query response", err) 22 | return 23 | } 24 | } 25 | } 26 | 27 | // UnmarshalMeta unmarshals response headers for the EC2 protocol. 28 | func UnmarshalMeta(r *aws.Request) { 29 | // TODO implement unmarshaling of request IDs 30 | } 31 | 32 | type xmlErrorResponse struct { 33 | XMLName xml.Name `xml:"Response"` 34 | Code string `xml:"Errors>Error>Code"` 35 | Message string `xml:"Errors>Error>Message"` 36 | RequestID string `xml:"RequestId"` 37 | } 38 | 39 | // UnmarshalError unmarshals a response error for the EC2 protocol. 40 | func UnmarshalError(r *aws.Request) { 41 | defer r.HTTPResponse.Body.Close() 42 | 43 | resp := &xmlErrorResponse{} 44 | err := xml.NewDecoder(r.HTTPResponse.Body).Decode(resp) 45 | if err != nil && err != io.EOF { 46 | r.Error = apierr.New("Unmarshal", "failed decoding EC2 Query error response", err) 47 | } else { 48 | r.Error = apierr.NewRequestError( 49 | apierr.New(resp.Code, resp.Message, nil), 50 | r.HTTPResponse.StatusCode, 51 | resp.RequestID, 52 | ) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /internal/protocol/json/jsonutil/build.go: -------------------------------------------------------------------------------- 1 | // Package jsonutil provides JSON serialisation of AWS requests and responses. 2 | package jsonutil 3 | 4 | import ( 5 | "bytes" 6 | "encoding/base64" 7 | "fmt" 8 | "reflect" 9 | "sort" 10 | "strconv" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | // BuildJSON builds a JSON string for a given object v. 16 | func BuildJSON(v interface{}) ([]byte, error) { 17 | var buf bytes.Buffer 18 | 19 | err := buildAny(reflect.ValueOf(v), &buf, "") 20 | return buf.Bytes(), err 21 | } 22 | 23 | func buildAny(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { 24 | value = reflect.Indirect(value) 25 | if !value.IsValid() { 26 | return nil 27 | } 28 | 29 | vtype := value.Type() 30 | 31 | t := tag.Get("type") 32 | if t == "" { 33 | switch vtype.Kind() { 34 | case reflect.Struct: 35 | // also it can't be a time object 36 | if _, ok := value.Interface().(time.Time); !ok { 37 | t = "structure" 38 | } 39 | case reflect.Slice: 40 | // also it can't be a byte slice 41 | if _, ok := value.Interface().([]byte); !ok { 42 | t = "list" 43 | } 44 | case reflect.Map: 45 | t = "map" 46 | } 47 | } 48 | 49 | switch t { 50 | case "structure": 51 | if field, ok := vtype.FieldByName("SDKShapeTraits"); ok { 52 | tag = field.Tag 53 | } 54 | return buildStruct(value, buf, tag) 55 | case "list": 56 | return buildList(value, buf, tag) 57 | case "map": 58 | return buildMap(value, buf, tag) 59 | default: 60 | return buildScalar(value, buf, tag) 61 | } 62 | } 63 | 64 | func buildStruct(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { 65 | if !value.IsValid() { 66 | return nil 67 | } 68 | 69 | buf.WriteString("{") 70 | 71 | t, fields := value.Type(), []*reflect.StructField{} 72 | for i := 0; i < t.NumField(); i++ { 73 | field := t.Field(i) 74 | member := value.FieldByName(field.Name) 75 | if (member.Kind() == reflect.Ptr || member.Kind() == reflect.Slice || member.Kind() == reflect.Map) && member.IsNil() { 76 | continue // ignore unset fields 77 | } 78 | if c := field.Name[0:1]; strings.ToLower(c) == c { 79 | continue // ignore unexported fields 80 | } 81 | if field.Tag.Get("location") != "" { 82 | continue // ignore non-body elements 83 | } 84 | 85 | fields = append(fields, &field) 86 | } 87 | 88 | for i, field := range fields { 89 | member := value.FieldByName(field.Name) 90 | 91 | // figure out what this field is called 92 | name := field.Name 93 | if locName := field.Tag.Get("locationName"); locName != "" { 94 | name = locName 95 | } 96 | 97 | buf.WriteString(fmt.Sprintf("%q:", name)) 98 | 99 | err := buildAny(member, buf, field.Tag) 100 | if err != nil { 101 | return err 102 | } 103 | 104 | if i < len(fields)-1 { 105 | buf.WriteString(",") 106 | } 107 | } 108 | 109 | buf.WriteString("}") 110 | 111 | return nil 112 | } 113 | 114 | func buildList(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { 115 | buf.WriteString("[") 116 | 117 | for i := 0; i < value.Len(); i++ { 118 | buildAny(value.Index(i), buf, "") 119 | 120 | if i < value.Len()-1 { 121 | buf.WriteString(",") 122 | } 123 | } 124 | 125 | buf.WriteString("]") 126 | 127 | return nil 128 | } 129 | 130 | func buildMap(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { 131 | buf.WriteString("{") 132 | 133 | keys := make([]string, value.Len()) 134 | for i, n := range value.MapKeys() { 135 | keys[i] = n.String() 136 | } 137 | sort.Strings(keys) 138 | 139 | for i, k := range keys { 140 | buf.WriteString(fmt.Sprintf("%q:", k)) 141 | buildAny(value.MapIndex(reflect.ValueOf(k)), buf, "") 142 | 143 | if i < len(keys)-1 { 144 | buf.WriteString(",") 145 | } 146 | } 147 | 148 | buf.WriteString("}") 149 | 150 | return nil 151 | } 152 | 153 | func buildScalar(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error { 154 | switch converted := value.Interface().(type) { 155 | case string: 156 | writeString(converted, buf) 157 | case []byte: 158 | if !value.IsNil() { 159 | buf.WriteString(fmt.Sprintf("%q", base64.StdEncoding.EncodeToString(converted))) 160 | } 161 | case bool: 162 | buf.WriteString(strconv.FormatBool(converted)) 163 | case int64: 164 | buf.WriteString(strconv.FormatInt(converted, 10)) 165 | case float64: 166 | buf.WriteString(strconv.FormatFloat(converted, 'f', -1, 64)) 167 | case time.Time: 168 | buf.WriteString(strconv.FormatInt(converted.UTC().Unix(), 10)) 169 | default: 170 | return fmt.Errorf("unsupported JSON value %v (%s)", value.Interface(), value.Type()) 171 | } 172 | return nil 173 | } 174 | 175 | func writeString(s string, buf *bytes.Buffer) { 176 | buf.WriteByte('"') 177 | for _, r := range s { 178 | if r == '"' { 179 | buf.WriteString(`\"`) 180 | } else if r == '\\' { 181 | buf.WriteString(`\\`) 182 | } else if r == '\b' { 183 | buf.WriteString(`\b`) 184 | } else if r == '\f' { 185 | buf.WriteString(`\f`) 186 | } else if r == '\r' { 187 | buf.WriteString(`\r`) 188 | } else if r == '\t' { 189 | buf.WriteString(`\t`) 190 | } else if r == '\n' { 191 | buf.WriteString(`\n`) 192 | } else if r < 32 { 193 | fmt.Fprintf(buf, "\\u%0.4x", r) 194 | } else { 195 | buf.WriteRune(r) 196 | } 197 | } 198 | buf.WriteByte('"') 199 | } 200 | -------------------------------------------------------------------------------- /internal/protocol/json/jsonutil/build_test.go: -------------------------------------------------------------------------------- 1 | package jsonutil_test 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | "time" 7 | 8 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/json/jsonutil" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func S(s string) *string { 13 | return &s 14 | } 15 | 16 | func D(s int64) *int64 { 17 | return &s 18 | } 19 | 20 | func F(s float64) *float64 { 21 | return &s 22 | } 23 | 24 | func T(s time.Time) *time.Time { 25 | return &s 26 | } 27 | 28 | type J struct { 29 | S *string 30 | SS []string 31 | D *int64 32 | F *float64 33 | T *time.Time 34 | } 35 | 36 | var jsonTests = []struct { 37 | in interface{} 38 | out string 39 | err string 40 | }{ 41 | { 42 | J{}, 43 | `{}`, 44 | ``, 45 | }, 46 | { 47 | J{ 48 | S: S("str"), 49 | SS: []string{"A", "B", "C"}, 50 | D: D(123), 51 | F: F(4.56), 52 | T: T(time.Unix(987, 0)), 53 | }, 54 | `{"S":"str","SS":["A","B","C"],"D":123,"F":4.56,"T":987}`, 55 | ``, 56 | }, 57 | { 58 | J{ 59 | S: S(`"''"`), 60 | }, 61 | `{"S":"\"''\""}`, 62 | ``, 63 | }, 64 | { 65 | J{ 66 | S: S("\x00føø\u00FF\n\\\"\r\t\b\f"), 67 | }, 68 | `{"S":"\u0000føøÿ\n\\\"\r\t\b\f"}`, 69 | ``, 70 | }, 71 | } 72 | 73 | func TestBuildJSON(t *testing.T) { 74 | for _, test := range jsonTests { 75 | out, err := jsonutil.BuildJSON(test.in) 76 | if test.err != "" { 77 | assert.Error(t, err) 78 | assert.Contains(t, err.Error(), test.err) 79 | } else { 80 | assert.NoError(t, err) 81 | assert.Equal(t, string(out), test.out) 82 | } 83 | } 84 | } 85 | 86 | func BenchmarkBuildJSON(b *testing.B) { 87 | for i := 0; i < b.N; i++ { 88 | for _, test := range jsonTests { 89 | jsonutil.BuildJSON(test.in) 90 | } 91 | } 92 | } 93 | 94 | func BenchmarkStdlibJSON(b *testing.B) { 95 | for i := 0; i < b.N; i++ { 96 | for _, test := range jsonTests { 97 | json.Marshal(test.in) 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /internal/protocol/json/jsonutil/unmarshal.go: -------------------------------------------------------------------------------- 1 | package jsonutil 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "reflect" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | // UnmarshalJSON reads a stream and unmarshals the results in object v. 15 | func UnmarshalJSON(v interface{}, stream io.Reader) error { 16 | var out interface{} 17 | 18 | b, err := ioutil.ReadAll(stream) 19 | if err != nil { 20 | return err 21 | } 22 | 23 | if len(b) == 0 { 24 | return nil 25 | } 26 | 27 | if err := json.Unmarshal(b, &out); err != nil { 28 | return err 29 | } 30 | 31 | return unmarshalAny(reflect.ValueOf(v), out, "") 32 | } 33 | 34 | func unmarshalAny(value reflect.Value, data interface{}, tag reflect.StructTag) error { 35 | vtype := value.Type() 36 | if vtype.Kind() == reflect.Ptr { 37 | vtype = vtype.Elem() // check kind of actual element type 38 | } 39 | 40 | t := tag.Get("type") 41 | if t == "" { 42 | switch vtype.Kind() { 43 | case reflect.Struct: 44 | // also it can't be a time object 45 | if _, ok := value.Interface().(*time.Time); !ok { 46 | t = "structure" 47 | } 48 | case reflect.Slice: 49 | // also it can't be a byte slice 50 | if _, ok := value.Interface().([]byte); !ok { 51 | t = "list" 52 | } 53 | case reflect.Map: 54 | t = "map" 55 | } 56 | } 57 | 58 | switch t { 59 | case "structure": 60 | if field, ok := vtype.FieldByName("SDKShapeTraits"); ok { 61 | tag = field.Tag 62 | } 63 | return unmarshalStruct(value, data, tag) 64 | case "list": 65 | return unmarshalList(value, data, tag) 66 | case "map": 67 | return unmarshalMap(value, data, tag) 68 | default: 69 | return unmarshalScalar(value, data, tag) 70 | } 71 | } 72 | 73 | func unmarshalStruct(value reflect.Value, data interface{}, tag reflect.StructTag) error { 74 | if data == nil { 75 | return nil 76 | } 77 | mapData, ok := data.(map[string]interface{}) 78 | if !ok { 79 | return fmt.Errorf("JSON value is not a structure (%#v)", data) 80 | } 81 | 82 | t := value.Type() 83 | if value.Kind() == reflect.Ptr { 84 | if value.IsNil() { // create the structure if it's nil 85 | s := reflect.New(value.Type().Elem()) 86 | value.Set(s) 87 | value = s 88 | } 89 | 90 | value = value.Elem() 91 | t = t.Elem() 92 | } 93 | 94 | // unwrap any payloads 95 | if payload := tag.Get("payload"); payload != "" { 96 | field, _ := t.FieldByName(payload) 97 | return unmarshalAny(value.FieldByName(payload), data, field.Tag) 98 | } 99 | 100 | for i := 0; i < t.NumField(); i++ { 101 | field := t.Field(i) 102 | if c := field.Name[0:1]; strings.ToLower(c) == c { 103 | continue // ignore unexported fields 104 | } 105 | 106 | // figure out what this field is called 107 | name := field.Name 108 | if locName := field.Tag.Get("locationName"); locName != "" { 109 | name = locName 110 | } 111 | 112 | member := value.FieldByName(field.Name) 113 | err := unmarshalAny(member, mapData[name], field.Tag) 114 | if err != nil { 115 | return err 116 | } 117 | } 118 | return nil 119 | } 120 | 121 | func unmarshalList(value reflect.Value, data interface{}, tag reflect.StructTag) error { 122 | if data == nil { 123 | return nil 124 | } 125 | listData, ok := data.([]interface{}) 126 | if !ok { 127 | return fmt.Errorf("JSON value is not a list (%#v)", data) 128 | } 129 | 130 | if value.IsNil() { 131 | l := len(listData) 132 | value.Set(reflect.MakeSlice(value.Type(), l, l)) 133 | } 134 | 135 | for i, c := range listData { 136 | err := unmarshalAny(value.Index(i), c, "") 137 | if err != nil { 138 | return err 139 | } 140 | } 141 | 142 | return nil 143 | } 144 | 145 | func unmarshalMap(value reflect.Value, data interface{}, tag reflect.StructTag) error { 146 | if data == nil { 147 | return nil 148 | } 149 | mapData, ok := data.(map[string]interface{}) 150 | if !ok { 151 | return fmt.Errorf("JSON value is not a map (%#v)", data) 152 | } 153 | 154 | if value.IsNil() { 155 | value.Set(reflect.MakeMap(value.Type())) 156 | } 157 | 158 | for k, v := range mapData { 159 | kvalue := reflect.ValueOf(k) 160 | vvalue := reflect.New(value.Type().Elem()).Elem() 161 | 162 | unmarshalAny(vvalue, v, "") 163 | value.SetMapIndex(kvalue, vvalue) 164 | } 165 | 166 | return nil 167 | } 168 | 169 | func unmarshalScalar(value reflect.Value, data interface{}, tag reflect.StructTag) error { 170 | errf := func() error { 171 | return fmt.Errorf("unsupported value: %v (%s)", value.Interface(), value.Type()) 172 | } 173 | 174 | switch d := data.(type) { 175 | case nil: 176 | return nil // nothing to do here 177 | case string: 178 | switch value.Interface().(type) { 179 | case *string: 180 | value.Set(reflect.ValueOf(&d)) 181 | case []byte: 182 | b, err := base64.StdEncoding.DecodeString(d) 183 | if err != nil { 184 | return err 185 | } 186 | value.Set(reflect.ValueOf(b)) 187 | default: 188 | return errf() 189 | } 190 | case float64: 191 | switch value.Interface().(type) { 192 | case *int64: 193 | di := int64(d) 194 | value.Set(reflect.ValueOf(&di)) 195 | case *float64: 196 | value.Set(reflect.ValueOf(&d)) 197 | case *time.Time: 198 | t := time.Unix(int64(d), 0).UTC() 199 | value.Set(reflect.ValueOf(&t)) 200 | default: 201 | return errf() 202 | } 203 | case bool: 204 | switch value.Interface().(type) { 205 | case *bool: 206 | value.Set(reflect.ValueOf(&d)) 207 | default: 208 | return errf() 209 | } 210 | default: 211 | return fmt.Errorf("unsupported JSON value (%v)", data) 212 | } 213 | return nil 214 | } 215 | -------------------------------------------------------------------------------- /internal/protocol/jsonrpc/jsonrpc.go: -------------------------------------------------------------------------------- 1 | // Package jsonrpc provides JSON RPC utilities for serialisation of AWS 2 | // requests and responses. 3 | package jsonrpc 4 | 5 | //go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/input/json.json build_test.go 6 | //go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/output/json.json unmarshal_test.go 7 | 8 | import ( 9 | "encoding/json" 10 | "io/ioutil" 11 | "reflect" 12 | "strings" 13 | 14 | "github.com/ks3sdklib/aws-sdk-go/aws" 15 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 16 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/json/jsonutil" 17 | ) 18 | 19 | var emptyJSON = []byte("{}") 20 | 21 | // Build builds a JSON payload for a JSON RPC request. 22 | func Build(req *aws.Request) { 23 | var buf []byte 24 | var err error 25 | if req.ParamsFilled() { 26 | v := reflect.ValueOf(req.Params).Elem() 27 | if field, ok := v.Type().FieldByName("SDKShapeTraits"); ok { 28 | if payloadName := field.Tag.Get("payload"); payloadName != "" { 29 | pfield, _ := v.Type().FieldByName(payloadName) 30 | if ptag := pfield.Tag.Get("type"); ptag == "" || ptag == "structure" { 31 | payload := reflect.Indirect(v.FieldByName(payloadName)) 32 | if payload.IsValid() && payload.Interface() != nil { 33 | buf, err = jsonutil.BuildJSON(payload.Interface()) 34 | } 35 | } 36 | } 37 | } 38 | if err != nil { 39 | req.Error = apierr.New("Marshal", "failed encoding JSON RPC request", err) 40 | return 41 | } 42 | } else { 43 | buf = emptyJSON 44 | } 45 | 46 | if req.Service.TargetPrefix != "" || string(buf) != "{}" { 47 | req.SetBufferBody(buf) 48 | } 49 | 50 | if req.Service.TargetPrefix != "" { 51 | target := req.Service.TargetPrefix + "." + req.Operation.Name 52 | req.HTTPRequest.Header.Add("X-Amz-Target", target) 53 | } 54 | if req.Service.JSONVersion != "" { 55 | jsonVersion := req.Service.JSONVersion 56 | req.HTTPRequest.Header.Add("Content-Type", "application/x-amz-json-"+jsonVersion) 57 | } 58 | } 59 | 60 | // Unmarshal unmarshals a response for a JSON RPC service. 61 | func Unmarshal(req *aws.Request) { 62 | defer req.HTTPResponse.Body.Close() 63 | if req.DataFilled() { 64 | v := reflect.ValueOf(req.Data).Elem() 65 | if field, ok := v.Type().FieldByName("SDKShapeTraits"); ok { 66 | if payloadName := field.Tag.Get("payload"); payloadName != "" { 67 | pfield, _ := v.Type().FieldByName(payloadName) 68 | if ptag := pfield.Tag.Get("type"); ptag == "" || ptag == "structure" { 69 | payload := v.FieldByName(payloadName) 70 | if payload.IsValid() && payload.Interface() != nil { 71 | err := jsonutil.UnmarshalJSON(payload.Interface(), req.HTTPResponse.Body) 72 | if err != nil { 73 | req.Error = apierr.New("Unmarshal", "failed decoding JSON RPC response", err) 74 | } 75 | } 76 | } 77 | } 78 | } 79 | } 80 | return 81 | } 82 | 83 | // UnmarshalMeta unmarshals headers from a response for a JSON RPC service. 84 | func UnmarshalMeta(req *aws.Request) { 85 | req.RequestID = req.HTTPResponse.Header.Get("x-amzn-requestid") 86 | } 87 | 88 | // UnmarshalError unmarshals an error response for a JSON RPC service. 89 | func UnmarshalError(req *aws.Request) { 90 | defer req.HTTPResponse.Body.Close() 91 | bodyBytes, err := ioutil.ReadAll(req.HTTPResponse.Body) 92 | if err != nil { 93 | req.Error = apierr.New("Unmarshal", "failed reading JSON RPC error response", err) 94 | return 95 | } 96 | if len(bodyBytes) == 0 { 97 | req.Error = apierr.NewRequestError( 98 | apierr.New("Unmarshal", req.HTTPResponse.Status, nil), 99 | req.HTTPResponse.StatusCode, 100 | "", 101 | ) 102 | return 103 | } 104 | var jsonErr jsonErrorResponse 105 | if err := json.Unmarshal(bodyBytes, &jsonErr); err != nil { 106 | req.Error = apierr.New("Unmarshal", "failed decoding JSON RPC error response", err) 107 | return 108 | } 109 | 110 | codes := strings.SplitN(jsonErr.Code, "#", 2) 111 | req.Error = apierr.NewRequestError( 112 | apierr.New(codes[len(codes)-1], jsonErr.Message, nil), 113 | req.HTTPResponse.StatusCode, 114 | "", 115 | ) 116 | } 117 | 118 | type jsonErrorResponse struct { 119 | Code string `json:"__type"` 120 | Message string `json:"message"` 121 | } 122 | -------------------------------------------------------------------------------- /internal/protocol/query/build.go: -------------------------------------------------------------------------------- 1 | // Package query provides serialisation of AWS query requests, and responses. 2 | package query 3 | 4 | //go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/input/query.json build_test.go 5 | 6 | import ( 7 | "net/url" 8 | 9 | "github.com/ks3sdklib/aws-sdk-go/aws" 10 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 11 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/query/queryutil" 12 | ) 13 | 14 | // Build builds a request for an AWS Query service. 15 | func Build(r *aws.Request) { 16 | body := url.Values{ 17 | "Action": {r.Operation.Name}, 18 | "Version": {r.Service.APIVersion}, 19 | } 20 | if err := queryutil.Parse(body, r.Params, false); err != nil { 21 | r.Error = apierr.New("Marshal", "failed encoding Query request", err) 22 | return 23 | } 24 | 25 | if r.ExpireTime == 0 { 26 | r.HTTPRequest.Method = "POST" 27 | r.HTTPRequest.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8") 28 | r.SetBufferBody([]byte(body.Encode())) 29 | } else { // This is a pre-signed request 30 | r.HTTPRequest.Method = "GET" 31 | r.HTTPRequest.URL.RawQuery = body.Encode() 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /internal/protocol/query/queryutil/queryutil.go: -------------------------------------------------------------------------------- 1 | package queryutil 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "net/url" 7 | "reflect" 8 | "sort" 9 | "strconv" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | // Parse parses an object i and fills a url.Values object. The isEC2 flag 15 | // indicates if this is the EC2 Query sub-protocol. 16 | func Parse(body url.Values, i interface{}, isEC2 bool) error { 17 | q := queryParser{isEC2: isEC2} 18 | return q.parseValue(body, reflect.ValueOf(i), "", "") 19 | } 20 | 21 | func elemOf(value reflect.Value) reflect.Value { 22 | for value.Kind() == reflect.Ptr { 23 | value = value.Elem() 24 | } 25 | return value 26 | } 27 | 28 | type queryParser struct { 29 | isEC2 bool 30 | } 31 | 32 | func (q *queryParser) parseValue(v url.Values, value reflect.Value, prefix string, tag reflect.StructTag) error { 33 | value = elemOf(value) 34 | 35 | // no need to handle zero values 36 | if !value.IsValid() { 37 | return nil 38 | } 39 | 40 | t := tag.Get("type") 41 | if t == "" { 42 | switch value.Kind() { 43 | case reflect.Struct: 44 | t = "structure" 45 | case reflect.Slice: 46 | t = "list" 47 | case reflect.Map: 48 | t = "map" 49 | } 50 | } 51 | 52 | switch t { 53 | case "structure": 54 | return q.parseStruct(v, value, prefix) 55 | case "list": 56 | return q.parseList(v, value, prefix, tag) 57 | case "map": 58 | return q.parseMap(v, value, prefix, tag) 59 | default: 60 | return q.parseScalar(v, value, prefix, tag) 61 | } 62 | } 63 | 64 | func (q *queryParser) parseStruct(v url.Values, value reflect.Value, prefix string) error { 65 | if !value.IsValid() { 66 | return nil 67 | } 68 | 69 | t := value.Type() 70 | for i := 0; i < value.NumField(); i++ { 71 | if c := t.Field(i).Name[0:1]; strings.ToLower(c) == c { 72 | continue // ignore unexported fields 73 | } 74 | 75 | value := elemOf(value.Field(i)) 76 | field := t.Field(i) 77 | var name string 78 | 79 | if q.isEC2 { 80 | name = field.Tag.Get("queryName") 81 | } 82 | if name == "" { 83 | if field.Tag.Get("flattened") != "" && field.Tag.Get("locationNameList") != "" { 84 | name = field.Tag.Get("locationNameList") 85 | } else if locName := field.Tag.Get("locationName"); locName != "" { 86 | name = locName 87 | } 88 | if name != "" && q.isEC2 { 89 | name = strings.ToUpper(name[0:1]) + name[1:] 90 | } 91 | } 92 | if name == "" { 93 | name = field.Name 94 | } 95 | 96 | if prefix != "" { 97 | name = prefix + "." + name 98 | } 99 | 100 | if err := q.parseValue(v, value, name, field.Tag); err != nil { 101 | return err 102 | } 103 | } 104 | return nil 105 | } 106 | 107 | func (q *queryParser) parseList(v url.Values, value reflect.Value, prefix string, tag reflect.StructTag) error { 108 | // If it's empty, generate an empty value 109 | if !value.IsNil() && value.Len() == 0 { 110 | v.Set(prefix, "") 111 | return nil 112 | } 113 | 114 | // check for unflattened list member 115 | if !q.isEC2 && tag.Get("flattened") == "" { 116 | prefix += ".member" 117 | } 118 | 119 | for i := 0; i < value.Len(); i++ { 120 | slicePrefix := prefix 121 | if slicePrefix == "" { 122 | slicePrefix = strconv.Itoa(i + 1) 123 | } else { 124 | slicePrefix = slicePrefix + "." + strconv.Itoa(i+1) 125 | } 126 | if err := q.parseValue(v, value.Index(i), slicePrefix, ""); err != nil { 127 | return err 128 | } 129 | } 130 | return nil 131 | } 132 | 133 | func (q *queryParser) parseMap(v url.Values, value reflect.Value, prefix string, tag reflect.StructTag) error { 134 | // If it's empty, generate an empty value 135 | if !value.IsNil() && value.Len() == 0 { 136 | v.Set(prefix, "") 137 | return nil 138 | } 139 | 140 | // check for unflattened list member 141 | if !q.isEC2 && tag.Get("flattened") == "" { 142 | prefix += ".entry" 143 | } 144 | 145 | // sort keys for improved serialization consistency. 146 | // this is not strictly necessary for protocol support. 147 | mapKeyValues := value.MapKeys() 148 | mapKeys := map[string]reflect.Value{} 149 | mapKeyNames := make([]string, len(mapKeyValues)) 150 | for i, mapKey := range mapKeyValues { 151 | name := mapKey.String() 152 | mapKeys[name] = mapKey 153 | mapKeyNames[i] = name 154 | } 155 | sort.Strings(mapKeyNames) 156 | 157 | for i, mapKeyName := range mapKeyNames { 158 | mapKey := mapKeys[mapKeyName] 159 | mapValue := value.MapIndex(mapKey) 160 | 161 | kname := tag.Get("locationNameKey") 162 | if kname == "" { 163 | kname = "key" 164 | } 165 | vname := tag.Get("locationNameValue") 166 | if vname == "" { 167 | vname = "value" 168 | } 169 | 170 | // serialize key 171 | var keyName string 172 | if prefix == "" { 173 | keyName = strconv.Itoa(i+1) + "." + kname 174 | } else { 175 | keyName = prefix + "." + strconv.Itoa(i+1) + "." + kname 176 | } 177 | 178 | if err := q.parseValue(v, mapKey, keyName, ""); err != nil { 179 | return err 180 | } 181 | 182 | // serialize value 183 | var valueName string 184 | if prefix == "" { 185 | valueName = strconv.Itoa(i+1) + "." + vname 186 | } else { 187 | valueName = prefix + "." + strconv.Itoa(i+1) + "." + vname 188 | } 189 | 190 | if err := q.parseValue(v, mapValue, valueName, ""); err != nil { 191 | return err 192 | } 193 | } 194 | 195 | return nil 196 | } 197 | 198 | func (q *queryParser) parseScalar(v url.Values, r reflect.Value, name string, tag reflect.StructTag) error { 199 | switch value := r.Interface().(type) { 200 | case string: 201 | v.Set(name, value) 202 | case []byte: 203 | if !r.IsNil() { 204 | v.Set(name, base64.StdEncoding.EncodeToString(value)) 205 | } 206 | case bool: 207 | v.Set(name, strconv.FormatBool(value)) 208 | case int64: 209 | v.Set(name, strconv.FormatInt(value, 10)) 210 | case int: 211 | v.Set(name, strconv.Itoa(value)) 212 | case float64: 213 | v.Set(name, strconv.FormatFloat(value, 'f', -1, 64)) 214 | case float32: 215 | v.Set(name, strconv.FormatFloat(float64(value), 'f', -1, 32)) 216 | case time.Time: 217 | const ISO8601UTC = "2006-01-02T15:04:05Z" 218 | v.Set(name, value.UTC().Format(ISO8601UTC)) 219 | default: 220 | return fmt.Errorf("unsupported value for param %s: %v (%s)", name, r.Interface(), r.Type().Name()) 221 | } 222 | return nil 223 | } 224 | -------------------------------------------------------------------------------- /internal/protocol/query/unmarshal.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | //go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/output/query.json unmarshal_test.go 4 | 5 | import ( 6 | "encoding/xml" 7 | 8 | "github.com/ks3sdklib/aws-sdk-go/aws" 9 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 10 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil" 11 | ) 12 | 13 | // Unmarshal unmarshals a response for an AWS Query service. 14 | func Unmarshal(r *aws.Request) { 15 | defer r.HTTPResponse.Body.Close() 16 | if r.DataFilled() { 17 | decoder := xml.NewDecoder(r.HTTPResponse.Body) 18 | err := xmlutil.UnmarshalXML(r.Data, decoder, r.Operation.Name+"Result") 19 | if err != nil { 20 | r.Error = apierr.New("Unmarshal", "failed decoding Query response", err) 21 | return 22 | } 23 | } 24 | } 25 | 26 | // UnmarshalMeta unmarshals header response values for an AWS Query service. 27 | func UnmarshalMeta(r *aws.Request) { 28 | // TODO implement unmarshaling of request IDs 29 | } 30 | -------------------------------------------------------------------------------- /internal/protocol/query/unmarshal_error.go: -------------------------------------------------------------------------------- 1 | package query 2 | 3 | import ( 4 | "encoding/xml" 5 | "github.com/ks3sdklib/aws-sdk-go/aws" 6 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 7 | "io" 8 | "regexp" 9 | "strings" 10 | ) 11 | 12 | type XmlErrorResponse struct { 13 | XMLName xml.Name `xml:"Error"` 14 | Code string `xml:"Code"` 15 | StatusCode int `xml:"StatusCode"` 16 | Message string `xml:"Message"` 17 | Resource string `xml:"Resource"` 18 | RequestID string `xml:"RequestId"` 19 | } 20 | 21 | // UnmarshalError unmarshal an error response for an AWS Query service. 22 | func UnmarshalError(r *aws.Request) { 23 | defer r.HTTPResponse.Body.Close() 24 | 25 | resp := &XmlErrorResponse{} 26 | body, err := io.ReadAll(r.HTTPResponse.Body) 27 | if err != nil { 28 | r.Error = apierr.New("Unmarshal", "failed to read body", err) 29 | return 30 | } 31 | 32 | // 如果响应类型是html，则解析html文本 33 | if strings.Contains(r.HTTPResponse.Header.Get("Content-Type"), "text/html") { 34 | // 获取HTML文本中title标签的内容 35 | re := regexp.MustCompile(`(.*?)`) 36 | matches := re.FindStringSubmatch(string(body)) 37 | 38 | title := "" 39 | if len(matches) > 1 { 40 | title = matches[1] 41 | } 42 | 43 | r.Error = apierr.NewRequestError(apierr.New(title, "", nil), r.HTTPResponse.StatusCode, "") 44 | return 45 | } 46 | 47 | err = xml.Unmarshal(body, &resp) 48 | resp.StatusCode = r.HTTPResponse.StatusCode 49 | 50 | // head请求无法从body中获取request id，如果是head请求，则从header中获取 51 | if resp.RequestID == "" && r.HTTPRequest.Method == "HEAD" { 52 | resp.RequestID = r.HTTPResponse.Header.Get("X-Kss-Request-Id") 53 | } 54 | 55 | if err != nil && err != io.EOF { 56 | r.Error = apierr.New("Unmarshal", "failed to decode query XML error response", err) 57 | } else { 58 | r.Error = apierr.NewRequestError( 59 | apierr.New(resp.Code, resp.Message, nil), 60 | r.HTTPResponse.StatusCode, 61 | resp.RequestID, 62 | ) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /internal/protocol/rest/payload.go: -------------------------------------------------------------------------------- 1 | package rest 2 | 3 | import ( 4 | "reflect" 5 | ) 6 | 7 | // PayloadMember returns the payload field member of i if there is one, or nil. 8 | func PayloadMember(i interface{}) interface{} { 9 | if i == nil { 10 | return nil 11 | } 12 | 13 | v := reflect.ValueOf(i).Elem() 14 | if !v.IsValid() { 15 | return nil 16 | } 17 | if field, ok := v.Type().FieldByName("SDKShapeTraits"); ok { 18 | if payloadName := field.Tag.Get("payload"); payloadName != "" { 19 | field, _ := v.Type().FieldByName(payloadName) 20 | if field.Tag.Get("type") != "structure" { 21 | return nil 22 | } 23 | 24 | payload := v.FieldByName(payloadName) 25 | if payload.IsValid() || (payload.Kind() == reflect.Ptr && !payload.IsNil()) { 26 | return payload.Interface() 27 | } 28 | } 29 | } 30 | return nil 31 | } 32 | 33 | // PayloadType returns the type of a payload field member of i if there is one, or "". 34 | func PayloadType(i interface{}) string { 35 | v := reflect.Indirect(reflect.ValueOf(i)) 36 | if !v.IsValid() { 37 | return "" 38 | } 39 | if field, ok := v.Type().FieldByName("SDKShapeTraits"); ok { 40 | if payloadName := field.Tag.Get("payload"); payloadName != "" { 41 | if payloadName == "GetBucketCORSInput" { 42 | return field.Tag.Get("type") 43 | } 44 | if member, ok := v.Type().FieldByName(payloadName); ok { 45 | return member.Tag.Get("type") 46 | } 47 | } 48 | } 49 | return "" 50 | } 51 | 52 | // PayloadMd5 判断给定结构体 i 中是否有 AutoFillMD5 字段 53 | func PayloadMd5(i interface{}) (hasField bool) { 54 | // 获取结构体指针的 Value 55 | v := reflect.Indirect(reflect.ValueOf(i)) 56 | // 如果结构体不存在或为空，则直接返回 false 57 | if !v.IsValid() { 58 | return 59 | } 60 | // 判断是否存在 AutoFillMD5 字段 61 | _, hasField = v.Type().FieldByName("AutoFillMD5") 62 | return 63 | } 64 | -------------------------------------------------------------------------------- /internal/protocol/rest/unmarshal.go: -------------------------------------------------------------------------------- 1 | package rest 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "reflect" 9 | "strconv" 10 | "strings" 11 | "time" 12 | 13 | "github.com/ks3sdklib/aws-sdk-go/aws" 14 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 15 | ) 16 | 17 | // Unmarshal unmarshals the REST component of a response in a REST service. 18 | func Unmarshal(r *aws.Request) { 19 | if r.DataFilled() { 20 | v := reflect.Indirect(reflect.ValueOf(r.Data)) 21 | unmarshalBody(r, v) 22 | } 23 | } 24 | 25 | func UnmarshalMeta(r *aws.Request) { 26 | if r.DataFilled() { 27 | v := reflect.Indirect(reflect.ValueOf(r.Data)) 28 | unmarshalLocationElements(r, v) 29 | } 30 | } 31 | 32 | func unmarshalBody(r *aws.Request, v reflect.Value) { 33 | if field, ok := v.Type().FieldByName("SDKShapeTraits"); ok { 34 | if payloadName := field.Tag.Get("payload"); payloadName != "" { 35 | pfield, _ := v.Type().FieldByName(payloadName) 36 | if ptag := pfield.Tag.Get("type"); ptag != "" && ptag != "structure" { 37 | payload := v.FieldByName(payloadName) 38 | if payload.IsValid() { 39 | switch payload.Interface().(type) { 40 | case []byte: 41 | b, err := ioutil.ReadAll(r.HTTPResponse.Body) 42 | if err != nil { 43 | r.Error = apierr.New("Unmarshal", "failed to decode REST response", err) 44 | } else { 45 | payload.Set(reflect.ValueOf(b)) 46 | } 47 | case *string: 48 | b, err := ioutil.ReadAll(r.HTTPResponse.Body) 49 | if err != nil { 50 | r.Error = apierr.New("Unmarshal", "failed to decode REST response", err) 51 | } else { 52 | str := string(b) 53 | payload.Set(reflect.ValueOf(&str)) 54 | } 55 | default: 56 | switch payload.Type().String() { 57 | case "io.ReadSeeker": 58 | payload.Set(reflect.ValueOf(aws.ReadSeekCloser(r.HTTPResponse.Body))) 59 | case "aws.ReadSeekCloser", "io.ReadCloser": 60 | payload.Set(reflect.ValueOf(r.HTTPResponse.Body)) 61 | default: 62 | r.Error = apierr.New("Unmarshal", 63 | "failed to decode REST response", 64 | fmt.Errorf("unknown payload type %s", payload.Type())) 65 | } 66 | } 67 | } 68 | } 69 | } 70 | } 71 | } 72 | 73 | func unmarshalLocationElements(r *aws.Request, v reflect.Value) { 74 | for i := 0; i < v.NumField(); i++ { 75 | m, field := v.Field(i), v.Type().Field(i) 76 | if n := field.Name; n[0:1] == strings.ToLower(n[0:1]) { 77 | continue 78 | } 79 | 80 | if m.IsValid() { 81 | name := field.Tag.Get("locationName") 82 | if name == "" { 83 | name = field.Name 84 | } 85 | 86 | switch field.Tag.Get("location") { 87 | case "statusCode": 88 | unmarshalStatusCode(m, r.HTTPResponse.StatusCode) 89 | case "header": 90 | err := unmarshalHeader(m, r.HTTPResponse.Header.Get(name)) 91 | if err != nil { 92 | r.Error = apierr.New("Unmarshal", "failed to decode REST response", err) 93 | break 94 | } 95 | case "headers": 96 | prefix := field.Tag.Get("locationName") 97 | err := unmarshalHeaderMap(m, r.HTTPResponse.Header, prefix) 98 | if err != nil { 99 | r.Error = apierr.New("Unmarshal", "failed to decode REST response", err) 100 | break 101 | } 102 | } 103 | } 104 | if r.Error != nil { 105 | return 106 | } 107 | } 108 | } 109 | 110 | func unmarshalStatusCode(v reflect.Value, statusCode int) { 111 | if !v.IsValid() { 112 | return 113 | } 114 | 115 | switch v.Interface().(type) { 116 | case *int64: 117 | s := int64(statusCode) 118 | v.Set(reflect.ValueOf(&s)) 119 | } 120 | } 121 | 122 | func unmarshalHeaderMap(r reflect.Value, headers http.Header, prefix string) error { 123 | switch r.Interface().(type) { 124 | case map[string]*string: // we only support string map value types 125 | out := map[string]*string{} 126 | for k, v := range headers { 127 | k = http.CanonicalHeaderKey(k) 128 | if strings.HasPrefix(strings.ToLower(k), strings.ToLower(prefix)) { 129 | out[k[len(prefix):]] = &v[0] 130 | } 131 | } 132 | r.Set(reflect.ValueOf(out)) 133 | } 134 | return nil 135 | } 136 | 137 | func unmarshalHeader(v reflect.Value, header string) error { 138 | if !v.IsValid() || (header == "" && v.Elem().Kind() != reflect.String) { 139 | return nil 140 | } 141 | 142 | switch v.Interface().(type) { 143 | case *string: 144 | v.Set(reflect.ValueOf(&header)) 145 | case []byte: 146 | b, err := base64.StdEncoding.DecodeString(header) 147 | if err != nil { 148 | return err 149 | } 150 | v.Set(reflect.ValueOf(&b)) 151 | case *bool: 152 | b, err := strconv.ParseBool(header) 153 | if err != nil { 154 | return err 155 | } 156 | v.Set(reflect.ValueOf(&b)) 157 | case *int64: 158 | i, err := strconv.ParseInt(header, 10, 64) 159 | if err != nil { 160 | return err 161 | } 162 | v.Set(reflect.ValueOf(&i)) 163 | case *float64: 164 | f, err := strconv.ParseFloat(header, 64) 165 | if err != nil { 166 | return err 167 | } 168 | v.Set(reflect.ValueOf(&f)) 169 | case *time.Time: 170 | t, err := time.Parse(RFC822, header) 171 | if err != nil { 172 | return err 173 | } 174 | v.Set(reflect.ValueOf(&t)) 175 | default: 176 | err := fmt.Errorf("Unsupported value for param %v (%s)", v.Interface(), v.Type()) 177 | return err 178 | } 179 | return nil 180 | } 181 | -------------------------------------------------------------------------------- /internal/protocol/restjson/restjson.go: -------------------------------------------------------------------------------- 1 | // Package restjson provides RESTful JSON serialisation of AWS 2 | // requests and responses. 3 | package restjson 4 | 5 | //go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/input/rest-json.json build_test.go 6 | //go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/output/rest-json.json unmarshal_test.go 7 | 8 | import ( 9 | "encoding/json" 10 | "io/ioutil" 11 | "strings" 12 | 13 | "github.com/ks3sdklib/aws-sdk-go/aws" 14 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 15 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/jsonrpc" 16 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest" 17 | ) 18 | 19 | // Build builds a request for the REST JSON protocol. 20 | func Build(r *aws.Request) { 21 | rest.Build(r) 22 | 23 | if t := rest.PayloadType(r.Params); t == "structure" || t == "" { 24 | jsonrpc.Build(r) 25 | } 26 | } 27 | 28 | // Unmarshal unmarshals a response body for the REST JSON protocol. 29 | func Unmarshal(r *aws.Request) { 30 | if t := rest.PayloadType(r.Data); t == "structure" || t == "" { 31 | jsonrpc.Unmarshal(r) 32 | } 33 | } 34 | 35 | // UnmarshalMeta unmarshals response headers for the REST JSON protocol. 36 | func UnmarshalMeta(r *aws.Request) { 37 | rest.Unmarshal(r) 38 | } 39 | 40 | // UnmarshalError unmarshals a response error for the REST JSON protocol. 41 | func UnmarshalError(r *aws.Request) { 42 | code := r.HTTPResponse.Header.Get("X-Amzn-Errortype") 43 | bodyBytes, err := ioutil.ReadAll(r.HTTPResponse.Body) 44 | if err != nil { 45 | r.Error = apierr.New("Unmarshal", "failed reading REST JSON error response", err) 46 | return 47 | } 48 | if len(bodyBytes) == 0 { 49 | r.Error = apierr.NewRequestError( 50 | apierr.New("Unmarshal", r.HTTPResponse.Status, nil), 51 | r.HTTPResponse.StatusCode, 52 | "", 53 | ) 54 | return 55 | } 56 | var jsonErr jsonErrorResponse 57 | if err := json.Unmarshal(bodyBytes, &jsonErr); err != nil { 58 | r.Error = apierr.New("Unmarshal", "failed decoding REST JSON error response", err) 59 | return 60 | } 61 | 62 | if code == "" { 63 | code = jsonErr.Code 64 | } 65 | 66 | codes := strings.SplitN(code, ":", 2) 67 | r.Error = apierr.NewRequestError( 68 | apierr.New(codes[0], jsonErr.Message, nil), 69 | r.HTTPResponse.StatusCode, 70 | "", 71 | ) 72 | } 73 | 74 | type jsonErrorResponse struct { 75 | Code string `json:"code"` 76 | Message string `json:"message"` 77 | } 78 | -------------------------------------------------------------------------------- /internal/protocol/restxml/restxml.go: -------------------------------------------------------------------------------- 1 | // Package restxml provides RESTful XML serialisation of AWS 2 | // requests and responses. 3 | package restxml 4 | 5 | //go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/input/rest-xml.json build_test.go 6 | //go:generate go run ../../fixtures/protocol/generate.go ../../fixtures/protocol/output/rest-xml.json unmarshal_test.go 7 | 8 | import ( 9 | "bytes" 10 | "encoding/xml" 11 | "github.com/ks3sdklib/aws-sdk-go/aws" 12 | "github.com/ks3sdklib/aws-sdk-go/aws/awsutil" 13 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 14 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/query" 15 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/rest" 16 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/xml/xmlutil" 17 | "io/ioutil" 18 | ) 19 | 20 | // Build builds a request payload for the REST XML protocol. 21 | func Build(r *aws.Request) { 22 | rest.Build(r) 23 | 24 | if t := rest.PayloadType(r.Params); t == "structure" || t == "" { 25 | var buf bytes.Buffer 26 | err := xmlutil.BuildXML(r.Params, xml.NewEncoder(&buf)) 27 | if err != nil { 28 | r.Error = apierr.New("Marshal", "failed to encode rest XML request", err) 29 | return 30 | } 31 | r.SetBufferBody(buf.Bytes()) 32 | if rest.PayloadMd5(r.Params) { 33 | //增加md5 34 | r.HTTPRequest.Header.Set("Content-MD5", awsutil.EncodeAsString(awsutil.ComputeMD5Hash(buf.Bytes()))) 35 | } 36 | } 37 | } 38 | 39 | // Unmarshal unmarshals a payload response for the REST XML protocol. 40 | func Unmarshal(r *aws.Request) { 41 | if t := rest.PayloadType(r.Data); t == "structure" || t == "" { 42 | defer r.HTTPResponse.Body.Close() 43 | data, err := ioutil.ReadAll(r.HTTPResponse.Body) 44 | if err != nil { 45 | r.Error = apierr.New("ReadBody", "failed to read response body", err) 46 | return 47 | } 48 | decoder := xml.NewDecoder(bytes.NewReader(data)) 49 | err = xmlutil.UnmarshalXML(r.Data, decoder, "") 50 | if err != nil { 51 | r.Error = apierr.New("Unmarshal", "failed to decode REST XML response", err) 52 | return 53 | } 54 | return 55 | } 56 | } 57 | 58 | type GetBucketCORSOutput struct { 59 | CORSConfiguration *CORSConfiguration `xml:"CORSConfiguration"` 60 | Metadata map[string]*string `xml:"-"` 61 | } 62 | 63 | type CORSConfiguration struct { 64 | Rules []*CORSRule `xml:"CORSRule"` 65 | } 66 | 67 | type CORSRule struct { 68 | AllowedHeader []string `xml:"AllowedHeader"` 69 | AllowedMethod []string `xml:"AllowedMethod"` 70 | AllowedOrigin []string `xml:"AllowedOrigin"` 71 | ExposeHeader []string `xml:"ExposeHeader"` 72 | MaxAgeSeconds int `xml:"MaxAgeSeconds"` 73 | } 74 | 75 | // UnmarshalMeta unmarshals response headers for the REST XML protocol. 76 | func UnmarshalMeta(r *aws.Request) { 77 | rest.Unmarshal(r) 78 | } 79 | 80 | // UnmarshalError unmarshals a response error for the REST XML protocol. 81 | func UnmarshalError(r *aws.Request) { 82 | query.UnmarshalError(r) 83 | } 84 | -------------------------------------------------------------------------------- /internal/protocol/xml/xmlutil/xml_to_struct.go: -------------------------------------------------------------------------------- 1 | package xmlutil 2 | 3 | import ( 4 | "encoding/xml" 5 | "io" 6 | "sort" 7 | ) 8 | 9 | // A XMLNode contains the values to be encoded or decoded. 10 | type XMLNode struct { 11 | Name xml.Name `json:",omitempty"` 12 | Children map[string][]*XMLNode `json:",omitempty"` 13 | Text string `json:",omitempty"` 14 | Attr []xml.Attr `json:",omitempty"` 15 | } 16 | 17 | // NewXMLElement returns a pointer to a new XMLNode initialized to default values. 18 | func NewXMLElement(name xml.Name) *XMLNode { 19 | return &XMLNode{ 20 | Name: name, 21 | Children: map[string][]*XMLNode{}, 22 | Attr: []xml.Attr{}, 23 | } 24 | } 25 | 26 | // AddChild adds child to the XMLNode. 27 | func (n *XMLNode) AddChild(child *XMLNode) { 28 | if _, ok := n.Children[child.Name.Local]; !ok { 29 | n.Children[child.Name.Local] = []*XMLNode{} 30 | } 31 | n.Children[child.Name.Local] = append(n.Children[child.Name.Local], child) 32 | } 33 | 34 | // XMLToStruct converts a xml.Decoder stream to XMLNode with nested values. 35 | func XMLToStruct(d *xml.Decoder, s *xml.StartElement) (*XMLNode, error) { 36 | out := &XMLNode{} 37 | for { 38 | tok, err := d.Token() 39 | if tok == nil || err == io.EOF { 40 | break 41 | } 42 | if err != nil { 43 | return out, err 44 | } 45 | 46 | switch typed := tok.(type) { 47 | case xml.CharData: 48 | out.Text = string(typed.Copy()) 49 | case xml.StartElement: 50 | el := typed.Copy() 51 | out.Attr = el.Attr 52 | if out.Children == nil { 53 | out.Children = map[string][]*XMLNode{} 54 | } 55 | 56 | name := typed.Name.Local 57 | slice := out.Children[name] 58 | if slice == nil { 59 | slice = []*XMLNode{} 60 | } 61 | node, e := XMLToStruct(d, &el) 62 | if e != nil { 63 | return out, e 64 | } 65 | node.Name = typed.Name 66 | slice = append(slice, node) 67 | out.Children[name] = slice 68 | case xml.EndElement: 69 | if s != nil && s.Name.Local == typed.Name.Local { // matching end token 70 | return out, nil 71 | } 72 | } 73 | } 74 | return out, nil 75 | } 76 | 77 | // StructToXML writes an XMLNode to a xml.Encoder as tokens. 78 | func StructToXML(e *xml.Encoder, node *XMLNode, sorted bool) error { 79 | e.EncodeToken(xml.StartElement{Name: node.Name, Attr: node.Attr}) 80 | 81 | if node.Text != "" { 82 | e.EncodeToken(xml.CharData(node.Text)) 83 | } else if sorted { 84 | sortedNames := []string{} 85 | for k := range node.Children { 86 | sortedNames = append(sortedNames, k) 87 | } 88 | sort.Strings(sortedNames) 89 | 90 | for _, k := range sortedNames { 91 | for _, v := range node.Children[k] { 92 | StructToXML(e, v, sorted) 93 | } 94 | } 95 | } else { 96 | for _, c := range node.Children { 97 | for _, v := range c { 98 | StructToXML(e, v, sorted) 99 | } 100 | } 101 | } 102 | 103 | e.EncodeToken(xml.EndElement{Name: node.Name}) 104 | return e.Flush() 105 | } 106 | -------------------------------------------------------------------------------- /internal/signer/v4/v4_test.go: -------------------------------------------------------------------------------- 1 | package v4 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | "testing" 7 | "time" 8 | 9 | "github.com/ks3sdklib/aws-sdk-go/aws" 10 | "github.com/ks3sdklib/aws-sdk-go/aws/credentials" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func buildSigner(serviceName string, region string, signTime time.Time, expireTime time.Duration, body string) signer { 15 | endpoint := "https://" + serviceName + "." + region + ".amazonaws.com" 16 | reader := strings.NewReader(body) 17 | req, _ := http.NewRequest("POST", endpoint, reader) 18 | req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()" 19 | req.Header.Add("X-Amz-Target", "prefix.Operation") 20 | req.Header.Add("Content-Type", "application/x-amz-json-1.0") 21 | req.Header.Add("Content-Length", string(len(body))) 22 | req.Header.Add("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)") 23 | 24 | return signer{ 25 | Request: req, 26 | Time: signTime, 27 | ExpireTime: int64(expireTime), 28 | Query: req.URL.Query(), 29 | Body: reader, 30 | ServiceName: serviceName, 31 | Region: region, 32 | Credentials: credentials.NewStaticCredentials("AKID", "SECRET", "SESSION"), 33 | } 34 | } 35 | 36 | func removeWS(text string) string { 37 | text = strings.Replace(text, " ", "", -1) 38 | text = strings.Replace(text, "\n", "", -1) 39 | text = strings.Replace(text, "\t", "", -1) 40 | return text 41 | } 42 | 43 | func assertEqual(t *testing.T, expected, given string) { 44 | if removeWS(expected) != removeWS(given) { 45 | t.Errorf("\nExpected: %s\nGiven: %s", expected, given) 46 | } 47 | } 48 | 49 | func TestPresignRequest(t *testing.T) { 50 | signer := buildSigner("dynamodb", "us-east-1", time.Unix(0, 0), 300*time.Second, "{}") 51 | signer.sign() 52 | 53 | expectedDate := "19700101T000000Z" 54 | expectedHeaders := "host;x-amz-meta-other-header;x-amz-target" 55 | expectedSig := "5eeedebf6f995145ce56daa02902d10485246d3defb34f97b973c1f40ab82d36" 56 | expectedCred := "AKID/19700101/us-east-1/dynamodb/aws4_request" 57 | 58 | q := signer.Request.URL.Query() 59 | assert.Equal(t, expectedSig, q.Get("X-Amz-Signature")) 60 | assert.Equal(t, expectedCred, q.Get("X-Amz-Credential")) 61 | assert.Equal(t, expectedHeaders, q.Get("X-Amz-SignedHeaders")) 62 | assert.Equal(t, expectedDate, q.Get("X-Amz-Date")) 63 | } 64 | 65 | func TestSignRequest(t *testing.T) { 66 | signer := buildSigner("dynamodb", "us-east-1", time.Unix(0, 0), 0, "{}") 67 | signer.sign() 68 | 69 | expectedDate := "19700101T000000Z" 70 | expectedSig := "AWS4-HMAC-SHA256 Credential=AKID/19700101/us-east-1/dynamodb/aws4_request, SignedHeaders=host;x-amz-date;x-amz-meta-other-header;x-amz-security-token;x-amz-target, Signature=69ada33fec48180dab153576e4dd80c4e04124f80dda3eccfed8a67c2b91ed5e" 71 | 72 | q := signer.Request.Header 73 | assert.Equal(t, expectedSig, q.Get("Authorization")) 74 | assert.Equal(t, expectedDate, q.Get("X-Amz-Date")) 75 | } 76 | 77 | func TestSignEmptyBody(t *testing.T) { 78 | signer := buildSigner("dynamodb", "us-east-1", time.Now(), 0, "") 79 | signer.Body = nil 80 | signer.sign() 81 | hash := signer.Request.Header.Get("X-Amz-Content-Sha256") 82 | assert.Equal(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", hash) 83 | } 84 | 85 | func TestSignBody(t *testing.T) { 86 | signer := buildSigner("dynamodb", "us-east-1", time.Now(), 0, "hello") 87 | signer.sign() 88 | hash := signer.Request.Header.Get("X-Amz-Content-Sha256") 89 | assert.Equal(t, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", hash) 90 | } 91 | 92 | func TestSignSeekedBody(t *testing.T) { 93 | signer := buildSigner("dynamodb", "us-east-1", time.Now(), 0, " hello") 94 | signer.Body.Read(make([]byte, 3)) // consume first 3 bytes so body is now "hello" 95 | signer.sign() 96 | hash := signer.Request.Header.Get("X-Amz-Content-Sha256") 97 | assert.Equal(t, "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824", hash) 98 | 99 | start, _ := signer.Body.Seek(0, 1) 100 | assert.Equal(t, int64(3), start) 101 | } 102 | 103 | func TestPresignEmptyBodyS3(t *testing.T) { 104 | signer := buildSigner("s3", "us-east-1", time.Now(), 5*time.Minute, "hello") 105 | signer.sign() 106 | hash := signer.Request.Header.Get("X-Amz-Content-Sha256") 107 | assert.Equal(t, "UNSIGNED-PAYLOAD", hash) 108 | } 109 | 110 | func TestSignPrecomputedBodyChecksum(t *testing.T) { 111 | signer := buildSigner("dynamodb", "us-east-1", time.Now(), 0, "hello") 112 | signer.Request.Header.Set("X-Amz-Content-Sha256", "PRECOMPUTED") 113 | signer.sign() 114 | hash := signer.Request.Header.Get("X-Amz-Content-Sha256") 115 | assert.Equal(t, "PRECOMPUTED", hash) 116 | } 117 | 118 | func TestAnonymousCredentials(t *testing.T) { 119 | r := aws.NewRequest( 120 | aws.NewService(&aws.Config{Credentials: credentials.AnonymousCredentials}), 121 | &aws.Operation{ 122 | Name: "BatchGetItem", 123 | HTTPMethod: "POST", 124 | HTTPPath: "/", 125 | }, 126 | nil, 127 | nil, 128 | ) 129 | Sign(r) 130 | 131 | urlQ := r.HTTPRequest.URL.Query() 132 | assert.Empty(t, urlQ.Get("X-Amz-Signature")) 133 | assert.Empty(t, urlQ.Get("X-Amz-Credential")) 134 | assert.Empty(t, urlQ.Get("X-Amz-SignedHeaders")) 135 | assert.Empty(t, urlQ.Get("X-Amz-Date")) 136 | 137 | hQ := r.HTTPRequest.Header 138 | assert.Empty(t, hQ.Get("Authorization")) 139 | assert.Empty(t, hQ.Get("X-Amz-Date")) 140 | } 141 | 142 | func TestIgnoreResignRequestWithValidCreds(t *testing.T) { 143 | r := aws.NewRequest( 144 | aws.NewService(&aws.Config{ 145 | Credentials: credentials.NewStaticCredentials("AKID", "SECRET", "SESSION"), 146 | Region: "us-west-2", 147 | }), 148 | &aws.Operation{ 149 | Name: "BatchGetItem", 150 | HTTPMethod: "POST", 151 | HTTPPath: "/", 152 | }, 153 | nil, 154 | nil, 155 | ) 156 | 157 | Sign(r) 158 | sig := r.HTTPRequest.Header.Get("Authorization") 159 | 160 | Sign(r) 161 | assert.Equal(t, sig, r.HTTPRequest.Header.Get("Authorization")) 162 | } 163 | 164 | func TestIgnorePreResignRequestWithValidCreds(t *testing.T) { 165 | r := aws.NewRequest( 166 | aws.NewService(&aws.Config{ 167 | Credentials: credentials.NewStaticCredentials("AKID", "SECRET", "SESSION"), 168 | Region: "us-west-2", 169 | }), 170 | &aws.Operation{ 171 | Name: "BatchGetItem", 172 | HTTPMethod: "POST", 173 | HTTPPath: "/", 174 | }, 175 | nil, 176 | nil, 177 | ) 178 | r.ExpireTime = 10 179 | 180 | Sign(r) 181 | sig := r.HTTPRequest.Header.Get("X-Amz-Signature") 182 | 183 | Sign(r) 184 | assert.Equal(t, sig, r.HTTPRequest.Header.Get("X-Amz-Signature")) 185 | } 186 | 187 | func TestResignRequestExpiredCreds(t *testing.T) { 188 | creds := credentials.NewStaticCredentials("AKID", "SECRET", "SESSION") 189 | r := aws.NewRequest( 190 | aws.NewService(&aws.Config{Credentials: creds}), 191 | &aws.Operation{ 192 | Name: "BatchGetItem", 193 | HTTPMethod: "POST", 194 | HTTPPath: "/", 195 | }, 196 | nil, 197 | nil, 198 | ) 199 | Sign(r) 200 | querySig := r.HTTPRequest.Header.Get("Authorization") 201 | 202 | creds.Expire() 203 | 204 | Sign(r) 205 | assert.NotEqual(t, querySig, r.HTTPRequest.Header.Get("Authorization")) 206 | } 207 | 208 | func BenchmarkPresignRequest(b *testing.B) { 209 | signer := buildSigner("dynamodb", "us-east-1", time.Now(), 300*time.Second, "{}") 210 | for i := 0; i < b.N; i++ { 211 | signer.sign() 212 | } 213 | } 214 | 215 | func BenchmarkSignRequest(b *testing.B) { 216 | signer := buildSigner("dynamodb", "us-east-1", time.Now(), 0, "{}") 217 | for i := 0; i < b.N; i++ { 218 | signer.sign() 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /internal/util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "regexp" 5 | "sort" 6 | "strings" 7 | ) 8 | 9 | var reTrim = regexp.MustCompile(`\s{2,}`) 10 | 11 | func Trim(s string) string { 12 | return strings.TrimSpace(reTrim.ReplaceAllString(s, " ")) 13 | } 14 | 15 | // SortedKeys returns a sorted slice of keys of a map. 16 | func SortedKeys(m map[string]interface{}) []string { 17 | i, sorted := 0, make([]string, len(m)) 18 | for k := range m { 19 | sorted[i] = k 20 | i++ 21 | } 22 | sort.Strings(sorted) 23 | return sorted 24 | } 25 | -------------------------------------------------------------------------------- /sdk.go: -------------------------------------------------------------------------------- 1 | // Package sdk is the official AWS SDK for the Go programming language. 2 | // 3 | // See our Developer Guide for information for on getting started and using 4 | // the SDK. 5 | // 6 | // https://github.com/ks3sdklib/aws-sdk-go/wiki 7 | package sdk 8 | -------------------------------------------------------------------------------- /service/generate.go: -------------------------------------------------------------------------------- 1 | // Package service contains automatically generated AWS clients. 2 | package service 3 | 4 | //go:generate go run ../internal/model/cli/gen-api/main.go -path=../service ../apis/*/*.normal.json 5 | //go:generate gofmt -s -w ../service 6 | -------------------------------------------------------------------------------- /service/s3/bucket_location.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "github.com/ks3sdklib/aws-sdk-go/aws" 5 | "github.com/ks3sdklib/aws-sdk-go/aws/awserr" 6 | "io" 7 | "regexp" 8 | ) 9 | 10 | var reBucketLocation = regexp.MustCompile(`>([^<>]+)<\/LocationConstraint`) 11 | 12 | func buildGetBucketLocation(r *aws.Request) { 13 | if r.DataFilled() { 14 | out := r.Data.(*GetBucketLocationOutput) 15 | b, err := io.ReadAll(r.HTTPResponse.Body) 16 | if err != nil { 17 | r.Error = awserr.New("Unmarshal", 18 | "failed reading response body", err) 19 | return 20 | } 21 | match := reBucketLocation.FindSubmatch(b) 22 | if len(match) > 1 { 23 | loc := string(match[1]) 24 | out.LocationConstraint = aws.String(loc) 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /service/s3/const.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | // HTTP headers 4 | const ( 5 | HTTPHeaderAcceptEncoding string = "Accept-Encoding" 6 | HTTPHeaderAuthorization = "Authorization" 7 | HTTPHeaderCacheControl = "Cache-Control" 8 | HTTPHeaderContentDisposition = "Content-Disposition" 9 | HTTPHeaderContentEncoding = "Content-Encoding" 10 | HTTPHeaderContentLength = "Content-Length" 11 | HTTPHeaderContentMD5 = "Content-MD5" 12 | HTTPHeaderContentType = "Content-Type" 13 | HTTPHeaderContentLanguage = "Content-Language" 14 | HTTPHeaderLastModified = "Last-Modified" 15 | HTTPHeaderDate = "Date" 16 | HTTPHeaderEtag = "Etag" 17 | HTTPHeaderExpires = "Expires" 18 | HTTPHeaderHost = "Host" 19 | HTTPHeaderAmzACL = "X-Amz-Acl" 20 | HTTPHeaderAmzChecksumCrc64ecma = "X-Amz-Checksum-Crc64ecma" 21 | HTTPHeaderAmzStorageClass = "X-Amz-Storage-Class" 22 | HTTPHeaderAmzDataRedundancyType = "X-Amz-Data-Redundancy-Type" 23 | HTTPHeaderAmzZRSSwitchEnable = "X-Amz-Zrs-Switch-Enable" 24 | ) 25 | 26 | // ACL 27 | const ( 28 | ACLPrivate string = "private" 29 | ACLPublicRead string = "public-read" 30 | ACLPublicReadWrite string = "public-read-write" 31 | ) 32 | 33 | // StorageClass 34 | const ( 35 | StorageClassExtremePL3 string = "EXTREME_PL3" 36 | StorageClassExtremePL2 string = "EXTREME_PL2" 37 | StorageClassExtremePL1 string = "EXTREME_PL1" 38 | StorageClassStandard string = "STANDARD" 39 | StorageClassIA string = "STANDARD_IA" 40 | StorageClassDeepIA string = "DEEP_IA" 41 | StorageClassArchive string = "ARCHIVE" 42 | StorageClassDeepColdArchive string = "DEEP_COLD_ARCHIVE" 43 | ) 44 | 45 | // BucketType 46 | const ( 47 | BucketTypeExtremePL3 string = "EXTREME_PL3" 48 | BucketTypeExtremePL2 string = "EXTREME_PL2" 49 | BucketTypeExtremePL1 string = "EXTREME_PL1" 50 | BucketTypeNormal string = "NORMAL" 51 | BucketTypeIA string = "IA" 52 | BucketTypeDeepIA string = "DEEP_IA" 53 | BucketTypeArchive string = "ARCHIVE" 54 | ) 55 | 56 | type HTTPMethod string 57 | 58 | const ( 59 | PUT HTTPMethod = "PUT" 60 | GET HTTPMethod = "GET" 61 | DELETE HTTPMethod = "DELETE" 62 | HEAD HTTPMethod = "HEAD" 63 | POST HTTPMethod = "POST" 64 | ) 65 | 66 | const ( 67 | AllUsersUri = "http://acs.amazonaws.com/groups/global/AllUsers" 68 | MetaPrefix = "x-amz-meta-" 69 | ) 70 | 71 | const ( 72 | DataRedundancyTypeLRS string = "LRS" 73 | DataRedundancyTypeZRS string = "ZRS" 74 | ) 75 | -------------------------------------------------------------------------------- /service/s3/cors.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "github.com/ks3sdklib/aws-sdk-go/aws" 5 | ) 6 | 7 | // GetBucketCORSRequest generates a request for the GetBucketCORS operation. 8 | func (c *S3) GetBucketCORSRequest(input *GetBucketCORSInput) (req *aws.Request, output *GetBucketCORSOutput) { 9 | oprw.Lock() 10 | defer oprw.Unlock() 11 | 12 | if opGetBucketCORS == nil { 13 | opGetBucketCORS = &aws.Operation{ 14 | Name: "GetBucketCors", 15 | HTTPMethod: "GET", 16 | HTTPPath: "/{Bucket}?cors", 17 | } 18 | } 19 | 20 | if input == nil { 21 | input = &GetBucketCORSInput{} 22 | } 23 | 24 | req = c.newRequest(opGetBucketCORS, input, output) 25 | output = &GetBucketCORSOutput{} 26 | req.Data = output 27 | return 28 | } 29 | 30 | // GetBucketCORS Returns the cors configuration for the bucket. 31 | func (c *S3) GetBucketCORS(input *GetBucketCORSInput) (*GetBucketCORSOutput, error) { 32 | req, out := c.GetBucketCORSRequest(input) 33 | err := req.Send() 34 | if req.Data != nil { 35 | out = req.Data.(*GetBucketCORSOutput) 36 | } 37 | return out, err 38 | } 39 | 40 | func (c *S3) GetBucketCORSWithContext(ctx aws.Context, input *GetBucketCORSInput) (*GetBucketCORSOutput, error) { 41 | req, out := c.GetBucketCORSRequest(input) 42 | req.SetContext(ctx) 43 | err := req.Send() 44 | if req.Data != nil { 45 | out = req.Data.(*GetBucketCORSOutput) 46 | } 47 | return out, err 48 | } 49 | 50 | var opGetBucketCORS *aws.Operation 51 | 52 | // DeleteBucketCORSRequest generates a request for the DeleteBucketCORS operation. 53 | func (c *S3) DeleteBucketCORSRequest(input *DeleteBucketCORSInput) (req *aws.Request, output *DeleteBucketCORSOutput) { 54 | oprw.Lock() 55 | defer oprw.Unlock() 56 | 57 | if opDeleteBucketCORS == nil { 58 | opDeleteBucketCORS = &aws.Operation{ 59 | Name: "DeleteBucketCors", 60 | HTTPMethod: "DELETE", 61 | HTTPPath: "/{Bucket}?cors", 62 | } 63 | } 64 | 65 | if input == nil { 66 | input = &DeleteBucketCORSInput{} 67 | } 68 | 69 | req = c.newRequest(opDeleteBucketCORS, input, output) 70 | output = &DeleteBucketCORSOutput{} 71 | req.Data = output 72 | return 73 | } 74 | 75 | // DeleteBucketCORS Deletes the cors configuration information set for the bucket. 76 | func (c *S3) DeleteBucketCORS(input *DeleteBucketCORSInput) (*DeleteBucketCORSOutput, error) { 77 | req, out := c.DeleteBucketCORSRequest(input) 78 | err := req.Send() 79 | return out, err 80 | } 81 | 82 | func (c *S3) DeleteBucketCORSWithContext(ctx aws.Context, input *DeleteBucketCORSInput) (*DeleteBucketCORSOutput, error) { 83 | req, out := c.DeleteBucketCORSRequest(input) 84 | req.SetContext(ctx) 85 | err := req.Send() 86 | return out, err 87 | } 88 | 89 | var opDeleteBucketCORS *aws.Operation 90 | 91 | // PutBucketCORSRequest generates a request for the PutBucketCORS operation. 92 | func (c *S3) PutBucketCORSRequest(input *PutBucketCORSInput) (req *aws.Request, output *PutBucketCORSOutput) { 93 | oprw.Lock() 94 | defer oprw.Unlock() 95 | if opPutBucketCORS == nil { 96 | opPutBucketCORS = &aws.Operation{ 97 | Name: "PutBucketCors", 98 | HTTPMethod: "PUT", 99 | HTTPPath: "/{Bucket}?cors", 100 | } 101 | } 102 | if input == nil { 103 | input = &PutBucketCORSInput{} 104 | } 105 | //目前默认为true 106 | input.AutoFillMD5 = true 107 | req = c.newRequest(opPutBucketCORS, input, output) 108 | output = &PutBucketCORSOutput{} 109 | req.Data = output 110 | return 111 | } 112 | 113 | // PutBucketCORS Sets the cors configuration for a bucket. 114 | func (c *S3) PutBucketCORS(input *PutBucketCORSInput) (*PutBucketCORSOutput, error) { 115 | req, out := c.PutBucketCORSRequest(input) 116 | err := req.Send() 117 | return out, err 118 | } 119 | 120 | func (c *S3) PutBucketCORSWithContext(ctx aws.Context, input *PutBucketCORSInput) (*PutBucketCORSOutput, error) { 121 | req, out := c.PutBucketCORSRequest(input) 122 | req.SetContext(ctx) 123 | err := req.Send() 124 | return out, err 125 | } 126 | 127 | var opPutBucketCORS *aws.Operation 128 | 129 | type PutBucketCORSInput struct { 130 | Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` 131 | 132 | CORSConfiguration *CORSConfiguration `locationName:"CORSConfiguration" type:"structure" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/" ` 133 | 134 | ContentType *string `location:"header" locationName:"Content-Type" type:"string"` 135 | 136 | metadataPutBucketCORSInput `json:"-" xml:"-"` 137 | } 138 | 139 | type metadataPutBucketCORSInput struct { 140 | SDKShapeTraits bool `type:"structure" payload:"CORSConfiguration"` 141 | AutoFillMD5 bool 142 | } 143 | 144 | type GetBucketCORSInput struct { 145 | Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` 146 | 147 | ContentType *string `location:"header" locationName:"Content-Type" type:"string"` 148 | 149 | metadataInput `json:"-" xml:"-"` 150 | } 151 | 152 | type metadataInput struct { 153 | SDKShapeTraits bool `type:"structure"` 154 | } 155 | 156 | type GetBucketCORSOutput struct { 157 | Metadata map[string]*string `location:"headers" type:"map"` 158 | Rules []*GetCORSRule `locationName:"CORSRule" type:"list" flattened:"true" xml:"CORSRule"` 159 | StatusCode *int64 `location:"statusCode" type:"integer"` 160 | } 161 | type GetCORSRule struct { 162 | AllowedHeader []*string `locationName:"AllowedHeader" type:"list" flattened:"true" ` 163 | AllowedMethod []*string `locationName:"AllowedMethod" type:"list" flattened:"true"` 164 | AllowedOrigin []*string `locationName:"AllowedOrigin" type:"list" flattened:"true"` 165 | ExposeHeader []*string `locationName:"ExposeHeader" type:"list" flattened:"true"` 166 | MaxAgeSeconds *int64 `locationName:"MaxAgeSeconds" flattened:"true"` // Max cache ages in seconds 167 | } 168 | type CORSConfiguration struct { 169 | Rules []*CORSRule `locationName:"CORSRule" type:"list" flattened:"true" xml:"CORSRule"` 170 | } 171 | 172 | type PutBucketCORSOutput struct { 173 | metadataPutBucketCORSOutput `json:"-" xml:"-"` 174 | 175 | Metadata map[string]*string `location:"headers" type:"map"` 176 | 177 | StatusCode *int64 `location:"statusCode" type:"integer"` 178 | } 179 | 180 | type metadataPutBucketCORSOutput struct { 181 | SDKShapeTraits bool `type:"structure"` 182 | } 183 | 184 | type CORSRule struct { 185 | AllowedHeader []string `locationName:"AllowedHeader" type:"list" flattened:"true"` 186 | AllowedMethod []string `locationName:"AllowedMethod" type:"list" flattened:"true"` 187 | AllowedOrigin []string `locationName:"AllowedOrigin" type:"list" flattened:"true"` 188 | ExposeHeader []string `locationName:"ExposeHeader" type:"list" flattened:"true"` 189 | MaxAgeSeconds int64 `locationName:"MaxAgeSeconds" flattened:"true"` // Max cache ages in seconds 190 | } 191 | 192 | type DeleteBucketCORSInput struct { 193 | Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` 194 | 195 | ContentType *string `location:"header" locationName:"Content-Type" type:"string"` 196 | 197 | metadataDeleteBucketCORSInput `json:"-" xml:"-"` 198 | } 199 | 200 | type metadataDeleteBucketCORSInput struct { 201 | SDKShapeTraits bool `type:"structure"` 202 | } 203 | 204 | type DeleteBucketCORSOutput struct { 205 | metadataDeleteBucketCORSOutput `json:"-" xml:"-"` 206 | 207 | Metadata map[string]*string `location:"headers" type:"map"` 208 | 209 | StatusCode *int64 `location:"statusCode" type:"integer"` 210 | } 211 | 212 | type metadataDeleteBucketCORSOutput struct { 213 | SDKShapeTraits bool `type:"structure"` 214 | } 215 | -------------------------------------------------------------------------------- /service/s3/crc_check.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "fmt" 5 | "github.com/ks3sdklib/aws-sdk-go/aws" 6 | "github.com/ks3sdklib/aws-sdk-go/internal/apierr" 7 | "hash" 8 | "strconv" 9 | ) 10 | 11 | func CheckUploadCrc64(r *aws.Request) { 12 | clientCrc := r.Crc64.Sum64() 13 | serverCrc := uint64(0) 14 | if r.HTTPResponse.Header.Get("X-Amz-Checksum-Crc64ecma") != "" { 15 | serverCrc, _ = strconv.ParseUint(r.HTTPResponse.Header.Get("X-Amz-Checksum-Crc64ecma"), 10, 64) 16 | } 17 | 18 | r.Config.LogInfo("client crc:%d, server crc:%d", clientCrc, serverCrc) 19 | 20 | if serverCrc != 0 && clientCrc != serverCrc { 21 | r.Error = apierr.New("CRCCheckError", fmt.Sprintf("client crc and server crc do not match, request id:[%s]", r.HTTPResponse.Header.Get("X-Kss-Request-Id")), nil) 22 | r.Config.LogError("%s", r.Error.Error()) 23 | } 24 | } 25 | 26 | func CheckDownloadCrc64(s3 *S3, res *GetObjectOutput, crc hash.Hash64) error { 27 | var err error 28 | clientCrc := crc.Sum64() 29 | serverCrc := uint64(0) 30 | if res.Metadata["X-Amz-Checksum-Crc64ecma"] != nil { 31 | serverCrc, _ = strconv.ParseUint(*res.Metadata["X-Amz-Checksum-Crc64ecma"], 10, 64) 32 | } 33 | 34 | s3.Config.LogInfo("client crc:%d, server crc:%d", clientCrc, serverCrc) 35 | 36 | if serverCrc != 0 && clientCrc != serverCrc { 37 | err = apierr.New("CRCCheckError", fmt.Sprintf("client crc and server crc do not match, request id:[%s]", *res.Metadata["X-Kss-Request-Id"]), nil) 38 | s3.Config.LogError("%s", err.Error()) 39 | } 40 | 41 | return err 42 | } 43 | -------------------------------------------------------------------------------- /service/s3/data_redundancy_switch.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "github.com/ks3sdklib/aws-sdk-go/aws" 5 | "time" 6 | ) 7 | 8 | var opPutBucketDataRedundancySwitch *aws.Operation 9 | 10 | type PutBucketDataRedundancySwitchInput struct { 11 | // The name of the bucket. 12 | Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` 13 | // The bucket data redundancy type. 14 | // Valid value: LRS丨ZRS 15 | // LRS: local redundancy storage 16 | // ZRS: zone redundancy storage 17 | DataRedundancyType *string `location:"header" locationName:"x-amz-data-redundancy-type" type:"string" required:"true"` 18 | } 19 | 20 | type PutBucketDataRedundancySwitchOutput struct { 21 | // The HTTP headers of the response. 22 | Metadata map[string]*string `location:"headers" type:"map"` 23 | // The HTTP status code of the response. 24 | StatusCode *int64 `location:"statusCode" type:"integer"` 25 | } 26 | 27 | // PutBucketDataRedundancySwitchRequest generates a request for the PutBucketDataRedundancySwitch operation. 28 | func (c *S3) PutBucketDataRedundancySwitchRequest(input *PutBucketDataRedundancySwitchInput) (req *aws.Request, output *PutBucketDataRedundancySwitchOutput) { 29 | oprw.Lock() 30 | defer oprw.Unlock() 31 | if opPutBucketDataRedundancySwitch == nil { 32 | opPutBucketDataRedundancySwitch = &aws.Operation{ 33 | Name: "PutBucketDataRedundancySwitch", 34 | HTTPMethod: "PUT", 35 | HTTPPath: "/{Bucket}?dataRedundancySwitch", 36 | } 37 | } 38 | if input == nil { 39 | input = &PutBucketDataRedundancySwitchInput{} 40 | } 41 | req = c.newRequest(opPutBucketDataRedundancySwitch, input, output) 42 | output = &PutBucketDataRedundancySwitchOutput{} 43 | req.Data = output 44 | return 45 | } 46 | 47 | // PutBucketDataRedundancySwitch sets the data redundancy type for the bucket. 48 | func (c *S3) PutBucketDataRedundancySwitch(input *PutBucketDataRedundancySwitchInput) (*PutBucketDataRedundancySwitchOutput, error) { 49 | req, out := c.PutBucketDataRedundancySwitchRequest(input) 50 | err := req.Send() 51 | return out, err 52 | } 53 | 54 | func (c *S3) PutBucketDataRedundancySwitchWithContext(ctx aws.Context, input *PutBucketDataRedundancySwitchInput) (*PutBucketDataRedundancySwitchOutput, error) { 55 | req, out := c.PutBucketDataRedundancySwitchRequest(input) 56 | req.SetContext(ctx) 57 | err := req.Send() 58 | return out, err 59 | } 60 | 61 | var opGetBucketDataRedundancySwitch *aws.Operation 62 | 63 | type GetBucketDataRedundancySwitchInput struct { 64 | // The name of the bucket. 65 | Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` 66 | } 67 | 68 | type GetBucketDataRedundancySwitchOutput struct { 69 | // The bucket data redundancy switch configuration. 70 | DataRedundancySwitch *DataRedundancySwitch `locationName:"DataRedundancySwitch" type:"structure"` 71 | // The HTTP headers of the response. 72 | Metadata map[string]*string `location:"headers" type:"map"` 73 | // The HTTP status code of the response. 74 | StatusCode *int64 `location:"statusCode" type:"integer"` 75 | 76 | metadataGetBucketDataRedundancySwitchOutput `json:"-" xml:"-"` 77 | } 78 | 79 | type DataRedundancySwitch struct { 80 | // The bucket data redundancy type. 81 | DataRedundancyType *string `locationName:"DataRedundancyType" type:"string"` 82 | // Time when zone redundancy is enabled. 83 | SwitchTime *time.Time `locationName:"SwitchTime" type:"timestamp" timestampFormat:"iso8601"` 84 | } 85 | 86 | type metadataGetBucketDataRedundancySwitchOutput struct { 87 | SDKShapeTraits bool `type:"structure" payload:"DataRedundancySwitch"` 88 | } 89 | 90 | // GetBucketDataRedundancySwitchRequest generates a request for the GetBucketDataRedundancySwitch operation. 91 | func (c *S3) GetBucketDataRedundancySwitchRequest(input *GetBucketDataRedundancySwitchInput) (req *aws.Request, output *GetBucketDataRedundancySwitchOutput) { 92 | oprw.Lock() 93 | defer oprw.Unlock() 94 | if opGetBucketDataRedundancySwitch == nil { 95 | opGetBucketDataRedundancySwitch = &aws.Operation{ 96 | Name: "GetBucketDataRedundancySwitch", 97 | HTTPMethod: "GET", 98 | HTTPPath: "/{Bucket}?dataRedundancySwitch", 99 | } 100 | } 101 | if input == nil { 102 | input = &GetBucketDataRedundancySwitchInput{} 103 | } 104 | req = c.newRequest(opGetBucketDataRedundancySwitch, input, output) 105 | output = &GetBucketDataRedundancySwitchOutput{} 106 | req.Data = output 107 | return 108 | } 109 | 110 | // GetBucketDataRedundancySwitch gets the data redundancy switch configuration for the bucket. 111 | func (c *S3) GetBucketDataRedundancySwitch(input *GetBucketDataRedundancySwitchInput) (*GetBucketDataRedundancySwitchOutput, error) { 112 | req, out := c.GetBucketDataRedundancySwitchRequest(input) 113 | err := req.Send() 114 | return out, err 115 | } 116 | 117 | func (c *S3) GetBucketDataRedundancySwitchWithContext(ctx aws.Context, input *GetBucketDataRedundancySwitchInput) (*GetBucketDataRedundancySwitchOutput, error) { 118 | req, out := c.GetBucketDataRedundancySwitchRequest(input) 119 | req.SetContext(ctx) 120 | err := req.Send() 121 | return out, err 122 | } 123 | -------------------------------------------------------------------------------- /service/s3/errors.go: -------------------------------------------------------------------------------- 1 | // Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT. 2 | 3 | package s3 4 | 5 | const ( 6 | 7 | // ErrCodeBucketAlreadyExists for service response error code 8 | // "BucketAlreadyExists". 9 | // 10 | // The requested bucket name is not available. The bucket namespace is shared 11 | // by all users of the system. Select a different name and try again. 12 | ErrCodeBucketAlreadyExists = "BucketAlreadyExists" 13 | 14 | // ErrCodeBucketAlreadyOwnedByYou for service response error code 15 | // "BucketAlreadyOwnedByYou". 16 | // 17 | // The bucket you tried to create already exists, and you own it. Amazon S3 18 | // returns this error in all AWS Regions except in the North Virginia Region. 19 | // For legacy compatibility, if you re-create an existing bucket that you already 20 | // own in the North Virginia Region, Amazon S3 returns 200 OK and resets the 21 | // bucket access control lists (ACLs). 22 | ErrCodeBucketAlreadyOwnedByYou = "BucketAlreadyOwnedByYou" 23 | 24 | // ErrCodeInvalidObjectState for service response error code 25 | // "InvalidObjectState". 26 | // 27 | // Object is archived and inaccessible until restored. 28 | ErrCodeInvalidObjectState = "InvalidObjectState" 29 | 30 | // ErrCodeNoSuchBucket for service response error code 31 | // "NoSuchBucket". 32 | // 33 | // The specified bucket does not exist. 34 | ErrCodeNoSuchBucket = "NoSuchBucket" 35 | 36 | // ErrCodeNoSuchKey for service response error code 37 | // "NoSuchKey". 38 | // 39 | // The specified key does not exist. 40 | ErrCodeNoSuchKey = "NoSuchKey" 41 | 42 | // ErrCodeNoSuchUpload for service response error code 43 | // "NoSuchUpload". 44 | // 45 | // The specified multipart upload does not exist. 46 | ErrCodeNoSuchUpload = "NoSuchUpload" 47 | 48 | // ErrCodeObjectAlreadyInActiveTierError for service response error code 49 | // "ObjectAlreadyInActiveTierError". 50 | // 51 | // This action is not allowed against this storage tier. 52 | ErrCodeObjectAlreadyInActiveTierError = "ObjectAlreadyInActiveTierError" 53 | 54 | // ErrCodeObjectNotInActiveTierError for service response error code 55 | // "ObjectNotInActiveTierError". 56 | // 57 | // The source object of the COPY action is not in the active tier and is only 58 | // stored in Amazon S3 Glacier. 59 | ErrCodeObjectNotInActiveTierError = "ObjectNotInActiveTierError" 60 | ) 61 | -------------------------------------------------------------------------------- /service/s3/host_style_bucket.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | 7 | "github.com/ks3sdklib/aws-sdk-go/aws" 8 | "github.com/ks3sdklib/aws-sdk-go/aws/awsutil" 9 | ) 10 | 11 | var reDomain = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`) 12 | var reIPAddress = regexp.MustCompile(`^(\d+\.){3}\d+$`) 13 | 14 | // dnsCompatibleBucketName returns true if the bucket name is DNS compatible. 15 | // Buckets created outside the classic region MUST be DNS compatible. 16 | func dnsCompatibleBucketName(bucket string) bool { 17 | return reDomain.MatchString(bucket) && 18 | !reIPAddress.MatchString(bucket) && 19 | !strings.Contains(bucket, "..") 20 | } 21 | 22 | // hostStyleBucketName returns true if the request should put the bucket in 23 | // the host. This is false if S3ForcePathStyle is explicitly set or if the 24 | // bucket is not DNS compatible. 25 | func hostStyleBucketName(r *aws.Request, bucket string) bool { 26 | if r.Config.S3ForcePathStyle { 27 | return false 28 | } 29 | 30 | // Bucket might be DNS compatible but dots in the hostname will fail 31 | // certificate validation, so do not use host-style. 32 | if r.HTTPRequest.URL.Scheme == "https" && strings.Contains(bucket, ".") { 33 | return false 34 | } 35 | 36 | // Use host-style if the bucket is DNS compatible 37 | return dnsCompatibleBucketName(bucket) 38 | } 39 | 40 | func updateHostWithBucket(r *aws.Request) { 41 | b := awsutil.ValuesAtPath(r.Params, "Bucket") 42 | if len(b) == 0 { 43 | return 44 | } 45 | 46 | if bucket := b[0].(string); bucket != "" && hostStyleBucketName(r, bucket) { 47 | r.HTTPRequest.URL.Host = bucket + "." + r.HTTPRequest.URL.Host 48 | r.HTTPRequest.URL.Path = strings.Replace(r.HTTPRequest.URL.Path, "/{Bucket}", "", -1) 49 | if r.HTTPRequest.URL.Path == "" { 50 | r.HTTPRequest.URL.Path = "/" 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /service/s3/replication.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import "github.com/ks3sdklib/aws-sdk-go/aws" 4 | 5 | // PutBucketReplicationRequest generates a request for the PutBucketReplication operation. 6 | func (c *S3) PutBucketReplicationRequest(input *PutBucketReplicationInput) (req *aws.Request, output *PutBucketReplicationOutput) { 7 | oprw.Lock() 8 | defer oprw.Unlock() 9 | 10 | if opPutBucketReplication == nil { 11 | opPutBucketReplication = &aws.Operation{ 12 | Name: "PutBucketReplication", 13 | HTTPMethod: "PUT", 14 | HTTPPath: "/{Bucket}?crr", 15 | } 16 | } 17 | 18 | if input == nil { 19 | input = &PutBucketReplicationInput{} 20 | } 21 | 22 | input.AutoFillMD5 = true 23 | req = c.newRequest(opPutBucketReplication, input, output) 24 | output = &PutBucketReplicationOutput{} 25 | req.Data = output 26 | return 27 | } 28 | 29 | // PutBucketReplication creates a new replication configuration. 30 | func (c *S3) PutBucketReplication(input *PutBucketReplicationInput) (*PutBucketReplicationOutput, error) { 31 | req, out := c.PutBucketReplicationRequest(input) 32 | err := req.Send() 33 | return out, err 34 | } 35 | 36 | func (c *S3) PutBucketReplicationWithContext(ctx aws.Context, input *PutBucketReplicationInput) (*PutBucketReplicationOutput, error) { 37 | req, out := c.PutBucketReplicationRequest(input) 38 | req.SetContext(ctx) 39 | err := req.Send() 40 | return out, err 41 | } 42 | 43 | var opPutBucketReplication *aws.Operation 44 | 45 | type PutBucketReplicationInput struct { 46 | Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` 47 | 48 | ReplicationConfiguration *ReplicationConfiguration `locationName:"Replication" type:"structure" required:"true" xmlURI:"http://s3.amazonaws.com/doc/2006-03-01/"` 49 | 50 | ContentType *string `location:"header" locationName:"Content-Type" type:"string"` 51 | 52 | metadataPutBucketReplicationInput `json:"-" xml:"-"` 53 | } 54 | 55 | type metadataPutBucketReplicationInput struct { 56 | SDKShapeTraits bool `type:"structure" payload:"ReplicationConfiguration"` 57 | 58 | AutoFillMD5 bool 59 | } 60 | 61 | type PutBucketReplicationOutput struct { 62 | Metadata map[string]*string `location:"headers" type:"map"` 63 | 64 | StatusCode *int64 `location:"statusCode" type:"integer"` 65 | } 66 | 67 | type ReplicationConfiguration struct { 68 | // Prefix matching, only objects that match prefix rules will be copied. Each copying rule 69 | // can add up to 10 prefix matching rules, and prefixes cannot overlap with each other. 70 | Prefix []*string `locationName:"prefix" type:"list" flattened:"true"` 71 | 72 | // Indicate whether to enable delete replication. If set to Enabled, it means enabled; if set to 73 | // Disabled or not, it means disabled. If set to delete replication, when the source bucket deletes 74 | // an object, the replica of that object in the target bucket will also be deleted. 75 | DeleteMarkerStatus *string `locationName:"DeleteMarkerStatus" type:"string" required:"true"` 76 | 77 | // Target bucket for copying rules. 78 | TargetBucket *string `locationName:"targetBucket" type:"string" required:"true"` 79 | 80 | // Specify whether to copy historical data. Whether to copy the data from the source bucket 81 | // to the target bucket before enabling data replication. 82 | // Enabled: Copy historical data to the target bucket (default value) 83 | // Disabled: Do not copy historical data, only copy new data after enabling the rule to the target bucket. 84 | HistoricalObjectReplication *string `locationName:"HistoricalObjectReplication" type:"string"` 85 | 86 | // Region of the target bucket. 87 | Region *string `locationName:"region" type:"string"` 88 | } 89 | 90 | // GetBucketReplicationRequest generates a request for the GetBucketReplication operation. 91 | func (c *S3) GetBucketReplicationRequest(input *GetBucketReplicationInput) (req *aws.Request, output *GetBucketReplicationOutput) { 92 | oprw.Lock() 93 | defer oprw.Unlock() 94 | 95 | if opGetBucketReplication == nil { 96 | opGetBucketReplication = &aws.Operation{ 97 | Name: "GetBucketReplication", 98 | HTTPMethod: "GET", 99 | HTTPPath: "/{Bucket}?crr", 100 | } 101 | } 102 | 103 | if input == nil { 104 | input = &GetBucketReplicationInput{} 105 | } 106 | 107 | req = c.newRequest(opGetBucketReplication, input, output) 108 | output = &GetBucketReplicationOutput{} 109 | req.Data = output 110 | return 111 | } 112 | 113 | // GetBucketReplication gets the replication configuration for the bucket. 114 | func (c *S3) GetBucketReplication(input *GetBucketReplicationInput) (*GetBucketReplicationOutput, error) { 115 | req, out := c.GetBucketReplicationRequest(input) 116 | err := req.Send() 117 | return out, err 118 | } 119 | 120 | func (c *S3) GetBucketReplicationWithContext(ctx aws.Context, input *GetBucketReplicationInput) (*GetBucketReplicationOutput, error) { 121 | req, out := c.GetBucketReplicationRequest(input) 122 | req.SetContext(ctx) 123 | err := req.Send() 124 | return out, err 125 | } 126 | 127 | var opGetBucketReplication *aws.Operation 128 | 129 | type GetBucketReplicationInput struct { 130 | Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` 131 | } 132 | 133 | type GetBucketReplicationOutput struct { 134 | ReplicationConfiguration *ReplicationConfiguration `locationName:"Replication" type:"structure"` 135 | 136 | Metadata map[string]*string `location:"headers" type:"map"` 137 | 138 | StatusCode *int64 `location:"statusCode" type:"integer"` 139 | 140 | metadataGetBucketReplicationOutput `json:"-" xml:"-"` 141 | } 142 | 143 | type metadataGetBucketReplicationOutput struct { 144 | SDKShapeTraits bool `type:"structure" payload:"ReplicationConfiguration"` 145 | } 146 | 147 | // DeleteBucketReplicationRequest generates a request for the DeleteBucketReplication operation. 148 | func (c *S3) DeleteBucketReplicationRequest(input *DeleteBucketReplicationInput) (req *aws.Request, output *DeleteBucketReplicationOutput) { 149 | oprw.Lock() 150 | defer oprw.Unlock() 151 | 152 | if opDeleteBucketReplication == nil { 153 | opDeleteBucketReplication = &aws.Operation{ 154 | Name: "DeleteBucketReplication", 155 | HTTPMethod: "DELETE", 156 | HTTPPath: "/{Bucket}?crr", 157 | } 158 | } 159 | 160 | if input == nil { 161 | input = &DeleteBucketReplicationInput{} 162 | } 163 | 164 | req = c.newRequest(opDeleteBucketReplication, input, output) 165 | output = &DeleteBucketReplicationOutput{} 166 | req.Data = output 167 | return 168 | } 169 | 170 | // DeleteBucketReplication deletes the replication configuration for the bucket. 171 | func (c *S3) DeleteBucketReplication(input *DeleteBucketReplicationInput) (*DeleteBucketReplicationOutput, error) { 172 | req, out := c.DeleteBucketReplicationRequest(input) 173 | err := req.Send() 174 | return out, err 175 | } 176 | 177 | func (c *S3) DeleteBucketReplicationWithContext(ctx aws.Context, input *DeleteBucketReplicationInput) (*DeleteBucketReplicationOutput, error) { 178 | req, out := c.DeleteBucketReplicationRequest(input) 179 | req.SetContext(ctx) 180 | err := req.Send() 181 | return out, err 182 | } 183 | 184 | var opDeleteBucketReplication *aws.Operation 185 | 186 | type DeleteBucketReplicationInput struct { 187 | Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` 188 | } 189 | type DeleteBucketReplicationOutput struct { 190 | Metadata map[string]*string `location:"headers" type:"map"` 191 | 192 | StatusCode *int64 `location:"statusCode" type:"integer"` 193 | } 194 | -------------------------------------------------------------------------------- /service/s3/s3iface/interface.go: -------------------------------------------------------------------------------- 1 | // THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT. 2 | 3 | // Package s3iface provides an interface for the Amazon Simple Storage Service. 4 | package s3iface 5 | 6 | import ( 7 | "github.com/ks3sdklib/aws-sdk-go/service/s3" 8 | ) 9 | 10 | // S3API is the interface type for s3.S3. 11 | type S3API interface { 12 | AbortMultipartUpload(*s3.AbortMultipartUploadInput) (*s3.AbortMultipartUploadOutput, error) 13 | 14 | AppendObject(input *s3.AppendObjectInput) (*s3.AppendObjectOutput, error) 15 | 16 | CompleteMultipartUpload(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) 17 | 18 | CopyObject(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error) 19 | 20 | CreateBucket(*s3.CreateBucketInput) (*s3.CreateBucketOutput, error) 21 | 22 | CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) 23 | 24 | ClearObject(*s3.ClearObjectInput) (*s3.ClearObjectOutput, error) 25 | 26 | DeleteBucket(*s3.DeleteBucketInput) (*s3.DeleteBucketOutput, error) 27 | 28 | DeleteBucketCORS(*s3.DeleteBucketCORSInput) (*s3.DeleteBucketCORSOutput, error) 29 | 30 | DeleteBucketLifecycle(*s3.DeleteBucketLifecycleInput) (*s3.DeleteBucketLifecycleOutput, error) 31 | 32 | DeleteBucketPolicy(*s3.DeleteBucketPolicyInput) (*s3.DeleteBucketPolicyOutput, error) 33 | 34 | DeleteBucketReplication(*s3.DeleteBucketReplicationInput) (*s3.DeleteBucketReplicationOutput, error) 35 | 36 | DeleteBucketTagging(*s3.DeleteBucketTaggingInput) (*s3.DeleteBucketTaggingOutput, error) 37 | 38 | DeleteBucketWebsite(*s3.DeleteBucketWebsiteInput) (*s3.DeleteBucketWebsiteOutput, error) 39 | 40 | DeleteBucketDecompressPolicy(*s3.DeleteBucketDecompressPolicyInput) (*s3.DeleteBucketDecompressPolicyOutput, error) 41 | 42 | DeleteBucketInventory(*s3.DeleteBucketInventoryInput) (*s3.DeleteBucketInventoryOutput, error) 43 | 44 | DeleteObject(*s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) 45 | 46 | DeleteObjects(*s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) 47 | 48 | GetBucketACL(*s3.GetBucketACLInput) (*s3.GetBucketACLOutput, error) 49 | 50 | GetBucketCORS(*s3.GetBucketCORSInput) (*s3.GetBucketCORSOutput, error) 51 | 52 | GetBucketLifecycle(*s3.GetBucketLifecycleInput) (*s3.GetBucketLifecycleOutput, error) 53 | 54 | GetBucketLocation(*s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error) 55 | 56 | GetBucketLogging(*s3.GetBucketLoggingInput) (*s3.GetBucketLoggingOutput, error) 57 | 58 | GetBucketNotification(*s3.GetBucketNotificationConfigurationRequest) (*s3.NotificationConfigurationDeprecated, error) 59 | 60 | GetBucketNotificationConfiguration(*s3.GetBucketNotificationConfigurationRequest) (*s3.NotificationConfiguration, error) 61 | 62 | GetBucketPolicy(*s3.GetBucketPolicyInput) (*s3.GetBucketPolicyOutput, error) 63 | 64 | GetBucketReplication(*s3.GetBucketReplicationInput) (*s3.GetBucketReplicationOutput, error) 65 | 66 | GetBucketRequestPayment(*s3.GetBucketRequestPaymentInput) (*s3.GetBucketRequestPaymentOutput, error) 67 | 68 | GetBucketTagging(*s3.GetBucketTaggingInput) (*s3.GetBucketTaggingOutput, error) 69 | 70 | GetBucketVersioning(*s3.GetBucketVersioningInput) (*s3.GetBucketVersioningOutput, error) 71 | 72 | GetBucketWebsite(*s3.GetBucketWebsiteInput) (*s3.GetBucketWebsiteOutput, error) 73 | 74 | GetBucketDecompressPolicy(*s3.GetBucketDecompressPolicyInput) (*s3.GetBucketDecompressPolicyOutput, error) 75 | 76 | GetBucketRetention(*s3.GetBucketRetentionInput) (*s3.GetBucketRetentionOutput, error) 77 | 78 | GetBucketInventory(*s3.GetBucketInventoryInput) (*s3.GetBucketInventoryOutput, error) 79 | 80 | GetObject(*s3.GetObjectInput) (*s3.GetObjectOutput, error) 81 | 82 | GetObjectACL(*s3.GetObjectACLInput) (*s3.GetObjectACLOutput, error) 83 | 84 | GetObjectTorrent(*s3.GetObjectTorrentInput) (*s3.GetObjectTorrentOutput, error) 85 | 86 | HeadBucket(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) 87 | 88 | HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) 89 | 90 | ListBuckets(*s3.ListBucketsInput) (*s3.ListBucketsOutput, error) 91 | 92 | ListMultipartUploads(*s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) 93 | 94 | ListObjectVersions(*s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) 95 | 96 | ListObjects(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) 97 | 98 | ListParts(*s3.ListPartsInput) (*s3.ListPartsOutput, error) 99 | 100 | ListRetention(*s3.ListRetentionInput) (*s3.ListRetentionOutput, error) 101 | 102 | ListBucketInventory(*s3.ListBucketInventoryInput) (*s3.ListBucketInventoryOutput, error) 103 | 104 | PutBucketACL(*s3.PutBucketACLInput) (*s3.PutBucketACLOutput, error) 105 | 106 | PutBucketCORS(*s3.PutBucketCORSInput) (*s3.PutBucketCORSOutput, error) 107 | 108 | PutBucketLifecycle(*s3.PutBucketLifecycleInput) (*s3.PutBucketLifecycleOutput, error) 109 | 110 | PutBucketLogging(*s3.PutBucketLoggingInput) (*s3.PutBucketLoggingOutput, error) 111 | 112 | PutBucketNotification(*s3.PutBucketNotificationInput) (*s3.PutBucketNotificationOutput, error) 113 | 114 | PutBucketNotificationConfiguration(*s3.PutBucketNotificationConfigurationInput) (*s3.PutBucketNotificationConfigurationOutput, error) 115 | 116 | PutBucketPolicy(*s3.PutBucketPolicyInput) (*s3.PutBucketPolicyOutput, error) 117 | 118 | PutBucketReplication(*s3.PutBucketReplicationInput) (*s3.PutBucketReplicationOutput, error) 119 | 120 | PutBucketRequestPayment(*s3.PutBucketRequestPaymentInput) (*s3.PutBucketRequestPaymentOutput, error) 121 | 122 | PutBucketTagging(*s3.PutBucketTaggingInput) (*s3.PutBucketTaggingOutput, error) 123 | 124 | PutBucketVersioning(*s3.PutBucketVersioningInput) (*s3.PutBucketVersioningOutput, error) 125 | 126 | PutBucketWebsite(*s3.PutBucketWebsiteInput) (*s3.PutBucketWebsiteOutput, error) 127 | 128 | PutBucketDecompressPolicy(*s3.PutBucketDecompressPolicyInput) (*s3.PutBucketDecompressPolicyOutput, error) 129 | 130 | PutBucketRetention(*s3.PutBucketRetentionInput) (*s3.PutBucketRetentionOutput, error) 131 | 132 | PutBucketInventory(*s3.PutBucketInventoryInput) (*s3.PutBucketInventoryOutput, error) 133 | 134 | PutObject(*s3.PutObjectInput) (*s3.PutObjectOutput, error) 135 | 136 | PutObjectACL(*s3.PutObjectACLInput) (*s3.PutObjectACLOutput, error) 137 | 138 | RestoreObject(*s3.RestoreObjectInput) (*s3.RestoreObjectOutput, error) 139 | 140 | RecoverObject(*s3.RecoverObjectInput) (*s3.RecoverObjectOutput, error) 141 | 142 | UploadPart(*s3.UploadPartInput) (*s3.UploadPartOutput, error) 143 | 144 | UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) 145 | } 146 | -------------------------------------------------------------------------------- /service/s3/s3manager/fileInfo.go: -------------------------------------------------------------------------------- 1 | package s3manager 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/user" 7 | "runtime" 8 | "strings" 9 | ) 10 | 11 | type fileInfoType struct { 12 | filePath string 13 | name string 14 | bucket string 15 | objectKey string 16 | size int64 17 | dir string 18 | acl string 19 | storageClass string 20 | } 21 | 22 | // CloudURL describes ks3 url 23 | type CloudURL struct { 24 | urlStr string 25 | bucket string 26 | object string 27 | } 28 | 29 | // StorageURLer is the interface for all url 30 | type StorageURLer interface { 31 | IsCloudURL() bool 32 | IsFileURL() bool 33 | ToString() string 34 | } 35 | 36 | // FileURL describes file url 37 | type FileURL struct { 38 | urlStr string 39 | } 40 | 41 | // Init simulate inheritance, and polymorphism 42 | func (fu *FileURL) Init(urlStr string) error { 43 | 44 | if len(urlStr) >= 2 && urlStr[:2] == "~"+string(os.PathSeparator) { 45 | homeDir := currentHomeDir() 46 | if homeDir != "" { 47 | urlStr = strings.Replace(urlStr, "~", homeDir, 1) 48 | } else { 49 | return fmt.Errorf("current home dir is empty") 50 | } 51 | } 52 | fu.urlStr = urlStr 53 | return nil 54 | } 55 | 56 | // IsCloudURL simulate inheritance, and polymorphism 57 | func (fu *FileURL) IsCloudURL() bool { 58 | return false 59 | } 60 | 61 | // IsFileURL simulate inheritance, and polymorphism 62 | func (fu *FileURL) IsFileURL() bool { 63 | return true 64 | } 65 | 66 | // ToString simulate inheritance, and polymorphism 67 | func (fu *FileURL) ToString() string { 68 | return fu.urlStr 69 | } 70 | 71 | // StorageURLFromString analysis input url type and build a storage url from the url 72 | func StorageURLFromString(urlStr string) (StorageURLer, error) { 73 | var fileURL *FileURL 74 | if err := fileURL.Init(urlStr); err != nil { 75 | return nil, err 76 | } 77 | return fileURL, nil 78 | } 79 | func currentHomeDir() string { 80 | homeDir := "" 81 | homeDrive := os.Getenv("HOMEDRIVE") 82 | homePath := os.Getenv("HOMEPATH") 83 | if runtime.GOOS == "windows" && homeDrive != "" && homePath != "" { 84 | homeDir = homeDrive + string(os.PathSeparator) + homePath 85 | } 86 | 87 | if homeDir != "" { 88 | return homeDir 89 | } 90 | 91 | usr, _ := user.Current() 92 | if usr != nil { 93 | homeDir = usr.HomeDir 94 | } else { 95 | homeDir = os.Getenv("HOME") 96 | } 97 | return homeDir 98 | } 99 | -------------------------------------------------------------------------------- /service/s3/s3manager/file_counter.go: -------------------------------------------------------------------------------- 1 | package s3manager 2 | 3 | import "sync/atomic" 4 | 5 | type FileCounter struct { 6 | TotalNum int64 7 | SuccessNum int64 8 | FailNum int64 9 | } 10 | 11 | func (fc *FileCounter) addTotalNum(num int64) { 12 | atomic.AddInt64(&fc.TotalNum, num) 13 | } 14 | 15 | func (fc *FileCounter) addSuccessNum(num int64) { 16 | atomic.AddInt64(&fc.SuccessNum, num) 17 | } 18 | 19 | func (fc *FileCounter) addFailNum(num int64) { 20 | atomic.AddInt64(&fc.FailNum, num) 21 | } 22 | -------------------------------------------------------------------------------- /service/s3/service.go: -------------------------------------------------------------------------------- 1 | // THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT. 2 | 3 | package s3 4 | 5 | import ( 6 | "github.com/ks3sdklib/aws-sdk-go/aws" 7 | "github.com/ks3sdklib/aws-sdk-go/internal/protocol/body" 8 | "github.com/ks3sdklib/aws-sdk-go/internal/signer/v2" 9 | "github.com/ks3sdklib/aws-sdk-go/internal/signer/v4" 10 | "strings" 11 | ) 12 | 13 | // S3 is a client for Amazon S3. 14 | type S3 struct { 15 | *aws.Service 16 | } 17 | 18 | // Used for custom service initialization logic 19 | var initService func(*aws.Service) 20 | 21 | // Used for custom request initialization logic 22 | var initRequest func(*aws.Request) 23 | 24 | // New returns a new S3 client. 25 | func New(config *aws.Config) *S3 { 26 | service := &aws.Service{ 27 | Config: aws.DefaultConfig.Merge(config), 28 | ServiceName: "s3", 29 | APIVersion: "2006-03-01", 30 | } 31 | service.Initialize() 32 | 33 | // Handlers 34 | if service.Config.SignerVersion == "V4" || service.Config.SignerVersion == "V4_UNSIGNED_PAYLOAD_SIGNER" { 35 | service.Handlers.Sign.PushBack(v4.Sign) 36 | } else { 37 | service.Handlers.Sign.PushBack(v2.Sign) 38 | } 39 | 40 | service.Handlers.Build.PushBack(body.Build) 41 | service.Handlers.Unmarshal.PushBack(body.UnmarshalBody) 42 | service.Handlers.UnmarshalMeta.PushBack(body.UnmarshalMeta) 43 | service.Handlers.UnmarshalError.PushBack(body.UnmarshalError) 44 | 45 | // Run custom service initialization if present 46 | if initService != nil { 47 | initService(service) 48 | } 49 | 50 | return &S3{service} 51 | } 52 | 53 | // newRequest creates a new request for a S3 operation and runs any 54 | // custom request initialization. 55 | func (c *S3) newRequest(op *aws.Operation, params, data interface{}) *aws.Request { 56 | r := aws.NewRequest(c.Service, op, params, data) 57 | if r.Config.DomainMode { 58 | r.HTTPRequest.URL.Path = strings.Replace(r.HTTPRequest.URL.Path, "/{Bucket}", "", -1) 59 | if r.HTTPRequest.URL.Path == "" { 60 | r.HTTPRequest.URL.Path = "/" 61 | } 62 | } else { 63 | updateHostWithBucket(r) 64 | } 65 | // Run custom request initialization if present 66 | if initRequest != nil { 67 | initRequest(r) 68 | } 69 | return r 70 | } 71 | -------------------------------------------------------------------------------- /service/s3/try.go: -------------------------------------------------------------------------------- 1 | package s3 2 | import "errors" 3 | 4 | // MaxRetries is the maximum number of retries before bailing. 5 | var MaxRetries = 10 6 | 7 | var errMaxRetriesReached = errors.New("exceeded retry limit") 8 | 9 | // Func represents functions that can be retried. 10 | type Func func(attempt int) (retry bool, err error) 11 | 12 | // Do keeps trying the function until the second argument 13 | // returns false, or no error is returned. 14 | func Do(fn Func) error { 15 | var err error 16 | var cont bool 17 | attempt := 1 18 | for { 19 | cont, err = fn(attempt) 20 | if !cont || err == nil { 21 | break 22 | } 23 | attempt++ 24 | if attempt > MaxRetries { 25 | return errMaxRetriesReached 26 | } 27 | } 28 | return err 29 | } 30 | 31 | // IsMaxRetries checks whether the error is due to hitting the 32 | // maximum number of retries or not. 33 | func IsMaxRetries(err error) bool { 34 | return err == errMaxRetriesReached 35 | } 36 | -------------------------------------------------------------------------------- /service/s3/util.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/base64" 6 | "io" 7 | "net/url" 8 | "os" 9 | ) 10 | 11 | // GetBase64MD5Str 计算Base64格式字符串的MD5值 12 | func GetBase64MD5Str(str string) string { 13 | // 创建一个MD5哈希对象 14 | hash := md5.New() 15 | 16 | // 将字符串转换为字节数组并计算MD5哈希值 17 | hash.Write([]byte(str)) 18 | md5Hash := hash.Sum(nil) 19 | 20 | // 将MD5哈希值转换为Base64格式 21 | base64Str := base64.StdEncoding.EncodeToString(md5Hash) 22 | 23 | return base64Str 24 | } 25 | 26 | // GetBase64Str 计算Base64格式字符串 27 | func GetBase64Str(str string) string { 28 | return base64.StdEncoding.EncodeToString([]byte(str)) 29 | } 30 | 31 | // GetBase64FileMD5Str 计算Base64格式文件的MD5值 32 | func GetBase64FileMD5Str(filePath string) (string, error) { 33 | file, err := os.Open(filePath) 34 | if err != nil { 35 | return "", err 36 | } 37 | defer file.Close() 38 | 39 | hash := md5.New() 40 | if _, err := io.Copy(hash, file); err != nil { 41 | return "", err 42 | } 43 | 44 | md5Hash := hash.Sum(nil) 45 | 46 | // 将MD5哈希值转换为Base64格式 47 | base64Str := base64.StdEncoding.EncodeToString(md5Hash) 48 | 49 | return base64Str, err 50 | } 51 | 52 | // BuildCopySource 构建拷贝源 53 | func BuildCopySource(bucket *string, key *string) string { 54 | if bucket == nil || key == nil { 55 | return "" 56 | } 57 | return "/" + *bucket + "/" + url.QueryEscape(*key) 58 | } 59 | 60 | // GetCannedACL 获取访问控制权限 61 | func GetCannedACL(Grants []*Grant) string { 62 | allUsersPermissions := map[string]*string{} 63 | for _, value := range Grants { 64 | if value.Grantee.URI != nil && *value.Grantee.URI == AllUsersUri { 65 | allUsersPermissions[*value.Permission] = value.Permission 66 | } 67 | } 68 | _, read := allUsersPermissions["READ"] 69 | _, write := allUsersPermissions["WRITE"] 70 | if read && write { 71 | return ACLPublicReadWrite 72 | } else if read { 73 | return ACLPublicRead 74 | } else { 75 | return ACLPrivate 76 | } 77 | } 78 | 79 | // FileExists returns whether the given file exists or not 80 | func FileExists(filename string) bool { 81 | info, err := os.Stat(filename) 82 | if os.IsNotExist(err) { 83 | return false 84 | } 85 | return !info.IsDir() 86 | } 87 | 88 | // DirExists returns whether the given directory exists or not 89 | func DirExists(dir string) bool { 90 | info, err := os.Stat(dir) 91 | if os.IsNotExist(err) { 92 | return false 93 | } 94 | return info.IsDir() 95 | } 96 | 97 | // Min returns the smaller of two integers 98 | func Min(a, b int64) int64 { 99 | if a < b { 100 | return a 101 | } 102 | return b 103 | } 104 | --------------------------------------------------------------------------------