├── .github └── workflows │ └── ci.yml ├── .gitignore ├── LICENSE ├── README.md ├── config.go ├── config_test.go ├── constants.go ├── cookie.go ├── cookie_test.go ├── default.go ├── delete.go ├── delete_test.go ├── download.go ├── download_test.go ├── execute.go ├── fetch.go ├── fetch_test.go ├── get.go ├── get_test.go ├── global.go ├── global_test.go ├── go.mod ├── go.sum ├── head.go ├── head_test.go ├── methods.go ├── methods_test.go ├── patch.go ├── patch_test.go ├── post.go ├── post_test.go ├── progress.go ├── put.go ├── put_test.go ├── response.go ├── response_test.go ├── session.go ├── session_test.go ├── stream.go ├── stream_test.go ├── tag.go ├── tag_test.go ├── upload.go ├── upload_test.go └── version.go /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | tags: 7 | - v* 8 | pull_request: 9 | branches: [ master ] 10 | 11 | jobs: 12 | 13 | ci: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v2 17 | 18 | - name: Set up Go 19 | uses: zmicro-design/action-setup-go@v1 20 | with: 21 | go-version: v1.24.4 22 | 23 | - name: install deps 24 | run: | 25 | go mod tidy 26 | go install golang.org/x/tools/cmd/goimports@latest 27 | go install golang.org/x/lint/golint@latest 28 | go install github.com/mattn/goveralls@latest 29 | - name: static analysis 30 | run: | 31 | golint -set_exit_status 32 | # go vet 33 | # test -z "$(goimports -l .)" 34 | - name: Test 35 | run: goveralls -service=github 36 | env: 37 | COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | 3 | # If you prefer the allow list template instead of the deny list, see community template: 4 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 5 | # 6 | # Binaries for programs and plugins 7 | *.exe 8 | *.exe~ 9 | *.dll 10 | *.so 11 | *.dylib 12 | 13 | # Test binary, built with `go test -c` 14 | *.test 15 | 16 | # Output of the go coverage tool, specifically when used with LiteIDE 17 | *.out 18 | 19 | # Dependency directories (remove the comment below to include it) 20 | # vendor/ 21 | 22 | # Go workspace file 23 | go.work 24 | 25 | # Common 26 | .env -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 GoZooX 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Fetch - HTTP Client 2 | 3 | `HTTP Client` for Go, inspired by the [Fetch API](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API), and [Axios](https://github.com/axios/axios) + [Got](https://github.com/sindresorhus/got) (Sindre Sorhus). 4 | 5 | [![PkgGoDev](https://pkg.go.dev/badge/github.com/go-zoox/fetch)](https://pkg.go.dev/github.com/go-zoox/fetch) 6 | [![Build Status](https://github.com/go-zoox/fetch/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/go-zoox/fetch/actions/workflows/ci.yml) 7 | [![Go Report Card](https://goreportcard.com/badge/github.com/go-zoox/fetch)](https://goreportcard.com/report/github.com/go-zoox/fetch) 8 | [![Coverage Status](https://coveralls.io/repos/github/go-zoox/fetch/badge.svg?branch=master)](https://coveralls.io/github/go-zoox/fetch?branch=master) 9 | [![GitHub issues](https://img.shields.io/github/issues/go-zoox/fetch.svg)](https://github.com/go-zoox/fetch/issues) 10 | [![Release](https://img.shields.io/github/tag/go-zoox/fetch.svg?label=Release)](https://github.com/go-zoox/fetch/releases) 11 | 12 | ## Features 13 | ### Main API 14 | - [x] Make HTTP requests 15 | - [x] Easy JSON Response 16 | - [ ] GZip support 17 | - [x] Decode GZip response 18 | - [ ] Encode GZip request (Upload File with GZip) 19 | - [x] HTTP/2 support 20 | - [x] TLS 21 | - [x] Custom TLS Ca Certificate (Self signed certificate) [Example](https://github.com/go-zoox/examples/tree/master/https/fetch) 22 | - [x] Custom Client Cert and Key for two-way authentication (Client Cert and Key) 23 | - [x] Simple Auth Methods 24 | - [x] Basic Auth 25 | - [x] Bearer Auth 26 | - [x] Support cancel (using context) 27 | 28 | ### Timeouts and retries 29 | - [x] Support timeout 30 | - [x] Support retry on failure 31 | 32 | ### Progress 33 | - [x] Support progress and progress events 34 | 35 | ### File upload and download 36 | - [x] Download files easily 37 | - [x] Upload files easily 38 | 39 | ### Cache, Proxy and UNIX sockets 40 | - [ ] [RFC compliant caching](https://github.com/sindresorhus/got/blob/main/documentation/cache.md) 41 | - [x] Proxy support 42 | - [x] Environment variables (HTTP_PROXY/HTTPS_PROXY/SOCKS_PROXY) 43 | - [x] Custom proxy 44 | - [x] UNIX Domain Sockets 45 | - [Example: HTTP](https://github.com/go-zoox/examples/tree/master/unix-domain-socket/http) 46 | - [Example: HTTPs](https://github.com/go-zoox/examples/tree/master/unix-domain-socket/https) 47 | 48 | ### WebDAV 49 | - [ ] WebDAV protocol support 50 | 51 | ### Advanced creation 52 | - [ ] Plugin system 53 | - [ ] Middleware system 54 | 55 | ## Installation 56 | 57 | To install the package, run: 58 | 59 | ```bash 60 | go get github.com/go-zoox/fetch 61 | ``` 62 | 63 | ## Methods 64 | - [x] GET 65 | - [x] POST 66 | - [x] PUT 67 | - [x] PATCH 68 | - [x] DELETE 69 | - [x] HEAD 70 | - [ ] OPTIONS 71 | - [ ] TRACE 72 | - [ ] CONNECT 73 | 74 | ## Getting Started 75 | 76 | ```go 77 | package main 78 | 79 | import ( 80 | "github.com/go-zoox/fetch" 81 | ) 82 | 83 | func main() { 84 | response, _ := fetch.Get("https://httpbin.zcorky.com/get") 85 | url := response.Get("url") 86 | method := response.Get("method") 87 | 88 | fmt.Println(url, method) 89 | } 90 | ``` 91 | 92 | ## Examples 93 | 94 | ### Get 95 | 96 | ```go 97 | package main 98 | 99 | import ( 100 | "github.com/go-zoox/fetch" 101 | ) 102 | 103 | func main() { 104 | response, err := fetch.Get("https://httpbin.zcorky.com/get") 105 | if err != nil { 106 | panic(err) 107 | } 108 | 109 | fmt.Println(response.JSON()) 110 | } 111 | ``` 112 | 113 | ### Post 114 | 115 | ```go 116 | package main 117 | 118 | import ( 119 | "github.com/go-zoox/fetch" 120 | ) 121 | 122 | func main() { 123 | response, err := fetch.Post("https://httpbin.zcorky.com/post", &fetch.Config{ 124 | Body: map[string]interface{}{ 125 | "foo": "bar", 126 | "foo2": "bar2", 127 | "number": 1, 128 | "boolean": true, 129 | "array": []string{ 130 | "foo3", 131 | "bar3", 132 | }, 133 | "nest": map[string]string{ 134 | "foo4": "bar4", 135 | }, 136 | }, 137 | }) 138 | if err != nil { 139 | panic(err) 140 | } 141 | 142 | fmt.Println(response.JSON()) 143 | } 144 | ``` 145 | 146 | ### Put 147 | 148 | ```go 149 | package main 150 | 151 | import ( 152 | "github.com/go-zoox/fetch" 153 | ) 154 | 155 | func main() { 156 | response, err := fetch.Put("https://httpbin.zcorky.com/put", &fetch.Config{ 157 | Body: map[string]interface{}{ 158 | "foo": "bar", 159 | "foo2": "bar2", 160 | "number": 1, 161 | "boolean": true, 162 | "array": []string{ 163 | "foo3", 164 | "bar3", 165 | }, 166 | "nest": map[string]string{ 167 | "foo4": "bar4", 168 | }, 169 | }, 170 | }) 171 | if err != nil { 172 | panic(err) 173 | } 174 | 175 | 176 | fmt.Println(response.JSON()) 177 | } 178 | ``` 179 | ### Delete 180 | 181 | ```go 182 | package main 183 | 184 | import ( 185 | "github.com/go-zoox/fetch" 186 | ) 187 | 188 | func main() { 189 | response, err := fetch.Delete("https://httpbin.zcorky.com/Delete", &fetch.Config{ 190 | Body: map[string]interface{}{ 191 | "foo": "bar", 192 | "foo2": "bar2", 193 | "number": 1, 194 | "boolean": true, 195 | "array": []string{ 196 | "foo3", 197 | "bar3", 198 | }, 199 | "nest": map[string]string{ 200 | "foo4": "bar4", 201 | }, 202 | }, 203 | }) 204 | if err != nil { 205 | panic(err) 206 | } 207 | 208 | fmt.Println(response.JSON()) 209 | } 210 | ``` 211 | 212 | ### Timeout 213 | 214 | ```go 215 | package main 216 | 217 | import ( 218 | "github.com/go-zoox/fetch" 219 | ) 220 | 221 | func main() { 222 | response, err := fetch.Get("https://httpbin.zcorky.com/get", &fetch.Config{ 223 | Timeout: 5 * time.Second, 224 | }) 225 | if err != nil { 226 | panic(err) 227 | } 228 | 229 | fmt.Println(response.JSON()) 230 | } 231 | ``` 232 | 233 | ### Proxy 234 | 235 | ```go 236 | package main 237 | 238 | import ( 239 | "github.com/go-zoox/fetch" 240 | ) 241 | 242 | func main() { 243 | response, err := fetch.Get("https://httpbin.zcorky.com/ip", &fetch.Config{ 244 | Proxy: "http://127.0.0.1:17890", 245 | }) 246 | if err != nil { 247 | panic(err) 248 | } 249 | 250 | fmt.Println(response.JSON()) 251 | } 252 | ``` 253 | 254 | ### Basic Auth 255 | 256 | ```go 257 | package main 258 | 259 | import ( 260 | "github.com/go-zoox/fetch" 261 | ) 262 | 263 | func main() { 264 | response, err := fetch.Get("https://httpbin.zcorky.com/ip", &fetch.Config{ 265 | BasicAuth: &fetch.BasicAuth{ 266 | Username: "foo", 267 | Password: "bar", 268 | }, 269 | }) 270 | if err != nil { 271 | panic(err) 272 | } 273 | 274 | fmt.Println(response.JSON()) 275 | } 276 | ``` 277 | 278 | ### Download 279 | 280 | ```go 281 | package main 282 | 283 | import ( 284 | "github.com/go-zoox/fetch" 285 | ) 286 | 287 | func main() { 288 | response, err := fetch.Download("https://httpbin.zcorky.com/image", "/tmp/image.webp") 289 | if err != nil { 290 | panic(err) 291 | } 292 | } 293 | ``` 294 | 295 | ### Upload 296 | 297 | ```go 298 | package main 299 | 300 | import ( 301 | "github.com/go-zoox/fetch" 302 | ) 303 | 304 | func main() { 305 | file, _ := os.Open("go.mod") 306 | 307 | response, err := Upload("https://httpbin.zcorky.com/upload", file) 308 | if err != nil { 309 | panic(err) 310 | } 311 | 312 | fmt.Println(response.JSON()) 313 | } 314 | ``` 315 | 316 | ### Cancel 317 | 318 | ```go 319 | package main 320 | 321 | import ( 322 | "github.com/go-zoox/fetch" 323 | ) 324 | 325 | func main() { 326 | file, _ := os.Open("go.mod") 327 | 328 | ctx, cancel := context.WithCancel(context.Background()) 329 | 330 | f := fetch.New() 331 | f.SetBaseURL("https://httpbin.zcorky.com") 332 | f.SetURL("/delay/3") 333 | f.SetContext(ctx) 334 | 335 | go func() { 336 | _, err := f.Execute() 337 | fmt.Println(err) 338 | }() 339 | 340 | cancel() 341 | } 342 | ``` 343 | 344 | ## Depencencies 345 | 346 | - [gjson](github.com/tidwall/gjson) - Get JSON Whenever You Need, you don't 347 | define type first。 348 | 349 | ## Inspired By 350 | 351 | - [sindresorhus/got](https://github.com/sindresorhus/got) - 🌐 Human-friendly and powerful HTTP request library for Node.js 352 | - [axios/axios](https://github.com/axios/axios) - Promise based HTTP client for the browser and node.js 353 | - [mozillazg/request](https://github.com/mozillazg/request) - A 354 | developer-friendly HTTP request library for Gopher 355 | - [monaco-io/request](https://github.com/monaco-io/request) - go request, go http client 356 | 357 | ## License 358 | 359 | GoZoox is released under the [MIT License](./LICENSE). 360 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "time" 7 | ) 8 | 9 | // Config is the configuration for the fetch 10 | type Config struct { 11 | URL string 12 | Method string 13 | Headers Headers 14 | Query Query 15 | Params Params 16 | Body Body 17 | // 18 | BaseURL string 19 | Timeout time.Duration 20 | // 21 | DownloadFilePath string 22 | // 23 | Proxy string 24 | // 25 | IsStream bool 26 | // 27 | IsSession bool 28 | // 29 | HTTP2 bool 30 | 31 | // TLS Ca Cert 32 | TLSCaCert []byte 33 | TLSCaCertFile string 34 | // TLS Cert 35 | TLSCert []byte 36 | TLSCertFile string 37 | // TLS Client Private Key 38 | TLSKey []byte 39 | TLSKeyFile string 40 | 41 | // TLSInsecureSkipVerify means ignore verify tls certificate 42 | // use carefully, becuase it may cause security problems, 43 | // which means maybe server certificate has been hacked 44 | TLSInsecureSkipVerify bool 45 | // UnixDomainSocket socket like /var/run/docker.sock 46 | UnixDomainSocket string 47 | // 48 | Context context.Context 49 | // 50 | OnProgress OnProgress 51 | // 52 | BasicAuth BasicAuth 53 | // 54 | Username string 55 | Password string 56 | } 57 | 58 | // BasicAuth is the basic auth 59 | type BasicAuth struct { 60 | Username string 61 | Password string 62 | } 63 | 64 | // OnProgress is the progress callback 65 | type OnProgress func(percent int64, current, total int64) 66 | 67 | // MarshalJSON returns the json string 68 | func (op *OnProgress) MarshalJSON() ([]byte, error) { 69 | return []byte("null"), nil 70 | } 71 | 72 | // UnmarshalJSON unmarshals the json string 73 | func (op *OnProgress) UnmarshalJSON(data []byte) error { 74 | return nil 75 | } 76 | 77 | // Merge merges the config with the given config 78 | func (c *Config) Merge(config *Config) { 79 | if config == nil { 80 | return 81 | } 82 | 83 | if config.URL != "" { 84 | c.URL = config.URL 85 | } 86 | 87 | if config.Method != "" { 88 | c.Method = config.Method 89 | } 90 | 91 | if config.Headers != nil { 92 | if c.Headers == nil { 93 | c.Headers = make(Headers) 94 | } 95 | 96 | for header := range config.Headers { 97 | if _, ok := c.Headers[header]; !ok { 98 | // fmt.Printf("%s origin(%s) => new(%s)", header, cfg.Headers[header], config.Headers[header]) 99 | c.Headers[header] = config.Headers[header] 100 | } 101 | } 102 | } 103 | 104 | if config.Query != nil { 105 | if c.Query == nil { 106 | c.Query = make(Query) 107 | } 108 | 109 | for query := range config.Query { 110 | if _, ok := c.Query[query]; !ok { 111 | c.Query[query] = config.Query[query] 112 | } 113 | } 114 | } 115 | 116 | if config.Params != nil { 117 | if c.Params == nil { 118 | c.Params = make(Params) 119 | } 120 | 121 | for param := range config.Params { 122 | if _, ok := c.Params[param]; !ok { 123 | c.Params[param] = config.Params[param] 124 | } 125 | } 126 | } 127 | 128 | if config.Body != nil { 129 | c.Body = config.Body 130 | } 131 | 132 | if config.BaseURL != "" { 133 | c.BaseURL = config.BaseURL 134 | } 135 | 136 | if config.Timeout != 0 { 137 | c.Timeout = config.Timeout 138 | } 139 | 140 | if config.DownloadFilePath != "" { 141 | c.DownloadFilePath = config.DownloadFilePath 142 | } 143 | 144 | if config.Proxy != "" { 145 | c.Proxy = config.Proxy 146 | } 147 | 148 | if config.TLSCaCert != nil { 149 | c.TLSCaCert = config.TLSCaCert 150 | } 151 | 152 | if config.TLSCaCertFile != "" { 153 | c.TLSCaCertFile = config.TLSCaCertFile 154 | } 155 | 156 | if config.TLSCert != nil { 157 | c.TLSCert = config.TLSCert 158 | } 159 | 160 | if config.TLSCertFile != "" { 161 | c.TLSCertFile = config.TLSCertFile 162 | } 163 | 164 | if config.TLSKey != nil { 165 | c.TLSKey = config.TLSKey 166 | } 167 | 168 | if config.TLSKeyFile != "" { 169 | c.TLSKeyFile = config.TLSKeyFile 170 | } 171 | 172 | if config.TLSInsecureSkipVerify { 173 | c.TLSInsecureSkipVerify = config.TLSInsecureSkipVerify 174 | } 175 | 176 | if config.UnixDomainSocket != "" { 177 | c.UnixDomainSocket = config.UnixDomainSocket 178 | } 179 | 180 | if config.IsStream { 181 | c.IsStream = config.IsStream 182 | } 183 | 184 | if config.BasicAuth.Username != "" || config.BasicAuth.Password != "" { 185 | c.BasicAuth = config.BasicAuth 186 | } 187 | 188 | if config.Username != "" || config.Password != "" { 189 | c.Username = config.Username 190 | c.Password = config.Password 191 | } 192 | 193 | if config.Context != nil { 194 | c.Context = config.Context 195 | } 196 | } 197 | 198 | // Clone returns a clone of the config 199 | func (c *Config) Clone() *Config { 200 | nc := DefaultConfig() 201 | nc.Merge(c) 202 | return c 203 | } 204 | 205 | // Body is the body of the request 206 | type Body interface{} 207 | 208 | // Headers is the headers of the request 209 | type Headers map[string]string 210 | 211 | // Get returns the value of the given key 212 | func (h Headers) Get(key string) string { 213 | for k, v := range h { 214 | if strings.EqualFold(k, key) { 215 | return v 216 | } 217 | } 218 | 219 | return "" 220 | } 221 | 222 | // Set sets the value of the given key 223 | func (h Headers) Set(key, value string) { 224 | h[strings.ToLower(key)] = value 225 | } 226 | 227 | // Query is the query of the request 228 | type Query map[string]string 229 | 230 | // Get returns the value of the given key 231 | func (q Query) Get(key string) string { 232 | return q[key] 233 | } 234 | 235 | // Set sets the value of the given key 236 | func (q Query) Set(key, value string) { 237 | q[key] = value 238 | } 239 | 240 | // Params is the params of the request 241 | type Params map[string]string 242 | 243 | // Get returns the value of the given key 244 | func (p Params) Get(key string) string { 245 | return p[key] 246 | } 247 | 248 | // Set sets the value of the given key 249 | func (p Params) Set(key, value string) { 250 | p[key] = value 251 | } 252 | -------------------------------------------------------------------------------- /config_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/go-zoox/testify" 7 | ) 8 | 9 | func TestConfigMerge(t *testing.T) { 10 | // nil 11 | cfg := Config{} 12 | cfg.Merge(nil) 13 | 14 | // merge params 15 | testify.Equal(t, "", cfg.Headers.Get("key")) 16 | cfg.Merge(&Config{ 17 | Params: Params{ 18 | "key": "value", 19 | }, 20 | }) 21 | testify.Equal(t, "value", cfg.Params.Get("key")) 22 | 23 | // merge query 24 | testify.Equal(t, "", cfg.Query.Get("key")) 25 | cfg.Merge(&Config{ 26 | Query: Query{ 27 | "key": "value", 28 | }, 29 | }) 30 | testify.Equal(t, "value", cfg.Query.Get("key")) 31 | 32 | // merge headers 33 | testify.Equal(t, "", cfg.Headers.Get("key")) 34 | cfg.Merge(&Config{ 35 | Headers: Headers{ 36 | "key": "value", 37 | }, 38 | }) 39 | testify.Equal(t, "value", cfg.Headers.Get("key")) 40 | 41 | // merge base url 42 | testify.Equal(t, "", cfg.BaseURL) 43 | cfg.Merge(&Config{ 44 | BaseURL: "http://example.com", 45 | }) 46 | testify.Equal(t, "http://example.com", cfg.BaseURL) 47 | 48 | // merge timeout 49 | testify.Equal(t, 0, cfg.Timeout) 50 | cfg.Merge(&Config{ 51 | Timeout: 1, 52 | }) 53 | testify.Equal(t, 1, cfg.Timeout) 54 | 55 | // merge download file path 56 | testify.Equal(t, "", cfg.DownloadFilePath) 57 | cfg.Merge(&Config{ 58 | DownloadFilePath: "path", 59 | }) 60 | testify.Equal(t, "path", cfg.DownloadFilePath) 61 | 62 | // merge proxy 63 | testify.Equal(t, "", cfg.Proxy) 64 | cfg.Merge(&Config{ 65 | Proxy: "http://example.com", 66 | }) 67 | testify.Equal(t, "http://example.com", cfg.Proxy) 68 | 69 | // merge is stream 70 | testify.Equal(t, false, cfg.IsStream) 71 | cfg.Merge(&Config{ 72 | IsStream: true, 73 | }) 74 | testify.Equal(t, true, cfg.IsStream) 75 | } 76 | 77 | func TestConfigHeaders(t *testing.T) { 78 | headers := Headers{} 79 | testify.Equal(t, headers.Get("key"), "", "Expected empty string") 80 | 81 | headers.Set("key", "value") 82 | testify.Equal(t, headers.Get("key"), "value", "Expected value") 83 | } 84 | 85 | func TestConfigQuery(t *testing.T) { 86 | query := Query{} 87 | testify.Equal(t, query.Get("key"), "", "Expected empty string") 88 | 89 | query.Set("key", "value") 90 | testify.Equal(t, query.Get("key"), "value", "Expected value") 91 | } 92 | 93 | func TestConfigParams(t *testing.T) { 94 | params := Params{} 95 | testify.Equal(t, params.Get("key"), "", "Expected empty string") 96 | 97 | params.Set("key", "value") 98 | testify.Equal(t, params.Get("key"), "value", "Expected value") 99 | } 100 | -------------------------------------------------------------------------------- /constants.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import "errors" 4 | 5 | // HEAD is request method HEAD 6 | const HEAD = "HEAD" 7 | 8 | // GET is request method GET 9 | const GET = "GET" 10 | 11 | // POST is request method POST 12 | const POST = "POST" 13 | 14 | // PUT is request method PUT 15 | const PUT = "PUT" 16 | 17 | // DELETE is request method DELETE 18 | const DELETE = "DELETE" 19 | 20 | // PATCH is request method PATCH 21 | const PATCH = "PATCH" 22 | 23 | // METHODS is the list of supported methods 24 | var METHODS = []string{ 25 | HEAD, 26 | GET, 27 | POST, 28 | PUT, 29 | DELETE, 30 | PATCH, 31 | } 32 | 33 | // // headers.ContentType is the content type header name 34 | // const headers.ContentType = "Content-Type" 35 | 36 | // // headers.Accept is the accept header name 37 | // const headers.Accept = "Accept" 38 | 39 | // // headers.Referrer is the referrer header name 40 | // const headers.Referrer = "Referer" 41 | 42 | // // headers.UserAgent ... 43 | // const headers.UserAgent = "User-Agent" 44 | 45 | // // headers.Authorization ... 46 | // const headers.Authorization = "Authorization" 47 | 48 | // // headers.CacheControl ... 49 | // const headers.CacheControl = "Cache-Control" 50 | 51 | // // headers.AcceptEncoding ... 52 | // const headers.AcceptEncoding = "Accept-Encoding" 53 | 54 | // // headers.AcceptLanguage ... 55 | // const headers.AcceptLanguage = "Accept-Language" 56 | 57 | // // headers.Cookie ... 58 | // const headers.Cookie = "Cookie" 59 | 60 | // // headers.Location ... 61 | // const headers.Location = "Location" 62 | 63 | // // headers.ContentLength ... 64 | // const headers.ContentLength = "Content-Length" 65 | 66 | // // headers.ContentEncoding ... 67 | // const headers.ContentEncoding = "Content-Encoding" 68 | 69 | // // headers.TransferEncoding ... 70 | // const headers.TransferEncoding = "Transfer-Encoding" 71 | 72 | // // headers.ContentLanguage ... 73 | // const headers.ContentLanguage = "Content-Language" 74 | 75 | // // headers.SetCookie ... 76 | // const headers.SetCookie = "Set-Cookie" 77 | 78 | // // headers.XPoweredBy ... 79 | // const headers.XPoweredBy = "X-Powered-By" 80 | 81 | // // headers.XRequestID ... 82 | // const headers.XRequestID = "X-Request-ID" 83 | 84 | // // headers.AcceptRanges ... 85 | // const headers.AcceptRanges = "Accept-Ranges" 86 | 87 | // EnvDEBUG is the DEBUG env name 88 | const EnvDEBUG = "GO_ZOOX_FETCH_DEBUG" 89 | 90 | // ErrTooManyArguments is the error when the number of arguments is too many 91 | var ErrTooManyArguments = errors.New("too many arguments") 92 | 93 | // ErrInvalidMethod is the error when the method is invalid 94 | var ErrInvalidMethod = errors.New("invalid method") 95 | 96 | // ErrCannotCreateRequest is the error when the request cannot be created 97 | var ErrCannotCreateRequest = errors.New("cannot create request") 98 | 99 | // ErrCannotSendBodyWithGet is the error when the body cannot be sent with GET method 100 | var ErrCannotSendBodyWithGet = errors.New("cannot send body with GET method") 101 | 102 | // ErrInvalidJSONBody is the error when the body is not a valid JSON 103 | var ErrInvalidJSONBody = errors.New("error marshalling body") 104 | 105 | // ErrSendingRequest is the error when the request cannot be sent 106 | var ErrSendingRequest = errors.New("error sending request") 107 | 108 | // ErrReadingResponse is the error when the response cannot be read 109 | var ErrReadingResponse = errors.New("error reading response") 110 | 111 | // ErrInvalidContentType is the error when the content type is invalid 112 | var ErrInvalidContentType = errors.New("invalid content type") 113 | 114 | // ErrorInvalidBody is the error when the body is invalid 115 | var ErrorInvalidBody = errors.New("invalid body") 116 | 117 | // ErrInvalidBodyMultipart is the error when the body is invalid for multipart 118 | var ErrInvalidBodyMultipart = errors.New("invalid body multipart") 119 | 120 | // ErrCannotCreateFormFile is the error when the form file cannot be created 121 | var ErrCannotCreateFormFile = errors.New("cannot create form file") 122 | 123 | // ErrCannotCopyFile is the error when the file cannot be copied 124 | var ErrCannotCopyFile = errors.New("cannot copy file") 125 | 126 | // ErrInvalidURLFormEncodedBody is the error when the body is invalid for url form encoded 127 | var ErrInvalidURLFormEncodedBody = errors.New("invalid url form encoded body") 128 | 129 | // ErrCookieEmptyKey is the error when the key is empty 130 | var ErrCookieEmptyKey = errors.New("empty key") 131 | -------------------------------------------------------------------------------- /cookie.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import "strings" 4 | 5 | type cookie struct { 6 | data []cookieItem 7 | } 8 | 9 | type cookieItem struct { 10 | Key string 11 | Value string 12 | } 13 | 14 | func newCookie(origin ...string) *cookie { 15 | c := &cookie{} 16 | 17 | if len(origin) > 0 { 18 | if origin[0] != "" { 19 | c.Parse(origin[0]) 20 | } 21 | } 22 | 23 | return c 24 | } 25 | 26 | func (c *cookie) Parse(str string) (err error) { 27 | str = strings.TrimSpace(str) 28 | str = strings.TrimSuffix(str, ";") 29 | str = strings.TrimSpace(str) 30 | 31 | items := strings.Split(str, ";") 32 | 33 | for _, item := range items { 34 | item = strings.TrimSpace(item) 35 | if item == "" { 36 | continue 37 | } 38 | 39 | kv := strings.Split(item, "=") 40 | if len(kv) != 2 { 41 | continue 42 | } 43 | 44 | if err := c.Add(kv[0], kv[1]); err != nil { 45 | return err 46 | } 47 | } 48 | 49 | return 50 | } 51 | 52 | func (c *cookie) Add(key, value string) (err error) { 53 | if key == "" { 54 | return ErrCookieEmptyKey 55 | } 56 | 57 | c.data = append(c.data, cookieItem{key, value}) 58 | return nil 59 | } 60 | 61 | func (c *cookie) Get(key string) string { 62 | for _, item := range c.data { 63 | if item.Key == key { 64 | return item.Value 65 | } 66 | } 67 | return "" 68 | } 69 | 70 | func (c *cookie) Set(key, value string) (err error) { 71 | for i, item := range c.data { 72 | if item.Key == key { 73 | c.data[i].Value = value 74 | return nil 75 | } 76 | } 77 | 78 | return c.Add(key, value) 79 | } 80 | 81 | func (c *cookie) Remove(key string) (err error) { 82 | for i, item := range c.data { 83 | if item.Key == key { 84 | c.data = append(c.data[:i], c.data[i+1:]...) 85 | return 86 | } 87 | } 88 | 89 | return 90 | } 91 | 92 | func (c *cookie) Clear() error { 93 | c.data = nil 94 | return nil 95 | } 96 | 97 | func (c *cookie) Items() []cookieItem { 98 | return c.data 99 | } 100 | 101 | func (c *cookie) String() string { 102 | var ss []string 103 | for _, item := range c.data { 104 | ss = append(ss, item.Key+"="+item.Value) 105 | } 106 | 107 | return strings.Join(ss, "; ") 108 | } 109 | -------------------------------------------------------------------------------- /cookie_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestCookieEmpty(t *testing.T) { 8 | cookie := newCookie() 9 | if cookie.Get("") != "" { 10 | t.Error("Expected empty string, got", cookie.Get("")) 11 | } 12 | 13 | if err := cookie.Add("", ""); err == nil { 14 | t.Fatal("Expected error, got nil") 15 | } 16 | 17 | if items := cookie.Items(); len(items) != 0 { 18 | t.Fatal("Expected 0 items, got", len(items)) 19 | } 20 | 21 | if err := cookie.Set("key", "value"); err != nil { 22 | t.Fatal("Expected error, got nil") 23 | } 24 | 25 | if items := cookie.Items(); len(items) != 1 { 26 | t.Fatal("Expected 1 items, got", len(items)) 27 | } 28 | 29 | if cookie.Get("key") != "value" { 30 | t.Fatal("Expected value, got", cookie.Get("key")) 31 | } 32 | 33 | if cookie.String() != "key=value" { 34 | t.Fatal("Expected key=value, got", cookie.String()) 35 | } 36 | 37 | if err := cookie.Set("key1", "value1"); err != nil { 38 | t.Fatal("Expected error, got nil") 39 | } 40 | 41 | if items := cookie.Items(); len(items) != 2 { 42 | t.Fatal("Expected 2 items, got", len(items)) 43 | } 44 | 45 | if cookie.Get("key1") != "value1" { 46 | t.Fatal("Expected value, got", cookie.Get("key1")) 47 | } 48 | 49 | if cookie.String() != "key=value; key1=value1" { 50 | t.Fatal("Expected string, got", cookie.String()) 51 | } 52 | 53 | if err := cookie.Remove("key"); err != nil { 54 | t.Fatal("Expected error, got nil") 55 | } 56 | 57 | if cookie.Get("key") != "" { 58 | t.Fatal("Expected empty string, got", cookie.Get("key1")) 59 | } 60 | 61 | if cookie.Get("key1") != "value1" { 62 | t.Fatal("Expected value, got", cookie.Get("key1")) 63 | } 64 | 65 | if cookie.String() != "key1=value1" { 66 | t.Fatal("Expected key1=value1, got", cookie.String()) 67 | } 68 | 69 | if err := cookie.Clear(); err != nil { 70 | t.Fatal("Expected error, got nil") 71 | } 72 | 73 | if cookie.Get("key1") != "" { 74 | t.Fatal("Expected empty string, got", cookie.Get("key1")) 75 | } 76 | 77 | if cookie.String() != "" { 78 | t.Fatal("Expected empty string, got", cookie.String()) 79 | } 80 | } 81 | 82 | func TestCookieParse(t *testing.T) { 83 | cookie := newCookie() 84 | if err := cookie.Parse("key=value; key1=value1"); err != nil { 85 | t.Fatal("Expected error, got nil") 86 | } 87 | 88 | if cookie.Get("key") != "value" { 89 | t.Fatal("Expected value, got", cookie.Get("key")) 90 | } 91 | 92 | if cookie.Get("key1") != "value1" { 93 | t.Fatal("Expected value, got", cookie.Get("key1")) 94 | } 95 | 96 | if cookie.String() != "key=value; key1=value1" { 97 | t.Fatal("Expected string, got", cookie.String()) 98 | } 99 | } 100 | 101 | func TestCookie(t *testing.T) { 102 | response, err := New(). 103 | SetCookie("key", "value"). 104 | SetCookie("key1", "value1"). 105 | SetURL("https://httpbin.zcorky.com/headers"). 106 | Send() 107 | if err != nil { 108 | t.Fatal(err) 109 | } 110 | 111 | // fmt.Println(response.Get("headers").String()) 112 | 113 | if response.Get("headers.cookie").String() != "key=value; key1=value1" { 114 | t.Fatal("Expected cookie, got", response.Get("header.cookie").String()) 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /default.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import "github.com/go-zoox/headers" 4 | 5 | // DefaultConfig returns the default config 6 | func DefaultConfig() *Config { 7 | config := &Config{ 8 | Headers: make(Headers), 9 | Query: make(Query), 10 | Params: make(Params), 11 | BaseURL: BaseURL, 12 | Timeout: Timeout, 13 | } 14 | 15 | config.Headers[headers.UserAgent] = DefaultUserAgent() 16 | 17 | return config 18 | } 19 | 20 | // DefaultUserAgent returns the default user agent 21 | func DefaultUserAgent() string { 22 | return UserAgent 23 | } 24 | -------------------------------------------------------------------------------- /delete.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | // Delete is a wrapper for the Delete method of the Client 4 | func Delete(url string, config *Config) (*Response, error) { 5 | return New().Delete(url, config).Execute() 6 | } 7 | -------------------------------------------------------------------------------- /delete_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func Test_Delete(t *testing.T) { 9 | response, err := Delete("https://httpbin.zcorky.com/Delete", &Config{ 10 | Body: map[string]interface{}{ 11 | "foo": "bar", 12 | "foo2": "bar2", 13 | "number": 1, 14 | "boolean": true, 15 | "array": []string{ 16 | "foo3", 17 | "bar3", 18 | }, 19 | "nest": map[string]string{ 20 | "foo4": "bar4", 21 | }, 22 | }, 23 | }) 24 | if err != nil { 25 | t.Error(err) 26 | } 27 | 28 | if response.Status != 200 { 29 | t.Error("Expected status code 200, got", response.Status) 30 | } 31 | 32 | if response.Headers.Get("content-type") != "application/json; charset=utf-8" { 33 | t.Error("Expected content-type application/json; charset=utf-8, got", response.Headers.Get("content-type")) 34 | } 35 | 36 | if response.Headers.Get("server") != "openresty" { 37 | t.Error("Expected server openresty, got", response.Headers.Get("server")) 38 | } 39 | 40 | if response.Get("url").String() != "/Delete" { 41 | t.Error("Expected url /Delete, got", response.Get("url").String()) 42 | } 43 | 44 | if response.Get("method").String() != "DELETE" { 45 | t.Error("Expected method DELETE, got", response.Get("method").String()) 46 | } 47 | 48 | if response.Get("headers.host").String() != "httpbin.zcorky.com" { 49 | t.Error("Expected Host httpbin.zcorky.com, got", response.Get("headers.Host").String()) 50 | } 51 | 52 | if response.Get("headers.accept-encoding").String() != "gzip" { 53 | t.Error("Expected accept-encoding gzip, got", response.Get("headers.accept-encoding").String()) 54 | } 55 | 56 | if response.Get("headers.user-agent").String() != DefaultUserAgent() { 57 | t.Error(fmt.Sprintf("Expected user-agent %s, got", DefaultUserAgent()), response.Get("headers.user-agent").String()) 58 | } 59 | 60 | if response.Get("headers.connection").String() != "close" { 61 | t.Error("Expected connection close, got", response.Get("headers.connection").String()) 62 | } 63 | 64 | if response.Get("origin").String() != "https://httpbin.zcorky.com" { 65 | t.Error("Expected origin https://httpbin.zcorky.com, got", response.Get("origin").String()) 66 | } 67 | 68 | if response.Get("body.foo").String() != "bar" { 69 | t.Error("Expected body.foo bar, got", response.Get("body.foo").String()) 70 | } 71 | 72 | if response.Get("body.foo2").String() != "bar2" { 73 | t.Error("Expected body.foo2 bar2, got", response.Get("body.foo2").String()) 74 | } 75 | 76 | if response.Get("body.number").Int() != 1 { 77 | t.Error("Expected body.number 1, got", response.Get("body.number").String()) 78 | } 79 | 80 | if response.Get("body.boolean").Bool() != true { 81 | t.Error("Expected body.boolean true, got", response.Get("body.boolean").String()) 82 | } 83 | 84 | if response.Get("body.array.0").String() != "foo3" { 85 | t.Error("Expected body.array foo3, got", response.Get("body.array").String()) 86 | } 87 | 88 | if response.Get("body.array.1").String() != "bar3" { 89 | t.Error("Expected body.array bar3, got", response.Get("body.array").String()) 90 | } 91 | 92 | if response.Get("body.nest.foo4").String() != "bar4" { 93 | t.Error("Expected body.nest.foo4 bar4, got", response.Get("body.nest.foo4").String()) 94 | } 95 | } 96 | 97 | func Test_Delete_With_Header(t *testing.T) { 98 | response, err := Delete("https://httpbin.zcorky.com/Delete", &Config{ 99 | Headers: map[string]string{ 100 | "X-CUSTOM-VAR": "custom-value", 101 | "x-custom-var-2": "custom-value-2", 102 | }, 103 | }) 104 | if err != nil { 105 | t.Error(err) 106 | } 107 | 108 | if response.Get("headers.x-custom-var").String() != "custom-value" { 109 | t.Error("Expected x-custom-var custom-value, got", response.Get("headers.x-custom-var").String()) 110 | } 111 | 112 | if response.Get("headers.x-custom-var-2").String() != "custom-value-2" { 113 | t.Error("Expected x-custom-var-2 custom-value, got", response.Get("headers.x-custom-var").String()) 114 | } 115 | } 116 | 117 | func Test_Delete_With_Query(t *testing.T) { 118 | response, err := Delete("https://httpbin.zcorky.com/Delete", &Config{ 119 | Query: map[string]string{ 120 | "foo": "bar", 121 | "foo2": "bar2", 122 | }, 123 | }) 124 | if err != nil { 125 | t.Error(err) 126 | } 127 | 128 | if response.Get("query.foo").String() != "bar" { 129 | t.Error("Expected foo bar, got", response.Get("query.foo").String()) 130 | } 131 | 132 | if response.Get("query.foo2").String() != "bar2" { 133 | t.Error("Expected foo2 bar2, got", response.Get("query.foo2").String()) 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /download.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | // Download is a wrapper for the Download method of the Client 4 | func Download(url string, filepath string, config ...interface{}) (*Response, error) { 5 | c := &Config{} 6 | if len(config) == 1 { 7 | c = config[0].(*Config) 8 | } else if len(config) > 1 { 9 | return nil, ErrTooManyArguments 10 | } 11 | 12 | return New().Download(url, filepath, c).Execute() 13 | } 14 | -------------------------------------------------------------------------------- /download_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/go-zoox/testify" 8 | ) 9 | 10 | func TestDownload(t *testing.T) { 11 | response, err := Download("https://httpbin.zcorky.com/image", "/tmp/image.webp") 12 | if err != nil { 13 | t.Error(err) 14 | } 15 | 16 | if response.Status != 200 { 17 | t.Error("Expected status code 200, got", response.Status) 18 | } 19 | 20 | stat, err := os.Stat("/tmp/image.webp") 21 | if err != nil { 22 | t.Error(err) 23 | } 24 | 25 | if stat.Size() == 0 { 26 | t.Error("Expected file size not 0, got 0") 27 | } 28 | } 29 | 30 | func TestDownloadParamsError(t *testing.T) { 31 | _, err := Download("", "/tmp/image.webp") 32 | testify.Assert(t, err != nil, "Expected error, got nil") 33 | 34 | _, err = Download("https://httpbin.zcorky.com/image", "/tmp/image.webp", &Config{}, &Config{}) 35 | testify.Assert(t, err != nil, "Expected error, got nil") 36 | 37 | _, err = Download("https://httpbin.zcorky.com/image", "/tmp/image.webp", &Config{}) 38 | testify.Assert(t, err == nil, "Expected nil, got error") 39 | } 40 | -------------------------------------------------------------------------------- /execute.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "context" 7 | "crypto/tls" 8 | "crypto/x509" 9 | "encoding/json" 10 | "errors" 11 | "io" 12 | "io/ioutil" 13 | "mime/multipart" 14 | "net" 15 | "net/http" 16 | "net/textproto" 17 | "net/url" 18 | "os" 19 | "strings" 20 | "time" 21 | 22 | "github.com/go-zoox/core-utils/fmt" 23 | "github.com/go-zoox/headers" 24 | "github.com/tidwall/gjson" 25 | 26 | "golang.org/x/net/proxy" 27 | ) 28 | 29 | // Execute executes the request 30 | func (f *Fetch) Execute() (*Response, error) { 31 | if len(f.Errors) > 0 { 32 | return nil, f.Errors[0] 33 | } 34 | 35 | config, err := f.Config() 36 | if err != nil { 37 | return nil, fmt.Errorf("failed to get config: %v", err) 38 | } 39 | 40 | if os.Getenv(EnvDEBUG) != "" { 41 | if err := fmt.PrintJSON("[GOZOOX_FETCH][DEBUG][Request]", config); err != nil { 42 | fmt.Println("[warn] failed to fmt.PrintJSON:", err, config) 43 | } 44 | } 45 | 46 | fullURL := config.URL 47 | methodOrigin := config.Method 48 | // @ORIGIN QUERY 49 | var urlQueryOrigin url.Values 50 | if strings.ContainsAny(fullURL, "?") { 51 | u, err := url.Parse(config.BaseURL) 52 | if err != nil { 53 | return nil, errors.New("failed to parsed origin url") 54 | } 55 | 56 | urlQueryOrigin = u.Query() 57 | } 58 | 59 | if config.TLSCaCertFile != "" { 60 | caCrt, err := ioutil.ReadFile(config.TLSCaCertFile) 61 | if err != nil { 62 | return nil, fmt.Errorf("failed to read tls certificate file(%s): %v", config.TLSCaCertFile, err) 63 | } 64 | 65 | config.TLSCaCert = caCrt 66 | } 67 | 68 | if config.TLSCertFile != "" { 69 | clientCrt, err := ioutil.ReadFile(config.TLSCertFile) 70 | if err != nil { 71 | return nil, fmt.Errorf("failed to read tls certificate file(%s): %v", config.TLSCertFile, err) 72 | } 73 | 74 | config.TLSCert = clientCrt 75 | } 76 | 77 | if config.TLSKeyFile != "" { 78 | clientKey, err := ioutil.ReadFile(config.TLSKeyFile) 79 | if err != nil { 80 | return nil, fmt.Errorf("failed to read tls certificate file(%s): %v", config.TLSKeyFile, err) 81 | } 82 | 83 | config.TLSKey = clientKey 84 | } 85 | 86 | transport := http.DefaultTransport 87 | if config.TLSCaCert != nil { 88 | pool := x509.NewCertPool() 89 | pool.AppendCertsFromPEM(config.TLSCaCert) 90 | 91 | // defaultTransportDialContext := func(dialer *net.Dialer) func(context.Context, string, string) (net.Conn, error) { 92 | // return dialer.DialContext 93 | // } 94 | 95 | // transport = &http.Transport{ 96 | // Proxy: http.ProxyFromEnvironment, 97 | // DialContext: defaultTransportDialContext(&net.Dialer{ 98 | // Timeout: 30 * time.Second, 99 | // KeepAlive: 30 * time.Second, 100 | // }), 101 | // ForceAttemptHTTP2: true, 102 | // MaxIdleConns: 100, 103 | // IdleConnTimeout: 90 * time.Second, 104 | // TLSHandshakeTimeout: 10 * time.Second, 105 | // ExpectContinueTimeout: 1 * time.Second, 106 | // // https://stackoverflow.com/questions/38822764/how-to-send-a-https-request-with-a-certificate-golang 107 | // TLSClientConfig: &tls.Config{ 108 | // RootCAs: pool, 109 | // }, 110 | // } 111 | 112 | tr := transport.(*http.Transport) 113 | if tr.TLSClientConfig == nil { 114 | tr.TLSClientConfig = &tls.Config{} 115 | } 116 | tr.TLSClientConfig.RootCAs = pool 117 | } 118 | 119 | if config.TLSCert != nil && config.TLSKey != nil { 120 | tr := transport.(*http.Transport) 121 | if tr.TLSClientConfig == nil { 122 | tr.TLSClientConfig = &tls.Config{} 123 | } 124 | 125 | clientCrt, err := tls.X509KeyPair(config.TLSCert, config.TLSKey) 126 | if err != nil { 127 | return nil, fmt.Errorf("failed to load client cert and key: %v", err) 128 | } 129 | 130 | tr.TLSClientConfig.Certificates = []tls.Certificate{clientCrt} 131 | } 132 | 133 | if config.TLSInsecureSkipVerify { 134 | tr := transport.(*http.Transport) 135 | if tr.TLSClientConfig == nil { 136 | tr.TLSClientConfig = &tls.Config{} 137 | } 138 | tr.TLSClientConfig.InsecureSkipVerify = config.TLSInsecureSkipVerify 139 | } 140 | 141 | // if f.config.HTTP2 { 142 | // if err := http2.ConfigureTransport(&transport); err != nil { 143 | // return nil, fmt.Errorf("failed to configure http2: %v", err) 144 | // } 145 | // } 146 | 147 | client := &http.Client{ 148 | Timeout: config.Timeout, 149 | Transport: transport, 150 | } 151 | 152 | // apply proxy 153 | if config.Proxy != "" { 154 | // fmt.Println("proxy:", config.Proxy) 155 | proxyURL, err := url.Parse(config.Proxy) 156 | if err != nil { 157 | return nil, fmt.Errorf("invalid proxy: %s", config.Proxy) 158 | } 159 | 160 | switch proxyURL.Scheme { 161 | case "http", "https": 162 | client.Transport = &http.Transport{ 163 | Proxy: http.ProxyURL(proxyURL), 164 | Dial: (&net.Dialer{ 165 | Timeout: 30 * time.Second, 166 | KeepAlive: 30 * time.Second, 167 | }).Dial, 168 | // default transport 169 | ForceAttemptHTTP2: true, 170 | MaxIdleConns: 100, 171 | IdleConnTimeout: 90 * time.Second, 172 | TLSHandshakeTimeout: 10 * time.Second, 173 | ExpectContinueTimeout: 1 * time.Second, 174 | } 175 | case "socks5", "socks5h": 176 | dialer, err := proxy.FromURL(proxyURL, proxy.Direct) 177 | if err != nil { 178 | return nil, fmt.Errorf("invalid socks5 proxy: %s", config.Proxy) 179 | } 180 | 181 | client.Transport = &http.Transport{ 182 | Proxy: http.ProxyFromEnvironment, 183 | Dial: dialer.Dial, 184 | // default transport 185 | ForceAttemptHTTP2: true, 186 | MaxIdleConns: 100, 187 | IdleConnTimeout: 90 * time.Second, 188 | TLSHandshakeTimeout: 10 * time.Second, 189 | ExpectContinueTimeout: 1 * time.Second, 190 | } 191 | default: 192 | return nil, fmt.Errorf("unsupport proxy(%s)", config.Proxy) 193 | } 194 | } 195 | 196 | req, err := http.NewRequestWithContext(f.config.Context, methodOrigin, fullURL, nil) 197 | if err != nil { 198 | // panic("error creating request: " + err.Error()) 199 | return nil, errors.New("ErrCannotCreateRequest(1): " + ErrCannotCreateRequest.Error() + ", err: " + err.Error()) 200 | } 201 | 202 | // @TODO 203 | if _, ok := config.Body.(string); ok { 204 | req.Header.Set(headers.ContentType, "text/plain") 205 | } 206 | 207 | for k, v := range config.Headers { 208 | // ignore empty value 209 | if v != "" { 210 | req.Header.Set(k, v) 211 | } 212 | } 213 | 214 | query := req.URL.Query() 215 | // apply origin query 216 | for k, v := range urlQueryOrigin { 217 | query.Add(k, v[0]) 218 | } 219 | // apply custom query 220 | for k, v := range config.Query { 221 | // ignore empty value 222 | if v != "" { 223 | query.Add(k, v) 224 | } 225 | } 226 | req.URL.RawQuery = query.Encode() 227 | if config.Username != "" || config.Password != "" { 228 | req.URL.User = url.UserPassword(config.Username, config.Password) 229 | } 230 | 231 | // if GET, ignore Body 232 | if config.Body != nil && config.Method == GET { 233 | // // panic("Cannot set body for GET request") 234 | // return nil, ErrCannotSendBodyWithGet 235 | config.Body = nil 236 | } 237 | 238 | if config.Body != nil { 239 | if req.Header.Get(headers.ContentType) == "" { 240 | req.Header.Set(headers.ContentType, "application/json") 241 | } 242 | 243 | if strings.Contains(req.Header.Get(headers.ContentType), "application/json") { 244 | body, err := json.Marshal(config.Body) 245 | if err != nil { 246 | // panic("error marshalling body: " + err.Error()) 247 | return nil, errors.New("ErrInvalidJSONBody(2): " + ErrInvalidJSONBody.Error() + ", err: " + err.Error()) 248 | } 249 | 250 | // req.Header.Set(HeaderContentTye, "application/json") 251 | req.Body = ioutil.NopCloser(bytes.NewReader(body)) 252 | } else if strings.Contains(req.Header.Get(headers.ContentType), "application/x-www-form-urlencoded") { 253 | body := url.Values{} 254 | if kv, ok := config.Body.(map[string]string); ok { 255 | for k, v := range kv { 256 | body.Add(k, v) 257 | } 258 | } else { 259 | return nil, errors.New(ErrInvalidURLFormEncodedBody.Error() + ": must be map[string]string") 260 | } 261 | 262 | // req.Header.Set(HeaderContentTye, "application/x-www-form-urlencoded") 263 | // req.Body = ioutil.NopCloser(bytes.NewReader(body)) 264 | req.Body = ioutil.NopCloser(strings.NewReader(body.Encode())) 265 | } else if strings.Contains(req.Header.Get(headers.ContentType), "multipart/form-data") { 266 | if values, ok := config.Body.(map[string]interface{}); ok { 267 | var b bytes.Buffer 268 | w := multipart.NewWriter(&b) 269 | for k, v := range values { 270 | if v == nil { 271 | continue 272 | } 273 | 274 | var fw io.Writer 275 | if text, ok := v.(string); ok { 276 | if fw, err = w.CreateFormField(k); err != nil { 277 | return nil, err 278 | } 279 | 280 | if _, err = io.Copy(fw, strings.NewReader(text)); err != nil { 281 | return nil, err 282 | } 283 | 284 | continue 285 | } 286 | 287 | if f, ok := v.(io.ReadCloser); ok { 288 | // fix multipart form file content type 289 | if err := createFormFile(w, f, k); err != nil { 290 | return nil, err 291 | } 292 | 293 | continue 294 | } 295 | } 296 | w.Close() 297 | req.Header.Set(headers.ContentType, w.FormDataContentType()) 298 | req.Body = ioutil.NopCloser(&b) 299 | } else if values, ok := config.Body.(map[string]string); ok { 300 | var b bytes.Buffer 301 | w := multipart.NewWriter(&b) 302 | for k, v := range values { 303 | var fw io.Writer 304 | if fw, err = w.CreateFormField(k); err != nil { 305 | return nil, err 306 | } 307 | 308 | if _, err = io.Copy(fw, strings.NewReader(v)); err != nil { 309 | return nil, err 310 | } 311 | 312 | continue 313 | } 314 | w.Close() 315 | req.Header.Set(headers.ContentType, w.FormDataContentType()) 316 | req.Body = ioutil.NopCloser(&b) 317 | } else { 318 | return nil, errors.New(ErrInvalidBodyMultipart.Error() + ": must be map[string]interface{} or map[string]string") 319 | } 320 | } else if strings.Contains(req.Header.Get(headers.ContentType), "application/octet-stream") { 321 | if config.Body == nil { 322 | return nil, fmt.Errorf("octet-stream body is required") 323 | } 324 | 325 | body, ok := config.Body.(io.ReadCloser) 326 | if !ok && body != nil { 327 | body = io.NopCloser(body) 328 | } 329 | 330 | req.Body = body 331 | } else { 332 | if _, ok := config.Body.(string); !ok { 333 | return nil, ErrorInvalidBody 334 | } 335 | 336 | req.Body = ioutil.NopCloser(bytes.NewReader([]byte(config.Body.(string)))) 337 | } 338 | } 339 | 340 | // unix domain socket: https://gist.github.com/teknoraver/5ffacb8757330715bcbcc90e6d46ac74 341 | if config.UnixDomainSocket != "" { 342 | // remove unix:// 343 | // if strings.HasPrefix(config.UnixDomainSocket, "unix://") { 344 | // config.UnixDomainSocket = config.UnixDomainSocket[7:] 345 | // } 346 | config.UnixDomainSocket = strings.TrimPrefix(config.UnixDomainSocket, "unix://") 347 | 348 | tr := client.Transport.(*http.Transport) 349 | tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { 350 | return net.Dial("unix", config.UnixDomainSocket) 351 | } 352 | } 353 | 354 | resp, err := client.Do(req) 355 | 356 | if err != nil { 357 | // panic("error sending request: " + err.Error()) 358 | return nil, errors.New("ErrSendingRequest(3): " + ErrSendingRequest.Error() + ", err: " + err.Error() + "(Please check your network, maybe use bad proxy or network offline)") 359 | } 360 | 361 | // Check that the server actually sent compressed data 362 | var reader io.ReadCloser 363 | switch resp.Header.Get(headers.ContentEncoding) { 364 | case "gzip": 365 | reader, err = gzip.NewReader(resp.Body) 366 | if err != nil { 367 | return nil, fmt.Errorf("gzip decode error: %s", err) 368 | } 369 | // defer reader.Close() 370 | default: 371 | reader = resp.Body 372 | } 373 | 374 | if !config.IsStream { 375 | defer reader.Close() 376 | } 377 | 378 | if config.BasicAuth.Username != "" || config.BasicAuth.Password != "" { 379 | f.SetBasicAuth(config.BasicAuth.Username, config.BasicAuth.Password) 380 | } 381 | 382 | if config.IsSession { 383 | cookies := resp.Cookies() 384 | for _, cookie := range cookies { 385 | f.SetCookie(cookie.Name, cookie.Value) 386 | } 387 | } 388 | 389 | if config.DownloadFilePath != "" { 390 | file, err := os.OpenFile(config.DownloadFilePath, os.O_CREATE|os.O_WRONLY, 0644) 391 | if err != nil { 392 | return nil, err 393 | } 394 | defer file.Close() 395 | 396 | res := &Response{ 397 | Status: resp.StatusCode, 398 | Headers: resp.Header, 399 | // 400 | Request: config, 401 | } 402 | 403 | if f.config.OnProgress != nil { 404 | progress := &Progress{ 405 | Total: resp.ContentLength, 406 | Current: 0, 407 | Reporter: f.config.OnProgress, 408 | } 409 | 410 | _, err = io.Copy(io.MultiWriter(file, progress), reader) 411 | if err != nil { 412 | return nil, err 413 | } 414 | } else { 415 | _, err = io.Copy(file, reader) 416 | if err != nil { 417 | return nil, err 418 | } 419 | } 420 | 421 | return res, nil 422 | } 423 | 424 | if config.IsStream { 425 | return &Response{ 426 | Status: resp.StatusCode, 427 | Headers: resp.Header, 428 | // 429 | Request: config, 430 | // 431 | Stream: reader, 432 | }, nil 433 | } 434 | 435 | body, err := ioutil.ReadAll(reader) 436 | if err != nil { 437 | // panic("error reading response: " + err.Error()) 438 | return nil, errors.New("ErrReadingResponse(4): " + ErrReadingResponse.Error() + ", err: " + err.Error()) 439 | } 440 | 441 | // fmt.Println("response: ", string(body)) 442 | 443 | if os.Getenv(EnvDEBUG) != "" { 444 | if strings.Contains(resp.Header.Get(headers.ContentType), "application/json") { 445 | b, err := json.MarshalIndent(gjson.Parse(string(body)).Value(), "", " ") 446 | if err != nil { 447 | fmt.Println("[GOZOOX_FETCH][DEBUG][Response]", string(body)) 448 | } else { 449 | fmt.Println("[GOZOOX_FETCH][DEBUG][Response]", string(b)) 450 | } 451 | } else { 452 | fmt.Println("[GOZOOX_FETCH][DEBUG][Response]", string(body)) 453 | } 454 | } 455 | 456 | return &Response{ 457 | Status: resp.StatusCode, 458 | Headers: resp.Header, 459 | Body: body, 460 | // 461 | Request: config, 462 | }, nil 463 | } 464 | 465 | // @TODO for multipart/form-data with file 466 | // 467 | // Issue: 468 | // mime/multipart:CreateFormFile has a fixed content type(application/octet-stream), 469 | // does not support auto-detect content type 470 | // 471 | // Need: Create MIME encoded form files that auto-detect the content type. 472 | // 473 | // Reference: 474 | // 1. https://groups.google.com/g/golang-nuts/c/HwOYproYQqA 475 | // 2. https://github.com/go-openapi/runtime/pull/170/files 476 | 477 | // NamedReadCloser is a named reader 478 | type NamedReadCloser interface { 479 | io.ReadCloser 480 | 481 | Name() string 482 | } 483 | 484 | func escapeQuotes(s string) string { 485 | return strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace(s) 486 | } 487 | 488 | func createFormFile(w *multipart.Writer, reader io.ReadCloser, fieldname string) error { 489 | buf := bytes.NewBuffer([]byte{}) 490 | filename := "" 491 | if f, ok := reader.(NamedReadCloser); ok { 492 | filename = f.Name() 493 | } 494 | 495 | // Need to read the data so that we can detect the content type 496 | if _, err := io.Copy(buf, reader); err != nil { 497 | return err 498 | } 499 | fileBytes := buf.Bytes() 500 | fileContentType := http.DetectContentType(fileBytes) 501 | 502 | newFi := CreateNamedReader(filename, buf) 503 | 504 | h := make(textproto.MIMEHeader) 505 | if filename == "" { 506 | h.Set("Content-Disposition", 507 | fmt.Sprintf(`form-data; name="%s"`, escapeQuotes(fieldname))) 508 | } else { 509 | h.Set("Content-Disposition", 510 | fmt.Sprintf(`form-data; name="%s"; filename="%s"`, 511 | escapeQuotes(fieldname), escapeQuotes(filename))) 512 | } 513 | h.Set("Content-Type", fileContentType) 514 | 515 | fw, err := w.CreatePart(h) 516 | if err != nil { 517 | return err 518 | } 519 | 520 | if _, err = io.Copy(fw, newFi); err != nil { 521 | return err 522 | } 523 | 524 | return nil 525 | } 526 | 527 | // CreateNamedReader creates a named reader 528 | // 529 | // multipart.File, that is Request.ParseMultipartForm, does not have a name 530 | // so we need to create a named reader to get the file name 531 | // when uploading a file with multipart/form-data 532 | func CreateNamedReader(name string, rdr io.Reader) NamedReadCloser { 533 | rc, ok := rdr.(io.ReadCloser) 534 | if !ok { 535 | rc = io.NopCloser(rdr) 536 | } 537 | return &namedReadCloser{ 538 | name: name, 539 | cr: rc, 540 | } 541 | } 542 | 543 | type namedReadCloser struct { 544 | name string 545 | cr io.ReadCloser 546 | } 547 | 548 | func (n *namedReadCloser) Close() error { 549 | return n.cr.Close() 550 | } 551 | func (n *namedReadCloser) Read(p []byte) (int, error) { 552 | return n.cr.Read(p) 553 | } 554 | func (n *namedReadCloser) Name() string { 555 | return n.name 556 | } 557 | -------------------------------------------------------------------------------- /fetch.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "errors" 7 | "fmt" 8 | "net/url" 9 | "path" 10 | "strings" 11 | "time" 12 | 13 | "github.com/go-zoox/headers" 14 | ) 15 | 16 | // Fetch is the Fetch Client 17 | type Fetch struct { 18 | config *Config 19 | Errors []error 20 | } 21 | 22 | // New creates a fetch client 23 | func New(cfg ...*Config) *Fetch { 24 | config := DefaultConfig() 25 | if len(cfg) > 1 { 26 | panic("Too many arguments") 27 | } 28 | 29 | if len(cfg) == 1 { 30 | config.Merge(cfg[0]) 31 | } 32 | 33 | if config.Context == nil { 34 | config.Context = context.Background() 35 | } 36 | 37 | return &Fetch{ 38 | config: config, 39 | } 40 | } 41 | 42 | // Create creates a new fetch with base url 43 | // Specially useful for Client SDK 44 | func Create(baseURL string) *Fetch { 45 | return New().SetBaseURL(baseURL) 46 | } 47 | 48 | // SetContext sets the context 49 | func (f *Fetch) SetContext(ctx context.Context) *Fetch { 50 | f.config.Context = ctx 51 | return f 52 | } 53 | 54 | // SetConfig sets the config of fetch 55 | func (f *Fetch) SetConfig(configs ...*Config) *Fetch { 56 | for _, config := range configs { 57 | f.config.Merge(config) 58 | } 59 | 60 | return f 61 | } 62 | 63 | // SetURL sets the url of fetch 64 | func (f *Fetch) SetURL(url string) *Fetch { 65 | f.config.URL = url 66 | return f 67 | } 68 | 69 | // SetDownloadFilePath sets the download file path 70 | func (f *Fetch) SetDownloadFilePath(filepath string) *Fetch { 71 | f.config.DownloadFilePath = filepath 72 | return f 73 | } 74 | 75 | // SetProgressCallback sets the progress callback 76 | func (f *Fetch) SetProgressCallback(callback func(percent int64, current, total int64)) *Fetch { 77 | f.config.OnProgress = callback 78 | return f 79 | } 80 | 81 | // SetMethod sets the method 82 | func (f *Fetch) SetMethod(method string) *Fetch { 83 | for m := range METHODS { 84 | if method == METHODS[m] { 85 | f.config.Method = method 86 | return f 87 | } 88 | } 89 | 90 | f.Errors = append(f.Errors, ErrInvalidMethod) 91 | return f 92 | } 93 | 94 | // SetHeader sets the header key and value 95 | func (f *Fetch) SetHeader(key, value string) *Fetch { 96 | f.config.Headers[key] = value 97 | return f 98 | } 99 | 100 | // SetQuery sets the query key and value 101 | func (f *Fetch) SetQuery(key, value string) *Fetch { 102 | f.config.Query[key] = value 103 | return f 104 | } 105 | 106 | // SetParam sets the param key and value 107 | func (f *Fetch) SetParam(key, value string) *Fetch { 108 | f.config.Params[key] = value 109 | return f 110 | } 111 | 112 | // SetBody sets the body 113 | func (f *Fetch) SetBody(body Body) *Fetch { 114 | f.config.Body = body 115 | return f 116 | } 117 | 118 | // SetBaseURL sets the base url 119 | func (f *Fetch) SetBaseURL(url string) *Fetch { 120 | f.config.BaseURL = url 121 | return f 122 | } 123 | 124 | // SetTimeout sets the timeout 125 | func (f *Fetch) SetTimeout(timeout time.Duration) *Fetch { 126 | f.config.Timeout = timeout 127 | return f 128 | } 129 | 130 | // SetUserAgent sets the user agent 131 | func (f *Fetch) SetUserAgent(userAgent string) *Fetch { 132 | return f.SetHeader(headers.UserAgent, userAgent) 133 | } 134 | 135 | // SetBasicAuth sets the basic auth username and password 136 | func (f *Fetch) SetBasicAuth(username, password string) *Fetch { 137 | return f.SetAuthorization("Basic " + base64.StdEncoding.EncodeToString([]byte(username+":"+password))) 138 | } 139 | 140 | // SetBearToken sets the bear token 141 | func (f *Fetch) SetBearToken(token string) *Fetch { 142 | return f.SetAuthorization("Bearer " + token) 143 | } 144 | 145 | // SetAuthorization sets the authorization token 146 | func (f *Fetch) SetAuthorization(token string) *Fetch { 147 | return f.SetHeader(headers.Authorization, token) 148 | } 149 | 150 | // SetCookie sets the cookie 151 | func (f *Fetch) SetCookie(key, value string) *Fetch { 152 | origin := f.config.Headers.Get(headers.Cookie) 153 | 154 | cookie := newCookie(origin) 155 | cookie.Set(key, value) 156 | 157 | return f.SetHeader(headers.Cookie, cookie.String()) 158 | } 159 | 160 | // SetProxy sets the proxy 161 | // 162 | // support http, https, socks5 163 | // example: 164 | // http://127.0.0.1:17890 165 | // https://127.0.0.1:17890 166 | // socks5://127.0.0.1:17890 167 | func (f *Fetch) SetProxy(proxy string) *Fetch { 168 | // validdate proxy 169 | _, err := url.Parse(proxy) 170 | if err != nil { 171 | panic(fmt.Sprintf("invalid proxy %s", proxy)) 172 | } 173 | 174 | f.config.Proxy = proxy 175 | 176 | return f 177 | } 178 | 179 | // SetAccept sets the accept header 180 | func (f *Fetch) SetAccept(accept string) *Fetch { 181 | return f.SetHeader(headers.Accept, accept) 182 | } 183 | 184 | // SetContentType ... 185 | func (f *Fetch) SetContentType(contentType string) *Fetch { 186 | return f.SetHeader(headers.ContentType, contentType) 187 | } 188 | 189 | // SetReferrer sets the referrer 190 | func (f *Fetch) SetReferrer(referrer string) *Fetch { 191 | return f.SetHeader(headers.Referrer, referrer) 192 | } 193 | 194 | // SetCacheControl sets the cache control 195 | func (f *Fetch) SetCacheControl(cacheControl string) *Fetch { 196 | return f.SetHeader(headers.CacheControl, cacheControl) 197 | } 198 | 199 | // SetAcceptEncoding sets the accept encoding 200 | func (f *Fetch) SetAcceptEncoding(acceptEncoding string) *Fetch { 201 | return f.SetHeader(headers.AcceptEncoding, acceptEncoding) 202 | } 203 | 204 | // SetAcceptLanguage sets the accept language 205 | func (f *Fetch) SetAcceptLanguage(acceptLanguage string) *Fetch { 206 | return f.SetHeader(headers.AcceptLanguage, acceptLanguage) 207 | } 208 | 209 | // Config returns the config of fetch 210 | func (f *Fetch) Config() (*Config, error) { 211 | cfg := f.config.Clone() 212 | 213 | // if f.isConfigBuilt { 214 | // return f.config, nil 215 | // } 216 | // f.isConfigBuilt = true 217 | 218 | newURL := f.config.URL 219 | if f.config.Params != nil { 220 | for k, v := range f.config.Params { 221 | vEscaped := url.QueryEscape(v) 222 | // support /:id/:name 223 | newURL = strings.Replace(newURL, ":"+k, vEscaped, -1) 224 | // support /{id}/{name} 225 | newURL = strings.Replace(newURL, "{"+k+"}", vEscaped, -1) 226 | } 227 | } 228 | 229 | // @BASEURL 230 | if f.config.BaseURL != "" { 231 | uNewURL, err := url.Parse(newURL) 232 | if err != nil { 233 | return cfg, errors.New("invalid NewURL") 234 | } 235 | 236 | if uNewURL.Host == "" { 237 | parsedBaseURL, err := url.Parse(f.config.BaseURL) 238 | if err != nil { 239 | return cfg, errors.New("invalid base URL") 240 | } 241 | 242 | parsedBaseURL.Path = path.Join(parsedBaseURL.Path, newURL) 243 | newURL = parsedBaseURL.String() 244 | } 245 | } 246 | 247 | cfg.URL = newURL 248 | 249 | return cfg, nil 250 | } 251 | 252 | // Send sends the request 253 | func (f *Fetch) Send() (*Response, error) { 254 | return f.Execute() 255 | } 256 | 257 | // Clone creates a new fetch 258 | func (f *Fetch) Clone() *Fetch { 259 | return New(f.config) 260 | } 261 | 262 | // Retry retries the request 263 | func (f *Fetch) Retry(before func(f *Fetch)) (*Response, error) { 264 | nf := f.Clone() 265 | 266 | if before != nil { 267 | before(nf) 268 | } 269 | 270 | return nf.Send() 271 | } 272 | -------------------------------------------------------------------------------- /fetch_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "testing" 8 | "time" 9 | 10 | "github.com/go-zoox/headers" 11 | "github.com/go-zoox/testify" 12 | ) 13 | 14 | func TestNew(t *testing.T) { 15 | f := New() 16 | // testify.NotNil(t, f) 17 | testify.Assert(t, f != nil, "Expected not nil, got nil") 18 | } 19 | 20 | func TestSetMethod(t *testing.T) { 21 | f := New() 22 | f.SetMethod("GET") 23 | testify.Equal(t, "GET", f.config.Method) 24 | 25 | // invalid method 26 | f.SetMethod("INVALID") 27 | testify.Assert(t, &f.Errors[0] != nil) 28 | } 29 | 30 | func TestBaseURL(t *testing.T) { 31 | BaseURL := "https://httpbin.zcorky.com" 32 | 33 | f := New() 34 | 35 | response, err := f.Get("/get", &Config{BaseURL: BaseURL}).Send() 36 | if err != nil { 37 | t.Error(err) 38 | } 39 | 40 | if response.Get("origin").String() != BaseURL { 41 | t.Fatal("Expected BaseURL https://httpbin.zcorky.com, got", response.Get("origin").String()) 42 | } 43 | } 44 | 45 | func TestTimeout(t *testing.T) { 46 | BaseURL := "https://httpbin.zcorky.com" 47 | 48 | f := New() 49 | 50 | _, err := f.Get("/get", &Config{ 51 | BaseURL: BaseURL, 52 | Timeout: 1 * time.Microsecond, 53 | }).Send() 54 | if err == nil { 55 | t.Error(errors.New("Expected timeout error, got nil")) 56 | } 57 | } 58 | 59 | func TestResponseUnmarshal(t *testing.T) { 60 | type body struct { 61 | URL string `alias:"url"` 62 | Method string `alias:"method"` 63 | } 64 | 65 | var b body 66 | response, _ := Get("https://httpbin.zcorky.com/get") 67 | if err := response.UnmarshalJSON(&b); err != nil { 68 | t.Error(err) 69 | } 70 | 71 | if b.URL != "/get" { 72 | t.Error("Expected url /get, got", b.URL) 73 | } 74 | 75 | if b.Method != "GET" { 76 | t.Error("Expected method GET, got", b.Method) 77 | } 78 | } 79 | 80 | func TestSetBasicAuth(t *testing.T) { 81 | f := New() 82 | 83 | response, err := f.Get("https://httpbin.zcorky.com/basic-auth/user/passwd"). 84 | SetBasicAuth("user", "passwd"). 85 | Send() 86 | if err != nil { 87 | t.Error(err) 88 | } 89 | 90 | if response.Status != 200 { 91 | t.Error("Expected authenticated 200, got", response.Status) 92 | } 93 | } 94 | 95 | func TestSetBearToken(t *testing.T) { 96 | f := New() 97 | 98 | response, err := f.Get("https://httpbin.zcorky.com/headers"). 99 | SetBearToken("token"). 100 | Send() 101 | if err != nil { 102 | t.Error(err) 103 | } 104 | 105 | if response.Status != 200 { 106 | t.Error("Expected authenticated 200, got", response.Status) 107 | } 108 | 109 | if response.Get("headers.authorization").String() != "Bearer token" { 110 | t.Error("Expected Authorization Bearer token, got", response.Get("headers.authorization").String()) 111 | } 112 | } 113 | 114 | func TestProxy(t *testing.T) { 115 | f := New() 116 | 117 | response, err := f.Get("https://httpbin.org/ip"). 118 | // SetProxy("http://127.0.0.1:17890"). 119 | // SetProxy("https://127.0.0.1:17890"). 120 | // SetProxy("socks5://127.0.0.1:17890"). 121 | Send() 122 | 123 | if err != nil { 124 | t.Fatal(err) 125 | } 126 | 127 | fmt.Println("response:", response.String()) 128 | } 129 | 130 | func TestRetryManual(t *testing.T) { 131 | f := New() 132 | 133 | response, err := f.Get("https://httpbin.zcorky.com/headers"). 134 | SetBearToken("zzz"). 135 | Send() 136 | if err != nil { 137 | t.Fatal(err) 138 | } 139 | 140 | // j, _ := json.MarshalIndent(f.config, " ", " ") 141 | // fmt.Println(string(j)) 142 | 143 | if response.Get("headers.authorization").String() != "Bearer zzz" { 144 | t.Fatal("Expected Authorization Bearer zzz, got", response.Get("headers.authorization").String()) 145 | } 146 | 147 | response, err = f.Retry(func(f *Fetch) { 148 | f.SetBearToken("another") 149 | }) 150 | if err != nil { 151 | t.Fatal(err) 152 | } 153 | 154 | if response.Get("headers.authorization").String() != "Bearer another" { 155 | t.Fatal("Expected Authorization Bearer zzz, got", response.Get("headers.authorization").String()) 156 | } 157 | } 158 | 159 | func TestCreate(t *testing.T) { 160 | baseURL := "https://httpbin.zcorky.com" 161 | f := Create(baseURL) 162 | testify.Equal(t, baseURL, f.config.BaseURL) 163 | } 164 | 165 | func TestSetQuery(t *testing.T) { 166 | f := New() 167 | f.SetQuery("a", "b") 168 | f.SetQuery("c", "d") 169 | 170 | testify.Equal(t, "b", f.config.Query.Get("a")) 171 | testify.Equal(t, "d", f.config.Query.Get("c")) 172 | } 173 | 174 | func TestSetHeader(t *testing.T) { 175 | f := New() 176 | f.SetHeader("a", "b") 177 | f.SetHeader("c", "d") 178 | 179 | testify.Equal(t, "b", f.config.Headers.Get("a")) 180 | testify.Equal(t, "d", f.config.Headers.Get("c")) 181 | } 182 | 183 | func TestSetParams(t *testing.T) { 184 | f := New() 185 | f.SetParam("a", "b") 186 | f.SetParam("c", "d") 187 | 188 | testify.Equal(t, "b", f.config.Params.Get("a")) 189 | testify.Equal(t, "d", f.config.Params.Get("c")) 190 | } 191 | 192 | func TestSetBody(t *testing.T) { 193 | f := New() 194 | f.SetBody("a") 195 | testify.Equal(t, "a", f.config.Body.(string)) 196 | 197 | // body := map[string]string{ 198 | // "a": "b", 199 | // } 200 | // f.SetBody(body) 201 | // testify.Equal(t, body, f.config.Body.(map[string]string)) 202 | } 203 | 204 | func TestSetBaseURL(t *testing.T) { 205 | f := New() 206 | f.SetBaseURL("https://httpbin.zcorky.com") 207 | testify.Equal(t, "https://httpbin.zcorky.com", f.config.BaseURL) 208 | } 209 | 210 | func TestSetTimeout(t *testing.T) { 211 | f := New() 212 | f.SetTimeout(1 * time.Second) 213 | testify.Equal(t, 1*time.Second, f.config.Timeout) 214 | } 215 | 216 | func TestSetUserAgent(t *testing.T) { 217 | f := New() 218 | f.SetUserAgent("test") 219 | testify.Equal(t, "test", f.config.Headers.Get(headers.UserAgent)) 220 | } 221 | 222 | func TestConfigSetBasicAuth(t *testing.T) { 223 | f := New() 224 | f.SetBasicAuth("user", "passwd") 225 | testify.Equal(t, "Basic dXNlcjpwYXNzd2Q=", f.config.Headers.Get(headers.Authorization)) 226 | } 227 | 228 | func TestConfigSetBearToken(t *testing.T) { 229 | f := New() 230 | f.SetBearToken("token") 231 | testify.Equal(t, "Bearer token", f.config.Headers.Get(headers.Authorization)) 232 | } 233 | 234 | func TestSetAuthorization(t *testing.T) { 235 | f := New() 236 | f.SetAuthorization("token") 237 | testify.Equal(t, "token", f.config.Headers.Get(headers.Authorization)) 238 | } 239 | 240 | func TestSetAccept(t *testing.T) { 241 | f := New() 242 | f.SetAccept("application/json") 243 | testify.Equal(t, "application/json", f.config.Headers.Get(headers.Accept)) 244 | } 245 | 246 | func TestSetContentType(t *testing.T) { 247 | f := New() 248 | f.SetContentType("application/json") 249 | testify.Equal(t, "application/json", f.config.Headers.Get(headers.ContentType)) 250 | } 251 | 252 | func TestSetReferrer(t *testing.T) { 253 | f := New() 254 | f.SetReferrer("https://httpbin.zcorky.com") 255 | testify.Equal(t, "https://httpbin.zcorky.com", f.config.Headers.Get(headers.Referrer)) 256 | } 257 | 258 | func TestSetCacheControl(t *testing.T) { 259 | f := New() 260 | f.SetCacheControl("no-cache") 261 | testify.Equal(t, "no-cache", f.config.Headers.Get(headers.CacheControl)) 262 | } 263 | 264 | func TestSetAcceptEncoding(t *testing.T) { 265 | f := New() 266 | f.SetAcceptEncoding("gzip") 267 | testify.Equal(t, "gzip", f.config.Headers.Get(headers.AcceptEncoding)) 268 | } 269 | 270 | func TestSetAcceptLanguage(t *testing.T) { 271 | f := New() 272 | f.SetAcceptLanguage("zh-CN") 273 | testify.Equal(t, "zh-CN", f.config.Headers.Get(headers.AcceptLanguage)) 274 | } 275 | 276 | func TestSetProxy(t *testing.T) { 277 | f := New() 278 | f.SetProxy("https://example.com") 279 | testify.Equal(t, "https://example.com", f.config.Proxy) 280 | } 281 | 282 | func TestFetchConfig(t *testing.T) { 283 | f := New() 284 | f.SetBaseURL("https://httpbin.zcorky.com") 285 | f.SetURL("/get/:id/:name") 286 | f.SetParam("id", "1") 287 | f.SetParam("name", "Zero") 288 | cfg, err := f.Config() 289 | testify.Assert(t, err == nil, "err should be nil") 290 | testify.Equal(t, "https://httpbin.zcorky.com/get/1/Zero", cfg.URL) 291 | } 292 | 293 | func TestFetchCancel(t *testing.T) { 294 | f := New() 295 | f.SetBaseURL("https://httpbin.zcorky.com") 296 | f.SetURL("/delay/3") 297 | ctx, cancel := context.WithCancel(context.Background()) 298 | f.SetContext(ctx) 299 | cancel() 300 | _, err := f.Execute() 301 | // fmt.Println(err) 302 | testify.Assert(t, err != nil, "err should not be nil") 303 | } 304 | -------------------------------------------------------------------------------- /get.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | // Get is a wrapper for the Get method of the Client 4 | func Get(url string, config ...interface{}) (*Response, error) { 5 | c := &Config{} 6 | if len(config) == 1 { 7 | c = config[0].(*Config) 8 | } else if len(config) > 1 { 9 | return nil, ErrTooManyArguments 10 | } 11 | 12 | if c.Body != nil { 13 | panic("Request with GET method cannot have body") 14 | } 15 | 16 | return New().Get(url, c).Execute() 17 | } 18 | -------------------------------------------------------------------------------- /get_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func Test_Get(t *testing.T) { 9 | response, _ := Get("https://httpbin.zcorky.com/get") 10 | 11 | if response.Status != 200 { 12 | t.Error("Expected status code 200, got", response.Status) 13 | } 14 | 15 | if response.Headers.Get("content-type") != "application/json; charset=utf-8" { 16 | t.Error("Expected content-type application/json; charset=utf-8, got", response.Headers.Get("content-type")) 17 | } 18 | 19 | if response.Headers.Get("server") != "openresty" { 20 | t.Error("Expected server openresty, got", response.Headers.Get("server")) 21 | } 22 | 23 | if response.Get("url").String() != "/get" { 24 | t.Error("Expected url /get, got", response.Get("url").String()) 25 | } 26 | 27 | if response.Get("method").String() != "GET" { 28 | t.Error("Expected method GET, got", response.Get("method").String()) 29 | } 30 | 31 | if response.Get("headers.host").String() != "httpbin.zcorky.com" { 32 | t.Error("Expected Host httpbin.zcorky.com, got", response.Get("headers.Host").String()) 33 | } 34 | 35 | if response.Get("headers.accept-encoding").String() != "gzip" { 36 | t.Error("Expected accept-encoding gzip, got", response.Get("headers.accept-encoding").String()) 37 | } 38 | 39 | if response.Get("headers.user-agent").String() != DefaultUserAgent() { 40 | t.Error(fmt.Sprintf("Expected user-agent %s, got", DefaultUserAgent()), response.Get("headers.user-agent").String()) 41 | } 42 | 43 | if response.Get("headers.connection").String() != "close" { 44 | t.Error("Expected connection close, got", response.Get("headers.connection").String()) 45 | } 46 | 47 | if response.Get("origin").String() != "https://httpbin.zcorky.com" { 48 | t.Error("Expected origin https://httpbin.zcorky.com, got", response.Get("origin").String()) 49 | } 50 | } 51 | 52 | func Test_Get_With_Header(t *testing.T) { 53 | response, _ := Get("https://httpbin.zcorky.com/get", &Config{ 54 | Headers: map[string]string{ 55 | "X-CUSTOM-VAR": "custom-value", 56 | "x-custom-var-2": "custom-value-2", 57 | }, 58 | }) 59 | 60 | // fmt.Println("raw: ", response.JSON()) 61 | 62 | if response.Get("headers.x-custom-var").String() != "custom-value" { 63 | t.Error("Expected x-custom-var custom-value, got", response.Get("headers.x-custom-var").String()) 64 | } 65 | 66 | if response.Get("headers.x-custom-var-2").String() != "custom-value-2" { 67 | t.Error("Expected x-custom-var-2 custom-value, got", response.Get("headers.x-custom-var").String()) 68 | } 69 | } 70 | 71 | func Test_Get_With_Query(t *testing.T) { 72 | response, _ := Get("https://httpbin.zcorky.com/get", &Config{ 73 | Query: map[string]string{ 74 | "foo": "bar", 75 | "foo2": "bar2", 76 | }, 77 | }) 78 | 79 | if response.Get("query.foo").String() != "bar" { 80 | t.Error("Expected foo bar, got", response.Get("query.foo").String()) 81 | } 82 | 83 | if response.Get("query.foo2").String() != "bar2" { 84 | t.Error("Expected foo2 bar2, got", response.Get("query.foo2").String()) 85 | } 86 | } 87 | 88 | func Test_Get_With_BasicAuth_In_URL(t *testing.T) { 89 | response, err := Get("https://user1:pass1@httpbin.zcorky.com/headers") 90 | if err != nil { 91 | t.Error("Expected no error, got", err) 92 | } 93 | 94 | if response.Status != 200 { 95 | t.Error("Expected status code 200, got", response.Status) 96 | } 97 | 98 | if response.Get("headers.authorization").String() != "Basic dXNlcjE6cGFzczE=" { 99 | t.Error("Expected Authorization Basic dXNlcjE6cGFzczE=, got", response.Get("headers.authorization").String()) 100 | } 101 | 102 | fmt.Println(response.String()) 103 | } 104 | -------------------------------------------------------------------------------- /global.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // BaseURL is the default base url 9 | var BaseURL = "" 10 | 11 | // Timeout is the default timeout 12 | var Timeout = 60 * time.Second 13 | 14 | // UserAgent is the default user agent 15 | var UserAgent = fmt.Sprintf("GoFetch/%s (github.com/go-zoox/fetch)", Version) 16 | 17 | // @TODO 18 | // var Headers = make(ConfigHeaders) 19 | 20 | // SetBaseURL sets the base url 21 | func SetBaseURL(url string) { 22 | BaseURL = url 23 | } 24 | 25 | // SetTimeout sets the timeout 26 | func SetTimeout(timeout time.Duration) { 27 | Timeout = timeout 28 | } 29 | 30 | // SetUserAgent sets the user agent 31 | func SetUserAgent(userAgent string) { 32 | UserAgent = userAgent 33 | } 34 | 35 | // func SetHeader(key, value string) { 36 | // Headers[key] = value 37 | // } 38 | -------------------------------------------------------------------------------- /global_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestGlobalSetBaseURL(t *testing.T) { 10 | if BaseURL != "" { 11 | t.Errorf("global BaseURL should be empty string, but got %s", BaseURL) 12 | } 13 | 14 | SetBaseURL("https://example.com") 15 | if BaseURL != "https://example.com" { 16 | t.Errorf("global BaseURL should be https://example.com, but got %s", BaseURL) 17 | } 18 | 19 | SetBaseURL("") 20 | if BaseURL != "" { 21 | t.Errorf("global BaseURL should be empty string, but got %s", BaseURL) 22 | } 23 | } 24 | 25 | func TestGlobalUserAgent(t *testing.T) { 26 | if UserAgent != fmt.Sprintf("GoFetch/%s (github.com/go-zoox/fetch)", Version) { 27 | t.Errorf("global UserAgent should be empty string, but got %s", UserAgent) 28 | } 29 | 30 | SetUserAgent("test user agent") 31 | if UserAgent != "test user agent" { 32 | t.Errorf("global UserAgent should be test user agent, but got %s", UserAgent) 33 | } 34 | 35 | SetUserAgent(fmt.Sprintf("GoFetch/%s (github.com/go-zoox/fetch)", Version)) 36 | if UserAgent != fmt.Sprintf("GoFetch/%s (github.com/go-zoox/fetch)", Version) { 37 | t.Errorf("global UserAgent should be empty string, but got %s", UserAgent) 38 | } 39 | } 40 | 41 | func TestGlobalTimeout(t *testing.T) { 42 | if Timeout != 60*time.Second { 43 | t.Errorf("global Timeout should be empty string, but got %s", Timeout) 44 | } 45 | 46 | SetTimeout(30 * time.Second) 47 | if Timeout != 30*time.Second { 48 | t.Errorf("global Timeout should be test user agent, but got %s", Timeout) 49 | } 50 | 51 | SetTimeout(60 * time.Second) 52 | if Timeout != 60*time.Second { 53 | t.Errorf("global Timeout should be empty string, but got %s", Timeout) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/go-zoox/fetch 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/go-zoox/core-utils v1.2.11 7 | github.com/go-zoox/headers v1.0.6 8 | github.com/go-zoox/testify v1.0.0 9 | github.com/tidwall/gjson v1.14.4 10 | golang.org/x/net v0.23.0 11 | gopkg.in/yaml.v3 v3.0.1 12 | ) 13 | 14 | require ( 15 | github.com/tidwall/match v1.1.1 // indirect 16 | github.com/tidwall/pretty v1.2.1 // indirect 17 | github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/go-zoox/core-utils v1.2.11 h1:3h8P4d+P1XTEzi6M68CywUfy4p8WEZOFuWME8uIYJJ4= 2 | github.com/go-zoox/core-utils v1.2.11/go.mod h1:Y6izFcxuELrkOen5mTQccCJxJqqPJaZV5dQtUMBdkBM= 3 | github.com/go-zoox/headers v1.0.6 h1:LJvVaqs6d+QUvV0sNU8qHFkeyQlECu0mJau1nVFsEQU= 4 | github.com/go-zoox/headers v1.0.6/go.mod h1:WEgEbewswEw4n4qS1iG68Kn/vOQVCAKGwwuZankc6so= 5 | github.com/go-zoox/testify v1.0.0 h1:zXuj+JMcudM/dWk8HgMfCKpGYDcyHbTUBGxH35SGubU= 6 | github.com/go-zoox/testify v1.0.0/go.mod h1:6+UZ2gOcwcnUvR5lclGRnLrE3/mLoQMAGExjrZgs3aA= 7 | github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= 8 | github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 9 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 10 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 11 | github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 12 | github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= 13 | github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 14 | github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 h1:OXcKh35JaYsGMRzpvFkLv/MEyPuL49CThT1pZ8aSml4= 15 | github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= 16 | golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= 17 | golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= 18 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 19 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 20 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 21 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 22 | -------------------------------------------------------------------------------- /head.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | // Head is a wrapper for the Head method of the Client 4 | func Head(url string, config ...interface{}) (*Response, error) { 5 | c := &Config{} 6 | if len(config) == 1 { 7 | c = config[0].(*Config) 8 | } else if len(config) > 1 { 9 | return nil, ErrTooManyArguments 10 | } 11 | 12 | if c.Body != nil { 13 | panic("Request with HEAD method cannot have body") 14 | } 15 | 16 | return New().Head(url, c).Execute() 17 | } 18 | -------------------------------------------------------------------------------- /head_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/go-zoox/testify" 7 | ) 8 | 9 | func TestHead(t *testing.T) { 10 | response, err := Head("https://httpbin.zcorky.com") 11 | if err != nil { 12 | t.Error(err) 13 | } 14 | 15 | if response.Status != 200 { 16 | t.Error("Expected status code 200, got", response.Status) 17 | } 18 | 19 | if response.Headers.Get("content-type") != "text/plain; charset=utf-8" { 20 | t.Error("Expected content-type text/plain; charset=utf-8, got", response.Headers.Get("content-type")) 21 | } 22 | 23 | if response.Headers.Get("server") == "" { 24 | t.Error("Expected server not empty, got empty") 25 | } 26 | } 27 | 28 | func TestHeadParamsError(t *testing.T) { 29 | _, err := Head("") 30 | testify.Assert(t, err != nil, "Expected error, got nil") 31 | 32 | _, err = Head("https://httpbin.zcorky.com/image", &Config{}, &Config{}) 33 | testify.Assert(t, err != nil, "Expected error, got nil") 34 | 35 | _, err = Head("https://httpbin.zcorky.com/image", &Config{}) 36 | testify.Assert(t, err == nil, "Expected nil, got error") 37 | } 38 | -------------------------------------------------------------------------------- /methods.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/go-zoox/headers" 7 | ) 8 | 9 | // Head is http.head 10 | func (f *Fetch) Head(url string, config ...*Config) *Fetch { 11 | return f. 12 | SetConfig(config...). 13 | SetMethod(HEAD). 14 | SetURL(url) 15 | } 16 | 17 | // Get is http.get 18 | func (f *Fetch) Get(url string, config ...*Config) *Fetch { 19 | return f. 20 | SetConfig(config...). 21 | SetMethod(GET). 22 | SetURL(url) 23 | } 24 | 25 | // Post is http.post 26 | func (f *Fetch) Post(url string, config ...*Config) *Fetch { 27 | return f. 28 | SetConfig(config...). 29 | SetMethod(POST). 30 | SetURL(url) 31 | } 32 | 33 | // Put is http.put 34 | func (f *Fetch) Put(url string, config ...*Config) *Fetch { 35 | return f. 36 | SetConfig(config...). 37 | SetMethod(PUT). 38 | SetURL(url) 39 | } 40 | 41 | // Patch is http.patch 42 | func (f *Fetch) Patch(url string, config ...*Config) *Fetch { 43 | return f. 44 | SetConfig(config...). 45 | SetMethod(PATCH). 46 | SetURL(url) 47 | } 48 | 49 | // Delete is http.delete 50 | func (f *Fetch) Delete(url string, config ...*Config) *Fetch { 51 | return f. 52 | SetConfig(config...). 53 | SetMethod(DELETE). 54 | SetURL(url) 55 | } 56 | 57 | // Download downloads file by url 58 | func (f *Fetch) Download(url string, filepath string, config ...*Config) *Fetch { 59 | return f. 60 | SetHeader(headers.AcceptEncoding, "gzip"). 61 | SetConfig(config...). 62 | SetMethod(GET). 63 | SetURL(url). 64 | SetDownloadFilePath(filepath) 65 | } 66 | 67 | // Upload upload a file 68 | func (f *Fetch) Upload(url string, file io.Reader, config ...*Config) *Fetch { 69 | return f. 70 | SetConfig(config...). 71 | SetMethod(POST). 72 | SetURL(url). 73 | SetHeader(headers.ContentType, "multipart/form-data"). 74 | SetBody(map[string]interface{}{ 75 | "file": file, 76 | }) 77 | } 78 | 79 | // Stream ... 80 | func (f *Fetch) Stream(url string, config ...*Config) *Fetch { 81 | var cfg *Config = &Config{} 82 | if len(config) > 0 { 83 | cfg = config[0] 84 | } 85 | 86 | if cfg.Method == "" { 87 | cfg.Method = GET 88 | } 89 | 90 | cfg.IsStream = true 91 | 92 | return f. 93 | SetConfig(cfg). 94 | SetURL(url) 95 | } 96 | 97 | // func (f *Fetch) JSON() *Response { 98 | // f.SetHeader("accept", "application/json") 99 | // return f.Execute() 100 | // } 101 | -------------------------------------------------------------------------------- /methods_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import "testing" 4 | 5 | func TestMethodStream(t *testing.T) { 6 | f := New() 7 | f.Stream("http://example.com") 8 | } 9 | -------------------------------------------------------------------------------- /patch.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | // Patch is a wrapper for the Patch method of the Client 4 | func Patch(url string, config *Config) (*Response, error) { 5 | return New().Patch(url, config).Execute() 6 | } 7 | -------------------------------------------------------------------------------- /patch_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func Test_Patch(t *testing.T) { 9 | response, _ := Patch("https://httpbin.zcorky.com/patch", &Config{ 10 | Body: map[string]interface{}{ 11 | "foo": "bar", 12 | "foo2": "bar2", 13 | "number": 1, 14 | "boolean": true, 15 | "array": []string{ 16 | "foo3", 17 | "bar3", 18 | }, 19 | "nest": map[string]string{ 20 | "foo4": "bar4", 21 | }, 22 | }, 23 | }) 24 | 25 | if response.Status != 200 { 26 | t.Error("Expected status code 200, got", response.Status) 27 | } 28 | 29 | if response.Headers.Get("content-type") != "application/json; charset=utf-8" { 30 | t.Error("Expected content-type application/json; charset=utf-8, got", response.Headers.Get("content-type")) 31 | } 32 | 33 | if response.Headers.Get("server") != "openresty" { 34 | t.Error("Expected server openresty, got", response.Headers.Get("server")) 35 | } 36 | 37 | if response.Get("url").String() != "/patch" { 38 | t.Error("Expected url /patch, got", response.Get("url").String()) 39 | } 40 | 41 | if response.Get("method").String() != "PATCH" { 42 | t.Error("Expected method PATCH, got", response.Get("method").String()) 43 | } 44 | 45 | if response.Get("headers.host").String() != "httpbin.zcorky.com" { 46 | t.Error("Expected Host httpbin.zcorky.com, got", response.Get("headers.Host").String()) 47 | } 48 | 49 | if response.Get("headers.accept-encoding").String() != "gzip" { 50 | t.Error("Expected accept-encoding gzip, got", response.Get("headers.accept-encoding").String()) 51 | } 52 | 53 | if response.Get("headers.user-agent").String() != DefaultUserAgent() { 54 | t.Error(fmt.Sprintf("Expected user-agent %s, got", DefaultUserAgent()), response.Get("headers.user-agent").String()) 55 | } 56 | 57 | if response.Get("headers.connection").String() != "close" { 58 | t.Error("Expected connection close, got", response.Get("headers.connection").String()) 59 | } 60 | 61 | if response.Get("origin").String() != "https://httpbin.zcorky.com" { 62 | t.Error("Expected origin https://httpbin.zcorky.com, got", response.Get("origin").String()) 63 | } 64 | 65 | if response.Get("body.foo").String() != "bar" { 66 | t.Error("Expected body.foo bar, got", response.Get("body.foo").String()) 67 | } 68 | 69 | if response.Get("body.foo2").String() != "bar2" { 70 | t.Error("Expected body.foo2 bar2, got", response.Get("body.foo2").String()) 71 | } 72 | 73 | if response.Get("body.number").Int() != 1 { 74 | t.Error("Expected body.number 1, got", response.Get("body.number").String()) 75 | } 76 | 77 | if response.Get("body.boolean").Bool() != true { 78 | t.Error("Expected body.boolean true, got", response.Get("body.boolean").String()) 79 | } 80 | 81 | if response.Get("body.array.0").String() != "foo3" { 82 | t.Error("Expected body.array foo3, got", response.Get("body.array").String()) 83 | } 84 | 85 | if response.Get("body.array.1").String() != "bar3" { 86 | t.Error("Expected body.array bar3, got", response.Get("body.array").String()) 87 | } 88 | 89 | if response.Get("body.nest.foo4").String() != "bar4" { 90 | t.Error("Expected body.nest.foo4 bar4, got", response.Get("body.nest.foo4").String()) 91 | } 92 | } 93 | 94 | func Test_Patch_With_Header(t *testing.T) { 95 | response, _ := Patch("https://httpbin.zcorky.com/patch", &Config{ 96 | Headers: map[string]string{ 97 | "X-CUSTOM-VAR": "custom-value", 98 | "x-custom-var-2": "custom-value-2", 99 | }, 100 | }) 101 | 102 | if response.Get("headers.x-custom-var").String() != "custom-value" { 103 | t.Error("Expected x-custom-var custom-value, got", response.Get("headers.x-custom-var").String()) 104 | } 105 | 106 | if response.Get("headers.x-custom-var-2").String() != "custom-value-2" { 107 | t.Error("Expected x-custom-var-2 custom-value, got", response.Get("headers.x-custom-var").String()) 108 | } 109 | } 110 | 111 | func Test_Patch_With_Query(t *testing.T) { 112 | response, _ := Patch("https://httpbin.zcorky.com/patch", &Config{ 113 | Query: map[string]string{ 114 | "foo": "bar", 115 | "foo2": "bar2", 116 | }, 117 | }) 118 | 119 | if response.Get("query.foo").String() != "bar" { 120 | t.Error("Expected foo bar, got", response.Get("query.foo").String()) 121 | } 122 | 123 | if response.Get("query.foo2").String() != "bar2" { 124 | t.Error("Expected foo2 bar2, got", response.Get("query.foo2").String()) 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /post.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | // Post is a wrapper for the Post method of the Client 4 | func Post(url string, config *Config) (*Response, error) { 5 | return New().Post(url, config).Execute() 6 | } 7 | -------------------------------------------------------------------------------- /post_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func Test_Post(t *testing.T) { 10 | response, _ := Post("https://httpbin.zcorky.com/post", &Config{ 11 | Body: map[string]interface{}{ 12 | "foo": "bar", 13 | "foo2": "bar2", 14 | "number": 1, 15 | "boolean": true, 16 | "array": []string{ 17 | "foo3", 18 | "bar3", 19 | }, 20 | "nest": map[string]string{ 21 | "foo4": "bar4", 22 | }, 23 | }, 24 | }) 25 | 26 | if response.Status != 200 { 27 | t.Error("Expected status code 200, got", response.Status) 28 | } 29 | 30 | if response.Headers.Get("content-type") != "application/json; charset=utf-8" { 31 | t.Error("Expected content-type application/json; charset=utf-8, got", response.Headers.Get("content-type")) 32 | } 33 | 34 | if response.Headers.Get("server") != "openresty" { 35 | t.Error("Expected server openresty, got", response.Headers.Get("server")) 36 | } 37 | 38 | if response.Get("url").String() != "/post" { 39 | t.Error("Expected url /post, got", response.Get("url").String()) 40 | } 41 | 42 | if response.Get("method").String() != "POST" { 43 | t.Error("Expected method POST, got", response.Get("method").String()) 44 | } 45 | 46 | if response.Get("headers.host").String() != "httpbin.zcorky.com" { 47 | t.Error("Expected Host httpbin.zcorky.com, got", response.Get("headers.Host").String()) 48 | } 49 | 50 | if response.Get("headers.accept-encoding").String() != "gzip" { 51 | t.Error("Expected accept-encoding gzip, got", response.Get("headers.accept-encoding").String()) 52 | } 53 | 54 | if response.Get("headers.user-agent").String() != DefaultUserAgent() { 55 | t.Error(fmt.Sprintf("Expected user-agent %s, got", DefaultUserAgent()), response.Get("headers.user-agent").String()) 56 | } 57 | 58 | if response.Get("headers.connection").String() != "close" { 59 | t.Error("Expected connection close, got", response.Get("headers.connection").String()) 60 | } 61 | 62 | if response.Get("origin").String() != "https://httpbin.zcorky.com" { 63 | t.Error("Expected origin https://httpbin.zcorky.com, got", response.Get("origin").String()) 64 | } 65 | 66 | if response.Get("body.foo").String() != "bar" { 67 | t.Error("Expected body.foo bar, got", response.Get("body.foo").String()) 68 | } 69 | 70 | if response.Get("body.foo2").String() != "bar2" { 71 | t.Error("Expected body.foo2 bar2, got", response.Get("body.foo2").String()) 72 | } 73 | 74 | if response.Get("body.number").Int() != 1 { 75 | t.Error("Expected body.number 1, got", response.Get("body.number").String()) 76 | } 77 | 78 | if response.Get("body.boolean").Bool() != true { 79 | t.Error("Expected body.boolean true, got", response.Get("body.boolean").String()) 80 | } 81 | 82 | if response.Get("body.array.0").String() != "foo3" { 83 | t.Error("Expected body.array foo3, got", response.Get("body.array").String()) 84 | } 85 | 86 | if response.Get("body.array.1").String() != "bar3" { 87 | t.Error("Expected body.array bar3, got", response.Get("body.array").String()) 88 | } 89 | 90 | if response.Get("body.nest.foo4").String() != "bar4" { 91 | t.Error("Expected body.nest.foo4 bar4, got", response.Get("body.nest.foo4").String()) 92 | } 93 | } 94 | 95 | func Test_Post_With_Header(t *testing.T) { 96 | response, _ := Post("https://httpbin.zcorky.com/post", &Config{ 97 | Headers: map[string]string{ 98 | "X-CUSTOM-VAR": "custom-value", 99 | "x-custom-var-2": "custom-value-2", 100 | }, 101 | }) 102 | 103 | if response.Get("headers.x-custom-var").String() != "custom-value" { 104 | t.Error("Expected x-custom-var custom-value, got", response.Get("headers.x-custom-var").String()) 105 | } 106 | 107 | if response.Get("headers.x-custom-var-2").String() != "custom-value-2" { 108 | t.Error("Expected x-custom-var-2 custom-value, got", response.Get("headers.x-custom-var").String()) 109 | } 110 | } 111 | 112 | func Test_Post_With_Query(t *testing.T) { 113 | response, _ := Post("https://httpbin.zcorky.com/post", &Config{ 114 | Query: map[string]string{ 115 | "foo": "bar", 116 | "foo2": "bar2", 117 | }, 118 | }) 119 | 120 | if response.Get("query.foo").String() != "bar" { 121 | t.Error("Expected foo bar, got", response.Get("query.foo").String()) 122 | } 123 | 124 | if response.Get("query.foo2").String() != "bar2" { 125 | t.Error("Expected foo2 bar2, got", response.Get("query.foo2").String()) 126 | } 127 | } 128 | 129 | func Test_Post_With_UrlFormEncoded(t *testing.T) { 130 | response, _ := Post("https://httpbin.zcorky.com/post", &Config{ 131 | Headers: map[string]string{ 132 | "Content-Type": "application/x-www-form-urlencoded", 133 | }, 134 | Body: map[string]string{ 135 | "foo": "bar", 136 | "foo2": "bar2", 137 | }, 138 | }) 139 | 140 | // fmt.Println("response:", response.String()) 141 | 142 | if response.Get("body.foo").String() != "bar" { 143 | t.Error("Expected foo bar, got", response.Get("body.foo").String()) 144 | } 145 | 146 | if response.Get("body.foo2").String() != "bar2" { 147 | t.Error("Expected foo2 bar2, got", response.Get("body.foo2").String()) 148 | } 149 | } 150 | 151 | func Test_Post_With_FormData(t *testing.T) { 152 | response, err := Post("https://httpbin.zcorky.com/post", &Config{ 153 | Headers: map[string]string{ 154 | "Content-Type": "multipart/form-data", 155 | }, 156 | Body: map[string]string{ 157 | "foo": "bar", 158 | "foo2": "bar2", 159 | }, 160 | }) 161 | if err != nil { 162 | t.Error(err) 163 | } 164 | 165 | // fmt.Println("response:", response.String()) 166 | 167 | if response.Get("body.foo").String() != "bar" { 168 | t.Error("Expected foo bar, got", response.Get("body.foo").String()) 169 | } 170 | 171 | if response.Get("body.foo2").String() != "bar2" { 172 | t.Error("Expected foo2 bar2, got", response.Get("body.foo2").String()) 173 | } 174 | } 175 | 176 | func Test_Post_With_FormData_Upload_File(t *testing.T) { 177 | file, _ := os.Open("go.mod") 178 | 179 | response, err := Post("https://httpbin.zcorky.com/upload", &Config{ 180 | Headers: map[string]string{ 181 | "Content-Type": "multipart/form-data", 182 | }, 183 | Body: map[string]interface{}{ 184 | "foo": "bar", 185 | "thefilename": file, 186 | }, 187 | }) 188 | if err != nil { 189 | t.Error(err) 190 | return 191 | } 192 | 193 | // fmt.Println("response:", response.String()) 194 | 195 | if response.Get("body.foo").String() != "bar" { 196 | t.Error("Expected foo bar, got", response.Get("body.foo").String()) 197 | } 198 | 199 | if response.Get("files.thefilename.name").String() != "go.mod" { 200 | t.Error("Expected thefilename go.mod, got", response.Get("files.thefilename.name").String()) 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /progress.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | // inspired by: 4 | // https://github.com/schollz/progressbar/blob/master/progressbar.go 5 | // https://stackoverflow.com/questions/26050380/go-tracking-post-request-progress 6 | 7 | // Progress is a progress event 8 | type Progress struct { 9 | Reporter func(percent int64, current, total int64) 10 | Total int64 11 | Current int64 12 | } 13 | 14 | // Write writes the data to the progress event 15 | func (p *Progress) Write(b []byte) (n int, err error) { 16 | n = len(b) 17 | p.Current += int64(n) 18 | p.Reporter(p.Current/p.Total, p.Current, p.Total) 19 | return 20 | } 21 | -------------------------------------------------------------------------------- /put.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | // Put is a wrapper for the Put method of the Client 4 | func Put(url string, config *Config) (*Response, error) { 5 | return New().Put(url, config).Execute() 6 | } 7 | -------------------------------------------------------------------------------- /put_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func Test_Put(t *testing.T) { 9 | response, err := Put("https://httpbin.zcorky.com/put", &Config{ 10 | Body: map[string]interface{}{ 11 | "foo": "bar", 12 | "foo2": "bar2", 13 | "number": 1, 14 | "boolean": true, 15 | "array": []string{ 16 | "foo3", 17 | "bar3", 18 | }, 19 | "nest": map[string]string{ 20 | "foo4": "bar4", 21 | }, 22 | }, 23 | }) 24 | 25 | if err != nil { 26 | t.Error(err) 27 | return 28 | } 29 | 30 | if response.Status != 200 { 31 | t.Error("Expected status code 200, got", response.Status) 32 | } 33 | 34 | if response.Headers.Get("content-type") != "application/json; charset=utf-8" { 35 | t.Error("Expected content-type application/json; charset=utf-8, got", response.Headers.Get("content-type")) 36 | return 37 | } 38 | 39 | if response.Headers.Get("server") != "openresty" { 40 | t.Error("Expected server openresty, got", response.Headers.Get("server")) 41 | } 42 | 43 | if response.Get("url").String() != "/put" { 44 | t.Error("Expected url /put, got", response.Get("url").String()) 45 | } 46 | 47 | if response.Get("method").String() != "PUT" { 48 | t.Error("Expected method PUT, got", response.Get("method").String()) 49 | } 50 | 51 | if response.Get("headers.host").String() != "httpbin.zcorky.com" { 52 | t.Error("Expected Host httpbin.zcorky.com, got", response.Get("headers.Host").String()) 53 | } 54 | 55 | if response.Get("headers.accept-encoding").String() != "gzip" { 56 | t.Error("Expected accept-encoding gzip, got", response.Get("headers.accept-encoding").String()) 57 | } 58 | 59 | if response.Get("headers.user-agent").String() != DefaultUserAgent() { 60 | t.Error(fmt.Sprintf("Expected user-agent %s, got", DefaultUserAgent()), response.Get("headers.user-agent").String()) 61 | } 62 | 63 | if response.Get("headers.connection").String() != "close" { 64 | t.Error("Expected connection close, got", response.Get("headers.connection").String()) 65 | } 66 | 67 | if response.Get("origin").String() != "https://httpbin.zcorky.com" { 68 | t.Error("Expected origin https://httpbin.zcorky.com, got", response.Get("origin").String()) 69 | } 70 | 71 | if response.Get("body.foo").String() != "bar" { 72 | t.Error("Expected body.foo bar, got", response.Get("body.foo").String()) 73 | } 74 | 75 | if response.Get("body.foo2").String() != "bar2" { 76 | t.Error("Expected body.foo2 bar2, got", response.Get("body.foo2").String()) 77 | } 78 | 79 | if response.Get("body.number").Int() != 1 { 80 | t.Error("Expected body.number 1, got", response.Get("body.number").String()) 81 | } 82 | 83 | if response.Get("body.boolean").Bool() != true { 84 | t.Error("Expected body.boolean true, got", response.Get("body.boolean").String()) 85 | } 86 | 87 | if response.Get("body.array.0").String() != "foo3" { 88 | t.Error("Expected body.array foo3, got", response.Get("body.array").String()) 89 | } 90 | 91 | if response.Get("body.array.1").String() != "bar3" { 92 | t.Error("Expected body.array bar3, got", response.Get("body.array").String()) 93 | } 94 | 95 | if response.Get("body.nest.foo4").String() != "bar4" { 96 | t.Error("Expected body.nest.foo4 bar4, got", response.Get("body.nest.foo4").String()) 97 | } 98 | } 99 | 100 | func Test_Put_With_Header(t *testing.T) { 101 | response, _ := Put("https://httpbin.zcorky.com/put", &Config{ 102 | Headers: map[string]string{ 103 | "X-CUSTOM-VAR": "custom-value", 104 | "x-custom-var-2": "custom-value-2", 105 | }, 106 | }) 107 | 108 | if response.Get("headers.x-custom-var").String() != "custom-value" { 109 | t.Error("Expected x-custom-var custom-value, got", response.Get("headers.x-custom-var").String()) 110 | } 111 | 112 | if response.Get("headers.x-custom-var-2").String() != "custom-value-2" { 113 | t.Error("Expected x-custom-var-2 custom-value, got", response.Get("headers.x-custom-var").String()) 114 | } 115 | } 116 | 117 | func Test_Put_With_Query(t *testing.T) { 118 | response, _ := Put("https://httpbin.zcorky.com/put", &Config{ 119 | Query: map[string]string{ 120 | "foo": "bar", 121 | "foo2": "bar2", 122 | }, 123 | }) 124 | 125 | if response.Get("query.foo").String() != "bar" { 126 | t.Error("Expected foo bar, got", response.Get("query.foo").String()) 127 | } 128 | 129 | if response.Get("query.foo2").String() != "bar2" { 130 | t.Error("Expected foo2 bar2, got", response.Get("query.foo2").String()) 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /response.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "strconv" 10 | 11 | "github.com/go-zoox/headers" 12 | "github.com/tidwall/gjson" 13 | "gopkg.in/yaml.v3" 14 | ) 15 | 16 | // Response is the fetch response 17 | type Response struct { 18 | Status int 19 | Headers http.Header 20 | Body []byte 21 | resultCache gjson.Result 22 | parsed bool 23 | // 24 | Request *Config 25 | // 26 | Stream io.ReadCloser 27 | } 28 | 29 | // String returns the body as string 30 | func (r *Response) String() string { 31 | return string(r.Body) 32 | } 33 | 34 | // Value returns the body as gjson.Result 35 | func (r *Response) Value() gjson.Result { 36 | if !r.parsed { 37 | r.resultCache = gjson.Parse(r.String()) 38 | r.parsed = true 39 | } 40 | 41 | return r.resultCache 42 | } 43 | 44 | // Get returns the value of the key 45 | func (r *Response) Get(key string) gjson.Result { 46 | return r.Value().Get(key) 47 | } 48 | 49 | // JSON returns the body as json string 50 | func (r *Response) JSON() (string, error) { 51 | raw := r.String() 52 | b, err := json.MarshalIndent(gjson.Parse(raw).Value(), "", " ") 53 | if err != nil { 54 | return "", errors.New("invalid json: " + raw) 55 | } 56 | 57 | return string(b), nil 58 | } 59 | 60 | // func (r *Response) Unmarshal(v interface{}) error { 61 | // return json.Unmarshal(r.Body, v) 62 | // // return decode(v, r) 63 | // } 64 | 65 | // UnmarshalJSON unmarshals body to json struct 66 | // 67 | // @TODO bug when lint (go vet) method UnmarshalJSON(v interface{}) error should have signature UnmarshalJSON([]byte) error 68 | func (r *Response) UnmarshalJSON(v interface{}) error { 69 | return json.Unmarshal(r.Body, v) 70 | } 71 | 72 | // UnmarshalYAML unmarshals body to yaml struct 73 | func (r *Response) UnmarshalYAML(v interface{}) error { 74 | return yaml.Unmarshal(r.Body, v) 75 | } 76 | 77 | // Ok returns true if status code is 2xx 78 | func (r *Response) Ok() bool { 79 | return r.Status >= 200 && r.Status < 300 80 | } 81 | 82 | // Error returns error with status and response string. 83 | func (r *Response) Error() error { 84 | return fmt.Errorf("[%d] %s", r.Status, r.String()) 85 | } 86 | 87 | // StatusCode returns status code of the response 88 | func (r *Response) StatusCode() int { 89 | return r.Status 90 | } 91 | 92 | // StatusText returns status text of the response 93 | func (r *Response) StatusText() string { 94 | return http.StatusText(r.Status) 95 | } 96 | 97 | // ContentType returns content type of the response 98 | func (r *Response) ContentType() string { 99 | return r.Headers.Get(headers.ContentType) 100 | } 101 | 102 | // Location returns location of the response 103 | func (r *Response) Location() string { 104 | return r.Headers.Get(headers.Location) 105 | } 106 | 107 | // ContentLength returns content length of the response 108 | func (r *Response) ContentLength() int { 109 | vs := r.Headers.Get(headers.ContentLength) 110 | if vs == "" { 111 | return 0 112 | } 113 | 114 | value, err := strconv.Atoi(vs) 115 | if err != nil { 116 | return 0 117 | } 118 | 119 | return value 120 | } 121 | 122 | // ContentEncoding returns content encoding of the response 123 | func (r *Response) ContentEncoding() string { 124 | return r.Headers.Get(headers.ContentEncoding) 125 | } 126 | 127 | // TransferEncoding returns transfer encoding of the response 128 | func (r *Response) TransferEncoding() string { 129 | return r.Headers.Get(headers.TransferEncoding) 130 | } 131 | 132 | // ContentLanguage returns content language of the response 133 | func (r *Response) ContentLanguage() string { 134 | return r.Headers.Get(headers.ContentLanguage) 135 | } 136 | 137 | // XPoweredBy returns x-powered-by of the response 138 | func (r *Response) XPoweredBy() string { 139 | return r.Headers.Get(headers.XPoweredBy) 140 | } 141 | 142 | // XRequestID returns x-request-id of the response 143 | func (r *Response) XRequestID() string { 144 | return r.Headers.Get(headers.XRequestID) 145 | } 146 | 147 | // AcceptRanges returns x-accept-ranges of the response 148 | func (r *Response) AcceptRanges() string { 149 | return r.Headers.Get(headers.AcceptRanges) 150 | } 151 | 152 | // SetCookie returns set-cookie of the response 153 | func (r *Response) SetCookie() string { 154 | return r.Headers.Get(headers.SetCookie) 155 | } 156 | -------------------------------------------------------------------------------- /response_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "os" 7 | "testing" 8 | 9 | "github.com/go-zoox/testify" 10 | ) 11 | 12 | func TestResponse(t *testing.T) { 13 | os.Setenv(EnvDEBUG, "true") 14 | 15 | r := &Response{} 16 | testify.Equal(t, r.Status, 0) 17 | 18 | r.Body, _ = json.Marshal(map[string]string{ 19 | "key": "value", 20 | }) 21 | testify.Equal(t, string(r.Body), `{"key":"value"}`) 22 | 23 | testify.Equal(t, r.String(), `{"key":"value"}`) 24 | 25 | jsonStr, _ := r.JSON() 26 | testify.Equal(t, "{\n \"key\": \"value\"\n}", jsonStr) 27 | 28 | r.Status = 400 29 | testify.Equal(t, "[400] {\"key\":\"value\"}", r.Error().Error()) 30 | 31 | testify.Equal(t, "Bad Request", r.StatusText()) 32 | 33 | // content type 34 | testify.Equal(t, "", r.ContentType()) 35 | r.Headers = http.Header{} 36 | r.Headers.Set("content-type", "text/plain; charset=utf-8") 37 | testify.Equal(t, "text/plain; charset=utf-8", r.ContentType()) 38 | 39 | // location 40 | testify.Equal(t, "", r.Location()) 41 | r.Headers.Set("location", "https://httpbin.zcorky.com/image") 42 | testify.Equal(t, "https://httpbin.zcorky.com/image", r.Location()) 43 | 44 | // content length 45 | testify.Equal(t, 0, r.ContentLength()) 46 | r.Headers.Set("content-length", "10") 47 | testify.Equal(t, 10, r.ContentLength()) 48 | 49 | // transfer encoding 50 | testify.Equal(t, "", r.TransferEncoding()) 51 | r.Headers.Set("transfer-encoding", "chunked") 52 | testify.Equal(t, "chunked", r.TransferEncoding()) 53 | 54 | // content language 55 | testify.Equal(t, "", r.ContentLanguage()) 56 | r.Headers.Set("content-language", "en-US") 57 | testify.Equal(t, "en-US", r.ContentLanguage()) 58 | 59 | // content encoding 60 | testify.Equal(t, "", r.ContentEncoding()) 61 | r.Headers.Set("content-encoding", "gzip") 62 | testify.Equal(t, "gzip", r.ContentEncoding()) 63 | 64 | // x-powered-by 65 | testify.Equal(t, "", r.XPoweredBy()) 66 | r.Headers.Set("x-powered-by", "Go") 67 | testify.Equal(t, "Go", r.XPoweredBy()) 68 | 69 | // x-request-id 70 | testify.Equal(t, "", r.XRequestID()) 71 | r.Headers.Set("x-request-id", "12345") 72 | testify.Equal(t, "12345", r.XRequestID()) 73 | 74 | // accept-ranges 75 | testify.Equal(t, "", r.AcceptRanges()) 76 | r.Headers.Set("accept-ranges", "bytes") 77 | testify.Equal(t, "bytes", r.AcceptRanges()) 78 | 79 | // set cookie 80 | testify.Equal(t, "", r.SetCookie()) 81 | r.Headers.Set("set-cookie", "key=value") 82 | testify.Equal(t, "key=value", r.SetCookie()) 83 | 84 | // ok 85 | r.Status = 200 86 | testify.Equal(t, true, r.Ok()) 87 | r.Status = 400 88 | testify.Equal(t, false, r.Ok()) 89 | r.Status = 500 90 | testify.Equal(t, false, r.Ok()) 91 | r.Status = 201 92 | testify.Equal(t, true, r.Ok()) 93 | r.Status = 202 94 | testify.Equal(t, true, r.Ok()) 95 | 96 | // status code 97 | testify.Equal(t, 202, r.StatusCode()) 98 | 99 | // DEBUG 100 | os.Setenv(EnvDEBUG, "") 101 | } 102 | -------------------------------------------------------------------------------- /session.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | // Session is remembered between requests 4 | func Session() *Fetch { 5 | f := New() 6 | 7 | f.config.IsSession = true 8 | 9 | return f 10 | } 11 | -------------------------------------------------------------------------------- /session_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/go-zoox/testify" 7 | ) 8 | 9 | func TestSession(t *testing.T) { 10 | f := Session() 11 | testify.Assert(t, f != nil, "Expected error, got nil") 12 | } 13 | -------------------------------------------------------------------------------- /stream.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | // Stream is a wrapper for the Stream method of the Client 4 | func Stream(url string, config ...interface{}) (*Response, error) { 5 | c := &Config{} 6 | if len(config) == 1 { 7 | c = config[0].(*Config) 8 | } else if len(config) > 1 { 9 | return nil, ErrTooManyArguments 10 | } 11 | 12 | return New().Stream(url, c).Execute() 13 | } 14 | -------------------------------------------------------------------------------- /stream_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/go-zoox/testify" 7 | ) 8 | 9 | func TestStream(t *testing.T) { 10 | _, err := Stream("") 11 | testify.Assert(t, err != nil, "Expected error, got nil") 12 | 13 | _, err = Stream("https://httpbin.zcorky.com/image", &Config{}, &Config{}) 14 | testify.Assert(t, err != nil, "Expected error, got nil") 15 | 16 | _, err = Stream("https://httpbin.zcorky.com/image", &Config{}) 17 | testify.Assert(t, err == nil, "Expected nil, got error") 18 | } 19 | -------------------------------------------------------------------------------- /tag.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | 7 | "github.com/tidwall/gjson" 8 | ) 9 | 10 | // DataSource defines the interface for loading data from a data source. 11 | type DataSource interface { 12 | Get(key string) string 13 | } 14 | 15 | func decodeWithTagFromDataSource(ptr interface{}, tagName string, dataSource gjson.Result) error { 16 | t := reflect.TypeOf(ptr).Elem() 17 | v := reflect.ValueOf(ptr).Elem() 18 | 19 | for i := 0; i < t.NumField(); i++ { 20 | typ := t.Field(i) 21 | val := v.Field(i) 22 | 23 | kind := val.Kind() 24 | 25 | tagValueName := typ.Tag.Get(tagName) 26 | tabValueDfeault := typ.Tag.Get("default") 27 | tagValueRequired := typ.Tag.Get("required") 28 | 29 | switch kind { 30 | case reflect.String: 31 | tagValue := dataSource.Get(tagValueName).String() 32 | if tagValue == "" && tabValueDfeault != "" { 33 | tagValue = tabValueDfeault 34 | } 35 | 36 | if tagValueRequired == "true" && tagValue == "" { 37 | return fmt.Errorf("%s is required", tagValueName) 38 | } 39 | 40 | val.SetString(tagValue) 41 | case reflect.Bool: 42 | tagValue := dataSource.Get(tagValueName).Bool() 43 | val.SetBool(tagValue) 44 | case reflect.Int, reflect.Int64: 45 | tagValue := dataSource.Get(tagValueName).Int() 46 | if tagValueRequired == "true" && tagValue == 0 { 47 | return fmt.Errorf("%s is required", tagValueName) 48 | } 49 | 50 | val.SetInt(tagValue) 51 | case reflect.Float64: 52 | tagValue := dataSource.Get(tagValueName).Float() 53 | if tagValueRequired == "true" && tagValue == 0.0 { 54 | return fmt.Errorf("%s is required", tagValueName) 55 | } 56 | 57 | val.SetFloat(tagValue) 58 | } 59 | } 60 | 61 | return nil 62 | } 63 | 64 | // Decode decodes the given struct pointer from the environment. 65 | func decode(ptr interface{}, response *Response) error { 66 | return decodeWithTagFromDataSource(ptr, "alias", response.Value()) 67 | } 68 | -------------------------------------------------------------------------------- /tag_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import "testing" 4 | 5 | func TestTagDecode(t *testing.T) { 6 | type body struct { 7 | URL string `alias:"url"` 8 | Method string `alias:"method"` 9 | Num int `alias:"number"` 10 | IsBase64 bool `alias:"is_base64"` 11 | Float float64 `alias:"float"` 12 | } 13 | 14 | var b body 15 | response := &Response{ 16 | Status: 200, 17 | Body: []byte(`{ 18 | "url": "/get", 19 | "method":"GET", 20 | "number": 10, 21 | "is_base64": true, 22 | "float": 1.1 23 | }`), 24 | } 25 | 26 | if err := decode(&b, response); err != nil { 27 | t.Error(err) 28 | } 29 | 30 | if b.URL != "/get" { 31 | t.Error("Expected url /get, got", b.URL) 32 | } 33 | 34 | if b.Method != "GET" { 35 | t.Error("Expected method GET, got", b.Method) 36 | } 37 | 38 | if b.Num != 10 { 39 | t.Error("Expected num 10, got", b.Num) 40 | } 41 | 42 | if b.IsBase64 != true { 43 | t.Error("Expected IsBase64 false, got", b.IsBase64) 44 | } 45 | 46 | if b.Float != 1.1 { 47 | t.Error("Expected Float 1.1, got", b.Float) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /upload.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import "io" 4 | 5 | // Upload is a wrapper for the Upload method of the Client 6 | func Upload(url string, file io.Reader, config ...interface{}) (*Response, error) { 7 | c := &Config{} 8 | if len(config) == 1 { 9 | c = config[0].(*Config) 10 | } else if len(config) > 1 { 11 | return nil, ErrTooManyArguments 12 | } 13 | 14 | return New().Upload(url, file, c).Execute() 15 | } 16 | -------------------------------------------------------------------------------- /upload_test.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestUpload(t *testing.T) { 9 | file, _ := os.Open("go.mod") 10 | 11 | response, err := Upload("https://httpbin.zcorky.com/upload", file) 12 | if err != nil { 13 | t.Error(err) 14 | } 15 | 16 | if response.Status != 200 { 17 | t.Error("Expected status code 200, got", response.Status) 18 | } 19 | 20 | if response.Get("files.file.name").String() != "go.mod" { 21 | t.Error("Expected file go.mod, got", response.Get("files.file.name").String()) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package fetch 2 | 3 | // Version is the version of this package 4 | var Version = "1.8.6" 5 | --------------------------------------------------------------------------------