├── .gitignore ├── docs └── assets │ ├── avatar.jpg │ └── cover.jpg ├── error_test.go ├── go.mod ├── .travis.yml ├── benchmark_test.go ├── LICENSE ├── formdata.go ├── request.go ├── error.go ├── README.md ├── datatype_test.go ├── api_test.go ├── response_test.go ├── api.go ├── download_test.go ├── download.go ├── session_test.go ├── response.go ├── go.sum ├── datatype.go └── session.go /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | *.DS_Store 3 | *.idea 4 | -------------------------------------------------------------------------------- /docs/assets/avatar.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wnanbei/direwolf/HEAD/docs/assets/avatar.jpg -------------------------------------------------------------------------------- /docs/assets/cover.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wnanbei/direwolf/HEAD/docs/assets/cover.jpg -------------------------------------------------------------------------------- /error_test.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | ) 7 | 8 | func TestError(t *testing.T) { 9 | err1 := ErrRequestBody 10 | err2 := WrapErr(err1, "second testing") 11 | err3 := WrapErrf(err2, "==%s==", "third testing") 12 | 13 | if !errors.Is(err3, ErrRequestBody) { 14 | t.Fatal("Test errors.Is failed.") 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/wnanbei/direwolf 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/PuerkitoBio/goquery v1.5.0 7 | github.com/andybalholm/cascadia v1.1.0 // indirect 8 | github.com/gin-gonic/gin v1.7.7 9 | github.com/json-iterator/go v1.1.9 10 | github.com/tidwall/gjson v1.14.0 11 | github.com/valyala/fasthttp v1.35.0 12 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f 13 | golang.org/x/text v0.3.7 14 | ) 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | os: 4 | - linux 5 | - osx 6 | 7 | go: 8 | - 1.13.x 9 | 10 | matrix: 11 | fast_finish: true 12 | 13 | script: 14 | - go test -v -race -coverprofile=coverage.txt -covermode=atomic 15 | 16 | after_success: 17 | - bash <(curl -s https://codecov.io/bash) 18 | 19 | notifications: 20 | email: 21 | recipients: 22 | - wnanbei@gmail.com 23 | on_success: never # default: change 24 | on_failure: always # default: always -------------------------------------------------------------------------------- /benchmark_test.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "testing" 7 | 8 | "github.com/valyala/fasthttp" 9 | ) 10 | 11 | func BenchmarkDirewolfGet(b *testing.B) { 12 | ts := newTestResponseServer() 13 | defer ts.Close() 14 | b.ResetTimer() 15 | for i := 0; i < b.N; i++ { 16 | resp, _ := Get(ts.URL) 17 | resp.Text() 18 | } 19 | } 20 | 21 | func BenchmarkGolangGet(b *testing.B) { 22 | ts := newTestResponseServer() 23 | defer ts.Close() 24 | b.ResetTimer() 25 | for i := 0; i < b.N; i++ { 26 | resp, _ := http.Get(ts.URL) 27 | ioutil.ReadAll(resp.Body) 28 | resp.Body.Close() 29 | } 30 | } 31 | 32 | func BenchmarkFasthttpGet(b *testing.B) { 33 | ts := newTestResponseServer() 34 | defer ts.Close() 35 | b.ResetTimer() 36 | client := fasthttp.Client{} 37 | for i := 0; i < b.N; i++ { 38 | req := fasthttp.AcquireRequest() 39 | resp := fasthttp.AcquireResponse() 40 | req.SetRequestURI(ts.URL) 41 | client.Do(req, resp) 42 | req.Reset() 43 | resp.Reset() 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 wnanbei 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /formdata.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "mime/multipart" 7 | "os" 8 | "path/filepath" 9 | ) 10 | 11 | type MultipartForm struct { 12 | body *bytes.Buffer 13 | m *multipart.Writer 14 | } 15 | 16 | func NewMultipartForm() *MultipartForm { 17 | body := bytes.Buffer{} 18 | w := multipart.NewWriter(&body) 19 | 20 | mf := MultipartForm{ 21 | body: &body, 22 | m: w, 23 | } 24 | return &mf 25 | } 26 | 27 | func (mf *MultipartForm) WriteField(key, value string) error { 28 | return mf.m.WriteField(key, value) 29 | } 30 | 31 | func (mf *MultipartForm) WriteFile(key, filePath string) error { 32 | _, fileName := filepath.Split(filePath) 33 | 34 | fw, err := mf.m.CreateFormFile(key, fileName) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | f, err := os.Open(filePath) 40 | if err != nil { 41 | return err 42 | } 43 | defer f.Close() 44 | 45 | _, err = io.Copy(fw, f) 46 | return err 47 | } 48 | 49 | func (mf *MultipartForm) Close() error { 50 | return mf.m.Close() 51 | } 52 | 53 | func (mf *MultipartForm) Reader() *bytes.Reader { 54 | return bytes.NewReader(mf.body.Bytes()) 55 | } 56 | 57 | func (mf *MultipartForm) ContentType() string { 58 | return mf.m.FormDataContentType() 59 | } 60 | 61 | func (mf *MultipartForm) bindRequest(request *Request) error { 62 | request.MultipartForm = mf 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /request.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | ) 7 | 8 | // Request is a prepared request setting, you should construct it by using 9 | // NewRequest(). 10 | type Request struct { 11 | Method string 12 | URL string 13 | Headers http.Header 14 | Body []byte 15 | JsonBody []byte 16 | Params *Params 17 | PostForm *PostForm 18 | Cookies Cookies 19 | Proxy *Proxy 20 | RedirectNum int 21 | Timeout int 22 | MultipartForm *MultipartForm 23 | } 24 | 25 | // NewRequest construct a Request by passing the parameters. 26 | // 27 | // You can construct this request by passing the following parameters: 28 | // method: Method for the request. 29 | // url: URL for the request. 30 | // direwolf.Header: HTTP Headers to send. 31 | // direwolf.Params: Parameters to send in the query string. 32 | // direwolf.Cookies: Cookies to send. 33 | // direwolf.PostForm: Post data form to send. 34 | // direwolf.Body: Post body to send. 35 | // direwolf.Proxy: Proxy url to use. 36 | // direwolf.Timeout: Request Timeout. 37 | // direwolf.RedirectNum: Number of Request allowed to redirect. 38 | func NewRequest(method string, URL string, args ...RequestOption) (req *Request, err error) { 39 | req = &Request{} // new a Request and set default field 40 | req.Method = strings.ToUpper(method) // Upper the method string 41 | req.URL = URL 42 | 43 | // Check the type of the parameter and handle it. 44 | for _, arg := range args { 45 | if err := arg.bindRequest(req); err != nil { 46 | return nil, WrapErr(err, "set request parameters failed.") 47 | } 48 | } 49 | return req, nil 50 | } 51 | -------------------------------------------------------------------------------- /error.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "runtime" 7 | "strconv" 8 | "time" 9 | ) 10 | 11 | var ( 12 | ErrRequestBody = errors.New("request body can`t coexists with PostForm") 13 | ErrTimeout = errors.New("reqeust timeout") 14 | ) 15 | 16 | type RedirectError struct { 17 | RedirectNum int 18 | } 19 | 20 | func (e *RedirectError) Error() string { 21 | return "exceeded the maximum number of redirects: " + strconv.Itoa(e.RedirectNum) 22 | } 23 | 24 | type Error struct { 25 | // wrapped error 26 | err error 27 | msg string 28 | // file path and name 29 | file string 30 | fileLine int 31 | time string 32 | } 33 | 34 | func (e *Error) Error() string { 35 | _, ok := e.err.(interface { 36 | Unwrap() error 37 | }) 38 | if ok { 39 | return fmt.Sprintf("%s - %s:%d\n%s\n%s", e.time, e.file, e.fileLine, e.msg, e.err.Error()) 40 | } 41 | return fmt.Sprintf("%s - %s:%d\n%s\n\n%s\n", e.time, e.file, e.fileLine, e.msg, e.err.Error()) 42 | } 43 | 44 | func (e *Error) Unwrap() error { 45 | if e.err != nil { 46 | return e.err 47 | } 48 | return nil 49 | } 50 | 51 | // WrapErr will wrap a error with some information: filename, line, time and some message. 52 | func WrapErr(err error, msg string) error { 53 | _, file, line, _ := runtime.Caller(1) 54 | return &Error{ 55 | err: err, 56 | msg: msg, 57 | file: file, 58 | fileLine: line, 59 | time: time.Now().Format("2006-01-02 15:04:05"), 60 | } 61 | } 62 | 63 | // WrapErrf will wrap a error with some information: filename, line, time and some message. 64 | // You can format message of error. 65 | func WrapErrf(err error, format string, args ...interface{}) error { 66 | msg := fmt.Sprintf(format, args...) 67 | _, file, line, _ := runtime.Caller(1) 68 | return &Error{ 69 | err: err, 70 | msg: msg, 71 | file: file, 72 | fileLine: line, 73 | time: time.Now().Format("2006-01-02 15:04:05"), 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Direwolf HTTP Client: Save your time 2 | 3 | [![Build Status](https://travis-ci.org/wnanbei/direwolf.svg?branch=master)](https://travis-ci.org/wnanbei/direwolf) 4 | [![codecov](https://codecov.io/gh/wnanbei/direwolf/branch/dev/graph/badge.svg)](https://codecov.io/gh/wnanbei/direwolf) 5 | ![GitHub release (latest by date)](https://img.shields.io/github/v/release/wnanbei/direwolf) 6 | ![language](https://img.shields.io/badge/language-Golang%201.13%2B-blue) 7 | ![GitHub](https://img.shields.io/github/license/wnanbei/direwolf) 8 | 9 | Package direwolf is a convenient and easy to use http client written in Golang. 10 | 11 | If you want find more info, please go here: [Direwolf HTTP Client: Save your time](https://wnanbei.github.io/direwolf/)，内有中文文档。 12 | 13 | ![direwolf](docs/assets/cover.jpg) 14 | 15 | ## Feature Support 16 | 17 | - Clean and Convenient API 18 | - Simple to Set Headers, Cookies, Parameters, Post Forms 19 | - Sessions Control 20 | - Keep-Alive & Connection Pooling 21 | - HTTP(S) Proxy Support 22 | - Redirect Control 23 | - Timeout Control 24 | - Support extract result from response body with css selector, regexp, json 25 | - Content Decoding 26 | - More to come... 27 | 28 | ## Installation 29 | 30 | ```text 31 | go get github.com/wnanbei/direwolf 32 | ``` 33 | 34 | ## Take a glance 35 | 36 | You can easily send a request like this: 37 | 38 | ```go 39 | import ( 40 | "fmt" 41 | 42 | dw "github.com/wnanbei/direwolf" 43 | ) 44 | 45 | func main() { 46 | resp, err := dw.Get("https://www.google.com") 47 | if err != nil { 48 | return 49 | } 50 | fmt.Println(resp.Text()) 51 | } 52 | ``` 53 | 54 | Besides, direwolf provide a convenient way to add parameters to request. Such 55 | as Headers, Cookies, Params, etc. 56 | 57 | ```go 58 | import ( 59 | "fmt" 60 | 61 | dw "github.com/wnanbei/direwolf" 62 | ) 63 | 64 | func main() { 65 | headers := dw.NewHeaders( 66 | "User-Agent", "direwolf", 67 | ) 68 | params := dw.NewParams( 69 | "name", "wnanbei", 70 | "age", "18", 71 | ) 72 | cookies := dw.NewCookies( 73 | "sign", "kzhxciuvyqwekhiuxcyvnkjdhiue", 74 | ) 75 | resp, err := dw.Get("https://httpbin.org/get", headers, params, cookies) 76 | if err != nil { 77 | return 78 | } 79 | fmt.Println(resp.Text()) 80 | } 81 | ``` 82 | 83 | Output: 84 | 85 | ```json 86 | { 87 | "args": { 88 | "age": "18", 89 | "name": "wnanbei" 90 | }, 91 | "headers": { 92 | "Accept-Encoding": "gzip", 93 | "Cookie": "sign=kzhxciuvyqwekhiuxcyvnkjdhiue", 94 | "Host": "httpbin.org", 95 | "User-Agent": "direwolf" 96 | }, 97 | "origin": "1.1.1.1, 1.1.1.1", 98 | "url": "https://httpbin.org/get?age=18&name=wnanbei" 99 | } 100 | ``` 101 | 102 | ## Contribute 103 | 104 | Direwolf is a personal project now, but all contributions, bug reports, bug fixes, documentation improvements, enhancements, and ideas are welcome. 105 | 106 | If you find a bug in direwolf or have some good ideas: 107 | 108 | - Go to [GitHub “issues” tab](https://github.com/wnanbei/direwolf/issues) and open a fresh issue to start a discussion around a feature idea or a bug. 109 | - Send a pull request and bug the maintainer until it gets merged and published. 110 | - Write a test which shows that the bug was fixed or that the feature works as expected. 111 | 112 | If you need to discuss about direwolf with me, you can send me a e-mail. 113 | -------------------------------------------------------------------------------- /datatype_test.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/gin-gonic/gin" 10 | ) 11 | 12 | func TestHeaders(t *testing.T) { 13 | headers := NewHeaders( 14 | "key1", "value1", 15 | "key2", "value2", 16 | ) 17 | if headers.Get("key1") != "value1" { 18 | t.Fatal("Headers.Get() failed.") 19 | } 20 | if headers.Get("key3") != "" { 21 | t.Fatal("Headers.Get() failed.") 22 | } 23 | } 24 | 25 | func TestStringSliceMap(t *testing.T) { 26 | params := NewParams( 27 | "key1", "value1", 28 | "key2", "value2", 29 | ) 30 | if params.Get("key1") != "value1" { 31 | t.Fatal("params.Get() failed.") 32 | } 33 | if params.Get("key3") != "" { 34 | t.Fatal("params.Get() failed.") 35 | } 36 | 37 | if params.URLEncode() != "key1=value1&key2=value2" { 38 | t.Fatal("params.URLEncode() failed.") 39 | } 40 | 41 | params.Add("key1", "value3") 42 | if params.Get("key1", 1) != "value3" { 43 | t.Fatal("params.Add() failed.") 44 | } 45 | 46 | params.Set("key1", "value4") 47 | if params.Get("key1") != "value4" { 48 | t.Fatal("params.Set() failed.") 49 | } 50 | 51 | params.Del("key2") 52 | if params.Get("key2") != "" { 53 | t.Fatal("params.Del() failed.") 54 | } 55 | 56 | postForm := NewPostForm( 57 | "key1", "value1", 58 | "key2", "value2", 59 | ) 60 | if postForm.Get("key1") != "value1" { 61 | t.Fatal("PostForm.Get() failed.") 62 | } 63 | } 64 | 65 | func TestParams(t *testing.T) { 66 | URL := "http://test.com" 67 | params := NewParams( 68 | "key1", "value2", 69 | "key2", "value2") 70 | req, err := NewRequest("GET", URL, params) 71 | if err != nil { 72 | t.Fatal(err) 73 | } 74 | if req.URL != "http://test.com?key1=value2&key2=value2" { 75 | t.Fatal("Test params failed.") 76 | } 77 | 78 | URL = "http://test.com?" 79 | params = NewParams( 80 | "key1", "value2", 81 | "key2", "value2") 82 | req, err = NewRequest("GET", URL, params) 83 | if err != nil { 84 | t.Fatal(err) 85 | } 86 | if req.URL != "http://test.com?key1=value2&key2=value2" { 87 | t.Fatal("Test params failed.") 88 | } 89 | 90 | URL = "http://test.com?xxx=yyy" 91 | params = NewParams( 92 | "key1", "value2", 93 | "key2", "value2") 94 | req, err = NewRequest("GET", URL, params) 95 | if err != nil { 96 | t.Fatal(err) 97 | } 98 | if req.URL != "http://test.com?xxx=yyy&key1=value2&key2=value2" { 99 | t.Fatal("Test params failed.") 100 | } 101 | } 102 | 103 | func GetJsonTestServer() *httptest.Server { 104 | gin.SetMode(gin.ReleaseMode) 105 | router := gin.New() 106 | router.GET("/json", func(c *gin.Context) { 107 | data, err := ioutil.ReadAll(c.Request.Body) 108 | if err != nil { 109 | fmt.Println(err) 110 | } 111 | c.String(200, string(data)) 112 | }) 113 | ts := httptest.NewServer(router) 114 | return ts 115 | } 116 | 117 | func TestJsonBody(t *testing.T) { 118 | ts := GetJsonTestServer() 119 | 120 | type Student struct { 121 | Name string 122 | Age int 123 | Guake bool 124 | } 125 | jsonText := &Student{ 126 | "Xiao Ming", 127 | 16, 128 | true, 129 | } 130 | 131 | jsonBody := NewJsonBody(jsonText) 132 | resp, err := Get(ts.URL+"/json", jsonBody) 133 | if err != nil { 134 | t.Fatal("TestJsonBody Failed.") 135 | } 136 | 137 | if resp.JsonGet("Name").String() != "Xiao Ming" { 138 | t.Fatal("TestJsonBody Failed.") 139 | } 140 | 141 | jsonRespBody := &Student{} 142 | if err := resp.Json(jsonRespBody); err != nil { 143 | t.Fatal("TestJsonBody Failed.") 144 | } 145 | if jsonRespBody.Name != "Xiao Ming" { 146 | t.Fatal("TestJsonBody Failed.") 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /api_test.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestGet(t *testing.T) { 8 | ts := newTestSessionServer() 9 | defer ts.Close() 10 | 11 | resp, err := Get(ts.URL + "/test") 12 | if err != nil { 13 | t.Fatal(err) 14 | } 15 | text := resp.Text() 16 | if text != "GET" { 17 | t.Fatal("Get test failed") 18 | } 19 | } 20 | 21 | func TestPost(t *testing.T) { 22 | ts := newTestSessionServer() 23 | defer ts.Close() 24 | 25 | postForm := NewPostForm( 26 | "key", "value", 27 | ) 28 | resp, err := Post(ts.URL+"/test", postForm) 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | text := resp.Text() 33 | if text != "POST" { 34 | t.Fatal("Post test failed") 35 | } 36 | 37 | body := Body("key=value") 38 | resp2, err := Post(ts.URL+"/test", body) 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | text2 := resp2.Text() 43 | if text2 != "POST" { 44 | t.Fatal("Post test failed") 45 | } 46 | } 47 | 48 | func TestPut(t *testing.T) { 49 | ts := newTestSessionServer() 50 | defer ts.Close() 51 | 52 | resp, err := Put(ts.URL) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | text := resp.Text() 57 | if text != "PUT" { 58 | t.Fatal("Put test failed") 59 | } 60 | } 61 | 62 | func TestPatch(t *testing.T) { 63 | ts := newTestSessionServer() 64 | defer ts.Close() 65 | 66 | resp, err := Patch(ts.URL) 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | text := resp.Text() 71 | if text != "PATCH" { 72 | t.Fatal("Patch test failed") 73 | } 74 | } 75 | 76 | func TestDelete(t *testing.T) { 77 | ts := newTestSessionServer() 78 | defer ts.Close() 79 | 80 | resp, err := Delete(ts.URL) 81 | if err != nil { 82 | t.Fatal(err) 83 | } 84 | text := resp.Text() 85 | if text != "DELETE" { 86 | t.Fatal("Delete test failed") 87 | } 88 | } 89 | 90 | func TestHead(t *testing.T) { 91 | ts := newTestSessionServer() 92 | defer ts.Close() 93 | 94 | resp, err := Head(ts.URL) 95 | if err != nil { 96 | t.Fatal(err) 97 | } 98 | cookies := resp.Cookies 99 | if cookies[0].Name != "HEAD" { 100 | t.Fatal("Head test failed") 101 | } 102 | } 103 | 104 | func TestRequest(t *testing.T) { 105 | ts := newTestSessionServer() 106 | defer ts.Close() 107 | 108 | req, err := NewRequest("Get", ts.URL+"/test") 109 | if err != nil { 110 | t.Fatal(err) 111 | } 112 | 113 | resp, err := Send(req) 114 | if err != nil { 115 | t.Fatal(err) 116 | } 117 | text := resp.Text() 118 | if text != "GET" { 119 | t.Fatal("Request test failed") 120 | } 121 | } 122 | 123 | func TestSendCookie(t *testing.T) { 124 | ts := newTestSessionServer() 125 | defer ts.Close() 126 | 127 | cookies := NewCookies( 128 | "name", "direwolf", 129 | ) 130 | resp, err := Get(ts.URL+"/getCookie", cookies) 131 | if err != nil { 132 | t.Fatal(err) 133 | } 134 | if resp.Text() != "name=direwolf" { 135 | t.Fatal("request cookies test failed") 136 | } 137 | } 138 | 139 | func TestSendHeaders(t *testing.T) { 140 | ts := newTestSessionServer() 141 | defer ts.Close() 142 | 143 | headers := NewHeaders( 144 | "User-Agent", "direwolf", 145 | ) 146 | resp, err := Get(ts.URL+"/getHeader", headers) 147 | if err != nil { 148 | t.Fatal(err) 149 | } 150 | text := resp.Text() 151 | if text != "direwolf" { 152 | t.Fatal("request headers test failed") 153 | } 154 | } 155 | 156 | func TestSendParams(t *testing.T) { 157 | ts := newTestSessionServer() 158 | defer ts.Close() 159 | 160 | params := NewParams( 161 | "key", "value", 162 | ) 163 | resp, err := Get(ts.URL+"/getParams", params) 164 | if err != nil { 165 | t.Fatal(err) 166 | } 167 | if resp.Text() != "value" { 168 | t.Fatal("request params test failed") 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /response_test.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "net/http/httptest" 5 | "testing" 6 | 7 | "github.com/gin-gonic/gin" 8 | 9 | "golang.org/x/text/encoding/charmap" 10 | "golang.org/x/text/encoding/simplifiedchinese" 11 | ) 12 | 13 | func newTestResponseServer() *httptest.Server { 14 | respString := ` 15 | 16 | Direwolf 17 | 18 | 19 |
  • is a convenient
    • 20 |
  • and easy to use http client with Golang
    • 21 |
  • 南北
    • 22 |
  • 2019-06-21
    • 23 | 24 | ` 25 | 26 | gin.SetMode(gin.ReleaseMode) 27 | router := gin.New() 28 | router.GET("/", func(c *gin.Context) { 29 | c.String(200, respString) 30 | }) 31 | router.GET("/GBK", func(c *gin.Context) { 32 | content, _ := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(respString)) 33 | c.Data(200, "text/html", content) 34 | }) 35 | router.GET("/GB18030", func(c *gin.Context) { 36 | content, _ := simplifiedchinese.GB18030.NewEncoder().Bytes([]byte(respString)) 37 | c.Data(200, "text/html", content) 38 | }) 39 | router.GET("/latin1", func(c *gin.Context) { 40 | content, _ := charmap.ISO8859_1.NewEncoder().Bytes([]byte(`
  • ...
    • `)) 41 | c.Data(200, "text/html", content) 42 | }) 43 | ts := httptest.NewServer(router) 44 | return ts 45 | } 46 | 47 | func TestReExtract(t *testing.T) { 48 | ts := newTestResponseServer() 49 | defer ts.Close() 50 | 51 | resp, err := Get(ts.URL) 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | result1 := resp.Re(`\d{4}-\d{2}-\d{2}`) 56 | if result1[0] != "2019-06-21" { 57 | t.Fatal("Response.Re() failed.") 58 | } 59 | 60 | result2 := resp.ReSubMatch(`(.*?)`) 61 | if len(result2) != 4 { 62 | t.Fatal("Response.ReSubMatch() failed.") 63 | } 64 | if result2[3][0] != "2019-06-21" { 65 | t.Fatal("Response.ReSubMatch() failed.") 66 | } 67 | } 68 | 69 | func TestCssExtract(t *testing.T) { 70 | ts := newTestResponseServer() 71 | defer ts.Close() 72 | 73 | resp, err := Get(ts.URL) 74 | if err != nil { 75 | t.Fatal(err) 76 | } 77 | result1 := resp.CSS(`a`).First().Text() 78 | if result1 != "is a convenient" { 79 | t.Fatal("Response.CSS().First().Text() failed.") 80 | } 81 | 82 | result2 := resp.CSS(`body`).CSS(`li`).CSS(`a[href=\/time\/]`).First().Text() 83 | if result2 != "2019-06-21" { 84 | t.Fatal("Response.CSS() failed.") 85 | } 86 | 87 | result3 := resp.CSS(`a`).First().Attr("href") 88 | if result3 != "/convenient/" { 89 | t.Fatal("Response.CSS().First().Attr() failed.") 90 | } 91 | 92 | result5 := resp.CSS(`a`).At(2).Text() 93 | if result5 != "南北" { 94 | t.Fatal("Response.CSS().At() failed.") 95 | } 96 | } 97 | 98 | func TestResponseEncoding(t *testing.T) { 99 | ts := newTestResponseServer() 100 | defer ts.Close() 101 | 102 | resp3, err := Get(ts.URL + "/latin1") 103 | if err != nil { 104 | t.Fatal(err) 105 | } 106 | resp3.Encoding("latin1") 107 | result3 := resp3.ReSubMatch(`(.*?)`) 108 | if result3[0][0] != "..." { 109 | t.Fatal("Response latin1 failed.") 110 | } 111 | 112 | resp, err := Get(ts.URL + "/GBK") 113 | if err != nil { 114 | t.Fatal(err) 115 | } 116 | resp.Encoding("GBK") 117 | result1 := resp.ReSubMatch(`(.*?)`) 118 | if result1[0][0] != "南北" { 119 | t.Fatal("Response GBK failed.") 120 | } 121 | 122 | resp2, err := Get(ts.URL + "/GB18030") 123 | if err != nil { 124 | t.Fatal(err) 125 | } 126 | resp2.Encoding("GB18030") 127 | result2 := resp2.ReSubMatch(`(.*?)`) 128 | if result2[0][0] != "南北" { 129 | t.Fatal("Response GB18030 failed.") 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package direwolf is a convenient and easy to use http client written in Golang. 3 | */ 4 | package direwolf 5 | 6 | // Default global session 7 | var defaultSession *Session 8 | 9 | func init() { 10 | sessionOptions := DefaultSessionOptions() // New default global session 11 | sessionOptions.DisableCookieJar = true 12 | defaultSession = NewSession(sessionOptions) 13 | } 14 | 15 | // Send is different with Get and Post method, you should pass a 16 | // Request to it. You can construct Request by use NewRequest 17 | // method. 18 | func Send(req *Request) (*Response, error) { 19 | resp, err := defaultSession.Send(req) 20 | if err != nil { 21 | return nil, err 22 | } 23 | return resp, nil 24 | } 25 | 26 | // Get is the most common method of direwolf to constructs and sends a 27 | // Get request. 28 | // 29 | // You can construct this request by passing the following parameters: 30 | // url: URL for the request. 31 | // http.Header: HTTP Headers to send. 32 | // direwolf.Params: Parameters to send in the query string. 33 | // direwolf.Cookies: Cookies to send. 34 | // direwolf.PostForm: Post data form to send. 35 | // direwolf.Body: Post body to send. 36 | // direwolf.Proxy: Proxy url to use. 37 | // direwolf.Timeout: Request Timeout. Default value is 30. 38 | // direwolf.RedirectNum: Number of Request allowed to redirect. Default value is 5. 39 | func Get(URL string, args ...RequestOption) (*Response, error) { 40 | req, err := NewRequest("GET", URL, args...) 41 | if err != nil { 42 | return nil, err 43 | } 44 | resp, err := Send(req) 45 | if err != nil { 46 | return nil, err 47 | } 48 | return resp, nil 49 | } 50 | 51 | // Post is the method to constructs and sends a Post request. Parameters are 52 | // the same with direwolf.Get() 53 | // 54 | // Note: direwolf.Body can`t existed with direwolf.PostForm. 55 | func Post(URL string, args ...RequestOption) (*Response, error) { 56 | req, err := NewRequest("POST", URL, args...) 57 | if err != nil { 58 | return nil, err 59 | } 60 | resp, err := Send(req) 61 | if err != nil { 62 | return nil, err 63 | } 64 | return resp, nil 65 | } 66 | 67 | // Head is the method to constructs and sends a Head request. Parameters are 68 | // the same with direwolf.Get() 69 | func Head(URL string, args ...RequestOption) (*Response, error) { 70 | req, err := NewRequest("HEAD", URL, args...) 71 | if err != nil { 72 | return nil, err 73 | } 74 | resp, err := Send(req) 75 | if err != nil { 76 | return nil, err 77 | } 78 | return resp, nil 79 | } 80 | 81 | // Put is the method to constructs and sends a Put request. Parameters are 82 | // the same with direwolf.Get() 83 | func Put(URL string, args ...RequestOption) (*Response, error) { 84 | req, err := NewRequest("Put", URL, args...) 85 | if err != nil { 86 | return nil, err 87 | } 88 | resp, err := Send(req) 89 | if err != nil { 90 | return nil, err 91 | } 92 | return resp, nil 93 | } 94 | 95 | // Patch is the method to constructs and sends a Patch request. Parameters are 96 | // the same with direwolf.Get() 97 | func Patch(URL string, args ...RequestOption) (*Response, error) { 98 | req, err := NewRequest("Patch", URL, args...) 99 | if err != nil { 100 | return nil, err 101 | } 102 | resp, err := Send(req) 103 | if err != nil { 104 | return nil, err 105 | } 106 | return resp, nil 107 | } 108 | 109 | // Delete is the method to constructs and sends a Delete request. Parameters are 110 | // the same with direwolf.Get() 111 | func Delete(URL string, args ...RequestOption) (*Response, error) { 112 | req, err := NewRequest("Delete", URL, args...) 113 | if err != nil { 114 | return nil, err 115 | } 116 | resp, err := Send(req) 117 | if err != nil { 118 | return nil, err 119 | } 120 | return resp, nil 121 | } 122 | -------------------------------------------------------------------------------- /download_test.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "net/http" 8 | "net/http/httptest" 9 | "strconv" 10 | "testing" 11 | "time" 12 | 13 | "github.com/gin-gonic/gin" 14 | ) 15 | 16 | func newTestProxyServer() *httptest.Server { 17 | gin.SetMode(gin.ReleaseMode) 18 | router := gin.New() 19 | router.GET("/:a", func(c *gin.Context) { 20 | reqUrl := c.Request.URL.String() // Get request url 21 | req, err := http.NewRequest(c.Request.Method, reqUrl, nil) 22 | if err != nil { 23 | c.AbortWithStatus(404) 24 | return 25 | } 26 | 27 | // Forwarding requests from client. 28 | cli := &http.Client{} 29 | resp, err := cli.Do(req) 30 | if err != nil { 31 | c.AbortWithStatus(404) 32 | return 33 | } 34 | defer resp.Body.Close() 35 | 36 | body, err := ioutil.ReadAll(resp.Body) 37 | c.Data(200, "text/plain", body) // write response body to response. 38 | c.String(200, "This is proxy Server.") // add proxy info. 39 | }) 40 | ts := httptest.NewServer(router) 41 | return ts 42 | } 43 | 44 | func TestSetProxy(t *testing.T) { 45 | proxyServer := newTestProxyServer() 46 | defer proxyServer.Close() 47 | targetServer := newTestSessionServer() 48 | defer targetServer.Close() 49 | 50 | proxy := &Proxy{ 51 | HTTP: proxyServer.URL, 52 | HTTPS: proxyServer.URL, 53 | } 54 | resp, err := Get(targetServer.URL+"/proxy", proxy) 55 | if err != nil { 56 | t.Fatal("SetProxy failed: ", err) 57 | } 58 | if resp.Text() != "This is target website.This is proxy Server." { 59 | t.Fatal("SetProxy failed: ", err) 60 | } 61 | } 62 | 63 | func newTestTimeoutServer() *httptest.Server { 64 | gin.SetMode(gin.ReleaseMode) 65 | router := gin.New() 66 | router.GET("/", func(c *gin.Context) { 67 | time.Sleep(time.Second * 2) 68 | c.String(200, "successed") 69 | }) 70 | ts := httptest.NewServer(router) 71 | return ts 72 | } 73 | 74 | func TestTimeout(t *testing.T) { 75 | timeoutServer := newTestTimeoutServer() 76 | defer timeoutServer.Close() 77 | 78 | _, err := Get(timeoutServer.URL, Timeout(1)) 79 | if err != nil { 80 | if !errors.Is(err, ErrTimeout) { 81 | t.Fatal("TestTimeout failed: ", err) 82 | } 83 | } 84 | 85 | _, err = Get(timeoutServer.URL, Timeout(3)) 86 | if err != nil { 87 | t.Fatal("TestTimeout failed: ", err) 88 | } 89 | } 90 | 91 | func newTestRedirectServer() *httptest.Server { 92 | gin.SetMode(gin.ReleaseMode) 93 | router := gin.New() 94 | router.GET("/", func(c *gin.Context) { 95 | c.String(200, "successed") 96 | }) 97 | router.GET("/:number", func(c *gin.Context) { 98 | number, _ := strconv.Atoi(c.Param("number")) 99 | if number == 1 { 100 | c.Redirect(302, "/") 101 | } 102 | number = number - 1 103 | c.Redirect(302, strconv.Itoa(number)) 104 | }) 105 | ts := httptest.NewServer(router) 106 | return ts 107 | } 108 | 109 | func TestRedirect(t *testing.T) { 110 | redirectServer := newTestRedirectServer() 111 | defer redirectServer.Close() 112 | 113 | redirect := RedirectNum(-1) // ban redirect 114 | resp, err := Get(redirectServer.URL+"/", redirect) 115 | if err != nil { 116 | t.Fatal("Test TestRedirectError failed.") 117 | } 118 | if resp.Text() != "successed" { 119 | t.Fatal("Test TestRedirectError failed.") 120 | } 121 | 122 | redirect = RedirectNum(1) // allow 1 redirect 123 | resp, err = Get(redirectServer.URL+"/1", redirect) 124 | if err != nil { 125 | t.Fatal("Test TestRedirectError failed.") 126 | } 127 | if resp.Text() != "successed" { 128 | t.Fatal("Test TestRedirectError failed.") 129 | } 130 | 131 | redirect = RedirectNum(1) // allow 1 redirect 132 | resp, err = Get(redirectServer.URL+"/2", redirect) 133 | if err != nil { 134 | var errType *RedirectError 135 | if !errors.As(err, &errType) { // check RedirectError 136 | fmt.Println("yyyyy") 137 | t.Fatal("Test TestRedirectError failed.") 138 | } 139 | } else { 140 | t.Fatal("Test TestRedirectError failed.") 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /download.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | // send is low level request method. 15 | func send(session *Session, req *Request) (*Response, error) { 16 | // Set timeout to request context. 17 | // Default timeout is 30s. 18 | timeout := time.Second * 30 19 | if req.Timeout > 0 { 20 | timeout = time.Second * time.Duration(req.Timeout) 21 | } else if session.Timeout > 0 { 22 | timeout = time.Second * time.Duration(session.Timeout) 23 | } 24 | ctx, timeoutCancel := context.WithTimeout(context.Background(), timeout) 25 | 26 | // set proxy to request context. 27 | if req.Proxy != nil { 28 | ctx = context.WithValue(ctx, "http", req.Proxy.HTTP) 29 | ctx = context.WithValue(ctx, "https", req.Proxy.HTTPS) 30 | } else if session.Proxy != nil { 31 | ctx = context.WithValue(ctx, "http", session.Proxy.HTTP) 32 | ctx = context.WithValue(ctx, "https", session.Proxy.HTTPS) 33 | } 34 | 35 | // set RedirectNum to request context. 36 | // default RedirectNum is 10. 37 | if req.RedirectNum == 0 { 38 | ctx = context.WithValue(ctx, "redirectNum", 10) 39 | } else if req.RedirectNum > 0 { 40 | ctx = context.WithValue(ctx, "redirectNum", req.RedirectNum) 41 | } else { 42 | ctx = context.WithValue(ctx, "redirectNum", 0) 43 | } 44 | 45 | // Make new http.Request with context 46 | httpReq, err := http.NewRequestWithContext(ctx, req.Method, req.URL, nil) 47 | if err != nil { 48 | timeoutCancel() 49 | return nil, WrapErr(err, "build Request error, please check request url or request method") 50 | } 51 | 52 | // Handle the Headers. 53 | httpReq.Header = mergeHeaders(req.Headers, session.Headers) 54 | 55 | // Handle the DataForm, Body or JsonBody. 56 | // Set right Content-Type. 57 | if req.PostForm != nil { 58 | httpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded") 59 | data := req.PostForm.URLEncode() 60 | httpReq.Body = ioutil.NopCloser(strings.NewReader(data)) 61 | } else if req.Body != nil { 62 | httpReq.Body = ioutil.NopCloser(bytes.NewReader(req.Body)) 63 | } else if req.JsonBody != nil { 64 | httpReq.Header.Set("Content-Type", "application/json") 65 | httpReq.Body = ioutil.NopCloser(bytes.NewReader(req.JsonBody)) 66 | } else if req.MultipartForm != nil { 67 | httpReq.Header.Set("Content-Type", req.MultipartForm.ContentType()) 68 | httpReq.Body = ioutil.NopCloser(req.MultipartForm.Reader()) 69 | } 70 | 71 | // Handle Cookies 72 | if req.Cookies != nil { 73 | for _, cookie := range req.Cookies { 74 | httpReq.AddCookie(cookie) 75 | } 76 | } 77 | 78 | resp, err := session.client.Do(httpReq) // do request 79 | if err != nil { 80 | if strings.Contains(err.Error(), "context deadline exceeded") { // check timeout error 81 | timeoutCancel() 82 | return nil, WrapErr(ErrTimeout, err.Error()) 83 | } 84 | timeoutCancel() 85 | return nil, WrapErr(err, "Request Error") 86 | } 87 | defer func() { 88 | if err := resp.Body.Close(); err != nil { 89 | panic(err) 90 | } 91 | }() 92 | 93 | response, err := buildResponse(req, resp) 94 | if err != nil { 95 | timeoutCancel() 96 | return nil, WrapErr(err, "build Response Error") 97 | } 98 | 99 | timeoutCancel() // cancel the timeout context after request succeed. 100 | return response, nil 101 | } 102 | 103 | // buildResponse build response with http.Response after do request. 104 | func buildResponse(httpReq *Request, httpResp *http.Response) (*Response, error) { 105 | content, err := ioutil.ReadAll(httpResp.Body) 106 | if err != nil { 107 | if !errors.Is(err, io.ErrUnexpectedEOF) { // Ignore Unexpected EOF error 108 | return nil, WrapErr(err, "read Response.Body failed") 109 | } 110 | } 111 | return &Response{ 112 | URL: httpReq.URL, 113 | StatusCode: httpResp.StatusCode, 114 | Proto: httpResp.Proto, 115 | Headers: httpResp.Header, 116 | Cookies: httpResp.Cookies(), 117 | Request: httpReq, 118 | ContentLength: httpResp.ContentLength, 119 | Content: content, 120 | encoding: "UTF-8", 121 | }, nil 122 | } 123 | 124 | // mergeHeaders merge Request headers and Session Headers. 125 | // Request has higher priority. 126 | func mergeHeaders(h1, h2 http.Header) http.Header { 127 | h := http.Header{} 128 | for key, values := range h2 { 129 | for _, value := range values { 130 | h.Set(key, value) 131 | } 132 | } 133 | for key, values := range h1 { 134 | for _, value := range values { 135 | h.Set(key, value) 136 | } 137 | } 138 | return h 139 | } 140 | -------------------------------------------------------------------------------- /session_test.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | "time" 8 | 9 | "github.com/gin-gonic/gin" 10 | ) 11 | 12 | func newTestSessionServer() *httptest.Server { 13 | gin.SetMode(gin.ReleaseMode) 14 | router := gin.New() 15 | router.GET("/test", func(c *gin.Context) { 16 | c.String(200, "GET") 17 | }) 18 | router.GET("/setCookie", func(c *gin.Context) { 19 | http.SetCookie(c.Writer, &http.Cookie{ 20 | Name: "key", 21 | Value: "value", 22 | Path: "/", 23 | Expires: time.Now().Add(30 * time.Second), 24 | }) 25 | }) 26 | router.GET("/getCookie", func(c *gin.Context) { 27 | cookies := c.Request.Cookies() 28 | for _, cookie := range cookies { 29 | c.String(200, cookie.Name+"="+cookie.Value) 30 | } 31 | }) 32 | router.GET("/getHeader", func(c *gin.Context) { 33 | header := c.GetHeader("user-agent") 34 | c.String(200, header) 35 | }) 36 | router.GET("/getParams", func(c *gin.Context) { 37 | value := c.Query("key") 38 | c.String(200, value) 39 | }) 40 | router.GET("/proxy", func(c *gin.Context) { 41 | c.String(200, "This is target website.") 42 | }) 43 | router.POST("/test", func(c *gin.Context) { 44 | data, err := c.GetRawData() 45 | if err != nil { 46 | c.AbortWithStatus(404) 47 | return 48 | } 49 | if string(data) == "key=value" { 50 | c.String(200, "POST") 51 | } else { 52 | c.String(200, "Failed") 53 | } 54 | }) 55 | router.HEAD("/", func(c *gin.Context) { 56 | c.SetCookie("HEAD", "RIGHT", 1000, "/", "localhost", false, false) 57 | }) 58 | router.PUT("/", func(c *gin.Context) { 59 | c.String(200, "PUT") 60 | }) 61 | router.DELETE("/", func(c *gin.Context) { 62 | c.String(200, "DELETE") 63 | }) 64 | router.PATCH("/", func(c *gin.Context) { 65 | c.String(200, "PATCH") 66 | }) 67 | 68 | ts := httptest.NewServer(router) 69 | return ts 70 | 71 | //ys := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 72 | // // check method is GET before going to check other features 73 | // if r.Method == "GET" { 74 | // if r.URL.Path == "/test" { 75 | // if _, err := w.Write([]byte("GET")); err != nil { 76 | // } 77 | // } 78 | // if r.URL.Path == "/setCookie" { 79 | // http.SetCookie(w, &http.Cookie{Name: "key", Value: "value"}) 80 | // } 81 | // if r.URL.Path == "/getCookie" { 82 | // cookies := r.Cookies() 83 | // for _, cookie := range cookies { 84 | // if _, err := w.Write([]byte(cookie.Name + "=" + cookie.Value)); err != nil { 85 | // } 86 | // } 87 | // } 88 | // if r.URL.Path == "/getHeader" { 89 | // header := r.Header 90 | // value := header.Get("user-agent") 91 | // if _, err := w.Write([]byte(value)); err != nil { 92 | // } 93 | // } 94 | // if r.URL.Path == "/getParams" { 95 | // if err := r.ParseForm(); err != nil { 96 | // } 97 | // params := r.Form 98 | // value := params.Get("key") 99 | // if _, err := w.Write([]byte(value)); err != nil { 100 | // } 101 | // } 102 | // if r.URL.Path == "/proxy" { 103 | // if _, err := w.Write([]byte("This is target website.")); err != nil { 104 | // } 105 | // } 106 | // } 107 | // if r.Method == "POST" { 108 | // if r.URL.Path == "/test" { 109 | // body, _ := ioutil.ReadAll(r.Body) 110 | // if string(body) != "key=value" { 111 | // if _, err := w.Write([]byte("Failed")); err != nil { 112 | // } 113 | // } else { 114 | // if _, err := w.Write([]byte("POST")); err != nil { 115 | // } 116 | // } 117 | // } 118 | // } 119 | // if r.Method == "HEAD" { 120 | // http.SetCookie(w, &http.Cookie{Name: "HEAD", Value: "RIGHT"}) 121 | // } 122 | // if r.Method == "PUT" { 123 | // if _, err := w.Write([]byte("PUT")); err != nil { 124 | // } 125 | // } 126 | // if r.Method == "PATCH" { 127 | // if _, err := w.Write([]byte("PATCH")); err != nil { 128 | // } 129 | // } 130 | // if r.Method == "DELETE" { 131 | // if _, err := w.Write([]byte("DELETE")); err != nil { 132 | // } 133 | // } 134 | //})) 135 | //return ys 136 | } 137 | 138 | func TestSessionGet(t *testing.T) { 139 | ts := newTestSessionServer() 140 | defer ts.Close() 141 | 142 | session := NewSession() 143 | resp, err := session.Get(ts.URL + "/test") 144 | if err != nil { 145 | t.Fatal(err) 146 | } 147 | text := resp.Text() 148 | if text != "GET" { 149 | t.Fatal("Session.Get test failed") 150 | } 151 | } 152 | 153 | func TestSessionPost(t *testing.T) { 154 | ts := newTestSessionServer() 155 | defer ts.Close() 156 | 157 | session := NewSession() 158 | postForm := NewPostForm( 159 | "key", "value", 160 | ) 161 | resp, err := session.Post(ts.URL+"/test", postForm) 162 | if err != nil { 163 | t.Fatal(err) 164 | } 165 | text := resp.Text() 166 | if text != "POST" { 167 | t.Fatal("Session.Post test failed") 168 | } 169 | 170 | body := Body("key=value") 171 | resp2, err := session.Post(ts.URL+"/test", body) 172 | if err != nil { 173 | t.Fatal(err) 174 | } 175 | text2 := resp2.Text() 176 | if text2 != "POST" { 177 | t.Fatal("Session.Post test failed") 178 | } 179 | } 180 | 181 | func TestSessionPut(t *testing.T) { 182 | ts := newTestSessionServer() 183 | defer ts.Close() 184 | 185 | session := NewSession() 186 | resp, err := session.Put(ts.URL) 187 | if err != nil { 188 | t.Fatal(err) 189 | } 190 | text := resp.Text() 191 | if text != "PUT" { 192 | t.Fatal("Session.Put test failed") 193 | } 194 | } 195 | 196 | func TestSessionPatch(t *testing.T) { 197 | ts := newTestSessionServer() 198 | defer ts.Close() 199 | 200 | session := NewSession() 201 | resp, err := session.Patch(ts.URL) 202 | if err != nil { 203 | t.Fatal(err) 204 | } 205 | text := resp.Text() 206 | if text != "PATCH" { 207 | t.Fatal("Session.Patch test failed") 208 | } 209 | } 210 | 211 | func TestSessionDelete(t *testing.T) { 212 | ts := newTestSessionServer() 213 | defer ts.Close() 214 | 215 | session := NewSession() 216 | resp, err := session.Delete(ts.URL) 217 | if err != nil { 218 | t.Fatal(err) 219 | } 220 | text := resp.Text() 221 | if text != "DELETE" { 222 | t.Fatal("Session.Delete test failed") 223 | } 224 | } 225 | 226 | func TestSessionHead(t *testing.T) { 227 | ts := newTestSessionServer() 228 | defer ts.Close() 229 | 230 | session := NewSession() 231 | resp, err := session.Head(ts.URL) 232 | if err != nil { 233 | t.Fatal(err) 234 | } 235 | cookies := resp.Cookies 236 | if cookies[0].Name != "HEAD" { 237 | t.Fatal("Session.Head test failed") 238 | } 239 | } 240 | 241 | func TestSessionCookieJar(t *testing.T) { 242 | ts := newTestSessionServer() 243 | defer ts.Close() 244 | 245 | session := NewSession() 246 | _, err := session.Get(ts.URL + "/setCookie") 247 | if err != nil { 248 | t.Fatal(err) 249 | } 250 | resp, err := session.Get(ts.URL + "/getCookie") 251 | if err != nil { 252 | t.Fatal(err) 253 | } 254 | if resp.Text() != "key=value" { 255 | t.Fatal("Session.CookieJar failed.") 256 | return 257 | } 258 | } 259 | 260 | func TestSessionSetCookie(t *testing.T) { 261 | ts := newTestSessionServer() 262 | defer ts.Close() 263 | 264 | session := NewSession() 265 | cookie := NewCookies("key", "value") 266 | session.SetCookies(ts.URL, cookie) 267 | resp, err := session.Get(ts.URL + "/getCookie") 268 | if err != nil { 269 | t.Fatal(err) 270 | } 271 | if resp.Text() != "key=value" { 272 | t.Fatal("Session.SetCookies() failed.") 273 | return 274 | } 275 | } 276 | 277 | func TestSessionCookies(t *testing.T) { 278 | ts := newTestSessionServer() 279 | defer ts.Close() 280 | 281 | session := NewSession() 282 | _, err := session.Get(ts.URL + "/setCookie") 283 | if err != nil { 284 | t.Fatal(err) 285 | } 286 | cookies := session.Cookies(ts.URL) 287 | if cookies[0].Name != "key" { 288 | t.Fatal("Session.Cookies() failed.") 289 | } 290 | } 291 | -------------------------------------------------------------------------------- /response.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "net/http" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/PuerkitoBio/goquery" 9 | jsoniter "github.com/json-iterator/go" 10 | "github.com/tidwall/gjson" 11 | "golang.org/x/text/encoding/charmap" 12 | "golang.org/x/text/encoding/simplifiedchinese" 13 | ) 14 | 15 | // Response is the response from request. 16 | type Response struct { 17 | URL string 18 | StatusCode int 19 | Proto string 20 | Headers http.Header 21 | Cookies Cookies 22 | Request *Request 23 | Content []byte 24 | ContentLength int64 25 | encoding string 26 | text string 27 | dom *goquery.Document 28 | } 29 | 30 | // Encoding can change and return the encoding type of response. Like this: 31 | // encoding := resp.Encoding("GBK") 32 | // You can specified encoding type. Such as GBK, GB18030, latin1. Default is UTF-8. 33 | // It will decode the content to string if you specified encoding type. 34 | // It will just return the encoding type of response if you do not pass parameter. 35 | func (resp *Response) Encoding(encoding ...string) string { 36 | if len(encoding) > 0 { 37 | resp.encoding = strings.ToUpper(encoding[0]) 38 | resp.text = decodeContent(resp.encoding, resp.Content) 39 | } 40 | return resp.encoding 41 | } 42 | 43 | // Text return the text of Response. It will decode the content to string the first time 44 | // it is called. 45 | func (resp *Response) Text() string { 46 | if resp.text == "" { 47 | resp.text = decodeContent(resp.encoding, resp.Content) 48 | } 49 | return resp.text 50 | } 51 | 52 | // Re extract required data with regexp. 53 | // It return a slice of string. 54 | // Every time you call this method, it will transcode the Response.content to text once. 55 | // So please try to extract required data at once. 56 | func (resp *Response) Re(queryStr string) []string { 57 | text := resp.Text() 58 | return regexp.MustCompile(queryStr).FindAllString(text, -1) 59 | } 60 | 61 | // ReSubMatch extract required data with regexp. 62 | // It return a slice of string from FindAllStringSubmatch. 63 | // Every time you call this method, it will transcode the Response.content to text once. 64 | // So please try to extract required data at once. 65 | func (resp *Response) ReSubMatch(queryStr string) [][]string { 66 | text := resp.Text() 67 | data := regexp.MustCompile(queryStr).FindAllStringSubmatch(text, -1) 68 | var subMatchResult [][]string 69 | for _, match := range data { 70 | if len(match) > 1 { // In case that query has no sub match part 71 | subMatchResult = append(subMatchResult, match[1:]) 72 | } 73 | } 74 | return subMatchResult 75 | } 76 | 77 | // CSS is a method to extract data with css selector, it returns a CSSNodeList. 78 | func (resp *Response) CSS(queryStr string) *CSSNodeList { 79 | if resp.dom == nil { // New the dom if resp.dom not exists. 80 | text := strings.NewReader(resp.Text()) 81 | dom, err := goquery.NewDocumentFromReader(text) 82 | if err != nil { 83 | return nil 84 | } 85 | resp.dom = dom 86 | } 87 | 88 | newNodeList := make([]CSSNode, 0) 89 | resp.dom.Find(queryStr).Each(func(i int, selection *goquery.Selection) { 90 | newNode := CSSNode{selection: selection} 91 | newNodeList = append(newNodeList, newNode) 92 | }) 93 | return &CSSNodeList{container: newNodeList} 94 | } 95 | 96 | // Json can unmarshal json type response body to a struct. 97 | func (resp *Response) Json(output interface{}) error { 98 | if err := jsoniter.Unmarshal(resp.Content, output); err != nil { 99 | return err 100 | } 101 | return nil 102 | } 103 | 104 | // JsonGet can get a value from json type response body with path. 105 | func (resp *Response) JsonGet(path string) gjson.Result { 106 | return gjson.GetBytes(resp.Content, path) 107 | } 108 | 109 | // decodeContent decode the content with the encodingType. It just support 110 | // UTF-8, GBK, GB18030, Lantin1 now. 111 | func decodeContent(encodingType string, content []byte) (decodedText string) { 112 | var text = "" 113 | switch encodingType { 114 | case "UTF-8", "UTF8": 115 | text = string(content) 116 | case "GBK": 117 | decodeBytes, err := simplifiedchinese.GBK.NewDecoder().Bytes(content) 118 | if err != nil { 119 | return "" 120 | } 121 | text = string(decodeBytes) 122 | case "GB18030": 123 | decodeBytes, err := simplifiedchinese.GB18030.NewDecoder().Bytes(content) 124 | if err != nil { 125 | return "" 126 | } 127 | text = string(decodeBytes) 128 | case "LATIN1": 129 | decodeBytes, err := charmap.ISO8859_1.NewDecoder().Bytes(content) 130 | if err != nil { 131 | return "" 132 | } 133 | text = string(decodeBytes) 134 | } 135 | return text 136 | } 137 | 138 | // CSSNode is a container that stores single selected results 139 | type CSSNode struct { 140 | selection *goquery.Selection 141 | } 142 | 143 | // Text return the text of the CSSNode. Only include straight children node text 144 | func (node *CSSNode) Text() string { 145 | if node.selection != nil { 146 | var text string 147 | node.selection.Contents().Each(func(i int, s *goquery.Selection) { 148 | if goquery.NodeName(s) == "#text" { 149 | t := s.Text() 150 | text = text + t 151 | } 152 | }) 153 | return text 154 | } 155 | return "" 156 | } 157 | 158 | // TextAll return the text of the CSSNode. Include all children node text 159 | func (node *CSSNode) TextAll() string { 160 | if node.selection != nil { 161 | return node.selection.Text() 162 | } 163 | return "" 164 | } 165 | 166 | // Attr return the attribute value of the CSSNode. 167 | // You can set default value, if value isn`t exists, return default value. 168 | func (node *CSSNode) Attr(attrName string, defaultValue ...string) string { 169 | var d, attrValue string 170 | if node.selection != nil { 171 | if len(defaultValue) > 0 { 172 | d = defaultValue[0] 173 | } 174 | attrValue = node.selection.AttrOr(attrName, d) 175 | } 176 | return attrValue 177 | } 178 | 179 | // CSSNodeList is a container that stores selected results 180 | type CSSNodeList struct { 181 | container []CSSNode 182 | } 183 | 184 | // Text return a list of text. Only include straight children node text 185 | func (nodeList *CSSNodeList) Text() (textList []string) { 186 | for _, node := range nodeList.container { 187 | text := node.Text() 188 | if text != "" { 189 | textList = append(textList, text) 190 | } 191 | } 192 | return 193 | } 194 | 195 | // TextAll return a list of text. Include all children node text 196 | func (nodeList *CSSNodeList) TextAll() (textList []string) { 197 | for _, node := range nodeList.container { 198 | text := node.TextAll() 199 | if text != "" { 200 | textList = append(textList, text) 201 | } 202 | } 203 | return 204 | } 205 | 206 | // Attr return a list of attribute value 207 | func (nodeList *CSSNodeList) Attr(attrName string, defaultValue ...string) (valueList []string) { 208 | for _, node := range nodeList.container { 209 | value := node.Attr(attrName, defaultValue...) 210 | if value != "" { 211 | valueList = append(valueList, value) 212 | } 213 | } 214 | return 215 | } 216 | 217 | // CSS return a CSSNodeList, so you can chain CSS 218 | func (nodeList *CSSNodeList) CSS(queryStr string) *CSSNodeList { 219 | newNodeList := make([]CSSNode, 0) 220 | for _, node := range nodeList.container { 221 | node.selection.Find(queryStr).Each(func(i int, selection *goquery.Selection) { 222 | newNode := CSSNode{selection: selection} 223 | newNodeList = append(newNodeList, newNode) 224 | }) 225 | } 226 | return &CSSNodeList{container: newNodeList} 227 | } 228 | 229 | // First return the first cssNode of CSSNodeList. 230 | // Return a empty cssNode if there is no cssNode in CSSNodeList 231 | func (nodeList *CSSNodeList) First() *CSSNode { 232 | if len(nodeList.container) > 0 { 233 | return &nodeList.container[0] 234 | } 235 | return &CSSNode{} 236 | } 237 | 238 | // At return the cssNode of specified index position. 239 | // Return a empty cssNode if there is no cssNode in CSSNodeList 240 | func (nodeList *CSSNodeList) At(index int) *CSSNode { 241 | if len(nodeList.container) > index { 242 | return &nodeList.container[index] 243 | } 244 | return &CSSNode{} 245 | } 246 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= 2 | github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= 3 | github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= 4 | github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= 5 | github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= 6 | github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= 7 | github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= 12 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= 13 | github.com/gin-gonic/gin v1.7.7 h1:3DoBmSbJbZAWqXJC3SLjAPfutPJJRN1U5pALB7EeTTs= 14 | github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= 15 | github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= 16 | github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 17 | github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= 18 | github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= 19 | github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= 20 | github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= 21 | github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= 22 | github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= 23 | github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= 24 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 25 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 26 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 27 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 28 | github.com/klauspost/compress v1.15.0 h1:xqfchp4whNFxn5A4XFyyYtitiWI8Hy5EW59jEwcyL6U= 29 | github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 30 | github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= 31 | github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= 32 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 33 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 34 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 35 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 36 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 37 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 38 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 39 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 40 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 41 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 42 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 43 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 44 | github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= 45 | github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 46 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 47 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 48 | github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= 49 | github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 50 | github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= 51 | github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= 52 | github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= 53 | github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= 54 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 55 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 56 | github.com/valyala/fasthttp v1.35.0 h1:wwkR8mZn2NbigFsaw2Zj5r+xkmzjbrA/lyTmiSlal/Y= 57 | github.com/valyala/fasthttp v1.35.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= 58 | github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= 59 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 60 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 61 | golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= 62 | golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 63 | golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 64 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 65 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 66 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 67 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= 68 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 69 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 70 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 71 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 72 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 73 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 74 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 75 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 76 | golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= 77 | golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 78 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 79 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 80 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 81 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 82 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 83 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 84 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 85 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 86 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 87 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 88 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 89 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 90 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 91 | -------------------------------------------------------------------------------- /datatype.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | "sort" 7 | "strings" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | ) 11 | 12 | // RequestOption is the interface of Request Options. Use to bind the options to Request. 13 | type RequestOption interface { 14 | // Bind Options to Request 15 | bindRequest(request *Request) error 16 | } 17 | 18 | // Body is the data you want to post, one of the Request Options. 19 | type Body []byte 20 | 21 | // RequestOption interface method, bind request option to request. 22 | func (options Body) bindRequest(request *Request) error { 23 | request.Body = options 24 | return nil 25 | } 26 | 27 | // JsonBody is the json data you want to post. 28 | type JsonBody []byte 29 | 30 | // NewJsonBody new a json type body. 31 | func NewJsonBody(v interface{}) JsonBody { 32 | body, err := jsoniter.Marshal(v) 33 | if err != nil { 34 | return nil 35 | } 36 | return body 37 | } 38 | 39 | // RequestOption interface method, bind request option to request. 40 | func (options JsonBody) bindRequest(request *Request) error { 41 | request.Body = options 42 | return nil 43 | } 44 | 45 | // RedirectNum is the number of request redirect allowed. 46 | // If RedirectNum > 0, it means a redirect number limit for requests. 47 | // If RedirectNum <= 0, it means ban redirect. 48 | // If RedirectNum is not set, it means default 5 times redirect limit. 49 | type RedirectNum int 50 | 51 | // RequestOption interface method, bind request option to request. 52 | func (options RedirectNum) bindRequest(request *Request) error { 53 | request.RedirectNum = int(options) 54 | return nil 55 | } 56 | 57 | // Timeout is the number of time to timeout request. 58 | // if timeout > 0, it means a time limit for requests. 59 | // if timeout < 0, it means no limit. 60 | // if timeout = 0, it means keep default 30 second timeout. 61 | type Timeout int 62 | 63 | // RequestOption interface method, bind request option to request. 64 | func (options Timeout) bindRequest(request *Request) error { 65 | request.Timeout = int(options) 66 | return nil 67 | } 68 | 69 | // Proxy is the proxy server address, like "http://127.0.0.1:1080". 70 | // You can set different proxies for HTTP and HTTPS sites. 71 | type Proxy struct { 72 | HTTP string 73 | HTTPS string 74 | } 75 | 76 | // RequestOption interface method, bind request option to request. 77 | func (options *Proxy) bindRequest(request *Request) error { 78 | request.Proxy = options 79 | return nil 80 | } 81 | 82 | // strSliceMap type is map[string][]string, used for Params, PostForm. 83 | type strSliceMap struct { 84 | data map[string][]string 85 | } 86 | 87 | // New is the way to create a strSliceMap. 88 | // You can set key-value pair when you init it by sent params. Just like this: 89 | // stringSliceMap{}.New( 90 | // "key1", "value1", 91 | // "key2", "value2", 92 | // ) 93 | // But be careful, between the key and value is a comma. 94 | // And if the number of parameters is not a multiple of 2, it will panic. 95 | func (ssm *strSliceMap) New(keyValue ...string) { 96 | ssm.data = make(map[string][]string) 97 | if keyValue != nil { 98 | if len(keyValue)%2 != 0 { 99 | panic("key and value must be pair") 100 | } 101 | 102 | for i := 0; i < len(keyValue)/2; i++ { 103 | key := keyValue[i*2] 104 | value := keyValue[i*2+1] 105 | ssm.data[key] = append(ssm.data[key], value) 106 | } 107 | } 108 | } 109 | 110 | // Add key and value to stringSliceMap. 111 | // If key exists, value will append to slice. 112 | func (ssm *strSliceMap) Add(key, value string) { 113 | ssm.data[key] = append(ssm.data[key], value) 114 | } 115 | 116 | // Set key and value to stringSliceMap. 117 | // If key exists, existed value will drop and new value will set. 118 | func (ssm *strSliceMap) Set(key, value string) { 119 | ssm.data[key] = []string{value} 120 | } 121 | 122 | // Del delete the given key. 123 | func (ssm *strSliceMap) Del(key string) { 124 | delete(ssm.data, key) 125 | } 126 | 127 | // Get get the value pair to given key. 128 | // You can pass index to assign which value to get, when there are multiple values. 129 | func (ssm *strSliceMap) Get(key string, index ...int) string { 130 | if ssm.data == nil { 131 | return "" 132 | } 133 | ssmValue := ssm.data[key] 134 | if len(ssmValue) == 0 { 135 | return "" 136 | } 137 | if index != nil { 138 | return ssmValue[index[0]] 139 | } 140 | return ssmValue[0] 141 | } 142 | 143 | // URLEncode encodes the values into ``URL encoded'' form 144 | // ("bar=baz&foo=qux") sorted by key. 145 | func (ssm *strSliceMap) URLEncode() string { 146 | if ssm.data == nil { 147 | return "" 148 | } 149 | var buf strings.Builder 150 | keys := make([]string, 0, len(ssm.data)) 151 | for k := range ssm.data { 152 | keys = append(keys, k) 153 | } 154 | sort.Strings(keys) 155 | for _, k := range keys { 156 | ssmValue := ssm.data[k] 157 | keyEscaped := url.QueryEscape(k) 158 | for _, v := range ssmValue { 159 | if buf.Len() > 0 { 160 | buf.WriteByte('&') 161 | } 162 | buf.WriteString(keyEscaped) 163 | buf.WriteByte('=') 164 | buf.WriteString(url.QueryEscape(v)) 165 | } 166 | } 167 | return buf.String() 168 | } 169 | 170 | // Params is url params you want to join to url, as parameter in Request method. 171 | // You should init it by using NewParams like this: 172 | // params := dw.NewParams( 173 | // "key1", "value1", 174 | // "key2", "value2", 175 | // ) 176 | // Note: mid symbol is comma. 177 | type Params struct { 178 | strSliceMap 179 | } 180 | 181 | // NewParams new a Params type. 182 | // 183 | // You can set key-value pair when you init it by sent parameters. Just like this: 184 | // params := NewParams( 185 | // "key1", "value1", 186 | // "key2", "value2", 187 | // ) 188 | // But be careful, between the key and value is a comma. 189 | // And if the number of parameters is not a multiple of 2, it will panic. 190 | func NewParams(keyValue ...string) *Params { 191 | var p = &Params{} 192 | p.New(keyValue...) 193 | return p 194 | } 195 | 196 | // RequestOption interface method, bind request option to request. 197 | func (options *Params) bindRequest(request *Request) error { 198 | request.Params = options 199 | u, err := url.Parse(request.URL) 200 | if err != nil { 201 | return WrapErrf(err, "URL error") 202 | } 203 | 204 | // check whether parameters is existed in url. 205 | if u.RawQuery == "" && u.ForceQuery == false { 206 | request.URL = request.URL + "?" + request.Params.URLEncode() 207 | } else if u.RawQuery == "" && u.ForceQuery == true { 208 | request.URL = request.URL + request.Params.URLEncode() 209 | } else { 210 | request.URL = request.URL + "&" + request.Params.URLEncode() 211 | } 212 | return nil 213 | } 214 | 215 | // PostForm is the form you want to post, as parameter in Request method. 216 | // You should init it by using NewPostForm like this: 217 | // postForm := dw.NewPostForm( 218 | // "key1", "value1", 219 | // "key2", "value2", 220 | // ) 221 | // Note: mid symbol is comma. 222 | type PostForm struct { 223 | strSliceMap 224 | } 225 | 226 | // NewPostForm new a PostForm type. 227 | // 228 | // You can set key-value pair when you init it by sent parameters. Just like this: 229 | // postForm := NewPostForm( 230 | // "key1", "value1", 231 | // "key2", "value2", 232 | // ) 233 | // But be careful, between the key and value is a comma. 234 | // And if the number of parameters is not a multiple of 2, it will panic. 235 | func NewPostForm(keyValue ...string) *PostForm { 236 | var p = &PostForm{} 237 | p.New(keyValue...) 238 | return p 239 | } 240 | 241 | // RequestOption interface method, bind request option to request. 242 | func (options *PostForm) bindRequest(request *Request) error { 243 | request.PostForm = options 244 | return nil 245 | } 246 | 247 | type Headers struct { 248 | http.Header 249 | } 250 | 251 | // RequestOption interface method, bind request option to request. 252 | func (options Headers) bindRequest(request *Request) error { 253 | request.Headers = options.Header 254 | return nil 255 | } 256 | 257 | // NewHeaders new a http.Header type. 258 | // 259 | // You can set key-value pair when you init it by sent parameters. Just like this: 260 | // headers := NewHeaders( 261 | // "key1", "value1", 262 | // "key2", "value2", 263 | // ) 264 | // But be careful, between the key and value is a comma. 265 | // And if the number of parameters is not a multiple of 2, it will panic. 266 | func NewHeaders(keyValue ...string) *Headers { 267 | h := new(Headers) 268 | h.Header = http.Header{} 269 | if keyValue != nil { 270 | if len(keyValue)%2 != 0 { 271 | panic("key and value must be part") 272 | } 273 | 274 | for i := 0; i < len(keyValue)/2; i++ { 275 | key := keyValue[i*2] 276 | value := keyValue[i*2+1] 277 | h.Add(key, value) 278 | } 279 | } 280 | return h 281 | } 282 | 283 | // Cookies is request cookies, as parameter in Request method. 284 | // You should init it by using NewCookies like this: 285 | // cookies := dw.NewCookies( 286 | // "key1", "value1", 287 | // "key2", "value2", 288 | // ) 289 | // Note: mid symbol is comma. 290 | type Cookies []*http.Cookie 291 | 292 | // NewCookies new a Cookies type. 293 | // 294 | // You can set key-value pair when you init it by sent parameters. Just like this: 295 | // cookies := NewCookies( 296 | // "key1", "value1", 297 | // "key2", "value2", 298 | // ) 299 | // But be careful, between the key and value is a comma. 300 | // And if the number of parameters is not a multiple of 2, it will panic. 301 | func NewCookies(keyValue ...string) Cookies { 302 | c := make(Cookies, 0) 303 | if keyValue != nil { 304 | if len(keyValue)%2 != 0 { 305 | panic("key and value must be part") 306 | } 307 | 308 | for i := 0; i < len(keyValue)/2; i++ { 309 | key := keyValue[i*2] 310 | value := keyValue[i*2+1] 311 | cookie := &http.Cookie{Name: key, Value: value} 312 | c = append(c, cookie) 313 | } 314 | } 315 | return c 316 | } 317 | 318 | // RequestOption interface method, bind request option to request. 319 | func (c Cookies) bindRequest(request *Request) error { 320 | request.Cookies = c 321 | return nil 322 | } 323 | 324 | // Add append a new cookie to Cookies. 325 | func (c Cookies) Add(key, value string) { 326 | c = append(c, &http.Cookie{Name: key, Value: value}) 327 | } 328 | -------------------------------------------------------------------------------- /session.go: -------------------------------------------------------------------------------- 1 | package direwolf 2 | 3 | import ( 4 | "net" 5 | "net/http" 6 | "net/http/cookiejar" 7 | "net/url" 8 | "sync" 9 | "time" 10 | 11 | "golang.org/x/net/http/httpproxy" 12 | 13 | "golang.org/x/net/publicsuffix" 14 | ) 15 | 16 | // Session is the main object in direwolf. This is its main features: 17 | // 1. handling redirects 18 | // 2. automatically managing cookies 19 | type Session struct { 20 | client *http.Client 21 | transport *http.Transport 22 | Headers http.Header 23 | Proxy *Proxy 24 | Timeout int 25 | } 26 | 27 | // NewSession new a Session object, and set a default Client and Transport. 28 | func NewSession(options ...*SessionOptions) *Session { 29 | var sessionOptions *SessionOptions 30 | if len(options) > 0 { 31 | sessionOptions = options[0] 32 | } else { 33 | sessionOptions = DefaultSessionOptions() 34 | } 35 | 36 | // set transport parameters. 37 | trans := &http.Transport{ 38 | DialContext: (&net.Dialer{ 39 | Timeout: sessionOptions.DialTimeout, 40 | KeepAlive: sessionOptions.DialKeepAlive, 41 | }).DialContext, 42 | MaxIdleConns: sessionOptions.MaxIdleConns, 43 | MaxIdleConnsPerHost: sessionOptions.MaxIdleConnsPerHost, 44 | MaxConnsPerHost: sessionOptions.MaxConnsPerHost, 45 | IdleConnTimeout: sessionOptions.IdleConnTimeout, 46 | TLSHandshakeTimeout: sessionOptions.TLSHandshakeTimeout, 47 | ExpectContinueTimeout: sessionOptions.ExpectContinueTimeout, 48 | Proxy: proxyFunc, 49 | } 50 | if sessionOptions.DisableDialKeepAlives { 51 | trans.DisableKeepAlives = true 52 | } 53 | 54 | client := &http.Client{ 55 | Transport: trans, 56 | CheckRedirect: redirectFunc, 57 | } 58 | 59 | // set CookieJar 60 | if sessionOptions.DisableCookieJar == false { 61 | cookieJarOptions := cookiejar.Options{ 62 | PublicSuffixList: publicsuffix.List, 63 | } 64 | jar, err := cookiejar.New(&cookieJarOptions) 65 | if err != nil { 66 | return nil 67 | } 68 | client.Jar = jar 69 | } 70 | 71 | // Set default user agent 72 | headers := http.Header{} 73 | headers.Add("User-Agent", "direwolf - winter is coming") 74 | 75 | return &Session{ 76 | client: client, 77 | transport: trans, 78 | Headers: headers, 79 | } 80 | } 81 | 82 | // Send is a generic request method. 83 | func (session *Session) Send(req *Request) (*Response, error) { 84 | resp, err := send(session, req) 85 | if err != nil { 86 | return nil, WrapErr(err, "session send failed") 87 | } 88 | return resp, nil 89 | } 90 | 91 | // Get is a get method. 92 | func (session *Session) Get(URL string, args ...RequestOption) (*Response, error) { 93 | req, err := NewRequest("GET", URL, args...) 94 | if err != nil { 95 | return nil, err 96 | } 97 | resp, err := session.Send(req) 98 | if err != nil { 99 | return nil, err 100 | } 101 | return resp, nil 102 | } 103 | 104 | // Post is a post method. 105 | func (session *Session) Post(URL string, args ...RequestOption) (*Response, error) { 106 | req, err := NewRequest("POST", URL, args...) 107 | if err != nil { 108 | return nil, err 109 | } 110 | resp, err := session.Send(req) 111 | if err != nil { 112 | return nil, err 113 | } 114 | return resp, nil 115 | } 116 | 117 | // Head is a post method. 118 | func (session *Session) Head(URL string, args ...RequestOption) (*Response, error) { 119 | req, err := NewRequest("HEAD", URL, args...) 120 | if err != nil { 121 | return nil, err 122 | } 123 | resp, err := session.Send(req) 124 | if err != nil { 125 | return nil, err 126 | } 127 | return resp, nil 128 | } 129 | 130 | // Put is a post method. 131 | func (session *Session) Put(URL string, args ...RequestOption) (*Response, error) { 132 | req, err := NewRequest("PUT", URL, args...) 133 | if err != nil { 134 | return nil, err 135 | } 136 | resp, err := session.Send(req) 137 | if err != nil { 138 | return nil, err 139 | } 140 | return resp, nil 141 | } 142 | 143 | // Patch is a post method. 144 | func (session *Session) Patch(URL string, args ...RequestOption) (*Response, error) { 145 | req, err := NewRequest("PATCH", URL, args...) 146 | if err != nil { 147 | return nil, err 148 | } 149 | resp, err := session.Send(req) 150 | if err != nil { 151 | return nil, err 152 | } 153 | return resp, nil 154 | } 155 | 156 | // Delete is a post method. 157 | func (session *Session) Delete(URL string, args ...RequestOption) (*Response, error) { 158 | req, err := NewRequest("DELETE", URL, args...) 159 | if err != nil { 160 | return nil, err 161 | } 162 | resp, err := session.Send(req) 163 | if err != nil { 164 | return nil, err 165 | } 166 | return resp, nil 167 | } 168 | 169 | // Cookies returns the cookies of the given url in Session. 170 | func (session *Session) Cookies(URL string) Cookies { 171 | if session.client.Jar == nil { 172 | return nil 173 | } 174 | parsedURL, err := url.Parse(URL) 175 | if err != nil { 176 | return nil 177 | } 178 | return session.client.Jar.Cookies(parsedURL) 179 | } 180 | 181 | // SetCookies set cookies of the url in Session. 182 | func (session *Session) SetCookies(URL string, cookies Cookies) { 183 | if session.client.Jar == nil { 184 | return 185 | } 186 | parsedURL, err := url.Parse(URL) 187 | if err != nil { 188 | return 189 | } 190 | session.client.Jar.SetCookies(parsedURL, cookies) 191 | } 192 | 193 | type SessionOptions struct { 194 | // DialTimeout is the maximum amount of time a dial will wait for 195 | // a connect to complete. 196 | // 197 | // When using TCP and dialing a host name with multiple IP 198 | // addresses, the timeout may be divided between them. 199 | // 200 | // With or without a timeout, the operating system may impose 201 | // its own earlier timeout. For instance, TCP timeouts are 202 | // often around 3 minutes. 203 | DialTimeout time.Duration 204 | 205 | // DialKeepAlive specifies the interval between keep-alive 206 | // probes for an active network connection. 207 | // 208 | // Network protocols or operating systems that do 209 | // not support keep-alives ignore this field. 210 | // If negative, keep-alive probes are disabled. 211 | DialKeepAlive time.Duration 212 | 213 | // MaxConnsPerHost optionally limits the total number of 214 | // connections per host, including connections in the dialing, 215 | // active, and idle states. On limit violation, dials will block. 216 | // 217 | // Zero means no limit. 218 | MaxConnsPerHost int 219 | 220 | // MaxIdleConns controls the maximum number of idle (keep-alive) 221 | // connections across all hosts. Zero means no limit. 222 | MaxIdleConns int 223 | 224 | // MaxIdleConnsPerHost, if non-zero, controls the maximum idle 225 | // (keep-alive) connections to keep per-host. If zero, 226 | // DefaultMaxIdleConnsPerHost is used. 227 | MaxIdleConnsPerHost int 228 | 229 | // IdleConnTimeout is the maximum amount of time an idle 230 | // (keep-alive) connection will remain idle before closing 231 | // itself. 232 | // Zero means no limit. 233 | IdleConnTimeout time.Duration 234 | 235 | // TLSHandshakeTimeout specifies the maximum amount of time waiting to 236 | // wait for a TLS handshake. Zero means no timeout. 237 | TLSHandshakeTimeout time.Duration 238 | 239 | // ExpectContinueTimeout, if non-zero, specifies the amount of 240 | // time to wait for a server's first response headers after fully 241 | // writing the request headers if the request has an 242 | // "Expect: 100-continue" header. Zero means no timeout and 243 | // causes the body to be sent immediately, without 244 | // waiting for the server to approve. 245 | // This time does not include the time to send the request header. 246 | ExpectContinueTimeout time.Duration 247 | 248 | // DisableCookieJar specifies whether disable session cookiejar. 249 | DisableCookieJar bool 250 | 251 | // DisableDialKeepAlives, if true, disables HTTP keep-alives and 252 | // will only use the connection to the server for a single 253 | // HTTP request. 254 | // 255 | // This is unrelated to the similarly named TCP keep-alives. 256 | DisableDialKeepAlives bool 257 | } 258 | 259 | // DefaultSessionOptions return a default SessionOptions object. 260 | func DefaultSessionOptions() *SessionOptions { 261 | return &SessionOptions{ 262 | DialTimeout: 30 * time.Second, 263 | DialKeepAlive: 30 * time.Second, 264 | MaxConnsPerHost: 0, 265 | MaxIdleConns: 100, 266 | MaxIdleConnsPerHost: 2, 267 | IdleConnTimeout: 90 * time.Second, 268 | TLSHandshakeTimeout: 10 * time.Second, 269 | ExpectContinueTimeout: 1 * time.Second, 270 | DisableCookieJar: false, 271 | DisableDialKeepAlives: false, 272 | } 273 | } 274 | 275 | var ( 276 | // proxyConfigOnce guards proxyConfig 277 | envProxyOnce sync.Once 278 | envProxyFuncValue func(*url.URL) (*url.URL, error) 279 | ) 280 | 281 | // proxyFunc get proxy from request context. 282 | // If there is no proxy set, use default proxy from environment. 283 | func proxyFunc(req *http.Request) (*url.URL, error) { 284 | httpURLStr := req.Context().Value("http") // get http proxy url form context 285 | httpsURLStr := req.Context().Value("https") // get https proxy url form context 286 | 287 | // If there is no proxy set, use default proxy from environment. 288 | // This mitigates expensive lookups on some platforms (e.g. Windows). 289 | envProxyOnce.Do(func() { 290 | envProxyFuncValue = httpproxy.FromEnvironment().ProxyFunc() 291 | }) 292 | 293 | if req.URL.Scheme == "http" { // set proxy for http site 294 | if httpURLStr != nil { 295 | httpURL, err := url.Parse(httpURLStr.(string)) 296 | if err != nil { 297 | return nil, WrapErr(err, "HTTP Proxy error, please check proxy url") 298 | } 299 | return httpURL, nil 300 | } 301 | } else if req.URL.Scheme == "https" { // set proxy for https site 302 | if httpsURLStr != nil { 303 | httpsURL, err := url.Parse(httpsURLStr.(string)) 304 | if err != nil { 305 | return nil, WrapErr(err, "HTTPS Proxy error, please check proxy url") 306 | } 307 | return httpsURL, nil 308 | } 309 | } 310 | 311 | return envProxyFuncValue(req.URL) 312 | } 313 | 314 | // redirectFunc get redirectNum from request context and check redirect number. 315 | func redirectFunc(req *http.Request, via []*http.Request) error { 316 | redirectNum := req.Context().Value("redirectNum").(int) 317 | if len(via) > redirectNum { 318 | err := &RedirectError{redirectNum} 319 | return WrapErr(err, "RedirectError") 320 | } 321 | return nil 322 | } 323 | --------------------------------------------------------------------------------