├── LICENSE ├── README.md ├── body.go ├── go.mod ├── go.sum ├── multipart.go ├── options.go ├── reader_with_cancel.go ├── requests.go ├── response.go ├── session.go ├── tools └── tools.go ├── util.go └── zhttp.go /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [2020] [Greyh4t] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # zhttp 2 | 3 | zhttp 是一个对 net/http 标准库的封装，参考了 python 中著名的 requests 库，支持标准库中的连接池，支持dns缓存，支持流式文件上传，支持多种body格式，很多代码及思路来自[grequests](https://github.com/levigross/grequests) 4 | 5 | [![PkgGoDev](https://pkg.go.dev/badge/github.com/greyh4t/zhttp)](https://pkg.go.dev/github.com/greyh4t/zhttp) 6 | 7 | ## Installation 8 | 9 | ``` 10 | go get github.com/greyh4t/zhttp 11 | ``` 12 | 13 | ## Usage 14 | 15 | #### 直接使用默认client 16 | ```go 17 | import "github.com/greyh4t/zhttp" 18 | 19 | func main() { 20 | resp, err := zhttp.Get("http://www.example.com/", nil) 21 | if err != nil { 22 | return 23 | } 24 | resp.Close() 25 | } 26 | ``` 27 | 28 | #### 更改默认client配置 29 | ```go 30 | import "github.com/greyh4t/zhttp" 31 | 32 | func main() { 33 | zhttp.InitDefaultClient(&zhttp.HTTPOptions{ 34 | Proxies: zhttp.MustProxy(zhttp.M{ 35 | "http": "http://127.0.0.1:8080", 36 | "https": "http://127.0.0.1:8080", 37 | }), 38 | }) 39 | 40 | resp, err := zhttp.Get("http://www.example.com/", nil) 41 | if err != nil { 42 | return 43 | } 44 | resp.Close() 45 | } 46 | ``` 47 | 48 | #### 创建独立的client使用 49 | ```go 50 | import "github.com/greyh4t/zhttp" 51 | 52 | func main() { 53 | z := zhttp.New(&zhttp.HTTPOptions{ 54 | Proxies: zhttp.MustProxy(zhttp.M{ 55 | "http": "http://127.0.0.1:8080", 56 | "https": "http://127.0.0.1:8080", 57 | }), 58 | }) 59 | 60 | resp, err := z.Get("http://www.example.com/", nil) 61 | if err != nil { 62 | return 63 | } 64 | resp.Close() 65 | } 66 | ``` 67 | 68 | ## Example 69 | 70 | 如下为简单示例，更多使用方法请参考godoc 71 | 72 | ```go 73 | package main 74 | 75 | import ( 76 | "log" 77 | "net/url" 78 | "os" 79 | "time" 80 | 81 | "github.com/greyh4t/zhttp" 82 | "github.com/greyh4t/zhttp/tools" 83 | ) 84 | 85 | func main() { 86 | z := zhttp.New(&zhttp.HTTPOptions{ 87 | UserAgent: "global-useragent", 88 | Headers: zhttp.M{ 89 | "globalheader1": "value1", 90 | "globalheader2": "value2", 91 | }, 92 | DNSCacheExpire: time.Minute * 10, 93 | DNSServer: "8.8.8.8:25", 94 | InsecureSkipVerify: true, 95 | DialTimeout: time.Second * 5, 96 | TLSHandshakeTimeout: time.Second * 5, 97 | KeepAlive: time.Second * 10, 98 | MaxIdleConns: 10, 99 | }) 100 | 101 | // 请求1 102 | resp, err := z.Get("http://www.example.com/", nil) 103 | if err != nil { 104 | log.Fatal(err) 105 | } 106 | log.Println(resp.StatusCode, resp.Status, resp.ContentLength) 107 | log.Println(resp.Headers.String()) 108 | log.Println(resp.Cookies().String()) 109 | log.Println(resp.DumpRequest()) 110 | log.Println(resp.Body.String()) 111 | resp.Close() 112 | 113 | // 请求2 post表单 114 | resp, err = z.Post("http://www.example.com/?query1=value3", &zhttp.ReqOptions{ 115 | DisableRedirect: true, 116 | Timeout: time.Second * 10, 117 | Proxies: zhttp.MustProxy(zhttp.M{ 118 | "http": "http://127.0.0.1:8080", 119 | "https": "http://127.0.0.1:8080", 120 | }), 121 | Headers: zhttp.M{ 122 | "header1": "value1", 123 | "header2": "value2", 124 | }, 125 | Cookies: zhttp.M{ 126 | "k1": "v1", 127 | "k2": "v2", 128 | }, 129 | Body: zhttp.Form(zhttp.M{ 130 | "key1": "value1", 131 | "key2": "value2", 132 | }), 133 | Query: url.Values{ 134 | "query1": {"value1"}, 135 | "query2": {"value2"}, 136 | }, 137 | }) 138 | if err != nil { 139 | log.Fatal(err) 140 | } 141 | body := resp.Body.Bytes() 142 | if resp.Body.Err != nil { 143 | log.Fatal(resp.Body.Err) 144 | } 145 | resp.Close() 146 | log.Println(body) 147 | 148 | // 请求3 post表单 149 | resp, err = z.Post("http://www.example.com/?query1=value1&query2=value2", &zhttp.ReqOptions{ 150 | Body: zhttp.FormString(`fk1=fv1&fk2=fv2`), 151 | Headers: zhttp.M{ 152 | "Cookie": "k1=v1; k2=v2", 153 | }, 154 | UserAgent: "zhttp-ua-test", 155 | }) 156 | if err != nil { 157 | log.Fatal(err) 158 | } 159 | resp.Close() 160 | 161 | // 请求4 post json 162 | resp, err = z.Post("http://www.example.com/", &zhttp.ReqOptions{ 163 | Body: zhttp.JSONString(`{"jk1":"jv","jk2":2}`), 164 | Headers: zhttp.M{ 165 | "Cookie": "k1=v1; k2=v2", 166 | }, 167 | UserAgent: "zhttp-ua-test", 168 | IsAjax: true, 169 | }) 170 | if err != nil { 171 | log.Fatal(err) 172 | } 173 | resp.Close() 174 | 175 | // 请求5 文件上传 176 | f, err := os.Open("test.txt") 177 | if err != nil { 178 | log.Fatal(err) 179 | } 180 | defer f.Close() 181 | 182 | resp, err = z.Post("http://www.example.com/", &zhttp.ReqOptions{ 183 | Body: zhttp.Reader(f), 184 | ContentType: "text/plain", 185 | Headers: zhttp.M{ 186 | "h1": "v1", 187 | "h2": "v2", 188 | }, 189 | Auth: zhttp.Auth{ 190 | Username: "username", 191 | Password: "password", 192 | }, 193 | }) 194 | if err != nil { 195 | log.Fatal(err) 196 | } 197 | resp.Close() 198 | 199 | // 请求6 文件上传 200 | file1, err := tools.FileFromDisk("file1.txt") 201 | if err != nil { 202 | log.Fatal(err) 203 | } 204 | 205 | file2, err := tools.FileFromDisk("file2.txt") 206 | if err != nil { 207 | log.Fatal(err) 208 | } 209 | 210 | resp, err = z.Post("http://www.example.com/", &zhttp.ReqOptions{ 211 | Body: zhttp.Multipart([]*zhttp.File{file1, file2}, zhttp.M{ 212 | "field1": "value1", 213 | "field2": "value2", 214 | }), 215 | Host: "file.example.com", 216 | }) 217 | if err != nil { 218 | log.Fatal(err) 219 | } 220 | resp.Close() 221 | 222 | // 请求7 session的使用 223 | s := z.NewSession() 224 | resp, err = s.Post("http://www.example.com/login", &zhttp.ReqOptions{ 225 | Body: zhttp.Form(zhttp.M{ 226 | "username": "username", 227 | "password": "password", 228 | }), 229 | Timeout: time.Second * 10, 230 | }) 231 | if err != nil { 232 | log.Fatal(err) 233 | } 234 | resp.Close() 235 | 236 | resp, err = s.Get("http://www.example.com/userinfo", nil) 237 | if err != nil { 238 | log.Fatal(err) 239 | } 240 | resp.Close() 241 | } 242 | ``` 243 | -------------------------------------------------------------------------------- /body.go: -------------------------------------------------------------------------------- 1 | package zhttp 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "encoding/xml" 7 | "io" 8 | "net/url" 9 | "strings" 10 | ) 11 | 12 | // Body is used to define the body part of the http request 13 | type Body interface { 14 | Content() (io.Reader, string, error) 15 | } 16 | 17 | // String used to create Body from string, need to set the Content-Type yourself 18 | func String(body string) Body { 19 | return &StringBody{Body: body} 20 | } 21 | 22 | // Bytes used to create Body from []byte, need to set the Content-Type yourself 23 | func Bytes(body []byte) Body { 24 | return &BytesBody{Body: body} 25 | } 26 | 27 | // Reader used to create Body from io.Reader, need to set the Content-Type yourself 28 | func Reader(body io.Reader) Body { 29 | return &ReaderBody{Body: body} 30 | } 31 | 32 | // JSONString used to create Body from string, and set json Content-Type 33 | func JSONString(body string) Body { 34 | return &StringBody{ 35 | ContentType: "application/json", 36 | Body: body, 37 | } 38 | } 39 | 40 | // JSONBytes used to create Body from []byte, and set json Content-Type 41 | func JSONBytes(body []byte) Body { 42 | return &BytesBody{ 43 | ContentType: "application/json", 44 | Body: body, 45 | } 46 | } 47 | 48 | // JSON used to create Body from map, struct and so on, and set json Content-Type 49 | func JSON(body interface{}) Body { 50 | return &JSONBody{body} 51 | } 52 | 53 | // XMLString used to create Body from string, and set xml Content-Type 54 | func XMLString(body string) Body { 55 | return &StringBody{ 56 | ContentType: "application/xml", 57 | Body: body, 58 | } 59 | } 60 | 61 | // XMLBytes used to create Body from []byte, and set xml Content-Type 62 | func XMLBytes(body []byte) Body { 63 | return &BytesBody{ 64 | ContentType: "application/xml", 65 | Body: body, 66 | } 67 | } 68 | 69 | // XML used to create Body from struct, and set xml Content-Type 70 | func XML(body interface{}) Body { 71 | return &XMLBody{body} 72 | } 73 | 74 | // FormString used to create Body from string, and set form Content-Type 75 | func FormString(body string) Body { 76 | return &StringBody{ 77 | ContentType: "application/x-www-form-urlencoded", 78 | Body: body, 79 | } 80 | } 81 | 82 | // FormBytes used to create Body from []byte, and set form Content-Type 83 | func FormBytes(body []byte) Body { 84 | return &BytesBody{ 85 | ContentType: "application/x-www-form-urlencoded", 86 | Body: body, 87 | } 88 | } 89 | 90 | // Form used to create Body from map, and set form Content-Type 91 | func Form(body map[string]string) Body { 92 | values := url.Values{} 93 | for key, value := range body { 94 | values.Set(key, value) 95 | } 96 | return &StringBody{ 97 | ContentType: "application/x-www-form-urlencoded", 98 | Body: values.Encode(), 99 | } 100 | } 101 | 102 | // FormValues used to create Body from map, and set form Content-Type 103 | // The difference with form is that it supports setting multiple parameters with the same name 104 | func FormValues(body map[string][]string) Body { 105 | return &StringBody{ 106 | ContentType: "application/x-www-form-urlencoded", 107 | Body: url.Values(body).Encode(), 108 | } 109 | } 110 | 111 | type ReaderBody struct { 112 | ContentType string 113 | Body io.Reader 114 | } 115 | 116 | func (body *ReaderBody) Content() (io.Reader, string, error) { 117 | return body.Body, body.ContentType, nil 118 | } 119 | 120 | type StringBody struct { 121 | ContentType string 122 | Body string 123 | } 124 | 125 | func (body *StringBody) Content() (io.Reader, string, error) { 126 | return strings.NewReader(body.Body), body.ContentType, nil 127 | } 128 | 129 | type BytesBody struct { 130 | ContentType string 131 | Body []byte 132 | } 133 | 134 | func (body *BytesBody) Content() (io.Reader, string, error) { 135 | return bytes.NewReader(body.Body), body.ContentType, nil 136 | } 137 | 138 | type JSONBody struct { 139 | Body interface{} 140 | } 141 | 142 | func (body *JSONBody) Content() (io.Reader, string, error) { 143 | contentType := "application/json" 144 | data, err := json.Marshal(body.Body) 145 | if err != nil { 146 | return nil, "", err 147 | } 148 | return bytes.NewReader(data), contentType, nil 149 | } 150 | 151 | type XMLBody struct { 152 | Body interface{} 153 | } 154 | 155 | func (body *XMLBody) Content() (io.Reader, string, error) { 156 | contentType := "application/xml" 157 | data, err := xml.Marshal(body.Body) 158 | if err != nil { 159 | return nil, "", err 160 | } 161 | return bytes.NewReader(data), contentType, nil 162 | } 163 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/greyh4t/zhttp 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/greyh4t/dnscache v0.0.0-20200422032442-29453c061c08 7 | golang.org/x/net v0.17.0 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/greyh4t/dnscache v0.0.0-20200422032442-29453c061c08 h1:eTP0Lb9KHlWWys5GBjWWBd74VNvzxJmRdqeLy134dSE= 2 | github.com/greyh4t/dnscache v0.0.0-20200422032442-29453c061c08/go.mod h1:QtTdAWVz7zSfKy/zH9+YOQlFDuUC8alCiAOt0zFf6/o= 3 | golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= 4 | golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= 5 | -------------------------------------------------------------------------------- /multipart.go: -------------------------------------------------------------------------------- 1 | package zhttp 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "mime/multipart" 8 | "net/textproto" 9 | "strconv" 10 | ) 11 | 12 | type File struct { 13 | // Name is the name of the file that you wish to upload. 14 | // We use this to guess the mimetype as well as pass it onto the server 15 | Name string 16 | 17 | // Contents is happy as long as you pass it a io.ReadCloser (which most file use anyways) 18 | Contents io.ReadCloser 19 | 20 | // FieldName is form field name 21 | FieldName string 22 | 23 | // Mime represents which mime should be sent along with the file. 24 | // When empty, defaults to application/octet-stream 25 | Mime string 26 | } 27 | 28 | // Multipart used to create Body object 29 | func Multipart(files []*File, form map[string]string) Body { 30 | return &MultipartBody{ 31 | Files: files, 32 | Form: form, 33 | } 34 | } 35 | 36 | // MultipartStream used to create Body object 37 | // Use streaming upload to prevent all files from being loaded into memory 38 | func MultipartStream(files []*File, form map[string]string) Body { 39 | return &MultipartBody{ 40 | Files: files, 41 | Form: form, 42 | Stream: true, 43 | } 44 | } 45 | 46 | type MultipartBody struct { 47 | Files []*File 48 | Form map[string]string 49 | Stream bool 50 | } 51 | 52 | func (body *MultipartBody) Close() { 53 | for _, f := range body.Files { 54 | if f.Contents != nil { 55 | f.Contents.Close() 56 | } 57 | } 58 | } 59 | 60 | func (body *MultipartBody) Content() (io.Reader, string, error) { 61 | if body.Stream { 62 | return body.streamContent() 63 | } 64 | 65 | var buf bytes.Buffer 66 | multipartWriter := multipart.NewWriter(&buf) 67 | err := body.writeMultipart(multipartWriter) 68 | if err != nil { 69 | body.Close() 70 | return nil, "", err 71 | } 72 | 73 | return &buf, multipartWriter.FormDataContentType(), nil 74 | } 75 | 76 | func (body *MultipartBody) writeMultipart(multipartWriter *multipart.Writer) (err error) { 77 | for i, f := range body.Files { 78 | fieldName := f.FieldName 79 | 80 | if fieldName == "" { 81 | if len(body.Files) > 1 { 82 | fieldName = "file" + strconv.Itoa(i+1) 83 | } else { 84 | fieldName = "file" 85 | } 86 | } 87 | 88 | var writer io.Writer 89 | if f.Mime != "" { 90 | h := make(textproto.MIMEHeader) 91 | h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, 92 | escapeQuotes(fieldName), escapeQuotes(f.Name))) 93 | h.Set("Content-Type", f.Mime) 94 | writer, err = multipartWriter.CreatePart(h) 95 | } else { 96 | writer, err = multipartWriter.CreateFormFile(fieldName, f.Name) 97 | } 98 | if err != nil { 99 | return 100 | } 101 | 102 | if f.Contents != nil { 103 | _, err = io.Copy(writer, f.Contents) 104 | if err != nil && err != io.EOF { 105 | return 106 | } 107 | err = f.Contents.Close() 108 | if err != nil { 109 | return 110 | } 111 | } 112 | } 113 | 114 | // Populate the other parts of the form (if there are any) 115 | for key, value := range body.Form { 116 | multipartWriter.WriteField(key, value) 117 | } 118 | 119 | // Close just write last boundary, so we only need to close it when all processes successful. 120 | err = multipartWriter.Close() 121 | 122 | return 123 | } 124 | 125 | func (body *MultipartBody) streamContent() (io.Reader, string, error) { 126 | pr, pw := io.Pipe() 127 | multipartWriter := multipart.NewWriter(pw) 128 | 129 | go func() { 130 | var err error 131 | defer func() { 132 | if err != nil { 133 | body.Close() 134 | pw.CloseWithError(err) 135 | } else { 136 | pw.Close() 137 | } 138 | }() 139 | err = body.writeMultipart(multipartWriter) 140 | }() 141 | 142 | return pr, multipartWriter.FormDataContentType(), nil 143 | } 144 | -------------------------------------------------------------------------------- /options.go: -------------------------------------------------------------------------------- 1 | package zhttp 2 | 3 | import ( 4 | "net/url" 5 | "time" 6 | ) 7 | 8 | // Auth is used to perform HTTP Basic authentication 9 | type Auth struct { 10 | Username string 11 | Password string 12 | } 13 | 14 | // M is short for map[string] string 15 | type M map[string]string 16 | 17 | // P is short for map[string]*url.URL 18 | type P map[string]*url.URL 19 | 20 | // HTTPOptions is the options for zhttp.Zhttp, 21 | // it will effective on per request 22 | type HTTPOptions struct { 23 | // UserAgent allows you to set an arbitrary custom user agent 24 | UserAgent string 25 | 26 | // Cookie allows you to attach cookies to every request. 27 | Cookies map[string]string 28 | 29 | // Headers uses to set custom HTTP headers to every request 30 | // The header name is case-sensitive 31 | Headers map[string]string 32 | 33 | // Proxies is a map in the following format 34 | // *protocol* => proxy address e.g http => http://127.0.0.1:8080, 35 | Proxies map[string]*url.URL 36 | 37 | // InsecureSkipVerify is a flag that specifies if we should validate the 38 | // server's TLS certificate. It should be noted that Go's TLS verify mechanism 39 | // doesn't validate if a certificate has been revoked 40 | InsecureSkipVerify bool 41 | 42 | // RequestTimeout is the maximum amount of time a whole request(include dial / request / redirect) will wait 43 | RequestTimeout time.Duration 44 | 45 | // Timeout is the time that the client will wait between bytes sent from the server. 46 | Timeout time.Duration 47 | 48 | // DialTimeout is the maximum amount of time a dial will wait for a connect to complete 49 | DialTimeout time.Duration 50 | 51 | // TLSHandshakeTimeout specifies the maximum amount of time waiting to 52 | // wait for a TLS handshake. Zero means no timeout. 53 | TLSHandshakeTimeout time.Duration 54 | 55 | // KeepAlive specifies the interval between keep-alive 56 | // probes for an active network connection. 57 | // If zero, keep-alive probes are sent with a default value 58 | // (currently 15 seconds), if supported by the protocol and operating 59 | // system. Network protocols or operating systems that do 60 | // not support keep-alives ignore this field. 61 | // If negative, keep-alive probes are disabled. 62 | KeepAlive time.Duration 63 | 64 | // DisableKeepAlives, if true, disables HTTP keep-alives and 65 | // will only use the connection to the server for a single 66 | // HTTP request. 67 | // 68 | // This is unrelated to the similarly named TCP keep-alives. 69 | DisableKeepAlives bool 70 | 71 | // DisableCompression, if true, prevents the Transport from 72 | // requesting compression with an "Accept-Encoding: gzip" 73 | // request header when the Request contains no existing 74 | // Accept-Encoding value. If the Transport requests gzip on 75 | // its own and gets a gzipped response, it's transparently 76 | // decoded in the Response.Body. However, if the user 77 | // explicitly requested gzip it is not automatically 78 | // uncompressed. 79 | DisableCompression bool 80 | 81 | // MaxIdleConns controls the maximum number of idle (keep-alive) 82 | // connections across all hosts. Zero means no limit. 83 | MaxIdleConns int 84 | 85 | // MaxIdleConnsPerHost, if non-zero, controls the maximum idle 86 | // (keep-alive) connections to keep per-host. If zero, 87 | // DefaultMaxIdleConnsPerHost is used. 88 | MaxIdleConnsPerHost int 89 | 90 | // MaxConnsPerHost optionally limits the total number of 91 | // connections per host, including connections in the dialing, 92 | // active, and idle states. On limit violation, dials will block. 93 | // 94 | // Zero means no limit. 95 | MaxConnsPerHost int 96 | 97 | // IdleConnTimeout is the maximum amount of time an idle 98 | // (keep-alive) connection will remain idle before closing 99 | // itself. 100 | // Zero means no limit. 101 | IdleConnTimeout time.Duration 102 | 103 | // DNSCacheExpire is the timeout of dns cache , if zero, not use dns cache 104 | DNSCacheExpire time.Duration 105 | 106 | // DNSServer allows you to set an custom dns host, like 1.1.1.1:25, only effective in linux 107 | DNSServer string 108 | 109 | // NoUA is a flag that means do not set default UserAgent 110 | NoUA bool 111 | } 112 | 113 | // ReqOptions is the options for single request 114 | type ReqOptions struct { 115 | // RequestTimeout is the maximum amount of time a whole request(include dial / request / redirect) will wait. 116 | // if non-zero, overwrite HTTPOptions.Timeout in current request. 117 | RequestTimeout time.Duration 118 | 119 | // Timeout is the time that the client will wait between bytes sent from the server. 120 | Timeout time.Duration 121 | 122 | // ContentType allows you to set an arbitrary custom content type 123 | ContentType string 124 | 125 | // UserAgent allows you to set an arbitrary custom user agent 126 | UserAgent string 127 | 128 | // Proxies is a map in the following format 129 | // *protocol* => proxy address e.g http => http://127.0.0.1:8080, 130 | // If setted, overwrite HTTPOptions.Proxies in current request. 131 | Proxies map[string]*url.URL 132 | 133 | // DisableRedirect will disable redirect for request 134 | DisableRedirect bool 135 | 136 | // Query will be encode to query string that may be used within a GET request 137 | Query url.Values 138 | 139 | // Body is a interface{} that will eventually convert into the the body of a POST request 140 | Body Body 141 | 142 | // Cookie allows you to attach cookies to your request. 143 | // Only effective in current request 144 | Cookies map[string]string 145 | 146 | // Headers uses to set custom HTTP headers to the request 147 | // The header name is case-sensitive 148 | Headers map[string]string 149 | 150 | // Host allows you to set an arbitrary custom host 151 | Host string 152 | 153 | // HostIP allows you to set an custom dns resolution for current request. 154 | // The value should be an IP. 155 | // When proxy usable, this value does not take effect 156 | HostIP string 157 | 158 | // Auth allows you to specify a user name and password that you wish to 159 | // use when requesting the URL. It will use basic HTTP authentication 160 | // formatting the username and password in base64. 161 | Auth Auth 162 | 163 | // IsAjax is a flag that can be set to make the request appear 164 | // to be generated by browser Javascript. 165 | IsAjax bool 166 | 167 | // NoUA is a flag that means do not set default UserAgent 168 | NoUA bool 169 | } 170 | -------------------------------------------------------------------------------- /reader_with_cancel.go: -------------------------------------------------------------------------------- 1 | package zhttp 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "time" 8 | ) 9 | 10 | type ReaderWithCancel struct { 11 | rc io.ReadCloser 12 | cancel context.CancelFunc 13 | timeout time.Duration 14 | timer *time.Timer 15 | } 16 | 17 | func (r *ReaderWithCancel) readWithTimeout(p []byte) (int, error) { 18 | if r.timer != nil { 19 | r.timer.Reset(r.timeout) 20 | } else { 21 | r.timer = time.AfterFunc(r.timeout, r.cancel) 22 | } 23 | 24 | n, err := r.rc.Read(p) 25 | 26 | r.timer.Stop() 27 | 28 | if err == context.Canceled { 29 | err = fmt.Errorf("%w (timeout exceeded while read body)", err) 30 | } 31 | 32 | return n, err 33 | } 34 | 35 | func (r *ReaderWithCancel) Read(p []byte) (n int, err error) { 36 | if r.timeout > 0 { 37 | return r.readWithTimeout(p) 38 | } 39 | 40 | return r.rc.Read(p) 41 | } 42 | 43 | func (r *ReaderWithCancel) Close() error { 44 | r.cancel() 45 | return r.rc.Close() 46 | } 47 | -------------------------------------------------------------------------------- /requests.go: -------------------------------------------------------------------------------- 1 | package zhttp 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "fmt" 7 | "net" 8 | "net/http" 9 | "net/url" 10 | "time" 11 | 12 | "github.com/greyh4t/dnscache" 13 | ) 14 | 15 | var ctxOptionKey = struct{}{} 16 | 17 | func disableRedirect(req *http.Request, via []*http.Request) error { 18 | return http.ErrUseLastResponse 19 | } 20 | 21 | // buildClient make a new client 22 | func (z *Zhttp) buildClient(httpOptions *HTTPOptions, reqOptions *ReqOptions, cookieJar http.CookieJar) *http.Client { 23 | client := &http.Client{ 24 | Transport: z.transport, 25 | Jar: cookieJar, 26 | Timeout: httpOptions.RequestTimeout, 27 | } 28 | 29 | if reqOptions.RequestTimeout > 0 { 30 | client.Timeout = reqOptions.RequestTimeout 31 | } 32 | 33 | if reqOptions.DisableRedirect { 34 | client.CheckRedirect = disableRedirect 35 | } 36 | 37 | return client 38 | } 39 | 40 | // createTransport create a global *http.Transport for all http client 41 | func createTransport(options *HTTPOptions, cache *dnscache.Cache) *http.Transport { 42 | transport := http.DefaultTransport.(*http.Transport).Clone() 43 | 44 | transport.MaxIdleConnsPerHost = options.MaxIdleConnsPerHost 45 | transport.MaxConnsPerHost = options.MaxConnsPerHost 46 | transport.DisableKeepAlives = options.DisableKeepAlives 47 | transport.DisableCompression = options.DisableCompression 48 | 49 | if options.InsecureSkipVerify { 50 | transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: options.InsecureSkipVerify} 51 | } 52 | if options.IdleConnTimeout > 0 { 53 | transport.IdleConnTimeout = options.IdleConnTimeout 54 | } 55 | if options.MaxIdleConns > 0 { 56 | transport.MaxIdleConns = options.MaxIdleConns 57 | } 58 | if options.TLSHandshakeTimeout > 0 { 59 | transport.TLSHandshakeTimeout = options.TLSHandshakeTimeout 60 | } 61 | 62 | transport.Proxy = func(req *http.Request) (*url.URL, error) { 63 | reqOptions, ok := req.Context().Value(ctxOptionKey).(*ReqOptions) 64 | if ok && len(reqOptions.Proxies) > 0 { 65 | if p, ok := reqOptions.Proxies[req.URL.Scheme]; ok { 66 | return p, nil 67 | } 68 | } else if len(options.Proxies) > 0 { 69 | if p, ok := options.Proxies[req.URL.Scheme]; ok { 70 | return p, nil 71 | } 72 | } 73 | // get proxy from environment 74 | return http.ProxyFromEnvironment(req) 75 | } 76 | 77 | dialer := &net.Dialer{ 78 | Timeout: 30 * time.Second, 79 | KeepAlive: 30 * time.Second, 80 | } 81 | if options.DialTimeout > 0 { 82 | dialer.Timeout = options.DialTimeout 83 | } 84 | if options.KeepAlive != 0 { 85 | dialer.KeepAlive = options.KeepAlive 86 | } 87 | 88 | transport.DialContext = makeDialContext(dialer, cache) 89 | 90 | return transport 91 | } 92 | 93 | func makeDialContext(dialer *net.Dialer, cache *dnscache.Cache) func(ctx context.Context, network, addr string) (net.Conn, error) { 94 | return func(ctx context.Context, network string, address string) (net.Conn, error) { 95 | reqOptions, ok := ctx.Value(ctxOptionKey).(*ReqOptions) 96 | if ok && reqOptions.HostIP != "" { 97 | _, port, _ := net.SplitHostPort(address) 98 | address = net.JoinHostPort(reqOptions.HostIP, port) 99 | } else if cache != nil { 100 | host, port, _ := net.SplitHostPort(address) 101 | ip, err := cache.FetchOneV4String(host) 102 | if err != nil { 103 | return nil, err 104 | } 105 | address = net.JoinHostPort(ip, port) 106 | } 107 | 108 | return dialer.DialContext(ctx, network, address) 109 | } 110 | } 111 | 112 | // doRequest send request with http client to server 113 | func (z *Zhttp) doRequest(method, rawURL string, options *ReqOptions, jar http.CookieJar) (*Response, error) { 114 | if options == nil { 115 | options = &ReqOptions{} 116 | } 117 | 118 | rawURL, err := z.buildURL(rawURL, options) 119 | if err != nil { 120 | return nil, err 121 | } 122 | 123 | ctx, cancel := context.WithCancel(context.Background()) 124 | req, err := z.buildRequest(ctx, method, rawURL, options) 125 | if err != nil { 126 | cancel() 127 | return nil, err 128 | } 129 | 130 | z.addCookies(req, options) 131 | z.addHeaders(req, options) 132 | 133 | client := z.buildClient(z.options, options, jar) 134 | 135 | timeout := z.options.Timeout 136 | if options.Timeout > 0 { 137 | timeout = options.Timeout 138 | } 139 | 140 | resp, err := z.do(client, req, cancel, timeout) 141 | if err != nil { 142 | cancel() 143 | return nil, err 144 | } 145 | 146 | return &Response{ 147 | RawResponse: resp, 148 | StatusCode: resp.StatusCode, 149 | Status: resp.Status, 150 | ContentLength: resp.ContentLength, 151 | Headers: Headers(resp.Header), 152 | Body: &ZBody{ 153 | rawBody: &ReaderWithCancel{ 154 | rc: resp.Body, 155 | cancel: cancel, 156 | timeout: timeout, 157 | }, 158 | }, 159 | }, nil 160 | } 161 | 162 | func (z *Zhttp) do(client *http.Client, req *http.Request, cancel context.CancelFunc, 163 | timeout time.Duration) (*http.Response, error) { 164 | if timeout > 0 { 165 | timer := time.AfterFunc(timeout, cancel) 166 | resp, err := client.Do(req) 167 | timer.Stop() 168 | if err == context.Canceled { 169 | err = fmt.Errorf("%w (timeout exceeded while send request)", err) 170 | } 171 | return resp, err 172 | } 173 | 174 | return client.Do(req) 175 | } 176 | 177 | // buildRequest build request with body and other 178 | func (z *Zhttp) buildRequest(ctx context.Context, method, rawURL string, options *ReqOptions) (*http.Request, error) { 179 | if len(options.Proxies) > 0 || options.HostIP != "" { 180 | ctx = context.WithValue(ctx, ctxOptionKey, options) 181 | } 182 | 183 | if options.Body == nil { 184 | return http.NewRequestWithContext(ctx, method, rawURL, nil) 185 | } 186 | 187 | bodyReader, contentType, err := options.Body.Content() 188 | if err != nil { 189 | return nil, err 190 | } 191 | 192 | req, err := http.NewRequestWithContext(ctx, method, rawURL, bodyReader) 193 | if err != nil { 194 | return nil, err 195 | } 196 | 197 | if contentType != "" { 198 | req.Header.Set("Content-Type", contentType) 199 | } 200 | 201 | return req, nil 202 | } 203 | 204 | // buildURL make url and set custom query 205 | func (z *Zhttp) buildURL(rawURL string, options *ReqOptions) (string, error) { 206 | if len(options.Query) > 0 { 207 | parsedURL, err := url.Parse(rawURL) 208 | if err != nil { 209 | return "", err 210 | } 211 | 212 | if parsedURL.RawQuery != "" { 213 | parsedURL.RawQuery += "&" + options.Query.Encode() 214 | } else { 215 | parsedURL.RawQuery = options.Query.Encode() 216 | } 217 | 218 | rawURL = parsedURL.String() 219 | } 220 | 221 | return rawURL, nil 222 | } 223 | 224 | // addHeaders handle custom headers 225 | func (z *Zhttp) addHeaders(req *http.Request, options *ReqOptions) { 226 | z.setDefaultHeaders(req, options) 227 | 228 | // set global headers 229 | z.setHeaders(req, z.options.Headers) 230 | 231 | if z.options.UserAgent != "" { 232 | req.Header.Set("User-Agent", z.options.UserAgent) 233 | } 234 | 235 | // set headers of each request 236 | z.setHeaders(req, options.Headers) 237 | 238 | if options.Host != "" { 239 | req.Host = options.Host 240 | } 241 | 242 | if options.Auth.Username != "" { 243 | req.SetBasicAuth(options.Auth.Username, options.Auth.Password) 244 | } 245 | 246 | if options.IsAjax { 247 | req.Header.Set("X-Requested-With", "XMLHttpRequest") 248 | } 249 | 250 | if options.ContentType != "" { 251 | req.Header.Set("Content-Type", options.ContentType) 252 | } 253 | 254 | if options.UserAgent != "" { 255 | req.Header.Set("User-Agent", options.UserAgent) 256 | } 257 | } 258 | 259 | func (z *Zhttp) setDefaultHeaders(req *http.Request, options *ReqOptions) { 260 | if z.options.NoUA || options.NoUA { 261 | // set empty string to prevent go client set default UA 262 | req.Header.Set("User-Agent", "") 263 | } else { 264 | req.Header.Set("User-Agent", "Zhttp/2.0") 265 | } 266 | } 267 | 268 | func (z *Zhttp) setHeaders(req *http.Request, headers map[string]string) { 269 | for key, value := range headers { 270 | req.Header[key] = []string{value} 271 | } 272 | } 273 | 274 | // addCookies handle custom cookies 275 | func (z *Zhttp) addCookies(req *http.Request, options *ReqOptions) { 276 | for k, v := range z.options.Cookies { 277 | if _, ok := options.Cookies[k]; !ok { 278 | req.AddCookie(&http.Cookie{Name: k, Value: v}) 279 | } 280 | } 281 | 282 | for k, v := range options.Cookies { 283 | req.AddCookie(&http.Cookie{Name: k, Value: v}) 284 | } 285 | } 286 | -------------------------------------------------------------------------------- /response.go: -------------------------------------------------------------------------------- 1 | package zhttp 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "net/http/httputil" 10 | "strings" 11 | ) 12 | 13 | // ZBody is a wrapper for http.Response.ZBody 14 | type ZBody struct { 15 | rawBody io.ReadCloser 16 | buf bytes.Buffer 17 | bufCached bool 18 | Err error 19 | } 20 | 21 | // Read is the implementation of the reader interface 22 | func (b *ZBody) Read(p []byte) (int, error) { 23 | if b.Err != nil { 24 | return 0, b.Err 25 | } 26 | 27 | return b.rawBody.Read(p) 28 | } 29 | 30 | // ReadN read and return n byte of body, and cache them 31 | func (b *ZBody) ReadN(n int64) []byte { 32 | if b.Err != nil { 33 | return nil 34 | } 35 | 36 | lr := io.LimitReader(b.rawBody, n) 37 | tr := io.TeeReader(lr, &(b.buf)) 38 | 39 | data, err := ioutil.ReadAll(tr) 40 | if err != nil && err != io.EOF { 41 | b.Err = err 42 | b.ClearCache() 43 | b.rawBody.Close() 44 | return nil 45 | } 46 | 47 | return data 48 | } 49 | 50 | // fillBuffer cache the body content – this is largely used for .String() and .Bytes() 51 | func (b *ZBody) fillBuffer() { 52 | if b.bufCached { 53 | return 54 | } 55 | 56 | _, err := io.Copy(&b.buf, b.rawBody) 57 | b.bufCached = true 58 | 59 | if err != nil && err != io.EOF { 60 | b.Err = err 61 | b.ClearCache() 62 | } 63 | 64 | b.rawBody.Close() 65 | } 66 | 67 | // String return the body in string type 68 | func (b *ZBody) String() string { 69 | if b.Err != nil { 70 | return "" 71 | } 72 | 73 | b.fillBuffer() 74 | 75 | return b.buf.String() 76 | } 77 | 78 | // Bytes return the body with []byte type 79 | func (b *ZBody) Bytes() []byte { 80 | if b.Err != nil { 81 | return nil 82 | } 83 | 84 | b.fillBuffer() 85 | 86 | if b.buf.Len() == 0 { 87 | return nil 88 | } 89 | 90 | return b.buf.Bytes() 91 | } 92 | 93 | // Close close the body. Must be called when the response is used 94 | func (b *ZBody) Close() error { 95 | if b.Err != nil { 96 | return b.Err 97 | } 98 | 99 | return b.rawBody.Close() 100 | } 101 | 102 | // ClearCache clear the cache of body 103 | func (b *ZBody) ClearCache() { 104 | if b.buf.Len() > 0 { 105 | b.buf.Reset() 106 | } 107 | } 108 | 109 | // Headers is a wrapper for http.Header 110 | type Headers map[string][]string 111 | 112 | // String return a header in wire format. 113 | func (h Headers) String() string { 114 | if h == nil { 115 | return "" 116 | } 117 | 118 | var buf strings.Builder 119 | http.Header(h).Write(&buf) 120 | return buf.String() 121 | } 122 | 123 | // Get gets the value associated with the given key. If 124 | // there are no values associated with the key, Get returns "". 125 | // multiple header fields with the same name will be join with ", ". 126 | // It is case insensitive; textproto.CanonicalMIMEHeaderKey is 127 | // used to canonicalize the provided key. To access multiple 128 | // values of a key, or to use non-canonical keys, access the 129 | // map directly. 130 | func (h Headers) Get(key string) string { 131 | v := http.Header(h).Values(key) 132 | if len(v) == 0 { 133 | return "" 134 | } 135 | 136 | return strings.Join(v, ", ") 137 | } 138 | 139 | // Has will return information about whether a response header 140 | // with the given name exists. If not exist, Has returns false. 141 | // It is case insensitive; 142 | func (h Headers) Has(key string) bool { 143 | if h == nil { 144 | return false 145 | } 146 | 147 | _, ok := h[http.CanonicalHeaderKey(key)] 148 | return ok 149 | } 150 | 151 | // Cookies is a wrapper for []*http.Cookie 152 | type Cookies []*http.Cookie 153 | 154 | // String return the cookies in string type. 155 | // like key1=value1; key2=value2 156 | func (c Cookies) String() string { 157 | if len(c) == 0 { 158 | return "" 159 | } 160 | 161 | var buf strings.Builder 162 | for i, cookie := range c { 163 | buf.WriteString(cookie.Name) 164 | buf.WriteRune('=') 165 | buf.WriteString(cookie.Value) 166 | if i < len(c)-1 { 167 | buf.WriteString("; ") 168 | } 169 | } 170 | 171 | return buf.String() 172 | } 173 | 174 | // Get gets the cookie value with the given name. If 175 | // there are no values associated with the name, Get returns "". 176 | func (c Cookies) Get(name string) string { 177 | for _, cookie := range c { 178 | if cookie.Name == name { 179 | return cookie.Value 180 | } 181 | } 182 | 183 | return "" 184 | } 185 | 186 | // Has will return whether the specified cookie is set in response. 187 | func (c Cookies) Has(name string) bool { 188 | for _, cookie := range c { 189 | if cookie.Name == name { 190 | return true 191 | } 192 | } 193 | 194 | return false 195 | } 196 | 197 | // Response is a wrapper for *http.Response 198 | type Response struct { 199 | StatusCode int 200 | Status string 201 | ContentLength int64 202 | Headers Headers 203 | Body *ZBody 204 | RawResponse *http.Response 205 | cookies Cookies 206 | } 207 | 208 | // Cookies parses and returns the cookies set in the Set-Cookie headers. 209 | func (resp *Response) Cookies() Cookies { 210 | if resp.cookies == nil { 211 | resp.cookies = resp.RawResponse.Cookies() 212 | } 213 | 214 | return resp.cookies 215 | } 216 | 217 | // OK validates that the server returned a 2xx code. 218 | func (resp *Response) OK() bool { 219 | return resp.StatusCode >= 200 && resp.StatusCode < 300 220 | } 221 | 222 | // Err returns the first non-EOF error that was encountered by read body. 223 | func (resp *Response) Err() error { 224 | return resp.Body.Err 225 | } 226 | 227 | // Close close the http response body. 228 | func (resp *Response) Close() error { 229 | resp.Body.ClearCache() 230 | return resp.Body.Close() 231 | } 232 | 233 | // DumpRequest format the last http.Request to string. 234 | // Notice, the order of headers is not strictly consistent 235 | func (resp *Response) DumpRequest() string { 236 | var buf strings.Builder 237 | req := resp.RawResponse.Request.Clone(context.Background()) 238 | 239 | data, _ := httputil.DumpRequestOut(req, false) 240 | buf.Write(data) 241 | 242 | if req.GetBody != nil { 243 | rc, err := req.GetBody() 244 | if err == nil { 245 | io.Copy(&buf, rc) 246 | rc.Close() 247 | } 248 | } 249 | 250 | return buf.String() 251 | } 252 | 253 | // DumpResponse format the last http.Response to string. 254 | // Notice, the order of headers is not strictly consistent 255 | func (resp *Response) DumpResponse() string { 256 | var buf strings.Builder 257 | 258 | data, _ := httputil.DumpResponse(resp.RawResponse, false) 259 | buf.Write(data) 260 | 261 | buf.Write(resp.Body.Bytes()) 262 | 263 | return buf.String() 264 | } 265 | -------------------------------------------------------------------------------- /session.go: -------------------------------------------------------------------------------- 1 | package zhttp 2 | 3 | import ( 4 | "net/http/cookiejar" 5 | ) 6 | 7 | // Session is a client used to send http requests. 8 | // Unlike Zhttp, it handle session for all requests 9 | type Session struct { 10 | z *Zhttp 11 | CookieJar *cookiejar.Jar 12 | } 13 | 14 | func (s *Session) Get(url string, options *ReqOptions) (*Response, error) { 15 | return s.z.doRequest("GET", url, options, s.CookieJar) 16 | } 17 | 18 | func (s *Session) Post(url string, options *ReqOptions) (*Response, error) { 19 | return s.z.doRequest("POST", url, options, s.CookieJar) 20 | } 21 | 22 | func (s *Session) Head(url string, options *ReqOptions) (*Response, error) { 23 | return s.z.doRequest("HEAD", url, options, s.CookieJar) 24 | } 25 | 26 | func (s *Session) Put(url string, options *ReqOptions) (*Response, error) { 27 | return s.z.doRequest("PUT", url, options, s.CookieJar) 28 | } 29 | 30 | func (s *Session) Delete(url string, options *ReqOptions) (*Response, error) { 31 | return s.z.doRequest("DELETE", url, options, s.CookieJar) 32 | } 33 | 34 | func (s *Session) Patch(url string, options *ReqOptions) (*Response, error) { 35 | return s.z.doRequest("PATCH", url, options, s.CookieJar) 36 | } 37 | 38 | func (s *Session) Options(url string, options *ReqOptions) (*Response, error) { 39 | return s.z.doRequest("OPTIONS", url, options, s.CookieJar) 40 | } 41 | 42 | func (s *Session) Request(method string, url string, options *ReqOptions) (*Response, error) { 43 | return s.z.doRequest(method, url, options, s.CookieJar) 44 | } 45 | -------------------------------------------------------------------------------- /tools/tools.go: -------------------------------------------------------------------------------- 1 | package tools 2 | 3 | import ( 4 | "io" 5 | "mime" 6 | "net/http" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | "github.com/greyh4t/zhttp" 12 | ) 13 | 14 | // DeepGetCookie check all responses in the redirect and return the first matching url and cookie 15 | func DeepGetCookie(resp *http.Response, name string) (string, string, bool) { 16 | if resp == nil { 17 | return "", "", false 18 | } 19 | 20 | req := resp.Request 21 | cookies := resp.Cookies() 22 | for _, cookie := range cookies { 23 | if cookie.Name == name { 24 | return req.URL.String(), cookie.Value, true 25 | } 26 | } 27 | 28 | return DeepGetCookie(req.Response, name) 29 | } 30 | 31 | // DeepGetHeader check all responses in the redirect and return the first matching url and header 32 | func DeepGetHeader(resp *http.Response, key string) (string, string, bool) { 33 | if resp == nil { 34 | return "", "", false 35 | } 36 | 37 | req := resp.Request 38 | values, ok := resp.Header[http.CanonicalHeaderKey(key)] 39 | if ok { 40 | return req.URL.String(), strings.Join(values, ", "), true 41 | } 42 | 43 | return DeepGetHeader(req.Response, key) 44 | } 45 | 46 | // CookiesFromRaw parse a cookie in string format to []*http.Cookie 47 | func CookiesFromRaw(rawCookie string, domain string) []*http.Cookie { 48 | list := strings.Split(rawCookie, ";") 49 | if len(list) == 0 { 50 | return nil 51 | } 52 | 53 | cookies := make([]*http.Cookie, len(list)) 54 | for i, item := range list { 55 | pairs := strings.SplitN(strings.TrimSpace(item), "=", 2) 56 | cookie := &http.Cookie{ 57 | Name: pairs[0], 58 | Domain: domain, 59 | } 60 | 61 | if len(pairs) == 2 { 62 | cookie.Value = pairs[1] 63 | } 64 | 65 | cookies[i] = cookie 66 | } 67 | 68 | return cookies 69 | } 70 | 71 | // FileFromDisk read file from disk and detect mime with filename 72 | func FileFromDisk(filePath string) (*zhttp.File, error) { 73 | filePath = filepath.Clean(filePath) 74 | fd, err := os.Open(filePath) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | file := &zhttp.File{ 80 | Name: fd.Name(), 81 | Contents: fd, 82 | } 83 | file.Mime = mime.TypeByExtension(filepath.Ext(file.Name)) 84 | 85 | return file, nil 86 | } 87 | 88 | // SaveToFile save reader's content to file 89 | func SaveToFile(r io.Reader, filename string) error { 90 | f, err := os.Create(filename) 91 | if err != nil { 92 | return err 93 | } 94 | 95 | _, err = io.Copy(f, r) 96 | f.Close() 97 | 98 | if err != nil { 99 | os.Remove(filename) 100 | return err 101 | } 102 | 103 | return nil 104 | } 105 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package zhttp 2 | 3 | import ( 4 | "net/url" 5 | "strings" 6 | ) 7 | 8 | var quoteEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`) 9 | 10 | func escapeQuotes(s string) string { 11 | return quoteEscaper.Replace(s) 12 | } 13 | 14 | // MustProxy convert scheme and url string to map[string]*url.URL. 15 | // If there have any error, will panic 16 | func MustProxy(proxies map[string]string) map[string]*url.URL { 17 | if len(proxies) > 0 { 18 | proxiesMap := map[string]*url.URL{} 19 | for scheme, proxyURL := range proxies { 20 | urlObj, err := url.Parse(proxyURL) 21 | if err != nil { 22 | panic(err) 23 | } 24 | proxiesMap[scheme] = urlObj 25 | } 26 | return proxiesMap 27 | } 28 | return nil 29 | } 30 | 31 | // CookieMapFromRaw parse a cookie in string format to map[string]string 32 | func CookieMapFromRaw(rawCookie string) map[string]string { 33 | list := strings.Split(rawCookie, ";") 34 | if len(list) == 0 { 35 | return nil 36 | } 37 | 38 | cookies := make(map[string]string, len(list)) 39 | for _, item := range list { 40 | pairs := strings.SplitN(strings.TrimSpace(item), "=", 2) 41 | 42 | if len(pairs) == 2 { 43 | cookies[pairs[0]] = pairs[1] 44 | } else { 45 | cookies[pairs[0]] = "" 46 | } 47 | } 48 | 49 | return cookies 50 | } 51 | -------------------------------------------------------------------------------- /zhttp.go: -------------------------------------------------------------------------------- 1 | package zhttp 2 | 3 | import ( 4 | "net/http" 5 | "net/http/cookiejar" 6 | "runtime" 7 | "time" 8 | 9 | "github.com/greyh4t/dnscache" 10 | "golang.org/x/net/publicsuffix" 11 | ) 12 | 13 | type Zhttp struct { 14 | options *HTTPOptions 15 | dnsCache *dnscache.Cache 16 | transport *http.Transport 17 | } 18 | 19 | // New generate an *Zhttp client to send request 20 | func New(options *HTTPOptions) *Zhttp { 21 | z := &Zhttp{options: options} 22 | if z.options == nil { 23 | z.options = &HTTPOptions{} 24 | } 25 | 26 | var cache *dnscache.Cache 27 | if z.options.DNSCacheExpire > 0 { 28 | if z.options.DNSServer != "" { 29 | cache = dnscache.NewWithServer(z.options.DNSCacheExpire, z.options.DNSServer) 30 | } else { 31 | cache = dnscache.New(z.options.DNSCacheExpire) 32 | } 33 | z.dnsCache = cache 34 | } 35 | 36 | z.transport = createTransport(z.options, cache) 37 | 38 | ensureResourcesFinalized(z, z.dnsCache != nil) 39 | 40 | return z 41 | } 42 | 43 | // NewWithDNSCache generate an *Zhttp client that uses an external DNSCache. 44 | // This will ignore HTTPOptions.DNSCacheExpire and HTTPOptions.DNSServer 45 | func NewWithDNSCache(options *HTTPOptions, cache *dnscache.Cache) *Zhttp { 46 | z := &Zhttp{options: options} 47 | if z.options == nil { 48 | z.options = &HTTPOptions{} 49 | } 50 | 51 | if cache != nil { 52 | z.dnsCache = cache 53 | } 54 | 55 | z.transport = createTransport(z.options, cache) 56 | 57 | ensureResourcesFinalized(z, false) 58 | 59 | return z 60 | } 61 | 62 | func ensureResourcesFinalized(zhttp *Zhttp, finalizeDNSCache bool) { 63 | runtime.SetFinalizer(zhttp, func(z *Zhttp) { 64 | z.transport.CloseIdleConnections() 65 | if finalizeDNSCache { 66 | z.dnsCache.Close() 67 | } 68 | }) 69 | } 70 | 71 | // NewSession generate an client that will handle session for all requests 72 | func (z *Zhttp) NewSession() *Session { 73 | s := &Session{z: z} 74 | s.CookieJar, _ = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) 75 | return s 76 | } 77 | 78 | func (z *Zhttp) Request(method, url string, options *ReqOptions) (*Response, error) { 79 | return z.doRequest(method, url, options, nil) 80 | } 81 | 82 | func (z *Zhttp) Get(url string, options *ReqOptions) (*Response, error) { 83 | return z.doRequest("GET", url, options, nil) 84 | } 85 | 86 | func (z *Zhttp) Delete(url string, options *ReqOptions) (*Response, error) { 87 | return z.doRequest("DELETE", url, options, nil) 88 | } 89 | 90 | func (z *Zhttp) Head(url string, options *ReqOptions) (*Response, error) { 91 | return z.doRequest("HEAD", url, options, nil) 92 | } 93 | 94 | func (z *Zhttp) Patch(url string, options *ReqOptions) (*Response, error) { 95 | return z.doRequest("PATCH", url, options, nil) 96 | } 97 | 98 | func (z *Zhttp) Post(url string, options *ReqOptions) (*Response, error) { 99 | return z.doRequest("POST", url, options, nil) 100 | } 101 | 102 | func (z *Zhttp) Put(url string, options *ReqOptions) (*Response, error) { 103 | return z.doRequest("PUT", url, options, nil) 104 | } 105 | 106 | func (z *Zhttp) Options(url string, options *ReqOptions) (*Response, error) { 107 | return z.doRequest("OPTIONS", url, options, nil) 108 | } 109 | 110 | var defaultZ *Zhttp 111 | 112 | // InitDefaultClient initialization the default zhttp client with options 113 | func InitDefaultClient(options *HTTPOptions) { 114 | defaultZ = New(options) 115 | } 116 | 117 | func prepareDefaultZ() { 118 | if defaultZ == nil { 119 | defaultZ = New(&HTTPOptions{ 120 | Timeout: time.Second * 30, 121 | }) 122 | } 123 | } 124 | 125 | // NewSession generate an default client that will handle session for all requests 126 | func NewSession() *Session { 127 | prepareDefaultZ() 128 | return defaultZ.NewSession() 129 | } 130 | 131 | func Request(method, url string, options *ReqOptions) (*Response, error) { 132 | prepareDefaultZ() 133 | return defaultZ.doRequest(method, url, options, nil) 134 | } 135 | 136 | func Get(url string, options *ReqOptions) (*Response, error) { 137 | prepareDefaultZ() 138 | return defaultZ.doRequest("GET", url, options, nil) 139 | } 140 | 141 | func Delete(url string, options *ReqOptions) (*Response, error) { 142 | prepareDefaultZ() 143 | return defaultZ.doRequest("DELETE", url, options, nil) 144 | } 145 | 146 | func Head(url string, options *ReqOptions) (*Response, error) { 147 | prepareDefaultZ() 148 | return defaultZ.doRequest("HEAD", url, options, nil) 149 | } 150 | 151 | func Patch(url string, options *ReqOptions) (*Response, error) { 152 | prepareDefaultZ() 153 | return defaultZ.doRequest("PATCH", url, options, nil) 154 | } 155 | 156 | func Post(url string, options *ReqOptions) (*Response, error) { 157 | prepareDefaultZ() 158 | return defaultZ.doRequest("POST", url, options, nil) 159 | } 160 | 161 | func Put(url string, options *ReqOptions) (*Response, error) { 162 | prepareDefaultZ() 163 | return defaultZ.doRequest("PUT", url, options, nil) 164 | } 165 | 166 | func Options(url string, options *ReqOptions) (*Response, error) { 167 | prepareDefaultZ() 168 | return defaultZ.doRequest("OPTIONS", url, options, nil) 169 | } 170 | --------------------------------------------------------------------------------