├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── examples └── httpbin.go ├── full_test.go ├── go.mod ├── request.go └── response.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.8 5 | - 1.9 6 | - tip 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Tom Hudson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rawhttp 2 | 3 | [![Build Status](https://img.shields.io/travis/tomnomnom/rawhttp/master.svg?style=flat)](https://travis-ci.org/tomnomnom/rawhttp) 4 | [![Documentation](https://img.shields.io/badge/godoc-reference-brightgreen.svg?style=flat)](https://godoc.org/github.com/tomnomnom/rawhttp) 5 | 6 | rawhttp is a [Go](https://golang.org/) package for making HTTP requests. 7 | It intends to fill a niche that [https://golang.org/pkg/net/http/](net/http) does not cover: 8 | having *complete* control over the requests being sent to the server. 9 | 10 | rawhttp purposefully does as little validation as possible, and you can override just about 11 | anything about the request; even the line endings. 12 | 13 | **Warning:** This is a work in progress. The API isn't fixed yet. 14 | 15 | Full documentation can be found on [GoDoc](https://godoc.org/github.com/tomnomnom/rawhttp). 16 | 17 | ## Example 18 | 19 | ```go 20 | req, err := rawhttp.FromURL("POST", "https://httpbin.org") 21 | if err != nil { 22 | log.Fatal(err) 23 | } 24 | 25 | // automatically set the host header 26 | req.AutoSetHost() 27 | 28 | req.Method = "PUT" 29 | req.Hostname = "httpbin.org" 30 | req.Port = "443" 31 | req.Path = "/anything" 32 | req.Query = "one=1&two=2" 33 | req.Fragment = "anchor" 34 | req.Proto = "HTTP/1.1" 35 | req.EOL = "\r\n" 36 | 37 | req.AddHeader("Content-Type: application/x-www-form-urlencoded") 38 | 39 | req.Body = "username=AzureDiamond&password=hunter2" 40 | 41 | // automatically set the Content-Length header 42 | req.AutoSetContentLength() 43 | 44 | fmt.Printf("%s\n\n", req.String()) 45 | 46 | resp, err := rawhttp.Do(req) 47 | if err != nil { 48 | log.Fatal(err) 49 | } 50 | 51 | fmt.Printf("< %s\n", resp.StatusLine()) 52 | for _, h := range resp.Headers() { 53 | fmt.Printf("< %s\n", h) 54 | } 55 | 56 | fmt.Printf("\n%s\n", resp.Body()) 57 | ``` 58 | 59 | ``` 60 | PUT /anything?one=1&two=2#anchor HTTP/1.1 61 | Host: httpbin.org 62 | Content-Type: application/x-www-form-urlencoded 63 | Content-Length: 38 64 | 65 | username=AzureDiamond&password=hunter2 66 | 67 | < HTTP/1.1 200 OK 68 | < Connection: keep-alive 69 | < Server: meinheld/0.6.1 70 | < Date: Sat, 02 Sep 2017 13:22:06 GMT 71 | < Content-Type: application/json 72 | < Access-Control-Allow-Origin: * 73 | < Access-Control-Allow-Credentials: true 74 | < X-Powered-By: Flask 75 | < X-Processed-Time: 0.000869989395142 76 | < Content-Length: 443 77 | < Via: 1.1 vegur 78 | 79 | { 80 | "args": { 81 | "one": "1", 82 | "two": "2" 83 | }, 84 | "data": "", 85 | "files": {}, 86 | "form": { 87 | "password": "hunter2", 88 | "username": "AzureDiamond" 89 | }, 90 | "headers": { 91 | "Connection": "close", 92 | "Content-Length": "38", 93 | "Content-Type": "application/x-www-form-urlencoded", 94 | "Host": "httpbin.org" 95 | }, 96 | "json": null, 97 | "method": "PUT", 98 | "origin": "123.123.123.123", 99 | "url": "https://httpbin.org/anything?one=1&two=2" 100 | } 101 | ``` 102 | 103 | -------------------------------------------------------------------------------- /examples/httpbin.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | 7 | "github.com/tomnomnom/rawhttp" 8 | ) 9 | 10 | func main() { 11 | req, err := rawhttp.FromURL("POST", "https://httpbin.org") 12 | if err != nil { 13 | log.Fatal(err) 14 | } 15 | 16 | // automatically set the host header 17 | req.AutoSetHost() 18 | 19 | req.Method = "PUT" 20 | req.Hostname = "httpbin.org" 21 | req.Port = "443" 22 | req.Path = "/anything" 23 | req.Query = "one=1&two=2" 24 | req.Fragment = "anchor" 25 | req.Proto = "HTTP/1.1" 26 | req.EOL = "\r\n" 27 | 28 | req.AddHeader("Content-Type: application/x-www-form-urlencoded") 29 | 30 | req.Body = "username=AzureDiamond&password=hunter2" 31 | 32 | // automatically set the Content-Length header 33 | req.AutoSetContentLength() 34 | 35 | fmt.Printf("%s\n\n", req.String()) 36 | 37 | resp, err := rawhttp.Do(req) 38 | if err != nil { 39 | log.Fatal(err) 40 | } 41 | 42 | fmt.Printf("< %s\n", resp.StatusLine()) 43 | for _, h := range resp.Headers() { 44 | fmt.Printf("< %s\n", h) 45 | } 46 | 47 | fmt.Printf("\n%s\n", resp.Body()) 48 | } 49 | -------------------------------------------------------------------------------- /full_test.go: -------------------------------------------------------------------------------- 1 | package rawhttp 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/httptest" 7 | "net/url" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func TestRaw(t *testing.T) { 13 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 14 | w.Header().Set("Response", "check") 15 | fmt.Fprintln(w, "the response") 16 | })) 17 | defer ts.Close() 18 | 19 | u, err := url.Parse(ts.URL) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | 24 | req := RawRequest{ 25 | Hostname: u.Hostname(), 26 | Port: u.Port(), 27 | Request: "GET /anything HTTP/1.1\r\n" + "Host: localhost\r\n", 28 | } 29 | 30 | resp, err := Do(req) 31 | if err != nil { 32 | t.Errorf("want nil error, have %s", err) 33 | } 34 | 35 | have := strings.TrimSpace(string(resp.Body())) 36 | if have != "the response" { 37 | t.Errorf("want body to be 'the response'; have '%s'", have) 38 | } 39 | 40 | if resp.Header("Response") != "check" { 41 | t.Errorf("want response header to be 'check' have '%s'", resp.Header("Response")) 42 | } 43 | 44 | if resp.StatusCode() != "200" { 45 | t.Errorf("want 200 response; have %s", resp.StatusCode()) 46 | } 47 | } 48 | 49 | func TestFromURL(t *testing.T) { 50 | ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 51 | w.Header().Set("Response", "check") 52 | fmt.Fprintln(w, "the response") 53 | })) 54 | defer ts.Close() 55 | 56 | req, err := FromURL("POST", ts.URL) 57 | if err != nil { 58 | t.Fatalf("want nil error, have %s", err) 59 | } 60 | req.AutoSetHost() 61 | req.Body = "This is some POST data" 62 | 63 | resp, err := Do(req) 64 | 65 | if err != nil { 66 | t.Fatalf("want nil error, have %s", err) 67 | } 68 | 69 | have := strings.TrimSpace(string(resp.Body())) 70 | if have != "the response" { 71 | t.Errorf("want body to be 'the response'; have '%s'", have) 72 | } 73 | 74 | if resp.Header("Response") != "check" { 75 | t.Errorf("want response header to be 'check' have '%s'", resp.Header("Response")) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tomnomnom/rawhttp 2 | 3 | go 1.18 4 | -------------------------------------------------------------------------------- /request.go: -------------------------------------------------------------------------------- 1 | package rawhttp 2 | 3 | import ( 4 | "bytes" 5 | "crypto/tls" 6 | "crypto/x509" 7 | "fmt" 8 | "io" 9 | "net" 10 | "net/url" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | // A Requester defines the bare minimum set of methods needed to make an HTTP request. 16 | type Requester interface { 17 | // IsTLS should return true if the connection should be made using TLS 18 | IsTLS() bool 19 | 20 | // Host should return a hostname:port pair 21 | Host() string 22 | 23 | // String should return the request as a string E.g: 24 | // GET / HTTP/1.1\r\nHost:... 25 | String() string 26 | 27 | // GetTimeout returns the timeout for a request 28 | GetTimeout() time.Duration 29 | } 30 | 31 | // Request is the main implementation of Requester. It gives you 32 | // fine-grained control over just about everything to do with the 33 | // request, but with the posibility of sane defaults. 34 | type Request struct { 35 | // TLS should be true if TLS should be used 36 | TLS bool 37 | 38 | // Method is the HTTP verb. E.g. GET 39 | Method string 40 | 41 | // Scheme is the protocol scheme. E.g. https 42 | Scheme string 43 | 44 | // Hostname is the hostname to connect to. E.g. localhost 45 | Hostname string 46 | 47 | // Port is the port to connect to. E.g. 80 48 | Port string 49 | 50 | // Path is the path to request. E.g. /security.txt 51 | Path string 52 | 53 | // Query is the query string of the path. E.g. q=searchterm&page=3 54 | Query string 55 | 56 | // Fragment is the bit after the '#'. E.g. pagesection 57 | Fragment string 58 | 59 | // Proto is the protocol specifier in the first line of the request. 60 | // E.g. HTTP/1.1 61 | Proto string 62 | 63 | // Headers is a slice of headers to send. E.g: 64 | // []string{"Host: localhost", "Accept: text/plain"} 65 | Headers []string 66 | 67 | // Body is the 'POST' data to send. E.g: 68 | // username=AzureDiamond&password=hunter2 69 | Body string 70 | 71 | // EOL is the string that should be used for line endings. E.g. \r\n 72 | EOL string 73 | 74 | // Deadline 75 | Timeout time.Duration 76 | } 77 | 78 | // FromURL returns a *Request for a given method and URL and any 79 | // error that occured during parsing the URL. Sane defaults are 80 | // set for all of *Request's fields. 81 | func FromURL(method, rawurl string) (*Request, error) { 82 | r := &Request{} 83 | 84 | u, err := url.Parse(rawurl) 85 | if err != nil { 86 | return r, err 87 | } 88 | 89 | // url.Parse() tends to mess with the path, so we need to 90 | // try and fix that. 91 | schemeEtc := strings.SplitN(rawurl, "//", 2) 92 | if len(schemeEtc) != 2 { 93 | return nil, fmt.Errorf("invalid url: %s", rawurl) 94 | } 95 | 96 | pathEtc := strings.SplitN(schemeEtc[1], "/", 2) 97 | path := "/" 98 | if len(pathEtc) == 2 { 99 | // Remove any query string or fragment 100 | path = "/" + pathEtc[1] 101 | noQuery := strings.Split(path, "?") 102 | noFragment := strings.Split(noQuery[0], "#") 103 | path = noFragment[0] 104 | } 105 | 106 | r.TLS = u.Scheme == "https" 107 | r.Method = method 108 | r.Scheme = u.Scheme 109 | r.Hostname = u.Hostname() 110 | r.Port = u.Port() 111 | r.Path = path 112 | r.Query = u.RawQuery 113 | r.Fragment = u.Fragment 114 | r.Proto = "HTTP/1.1" 115 | r.EOL = "\r\n" 116 | r.Timeout = time.Second * 30 117 | 118 | if r.Path == "" { 119 | r.Path = "/" 120 | } 121 | 122 | if r.Port == "" { 123 | if r.TLS { 124 | r.Port = "443" 125 | } else { 126 | r.Port = "80" 127 | } 128 | } 129 | 130 | return r, nil 131 | 132 | } 133 | 134 | // IsTLS returns true if TLS should be used 135 | func (r Request) IsTLS() bool { 136 | return r.TLS 137 | } 138 | 139 | // Host returns the hostname:port pair to connect to 140 | func (r Request) Host() string { 141 | return r.Hostname + ":" + r.Port 142 | } 143 | 144 | // AddHeader adds a header to the *Request 145 | func (r *Request) AddHeader(h string) { 146 | r.Headers = append(r.Headers, h) 147 | } 148 | 149 | // Header finds and returns the value of a header on the request. 150 | // An empty string is returned if no match is found. 151 | func (r Request) Header(search string) string { 152 | search = strings.ToLower(search) 153 | 154 | for _, header := range r.Headers { 155 | 156 | p := strings.SplitN(header, ":", 2) 157 | if len(p) != 2 { 158 | continue 159 | } 160 | 161 | if strings.ToLower(p[0]) == search { 162 | return strings.TrimSpace(p[1]) 163 | } 164 | } 165 | return "" 166 | } 167 | 168 | // AutoSetHost adds a Host header to the request 169 | // using the value of Request.Hostname 170 | func (r *Request) AutoSetHost() { 171 | r.AddHeader(fmt.Sprintf("Host: %s", r.Hostname)) 172 | } 173 | 174 | // AutoSetContentLength adds a Content-Length header 175 | // to the request with the length of Request.Body as the value 176 | func (r *Request) AutoSetContentLength() { 177 | r.AddHeader(fmt.Sprintf("Content-Length: %d", len(r.Body))) 178 | } 179 | 180 | // fullPath returns the path including query string and fragment 181 | func (r Request) fullPath() string { 182 | q := "" 183 | if r.Query != "" { 184 | q = "?" + r.Query 185 | } 186 | 187 | f := "" 188 | if r.Fragment != "" { 189 | f = "#" + r.Fragment 190 | } 191 | return r.Path + q + f 192 | } 193 | 194 | // URL forms and returns a complete URL for the request 195 | func (r Request) URL() string { 196 | return fmt.Sprintf( 197 | "%s://%s%s", 198 | r.Scheme, 199 | r.Host(), 200 | r.fullPath(), 201 | ) 202 | } 203 | 204 | // RequestLine returns the request line. E.g. GET / HTTP/1.1 205 | func (r Request) RequestLine() string { 206 | return fmt.Sprintf("%s %s %s", r.Method, r.fullPath(), r.Proto) 207 | } 208 | 209 | // String returns a plain-text version of the request to be sent to the server 210 | func (r Request) String() string { 211 | var b bytes.Buffer 212 | 213 | b.WriteString(fmt.Sprintf("%s%s", r.RequestLine(), r.EOL)) 214 | 215 | for _, h := range r.Headers { 216 | b.WriteString(fmt.Sprintf("%s%s", h, r.EOL)) 217 | } 218 | 219 | b.WriteString(r.EOL) 220 | 221 | b.WriteString(r.Body) 222 | 223 | return b.String() 224 | } 225 | 226 | // GetTimeout returns the timeout for a request 227 | func (r Request) GetTimeout() time.Duration { 228 | // default 30 seconds 229 | if r.Timeout == 0 { 230 | return time.Second * 30 231 | } 232 | return r.Timeout 233 | } 234 | 235 | // RawRequest is the most basic implementation of Requester. You should 236 | // probably only use it if you're doing something *really* weird 237 | type RawRequest struct { 238 | // TLS should be true if TLS should be used 239 | TLS bool 240 | 241 | // Hostname is the name of the host to connect to. E.g: localhost 242 | Hostname string 243 | 244 | // Port is the port to connect to. E.g.: 80 245 | Port string 246 | 247 | // Request is the actual message to send to the server. E.g: 248 | // GET / HTTP/1.1\r\nHost:... 249 | Request string 250 | 251 | // Timeout for the request 252 | Timeout time.Duration 253 | } 254 | 255 | // IsTLS returns true if the connection should use TLS 256 | func (r RawRequest) IsTLS() bool { 257 | return r.TLS 258 | } 259 | 260 | // Host returns the hostname:port pair 261 | func (r RawRequest) Host() string { 262 | return r.Hostname + ":" + r.Port 263 | } 264 | 265 | // String returns the message to send to the server 266 | func (r RawRequest) String() string { 267 | return r.Request 268 | } 269 | 270 | // GetTimeout returns the timeout for the request 271 | func (r RawRequest) GetTimeout() time.Duration { 272 | // default 30 seconds 273 | if r.Timeout == 0 { 274 | return time.Second * 30 275 | } 276 | return r.Timeout 277 | } 278 | 279 | // Do performs the HTTP request for the given Requester and returns 280 | // a *Response and any error that occured 281 | func Do(req Requester) (*Response, error) { 282 | var conn io.ReadWriter 283 | var connerr error 284 | 285 | // This needs timeouts because it's fairly likely 286 | // that something will go wrong :) 287 | if req.IsTLS() { 288 | roots, err := x509.SystemCertPool() 289 | if err != nil { 290 | return nil, err 291 | } 292 | 293 | // This library is meant for doing stupid stuff, so skipping cert 294 | // verification is actually the right thing to do 295 | conf := &tls.Config{RootCAs: roots, InsecureSkipVerify: true} 296 | conn, connerr = tls.DialWithDialer(&net.Dialer{ 297 | Timeout: req.GetTimeout(), 298 | }, "tcp", req.Host(), conf) 299 | 300 | } else { 301 | d := net.Dialer{Timeout: req.GetTimeout()} 302 | conn, connerr = d.Dial("tcp", req.Host()) 303 | } 304 | 305 | if connerr != nil { 306 | return nil, connerr 307 | } 308 | 309 | fmt.Fprint(conn, req.String()) 310 | fmt.Fprint(conn, "\r\n") 311 | 312 | return newResponse(conn) 313 | } 314 | -------------------------------------------------------------------------------- /response.go: -------------------------------------------------------------------------------- 1 | package rawhttp 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "io/ioutil" 7 | "strconv" 8 | "strings" 9 | ) 10 | 11 | // A Response wraps the HTTP response from the server 12 | type Response struct { 13 | rawStatus string 14 | headers []string 15 | body []byte 16 | } 17 | 18 | // Header finds and returns the value of a header on the response. 19 | // An empty string is returned if no match is found. 20 | func (r Response) Header(search string) string { 21 | search = strings.ToLower(search) 22 | 23 | for _, header := range r.headers { 24 | 25 | p := strings.SplitN(header, ":", 2) 26 | if len(p) != 2 { 27 | continue 28 | } 29 | 30 | if strings.ToLower(p[0]) == search { 31 | return strings.TrimSpace(p[1]) 32 | } 33 | } 34 | return "" 35 | } 36 | 37 | // ParseLocation parses the Location header of a response, 38 | // using the initial request for context on relative URLs 39 | func (r Response) ParseLocation(req *Request) string { 40 | loc := r.Header("Location") 41 | 42 | if loc == "" { 43 | return "" 44 | } 45 | 46 | // Relative locations need the context of the request 47 | if len(loc) > 2 && loc[:2] == "//" { 48 | return req.Scheme + ":" + loc 49 | } 50 | 51 | if len(loc) > 0 && loc[0] == '/' { 52 | return req.Scheme + "://" + req.Hostname + loc 53 | } 54 | 55 | return loc 56 | } 57 | 58 | // StatusLine returns the HTTP status line from the response 59 | func (r Response) StatusLine() string { 60 | return r.rawStatus 61 | } 62 | 63 | // StatusCode returns the HTTP status code as a string; e.g. 200 64 | func (r Response) StatusCode() string { 65 | parts := strings.SplitN(r.rawStatus, " ", 3) 66 | if len(parts) != 3 { 67 | return "" 68 | } 69 | 70 | return parts[1] 71 | } 72 | 73 | // Headers returns the response headers 74 | func (r Response) Headers() []string { 75 | return r.headers 76 | } 77 | 78 | // Body returns the response body 79 | func (r Response) Body() []byte { 80 | return r.body 81 | } 82 | 83 | // addHeader adds a header to the *Response 84 | func (r *Response) addHeader(header string) { 85 | r.headers = append(r.headers, header) 86 | } 87 | 88 | // newResponse accepts an io.Reader, reads the response 89 | // headers and body and returns a new *Response and any 90 | // error that occured. 91 | func newResponse(conn io.Reader) (*Response, error) { 92 | 93 | r := bufio.NewReader(conn) 94 | resp := &Response{} 95 | 96 | s, err := r.ReadString('\n') 97 | if err != nil { 98 | return nil, err 99 | } 100 | resp.rawStatus = strings.TrimSpace(s) 101 | 102 | for { 103 | line, err := r.ReadString('\n') 104 | line = strings.TrimSpace(line) 105 | 106 | if err != nil || line == "" { 107 | break 108 | } 109 | 110 | resp.addHeader(line) 111 | } 112 | 113 | if cl := resp.Header("Content-Length"); cl != "" { 114 | length, err := strconv.Atoi(cl) 115 | 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | if length > 0 { 121 | b := make([]byte, length) 122 | _, err = io.ReadAtLeast(r, b, length) 123 | if err != nil { 124 | return nil, err 125 | } 126 | resp.body = b 127 | } 128 | 129 | } else { 130 | b, err := ioutil.ReadAll(r) 131 | if err != nil { 132 | return nil, err 133 | } 134 | 135 | resp.body = b 136 | } 137 | 138 | return resp, nil 139 | } 140 | --------------------------------------------------------------------------------