├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── README.md ├── cmd └── httpcache │ └── main.go ├── internal ├── cache │ └── cache.go ├── handler │ ├── ping.go │ ├── proxycache.go │ └── stats.go ├── middleware │ ├── panic.go │ └── panic_test.go ├── roundtripper │ ├── cacher.go │ ├── cacher_test.go │ ├── logger.go │ └── reponse_body_limit.go ├── size │ └── size.go └── xhttp │ └── server.go └── tests └── api_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | .idea -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: go 3 | go: 4 | - 1.10.x 5 | - 1.11.x 6 | - tip 7 | 8 | before_install: 9 | - go get github.com/mattn/goveralls 10 | 11 | script: 12 | - go test -race -v ./... 13 | - $HOME/gopath/bin/goveralls -service=travis-ci 14 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG GO_VERSION=1.11 2 | 3 | FROM golang:${GO_VERSION}-alpine AS builder 4 | 5 | ENV PACKAGE=github.com/donutloop/httpcache 6 | 7 | RUN apk add --update --no-cache ca-certificates make git curl mercurial 8 | 9 | RUN mkdir -p /go/src/${PACKAGE} 10 | WORKDIR /go/src/${PACKAGE} 11 | COPY . /go/src/${PACKAGE} 12 | 13 | RUN CGO_ENABLED=0 go build ${PACKAGE}/cmd/httpcache 14 | 15 | FROM alpine:3.7 16 | 17 | ENV PACKAGE=github.com/donutloop/httpcache 18 | 19 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 20 | COPY --from=builder /go/src/${PACKAGE}/httpcache /httpcache 21 | 22 | USER nobody:nobody 23 | 24 | EXPOSE 8000 25 | ENTRYPOINT ["./httpcache"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Marcel Edmund Franke 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # httpcache 2 | 3 | [![Build Status](https://travis-ci.org/donutloop/httpcache.svg?branch=master)](https://travis-ci.org/donutloop/httpcache) 4 | [![Coverage Status](https://coveralls.io/repos/github/donutloop/httpcache/badge.svg)](https://coveralls.io/github/donutloop/httpcache) 5 | 6 | An HTTP server that proxies all requests to other HTTP servers and this servers caches all incoming responses objects 7 | 8 | ## Backend Requirements 9 | 10 | * [golang](https://golang.org/) - The Go Programming Language 11 | * [docker](https://www.docker.com/) - Build, Manage and Secure Your Apps Anywhere. Your Way. 12 | 13 | ## Prepare GO development environment 14 | 15 | Follow [install guide](https://golang.org/doc/install) to install golang. 16 | 17 | ## Build without docker 18 | 19 | ```bash 20 | mkdir -p $GOPATH/src/github.com/donutloop/ && cd $GOPATH/src/github.com/donutloop/ 21 | 22 | git clone git@github.com:donutloop/httpcache.git 23 | 24 | cd httpcache 25 | 26 | go build ./cmd/httpcache 27 | ``` 28 | 29 | ## Build with docker 30 | 31 | ```bash 32 | mkdir -p $GOPATH/src/github.com/donutloop/ && cd $GOPATH/src/github.com/donutloop/ 33 | 34 | git clone git@github.com:donutloop/httpcache.git 35 | 36 | docker build . 37 | ``` 38 | 39 | ## Usage 40 | 41 | ```bash 42 | USAGE 43 | httpcache [flags] 44 | 45 | FLAGS 46 | -cap 100 capacity of cache 47 | -cert server.crt TLS certificate 48 | -expire 5 the items in the cache expire after or expire never 49 | -http :80 serve HTTP on this address (optional) 50 | -key server.key TLS key 51 | -rbcl 524288000 response size limit 52 | -tls serve TLS on this address (optional) 53 | ``` 54 | 55 | ## Usage of cache from outside (GO Example) 56 | 57 | ```golang 58 | ... 59 | transport := &http.Transport{ 60 | Proxy: SetProxyURL(proxyServer.URL), // Set url of http cache 61 | DialContext: (&net.Dialer{ 62 | Timeout: 30 * time.Second, 63 | KeepAlive: 30 * time.Second, 64 | DualStack: true, 65 | }).DialContext, 66 | MaxIdleConns: 100, 67 | IdleConnTimeout: 90 * time.Second, 68 | TLSHandshakeTimeout: 10 * time.Second, 69 | ExpectContinueTimeout: 1 * time.Second, 70 | } 71 | 72 | client = &http.Client{ 73 | Transport: transport, 74 | } 75 | 76 | client.Do(req) 77 | ... 78 | ``` 79 | 80 | ## Run container 81 | It's expose port 8000 and run a spefici container by id 82 | ```bash 83 | sudo docker run -p 8000:8000 {{container_id}} 84 | ``` 85 | -------------------------------------------------------------------------------- /cmd/httpcache/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/donutloop/httpcache/internal/cache" 7 | "github.com/donutloop/httpcache/internal/handler" 8 | "github.com/donutloop/httpcache/internal/middleware" 9 | "github.com/donutloop/httpcache/internal/size" 10 | "github.com/donutloop/httpcache/internal/xhttp" 11 | "log" 12 | "net" 13 | "net/http" 14 | "os" 15 | "text/tabwriter" 16 | "time" 17 | ) 18 | 19 | func main() { 20 | log.SetFlags(log.Ldate | log.Lshortfile | log.Ltime) 21 | 22 | fs := flag.NewFlagSet("http-proxy", flag.ExitOnError) 23 | var ( 24 | httpAddr = fs.String("http", ":8000", "serve HTTP on this address (optional)") 25 | tlsAddr = fs.String("tls", "", "serve TLS on this address (optional)") 26 | cert = fs.String("cert", "server.crt", "TLS certificate") 27 | key = fs.String("key", "server.key", "TLS key") 28 | cap = fs.Int64("cap", 100, "capacity of cache") 29 | responseBodyContentLenghtLimit = fs.Int64("rbcl", 500*size.MB, "response size limit") 30 | expire = fs.Int64("expire", 5, "the items in the cache expire after or expire never") 31 | ) 32 | fs.Usage = usageFor(fs, "httpcache [flags]") 33 | fs.Parse(os.Args[1:]) 34 | 35 | logger := log.New(os.Stderr, "", log.LstdFlags) 36 | 37 | logger.Print( 38 | "\n", 39 | fmt.Sprintf("http addr: %v \n", *httpAddr), 40 | fmt.Sprintf("tls addr: %v \n", *tlsAddr), 41 | fmt.Sprintf("cap: %v \n", *cap), 42 | fmt.Sprintf("responseBodyContentLenghtLimit: %v \n", *responseBodyContentLenghtLimit), 43 | fmt.Sprintf("expire: %v \n", *expire), 44 | ) 45 | 46 | e := time.Duration(*expire) * (time.Hour * 24) 47 | c := cache.NewLRUCache(*cap, e) 48 | { 49 | c.OnEviction = func(key string) { 50 | logger.Println(fmt.Sprintf("cache item is older then %v dayes (key: %s)", e, key)) 51 | c.Delete(key) 52 | } 53 | } 54 | 55 | stats := handler.NewStats(c, logger.Println) 56 | ping := handler.NewPing(logger.Println) 57 | proxy := handler.NewProxy( 58 | c, 59 | logger.Println, 60 | *responseBodyContentLenghtLimit, 61 | ping, 62 | stats, 63 | ) 64 | 65 | stack := middleware.NewPanic(proxy, logger.Println) 66 | 67 | if *httpAddr != "" { 68 | listener, err := net.Listen("tcp", *httpAddr) 69 | if err != nil { 70 | log.Fatal(err) 71 | } 72 | 73 | xserver := xhttp.Server{ 74 | Server: &http.Server{Addr: *httpAddr, Handler: stack}, 75 | Logger: logger, 76 | Listener: listener, 77 | ShutdownTimeout: 3 * time.Second, 78 | } 79 | if err := xserver.Start(); err != nil { 80 | xserver.Stop() 81 | } 82 | } else { 83 | logger.Printf("not serving HTTP") 84 | } 85 | 86 | if *tlsAddr != "" { 87 | 88 | listener, err := net.Listen("tcp", *tlsAddr) 89 | if err != nil { 90 | logger.Fatal(err) 91 | } 92 | 93 | xserver := xhttp.Server{ 94 | Server: &http.Server{Addr: *tlsAddr, Handler: stack}, 95 | Logger: logger, 96 | Listener: listener, 97 | ShutdownTimeout: 3 * time.Second, 98 | } 99 | if err := xserver.StartTLS(*cert, *key); err != nil { 100 | xserver.Stop() 101 | } 102 | } else { 103 | logger.Printf("not serving TLS") 104 | } 105 | } 106 | 107 | func usageFor(fs *flag.FlagSet, short string) func() { 108 | return func() { 109 | fmt.Fprintf(os.Stdout, "USAGE\n") 110 | fmt.Fprintf(os.Stdout, " %s\n", short) 111 | fmt.Fprintf(os.Stdout, "\n") 112 | fmt.Fprintf(os.Stdout, "FLAGS\n") 113 | tw := tabwriter.NewWriter(os.Stdout, 0, 2, 2, ' ', 0) 114 | fs.VisitAll(func(f *flag.Flag) { 115 | def := f.DefValue 116 | if def == "" { 117 | def = "..." 118 | } 119 | fmt.Fprintf(tw, " -%s %s\t%s\n", f.Name, f.DefValue, f.Usage) 120 | }) 121 | tw.Flush() 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /internal/cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "container/list" 5 | "encoding/binary" 6 | "net/http" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | // LRUCache is a typical LRU cache implementation. If the cache 12 | // reaches the capacity, the least recently used item is deleted from 13 | // the cache. Note the capacity is not the number of items, but the 14 | // total sum of the Size() of each item. 15 | type LRUCache struct { 16 | mu sync.Mutex 17 | 18 | // list & table of *entry objects 19 | list *list.List 20 | table map[string]*list.Element 21 | 22 | // Our current size. Obviously a gross simplification and 23 | // low-grade approximation. 24 | size int64 25 | 26 | // How much we are limiting the cache to. 27 | capacity int64 28 | 29 | expiry time.Duration 30 | 31 | // Stop garbage collection routine, stops any running GC routine. 32 | stopGC chan struct{} 33 | 34 | OnEviction func(key string) 35 | } 36 | 37 | type CachedResponse struct { 38 | Resp *http.Response 39 | } 40 | 41 | func (cp *CachedResponse) Size() int { 42 | return binary.Size(cp.Resp) 43 | } 44 | 45 | // Item is what is stored in the cache 46 | type Item struct { 47 | Key string 48 | Value CachedResponse 49 | } 50 | 51 | type entry struct { 52 | key string 53 | value *CachedResponse 54 | size int64 55 | timeAccessed time.Time 56 | } 57 | 58 | // NewLRUCache creates a new empty cache with the given capacity. 59 | func NewLRUCache(capacity int64, expiry time.Duration) *LRUCache { 60 | cache := &LRUCache{ 61 | list: list.New(), 62 | table: make(map[string]*list.Element), 63 | capacity: capacity, 64 | expiry: expiry, 65 | } 66 | 67 | // We have expiry start the janitor routine. 68 | if expiry > 0 { 69 | // Initialize a new stop GC channel. 70 | cache.stopGC = make(chan struct{}) 71 | 72 | // Start garbage collection routine to expire objects. 73 | cache.StartGC() 74 | } 75 | 76 | return cache 77 | } 78 | 79 | // Get returns a value from the cache, and marks the entry as most 80 | // recently used. 81 | func (lru *LRUCache) Get(key string) (v *CachedResponse, ok bool) { 82 | lru.mu.Lock() 83 | defer lru.mu.Unlock() 84 | 85 | element := lru.table[key] 86 | if element == nil { 87 | return nil, false 88 | } 89 | lru.moveToFront(element) 90 | return element.Value.(*entry).value, true 91 | } 92 | 93 | // Set sets a value in the cache. 94 | func (lru *LRUCache) Set(key string, value *CachedResponse) { 95 | lru.mu.Lock() 96 | defer lru.mu.Unlock() 97 | 98 | if element := lru.table[key]; element != nil { 99 | lru.updateInplace(element, value) 100 | } else { 101 | lru.addNew(key, value) 102 | } 103 | } 104 | 105 | // SetIfAbsent will set the value in the cache if not present. If the 106 | // value exists in the cache, we don't set it. 107 | func (lru *LRUCache) SetIfAbsent(key string, value *CachedResponse) { 108 | lru.mu.Lock() 109 | defer lru.mu.Unlock() 110 | 111 | if element := lru.table[key]; element != nil { 112 | lru.moveToFront(element) 113 | } else { 114 | lru.addNew(key, value) 115 | } 116 | } 117 | 118 | // Delete removes an entry from the cache, and returns if the entry existed. 119 | func (lru *LRUCache) Delete(key string) bool { 120 | lru.mu.Lock() 121 | defer lru.mu.Unlock() 122 | 123 | element := lru.table[key] 124 | if element == nil { 125 | return false 126 | } 127 | 128 | lru.list.Remove(element) 129 | delete(lru.table, key) 130 | lru.size -= element.Value.(*entry).size 131 | return true 132 | } 133 | 134 | // Stats returns a few stats on the cache. 135 | func (lru *LRUCache) Stats() (length, size, capacity int64, oldest time.Time) { 136 | lru.mu.Lock() 137 | defer lru.mu.Unlock() 138 | if lastElem := lru.list.Back(); lastElem != nil { 139 | oldest = lastElem.Value.(*entry).timeAccessed 140 | } 141 | return int64(lru.list.Len()), lru.size, lru.capacity, oldest 142 | } 143 | 144 | // Length returns how many elements are in the cache 145 | func (lru *LRUCache) Length() int64 { 146 | lru.mu.Lock() 147 | defer lru.mu.Unlock() 148 | return int64(lru.list.Len()) 149 | } 150 | 151 | // Size returns the sum of the objects' Size() method. 152 | func (lru *LRUCache) Size() int64 { 153 | lru.mu.Lock() 154 | defer lru.mu.Unlock() 155 | return lru.size 156 | } 157 | 158 | // Capacity returns the cache maximum capacity. 159 | func (lru *LRUCache) Capacity() int64 { 160 | lru.mu.Lock() 161 | defer lru.mu.Unlock() 162 | return lru.capacity 163 | } 164 | 165 | func (lru *LRUCache) updateInplace(element *list.Element, value *CachedResponse) { 166 | valueSize := int64(value.Size()) 167 | sizeDiff := valueSize - element.Value.(*entry).size 168 | element.Value.(*entry).value = value 169 | element.Value.(*entry).size = valueSize 170 | lru.size += sizeDiff 171 | lru.moveToFront(element) 172 | lru.checkCapacity() 173 | } 174 | 175 | func (lru *LRUCache) moveToFront(element *list.Element) { 176 | lru.list.MoveToFront(element) 177 | element.Value.(*entry).timeAccessed = time.Now() 178 | } 179 | 180 | func (lru *LRUCache) addNew(key string, value *CachedResponse) { 181 | newEntry := &entry{key, value, int64(value.Size()), time.Now()} 182 | element := lru.list.PushFront(newEntry) 183 | lru.table[key] = element 184 | lru.size += newEntry.size 185 | lru.checkCapacity() 186 | } 187 | 188 | func (lru *LRUCache) checkCapacity() { 189 | // Partially duplicated from Delete 190 | for lru.size > lru.capacity { 191 | delElem := lru.list.Back() 192 | delValue := delElem.Value.(*entry) 193 | lru.list.Remove(delElem) 194 | delete(lru.table, delValue.key) 195 | lru.size -= delValue.size 196 | } 197 | } 198 | 199 | // gc garbage collect all the expired entries from the cache. 200 | func (lru *LRUCache) gc() { 201 | var evictedEntries []string 202 | lru.mu.Lock() 203 | for k, elem := range lru.table { 204 | if lru.expiry > 0 && time.Now().UTC().Sub(elem.Value.(*entry).timeAccessed) > lru.expiry { 205 | evictedEntries = append(evictedEntries, k) 206 | } 207 | } 208 | lru.mu.Unlock() 209 | for _, k := range evictedEntries { 210 | if lru.OnEviction != nil { 211 | lru.OnEviction(k) 212 | } 213 | } 214 | } 215 | 216 | // rest deletes all the entries from the cache. 217 | func (lru *LRUCache) Reset() { 218 | for k := range lru.table { 219 | lru.Delete(k) 220 | } 221 | } 222 | 223 | // StopGC sends a message to the expiry routine to stop 224 | // expiring cached entries. NOTE: once this is called, cached 225 | // entries will not be expired, be careful if you are using this. 226 | func (c *LRUCache) StopGC() { 227 | if c.stopGC != nil { 228 | c.stopGC <- struct{}{} 229 | } 230 | } 231 | 232 | // StartGC starts running a routine ticking at expiry interval, 233 | // on each interval this routine does a sweep across the cache 234 | // entries and garbage collects all the expired entries. 235 | func (c *LRUCache) StartGC() { 236 | go func() { 237 | for { 238 | select { 239 | // Wait till cleanup interval and initiate delete expired entries. 240 | case <-time.After(c.expiry / 4): 241 | c.gc() 242 | // Stop the routine, usually called by the user of object cache during cleanup. 243 | case <-c.stopGC: 244 | return 245 | } 246 | } 247 | }() 248 | } 249 | -------------------------------------------------------------------------------- /internal/handler/ping.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | func NewPing(logger func(v ...interface{})) *Ping { 8 | return &Ping{ 9 | logger: logger, 10 | } 11 | } 12 | 13 | type Ping struct { 14 | logger func(v ...interface{}) 15 | } 16 | 17 | func (s *Ping) ServeHTTP(resp http.ResponseWriter, req *http.Request) { 18 | s.logger("pinged cache") 19 | resp.WriteHeader(http.StatusOK) 20 | resp.Write([]byte("ok")) 21 | } 22 | -------------------------------------------------------------------------------- /internal/handler/proxycache.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "fmt" 5 | "github.com/donutloop/httpcache/internal/cache" 6 | "github.com/donutloop/httpcache/internal/roundtripper" 7 | "io" 8 | "io/ioutil" 9 | "net" 10 | "net/http" 11 | "net/http/httputil" 12 | "strings" 13 | ) 14 | 15 | func NewProxy(cache *cache.LRUCache, logger func(v ...interface{}), contentLength int64, ping *Ping, stats *Stats) *Proxy { 16 | return &Proxy{ 17 | client: &http.Client{ 18 | Transport: &roundtripper.LoggedTransport{ 19 | Transport: &roundtripper.CacheTransport{ 20 | Transport: &roundtripper.ResponseBodyLimitRoundTripper{ 21 | Transport: http.DefaultTransport, 22 | Limit: contentLength, 23 | }, 24 | Cache: cache, 25 | }, 26 | Logger: logger, 27 | }}, 28 | logger: logger, 29 | ping: ping, 30 | stats: stats, 31 | } 32 | } 33 | 34 | type Proxy struct { 35 | client *http.Client 36 | logger func(v ...interface{}) 37 | ping *Ping 38 | stats *Stats 39 | } 40 | 41 | func (p *Proxy) ServeHTTP(resp http.ResponseWriter, req *http.Request) { 42 | 43 | if req.URL.Path == "/ping" { 44 | p.ping.ServeHTTP(resp, req) 45 | return 46 | } 47 | 48 | if req.URL.Path == "/stats" { 49 | p.stats.ServeHTTP(resp, req) 50 | return 51 | } 52 | 53 | req.RequestURI = "" 54 | if req.Method == http.MethodConnect { 55 | p.ProxyHTTPS(resp, req) 56 | return 57 | } 58 | 59 | proxyResponse, err := p.client.Do(req) 60 | if err != nil { 61 | if strings.Contains(err.Error(), roundtripper.ResponseIsToLarge.Error()) { 62 | resp.WriteHeader(http.StatusRequestEntityTooLarge) 63 | return 64 | } 65 | resp.WriteHeader(http.StatusInternalServerError) 66 | return 67 | } 68 | 69 | for k, vv := range proxyResponse.Header { 70 | for _, v := range vv { 71 | resp.Header().Add(k, v) 72 | } 73 | } 74 | 75 | body, err := ioutil.ReadAll(proxyResponse.Body) 76 | if err != nil { 77 | p.logger(fmt.Sprintf("proxy couldn't read body of response (%v)", err)) 78 | requestDumped, responseDumped, err := dump(req, proxyResponse) 79 | if err == nil { 80 | p.logger(fmt.Sprintf("request: %#v", requestDumped)) 81 | p.logger(fmt.Sprintf("response: %#v", responseDumped)) 82 | } 83 | resp.WriteHeader(http.StatusInternalServerError) 84 | return 85 | } 86 | resp.WriteHeader(proxyResponse.StatusCode) 87 | resp.Write(body) 88 | } 89 | 90 | func (p *Proxy) ProxyHTTPS(rw http.ResponseWriter, req *http.Request) { 91 | hij, ok := rw.(http.Hijacker) 92 | if !ok { 93 | p.logger("proxy https error: http server does not support hijacker") 94 | return 95 | } 96 | 97 | clientConn, _, err := hij.Hijack() 98 | if err != nil { 99 | p.logger("proxy https error: %v", err) 100 | return 101 | } 102 | 103 | proxyConn, err := net.Dial("tcp", req.URL.Host) 104 | if err != nil { 105 | p.logger("proxy https error: %v", err) 106 | return 107 | } 108 | 109 | _, err = clientConn.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) 110 | if err != nil { 111 | p.logger("proxy https error: %v", err) 112 | return 113 | } 114 | 115 | go func() { 116 | io.Copy(clientConn, proxyConn) 117 | clientConn.Close() 118 | proxyConn.Close() 119 | }() 120 | 121 | io.Copy(proxyConn, clientConn) 122 | proxyConn.Close() 123 | clientConn.Close() 124 | } 125 | 126 | type requestDump []byte 127 | 128 | type responseDump []byte 129 | 130 | func dump(request *http.Request, response *http.Response) (requestDump, responseDump, error) { 131 | dumpedResponse, err := httputil.DumpResponse(response, true) 132 | if err != nil { 133 | return nil, nil, err 134 | } 135 | dumpedRequest, err := httputil.DumpRequest(request, true) 136 | if err != nil { 137 | return nil, nil, err 138 | } 139 | return dumpedRequest, dumpedResponse, nil 140 | } 141 | -------------------------------------------------------------------------------- /internal/handler/stats.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/donutloop/httpcache/internal/cache" 7 | "net/http" 8 | "time" 9 | ) 10 | 11 | func NewStats(c *cache.LRUCache, logger func(v ...interface{})) *Stats { 12 | return &Stats{ 13 | c: c, 14 | logger: logger, 15 | } 16 | } 17 | 18 | type StatsResponse struct { 19 | Length int64 `json:"length"` 20 | Size int64 `json:"size"` 21 | Capacity int64 `json:"capacity"` 22 | Oldest time.Time `json:"oldest"` 23 | } 24 | 25 | type Stats struct { 26 | c *cache.LRUCache 27 | logger func(v ...interface{}) 28 | } 29 | 30 | func (s *Stats) ServeHTTP(resp http.ResponseWriter, req *http.Request) { 31 | 32 | domainResp := s.Endpoint() 33 | 34 | v, err := json.Marshal(domainResp) 35 | if err != nil { 36 | s.logger(fmt.Sprintf("could not marshal response (%v)", err)) 37 | resp.WriteHeader(http.StatusInternalServerError) 38 | return 39 | } 40 | 41 | resp.WriteHeader(http.StatusOK) 42 | resp.Write(v) 43 | } 44 | 45 | func (s *Stats) Endpoint() *StatsResponse { 46 | 47 | length, size, capacity, oldest := s.c.Stats() 48 | 49 | resp := &StatsResponse{ 50 | Length: length, 51 | Size: size, 52 | Capacity: capacity, 53 | Oldest: oldest, 54 | } 55 | 56 | return resp 57 | } 58 | -------------------------------------------------------------------------------- /internal/middleware/panic.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "runtime/debug" 7 | ) 8 | 9 | func NewPanic(next http.Handler, loggerFunc func(v ...interface{})) *Panic { 10 | return &Panic{ 11 | Next: next, 12 | loggerFunc: loggerFunc, 13 | } 14 | } 15 | 16 | // Panic recovers from API panics and logs encountered panics 17 | type Panic struct { 18 | Next http.Handler 19 | loggerFunc func(v ...interface{}) 20 | } 21 | 22 | // It recovers from panics of all next handlers and logs them 23 | func (h *Panic) ServeHTTP(r http.ResponseWriter, req *http.Request) { 24 | defer func() { 25 | if r := recover(); r != nil { 26 | h.loggerFunc("begin: recovered from panic") 27 | h.loggerFunc(fmt.Sprintf("unkown value of recover (%v)", r)) 28 | h.loggerFunc(fmt.Sprintf("url %v", req.URL.String())) 29 | h.loggerFunc(fmt.Sprintf("method %v", req.Method)) 30 | h.loggerFunc(fmt.Sprintf("remote address %v", req.RemoteAddr)) 31 | h.loggerFunc(fmt.Sprintf("stack strace of cause \n %v", string(debug.Stack()))) 32 | h.loggerFunc("end: recovered from panic") 33 | } 34 | }() 35 | h.Next.ServeHTTP(r, req) 36 | } 37 | -------------------------------------------------------------------------------- /internal/middleware/panic_test.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | ) 8 | 9 | func TestPanic(t *testing.T) { 10 | crashedHandler := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { 11 | panic("hello world") 12 | }) 13 | 14 | middleware := NewPanic(crashedHandler, t.Log) 15 | 16 | resp := httptest.NewRecorder() 17 | req := httptest.NewRequest(http.MethodGet, "/panic", nil) 18 | 19 | middleware.ServeHTTP(resp, req) 20 | } 21 | -------------------------------------------------------------------------------- /internal/roundtripper/cacher.go: -------------------------------------------------------------------------------- 1 | package roundtripper 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/hex" 6 | "github.com/donutloop/httpcache/internal/cache" 7 | "net/http" 8 | "net/http/httputil" 9 | ) 10 | 11 | type CacheTransport struct { 12 | Cache *cache.LRUCache 13 | Transport http.RoundTripper // underlying transport (or default if nil) 14 | } 15 | 16 | func (t *CacheTransport) RoundTrip(req *http.Request) (*http.Response, error) { 17 | clonedRequest, err := makeHashFromRequest(req) 18 | if err != nil { 19 | return nil, err 20 | } 21 | cachedResponse, ok := t.Cache.Get(clonedRequest) 22 | if !ok { 23 | proxyResponse, err := t.Transport.RoundTrip(req) 24 | if err != nil { 25 | return nil, err 26 | } 27 | cachedResponse = &cache.CachedResponse{Resp: proxyResponse} 28 | t.Cache.Set(clonedRequest, cachedResponse) 29 | return cachedResponse.Resp, nil 30 | } 31 | return cachedResponse.Resp, nil 32 | } 33 | 34 | // CloneRequest returns a clone of the provided *http.Request. The clone is a 35 | // shallow copy of the struct and its Header map. 36 | func makeHashFromRequest(r *http.Request) (string, error) { 37 | // shallow copy of the struct 38 | r2 := new(http.Request) 39 | *r2 = *r 40 | // deep copy of the Header 41 | r2.Header = make(http.Header) 42 | for k, s := range r.Header { 43 | r2.Header[k] = s 44 | } 45 | 46 | d, err := httputil.DumpRequest(r, true) 47 | if err != nil { 48 | return "", err 49 | } 50 | 51 | hasher := md5.New() 52 | hasher.Write([]byte(d)) 53 | return hex.EncodeToString(hasher.Sum(nil)), nil 54 | } 55 | -------------------------------------------------------------------------------- /internal/roundtripper/cacher_test.go: -------------------------------------------------------------------------------- 1 | package roundtripper 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | ) 7 | 8 | func TestMakeHashFromRequest(t *testing.T) { 9 | 10 | req, err := http.NewRequest(http.MethodGet, "http://test.de", nil) 11 | if err != nil { 12 | t.Fatal(err) 13 | } 14 | 15 | hash1, err := makeHashFromRequest(req) 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | 20 | hash2, err := makeHashFromRequest(req) 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | 25 | if hash1 != hash2 { 26 | t.Log("hash 1: " + hash1) 27 | t.Log("hash 2: " + hash1) 28 | t.Error("hash are not equal") 29 | } 30 | 31 | t.Log("hash 1: " + hash1) 32 | t.Log("hash 2: " + hash1) 33 | } 34 | -------------------------------------------------------------------------------- /internal/roundtripper/logger.go: -------------------------------------------------------------------------------- 1 | package roundtripper 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | // A LoggedTransport prints URLs and timings for each HTTP request. 10 | type LoggedTransport struct { 11 | Logger func(v ...interface{}) 12 | Transport http.RoundTripper // underlying transport (or default if nil) 13 | } 14 | 15 | func (t *LoggedTransport) RoundTrip(req *http.Request) (*http.Response, error) { 16 | 17 | start := time.Now() 18 | resp, err := t.Transport.RoundTrip(req) 19 | if err != nil { 20 | t.Logger(fmt.Sprintf("HTTP %s %s: error: %s\n", req.Method, req.URL, err)) 21 | return nil, err 22 | } 23 | 24 | t.Logger(fmt.Sprintf("HTTP %s %s %d [%s rtt]\n", req.Method, req.URL, resp.StatusCode, time.Since(start))) 25 | 26 | return resp, nil 27 | } 28 | -------------------------------------------------------------------------------- /internal/roundtripper/reponse_body_limit.go: -------------------------------------------------------------------------------- 1 | package roundtripper 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | ) 7 | 8 | var ResponseIsToLarge = errors.New("response body is to large for the cache") 9 | 10 | type ResponseBodyLimitRoundTripper struct { 11 | Limit int64 12 | Transport http.RoundTripper // underlying transport (or default if nil) 13 | } 14 | 15 | func (t *ResponseBodyLimitRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { 16 | response, err := t.Transport.RoundTrip(req) 17 | if err != nil { 18 | return nil, err 19 | } 20 | 21 | if response.ContentLength > t.Limit { 22 | return nil, ResponseIsToLarge 23 | } 24 | 25 | return response, nil 26 | } 27 | -------------------------------------------------------------------------------- /internal/size/size.go: -------------------------------------------------------------------------------- 1 | package size 2 | 3 | const ( 4 | _ = iota // ignore first value by assigning to blank identifier 5 | KB int64 = 1 << (10 * iota) 6 | MB 7 | GB 8 | ) 9 | -------------------------------------------------------------------------------- /internal/xhttp/server.go: -------------------------------------------------------------------------------- 1 | package xhttp 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "net" 8 | "net/http" 9 | "time" 10 | ) 11 | 12 | type Server struct { 13 | *http.Server 14 | Listener net.Listener 15 | 16 | ShutdownTimeout time.Duration 17 | 18 | Logger *log.Logger 19 | } 20 | 21 | // Start starts the server and waits for it to return. 22 | func (s *Server) Start() error { 23 | s.Logger.Println(fmt.Sprintf("starting server on (%v)", s.Listener.Addr())) 24 | 25 | return s.Serve(s.Listener) 26 | } 27 | 28 | // Start starts the server and waits for it to return. 29 | func (s *Server) StartTLS(certFile, keyFile string) error { 30 | s.Logger.Println(fmt.Sprintf("starting server on (%v)", s.Listener.Addr())) 31 | 32 | return s.ServeTLS(s.Listener, certFile, keyFile) 33 | } 34 | 35 | // Stop tries to shut the server down gracefully first, then forcefully closes it. 36 | func (s *Server) Stop() { 37 | ctx := context.Background() 38 | if s.ShutdownTimeout > 0 { 39 | var cancel context.CancelFunc 40 | ctx, cancel = context.WithTimeout(context.Background(), s.ShutdownTimeout) 41 | 42 | defer cancel() 43 | } 44 | 45 | s.Logger.Println("shutting server down") 46 | err := s.Server.Shutdown(ctx) 47 | if err != nil { 48 | s.Logger.Println(err) 49 | } 50 | 51 | s.Server.Close() 52 | } 53 | -------------------------------------------------------------------------------- /tests/api_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "crypto/tls" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/donutloop/httpcache/internal/cache" 8 | "github.com/donutloop/httpcache/internal/handler" 9 | "github.com/donutloop/httpcache/internal/middleware" 10 | "github.com/donutloop/httpcache/internal/size" 11 | "github.com/donutloop/httpcache/internal/xhttp" 12 | "log" 13 | "math/rand" 14 | "net" 15 | "net/http" 16 | "net/http/httptest" 17 | "net/http/httputil" 18 | "net/url" 19 | "os" 20 | "testing" 21 | "time" 22 | ) 23 | 24 | var client *http.Client 25 | var clientTls *http.Client 26 | var c *cache.LRUCache 27 | 28 | func TestMain(m *testing.M) { 29 | c = cache.NewLRUCache(100, 0) 30 | stats := handler.NewStats(c, log.Println) 31 | ping := handler.NewPing(log.Println) 32 | proxy := handler.NewProxy( 33 | c, 34 | log.Println, 35 | 500*size.MB, 36 | ping, 37 | stats, 38 | ) 39 | 40 | stack := middleware.NewPanic(proxy, log.Println) 41 | 42 | proxyServer := httptest.NewServer(stack) 43 | proxyServerTLS := httptest.NewTLSServer(proxy) 44 | 45 | transport := &http.Transport{ 46 | Proxy: SetProxyURL(proxyServer.URL), 47 | DialContext: (&net.Dialer{ 48 | Timeout: 30 * time.Second, 49 | KeepAlive: 30 * time.Second, 50 | DualStack: true, 51 | }).DialContext, 52 | MaxIdleConns: 100, 53 | IdleConnTimeout: 90 * time.Second, 54 | TLSHandshakeTimeout: 10 * time.Second, 55 | ExpectContinueTimeout: 1 * time.Second, 56 | } 57 | 58 | client = &http.Client{ 59 | Transport: transport, 60 | } 61 | 62 | transportTls := &http.Transport{ 63 | Proxy: SetProxyURL(proxyServerTLS.URL + "/"), 64 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 65 | DialContext: (&net.Dialer{ 66 | Timeout: 30 * time.Second, 67 | KeepAlive: 30 * time.Second, 68 | DualStack: true, 69 | }).DialContext, 70 | MaxIdleConns: 100, 71 | IdleConnTimeout: 90 * time.Second, 72 | TLSHandshakeTimeout: 10 * time.Second, 73 | ExpectContinueTimeout: 1 * time.Second, 74 | } 75 | 76 | log.Println("tls proxy " + proxyServerTLS.URL) 77 | 78 | clientTls = &http.Client{ 79 | Transport: transportTls, 80 | } 81 | 82 | testtransport := &http.Transport{ 83 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 84 | DialContext: (&net.Dialer{ 85 | Timeout: 30 * time.Second, 86 | KeepAlive: 30 * time.Second, 87 | DualStack: true, 88 | }).DialContext, 89 | MaxIdleConns: 100, 90 | IdleConnTimeout: 90 * time.Second, 91 | TLSHandshakeTimeout: 10 * time.Second, 92 | ExpectContinueTimeout: 1 * time.Second, 93 | } 94 | 95 | testclient := http.Client{ 96 | Transport: testtransport, 97 | } 98 | 99 | resp, err := testclient.Get(fmt.Sprintf("%v/ping", proxyServerTLS.URL)) 100 | if err != nil { 101 | log.Fatalln(fmt.Sprintf("tls proxy (%v)", err)) 102 | } 103 | 104 | if resp.StatusCode != http.StatusOK { 105 | log.Fatalln(fmt.Sprintf("status code is bad (%v)", resp.StatusCode)) 106 | } 107 | 108 | resp, err = testclient.Get(fmt.Sprintf("%v/ping", proxyServer.URL)) 109 | if err != nil { 110 | log.Fatalln(fmt.Sprintf("proxy (%v)", err)) 111 | } 112 | 113 | if resp.StatusCode != http.StatusOK { 114 | log.Fatalln(fmt.Sprintf("status code is bad (%v)", resp.StatusCode)) 115 | } 116 | 117 | // call flag.Parse() here if TestMain uses flags 118 | os.Exit(m.Run()) 119 | } 120 | 121 | func SetProxyURL(proxy string) func(req *http.Request) (*url.URL, error) { 122 | return func(req *http.Request) (*url.URL, error) { 123 | proxyURL, err := url.Parse(proxy) 124 | if err != nil { 125 | return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) 126 | } 127 | return proxyURL, nil 128 | } 129 | } 130 | 131 | func TestProxyHTTPSHandler(t *testing.T) { 132 | defer c.Reset() 133 | 134 | handler := func(w http.ResponseWriter, r *http.Request) { 135 | w.WriteHeader(http.StatusOK) 136 | w.Write([]byte(`{"count": 10}`)) 137 | return 138 | } 139 | 140 | testHandler := httptest.NewTLSServer(http.HandlerFunc(handler)) 141 | 142 | req, err := http.NewRequest(http.MethodGet, testHandler.URL, nil) 143 | if err != nil { 144 | t.Fatal(err) 145 | } 146 | 147 | resp, err := clientTls.Do(req) 148 | if err != nil { 149 | t.Fatal(err) 150 | } 151 | 152 | if resp.StatusCode != http.StatusOK { 153 | t.Fatalf("status code is bad (%v)", resp.StatusCode) 154 | } 155 | 156 | if c.Length() != 0 { 157 | t.Fatalf("cache length is bad, got=%d", c.Length()) 158 | } 159 | } 160 | 161 | func TestProxyHandler(t *testing.T) { 162 | defer c.Reset() 163 | 164 | handler := func(w http.ResponseWriter, r *http.Request) { 165 | w.WriteHeader(http.StatusOK) 166 | w.Write([]byte(`{"count": 10}`)) 167 | return 168 | } 169 | 170 | server := httptest.NewServer(http.HandlerFunc(handler)) 171 | req, err := http.NewRequest(http.MethodGet, server.URL, nil) 172 | if err != nil { 173 | t.Fatal(err) 174 | } 175 | 176 | resp, err := client.Do(req) 177 | if err != nil { 178 | t.Fatal(err) 179 | } 180 | 181 | if resp.StatusCode != http.StatusOK { 182 | t.Fatalf("status code is bad (%v)", resp.StatusCode) 183 | } 184 | 185 | if c.Length() != 1 { 186 | t.Fatalf("cache length is bad, got=%d", c.Length()) 187 | } 188 | } 189 | 190 | func TestStatsHandler(t *testing.T) { 191 | defer c.Reset() 192 | 193 | testHandler := func(w http.ResponseWriter, r *http.Request) { 194 | w.WriteHeader(http.StatusOK) 195 | w.Write([]byte(`{"count": 10}`)) 196 | return 197 | } 198 | 199 | server := httptest.NewServer(http.HandlerFunc(testHandler)) 200 | 201 | req, err := http.NewRequest(http.MethodGet, server.URL, nil) 202 | if err != nil { 203 | t.Fatal(err) 204 | } 205 | 206 | t.Log(req.URL) 207 | resp, err := client.Do(req) 208 | if err != nil { 209 | t.Fatal(err) 210 | } 211 | 212 | if resp.StatusCode != http.StatusOK { 213 | t.Fatalf("status code is bad (%v)", resp.StatusCode) 214 | } 215 | 216 | if c.Length() != 1 { 217 | t.Fatalf("cache length is bad, got=%d", c.Length()) 218 | } 219 | 220 | req, err = http.NewRequest(http.MethodGet, server.URL+"/stats", nil) 221 | if err != nil { 222 | t.Fatal(err) 223 | } 224 | 225 | t.Log(req.URL) 226 | resp, err = client.Do(req) 227 | if err != nil { 228 | t.Fatal(err) 229 | } 230 | 231 | if resp.StatusCode != http.StatusOK { 232 | t.Fatalf("status code is bad (%v)", resp.StatusCode) 233 | } 234 | 235 | statsResponse := &handler.StatsResponse{} 236 | if err := json.NewDecoder(resp.Body).Decode(statsResponse); err != nil { 237 | b, err := httputil.DumpResponse(resp, true) 238 | if err == nil { 239 | t.Log(string(b)) 240 | } 241 | t.Fatalf("could not decode incoming response (%v)", err) 242 | } 243 | 244 | if statsResponse.Length != 1 { 245 | t.Fatalf("cache length is bad, got=%d", c.Length()) 246 | } 247 | 248 | t.Log(fmt.Sprintf("%#v", statsResponse)) 249 | } 250 | 251 | func TestProxyHandler_ResponseBodyContentLengthLimit(t *testing.T) { 252 | c1 := cache.NewLRUCache(100, 1*time.Second) 253 | { 254 | c1.OnEviction = func(key string) { 255 | c1.Delete(key) 256 | } 257 | } 258 | cl := 1 * size.KB 259 | t.Log("size: ", cl) 260 | 261 | go func() { 262 | logger := log.New(os.Stderr, "", log.LstdFlags) 263 | 264 | stats := handler.NewStats(c, logger.Println) 265 | ping := handler.NewPing(logger.Println) 266 | proxy := handler.NewProxy( 267 | c, 268 | logger.Println, 269 | cl, 270 | ping, 271 | stats, 272 | ) 273 | 274 | mux := http.NewServeMux() 275 | mux.Handle("/", proxy) 276 | 277 | listener, err := net.Listen("tcp", "localhost:4528") 278 | if err != nil { 279 | logger.Fatal(err) 280 | } 281 | 282 | xserver := xhttp.Server{ 283 | Server: &http.Server{Addr: "localhost:4528", Handler: proxy}, 284 | Logger: logger, 285 | Listener: listener, 286 | } 287 | if err := xserver.Start(); err != nil { 288 | xserver.Stop() 289 | } 290 | }() 291 | 292 | <-time.After(1 * time.Second) 293 | 294 | transport := &http.Transport{ 295 | Proxy: SetProxyURL("http://localhost:4528"), 296 | DialContext: (&net.Dialer{ 297 | Timeout: 30 * time.Second, 298 | KeepAlive: 30 * time.Second, 299 | DualStack: true, 300 | }).DialContext, 301 | MaxIdleConns: 100, 302 | IdleConnTimeout: 90 * time.Second, 303 | TLSHandshakeTimeout: 10 * time.Second, 304 | ExpectContinueTimeout: 1 * time.Second, 305 | } 306 | 307 | client := &http.Client{ 308 | Transport: transport, 309 | } 310 | 311 | testHandler := func(w http.ResponseWriter, r *http.Request) { 312 | w.WriteHeader(http.StatusOK) 313 | data := make([]byte, 2*size.KB, 2*size.KB) 314 | w.Write(data) 315 | return 316 | } 317 | 318 | server := httptest.NewServer(http.HandlerFunc(testHandler)) 319 | 320 | req, err := http.NewRequest(http.MethodGet, server.URL, nil) 321 | if err != nil { 322 | t.Fatal(err) 323 | } 324 | 325 | t.Log(req.URL) 326 | resp, err := client.Do(req) 327 | if err != nil { 328 | t.Fatal(err) 329 | } 330 | 331 | if resp.StatusCode != http.StatusRequestEntityTooLarge { 332 | t.Fatalf("status code is bad (%v)", resp.StatusCode) 333 | } 334 | 335 | <-time.After(2 * time.Second) 336 | 337 | if c1.Length() != 0 { 338 | t.Fatalf("cache length is bad, got=%d", c1.Length()) 339 | } 340 | } 341 | 342 | func TestProxyHandler_GC(t *testing.T) { 343 | c1 := cache.NewLRUCache(100, 1*time.Second) 344 | { 345 | c1.OnEviction = func(key string) { 346 | c1.Delete(key) 347 | } 348 | } 349 | 350 | go func() { 351 | logger := log.New(os.Stderr, "", log.LstdFlags) 352 | stats := handler.NewStats(c, logger.Println) 353 | ping := handler.NewPing(logger.Println) 354 | proxy := handler.NewProxy( 355 | c, 356 | logger.Println, 357 | 3*size.MB, 358 | ping, 359 | stats, 360 | ) 361 | 362 | mux := http.NewServeMux() 363 | mux.Handle("/", proxy) 364 | 365 | listener, err := net.Listen("tcp", "localhost:4568") 366 | if err != nil { 367 | logger.Fatal(err) 368 | } 369 | 370 | xserver := xhttp.Server{ 371 | Server: &http.Server{Addr: "localhost:4568", Handler: proxy}, 372 | Logger: logger, 373 | Listener: listener, 374 | } 375 | if err := xserver.Start(); err != nil { 376 | xserver.Stop() 377 | } 378 | }() 379 | 380 | <-time.After(1 * time.Second) 381 | 382 | transport := &http.Transport{ 383 | Proxy: SetProxyURL("http://localhost:4568"), 384 | DialContext: (&net.Dialer{ 385 | Timeout: 30 * time.Second, 386 | KeepAlive: 30 * time.Second, 387 | DualStack: true, 388 | }).DialContext, 389 | MaxIdleConns: 100, 390 | IdleConnTimeout: 90 * time.Second, 391 | TLSHandshakeTimeout: 10 * time.Second, 392 | ExpectContinueTimeout: 1 * time.Second, 393 | } 394 | 395 | client := &http.Client{ 396 | Transport: transport, 397 | } 398 | 399 | testHandler := func(w http.ResponseWriter, r *http.Request) { 400 | w.WriteHeader(http.StatusOK) 401 | w.Write([]byte(`{"count": 10}`)) 402 | return 403 | } 404 | 405 | server := httptest.NewServer(http.HandlerFunc(testHandler)) 406 | 407 | req, err := http.NewRequest(http.MethodGet, server.URL, nil) 408 | if err != nil { 409 | t.Fatal(err) 410 | } 411 | 412 | t.Log(req.URL) 413 | resp, err := client.Do(req) 414 | if err != nil { 415 | t.Fatal(err) 416 | } 417 | 418 | if resp.StatusCode != http.StatusOK { 419 | t.Fatalf("status code is bad (%v)", resp.StatusCode) 420 | } 421 | 422 | <-time.After(2 * time.Second) 423 | 424 | if c1.Length() != 0 { 425 | t.Fatalf("cache length is bad, got=%d", c1.Length()) 426 | } 427 | } 428 | 429 | func TestProxyHttpServer(t *testing.T) { 430 | 431 | c1 := cache.NewLRUCache(100, 0) 432 | go func() { 433 | logger := log.New(os.Stderr, "", log.LstdFlags) 434 | 435 | stats := handler.NewStats(c, logger.Println) 436 | ping := handler.NewPing(logger.Println) 437 | proxy := handler.NewProxy( 438 | c1, 439 | logger.Println, 440 | 5*size.MB, 441 | ping, 442 | stats, 443 | ) 444 | mux := http.NewServeMux() 445 | mux.Handle("/", proxy) 446 | 447 | listener, err := net.Listen("tcp", "localhost:4567") 448 | if err != nil { 449 | logger.Fatal(err) 450 | } 451 | 452 | xserver := xhttp.Server{ 453 | Server: &http.Server{Addr: "localhost:4567", Handler: proxy}, 454 | Logger: logger, 455 | Listener: listener, 456 | } 457 | if err := xserver.Start(); err != nil { 458 | xserver.Stop() 459 | } 460 | }() 461 | 462 | <-time.After(1 * time.Second) 463 | 464 | transport := &http.Transport{ 465 | Proxy: SetProxyURL("http://localhost:4567"), 466 | DialContext: (&net.Dialer{ 467 | Timeout: 30 * time.Second, 468 | KeepAlive: 30 * time.Second, 469 | DualStack: true, 470 | }).DialContext, 471 | MaxIdleConns: 100, 472 | IdleConnTimeout: 90 * time.Second, 473 | TLSHandshakeTimeout: 10 * time.Second, 474 | ExpectContinueTimeout: 1 * time.Second, 475 | } 476 | 477 | client := &http.Client{ 478 | Transport: transport, 479 | } 480 | 481 | handler := func(w http.ResponseWriter, r *http.Request) { 482 | w.WriteHeader(http.StatusOK) 483 | w.Write([]byte(`{"count": 10}`)) 484 | return 485 | } 486 | server := httptest.NewServer(http.HandlerFunc(handler)) 487 | 488 | req, err := http.NewRequest(http.MethodGet, server.URL, nil) 489 | if err != nil { 490 | t.Fatal(err) 491 | } 492 | 493 | resp, err := client.Do(req) 494 | if err != nil { 495 | t.Fatal(err) 496 | } 497 | 498 | if resp.StatusCode != http.StatusOK { 499 | t.Fatalf("status code is bad (%v)", resp.StatusCode) 500 | } 501 | 502 | v := struct { 503 | Count int 504 | }{} 505 | if err := json.NewDecoder(resp.Body).Decode(&v); err != nil { 506 | t.Fatal(err) 507 | } 508 | 509 | if v.Count != 10 { 510 | t.Fatalf("count is bad, got=%d", v.Count) 511 | } 512 | 513 | if c1.Length() != 1 { 514 | t.Fatalf("cache length is bad, got=%d", c1.Length()) 515 | } 516 | } 517 | 518 | func BenchmarkProxy(b *testing.B) { 519 | defer c.Reset() 520 | 521 | servers := make([]*httptest.Server, 0) 522 | for i := 0; i < 10; i++ { 523 | handler := func(w http.ResponseWriter, r *http.Request) { 524 | w.WriteHeader(http.StatusOK) 525 | w.Write([]byte(`{"data": "` + generateData(256) + `"}`)) 526 | return 527 | } 528 | server := httptest.NewServer(http.HandlerFunc(handler)) 529 | servers = append(servers, server) 530 | } 531 | 532 | b.N = 10 533 | 534 | for n := 0; n < b.N; n++ { 535 | req, err := http.NewRequest(http.MethodGet, servers[rand.Intn(9)].URL, nil) 536 | if err != nil { 537 | continue 538 | } 539 | 540 | resp, err := client.Do(req) 541 | if err != nil { 542 | continue 543 | } 544 | 545 | if resp.StatusCode != http.StatusOK { 546 | b.Logf("status code is bad (%v)", resp.StatusCode) 547 | } 548 | } 549 | } 550 | 551 | func generateData(n int) string { 552 | const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" 553 | b := make([]byte, n) 554 | for i := range b { 555 | b[i] = letterBytes[rand.Intn(len(letterBytes))] 556 | } 557 | return string(b) 558 | } 559 | --------------------------------------------------------------------------------