├── .gitignore ├── LICENSE.md ├── README.md ├── buffer_pool.go ├── buffer_pool_test.go ├── compression ├── compression.go └── compression_test.go ├── continue.go ├── doc.go ├── etag ├── etag.go └── etag_test.go ├── examples ├── basic_file_server │ └── server.go ├── blog_example │ └── blog_example.go ├── hello_world │ └── hello_world.go └── hot_restart │ └── hot_restart.go ├── filter.go ├── handler_filter.go ├── handler_filter_test.go ├── logger.go ├── pipeline.go ├── pipeline_test.go ├── request.go ├── response.go ├── router.go ├── router_test.go ├── server.go ├── server_notwindows.go ├── server_windows.go ├── static_file ├── file_filter.go └── file_filter_test.go ├── string_body.go ├── string_body_test.go ├── test ├── custom_type.foo ├── foo.json ├── hello │ └── world.txt ├── images │ └── face.png └── index.html └── upstream ├── upstream.go └── upstream_pool.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.6 2 | _* 3 | 6.out 4 | *.swp 5 | tags 6 | bin 7 | examples/hot_restart/hot_restart 8 | examples/hello_world/hello_world 9 | examples/basic_file_server/basic_file_server 10 | 11 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 ngmoco:) inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Falcore has Moved 2 | 3 | [Fitstar Falcore »](https://github.com/fitstar/falcore) 4 | 5 | The Fitstar fork has tons of updates that break backwards compatibility so we are leaving this repo live for reference of the older interfaces. It will no longer be maintained here, though so you should upgrade to the Fitstar fork. 6 | 7 | # Falcore 8 | 9 | Falcore is a framework for constructing high performance, modular HTTP servers in Golang. 10 | 11 | [Read more on our blog »](http://ngenuity.ngmoco.com/2012/01/introducing-falcore-and-timber.html) 12 | 13 | [GoPkgDoc](http://gopkgdoc.appspot.com/pkg/github.com/ngmoco/falcore) hosts code documentation for this project. 14 | 15 | ## Features 16 | * Modular and flexible design 17 | * Hot restart hooks for zero-downtime deploys 18 | * Builtin statistics framework 19 | * Builtin logging framework 20 | 21 | ## Design 22 | 23 | Falcore is a filter pipeline based HTTP server library. You can build arbitrarily complicated HTTP services by chaining just a few simple components: 24 | 25 | * `RequestFilters` are the core component. A request filter takes a request and returns a response or nil. Request filters can modify the request as it passes through. 26 | * `ResponseFilters` can modify a response on its way out the door. An example response filter, `compression_filter`, is included. It applies `deflate` or `gzip` compression to the response if the request supplies the proper headers. 27 | * `Pipelines` form one of the two logic components. A pipeline contains a list of `RequestFilters` and a list of `ResponseFilters`. A request is processed through the request filters, in order, until one returns a response. It then passes the response through each of the response filters, in order. A pipeline is a valid `RequestFilter`. 28 | * `Routers` allow you to conditionally follow different pipelines. A router chooses from a set of pipelines. A few basic routers are included, including routing by hostname or requested path. You can implement your own router by implementing `falcore.Router`. `Routers` are not `RequestFilters`, but they can be put into pipelines. 29 | 30 | ## Building 31 | 32 | Falcore is currently targeted at Go 1.0. If you're still using Go r.60.x, you can get the last working version of falcore for r.60 using the tag `last_r60`. 33 | 34 | Check out the project into $GOROOT/src/pkg/github.com/ngmoco/falcore. Build using the `go build` command. 35 | 36 | ## Usage 37 | 38 | See the `examples` directory for usage examples. 39 | 40 | ## HTTPS 41 | 42 | To use falcore to serve HTTPS, simply call `ListenAndServeTLS` instead of `ListenAndServe`. If you want to host SSL and nonSSL out of the same process, simply create two instances of `falcore.Server`. You can give them the same pipeline or share pipeline components. 43 | 44 | ## Maintainers 45 | 46 | * [Dave Grijalva](http://www.github.com/dgrijalva) 47 | * [Scott White](http://www.github.com/smw1218) 48 | 49 | ## Contributors 50 | 51 | * [Graham Anderson](http://www.github.com/gnanderson) 52 | * [Amir Mohammad Saied](http://github.com/amir) 53 | * [James Wynn](https://github.com/jameswynn) 54 | 55 | [gb]: http://code.google.com/p/go-gb/ 56 | -------------------------------------------------------------------------------- /buffer_pool.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "io/ioutil" 7 | ) 8 | 9 | // uses a chan as a leaky bucket buffer pool 10 | type bufferPool struct { 11 | // size of buffer when creating new ones 12 | bufSize int 13 | // the actual pool of buffers ready for reuse 14 | pool chan *bufferPoolEntry 15 | } 16 | 17 | // This is what's stored in the buffer. It allows 18 | // for the underlying io.Reader to be changed out 19 | // inside a bufio.Reader. This is required for reuse. 20 | type bufferPoolEntry struct { 21 | br *bufio.Reader 22 | source io.Reader 23 | } 24 | 25 | // make bufferPoolEntry a passthrough io.Reader 26 | func (bpe *bufferPoolEntry) Read(p []byte) (n int, err error) { 27 | return bpe.source.Read(p) 28 | } 29 | 30 | func newBufferPool(poolSize, bufferSize int) *bufferPool { 31 | return &bufferPool{ 32 | bufSize: bufferSize, 33 | pool: make(chan *bufferPoolEntry, poolSize), 34 | } 35 | } 36 | 37 | // Take a buffer from the pool and set 38 | // it up to read from r 39 | func (p *bufferPool) take(r io.Reader) (bpe *bufferPoolEntry) { 40 | select { 41 | case bpe = <-p.pool: 42 | // prepare for reuse 43 | if a := bpe.br.Buffered(); a > 0 { 44 | // drain the internal buffer 45 | io.CopyN(ioutil.Discard, bpe.br, int64(a)) 46 | } 47 | // swap out the underlying reader 48 | bpe.source = r 49 | default: 50 | // none available. create a new one 51 | bpe = &bufferPoolEntry{nil, r} 52 | bpe.br = bufio.NewReaderSize(bpe, p.bufSize) 53 | } 54 | return 55 | } 56 | 57 | // Return a buffer to the pool 58 | func (p *bufferPool) give(bpe *bufferPoolEntry) { 59 | select { 60 | case p.pool <- bpe: // return to pool 61 | default: // discard 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /buffer_pool_test.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | /* "io"*/ 5 | "bytes" 6 | "testing" 7 | ) 8 | 9 | func TestBufferPool(t *testing.T) { 10 | pool := newBufferPool(10, 1024) 11 | 12 | text := []byte("Hello World") 13 | 14 | // get one 15 | bpe := pool.take(bytes.NewBuffer(text)) 16 | // read all 17 | out := make([]byte, 1024) 18 | l, _ := bpe.br.Read(out) 19 | if bytes.Compare(out[0:l], text) != 0 { 20 | t.Errorf("Read invalid data: %v", out) 21 | } 22 | if l != len(text) { 23 | t.Errorf("Expected length %v got %v", len(text), l) 24 | } 25 | pool.give(bpe) 26 | 27 | // get one 28 | bpe = pool.take(bytes.NewBuffer(text)) 29 | // read all 30 | out = make([]byte, 1024) 31 | l, _ = bpe.br.Read(out) 32 | if bytes.Compare(out[0:l], text) != 0 { 33 | t.Errorf("Read invalid data: %v", out) 34 | } 35 | if l != len(text) { 36 | t.Errorf("Expected length %v got %v", len(text), l) 37 | } 38 | pool.give(bpe) 39 | 40 | // get one 41 | bpe = pool.take(bytes.NewBuffer(text)) 42 | // read 1 byte 43 | out = make([]byte, 1) 44 | bpe.br.Read(out) 45 | pool.give(bpe) 46 | 47 | // get one 48 | bpe = pool.take(bytes.NewBuffer(text)) 49 | // read all 50 | out = make([]byte, 1024) 51 | l, _ = bpe.br.Read(out) 52 | if bytes.Compare(out[0:l], text) != 0 { 53 | t.Errorf("Read invalid data: %v", out) 54 | } 55 | if l != len(text) { 56 | t.Errorf("Expected length %v got %v", len(text), l) 57 | } 58 | pool.give(bpe) 59 | 60 | } 61 | -------------------------------------------------------------------------------- /compression/compression.go: -------------------------------------------------------------------------------- 1 | package compression 2 | 3 | import ( 4 | "bytes" 5 | "compress/flate" 6 | "compress/gzip" 7 | "github.com/ngmoco/falcore" 8 | "io" 9 | "net/http" 10 | "strings" 11 | ) 12 | 13 | var DefaultTypes = []string{"text/plain", "text/html", "application/json", "text/xml"} 14 | 15 | type Filter struct { 16 | types []string 17 | } 18 | 19 | func NewFilter(types []string) *Filter { 20 | f := new(Filter) 21 | if types != nil { 22 | f.types = types 23 | } else { 24 | f.types = DefaultTypes 25 | } 26 | return f 27 | } 28 | 29 | func (c *Filter) FilterResponse(request *falcore.Request, res *http.Response) { 30 | req := request.HttpRequest 31 | if accept := req.Header.Get("Accept-Encoding"); accept != "" { 32 | 33 | // Is content an acceptable type for encoding? 34 | var compress = false 35 | var content_type = res.Header.Get("Content-Type") 36 | for _, t := range c.types { 37 | if content_type == t { 38 | compress = true 39 | break 40 | } 41 | } 42 | 43 | // Is the content already compressed 44 | if res.Header.Get("Content-Encoding") != "" { 45 | compress = false 46 | } 47 | 48 | if !compress { 49 | request.CurrentStage.Status = 1 // Skip 50 | return 51 | } 52 | 53 | // Figure out which encoding to use 54 | options := strings.Split(accept, ",") 55 | var mode string 56 | for _, opt := range options { 57 | if m := strings.TrimSpace(opt); m == "gzip" || m == "deflate" { 58 | mode = m 59 | break 60 | } 61 | } 62 | 63 | var compressor io.WriteCloser 64 | var buf = bytes.NewBuffer(make([]byte, 0, 1024)) 65 | switch mode { 66 | case "gzip": 67 | compressor = gzip.NewWriter(buf) 68 | case "deflate": 69 | comp, err := flate.NewWriter(buf, -1) 70 | if err != nil { 71 | falcore.Error("Compression Error: %v", err) 72 | request.CurrentStage.Status = 1 // Skip 73 | return 74 | } 75 | compressor = comp 76 | default: 77 | request.CurrentStage.Status = 1 // Skip 78 | return 79 | } 80 | 81 | // Perform compression 82 | r := make([]byte, 1024) 83 | var err error 84 | var i int 85 | for err == nil { 86 | i, err = res.Body.Read(r) 87 | compressor.Write(r[0:i]) 88 | } 89 | compressor.Close() 90 | res.Body.Close() 91 | 92 | res.ContentLength = int64(buf.Len()) 93 | res.Body = (*filteredBody)(buf) 94 | res.Header.Set("Content-Encoding", mode) 95 | } else { 96 | request.CurrentStage.Status = 1 // Skip 97 | } 98 | } 99 | 100 | // wrapper type for Response struct 101 | 102 | type filteredBody bytes.Buffer 103 | 104 | func (b *filteredBody) Read(byt []byte) (int, error) { 105 | i, err := (*bytes.Buffer)(b).Read(byt) 106 | return i, err 107 | } 108 | 109 | func (b filteredBody) Close() error { 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /compression/compression_test.go: -------------------------------------------------------------------------------- 1 | package compression 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "compress/flate" 7 | "compress/gzip" 8 | "fmt" 9 | "github.com/ngmoco/falcore" 10 | "io" 11 | "io/ioutil" 12 | "net" 13 | "net/http" 14 | "path" 15 | "testing" 16 | "time" 17 | ) 18 | 19 | var srv *falcore.Server 20 | 21 | func init() { 22 | go func() { 23 | // falcore setup 24 | pipeline := falcore.NewPipeline() 25 | pipeline.Upstream.PushBack(falcore.NewRequestFilter(func(req *falcore.Request) *http.Response { 26 | for _, data := range serverData { 27 | if data.path == req.HttpRequest.URL.Path { 28 | header := make(http.Header) 29 | header.Set("Content-Type", data.mime) 30 | header.Set("Content-Encoding", data.encoding) 31 | return falcore.SimpleResponse(req.HttpRequest, 200, header, string(data.body)) 32 | } 33 | } 34 | return falcore.SimpleResponse(req.HttpRequest, 404, nil, "Not Found") 35 | })) 36 | 37 | pipeline.Downstream.PushBack(NewFilter(nil)) 38 | 39 | srv = falcore.NewServer(0, pipeline) 40 | if err := srv.ListenAndServe(); err != nil { 41 | panic("Could not start falcore") 42 | } 43 | }() 44 | } 45 | 46 | func port() int { 47 | for srv.Port() == 0 { 48 | time.Sleep(1e7) 49 | } 50 | return srv.Port() 51 | } 52 | 53 | var serverData = []struct { 54 | path string 55 | mime string 56 | encoding string 57 | body []byte 58 | }{ 59 | { 60 | "/hello", 61 | "text/plain", 62 | "", 63 | []byte("hello world"), 64 | }, 65 | { 66 | "/hello.gz", 67 | "text/plain", 68 | "gzip", 69 | compress_gzip([]byte("hello world")), 70 | }, 71 | { 72 | "/images/face.png", 73 | "image/png", 74 | "", 75 | readfile("../test/images/face.png"), 76 | }, 77 | } 78 | 79 | var testData = []struct { 80 | name string 81 | // input 82 | path string 83 | accept string 84 | // output 85 | encoding string 86 | encoded_body []byte 87 | }{ 88 | { 89 | "no compression", 90 | "/hello", 91 | "", 92 | "", 93 | []byte("hello world"), 94 | }, 95 | { 96 | "gzip", 97 | "/hello", 98 | "gzip", 99 | "gzip", 100 | compress_gzip([]byte("hello world")), 101 | }, 102 | { 103 | "deflate", 104 | "/hello", 105 | "deflate", 106 | "deflate", 107 | compress_deflate([]byte("hello world")), 108 | }, 109 | { 110 | "preference", 111 | "/hello", 112 | "gzip, deflate", 113 | "gzip", 114 | compress_gzip([]byte("hello world")), 115 | }, 116 | { 117 | "precompressed", 118 | "/hello.gz", 119 | "gzip", 120 | "gzip", 121 | compress_gzip([]byte("hello world")), 122 | }, 123 | { 124 | "image", 125 | "/images/face.png", 126 | "gzip", 127 | "", 128 | readfile("../test/images/face.png"), 129 | }, 130 | } 131 | 132 | func compress_gzip(body []byte) []byte { 133 | buf := new(bytes.Buffer) 134 | comp := gzip.NewWriter(buf) 135 | comp.Write(body) 136 | comp.Close() 137 | b := buf.Bytes() 138 | // fmt.Println(b) 139 | return b 140 | } 141 | 142 | func compress_deflate(body []byte) []byte { 143 | buf := new(bytes.Buffer) 144 | comp, err := flate.NewWriter(buf, -1) 145 | if err != nil { 146 | panic(fmt.Sprintf("Error using compress/flate.NewWriter() %v", err)) 147 | } 148 | comp.Write(body) 149 | comp.Close() 150 | b := buf.Bytes() 151 | // fmt.Println(b) 152 | return b 153 | } 154 | 155 | func readfile(path string) []byte { 156 | if data, err := ioutil.ReadFile(path); err == nil { 157 | return data 158 | } else { 159 | panic(fmt.Sprintf("Error reading file %v: %v", path, err)) 160 | } 161 | return nil 162 | } 163 | 164 | func get(p string, accept string) (r *http.Response, err error) { 165 | var conn net.Conn 166 | if conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%v", port())); err == nil { 167 | req, _ := http.NewRequest("GET", fmt.Sprintf("http://%v", path.Join(fmt.Sprintf("localhost:%v/", port()), p)), nil) 168 | req.Header.Set("Accept-Encoding", accept) 169 | req.Write(conn) 170 | buf := bufio.NewReader(conn) 171 | r, err = http.ReadResponse(buf, req) 172 | } 173 | return 174 | } 175 | 176 | func TestCompressionFilter(t *testing.T) { 177 | // select{} 178 | for _, test := range testData { 179 | if res, err := get(test.path, test.accept); err == nil { 180 | bodyBuf := new(bytes.Buffer) 181 | io.Copy(bodyBuf, res.Body) 182 | body := bodyBuf.Bytes() 183 | if enc := res.Header.Get("Content-Encoding"); enc != test.encoding { 184 | t.Errorf("%v Header mismatch. Expecting: %v Got: %v", test.name, test.encoding, enc) 185 | } 186 | if !bytes.Equal(body, test.encoded_body) { 187 | t.Errorf("%v Body mismatch.\n\tExpecting:\n\t%v\n\tGot:\n\t%v", test.name, test.encoded_body, body) 188 | } 189 | } else { 190 | t.Errorf("%v HTTP Error %v", test.name, err) 191 | } 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /continue.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | type continueReader struct { 8 | req *Request 9 | r io.ReadCloser 10 | opened bool 11 | } 12 | 13 | var _ io.ReadCloser = new(continueReader) 14 | 15 | func (r *continueReader) Read(p []byte) (int, error) { 16 | // sent 100 continue the first time we try to read the body 17 | if !r.opened { 18 | resp := SimpleResponse(r.req.HttpRequest, 100, nil, "") 19 | if err := resp.Write(r.req.Connection); err != nil { 20 | return 0, err 21 | } 22 | r.req = nil 23 | r.opened = true 24 | } 25 | return r.r.Read(p) 26 | } 27 | 28 | func (r *continueReader) Close() error { 29 | r.req = nil 30 | return r.r.Close() 31 | } 32 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package falcore is a framework for constructing high performance, modular HTTP servers. 2 | // For more information, see README.md 3 | package falcore 4 | -------------------------------------------------------------------------------- /etag/etag.go: -------------------------------------------------------------------------------- 1 | package etag 2 | 3 | import ( 4 | "github.com/ngmoco/falcore" 5 | "net/http" 6 | ) 7 | 8 | // falcore/etag.Filter is a falcore.ResponseFilter that matches 9 | // the response's Etag header against the request's If-None-Match 10 | // header. If they match, the filter will return a '304 Not Modifed' 11 | // status and no body. 12 | // 13 | // Ideally, Etag filtering is performed as soon as possible as 14 | // you may be able to skip generating the response body at all. 15 | // Even as a last step, you will see a significant benefit if 16 | // clients are well behaved. 17 | // 18 | type Filter struct { 19 | } 20 | 21 | func (f *Filter) FilterResponse(request *falcore.Request, res *http.Response) { 22 | request.CurrentStage.Status = 1 // Skipped (default) 23 | if if_none_match := request.HttpRequest.Header.Get("If-None-Match"); if_none_match != "" { 24 | if res.StatusCode == 200 && res.Header.Get("Etag") == if_none_match { 25 | res.StatusCode = 304 26 | res.Status = "304 Not Modified" 27 | res.Body.Close() 28 | res.Body = nil 29 | res.ContentLength = 0 30 | request.CurrentStage.Status = 0 // Success 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /etag/etag_test.go: -------------------------------------------------------------------------------- 1 | package etag 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "github.com/ngmoco/falcore" 8 | "io" 9 | "net" 10 | "net/http" 11 | "path" 12 | "testing" 13 | "time" 14 | ) 15 | 16 | var srv *falcore.Server 17 | 18 | func init() { 19 | go func() { 20 | // falcore setup 21 | pipeline := falcore.NewPipeline() 22 | pipeline.Upstream.PushBack(falcore.NewRequestFilter(func(req *falcore.Request) *http.Response { 23 | for _, data := range serverData { 24 | if data.path == req.HttpRequest.URL.Path { 25 | header := make(http.Header) 26 | header.Set("Etag", data.etag) 27 | return falcore.SimpleResponse(req.HttpRequest, data.status, header, string(data.body)) 28 | } 29 | } 30 | return falcore.SimpleResponse(req.HttpRequest, 404, nil, "Not Found") 31 | })) 32 | 33 | pipeline.Downstream.PushBack(new(Filter)) 34 | 35 | srv = falcore.NewServer(0, pipeline) 36 | if err := srv.ListenAndServe(); err != nil { 37 | panic("Could not start falcore") 38 | } 39 | }() 40 | } 41 | 42 | func port() int { 43 | for srv.Port() == 0 { 44 | time.Sleep(1e7) 45 | } 46 | return srv.Port() 47 | } 48 | 49 | var serverData = []struct { 50 | path string 51 | status int 52 | etag string 53 | body []byte 54 | }{ 55 | { 56 | "/hello", 57 | 200, 58 | "abc123", 59 | []byte("hello world"), 60 | }, 61 | { 62 | "/pre", 63 | 304, 64 | "abc123", 65 | []byte{}, 66 | }, 67 | } 68 | 69 | var testData = []struct { 70 | name string 71 | // input 72 | path string 73 | etag string 74 | // output 75 | status int 76 | body []byte 77 | }{ 78 | { 79 | "no etag", 80 | "/hello", 81 | "", 82 | 200, 83 | []byte("hello world"), 84 | }, 85 | { 86 | "match", 87 | "/hello", 88 | "abc123", 89 | 304, 90 | []byte{}, 91 | }, 92 | { 93 | "pre-filtered", 94 | "/pre", 95 | "abc123", 96 | 304, 97 | []byte{}, 98 | }, 99 | } 100 | 101 | func get(p string, etag string) (r *http.Response, err error) { 102 | var conn net.Conn 103 | if conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%v", port())); err == nil { 104 | req, _ := http.NewRequest("GET", fmt.Sprintf("http://%v", path.Join(fmt.Sprintf("localhost:%v/", port()), p)), nil) 105 | req.Header.Set("If-None-Match", etag) 106 | req.Write(conn) 107 | buf := bufio.NewReader(conn) 108 | r, err = http.ReadResponse(buf, req) 109 | } 110 | return 111 | } 112 | 113 | func TestEtagFilter(t *testing.T) { 114 | // select{} 115 | for _, test := range testData { 116 | if res, err := get(test.path, test.etag); err == nil { 117 | bodyBuf := new(bytes.Buffer) 118 | io.Copy(bodyBuf, res.Body) 119 | body := bodyBuf.Bytes() 120 | if st := res.StatusCode; st != test.status { 121 | t.Errorf("%v StatusCode mismatch. Expecting: %v Got: %v", test.name, test.status, st) 122 | } 123 | if !bytes.Equal(body, test.body) { 124 | t.Errorf("%v Body mismatch.\n\tExpecting:\n\t%v\n\tGot:\n\t%v", test.name, test.body, body) 125 | } 126 | } else { 127 | t.Errorf("%v HTTP Error %v", test.name, err) 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /examples/basic_file_server/server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/ngmoco/falcore" 7 | "github.com/ngmoco/falcore/compression" 8 | "github.com/ngmoco/falcore/static_file" 9 | "net/http" 10 | ) 11 | 12 | // Command line options 13 | var ( 14 | port = flag.Int("port", 8000, "the port to listen on") 15 | path = flag.String("base", "./test", "the path to serve files from") 16 | ) 17 | 18 | func main() { 19 | // parse command line options 20 | flag.Parse() 21 | 22 | // setup pipeline 23 | pipeline := falcore.NewPipeline() 24 | 25 | // upstream filters 26 | 27 | // Serve index.html for root requests 28 | pipeline.Upstream.PushBack(falcore.NewRequestFilter(func(req *falcore.Request) *http.Response { 29 | if req.HttpRequest.URL.Path == "/" { 30 | req.HttpRequest.URL.Path = "/index.html" 31 | } 32 | return nil 33 | })) 34 | // Serve files 35 | pipeline.Upstream.PushBack(&static_file.Filter{ 36 | BasePath: *path, 37 | }) 38 | 39 | // downstream 40 | pipeline.Downstream.PushBack(compression.NewFilter(nil)) 41 | 42 | // setup server 43 | server := falcore.NewServer(*port, pipeline) 44 | 45 | // start the server 46 | // this is normally blocking forever unless you send lifecycle commands 47 | if err := server.ListenAndServe(); err != nil { 48 | fmt.Println("Could not start server:", err) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /examples/blog_example/blog_example.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/ngmoco/falcore" 6 | "math/rand" 7 | "net/http" 8 | "time" 9 | ) 10 | 11 | func main() { 12 | // create pipeline 13 | pipeline := falcore.NewPipeline() 14 | 15 | // add upstream pipeline stages 16 | var filter1 delayFilter 17 | pipeline.Upstream.PushBack(filter1) 18 | var filter2 helloFilter 19 | pipeline.Upstream.PushBack(filter2) 20 | 21 | // add request done callback stage 22 | pipeline.RequestDoneCallback = reqCB 23 | 24 | // create server on port 8000 25 | server := falcore.NewServer(8000, pipeline) 26 | 27 | // start the server 28 | // this is normally blocking forever unless you send lifecycle commands 29 | if err := server.ListenAndServe(); err != nil { 30 | fmt.Println("Could not start server:", err) 31 | } 32 | } 33 | 34 | // Example filter to show timing features 35 | type delayFilter int 36 | 37 | func (f delayFilter) FilterRequest(req *falcore.Request) *http.Response { 38 | status := rand.Intn(2) // random status 0 or 1 39 | if status == 0 { 40 | time.Sleep(time.Duration(rand.Int63n(100e6))) // random sleep between 0 and 100 ms 41 | } 42 | req.CurrentStage.Status = byte(status) // set the status to produce a unique signature 43 | return nil 44 | } 45 | 46 | // Example filter that returns a Response 47 | type helloFilter int 48 | 49 | func (f helloFilter) FilterRequest(req *falcore.Request) *http.Response { 50 | return falcore.SimpleResponse(req.HttpRequest, 200, nil, "hello world!\n") 51 | } 52 | 53 | var reqCB = falcore.NewRequestFilter(func(req *falcore.Request) *http.Response { 54 | req.Trace() // Prints detailed stats about the request to the log 55 | return nil 56 | }) 57 | -------------------------------------------------------------------------------- /examples/hello_world/hello_world.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/ngmoco/falcore" 7 | "net/http" 8 | ) 9 | 10 | // Command line options 11 | var ( 12 | port = flag.Int("port", 8000, "the port to listen on") 13 | ) 14 | 15 | func main() { 16 | // parse command line options 17 | flag.Parse() 18 | 19 | // setup pipeline 20 | pipeline := falcore.NewPipeline() 21 | 22 | // upstream 23 | pipeline.Upstream.PushBack(helloFilter) 24 | 25 | // setup server 26 | server := falcore.NewServer(*port, pipeline) 27 | 28 | // start the server 29 | // this is normally blocking forever unless you send lifecycle commands 30 | if err := server.ListenAndServe(); err != nil { 31 | fmt.Println("Could not start server:", err) 32 | } 33 | } 34 | 35 | var helloFilter = falcore.NewRequestFilter(func(req *falcore.Request) *http.Response { 36 | return falcore.SimpleResponse(req.HttpRequest, 200, nil, "hello world!") 37 | }) 38 | -------------------------------------------------------------------------------- /examples/hot_restart/hot_restart.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/ngmoco/falcore" 7 | "net/http" 8 | "os" 9 | "os/signal" 10 | "runtime" 11 | "strconv" 12 | "strings" 13 | "syscall" 14 | ) 15 | 16 | // very simple request filter 17 | func Filter(request *falcore.Request) *http.Response { 18 | return falcore.SimpleResponse(request.HttpRequest, 200, nil, "OK\n") 19 | } 20 | 21 | // flag to accept a socket file descriptor 22 | var socketFd = flag.Int("socket", -1, "Socket file descriptor") 23 | 24 | func main() { 25 | pid := syscall.Getpid() 26 | flag.Parse() 27 | 28 | // create the pipeline 29 | pipeline := falcore.NewPipeline() 30 | pipeline.Upstream.PushBack(falcore.NewRequestFilter(Filter)) 31 | 32 | // create the server with the pipeline 33 | srv := falcore.NewServer(8090, pipeline) 34 | 35 | // if passed the socket file descriptor, setup the listener that way 36 | // if you don't have it, the default is to create the socket listener 37 | // with the data passed to falcore.NewServer above (happens in ListenAndServer()) 38 | if *socketFd != -1 { 39 | // I know I'm a child process if I get here so I can signal the parent when I'm ready to take over 40 | go childReady(srv) 41 | fmt.Printf("%v Got socket FD: %v\n", pid, *socketFd) 42 | srv.FdListen(*socketFd) 43 | } 44 | 45 | // using signals to manage the restart lifecycle 46 | go handleSignals(srv) 47 | 48 | // start the server 49 | // this is normally blocking forever unless you send lifecycle commands 50 | fmt.Printf("%v Starting Listener on 8090\n", pid) 51 | if err := srv.ListenAndServe(); err != nil { 52 | fmt.Printf("%v Could not start server: %v", pid, err) 53 | } 54 | fmt.Printf("%v Exiting now\n", pid) 55 | } 56 | 57 | // blocks on the server ready and when ready, it sends 58 | // a signal to the parent so that it knows it cna now exit 59 | func childReady(srv *falcore.Server) { 60 | pid := syscall.Getpid() 61 | // wait for the ready signal 62 | <-srv.AcceptReady 63 | // grab the parent and send a signal that the child is ready 64 | parent := syscall.Getppid() 65 | fmt.Printf("%v Kill parent %v with SIGUSR1\n", pid, parent) 66 | syscall.Kill(parent, syscall.SIGUSR1) 67 | } 68 | 69 | // setup and fork/exec myself. Make sure to keep open important FD's that won't get re-created by the child 70 | // specifically, std* and your listen socket 71 | func forker(srv *falcore.Server) (pid int, err error) { 72 | var socket string 73 | // At version 1.0.3 the socket FD behavior changed and the fork socket is always 3 74 | // 0 = stdin, 1 = stdout, 2 = stderr, 3 = acceptor socket 75 | // This is because the ForkExec dups all the saved FDs down to 76 | // start at 0. This is also why you MUST include 0,1,2 in the 77 | // attr.Files 78 | if goVersion103OrAbove() { 79 | socket = "3" 80 | } else { 81 | socket = fmt.Sprintf("%v", srv.SocketFd()) 82 | } 83 | fmt.Printf("Forking now with socket: %v\n", socket) 84 | mypath := os.Args[0] 85 | args := []string{mypath, "-socket", socket} 86 | attr := new(syscall.ProcAttr) 87 | attr.Files = append([]uintptr(nil), 0, 1, 2, uintptr(srv.SocketFd())) 88 | pid, err = syscall.ForkExec(mypath, args, attr) 89 | return 90 | } 91 | 92 | func goVersion103OrAbove() bool { 93 | ver := strings.Split(runtime.Version(), ".") 94 | // Go versioning is weird so this only works for common go1 cases: 95 | // current as of patch: 96 | // go1.0.3 13678:2d8bc3c94ecb : true 97 | // go1.0.2 13278:5e806355a9e1 : false 98 | // go1.0.1 12994:2ccfd4b451d3 : false 99 | // go1 12872:920e9d1ffd1f : false 100 | // go1.1+/go2+ : true 101 | // release* : true (this is possibly broken) 102 | // weekly* : true (this is possibly broken) 103 | // tip : true 104 | if len(ver) > 0 && strings.Index(ver[0], "go") == 0 { 105 | if ver[0] == "go1" && len(ver) == 1 { 106 | // just go1 107 | return false 108 | } else if ver[0] == "go1" && len(ver) == 3 && ver[1] == "0" { 109 | if patchVer, _ := strconv.ParseInt(ver[2], 10, 64); patchVer < 3 { 110 | return false 111 | } 112 | } 113 | } 114 | return true 115 | } 116 | 117 | // Handle lifecycle events 118 | func handleSignals(srv *falcore.Server) { 119 | var sig os.Signal 120 | var sigChan = make(chan os.Signal) 121 | signal.Notify(sigChan, syscall.SIGHUP, syscall.SIGUSR1, syscall.SIGINT, syscall.SIGTERM, syscall.SIGTSTP) 122 | pid := syscall.Getpid() 123 | for { 124 | sig = <-sigChan 125 | switch sig { 126 | case syscall.SIGHUP: 127 | // send this to the paraent process to initiate the restart 128 | fmt.Println(pid, "Received SIGHUP. forking.") 129 | cpid, err := forker(srv) 130 | fmt.Println(pid, "Forked pid:", cpid, "errno:", err) 131 | case syscall.SIGUSR1: 132 | // child sends this back to the parent when it's ready to Accept 133 | fmt.Println(pid, "Received SIGUSR1. Stopping accept.") 134 | srv.StopAccepting() 135 | case syscall.SIGINT: 136 | fmt.Println(pid, "Received SIGINT. Shutting down.") 137 | os.Exit(0) 138 | case syscall.SIGTERM: 139 | fmt.Println(pid, "Received SIGTERM. Terminating.") 140 | os.Exit(0) 141 | case syscall.SIGTSTP: 142 | fmt.Println(pid, "Received SIGTSTP. Stopping.") 143 | syscall.Kill(pid, syscall.SIGSTOP) 144 | default: 145 | fmt.Println(pid, "Received", sig, ": ignoring") 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /filter.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | // Filter incomming requests and optionally return a response or nil. 8 | // Filters are chained together into a flow (the Pipeline) which will terminate 9 | // if the Filter returns a response. 10 | type RequestFilter interface { 11 | FilterRequest(req *Request) *http.Response 12 | } 13 | 14 | // Helper to create a Filter by just passing in a func 15 | // filter = NewRequestFilter(func(req *Request) *http.Response { 16 | // req.Headers.Add("X-Falcore", "is_cool") 17 | // return 18 | // }) 19 | func NewRequestFilter(f func(req *Request) *http.Response) RequestFilter { 20 | rf := new(genericRequestFilter) 21 | rf.f = f 22 | return rf 23 | } 24 | 25 | type genericRequestFilter struct { 26 | f func(req *Request) *http.Response 27 | } 28 | 29 | func (f *genericRequestFilter) FilterRequest(req *Request) *http.Response { 30 | return f.f(req) 31 | } 32 | 33 | // Filter outgoing responses. This can be used to modify the response 34 | // before it is sent. Modifying the request at this point will have no 35 | // effect. 36 | type ResponseFilter interface { 37 | FilterResponse(req *Request, res *http.Response) 38 | } 39 | 40 | // Helper to create a Filter by just passing in a func 41 | // filter = NewResponseFilter(func(req *Request, res *http.Response) { 42 | // // some crazy response magic 43 | // return 44 | // }) 45 | func NewResponseFilter(f func(req *Request, res *http.Response)) ResponseFilter { 46 | rf := new(genericResponseFilter) 47 | rf.f = f 48 | return rf 49 | } 50 | 51 | type genericResponseFilter struct { 52 | f func(req *Request, res *http.Response) 53 | } 54 | 55 | func (f *genericResponseFilter) FilterResponse(req *Request, res *http.Response) { 56 | f.f(req, res) 57 | } 58 | -------------------------------------------------------------------------------- /handler_filter.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | ) 8 | 9 | // Implements a RequestFilter using a http.Handler to produce the response 10 | // This will always return a response due to the requirements of the http.Handler 11 | // interface so it should be placed at the end of the Upstream pipeline. 12 | type HandlerFilter struct { 13 | handler http.Handler 14 | } 15 | 16 | func NewHandlerFilter(handler http.Handler) *HandlerFilter { 17 | return &HandlerFilter{handler: handler} 18 | } 19 | 20 | func (h *HandlerFilter) FilterRequest(req *Request) *http.Response { 21 | rw, respc := newPopulateResponseWriter(req.HttpRequest) 22 | // this must be done concurrently so that the HandlerFunc can write the response 23 | // while falcore is copying it to the socket 24 | go func() { 25 | h.handler.ServeHTTP(rw, req.HttpRequest) 26 | rw.finish() 27 | }() 28 | return <-respc 29 | } 30 | 31 | // copied from net/http/filetransport.go 32 | func newPopulateResponseWriter(req *http.Request) (*populateResponse, <-chan *http.Response) { 33 | pr, pw := io.Pipe() 34 | rw := &populateResponse{ 35 | ch: make(chan *http.Response), 36 | pw: pw, 37 | res: &http.Response{ 38 | Proto: "HTTP/1.0", 39 | ProtoMajor: 1, 40 | Header: make(http.Header), 41 | Close: true, 42 | Body: pr, 43 | Request: req, 44 | }, 45 | } 46 | return rw, rw.ch 47 | } 48 | 49 | // populateResponse is a ResponseWriter that populates the *Response 50 | // in res, and writes its body to a pipe connected to the response 51 | // body. Once writes begin or finish() is called, the response is sent 52 | // on ch. 53 | type populateResponse struct { 54 | res *http.Response 55 | ch chan *http.Response 56 | wroteHeader bool 57 | hasContent bool 58 | sentResponse bool 59 | pw *io.PipeWriter 60 | } 61 | 62 | func (pr *populateResponse) finish() { 63 | if !pr.wroteHeader { 64 | pr.WriteHeader(500) 65 | } 66 | if !pr.sentResponse { 67 | pr.sendResponse() 68 | } 69 | pr.pw.Close() 70 | } 71 | 72 | func (pr *populateResponse) sendResponse() { 73 | if pr.sentResponse { 74 | return 75 | } 76 | pr.sentResponse = true 77 | 78 | if pr.hasContent { 79 | pr.res.ContentLength = -1 80 | } 81 | pr.ch <- pr.res 82 | } 83 | 84 | func (pr *populateResponse) Header() http.Header { 85 | return pr.res.Header 86 | } 87 | 88 | func (pr *populateResponse) WriteHeader(code int) { 89 | if pr.wroteHeader { 90 | return 91 | } 92 | pr.wroteHeader = true 93 | 94 | pr.res.StatusCode = code 95 | pr.res.Status = fmt.Sprintf("%d %s", code, http.StatusText(code)) 96 | } 97 | 98 | func (pr *populateResponse) Write(p []byte) (n int, err error) { 99 | if !pr.wroteHeader { 100 | pr.WriteHeader(http.StatusOK) 101 | } 102 | pr.hasContent = true 103 | if !pr.sentResponse { 104 | pr.sendResponse() 105 | } 106 | return pr.pw.Write(p) 107 | } 108 | -------------------------------------------------------------------------------- /handler_filter_test.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "testing" 8 | ) 9 | 10 | func TestHandlerFilter(t *testing.T) { 11 | reply := "Hello, World" 12 | handler := func(w http.ResponseWriter, r *http.Request) { 13 | fmt.Fprintf(w, reply) 14 | } 15 | 16 | hff := NewHandlerFilter(http.HandlerFunc(handler)) 17 | 18 | tmp, _ := http.NewRequest("GET", "/hello", nil) 19 | _, res := TestWithRequest(tmp, hff, nil) 20 | 21 | if res == nil { 22 | t.Errorf("Response is nil") 23 | } 24 | 25 | if replyGot, err := ioutil.ReadAll(res.Body); err != nil { 26 | t.Errorf("Error reading body: %v", err) 27 | } else if string(replyGot) != reply { 28 | t.Errorf("Expected body does not match") 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | "time" 7 | ) 8 | 9 | // I really want to use log4go... but i need to support falling back to standard (shitty) logger :( 10 | // I suggest using go-timber for the real logger 11 | type Logger interface { 12 | // Matches the log4go interface 13 | Finest(arg0 interface{}, args ...interface{}) 14 | Fine(arg0 interface{}, args ...interface{}) 15 | Debug(arg0 interface{}, args ...interface{}) 16 | Trace(arg0 interface{}, args ...interface{}) 17 | Info(arg0 interface{}, args ...interface{}) 18 | Warn(arg0 interface{}, args ...interface{}) error 19 | Error(arg0 interface{}, args ...interface{}) error 20 | Critical(arg0 interface{}, args ...interface{}) error 21 | } 22 | 23 | var logger Logger = NewStdLibLogger() 24 | 25 | func SetLogger(newLogger Logger) { 26 | logger = newLogger 27 | } 28 | 29 | // Helper for calculating times 30 | func TimeDiff(startTime time.Time, endTime time.Time) float32 { 31 | return float32(endTime.Sub(startTime)) / 1.0e9 32 | } 33 | 34 | // Global Logging 35 | func Finest(arg0 interface{}, args ...interface{}) { 36 | logger.Finest(arg0, args...) 37 | } 38 | 39 | func Fine(arg0 interface{}, args ...interface{}) { 40 | logger.Fine(arg0, args...) 41 | } 42 | 43 | func Debug(arg0 interface{}, args ...interface{}) { 44 | logger.Debug(arg0, args...) 45 | } 46 | 47 | func Trace(arg0 interface{}, args ...interface{}) { 48 | logger.Trace(arg0, args...) 49 | } 50 | 51 | func Info(arg0 interface{}, args ...interface{}) { 52 | logger.Info(arg0, args...) 53 | } 54 | 55 | func Warn(arg0 interface{}, args ...interface{}) error { 56 | return logger.Warn(arg0, args...) 57 | } 58 | 59 | func Error(arg0 interface{}, args ...interface{}) error { 60 | return logger.Error(arg0, args...) 61 | } 62 | 63 | func Critical(arg0 interface{}, args ...interface{}) error { 64 | return logger.Critical(arg0, args...) 65 | } 66 | 67 | // This is a simple Logger implementation that 68 | // uses the go log package for output. It's not 69 | // really meant for production use since it isn't 70 | // very configurable. It is a sane default alternative 71 | // that allows us to not have any external dependencies. 72 | // Use timber or log4go as a real alternative. 73 | type StdLibLogger struct{} 74 | 75 | func NewStdLibLogger() Logger { 76 | return new(StdLibLogger) 77 | } 78 | 79 | type level int 80 | 81 | const ( 82 | FINEST level = iota 83 | FINE 84 | DEBUG 85 | TRACE 86 | INFO 87 | WARNING 88 | ERROR 89 | CRITICAL 90 | ) 91 | 92 | var ( 93 | levelStrings = [...]string{"[FNST]", "[FINE]", "[DEBG]", "[TRAC]", "[INFO]", "[WARN]", "[EROR]", "[CRIT]"} 94 | ) 95 | 96 | func (fl StdLibLogger) Finest(arg0 interface{}, args ...interface{}) { 97 | fl.Log(FINEST, arg0, args...) 98 | } 99 | 100 | func (fl StdLibLogger) Fine(arg0 interface{}, args ...interface{}) { 101 | fl.Log(FINE, arg0, args...) 102 | } 103 | 104 | func (fl StdLibLogger) Debug(arg0 interface{}, args ...interface{}) { 105 | fl.Log(DEBUG, arg0, args...) 106 | } 107 | 108 | func (fl StdLibLogger) Trace(arg0 interface{}, args ...interface{}) { 109 | fl.Log(TRACE, arg0, args...) 110 | } 111 | 112 | func (fl StdLibLogger) Info(arg0 interface{}, args ...interface{}) { 113 | fl.Log(INFO, arg0, args...) 114 | } 115 | 116 | func (fl StdLibLogger) Warn(arg0 interface{}, args ...interface{}) error { 117 | return fl.Log(WARNING, arg0, args...) 118 | } 119 | 120 | func (fl StdLibLogger) Error(arg0 interface{}, args ...interface{}) error { 121 | return fl.Log(ERROR, arg0, args...) 122 | } 123 | 124 | func (fl StdLibLogger) Critical(arg0 interface{}, args ...interface{}) error { 125 | return fl.Log(CRITICAL, arg0, args...) 126 | } 127 | 128 | func (fl StdLibLogger) Log(lvl level, arg0 interface{}, args ...interface{}) (e error) { 129 | defer func() { 130 | if x := recover(); x != nil { 131 | var ok bool 132 | if e, ok = x.(error); ok { 133 | return 134 | } 135 | e = errors.New("Um... barf") 136 | } 137 | }() 138 | switch first := arg0.(type) { 139 | case string: 140 | // Use the string as a format string 141 | argsNew := append([]interface{}{levelStrings[lvl]}, args...) 142 | log.Printf("%s "+first, argsNew...) 143 | case func() string: 144 | // Log the closure (no other arguments used) 145 | argsNew := append([]interface{}{levelStrings[lvl]}, first()) 146 | log.Println(argsNew...) 147 | default: 148 | // Build a format string so that it will be similar to Sprint 149 | argsNew := append([]interface{}{levelStrings[lvl]}, args...) 150 | log.Println(argsNew...) 151 | } 152 | return nil 153 | } 154 | -------------------------------------------------------------------------------- /pipeline.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "container/list" 5 | "log" 6 | "net/http" 7 | "reflect" 8 | ) 9 | 10 | // Pipelines have an upstream and downstream list of filters. 11 | // A request is passed through the upstream items in order UNTIL 12 | // a Response is returned. Once a request is returned, it is passed 13 | // through ALL ResponseFilters in the Downstream list, in order. 14 | // 15 | // If no response is generated by any Filters a default 404 response is 16 | // returned. 17 | // 18 | // The RequestDoneCallback (if set) will be called after the request 19 | // has completed. The finished request object will be passed to 20 | // the FilterRequest method for inspection. Changes to the request 21 | // will have no effect and the return value is ignored. 22 | // 23 | // 24 | type Pipeline struct { 25 | Upstream *list.List 26 | Downstream *list.List 27 | RequestDoneCallback RequestFilter 28 | } 29 | 30 | func NewPipeline() (l *Pipeline) { 31 | l = new(Pipeline) 32 | l.Upstream = list.New() 33 | l.Downstream = list.New() 34 | return 35 | } 36 | 37 | // Pipelines are also RequestFilters... wacky eh? 38 | // Be careful though because a Pipeline will always returns a 39 | // response so no Filters after a Pipeline filter will be run. 40 | func (p *Pipeline) FilterRequest(req *Request) *http.Response { 41 | return p.execute(req) 42 | } 43 | 44 | func (p *Pipeline) execute(req *Request) (res *http.Response) { 45 | for e := p.Upstream.Front(); e != nil && res == nil; e = e.Next() { 46 | switch filter := e.Value.(type) { 47 | case Router: 48 | t := reflect.TypeOf(filter) 49 | req.startPipelineStage(t.String()) 50 | pipe := filter.SelectPipeline(req) 51 | req.finishPipelineStage() 52 | if pipe != nil { 53 | res = p.execFilter(req, pipe) 54 | if res != nil { 55 | break 56 | } 57 | } 58 | case RequestFilter: 59 | res = p.execFilter(req, filter) 60 | if res != nil { 61 | break 62 | } 63 | default: 64 | log.Printf("%v is not a RequestFilter\n", e.Value) 65 | break 66 | } 67 | } 68 | 69 | if res == nil { 70 | // Error: No response was generated 71 | res = SimpleResponse(req.HttpRequest, 404, nil, "Not found\n") 72 | } 73 | 74 | p.down(req, res) 75 | return 76 | } 77 | 78 | func (p *Pipeline) execFilter(req *Request, filter RequestFilter) *http.Response { 79 | if _, skipTracking := filter.(*Pipeline); !skipTracking { 80 | t := reflect.TypeOf(filter) 81 | req.startPipelineStage(t.String()) 82 | defer req.finishPipelineStage() 83 | } 84 | return filter.FilterRequest(req) 85 | } 86 | 87 | func (p *Pipeline) down(req *Request, res *http.Response) { 88 | for e := p.Downstream.Front(); e != nil; e = e.Next() { 89 | if filter, ok := e.Value.(ResponseFilter); ok { 90 | t := reflect.TypeOf(filter) 91 | req.startPipelineStage(t.String()) 92 | filter.FilterResponse(req, res) 93 | req.finishPipelineStage() 94 | } else { 95 | // TODO 96 | break 97 | } 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /pipeline_test.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "bytes" 5 | "container/list" 6 | "net/http" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func validGetRequest() *Request { 12 | tmp, _ := http.NewRequest("GET", "/hello", bytes.NewBuffer(make([]byte, 0))) 13 | return newRequest(tmp, nil, time.Now()) 14 | } 15 | 16 | var stageTrack *list.List 17 | 18 | func doStageTrack() { 19 | i := 0 20 | if stageTrack.Len() > 0 { 21 | i = stageTrack.Back().Value.(int) 22 | } 23 | stageTrack.PushBack(i + 1) 24 | } 25 | 26 | func sumFilter(req *Request) *http.Response { 27 | doStageTrack() 28 | return nil 29 | } 30 | 31 | func sumResponseFilter(*Request, *http.Response) { 32 | doStageTrack() 33 | } 34 | 35 | func successFilter(req *Request) *http.Response { 36 | doStageTrack() 37 | return SimpleResponse(req.HttpRequest, 200, nil, "OK") 38 | } 39 | 40 | func TestPipelineNoResponse(t *testing.T) { 41 | p := NewPipeline() 42 | 43 | stageTrack = list.New() 44 | f := NewRequestFilter(sumFilter) 45 | 46 | p.Upstream.PushBack(f) 47 | p.Upstream.PushBack(f) 48 | p.Upstream.PushBack(f) 49 | 50 | //response := new(http.Response) 51 | response := p.execute(validGetRequest()) 52 | 53 | if stageTrack.Len() != 3 { 54 | t.Fatalf("Wrong number of stages executed: %v expected %v", stageTrack.Len(), 3) 55 | } 56 | if sum, ok := stageTrack.Back().Value.(int); ok { 57 | if sum != 3 { 58 | t.Errorf("Pipeline stages did not complete %v expected %v", sum, 3) 59 | } 60 | } 61 | if response.StatusCode != 404 { 62 | t.Errorf("Pipeline response code wrong: %v expected %v", response.StatusCode, 404) 63 | } 64 | } 65 | 66 | func TestPipelineOKResponse(t *testing.T) { 67 | p := NewPipeline() 68 | 69 | stageTrack = list.New() 70 | f := NewRequestFilter(sumFilter) 71 | 72 | p.Upstream.PushBack(f) 73 | p.Upstream.PushBack(f) 74 | p.Upstream.PushBack(NewRequestFilter(successFilter)) 75 | p.Upstream.PushBack(f) 76 | 77 | response := p.execute(validGetRequest()) 78 | 79 | if stageTrack.Len() != 3 { 80 | t.Fatalf("Wrong number of stages executed: %v expected %v", stageTrack.Len(), 3) 81 | } 82 | if sum, ok := stageTrack.Back().Value.(int); ok { 83 | if sum != 3 { 84 | t.Errorf("Pipeline stages did not complete %v expected %v", sum, 3) 85 | } 86 | } 87 | if response.StatusCode != 200 { 88 | t.Errorf("Pipeline response code wrong: %v expected %v", response.StatusCode, 200) 89 | } 90 | } 91 | 92 | func TestPipelineResponseFilter(t *testing.T) { 93 | p := NewPipeline() 94 | 95 | stageTrack = list.New() 96 | f := NewRequestFilter(sumFilter) 97 | 98 | p.Upstream.PushBack(f) 99 | p.Upstream.PushBack(NewRequestFilter(successFilter)) 100 | p.Upstream.PushBack(f) 101 | p.Downstream.PushBack(NewResponseFilter(sumResponseFilter)) 102 | p.Downstream.PushBack(NewResponseFilter(sumResponseFilter)) 103 | 104 | //response := new(http.Response) 105 | req := validGetRequest() 106 | response := p.execute(req) 107 | 108 | stages := 4 109 | // check basic execution 110 | if stageTrack.Len() != stages { 111 | t.Fatalf("Wrong number of stages executed: %v expected %v", stageTrack.Len(), stages) 112 | } 113 | if sum, ok := stageTrack.Back().Value.(int); ok { 114 | if sum != stages { 115 | t.Errorf("Pipeline stages did not complete %v expected %v", sum, stages) 116 | } 117 | } 118 | // check status 119 | if response.StatusCode != 200 { 120 | t.Errorf("Pipeline response code wrong: %v expected %v", response.StatusCode, 200) 121 | } 122 | req.finishRequest() 123 | if req.Signature() != "F7F5165F" { 124 | t.Errorf("Signature failed: %v expected %v", req.Signature(), "F7F5165F") 125 | } 126 | if req.PipelineStageStats.Len() != stages { 127 | t.Errorf("PipelineStageStats incomplete: %v expected %v", req.PipelineStageStats.Len(), stages) 128 | } 129 | //req.Trace() 130 | 131 | } 132 | 133 | func TestPipelineStatsChecksum(t *testing.T) { 134 | p := NewPipeline() 135 | 136 | stageTrack = list.New() 137 | f := NewRequestFilter(sumFilter) 138 | 139 | p.Upstream.PushBack(f) 140 | p.Upstream.PushBack(NewRequestFilter(func(req *Request) *http.Response { 141 | doStageTrack() 142 | req.CurrentStage.Status = 1 143 | return nil 144 | })) 145 | p.Upstream.PushBack(NewRequestFilter(successFilter)) 146 | p.Downstream.PushBack(NewResponseFilter(sumResponseFilter)) 147 | p.Downstream.PushBack(NewResponseFilter(sumResponseFilter)) 148 | 149 | //response := new(http.Response) 150 | req := validGetRequest() 151 | response := p.execute(req) 152 | 153 | stages := 5 154 | // check basic execution 155 | if stageTrack.Len() != stages { 156 | t.Fatalf("Wrong number of stages executed: %v expected %v", stageTrack.Len(), stages) 157 | } 158 | if sum, ok := stageTrack.Back().Value.(int); ok { 159 | if sum != stages { 160 | t.Errorf("Pipeline stages did not complete %v expected %v", sum, stages) 161 | } 162 | } 163 | // check status 164 | if response.StatusCode != 200 { 165 | t.Errorf("Pipeline response code wrong: %v expected %v", response.StatusCode, 200) 166 | } 167 | req.finishRequest() 168 | if req.Signature() != "CA843113" { 169 | t.Errorf("Signature failed: %v expected %v", req.Signature(), "CA843113") 170 | } 171 | if req.PipelineStageStats.Len() != stages { 172 | t.Errorf("PipelineStageStats incomplete: %v expected %v", req.PipelineStageStats.Len(), stages) 173 | } 174 | //req.Trace() 175 | 176 | } 177 | -------------------------------------------------------------------------------- /request.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "container/list" 5 | "fmt" 6 | "hash" 7 | "hash/crc32" 8 | "math/rand" 9 | "net" 10 | "net/http" 11 | "reflect" 12 | "time" 13 | ) 14 | 15 | // Request wrapper 16 | // 17 | // The request is wrapped so that useful information can be kept 18 | // with the request as it moves through the pipeline. 19 | // 20 | // A pointer is kept to the originating Connection. 21 | // 22 | // There is a unique ID assigned to each request. This ID is not 23 | // globally unique to keep it shorter for logging purposes. It is 24 | // possible to have duplicates though very unlikely over the period 25 | // of a day or so. It is a good idea to log the ID in any custom 26 | // log statements so that individual requests can easily be grepped 27 | // from busy log files. 28 | // 29 | // Falcore collects performance statistics on every stage of the 30 | // pipeline. The stats for the request are kept in PipelineStageStats. 31 | // This structure will only be complete in the Request passed to the 32 | // pipeline RequestDoneCallback. Overhead will only be available in 33 | // the RequestDoneCallback and it's the difference between the total 34 | // request time and the sums of the stage times. It will include things 35 | // like pipeline iteration and the stat collection itself. 36 | // 37 | // See falcore.PipelineStageStat docs for more info. 38 | // 39 | // The Signature is also a cool feature. See the 40 | type Request struct { 41 | ID string 42 | StartTime time.Time 43 | EndTime time.Time 44 | HttpRequest *http.Request 45 | Connection net.Conn 46 | RemoteAddr *net.TCPAddr 47 | PipelineStageStats *list.List 48 | CurrentStage *PipelineStageStat 49 | pipelineHash hash.Hash32 50 | piplineTot time.Duration 51 | Overhead time.Duration 52 | Context map[string]interface{} 53 | } 54 | 55 | // Used internally to create and initialize a new request. 56 | func newRequest(request *http.Request, conn net.Conn, startTime time.Time) *Request { 57 | fReq := new(Request) 58 | fReq.Context = make(map[string]interface{}) 59 | fReq.HttpRequest = request 60 | fReq.StartTime = startTime 61 | fReq.Connection = conn 62 | if conn != nil { 63 | fReq.RemoteAddr = conn.RemoteAddr().(*net.TCPAddr) 64 | } 65 | // create a semi-unique id to track a connection in the logs 66 | // ID is the least significant decimal digits of time with some randomization 67 | // the last 3 zeros of time.Nanoseconds appear to always be zero 68 | var ut = fReq.StartTime.UnixNano() 69 | fReq.ID = fmt.Sprintf("%010x", (ut-(ut-(ut%1e12)))+int64(rand.Intn(999))) 70 | fReq.PipelineStageStats = list.New() 71 | fReq.pipelineHash = crc32.NewIEEE() 72 | 73 | // Support for 100-continue requests 74 | // http.Server (and presumably google app engine) already handle this 75 | // case. So we don't need to do anything if we don't own the 76 | // connection. 77 | if conn != nil && request.Header.Get("Expect") == "100-continue" { 78 | request.Body = &continueReader{req: fReq, r: request.Body} 79 | } 80 | 81 | return fReq 82 | } 83 | 84 | // Returns a completed falcore.Request and response after running the single filter stage 85 | // The PipelineStageStats is completed in the returned Request 86 | // The falcore.Request.Connection and falcore.Request.RemoteAddr are nil 87 | func TestWithRequest(request *http.Request, filter RequestFilter, context map[string]interface{}) (*Request, *http.Response) { 88 | r := newRequest(request, nil, time.Now()) 89 | if context == nil { 90 | context = make(map[string]interface{}) 91 | } 92 | r.Context = context 93 | t := reflect.TypeOf(filter) 94 | r.startPipelineStage(t.String()) 95 | res := filter.FilterRequest(r) 96 | r.finishPipelineStage() 97 | r.finishRequest() 98 | return r, res 99 | } 100 | 101 | // Starts a new pipeline stage and makes it the CurrentStage. 102 | func (fReq *Request) startPipelineStage(name string) { 103 | fReq.CurrentStage = NewPiplineStage(name) 104 | fReq.PipelineStageStats.PushBack(fReq.CurrentStage) 105 | } 106 | 107 | // Finishes the CurrentStage. 108 | func (fReq *Request) finishPipelineStage() { 109 | fReq.CurrentStage.EndTime = time.Now() 110 | fReq.finishCommon() 111 | } 112 | 113 | // Appends an already completed PipelineStageStat directly to the list 114 | func (fReq *Request) appendPipelineStage(pss *PipelineStageStat) { 115 | fReq.PipelineStageStats.PushBack(pss) 116 | fReq.CurrentStage = pss 117 | fReq.finishCommon() 118 | } 119 | 120 | // Does some required bookeeping for the pipeline and the pipeline signature 121 | func (fReq *Request) finishCommon() { 122 | fReq.pipelineHash.Write([]byte(fReq.CurrentStage.Name)) 123 | fReq.pipelineHash.Write([]byte{fReq.CurrentStage.Status}) 124 | fReq.piplineTot += fReq.CurrentStage.EndTime.Sub(fReq.CurrentStage.StartTime) 125 | } 126 | 127 | // The Signature will only be complete in the RequestDoneCallback. At 128 | // any given time, the Signature is a crc32 sum of all the finished 129 | // pipeline stages combining PipelineStageStat.Name and PipelineStageStat.Status. 130 | // This gives a unique signature for each unique path through the pipeline. 131 | // To modify the signature for your own use, just set the 132 | // request.CurrentStage.Status in your RequestFilter or ResponseFilter. 133 | func (fReq *Request) Signature() string { 134 | return fmt.Sprintf("%X", fReq.pipelineHash.Sum32()) 135 | } 136 | 137 | // Call from RequestDoneCallback. Logs a bunch of information about the 138 | // request to the falcore logger. This is a pretty big hit to performance 139 | // so it should only be used for debugging or development. The source is a 140 | // good example of how to get useful information out of the Request. 141 | func (fReq *Request) Trace() { 142 | reqTime := TimeDiff(fReq.StartTime, fReq.EndTime) 143 | req := fReq.HttpRequest 144 | Trace("%s [%s] %s%s Sig=%s Tot=%.4f", fReq.ID, req.Method, req.Host, req.URL, fReq.Signature(), reqTime) 145 | l := fReq.PipelineStageStats 146 | for e := l.Front(); e != nil; e = e.Next() { 147 | pss, _ := e.Value.(*PipelineStageStat) 148 | dur := TimeDiff(pss.StartTime, pss.EndTime) 149 | Trace("%s %-30s S=%d Tot=%.4f %%=%.2f", fReq.ID, pss.Name, pss.Status, dur, dur/reqTime*100.0) 150 | } 151 | Trace("%s %-30s S=0 Tot=%.4f %%=%.2f", fReq.ID, "Overhead", float32(fReq.Overhead)/1.0e9, float32(fReq.Overhead)/1.0e9/reqTime*100.0) 152 | } 153 | 154 | func (fReq *Request) finishRequest() { 155 | fReq.EndTime = time.Now() 156 | fReq.Overhead = fReq.EndTime.Sub(fReq.StartTime) - fReq.piplineTot 157 | } 158 | 159 | // Container for keeping stats per pipeline stage 160 | // Name for filter stages is reflect.TypeOf(filter).String()[1:] and the Status is 0 unless 161 | // it is changed explicitly in the Filter or Router. 162 | // 163 | // For the Status, the falcore library will not apply any specific meaning to the status 164 | // codes but the following are suggested conventional usages that we have found useful 165 | // 166 | // type PipelineStatus byte 167 | // const ( 168 | // Success PipelineStatus = iota // General Run successfully 169 | // Skip // Skipped (all or most of the work of this stage) 170 | // Fail // General Fail 171 | // // All others may be used as custom status codes 172 | // ) 173 | type PipelineStageStat struct { 174 | Name string 175 | Status byte 176 | StartTime time.Time 177 | EndTime time.Time 178 | } 179 | 180 | func NewPiplineStage(name string) *PipelineStageStat { 181 | pss := new(PipelineStageStat) 182 | pss.Name = name 183 | pss.StartTime = time.Now() 184 | return pss 185 | } 186 | -------------------------------------------------------------------------------- /response.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "net/http" 5 | "strings" 6 | ) 7 | 8 | func SimpleResponse(req *http.Request, status int, headers http.Header, body string) *http.Response { 9 | res := new(http.Response) 10 | body_rdr := (*fixedResBody)(strings.NewReader(body)) 11 | res.StatusCode = status 12 | res.ProtoMajor = 1 13 | res.ProtoMinor = 1 14 | res.ContentLength = int64((*strings.Reader)(body_rdr).Len()) 15 | res.Request = req 16 | res.Header = make(map[string][]string) 17 | res.Body = body_rdr 18 | if headers != nil { 19 | res.Header = headers 20 | } 21 | return res 22 | } 23 | 24 | // string type for response objects 25 | 26 | type fixedResBody strings.Reader 27 | 28 | func (s *fixedResBody) Close() error { 29 | return nil 30 | } 31 | 32 | func (s *fixedResBody) Read(b []byte) (int, error) { 33 | return (*strings.Reader)(s).Read(b) 34 | } 35 | 36 | func RedirectResponse(req *http.Request, url string) *http.Response { 37 | res := new(http.Response) 38 | res.StatusCode = 302 39 | res.ProtoMajor = 1 40 | res.ProtoMinor = 1 41 | res.ContentLength = 0 42 | res.Request = req 43 | res.Header = make(map[string][]string) 44 | res.Header.Set("Location", url) 45 | return res 46 | } 47 | -------------------------------------------------------------------------------- /router.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "container/list" 5 | "regexp" 6 | ) 7 | 8 | // Interface for defining routers 9 | type Router interface { 10 | // Returns a Pipeline or nil if one can't be found 11 | SelectPipeline(req *Request) (pipe RequestFilter) 12 | } 13 | 14 | // Interface for defining individual routes 15 | type Route interface { 16 | // Returns the route's filter if there's a match. nil if there isn't 17 | MatchString(str string) RequestFilter 18 | } 19 | 20 | // Generate a new Router instance using f for SelectPipeline 21 | func NewRouter(f genericRouter) Router { 22 | return f 23 | } 24 | 25 | type genericRouter func(req *Request) (pipe RequestFilter) 26 | 27 | func (f genericRouter) SelectPipeline(req *Request) (pipe RequestFilter) { 28 | return f(req) 29 | } 30 | 31 | // Will match any request. Useful for fallthrough filters. 32 | type MatchAnyRoute struct { 33 | Filter RequestFilter 34 | } 35 | 36 | func (r *MatchAnyRoute) MatchString(str string) RequestFilter { 37 | return r.Filter 38 | } 39 | 40 | // Will match based on a regular expression 41 | type RegexpRoute struct { 42 | Match *regexp.Regexp 43 | Filter RequestFilter 44 | } 45 | 46 | func (r *RegexpRoute) MatchString(str string) RequestFilter { 47 | if r.Match.MatchString(str) { 48 | return r.Filter 49 | } 50 | return nil 51 | } 52 | 53 | // Route requsts based on hostname 54 | type HostRouter struct { 55 | hosts map[string]RequestFilter 56 | } 57 | 58 | // Generate a new HostRouter instance 59 | func NewHostRouter() *HostRouter { 60 | r := new(HostRouter) 61 | r.hosts = make(map[string]RequestFilter) 62 | return r 63 | } 64 | 65 | // TODO: support for non-exact matches 66 | func (r *HostRouter) AddMatch(host string, pipe RequestFilter) { 67 | r.hosts[host] = pipe 68 | } 69 | 70 | func (r *HostRouter) SelectPipeline(req *Request) (pipe RequestFilter) { 71 | return r.hosts[req.HttpRequest.Host] 72 | } 73 | 74 | // Route requests based on path 75 | type PathRouter struct { 76 | Routes *list.List 77 | } 78 | 79 | // Generate a new instance of PathRouter 80 | func NewPathRouter() *PathRouter { 81 | r := new(PathRouter) 82 | r.Routes = list.New() 83 | return r 84 | } 85 | 86 | func (r *PathRouter) AddRoute(route Route) { 87 | r.Routes.PushBack(route) 88 | } 89 | 90 | // convenience method for adding RegexpRoutes 91 | func (r *PathRouter) AddMatch(match string, filter RequestFilter) (err error) { 92 | route := &RegexpRoute{Filter: filter} 93 | if route.Match, err = regexp.Compile(match); err == nil { 94 | r.Routes.PushBack(route) 95 | } 96 | return 97 | } 98 | 99 | // Will panic if r.Routes contains an object that isn't a Route 100 | func (r *PathRouter) SelectPipeline(req *Request) (pipe RequestFilter) { 101 | var route Route 102 | for r := r.Routes.Front(); r != nil; r = r.Next() { 103 | route = r.Value.(Route) 104 | if f := route.MatchString(req.HttpRequest.URL.Path); f != nil { 105 | return f 106 | } 107 | } 108 | return nil 109 | } 110 | -------------------------------------------------------------------------------- /router_test.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "net/http" 5 | "regexp" 6 | "testing" 7 | ) 8 | 9 | type SimpleFilter int 10 | 11 | func (sf SimpleFilter) FilterRequest(req *Request) *http.Response { 12 | sf = -sf 13 | return nil 14 | } 15 | 16 | func TestRegexpRoute(t *testing.T) { 17 | r := new(RegexpRoute) 18 | 19 | var sf1 SimpleFilter = 1 20 | r.Filter = sf1 21 | r.Match = regexp.MustCompile(`one`) 22 | 23 | if r.MatchString("http://tester.com/one") != sf1 { 24 | t.Errorf("Failed to match regexp") 25 | } 26 | if r.MatchString("http://tester.com/two") != nil { 27 | t.Errorf("False regexp match") 28 | } 29 | 30 | } 31 | 32 | func TestHostRouter(t *testing.T) { 33 | hr := NewHostRouter() 34 | 35 | var sf1 SimpleFilter = 1 36 | var sf2 SimpleFilter = 2 37 | hr.AddMatch("www.ngmoco.com", sf1) 38 | hr.AddMatch("developer.ngmoco.com", sf2) 39 | 40 | req := validGetRequest() 41 | req.HttpRequest.Host = "developer.ngmoco.com" 42 | 43 | filt := hr.SelectPipeline(req) 44 | if filt != sf2 { 45 | t.Errorf("Host router didn't get the right pipeline") 46 | } 47 | 48 | req.HttpRequest.Host = "ngmoco.com" 49 | filt = hr.SelectPipeline(req) 50 | if filt != nil { 51 | t.Errorf("Host router got currently unsupported fuzzy match so you should update this test") 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /server.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "bufio" 5 | "crypto/rand" 6 | "crypto/tls" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "net" 11 | "net/http" 12 | "os" 13 | "runtime" 14 | "strconv" 15 | "sync" 16 | "syscall" 17 | "time" 18 | ) 19 | 20 | type Server struct { 21 | Addr string 22 | Pipeline *Pipeline 23 | listener net.Listener 24 | listenerFile *os.File 25 | stopAccepting chan int 26 | handlerWaitGroup *sync.WaitGroup 27 | logPrefix string 28 | AcceptReady chan int 29 | sendfile bool 30 | sockOpt int 31 | bufferPool *bufferPool 32 | } 33 | 34 | func NewServer(port int, pipeline *Pipeline) *Server { 35 | s := new(Server) 36 | s.Addr = fmt.Sprintf(":%v", port) 37 | s.Pipeline = pipeline 38 | s.stopAccepting = make(chan int) 39 | s.AcceptReady = make(chan int, 1) 40 | s.handlerWaitGroup = new(sync.WaitGroup) 41 | s.logPrefix = fmt.Sprintf("%d", syscall.Getpid()) 42 | 43 | // openbsd/netbsd don't have TCP_NOPUSH so it's likely sendfile will be slower 44 | // without these socket options, just enable for linux, mac and freebsd. 45 | // TODO (Graham) windows has TransmitFile zero-copy mechanism, try to use it 46 | switch runtime.GOOS { 47 | case "linux": 48 | s.sendfile = true 49 | s.sockOpt = 0x3 // syscall.TCP_CORK 50 | case "freebsd", "darwin": 51 | s.sendfile = true 52 | s.sockOpt = 0x4 // syscall.TCP_NOPUSH 53 | default: 54 | s.sendfile = false 55 | } 56 | 57 | // buffer pool for reusing connection bufio.Readers 58 | s.bufferPool = newBufferPool(100, 8192) 59 | 60 | return s 61 | } 62 | 63 | func (srv *Server) FdListen(fd int) error { 64 | var err error 65 | srv.listenerFile = os.NewFile(uintptr(fd), "") 66 | if srv.listener, err = net.FileListener(srv.listenerFile); err != nil { 67 | return err 68 | } 69 | if _, ok := srv.listener.(*net.TCPListener); !ok { 70 | return errors.New("Broken listener isn't TCP") 71 | } 72 | return nil 73 | } 74 | 75 | func (srv *Server) socketListen() error { 76 | var la *net.TCPAddr 77 | var err error 78 | if la, err = net.ResolveTCPAddr("tcp", srv.Addr); err != nil { 79 | return err 80 | } 81 | 82 | var l *net.TCPListener 83 | if l, err = net.ListenTCP("tcp", la); err != nil { 84 | return err 85 | } 86 | srv.listener = l 87 | // setup listener to be non-blocking if we're not on windows. 88 | // this is required for hot restart to work. 89 | return srv.setupNonBlockingListener(err, l) 90 | } 91 | 92 | func (srv *Server) ListenAndServe() error { 93 | if srv.Addr == "" { 94 | srv.Addr = ":http" 95 | } 96 | if srv.listener == nil { 97 | if err := srv.socketListen(); err != nil { 98 | return err 99 | } 100 | } 101 | return srv.serve() 102 | } 103 | 104 | func (srv *Server) SocketFd() int { 105 | return int(srv.listenerFile.Fd()) 106 | } 107 | 108 | func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { 109 | if srv.Addr == "" { 110 | srv.Addr = ":https" 111 | } 112 | config := &tls.Config{ 113 | Rand: rand.Reader, 114 | Time: time.Now, 115 | NextProtos: []string{"http/1.1"}, 116 | } 117 | 118 | var err error 119 | config.Certificates = make([]tls.Certificate, 1) 120 | config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile) 121 | if err != nil { 122 | return err 123 | } 124 | 125 | if srv.listener == nil { 126 | if err := srv.socketListen(); err != nil { 127 | return err 128 | } 129 | } 130 | 131 | srv.listener = tls.NewListener(srv.listener, config) 132 | 133 | return srv.serve() 134 | } 135 | 136 | func (srv *Server) StopAccepting() { 137 | close(srv.stopAccepting) 138 | } 139 | 140 | func (srv *Server) Port() int { 141 | if l := srv.listener; l != nil { 142 | a := l.Addr() 143 | if _, p, e := net.SplitHostPort(a.String()); e == nil && p != "" { 144 | server_port, _ := strconv.Atoi(p) 145 | return server_port 146 | } 147 | } 148 | return 0 149 | } 150 | 151 | func (srv *Server) serve() (e error) { 152 | var accept = true 153 | srv.AcceptReady <- 1 154 | for accept { 155 | var c net.Conn 156 | if l, ok := srv.listener.(*net.TCPListener); ok { 157 | l.SetDeadline(time.Now().Add(3e9)) 158 | } 159 | c, e = srv.listener.Accept() 160 | if e != nil { 161 | if ope, ok := e.(*net.OpError); ok { 162 | if !(ope.Timeout() && ope.Temporary()) { 163 | Error("%s SERVER Accept Error: %v", srv.serverLogPrefix(), ope) 164 | } 165 | } else { 166 | Error("%s SERVER Accept Error: %v", srv.serverLogPrefix(), e) 167 | } 168 | } else { 169 | //Trace("Handling!") 170 | srv.handlerWaitGroup.Add(1) 171 | go srv.handler(c) 172 | } 173 | select { 174 | case <-srv.stopAccepting: 175 | accept = false 176 | default: 177 | } 178 | } 179 | Trace("Stopped accepting, waiting for handlers") 180 | // wait for handlers 181 | srv.handlerWaitGroup.Wait() 182 | return nil 183 | } 184 | 185 | func (srv *Server) sentinel(c net.Conn, connClosed chan int) { 186 | select { 187 | case <-srv.stopAccepting: 188 | c.SetReadDeadline(time.Now().Add(3 * time.Second)) 189 | case <-connClosed: 190 | } 191 | } 192 | 193 | func (srv *Server) handler(c net.Conn) { 194 | startTime := time.Now() 195 | bpe := srv.bufferPool.take(c) 196 | defer srv.bufferPool.give(bpe) 197 | var closeSentinelChan = make(chan int) 198 | go srv.sentinel(c, closeSentinelChan) 199 | defer srv.connectionFinished(c, closeSentinelChan) 200 | var err error 201 | var req *http.Request 202 | // no keepalive (for now) 203 | reqCount := 0 204 | keepAlive := true 205 | for err == nil && keepAlive { 206 | if req, err = http.ReadRequest(bpe.br); err == nil { 207 | if req.Header.Get("Connection") != "Keep-Alive" { 208 | keepAlive = false 209 | } 210 | request := newRequest(req, c, startTime) 211 | reqCount++ 212 | var res *http.Response 213 | 214 | pssInit := new(PipelineStageStat) 215 | pssInit.Name = "server.Init" 216 | pssInit.StartTime = startTime 217 | pssInit.EndTime = time.Now() 218 | request.appendPipelineStage(pssInit) 219 | // execute the pipeline 220 | if res = srv.Pipeline.execute(request); res == nil { 221 | res = SimpleResponse(req, 404, nil, "Not Found") 222 | } 223 | // cleanup 224 | request.startPipelineStage("server.ResponseWrite") 225 | req.Body.Close() 226 | 227 | // shutting down? 228 | select { 229 | case <-srv.stopAccepting: 230 | keepAlive = false 231 | res.Close = true 232 | default: 233 | } 234 | // The res.Write omits Content-length on 0 length bodies, and by spec, 235 | // it SHOULD. While this is not MUST, it's kinda broken. See sec 4.4 236 | // of rfc2616 and a 200 with a zero length does not satisfy any of the 237 | // 5 conditions if Connection: keep-alive is set :( 238 | // I'm forcing chunked which seems to work because I couldn't get the 239 | // content length to write if it was 0. 240 | // Specifically, the android http client waits forever if there's no 241 | // content-length instead of assuming zero at the end of headers. der. 242 | if res.ContentLength == 0 && len(res.TransferEncoding) == 0 && !((res.StatusCode-100 < 100) || res.StatusCode == 204 || res.StatusCode == 304) { 243 | res.TransferEncoding = []string{"identity"} 244 | } 245 | if res.ContentLength < 0 { 246 | res.TransferEncoding = []string{"chunked"} 247 | } 248 | 249 | // For HTTP/1.0 and Keep-Alive, sending the Connection: Keep-Alive response header is required 250 | // because close is default (opposite of 1.1) 251 | if keepAlive && !req.ProtoAtLeast(1, 1) { 252 | res.Header.Add("Connection", "Keep-Alive") 253 | } 254 | 255 | // write response 256 | if srv.sendfile { 257 | res.Write(c) 258 | srv.cycleNonBlock(c) 259 | } else { 260 | wbuf := bufio.NewWriter(c) 261 | res.Write(wbuf) 262 | wbuf.Flush() 263 | } 264 | if res.Body != nil { 265 | res.Body.Close() 266 | } 267 | request.finishPipelineStage() 268 | request.finishRequest() 269 | srv.requestFinished(request) 270 | 271 | if res.Close { 272 | keepAlive = false 273 | } 274 | 275 | // Reset the startTime 276 | // this isn't great since there may be lag between requests; but it's the best we've got 277 | startTime = time.Now() 278 | } else { 279 | // EOF is socket closed 280 | if nerr, ok := err.(net.Error); err != io.EOF && !(ok && nerr.Timeout()) { 281 | Error("%s %v ERROR reading request: <%T %v>", srv.serverLogPrefix(), c.RemoteAddr(), err, err) 282 | } 283 | } 284 | } 285 | //Debug("%s Processed %v requests on connection %v", srv.serverLogPrefix(), reqCount, c.RemoteAddr()) 286 | } 287 | 288 | func (srv *Server) serverLogPrefix() string { 289 | return srv.logPrefix 290 | } 291 | 292 | func (srv *Server) requestFinished(request *Request) { 293 | if srv.Pipeline.RequestDoneCallback != nil { 294 | // Don't block the connecion for this 295 | go srv.Pipeline.RequestDoneCallback.FilterRequest(request) 296 | } 297 | } 298 | 299 | func (srv *Server) connectionFinished(c net.Conn, closeChan chan int) { 300 | c.Close() 301 | close(closeChan) 302 | srv.handlerWaitGroup.Done() 303 | } 304 | -------------------------------------------------------------------------------- /server_notwindows.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | package falcore 4 | 5 | import ( 6 | "net" 7 | "syscall" 8 | ) 9 | 10 | // only valid on non-windows 11 | func (srv *Server) setupNonBlockingListener(err error, l *net.TCPListener) error { 12 | // FIXME: File() returns a copied pointer. we're leaking it. probably doesn't matter 13 | if srv.listenerFile, err = l.File(); err != nil { 14 | return err 15 | } 16 | fd := int(srv.listenerFile.Fd()) 17 | if e := syscall.SetNonblock(fd, true); e != nil { 18 | return e 19 | } 20 | if srv.sendfile { 21 | if e := syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, srv.sockOpt, 1); e != nil { 22 | return e 23 | } 24 | } 25 | return nil 26 | } 27 | 28 | func (srv *Server) cycleNonBlock(c net.Conn) { 29 | if srv.sendfile { 30 | if tcpC, ok := c.(*net.TCPConn); ok { 31 | if f, err := tcpC.File(); err == nil { 32 | // f is a copy. must be closed 33 | defer f.Close() 34 | fd := int(f.Fd()) 35 | // Disable TCP_CORK/TCP_NOPUSH 36 | syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, srv.sockOpt, 0) 37 | // For TCP_NOPUSH, we need to force flush 38 | c.Write([]byte{}) 39 | // Re-enable TCP_CORK/TCP_NOPUSH 40 | syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, srv.sockOpt, 1) 41 | syscall.SetNonblock(fd, true) 42 | } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /server_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | package falcore 3 | 4 | import ( 5 | "net" 6 | ) 7 | 8 | // only valid on non-windows 9 | func (srv *Server) setupNonBlockingListener(err error, l *net.TCPListener) error { 10 | return nil 11 | } 12 | 13 | func (srv *Server) cycleNonBlock(c net.Conn) { 14 | // nuthin 15 | } 16 | -------------------------------------------------------------------------------- /static_file/file_filter.go: -------------------------------------------------------------------------------- 1 | package static_file 2 | 3 | import ( 4 | "github.com/ngmoco/falcore" 5 | "mime" 6 | "net/http" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | ) 11 | 12 | // A falcore RequestFilter for serving static files 13 | // from the filesystem. 14 | type Filter struct { 15 | // File system base path for serving files 16 | BasePath string 17 | // Prefix in URL path 18 | PathPrefix string 19 | } 20 | 21 | func (f *Filter) FilterRequest(req *falcore.Request) (res *http.Response) { 22 | // Clean asset path 23 | asset_path := filepath.Clean(filepath.FromSlash(req.HttpRequest.URL.Path)) 24 | 25 | // Resolve PathPrefix 26 | if strings.HasPrefix(asset_path, f.PathPrefix) { 27 | asset_path = asset_path[len(f.PathPrefix):] 28 | } else { 29 | falcore.Debug("%v doesn't match prefix %v", asset_path, f.PathPrefix) 30 | res = falcore.SimpleResponse(req.HttpRequest, 404, nil, "Not found.") 31 | return 32 | } 33 | 34 | // Resolve FSBase 35 | if f.BasePath != "" { 36 | asset_path = filepath.Join(f.BasePath, asset_path) 37 | } else { 38 | falcore.Error("file_filter requires a BasePath") 39 | return falcore.SimpleResponse(req.HttpRequest, 500, nil, "Server Error\n") 40 | } 41 | 42 | // Open File 43 | if file, err := os.Open(asset_path); err == nil { 44 | // Make sure it's an actual file 45 | if stat, err := file.Stat(); err == nil && stat.Mode()&os.ModeType == 0 { 46 | res = &http.Response{ 47 | Request: req.HttpRequest, 48 | StatusCode: 200, 49 | Proto: "HTTP/1.1", 50 | ProtoMajor: 1, 51 | ProtoMinor: 1, 52 | Body: file, 53 | Header: make(http.Header), 54 | ContentLength: stat.Size(), 55 | } 56 | if ct := mime.TypeByExtension(filepath.Ext(asset_path)); ct != "" { 57 | res.Header.Set("Content-Type", ct) 58 | } 59 | } else { 60 | file.Close() 61 | } 62 | } else { 63 | falcore.Finest("Can't open %v: %v", asset_path, err) 64 | } 65 | return 66 | } 67 | -------------------------------------------------------------------------------- /static_file/file_filter_test.go: -------------------------------------------------------------------------------- 1 | package static_file 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/ngmoco/falcore" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "mime" 11 | "net/http" 12 | "strings" 13 | "testing" 14 | "time" 15 | ) 16 | 17 | var srv *falcore.Server 18 | 19 | func init() { 20 | // Silence log output 21 | log.SetOutput(nil) 22 | 23 | // setup mime 24 | mime.AddExtensionType(".foo", "foo/bar") 25 | mime.AddExtensionType(".json", "application/json") 26 | mime.AddExtensionType(".txt", "text/plain") 27 | mime.AddExtensionType(".png", "image/png") 28 | 29 | go func() { 30 | 31 | // falcore setup 32 | pipeline := falcore.NewPipeline() 33 | pipeline.Upstream.PushBack(&Filter{ 34 | PathPrefix: "/", 35 | BasePath: "../test/", 36 | }) 37 | srv = falcore.NewServer(0, pipeline) 38 | if err := srv.ListenAndServe(); err != nil { 39 | panic(fmt.Sprintf("Could not start falcore: %v", err)) 40 | } 41 | }() 42 | } 43 | 44 | func port() int { 45 | for srv.Port() == 0 { 46 | time.Sleep(1e7) 47 | } 48 | return srv.Port() 49 | } 50 | 51 | func get(p string) (r *http.Response, err error) { 52 | req, _ := http.NewRequest("GET", fmt.Sprintf("http://%v", fmt.Sprintf("localhost:%v/", port())), nil) 53 | req.URL.Path = p 54 | r, err = http.DefaultTransport.RoundTrip(req) 55 | return 56 | } 57 | 58 | var fourOhFourTests = []struct { 59 | name string 60 | url string 61 | }{ 62 | { 63 | name: "basic invalid path", 64 | url: "/this/path/doesnt/exist", 65 | }, 66 | { 67 | name: "realtive pathing out of sandbox", 68 | url: "/../README.md", 69 | }, 70 | { 71 | name: "directory", 72 | url: "/hello", 73 | }, 74 | } 75 | 76 | func TestFourOhFour(t *testing.T) { 77 | for _, test := range fourOhFourTests { 78 | r, err := get(test.url) 79 | if err != nil { 80 | t.Errorf("%v Error getting file:", test.name, err) 81 | continue 82 | } 83 | if r.StatusCode != 404 { 84 | t.Errorf("%v Expected status 404, got %v", test.name, r.StatusCode) 85 | } 86 | } 87 | } 88 | 89 | var basicTests = []struct { 90 | name string 91 | path string 92 | mime string 93 | data []byte 94 | file string 95 | url string 96 | }{ 97 | { 98 | name: "small text file", 99 | mime: "text/plain", 100 | path: "fsbase_test/hello/world.txt", 101 | data: []byte("Hello world!"), 102 | url: "/hello/world.txt", 103 | }, 104 | { 105 | name: "json file", 106 | mime: "application/json", 107 | path: "fsbase_test/foo.json", 108 | file: "../test/foo.json", 109 | url: "/foo.json", 110 | }, 111 | { 112 | name: "png file", 113 | mime: "image/png", 114 | path: "fsbase_test/images/face.png", 115 | file: "../test/images/face.png", 116 | url: "/images/face.png", 117 | }, 118 | { 119 | name: "relative paths", 120 | mime: "application/json", 121 | path: "fsbase_test/foo.json", 122 | file: "../test/foo.json", 123 | url: "/images/../foo.json", 124 | }, 125 | { 126 | name: "custom mime type", 127 | mime: "foo/bar", 128 | path: "fsbase_test/custom_type.foo", 129 | file: "../test/custom_type.foo", 130 | url: "/custom_type.foo", 131 | }, 132 | } 133 | 134 | func TestBasicFiles(t *testing.T) { 135 | rbody := new(bytes.Buffer) 136 | for _, test := range basicTests { 137 | // read in test file data 138 | if test.file != "" { 139 | test.data, _ = ioutil.ReadFile(test.file) 140 | } 141 | 142 | r, err := get(test.url) 143 | if err != nil { 144 | t.Errorf("%v Error GETting file:%v", test.name, err) 145 | continue 146 | } 147 | if r.StatusCode != 200 { 148 | t.Errorf("%v Expected status 200, got %v", test.name, r.StatusCode) 149 | continue 150 | } 151 | if strings.Split(r.Header.Get("Content-Type"), ";")[0] != test.mime { 152 | t.Errorf("%v Expected Content-Type: %v, got '%v'", test.name, test.mime, r.Header.Get("Content-Type")) 153 | } 154 | rbody.Reset() 155 | io.Copy(rbody, r.Body) 156 | if rbytes := rbody.Bytes(); !bytes.Equal(test.data, rbytes) { 157 | t.Errorf("%v Body doesn't match.\n\tExpected:\n\t%v\n\tReceived:\n\t%v", test.name, test.data, rbytes) 158 | } 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /string_body.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "net/http" 7 | "strings" 8 | ) 9 | 10 | // Keeps the body of a request in a string so it can be re-read at each stage of the pipeline 11 | // implements io.ReadCloser to match http.Request.Body 12 | 13 | type StringBody struct { 14 | BodyBuffer *bytes.Reader 15 | bpe *bufferPoolEntry 16 | } 17 | 18 | type StringBodyFilter struct { 19 | pool *bufferPool 20 | } 21 | 22 | func NewStringBodyFilter() *StringBodyFilter { 23 | sbf := &StringBodyFilter{} 24 | sbf.pool = newBufferPool(100, 1024) 25 | return sbf 26 | } 27 | func (sbf *StringBodyFilter) FilterRequest(request *Request) *http.Response { 28 | req := request.HttpRequest 29 | // This caches the request body so that multiple filters can iterate it 30 | if req.Method == "POST" || req.Method == "PUT" { 31 | sb, err := sbf.readRequestBody(req) 32 | if sb == nil || err != nil { 33 | request.CurrentStage.Status = 3 // Skip 34 | Debug("%s No Req Body or Ignored: %v", request.ID, err) 35 | } 36 | } else { 37 | request.CurrentStage.Status = 1 // Skip 38 | } 39 | return nil 40 | } 41 | 42 | // reads the request body and replaces the buffer with self 43 | // returns nil if the body is multipart and not replaced 44 | func (sbf *StringBodyFilter) readRequestBody(r *http.Request) (sb *StringBody, err error) { 45 | ct := r.Header.Get("Content-Type") 46 | // leave it on the buffer if we're multipart 47 | if strings.SplitN(ct, ";", 2)[0] != "multipart/form-data" && r.ContentLength > 0 { 48 | sb = &StringBody{} 49 | const maxFormSize = int64(10 << 20) // 10 MB is a lot of text. 50 | sb.bpe = sbf.pool.take(io.LimitReader(r.Body, maxFormSize+1)) 51 | 52 | // There shouldn't be a null byte so we should get EOF 53 | b, e := sb.bpe.br.ReadBytes(0) 54 | if e != nil && e != io.EOF { 55 | return nil, e 56 | } 57 | sb.BodyBuffer = bytes.NewReader(b) 58 | r.Body.Close() 59 | r.Body = sb 60 | return sb, nil 61 | } 62 | return nil, nil // ignore 63 | } 64 | 65 | // Returns a buffer used in the FilterRequest stage to a buffer pool 66 | // this speeds up this filter significantly by reusing buffers 67 | func (sbf *StringBodyFilter) ReturnBuffer(request *Request) { 68 | if sb, ok := request.HttpRequest.Body.(*StringBody); ok { 69 | sbf.pool.give(sb.bpe) 70 | } 71 | } 72 | 73 | // Insert this in the response pipeline to return the buffer pool for the request body 74 | // If there is an appropriate place in your flow, you can call ReturnBuffer explicitly 75 | func (sbf *StringBodyFilter) FilterResponse(request *Request, res *http.Response) { 76 | sbf.ReturnBuffer(request) 77 | } 78 | 79 | func (sb *StringBody) Read(b []byte) (n int, err error) { 80 | return sb.BodyBuffer.Read(b) 81 | } 82 | 83 | func (sb *StringBody) Close() error { 84 | // start over 85 | sb.BodyBuffer.Seek(0, 0) 86 | return nil 87 | } 88 | -------------------------------------------------------------------------------- /string_body_test.go: -------------------------------------------------------------------------------- 1 | package falcore 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "net/http" 7 | "testing" 8 | "time" 9 | //"io" 10 | ) 11 | 12 | func TestStringBody(t *testing.T) { 13 | expected := []byte("HOT HOT HOT!!!") 14 | tmp, _ := http.NewRequest("POST", "/hello", bytes.NewReader(expected)) 15 | tmp.Header.Set("Content-Type", "text/plain") 16 | tmp.ContentLength = int64(len(expected)) 17 | req := newRequest(tmp, nil, time.Now()) 18 | req.startPipelineStage("StringBodyTest") 19 | 20 | sbf := NewStringBodyFilter() 21 | //sbf := &StringBodyFilter{} 22 | sbf.FilterRequest(req) 23 | 24 | if sb, ok := req.HttpRequest.Body.(*StringBody); ok { 25 | readin, _ := ioutil.ReadAll(sb) 26 | sb.Close() 27 | if bytes.Compare(readin, expected) != 0 { 28 | t.Errorf("Body string not read %q expected %q", readin, expected) 29 | } 30 | } else { 31 | t.Errorf("Body not replaced with StringBody") 32 | } 33 | 34 | if req.CurrentStage.Status != 0 { 35 | t.Errorf("SBF failed to parse POST with status %d", req.CurrentStage.Status) 36 | } 37 | 38 | var body []byte = make([]byte, 100) 39 | l, _ := req.HttpRequest.Body.Read(body) 40 | if bytes.Compare(body[0:l], expected) != 0 { 41 | t.Errorf("Failed to read the right bytes %q expected %q", body, expected) 42 | 43 | } 44 | 45 | l, _ = req.HttpRequest.Body.Read(body) 46 | if l != 0 { 47 | t.Errorf("Should have read zero!") 48 | } 49 | 50 | // Close resets the buffer 51 | req.HttpRequest.Body.Close() 52 | 53 | l, _ = req.HttpRequest.Body.Read(body) 54 | if bytes.Compare(body[0:l], expected) != 0 { 55 | t.Errorf("Failed to read the right bytes after calling Close %q expected %q", body, expected) 56 | 57 | } 58 | 59 | } 60 | 61 | func BenchmarkStringBody(b *testing.B) { 62 | b.StopTimer() 63 | expected := []byte("test=123456&test2=987654&test3=somedatanstuff&test4=moredataontheend") 64 | expLen := int64(len(expected)) 65 | req := newRequest(nil, nil, time.Now()) 66 | req.startPipelineStage("StringBodyTest") 67 | 68 | sbf := NewStringBodyFilter() 69 | //sbf := &StringBodyFilter{} 70 | 71 | for i := 0; i < b.N; i++ { 72 | tmp, _ := http.NewRequest("POST", "/hello", bytes.NewReader(expected)) 73 | tmp.Header.Set("Content-Type", "application/x-www-form-urlencoded") 74 | tmp.ContentLength = expLen 75 | req.HttpRequest = tmp 76 | b.StartTimer() 77 | // replace the body 78 | sbf.FilterRequest(req) 79 | sbf.ReturnBuffer(req) 80 | // read the body twice 81 | /* nah, this isn't so useful 82 | io.CopyN(ioutil.Discard, req.HttpRequest.Body, req.HttpRequest.ContentLength) 83 | req.HttpRequest.Body .Close() 84 | io.CopyN(ioutil.Discard, req.HttpRequest.Body, req.HttpRequest.ContentLength) 85 | req.HttpRequest.Body .Close() 86 | */ 87 | b.StopTimer() 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /test/custom_type.foo: -------------------------------------------------------------------------------- 1 | custom type! -------------------------------------------------------------------------------- /test/foo.json: -------------------------------------------------------------------------------- 1 | {"all": "your", "base": [1, 2, 3]} -------------------------------------------------------------------------------- /test/hello/world.txt: -------------------------------------------------------------------------------- 1 | Hello world! -------------------------------------------------------------------------------- /test/images/face.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ngmoco/falcore/0b26499d6ab1fab10dfb0dfc15cc382736bcc9e6/test/images/face.png -------------------------------------------------------------------------------- /test/index.html: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | Example Page! 9 | 10 | 11 | 12 | 13 | 14 |

WOOOOOOO!

15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /upstream/upstream.go: -------------------------------------------------------------------------------- 1 | package upstream 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/ngmoco/falcore" 7 | "io" 8 | "net" 9 | "net/http" 10 | "time" 11 | ) 12 | 13 | type passThruReadCloser struct { 14 | io.Reader 15 | io.Closer 16 | } 17 | 18 | type connWrapper struct { 19 | conn net.Conn 20 | timeout time.Duration 21 | } 22 | 23 | func (cw *connWrapper) Write(b []byte) (int, error) { 24 | if err := cw.conn.SetDeadline(time.Now().Add(cw.timeout)); err != nil { 25 | return 0, err 26 | } 27 | return cw.conn.Write(b) 28 | } 29 | func (cw *connWrapper) Read(b []byte) (n int, err error) { return cw.conn.Read(b) } 30 | func (cw *connWrapper) Close() error { return cw.conn.Close() } 31 | func (cw *connWrapper) LocalAddr() net.Addr { return cw.conn.LocalAddr() } 32 | func (cw *connWrapper) RemoteAddr() net.Addr { return cw.conn.RemoteAddr() } 33 | func (cw *connWrapper) SetDeadline(t time.Time) error { return cw.conn.SetDeadline(t) } 34 | func (cw *connWrapper) SetReadDeadline(t time.Time) error { return cw.conn.SetReadDeadline(t) } 35 | func (cw *connWrapper) SetWriteDeadline(t time.Time) error { return cw.conn.SetWriteDeadline(t) } 36 | 37 | type Upstream struct { 38 | // The upstream host to connect to 39 | Host string 40 | // The port on the upstream host 41 | Port int 42 | // Default 60 seconds 43 | Timeout time.Duration 44 | // Will ignore https on the incoming request and always upstream http 45 | ForceHttp bool 46 | // Ping URL Path-only for checking upness 47 | PingPath string 48 | 49 | transport *http.Transport 50 | host string 51 | tcpaddr *net.TCPAddr 52 | } 53 | 54 | func NewUpstream(host string, port int, forceHttp bool) *Upstream { 55 | u := new(Upstream) 56 | u.Host = host 57 | u.Port = port 58 | u.ForceHttp = forceHttp 59 | ips, err := net.LookupIP(host) 60 | var ip net.IP = nil 61 | for i := range ips { 62 | ip = ips[i].To4() 63 | if ip != nil { 64 | break 65 | } 66 | } 67 | if err == nil && ip != nil { 68 | u.tcpaddr = &net.TCPAddr{} 69 | u.tcpaddr.Port = port 70 | u.tcpaddr.IP = ip 71 | } else { 72 | falcore.Warn("Can't get IP addr for %v: %v", host, err) 73 | } 74 | u.Timeout = 60 * time.Second 75 | u.host = fmt.Sprintf("%v:%v", u.Host, u.Port) 76 | 77 | u.transport = &http.Transport{} 78 | // This dial ignores the addr passed in and dials based on the upstream host and port 79 | u.transport.Dial = func(n, addr string) (c net.Conn, err error) { 80 | falcore.Fine("Dialing connection to %v", u.tcpaddr) 81 | var ctcp *net.TCPConn 82 | ctcp, err = net.DialTCP("tcp4", nil, u.tcpaddr) 83 | if err != nil { 84 | falcore.Error("Dial Failed: %v", err) 85 | return 86 | } 87 | c = &connWrapper{conn: ctcp, timeout: u.Timeout} 88 | return 89 | } 90 | u.transport.MaxIdleConnsPerHost = 15 91 | return u 92 | } 93 | 94 | // Alter the number of connections to multiplex with 95 | func (u *Upstream) SetPoolSize(size int) { 96 | u.transport.MaxIdleConnsPerHost = size 97 | } 98 | 99 | func (u *Upstream) FilterRequest(request *falcore.Request) (res *http.Response) { 100 | var err error 101 | req := request.HttpRequest 102 | 103 | // Force the upstream to use http 104 | if u.ForceHttp || req.URL.Scheme == "" { 105 | req.URL.Scheme = "http" 106 | req.URL.Host = req.Host 107 | } 108 | before := time.Now() 109 | req.Header.Set("Connection", "Keep-Alive") 110 | var upstrRes *http.Response 111 | upstrRes, err = u.transport.RoundTrip(req) 112 | diff := falcore.TimeDiff(before, time.Now()) 113 | if err == nil { 114 | // Copy response over to new record. Remove connection noise. Add some sanity. 115 | res = falcore.SimpleResponse(req, upstrRes.StatusCode, nil, "") 116 | if upstrRes.ContentLength > 0 && upstrRes.Body != nil { 117 | res.ContentLength = upstrRes.ContentLength 118 | res.Body = upstrRes.Body 119 | } else if upstrRes.ContentLength == 0 && upstrRes.Body != nil { 120 | // Any bytes? 121 | var testBuf [1]byte 122 | n, _ := io.ReadFull(upstrRes.Body, testBuf[:]) 123 | if n == 1 { 124 | // Yes there are. Chunked it is. 125 | res.TransferEncoding = []string{"chunked"} 126 | res.ContentLength = -1 127 | rc := &passThruReadCloser{ 128 | io.MultiReader(bytes.NewBuffer(testBuf[:]), upstrRes.Body), 129 | upstrRes.Body, 130 | } 131 | 132 | res.Body = rc 133 | } 134 | } else if upstrRes.Body != nil { 135 | res.Body = upstrRes.Body 136 | res.ContentLength = -1 137 | res.TransferEncoding = []string{"chunked"} 138 | } 139 | // Copy over headers with a few exceptions 140 | res.Header = make(http.Header) 141 | for hn, hv := range upstrRes.Header { 142 | switch hn { 143 | case "Content-Length": 144 | case "Connection": 145 | case "Transfer-Encoding": 146 | default: 147 | res.Header[hn] = hv 148 | } 149 | } 150 | } else { 151 | if nerr, ok := err.(net.Error); ok && nerr.Timeout() { 152 | falcore.Error("%s Upstream Timeout error: %v", request.ID, err) 153 | res = falcore.SimpleResponse(req, 504, nil, "Gateway Timeout\n") 154 | request.CurrentStage.Status = 2 // Fail 155 | } else { 156 | falcore.Error("%s Upstream error: %v", request.ID, err) 157 | res = falcore.SimpleResponse(req, 502, nil, "Bad Gateway\n") 158 | request.CurrentStage.Status = 2 // Fail 159 | } 160 | } 161 | falcore.Debug("%s [%s] [%s] %s s=%d Time=%.4f", request.ID, req.Method, u.host, req.URL, res.StatusCode, diff) 162 | return 163 | } 164 | 165 | func (u *Upstream) ping() (up bool, ok bool) { 166 | if u.PingPath != "" { 167 | // the url must be syntactically valid for this to work but the host will be ignored because we 168 | // are overriding the connection always 169 | request, err := http.NewRequest("GET", "http://localhost"+u.PingPath, nil) 170 | request.Header.Set("Connection", "Keep-Alive") // not sure if this should be here for a ping 171 | if err != nil { 172 | falcore.Error("Bad Ping request: %v", err) 173 | return false, true 174 | } 175 | res, err := u.transport.RoundTrip(request) 176 | 177 | if err != nil { 178 | falcore.Error("Failed Ping to %v:%v: %v", u.Host, u.Port, err) 179 | return false, true 180 | } else { 181 | res.Body.Close() 182 | } 183 | if res.StatusCode == 200 { 184 | return true, true 185 | } 186 | falcore.Error("Failed Ping to %v:%v: %v", u.Host, u.Port, res.Status) 187 | // bad status 188 | return false, true 189 | } 190 | return false, false 191 | } 192 | -------------------------------------------------------------------------------- /upstream/upstream_pool.go: -------------------------------------------------------------------------------- 1 | package upstream 2 | 3 | import ( 4 | "github.com/ngmoco/falcore" 5 | "net/http" 6 | "strconv" 7 | "strings" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | type UpstreamEntryConfig struct { 13 | HostPort string 14 | Weight int 15 | ForceHttp bool 16 | PingPath string 17 | } 18 | 19 | type UpstreamEntry struct { 20 | Upstream *Upstream 21 | Weight int 22 | } 23 | 24 | // An UpstreamPool is a list of upstream servers which are considered 25 | // functionally equivalent. The pool will round-robin the requests to the servers. 26 | type UpstreamPool struct { 27 | pool []*UpstreamEntry 28 | rr_count int 29 | ping_count int64 30 | Name string 31 | nextUpstream chan *UpstreamEntry 32 | shutdown chan int 33 | weightMutex *sync.RWMutex 34 | pinger *time.Ticker 35 | } 36 | 37 | // The config consists of a map of the servers in the pool in the format host_or_ip:port 38 | // where port is optional and defaults to 80. The map value is an int with the weight 39 | // only 0 and 1 are supported weights (0 disables a server and 1 enables it) 40 | func NewUpstreamPool(name string, config []UpstreamEntryConfig) *UpstreamPool { 41 | up := new(UpstreamPool) 42 | up.pool = make([]*UpstreamEntry, len(config)) 43 | up.Name = name 44 | up.nextUpstream = make(chan *UpstreamEntry) 45 | up.weightMutex = new(sync.RWMutex) 46 | up.shutdown = make(chan int) 47 | up.pinger = time.NewTicker(3e9) // 3s 48 | 49 | // create the pool 50 | for i, uec := range config { 51 | parts := strings.Split(uec.HostPort, ":") 52 | upstreamHost := parts[0] 53 | upstreamPort := 80 54 | if len(parts) > 1 { 55 | var err error 56 | upstreamPort, err = strconv.Atoi(parts[1]) 57 | if err != nil { 58 | upstreamPort = 80 59 | falcore.Error("UpstreamPool Error converting port to int for", upstreamHost, ":", err) 60 | } 61 | } 62 | ups := NewUpstream(upstreamHost, upstreamPort, uec.ForceHttp) 63 | ups.PingPath = uec.PingPath 64 | ue := new(UpstreamEntry) 65 | ue.Upstream = ups 66 | ue.Weight = uec.Weight 67 | up.pool[i] = ue 68 | } 69 | go up.nextServer() 70 | go up.pingUpstreams() 71 | return up 72 | } 73 | 74 | func (up UpstreamPool) Next() *UpstreamEntry { 75 | // TODO check in case all are down that we timeout 76 | return <-up.nextUpstream 77 | } 78 | 79 | func (up UpstreamPool) LogStatus() { 80 | weightsBuffer := make([]int, len(up.pool)) 81 | // loop and save the weights so we don't lock for logging 82 | up.weightMutex.RLock() 83 | for i, ue := range up.pool { 84 | weightsBuffer[i] = ue.Weight 85 | } 86 | up.weightMutex.RUnlock() 87 | // Now do the logging 88 | for i, ue := range up.pool { 89 | falcore.Info("Upstream %v: %v:%v\t%v", up.Name, ue.Upstream.Host, ue.Upstream.Port, weightsBuffer[i]) 90 | } 91 | } 92 | 93 | func (up UpstreamPool) FilterRequest(req *falcore.Request) (res *http.Response) { 94 | ue := up.Next() 95 | res = ue.Upstream.FilterRequest(req) 96 | if req.CurrentStage.Status == 2 { 97 | // this gets set by the upstream for errors 98 | // so mark this upstream as down 99 | up.updateUpstream(ue, 0) 100 | up.LogStatus() 101 | } 102 | return 103 | } 104 | 105 | func (up UpstreamPool) updateUpstream(ue *UpstreamEntry, wgt int) { 106 | up.weightMutex.Lock() 107 | ue.Weight = wgt 108 | up.weightMutex.Unlock() 109 | } 110 | 111 | // This should only be called if the upstream pool is no longer active or this may deadlock 112 | func (up UpstreamPool) Shutdown() { 113 | // ping and nextServer 114 | close(up.shutdown) 115 | 116 | // make sure we hit the shutdown code in the nextServer goroutine 117 | up.Next() 118 | } 119 | 120 | func (up UpstreamPool) nextServer() { 121 | loopCount := 0 122 | for { 123 | next := up.rr_count % len(up.pool) 124 | up.weightMutex.RLock() 125 | wgt := up.pool[next].Weight 126 | up.weightMutex.RUnlock() 127 | // just return a down host if we've gone through the list twice and nothing is up 128 | // be sure to never return negative wgt hosts 129 | if (wgt > 0 || (loopCount > 2*len(up.pool))) && wgt >= 0 { 130 | loopCount = 0 131 | select { 132 | case <-up.shutdown: 133 | return 134 | case up.nextUpstream <- up.pool[next]: 135 | } 136 | } else { 137 | loopCount++ 138 | } 139 | up.rr_count++ 140 | } 141 | } 142 | 143 | func (up UpstreamPool) pingUpstreams() { 144 | pingable := true 145 | for pingable { 146 | select { 147 | case <-up.shutdown: 148 | return 149 | case <-up.pinger.C: 150 | gotone := false 151 | for i, ups := range up.pool { 152 | if ups.Upstream.PingPath != "" { 153 | go up.pingUpstream(ups, i) 154 | gotone = true 155 | } 156 | } 157 | if !gotone { 158 | pingable = false 159 | } 160 | } 161 | } 162 | falcore.Warn("Stopping ping for %v", up.Name) 163 | } 164 | 165 | func (up UpstreamPool) pingUpstream(ups *UpstreamEntry, index int) { 166 | isUp, ok := ups.Upstream.ping() 167 | up.weightMutex.RLock() 168 | wgt := ups.Weight 169 | up.weightMutex.RUnlock() 170 | // change in status 171 | if ok && (wgt > 0) != isUp { 172 | if isUp { 173 | up.updateUpstream(ups, 1) 174 | } else { 175 | up.updateUpstream(ups, 0) 176 | } 177 | up.LogStatus() 178 | } 179 | } 180 | --------------------------------------------------------------------------------