├── test
├── custom_type.foo
├── hello
│ └── world.txt
├── foo.json
├── images
│ └── face.png
└── index.html
├── doc.go
├── .gitignore
├── server_windows.go
├── continue.go
├── handler_filter_test.go
├── examples
├── hello_world
│ └── hello_world.go
├── basic_file_server
│ └── server.go
├── blog_example
│ └── blog_example.go
└── hot_restart
│ └── hot_restart.go
├── LICENSE.md
├── etag
├── etag.go
└── etag_test.go
├── response.go
├── router_test.go
├── server_notwindows.go
├── buffer_pool_test.go
├── buffer_pool.go
├── filter.go
├── static_file
├── file_filter.go
└── file_filter_test.go
├── string_body_test.go
├── string_body.go
├── compression
├── compression.go
└── compression_test.go
├── handler_filter.go
├── router.go
├── pipeline.go
├── README.md
├── logger.go
├── upstream
├── upstream_pool.go
└── upstream.go
├── pipeline_test.go
├── request.go
└── server.go
/test/custom_type.foo:
--------------------------------------------------------------------------------
1 | custom type!
--------------------------------------------------------------------------------
/test/hello/world.txt:
--------------------------------------------------------------------------------
1 | Hello world!
--------------------------------------------------------------------------------
/test/foo.json:
--------------------------------------------------------------------------------
1 | {"all": "your", "base": [1, 2, 3]}
--------------------------------------------------------------------------------
/test/images/face.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ngmoco/falcore/HEAD/test/images/face.png
--------------------------------------------------------------------------------
/doc.go:
--------------------------------------------------------------------------------
1 | // Package falcore is a framework for constructing high performance, modular HTTP servers.
2 | // For more information, see README.md
3 | package falcore
4 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.6
2 | _*
3 | 6.out
4 | *.swp
5 | tags
6 | bin
7 | examples/hot_restart/hot_restart
8 | examples/hello_world/hello_world
9 | examples/basic_file_server/basic_file_server
10 |
11 |
--------------------------------------------------------------------------------
/server_windows.go:
--------------------------------------------------------------------------------
1 | // +build windows
2 | package falcore
3 |
4 | import (
5 | "net"
6 | )
7 |
8 | // only valid on non-windows
9 | func (srv *Server) setupNonBlockingListener(err error, l *net.TCPListener) error {
10 | return nil
11 | }
12 |
13 | func (srv *Server) cycleNonBlock(c net.Conn) {
14 | // nuthin
15 | }
16 |
--------------------------------------------------------------------------------
/test/index.html:
--------------------------------------------------------------------------------
1 |
3 |
4 |
5 |
6 |
7 |
8 | Example Page!
9 |
10 |
11 |
12 |
13 |
14 | WOOOOOOO!
15 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/continue.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "io"
5 | )
6 |
7 | type continueReader struct {
8 | req *Request
9 | r io.ReadCloser
10 | opened bool
11 | }
12 |
13 | var _ io.ReadCloser = new(continueReader)
14 |
15 | func (r *continueReader) Read(p []byte) (int, error) {
16 | // sent 100 continue the first time we try to read the body
17 | if !r.opened {
18 | resp := SimpleResponse(r.req.HttpRequest, 100, nil, "")
19 | if err := resp.Write(r.req.Connection); err != nil {
20 | return 0, err
21 | }
22 | r.req = nil
23 | r.opened = true
24 | }
25 | return r.r.Read(p)
26 | }
27 |
28 | func (r *continueReader) Close() error {
29 | r.req = nil
30 | return r.r.Close()
31 | }
32 |
--------------------------------------------------------------------------------
/handler_filter_test.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "fmt"
5 | "io/ioutil"
6 | "net/http"
7 | "testing"
8 | )
9 |
10 | func TestHandlerFilter(t *testing.T) {
11 | reply := "Hello, World"
12 | handler := func(w http.ResponseWriter, r *http.Request) {
13 | fmt.Fprintf(w, reply)
14 | }
15 |
16 | hff := NewHandlerFilter(http.HandlerFunc(handler))
17 |
18 | tmp, _ := http.NewRequest("GET", "/hello", nil)
19 | _, res := TestWithRequest(tmp, hff, nil)
20 |
21 | if res == nil {
22 | t.Errorf("Response is nil")
23 | }
24 |
25 | if replyGot, err := ioutil.ReadAll(res.Body); err != nil {
26 | t.Errorf("Error reading body: %v", err)
27 | } else if string(replyGot) != reply {
28 | t.Errorf("Expected body does not match")
29 | }
30 |
31 | }
32 |
--------------------------------------------------------------------------------
/examples/hello_world/hello_world.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "flag"
5 | "fmt"
6 | "github.com/ngmoco/falcore"
7 | "net/http"
8 | )
9 |
10 | // Command line options
11 | var (
12 | port = flag.Int("port", 8000, "the port to listen on")
13 | )
14 |
15 | func main() {
16 | // parse command line options
17 | flag.Parse()
18 |
19 | // setup pipeline
20 | pipeline := falcore.NewPipeline()
21 |
22 | // upstream
23 | pipeline.Upstream.PushBack(helloFilter)
24 |
25 | // setup server
26 | server := falcore.NewServer(*port, pipeline)
27 |
28 | // start the server
29 | // this is normally blocking forever unless you send lifecycle commands
30 | if err := server.ListenAndServe(); err != nil {
31 | fmt.Println("Could not start server:", err)
32 | }
33 | }
34 |
35 | var helloFilter = falcore.NewRequestFilter(func(req *falcore.Request) *http.Response {
36 | return falcore.SimpleResponse(req.HttpRequest, 200, nil, "hello world!")
37 | })
38 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | Copyright (c) 2012 ngmoco:) inc.
2 |
3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
4 |
5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
6 |
7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
8 |
9 |
--------------------------------------------------------------------------------
/etag/etag.go:
--------------------------------------------------------------------------------
1 | package etag
2 |
3 | import (
4 | "github.com/ngmoco/falcore"
5 | "net/http"
6 | )
7 |
8 | // falcore/etag.Filter is a falcore.ResponseFilter that matches
9 | // the response's Etag header against the request's If-None-Match
10 | // header. If they match, the filter will return a '304 Not Modifed'
11 | // status and no body.
12 | //
13 | // Ideally, Etag filtering is performed as soon as possible as
14 | // you may be able to skip generating the response body at all.
15 | // Even as a last step, you will see a significant benefit if
16 | // clients are well behaved.
17 | //
18 | type Filter struct {
19 | }
20 |
21 | func (f *Filter) FilterResponse(request *falcore.Request, res *http.Response) {
22 | request.CurrentStage.Status = 1 // Skipped (default)
23 | if if_none_match := request.HttpRequest.Header.Get("If-None-Match"); if_none_match != "" {
24 | if res.StatusCode == 200 && res.Header.Get("Etag") == if_none_match {
25 | res.StatusCode = 304
26 | res.Status = "304 Not Modified"
27 | res.Body.Close()
28 | res.Body = nil
29 | res.ContentLength = 0
30 | request.CurrentStage.Status = 0 // Success
31 | }
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/response.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "net/http"
5 | "strings"
6 | )
7 |
8 | func SimpleResponse(req *http.Request, status int, headers http.Header, body string) *http.Response {
9 | res := new(http.Response)
10 | body_rdr := (*fixedResBody)(strings.NewReader(body))
11 | res.StatusCode = status
12 | res.ProtoMajor = 1
13 | res.ProtoMinor = 1
14 | res.ContentLength = int64((*strings.Reader)(body_rdr).Len())
15 | res.Request = req
16 | res.Header = make(map[string][]string)
17 | res.Body = body_rdr
18 | if headers != nil {
19 | res.Header = headers
20 | }
21 | return res
22 | }
23 |
24 | // string type for response objects
25 |
26 | type fixedResBody strings.Reader
27 |
28 | func (s *fixedResBody) Close() error {
29 | return nil
30 | }
31 |
32 | func (s *fixedResBody) Read(b []byte) (int, error) {
33 | return (*strings.Reader)(s).Read(b)
34 | }
35 |
36 | func RedirectResponse(req *http.Request, url string) *http.Response {
37 | res := new(http.Response)
38 | res.StatusCode = 302
39 | res.ProtoMajor = 1
40 | res.ProtoMinor = 1
41 | res.ContentLength = 0
42 | res.Request = req
43 | res.Header = make(map[string][]string)
44 | res.Header.Set("Location", url)
45 | return res
46 | }
47 |
--------------------------------------------------------------------------------
/router_test.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "net/http"
5 | "regexp"
6 | "testing"
7 | )
8 |
9 | type SimpleFilter int
10 |
11 | func (sf SimpleFilter) FilterRequest(req *Request) *http.Response {
12 | sf = -sf
13 | return nil
14 | }
15 |
16 | func TestRegexpRoute(t *testing.T) {
17 | r := new(RegexpRoute)
18 |
19 | var sf1 SimpleFilter = 1
20 | r.Filter = sf1
21 | r.Match = regexp.MustCompile(`one`)
22 |
23 | if r.MatchString("http://tester.com/one") != sf1 {
24 | t.Errorf("Failed to match regexp")
25 | }
26 | if r.MatchString("http://tester.com/two") != nil {
27 | t.Errorf("False regexp match")
28 | }
29 |
30 | }
31 |
32 | func TestHostRouter(t *testing.T) {
33 | hr := NewHostRouter()
34 |
35 | var sf1 SimpleFilter = 1
36 | var sf2 SimpleFilter = 2
37 | hr.AddMatch("www.ngmoco.com", sf1)
38 | hr.AddMatch("developer.ngmoco.com", sf2)
39 |
40 | req := validGetRequest()
41 | req.HttpRequest.Host = "developer.ngmoco.com"
42 |
43 | filt := hr.SelectPipeline(req)
44 | if filt != sf2 {
45 | t.Errorf("Host router didn't get the right pipeline")
46 | }
47 |
48 | req.HttpRequest.Host = "ngmoco.com"
49 | filt = hr.SelectPipeline(req)
50 | if filt != nil {
51 | t.Errorf("Host router got currently unsupported fuzzy match so you should update this test")
52 | }
53 | }
54 |
--------------------------------------------------------------------------------
/server_notwindows.go:
--------------------------------------------------------------------------------
1 | // +build !windows
2 |
3 | package falcore
4 |
5 | import (
6 | "net"
7 | "syscall"
8 | )
9 |
10 | // only valid on non-windows
11 | func (srv *Server) setupNonBlockingListener(err error, l *net.TCPListener) error {
12 | // FIXME: File() returns a copied pointer. we're leaking it. probably doesn't matter
13 | if srv.listenerFile, err = l.File(); err != nil {
14 | return err
15 | }
16 | fd := int(srv.listenerFile.Fd())
17 | if e := syscall.SetNonblock(fd, true); e != nil {
18 | return e
19 | }
20 | if srv.sendfile {
21 | if e := syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, srv.sockOpt, 1); e != nil {
22 | return e
23 | }
24 | }
25 | return nil
26 | }
27 |
28 | func (srv *Server) cycleNonBlock(c net.Conn) {
29 | if srv.sendfile {
30 | if tcpC, ok := c.(*net.TCPConn); ok {
31 | if f, err := tcpC.File(); err == nil {
32 | // f is a copy. must be closed
33 | defer f.Close()
34 | fd := int(f.Fd())
35 | // Disable TCP_CORK/TCP_NOPUSH
36 | syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, srv.sockOpt, 0)
37 | // For TCP_NOPUSH, we need to force flush
38 | c.Write([]byte{})
39 | // Re-enable TCP_CORK/TCP_NOPUSH
40 | syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, srv.sockOpt, 1)
41 | syscall.SetNonblock(fd, true)
42 | }
43 | }
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/examples/basic_file_server/server.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "flag"
5 | "fmt"
6 | "github.com/ngmoco/falcore"
7 | "github.com/ngmoco/falcore/compression"
8 | "github.com/ngmoco/falcore/static_file"
9 | "net/http"
10 | )
11 |
12 | // Command line options
13 | var (
14 | port = flag.Int("port", 8000, "the port to listen on")
15 | path = flag.String("base", "./test", "the path to serve files from")
16 | )
17 |
18 | func main() {
19 | // parse command line options
20 | flag.Parse()
21 |
22 | // setup pipeline
23 | pipeline := falcore.NewPipeline()
24 |
25 | // upstream filters
26 |
27 | // Serve index.html for root requests
28 | pipeline.Upstream.PushBack(falcore.NewRequestFilter(func(req *falcore.Request) *http.Response {
29 | if req.HttpRequest.URL.Path == "/" {
30 | req.HttpRequest.URL.Path = "/index.html"
31 | }
32 | return nil
33 | }))
34 | // Serve files
35 | pipeline.Upstream.PushBack(&static_file.Filter{
36 | BasePath: *path,
37 | })
38 |
39 | // downstream
40 | pipeline.Downstream.PushBack(compression.NewFilter(nil))
41 |
42 | // setup server
43 | server := falcore.NewServer(*port, pipeline)
44 |
45 | // start the server
46 | // this is normally blocking forever unless you send lifecycle commands
47 | if err := server.ListenAndServe(); err != nil {
48 | fmt.Println("Could not start server:", err)
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/buffer_pool_test.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | /* "io"*/
5 | "bytes"
6 | "testing"
7 | )
8 |
9 | func TestBufferPool(t *testing.T) {
10 | pool := newBufferPool(10, 1024)
11 |
12 | text := []byte("Hello World")
13 |
14 | // get one
15 | bpe := pool.take(bytes.NewBuffer(text))
16 | // read all
17 | out := make([]byte, 1024)
18 | l, _ := bpe.br.Read(out)
19 | if bytes.Compare(out[0:l], text) != 0 {
20 | t.Errorf("Read invalid data: %v", out)
21 | }
22 | if l != len(text) {
23 | t.Errorf("Expected length %v got %v", len(text), l)
24 | }
25 | pool.give(bpe)
26 |
27 | // get one
28 | bpe = pool.take(bytes.NewBuffer(text))
29 | // read all
30 | out = make([]byte, 1024)
31 | l, _ = bpe.br.Read(out)
32 | if bytes.Compare(out[0:l], text) != 0 {
33 | t.Errorf("Read invalid data: %v", out)
34 | }
35 | if l != len(text) {
36 | t.Errorf("Expected length %v got %v", len(text), l)
37 | }
38 | pool.give(bpe)
39 |
40 | // get one
41 | bpe = pool.take(bytes.NewBuffer(text))
42 | // read 1 byte
43 | out = make([]byte, 1)
44 | bpe.br.Read(out)
45 | pool.give(bpe)
46 |
47 | // get one
48 | bpe = pool.take(bytes.NewBuffer(text))
49 | // read all
50 | out = make([]byte, 1024)
51 | l, _ = bpe.br.Read(out)
52 | if bytes.Compare(out[0:l], text) != 0 {
53 | t.Errorf("Read invalid data: %v", out)
54 | }
55 | if l != len(text) {
56 | t.Errorf("Expected length %v got %v", len(text), l)
57 | }
58 | pool.give(bpe)
59 |
60 | }
61 |
--------------------------------------------------------------------------------
/examples/blog_example/blog_example.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "github.com/ngmoco/falcore"
6 | "math/rand"
7 | "net/http"
8 | "time"
9 | )
10 |
11 | func main() {
12 | // create pipeline
13 | pipeline := falcore.NewPipeline()
14 |
15 | // add upstream pipeline stages
16 | var filter1 delayFilter
17 | pipeline.Upstream.PushBack(filter1)
18 | var filter2 helloFilter
19 | pipeline.Upstream.PushBack(filter2)
20 |
21 | // add request done callback stage
22 | pipeline.RequestDoneCallback = reqCB
23 |
24 | // create server on port 8000
25 | server := falcore.NewServer(8000, pipeline)
26 |
27 | // start the server
28 | // this is normally blocking forever unless you send lifecycle commands
29 | if err := server.ListenAndServe(); err != nil {
30 | fmt.Println("Could not start server:", err)
31 | }
32 | }
33 |
34 | // Example filter to show timing features
35 | type delayFilter int
36 |
37 | func (f delayFilter) FilterRequest(req *falcore.Request) *http.Response {
38 | status := rand.Intn(2) // random status 0 or 1
39 | if status == 0 {
40 | time.Sleep(time.Duration(rand.Int63n(100e6))) // random sleep between 0 and 100 ms
41 | }
42 | req.CurrentStage.Status = byte(status) // set the status to produce a unique signature
43 | return nil
44 | }
45 |
46 | // Example filter that returns a Response
47 | type helloFilter int
48 |
49 | func (f helloFilter) FilterRequest(req *falcore.Request) *http.Response {
50 | return falcore.SimpleResponse(req.HttpRequest, 200, nil, "hello world!\n")
51 | }
52 |
53 | var reqCB = falcore.NewRequestFilter(func(req *falcore.Request) *http.Response {
54 | req.Trace() // Prints detailed stats about the request to the log
55 | return nil
56 | })
57 |
--------------------------------------------------------------------------------
/buffer_pool.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "bufio"
5 | "io"
6 | "io/ioutil"
7 | )
8 |
9 | // uses a chan as a leaky bucket buffer pool
10 | type bufferPool struct {
11 | // size of buffer when creating new ones
12 | bufSize int
13 | // the actual pool of buffers ready for reuse
14 | pool chan *bufferPoolEntry
15 | }
16 |
17 | // This is what's stored in the buffer. It allows
18 | // for the underlying io.Reader to be changed out
19 | // inside a bufio.Reader. This is required for reuse.
20 | type bufferPoolEntry struct {
21 | br *bufio.Reader
22 | source io.Reader
23 | }
24 |
25 | // make bufferPoolEntry a passthrough io.Reader
26 | func (bpe *bufferPoolEntry) Read(p []byte) (n int, err error) {
27 | return bpe.source.Read(p)
28 | }
29 |
30 | func newBufferPool(poolSize, bufferSize int) *bufferPool {
31 | return &bufferPool{
32 | bufSize: bufferSize,
33 | pool: make(chan *bufferPoolEntry, poolSize),
34 | }
35 | }
36 |
37 | // Take a buffer from the pool and set
38 | // it up to read from r
39 | func (p *bufferPool) take(r io.Reader) (bpe *bufferPoolEntry) {
40 | select {
41 | case bpe = <-p.pool:
42 | // prepare for reuse
43 | if a := bpe.br.Buffered(); a > 0 {
44 | // drain the internal buffer
45 | io.CopyN(ioutil.Discard, bpe.br, int64(a))
46 | }
47 | // swap out the underlying reader
48 | bpe.source = r
49 | default:
50 | // none available. create a new one
51 | bpe = &bufferPoolEntry{nil, r}
52 | bpe.br = bufio.NewReaderSize(bpe, p.bufSize)
53 | }
54 | return
55 | }
56 |
57 | // Return a buffer to the pool
58 | func (p *bufferPool) give(bpe *bufferPoolEntry) {
59 | select {
60 | case p.pool <- bpe: // return to pool
61 | default: // discard
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
/filter.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "net/http"
5 | )
6 |
7 | // Filter incomming requests and optionally return a response or nil.
8 | // Filters are chained together into a flow (the Pipeline) which will terminate
9 | // if the Filter returns a response.
10 | type RequestFilter interface {
11 | FilterRequest(req *Request) *http.Response
12 | }
13 |
14 | // Helper to create a Filter by just passing in a func
15 | // filter = NewRequestFilter(func(req *Request) *http.Response {
16 | // req.Headers.Add("X-Falcore", "is_cool")
17 | // return
18 | // })
19 | func NewRequestFilter(f func(req *Request) *http.Response) RequestFilter {
20 | rf := new(genericRequestFilter)
21 | rf.f = f
22 | return rf
23 | }
24 |
25 | type genericRequestFilter struct {
26 | f func(req *Request) *http.Response
27 | }
28 |
29 | func (f *genericRequestFilter) FilterRequest(req *Request) *http.Response {
30 | return f.f(req)
31 | }
32 |
33 | // Filter outgoing responses. This can be used to modify the response
34 | // before it is sent. Modifying the request at this point will have no
35 | // effect.
36 | type ResponseFilter interface {
37 | FilterResponse(req *Request, res *http.Response)
38 | }
39 |
40 | // Helper to create a Filter by just passing in a func
41 | // filter = NewResponseFilter(func(req *Request, res *http.Response) {
42 | // // some crazy response magic
43 | // return
44 | // })
45 | func NewResponseFilter(f func(req *Request, res *http.Response)) ResponseFilter {
46 | rf := new(genericResponseFilter)
47 | rf.f = f
48 | return rf
49 | }
50 |
51 | type genericResponseFilter struct {
52 | f func(req *Request, res *http.Response)
53 | }
54 |
55 | func (f *genericResponseFilter) FilterResponse(req *Request, res *http.Response) {
56 | f.f(req, res)
57 | }
58 |
--------------------------------------------------------------------------------
/static_file/file_filter.go:
--------------------------------------------------------------------------------
1 | package static_file
2 |
3 | import (
4 | "github.com/ngmoco/falcore"
5 | "mime"
6 | "net/http"
7 | "os"
8 | "path/filepath"
9 | "strings"
10 | )
11 |
12 | // A falcore RequestFilter for serving static files
13 | // from the filesystem.
14 | type Filter struct {
15 | // File system base path for serving files
16 | BasePath string
17 | // Prefix in URL path
18 | PathPrefix string
19 | }
20 |
21 | func (f *Filter) FilterRequest(req *falcore.Request) (res *http.Response) {
22 | // Clean asset path
23 | asset_path := filepath.Clean(filepath.FromSlash(req.HttpRequest.URL.Path))
24 |
25 | // Resolve PathPrefix
26 | if strings.HasPrefix(asset_path, f.PathPrefix) {
27 | asset_path = asset_path[len(f.PathPrefix):]
28 | } else {
29 | falcore.Debug("%v doesn't match prefix %v", asset_path, f.PathPrefix)
30 | res = falcore.SimpleResponse(req.HttpRequest, 404, nil, "Not found.")
31 | return
32 | }
33 |
34 | // Resolve FSBase
35 | if f.BasePath != "" {
36 | asset_path = filepath.Join(f.BasePath, asset_path)
37 | } else {
38 | falcore.Error("file_filter requires a BasePath")
39 | return falcore.SimpleResponse(req.HttpRequest, 500, nil, "Server Error\n")
40 | }
41 |
42 | // Open File
43 | if file, err := os.Open(asset_path); err == nil {
44 | // Make sure it's an actual file
45 | if stat, err := file.Stat(); err == nil && stat.Mode()&os.ModeType == 0 {
46 | res = &http.Response{
47 | Request: req.HttpRequest,
48 | StatusCode: 200,
49 | Proto: "HTTP/1.1",
50 | ProtoMajor: 1,
51 | ProtoMinor: 1,
52 | Body: file,
53 | Header: make(http.Header),
54 | ContentLength: stat.Size(),
55 | }
56 | if ct := mime.TypeByExtension(filepath.Ext(asset_path)); ct != "" {
57 | res.Header.Set("Content-Type", ct)
58 | }
59 | } else {
60 | file.Close()
61 | }
62 | } else {
63 | falcore.Finest("Can't open %v: %v", asset_path, err)
64 | }
65 | return
66 | }
67 |
--------------------------------------------------------------------------------
/string_body_test.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "bytes"
5 | "io/ioutil"
6 | "net/http"
7 | "testing"
8 | "time"
9 | //"io"
10 | )
11 |
12 | func TestStringBody(t *testing.T) {
13 | expected := []byte("HOT HOT HOT!!!")
14 | tmp, _ := http.NewRequest("POST", "/hello", bytes.NewReader(expected))
15 | tmp.Header.Set("Content-Type", "text/plain")
16 | tmp.ContentLength = int64(len(expected))
17 | req := newRequest(tmp, nil, time.Now())
18 | req.startPipelineStage("StringBodyTest")
19 |
20 | sbf := NewStringBodyFilter()
21 | //sbf := &StringBodyFilter{}
22 | sbf.FilterRequest(req)
23 |
24 | if sb, ok := req.HttpRequest.Body.(*StringBody); ok {
25 | readin, _ := ioutil.ReadAll(sb)
26 | sb.Close()
27 | if bytes.Compare(readin, expected) != 0 {
28 | t.Errorf("Body string not read %q expected %q", readin, expected)
29 | }
30 | } else {
31 | t.Errorf("Body not replaced with StringBody")
32 | }
33 |
34 | if req.CurrentStage.Status != 0 {
35 | t.Errorf("SBF failed to parse POST with status %d", req.CurrentStage.Status)
36 | }
37 |
38 | var body []byte = make([]byte, 100)
39 | l, _ := req.HttpRequest.Body.Read(body)
40 | if bytes.Compare(body[0:l], expected) != 0 {
41 | t.Errorf("Failed to read the right bytes %q expected %q", body, expected)
42 |
43 | }
44 |
45 | l, _ = req.HttpRequest.Body.Read(body)
46 | if l != 0 {
47 | t.Errorf("Should have read zero!")
48 | }
49 |
50 | // Close resets the buffer
51 | req.HttpRequest.Body.Close()
52 |
53 | l, _ = req.HttpRequest.Body.Read(body)
54 | if bytes.Compare(body[0:l], expected) != 0 {
55 | t.Errorf("Failed to read the right bytes after calling Close %q expected %q", body, expected)
56 |
57 | }
58 |
59 | }
60 |
61 | func BenchmarkStringBody(b *testing.B) {
62 | b.StopTimer()
63 | expected := []byte("test=123456&test2=987654&test3=somedatanstuff&test4=moredataontheend")
64 | expLen := int64(len(expected))
65 | req := newRequest(nil, nil, time.Now())
66 | req.startPipelineStage("StringBodyTest")
67 |
68 | sbf := NewStringBodyFilter()
69 | //sbf := &StringBodyFilter{}
70 |
71 | for i := 0; i < b.N; i++ {
72 | tmp, _ := http.NewRequest("POST", "/hello", bytes.NewReader(expected))
73 | tmp.Header.Set("Content-Type", "application/x-www-form-urlencoded")
74 | tmp.ContentLength = expLen
75 | req.HttpRequest = tmp
76 | b.StartTimer()
77 | // replace the body
78 | sbf.FilterRequest(req)
79 | sbf.ReturnBuffer(req)
80 | // read the body twice
81 | /* nah, this isn't so useful
82 | io.CopyN(ioutil.Discard, req.HttpRequest.Body, req.HttpRequest.ContentLength)
83 | req.HttpRequest.Body .Close()
84 | io.CopyN(ioutil.Discard, req.HttpRequest.Body, req.HttpRequest.ContentLength)
85 | req.HttpRequest.Body .Close()
86 | */
87 | b.StopTimer()
88 | }
89 | }
90 |
--------------------------------------------------------------------------------
/string_body.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "bytes"
5 | "io"
6 | "net/http"
7 | "strings"
8 | )
9 |
10 | // Keeps the body of a request in a string so it can be re-read at each stage of the pipeline
11 | // implements io.ReadCloser to match http.Request.Body
12 |
13 | type StringBody struct {
14 | BodyBuffer *bytes.Reader
15 | bpe *bufferPoolEntry
16 | }
17 |
18 | type StringBodyFilter struct {
19 | pool *bufferPool
20 | }
21 |
22 | func NewStringBodyFilter() *StringBodyFilter {
23 | sbf := &StringBodyFilter{}
24 | sbf.pool = newBufferPool(100, 1024)
25 | return sbf
26 | }
27 | func (sbf *StringBodyFilter) FilterRequest(request *Request) *http.Response {
28 | req := request.HttpRequest
29 | // This caches the request body so that multiple filters can iterate it
30 | if req.Method == "POST" || req.Method == "PUT" {
31 | sb, err := sbf.readRequestBody(req)
32 | if sb == nil || err != nil {
33 | request.CurrentStage.Status = 3 // Skip
34 | Debug("%s No Req Body or Ignored: %v", request.ID, err)
35 | }
36 | } else {
37 | request.CurrentStage.Status = 1 // Skip
38 | }
39 | return nil
40 | }
41 |
42 | // reads the request body and replaces the buffer with self
43 | // returns nil if the body is multipart and not replaced
44 | func (sbf *StringBodyFilter) readRequestBody(r *http.Request) (sb *StringBody, err error) {
45 | ct := r.Header.Get("Content-Type")
46 | // leave it on the buffer if we're multipart
47 | if strings.SplitN(ct, ";", 2)[0] != "multipart/form-data" && r.ContentLength > 0 {
48 | sb = &StringBody{}
49 | const maxFormSize = int64(10 << 20) // 10 MB is a lot of text.
50 | sb.bpe = sbf.pool.take(io.LimitReader(r.Body, maxFormSize+1))
51 |
52 | // There shouldn't be a null byte so we should get EOF
53 | b, e := sb.bpe.br.ReadBytes(0)
54 | if e != nil && e != io.EOF {
55 | return nil, e
56 | }
57 | sb.BodyBuffer = bytes.NewReader(b)
58 | r.Body.Close()
59 | r.Body = sb
60 | return sb, nil
61 | }
62 | return nil, nil // ignore
63 | }
64 |
65 | // Returns a buffer used in the FilterRequest stage to a buffer pool
66 | // this speeds up this filter significantly by reusing buffers
67 | func (sbf *StringBodyFilter) ReturnBuffer(request *Request) {
68 | if sb, ok := request.HttpRequest.Body.(*StringBody); ok {
69 | sbf.pool.give(sb.bpe)
70 | }
71 | }
72 |
73 | // Insert this in the response pipeline to return the buffer pool for the request body
74 | // If there is an appropriate place in your flow, you can call ReturnBuffer explicitly
75 | func (sbf *StringBodyFilter) FilterResponse(request *Request, res *http.Response) {
76 | sbf.ReturnBuffer(request)
77 | }
78 |
79 | func (sb *StringBody) Read(b []byte) (n int, err error) {
80 | return sb.BodyBuffer.Read(b)
81 | }
82 |
83 | func (sb *StringBody) Close() error {
84 | // start over
85 | sb.BodyBuffer.Seek(0, 0)
86 | return nil
87 | }
88 |
--------------------------------------------------------------------------------
/compression/compression.go:
--------------------------------------------------------------------------------
1 | package compression
2 |
3 | import (
4 | "bytes"
5 | "compress/flate"
6 | "compress/gzip"
7 | "github.com/ngmoco/falcore"
8 | "io"
9 | "net/http"
10 | "strings"
11 | )
12 |
13 | var DefaultTypes = []string{"text/plain", "text/html", "application/json", "text/xml"}
14 |
15 | type Filter struct {
16 | types []string
17 | }
18 |
19 | func NewFilter(types []string) *Filter {
20 | f := new(Filter)
21 | if types != nil {
22 | f.types = types
23 | } else {
24 | f.types = DefaultTypes
25 | }
26 | return f
27 | }
28 |
29 | func (c *Filter) FilterResponse(request *falcore.Request, res *http.Response) {
30 | req := request.HttpRequest
31 | if accept := req.Header.Get("Accept-Encoding"); accept != "" {
32 |
33 | // Is content an acceptable type for encoding?
34 | var compress = false
35 | var content_type = res.Header.Get("Content-Type")
36 | for _, t := range c.types {
37 | if content_type == t {
38 | compress = true
39 | break
40 | }
41 | }
42 |
43 | // Is the content already compressed
44 | if res.Header.Get("Content-Encoding") != "" {
45 | compress = false
46 | }
47 |
48 | if !compress {
49 | request.CurrentStage.Status = 1 // Skip
50 | return
51 | }
52 |
53 | // Figure out which encoding to use
54 | options := strings.Split(accept, ",")
55 | var mode string
56 | for _, opt := range options {
57 | if m := strings.TrimSpace(opt); m == "gzip" || m == "deflate" {
58 | mode = m
59 | break
60 | }
61 | }
62 |
63 | var compressor io.WriteCloser
64 | var buf = bytes.NewBuffer(make([]byte, 0, 1024))
65 | switch mode {
66 | case "gzip":
67 | compressor = gzip.NewWriter(buf)
68 | case "deflate":
69 | comp, err := flate.NewWriter(buf, -1)
70 | if err != nil {
71 | falcore.Error("Compression Error: %v", err)
72 | request.CurrentStage.Status = 1 // Skip
73 | return
74 | }
75 | compressor = comp
76 | default:
77 | request.CurrentStage.Status = 1 // Skip
78 | return
79 | }
80 |
81 | // Perform compression
82 | r := make([]byte, 1024)
83 | var err error
84 | var i int
85 | for err == nil {
86 | i, err = res.Body.Read(r)
87 | compressor.Write(r[0:i])
88 | }
89 | compressor.Close()
90 | res.Body.Close()
91 |
92 | res.ContentLength = int64(buf.Len())
93 | res.Body = (*filteredBody)(buf)
94 | res.Header.Set("Content-Encoding", mode)
95 | } else {
96 | request.CurrentStage.Status = 1 // Skip
97 | }
98 | }
99 |
100 | // wrapper type for Response struct
101 |
102 | type filteredBody bytes.Buffer
103 |
104 | func (b *filteredBody) Read(byt []byte) (int, error) {
105 | i, err := (*bytes.Buffer)(b).Read(byt)
106 | return i, err
107 | }
108 |
109 | func (b filteredBody) Close() error {
110 | return nil
111 | }
112 |
--------------------------------------------------------------------------------
/handler_filter.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "fmt"
5 | "io"
6 | "net/http"
7 | )
8 |
9 | // Implements a RequestFilter using a http.Handler to produce the response
10 | // This will always return a response due to the requirements of the http.Handler
11 | // interface so it should be placed at the end of the Upstream pipeline.
12 | type HandlerFilter struct {
13 | handler http.Handler
14 | }
15 |
16 | func NewHandlerFilter(handler http.Handler) *HandlerFilter {
17 | return &HandlerFilter{handler: handler}
18 | }
19 |
20 | func (h *HandlerFilter) FilterRequest(req *Request) *http.Response {
21 | rw, respc := newPopulateResponseWriter(req.HttpRequest)
22 | // this must be done concurrently so that the HandlerFunc can write the response
23 | // while falcore is copying it to the socket
24 | go func() {
25 | h.handler.ServeHTTP(rw, req.HttpRequest)
26 | rw.finish()
27 | }()
28 | return <-respc
29 | }
30 |
31 | // copied from net/http/filetransport.go
32 | func newPopulateResponseWriter(req *http.Request) (*populateResponse, <-chan *http.Response) {
33 | pr, pw := io.Pipe()
34 | rw := &populateResponse{
35 | ch: make(chan *http.Response),
36 | pw: pw,
37 | res: &http.Response{
38 | Proto: "HTTP/1.0",
39 | ProtoMajor: 1,
40 | Header: make(http.Header),
41 | Close: true,
42 | Body: pr,
43 | Request: req,
44 | },
45 | }
46 | return rw, rw.ch
47 | }
48 |
49 | // populateResponse is a ResponseWriter that populates the *Response
50 | // in res, and writes its body to a pipe connected to the response
51 | // body. Once writes begin or finish() is called, the response is sent
52 | // on ch.
53 | type populateResponse struct {
54 | res *http.Response
55 | ch chan *http.Response
56 | wroteHeader bool
57 | hasContent bool
58 | sentResponse bool
59 | pw *io.PipeWriter
60 | }
61 |
62 | func (pr *populateResponse) finish() {
63 | if !pr.wroteHeader {
64 | pr.WriteHeader(500)
65 | }
66 | if !pr.sentResponse {
67 | pr.sendResponse()
68 | }
69 | pr.pw.Close()
70 | }
71 |
72 | func (pr *populateResponse) sendResponse() {
73 | if pr.sentResponse {
74 | return
75 | }
76 | pr.sentResponse = true
77 |
78 | if pr.hasContent {
79 | pr.res.ContentLength = -1
80 | }
81 | pr.ch <- pr.res
82 | }
83 |
84 | func (pr *populateResponse) Header() http.Header {
85 | return pr.res.Header
86 | }
87 |
88 | func (pr *populateResponse) WriteHeader(code int) {
89 | if pr.wroteHeader {
90 | return
91 | }
92 | pr.wroteHeader = true
93 |
94 | pr.res.StatusCode = code
95 | pr.res.Status = fmt.Sprintf("%d %s", code, http.StatusText(code))
96 | }
97 |
98 | func (pr *populateResponse) Write(p []byte) (n int, err error) {
99 | if !pr.wroteHeader {
100 | pr.WriteHeader(http.StatusOK)
101 | }
102 | pr.hasContent = true
103 | if !pr.sentResponse {
104 | pr.sendResponse()
105 | }
106 | return pr.pw.Write(p)
107 | }
108 |
--------------------------------------------------------------------------------
/router.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "container/list"
5 | "regexp"
6 | )
7 |
8 | // Interface for defining routers
9 | type Router interface {
10 | // Returns a Pipeline or nil if one can't be found
11 | SelectPipeline(req *Request) (pipe RequestFilter)
12 | }
13 |
14 | // Interface for defining individual routes
15 | type Route interface {
16 | // Returns the route's filter if there's a match. nil if there isn't
17 | MatchString(str string) RequestFilter
18 | }
19 |
20 | // Generate a new Router instance using f for SelectPipeline
21 | func NewRouter(f genericRouter) Router {
22 | return f
23 | }
24 |
25 | type genericRouter func(req *Request) (pipe RequestFilter)
26 |
27 | func (f genericRouter) SelectPipeline(req *Request) (pipe RequestFilter) {
28 | return f(req)
29 | }
30 |
31 | // Will match any request. Useful for fallthrough filters.
32 | type MatchAnyRoute struct {
33 | Filter RequestFilter
34 | }
35 |
36 | func (r *MatchAnyRoute) MatchString(str string) RequestFilter {
37 | return r.Filter
38 | }
39 |
40 | // Will match based on a regular expression
41 | type RegexpRoute struct {
42 | Match *regexp.Regexp
43 | Filter RequestFilter
44 | }
45 |
46 | func (r *RegexpRoute) MatchString(str string) RequestFilter {
47 | if r.Match.MatchString(str) {
48 | return r.Filter
49 | }
50 | return nil
51 | }
52 |
53 | // Route requsts based on hostname
54 | type HostRouter struct {
55 | hosts map[string]RequestFilter
56 | }
57 |
58 | // Generate a new HostRouter instance
59 | func NewHostRouter() *HostRouter {
60 | r := new(HostRouter)
61 | r.hosts = make(map[string]RequestFilter)
62 | return r
63 | }
64 |
65 | // TODO: support for non-exact matches
66 | func (r *HostRouter) AddMatch(host string, pipe RequestFilter) {
67 | r.hosts[host] = pipe
68 | }
69 |
70 | func (r *HostRouter) SelectPipeline(req *Request) (pipe RequestFilter) {
71 | return r.hosts[req.HttpRequest.Host]
72 | }
73 |
74 | // Route requests based on path
75 | type PathRouter struct {
76 | Routes *list.List
77 | }
78 |
79 | // Generate a new instance of PathRouter
80 | func NewPathRouter() *PathRouter {
81 | r := new(PathRouter)
82 | r.Routes = list.New()
83 | return r
84 | }
85 |
86 | func (r *PathRouter) AddRoute(route Route) {
87 | r.Routes.PushBack(route)
88 | }
89 |
90 | // convenience method for adding RegexpRoutes
91 | func (r *PathRouter) AddMatch(match string, filter RequestFilter) (err error) {
92 | route := &RegexpRoute{Filter: filter}
93 | if route.Match, err = regexp.Compile(match); err == nil {
94 | r.Routes.PushBack(route)
95 | }
96 | return
97 | }
98 |
99 | // Will panic if r.Routes contains an object that isn't a Route
100 | func (r *PathRouter) SelectPipeline(req *Request) (pipe RequestFilter) {
101 | var route Route
102 | for r := r.Routes.Front(); r != nil; r = r.Next() {
103 | route = r.Value.(Route)
104 | if f := route.MatchString(req.HttpRequest.URL.Path); f != nil {
105 | return f
106 | }
107 | }
108 | return nil
109 | }
110 |
--------------------------------------------------------------------------------
/pipeline.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "container/list"
5 | "log"
6 | "net/http"
7 | "reflect"
8 | )
9 |
10 | // Pipelines have an upstream and downstream list of filters.
11 | // A request is passed through the upstream items in order UNTIL
12 | // a Response is returned. Once a request is returned, it is passed
13 | // through ALL ResponseFilters in the Downstream list, in order.
14 | //
15 | // If no response is generated by any Filters a default 404 response is
16 | // returned.
17 | //
18 | // The RequestDoneCallback (if set) will be called after the request
19 | // has completed. The finished request object will be passed to
20 | // the FilterRequest method for inspection. Changes to the request
21 | // will have no effect and the return value is ignored.
22 | //
23 | //
24 | type Pipeline struct {
25 | Upstream *list.List
26 | Downstream *list.List
27 | RequestDoneCallback RequestFilter
28 | }
29 |
30 | func NewPipeline() (l *Pipeline) {
31 | l = new(Pipeline)
32 | l.Upstream = list.New()
33 | l.Downstream = list.New()
34 | return
35 | }
36 |
37 | // Pipelines are also RequestFilters... wacky eh?
38 | // Be careful though because a Pipeline will always returns a
39 | // response so no Filters after a Pipeline filter will be run.
40 | func (p *Pipeline) FilterRequest(req *Request) *http.Response {
41 | return p.execute(req)
42 | }
43 |
44 | func (p *Pipeline) execute(req *Request) (res *http.Response) {
45 | for e := p.Upstream.Front(); e != nil && res == nil; e = e.Next() {
46 | switch filter := e.Value.(type) {
47 | case Router:
48 | t := reflect.TypeOf(filter)
49 | req.startPipelineStage(t.String())
50 | pipe := filter.SelectPipeline(req)
51 | req.finishPipelineStage()
52 | if pipe != nil {
53 | res = p.execFilter(req, pipe)
54 | if res != nil {
55 | break
56 | }
57 | }
58 | case RequestFilter:
59 | res = p.execFilter(req, filter)
60 | if res != nil {
61 | break
62 | }
63 | default:
64 | log.Printf("%v is not a RequestFilter\n", e.Value)
65 | break
66 | }
67 | }
68 |
69 | if res == nil {
70 | // Error: No response was generated
71 | res = SimpleResponse(req.HttpRequest, 404, nil, "Not found\n")
72 | }
73 |
74 | p.down(req, res)
75 | return
76 | }
77 |
78 | func (p *Pipeline) execFilter(req *Request, filter RequestFilter) *http.Response {
79 | if _, skipTracking := filter.(*Pipeline); !skipTracking {
80 | t := reflect.TypeOf(filter)
81 | req.startPipelineStage(t.String())
82 | defer req.finishPipelineStage()
83 | }
84 | return filter.FilterRequest(req)
85 | }
86 |
87 | func (p *Pipeline) down(req *Request, res *http.Response) {
88 | for e := p.Downstream.Front(); e != nil; e = e.Next() {
89 | if filter, ok := e.Value.(ResponseFilter); ok {
90 | t := reflect.TypeOf(filter)
91 | req.startPipelineStage(t.String())
92 | filter.FilterResponse(req, res)
93 | req.finishPipelineStage()
94 | } else {
95 | // TODO
96 | break
97 | }
98 | }
99 | }
100 |
--------------------------------------------------------------------------------
/etag/etag_test.go:
--------------------------------------------------------------------------------
1 | package etag
2 |
3 | import (
4 | "bufio"
5 | "bytes"
6 | "fmt"
7 | "github.com/ngmoco/falcore"
8 | "io"
9 | "net"
10 | "net/http"
11 | "path"
12 | "testing"
13 | "time"
14 | )
15 |
16 | var srv *falcore.Server
17 |
18 | func init() {
19 | go func() {
20 | // falcore setup
21 | pipeline := falcore.NewPipeline()
22 | pipeline.Upstream.PushBack(falcore.NewRequestFilter(func(req *falcore.Request) *http.Response {
23 | for _, data := range serverData {
24 | if data.path == req.HttpRequest.URL.Path {
25 | header := make(http.Header)
26 | header.Set("Etag", data.etag)
27 | return falcore.SimpleResponse(req.HttpRequest, data.status, header, string(data.body))
28 | }
29 | }
30 | return falcore.SimpleResponse(req.HttpRequest, 404, nil, "Not Found")
31 | }))
32 |
33 | pipeline.Downstream.PushBack(new(Filter))
34 |
35 | srv = falcore.NewServer(0, pipeline)
36 | if err := srv.ListenAndServe(); err != nil {
37 | panic("Could not start falcore")
38 | }
39 | }()
40 | }
41 |
42 | func port() int {
43 | for srv.Port() == 0 {
44 | time.Sleep(1e7)
45 | }
46 | return srv.Port()
47 | }
48 |
49 | var serverData = []struct {
50 | path string
51 | status int
52 | etag string
53 | body []byte
54 | }{
55 | {
56 | "/hello",
57 | 200,
58 | "abc123",
59 | []byte("hello world"),
60 | },
61 | {
62 | "/pre",
63 | 304,
64 | "abc123",
65 | []byte{},
66 | },
67 | }
68 |
69 | var testData = []struct {
70 | name string
71 | // input
72 | path string
73 | etag string
74 | // output
75 | status int
76 | body []byte
77 | }{
78 | {
79 | "no etag",
80 | "/hello",
81 | "",
82 | 200,
83 | []byte("hello world"),
84 | },
85 | {
86 | "match",
87 | "/hello",
88 | "abc123",
89 | 304,
90 | []byte{},
91 | },
92 | {
93 | "pre-filtered",
94 | "/pre",
95 | "abc123",
96 | 304,
97 | []byte{},
98 | },
99 | }
100 |
101 | func get(p string, etag string) (r *http.Response, err error) {
102 | var conn net.Conn
103 | if conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%v", port())); err == nil {
104 | req, _ := http.NewRequest("GET", fmt.Sprintf("http://%v", path.Join(fmt.Sprintf("localhost:%v/", port()), p)), nil)
105 | req.Header.Set("If-None-Match", etag)
106 | req.Write(conn)
107 | buf := bufio.NewReader(conn)
108 | r, err = http.ReadResponse(buf, req)
109 | }
110 | return
111 | }
112 |
113 | func TestEtagFilter(t *testing.T) {
114 | // select{}
115 | for _, test := range testData {
116 | if res, err := get(test.path, test.etag); err == nil {
117 | bodyBuf := new(bytes.Buffer)
118 | io.Copy(bodyBuf, res.Body)
119 | body := bodyBuf.Bytes()
120 | if st := res.StatusCode; st != test.status {
121 | t.Errorf("%v StatusCode mismatch. Expecting: %v Got: %v", test.name, test.status, st)
122 | }
123 | if !bytes.Equal(body, test.body) {
124 | t.Errorf("%v Body mismatch.\n\tExpecting:\n\t%v\n\tGot:\n\t%v", test.name, test.body, body)
125 | }
126 | } else {
127 | t.Errorf("%v HTTP Error %v", test.name, err)
128 | }
129 | }
130 | }
131 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Falcore has Moved
2 |
3 | [Fitstar Falcore »](https://github.com/fitstar/falcore)
4 |
5 | The Fitstar fork has tons of updates that break backwards compatibility so we are leaving this repo live for reference of the older interfaces. It will no longer be maintained here, though so you should upgrade to the Fitstar fork.
6 |
7 | # Falcore
8 |
9 | Falcore is a framework for constructing high performance, modular HTTP servers in Golang.
10 |
11 | [Read more on our blog »](http://ngenuity.ngmoco.com/2012/01/introducing-falcore-and-timber.html)
12 |
13 | [GoPkgDoc](http://gopkgdoc.appspot.com/pkg/github.com/ngmoco/falcore) hosts code documentation for this project.
14 |
15 | ## Features
16 | * Modular and flexible design
17 | * Hot restart hooks for zero-downtime deploys
18 | * Builtin statistics framework
19 | * Builtin logging framework
20 |
21 | ## Design
22 |
23 | Falcore is a filter pipeline based HTTP server library. You can build arbitrarily complicated HTTP services by chaining just a few simple components:
24 |
25 | * `RequestFilters` are the core component. A request filter takes a request and returns a response or nil. Request filters can modify the request as it passes through.
26 | * `ResponseFilters` can modify a response on its way out the door. An example response filter, `compression_filter`, is included. It applies `deflate` or `gzip` compression to the response if the request supplies the proper headers.
27 | * `Pipelines` form one of the two logic components. A pipeline contains a list of `RequestFilters` and a list of `ResponseFilters`. A request is processed through the request filters, in order, until one returns a response. It then passes the response through each of the response filters, in order. A pipeline is a valid `RequestFilter`.
28 | * `Routers` allow you to conditionally follow different pipelines. A router chooses from a set of pipelines. A few basic routers are included, including routing by hostname or requested path. You can implement your own router by implementing `falcore.Router`. `Routers` are not `RequestFilters`, but they can be put into pipelines.
29 |
30 | ## Building
31 |
32 | Falcore is currently targeted at Go 1.0. If you're still using Go r.60.x, you can get the last working version of falcore for r.60 using the tag `last_r60`.
33 |
34 | Check out the project into $GOROOT/src/pkg/github.com/ngmoco/falcore. Build using the `go build` command.
35 |
36 | ## Usage
37 |
38 | See the `examples` directory for usage examples.
39 |
40 | ## HTTPS
41 |
42 | To use falcore to serve HTTPS, simply call `ListenAndServeTLS` instead of `ListenAndServe`. If you want to host SSL and nonSSL out of the same process, simply create two instances of `falcore.Server`. You can give them the same pipeline or share pipeline components.
43 |
44 | ## Maintainers
45 |
46 | * [Dave Grijalva](http://www.github.com/dgrijalva)
47 | * [Scott White](http://www.github.com/smw1218)
48 |
49 | ## Contributors
50 |
51 | * [Graham Anderson](http://www.github.com/gnanderson)
52 | * [Amir Mohammad Saied](http://github.com/amir)
53 | * [James Wynn](https://github.com/jameswynn)
54 |
55 | [gb]: http://code.google.com/p/go-gb/
56 |
--------------------------------------------------------------------------------
/static_file/file_filter_test.go:
--------------------------------------------------------------------------------
1 | package static_file
2 |
3 | import (
4 | "bytes"
5 | "fmt"
6 | "github.com/ngmoco/falcore"
7 | "io"
8 | "io/ioutil"
9 | "log"
10 | "mime"
11 | "net/http"
12 | "strings"
13 | "testing"
14 | "time"
15 | )
16 |
17 | var srv *falcore.Server
18 |
19 | func init() {
20 | // Silence log output
21 | log.SetOutput(nil)
22 |
23 | // setup mime
24 | mime.AddExtensionType(".foo", "foo/bar")
25 | mime.AddExtensionType(".json", "application/json")
26 | mime.AddExtensionType(".txt", "text/plain")
27 | mime.AddExtensionType(".png", "image/png")
28 |
29 | go func() {
30 |
31 | // falcore setup
32 | pipeline := falcore.NewPipeline()
33 | pipeline.Upstream.PushBack(&Filter{
34 | PathPrefix: "/",
35 | BasePath: "../test/",
36 | })
37 | srv = falcore.NewServer(0, pipeline)
38 | if err := srv.ListenAndServe(); err != nil {
39 | panic(fmt.Sprintf("Could not start falcore: %v", err))
40 | }
41 | }()
42 | }
43 |
44 | func port() int {
45 | for srv.Port() == 0 {
46 | time.Sleep(1e7)
47 | }
48 | return srv.Port()
49 | }
50 |
51 | func get(p string) (r *http.Response, err error) {
52 | req, _ := http.NewRequest("GET", fmt.Sprintf("http://%v", fmt.Sprintf("localhost:%v/", port())), nil)
53 | req.URL.Path = p
54 | r, err = http.DefaultTransport.RoundTrip(req)
55 | return
56 | }
57 |
58 | var fourOhFourTests = []struct {
59 | name string
60 | url string
61 | }{
62 | {
63 | name: "basic invalid path",
64 | url: "/this/path/doesnt/exist",
65 | },
66 | {
67 | name: "realtive pathing out of sandbox",
68 | url: "/../README.md",
69 | },
70 | {
71 | name: "directory",
72 | url: "/hello",
73 | },
74 | }
75 |
76 | func TestFourOhFour(t *testing.T) {
77 | for _, test := range fourOhFourTests {
78 | r, err := get(test.url)
79 | if err != nil {
80 | t.Errorf("%v Error getting file:", test.name, err)
81 | continue
82 | }
83 | if r.StatusCode != 404 {
84 | t.Errorf("%v Expected status 404, got %v", test.name, r.StatusCode)
85 | }
86 | }
87 | }
88 |
89 | var basicTests = []struct {
90 | name string
91 | path string
92 | mime string
93 | data []byte
94 | file string
95 | url string
96 | }{
97 | {
98 | name: "small text file",
99 | mime: "text/plain",
100 | path: "fsbase_test/hello/world.txt",
101 | data: []byte("Hello world!"),
102 | url: "/hello/world.txt",
103 | },
104 | {
105 | name: "json file",
106 | mime: "application/json",
107 | path: "fsbase_test/foo.json",
108 | file: "../test/foo.json",
109 | url: "/foo.json",
110 | },
111 | {
112 | name: "png file",
113 | mime: "image/png",
114 | path: "fsbase_test/images/face.png",
115 | file: "../test/images/face.png",
116 | url: "/images/face.png",
117 | },
118 | {
119 | name: "relative paths",
120 | mime: "application/json",
121 | path: "fsbase_test/foo.json",
122 | file: "../test/foo.json",
123 | url: "/images/../foo.json",
124 | },
125 | {
126 | name: "custom mime type",
127 | mime: "foo/bar",
128 | path: "fsbase_test/custom_type.foo",
129 | file: "../test/custom_type.foo",
130 | url: "/custom_type.foo",
131 | },
132 | }
133 |
134 | func TestBasicFiles(t *testing.T) {
135 | rbody := new(bytes.Buffer)
136 | for _, test := range basicTests {
137 | // read in test file data
138 | if test.file != "" {
139 | test.data, _ = ioutil.ReadFile(test.file)
140 | }
141 |
142 | r, err := get(test.url)
143 | if err != nil {
144 | t.Errorf("%v Error GETting file:%v", test.name, err)
145 | continue
146 | }
147 | if r.StatusCode != 200 {
148 | t.Errorf("%v Expected status 200, got %v", test.name, r.StatusCode)
149 | continue
150 | }
151 | if strings.Split(r.Header.Get("Content-Type"), ";")[0] != test.mime {
152 | t.Errorf("%v Expected Content-Type: %v, got '%v'", test.name, test.mime, r.Header.Get("Content-Type"))
153 | }
154 | rbody.Reset()
155 | io.Copy(rbody, r.Body)
156 | if rbytes := rbody.Bytes(); !bytes.Equal(test.data, rbytes) {
157 | t.Errorf("%v Body doesn't match.\n\tExpected:\n\t%v\n\tReceived:\n\t%v", test.name, test.data, rbytes)
158 | }
159 | }
160 | }
161 |
--------------------------------------------------------------------------------
/logger.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "errors"
5 | "log"
6 | "time"
7 | )
8 |
9 | // I really want to use log4go... but i need to support falling back to standard (shitty) logger :(
10 | // I suggest using go-timber for the real logger
11 | type Logger interface {
12 | // Matches the log4go interface
13 | Finest(arg0 interface{}, args ...interface{})
14 | Fine(arg0 interface{}, args ...interface{})
15 | Debug(arg0 interface{}, args ...interface{})
16 | Trace(arg0 interface{}, args ...interface{})
17 | Info(arg0 interface{}, args ...interface{})
18 | Warn(arg0 interface{}, args ...interface{}) error
19 | Error(arg0 interface{}, args ...interface{}) error
20 | Critical(arg0 interface{}, args ...interface{}) error
21 | }
22 |
23 | var logger Logger = NewStdLibLogger()
24 |
25 | func SetLogger(newLogger Logger) {
26 | logger = newLogger
27 | }
28 |
29 | // Helper for calculating times
30 | func TimeDiff(startTime time.Time, endTime time.Time) float32 {
31 | return float32(endTime.Sub(startTime)) / 1.0e9
32 | }
33 |
34 | // Global Logging
35 | func Finest(arg0 interface{}, args ...interface{}) {
36 | logger.Finest(arg0, args...)
37 | }
38 |
39 | func Fine(arg0 interface{}, args ...interface{}) {
40 | logger.Fine(arg0, args...)
41 | }
42 |
43 | func Debug(arg0 interface{}, args ...interface{}) {
44 | logger.Debug(arg0, args...)
45 | }
46 |
47 | func Trace(arg0 interface{}, args ...interface{}) {
48 | logger.Trace(arg0, args...)
49 | }
50 |
51 | func Info(arg0 interface{}, args ...interface{}) {
52 | logger.Info(arg0, args...)
53 | }
54 |
55 | func Warn(arg0 interface{}, args ...interface{}) error {
56 | return logger.Warn(arg0, args...)
57 | }
58 |
59 | func Error(arg0 interface{}, args ...interface{}) error {
60 | return logger.Error(arg0, args...)
61 | }
62 |
63 | func Critical(arg0 interface{}, args ...interface{}) error {
64 | return logger.Critical(arg0, args...)
65 | }
66 |
67 | // This is a simple Logger implementation that
68 | // uses the go log package for output. It's not
69 | // really meant for production use since it isn't
70 | // very configurable. It is a sane default alternative
71 | // that allows us to not have any external dependencies.
72 | // Use timber or log4go as a real alternative.
73 | type StdLibLogger struct{}
74 |
75 | func NewStdLibLogger() Logger {
76 | return new(StdLibLogger)
77 | }
78 |
79 | type level int
80 |
81 | const (
82 | FINEST level = iota
83 | FINE
84 | DEBUG
85 | TRACE
86 | INFO
87 | WARNING
88 | ERROR
89 | CRITICAL
90 | )
91 |
92 | var (
93 | levelStrings = [...]string{"[FNST]", "[FINE]", "[DEBG]", "[TRAC]", "[INFO]", "[WARN]", "[EROR]", "[CRIT]"}
94 | )
95 |
96 | func (fl StdLibLogger) Finest(arg0 interface{}, args ...interface{}) {
97 | fl.Log(FINEST, arg0, args...)
98 | }
99 |
100 | func (fl StdLibLogger) Fine(arg0 interface{}, args ...interface{}) {
101 | fl.Log(FINE, arg0, args...)
102 | }
103 |
104 | func (fl StdLibLogger) Debug(arg0 interface{}, args ...interface{}) {
105 | fl.Log(DEBUG, arg0, args...)
106 | }
107 |
108 | func (fl StdLibLogger) Trace(arg0 interface{}, args ...interface{}) {
109 | fl.Log(TRACE, arg0, args...)
110 | }
111 |
112 | func (fl StdLibLogger) Info(arg0 interface{}, args ...interface{}) {
113 | fl.Log(INFO, arg0, args...)
114 | }
115 |
116 | func (fl StdLibLogger) Warn(arg0 interface{}, args ...interface{}) error {
117 | return fl.Log(WARNING, arg0, args...)
118 | }
119 |
120 | func (fl StdLibLogger) Error(arg0 interface{}, args ...interface{}) error {
121 | return fl.Log(ERROR, arg0, args...)
122 | }
123 |
124 | func (fl StdLibLogger) Critical(arg0 interface{}, args ...interface{}) error {
125 | return fl.Log(CRITICAL, arg0, args...)
126 | }
127 |
128 | func (fl StdLibLogger) Log(lvl level, arg0 interface{}, args ...interface{}) (e error) {
129 | defer func() {
130 | if x := recover(); x != nil {
131 | var ok bool
132 | if e, ok = x.(error); ok {
133 | return
134 | }
135 | e = errors.New("Um... barf")
136 | }
137 | }()
138 | switch first := arg0.(type) {
139 | case string:
140 | // Use the string as a format string
141 | argsNew := append([]interface{}{levelStrings[lvl]}, args...)
142 | log.Printf("%s "+first, argsNew...)
143 | case func() string:
144 | // Log the closure (no other arguments used)
145 | argsNew := append([]interface{}{levelStrings[lvl]}, first())
146 | log.Println(argsNew...)
147 | default:
148 | // Build a format string so that it will be similar to Sprint
149 | argsNew := append([]interface{}{levelStrings[lvl]}, args...)
150 | log.Println(argsNew...)
151 | }
152 | return nil
153 | }
154 |
--------------------------------------------------------------------------------
/compression/compression_test.go:
--------------------------------------------------------------------------------
1 | package compression
2 |
3 | import (
4 | "bufio"
5 | "bytes"
6 | "compress/flate"
7 | "compress/gzip"
8 | "fmt"
9 | "github.com/ngmoco/falcore"
10 | "io"
11 | "io/ioutil"
12 | "net"
13 | "net/http"
14 | "path"
15 | "testing"
16 | "time"
17 | )
18 |
19 | var srv *falcore.Server
20 |
21 | func init() {
22 | go func() {
23 | // falcore setup
24 | pipeline := falcore.NewPipeline()
25 | pipeline.Upstream.PushBack(falcore.NewRequestFilter(func(req *falcore.Request) *http.Response {
26 | for _, data := range serverData {
27 | if data.path == req.HttpRequest.URL.Path {
28 | header := make(http.Header)
29 | header.Set("Content-Type", data.mime)
30 | header.Set("Content-Encoding", data.encoding)
31 | return falcore.SimpleResponse(req.HttpRequest, 200, header, string(data.body))
32 | }
33 | }
34 | return falcore.SimpleResponse(req.HttpRequest, 404, nil, "Not Found")
35 | }))
36 |
37 | pipeline.Downstream.PushBack(NewFilter(nil))
38 |
39 | srv = falcore.NewServer(0, pipeline)
40 | if err := srv.ListenAndServe(); err != nil {
41 | panic("Could not start falcore")
42 | }
43 | }()
44 | }
45 |
46 | func port() int {
47 | for srv.Port() == 0 {
48 | time.Sleep(1e7)
49 | }
50 | return srv.Port()
51 | }
52 |
53 | var serverData = []struct {
54 | path string
55 | mime string
56 | encoding string
57 | body []byte
58 | }{
59 | {
60 | "/hello",
61 | "text/plain",
62 | "",
63 | []byte("hello world"),
64 | },
65 | {
66 | "/hello.gz",
67 | "text/plain",
68 | "gzip",
69 | compress_gzip([]byte("hello world")),
70 | },
71 | {
72 | "/images/face.png",
73 | "image/png",
74 | "",
75 | readfile("../test/images/face.png"),
76 | },
77 | }
78 |
79 | var testData = []struct {
80 | name string
81 | // input
82 | path string
83 | accept string
84 | // output
85 | encoding string
86 | encoded_body []byte
87 | }{
88 | {
89 | "no compression",
90 | "/hello",
91 | "",
92 | "",
93 | []byte("hello world"),
94 | },
95 | {
96 | "gzip",
97 | "/hello",
98 | "gzip",
99 | "gzip",
100 | compress_gzip([]byte("hello world")),
101 | },
102 | {
103 | "deflate",
104 | "/hello",
105 | "deflate",
106 | "deflate",
107 | compress_deflate([]byte("hello world")),
108 | },
109 | {
110 | "preference",
111 | "/hello",
112 | "gzip, deflate",
113 | "gzip",
114 | compress_gzip([]byte("hello world")),
115 | },
116 | {
117 | "precompressed",
118 | "/hello.gz",
119 | "gzip",
120 | "gzip",
121 | compress_gzip([]byte("hello world")),
122 | },
123 | {
124 | "image",
125 | "/images/face.png",
126 | "gzip",
127 | "",
128 | readfile("../test/images/face.png"),
129 | },
130 | }
131 |
132 | func compress_gzip(body []byte) []byte {
133 | buf := new(bytes.Buffer)
134 | comp := gzip.NewWriter(buf)
135 | comp.Write(body)
136 | comp.Close()
137 | b := buf.Bytes()
138 | // fmt.Println(b)
139 | return b
140 | }
141 |
142 | func compress_deflate(body []byte) []byte {
143 | buf := new(bytes.Buffer)
144 | comp, err := flate.NewWriter(buf, -1)
145 | if err != nil {
146 | panic(fmt.Sprintf("Error using compress/flate.NewWriter() %v", err))
147 | }
148 | comp.Write(body)
149 | comp.Close()
150 | b := buf.Bytes()
151 | // fmt.Println(b)
152 | return b
153 | }
154 |
155 | func readfile(path string) []byte {
156 | if data, err := ioutil.ReadFile(path); err == nil {
157 | return data
158 | } else {
159 | panic(fmt.Sprintf("Error reading file %v: %v", path, err))
160 | }
161 | return nil
162 | }
163 |
164 | func get(p string, accept string) (r *http.Response, err error) {
165 | var conn net.Conn
166 | if conn, err = net.Dial("tcp", fmt.Sprintf("localhost:%v", port())); err == nil {
167 | req, _ := http.NewRequest("GET", fmt.Sprintf("http://%v", path.Join(fmt.Sprintf("localhost:%v/", port()), p)), nil)
168 | req.Header.Set("Accept-Encoding", accept)
169 | req.Write(conn)
170 | buf := bufio.NewReader(conn)
171 | r, err = http.ReadResponse(buf, req)
172 | }
173 | return
174 | }
175 |
176 | func TestCompressionFilter(t *testing.T) {
177 | // select{}
178 | for _, test := range testData {
179 | if res, err := get(test.path, test.accept); err == nil {
180 | bodyBuf := new(bytes.Buffer)
181 | io.Copy(bodyBuf, res.Body)
182 | body := bodyBuf.Bytes()
183 | if enc := res.Header.Get("Content-Encoding"); enc != test.encoding {
184 | t.Errorf("%v Header mismatch. Expecting: %v Got: %v", test.name, test.encoding, enc)
185 | }
186 | if !bytes.Equal(body, test.encoded_body) {
187 | t.Errorf("%v Body mismatch.\n\tExpecting:\n\t%v\n\tGot:\n\t%v", test.name, test.encoded_body, body)
188 | }
189 | } else {
190 | t.Errorf("%v HTTP Error %v", test.name, err)
191 | }
192 | }
193 | }
194 |
--------------------------------------------------------------------------------
/upstream/upstream_pool.go:
--------------------------------------------------------------------------------
1 | package upstream
2 |
3 | import (
4 | "github.com/ngmoco/falcore"
5 | "net/http"
6 | "strconv"
7 | "strings"
8 | "sync"
9 | "time"
10 | )
11 |
12 | type UpstreamEntryConfig struct {
13 | HostPort string
14 | Weight int
15 | ForceHttp bool
16 | PingPath string
17 | }
18 |
19 | type UpstreamEntry struct {
20 | Upstream *Upstream
21 | Weight int
22 | }
23 |
24 | // An UpstreamPool is a list of upstream servers which are considered
25 | // functionally equivalent. The pool will round-robin the requests to the servers.
26 | type UpstreamPool struct {
27 | pool []*UpstreamEntry
28 | rr_count int
29 | ping_count int64
30 | Name string
31 | nextUpstream chan *UpstreamEntry
32 | shutdown chan int
33 | weightMutex *sync.RWMutex
34 | pinger *time.Ticker
35 | }
36 |
37 | // The config consists of a map of the servers in the pool in the format host_or_ip:port
38 | // where port is optional and defaults to 80. The map value is an int with the weight
39 | // only 0 and 1 are supported weights (0 disables a server and 1 enables it)
40 | func NewUpstreamPool(name string, config []UpstreamEntryConfig) *UpstreamPool {
41 | up := new(UpstreamPool)
42 | up.pool = make([]*UpstreamEntry, len(config))
43 | up.Name = name
44 | up.nextUpstream = make(chan *UpstreamEntry)
45 | up.weightMutex = new(sync.RWMutex)
46 | up.shutdown = make(chan int)
47 | up.pinger = time.NewTicker(3e9) // 3s
48 |
49 | // create the pool
50 | for i, uec := range config {
51 | parts := strings.Split(uec.HostPort, ":")
52 | upstreamHost := parts[0]
53 | upstreamPort := 80
54 | if len(parts) > 1 {
55 | var err error
56 | upstreamPort, err = strconv.Atoi(parts[1])
57 | if err != nil {
58 | upstreamPort = 80
59 | falcore.Error("UpstreamPool Error converting port to int for", upstreamHost, ":", err)
60 | }
61 | }
62 | ups := NewUpstream(upstreamHost, upstreamPort, uec.ForceHttp)
63 | ups.PingPath = uec.PingPath
64 | ue := new(UpstreamEntry)
65 | ue.Upstream = ups
66 | ue.Weight = uec.Weight
67 | up.pool[i] = ue
68 | }
69 | go up.nextServer()
70 | go up.pingUpstreams()
71 | return up
72 | }
73 |
74 | func (up UpstreamPool) Next() *UpstreamEntry {
75 | // TODO check in case all are down that we timeout
76 | return <-up.nextUpstream
77 | }
78 |
79 | func (up UpstreamPool) LogStatus() {
80 | weightsBuffer := make([]int, len(up.pool))
81 | // loop and save the weights so we don't lock for logging
82 | up.weightMutex.RLock()
83 | for i, ue := range up.pool {
84 | weightsBuffer[i] = ue.Weight
85 | }
86 | up.weightMutex.RUnlock()
87 | // Now do the logging
88 | for i, ue := range up.pool {
89 | falcore.Info("Upstream %v: %v:%v\t%v", up.Name, ue.Upstream.Host, ue.Upstream.Port, weightsBuffer[i])
90 | }
91 | }
92 |
93 | func (up UpstreamPool) FilterRequest(req *falcore.Request) (res *http.Response) {
94 | ue := up.Next()
95 | res = ue.Upstream.FilterRequest(req)
96 | if req.CurrentStage.Status == 2 {
97 | // this gets set by the upstream for errors
98 | // so mark this upstream as down
99 | up.updateUpstream(ue, 0)
100 | up.LogStatus()
101 | }
102 | return
103 | }
104 |
105 | func (up UpstreamPool) updateUpstream(ue *UpstreamEntry, wgt int) {
106 | up.weightMutex.Lock()
107 | ue.Weight = wgt
108 | up.weightMutex.Unlock()
109 | }
110 |
111 | // This should only be called if the upstream pool is no longer active or this may deadlock
112 | func (up UpstreamPool) Shutdown() {
113 | // ping and nextServer
114 | close(up.shutdown)
115 |
116 | // make sure we hit the shutdown code in the nextServer goroutine
117 | up.Next()
118 | }
119 |
120 | func (up UpstreamPool) nextServer() {
121 | loopCount := 0
122 | for {
123 | next := up.rr_count % len(up.pool)
124 | up.weightMutex.RLock()
125 | wgt := up.pool[next].Weight
126 | up.weightMutex.RUnlock()
127 | // just return a down host if we've gone through the list twice and nothing is up
128 | // be sure to never return negative wgt hosts
129 | if (wgt > 0 || (loopCount > 2*len(up.pool))) && wgt >= 0 {
130 | loopCount = 0
131 | select {
132 | case <-up.shutdown:
133 | return
134 | case up.nextUpstream <- up.pool[next]:
135 | }
136 | } else {
137 | loopCount++
138 | }
139 | up.rr_count++
140 | }
141 | }
142 |
143 | func (up UpstreamPool) pingUpstreams() {
144 | pingable := true
145 | for pingable {
146 | select {
147 | case <-up.shutdown:
148 | return
149 | case <-up.pinger.C:
150 | gotone := false
151 | for i, ups := range up.pool {
152 | if ups.Upstream.PingPath != "" {
153 | go up.pingUpstream(ups, i)
154 | gotone = true
155 | }
156 | }
157 | if !gotone {
158 | pingable = false
159 | }
160 | }
161 | }
162 | falcore.Warn("Stopping ping for %v", up.Name)
163 | }
164 |
165 | func (up UpstreamPool) pingUpstream(ups *UpstreamEntry, index int) {
166 | isUp, ok := ups.Upstream.ping()
167 | up.weightMutex.RLock()
168 | wgt := ups.Weight
169 | up.weightMutex.RUnlock()
170 | // change in status
171 | if ok && (wgt > 0) != isUp {
172 | if isUp {
173 | up.updateUpstream(ups, 1)
174 | } else {
175 | up.updateUpstream(ups, 0)
176 | }
177 | up.LogStatus()
178 | }
179 | }
180 |
--------------------------------------------------------------------------------
/pipeline_test.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "bytes"
5 | "container/list"
6 | "net/http"
7 | "testing"
8 | "time"
9 | )
10 |
11 | func validGetRequest() *Request {
12 | tmp, _ := http.NewRequest("GET", "/hello", bytes.NewBuffer(make([]byte, 0)))
13 | return newRequest(tmp, nil, time.Now())
14 | }
15 |
16 | var stageTrack *list.List
17 |
18 | func doStageTrack() {
19 | i := 0
20 | if stageTrack.Len() > 0 {
21 | i = stageTrack.Back().Value.(int)
22 | }
23 | stageTrack.PushBack(i + 1)
24 | }
25 |
26 | func sumFilter(req *Request) *http.Response {
27 | doStageTrack()
28 | return nil
29 | }
30 |
31 | func sumResponseFilter(*Request, *http.Response) {
32 | doStageTrack()
33 | }
34 |
35 | func successFilter(req *Request) *http.Response {
36 | doStageTrack()
37 | return SimpleResponse(req.HttpRequest, 200, nil, "OK")
38 | }
39 |
40 | func TestPipelineNoResponse(t *testing.T) {
41 | p := NewPipeline()
42 |
43 | stageTrack = list.New()
44 | f := NewRequestFilter(sumFilter)
45 |
46 | p.Upstream.PushBack(f)
47 | p.Upstream.PushBack(f)
48 | p.Upstream.PushBack(f)
49 |
50 | //response := new(http.Response)
51 | response := p.execute(validGetRequest())
52 |
53 | if stageTrack.Len() != 3 {
54 | t.Fatalf("Wrong number of stages executed: %v expected %v", stageTrack.Len(), 3)
55 | }
56 | if sum, ok := stageTrack.Back().Value.(int); ok {
57 | if sum != 3 {
58 | t.Errorf("Pipeline stages did not complete %v expected %v", sum, 3)
59 | }
60 | }
61 | if response.StatusCode != 404 {
62 | t.Errorf("Pipeline response code wrong: %v expected %v", response.StatusCode, 404)
63 | }
64 | }
65 |
66 | func TestPipelineOKResponse(t *testing.T) {
67 | p := NewPipeline()
68 |
69 | stageTrack = list.New()
70 | f := NewRequestFilter(sumFilter)
71 |
72 | p.Upstream.PushBack(f)
73 | p.Upstream.PushBack(f)
74 | p.Upstream.PushBack(NewRequestFilter(successFilter))
75 | p.Upstream.PushBack(f)
76 |
77 | response := p.execute(validGetRequest())
78 |
79 | if stageTrack.Len() != 3 {
80 | t.Fatalf("Wrong number of stages executed: %v expected %v", stageTrack.Len(), 3)
81 | }
82 | if sum, ok := stageTrack.Back().Value.(int); ok {
83 | if sum != 3 {
84 | t.Errorf("Pipeline stages did not complete %v expected %v", sum, 3)
85 | }
86 | }
87 | if response.StatusCode != 200 {
88 | t.Errorf("Pipeline response code wrong: %v expected %v", response.StatusCode, 200)
89 | }
90 | }
91 |
92 | func TestPipelineResponseFilter(t *testing.T) {
93 | p := NewPipeline()
94 |
95 | stageTrack = list.New()
96 | f := NewRequestFilter(sumFilter)
97 |
98 | p.Upstream.PushBack(f)
99 | p.Upstream.PushBack(NewRequestFilter(successFilter))
100 | p.Upstream.PushBack(f)
101 | p.Downstream.PushBack(NewResponseFilter(sumResponseFilter))
102 | p.Downstream.PushBack(NewResponseFilter(sumResponseFilter))
103 |
104 | //response := new(http.Response)
105 | req := validGetRequest()
106 | response := p.execute(req)
107 |
108 | stages := 4
109 | // check basic execution
110 | if stageTrack.Len() != stages {
111 | t.Fatalf("Wrong number of stages executed: %v expected %v", stageTrack.Len(), stages)
112 | }
113 | if sum, ok := stageTrack.Back().Value.(int); ok {
114 | if sum != stages {
115 | t.Errorf("Pipeline stages did not complete %v expected %v", sum, stages)
116 | }
117 | }
118 | // check status
119 | if response.StatusCode != 200 {
120 | t.Errorf("Pipeline response code wrong: %v expected %v", response.StatusCode, 200)
121 | }
122 | req.finishRequest()
123 | if req.Signature() != "F7F5165F" {
124 | t.Errorf("Signature failed: %v expected %v", req.Signature(), "F7F5165F")
125 | }
126 | if req.PipelineStageStats.Len() != stages {
127 | t.Errorf("PipelineStageStats incomplete: %v expected %v", req.PipelineStageStats.Len(), stages)
128 | }
129 | //req.Trace()
130 |
131 | }
132 |
133 | func TestPipelineStatsChecksum(t *testing.T) {
134 | p := NewPipeline()
135 |
136 | stageTrack = list.New()
137 | f := NewRequestFilter(sumFilter)
138 |
139 | p.Upstream.PushBack(f)
140 | p.Upstream.PushBack(NewRequestFilter(func(req *Request) *http.Response {
141 | doStageTrack()
142 | req.CurrentStage.Status = 1
143 | return nil
144 | }))
145 | p.Upstream.PushBack(NewRequestFilter(successFilter))
146 | p.Downstream.PushBack(NewResponseFilter(sumResponseFilter))
147 | p.Downstream.PushBack(NewResponseFilter(sumResponseFilter))
148 |
149 | //response := new(http.Response)
150 | req := validGetRequest()
151 | response := p.execute(req)
152 |
153 | stages := 5
154 | // check basic execution
155 | if stageTrack.Len() != stages {
156 | t.Fatalf("Wrong number of stages executed: %v expected %v", stageTrack.Len(), stages)
157 | }
158 | if sum, ok := stageTrack.Back().Value.(int); ok {
159 | if sum != stages {
160 | t.Errorf("Pipeline stages did not complete %v expected %v", sum, stages)
161 | }
162 | }
163 | // check status
164 | if response.StatusCode != 200 {
165 | t.Errorf("Pipeline response code wrong: %v expected %v", response.StatusCode, 200)
166 | }
167 | req.finishRequest()
168 | if req.Signature() != "CA843113" {
169 | t.Errorf("Signature failed: %v expected %v", req.Signature(), "CA843113")
170 | }
171 | if req.PipelineStageStats.Len() != stages {
172 | t.Errorf("PipelineStageStats incomplete: %v expected %v", req.PipelineStageStats.Len(), stages)
173 | }
174 | //req.Trace()
175 |
176 | }
177 |
--------------------------------------------------------------------------------
/examples/hot_restart/hot_restart.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "flag"
5 | "fmt"
6 | "github.com/ngmoco/falcore"
7 | "net/http"
8 | "os"
9 | "os/signal"
10 | "runtime"
11 | "strconv"
12 | "strings"
13 | "syscall"
14 | )
15 |
16 | // very simple request filter
17 | func Filter(request *falcore.Request) *http.Response {
18 | return falcore.SimpleResponse(request.HttpRequest, 200, nil, "OK\n")
19 | }
20 |
21 | // flag to accept a socket file descriptor
22 | var socketFd = flag.Int("socket", -1, "Socket file descriptor")
23 |
24 | func main() {
25 | pid := syscall.Getpid()
26 | flag.Parse()
27 |
28 | // create the pipeline
29 | pipeline := falcore.NewPipeline()
30 | pipeline.Upstream.PushBack(falcore.NewRequestFilter(Filter))
31 |
32 | // create the server with the pipeline
33 | srv := falcore.NewServer(8090, pipeline)
34 |
35 | // if passed the socket file descriptor, setup the listener that way
36 | // if you don't have it, the default is to create the socket listener
37 | // with the data passed to falcore.NewServer above (happens in ListenAndServer())
38 | if *socketFd != -1 {
39 | // I know I'm a child process if I get here so I can signal the parent when I'm ready to take over
40 | go childReady(srv)
41 | fmt.Printf("%v Got socket FD: %v\n", pid, *socketFd)
42 | srv.FdListen(*socketFd)
43 | }
44 |
45 | // using signals to manage the restart lifecycle
46 | go handleSignals(srv)
47 |
48 | // start the server
49 | // this is normally blocking forever unless you send lifecycle commands
50 | fmt.Printf("%v Starting Listener on 8090\n", pid)
51 | if err := srv.ListenAndServe(); err != nil {
52 | fmt.Printf("%v Could not start server: %v", pid, err)
53 | }
54 | fmt.Printf("%v Exiting now\n", pid)
55 | }
56 |
57 | // blocks on the server ready and when ready, it sends
58 | // a signal to the parent so that it knows it cna now exit
59 | func childReady(srv *falcore.Server) {
60 | pid := syscall.Getpid()
61 | // wait for the ready signal
62 | <-srv.AcceptReady
63 | // grab the parent and send a signal that the child is ready
64 | parent := syscall.Getppid()
65 | fmt.Printf("%v Kill parent %v with SIGUSR1\n", pid, parent)
66 | syscall.Kill(parent, syscall.SIGUSR1)
67 | }
68 |
69 | // setup and fork/exec myself. Make sure to keep open important FD's that won't get re-created by the child
70 | // specifically, std* and your listen socket
71 | func forker(srv *falcore.Server) (pid int, err error) {
72 | var socket string
73 | // At version 1.0.3 the socket FD behavior changed and the fork socket is always 3
74 | // 0 = stdin, 1 = stdout, 2 = stderr, 3 = acceptor socket
75 | // This is because the ForkExec dups all the saved FDs down to
76 | // start at 0. This is also why you MUST include 0,1,2 in the
77 | // attr.Files
78 | if goVersion103OrAbove() {
79 | socket = "3"
80 | } else {
81 | socket = fmt.Sprintf("%v", srv.SocketFd())
82 | }
83 | fmt.Printf("Forking now with socket: %v\n", socket)
84 | mypath := os.Args[0]
85 | args := []string{mypath, "-socket", socket}
86 | attr := new(syscall.ProcAttr)
87 | attr.Files = append([]uintptr(nil), 0, 1, 2, uintptr(srv.SocketFd()))
88 | pid, err = syscall.ForkExec(mypath, args, attr)
89 | return
90 | }
91 |
92 | func goVersion103OrAbove() bool {
93 | ver := strings.Split(runtime.Version(), ".")
94 | // Go versioning is weird so this only works for common go1 cases:
95 | // current as of patch:
96 | // go1.0.3 13678:2d8bc3c94ecb : true
97 | // go1.0.2 13278:5e806355a9e1 : false
98 | // go1.0.1 12994:2ccfd4b451d3 : false
99 | // go1 12872:920e9d1ffd1f : false
100 | // go1.1+/go2+ : true
101 | // release* : true (this is possibly broken)
102 | // weekly* : true (this is possibly broken)
103 | // tip : true
104 | if len(ver) > 0 && strings.Index(ver[0], "go") == 0 {
105 | if ver[0] == "go1" && len(ver) == 1 {
106 | // just go1
107 | return false
108 | } else if ver[0] == "go1" && len(ver) == 3 && ver[1] == "0" {
109 | if patchVer, _ := strconv.ParseInt(ver[2], 10, 64); patchVer < 3 {
110 | return false
111 | }
112 | }
113 | }
114 | return true
115 | }
116 |
117 | // Handle lifecycle events
118 | func handleSignals(srv *falcore.Server) {
119 | var sig os.Signal
120 | var sigChan = make(chan os.Signal)
121 | signal.Notify(sigChan, syscall.SIGHUP, syscall.SIGUSR1, syscall.SIGINT, syscall.SIGTERM, syscall.SIGTSTP)
122 | pid := syscall.Getpid()
123 | for {
124 | sig = <-sigChan
125 | switch sig {
126 | case syscall.SIGHUP:
127 | // send this to the paraent process to initiate the restart
128 | fmt.Println(pid, "Received SIGHUP. forking.")
129 | cpid, err := forker(srv)
130 | fmt.Println(pid, "Forked pid:", cpid, "errno:", err)
131 | case syscall.SIGUSR1:
132 | // child sends this back to the parent when it's ready to Accept
133 | fmt.Println(pid, "Received SIGUSR1. Stopping accept.")
134 | srv.StopAccepting()
135 | case syscall.SIGINT:
136 | fmt.Println(pid, "Received SIGINT. Shutting down.")
137 | os.Exit(0)
138 | case syscall.SIGTERM:
139 | fmt.Println(pid, "Received SIGTERM. Terminating.")
140 | os.Exit(0)
141 | case syscall.SIGTSTP:
142 | fmt.Println(pid, "Received SIGTSTP. Stopping.")
143 | syscall.Kill(pid, syscall.SIGSTOP)
144 | default:
145 | fmt.Println(pid, "Received", sig, ": ignoring")
146 | }
147 | }
148 | }
149 |
--------------------------------------------------------------------------------
/upstream/upstream.go:
--------------------------------------------------------------------------------
1 | package upstream
2 |
3 | import (
4 | "bytes"
5 | "fmt"
6 | "github.com/ngmoco/falcore"
7 | "io"
8 | "net"
9 | "net/http"
10 | "time"
11 | )
12 |
13 | type passThruReadCloser struct {
14 | io.Reader
15 | io.Closer
16 | }
17 |
18 | type connWrapper struct {
19 | conn net.Conn
20 | timeout time.Duration
21 | }
22 |
23 | func (cw *connWrapper) Write(b []byte) (int, error) {
24 | if err := cw.conn.SetDeadline(time.Now().Add(cw.timeout)); err != nil {
25 | return 0, err
26 | }
27 | return cw.conn.Write(b)
28 | }
29 | func (cw *connWrapper) Read(b []byte) (n int, err error) { return cw.conn.Read(b) }
30 | func (cw *connWrapper) Close() error { return cw.conn.Close() }
31 | func (cw *connWrapper) LocalAddr() net.Addr { return cw.conn.LocalAddr() }
32 | func (cw *connWrapper) RemoteAddr() net.Addr { return cw.conn.RemoteAddr() }
33 | func (cw *connWrapper) SetDeadline(t time.Time) error { return cw.conn.SetDeadline(t) }
34 | func (cw *connWrapper) SetReadDeadline(t time.Time) error { return cw.conn.SetReadDeadline(t) }
35 | func (cw *connWrapper) SetWriteDeadline(t time.Time) error { return cw.conn.SetWriteDeadline(t) }
36 |
37 | type Upstream struct {
38 | // The upstream host to connect to
39 | Host string
40 | // The port on the upstream host
41 | Port int
42 | // Default 60 seconds
43 | Timeout time.Duration
44 | // Will ignore https on the incoming request and always upstream http
45 | ForceHttp bool
46 | // Ping URL Path-only for checking upness
47 | PingPath string
48 |
49 | transport *http.Transport
50 | host string
51 | tcpaddr *net.TCPAddr
52 | }
53 |
54 | func NewUpstream(host string, port int, forceHttp bool) *Upstream {
55 | u := new(Upstream)
56 | u.Host = host
57 | u.Port = port
58 | u.ForceHttp = forceHttp
59 | ips, err := net.LookupIP(host)
60 | var ip net.IP = nil
61 | for i := range ips {
62 | ip = ips[i].To4()
63 | if ip != nil {
64 | break
65 | }
66 | }
67 | if err == nil && ip != nil {
68 | u.tcpaddr = &net.TCPAddr{}
69 | u.tcpaddr.Port = port
70 | u.tcpaddr.IP = ip
71 | } else {
72 | falcore.Warn("Can't get IP addr for %v: %v", host, err)
73 | }
74 | u.Timeout = 60 * time.Second
75 | u.host = fmt.Sprintf("%v:%v", u.Host, u.Port)
76 |
77 | u.transport = &http.Transport{}
78 | // This dial ignores the addr passed in and dials based on the upstream host and port
79 | u.transport.Dial = func(n, addr string) (c net.Conn, err error) {
80 | falcore.Fine("Dialing connection to %v", u.tcpaddr)
81 | var ctcp *net.TCPConn
82 | ctcp, err = net.DialTCP("tcp4", nil, u.tcpaddr)
83 | if err != nil {
84 | falcore.Error("Dial Failed: %v", err)
85 | return
86 | }
87 | c = &connWrapper{conn: ctcp, timeout: u.Timeout}
88 | return
89 | }
90 | u.transport.MaxIdleConnsPerHost = 15
91 | return u
92 | }
93 |
94 | // Alter the number of connections to multiplex with
95 | func (u *Upstream) SetPoolSize(size int) {
96 | u.transport.MaxIdleConnsPerHost = size
97 | }
98 |
99 | func (u *Upstream) FilterRequest(request *falcore.Request) (res *http.Response) {
100 | var err error
101 | req := request.HttpRequest
102 |
103 | // Force the upstream to use http
104 | if u.ForceHttp || req.URL.Scheme == "" {
105 | req.URL.Scheme = "http"
106 | req.URL.Host = req.Host
107 | }
108 | before := time.Now()
109 | req.Header.Set("Connection", "Keep-Alive")
110 | var upstrRes *http.Response
111 | upstrRes, err = u.transport.RoundTrip(req)
112 | diff := falcore.TimeDiff(before, time.Now())
113 | if err == nil {
114 | // Copy response over to new record. Remove connection noise. Add some sanity.
115 | res = falcore.SimpleResponse(req, upstrRes.StatusCode, nil, "")
116 | if upstrRes.ContentLength > 0 && upstrRes.Body != nil {
117 | res.ContentLength = upstrRes.ContentLength
118 | res.Body = upstrRes.Body
119 | } else if upstrRes.ContentLength == 0 && upstrRes.Body != nil {
120 | // Any bytes?
121 | var testBuf [1]byte
122 | n, _ := io.ReadFull(upstrRes.Body, testBuf[:])
123 | if n == 1 {
124 | // Yes there are. Chunked it is.
125 | res.TransferEncoding = []string{"chunked"}
126 | res.ContentLength = -1
127 | rc := &passThruReadCloser{
128 | io.MultiReader(bytes.NewBuffer(testBuf[:]), upstrRes.Body),
129 | upstrRes.Body,
130 | }
131 |
132 | res.Body = rc
133 | }
134 | } else if upstrRes.Body != nil {
135 | res.Body = upstrRes.Body
136 | res.ContentLength = -1
137 | res.TransferEncoding = []string{"chunked"}
138 | }
139 | // Copy over headers with a few exceptions
140 | res.Header = make(http.Header)
141 | for hn, hv := range upstrRes.Header {
142 | switch hn {
143 | case "Content-Length":
144 | case "Connection":
145 | case "Transfer-Encoding":
146 | default:
147 | res.Header[hn] = hv
148 | }
149 | }
150 | } else {
151 | if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
152 | falcore.Error("%s Upstream Timeout error: %v", request.ID, err)
153 | res = falcore.SimpleResponse(req, 504, nil, "Gateway Timeout\n")
154 | request.CurrentStage.Status = 2 // Fail
155 | } else {
156 | falcore.Error("%s Upstream error: %v", request.ID, err)
157 | res = falcore.SimpleResponse(req, 502, nil, "Bad Gateway\n")
158 | request.CurrentStage.Status = 2 // Fail
159 | }
160 | }
161 | falcore.Debug("%s [%s] [%s] %s s=%d Time=%.4f", request.ID, req.Method, u.host, req.URL, res.StatusCode, diff)
162 | return
163 | }
164 |
165 | func (u *Upstream) ping() (up bool, ok bool) {
166 | if u.PingPath != "" {
167 | // the url must be syntactically valid for this to work but the host will be ignored because we
168 | // are overriding the connection always
169 | request, err := http.NewRequest("GET", "http://localhost"+u.PingPath, nil)
170 | request.Header.Set("Connection", "Keep-Alive") // not sure if this should be here for a ping
171 | if err != nil {
172 | falcore.Error("Bad Ping request: %v", err)
173 | return false, true
174 | }
175 | res, err := u.transport.RoundTrip(request)
176 |
177 | if err != nil {
178 | falcore.Error("Failed Ping to %v:%v: %v", u.Host, u.Port, err)
179 | return false, true
180 | } else {
181 | res.Body.Close()
182 | }
183 | if res.StatusCode == 200 {
184 | return true, true
185 | }
186 | falcore.Error("Failed Ping to %v:%v: %v", u.Host, u.Port, res.Status)
187 | // bad status
188 | return false, true
189 | }
190 | return false, false
191 | }
192 |
--------------------------------------------------------------------------------
/request.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "container/list"
5 | "fmt"
6 | "hash"
7 | "hash/crc32"
8 | "math/rand"
9 | "net"
10 | "net/http"
11 | "reflect"
12 | "time"
13 | )
14 |
15 | // Request wrapper
16 | //
17 | // The request is wrapped so that useful information can be kept
18 | // with the request as it moves through the pipeline.
19 | //
20 | // A pointer is kept to the originating Connection.
21 | //
22 | // There is a unique ID assigned to each request. This ID is not
23 | // globally unique to keep it shorter for logging purposes. It is
24 | // possible to have duplicates though very unlikely over the period
25 | // of a day or so. It is a good idea to log the ID in any custom
26 | // log statements so that individual requests can easily be grepped
27 | // from busy log files.
28 | //
29 | // Falcore collects performance statistics on every stage of the
30 | // pipeline. The stats for the request are kept in PipelineStageStats.
31 | // This structure will only be complete in the Request passed to the
32 | // pipeline RequestDoneCallback. Overhead will only be available in
33 | // the RequestDoneCallback and it's the difference between the total
34 | // request time and the sums of the stage times. It will include things
35 | // like pipeline iteration and the stat collection itself.
36 | //
37 | // See falcore.PipelineStageStat docs for more info.
38 | //
39 | // The Signature is also a cool feature. See the
40 | type Request struct {
41 | ID string
42 | StartTime time.Time
43 | EndTime time.Time
44 | HttpRequest *http.Request
45 | Connection net.Conn
46 | RemoteAddr *net.TCPAddr
47 | PipelineStageStats *list.List
48 | CurrentStage *PipelineStageStat
49 | pipelineHash hash.Hash32
50 | piplineTot time.Duration
51 | Overhead time.Duration
52 | Context map[string]interface{}
53 | }
54 |
55 | // Used internally to create and initialize a new request.
56 | func newRequest(request *http.Request, conn net.Conn, startTime time.Time) *Request {
57 | fReq := new(Request)
58 | fReq.Context = make(map[string]interface{})
59 | fReq.HttpRequest = request
60 | fReq.StartTime = startTime
61 | fReq.Connection = conn
62 | if conn != nil {
63 | fReq.RemoteAddr = conn.RemoteAddr().(*net.TCPAddr)
64 | }
65 | // create a semi-unique id to track a connection in the logs
66 | // ID is the least significant decimal digits of time with some randomization
67 | // the last 3 zeros of time.Nanoseconds appear to always be zero
68 | var ut = fReq.StartTime.UnixNano()
69 | fReq.ID = fmt.Sprintf("%010x", (ut-(ut-(ut%1e12)))+int64(rand.Intn(999)))
70 | fReq.PipelineStageStats = list.New()
71 | fReq.pipelineHash = crc32.NewIEEE()
72 |
73 | // Support for 100-continue requests
74 | // http.Server (and presumably google app engine) already handle this
75 | // case. So we don't need to do anything if we don't own the
76 | // connection.
77 | if conn != nil && request.Header.Get("Expect") == "100-continue" {
78 | request.Body = &continueReader{req: fReq, r: request.Body}
79 | }
80 |
81 | return fReq
82 | }
83 |
84 | // Returns a completed falcore.Request and response after running the single filter stage
85 | // The PipelineStageStats is completed in the returned Request
86 | // The falcore.Request.Connection and falcore.Request.RemoteAddr are nil
87 | func TestWithRequest(request *http.Request, filter RequestFilter, context map[string]interface{}) (*Request, *http.Response) {
88 | r := newRequest(request, nil, time.Now())
89 | if context == nil {
90 | context = make(map[string]interface{})
91 | }
92 | r.Context = context
93 | t := reflect.TypeOf(filter)
94 | r.startPipelineStage(t.String())
95 | res := filter.FilterRequest(r)
96 | r.finishPipelineStage()
97 | r.finishRequest()
98 | return r, res
99 | }
100 |
101 | // Starts a new pipeline stage and makes it the CurrentStage.
102 | func (fReq *Request) startPipelineStage(name string) {
103 | fReq.CurrentStage = NewPiplineStage(name)
104 | fReq.PipelineStageStats.PushBack(fReq.CurrentStage)
105 | }
106 |
107 | // Finishes the CurrentStage.
108 | func (fReq *Request) finishPipelineStage() {
109 | fReq.CurrentStage.EndTime = time.Now()
110 | fReq.finishCommon()
111 | }
112 |
113 | // Appends an already completed PipelineStageStat directly to the list
114 | func (fReq *Request) appendPipelineStage(pss *PipelineStageStat) {
115 | fReq.PipelineStageStats.PushBack(pss)
116 | fReq.CurrentStage = pss
117 | fReq.finishCommon()
118 | }
119 |
120 | // Does some required bookeeping for the pipeline and the pipeline signature
121 | func (fReq *Request) finishCommon() {
122 | fReq.pipelineHash.Write([]byte(fReq.CurrentStage.Name))
123 | fReq.pipelineHash.Write([]byte{fReq.CurrentStage.Status})
124 | fReq.piplineTot += fReq.CurrentStage.EndTime.Sub(fReq.CurrentStage.StartTime)
125 | }
126 |
127 | // The Signature will only be complete in the RequestDoneCallback. At
128 | // any given time, the Signature is a crc32 sum of all the finished
129 | // pipeline stages combining PipelineStageStat.Name and PipelineStageStat.Status.
130 | // This gives a unique signature for each unique path through the pipeline.
131 | // To modify the signature for your own use, just set the
132 | // request.CurrentStage.Status in your RequestFilter or ResponseFilter.
133 | func (fReq *Request) Signature() string {
134 | return fmt.Sprintf("%X", fReq.pipelineHash.Sum32())
135 | }
136 |
137 | // Call from RequestDoneCallback. Logs a bunch of information about the
138 | // request to the falcore logger. This is a pretty big hit to performance
139 | // so it should only be used for debugging or development. The source is a
140 | // good example of how to get useful information out of the Request.
141 | func (fReq *Request) Trace() {
142 | reqTime := TimeDiff(fReq.StartTime, fReq.EndTime)
143 | req := fReq.HttpRequest
144 | Trace("%s [%s] %s%s Sig=%s Tot=%.4f", fReq.ID, req.Method, req.Host, req.URL, fReq.Signature(), reqTime)
145 | l := fReq.PipelineStageStats
146 | for e := l.Front(); e != nil; e = e.Next() {
147 | pss, _ := e.Value.(*PipelineStageStat)
148 | dur := TimeDiff(pss.StartTime, pss.EndTime)
149 | Trace("%s %-30s S=%d Tot=%.4f %%=%.2f", fReq.ID, pss.Name, pss.Status, dur, dur/reqTime*100.0)
150 | }
151 | Trace("%s %-30s S=0 Tot=%.4f %%=%.2f", fReq.ID, "Overhead", float32(fReq.Overhead)/1.0e9, float32(fReq.Overhead)/1.0e9/reqTime*100.0)
152 | }
153 |
154 | func (fReq *Request) finishRequest() {
155 | fReq.EndTime = time.Now()
156 | fReq.Overhead = fReq.EndTime.Sub(fReq.StartTime) - fReq.piplineTot
157 | }
158 |
159 | // Container for keeping stats per pipeline stage
160 | // Name for filter stages is reflect.TypeOf(filter).String()[1:] and the Status is 0 unless
161 | // it is changed explicitly in the Filter or Router.
162 | //
163 | // For the Status, the falcore library will not apply any specific meaning to the status
164 | // codes but the following are suggested conventional usages that we have found useful
165 | //
166 | // type PipelineStatus byte
167 | // const (
168 | // Success PipelineStatus = iota // General Run successfully
169 | // Skip // Skipped (all or most of the work of this stage)
170 | // Fail // General Fail
171 | // // All others may be used as custom status codes
172 | // )
173 | type PipelineStageStat struct {
174 | Name string
175 | Status byte
176 | StartTime time.Time
177 | EndTime time.Time
178 | }
179 |
180 | func NewPiplineStage(name string) *PipelineStageStat {
181 | pss := new(PipelineStageStat)
182 | pss.Name = name
183 | pss.StartTime = time.Now()
184 | return pss
185 | }
186 |
--------------------------------------------------------------------------------
/server.go:
--------------------------------------------------------------------------------
1 | package falcore
2 |
3 | import (
4 | "bufio"
5 | "crypto/rand"
6 | "crypto/tls"
7 | "errors"
8 | "fmt"
9 | "io"
10 | "net"
11 | "net/http"
12 | "os"
13 | "runtime"
14 | "strconv"
15 | "sync"
16 | "syscall"
17 | "time"
18 | )
19 |
20 | type Server struct {
21 | Addr string
22 | Pipeline *Pipeline
23 | listener net.Listener
24 | listenerFile *os.File
25 | stopAccepting chan int
26 | handlerWaitGroup *sync.WaitGroup
27 | logPrefix string
28 | AcceptReady chan int
29 | sendfile bool
30 | sockOpt int
31 | bufferPool *bufferPool
32 | }
33 |
34 | func NewServer(port int, pipeline *Pipeline) *Server {
35 | s := new(Server)
36 | s.Addr = fmt.Sprintf(":%v", port)
37 | s.Pipeline = pipeline
38 | s.stopAccepting = make(chan int)
39 | s.AcceptReady = make(chan int, 1)
40 | s.handlerWaitGroup = new(sync.WaitGroup)
41 | s.logPrefix = fmt.Sprintf("%d", syscall.Getpid())
42 |
43 | // openbsd/netbsd don't have TCP_NOPUSH so it's likely sendfile will be slower
44 | // without these socket options, just enable for linux, mac and freebsd.
45 | // TODO (Graham) windows has TransmitFile zero-copy mechanism, try to use it
46 | switch runtime.GOOS {
47 | case "linux":
48 | s.sendfile = true
49 | s.sockOpt = 0x3 // syscall.TCP_CORK
50 | case "freebsd", "darwin":
51 | s.sendfile = true
52 | s.sockOpt = 0x4 // syscall.TCP_NOPUSH
53 | default:
54 | s.sendfile = false
55 | }
56 |
57 | // buffer pool for reusing connection bufio.Readers
58 | s.bufferPool = newBufferPool(100, 8192)
59 |
60 | return s
61 | }
62 |
63 | func (srv *Server) FdListen(fd int) error {
64 | var err error
65 | srv.listenerFile = os.NewFile(uintptr(fd), "")
66 | if srv.listener, err = net.FileListener(srv.listenerFile); err != nil {
67 | return err
68 | }
69 | if _, ok := srv.listener.(*net.TCPListener); !ok {
70 | return errors.New("Broken listener isn't TCP")
71 | }
72 | return nil
73 | }
74 |
75 | func (srv *Server) socketListen() error {
76 | var la *net.TCPAddr
77 | var err error
78 | if la, err = net.ResolveTCPAddr("tcp", srv.Addr); err != nil {
79 | return err
80 | }
81 |
82 | var l *net.TCPListener
83 | if l, err = net.ListenTCP("tcp", la); err != nil {
84 | return err
85 | }
86 | srv.listener = l
87 | // setup listener to be non-blocking if we're not on windows.
88 | // this is required for hot restart to work.
89 | return srv.setupNonBlockingListener(err, l)
90 | }
91 |
92 | func (srv *Server) ListenAndServe() error {
93 | if srv.Addr == "" {
94 | srv.Addr = ":http"
95 | }
96 | if srv.listener == nil {
97 | if err := srv.socketListen(); err != nil {
98 | return err
99 | }
100 | }
101 | return srv.serve()
102 | }
103 |
104 | func (srv *Server) SocketFd() int {
105 | return int(srv.listenerFile.Fd())
106 | }
107 |
108 | func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error {
109 | if srv.Addr == "" {
110 | srv.Addr = ":https"
111 | }
112 | config := &tls.Config{
113 | Rand: rand.Reader,
114 | Time: time.Now,
115 | NextProtos: []string{"http/1.1"},
116 | }
117 |
118 | var err error
119 | config.Certificates = make([]tls.Certificate, 1)
120 | config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile)
121 | if err != nil {
122 | return err
123 | }
124 |
125 | if srv.listener == nil {
126 | if err := srv.socketListen(); err != nil {
127 | return err
128 | }
129 | }
130 |
131 | srv.listener = tls.NewListener(srv.listener, config)
132 |
133 | return srv.serve()
134 | }
135 |
136 | func (srv *Server) StopAccepting() {
137 | close(srv.stopAccepting)
138 | }
139 |
140 | func (srv *Server) Port() int {
141 | if l := srv.listener; l != nil {
142 | a := l.Addr()
143 | if _, p, e := net.SplitHostPort(a.String()); e == nil && p != "" {
144 | server_port, _ := strconv.Atoi(p)
145 | return server_port
146 | }
147 | }
148 | return 0
149 | }
150 |
151 | func (srv *Server) serve() (e error) {
152 | var accept = true
153 | srv.AcceptReady <- 1
154 | for accept {
155 | var c net.Conn
156 | if l, ok := srv.listener.(*net.TCPListener); ok {
157 | l.SetDeadline(time.Now().Add(3e9))
158 | }
159 | c, e = srv.listener.Accept()
160 | if e != nil {
161 | if ope, ok := e.(*net.OpError); ok {
162 | if !(ope.Timeout() && ope.Temporary()) {
163 | Error("%s SERVER Accept Error: %v", srv.serverLogPrefix(), ope)
164 | }
165 | } else {
166 | Error("%s SERVER Accept Error: %v", srv.serverLogPrefix(), e)
167 | }
168 | } else {
169 | //Trace("Handling!")
170 | srv.handlerWaitGroup.Add(1)
171 | go srv.handler(c)
172 | }
173 | select {
174 | case <-srv.stopAccepting:
175 | accept = false
176 | default:
177 | }
178 | }
179 | Trace("Stopped accepting, waiting for handlers")
180 | // wait for handlers
181 | srv.handlerWaitGroup.Wait()
182 | return nil
183 | }
184 |
185 | func (srv *Server) sentinel(c net.Conn, connClosed chan int) {
186 | select {
187 | case <-srv.stopAccepting:
188 | c.SetReadDeadline(time.Now().Add(3 * time.Second))
189 | case <-connClosed:
190 | }
191 | }
192 |
193 | func (srv *Server) handler(c net.Conn) {
194 | startTime := time.Now()
195 | bpe := srv.bufferPool.take(c)
196 | defer srv.bufferPool.give(bpe)
197 | var closeSentinelChan = make(chan int)
198 | go srv.sentinel(c, closeSentinelChan)
199 | defer srv.connectionFinished(c, closeSentinelChan)
200 | var err error
201 | var req *http.Request
202 | // no keepalive (for now)
203 | reqCount := 0
204 | keepAlive := true
205 | for err == nil && keepAlive {
206 | if req, err = http.ReadRequest(bpe.br); err == nil {
207 | if req.Header.Get("Connection") != "Keep-Alive" {
208 | keepAlive = false
209 | }
210 | request := newRequest(req, c, startTime)
211 | reqCount++
212 | var res *http.Response
213 |
214 | pssInit := new(PipelineStageStat)
215 | pssInit.Name = "server.Init"
216 | pssInit.StartTime = startTime
217 | pssInit.EndTime = time.Now()
218 | request.appendPipelineStage(pssInit)
219 | // execute the pipeline
220 | if res = srv.Pipeline.execute(request); res == nil {
221 | res = SimpleResponse(req, 404, nil, "Not Found")
222 | }
223 | // cleanup
224 | request.startPipelineStage("server.ResponseWrite")
225 | req.Body.Close()
226 |
227 | // shutting down?
228 | select {
229 | case <-srv.stopAccepting:
230 | keepAlive = false
231 | res.Close = true
232 | default:
233 | }
234 | // The res.Write omits Content-length on 0 length bodies, and by spec,
235 | // it SHOULD. While this is not MUST, it's kinda broken. See sec 4.4
236 | // of rfc2616 and a 200 with a zero length does not satisfy any of the
237 | // 5 conditions if Connection: keep-alive is set :(
238 | // I'm forcing chunked which seems to work because I couldn't get the
239 | // content length to write if it was 0.
240 | // Specifically, the android http client waits forever if there's no
241 | // content-length instead of assuming zero at the end of headers. der.
242 | if res.ContentLength == 0 && len(res.TransferEncoding) == 0 && !((res.StatusCode-100 < 100) || res.StatusCode == 204 || res.StatusCode == 304) {
243 | res.TransferEncoding = []string{"identity"}
244 | }
245 | if res.ContentLength < 0 {
246 | res.TransferEncoding = []string{"chunked"}
247 | }
248 |
249 | // For HTTP/1.0 and Keep-Alive, sending the Connection: Keep-Alive response header is required
250 | // because close is default (opposite of 1.1)
251 | if keepAlive && !req.ProtoAtLeast(1, 1) {
252 | res.Header.Add("Connection", "Keep-Alive")
253 | }
254 |
255 | // write response
256 | if srv.sendfile {
257 | res.Write(c)
258 | srv.cycleNonBlock(c)
259 | } else {
260 | wbuf := bufio.NewWriter(c)
261 | res.Write(wbuf)
262 | wbuf.Flush()
263 | }
264 | if res.Body != nil {
265 | res.Body.Close()
266 | }
267 | request.finishPipelineStage()
268 | request.finishRequest()
269 | srv.requestFinished(request)
270 |
271 | if res.Close {
272 | keepAlive = false
273 | }
274 |
275 | // Reset the startTime
276 | // this isn't great since there may be lag between requests; but it's the best we've got
277 | startTime = time.Now()
278 | } else {
279 | // EOF is socket closed
280 | if nerr, ok := err.(net.Error); err != io.EOF && !(ok && nerr.Timeout()) {
281 | Error("%s %v ERROR reading request: <%T %v>", srv.serverLogPrefix(), c.RemoteAddr(), err, err)
282 | }
283 | }
284 | }
285 | //Debug("%s Processed %v requests on connection %v", srv.serverLogPrefix(), reqCount, c.RemoteAddr())
286 | }
287 |
288 | func (srv *Server) serverLogPrefix() string {
289 | return srv.logPrefix
290 | }
291 |
292 | func (srv *Server) requestFinished(request *Request) {
293 | if srv.Pipeline.RequestDoneCallback != nil {
294 | // Don't block the connecion for this
295 | go srv.Pipeline.RequestDoneCallback.FilterRequest(request)
296 | }
297 | }
298 |
299 | func (srv *Server) connectionFinished(c net.Conn, closeChan chan int) {
300 | c.Close()
301 | close(closeChan)
302 | srv.handlerWaitGroup.Done()
303 | }
304 |
--------------------------------------------------------------------------------