├── .gitignore ├── LICENSE.txt ├── Makefile ├── README.md ├── examples ├── chunked-streaming │ ├── README.md │ ├── get_client │ │ └── main.go │ ├── input_stream │ │ └── main.go │ └── post_client │ │ └── main.go ├── cors-example.json └── simple │ ├── README.md │ ├── get_client │ └── main.go │ └── post_client │ └── main.go ├── go.mod ├── go.sum ├── main.go ├── scripts └── create-server-certs.sh └── server ├── cors.go ├── file.go ├── handlers.go ├── server.go └── waiting_requests.go /.gitignore: -------------------------------------------------------------------------------- 1 | content/ 2 | bin/ 3 | certs/ 4 | .vscode/ -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Matthew Neil - Jorge Cenzano 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export GO111MODULE = on 2 | 3 | ifeq ($(shell uname),Darwin) 4 | BINDIR = binaries/darwin 5 | else ifeq ($(shell uname),Linux) 6 | BINDIR = binaries/linux_x86_64 7 | endif 8 | 9 | PATH := $(shell pwd)/$(BINDIR):$(PATH) 10 | 11 | LDFLAGS = -ldflags "-X main.gitSHA=$(shell git rev-parse HEAD)" 12 | 13 | .PHONY: all 14 | all: build test 15 | 16 | .PHONY: build 17 | build: 18 | if [ ! -d bin ]; then mkdir bin; fi 19 | if [ ! -d logs ]; then mkdir logs; fi 20 | go build -o bin/go-chunked-streaming-server $(LDFLAGS) main.go 21 | 22 | .PHONY: fmt 23 | fmt: 24 | find . -not -path "./vendor/*" -name '*.go' -type f | sed 's#\(.*\)/.*#\1#' | sort -u | xargs -n1 -I {} bash -c "cd {} && goimports -w *.go && gofmt -w -s -l *.go" 25 | 26 | .PHONY: test 27 | test: 28 | ifndef BINDIR 29 | $(error Unable to set PATH based on current platform.) 30 | endif 31 | #TODO go test $(V) ./handlers 32 | 33 | .PHONY: clean 34 | clean: 35 | go clean 36 | rm -f bin/go-chunked-streaming-server 37 | rm -rf content/* 38 | 39 | .PHONY: build-example-chunked-streaming 40 | build-example-chunked-streaming: 41 | if [ ! -d bin ]; then mkdir bin; fi 42 | go build -o bin/examples-chunked-streaming-post-client examples/chunked-streaming/post_client/main.go 43 | go build -o bin/examples-chunked-streaming-input-stream examples/chunked-streaming/input_stream/main.go 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-chunked-streaming-server 2 | 3 | This is simple webserver written in [GoLang](https://golang.org/) that allows you to chunk transfer from the ingress and egress sides. In other words: 4 | - If it receives a GET request for object A, and object A is still being ingested, we send all we have of object A and we keep that GET request open sending on real time (and in memory) all the chunks that we received for object A, until that object is closed. 5 | - If we receives a GET request for object B and this object is already written (and closed), we send the object B as any other static web server. 6 | 7 | Ideal for low latency media processing pipelines. 8 | 9 | # Usage 10 | ## Installation 11 | 1. Just download GO in your computer. See [GoLang](https://golang.org/) 12 | 2. Create a Go directory to be used as a workspace for all go code, i.e. 13 | ``` 14 | mkdir ~/MYDIR/go 15 | ``` 16 | 3. Add `GOPATH` to your `~/.bash_profile` or equivalent for your shell 17 | ``` 18 | export GOPATH="$HOME/MYDIR/go" 19 | ``` 20 | 4. Add `GOPATH/bin` to your path in `~/.bash_profile` or equivalent for your shell 21 | ``` 22 | export PATH="$PATH:$GOPATH/bin 23 | ``` 24 | 5. Restart your terminal or source your profile 25 | 6. Clone this repo: 26 | ``` 27 | go get github.com/mjneil/go-chunked-streaming-server 28 | ``` 29 | 7. Go the the source code dir ` 30 | ``` 31 | cd $HOME/MYDIR/go/src/github.com/mjneil/go-chunked-streaming-server 32 | ``` 33 | 8. Compile `main.go` doing: 34 | ``` 35 | make 36 | ``` 37 | 38 | ## Testing 39 | You can execute `./bin/./go-chunked-streaming-server -h` to see all the possible command arguments. 40 | ``` 41 | Usage of ./bin/go-chunked-streaming-server: 42 | -c string 43 | Certificate file path (only for https) 44 | -d Indicates to remove files from the server based on original Cache-Control (max-age) header 45 | -i int 46 | Port used for HTTP ingress/ egress (default 9094) 47 | -k string 48 | Key file path (only for https) 49 | -o string 50 | JSON file path with the CORS headers definition 51 | -p string 52 | Path used to store (default "./content") 53 | -r Indicates DO NOT use disc as persistent/fallback storage (only RAM) 54 | ``` 55 | 56 | ## Example simple HTTP 57 | - Start the server 58 | ``` 59 | ./bin/./go-chunked-streaming-server 60 | ``` 61 | - Upload a file 62 | ``` 63 | curl http://localhost:9094 --upload-file file.test.txt 64 | ``` 65 | - Consume the file and saved it to disc 66 | ``` 67 | curl http://localhost:9094/file.test.txt -o file.test.downloaded.txt 68 | ``` 69 | 70 | ## Example chunked HTTP 71 | We could build a low latency HLS pipeline in conjunction with [go-ts-segmenter](https://github.com/jordicenzano/go-ts-segmenter) by doing: 72 | 1. Start webserver 73 | ``` 74 | ./bin/./go-chunked-streaming-server 75 | ``` 76 | 2. Generate LHLS data 77 | ``` 78 | ffmpeg -f lavfi -re -i smptebars=size=320x200:rate=30 -f lavfi -i sine=frequency=1000:sample_rate=48000 -pix_fmt yuv420p -c:v libx264 -b:v 180k -g 60 -keyint_min 60 -profile:v baseline -preset veryfast -c:a aac -b:a 96k -f mpegts - | manifest-generator -l 3 -d 2 79 | ``` 80 | 3. Play that data with any HLS player (recommended any player that implements low latency mode) 81 | ``` 82 | ffplay http://localhost:9094/results/chunklist.m3u8 83 | ``` 84 | Or use Safari with this URL `http://localhost:9094/results/chunklist.m3u8` 85 | -------------------------------------------------------------------------------- /examples/chunked-streaming/README.md: -------------------------------------------------------------------------------- 1 | # Chunked Streaming Example 2 | 3 | This simple example demonstrates reading a stream of bytes from STDIN and segmenting the input data, uploading to each segment in a chunked fashion. This is meant to be similar to the behavior of chunked media streaming such as low latency CMAF where segments are further divided into chunks for each frame. 4 | It uses 2 clients, a POST and GET client. The POST client reads bytes from STDIN generated by ./input_stream one chunk at a time. When the segment delimeter `$` is seen, a new http POST request is created for a new segment, and the chunks read in are written to the request until the next delimeter. The GET client opens a GET request to the chunked-streaming-server for each numbered segment, logging chunks to STDOUT as they are recieved. 5 | 6 | 7 | In one terminal session, start the chunked-streaming-server 8 | ``` 9 | ~/go-chunked-streaming-server $ go run main.go 10 | ``` 11 | 12 | Open a second terminal session and start the GET client. 13 | ``` 14 | ~/go-chunked-streaming-server $ go run examples/chunked-streaming/get_client/main.go 15 | ``` 16 | 17 | Open a third terminal session and start the POST client. 18 | ``` 19 | # first build the input_stream and post_client binaries so we can pipe data 20 | ~/go-chunked-streaming-server $ make build-example-chunked-streaming 21 | ~/go-chunked-streaming-server $ bin/examples-chunked-streaming-input-stream | bin/examples-chunked-streaming-post-client 22 | ``` 23 | 24 | In the second terminal session running the GET client, you should see it log out the segment chunks as it receives them. 25 | -------------------------------------------------------------------------------- /examples/chunked-streaming/get_client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/url" 9 | "path" 10 | "time" 11 | ) 12 | 13 | const ( 14 | httpBase = "/segments" 15 | max = 5 16 | ) 17 | 18 | func main() { 19 | tr := http.DefaultTransport 20 | 21 | client := &http.Client{ 22 | Transport: tr, 23 | Timeout: 0, 24 | } 25 | 26 | var ( 27 | index int 28 | err error 29 | currentBody io.Closer 30 | currentReader *bufio.Reader 31 | ) 32 | 33 | endSegment := func() error { 34 | if currentBody == nil { 35 | return nil 36 | } 37 | fmt.Printf("\nSegment finished: %s\n", segmentName(httpBase, index-1)) 38 | e := currentBody.Close() 39 | currentReader = nil 40 | currentBody = nil 41 | return e 42 | } 43 | 44 | nextSegment := func() error { 45 | req := &http.Request{ 46 | Method: "GET", 47 | URL: &url.URL{ 48 | Scheme: "http", 49 | Host: "localhost:9094", 50 | Path: segmentName(httpBase, index), 51 | }, 52 | } 53 | 54 | fmt.Printf("Requesting segment: %s\n", segmentName(httpBase, index)) 55 | 56 | resp, err := client.Do(req) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | if resp.StatusCode == http.StatusNotFound { 62 | currentBody = nil 63 | currentReader = nil 64 | return nil 65 | } 66 | 67 | index++ 68 | 69 | currentBody = resp.Body 70 | currentReader = bufio.NewReader(resp.Body) 71 | return nil 72 | } 73 | 74 | // cleanup potential request body 75 | defer func() { 76 | if err != nil { 77 | printErr(err) 78 | } 79 | err = endSegment() 80 | if err != nil { 81 | panic(err) 82 | } 83 | }() 84 | 85 | // errs in loop are printed in the defer 86 | for { 87 | if currentReader == nil { 88 | // sleep until the segment is available 89 | time.Sleep(1 * time.Second) 90 | err = nextSegment() 91 | if err != nil { 92 | return 93 | } 94 | continue 95 | } 96 | 97 | b, err := currentReader.ReadByte() 98 | fmt.Print(string(b)) 99 | 100 | if err == io.EOF { 101 | err = endSegment() 102 | if err != nil { 103 | return 104 | } 105 | 106 | if index == max { 107 | // done 108 | break 109 | } 110 | err = nextSegment() 111 | } 112 | 113 | if err != nil { 114 | return 115 | } 116 | } 117 | } 118 | 119 | func segmentName(base string, index int) string { 120 | return path.Join(base, fmt.Sprintf("segment-%d", index)) 121 | } 122 | 123 | func printErr(err error) { 124 | fmt.Printf("Error: %v\n", err) 125 | } 126 | -------------------------------------------------------------------------------- /examples/chunked-streaming/input_stream/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os" 8 | "time" 9 | ) 10 | 11 | const ( 12 | segmentLen = 16 13 | chunkLen = 4 14 | max = 5 15 | ) 16 | 17 | func main() { 18 | out := os.Stdout 19 | 20 | for i := 0; i < max; i++ { 21 | currentSegment := newSegment(i) 22 | for offset := 0; offset < len(currentSegment); offset += chunkLen { 23 | b := currentSegment[offset : offset+chunkLen] 24 | buf := bytes.NewBuffer(b) 25 | io.Copy(out, buf) 26 | time.Sleep(1 * time.Second) 27 | } 28 | } 29 | } 30 | 31 | func newSegment(index int) []byte { 32 | str := fmt.Sprintf("$%015d", index) 33 | buf := make([]byte, 16) 34 | copy(buf, str) 35 | return buf 36 | } 37 | -------------------------------------------------------------------------------- /examples/chunked-streaming/post_client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/url" 9 | "os" 10 | "path" 11 | ) 12 | 13 | const ( 14 | fileBase = "./content/examples/chunked-streaming/post_client" 15 | httpBase = "/segments" 16 | chunkLen = 4 17 | ) 18 | 19 | func setupContentDir() { 20 | err := os.RemoveAll(fileBase) 21 | if err != nil { 22 | panic(err) 23 | } 24 | err = os.MkdirAll(fileBase, 0755) 25 | if err != nil { 26 | panic(err) 27 | } 28 | } 29 | 30 | // Allows use of io.TeeReader for http.Request.Body 31 | type TeeReadCloser struct { 32 | r io.Reader 33 | c io.Closer 34 | } 35 | 36 | func (t *TeeReadCloser) Read(p []byte) (n int, err error) { 37 | return t.r.Read(p) 38 | } 39 | 40 | func (t *TeeReadCloser) Close() error { 41 | return t.c.Close() 42 | } 43 | 44 | type Segment struct { 45 | file *os.File 46 | index int 47 | w io.WriteCloser 48 | r io.ReadCloser 49 | } 50 | 51 | func NewSegment(index int) (*Segment, error) { 52 | file, err := newSegmentFile(index) 53 | if err != nil { 54 | return nil, err 55 | } 56 | 57 | pipeR, pipeW := io.Pipe() 58 | 59 | // as the http client reads from the pipe to upload, tee will also write to file 60 | tee := io.TeeReader(pipeR, file) 61 | teerc := &TeeReadCloser{tee, pipeR} 62 | 63 | return &Segment{ 64 | file: file, 65 | index: index, 66 | w: pipeW, 67 | r: teerc, 68 | }, nil 69 | } 70 | 71 | func (s *Segment) Write(p []byte) (n int, err error) { 72 | fmt.Print(string(p)) 73 | return s.w.Write(p) 74 | } 75 | 76 | func (s *Segment) Close() error { 77 | s.w.Close() 78 | fmt.Printf("\nEnd Upload: %s\n", segmentName(httpBase, s.index)) 79 | return s.file.Close() 80 | } 81 | 82 | func (s *Segment) StartUpload(client *http.Client, errC chan error) { 83 | fmt.Printf("Starting Upload: %s\n", segmentName(httpBase, s.index)) 84 | 85 | go func() { 86 | err := uploadSegment(client, s.index, s.r) 87 | if err != nil { 88 | errC <- err 89 | } 90 | }() 91 | } 92 | 93 | func main() { 94 | setupContentDir() 95 | 96 | tr := http.DefaultTransport 97 | 98 | client := &http.Client{ 99 | Transport: tr, 100 | Timeout: 0, 101 | } 102 | 103 | var ( 104 | inputStream = bufio.NewReader(os.Stdin) 105 | chunk = make([]byte, chunkLen) 106 | err error 107 | index int 108 | n int 109 | currentSegment *Segment 110 | errC = make(chan error) 111 | ) 112 | 113 | endSegment := func() error { 114 | if currentSegment == nil { 115 | return nil 116 | } 117 | 118 | e := currentSegment.Close() 119 | currentSegment = nil 120 | return e 121 | } 122 | 123 | nextSegment := func() error { 124 | currentSegment, err = NewSegment(index) 125 | if err != nil { 126 | return err 127 | } 128 | currentSegment.StartUpload(client, errC) 129 | index++ 130 | return nil 131 | } 132 | 133 | // cleanup potential segment opened 134 | defer func() { 135 | if err != nil { 136 | printErr(err) 137 | } 138 | err = endSegment() 139 | if err != nil { 140 | panic(err) 141 | } 142 | }() 143 | 144 | // errs in loop are printed in the defer 145 | for { 146 | select { 147 | case err = <-errC: 148 | return 149 | default: 150 | } 151 | 152 | // Read from STDIN one chunk at a time 153 | n, err = io.ReadFull(inputStream, chunk) 154 | if err != nil { 155 | if err == io.EOF { 156 | err = nil 157 | } 158 | if n > 0 { 159 | fmt.Printf("io.ReadFull bytes on error: %s\n", string(chunk[:n])) 160 | } 161 | return 162 | } 163 | 164 | // check for new segment delim 165 | if chunk[0] == '$' { 166 | err = endSegment() 167 | if err != nil { 168 | return 169 | } 170 | 171 | err = nextSegment() 172 | if err != nil { 173 | return 174 | } 175 | } 176 | 177 | _, err = currentSegment.Write(chunk) 178 | if err != nil { 179 | return 180 | } 181 | } 182 | } 183 | 184 | func segmentName(base string, index int) string { 185 | return path.Join(base, fmt.Sprintf("segment-%d", index)) 186 | } 187 | 188 | func newSegmentFile(index int) (*os.File, error) { 189 | return os.OpenFile(segmentName(fileBase, index), os.O_APPEND|os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) 190 | } 191 | 192 | func uploadSegment(client *http.Client, index int, body io.ReadCloser) error { 193 | req := &http.Request{ 194 | Method: "POST", 195 | URL: &url.URL{ 196 | Scheme: "http", 197 | Host: "localhost:9094", 198 | Path: segmentName(httpBase, index), 199 | }, 200 | ProtoMajor: 1, 201 | ProtoMinor: 1, 202 | ContentLength: -1, 203 | Body: body, 204 | } 205 | _, err := client.Do(req) 206 | return err 207 | } 208 | 209 | func printErr(err error) { 210 | fmt.Printf("Error: %v\n", err) 211 | } 212 | -------------------------------------------------------------------------------- /examples/cors-example.json: -------------------------------------------------------------------------------- 1 | { 2 | "AllowedMethods": ["GET", "HEAD", "POST", "PUT", "DELETE", "OPTIONS"], 3 | "AllowedHeaders": ["Joc-Seq-Id", "Joc-Media-Type", "Joc-Timestamp", "Joc-Uniq-Id", "Joc-First-Frame-Clk", "Joc-Chunk-Type", "Expires"], 4 | "AllowedOrigins": ["*"] 5 | } 6 | -------------------------------------------------------------------------------- /examples/simple/README.md: -------------------------------------------------------------------------------- 1 | # Simple Example 2 | 3 | This simple example demonstrates chunked transfer of a single request. 4 | It uses 2 clients, a POST and GET client. The POST client opens a POST request to the chunked-streaming-server using STDIN as the request body. When a newline is read from STDIN, the POST client writes those bytes to the request. The GET client opens a GET request to the chunked-streaming-server for the same file as the POST request, logging lines to STDOUT as they are recieved. 5 | 6 | 7 | In one terminal session, start the chunked-streaming-server 8 | ``` 9 | ~/go-chunked-streaming-server $ go run main.go 10 | ``` 11 | 12 | Open a second terminal session and start the POST client. 13 | ``` 14 | ~/go-chunked-streaming-server $ go run examples/simple/post_client/main.go 15 | ``` 16 | 17 | Open a third terminal session and start the GET client. 18 | ``` 19 | ~/go-chunked-streaming-server $ go run examples/simple/get_client/main.go 20 | ``` 21 | 22 | In the second terminal session running the POST client, start typing whatever text you want. Each newline should be printed on the third terminal session runnining the GET client. 23 | -------------------------------------------------------------------------------- /examples/simple/get_client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "net/url" 9 | ) 10 | 11 | func main() { 12 | tr := http.DefaultTransport 13 | 14 | client := &http.Client{ 15 | Transport: tr, 16 | Timeout: 0, 17 | } 18 | 19 | req := &http.Request{ 20 | Method: "GET", 21 | URL: &url.URL{ 22 | Scheme: "http", 23 | Host: "localhost:9094", 24 | Path: "/post-client.txt", 25 | }, 26 | } 27 | 28 | resp, err := client.Do(req) 29 | 30 | if err != nil { 31 | panic(err) 32 | } 33 | defer resp.Body.Close() 34 | 35 | reader := bufio.NewReader(resp.Body) 36 | for { 37 | line, err := reader.ReadBytes('\n') 38 | 39 | if err != nil { 40 | if err == io.EOF { 41 | fmt.Println("EOF") 42 | break 43 | } 44 | fmt.Printf("Error: %v\n", err) 45 | break 46 | } 47 | 48 | fmt.Println(string(line)) 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /examples/simple/post_client/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/url" 7 | "os" 8 | ) 9 | 10 | func main() { 11 | tr := http.DefaultTransport 12 | 13 | client := &http.Client{ 14 | Transport: tr, 15 | Timeout: 0, 16 | } 17 | 18 | r := os.Stdin 19 | req := &http.Request{ 20 | Method: "POST", 21 | URL: &url.URL{ 22 | Scheme: "http", 23 | Host: "localhost:9094", 24 | Path: "/post-client.txt", 25 | }, 26 | ProtoMajor: 1, 27 | ProtoMinor: 1, 28 | ContentLength: -1, 29 | Body: r, 30 | } 31 | fmt.Printf("Doing request\n") 32 | _, err := client.Do(req) 33 | fmt.Printf("Done request. Err: %v\n", err) 34 | } 35 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mjneil/go-chunked-streaming-server 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/google/uuid v1.3.0 // indirect 7 | github.com/gorilla/mux v1.7.4 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= 2 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 3 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= 4 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 5 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 6 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 8 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 9 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 10 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 11 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 12 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 13 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 14 | github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= 15 | github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 16 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 17 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 18 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 19 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 20 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 21 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 22 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 23 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 24 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 25 | github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= 26 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 27 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 28 | github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= 29 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 30 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 31 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 32 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= 33 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 34 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 35 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 36 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 37 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94= 38 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 39 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= 40 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 41 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 42 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 43 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | 7 | "github.com/mjneil/go-chunked-streaming-server/server" 8 | ) 9 | 10 | var ( 11 | certFilePath = flag.String("c", "", "Certificate file path (only for https)") 12 | keyFilePath = flag.String("k", "", "Key file path (only for https)") 13 | baseOutPath = flag.String("p", "./content", "Path used to store") 14 | port = flag.Int("i", 9094, "Port used for HTTP ingress/ egress") 15 | corsConfigFilePath = flag.String("o", "", "JSON file path with the CORS headers definition") 16 | onlyRAM = flag.Bool("r", false, "Indicates DO NOT use disc as persistent/fallback storage (only RAM)") 17 | waitForDataToArrive = flag.Bool("w", false, "Indicates to GET request to wait for some specific if data is NOT present yet") 18 | doCleanupBasedOnCacheHeaders = flag.Bool("d", false, "Indicates to remove files from the server based on original Cache-Control (max-age) header") 19 | ) 20 | 21 | func checkError(err error) { 22 | if err != nil { 23 | log.Fatalln(err) 24 | } 25 | } 26 | 27 | func main() { 28 | flag.Parse() 29 | 30 | checkError(server.StartHTTPServer(*baseOutPath, *port, *certFilePath, *keyFilePath, *corsConfigFilePath, *onlyRAM, *doCleanupBasedOnCacheHeaders, *waitForDataToArrive)) 31 | } 32 | -------------------------------------------------------------------------------- /scripts/create-server-certs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Create certs dir if it does not exists 4 | mkdir -p ../certs 5 | 6 | # Generate private key 7 | openssl genrsa -out ../certs/server.key 2048 8 | openssl ecparam -genkey -name secp384r1 -out ../certs/server.key 9 | 10 | # Generation of self-signed(x509) public key (PEM-encodings .pem|.crt) based on the private (.key) 11 | openssl req -new -x509 -sha256 -key ../certs/server.key -out ../certs/server.crt -days 3650 12 | -------------------------------------------------------------------------------- /server/cors.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "os" 7 | ) 8 | 9 | // corsData Raw data of CORS config 10 | type CorsData struct { 11 | AllowedMethods []string `json:"AllowedMethods"` 12 | AllowedOrigins []string `json:"AllowedOrigins"` 13 | AllowedHeaders []string `json:"AllowedHeaders"` 14 | } 15 | 16 | // corsData Raw data of CORS config 17 | type Cors struct { 18 | Data CorsData 19 | Loaded bool 20 | } 21 | 22 | // NewCors Creates a new Cors object 23 | func NewCors() *Cors { 24 | c := new(Cors) 25 | 26 | c.loadDefault() 27 | 28 | return c 29 | } 30 | 31 | // LoadFromDisc Initializes CORS from config file 32 | func (c *Cors) LoadFromDisc(configFilePath string) error { 33 | data, errLoad := c.loadJSONDataFromDisc(configFilePath) 34 | if errLoad != nil { 35 | return errLoad 36 | } 37 | 38 | errJson := json.Unmarshal(data, &c.Data) 39 | if errJson != nil { 40 | return errJson 41 | } 42 | 43 | c.Loaded = true 44 | 45 | return nil 46 | } 47 | 48 | func (c *Cors) String() string { 49 | ret := "" 50 | 51 | b, err := json.Marshal(c.Data) 52 | if err == nil { 53 | ret = string(b) 54 | } 55 | 56 | return ret 57 | } 58 | 59 | func (c *Cors) GetAllowedOrigins() []string { 60 | return c.Data.AllowedOrigins 61 | } 62 | 63 | func (c *Cors) GetAllowedMethods() []string { 64 | return c.Data.AllowedMethods 65 | } 66 | 67 | func (c *Cors) GetAllowedHeaders() []string { 68 | return c.Data.AllowedHeaders 69 | } 70 | 71 | func (c *Cors) loadJSONDataFromDisc(configFilePath string) (data []byte, err error) { 72 | jsonFile, errOpen := os.Open(configFilePath) 73 | if errOpen != nil { 74 | err = errOpen 75 | return 76 | } 77 | defer jsonFile.Close() 78 | 79 | data, errRead := ioutil.ReadAll(jsonFile) 80 | if errRead != nil { 81 | err = errRead 82 | } 83 | 84 | return 85 | } 86 | 87 | func (c *Cors) loadDefault() { 88 | c.Data.AllowedMethods = []string{"GET", "HEAD", "POST", "PUT", "DELETE", "OPTIONS"} 89 | c.Data.AllowedHeaders = []string{""} 90 | c.Data.AllowedOrigins = []string{"*"} 91 | } 92 | -------------------------------------------------------------------------------- /server/file.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "os" 10 | "path" 11 | "path/filepath" 12 | "sync" 13 | "time" 14 | ) 15 | 16 | var ( 17 | // Files Array of on the fly files 18 | Files = map[string]*File{} 19 | 20 | // FilesLock Lock used to write / read files 21 | FilesLock = new(sync.RWMutex) 22 | ) 23 | 24 | // FileReader Defines a reader 25 | type FileReadCloser struct { 26 | offset int 27 | w http.ResponseWriter 28 | *File 29 | } 30 | 31 | // Read Reads bytes from filereader 32 | func (r *FileReadCloser) Read(p []byte) (int, error) { 33 | r.File.lock.RLock() 34 | defer r.File.lock.RUnlock() 35 | if r.offset >= len(r.File.buffer) { 36 | if r.File.eof { 37 | return 0, io.EOF 38 | } 39 | 40 | return 0, nil 41 | } 42 | n := copy(p, r.File.buffer[r.offset:]) 43 | r.offset += n 44 | // r.w.(http.Flusher).Flush() 45 | return n, nil 46 | } 47 | 48 | // File Definition of file 49 | type File struct { 50 | Name string 51 | headers http.Header 52 | lock *sync.RWMutex 53 | buffer []byte 54 | eof bool 55 | onDisk bool 56 | receivedAt time.Time 57 | maxAgeS int64 58 | } 59 | 60 | // NewFile Creates a new file 61 | func NewFile(name string, headers http.Header, maxAgeS int64) *File { 62 | f := File{ 63 | Name: name, 64 | headers: headers, 65 | lock: new(sync.RWMutex), 66 | buffer: []byte{}, 67 | eof: false, 68 | onDisk: false, 69 | receivedAt: time.Now(), 70 | maxAgeS: maxAgeS, 71 | } 72 | 73 | contentType := f.GetContentType() 74 | 75 | log.Println("NEW File Content-Type " + contentType) 76 | 77 | return &f 78 | } 79 | 80 | func (f *File) GetContentType() string { 81 | return f.headers.Get("Content-Type") 82 | } 83 | 84 | // NewReadCloser Crates a new filereader from a file 85 | func (f *File) NewReadCloser(baseDir string, w http.ResponseWriter) io.ReadCloser { 86 | f.lock.RLock() 87 | defer f.lock.RUnlock() 88 | 89 | if f.onDisk { 90 | name := path.Join(baseDir, f.Name) 91 | file, err := os.Open(name) 92 | if err != nil { 93 | panic(err) 94 | } 95 | fmt.Println("Skipping file reading and reading from disk") 96 | return file 97 | } 98 | 99 | fmt.Println("Reading from memory") 100 | return &FileReadCloser{ 101 | offset: 0, 102 | w: w, 103 | File: f, 104 | } 105 | } 106 | 107 | // Close Closes a file 108 | func (f *File) Close() error { 109 | f.lock.Lock() 110 | defer f.lock.Unlock() 111 | f.eof = true 112 | 113 | return nil 114 | } 115 | 116 | // Write Write bytes to a file 117 | func (f *File) Write(p []byte) (int, error) { 118 | f.lock.Lock() 119 | defer f.lock.Unlock() 120 | f.buffer = append(f.buffer, p...) 121 | return len(p), nil 122 | } 123 | 124 | // WriteToDisk Writes a file to disc 125 | func (f *File) WriteToDisk(baseDir string) error { 126 | f.lock.Lock() 127 | defer f.lock.Unlock() 128 | name := path.Join(baseDir, f.Name) 129 | 130 | if _, err := os.Stat(filepath.Dir(name)); os.IsNotExist(err) { 131 | err := os.MkdirAll(filepath.Dir(name), 0755) 132 | if err != nil { 133 | return err 134 | } 135 | } 136 | 137 | err := ioutil.WriteFile(name, f.buffer, 0644) 138 | if err != nil { 139 | return err 140 | } 141 | f.onDisk = true 142 | f.buffer = nil 143 | return nil 144 | } 145 | 146 | // RemoveFromDisk Removes file from disc 147 | func (f *File) RemoveFromDisk(baseDir string) error { 148 | f.lock.Lock() 149 | defer f.lock.Unlock() 150 | 151 | name := path.Join(baseDir, f.Name) 152 | err := os.Remove(name) 153 | 154 | // even if we get an error, lets act as if the file is completely removed 155 | f.onDisk = false 156 | f.buffer = nil 157 | 158 | return err 159 | } 160 | -------------------------------------------------------------------------------- /server/handlers.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net/http" 7 | "regexp" 8 | "strconv" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | // ChunkedResponseWriter Define a response writer 14 | type ChunkedResponseWriter struct { 15 | w http.ResponseWriter 16 | } 17 | 18 | // Write Writes few bytes 19 | func (rw ChunkedResponseWriter) Write(p []byte) (nn int, err error) { 20 | nn, err = rw.w.Write(p) 21 | rw.w.(http.Flusher).Flush() 22 | return 23 | } 24 | 25 | // GetHandler Sends file bytes 26 | func GetHandler(waitingRequests *WaitingRequests, cors *Cors, basePath string, w http.ResponseWriter, r *http.Request) { 27 | name := r.URL.String() 28 | 29 | FilesLock.RLock() 30 | f, ok := Files[name] 31 | FilesLock.RUnlock() 32 | 33 | if !ok { 34 | isFound := false 35 | waited := 0 * time.Millisecond 36 | if waitingRequests != nil { 37 | // Wait and return 38 | isFound, waited = waitingRequests.AddWaitingRequest(name, getHeadersFiltered(r.Header)) 39 | w.Header().Set("Waited-For-Data-Ms", strconv.FormatInt(int64(waited/time.Millisecond), 10)) 40 | if isFound { 41 | // Refresh file 42 | FilesLock.RLock() 43 | fnew, ok := Files[name] 44 | FilesLock.RUnlock() 45 | if !ok { 46 | // This should be very rare, file arrived but it is not in Files. It can happen if it expired just between arrived and this line 47 | isFound = false 48 | } else { 49 | f = fnew 50 | } 51 | } 52 | } 53 | if !isFound { 54 | addCors(w, cors) 55 | w.WriteHeader(http.StatusNotFound) 56 | return 57 | } 58 | } 59 | addCors(w, cors) 60 | addHeaders(w, f.headers) 61 | 62 | // Add chunked only if the file is not yet complete 63 | if !f.eof { 64 | w.Header().Set("Transfer-Encoding", "chunked") 65 | } 66 | 67 | w.WriteHeader(http.StatusOK) 68 | io.Copy(ChunkedResponseWriter{w}, f.NewReadCloser(basePath, w)) 69 | } 70 | 71 | // HeadHandler Sends if file exists 72 | func HeadHandler(cors *Cors, w http.ResponseWriter, r *http.Request) { 73 | FilesLock.RLock() 74 | f, ok := Files[r.URL.String()] 75 | FilesLock.RUnlock() 76 | 77 | addCors(w, cors) 78 | if !ok { 79 | w.WriteHeader(http.StatusNotFound) 80 | return 81 | } 82 | 83 | addHeaders(w, f.headers) 84 | w.Header().Set("Transfer-Encoding", "chunked") 85 | 86 | w.WriteHeader(http.StatusOK) 87 | } 88 | 89 | // PostHandler Writes a file 90 | func PostHandler(waitingRequests *WaitingRequests, onlyRAM bool, cors *Cors, basePath string, w http.ResponseWriter, r *http.Request) { 91 | // TODO: Add trigger blocking requests reusing/coping the code in Get 92 | name := r.URL.String() 93 | 94 | maxAgeS := getMaxAgeOr(r.Header.Get("Cache-Control"), -1) 95 | headers := getHeadersFiltered(r.Header) 96 | 97 | f := NewFile(name, headers, maxAgeS) 98 | 99 | FilesLock.Lock() 100 | Files[name] = f 101 | FilesLock.Unlock() 102 | 103 | // Start writing to file without holding lock so that GET requests can read from it 104 | io.Copy(f, r.Body) 105 | r.Body.Close() 106 | f.Close() 107 | 108 | if !onlyRAM { 109 | err := f.WriteToDisk(basePath) 110 | if err != nil { 111 | log.Fatalf("Error saving to disk: %v", err) 112 | } 113 | } 114 | addCors(w, cors) 115 | w.WriteHeader(http.StatusNoContent) 116 | 117 | // Awake GET requests waiting (if there are any) 118 | if waitingRequests != nil { 119 | waitingRequests.ReceivedDataFor(name) 120 | } 121 | } 122 | 123 | // PutHandler Writes a file 124 | func PutHandler(waitingRequests *WaitingRequests, onlyRAM bool, cors *Cors, basePath string, w http.ResponseWriter, r *http.Request) { 125 | PostHandler(waitingRequests, onlyRAM, cors, basePath, w, r) 126 | } 127 | 128 | // DeleteHandler Deletes a file 129 | func DeleteHandler(onlyRAM bool, cors *Cors, basePath string, w http.ResponseWriter, r *http.Request) { 130 | FilesLock.RLock() 131 | f, ok := Files[r.URL.String()] 132 | FilesLock.RUnlock() 133 | 134 | addCors(w, cors) 135 | if !ok { 136 | w.WriteHeader(http.StatusNotFound) 137 | return 138 | } 139 | 140 | FilesLock.Lock() 141 | delete(Files, r.URL.String()) 142 | FilesLock.Unlock() 143 | 144 | if !onlyRAM { 145 | f.RemoveFromDisk(basePath) 146 | } 147 | 148 | w.WriteHeader(http.StatusNoContent) 149 | } 150 | 151 | // OptionsHandler Returns CORS options 152 | func OptionsHandler(cors *Cors, w http.ResponseWriter, r *http.Request) { 153 | w.Header().Set("Transfer-Encoding", "chunked") 154 | 155 | addCors(w, cors) 156 | w.WriteHeader(http.StatusNoContent) 157 | } 158 | 159 | func addCors(w http.ResponseWriter, cors *Cors) { 160 | // Add Content-Type & Cache-Control automatically 161 | // Some features depends on those 162 | allowedHeaders := cors.GetAllowedHeaders() 163 | allowedHeaders = append(allowedHeaders, "Content-Type") 164 | allowedHeaders = append(allowedHeaders, "Cache-Control") 165 | 166 | w.Header().Set("Access-Control-Allow-Origin", strings.Join(cors.GetAllowedOrigins(), ", ")) 167 | w.Header().Set("Access-Control-Allow-Headers", strings.Join(allowedHeaders, ", ")) 168 | w.Header().Set("Access-Control-Allow-Methods", strings.Join(cors.GetAllowedMethods(), ", ")) 169 | w.Header().Set("Access-Control-Expose-Headers", strings.Join(allowedHeaders, ", ")) 170 | } 171 | 172 | func addHeaders(w http.ResponseWriter, headersSrc http.Header) { 173 | // Copy all headers 174 | for name, values := range headersSrc { 175 | // Loop over all values for the name. 176 | for _, value := range values { 177 | w.Header().Set(name, value) 178 | } 179 | } 180 | } 181 | 182 | func getMaxAgeOr(s string, def int64) int64 { 183 | ret := def 184 | r := regexp.MustCompile(`max-age=(?P\d*)`) 185 | match := r.FindStringSubmatch(s) 186 | for i, name := range r.SubexpNames() { 187 | if i > 0 && i <= len(match) { 188 | if name == "maxage" { 189 | valInt, err := strconv.ParseInt(match[i], 10, 64) 190 | if err == nil { 191 | ret = valInt 192 | break 193 | } 194 | } 195 | } 196 | } 197 | return ret 198 | } 199 | 200 | func getHeadersFiltered(headers http.Header) http.Header { 201 | ret := headers.Clone() 202 | 203 | // Clean up 204 | ret.Del("User-Agent") 205 | 206 | return ret 207 | } 208 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/gorilla/mux" 10 | ) 11 | 12 | var ( 13 | cleanUpChannel = make(chan bool) 14 | ) 15 | 16 | // StartHTTPServer Starts the webserver 17 | func StartHTTPServer(basePath string, port int, certFilePath string, keyFilePath string, corsConfigFilePath string, onlyRAM bool, doCleanupBasedOnCacheHeaders bool, waitForDataToArrive bool) error { 18 | var err error 19 | 20 | cors := NewCors() 21 | if corsConfigFilePath != "" { 22 | // Loads CORS config 23 | err = cors.LoadFromDisc(corsConfigFilePath) 24 | if err != nil { 25 | return err 26 | } 27 | } else { 28 | log.Printf("CORS default policy applied") 29 | } 30 | log.Printf("CORS: %s", cors.String()) 31 | 32 | var waitingRequests *WaitingRequests = nil 33 | if waitForDataToArrive { 34 | log.Printf("Using waiting requests map") 35 | waitingRequests = NewWaitingRequests() 36 | } 37 | 38 | r := mux.NewRouter() 39 | 40 | r.PathPrefix("/").Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 41 | defer w.(http.Flusher).Flush() 42 | log.Printf("%s %s", r.Method, r.URL.String()) 43 | switch r.Method { 44 | case http.MethodGet: 45 | GetHandler(waitingRequests, cors, basePath, w, r) 46 | case http.MethodHead: 47 | HeadHandler(cors, w, r) 48 | case http.MethodPost: 49 | PostHandler(waitingRequests, onlyRAM, cors, basePath, w, r) 50 | case http.MethodPut: 51 | PutHandler(waitingRequests, onlyRAM, cors, basePath, w, r) 52 | case http.MethodDelete: 53 | DeleteHandler(onlyRAM, cors, basePath, w, r) 54 | case http.MethodOptions: 55 | OptionsHandler(cors, w, r) 56 | default: 57 | w.WriteHeader(http.StatusMethodNotAllowed) 58 | } 59 | })).Methods(http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut, http.MethodDelete, http.MethodOptions) 60 | 61 | if doCleanupBasedOnCacheHeaders { 62 | startCleanUp(basePath, 1000) 63 | } 64 | 65 | if (certFilePath != "") && (keyFilePath != "") { 66 | // Try HTTPS 67 | log.Printf("HTTPS server running on port %d", port) 68 | err = http.ListenAndServeTLS(":"+strconv.Itoa(port), certFilePath, keyFilePath, r) 69 | } else { 70 | // Try HTTP 71 | log.Printf("HTTP server running on port %d", port) 72 | err = http.ListenAndServe(":"+strconv.Itoa(port), r) 73 | } 74 | 75 | if doCleanupBasedOnCacheHeaders { 76 | stopCleanUp() 77 | } 78 | if waitingRequests != nil { 79 | waitingRequests.Close() 80 | } 81 | 82 | return err 83 | } 84 | 85 | func startCleanUp(basePath string, periodMs int64) { 86 | go runCleanupEvery(basePath, periodMs, cleanUpChannel) 87 | 88 | log.Printf("HTTP Started clean up thread") 89 | } 90 | 91 | func stopCleanUp() { 92 | // Send finish signal 93 | cleanUpChannel <- true 94 | 95 | // Wait to finish 96 | <-cleanUpChannel 97 | 98 | log.Printf("HTTP Stopped clean up thread") 99 | } 100 | 101 | func runCleanupEvery(basePath string, periodMs int64, cleanUpChannelBidi chan bool) { 102 | timeCh := time.NewTicker(time.Millisecond * time.Duration(periodMs)) 103 | exit := false 104 | 105 | for !exit { 106 | select { 107 | // Wait for the next tick 108 | case tm := <-timeCh.C: 109 | cacheCleanUp(basePath, tm) 110 | 111 | case <-cleanUpChannelBidi: 112 | exit = true 113 | } 114 | } 115 | // Indicates finished 116 | cleanUpChannelBidi <- true 117 | 118 | log.Printf("HTTP Exited clean up thread") 119 | } 120 | 121 | func cacheCleanUp(basePath string, now time.Time) { 122 | filesToDel := map[string]*File{} 123 | 124 | // TODO: This is a brute force approach, optimization recommended 125 | 126 | FilesLock.Lock() 127 | defer FilesLock.Unlock() 128 | 129 | // Check for expired files 130 | for key, file := range Files { 131 | if file.maxAgeS >= 0 && file.eof { 132 | if file.receivedAt.Add(time.Second * time.Duration(file.maxAgeS)).Before(now) { 133 | filesToDel[key] = file 134 | } 135 | } 136 | } 137 | // Delete expired files 138 | for keyToDel, fileToDel := range filesToDel { 139 | // Delete from array 140 | delete(Files, keyToDel) 141 | if fileToDel.onDisk { 142 | fileToDel.RemoveFromDisk(basePath) 143 | } 144 | log.Printf("CLEANUP expired, deleted: %s", keyToDel) 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /server/waiting_requests.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "net/http" 5 | "regexp" 6 | "strconv" 7 | "sync" 8 | "time" 9 | 10 | "github.com/google/uuid" 11 | ) 12 | 13 | const defaultRequestExpiration time.Duration = 1000 * time.Millisecond 14 | const defaultRequestCleanUpEvery time.Duration = 100 * time.Millisecond 15 | 16 | const ( 17 | cancelSignal = 0 18 | isCancelled = 1 19 | dataArrived = 2 20 | ) 21 | 22 | // WaitingRequests stores data of blocked requests 23 | type WaitingRequests struct { 24 | requests map[string]*WaitingRequestArrayBlock 25 | requestsLock sync.RWMutex 26 | 27 | cleanUpChannelBidi chan bool 28 | } 29 | 30 | type WaitingRequestArrayBlock struct { 31 | requests []*WaitingRequest 32 | } 33 | 34 | // WaitingRequest Definition of blocked request waiting for data 35 | type WaitingRequest struct { 36 | uidStr string 37 | receivedAt time.Time 38 | expirationAt time.Time 39 | 40 | channelBidi chan int 41 | } 42 | 43 | // NewCors Creates a new Cors object 44 | func NewWaitingRequests() *WaitingRequests { 45 | brs := WaitingRequests{ 46 | requests: map[string]*WaitingRequestArrayBlock{}, 47 | cleanUpChannelBidi: make(chan bool), 48 | } 49 | 50 | go brs.runRequestCleanupEvery(defaultRequestCleanUpEvery) 51 | 52 | return &brs 53 | } 54 | 55 | // AddWaitingRequest Adds a new requests to wait for data, and blocks the execution 56 | func (brs *WaitingRequests) AddWaitingRequest(name string, headers http.Header) (found bool, waited time.Duration) { 57 | found = false 58 | nowStart := time.Now() 59 | // This is modified Expires, instead of HTTP-date timestamp uses duration in seconds (Ex: "Expires: in=10") 60 | expiration := brs.getExpiresInOr(headers.Get("Expires"), defaultRequestExpiration) 61 | 62 | uidStr := uuid.New().String() 63 | br := WaitingRequest{ 64 | uidStr: uidStr, 65 | receivedAt: nowStart, 66 | expirationAt: nowStart.Add(expiration), 67 | channelBidi: make(chan int), 68 | } 69 | 70 | brs.requestsLock.Lock() 71 | 72 | //Add waiting request 73 | reqArrayBlock, exists := brs.requests[name] 74 | if !exists { 75 | brs.requests[name] = &WaitingRequestArrayBlock{} 76 | reqArrayBlock = brs.requests[name] 77 | } 78 | reqArrayBlock.requests = append(reqArrayBlock.requests, &br) 79 | 80 | brs.requestsLock.Unlock() 81 | 82 | // Wait on signal 83 | msg := <-br.channelBidi 84 | if msg == dataArrived { 85 | found = true 86 | } 87 | 88 | // Remove request 89 | brs.requestsLock.Lock() 90 | 91 | brs.removeRequestByUID(name, uidStr) 92 | 93 | brs.requestsLock.Unlock() 94 | 95 | waited = time.Since(nowStart) 96 | 97 | return 98 | } 99 | 100 | // AddBlockedRequest Adds a new blocked requests 101 | func (brs *WaitingRequests) Close() { 102 | brs.cancelRemoveAllRequests() 103 | 104 | brs.stopCleanUp() 105 | } 106 | 107 | func (brs *WaitingRequests) ReceivedDataFor(name string) { 108 | now := time.Now() 109 | 110 | brs.requestsLock.Lock() 111 | defer brs.requestsLock.Unlock() 112 | 113 | for nameWaiting, reqArrayBlock := range brs.requests { 114 | if name == nameWaiting { 115 | for _, bReq := range reqArrayBlock.requests { 116 | if now.Before(bReq.expirationAt) { 117 | brs.responseRequest(bReq) 118 | } 119 | } 120 | } 121 | } 122 | } 123 | 124 | func (brs *WaitingRequests) getExpiresInOr(s string, def time.Duration) time.Duration { 125 | ret := def 126 | r := regexp.MustCompile(`in=(?P\d*)`) 127 | match := r.FindStringSubmatch(s) 128 | for i, name := range r.SubexpNames() { 129 | if i > 0 && i <= len(match) { 130 | if name == "in" { 131 | valInt, err := strconv.ParseInt(match[i], 10, 64) 132 | if err == nil { 133 | ret = time.Duration(valInt) * time.Second 134 | break 135 | } 136 | } 137 | } 138 | } 139 | return ret 140 | } 141 | 142 | func (brs *WaitingRequests) runRequestCleanupEvery(period time.Duration) { 143 | timeCh := time.NewTicker(period) 144 | exit := false 145 | 146 | for !exit { 147 | select { 148 | // Wait for the next tick 149 | case tm := <-timeCh.C: 150 | brs.expireRequests(tm) 151 | 152 | case <-brs.cleanUpChannelBidi: 153 | exit = true 154 | } 155 | } 156 | // Indicates finished 157 | brs.cleanUpChannelBidi <- true 158 | } 159 | 160 | func (brs *WaitingRequests) stopCleanUp() { 161 | // Send finish signal 162 | cleanUpChannel <- true 163 | 164 | // Wait to finish 165 | <-cleanUpChannel 166 | } 167 | 168 | func (brs *WaitingRequests) expireRequests(now time.Time) { 169 | 170 | brs.requestsLock.Lock() 171 | defer brs.requestsLock.Unlock() 172 | 173 | for name, reqArrayBlock := range brs.requests { 174 | for i, bReq := range reqArrayBlock.requests { 175 | // Add expired requests to delete array 176 | if now.After(bReq.expirationAt) { 177 | brs.cancelRequest(brs.requests[name].requests[i]) 178 | } 179 | } 180 | } 181 | } 182 | 183 | func (brs *WaitingRequests) cancelRemoveAllRequests() { 184 | for _, reqArrayBlock := range brs.requests { 185 | for _, bReq := range reqArrayBlock.requests { 186 | brs.cancelRequest(bReq) 187 | } 188 | } 189 | } 190 | 191 | func (brs *WaitingRequests) removeRequestByUID(name string, uidStr string) { 192 | reqArrayBlock, exists := brs.requests[name] 193 | if exists { 194 | for i, bReq := range reqArrayBlock.requests { 195 | if bReq.uidStr == uidStr { 196 | if len(reqArrayBlock.requests) > 1 { 197 | reqArrayBlock.requests = append(reqArrayBlock.requests[:i], reqArrayBlock.requests[i+1:]...) 198 | } else { 199 | reqArrayBlock.requests = []*WaitingRequest{} 200 | } 201 | } 202 | } 203 | // Remove name entry if no waiting requests 204 | if len(reqArrayBlock.requests) <= 0 { 205 | delete(brs.requests, name) 206 | } 207 | } 208 | } 209 | 210 | func (brs *WaitingRequests) cancelRequest(br *WaitingRequest) { 211 | br.channelBidi <- cancelSignal 212 | } 213 | 214 | func (brs *WaitingRequests) responseRequest(br *WaitingRequest) { 215 | br.channelBidi <- dataArrived 216 | } 217 | --------------------------------------------------------------------------------