├── .github ├── dependabot.yml └── workflows │ ├── release.yml │ └── test.yml ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── api ├── api.go ├── api.goconvey ├── api_test.go └── logger.go ├── auth ├── authentication.go ├── authentication_test.go ├── httpauth.go └── logger.go ├── cmd └── restreamer │ ├── logger.go │ ├── profile.go │ └── restreamer.go ├── configuration ├── config.go └── config_test.go ├── doc ├── architecture.odg ├── logo.png └── logo.svg ├── event ├── event.goconvey ├── handlers.go ├── heartbeat.go ├── logger.go ├── notifications.go ├── queue.go ├── queue_test.go └── urlhandler.go ├── examples ├── documented │ └── restreamer.json └── minimal │ └── restreamer.json ├── genpreamble.sh ├── go.mod ├── go.sum ├── metrics ├── logger.go ├── prom.go ├── stats.go └── stats_test.go ├── protocol ├── fork.go ├── logger.go ├── mpegts.goconvey ├── packet.go ├── packet_test.go ├── reader.go └── reader_test.go ├── streaming ├── acl.go ├── acl_test.go ├── client.go ├── connection.go ├── logger.go ├── manager.go ├── proxy.go ├── proxy_test.go └── streamer.go └── util ├── atomic.go ├── atomic_test.go ├── errors.go ├── log.go ├── log_test.go ├── set.go ├── shuffle.go ├── shuffle_test.go ├── signal_unix.go ├── signal_windows.go └── util.goconvey /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: [ "*" ] 6 | 7 | permissions: 8 | contents: read 9 | 10 | jobs: 11 | test: 12 | name: Test (Release Go) 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Check out code 16 | uses: actions/checkout@v4 17 | - name: Test 18 | run: make DOCKER=docker release-test 19 | build: 20 | name: Release Build 21 | runs-on: ubuntu-latest 22 | permissions: 23 | # for uploading artifacts 24 | actions: write 25 | steps: 26 | - name: Check out code 27 | uses: actions/checkout@v4 28 | - name: Build 29 | run: make DOCKER=docker release 30 | - name: Upload artifacts 31 | uses: actions/upload-artifact@v4 32 | with: 33 | name: restreamer 34 | path: | 35 | restreamer-* 36 | SHA256SUMS 37 | if-no-files-found: warn 38 | retention-days: 1 39 | release: 40 | name: Release Tag 41 | runs-on: ubuntu-latest 42 | needs: [ build, test ] 43 | permissions: 44 | contents: write 45 | steps: 46 | - name: Download artifacts 47 | uses: actions/download-artifact@v4 48 | with: 49 | name: restreamer 50 | - name: List artifacts 51 | run: ls -lR 52 | - name: Verify checksums 53 | run: sha256sum -c SHA256SUMS 54 | - name: Construct release version 55 | id: release_version 56 | # remove the leading v (if present) and match the relaxed SemVer part 57 | run: | 58 | export release_version=$(echo ${{ github.ref_name }} | sed -E 's/^v?(([0-9]+.)*[0-9]+)/\1/') 59 | echo "version=${release_version}" >> "${GITHUB_OUTPUT}" 60 | - name: Create GitHub release 61 | uses: swisstxt/github-action-release-artifacts@v1 62 | with: 63 | tag: ${{ github.ref_name }} 64 | create_release: true 65 | release_name: ${{ steps.release_version.outputs.version }} 66 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Build + Test 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | test: 14 | name: Build + Test (Latest Go) 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Set up Go 18 | uses: actions/setup-go@v5 19 | with: 20 | go-version: '^1' 21 | - name: Check out code 22 | uses: actions/checkout@v4 23 | - name: Build 24 | run: make 25 | - name: Test 26 | run: make test 27 | releast-test: 28 | name: Test (Release Go) 29 | runs-on: ubuntu-latest 30 | steps: 31 | - name: Set up Go 32 | uses: actions/setup-go@v5 33 | with: 34 | go-version: '^1' 35 | - name: Check out code 36 | uses: actions/checkout@v4 37 | - name: Test 38 | run: make DOCKER=docker release-test 39 | prerelease: 40 | name: Prerelease 41 | runs-on: ubuntu-latest 42 | permissions: 43 | contents: read 44 | # for uploading artifacts 45 | actions: write 46 | steps: 47 | - name: Check out code 48 | uses: actions/checkout@v4 49 | - name: Release build 50 | run: make DOCKER=docker restreamer-linux-amd64 51 | - name: Upload test artifacts 52 | uses: actions/upload-artifact@v4 53 | with: 54 | name: restreamer-linux-amd64 55 | path: restreamer-linux-amd64 56 | retention-days: 10 57 | overwrite: true 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /*.json 2 | restreamer 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - "1.x" 5 | - "1.15.x" 6 | - master 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1 AS build 2 | 3 | WORKDIR /build 4 | COPY . /build 5 | 6 | RUN make restreamer 7 | 8 | FROM scratch 9 | LABEL maintainer="Gregor Riepl " 10 | 11 | COPY --from=build /build/restreamer / 12 | COPY examples/minimal/restreamer.json / 13 | 14 | EXPOSE 8000 15 | ENTRYPOINT ["/restreamer"] 16 | CMD ["/restreamer.json"] 17 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # enable profiling 2 | #export GODEBUG = gctrace=1 3 | # use go netcode instead of libc 4 | export CGO_ENABLED = 0 5 | # set the build container Go version 6 | GO_VERSION = 1.22.1 7 | # for overriding the container runtime (needed for GitHub) 8 | DOCKER = podman 9 | 10 | # all release binaries to build by default 11 | RELEASE_BINARIES := restreamer-linux-amd64 restreamer-linux-386 restreamer-linux-arm restreamer-linux-arm64 restreamer-darwin-amd64 restreamer-darwin-arm64 restreamer-windows-amd64.exe restreamer-windows-386.exe restreamer-windows-arm64.exe 12 | 13 | # always force a rebuild of the main binary 14 | .PHONY: all clean test fmt vendor docker restreamer 15 | 16 | all: restreamer 17 | 18 | clean: 19 | rm -rf restreamer SHA256SUMS ${RELEASE_BINARIES} 20 | 21 | test: 22 | go vet ./... 23 | go test ./... 24 | 25 | fmt: 26 | go fmt ./... 27 | 28 | docker: 29 | podman build -t onitake/restreamer . 30 | 31 | restreamer: 32 | go build ./cmd/restreamer 33 | 34 | # release builds - use container runtime to cross-compile to various architectures 35 | 36 | release-test: 37 | $(DOCKER) run -e GOCACHE=/go/.cache -e CGO_ENABLED=0 --rm -v $(abspath .):/go/restreamer:ro -w /go/restreamer golang:${GO_VERSION} sh -c "go vet ./... && go test ./..." 38 | 39 | release: SHA256SUMS 40 | 41 | SHA256SUMS: $(RELEASE_BINARIES) 42 | sha256sum $^ > $@ 43 | 44 | restreamer-linux-amd64: 45 | $(DOCKER) run -e GOOS=linux -e GOARCH=amd64 -e GOCACHE=/go/.cache -e CGO_ENABLED=0 --rm -v $(abspath .):/go/restreamer -w /go/restreamer golang:${GO_VERSION} sh -c "git config --global --add safe.directory /go/restreamer; go build -o $@ ./cmd/restreamer" 46 | 47 | restreamer-linux-386: 48 | $(DOCKER) run -e GOOS=linux -e GOARCH=386 -e GOCACHE=/go/.cache -e CGO_ENABLED=0 --rm -v $(abspath .):/go/restreamer -w /go/restreamer golang:${GO_VERSION} sh -c "git config --global --add safe.directory /go/restreamer; go build -o $@ ./cmd/restreamer" 49 | 50 | restreamer-linux-arm: 51 | $(DOCKER) run -e GOOS=linux -e GOARCH=arm -e GOCACHE=/go/.cache -e CGO_ENABLED=0 --rm -v $(abspath .):/go/restreamer -w /go/restreamer golang:${GO_VERSION} sh -c "git config --global --add safe.directory /go/restreamer; go build -o $@ ./cmd/restreamer" 52 | 53 | restreamer-linux-arm64: 54 | $(DOCKER) run -e GOOS=linux -e GOARCH=arm64 -e GOCACHE=/go/.cache -e CGO_ENABLED=0 --rm -v $(abspath .):/go/restreamer -w /go/restreamer golang:${GO_VERSION} sh -c "git config --global --add safe.directory /go/restreamer; go build -o $@ ./cmd/restreamer" 55 | 56 | restreamer-darwin-amd64: 57 | $(DOCKER) run -e GOOS=darwin -e GOARCH=amd64 -e GOCACHE=/go/.cache -e CGO_ENABLED=0 --rm -v $(abspath .):/go/restreamer -w /go/restreamer golang:${GO_VERSION} sh -c "git config --global --add safe.directory /go/restreamer; go build -o $@ ./cmd/restreamer" 58 | 59 | restreamer-darwin-arm64: 60 | $(DOCKER) run -e GOOS=darwin -e GOARCH=arm64 -e GOCACHE=/go/.cache -e CGO_ENABLED=0 --rm -v $(abspath .):/go/restreamer -w /go/restreamer golang:${GO_VERSION} sh -c "git config --global --add safe.directory /go/restreamer; go build -o $@ ./cmd/restreamer" 61 | 62 | restreamer-windows-amd64.exe: 63 | $(DOCKER) run -e GOOS=windows -e GOARCH=amd64 -e GOCACHE=/go/.cache -e CGO_ENABLED=0 --rm -v $(abspath .):/go/restreamer -w /go/restreamer golang:${GO_VERSION} sh -c "git config --global --add safe.directory /go/restreamer; go build -o $@ ./cmd/restreamer" 64 | 65 | restreamer-windows-386.exe: 66 | $(DOCKER) run -e GOOS=windows -e GOARCH=386 -e GOCACHE=/go/.cache -e CGO_ENABLED=0 --rm -v $(abspath .):/go/restreamer -w /go/restreamer golang:${GO_VERSION} sh -c "git config --global --add safe.directory /go/restreamer; go build -o $@ ./cmd/restreamer" 67 | 68 | restreamer-windows-arm64.exe: 69 | $(DOCKER) run -e GOOS=windows -e GOARCH=arm64 -e GOCACHE=/go/.cache -e CGO_ENABLED=0 --rm -v $(abspath .):/go/restreamer -w /go/restreamer golang:${GO_VERSION} sh -c "git config --global --add safe.directory /go/restreamer; go build -o $@ ./cmd/restreamer" 70 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![language](https://img.shields.io/badge/language-go-%2300ADD8.svg?style=flat-square)](https://golang.org/) [![release](https://img.shields.io/github/release/onitake/restreamer.svg?style=flat-square)](https://github.com/onitake/restreamer/releases) [![goreport](https://goreportcard.com/badge/github.com/onitake/restreamer?style=flat-square)](https://goreportcard.com/report/github.com/onitake/restreamer) ![license](https://img.shields.io/github/license/onitake/restreamer?style=flat-square) 2 | 3 | ![restreamer logo](doc/logo.png) 4 | 5 | # restreamer 6 | 7 | HTTP transport stream proxy 8 | 9 | Copyright © 2016-2024 Gregor Riepl; 10 | All rights reserved. 11 | 12 | Please see the LICENSE file for details on permitted use of this software. 13 | 14 | 15 | ## Introduction 16 | 17 | restreamer is a proof-of-concept implementation of a streaming proxy 18 | that can fetch, buffer and distribute [MPEG transport streams](https://en.wikipedia.org/wiki/MPEG-TS). 19 | 20 | It serves as a non-transcoding streaming proxy for legacy HbbTV applications. 21 | 22 | Data sources can be local files, remote HTTP servers or raw TCP streams. 23 | Unix domain sockets are also supported. 24 | 25 | The proxy is stateless: Streams are transported in realtime 26 | and cached resources are only kept in memory. 27 | This makes restreamer perfectly suited for a multi-tiered architecture, 28 | that can be easily deployed on containers and expanded or shrunk as 29 | load demands. 30 | 31 | 32 | ## Architecture 33 | 34 | restreamer is written in [Go](https://golang.org/), a very versatile 35 | programming language. The built-in network layer makes it a first-class 36 | choice for a streaming server. 37 | 38 | These are the key components: 39 | * util - a small utility library 40 | * streaming/client - HTTP getter for fetching upstream data 41 | * streaming/connection - HTTP server that feeds data to clients 42 | * streaming/streamer - connection broker and data queue 43 | * api/api - web API for service monitoring 44 | * streaming/proxy - static web server and proxy 45 | * protocol - network protocol library 46 | * configuration - abstraction of the configuration file 47 | * metrics - a small wrapper around the Promethus client library 48 | * metrics/stats - the old, deprecated metrics collector; use Prometheus if possible 49 | * cmd/restreamer - core program that glues the components together 50 | 51 | 52 | ## Compilation 53 | 54 | restreamer is go-gettable. 55 | Just invoke: 56 | 57 | ``` 58 | CGO_ENABLED=0 go get github.com/onitake/restreamer/... 59 | ``` 60 | 61 | Disabling CGO is recommended, as this will produce a standalone binary that 62 | does not depend on libc. This is useful for running restreamer in a bare container. 63 | 64 | A makefile is also provided, as a quick build reference. 65 | Simply invoke `make` to build `restreamer`. 66 | 67 | Passing GOOS and/or GOARCH will yield cross-compiled binaries for the 68 | respective platform. For example: 69 | 70 | ``` 71 | GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build github.com/onitake/restreamer/cmd/restreamer 72 | ``` 73 | or 74 | ``` 75 | make GOOS=windows GOARCH=amd64 76 | ``` 77 | 78 | You can also use `make test` to run the test suite, or `make fmt` to run `go fmt` on all sources. 79 | 80 | 81 | ## Releases 82 | 83 | Release builds are automatically done by a GitHub Action whenever a tag is pushed. 84 | 85 | Make sure that your tags conform to semantic versioning and begin with a "v". 86 | Example: v0.9.1 87 | 88 | Note: The release upload my sometimes fail due to issues with the GitHub asset API. 89 | Retry the release step until all artifacts have been uploaded. 90 | 91 | Once the upload is complete, open the newly generated release and add the changelog 92 | to the description. 93 | 94 | 95 | ## Configuration 96 | 97 | All configuration is done through a configuration file named `restreamer.json`. 98 | 99 | See `examples/documented/restreamer.json` for a documented example. 100 | 101 | The input and output buffer sizes should be adapted to the expected stream 102 | bitrates and must account for unstable or slow client-side internet connections. 103 | 104 | Output buffers should cover at least 1-5 seconds of video and the input buffer 105 | should be 4 times as much. 106 | The amount of packets to buffer, based on the video+audio bitrates, can be 107 | calculated with this formula (overhead is not taken into account): 108 | 109 | ``` 110 | mpegts_packet_size = 188 111 | buffer_size_packets = (avg_video_bitrate + avg_audio_bitrate) * buffer_size_seconds / (mpegts_packet_size * 8) 112 | ``` 113 | 114 | It is also important to keep the bandwidth of the network interfaces 115 | in mind, so the connection limit should be set accordingly. 116 | 117 | Buffer memory usage is equally important, and can be roughly calculated as follows: 118 | 119 | ``` 120 | mpegts_packet_size = 188 121 | max_buffer_memory = mpegts_packet_size * (number_of_streams * input_buffer_size + max_connections * output_buffer_size) 122 | ``` 123 | 124 | It is possible to specify multiple upstream URLs per stream. 125 | These will be tested in a round-robin fashion in random order, 126 | with the first successful one being used. 127 | The list is only shuffled once, at program startup. 128 | 129 | If a connection is terminated, all URLs will be tried again after a delay. 130 | If delay is 0, the stream will stay offline. 131 | 132 | To protect against overload, both a soft and a hard limit on the number of 133 | downstream connections can be set. When the soft limit is reached, the health 134 | API will start reporting that the server is "full". Once the hard limit is 135 | reached, new connections will be responded with a 404. A 503 would be more 136 | appropriate, but this is not handled well by many legacy streaming clients. 137 | 138 | 139 | ## Logging 140 | 141 | restreamer has a JSON logging module built in. 142 | Output can be sent to stdout or written to a log file. 143 | 144 | It is highly recommended to log to stdout and collect logs using journald 145 | or a similar logging engine. 146 | 147 | 148 | ## Metrics 149 | 150 | Metrics are exported through the Prometheus client library. Enable a Prometheus 151 | API endpoint and expose it on /metrics to expose them. 152 | 153 | Supported metrics are: 154 | 155 | * _streaming_packets_sent_ 156 | Total number of MPEG-TS packets sent from the output queue. 157 | * _streaming_bytes_sent_ 158 | Total number of bytes sent from the output queue. 159 | * _streaming_packets_dropped_ 160 | Total number of MPEG-TS packets dropped from the output queue. 161 | * _streaming_bytes_dropped_ 162 | Total number of bytes dropped from the output queue. 163 | * _streaming_connections_ 164 | Number of active client connections. 165 | * _streaming_duration_ 166 | Total time spent streaming, summed over all client connections. In nanoseconds. 167 | * _streaming_source_connected_ 168 | Connection status, 0=disconnected 1=connected. 169 | * _streaming_packets_received_ 170 | Total number of MPEG-TS packets received. 171 | * _streaming_bytes_received_ 172 | Total number of bytes received. 173 | 174 | Additionally, the standard process metrics supported by the Prometheus client 175 | library are exported. Go runtime statistics are disabled, as they can have a 176 | considerable effect on realtime operation. To enable them, you need to turn on 177 | profiling. 178 | 179 | 180 | ## Optimisation 181 | 182 | ### Test Stream 183 | 184 | Using the example config and ffmpeg, a test stream can be set up. 185 | 186 | Create a pipe: 187 | ``` 188 | mkfifo /tmp/pipe.ts 189 | ``` 190 | Start feeding data into the pipe (note the -re parameter for real-time streaming): 191 | ``` 192 | ffmpeg -re -stream_loop -1 -i test.ts -fflags +genpts -c:a copy -c:v copy -y /tmp/pipe.ts 193 | ``` 194 | Start the proxy: 195 | ``` 196 | bin/restreamer 197 | ``` 198 | Start playing: 199 | ``` 200 | cvlc http://localhost:8000/pipe.ts 201 | ``` 202 | 203 | ### File Descriptors 204 | 205 | Continuous streaming services require a lot of open file descriptors, 206 | particularly when running a lot of low-bandwidth streams on a powerful 207 | streaming node. 208 | 209 | Many operating systems limit the number of file descriptors a single 210 | process can use, so care must be taken that servers aren't starved by 211 | this artificial limit. 212 | 213 | As a rough guideline, determine the maximum downstream bandwidth your 214 | server can handle, then divide by the bandwidth of each stream and 215 | add the number of upstream connections. 216 | Reserve some file descriptors for reconnects and fluctuations. 217 | 218 | For example, if your server has a dedicated 10Gbit/s downstream network 219 | connection and it serves 10 streams at 8Mbit/s, the required number 220 | of file descriptors would be: 221 | 222 | ``` 223 | descriptor_count = streams_count + downstream_bandwidth / stream_bandwidth * 200% 224 | => (10 + 10 Gbit/s / 8 Mbit/s) * 2 = 2520 225 | ``` 226 | 227 | On many Linux systems, the default is very low, at 1024 file 228 | descriptors, so restreamer would not be able to saturate all bandwidth. 229 | With the following systemd unit file, this would be remedied: 230 | ``` 231 | [Unit] 232 | Description=restreamer HTTP streaming proxy 233 | After=network.target 234 | 235 | [Service] 236 | Type=simple 237 | ExecStart=/usr/bin/restreamer 238 | Restart=always 239 | KillMode=mixed 240 | LimitNOFILE=3000 241 | 242 | [Install] 243 | WantedBy=multi-user.target 244 | ``` 245 | 246 | In most cases, it's safe to set a high value, so something like 247 | 64000 (or more) would be fine. 248 | 249 | ### Memory/CPU/Network Usage 250 | 251 | To test behaviour under heavy network load, it is recommended to run 252 | multiple concurrent HTTP connections that only read data occasionally. 253 | 254 | This will force restreamer to buffer excessively, increasing memory usage. 255 | 256 | You should monitor memory usage of the restreamer process closely, 257 | and change buffer sizes or system memory accordingly. 258 | 259 | Note that the Go runtime does not immediately return garbage collected 260 | memory to the operating system. This is done only about every 5 minutes 261 | by a special reclaiming task. 262 | 263 | Using the built-in Go profiler can be more useful to watch memory usage. 264 | You should check out the `profiler` branch, it opens a separate web 265 | server on port 6060 that Go pprof can connect to. See [net/http/pprof](https://golang.org/pkg/net/http/pprof/) 266 | for instructions on its usage. 267 | 268 | ### Syscall Load 269 | 270 | Live streaming services like restreamer require realtime or near-realtime 271 | operation. Latency should be minimised, highlighting the importance of 272 | small buffers. Unfortunately, small buffers will increase the number of 273 | syscalls needed to feed data to the remote clients. 274 | 275 | In light of the recently published [speculative execution exploits 276 | affecting Intel CPUs](https://meltdownattack.com/), high syscall rates will 277 | have a devastating effect on performance on these CPUs due to 278 | mitigations in the operating system. 279 | 280 | For this reason, restreamer is relying on buffered I/O, with default buffer 281 | sizes as set in the Go http package. In the current implementation, 282 | a 2KiB and a 4KiB buffer is used. This should give a good compromise 283 | between moderate latency and limited system load. 284 | 285 | Even with these buffers, performance loss compared with unmitigated systems 286 | is considerable. If this is unacceptable, it is highly recommended to run 287 | restreamer on a system that does not need/use Meltdown mitigations. 288 | -------------------------------------------------------------------------------- /api/api.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2016-2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package api 18 | 19 | import ( 20 | "encoding/json" 21 | "github.com/onitake/restreamer/auth" 22 | "github.com/onitake/restreamer/metrics" 23 | "net/http" 24 | ) 25 | 26 | // connectChecker represents a type that can report its "connected" status. 27 | type connectChecker interface { 28 | Connected() bool 29 | } 30 | 31 | // healthApi encapsulates a system status object and 32 | // provides an HTTP/JSON handler for reporting system health. 33 | type healthApi struct { 34 | stats metrics.Statistics 35 | // auth is an authentication verifier for client requests 36 | auth auth.Authenticator 37 | } 38 | 39 | // NewHealthApi creates a new health API object, 40 | // serving data from a system Statistics object. 41 | func NewHealthApi(stats metrics.Statistics, auth auth.Authenticator) http.Handler { 42 | return &healthApi{ 43 | stats: stats, 44 | auth: auth, 45 | } 46 | } 47 | 48 | // ServeHTTP is the http handler method. 49 | // It sends back information about system health. 50 | func (api *healthApi) ServeHTTP(writer http.ResponseWriter, request *http.Request) { 51 | // set the content type for all responses 52 | writer.Header().Add("Content-Type", "application/json") 53 | 54 | // fail-fast: verify that this user can access this resource first 55 | if !auth.HandleHttpAuthentication(api.auth, request, writer) { 56 | return 57 | } 58 | 59 | global := api.stats.GetGlobalStatistics() 60 | var stats struct { 61 | Status string `json:"status"` 62 | Viewer int `json:"viewer"` 63 | Limit int `json:"limit"` 64 | Max int `json:"max"` 65 | Bandwidth int `json:"bandwidth"` 66 | } 67 | // report for both hard and soft, respecting disabled limits 68 | if global.MaxConnections != 0 && global.Connections >= global.MaxConnections { 69 | stats.Status = "full" 70 | } else if global.FullConnections != 0 && global.Connections >= global.FullConnections { 71 | stats.Status = "full" 72 | } else { 73 | stats.Status = "ok" 74 | } 75 | stats.Viewer = int(global.Connections) 76 | stats.Limit = int(global.FullConnections) 77 | stats.Max = int(global.MaxConnections) 78 | stats.Bandwidth = int(global.BytesPerSecondSent * 8 / 1024) // kbit/s 79 | 80 | response, err := json.Marshal(&stats) 81 | if err == nil { 82 | writer.WriteHeader(http.StatusOK) 83 | if _, err := writer.Write(response); err != nil { 84 | logger.Logkv( 85 | "event", eventApiError, 86 | "error", errorApiWrite, 87 | "message", err.Error(), 88 | ) 89 | } 90 | } else { 91 | writer.WriteHeader(http.StatusInternalServerError) 92 | if _, err := writer.Write([]byte(http.StatusText(http.StatusInternalServerError))); err != nil { 93 | logger.Logkv( 94 | "event", eventApiError, 95 | "error", errorApiWrite, 96 | "message", err.Error(), 97 | ) 98 | } 99 | logger.Logkv( 100 | "event", eventApiError, 101 | "error", errorApiJsonEncode, 102 | "message", err.Error(), 103 | ) 104 | } 105 | } 106 | 107 | // statisticsApi encapsulates a system status object and 108 | // provides an HTTP/JSON handler for reporting total system statistics. 109 | type statisticsApi struct { 110 | stats metrics.Statistics 111 | // auth is an authentication verifier for client requests 112 | auth auth.Authenticator 113 | } 114 | 115 | // NewStatisticsApi creates a new statistics API object, 116 | // serving data from a system Statistics object. 117 | func NewStatisticsApi(stats metrics.Statistics, auth auth.Authenticator) http.Handler { 118 | return &statisticsApi{ 119 | stats: stats, 120 | auth: auth, 121 | } 122 | } 123 | 124 | // ServeHTTP is the http handler method. 125 | // It sends back information about system health. 126 | func (api *statisticsApi) ServeHTTP(writer http.ResponseWriter, request *http.Request) { 127 | // set the content type for all responses 128 | writer.Header().Add("Content-Type", "application/json") 129 | 130 | // fail-fast: verify that this user can access this resource first 131 | if !auth.HandleHttpAuthentication(api.auth, request, writer) { 132 | return 133 | } 134 | 135 | global := api.stats.GetGlobalStatistics() 136 | var stats struct { 137 | Status string `json:"status"` 138 | Connections int `json:"connections"` 139 | MaxConnections int `json:"max_connections"` 140 | FullConnections int `json:"full_connections"` 141 | TotalPacketsReceived uint64 `json:"total_packets_received"` 142 | TotalPacketsSent uint64 `json:"total_packets_sent"` 143 | TotalPacketsDropped uint64 `json:"total_packets_dropped"` 144 | TotalBytesReceived uint64 `json:"total_bytes_received"` 145 | TotalBytesSent uint64 `json:"total_bytes_sent"` 146 | TotalBytesDropped uint64 `json:"total_bytes_dropped"` 147 | TotalStreamTime int64 `json:"total_stream_time_ns"` 148 | PacketsPerSecondReceived uint64 `json:"packets_per_second_received"` 149 | PacketsPerSecondSent uint64 `json:"packets_per_second_sent"` 150 | PacketsPerSecondDropped uint64 `json:"packets_per_second_dropped"` 151 | BytesPerSecondReceived uint64 `json:"bytes_per_second_received"` 152 | BytesPerSecondSent uint64 `json:"bytes_per_second_sent"` 153 | BytesPerSecondDropped uint64 `json:"bytes_per_second_dropped"` 154 | } 155 | // report for both hard and soft, respecting disabled limits 156 | if global.MaxConnections != 0 && global.Connections >= global.MaxConnections { 157 | stats.Status = "overload" 158 | } else if global.FullConnections != 0 && global.Connections >= global.FullConnections { 159 | stats.Status = "full" 160 | } else { 161 | stats.Status = "ok" 162 | } 163 | stats.Connections = int(global.Connections) 164 | stats.MaxConnections = int(global.MaxConnections) 165 | stats.FullConnections = int(global.FullConnections) 166 | stats.TotalPacketsReceived = global.TotalPacketsReceived 167 | stats.TotalPacketsSent = global.TotalPacketsSent 168 | stats.TotalPacketsDropped = global.TotalPacketsDropped 169 | stats.TotalBytesReceived = global.TotalBytesReceived 170 | stats.TotalBytesSent = global.TotalBytesSent 171 | stats.TotalBytesDropped = global.TotalBytesDropped 172 | stats.TotalStreamTime = global.TotalStreamTime 173 | stats.PacketsPerSecondReceived = global.PacketsPerSecondReceived 174 | stats.PacketsPerSecondSent = global.PacketsPerSecondSent 175 | stats.PacketsPerSecondDropped = global.PacketsPerSecondDropped 176 | stats.BytesPerSecondReceived = global.BytesPerSecondReceived 177 | stats.BytesPerSecondSent = global.BytesPerSecondSent 178 | stats.BytesPerSecondDropped = global.BytesPerSecondDropped 179 | 180 | response, err := json.Marshal(&stats) 181 | if err == nil { 182 | writer.WriteHeader(http.StatusOK) 183 | if _, err := writer.Write(response); err != nil { 184 | logger.Logkv( 185 | "event", eventApiError, 186 | "error", errorApiWrite, 187 | "message", err.Error(), 188 | ) 189 | } 190 | } else { 191 | writer.WriteHeader(http.StatusInternalServerError) 192 | if _, err := writer.Write([]byte(http.StatusText(http.StatusInternalServerError))); err != nil { 193 | logger.Logkv( 194 | "event", eventApiError, 195 | "error", errorApiWrite, 196 | "message", err.Error(), 197 | ) 198 | } 199 | logger.Logkv( 200 | "event", eventApiError, 201 | "error", errorApiJsonEncode, 202 | "message", err.Error(), 203 | ) 204 | } 205 | } 206 | 207 | // streamStatApi provides an API for checking stream availability. 208 | // The HTTP handler returns status code 200 if a stream is connected 209 | // and 404 if not. 210 | type streamStateApi struct { 211 | client connectChecker 212 | // auth is an authentication verifier for client requests 213 | auth auth.Authenticator 214 | } 215 | 216 | // NewStreamStateApi creates a new stream status API object, 217 | // serving the "connected" status of a stream connection. 218 | func NewStreamStateApi(client connectChecker, auth auth.Authenticator) http.Handler { 219 | return &streamStateApi{ 220 | client: client, 221 | auth: auth, 222 | } 223 | } 224 | 225 | // ServeHTTP is the http handler method. 226 | // It sends back "200 ok" if the stream is connected and "404 not found" if not, 227 | // along with the corresponding HTTP status code. 228 | func (api *streamStateApi) ServeHTTP(writer http.ResponseWriter, request *http.Request) { 229 | // set the content type for all responses 230 | writer.Header().Add("Content-Type", "text/plain") 231 | 232 | // fail-fast: verify that this user can access this resource first 233 | if !auth.HandleHttpAuthentication(api.auth, request, writer) { 234 | return 235 | } 236 | 237 | if api.client.Connected() { 238 | writer.WriteHeader(http.StatusOK) 239 | if _, err := writer.Write([]byte("200 ok")); err != nil { 240 | logger.Logkv( 241 | "event", eventApiError, 242 | "error", errorApiWrite, 243 | "message", err.Error(), 244 | ) 245 | } 246 | } else { 247 | writer.WriteHeader(http.StatusNotFound) 248 | if _, err := writer.Write([]byte("404 not found")); err != nil { 249 | logger.Logkv( 250 | "event", eventApiError, 251 | "error", errorApiWrite, 252 | "message", err.Error(), 253 | ) 254 | } 255 | } 256 | } 257 | 258 | // inhibitor represents a type that can prevent or allow new connections. 259 | type inhibitor interface { 260 | SetInhibit(inhibit bool) 261 | } 262 | 263 | // streamControlApi allows manipulation of a stream's state. 264 | // If this API is enabled for a stream, requests to start and stop it externally 265 | // can be sent. Useful for testing or as an emergency kill switch. 266 | type streamControlApi struct { 267 | inhibit inhibitor 268 | // auth is an authentication verifier for client requests 269 | auth auth.Authenticator 270 | } 271 | 272 | // NewStreamControlApi creates a new stream status API object, 273 | // serving the "connected" status of a stream connection. 274 | func NewStreamControlApi(inhibit inhibitor, auth auth.Authenticator) http.Handler { 275 | return &streamControlApi{ 276 | inhibit: inhibit, 277 | auth: auth, 278 | } 279 | } 280 | 281 | // ServeHTTP is the http handler method. 282 | // It parses the query string and prohibits or allows new connections depending 283 | // on the existence of the "offline" or "online" parameter. 284 | // When the "offline" parameter is present, all existing downstream connections 285 | // are closed immediately. If both are present, the query is treated like 286 | // if there was only "offline". 287 | func (api *streamControlApi) ServeHTTP(writer http.ResponseWriter, request *http.Request) { 288 | // set the content type for all responses 289 | writer.Header().Add("Content-Type", "text/plain") 290 | 291 | // fail-fast: verify that this user can access this resource first 292 | if !auth.HandleHttpAuthentication(api.auth, request, writer) { 293 | return 294 | } 295 | 296 | query := request.URL.Query() 297 | if len(query["offline"]) > 0 { 298 | api.inhibit.SetInhibit(true) 299 | writer.WriteHeader(http.StatusAccepted) 300 | if _, err := writer.Write([]byte("202 accepted")); err != nil { 301 | logger.Logkv( 302 | "event", eventApiError, 303 | "error", errorApiWrite, 304 | "message", err.Error(), 305 | ) 306 | } 307 | } else if len(query["online"]) > 0 { 308 | api.inhibit.SetInhibit(false) 309 | writer.WriteHeader(http.StatusAccepted) 310 | if _, err := writer.Write([]byte("202 accepted")); err != nil { 311 | logger.Logkv( 312 | "event", eventApiError, 313 | "error", errorApiWrite, 314 | "message", err.Error(), 315 | ) 316 | } 317 | } else { 318 | writer.WriteHeader(http.StatusBadRequest) 319 | if _, err := writer.Write([]byte("400 bad request")); err != nil { 320 | logger.Logkv( 321 | "event", eventApiError, 322 | "error", errorApiWrite, 323 | "message", err.Error(), 324 | ) 325 | } 326 | } 327 | } 328 | 329 | // prometheusApi implements a handler for scraping Prometheus metrics. 330 | type prometheusApi struct { 331 | // auth is an authentication verifier for client requests 332 | auth auth.Authenticator 333 | // handler is the delegate HTTP handler 334 | handler http.Handler 335 | } 336 | 337 | // NewPrometheusApi creates a new Prometheus metrics API object, 338 | // serving metrics to a Prometheus instance. 339 | func NewPrometheusApi(auth auth.Authenticator) http.Handler { 340 | return &prometheusApi{ 341 | auth: auth, 342 | handler: metrics.PromHandler(), 343 | } 344 | } 345 | 346 | // ServeHTTP is the http handler method. 347 | func (api *prometheusApi) ServeHTTP(writer http.ResponseWriter, request *http.Request) { 348 | // fail-fast: verify that this user can access this resource first 349 | if !auth.HandleHttpAuthentication(api.auth, request, writer) { 350 | return 351 | } 352 | 353 | // authentication successful, forward the request to the promhttp handler 354 | api.handler.ServeHTTP(writer, request) 355 | } 356 | -------------------------------------------------------------------------------- /api/api.goconvey: -------------------------------------------------------------------------------- 1 | // Timeout to avoid failing tests that block forever 2 | -timeout=12s 3 | -------------------------------------------------------------------------------- /api/api_test.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package api 18 | 19 | import ( 20 | //"encoding/hex" 21 | "bytes" 22 | "encoding/json" 23 | "github.com/onitake/restreamer/auth" 24 | "github.com/onitake/restreamer/configuration" 25 | "github.com/onitake/restreamer/metrics" 26 | "net/http" 27 | "net/url" 28 | "testing" 29 | ) 30 | 31 | type Logger interface { 32 | Log(args ...interface{}) 33 | Logf(format string, args ...interface{}) 34 | } 35 | 36 | type mockWriter struct { 37 | bytes.Buffer 38 | header http.Header 39 | log Logger 40 | } 41 | 42 | func newMockWriter(logger Logger) *mockWriter { 43 | return &mockWriter{ 44 | header: make(http.Header), 45 | log: logger, 46 | } 47 | } 48 | func (writer *mockWriter) Header() http.Header { 49 | return writer.header 50 | } 51 | func (writer *mockWriter) Write(data []byte) (int, error) { 52 | //writer.log.Logf("Write data:\n%s", hex.Dump(data)) 53 | return writer.Buffer.Write(data) 54 | } 55 | func (writer *mockWriter) WriteHeader(status int) { 56 | //writer.log.Logf("Write header, status code %d:", status) 57 | //writer.log.Log(writer.header) 58 | } 59 | 60 | type mockStatistics struct { 61 | Streams map[string]*metrics.StreamStatistics 62 | Global metrics.StreamStatistics 63 | } 64 | 65 | func (*mockStatistics) Start() {} 66 | func (*mockStatistics) Stop() {} 67 | func (*mockStatistics) RegisterStream(name string) metrics.Collector { 68 | return nil 69 | } 70 | func (*mockStatistics) RemoveStream(name string) {} 71 | func (stats *mockStatistics) GetStreamStatistics(name string) *metrics.StreamStatistics { 72 | return stats.Streams[name] 73 | } 74 | func (stats *mockStatistics) GetAllStreamStatistics() map[string]*metrics.StreamStatistics { 75 | return stats.Streams 76 | } 77 | func (stats *mockStatistics) GetGlobalStatistics() *metrics.StreamStatistics { 78 | return &stats.Global 79 | } 80 | 81 | func testStatisticsConnections(t *testing.T, connections, full, max int64, status string) { 82 | stats := &mockStatistics{ 83 | Global: metrics.StreamStatistics{ 84 | Connections: connections, 85 | MaxConnections: max, 86 | FullConnections: full, 87 | }, 88 | } 89 | api := &statisticsApi{ 90 | stats: stats, 91 | auth: auth.NewAuthenticator(configuration.Authentication{}, nil), 92 | } 93 | writer := newMockWriter(t) 94 | testurl, _ := url.Parse("http://localhost/statistics") 95 | api.ServeHTTP(writer, &http.Request{Header: make(http.Header), URL: testurl}) 96 | var decoded map[string]interface{} 97 | err := json.Unmarshal(writer.Bytes(), &decoded) 98 | if err != nil { 99 | t.Fatalf("Error decoding JSON: %s", err.Error()) 100 | } 101 | retstatus, ok := decoded["status"].(string) 102 | if !ok { 103 | t.Fatalf("No status field or incorrect type returned") 104 | } 105 | if retstatus != status { 106 | t.Errorf("Invalid status returned: expected %s, got %s", status, retstatus) 107 | } 108 | } 109 | 110 | func testHealthConnections(t *testing.T, connections, full, max int64, status string) { 111 | stats := &mockStatistics{ 112 | Global: metrics.StreamStatistics{ 113 | Connections: connections, 114 | MaxConnections: max, 115 | FullConnections: full, 116 | }, 117 | } 118 | api := &healthApi{ 119 | stats: stats, 120 | auth: auth.NewAuthenticator(configuration.Authentication{}, nil), 121 | } 122 | writer := newMockWriter(t) 123 | testurl, _ := url.Parse("http://localhost/health") 124 | api.ServeHTTP(writer, &http.Request{Header: make(http.Header), URL: testurl}) 125 | var decoded map[string]interface{} 126 | err := json.Unmarshal(writer.Bytes(), &decoded) 127 | if err != nil { 128 | t.Fatalf("Error decoding JSON: %s", err.Error()) 129 | } 130 | retstatus, ok := decoded["status"].(string) 131 | if !ok { 132 | t.Fatalf("No status field or incorrect type returned") 133 | } 134 | if retstatus != status { 135 | t.Errorf("Invalid status returned: expected %s, got %s", status, retstatus) 136 | } 137 | } 138 | 139 | func TestStatisticsApi(t *testing.T) { 140 | testStatisticsConnections(t, 0, 0, 0, "ok") 141 | testStatisticsConnections(t, 1, 0, 0, "ok") 142 | testStatisticsConnections(t, 1, 1, 2, "full") 143 | testStatisticsConnections(t, 1, 1, 0, "full") 144 | testStatisticsConnections(t, 1, 0, 2, "ok") 145 | testStatisticsConnections(t, 2, 1, 2, "overload") 146 | testStatisticsConnections(t, 2, 1, 0, "full") 147 | testStatisticsConnections(t, 2, 0, 2, "overload") 148 | } 149 | 150 | func TestHealthApi(t *testing.T) { 151 | testHealthConnections(t, 0, 0, 0, "ok") 152 | testHealthConnections(t, 1, 0, 0, "ok") 153 | testHealthConnections(t, 1, 1, 2, "full") 154 | testHealthConnections(t, 1, 1, 0, "full") 155 | testHealthConnections(t, 1, 0, 2, "ok") 156 | testHealthConnections(t, 2, 1, 2, "full") 157 | testHealthConnections(t, 2, 1, 0, "full") 158 | testHealthConnections(t, 2, 0, 2, "full") 159 | } 160 | -------------------------------------------------------------------------------- /api/logger.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package api 18 | 19 | import ( 20 | "github.com/onitake/restreamer/util" 21 | ) 22 | 23 | const ( 24 | moduleApi = "api" 25 | // 26 | eventApiError = "error" 27 | // 28 | errorApiJsonEncode = "json_encode" 29 | errorApiWrite = "write" 30 | ) 31 | 32 | var logger = util.NewGlobalModuleLogger(moduleApi, nil) 33 | -------------------------------------------------------------------------------- /auth/authentication.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package auth 18 | 19 | import ( 20 | "encoding/base64" 21 | "strings" 22 | // "crypto/md5" 23 | "github.com/onitake/restreamer/configuration" 24 | ) 25 | 26 | // Authenticator represents any type that can authenticate users. 27 | type Authenticator interface { 28 | // Authenticate parses an Authorization header and tries to authenticate the request. 29 | // Returns true if the authentication succeeded, false otherwise. 30 | Authenticate(authorization string) bool 31 | // AddUser adds a new user to the list. 32 | // Implementations may interpret users and passwords differently. 33 | AddUser(user, password string) 34 | // RemoveUser removes a user from the list. 35 | // Implementations may interpret users differently. 36 | RemoveUser(user string) 37 | // GetLogin returns an authentication string that can be sent to a remote system. 38 | GetLogin(user string) string 39 | // GetAuthenticateRequest returns a realm or other response that can be sent with a WWW-Authenticate header. 40 | GetAuthenticateRequest() string 41 | } 42 | 43 | // NewAuthenticator creates an authentication service from a credential database and 44 | // an authentication specification. The implementation depends on the algorithm. 45 | // 46 | // If an invalid authentication type is specified, an authenticator that will always 47 | // deny requests is returned. 48 | // If an empty authentication type is specified, an authenticator that will accept 49 | // all requests is returned. 50 | // 51 | // Note: Empty whitelists allow no users at all! 52 | func NewAuthenticator(auth configuration.Authentication, credentials map[string]configuration.UserCredentials) Authenticator { 53 | switch auth.Type { 54 | case "": 55 | return newPassAuthenticator() 56 | case "basic": 57 | return newBasicAuthenticator(auth.Users, credentials, auth.Realm) 58 | case "bearer": 59 | return newTokenAuthenticator(auth.Users, credentials) 60 | default: 61 | return newDenyAuthenticator() 62 | } 63 | } 64 | 65 | type passAuthenticator struct{} 66 | 67 | func newPassAuthenticator() *passAuthenticator { 68 | return &passAuthenticator{} 69 | } 70 | 71 | func (auth *passAuthenticator) Authenticate(authorization string) bool { 72 | return true 73 | } 74 | 75 | func (auth *passAuthenticator) AddUser(user, password string) {} 76 | 77 | func (auth *passAuthenticator) RemoveUser(user string) {} 78 | 79 | func (auth *passAuthenticator) GetLogin(user string) string { 80 | return "" 81 | } 82 | func (auth *passAuthenticator) GetAuthenticateRequest() string { 83 | return "" 84 | } 85 | 86 | type denyAuthenticator struct{} 87 | 88 | func newDenyAuthenticator() *denyAuthenticator { 89 | return &denyAuthenticator{} 90 | } 91 | 92 | func (auth *denyAuthenticator) Authenticate(authorization string) bool { 93 | return false 94 | } 95 | 96 | func (auth *denyAuthenticator) AddUser(user, password string) {} 97 | 98 | func (auth *denyAuthenticator) RemoveUser(user string) {} 99 | 100 | func (auth *denyAuthenticator) GetLogin(user string) string { 101 | return "" 102 | } 103 | func (auth *denyAuthenticator) GetAuthenticateRequest() string { 104 | return "" 105 | } 106 | 107 | type basicAuthenticator struct { 108 | // tokens maps valid authentication strings to yes/no 109 | tokens map[string]bool 110 | // users maps user names to valid authentication strings 111 | users map[string]string 112 | // the authentication realm (unique string sent back with an unauthorized response 113 | realm string 114 | } 115 | 116 | // newBasicAuthenticator creates a new Authenticator that supports basic authentication. 117 | // If the whitelist is empty, no requests are allowed. 118 | func newBasicAuthenticator(allowlist []string, credentials map[string]configuration.UserCredentials, realm string) *basicAuthenticator { 119 | auth := &basicAuthenticator{ 120 | tokens: make(map[string]bool), 121 | users: make(map[string]string), 122 | realm: realm, 123 | } 124 | for _, user := range allowlist { 125 | cred, ok := credentials[user] 126 | if ok { 127 | auth.AddUser(user, cred.Password) 128 | } 129 | } 130 | return auth 131 | } 132 | 133 | func (auth *basicAuthenticator) Authenticate(authorization string) bool { 134 | if strings.HasPrefix(authorization, "Basic") { 135 | // cut off the hash at the end 136 | hash := strings.SplitN(authorization, " ", 2) 137 | if len(hash) >= 2 { 138 | // check if the hash is allowed 139 | return auth.tokens[hash[1]] 140 | } 141 | } 142 | // not basic auth 143 | return false 144 | } 145 | 146 | func (auth *basicAuthenticator) AddUser(user, password string) { 147 | // remove the old token if the user exists already 148 | if oldtoken, ok := auth.users[user]; ok { 149 | delete(auth.tokens, oldtoken) 150 | } 151 | // base64(username + ':' + password) 152 | // we only support UTF-8 153 | token := base64.StdEncoding.EncodeToString([]byte(user + ":" + password)) 154 | auth.tokens[token] = true 155 | auth.users[user] = token 156 | } 157 | 158 | func (auth *basicAuthenticator) RemoveUser(user string) { 159 | token, ok := auth.users[user] 160 | if ok { 161 | delete(auth.users, user) 162 | delete(auth.tokens, token) 163 | } 164 | } 165 | 166 | func (auth *basicAuthenticator) GetLogin(user string) string { 167 | if token, ok := auth.users[user]; ok { 168 | return "Basic " + token 169 | } 170 | return "" 171 | } 172 | 173 | func (auth *basicAuthenticator) GetAuthenticateRequest() string { 174 | return "Basic realm=\"" + auth.realm + "\" charset=\"UTF-8\"" 175 | } 176 | 177 | type tokenAuthenticator struct { 178 | // tokens maps valid authentication tokens to yes/no 179 | tokens map[string]bool 180 | // users maps user names to valid authentication tokens 181 | users map[string]string 182 | } 183 | 184 | // newTokenAuthenticator creates a new Authenticator that supports bearer token authentication. 185 | // The user name is only used as a unique identifier for the token list 186 | func newTokenAuthenticator(whitelist []string, credentials map[string]configuration.UserCredentials) *tokenAuthenticator { 187 | auth := &tokenAuthenticator{ 188 | tokens: make(map[string]bool), 189 | users: make(map[string]string), 190 | } 191 | for _, user := range whitelist { 192 | cred, ok := credentials[user] 193 | if ok { 194 | auth.AddUser(user, cred.Password) 195 | } 196 | } 197 | return auth 198 | } 199 | 200 | func (auth *tokenAuthenticator) Authenticate(authorization string) bool { 201 | if strings.HasPrefix(authorization, "Bearer") { 202 | // cut off the hash at the end 203 | hash := strings.SplitN(authorization, " ", 2) 204 | if len(hash) >= 2 { 205 | // check if the hash is allowed 206 | return auth.tokens[hash[1]] 207 | } 208 | } 209 | // not basic auth 210 | return false 211 | } 212 | 213 | func (auth *tokenAuthenticator) AddUser(user, password string) { 214 | // remove the old token if the user exists already 215 | if oldtoken, ok := auth.users[user]; ok { 216 | delete(auth.tokens, oldtoken) 217 | } 218 | // base64(password) 219 | // we expect that token is already base64 formatted - do nothing here 220 | auth.tokens[password] = true 221 | auth.users[user] = password 222 | } 223 | 224 | func (auth *tokenAuthenticator) RemoveUser(user string) { 225 | token, ok := auth.users[user] 226 | if ok { 227 | delete(auth.users, user) 228 | delete(auth.tokens, token) 229 | } 230 | } 231 | 232 | func (auth *tokenAuthenticator) GetLogin(user string) string { 233 | if token, ok := auth.users[user]; ok { 234 | return "Bearer " + token 235 | } 236 | return "" 237 | } 238 | 239 | func (auth *tokenAuthenticator) GetAuthenticateRequest() string { 240 | // token-based auth doesn't support challenge-response or similar, as token are generated externally 241 | // just send back a 403. 242 | return "" 243 | } 244 | 245 | // UserAuthenticator is an authenticator that is bound to a single user. 246 | // It does not implement the Authenticator interface because it doesn't support the user argument. 247 | type UserAuthenticator struct { 248 | Auth Authenticator 249 | User string 250 | } 251 | 252 | // NewUserAuthenticator creates a new user authenticator from an Authentication configuration and 253 | // and an authenticator. 254 | // If the Authentication does not contain any users, nil is returned. If it contains more than 255 | // one user, the first one is used. 256 | func NewUserAuthenticator(cred configuration.Authentication, auth Authenticator) *UserAuthenticator { 257 | if len(cred.Users) < 1 { 258 | return nil 259 | } 260 | return &UserAuthenticator{ 261 | Auth: auth, 262 | User: cred.Users[0], 263 | } 264 | } 265 | 266 | func (auth *UserAuthenticator) GetLogin() string { 267 | return auth.Auth.GetLogin(auth.User) 268 | } 269 | -------------------------------------------------------------------------------- /auth/authentication_test.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package auth 18 | 19 | import ( 20 | "encoding/base64" 21 | "github.com/onitake/restreamer/configuration" 22 | "math/rand" 23 | "testing" 24 | ) 25 | 26 | const alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-+.:,;$!#@%&/()=?'[]{}_<>" 27 | 28 | // https://stackoverflow.com/a/31832326 29 | func randStringBytes(n int) string { 30 | b := make([]byte, n) 31 | for i := range b { 32 | b[i] = alphabet[rand.Intn(len(alphabet))] 33 | } 34 | return string(b) 35 | } 36 | 37 | func TestPassAuthenticator01(t *testing.T) { 38 | auth := newPassAuthenticator() 39 | if !auth.Authenticate(randStringBytes(16)) { 40 | t.Errorf("Cannot pass random auth string") 41 | } 42 | } 43 | 44 | func TestDenyAuthenticator01(t *testing.T) { 45 | auth := newDenyAuthenticator() 46 | auth.AddUser("user", "password") 47 | str := base64.StdEncoding.EncodeToString([]byte("user:password")) 48 | if auth.Authenticate("Basic " + str) { 49 | t.Errorf("Deny authenticator incorrectly allowed basic auth") 50 | } 51 | } 52 | 53 | func TestBasicAuthenticator01(t *testing.T) { 54 | user := "user" 55 | password := randStringBytes(16) 56 | realm := "Test Realm" 57 | whitelist := []string{ 58 | user, 59 | } 60 | cred := map[string]configuration.UserCredentials{ 61 | user: { 62 | Password: password, 63 | }, 64 | } 65 | auth := newBasicAuthenticator(whitelist, cred, realm) 66 | str := base64.StdEncoding.EncodeToString([]byte(user + ":" + password)) 67 | if !auth.Authenticate("Basic " + str) { 68 | t.Errorf("Basic authenticator didn't allow valid user") 69 | } 70 | } 71 | 72 | func TestBasicAuthenticator02(t *testing.T) { 73 | user := "user" 74 | password := randStringBytes(16) 75 | realm := "Test Realm" 76 | var allowlist []string 77 | cred := map[string]configuration.UserCredentials{ 78 | user: { 79 | Password: password, 80 | }, 81 | } 82 | auth := newBasicAuthenticator(allowlist, cred, realm) 83 | str := base64.StdEncoding.EncodeToString([]byte(user + ":" + password)) 84 | if auth.Authenticate("Basic " + str) { 85 | t.Errorf("Basic authenticator allowed non-whitelisted user") 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /auth/httpauth.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package auth 18 | 19 | import ( 20 | "net/http" 21 | ) 22 | 23 | // HandleHttpAuthentication handles authentication headers and responses. 24 | // If it returns false, authenticaten has failed, an appropriate response was sent and the caller should immediately return. 25 | // A true return value indicates that authentication has succeeded and the caller should proceed with handling the request. 26 | func HandleHttpAuthentication(auth Authenticator, request *http.Request, writer http.ResponseWriter) bool { 27 | // fail-fast: verify that this user can access this resource first 28 | if !auth.Authenticate(request.Header.Get("Authorization")) { 29 | realm := auth.GetAuthenticateRequest() 30 | if len(realm) > 0 { 31 | if logger != nil { 32 | logger.Logkv( 33 | "event", eventProtocolAuthenticating, 34 | "statuscode", 401, 35 | "message", "Requesting user authentication", 36 | "url", request.URL.Path, 37 | "client", request.RemoteAddr, 38 | ) 39 | } 40 | // if the authenticator supports responses to invalid authentication headers, send 41 | writer.Header().Add("WWW-Authenticate", realm) 42 | writer.WriteHeader(http.StatusUnauthorized) 43 | } else { 44 | if logger != nil { 45 | logger.Logkv( 46 | "event", eventProtocolError, 47 | "error", errorProtocolForbidden, 48 | "statuscode", 403, 49 | "message", "Denying user access", 50 | "url", request.URL.Path, 51 | "client", request.RemoteAddr, 52 | ) 53 | } 54 | // otherwise, just respond with a 403 55 | writer.WriteHeader(http.StatusForbidden) 56 | } 57 | return false 58 | } 59 | if logger != nil { 60 | logger.Logkv( 61 | "event", eventProtocolAuthenticated, 62 | "message", "Request authenticated", 63 | "url", request.URL.Path, 64 | "client", request.RemoteAddr, 65 | ) 66 | } 67 | return true 68 | } 69 | -------------------------------------------------------------------------------- /auth/logger.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package auth 18 | 19 | import ( 20 | "github.com/onitake/restreamer/util" 21 | ) 22 | 23 | const ( 24 | moduleAuth = "auth" 25 | // 26 | eventProtocolError = "error" 27 | eventProtocolAuthenticating = "authenticating" 28 | eventProtocolAuthenticated = "authenticated" 29 | // 30 | errorProtocolForbidden = "forbidden" 31 | ) 32 | 33 | var logger = util.NewGlobalModuleLogger(moduleAuth, nil) 34 | -------------------------------------------------------------------------------- /cmd/restreamer/logger.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "github.com/onitake/restreamer/util" 21 | ) 22 | 23 | const ( 24 | moduleMain = "main" 25 | // 26 | eventMainError = "error" 27 | eventMainConfig = "config" 28 | eventMainConfigStream = "stream" 29 | eventMainConfigStatic = "static" 30 | eventMainConfigApi = "api" 31 | eventMainHandled = "handled" 32 | eventMainStartMonitor = "start_monitor" 33 | eventMainStartServer = "start_server" 34 | // 35 | errorMainStreamNotFound = "stream_notfound" 36 | errorMainInvalidApi = "invalid_api" 37 | errorMainInvalidResource = "invalid_resource" 38 | errorMainInvalidNotification = "invalid_notification" 39 | errorMainMissingNotificationUser = "missing_notification_user" 40 | errorMainMissingStreamUser = "missing_stream_user" 41 | errorMainInvalidAuthentication = "invalid_authentication" 42 | errorMainPreambleRead = "preamble_read" 43 | ) 44 | 45 | var logger = util.NewGlobalModuleLogger(moduleMain, nil) 46 | -------------------------------------------------------------------------------- /cmd/restreamer/profile.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2016-2017 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package main 18 | 19 | import _ "net/http/pprof" 20 | import ( 21 | "log" 22 | "net/http" 23 | "runtime" 24 | "runtime/debug" 25 | ) 26 | 27 | func EnableProfiling() { 28 | /*profile, err := os.Create("server.prof") 29 | if err != nil { 30 | log.Fatal(err) 31 | } 32 | pprof.WriteHeapProfile(profile)*/ 33 | // Enable block profiling (granularity: 100 ms) 34 | runtime.SetBlockProfileRate(100000000) 35 | // Register URL to force reclaiming memory 36 | http.HandleFunc("/reclaim", func(http.ResponseWriter, *http.Request) { 37 | log.Printf("Reclaiming memory") 38 | debug.FreeOSMemory() 39 | }) 40 | go func() { 41 | // Start profiling web server 42 | log.Println(http.ListenAndServe(":6060", nil)) 43 | }() 44 | } 45 | -------------------------------------------------------------------------------- /cmd/restreamer/restreamer.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2016-2019 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "github.com/onitake/restreamer/api" 23 | "github.com/onitake/restreamer/auth" 24 | "github.com/onitake/restreamer/configuration" 25 | "github.com/onitake/restreamer/event" 26 | "github.com/onitake/restreamer/metrics" 27 | "github.com/onitake/restreamer/streaming" 28 | "github.com/onitake/restreamer/util" 29 | "io" 30 | "log" 31 | "math/rand" 32 | "net/http" 33 | "os" 34 | "time" 35 | ) 36 | 37 | func main() { 38 | logbackend := &util.ModuleLogger{ 39 | Logger: &util.ConsoleLogger{}, 40 | AddTimestamp: true, 41 | } 42 | util.SetGlobalStandardLogger(logbackend) 43 | 44 | rnd := rand.New(rand.NewSource(time.Now().Unix())) 45 | 46 | var configname string 47 | if len(os.Args) > 1 { 48 | configname = os.Args[1] 49 | } else { 50 | configname = "restreamer.json" 51 | } 52 | 53 | config, err := configuration.LoadConfigurationFile(configname) 54 | if err != nil { 55 | log.Fatal("Error parsing configuration: ", err) 56 | } 57 | 58 | logger.Logkv( 59 | "event", eventMainConfig, 60 | "listen", config.Listen, 61 | "timeout", config.Timeout, 62 | ) 63 | 64 | if config.Profile { 65 | EnableProfiling() 66 | // If profiling is enabled, we also want the Go runtime metrics collector 67 | metrics.EnableGoRuntimeCollector() 68 | } 69 | 70 | if config.Log != "" { 71 | flogger, err := util.NewFileLogger(config.Log, true) 72 | if err != nil { 73 | log.Fatal("Error opening log: ", err) 74 | } 75 | logbackend.Logger = flogger 76 | } 77 | 78 | clients := make(map[string]*streaming.Client) 79 | 80 | var stats metrics.Statistics 81 | if config.NoStats { 82 | stats = &metrics.DummyStatistics{} 83 | } else { 84 | stats = metrics.NewStatistics(config.MaxConnections, config.FullConnections) 85 | } 86 | 87 | controller := streaming.NewAccessController(config.MaxConnections) 88 | 89 | enableheartbeat := false 90 | 91 | queue := event.NewQueue(int(config.FullConnections)) 92 | for _, note := range config.Notifications { 93 | var err error 94 | var typ event.Type 95 | switch note.Event { 96 | case "limit_hit": 97 | typ = event.TypeLimitHit 98 | case "limit_miss": 99 | typ = event.TypeLimitMiss 100 | case "heartbeat": 101 | typ = event.TypeHeartbeat 102 | default: 103 | err = errors.New(fmt.Sprintf("Unknown event type: %s", note.Event)) 104 | } 105 | var handler event.Handler 106 | switch note.Type { 107 | case "url": 108 | authenticator := auth.NewUserAuthenticator(note.Authentication, auth.NewAuthenticator(note.Authentication, config.UserList)) 109 | if authenticator == nil { 110 | logger.Logkv( 111 | "event", eventMainError, 112 | "error", errorMainInvalidAuthentication, 113 | "message", fmt.Sprintf("Invalid authentication configuration, possibly a missing user"), 114 | ) 115 | } 116 | urlhandler, err := event.NewUrlHandler(note.Url, authenticator) 117 | if err == nil { 118 | handler = urlhandler 119 | } 120 | default: 121 | err = errors.New(fmt.Sprintf("Unknown handler type: %s", note.Type)) 122 | } 123 | if err == nil { 124 | queue.RegisterEventHandler(typ, handler) 125 | if typ == event.TypeHeartbeat { 126 | enableheartbeat = true 127 | logger.Logkv( 128 | "event", "enable_heartbeat", 129 | "message", fmt.Sprintf("Enabling heartbeat API"), 130 | ) 131 | } 132 | } else { 133 | logger.Logkv( 134 | "event", eventMainError, 135 | "error", errorMainInvalidNotification, 136 | "message", fmt.Sprintf("Cannot configure notification: %v", err), 137 | ) 138 | } 139 | } 140 | queue.Start() 141 | 142 | if enableheartbeat { 143 | event.NewHeartbeat(time.Duration(config.HeartbeatInterval)*time.Second, queue) 144 | } 145 | 146 | i := 0 147 | mux := http.NewServeMux() 148 | for _, streamdef := range config.Resources { 149 | switch streamdef.Type { 150 | case "stream": 151 | logger.Logkv( 152 | "event", eventMainConfigStream, 153 | "serve", streamdef.Serve, 154 | "remote", streamdef.Remotes, 155 | "message", fmt.Sprintf("Connecting stream %s to %v", streamdef.Serve, streamdef.Remotes), 156 | ) 157 | 158 | reg := stats.RegisterStream(streamdef.Serve) 159 | 160 | authenticator := auth.NewAuthenticator(streamdef.Authentication, config.UserList) 161 | 162 | streamer := streaming.NewStreamer(streamdef.Serve, config.OutputBuffer, controller, authenticator) 163 | streamer.SetCollector(reg) 164 | streamer.SetNotifier(queue) 165 | 166 | if streamdef.Preamble != "" { 167 | prein, err := os.Open(streamdef.Preamble) 168 | if err != nil { 169 | logger.Logkv( 170 | "event", eventMainError, 171 | "error", errorMainPreambleRead, 172 | "message", fmt.Sprintf("Cannot open preamble file: %v", err), 173 | ) 174 | } 175 | preamble, err := io.ReadAll(prein) 176 | if err != nil { 177 | logger.Logkv( 178 | "event", eventMainError, 179 | "error", errorMainPreambleRead, 180 | "message", fmt.Sprintf("Cannot read preamble file: %v", err), 181 | ) 182 | } 183 | streamer.SetPreamble(preamble) 184 | } 185 | 186 | // shuffle the list here, not later 187 | // should give a bit more randomness 188 | remotes := util.ShuffleStrings(rnd, streamdef.Remotes) 189 | 190 | client, err := streaming.NewClient(streamdef.Serve, remotes, streamer, config.Timeout, config.Reconnect, config.ReadTimeout, config.InputBuffer, streamdef.ClientInterface, config.InputBuffer, streamdef.Mru) 191 | if err == nil { 192 | client.SetCollector(reg) 193 | client.Connect() 194 | clients[streamdef.Serve] = client 195 | mux.Handle(streamdef.Serve, streamer) 196 | 197 | logger.Logkv( 198 | "event", eventMainHandled, 199 | "number", i, 200 | "message", fmt.Sprintf("Handled connection %d", i), 201 | ) 202 | i++ 203 | } else { 204 | log.Print(err) 205 | } 206 | 207 | case "static": 208 | logger.Logkv( 209 | "event", eventMainConfigStatic, 210 | "serve", streamdef.Serve, 211 | "remote", streamdef.Remote, 212 | "message", fmt.Sprintf("Configuring static resource %s on %s", streamdef.Serve, streamdef.Remote), 213 | ) 214 | authenticator := auth.NewAuthenticator(streamdef.Authentication, config.UserList) 215 | proxy, err := streaming.NewProxy(streamdef.Remote, config.Timeout, streamdef.Cache, authenticator) 216 | if err != nil { 217 | log.Print(err) 218 | } else { 219 | proxy.SetStatistics(stats) 220 | proxy.Start() 221 | mux.Handle(streamdef.Serve, proxy) 222 | } 223 | 224 | case "api": 225 | authenticator := auth.NewAuthenticator(streamdef.Authentication, config.UserList) 226 | 227 | switch streamdef.Api { 228 | case "health": 229 | logger.Logkv( 230 | "event", eventMainConfigApi, 231 | "api", "health", 232 | "serve", streamdef.Serve, 233 | "message", fmt.Sprintf("Registering global health API on %s", streamdef.Serve), 234 | ) 235 | mux.Handle(streamdef.Serve, api.NewHealthApi(stats, authenticator)) 236 | case "statistics": 237 | logger.Logkv( 238 | "event", eventMainConfigApi, 239 | "api", "statistics", 240 | "serve", streamdef.Serve, 241 | "message", fmt.Sprintf("Registering global statistics API on %s", streamdef.Serve), 242 | ) 243 | mux.Handle(streamdef.Serve, api.NewStatisticsApi(stats, authenticator)) 244 | case "check": 245 | logger.Logkv( 246 | "event", eventMainConfigApi, 247 | "api", "check", 248 | "serve", streamdef.Serve, 249 | "message", fmt.Sprintf("Registering stream check API on %s", streamdef.Serve), 250 | ) 251 | client := clients[streamdef.Remote] 252 | if client != nil { 253 | mux.Handle(streamdef.Serve, api.NewStreamStateApi(client, authenticator)) 254 | } else { 255 | logger.Logkv( 256 | "event", eventMainError, 257 | "error", errorMainStreamNotFound, 258 | "api", "check", 259 | "remote", streamdef.Remote, 260 | "message", fmt.Sprintf("Error, stream not found: %s", streamdef.Remote), 261 | ) 262 | } 263 | case "control": 264 | logger.Logkv( 265 | "event", eventMainConfigApi, 266 | "api", "control", 267 | "serve", streamdef.Serve, 268 | "message", fmt.Sprintf("Registering stream control API on %s", streamdef.Serve), 269 | ) 270 | client := clients[streamdef.Remote] 271 | if client != nil { 272 | mux.Handle(streamdef.Serve, api.NewStreamControlApi(client, authenticator)) 273 | } else { 274 | logger.Logkv( 275 | "event", eventMainError, 276 | "error", errorMainStreamNotFound, 277 | "api", "control", 278 | "remote", streamdef.Remote, 279 | "message", fmt.Sprintf("Error, stream not found: %s", streamdef.Remote), 280 | ) 281 | } 282 | case "prometheus": 283 | logger.Logkv( 284 | "event", eventMainConfigApi, 285 | "api", "prometheus", 286 | "serve", streamdef.Serve, 287 | "message", fmt.Sprintf("Registering Prometheus API on %s", streamdef.Serve), 288 | ) 289 | mux.Handle(streamdef.Serve, api.NewPrometheusApi(authenticator)) 290 | default: 291 | logger.Logkv( 292 | "event", eventMainError, 293 | "error", errorMainInvalidApi, 294 | "api", streamdef.Api, 295 | "message", fmt.Sprintf("Invalid API type: %s", streamdef.Api), 296 | ) 297 | } 298 | 299 | default: 300 | logger.Logkv( 301 | "event", eventMainError, 302 | "error", errorMainInvalidResource, 303 | "type", streamdef.Type, 304 | "message", fmt.Sprintf("Invalid resource type: %s", streamdef.Type), 305 | ) 306 | } 307 | } 308 | 309 | if i == 0 { 310 | log.Fatal("No streams available") 311 | } else { 312 | logger.Logkv( 313 | "event", eventMainStartMonitor, 314 | "message", "Starting stats monitor", 315 | ) 316 | stats.Start() 317 | logger.Logkv( 318 | "event", eventMainStartServer, 319 | "message", "Starting server", 320 | ) 321 | log.Fatal(http.ListenAndServe(config.Listen, mux)) 322 | } 323 | } 324 | -------------------------------------------------------------------------------- /configuration/config.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2016-2019 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package configuration 18 | 19 | import ( 20 | "bytes" 21 | "encoding/json" 22 | "io" 23 | "os" 24 | ) 25 | 26 | // Authentication configures authentication for a resource. 27 | // The exact semantics depend on the resource. 28 | type Authentication struct { 29 | // Type specifies the authentication type. 30 | // Only the empty string, 'basic' and 'bearer' are currently supported. 31 | // The interpretation of the type is as follows: 32 | // '': Disable authentication and allow all requests to succeed. 33 | // 'basic': compare the string after the 'Authorization: Basic' header with 34 | // base64(md5sum(username + ':' + passwords[username])) and allow the request if they match. 35 | // 'bearer': compare the string after 'Authentication: Bearer' with 36 | // base64(passwords[username]) and allow the request if they match. 37 | Type string `json:"type"` 38 | // Realm specifies the authentication realm that is sent 39 | // back to the client if the authentication header was missing. 40 | Realm string `json:"realm"` 41 | // User specifies a valid user who can access the resource. 42 | // This is merged with Users. 43 | User string `json:"user"` 44 | // Users specifies the list of valid user names. 45 | // User is merged into this list. 46 | Users []string `json:"users"` 47 | } 48 | 49 | // Resource is a single HTTP endpoint. 50 | type Resource struct { 51 | // Type is the resource type. 52 | Type string `json:"type"` 53 | // Api is the API type. 54 | Api string `json:"api"` 55 | // Serve is the local URL to serve this stream under. 56 | Serve string `json:"serve"` 57 | // Remote is a single upstream URL or API argument; 58 | // it will be added to Remotes during parsing. 59 | Remote string `json:"remote"` 60 | // Remotes is the upstream URLs. 61 | Remotes []string `json:"remotes"` 62 | // ClientInterface denotes a specific network interface for the remote connection. 63 | // This is currently only supported for multicast UDP. 64 | // All interfaces will be used if this is not set. 65 | ClientInterface string `json:"clientinterface"` 66 | // Cache the cache time in seconds. 67 | Cache uint `json:"cache"` 68 | // Authentication specifies credentials required to access this resource. 69 | // If the authentication type is unset, no authentication is required. 70 | Authentication Authentication `json:"authentication"` 71 | // Mru (maximum receive unit) is the size of the datagram receive buffer. 72 | // Only used for UDP and RTP protocols. 73 | Mru uint `json:"mru"` 74 | // Preamble specifies the name of a file containing a static preamble, that is sent to each client before 75 | // actual data is streamed. It can be used to synchronize the decoder quickly, instead of needing to wait for 76 | // the next PAT, PMT, SPS and PPS packets. 77 | // Make sure that the format of the preamble content matches the stream, or you will end up with badly 78 | // configured decoder! 79 | Preamble string `json:"preamble"` 80 | } 81 | 82 | // UserCredentials is a set of credentials for a single user 83 | type UserCredentials struct { 84 | // Password is the key or password of this user. 85 | Password string `json:"password"` 86 | } 87 | 88 | // Notification is a single notification definition. 89 | type Notification struct { 90 | // Event is the event to watch for. 91 | Event string `json:"event"` 92 | // Type is the kind of callback to send. 93 | Type string `json:"type"` 94 | // Url is the remote to access (if Type is http). 95 | Url string `json:"url"` 96 | // Authentication specifies credentials to be sent with the notification. 97 | // If the authentication type is unset, no authentication is sent. 98 | // Only the first user from the list (or the single 'User') is used, all others are ignored. 99 | Authentication Authentication `json:"authentication"` 100 | } 101 | 102 | // Configuration is a representation of the configurable settings. 103 | // These are normally read from a JSON file and deserialized by 104 | // the builtin marshaler. 105 | type Configuration struct { 106 | // Listen is the interface to listen on. 107 | Listen string `json:"listen"` 108 | // Timeout is the connection timeout 109 | // (both input and output). 110 | Timeout uint `json:"timeout"` 111 | // Reconnect is the reconnect delay. 112 | Reconnect uint `json:"reconnect"` 113 | // ReadTimeout is the upstream read timeout. 114 | ReadTimeout uint `json:"readtimeout"` 115 | // InputBuffer is the maximum number of packets on the input buffer of each stream. 116 | // It also determines the socket buffer size for datagram-oriented connections. 117 | InputBuffer uint `json:"inputbuffer"` 118 | // OutputBuffer is the size of the output buffer per connection. 119 | // Note that each connection will eat at least OutputBuffer * 192 bytes 120 | // when the queue is full, so you should adjust the value according 121 | // to the amount of RAM available. 122 | OutputBuffer uint `json:"outputbuffer"` 123 | // MaxConnections is the maximum total number of concurrent connections. 124 | // If it is 0, no hard limit will be imposed. 125 | MaxConnections uint `json:"maxconnections"` 126 | // FullConnections is the soft limit on the total number of concurrent connections. 127 | // If it is 0, no soft limit will be imposed/reported. 128 | FullConnections uint `json:"fullconnections"` 129 | // NoStats disables statistics collection, if set. 130 | NoStats bool `json:"nostats"` 131 | // HeartbeatInterval defines the number of seconds between heartbeat notifications. 132 | // This setting has not effect if no notifications were defined. 133 | HeartbeatInterval uint `json:"heartbeatinterval"` 134 | // Log is the access log file name. 135 | Log string `json:"log"` 136 | // Profile determines if profiling should be enabled. 137 | // Set to true to turn on the pprof web server. 138 | Profile bool `json:"profile"` 139 | // UserList is the built-in list of user accounts, to be used with authentication stanzas. 140 | // It maps user names to authentication credentials. 141 | UserList map[string]UserCredentials `json:"userlist"` 142 | // Resources is the list of streams. 143 | Resources []Resource `json:"resources"` 144 | // Notifications defines event callbacks. 145 | Notifications []Notification `json:"notifications"` 146 | } 147 | 148 | // DefaultConfiguration creates and returns a configuration object 149 | // with default values. 150 | func DefaultConfiguration() *Configuration { 151 | return &Configuration{ 152 | Listen: "localhost:http", 153 | Timeout: 0, 154 | Reconnect: 10, 155 | InputBuffer: 1000, 156 | OutputBuffer: 400, 157 | NoStats: false, 158 | HeartbeatInterval: 60, 159 | } 160 | } 161 | 162 | // LoadConfigurationFile loads a configuration in JSON format from "filename". 163 | func LoadConfigurationFile(filename string) (*Configuration, error) { 164 | fd, err := os.Open(filename) 165 | if err == nil { 166 | //goland:noinspection GoUnhandledErrorResult 167 | defer fd.Close() 168 | return LoadConfiguration(fd) 169 | } else { 170 | return nil, err 171 | } 172 | } 173 | 174 | // LoadConfiguration reads JSON data from the Reader argument and returns a parsed configuration from it. 175 | func LoadConfiguration(reader io.Reader) (*Configuration, error) { 176 | config := DefaultConfiguration() 177 | 178 | decoder := json.NewDecoder(reader) 179 | err := decoder.Decode(&config) 180 | if err != nil { 181 | return nil, err 182 | } 183 | 184 | for i := range config.Resources { 185 | resource := &config.Resources[i] 186 | // add remote to remotes list, if given - but only if this is a stream 187 | if resource.Type == "stream" && len(resource.Remote) > 0 { 188 | length := len(resource.Remotes) 189 | remotes := make([]string, length+1) 190 | remotes[0] = resource.Remote 191 | copy(remotes[1:], resource.Remotes) 192 | resource.Remotes = remotes 193 | // reset 194 | resource.Remote = "" 195 | } 196 | // add user to users list, if given 197 | if len(resource.Authentication.Type) > 0 && len(resource.Authentication.User) > 0 { 198 | length := len(resource.Authentication.Users) 199 | users := make([]string, length+1) 200 | users[0] = resource.Authentication.User 201 | copy(users[1:], resource.Authentication.Users) 202 | resource.Authentication.Users = users 203 | // reset 204 | resource.Authentication.User = "" 205 | } 206 | // fix up UDP buffer size 207 | if resource.Mru == 0 { 208 | resource.Mru = 1500 209 | } 210 | } 211 | for i := range config.Notifications { 212 | notification := &config.Notifications[i] 213 | // add user to users list, if given 214 | if len(notification.Authentication.Type) > 0 && len(notification.Authentication.User) > 0 { 215 | length := len(notification.Authentication.Users) 216 | users := make([]string, length+1) 217 | users[0] = notification.Authentication.User 218 | copy(users[1:], notification.Authentication.Users) 219 | notification.Authentication.Users = users 220 | // reset 221 | notification.Authentication.User = "" 222 | } 223 | } 224 | 225 | return config, err 226 | } 227 | 228 | // LoadConfigurationBytes parses the byte array argument as JSON and initialises a configuration from it. 229 | func LoadConfigurationBytes(json []byte) (*Configuration, error) { 230 | return LoadConfiguration(bytes.NewReader(json)) 231 | } 232 | -------------------------------------------------------------------------------- /configuration/config_test.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018-2019 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package configuration 18 | 19 | import ( 20 | // "encoding/json" 21 | "reflect" 22 | "testing" 23 | ) 24 | 25 | func TestArrayRange(t *testing.T) { 26 | vals := []struct{ X int }{ 27 | {0}, 28 | } 29 | for i := range vals { 30 | val := &vals[i] 31 | val.X = 1 32 | } 33 | if vals[0].X == 0 { 34 | t.Errorf("Invalid value") 35 | } 36 | } 37 | 38 | func TestConfig01(t *testing.T) { 39 | t01 := &Configuration{ 40 | Listen: "localhost:http", 41 | Timeout: 0, 42 | Reconnect: 10, 43 | InputBuffer: 1000, 44 | OutputBuffer: 400, 45 | NoStats: false, 46 | HeartbeatInterval: 60, 47 | } 48 | r01 := DefaultConfiguration() 49 | if !reflect.DeepEqual(t01, r01) { 50 | t.Errorf("Default configuration does not match test case") 51 | } 52 | } 53 | 54 | func TestConfig02(t *testing.T) { 55 | t02 := &Configuration{ 56 | Listen: "testhost:9999", 57 | } 58 | c02 := `{ 59 | "listen": "testhost:9999" 60 | }` 61 | r02, e02 := LoadConfigurationBytes([]byte(c02)) 62 | if e02 != nil || t02.Listen != r02.Listen { 63 | t.Errorf("Variable loaded from JSON does not match expected result") 64 | } 65 | } 66 | 67 | func TestConfig03(t *testing.T) { 68 | t03 := DefaultConfiguration() 69 | t03.Listen = "testhost:9999" 70 | c03 := `{ 71 | "listen": "testhost:9999" 72 | }` 73 | r03, e03 := LoadConfigurationBytes([]byte(c03)) 74 | if e03 != nil || !reflect.DeepEqual(t03, r03) { 75 | t.Logf("t03: %v", t03) 76 | t.Logf("r03: %v", r03) 77 | t.Errorf("Loaded JSON configuration does not match default configuration plus variable") 78 | } 79 | } 80 | 81 | func TestConfig04(t *testing.T) { 82 | t04 := DefaultConfiguration() 83 | t04.Resources = []Resource{ 84 | { 85 | Type: "stream", 86 | Remotes: []string{ 87 | "t04", 88 | }, 89 | Mru: 1500, 90 | }, 91 | } 92 | c04 := `{ 93 | "resources": [ 94 | { 95 | "type": "stream", 96 | "remote": "t04" 97 | } 98 | ] 99 | }` 100 | r04, e04 := LoadConfigurationBytes([]byte(c04)) 101 | if e04 != nil || !reflect.DeepEqual(t04, r04) { 102 | t.Logf("t04: %v", t04) 103 | t.Logf("r04: %v", r04) 104 | t.Logf("e04: %v", e04) 105 | t.Errorf("Single remote for stream not parsed correctly") 106 | } 107 | 108 | t04b := DefaultConfiguration() 109 | t04b.Resources = []Resource{ 110 | { 111 | Type: "api", 112 | Remote: "t04b", 113 | Mru: 1500, 114 | }, 115 | } 116 | c04b := `{ 117 | "resources": [ 118 | { 119 | "type": "api", 120 | "remote": "t04b" 121 | } 122 | ] 123 | }` 124 | r04b, e04b := LoadConfigurationBytes([]byte(c04b)) 125 | if e04b != nil || !reflect.DeepEqual(t04b, r04b) { 126 | t.Logf("t04b: %v", t04b) 127 | t.Logf("r04b: %v", r04b) 128 | t.Logf("e04b: %v", e04b) 129 | t.Errorf("Single remote for API not parsed correctly") 130 | } 131 | } 132 | 133 | func TestConfig05(t *testing.T) { 134 | t05 := DefaultConfiguration() 135 | t05.Resources = []Resource{ 136 | { 137 | Authentication: Authentication{ 138 | Type: "basic", 139 | Users: []string{ 140 | "t05", 141 | }, 142 | }, 143 | Mru: 1500, 144 | }, 145 | } 146 | c05 := `{ 147 | "resources": [ 148 | { 149 | "authentication": { 150 | "type": "basic", 151 | "user": "t05" 152 | } 153 | } 154 | ] 155 | }` 156 | r05, e05 := LoadConfigurationBytes([]byte(c05)) 157 | if e05 != nil || !reflect.DeepEqual(t05, r05) { 158 | t.Logf("t05: %v", t05) 159 | t.Logf("r05: %v", r05) 160 | t.Logf("e05: %v", e05) 161 | t.Errorf("User list not parsed correctly") 162 | } 163 | } 164 | 165 | func TestConfig06(t *testing.T) { 166 | t06 := DefaultConfiguration() 167 | t06.Notifications = []Notification{ 168 | { 169 | Authentication: Authentication{ 170 | Type: "basic", 171 | Users: []string{ 172 | "t06", 173 | }, 174 | }, 175 | }, 176 | } 177 | c06 := `{ 178 | "notifications": [ 179 | { 180 | "authentication": { 181 | "type": "basic", 182 | "user": "t06" 183 | } 184 | } 185 | ] 186 | }` 187 | r06, e06 := LoadConfigurationBytes([]byte(c06)) 188 | if e06 != nil || !reflect.DeepEqual(t06, r06) { 189 | t.Logf("t06: %v", t06) 190 | t.Logf("r06: %v", r06) 191 | t.Logf("e06: %v", e06) 192 | t.Errorf("Notification user not parsed correctly") 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /doc/architecture.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onitake/restreamer/8e23424e8e8e2d8000dcd2c4df6b84ca12fce721/doc/architecture.odg -------------------------------------------------------------------------------- /doc/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/onitake/restreamer/8e23424e8e8e2d8000dcd2c4df6b84ca12fce721/doc/logo.png -------------------------------------------------------------------------------- /event/event.goconvey: -------------------------------------------------------------------------------- 1 | // Timeout to avoid failing tests that block forever 2 | -timeout=3s 3 | -------------------------------------------------------------------------------- /event/handlers.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018-2019 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package event 18 | 19 | type Type int 20 | 21 | const ( 22 | TypeLimitHit Type = iota 23 | TypeLimitMiss 24 | TypeHeartbeat 25 | ) 26 | 27 | type Handler interface { 28 | HandleEvent(Type, ...interface{}) 29 | } 30 | -------------------------------------------------------------------------------- /event/heartbeat.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2019 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package event 18 | 19 | import "time" 20 | 21 | type HeartbeatStopper interface { 22 | Stop() 23 | } 24 | 25 | type Heartbeat struct { 26 | // ticker fires a heartbeat at regular intervals. 27 | ticker *time.Ticker 28 | // target is the notification target. 29 | // NotifyHeartbeat will be called on each tick. 30 | target Notifiable 31 | } 32 | 33 | // NewHeartbeat creates a new heartbeat ticker. 34 | // 35 | // On each heartbeat, target.NotifyHeartbeat will be called with the current timestamp. 36 | // Note that this happens asynchronously from a separate goroutine. 37 | func NewHeartbeat(interval time.Duration, target Notifiable) *Heartbeat { 38 | heartbeat := &Heartbeat{ 39 | ticker: time.NewTicker(interval), 40 | target: target, 41 | } 42 | go heartbeat.loop() 43 | return heartbeat 44 | } 45 | 46 | // loop is the ticker run loop 47 | func (heartbeat *Heartbeat) loop() { 48 | logger.Logkv( 49 | "event", queueEventHeartbeatStart, 50 | "message", "Starting heartbeat goroutine", 51 | ) 52 | // process events (also drains when the channel is closed) 53 | for range heartbeat.ticker.C { 54 | logger.Logkv( 55 | "event", queueEventHeartbeatFire, 56 | "message", "Firing heartbeat", 57 | ) 58 | heartbeat.target.NotifyHeartbeat(time.Now()) 59 | } 60 | logger.Logkv( 61 | "event", queueEventHeartbeatStop, 62 | "message", "Stopping heartbeat goroutine", 63 | ) 64 | } 65 | 66 | func (heartbeat *Heartbeat) Stop() { 67 | heartbeat.ticker.Stop() 68 | } 69 | 70 | type DummyHeartbeat struct{} 71 | 72 | // Stop does nothing 73 | func (*DummyHeartbeat) Stop() { 74 | // do nothing 75 | } 76 | -------------------------------------------------------------------------------- /event/logger.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018-2019 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package event 18 | 19 | import ( 20 | "github.com/onitake/restreamer/util" 21 | ) 22 | 23 | const ( 24 | moduleEvent = "event" 25 | // 26 | queueEventError = "error" 27 | queueEventLimitHit = "hit" 28 | queueEventLimitMiss = "miss" 29 | queueEventStarting = "starting" 30 | queueEventStopping = "stopping" 31 | queueEventStarted = "started" 32 | queueEventReceived = "received" 33 | queueEventDraining = "draining" 34 | queueEventStopped = "stopped" 35 | queueEventConnect = "connect" 36 | queueEventHeartbeat = "heartbeat" 37 | queueEventHeartbeatStart = "heartbeat_start" 38 | queueEventHeartbeatStop = "heartbeat_stop" 39 | queueEventHeartbeatFire = "heartbeat_fire" 40 | // 41 | queueErrorAlreadyRunning = "already_running" 42 | queueErrorInvalidNotification = "invalid_notification" 43 | queueErrorUnderflow = "underflow" 44 | queueErrorOverflow = "overflow" 45 | queueErrorRegister = "register" 46 | queueErrorNotRegistered = "not_registered" 47 | // 48 | urlHandlerEventError = "error" 49 | urlHandlerEventNotify = "notify" 50 | // 51 | urlHandlerErrorGet = "get" 52 | ) 53 | 54 | var logger = util.NewGlobalModuleLogger(moduleEvent, nil) 55 | -------------------------------------------------------------------------------- /event/notifications.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018-2019 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package event 18 | 19 | import "time" 20 | 21 | // Notifiable defines the interface for a notification dispatcher. 22 | // 23 | // Inidividual calls cause state changes, which may trigger events. 24 | type Notifiable interface { 25 | // NotifyConnect reports new connections (if connected is positive) or 26 | // disconnects (if connected is negative). 27 | // 28 | // Connects and disconnects should be reported separately. 29 | NotifyConnect(connected int) 30 | // NotifyHeartbeat is called periodically when enabled, to allow sending 31 | // keepalive messages to a monitoring system 32 | NotifyHeartbeat(when time.Time) 33 | } 34 | -------------------------------------------------------------------------------- /event/queue.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018-2019 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package event 18 | 19 | import ( 20 | "fmt" 21 | "math" 22 | "sync" 23 | "time" 24 | ) 25 | 26 | const ( 27 | // queueSize is the maximum number of notifications to enqueue before we block 28 | queueSize int = 10 29 | ) 30 | 31 | // changeType enumerates all possible state change notifications 32 | type changeType int 33 | 34 | const ( 35 | changeConnect changeType = iota 36 | changeHeartbeat 37 | ) 38 | 39 | // stateChange encapsulates a state change notification 40 | type stateChange struct { 41 | // typ contains the notification type 42 | typ changeType 43 | // connected contains the number of new connections. 44 | // Can be negative if connections are dropped. 45 | connected int 46 | // when contains the point of time when the event was created 47 | when time.Time 48 | } 49 | 50 | // Queue encapsulates state for a connection load reporting callback. 51 | // 52 | // The hit/miss pairs define a hysteresis range to avoid "flapping" reports 53 | // when the number of connections changes quickly around a limit. 54 | type Queue struct { 55 | // limit sets the number of connections when a hit is reported 56 | limit int 57 | // handlers contains all event handlers 58 | handlers map[Type]map[Handler]bool 59 | // internal notification channel for the reporting thread 60 | notifier chan *stateChange 61 | // connections contains the number of active connections. 62 | // only accessed from the reporting thread 63 | connections int 64 | // shutdown is the internal shutdown notifier 65 | shutdown chan struct{} 66 | // running tells if the notifier is currently active 67 | running bool 68 | // waiter allows waiting for shutdown 69 | waiter *sync.WaitGroup 70 | } 71 | 72 | // NewQueue creates a new connection load report notifier. 73 | // 74 | // limit specifies the reporting threshold. 75 | func NewQueue(limit int) *Queue { 76 | if limit < 0 { 77 | panic("limit is out of range") 78 | } 79 | return &Queue{ 80 | limit: limit, 81 | handlers: make(map[Type]map[Handler]bool), 82 | waiter: &sync.WaitGroup{}, 83 | } 84 | } 85 | 86 | // Start launches the reporting goroutine. 87 | // 88 | // To stop the reporter, call Shutdown(). 89 | func (reporter *Queue) Start() { 90 | logger.Logkv( 91 | "event", "check_start", 92 | "message", "Checking if the handler can be started", 93 | ) 94 | // check if we're running already 95 | if !reporter.running { 96 | logger.Logkv( 97 | "event", queueEventStarting, 98 | "message", "Starting notification handler", 99 | ) 100 | // initialise the channels 101 | reporter.shutdown = make(chan struct{}) 102 | reporter.notifier = make(chan *stateChange, queueSize) 103 | // set running state 104 | reporter.running = true 105 | reporter.waiter.Add(1) 106 | // and start the handler 107 | go reporter.run() 108 | } else { 109 | logger.Logkv( 110 | "event", queueEventError, 111 | "error", queueErrorAlreadyRunning, 112 | "message", "Notification handler already running, won't start again", 113 | ) 114 | } 115 | } 116 | 117 | // Shutdown stops the load reporter and waits for completion. 118 | // 119 | // You must not send any notifications after calling this method. 120 | func (reporter *Queue) Shutdown() { 121 | logger.Logkv( 122 | "event", queueEventStopping, 123 | "message", "Stopping notification handler", 124 | ) 125 | // signal shutdown 126 | if reporter.running { 127 | close(reporter.shutdown) 128 | reporter.waiter.Wait() 129 | } 130 | } 131 | 132 | // run is the notification handling loop 133 | func (reporter *Queue) run() { 134 | logger.Logkv( 135 | "event", queueEventStarted, 136 | "message", "Notification handler started", 137 | ) 138 | running := true 139 | for running { 140 | select { 141 | case <-reporter.shutdown: 142 | running = false 143 | case message := <-reporter.notifier: 144 | reporter.handle(message) 145 | } 146 | } 147 | logger.Logkv( 148 | "event", queueEventDraining, 149 | "message", "Draining notification queue", 150 | ) 151 | // drain the notification channel and close it 152 | close(reporter.notifier) 153 | for range reporter.notifier { 154 | } 155 | logger.Logkv( 156 | "event", queueEventStopped, 157 | "message", "Stopped notification handler", 158 | ) 159 | // and we're done 160 | reporter.running = false 161 | reporter.waiter.Done() 162 | } 163 | 164 | // handle handles a single message 165 | func (reporter *Queue) handle(message *stateChange) { 166 | switch message.typ { 167 | case changeConnect: 168 | reporter.handleConnect(message.connected) 169 | case changeHeartbeat: 170 | reporter.handleHeartbeat(message.when) 171 | default: 172 | logger.Logkv( 173 | "event", queueEventError, 174 | "error", queueErrorInvalidNotification, 175 | "type", message.typ, 176 | ) 177 | } 178 | } 179 | 180 | // handleHeartbeat handles a periodic heartbeat 181 | func (reporter *Queue) handleHeartbeat(when time.Time) { 182 | logger.Logkv( 183 | "event", queueEventHeartbeat, 184 | "message", fmt.Sprintf("Periodic heartbeat at: %v", when), 185 | "when", when, 186 | ) 187 | for handler, ok := range reporter.handlers[TypeHeartbeat] { 188 | if ok { 189 | handler.HandleEvent(TypeHeartbeat, when) 190 | } 191 | } 192 | } 193 | 194 | // handleConnect handles a connected clients state change 195 | func (reporter *Queue) handleConnect(connected int) { 196 | logger.Logkv( 197 | "event", queueEventConnect, 198 | "message", fmt.Sprintf("Number of connections changed by %d, current number %d, new number %d", connected, reporter.connections, reporter.connections+connected), 199 | "connected", connected, 200 | "current_connections", reporter.connections, 201 | "new_connections", reporter.connections+connected, 202 | ) 203 | // calculate the new connection count 204 | var newconn int 205 | if connected < 0 && -connected > reporter.connections { 206 | logger.Logkv( 207 | "event", queueEventError, 208 | "error", queueErrorUnderflow, 209 | "message", "Number of disconnects exceeds number of connections, setting to 0", 210 | "connected", connected, 211 | "connections", reporter.connections, 212 | ) 213 | newconn = 0 214 | } else if connected > math.MaxInt32-reporter.connections { 215 | logger.Logkv( 216 | "event", queueEventError, 217 | "error", queueErrorOverflow, 218 | "message", "Number of connects exceeds counter range, clamping to limit", 219 | "connected", connected, 220 | "connections", reporter.connections, 221 | ) 222 | newconn = math.MaxInt32 223 | } else { 224 | newconn = reporter.connections + connected 225 | } 226 | // check if the limit is enabled 227 | if reporter.limit != 0 { 228 | // handle state transitions 229 | if reporter.connections >= reporter.limit { 230 | if newconn < reporter.limit { 231 | // hit -> miss 232 | logger.Logkv( 233 | "event", queueEventLimitMiss, 234 | "message", "Limit missed", 235 | "connections", reporter.connections, 236 | "new", newconn, 237 | "limit", reporter.limit, 238 | ) 239 | for handler, ok := range reporter.handlers[TypeLimitMiss] { 240 | if ok { 241 | handler.HandleEvent(TypeLimitMiss, reporter.connections, newconn, reporter.limit) 242 | } 243 | } 244 | } 245 | } else { 246 | if newconn >= reporter.limit { 247 | // miss -> hit 248 | logger.Logkv( 249 | "event", queueEventLimitHit, 250 | "message", "Limit hit", 251 | "connections", reporter.connections, 252 | "new", newconn, 253 | "limit", reporter.limit, 254 | ) 255 | for handler, ok := range reporter.handlers[TypeLimitHit] { 256 | if ok { 257 | handler.HandleEvent(TypeLimitHit, reporter.connections, newconn, reporter.limit) 258 | } 259 | } 260 | } 261 | } 262 | } 263 | // update the counter 264 | reporter.connections = newconn 265 | } 266 | 267 | func (reporter *Queue) RegisterEventHandler(typ Type, handler Handler) { 268 | if reporter.running { 269 | logger.Logkv( 270 | "event", queueEventError, 271 | "error", queueErrorRegister, 272 | "message", "Cannot register new handlers while the queue is running", 273 | ) 274 | } else { 275 | if _, ok := reporter.handlers[typ]; !ok { 276 | reporter.handlers[typ] = make(map[Handler]bool) 277 | } 278 | reporter.handlers[typ][handler] = true 279 | } 280 | } 281 | 282 | func (reporter *Queue) UnregisterEventHandler(typ Type, handler Handler) { 283 | if reporter.running { 284 | logger.Logkv( 285 | "event", queueEventError, 286 | "error", queueErrorRegister, 287 | "message", "Cannot unregister new handlers while the queue is running", 288 | ) 289 | } else { 290 | if _, ok := reporter.handlers[typ][handler]; ok { 291 | delete(reporter.handlers[typ], handler) 292 | } else { 293 | logger.Logkv( 294 | "event", queueEventError, 295 | "error", queueErrorNotRegistered, 296 | "message", "Event handler wasn't registered", 297 | ) 298 | } 299 | } 300 | } 301 | 302 | func (reporter *Queue) NotifyConnect(connected int) { 303 | // construct the notification message and pass it down the queue 304 | message := &stateChange{ 305 | typ: changeConnect, 306 | connected: connected, 307 | } 308 | reporter.notifier <- message 309 | } 310 | 311 | func (reporter *Queue) NotifyHeartbeat(when time.Time) { 312 | // construct the notification message and pass it down the queue 313 | message := &stateChange{ 314 | typ: changeHeartbeat, 315 | when: when, 316 | } 317 | reporter.notifier <- message 318 | } 319 | -------------------------------------------------------------------------------- /event/queue_test.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package event 18 | 19 | import ( 20 | "github.com/onitake/restreamer/util" 21 | "sync" 22 | "testing" 23 | ) 24 | 25 | type mockLogger struct { 26 | t *testing.T 27 | Stage string 28 | } 29 | 30 | func (l *mockLogger) Logd(lines ...util.Dict) { 31 | for _, line := range lines { 32 | l.t.Logf("%s: %v", l.Stage, line) 33 | } 34 | } 35 | 36 | func (l *mockLogger) Logkv(keyValues ...interface{}) { 37 | l.Logd(util.LogFunnel(keyValues)) 38 | } 39 | 40 | type mockLogConnectable struct { 41 | t *testing.T 42 | Stage string 43 | Waiter *sync.WaitGroup 44 | } 45 | 46 | func (l *mockLogConnectable) Logd(lines ...util.Dict) { 47 | for _, line := range lines { 48 | l.t.Logf("%s: %v", l.Stage, line) 49 | if line["event"] == queueEventConnect { 50 | l.Waiter.Done() 51 | } 52 | } 53 | } 54 | 55 | func (l *mockLogConnectable) Logkv(keyValues ...interface{}) { 56 | l.Logd(util.LogFunnel(keyValues)) 57 | } 58 | 59 | type mockLogDisconnectable struct { 60 | t *testing.T 61 | Stage string 62 | Waiter *sync.WaitGroup 63 | } 64 | 65 | func (l *mockLogDisconnectable) Logd(lines ...util.Dict) { 66 | for _, line := range lines { 67 | l.t.Logf("%s: %v", l.Stage, line) 68 | if line["event"] == queueEventStopped { 69 | l.Waiter.Done() 70 | } 71 | } 72 | } 73 | 74 | func (l *mockLogDisconnectable) Logkv(keyValues ...interface{}) { 75 | l.Logd(util.LogFunnel(keyValues)) 76 | } 77 | 78 | type mockHandler struct { 79 | t *testing.T 80 | Hit *sync.WaitGroup 81 | Miss *sync.WaitGroup 82 | } 83 | 84 | func (h *mockHandler) HandleEvent(t Type, args ...interface{}) { 85 | switch t { 86 | case TypeLimitHit: 87 | h.Hit.Done() 88 | case TypeLimitMiss: 89 | h.Miss.Done() 90 | } 91 | } 92 | 93 | func TestCreateLoadReporter00(t *testing.T) { 94 | l := &mockLogger{t, ""} 95 | 96 | l.Stage = "t00" 97 | c00 := NewQueue(0) 98 | logger = l 99 | c00.Start() 100 | c00.Shutdown() 101 | } 102 | 103 | func TestCreateLoadReporter01(t *testing.T) { 104 | l := &mockLogger{t, ""} 105 | 106 | l.Stage = "t01" 107 | c01 := NewQueue(0) 108 | logger = l 109 | c01.Start() 110 | c01.Start() 111 | c01.Shutdown() 112 | } 113 | 114 | func TestCreateLoadReporter02(t *testing.T) { 115 | c02 := NewQueue(0) 116 | l02 := &mockLogConnectable{ 117 | t, 118 | "t02", 119 | &sync.WaitGroup{}, 120 | } 121 | logger = l02 122 | c02.Start() 123 | l02.Waiter.Add(1) 124 | c02.NotifyConnect(1) 125 | l02.Waiter.Wait() 126 | c02.Shutdown() 127 | } 128 | 129 | func TestCreateLoadReporter03(t *testing.T) { 130 | l := &mockLogger{t, ""} 131 | 132 | l.Stage = "t03" 133 | c03 := NewQueue(0) 134 | logger = l 135 | c03.Start() 136 | c03.Shutdown() 137 | c03.Start() 138 | c03.Shutdown() 139 | } 140 | 141 | func TestCreateLoadReporter04(t *testing.T) { 142 | c04 := NewQueue(0) 143 | l04 := &mockLogConnectable{ 144 | t, 145 | "t04", 146 | &sync.WaitGroup{}, 147 | } 148 | logger = l04 149 | c04.Start() 150 | l04.Waiter.Add(1) 151 | c04.NotifyConnect(1) 152 | l04.Waiter.Wait() 153 | c04.Shutdown() 154 | c04.Start() 155 | l04.Waiter.Add(1) 156 | c04.NotifyConnect(1) 157 | l04.Waiter.Wait() 158 | c04.Shutdown() 159 | } 160 | 161 | func TestCreateLoadReporter05(t *testing.T) { 162 | l := &mockLogger{t, ""} 163 | 164 | c05 := NewQueue(10) 165 | l.Stage = "t05" 166 | logger = l 167 | h05 := &mockHandler{ 168 | t: t, 169 | Hit: &sync.WaitGroup{}, 170 | Miss: &sync.WaitGroup{}, 171 | } 172 | h05.Hit.Add(3) 173 | h05.Miss.Add(2) 174 | c05.RegisterEventHandler(TypeLimitHit, h05) 175 | c05.RegisterEventHandler(TypeLimitMiss, h05) 176 | c05.Start() 177 | c05.NotifyConnect(10) 178 | c05.NotifyConnect(-1) 179 | c05.NotifyConnect(-2) 180 | c05.NotifyConnect(4) 181 | c05.NotifyConnect(1) 182 | c05.NotifyConnect(-2) 183 | c05.NotifyConnect(-1) 184 | c05.NotifyConnect(1) 185 | h05.Hit.Wait() 186 | h05.Miss.Wait() 187 | c05.Shutdown() 188 | } 189 | -------------------------------------------------------------------------------- /event/urlhandler.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package event 18 | 19 | import ( 20 | "fmt" 21 | "github.com/onitake/restreamer/auth" 22 | "net/http" 23 | "net/url" 24 | ) 25 | 26 | // UrlHandler is an event handler that can send GET requests to a preconfigured HTTP URL. 27 | type UrlHandler struct { 28 | // Url is the parsed URL 29 | Url *url.URL 30 | // userauth will be used to generate credentials for client requests 31 | userauth *auth.UserAuthenticator 32 | } 33 | 34 | func NewUrlHandler(urly string, userauth *auth.UserAuthenticator) (*UrlHandler, error) { 35 | u, err := url.Parse(urly) 36 | if err == nil { 37 | return &UrlHandler{ 38 | Url: u, 39 | userauth: userauth, 40 | }, nil 41 | } else { 42 | return nil, err 43 | } 44 | } 45 | 46 | func (handler *UrlHandler) HandleEvent(typ Type, args ...interface{}) { 47 | logger.Logkv( 48 | "event", urlHandlerEventNotify, 49 | "message", fmt.Sprintf("Event received, notifying %s", handler.Url), 50 | "url", handler.Url.String(), 51 | "auth", handler.userauth != nil, 52 | "type", typ, 53 | ) 54 | req := &http.Request{ 55 | Method: "GET", 56 | URL: handler.Url, 57 | Header: make(http.Header), 58 | } 59 | if handler.userauth != nil { 60 | req.Header.Add("Authorization", handler.userauth.GetLogin()) 61 | } 62 | _, err := http.DefaultClient.Do(req) 63 | if err != nil { 64 | logger.Logkv( 65 | "event", urlHandlerEventError, 66 | "error", urlHandlerErrorGet, 67 | "message", fmt.Sprintf("Error sending GET request: %v", err), 68 | "url", handler.Url.String(), 69 | "type", typ, 70 | ) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /examples/documented/restreamer.json: -------------------------------------------------------------------------------- 1 | { 2 | "": "Listen on ::1 and 127.0.0.1, port 8000.", 3 | "": "You can also use identifiers like :http to listen on all interfaces on a standard service port", 4 | "listen": "localhost:8000", 5 | "": "Set connect and network protocol timeouts, in seconds.", 6 | "": "0 disables the timeout, i.e. means: wait forever.", 7 | "": "Note that the OS may still impose I/O timeouts even if this is 0.", 8 | "timeout": 0, 9 | "": "Set the number of seconds between reconnection attempts.", 10 | "": "This also affects round-robin scheduling.", 11 | "": "0 disables reconnecting altogether.", 12 | "reconnect": 10, 13 | "": "Set the packet read timeout, in seconds.", 14 | "": "0 disables the timeout, i.e. means: wait forever for data.", 15 | "": "If set, connections are closed automatically when they stop sending.", 16 | "readtimeout": 0, 17 | "": "Set to true to disable stats tracking.", 18 | "nostats": false, 19 | "": "Set to true to enable profiling.", 20 | "profile": false, 21 | "": "Size of the input buffer per stream in TS packets (= 188 bytes).", 22 | "": "Also used to determine the size of the kernel buffer for datagram sockets.", 23 | "inputbuffer": 1000, 24 | "": "Size of the output buffer per client connection in TS packets.", 25 | "outputbuffer": 400, 26 | "": "The global client connection limit.", 27 | "maxconnections": 100, 28 | "": "Soft limit for the number of client connections.", 29 | "": "Restreamer will start reporting that it is full when this limit is reached.", 30 | "": "It will still accept new connections until maxconnections is reached, however.", 31 | "fullconnections": 90, 32 | "": "Number of seconds between each heartbeat.", 33 | "": "Will be ignore if no heartbeat notifications are defined.", 34 | "heartbeatinterval": 60, 35 | "": "The JSON access log file name. If this option is empty, access logs are disabled.", 36 | "log": "", 37 | "": "The user database used for authentication stanzas", 38 | "userlist": { 39 | "username": { 40 | "": "The user's password", 41 | "password": "secret_password" 42 | } 43 | }, 44 | "": "List of resources; can be streams, static content or APIs.", 45 | "resources": [ 46 | { 47 | "": "Type of this resource: stream, static, api", 48 | "": "stream = HTTP stream", 49 | "": "static = static content from a local file or remote source", 50 | "": "api = builtin API", 51 | "type": "stream", 52 | "": "API endpoint, only used if type is api.", 53 | "": "health = reports system health.", 54 | "": "statistics = reports detailed system statistics. [deprecated, use prometheus]", 55 | "": "prometheus = reports detailed system statistics as a standard Prometheus scrape endpoint.", 56 | "": "check = reports the status of a stream. remote contains the serve path of the stream.", 57 | "": "control = allows setting a stream offline or online. The state is controlled by the presence of the query parameters 'offline' or 'online', respectively.", 58 | "api": "", 59 | "": "Path under which a resource is made available.", 60 | "serve": "/stream.ts", 61 | "": "The upstream URL. Supported protocols are: http, https, file, tcp, udp, unix, unixgram, unixpacket or fork.", 62 | "": "file must specify the URL in host-compatible format.", 63 | "": "For tcp and udp, a port is mandatory. Literal IPv6 addresses must be enclosed in []", 64 | "": "unix will autodetect the type of domain socket, but you can also be explicit with unixgram and unixpacket.", 65 | "": "This parameter is also required for API types 'check' and 'control', setting the stream they refer to.", 66 | "": "If the udp protocol is used, the address can be a unicast or multicast address.", 67 | "": "Multicast groups are joined automatically.", 68 | "": "fork is a special protocol that allows launching a local command. Stream data is captured from the command's standard output.", 69 | "": "Anything written to standard error will be logged through restreamer's logging mechanism.", 70 | "": "The URL format is: fork:///path/to/executable?argument1+argument2+argument3+etc", 71 | "": "Note: Special characters in the arguments must be escaped, and spaces in the command path or arguments are not supported.", 72 | "remote": "http://localhost:10000/stream.ts", 73 | "": "Instead of a single remote URL, a list of URLs can be specified with the remotes option.", 74 | "": "The same rules as for remote apply.", 75 | "": "If both are specified, both are used. This does not apply to API and proxy endpoints, where only a single remote is supported.", 76 | "remotes": [ ], 77 | "": "Cache time in seconds, use 0 to disable caching.", 78 | "": "Only supported for static content.", 79 | "cache": 0, 80 | "": "Maximum receive unit, the packet size for datagram sockets (UDP).", 81 | "": "This value is important, because individual datagrams can only be received as a whole. Excess data is discarded.", 82 | "mru": 1500, 83 | "": "Specify a file name to a static preamble that will be sent to each newly connected client.", 84 | "": "This can help when a decoder isn't capable of initializing in the middle of a transmission,", 85 | "": "but it can also make things much worse. You have been warned.", 86 | "preamble": "preamble.ts", 87 | "": "Access control for this resource. If not present, no authentication is necessary.", 88 | "": "Otherwise, an authentication token that matches one of the users is required.", 89 | "authentication": { 90 | "": "The authentication type: basic or bearer", 91 | "": "Basic authentication requires a valid Authorization: Basic base64(md5sum('user:password')) header.", 92 | "": "Bearer authentication requires a valid Authorization: Bearer base64('password') header.", 93 | "type": "", 94 | "": "Realm specifies the realm that is sent back to the client if no Authorization header was present.", 95 | "realm": "", 96 | "": "A single user that is allowed to access this resource. Concatenated with users.", 97 | "user": "", 98 | "": "A list of users that may access this resource. prepended with user.", 99 | "users": [ ] 100 | 101 | } 102 | }, 103 | { 104 | "type": "api", 105 | "api": "check", 106 | "serve": "/check/stream.ts", 107 | "remote": "/stream.ts" 108 | }, 109 | { 110 | "type": "api", 111 | "api": "control", 112 | "serve": "/control/stream.ts", 113 | "remote": "/stream.ts" 114 | }, 115 | { 116 | "type": "stream", 117 | "serve": "/pipe.ts", 118 | "remote": "file:///tmp/pipe.ts", 119 | "remotes": [ "unix:///tmp/pipe2.ts" ] 120 | }, 121 | { 122 | "type": "api", 123 | "api": "health", 124 | "serve": "/health" 125 | }, 126 | { 127 | "type": "api", 128 | "api": "prometheus", 129 | "serve": "/metrics" 130 | }, 131 | { 132 | "type": "static", 133 | "serve": "/test", 134 | "remote": "file:///tmp/test" 135 | }, 136 | { 137 | "type": "static", 138 | "serve": "/stats", 139 | "remote": "http://localhost:10000/stats", 140 | "cache": 60 141 | } 142 | ], 143 | "": "List of event handlers; currently only HTTP callbacks are supported.", 144 | "notifications": [ 145 | { 146 | "": "Event to watch for: limit_hit, limit_miss or heartbeat", 147 | "": "limit_hit notifies when the soft limit (fullconnections) is reached", 148 | "": "limit_miss notifies when the number of connections goes below this threshold", 149 | "": "heartbeat notifies once per heartbeatinterval", 150 | "event": "limit_hit", 151 | "": "The kind of notification that is generated. Only url is supported.", 152 | "type": "url", 153 | "": "A GET request is sent to this URL if type is url.", 154 | "url": "http://localhost:8001/hit", 155 | "": "Optional authentication settings to allow sending an Authorization header with the get request", 156 | "authentication": { 157 | "": "Authentication type: basic or bearer", 158 | "type": "", 159 | "": "The user account that is used with this notification. Must be contained in the userlist.", 160 | "user": "" 161 | } 162 | }, 163 | { 164 | "event": "limit_miss", 165 | "type": "url", 166 | "url": "http://localhost:8001/miss" 167 | }, 168 | { 169 | "event": "heartbeat", 170 | "type": "url", 171 | "url": "http://localhost:8001/ping" 172 | } 173 | ] 174 | } 175 | -------------------------------------------------------------------------------- /examples/minimal/restreamer.json: -------------------------------------------------------------------------------- 1 | {"listen":":8000","timeout":0,"maxconnections":1,"log":"","resources":[{"type":"api","api":"health","serve": "/health"},{"type":"stream","serve":"/stream.ts","remote":"http://localhost:8001/"}]} 2 | -------------------------------------------------------------------------------- /genpreamble.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ffmpeg \ 4 | -use_wallclock_as_timestamps 1 \ 5 | -t 1 -r 50 -f lavfi -i color=c=black:s=1280x720 \ 6 | -t 1 -f lavfi -i anullsrc=r=48000:cl=stereo \ 7 | -pix_fmt:v yuv420p -color_range:v tv -color_primaries:v bt709 -colorspace bt709 -color_trc bt709 -profile:v high -b:v 3500000 -c:v libx264 \ 8 | -b:a 128000 -c:a aac \ 9 | -f mpegts -y out.ts 10 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/onitake/restreamer 2 | 3 | require github.com/prometheus/client_golang v1.19.1 4 | 5 | require ( 6 | github.com/beorn7/perks v1.0.1 // indirect 7 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 8 | github.com/prometheus/client_model v0.6.1 // indirect 9 | github.com/prometheus/common v0.52.2 // indirect 10 | github.com/prometheus/procfs v0.13.0 // indirect 11 | golang.org/x/sys v0.19.0 // indirect 12 | google.golang.org/protobuf v1.33.0 // indirect 13 | ) 14 | 15 | go 1.21 16 | 17 | toolchain go1.22.1 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 2 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 3 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 4 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 5 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 6 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 8 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 9 | github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= 10 | github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= 11 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 12 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 13 | github.com/prometheus/common v0.52.2 h1:LW8Vk7BccEdONfrJBDffQGRtpSzi5CQaRZGtboOO2ck= 14 | github.com/prometheus/common v0.52.2/go.mod h1:lrWtQx+iDfn2mbH5GUzlH9TSHyfZpHkSiG1W7y3sF2Q= 15 | github.com/prometheus/procfs v0.13.0 h1:GqzLlQyfsPbaEHaQkO7tbDlriv/4o5Hudv6OXHGKX7o= 16 | github.com/prometheus/procfs v0.13.0/go.mod h1:cd4PFCR54QLnGKPaKGA6l+cfuNXtht43ZKY6tow0Y1g= 17 | golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= 18 | golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 19 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= 20 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 21 | -------------------------------------------------------------------------------- /metrics/logger.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2019 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "github.com/onitake/restreamer/util" 21 | ) 22 | 23 | const ( 24 | moduleMetrics = "metrics" 25 | // 26 | eventMetricsError = "error" 27 | // 28 | errorMetricsPrometheus = "prometheus" 29 | ) 30 | 31 | var logger = util.NewGlobalModuleLogger(moduleMetrics, nil) 32 | -------------------------------------------------------------------------------- /metrics/prom.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2019 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "fmt" 21 | "github.com/prometheus/client_golang/prometheus" 22 | "github.com/prometheus/client_golang/prometheus/collectors" 23 | "github.com/prometheus/client_golang/prometheus/promhttp" 24 | "net/http" 25 | ) 26 | 27 | var ( 28 | defaultRegistry = prometheus.NewRegistry() 29 | // DefaultRegisterer is a prometheus client registry that contains no 30 | // default metrics. See prometheus.Registry for more information. 31 | DefaultRegisterer prometheus.Registerer = defaultRegistry 32 | // DefaultGatherer points to the same registry as DefaultRegisterer. 33 | // See prometheus.Registry for more information. 34 | DefaultGatherer prometheus.Gatherer = defaultRegistry 35 | ) 36 | 37 | func init() { 38 | // register the standard metrics 39 | DefaultRegisterer.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) 40 | // the Go metrics are NOT enabled by default as they can have a serious performance impact. 41 | // if you want them, you need to register with a call to EnableGoRuntimeCollector(). 42 | } 43 | 44 | // EnableGoRuntimeCollector enables the Prometheus Go runtime collector. 45 | // Warning: This can have a serious impact on runtime performance. Enable at your own risk. 46 | func EnableGoRuntimeCollector() { 47 | DefaultRegisterer.MustRegister(collectors.NewGoCollector()) 48 | } 49 | 50 | // promErrorLogger is an internal error logger that prints to the kvl log. 51 | type promErrorLogger struct{} 52 | 53 | func (*promErrorLogger) Println(v ...interface{}) { 54 | logger.Logkv( 55 | "event", eventMetricsError, 56 | "error", errorMetricsPrometheus, 57 | "message", fmt.Sprintln(v...), 58 | ) 59 | } 60 | 61 | // PromHandler creates a prometheus HTTP handler that wraps DefaultGatherer 62 | // and logs to the standard kvl logger. 63 | func PromHandler() http.Handler { 64 | return promhttp.HandlerFor(DefaultGatherer, promhttp.HandlerOpts{ 65 | ErrorLog: &promErrorLogger{}, 66 | ErrorHandling: promhttp.ContinueOnError, 67 | }) 68 | } 69 | 70 | // MustRegister registers the provided Collectors with the DefaultRegisterer 71 | // and panics if any error occurs. 72 | // 73 | // MustRegister is a shortcut for DefaultRegisterer.MustRegister(cs...). See 74 | // there for more details. 75 | func MustRegister(cs ...prometheus.Collector) { 76 | DefaultRegisterer.MustRegister(cs...) 77 | } 78 | 79 | // Register registers the provided Collector with the DefaultRegisterer. 80 | // 81 | // Register is a shortcut for DefaultRegisterer.Register(c). See there for more 82 | // details. 83 | func Register(c prometheus.Collector) error { 84 | return DefaultRegisterer.Register(c) 85 | } 86 | -------------------------------------------------------------------------------- /metrics/stats_test.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | ) 23 | 24 | /* 25 | func (stats *DummyStatistics) Start() { 26 | func (stats *DummyStatistics) Stop() { 27 | func (stats *DummyStatistics) RegisterStream(name string) Collector { 28 | func (stats *DummyStatistics) RemoveStream(name string) { 29 | func (stats *DummyStatistics) GetStreamStatistics(name string) *StreamStatistics { 30 | func (stats *DummyStatistics) GetAllStreamStatistics() map[string]*StreamStatistics { 31 | func (stats *DummyStatistics) GetGlobalStatistics() *StreamStatistics { 32 | 33 | func (stats *DummyCollector) ConnectionAdded() { 34 | func (stats *DummyCollector) ConnectionRemoved() { 35 | func (stats *DummyCollector) PacketReceived() { 36 | func (stats *DummyCollector) PacketSent() { 37 | func (stats *DummyCollector) PacketDropped() { 38 | func (stats *DummyCollector) SourceConnected() { 39 | func (stats *DummyCollector) SourceDisconnected() { 40 | func (stats *DummyCollector) IsUpstreamConnected() bool { 41 | func (stats *DummyCollector) StreamDuration(duration time.Duration) { 42 | 43 | Connections int64 44 | MaxConnections int64 45 | FullConnections int64 46 | TotalPacketsReceived uint64 47 | TotalPacketsSent uint64 48 | TotalPacketsDropped uint64 49 | TotalBytesReceived uint64 50 | TotalBytesSent uint64 51 | TotalBytesDropped uint64 52 | TotalStreamTime int64 53 | PacketsPerSecondReceived uint64 54 | PacketsPerSecondSent uint64 55 | PacketsPerSecondDropped uint64 56 | BytesPerSecondReceived uint64 57 | BytesPerSecondSent uint64 58 | BytesPerSecondDropped uint64 59 | Connected bool 60 | */ 61 | 62 | func testStatisticsStartStop(t *testing.T, s Statistics) { 63 | s.Start() 64 | s.Stop() 65 | } 66 | 67 | func testStatisticsRegisterRemove(t *testing.T, s Statistics) { 68 | s.RegisterStream("testStatisticsRegisterRemove") 69 | s.RemoveStream("testStatisticsRegisterRemove") 70 | } 71 | 72 | func testStatisticsRegisterGetRemove(t *testing.T, s Statistics) { 73 | s.RegisterStream("testStatisticsRegisterGetRemove") 74 | s.GetStreamStatistics("testStatisticsRegisterGetRemove") 75 | s.RemoveStream("testStatisticsRegisterGetRemove") 76 | } 77 | 78 | func testStatisticsLimits(t *testing.T, s Statistics, max, full int64) { 79 | r := s.GetGlobalStatistics() 80 | if r.MaxConnections != max { 81 | t.Errorf("testStatisticsLimits: Max connection value (=%v) not matched (=%v)", r.MaxConnections, max) 82 | } 83 | if r.FullConnections != full { 84 | t.Errorf("testStatisticsLimits: Full connection value (=%v) not matched (=%v)", r.FullConnections, full) 85 | } 86 | } 87 | 88 | func testStatisticsStateChange(t *testing.T, s Statistics) { 89 | c := s.RegisterStream("testStatisticsStateChange") 90 | s.Start() 91 | <-time.After(1 * time.Second) 92 | c.ConnectionAdded() 93 | c.PacketReceived() 94 | <-time.After(2 * time.Second) 95 | r := s.GetStreamStatistics("testStatisticsStateChange") 96 | s.Stop() 97 | t.Logf("testStatisticsStateChange: %v", r) 98 | s.RemoveStream("testStatisticsStateChange") 99 | if r.Connections != 1 { 100 | t.Errorf("testStatisticsStateChange: Connected value (=%v) not matched (=%v)", r.Connections, 1) 101 | } 102 | } 103 | 104 | func TestDummyStatistics(t *testing.T) { 105 | testStatisticsStartStop(t, &DummyStatistics{}) 106 | testStatisticsRegisterRemove(t, &DummyStatistics{}) 107 | testStatisticsRegisterGetRemove(t, &DummyStatistics{}) 108 | } 109 | 110 | func TestRealStatistics(t *testing.T) { 111 | testStatisticsStartStop(t, NewStatistics(0, 0)) 112 | testStatisticsRegisterRemove(t, NewStatistics(0, 0)) 113 | testStatisticsRegisterGetRemove(t, NewStatistics(0, 0)) 114 | testStatisticsLimits(t, NewStatistics(10, 20), 10, 20) 115 | testStatisticsStateChange(t, NewStatistics(0, 0)) 116 | } 117 | -------------------------------------------------------------------------------- /protocol/fork.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "os/exec" 8 | ) 9 | 10 | type ForkReader struct { 11 | command *exec.Cmd 12 | dataInput io.ReadCloser 13 | errorInput io.ReadCloser 14 | } 15 | 16 | func (f *ForkReader) Read(p []byte) (n int, err error) { 17 | return f.dataInput.Read(p) 18 | } 19 | 20 | func (f *ForkReader) Close() error { 21 | return f.command.Process.Kill() 22 | } 23 | 24 | func NewForkReader(command string, arguments []string) (*ForkReader, error) { 25 | cmd := exec.Command(command, arguments...) 26 | stdout, err := cmd.StdoutPipe() 27 | if err != nil { 28 | return nil, err 29 | } 30 | stderr, err := cmd.StderrPipe() 31 | if err != nil { 32 | return nil, err 33 | } 34 | if err := cmd.Start(); err != nil { 35 | return nil, err 36 | } 37 | logger.Logkv( 38 | "event", eventForkStarted, 39 | "pid", cmd.Process.Pid, 40 | "command", cmd.Path, 41 | "message", fmt.Sprintf("Fork reader command started: %s %v", command, arguments), 42 | ) 43 | fr := &ForkReader{ 44 | command: cmd, 45 | dataInput: stdout, 46 | errorInput: stderr, 47 | } 48 | // Launch a goroutine that logs output to stderr 49 | go func(f *ForkReader) { 50 | buffer := bufio.NewReader(f.errorInput) 51 | for line, err := "", error(nil); err == nil; { 52 | line, err = buffer.ReadString('\n') 53 | if err != nil && err != io.EOF { 54 | logger.Logkv( 55 | "event", eventForkError, 56 | "error", errorForkStderrRead, 57 | "command", f.command.Path, 58 | "message", fmt.Sprintf("Error reading from stderr: %v", err), 59 | ) 60 | } 61 | logger.Logkv( 62 | "event", eventForkChildMessage, 63 | "command", f.command.Path, 64 | "message", line, 65 | ) 66 | } 67 | }(fr) 68 | // Wait for command exit in a goroutine, so we can report process exit asynchronously 69 | go func(f *ForkReader) { 70 | err := f.command.Wait() 71 | if err != nil { 72 | logger.Logkv( 73 | "event", eventForkError, 74 | "error", errorForkExit, 75 | "exitcode", cmd.ProcessState.ExitCode(), 76 | "command", f.command.Path, 77 | "message", fmt.Sprintf("Process exited with error: %v", err), 78 | ) 79 | } 80 | }(fr) 81 | return fr, nil 82 | } 83 | -------------------------------------------------------------------------------- /protocol/logger.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2022 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package protocol 18 | 19 | import ( 20 | "github.com/onitake/restreamer/util" 21 | ) 22 | 23 | const ( 24 | moduleProtocol = "protocol" 25 | // 26 | eventForkError = "error" 27 | eventForkStarted = "forked" 28 | eventForkChildMessage = "childmessage" 29 | // 30 | errorForkExit = "exit_error" 31 | errorForkStderrRead = "stderr_read" 32 | ) 33 | 34 | var logger = util.NewGlobalModuleLogger(moduleProtocol, nil) 35 | -------------------------------------------------------------------------------- /protocol/mpegts.goconvey: -------------------------------------------------------------------------------- 1 | // Timeout to avoid failing tests that block forever 2 | -timeout=3s 3 | -------------------------------------------------------------------------------- /protocol/packet.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2016-2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package protocol 18 | 19 | import ( 20 | "io" 21 | ) 22 | 23 | const ( 24 | // MpegTsPacketSize is the TS packet size (188 bytes) 25 | MpegTsPacketSize = 188 26 | // MpegTsSyncByte is the byte value of the TS synchronization code (0x47) 27 | MpegTsSyncByte = 0x47 28 | ) 29 | 30 | // MpegTsPacket is an alias to a byte slice and represents one TS packet. 31 | // It is 188 bytes long and starts with 0x47. 32 | type MpegTsPacket []byte 33 | 34 | // ReadMpegTsPacket reads data from the input stream, 35 | // scans for the sync byte and returns one packet from that point on. 36 | // 37 | // If a sync byte can't be found among the first 188 bytes, 38 | // no packets are returned 39 | func ReadMpegTsPacket(reader io.Reader) (MpegTsPacket, error) { 40 | garbage := make(MpegTsPacket, MpegTsPacketSize) 41 | offset := 0 42 | // read 188 bytes ahead (assume we are at the start of a packet) 43 | for offset < MpegTsPacketSize { 44 | nbytes, err := reader.Read(garbage[offset:]) 45 | // read error - bail out 46 | if err != nil { 47 | return nil, err 48 | } 49 | offset += nbytes 50 | //logger.Logkv("event", "read", "bytes", nbytes) 51 | } 52 | 53 | // quick check if it starts with the sync byte 0x47 54 | if garbage[0] != MpegTsSyncByte { 55 | //logger.Logkv("event", "partial") 56 | 57 | // nope, scan first 58 | sync := -1 59 | for i, bytes := range garbage { 60 | if bytes == MpegTsSyncByte { 61 | // found, very good 62 | sync = i 63 | break 64 | } 65 | } 66 | // nothing found, return nothing 67 | if sync == -1 { 68 | return nil, nil 69 | } 70 | //logger.Logkv("event", "sync", "position", sync) 71 | 72 | // if the sync byte was not at the beginning, 73 | // create a new packet and append the remaining data. 74 | // this should happen only when the stream is out of sync, 75 | // so performance impact is minimal 76 | packet := make(MpegTsPacket, MpegTsPacketSize) 77 | offset = len(packet) - sync 78 | //logger.Logkv("event", "offset", "offset", offset) 79 | copy(packet, garbage[sync:]) 80 | for offset < MpegTsPacketSize { 81 | nbytes, err := reader.Read(packet[offset:]) 82 | if err != nil { 83 | return nil, err 84 | } 85 | offset += nbytes 86 | //logger.Logkv("event", "append", "bytes", nbytes, "position", offset) 87 | } 88 | // return the assembled packet 89 | return packet, nil 90 | } 91 | 92 | // and done 93 | return garbage, nil 94 | } 95 | -------------------------------------------------------------------------------- /protocol/packet_test.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package protocol 18 | 19 | import ( 20 | "bytes" 21 | "io" 22 | "math/rand" 23 | "testing" 24 | ) 25 | 26 | func makeRandomPacket(t *testing.T, bytes int) []byte { 27 | packet := make([]byte, bytes) 28 | rnd := rand.New(rand.NewSource(18847)) 29 | if n, err := rnd.Read(packet); err != nil || n != bytes { 30 | t.Fatal("Error creating a random packet") 31 | } 32 | return packet 33 | } 34 | 35 | func TestPacketScan(t *testing.T) { 36 | c01 := bytes.NewBuffer([]byte{}) 37 | r01, err := ReadMpegTsPacket(c01) 38 | if err != io.EOF || r01 != nil { 39 | t.Error("t01: Expected EOF on empty buffer, got something else") 40 | } 41 | 42 | c02 := bytes.NewBuffer([]byte{ 43 | 0x00, 44 | }) 45 | r02, err := ReadMpegTsPacket(c02) 46 | if err != io.EOF || r02 != nil { 47 | t.Error("t02: Expected EOF on incomplete buffer without sync, got something else") 48 | } 49 | 50 | c03 := bytes.NewBuffer([]byte{ 51 | 0x47, 0x00, 52 | }) 53 | r03, err := ReadMpegTsPacket(c03) 54 | if err != io.EOF || r03 != nil { 55 | t.Error("t03: Expected EOF on incomplete buffer, got something else") 56 | } 57 | 58 | c04 := bytes.NewBuffer(make([]byte, 188)) 59 | r04, err := ReadMpegTsPacket(c04) 60 | if err != nil || len(r04) != 0 { 61 | t.Errorf("t04: Expected empty result on zero buffer, got something else") 62 | } 63 | 64 | b05 := makeRandomPacket(t, 188) 65 | b05[0] = 0x47 66 | c05 := bytes.NewBuffer(b05) 67 | r05, err := ReadMpegTsPacket(c05) 68 | if err != nil || !bytes.Equal(b05, r05) { 69 | t.Error("t05: Expected packet identical to buffer, got an error or something else") 70 | } 71 | 72 | b06 := makeRandomPacket(t, 189) 73 | b06[0] = 0x47 74 | c06 := bytes.NewBuffer(b06) 75 | r06, err := ReadMpegTsPacket(c06) 76 | if err != nil || !bytes.Equal(b06[:188], r06) { 77 | t.Error("t06: Expected packet identical to head of buffer, got an error or something else") 78 | } 79 | 80 | b07 := makeRandomPacket(t, 189) 81 | b07[0] = 0 82 | b07[1] = 0x47 83 | c07 := bytes.NewBuffer(b07) 84 | r07, err := ReadMpegTsPacket(c07) 85 | if err != nil || !bytes.Equal(b07[1:189], r07) { 86 | t.Error("t07: Expected packet identical to tail of buffer, got an error or something else") 87 | } 88 | 89 | b08 := makeRandomPacket(t, 188) 90 | b08[0] = 0 91 | b08[1] = 0x47 92 | c08 := bytes.NewBuffer(b08) 93 | r08, err := ReadMpegTsPacket(c08) 94 | if err != io.EOF || r08 != nil { 95 | t.Error("t08: Expected EOF on incomplete packet that didn't start at offset 0, got something else") 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /protocol/reader.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2019 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package protocol 18 | 19 | import ( 20 | "bytes" 21 | "io" 22 | ) 23 | 24 | // FixedReader implements a buffered reader that always reads a fixed amount 25 | // of data from an underlying io.Reader. 26 | // 27 | // It is intended to produce a stream from a packet-based network socket, such 28 | // as net.UDPConn. 29 | // 30 | // Note that packet-based sockets normally give no guarantee about the order of 31 | // incoming packets, and don't account for resends of dropped ones. 32 | // If order is important, reordering must be implemented by other means. 33 | // 34 | // If the underlying reader implements the io.Closer interface, Close() calls 35 | // will be forwarded. Otherwise, Close() is a no-op. 36 | type FixedReader struct { 37 | reader io.Reader 38 | packetSize int 39 | buffer *bytes.Buffer 40 | } 41 | 42 | // NewFixedReader creates a new buffered reader that pulls in data from an 43 | // io.Reader in chunks of psize bytes. 44 | func NewFixedReader(reader io.Reader, psize int) *FixedReader { 45 | return &FixedReader{ 46 | reader: reader, 47 | packetSize: psize, 48 | buffer: bytes.NewBuffer(make([]byte, 0, psize)), 49 | } 50 | } 51 | 52 | // Read reads as many bytes from the internal buffer as can fit into p. 53 | // 54 | // If the buffer has no data left, it tries to pull in a new packet from the 55 | // underlying reader. 56 | func (b *FixedReader) Read(p []byte) (n int, err error) { 57 | // check if we need to read another packet 58 | if b.buffer.Len() == 0 { 59 | // read the next packet 60 | p := make([]byte, b.packetSize) 61 | var m int 62 | // pass on err if the read fails 63 | m, err = b.reader.Read(p) 64 | // only buffer as many bytes as were received 65 | b.buffer.Write(p[:m]) 66 | } 67 | if err == nil { 68 | // if the was no I/O error, pass on any buffer errors 69 | n, err = b.buffer.Read(p) 70 | } else { 71 | // ignore buffer errors and pass on the I/O error instead 72 | n, _ = b.buffer.Read(p) 73 | } 74 | return n, err 75 | } 76 | 77 | // Close closes the underlying reader. 78 | // 79 | // Subsequent Read calls will succeed as long as the internal buffer still 80 | // has data. If the buffer is drained, Read returns an error. 81 | func (b *FixedReader) Close() error { 82 | if closer, ok := b.reader.(io.Closer); ok { 83 | return closer.Close() 84 | } 85 | return nil 86 | } 87 | -------------------------------------------------------------------------------- /protocol/reader_test.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package protocol 18 | 19 | import ( 20 | "bytes" 21 | "io" 22 | "testing" 23 | ) 24 | 25 | func TestFixedReaderNoData(t *testing.T) { 26 | d := make([]byte, 0) 27 | r := bytes.NewBuffer(d) 28 | f := NewFixedReader(r, 10) 29 | for i := 0; i < 6; i++ { 30 | g := make([]byte, 2) 31 | n, err := f.Read(g) 32 | if n != 0 || err == nil { 33 | t.Fatal("Expected 0 bytes and error") 34 | } 35 | } 36 | } 37 | 38 | func TestFixedReaderNotEnoughInput(t *testing.T) { 39 | d := make([]byte, 2) 40 | for i := 0; i < len(d); i++ { 41 | d[i] = byte(i) 42 | } 43 | r := bytes.NewBuffer(d) 44 | f := NewFixedReader(r, 10) 45 | g := make([]byte, 2) 46 | n, err := f.Read(g) 47 | if n != 2 || err != nil { 48 | t.Fatal("Expected 2 bytes and no error") 49 | } 50 | for i := 0; i < 6; i++ { 51 | g := make([]byte, 2) 52 | n, err := f.Read(g) 53 | if n != 0 || err == nil { 54 | t.Fatal("Expected 0 bytes and error") 55 | } 56 | } 57 | } 58 | 59 | func TestFixedReaderJustRight(t *testing.T) { 60 | d := make([]byte, 10) 61 | for i := 0; i < len(d); i++ { 62 | d[i] = byte(i) 63 | } 64 | r := bytes.NewBuffer(d) 65 | f := NewFixedReader(r, 10) 66 | for i := 0; i < 5; i++ { 67 | g := make([]byte, 2) 68 | n, err := f.Read(g) 69 | if n != 2 || err != nil { 70 | t.Fatal("Expected 2 bytes and no error") 71 | } 72 | if g[0] != byte(i*2) || g[1] != byte(i*2+1) { 73 | t.Fatal("Expected number sequence") 74 | } 75 | } 76 | g := make([]byte, 2) 77 | n, err := f.Read(g) 78 | if n != 0 || err == nil { 79 | t.Fatal("Expected 0 bytes and error") 80 | } 81 | } 82 | 83 | func TestFixedReaderMultiRead(t *testing.T) { 84 | d := make([]byte, 20) 85 | for i := 0; i < len(d); i++ { 86 | d[i] = byte(i) 87 | } 88 | r := bytes.NewBuffer(d) 89 | f := NewFixedReader(r, 10) 90 | for i := 0; i < 10; i++ { 91 | g := make([]byte, 2) 92 | n, err := f.Read(g) 93 | if n != 2 || err != nil { 94 | t.Fatal("Expected 2 bytes and no error") 95 | } 96 | if g[0] != byte(i*2) || g[1] != byte(i*2+1) { 97 | t.Fatal("Expected number sequence") 98 | } 99 | } 100 | g := make([]byte, 2) 101 | n, err := f.Read(g) 102 | if n != 0 || err == nil { 103 | t.Fatal("Expected 0 bytes and error") 104 | } 105 | } 106 | 107 | func TestFixedReaderTailTruncated(t *testing.T) { 108 | d := make([]byte, 4) 109 | for i := 0; i < len(d); i++ { 110 | d[i] = byte(i) 111 | } 112 | r := bytes.NewBuffer(d) 113 | f := NewFixedReader(r, 3) 114 | g := make([]byte, 2) 115 | n, err := f.Read(g) 116 | if n != 2 || err != nil { 117 | t.Fatal("Expected 2 bytes and no error") 118 | } 119 | if g[0] != 0 || g[1] != 1 { 120 | t.Fatal("Expected number sequence") 121 | } 122 | g2 := make([]byte, 2) 123 | n2, err2 := f.Read(g2) 124 | if n2 != 1 || err2 != nil { 125 | t.Fatal("Expected 1 byte and no error") 126 | } 127 | if g2[0] != 2 { 128 | t.Fatal("Expected number sequence") 129 | } 130 | g3 := make([]byte, 2) 131 | n3, err3 := f.Read(g3) 132 | if n3 != 1 || err3 != nil { 133 | t.Fatal("Expected 1 byte and no error") 134 | } 135 | if g3[0] != 3 { 136 | t.Fatal("Expected number sequence") 137 | } 138 | } 139 | 140 | type closeableBuffer struct { 141 | reader io.Reader 142 | closed bool 143 | } 144 | 145 | func (b *closeableBuffer) Read(p []byte) (int, error) { 146 | if b.closed { 147 | return 0, io.EOF 148 | } else { 149 | return b.reader.Read(p) 150 | } 151 | } 152 | 153 | func (b *closeableBuffer) Close() error { 154 | b.closed = true 155 | return nil 156 | } 157 | 158 | func TestFixedReaderClose(t *testing.T) { 159 | d := make([]byte, 4) 160 | for i := 0; i < len(d); i++ { 161 | d[i] = byte(i) 162 | } 163 | r := &closeableBuffer{bytes.NewBuffer(d), false} 164 | f := NewFixedReader(r, 2) 165 | g := make([]byte, 2) 166 | n, err := f.Read(g) 167 | if n != 2 || err != nil { 168 | t.Fatal("Expected 2 bytes and no error") 169 | } 170 | if g[0] != 0 || g[1] != 1 { 171 | t.Fatal("Expected number sequence") 172 | } 173 | //goland:noinspection GoUnhandledErrorResult 174 | f.Close() 175 | g3 := make([]byte, 2) 176 | n3, err3 := f.Read(g3) 177 | if n3 != 0 || err3 == nil { 178 | t.Fatal("Expected 0 bytes and error") 179 | } 180 | } 181 | 182 | func TestFixedReaderCloseRemain(t *testing.T) { 183 | d := make([]byte, 6) 184 | for i := 0; i < len(d); i++ { 185 | d[i] = byte(i) 186 | } 187 | r := &closeableBuffer{bytes.NewBuffer(d), false} 188 | f := NewFixedReader(r, 4) 189 | g := make([]byte, 2) 190 | n, err := f.Read(g) 191 | if n != 2 || err != nil { 192 | t.Fatal("Expected 2 bytes and no error") 193 | } 194 | if g[0] != 0 || g[1] != 1 { 195 | t.Fatal("Expected number sequence") 196 | } 197 | //goland:noinspection GoUnhandledErrorResult 198 | f.Close() 199 | g2 := make([]byte, 2) 200 | n2, err2 := f.Read(g2) 201 | if n2 != 2 || err2 != nil { 202 | t.Fatal("Expected 2 bytes and no error") 203 | } 204 | if g2[0] != 2 || g2[1] != 3 { 205 | t.Fatal("Expected number sequence") 206 | } 207 | g3 := make([]byte, 2) 208 | n3, err3 := f.Read(g3) 209 | if n3 != 0 || err3 == nil { 210 | t.Fatal("Expected 0 bytes and error") 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /streaming/acl.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2016-2017 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package streaming 18 | 19 | import ( 20 | "fmt" 21 | "sync" 22 | ) 23 | 24 | // AccessController implements a connection broker that limits 25 | // the maximum number of concurrent connections. 26 | type AccessController struct { 27 | // maxconnections is a global limit on the number of connections. 28 | maxconnections uint 29 | // lock to protect the connection counter 30 | lock sync.Mutex 31 | // connections contains the number of active connections. 32 | // must be accessed atomically. 33 | connections uint 34 | // inhibit is a global connection inhibitor flag. 35 | inhibit bool 36 | } 37 | 38 | // NewAccessController creates a connection broker object that 39 | // handles access control according to the number of connected clients. 40 | func NewAccessController(maxconnections uint) *AccessController { 41 | return &AccessController{ 42 | maxconnections: maxconnections, 43 | } 44 | } 45 | 46 | // SetInhibit allows setting and clearing the inhibit flag. 47 | // If it is set, no further connections are accepted, irrespective of the 48 | // maxconnections limit. 49 | func (control *AccessController) SetInhibit(inhibit bool) { 50 | // protect concurrent access 51 | control.lock.Lock() 52 | control.inhibit = inhibit 53 | control.lock.Unlock() 54 | } 55 | 56 | // Accept accepts an incoming connection when the maximum number of open connections 57 | // has not been reached yet. 58 | func (control *AccessController) Accept(remoteaddr string, streamer *Streamer) bool { 59 | accept := false 60 | // protect concurrent access 61 | control.lock.Lock() 62 | // check if the limit is disabled or unreached, and no inhibit is set 63 | if !control.inhibit && (control.maxconnections == 0 || control.connections < control.maxconnections) { 64 | // and increase the counter 65 | control.connections++ 66 | accept = true 67 | } 68 | control.lock.Unlock() 69 | // print some info 70 | if accept { 71 | logger.Logkv( 72 | "event", eventAclAccepted, 73 | "remote", remoteaddr, 74 | "connections", control.connections, 75 | "max", control.maxconnections, 76 | "message", fmt.Sprintf("Accepted connection from %s, active=%d, max=%d", remoteaddr, control.connections, control.maxconnections), 77 | ) 78 | } else { 79 | logger.Logkv( 80 | "event", eventAclDenied, 81 | "remote", remoteaddr, 82 | "connections", control.connections, 83 | "max", control.maxconnections, 84 | "message", fmt.Sprintf("Denied connection from %s, active=%d, max=%d", remoteaddr, control.connections, control.maxconnections), 85 | ) 86 | } 87 | // return the result 88 | return accept 89 | } 90 | 91 | // Release decrements the open connections count. 92 | func (control *AccessController) Release(streamer *Streamer) { 93 | remove := false 94 | // protect concurrent access 95 | control.lock.Lock() 96 | if control.connections > 0 { 97 | // and decrease the counter 98 | control.connections-- 99 | remove = true 100 | } 101 | control.lock.Unlock() 102 | if remove { 103 | logger.Logkv( 104 | "event", eventAclRemoved, 105 | "connections", control.connections, 106 | "max", control.maxconnections, 107 | "message", fmt.Sprintf("Removed connection, active=%d, max=%d", control.connections, control.maxconnections), 108 | ) 109 | } else { 110 | logger.Logkv( 111 | "event", eventAclError, 112 | "error", errorAclNoConnection, 113 | "message", fmt.Sprintf("Error, no connection to remove"), 114 | ) 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /streaming/acl_test.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2017 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package streaming 18 | 19 | import ( 20 | "github.com/onitake/restreamer/util" 21 | "sync" 22 | "sync/atomic" 23 | "testing" 24 | ) 25 | 26 | type mockAclLogger struct { 27 | t *testing.T 28 | Stage string 29 | } 30 | 31 | func (l *mockAclLogger) Logd(lines ...util.Dict) { 32 | for _, line := range lines { 33 | l.t.Logf("%s: %v", l.Stage, line) 34 | } 35 | } 36 | 37 | func (l *mockAclLogger) Logkv(keyValues ...interface{}) { 38 | l.Logd(util.LogFunnel(keyValues)) 39 | } 40 | 41 | func TestAccessController00(t *testing.T) { 42 | l := &mockAclLogger{t, ""} 43 | 44 | l.Stage = "t00" 45 | c00 := NewAccessController(1) 46 | logger = l 47 | if !c00.Accept("c00b", nil) { 48 | t.Error("t00: Incorrectly refused connection on free access controller") 49 | } 50 | } 51 | 52 | func TestAccessController01(t *testing.T) { 53 | l := &mockAclLogger{t, ""} 54 | 55 | l.Stage = "t01" 56 | c01 := NewAccessController(1) 57 | logger = l 58 | c01.Accept("", nil) 59 | if c01.Accept("", nil) { 60 | t.Error("t01: Incorrectly accepted connection on full access controller") 61 | } 62 | } 63 | 64 | func TestAccessController02(t *testing.T) { 65 | l := &mockAclLogger{t, ""} 66 | 67 | l.Stage = "t02" 68 | c02 := NewAccessController(1) 69 | logger = l 70 | c02.Accept("c02a", nil) 71 | c02.Release(nil) 72 | if !c02.Accept("c02b", nil) { 73 | t.Error("t02: Incorrectly refused connection on freed access controller") 74 | } 75 | 76 | l.Stage = "t03" 77 | c03 := NewAccessController(100) 78 | logger = l 79 | w03 := &sync.WaitGroup{} 80 | w03.Add(100) 81 | var a03 int32 82 | for i := 0; i < 100; i++ { 83 | go func() { 84 | if c03.Accept("", nil) { 85 | atomic.AddInt32(&a03, 1) 86 | } 87 | w03.Done() 88 | }() 89 | } 90 | w03.Wait() 91 | if atomic.LoadInt32(&a03) != 100 { 92 | t.Error("t03: Premature accept failure") 93 | } 94 | if c03.Accept("", nil) { 95 | t.Error("t03: Incorrectly accepted connection on full controller") 96 | } 97 | 98 | l.Stage = "t04" 99 | w04 := &sync.WaitGroup{} 100 | w04.Add(50) 101 | var a04 int32 102 | for i := 0; i < 50; i++ { 103 | go func() { 104 | c03.Release(nil) 105 | atomic.AddInt32(&a04, 1) 106 | w04.Done() 107 | }() 108 | } 109 | w04.Wait() 110 | if atomic.LoadInt32(&a04) != 50 { 111 | t.Error("t04: Cannot release half of the connections") 112 | } 113 | 114 | l.Stage = "t05" 115 | w05 := &sync.WaitGroup{} 116 | w05.Add(100) 117 | var a05 int32 118 | for i := 0; i < 50; i++ { 119 | go func() { 120 | if c03.Accept("", nil) { 121 | atomic.AddInt32(&a05, 1) 122 | } 123 | w05.Done() 124 | }() 125 | } 126 | for i := 0; i < 50; i++ { 127 | go func() { 128 | c03.Release(nil) 129 | atomic.AddInt32(&a05, -1) 130 | w05.Done() 131 | }() 132 | } 133 | w05.Wait() 134 | if atomic.LoadInt32(&a05) != 0 { 135 | t.Error("t05: Cannot release/accept all connections") 136 | } 137 | 138 | l.Stage = "t06" 139 | for i := 0; i < 50; i++ { 140 | if !c03.Accept("", nil) { 141 | t.Error("t06: Failed to fill up all connection pool") 142 | } 143 | } 144 | if c02.Accept("", nil) { 145 | t.Error("t06: Incorrectly accepted connection on full controller") 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /streaming/connection.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2016-2017 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package streaming 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "github.com/onitake/restreamer/protocol" 23 | "net/http" 24 | "time" 25 | ) 26 | 27 | // Connection is a single active client connection. 28 | // 29 | // This is meant to be called directly from a ServeHTTP handler. 30 | // No separate thread is created. 31 | type Connection struct { 32 | // Queue is the per-connection packet queue 33 | Queue chan protocol.MpegTsPacket 34 | // ClientAddress is the remote client address 35 | ClientAddress string 36 | // the destination socket 37 | writer http.ResponseWriter 38 | // Closed is true if Serve was ended because of a closed channel. 39 | // This is simply there to avoid a double close. 40 | Closed bool 41 | // context contains the cached context object for this connection 42 | context context.Context 43 | } 44 | 45 | // NewConnection creates a new connection object. 46 | // To start sending data to a client, call Serve(). 47 | // 48 | // clientaddr should point to the remote address of the connecting client 49 | // and will be used for logging. 50 | func NewConnection(destination http.ResponseWriter, qsize int, clientaddr string, ctx context.Context) *Connection { 51 | conn := &Connection{ 52 | Queue: make(chan protocol.MpegTsPacket, qsize), 53 | ClientAddress: clientaddr, 54 | writer: destination, 55 | context: ctx, 56 | } 57 | return conn 58 | } 59 | 60 | // Serve starts serving data to a client, continuously feeding packets from the queue. 61 | // An optional preamble buffer can be passed that will be sent before streaming the live payload 62 | // (but after the HTTP response headers). 63 | func (conn *Connection) Serve(preamble []byte) { 64 | // set the content type (important) 65 | conn.writer.Header().Set("Content-Type", "video/mpeg") 66 | // a stream is always current 67 | conn.writer.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat)) 68 | // other headers to comply with the specs 69 | conn.writer.Header().Set("Accept-Range", "none") 70 | // suppress caching by intermediate proxies 71 | conn.writer.Header().Set("Cache-Control", "no-cache,no-store,no-transform") 72 | // use Add and Set to set more headers here 73 | // chunked mode should be on by default 74 | conn.writer.WriteHeader(http.StatusOK) 75 | // try to flush the header 76 | flusher, ok := conn.writer.(http.Flusher) 77 | if !ok { 78 | logger.Logkv( 79 | "event", eventConnectionError, 80 | "error", errorConnectionNotFlushable, 81 | "message", "ResponseWriter is not flushable!", 82 | ) 83 | } else { 84 | flusher.Flush() 85 | } 86 | logger.Logkv( 87 | "event", eventHeaderSent, 88 | "message", "Sent header", 89 | ) 90 | 91 | running := true 92 | 93 | // send the preamble 94 | if len(preamble) > 0 { 95 | _, err := conn.writer.Write(preamble) 96 | if err != nil { 97 | logger.Logkv( 98 | "event", eventConnectionClosed, 99 | "message", "Downstream connection closed during preamble", 100 | ) 101 | running = false 102 | } 103 | } 104 | 105 | // start reading packets 106 | for running { 107 | select { 108 | case packet, ok := <-conn.Queue: 109 | if ok { 110 | // packet received, log 111 | //log.Printf("Sending packet (length %d):\n%s\n", len(packet), hex.Dump(packet)) 112 | // send the packet out 113 | _, err := conn.writer.Write(packet) 114 | // NOTE we shouldn't flush here, to avoid swamping the kernel with syscalls. 115 | // see https://golang.org/pkg/net/http/?m=all#response.Write for details 116 | // on how Go buffers HTTP responses (hint: a 2KiB bufio and a 4KiB bufio) 117 | if err != nil { 118 | logger.Logkv( 119 | "event", eventConnectionClosed, 120 | "message", "Downstream connection closed", 121 | ) 122 | running = false 123 | } 124 | //log.Printf("Wrote packet of %d bytes\n", bytes) 125 | } else { 126 | // channel closed, exit 127 | logger.Logkv( 128 | "event", eventConnectionShutdown, 129 | "message", "Shutting down client connection", 130 | ) 131 | running = false 132 | conn.Closed = true 133 | } 134 | case <-conn.context.Done(): 135 | // connection closed while we were waiting for more data 136 | logger.Logkv( 137 | "event", eventConnectionClosedWait, 138 | "message", "Downstream connection closed (while waiting)", 139 | "error", fmt.Sprintf("%v", conn.context.Err()), 140 | ) 141 | running = false 142 | } 143 | } 144 | 145 | // we cannot drain the channel here, as it might not be closed yet. 146 | // better let our caller handle closure and draining. 147 | 148 | logger.Logkv( 149 | "event", eventConnectionDone, 150 | "message", "Streaming finished", 151 | ) 152 | } 153 | 154 | // ServeStreamError returns an appropriate error response to the client. 155 | func ServeStreamError(writer http.ResponseWriter, status int) { 156 | // set the content type (important) 157 | writer.Header().Set("Content-Type", "video/mpeg") 158 | // a stream is always current 159 | writer.Header().Set("Last-Modified", time.Now().UTC().Format(http.TimeFormat)) 160 | // other headers to comply with the specs 161 | writer.Header().Set("Accept-Range", "none") 162 | // suppress caching by intermediate proxies 163 | writer.Header().Set("Cache-Control", "no-cache,no-store,no-transform") 164 | // ...and the application-supplied status code 165 | writer.WriteHeader(http.StatusNotFound) 166 | } 167 | -------------------------------------------------------------------------------- /streaming/logger.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package streaming 18 | 19 | import ( 20 | "github.com/onitake/restreamer/util" 21 | ) 22 | 23 | const ( 24 | moduleStreaming = "streaming" 25 | // 26 | eventAclError = "error" 27 | eventAclAccepted = "accepted" 28 | eventAclDenied = "denied" 29 | eventAclRemoved = "removed" 30 | // 31 | errorAclNoConnection = "noconnection" 32 | // 33 | eventClientDebug = "debug" 34 | eventClientError = "error" 35 | eventClientRetry = "retry" 36 | eventClientConnecting = "connecting" 37 | eventClientConnectionLoss = "loss" 38 | eventClientConnectTimeout = "connect_timeout" 39 | eventClientOffline = "offline" 40 | eventClientStarted = "started" 41 | eventClientStopped = "stopped" 42 | eventClientOpenPath = "open_path" 43 | eventClientOpenHttp = "open_http" 44 | eventClientOpenTcp = "open_tcp" 45 | eventClientOpenDomain = "open_domain" 46 | eventClientPull = "pull" 47 | eventClientClosed = "closed" 48 | eventClientTimerStop = "timer_stop" 49 | eventClientTimerStopped = "timer_stopped" 50 | eventClientNoPacket = "nopacket" 51 | eventClientTimerKill = "killed" 52 | eventClientReadTimeout = "read_timeout" 53 | eventClientOpenUdp = "open_udp" 54 | eventClientOpenUdpMulticast = "open_multicast" 55 | eventClientOpenFork = "open_fork" 56 | // 57 | errorClientConnect = "connect" 58 | errorClientParse = "parse" 59 | errorClientInterface = "interface" 60 | errorClientSetBufferSize = "buffersize" 61 | errorClientClose = "close" 62 | errorClientStream = "stream" 63 | // 64 | eventConnectionDebug = "debug" 65 | eventConnectionError = "error" 66 | eventHeaderSent = "headersent" 67 | eventConnectionClosed = "closed" 68 | eventConnectionClosedWait = "closedwait" 69 | eventConnectionShutdown = "shutdown" 70 | eventConnectionDone = "done" 71 | // 72 | errorConnectionNotFlushable = "noflush" 73 | errorConnectionNoCloseNotify = "noclosenotify" 74 | // 75 | eventProxyError = "error" 76 | eventProxyStart = "start" 77 | eventProxyShutdown = "shutdown" 78 | eventProxyRequest = "request" 79 | eventProxyOffline = "offline" 80 | eventProxyFetch = "fetch" 81 | eventProxyFetched = "fetched" 82 | eventProxyRequesting = "requesting" 83 | eventProxyRequestDone = "requestdone" 84 | eventProxyReplyNotChanged = "replynotchanged" 85 | eventProxyReplyContent = "replycontent" 86 | eventProxyStale = "stale" 87 | eventProxyReturn = "return" 88 | // 89 | errorProxyInvalidUrl = "invalidurl" 90 | errorProxyNoLength = "nolength" 91 | errorProxyLimitExceeded = "limitexceeded" 92 | errorProxyShortRead = "shortread" 93 | errorProxyGet = "get" 94 | errorProxyWrite = "write" 95 | errorProxyHash = "hash" 96 | eventStreamerError = "error" 97 | eventStreamerQueueStart = "queuestart" 98 | eventStreamerStart = "start" 99 | eventStreamerStop = "stop" 100 | eventStreamerClientAdd = "add" 101 | eventStreamerClientRemove = "remove" 102 | eventStreamerStreaming = "streaming" 103 | eventStreamerClosed = "closed" 104 | eventStreamerInhibit = "inhibit" 105 | eventStreamerAllow = "allow" 106 | // 107 | errorStreamerInvalidCommand = "invalidcmd" 108 | errorStreamerPoolFull = "poolfull" 109 | errorStreamerOffline = "offline" 110 | ) 111 | 112 | var logger = util.NewGlobalModuleLogger(moduleStreaming, nil) 113 | -------------------------------------------------------------------------------- /streaming/manager.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2017 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package streaming 18 | 19 | // StateManager maintains a list of disconnectable objects, sending them 20 | // a notification whenever state changes. 21 | // 22 | // After connection closure has been notified, the list is cleared and 23 | // further notifications have no effect. 24 | type StateManager struct { 25 | notifiables map[chan<- bool]bool 26 | } 27 | 28 | // NewStateManager creates a new state manager. 29 | // 30 | // Register notification channels with Register(), and submit state changes 31 | // with Notify(). Channels can be removed later with Unregister(). 32 | // After notify has been called, the list of registered channels is cleared. 33 | func NewStateManager() *StateManager { 34 | return &StateManager{ 35 | notifiables: make(map[chan<- bool]bool), 36 | } 37 | } 38 | 39 | // Register registers a new notification channel. 40 | // 41 | // It is not possible to register a channel twice. 42 | // Any additional registrations will be ignored. 43 | func (manager *StateManager) Register(channel chan<- bool) { 44 | manager.notifiables[channel] = true 45 | } 46 | 47 | // Unregister removes a registered channel. 48 | // 49 | // If this channel was not registered previously, no action is taken. 50 | func (manager *StateManager) Unregister(channel chan<- bool) { 51 | delete(manager.notifiables, channel) 52 | } 53 | 54 | // Notify sends a state change to all registered notification channels and clears the list. 55 | func (manager *StateManager) Notify() { 56 | for notifiable := range manager.notifiables { 57 | notifiable <- true 58 | } 59 | manager.notifiables = make(map[chan<- bool]bool) 60 | } 61 | -------------------------------------------------------------------------------- /streaming/proxy.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2017 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package streaming 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "github.com/onitake/restreamer/auth" 23 | "github.com/onitake/restreamer/metrics" 24 | "hash/fnv" 25 | "io" 26 | "mime" 27 | "net/http" 28 | "net/url" 29 | "os" 30 | "path" 31 | "strconv" 32 | "time" 33 | ) 34 | 35 | const ( 36 | proxyBufferSize = 1024 37 | proxyDefaultLimit = 10 * 1024 * 1024 38 | proxyDefaultMime = "application/octet-stream" 39 | proxyFetchQueue = 10 40 | ) 41 | 42 | var ( 43 | // headerList is a list of HTTP headers that are allowed to be sent through the proxy. 44 | headerList = []string{ 45 | "Content-Type", 46 | } 47 | ErrNoLength = errors.New("restreamer: Fetching of remote resource with unknown length not supported") 48 | ErrLimitExceeded = errors.New("restreamer: Resource too large for cache") 49 | ErrShortRead = errors.New("restreamer: Short read, not all data was transferred in one go") 50 | ) 51 | 52 | // fetchableResource contains a cachable resource and its metadata. 53 | // This encapsulated type is used to ship data between the fetcher and the server. 54 | type fetchableResource struct { 55 | // contents 56 | data []byte 57 | // upstream status code 58 | statusCode int 59 | // resource hash received from upstream, or computed 60 | etag string 61 | // upstream headers 62 | header http.Header 63 | // last update time (for aging) 64 | updated time.Time 65 | } 66 | 67 | // Proxy implements a caching HTTP proxy. 68 | type Proxy struct { 69 | // the upstream URL (file/http/https) 70 | url *url.URL 71 | // HTTP client timeout 72 | timeout time.Duration 73 | // the cache time 74 | stale time.Duration 75 | // maximum size of remote resource 76 | limit int64 77 | // fetcher data request channel 78 | fetcher chan chan<- *fetchableResource 79 | // the cached resource 80 | // NOTE do not access this dirctly, use the fetcher instead 81 | resource *fetchableResource 82 | // a channel to signal shutdown to the fetcher 83 | // this channel should never be written to - shutdown is signalled by closing the channel 84 | shutdown chan struct{} 85 | // the global stats collector 86 | stats metrics.Statistics 87 | // auth is an authentication verifier for client requests 88 | auth auth.Authenticator 89 | } 90 | 91 | // NewProxy constructs a new HTTP proxy. 92 | // The upstream resource is not fetched until the first request. 93 | // If cache is non-zero, the resource will be evicted from memory after these 94 | // number of seconds. If it is zero, the resource will be fetched from upstream 95 | // every time it is requested. 96 | // timeout sets the upstream HTTP connection timeout. 97 | func NewProxy(uri string, timeout uint, cache uint, auth auth.Authenticator) (*Proxy, error) { 98 | parsed, err := url.Parse(uri) 99 | if err != nil { 100 | return nil, err 101 | } 102 | 103 | return &Proxy{ 104 | url: parsed, 105 | timeout: time.Duration(timeout) * time.Second, 106 | stale: time.Duration(cache) * time.Second, 107 | // TODO make this configurable 108 | limit: proxyDefaultLimit, 109 | // TODO make queue length configurable 110 | fetcher: make(chan chan<- *fetchableResource, proxyFetchQueue), 111 | shutdown: make(chan struct{}), 112 | resource: nil, 113 | stats: &metrics.DummyStatistics{}, 114 | auth: auth, 115 | }, nil 116 | } 117 | 118 | // SetStatistics assigns a stats collector. 119 | func (proxy *Proxy) SetStatistics(stats metrics.Statistics) { 120 | proxy.stats = stats 121 | } 122 | 123 | // Get opens a remote or local resource specified by URL and returns a reader, 124 | // upstream HTTP headers, an HTTP status code and the resource data length, or -1 if no length is available. 125 | // Local resources contain guessed data. 126 | // Supported protocols: file, http and https. 127 | func Get(url *url.URL, timeout time.Duration) (reader io.Reader, header http.Header, status int, length int64, err error) { 128 | status = http.StatusNotFound 129 | reader = nil 130 | header = make(http.Header) 131 | err = nil 132 | length = 0 133 | 134 | if url.Scheme == "file" { 135 | reader, err = os.Open(url.Path) 136 | if err == nil { 137 | status = http.StatusOK 138 | 139 | // guess the size 140 | info, err2 := os.Stat(url.Path) 141 | if err2 == nil { 142 | length = info.Size() 143 | } else { 144 | // we can't stat, so the length is indefinite... 145 | length = -1 146 | } 147 | 148 | // guess the mime type 149 | mtype := mime.TypeByExtension(path.Ext(url.Path)) 150 | if mtype != "" { 151 | header.Set("Content-Type", mtype) 152 | } 153 | } else { 154 | if err == os.ErrPermission { 155 | status = http.StatusForbidden 156 | } else { 157 | status = http.StatusNotFound 158 | } 159 | } 160 | } else { 161 | getter := &http.Client{ 162 | Timeout: timeout, 163 | } 164 | response, err := getter.Get(url.String()) 165 | if err == nil { 166 | status = response.StatusCode 167 | reader = response.Body 168 | length = response.ContentLength 169 | header = response.Header 170 | } else { 171 | // TODO: check if we have a timeout and return other status codes here 172 | status = http.StatusBadGateway 173 | } 174 | } 175 | 176 | return reader, header, status, length, err 177 | } 178 | 179 | // Start launches the fetcher thread. 180 | // This should only be called once. 181 | func (proxy *Proxy) Start() { 182 | logger.Logkv( 183 | "event", eventProxyStart, 184 | "message", "Starting fetcher", 185 | ) 186 | go proxy.fetch() 187 | } 188 | 189 | // Shutdown stops the fetcher thread. 190 | func (proxy *Proxy) Shutdown() { 191 | logger.Logkv( 192 | "event", eventProxyShutdown, 193 | "message", "Shutting down fetcher", 194 | ) 195 | close(proxy.shutdown) 196 | } 197 | 198 | // fetch waits for fetch requests and handles them one-by-one. 199 | // If the resource is already cached and not stale, it replies very quickly. 200 | // Performance impact should be minimal in this case. 201 | // Blocks while the resource is fetched. 202 | func (proxy *Proxy) fetch() { 203 | running := true 204 | for running { 205 | select { 206 | case <-proxy.shutdown: 207 | running = false 208 | case request := <-proxy.fetcher: 209 | logger.Logkv( 210 | "event", eventProxyRequest, 211 | "message", "Handling request", 212 | "resource", proxy.resource, 213 | ) 214 | // verify if we need to refetch 215 | now := time.Now() 216 | if proxy.resource == nil || now.Sub(proxy.resource.updated) > proxy.stale { 217 | // stale, cache first 218 | logger.Logkv( 219 | "event", eventProxyStale, 220 | "message", "Resource is stale", 221 | ) 222 | proxy.resource = proxy.cache() 223 | } 224 | // and return 225 | logger.Logkv( 226 | "event", eventProxyReturn, 227 | "message", "Returning resource", 228 | ) 229 | request <- proxy.resource 230 | } 231 | } 232 | logger.Logkv( 233 | "event", eventProxyOffline, 234 | "message", "Fetcher is offline", 235 | ) 236 | } 237 | 238 | // Etag calculates a hash value of data and returns it as a hex string. 239 | // Suitable for HTTP Etags. 240 | func Etag(data []byte) string { 241 | // 64-bit FNV-1a checksum of the data, formatted as a hex string 242 | hash := fnv.New64a() 243 | if _, err := hash.Write(data); err != nil { 244 | logger.Logkv( 245 | "event", eventProxyError, 246 | "error", errorProxyHash, 247 | "message", err.Error(), 248 | ) 249 | return "" 250 | } 251 | return fmt.Sprintf("%016x", hash.Sum64()) 252 | } 253 | 254 | // cache fetches the remote resource into memory. 255 | // Does not return errors. Instead, the cached resource contains a suitable return code and error content. 256 | func (proxy *Proxy) cache() *fetchableResource { 257 | logger.Logkv( 258 | "event", eventProxyFetch, 259 | "message", "Fetching resource from upstream", 260 | ) 261 | 262 | // fetch from upstream 263 | getter, header, status, length, err := Get(proxy.url, proxy.timeout) 264 | if err != nil { 265 | logger.Logkv( 266 | "event", eventProxyError, 267 | "error", errorProxyGet, 268 | "message", err.Error(), 269 | ) 270 | } 271 | 272 | // construct the return value 273 | res := &fetchableResource{ 274 | header: header, 275 | statusCode: status, 276 | } 277 | 278 | if err == nil { 279 | // verify the length 280 | if length < 0 { 281 | // TODO maybe allow caching of resources without length? 282 | err = ErrNoLength 283 | logger.Logkv( 284 | "event", eventProxyError, 285 | "error", errorProxyNoLength, 286 | "message", ErrNoLength, 287 | ) 288 | res.statusCode = http.StatusBadGateway 289 | res.data = []byte(http.StatusText(res.statusCode)) 290 | res.header = make(http.Header) 291 | } else if length > proxy.limit { 292 | err = ErrLimitExceeded 293 | logger.Logkv( 294 | "event", eventProxyError, 295 | "error", errorProxyLimitExceeded, 296 | "message", ErrLimitExceeded, 297 | ) 298 | res.statusCode = http.StatusBadGateway 299 | res.data = []byte(http.StatusText(res.statusCode)) 300 | res.header = make(http.Header) 301 | } 302 | } 303 | 304 | if err == nil { 305 | res.data = make([]byte, length) 306 | 307 | // fetch the data 308 | // TODO we should probably read in chunks 309 | var bytes int 310 | bytes, err = getter.Read(res.data) 311 | 312 | if err == nil && int64(bytes) != length { 313 | err = ErrShortRead 314 | logger.Logkv( 315 | "event", eventProxyError, 316 | "error", errorProxyShortRead, 317 | "message", ErrShortRead, 318 | "length", length, 319 | "received", bytes, 320 | ) 321 | res.data = res.data[:bytes] 322 | } 323 | } 324 | 325 | res.updated = time.Now() 326 | // calculate the content hash 327 | res.etag = Etag(res.data) 328 | 329 | logger.Logkv( 330 | "event", eventProxyFetched, 331 | "message", "Fetched resource from upstream", 332 | "etag", res.etag, 333 | "length", len(res.data), 334 | "status", res.statusCode, 335 | ) 336 | 337 | return res 338 | } 339 | 340 | // ServeHTTP handles an incoming connection. 341 | // Satisfies the http.Handler interface, so it can be used in an HTTP server. 342 | func (proxy *Proxy) ServeHTTP(writer http.ResponseWriter, request *http.Request) { 343 | // fail-fast: verify that this user can access this resource first 344 | if !auth.HandleHttpAuthentication(proxy.auth, request, writer) { 345 | return 346 | } 347 | 348 | // create a return channel for the fetcher 349 | fetchable := make(chan *fetchableResource) 350 | 351 | // request and wait for completion 352 | // since the channels are unbuffered, they will block on read/write 353 | // timeout must happen in the fetcher! 354 | logger.Logkv( 355 | "event", eventProxyRequesting, 356 | "message", "Handling incoming request", 357 | ) 358 | proxy.fetcher <- fetchable 359 | logger.Logkv( 360 | "event", "waiting", 361 | "message", "Waiting for response", 362 | ) 363 | res := <-fetchable 364 | close(fetchable) 365 | logger.Logkv( 366 | "event", eventProxyRequestDone, 367 | "message", "Request complete", 368 | ) 369 | 370 | // copy (appropriate) headers 371 | for _, key := range headerList { 372 | value := res.header.Get(key) 373 | if value != "" { 374 | writer.Header().Set(key, value) 375 | } 376 | } 377 | 378 | // headers for cached data 379 | writer.Header().Set("ETag", res.etag) 380 | // TODO maybe use the actual resource stale time here (Since()) 381 | // TODO no-cache for errors! 382 | writer.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", int(proxy.stale.Seconds()))) 383 | 384 | // verify if ETag has matched 385 | if res.etag != "" && request.Header.Get("If-None-Match") == res.etag { 386 | logger.Logkv( 387 | "event", eventProxyReplyNotChanged, 388 | "message", "Returning 304", 389 | ) 390 | // send only a 304 391 | writer.WriteHeader(http.StatusNotModified) 392 | // no content here 393 | } else { 394 | logger.Logkv( 395 | "event", eventProxyReplyContent, 396 | "message", "Returning updated content", 397 | "updated", res.updated, 398 | ) 399 | // otherwise, send updated data 400 | writer.Header().Set("Content-Length", strconv.Itoa(len(res.data))) 401 | writer.WriteHeader(res.statusCode) 402 | // and push the content 403 | if _, err := writer.Write(res.data); err != nil { 404 | logger.Logkv( 405 | "event", eventProxyError, 406 | "error", errorProxyWrite, 407 | "message", err.Error(), 408 | ) 409 | } 410 | } 411 | } 412 | -------------------------------------------------------------------------------- /streaming/proxy_test.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2017 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package streaming 18 | 19 | import ( 20 | "encoding/hex" 21 | "github.com/onitake/restreamer/auth" 22 | "github.com/onitake/restreamer/configuration" 23 | "github.com/onitake/restreamer/util" 24 | "net/http" 25 | "net/url" 26 | "testing" 27 | "time" 28 | ) 29 | 30 | type mockProxyLogger struct { 31 | t *testing.T 32 | Closed chan bool 33 | } 34 | 35 | func (l *mockProxyLogger) Logd(lines ...util.Dict) { 36 | shutdown := false 37 | for _, line := range lines { 38 | l.t.Logf("%v", line) 39 | if line["event"] == eventProxyOffline { 40 | shutdown = true 41 | } 42 | } 43 | if shutdown { 44 | l.Closed <- true 45 | } 46 | } 47 | 48 | func (l *mockProxyLogger) Logkv(keyValues ...interface{}) { 49 | l.Logd(util.LogFunnel(keyValues)) 50 | } 51 | 52 | type Logger interface { 53 | Log(args ...interface{}) 54 | Logf(format string, args ...interface{}) 55 | } 56 | 57 | type MockWriter struct { 58 | header http.Header 59 | log Logger 60 | } 61 | 62 | func newMockWriter(logger Logger) *MockWriter { 63 | return &MockWriter{ 64 | header: make(http.Header), 65 | log: logger, 66 | } 67 | } 68 | func (writer *MockWriter) Header() http.Header { 69 | return writer.header 70 | } 71 | func (writer *MockWriter) Write(data []byte) (int, error) { 72 | writer.log.Log("Write data:") 73 | writer.log.Log(hex.Dump(data)) 74 | return len(data), nil 75 | } 76 | func (writer *MockWriter) WriteHeader(status int) { 77 | writer.log.Logf("Write header, status code %d:", status) 78 | writer.log.Log(writer.header) 79 | } 80 | 81 | func testWithProxy(t *testing.T, l *mockProxyLogger, proxy *Proxy) { 82 | logger = l 83 | writer := newMockWriter(t) 84 | writer.log = t 85 | uri, _ := url.ParseRequestURI("http://host/test.txt") 86 | request := &http.Request{ 87 | Method: "GET", 88 | URL: uri, 89 | Proto: "HTTP/1.0", 90 | ProtoMajor: 1, 91 | ProtoMinor: 0, 92 | Header: make(http.Header), 93 | } 94 | proxy.Start() 95 | proxy.ServeHTTP(writer, request) 96 | proxy.Shutdown() 97 | select { 98 | case <-l.Closed: 99 | if len(l.Closed) > 0 { 100 | t.Fatalf("Multiple shutdown messages received") 101 | } 102 | case <-time.After(5 * time.Second): 103 | t.Errorf("Timeout waiting for proxy shutdown") 104 | } 105 | } 106 | 107 | func TestProxy(t *testing.T) { 108 | l := &mockProxyLogger{t, make(chan bool)} 109 | 110 | authenticator := auth.NewAuthenticator(configuration.Authentication{}, nil) 111 | 112 | direct, _ := NewProxy("file:///tmp/test.txt", 10, 0, authenticator) 113 | testWithProxy(t, l, direct) 114 | 115 | cached, _ := NewProxy("file:///tmp/test.txt", 10, 1, authenticator) 116 | testWithProxy(t, l, cached) 117 | } 118 | -------------------------------------------------------------------------------- /util/atomic.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2017 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "sync/atomic" 21 | ) 22 | 23 | // AtomicBool is a type placeholder for atomic operations on booleans. 24 | // 25 | // It shares its type with int32 and relies on the atomic.*Int32 functions 26 | // to implement the underlying atomic operations. 27 | // 28 | // CompareAndSwapBool, LoadBool, StoreBool and SwapBool are provided as 29 | // convenience frontends. They automatically cast between int32 and bool 30 | // types and should hide the nitty-gritty details. 31 | type AtomicBool int32 32 | 33 | const ( 34 | AtomicFalse AtomicBool = 0 35 | AtomicTrue AtomicBool = 1 36 | ) 37 | 38 | // ToAtomicBool converts a bool value to the corresponding AtomicBool value 39 | func ToAtomicBool(value bool) AtomicBool { 40 | if value { 41 | return AtomicTrue 42 | } else { 43 | return AtomicFalse 44 | } 45 | } 46 | 47 | // CompareAndSwapBool executes the compare-and-swap operation for a boolean value. 48 | func CompareAndSwapBool(addr *AtomicBool, old, new bool) (swapped bool) { 49 | var o int32 50 | if old { 51 | o = int32(AtomicTrue) 52 | } 53 | var n int32 54 | if new { 55 | n = int32(AtomicTrue) 56 | } 57 | return atomic.CompareAndSwapInt32((*int32)(addr), o, n) 58 | } 59 | 60 | // LoadBool atomically loads *addr. 61 | func LoadBool(addr *AtomicBool) (val bool) { 62 | var r = atomic.LoadInt32((*int32)(addr)) 63 | return r != int32(AtomicFalse) 64 | } 65 | 66 | // StoreBool atomically stores val into *addr. 67 | func StoreBool(addr *AtomicBool, val bool) { 68 | var v int32 69 | if val { 70 | v = int32(AtomicTrue) 71 | } 72 | atomic.StoreInt32((*int32)(addr), v) 73 | } 74 | 75 | // SwapBool atomically stores new into *addr and returns the previous *addr value. 76 | func SwapBool(addr *AtomicBool, new bool) (old bool) { 77 | var n int32 78 | if new { 79 | n = int32(AtomicTrue) 80 | } 81 | var r = atomic.SwapInt32((*int32)(addr), n) 82 | return r != int32(AtomicFalse) 83 | } 84 | -------------------------------------------------------------------------------- /util/atomic_test.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "testing" 21 | ) 22 | 23 | func TestValue(t *testing.T) { 24 | var z AtomicBool 25 | if z != AtomicFalse { 26 | t.Error("Invalid AtomicBool default value, expected AtomicFalse") 27 | } 28 | a := AtomicFalse 29 | // DO NOT remove, this is a sanity check 30 | //goland:noinspection GoBoolExpressions 31 | if a != AtomicFalse { 32 | t.Error("Invalid AtomicBool value, expected AtomicFalse") 33 | } 34 | c := AtomicTrue 35 | // DO NOT remove, this is a sanity check 36 | //goland:noinspection GoBoolExpressions 37 | if c != AtomicTrue { 38 | t.Error("Invalid AtomicBool value, expected AtomicTrue") 39 | } 40 | } 41 | 42 | func TestLoad(t *testing.T) { 43 | b := AtomicFalse 44 | if LoadBool(&b) != false { 45 | t.Error("Invalid AtomicBool value, expected false") 46 | } 47 | d := AtomicTrue 48 | if LoadBool(&d) != true { 49 | t.Error("Invalid AtomicBool value, expected true") 50 | } 51 | } 52 | 53 | func TestLoadStore(t *testing.T) { 54 | var e AtomicBool 55 | StoreBool(&e, false) 56 | if e != AtomicFalse || LoadBool(&e) != false { 57 | t.Error("Invalid AtomicBool value, expected false") 58 | } 59 | var f AtomicBool 60 | StoreBool(&f, true) 61 | if f != AtomicTrue || LoadBool(&f) != true { 62 | t.Error("Invalid AtomicBool value, expected true") 63 | } 64 | } 65 | 66 | func TestConvert(t *testing.T) { 67 | g := ToAtomicBool(false) 68 | if g != AtomicFalse || LoadBool(&g) != false { 69 | t.Error("Invalid AtomicBool value, expected false") 70 | } 71 | h := ToAtomicBool(true) 72 | if h != AtomicTrue || LoadBool(&h) != true { 73 | t.Error("Invalid AtomicBool value, expected true") 74 | } 75 | } 76 | 77 | func TestCompareAndSwap(t *testing.T) { 78 | i := AtomicFalse 79 | if !CompareAndSwapBool(&i, false, false) || i != AtomicFalse { 80 | t.Error("Invalid AtomicBool value, expected swap with false") 81 | } 82 | j := AtomicFalse 83 | if !CompareAndSwapBool(&j, false, true) || j != AtomicTrue { 84 | t.Error("Invalid AtomicBool value, expected swap with true") 85 | } 86 | k := AtomicFalse 87 | if CompareAndSwapBool(&k, true, false) || k != AtomicFalse { 88 | t.Error("Invalid AtomicBool value, expected no swap with false") 89 | } 90 | l := AtomicFalse 91 | if CompareAndSwapBool(&l, true, true) || l != AtomicFalse { 92 | t.Error("Invalid AtomicBool value, expected no swap with true") 93 | } 94 | m := AtomicTrue 95 | if CompareAndSwapBool(&m, false, false) || m != AtomicTrue { 96 | t.Error("Invalid AtomicBool value, expected no swap with false") 97 | } 98 | n := AtomicTrue 99 | if CompareAndSwapBool(&n, false, true) || n != AtomicTrue { 100 | t.Error("Invalid AtomicBool value, expected no swap with true") 101 | } 102 | o := AtomicTrue 103 | if !CompareAndSwapBool(&o, true, false) || o != AtomicFalse { 104 | t.Error("Invalid AtomicBool value, expected swap with false") 105 | } 106 | p := AtomicTrue 107 | if !CompareAndSwapBool(&p, true, true) || p != AtomicTrue { 108 | t.Error("Invalid AtomicBool value, expected swap with true") 109 | } 110 | } 111 | 112 | func TestSwap(t *testing.T) { 113 | q := AtomicFalse 114 | if SwapBool(&q, false) != false && q == AtomicFalse { 115 | t.Error("Invalid AtomicBool value, expected false,false") 116 | } 117 | r := AtomicFalse 118 | if SwapBool(&r, true) != false && r == AtomicTrue { 119 | t.Error("Invalid AtomicBool value, expected false,true") 120 | } 121 | s := AtomicTrue 122 | if SwapBool(&s, false) != true && s == AtomicFalse { 123 | t.Error("Invalid AtomicBool value, expected true,false") 124 | } 125 | u := AtomicTrue 126 | if SwapBool(&u, true) != true && u == AtomicTrue { 127 | t.Error("Invalid AtomicBool value, expected true,true") 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /util/errors.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package util 18 | 19 | // New creates an error that also has an error code. 20 | func New(text string, code int) error { 21 | return &ErrorStringCode{ 22 | text, 23 | code, 24 | } 25 | } 26 | 27 | // ErrorStringCode is an error that contains both a message and a code. 28 | type ErrorStringCode struct { 29 | Message string 30 | Code int 31 | } 32 | 33 | // Error returns the error message. 34 | func (err *ErrorStringCode) Error() string { 35 | return err.Message 36 | } 37 | -------------------------------------------------------------------------------- /util/log.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2017 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "io" 23 | "os" 24 | "os/signal" 25 | "time" 26 | ) 27 | 28 | const ( 29 | // signalQueueLength specifies the maximum number of unhandled control signals 30 | signalQueueLength int = 100 31 | // logQueueLength specifies the maximum number of unwritten log messages 32 | logQueueLength int = 100 33 | // timeFormat configures the format for time strings 34 | timeFormat string = time.RFC3339 35 | // hupSignal is a signal identifier for a "reopen the log" notification. 36 | // Distinct from UserSignal. 37 | hupSignal = internalSignal("HUP") 38 | // shutdownSignal is a signal identifier for a "stop logging" notification. 39 | shutdownSignal = internalSignal("SDN") 40 | // srcFileUnknown is the string stored in KeySrcFile when the caller's file name cannot be determined 41 | srcFileUnknown string = "" 42 | // srcLineUnknown is the number stored in KeySrcLine when the caller's source code line number cannot be determined 43 | srcLineUnknown int = 0 44 | // 45 | // KeyModule is the standard key for a user-defined module name 46 | KeyModule string = "module" 47 | // KeyTime is the standard key for the time stamp when the log entry was generated 48 | KeyTime string = "time" 49 | ) 50 | 51 | var ( 52 | globalStandardLogger = MultiLogger{ 53 | &ConsoleLogger{}, 54 | } 55 | ) 56 | 57 | type internalSignal string 58 | 59 | func (s internalSignal) Signal() {} 60 | func (s internalSignal) String() string { 61 | return string(s) 62 | } 63 | 64 | // Dict is a generic string:any dictionary type, for more convenience 65 | // when creating structured logs. 66 | type Dict map[string]interface{} 67 | 68 | // Logger is an interface for loggers that can generate JSON-formatted logs 69 | // from structured data. 70 | // 71 | // It is recommended that logs follow some general guidelines, like adding 72 | // a reference to the module that generated them, or a flag to differentiate 73 | // various kinds of log messages. 74 | // 75 | // See ModuleLogger for an easy way to do this. 76 | // 77 | // Examples: 78 | // { "module": "client", "type": "connect", "stream": "http://test.url/" } 79 | // { "module": "connection", "type": "connect", "source": "1.2.3.4:49999", "url": "/stream" } 80 | // { "module": "connection", "type": "disconnect", "source": "1.2.3.4:49999", "url": "/stream", "duration": 61, "bytes": 12087832 } 81 | type Logger interface { 82 | // Logd writes one or multiple data structures to the log represented by this logger. 83 | // Each argument is processed through json.Marshal and generates one line in the log. 84 | // 85 | // Example usage: 86 | // logger.Logd(Dict{ "key": "value" }, Dict{ "key": "value2" }) 87 | Logd(lines ...Dict) 88 | // Logkv is a convenience function that sends a single log line to the logger. 89 | // The arguments are alternating key -> value pairs that are assembled into a dictionary. 90 | // 91 | // This function is slightliy easier to use in many cases, because it doesn't require 92 | // in-place creation of data structures. But it's usually also slower. 93 | // Simply call: 94 | // logger.Log("key", "value", "key2", 10) 95 | Logkv(keyValues ...interface{}) 96 | } 97 | 98 | // LogFunnel is a simple helper for converting variadic key-value pairs into a dictionary 99 | func LogFunnel(keyValues []interface{}) Dict { 100 | d := make(Dict) 101 | // we need an even number of additional args 102 | for i := 0; i+1 < len(keyValues); i += 2 { 103 | k, ok := keyValues[i].(string) 104 | // ignore if the key is not a string 105 | if ok { 106 | d[k] = keyValues[i+1] 107 | } 108 | } 109 | return d 110 | } 111 | 112 | // NewGlobalModuleLogger creates a global logger for the current package and 113 | // connects it to the global standard logger. 114 | // 115 | // The default output for standard logger is a JSON log with added timestamps, 116 | // but this can be changed by calling SetGlobalStandardLogger. 117 | // 118 | // An optional dictionary argument allows specifying additional keys that are 119 | // added to every log line. Can be nil if you don't need it. 120 | func NewGlobalModuleLogger(module string, dict Dict) Logger { 121 | more := make(Dict) 122 | for k, v := range dict { 123 | more[k] = v 124 | } 125 | more[KeyModule] = module 126 | logger := &ModuleLogger{ 127 | Logger: globalStandardLogger, 128 | Defaults: more, 129 | } 130 | return logger 131 | } 132 | 133 | // SetGlobalStandardLogger assigns a new backing logger to the global standard logger 134 | // 135 | // A reference to the old logger is returned. 136 | func SetGlobalStandardLogger(logger Logger) Logger { 137 | old := globalStandardLogger[0] 138 | globalStandardLogger[0] = logger 139 | return old 140 | } 141 | 142 | // ModuleLogger encapsulates default values for a JSON log. 143 | // 144 | // This simplifies log calls, as values like the current module can be 145 | // initialised once, and they will be reused on every log call. 146 | // 147 | // If AddTimestamp is true, each log line will contain the key 'time' with the 148 | // current time in RFC 3339 format (ex.: 2006-01-02T15:04:05Z07:00). 149 | // 150 | // The keys in the Defaults dictionary will always be added. 151 | // 152 | // It is highly recommended to add at least a 'module' key to the defaults, 153 | // so the logging module can be identified. 154 | type ModuleLogger struct { 155 | // Logger is the backing logger to send log lines to. 156 | Logger Logger 157 | // Defaults is a dictionary containing default keys. 158 | // It is highly recommended to add any immutable data here, 159 | // in particular the key 'module' with a unique name for the module sending the log 160 | // will be very useful. 161 | Defaults Dict 162 | // AddTimestamp determines if a "time" value with the current time in RFC 3339 format 163 | // is added to the dictionary before it is passed to the underlying logger. 164 | AddTimestamp bool 165 | } 166 | 167 | // Logd adds predefined values to each log line and writes it to the encapsulated log. 168 | func (logger *ModuleLogger) Logd(lines ...Dict) { 169 | proclines := make([]Dict, len(lines)) 170 | for i, line := range lines { 171 | processed := make(Dict) 172 | for key, value := range logger.Defaults { 173 | processed[key] = value 174 | } 175 | if logger.AddTimestamp { 176 | processed[KeyTime] = time.Now().Format(timeFormat) 177 | } 178 | for key, value := range line { 179 | processed[key] = value 180 | } 181 | proclines[i] = processed 182 | } 183 | logger.Logger.Logd(proclines...) 184 | } 185 | 186 | func (logger *ModuleLogger) Logkv(keyValues ...interface{}) { 187 | logger.Logd(LogFunnel(keyValues)) 188 | } 189 | 190 | // DummyLogger is a logger placeholder that doesn't actually log anything. 191 | // Just a placeholder for the real big boy loggers. 192 | type DummyLogger struct{} 193 | 194 | func (*DummyLogger) Logd(lines ...Dict) {} 195 | func (*DummyLogger) Logkv(keyValues ...interface{}) {} 196 | 197 | // MultiLogger logs to several backend loggers at once. 198 | type MultiLogger []Logger 199 | 200 | // Logd writes the same log lines to all backing loggers. 201 | func (logger MultiLogger) Logd(lines ...Dict) { 202 | for _, backer := range logger { 203 | backer.Logd(lines...) 204 | } 205 | } 206 | 207 | func (logger MultiLogger) Logkv(keyValues ...interface{}) { 208 | logger.Logd(LogFunnel(keyValues)) 209 | } 210 | 211 | // ConsoleLogger is a simple logger that prints to stdout. 212 | type ConsoleLogger struct{} 213 | 214 | // Logd writes a log line to stdout. 215 | // 216 | // Your best bet if you don't want/need a full-blown file logging queue with 217 | // signal-initiated reopening or a central logging server. 218 | func (*ConsoleLogger) Logd(lines ...Dict) { 219 | encoder := json.NewEncoder(os.Stdout) 220 | for _, line := range lines { 221 | err := encoder.Encode(line) 222 | if err != nil { 223 | fmt.Printf("{\"event\":\"error\",\"message\":\"Cannot encode log line\",\"line\":\"%v\"}\n", line) 224 | } 225 | } 226 | } 227 | 228 | func (logger *ConsoleLogger) Logkv(keyValues ...interface{}) { 229 | logger.Logd(LogFunnel(keyValues)) 230 | } 231 | 232 | // A FileLogger writes JSON-formatted log lines to a file. 233 | // 234 | // Log lines are prefixed with a timestamp in RFC3339 format, like this: 235 | // [2006-01-02T15:04:05Z07:00] 236 | type FileLogger struct { 237 | // notification channel 238 | // also used for system signals 239 | signals chan os.Signal 240 | // log file name 241 | name string 242 | // log file handle 243 | log io.WriteCloser 244 | // message queue 245 | messages chan interface{} 246 | // log line counter 247 | lines uint64 248 | // dropped line counter 249 | drops uint64 250 | // error counter (encoding errors or closed log file) 251 | errors uint64 252 | } 253 | 254 | // NewFileLogger creates a new FileLogger and optionally installs a SIGUSR1 handler; 255 | // pass sigusr=true for that purpose. This is useful for log rotation, etc. 256 | // 257 | // Signals are only fully supported on POSIX systems, so no SIGUSR1 is sent 258 | // when running on Microsoft Windows, for example. The signal handler is 259 | // still installed, but it is never notified. 260 | func NewFileLogger(logfile string, sigusr bool) (*FileLogger, error) { 261 | // create logger instance 262 | logger := &FileLogger{ 263 | signals: make(chan os.Signal, signalQueueLength), 264 | name: logfile, 265 | messages: make(chan interface{}, logQueueLength), 266 | } 267 | 268 | // open the log for the first time 269 | err := logger.reopenLog() 270 | if err != nil { 271 | return nil, err 272 | } 273 | 274 | // install signal handler and start listening thread 275 | RegisterUserSignalHandler(logger.signals) 276 | go logger.handle() 277 | 278 | return logger, nil 279 | } 280 | 281 | // Logd writes a series of log lines, prefixed by a time stamp in RFC3339 format. 282 | func (logger *FileLogger) Logd(lines ...Dict) { 283 | // send these down the queue 284 | for _, line := range lines { 285 | select { 286 | case logger.messages <- line: 287 | // ok 288 | default: 289 | fmt.Printf("{\"event\":\"error\",\"message\":\"Log queue is full, message dropped\",\"line\":\"%v\"}\n", line) 290 | logger.drops++ 291 | } 292 | } 293 | } 294 | 295 | func (logger *FileLogger) Logkv(keyValues ...interface{}) { 296 | logger.Logd(LogFunnel(keyValues)) 297 | } 298 | 299 | // Writes a single log line 300 | func (logger *FileLogger) writeLog(line interface{}) { 301 | // only log if the output is open 302 | if logger.log != nil { 303 | data, err := json.Marshal(line) 304 | if err == nil { 305 | format := fmt.Sprintf("[%s] %s\n", time.Now().Format(timeFormat), data) 306 | if _, err := logger.log.Write([]byte(format)); err != nil { 307 | fmt.Printf("{\"event\":\"error\",\"message\":\"Cannot write log line to file\",\"line\":\"%v\",\"goerror\":\"%v\"}\n", line, err) 308 | } 309 | logger.lines++ 310 | } else { 311 | fmt.Printf("{\"event\":\"error\",\"message\":\"Cannot encode log line\",\"line\":\"%v\"}\n", line) 312 | logger.errors++ 313 | } 314 | } else { 315 | fmt.Printf("{\"event\":\"error\",\"message\":\"Output is closed, dropping line\",\"line\":\"%v\"}\n", line) 316 | logger.errors++ 317 | } 318 | } 319 | 320 | // Close closes the log file and disables further logging. 321 | func (logger *FileLogger) Close() { 322 | fmt.Printf("{\"event\":\"close_signal\",\"message\":\"Closing log\"}\n") 323 | logger.signals <- hupSignal 324 | } 325 | 326 | // closeLog closes the log and stops/removes the signal handler 327 | func (logger *FileLogger) closeLog() error { 328 | fmt.Printf("{\"event\":\"close\",\"message\":\"Really closing log\"}\n") 329 | 330 | // uninstall the singal handler 331 | signal.Stop(logger.signals) 332 | // signal stop 333 | logger.signals <- shutdownSignal 334 | 335 | // close the log 336 | err := logger.log.Close() 337 | logger.log = nil 338 | 339 | return err 340 | } 341 | 342 | // (Re-)opens the log file. 343 | func (logger *FileLogger) reopenLog() error { 344 | fmt.Printf("{\"event\":\"reopen\",\"message\":\"Reopening log\"}\n") 345 | 346 | var err error = nil 347 | 348 | if logger.log != nil { 349 | // close first 350 | err = logger.log.Close() 351 | logger.log = nil 352 | } 353 | if err == nil { 354 | logger.log, err = os.OpenFile(logger.name, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(0666)) 355 | } 356 | 357 | return err 358 | } 359 | 360 | // Handles the log queue and the USR1 signal. 361 | // If USR1 is received the log file is closed and reopened. 362 | func (logger *FileLogger) handle() { 363 | running := true 364 | 365 | for running { 366 | select { 367 | case sig := <-logger.signals: 368 | // check signal type 369 | switch sig { 370 | case UserSignal: 371 | // reopen the log file 372 | err := logger.reopenLog() 373 | if err != nil { 374 | // if this fails, print a message to the standard log 375 | fmt.Printf("{\"event\":\"error\",\"message\":\"Error reopening log\",\"error\":\"reopen\",\"errmsg\":\"%s\"}\n", err.Error()) 376 | } 377 | case hupSignal: 378 | // reopen the log file 379 | err := logger.closeLog() 380 | if err != nil { 381 | // if this fails, print a message to the standard log 382 | fmt.Printf("{\"event\":\"error\",\"message\":\"Error reopening log\",\"error\":\"reopen\",\"errmsg\":\"%s\"}\n", err.Error()) 383 | } 384 | case shutdownSignal: 385 | // shutdown requested 386 | running = false 387 | fmt.Printf("{\"event\":\"shutdown\",\"message\":\"Shutting down logger\"}\n") 388 | } 389 | case line := <-logger.messages: 390 | // encode and write the next line 391 | logger.writeLog(line) 392 | } 393 | } 394 | } 395 | -------------------------------------------------------------------------------- /util/log_test.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "bufio" 21 | "bytes" 22 | "os" 23 | "sync" 24 | "testing" 25 | ) 26 | 27 | func TestInternalSignal00(t *testing.T) { 28 | t00 := internalSignal("t00") 29 | // nothing should happen here 30 | t00.Signal() 31 | if t00.String() != "t00" { 32 | t.Errorf("Signal does not convert to identity") 33 | } 34 | } 35 | 36 | func TestLogFunnel00(t *testing.T) { 37 | t00 := []interface{}{ 38 | "a", "b", 39 | "bb", 10, 40 | 100, "x", 41 | "cde", 42 | } 43 | r00 := LogFunnel(t00) 44 | if v, ok := r00["a"]; !ok || v != "b" { 45 | t.Errorf("Key a not represented correctly in dictionary") 46 | } 47 | if v, ok := r00["bb"]; !ok || v != 10 { 48 | t.Errorf("Key bb not represented correctly in dictionary") 49 | } 50 | // 100 cannot be in dict - type mismatch 51 | if _, ok := r00["100"]; ok { 52 | t.Errorf("Key 100 shouldn't be in dictionary") 53 | } 54 | if _, ok := r00["cde"]; ok { 55 | t.Errorf("Key cde shouldn't be in dictionary") 56 | } 57 | } 58 | 59 | type mockLogger struct { 60 | t *testing.T 61 | lines []Dict 62 | } 63 | 64 | func (l *mockLogger) Logd(lines ...Dict) { 65 | l.lines = append(l.lines, lines...) 66 | } 67 | func (l *mockLogger) Logkv(keyValues ...interface{}) { 68 | l.Logd(LogFunnel(keyValues)) 69 | } 70 | 71 | func TestGlobalStdLogger00(t *testing.T) { 72 | m00a := &mockLogger{ 73 | t: t, 74 | } 75 | SetGlobalStandardLogger(m00a) 76 | m00b := &mockLogger{ 77 | t: t, 78 | } 79 | r00 := SetGlobalStandardLogger(m00b) 80 | if r00 != m00a { 81 | t.Errorf("Original logger is not the same as the returned logger") 82 | } 83 | } 84 | 85 | func TestGlobalModuleLogger00(t *testing.T) { 86 | m00 := &mockLogger{ 87 | t: t, 88 | } 89 | SetGlobalStandardLogger(m00) 90 | logger := NewGlobalModuleLogger("t00", Dict{ 91 | "cde": "v00", 92 | }) 93 | logger.Logkv("aa", "bb") 94 | if len(m00.lines) != 1 { 95 | t.Fatalf("Couldn't find the correct number of log lines in output") 96 | } 97 | if m00.lines[0]["aa"] != "bb" { 98 | t.Errorf("Didn't find test key in log line") 99 | } 100 | if m00.lines[0][KeyModule] != "t00" { 101 | t.Errorf("Didn't find module key in log line") 102 | } 103 | if m00.lines[0]["cde"] != "v00" { 104 | t.Errorf("Didn't find custom module key in log line") 105 | } 106 | } 107 | 108 | func TestModuleLogger00(t *testing.T) { 109 | m00 := &mockLogger{ 110 | t: t, 111 | } 112 | l00 := &ModuleLogger{ 113 | Logger: m00, 114 | AddTimestamp: true, 115 | } 116 | l00.Logkv("a", "b") 117 | if len(m00.lines) != 1 { 118 | t.Fatalf("Couldn't find the correct number of log lines in output") 119 | } 120 | if m00.lines[0]["a"] != "b" { 121 | t.Errorf("Didn't find test key in log line") 122 | } 123 | if len(m00.lines[0][KeyTime].(string)) < 20 { 124 | t.Errorf("Missing time key, or formatted time too short") 125 | } 126 | } 127 | 128 | func TestMultiLogger00(t *testing.T) { 129 | m00a := &mockLogger{ 130 | t: t, 131 | } 132 | m00b := &mockLogger{ 133 | t: t, 134 | } 135 | l00 := MultiLogger{ 136 | m00a, 137 | m00b, 138 | } 139 | l00.Logkv("ml00", "vl00") 140 | if len(m00a.lines) != 1 || len(m00b.lines) != 1 { 141 | t.Fatalf("Couldn't find the correct number of log lines in output") 142 | } 143 | if m00a.lines[0]["ml00"] != "vl00" { 144 | t.Errorf("Didn't find test key in log line of first logger") 145 | } 146 | if m00b.lines[0]["ml00"] != "vl00" { 147 | t.Errorf("Didn't find test key in log line of second logger") 148 | } 149 | } 150 | 151 | func TestConsoleLogger00(t *testing.T) { 152 | m00 := &mockLogger{ 153 | t: t, 154 | } 155 | r00, w00, err := os.Pipe() 156 | if err != nil { 157 | t.Fatalf("Cannot create pipe: %v", err) 158 | } 159 | wg := &sync.WaitGroup{} 160 | go func(l Logger, r *os.File, wg *sync.WaitGroup) { 161 | br := bufio.NewReader(r) 162 | for { 163 | line, err := br.ReadBytes('\n') 164 | if err != nil { 165 | break 166 | } 167 | l.Logkv("line", line) 168 | wg.Done() 169 | } 170 | }(m00, r00, wg) 171 | o00 := os.Stdout 172 | defer func(r, w, o *os.File) { 173 | //goland:noinspection GoUnhandledErrorResult 174 | defer r.Close() 175 | //goland:noinspection GoUnhandledErrorResult 176 | defer w.Close() 177 | os.Stdout = o 178 | }(r00, w00, o00) 179 | os.Stdout = w00 180 | l00 := &ConsoleLogger{} 181 | wg.Add(1) 182 | l00.Logkv("key00", "value00") 183 | wg.Wait() 184 | if len(m00.lines) != 1 { 185 | t.Fatalf("Couldn't find the correct number of log lines in output") 186 | } 187 | if !bytes.Contains(m00.lines[0]["line"].([]byte), []byte("key00")) { 188 | t.Errorf("Didn't find test key in log line: %s", m00.lines[0]) 189 | } 190 | if !bytes.Contains(m00.lines[0]["line"].([]byte), []byte("value00")) { 191 | t.Errorf("Didn't find test value in log line: %s", m00.lines[0]) 192 | } 193 | } 194 | -------------------------------------------------------------------------------- /util/set.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2017 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package util 18 | 19 | // Set is a set type, based on a map where keys represent the values. 20 | // 21 | // You can use both map semantics or the provided convenience functions 22 | // for basic set operations. 23 | // 24 | // Map semantics: 25 | // Create: 26 | // set := make(Set) 27 | // Add: 28 | // set[value] = true 29 | // Remove: 30 | // delete(set, value) 31 | // Test: 32 | // if set[value] { } 33 | // 34 | // Convenience functions: 35 | // Create: 36 | // set := MakeSet() 37 | // Add: 38 | // set.Add(value) 39 | // Remove: 40 | // set.Remove(value) 41 | // Test: 42 | // if set.Contains(value) 43 | type Set map[interface{}]bool 44 | 45 | // MakeSet creates a new empty set. 46 | func MakeSet() Set { 47 | return make(Set) 48 | } 49 | 50 | // Add adds a value to the set. 51 | func (set Set) Add(value interface{}) { 52 | set[value] = true 53 | } 54 | 55 | // Remove removes a value from the set. 56 | func (set Set) Remove(value interface{}) { 57 | delete(set, value) 58 | } 59 | 60 | // Contains tests if a value is in the set and returns true if this is the case. 61 | func (set Set) Contains(value interface{}) bool { 62 | return set[value] 63 | } 64 | -------------------------------------------------------------------------------- /util/shuffle.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "math/rand" 21 | ) 22 | 23 | // Shuffle shuffles a slice using Knuth's version of the Fisher-Yates algorithm. 24 | func Shuffle(rnd *rand.Rand, list []interface{}) []interface{} { 25 | N := len(list) 26 | ret := make([]interface{}, N) 27 | copy(ret, list) 28 | for i := 0; i < N; i++ { 29 | // choose index uniformly in [i, N-1] 30 | r := i + rnd.Intn(N-i) 31 | ret[r], ret[i] = ret[i], ret[r] 32 | } 33 | return ret 34 | } 35 | 36 | // ShuffleStrings shuffles a string slice using Knuth's version of the Fisher-Yates algorithm. 37 | func ShuffleStrings(rnd *rand.Rand, list []string) []string { 38 | N := len(list) 39 | ret := make([]string, N) 40 | copy(ret, list) 41 | for i := 0; i < N; i++ { 42 | // choose index uniformly in [i, N-1] 43 | r := i + rnd.Intn(N-i) 44 | ret[r], ret[i] = ret[i], ret[r] 45 | } 46 | return ret 47 | } 48 | -------------------------------------------------------------------------------- /util/shuffle_test.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2018 Gregor Riepl 2 | * 3 | * This program is free software: you can redistribute it and/or modify 4 | * it under the terms of the GNU General Public License as published by 5 | * the Free Software Foundation, either version 3 of the License, or 6 | * (at your option) any later version. 7 | * 8 | * This program is distributed in the hope that it will be useful, 9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | * GNU General Public License for more details. 12 | * 13 | * You should have received a copy of the GNU General Public License 14 | * along with this program. If not, see . 15 | */ 16 | 17 | package util 18 | 19 | import ( 20 | "fmt" 21 | "math/rand" 22 | "testing" 23 | ) 24 | 25 | const ( 26 | randSeed int64 = 0xc01df00d 27 | samples int = 10 28 | tries = 10 29 | ) 30 | 31 | func TestShuffleStrings(t *testing.T) { 32 | rnd := rand.New(rand.NewSource(randSeed)) 33 | 34 | values := make([]string, samples) 35 | results := make([]map[string]int, len(values)) 36 | for i := 0; i < len(values); i++ { 37 | values[i] = fmt.Sprintf("%d", i) 38 | results[i] = make(map[string]int, len(values)) 39 | } 40 | 41 | ShuffleStrings(rnd, values) 42 | } 43 | -------------------------------------------------------------------------------- /util/signal_unix.go: -------------------------------------------------------------------------------- 1 | //go:build !windows 2 | 3 | /* Copyright (c) 2018 Gregor Riepl 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | package util 20 | 21 | import ( 22 | "os" 23 | "os/signal" 24 | "syscall" 25 | ) 26 | 27 | const ( 28 | // UserSignal is a unique identifier for the signal that is sent through the 29 | // notification channel when a user event occurs. 30 | UserSignal = syscall.SIGUSR1 31 | ) 32 | 33 | // RegisterUserSignalHandler registers a process signal handler that reacts to 34 | // and notifies on external user events, like SIGUSR1 on Unix. 35 | // NOTE: Unsupported and ignored on Microsoft Windows. 36 | func RegisterUserSignalHandler(notify chan os.Signal) { 37 | signal.Notify(notify, UserSignal) 38 | } 39 | -------------------------------------------------------------------------------- /util/signal_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | /* Copyright (c) 2018 Gregor Riepl 4 | * 5 | * This program is free software: you can redistribute it and/or modify 6 | * it under the terms of the GNU General Public License as published by 7 | * the Free Software Foundation, either version 3 of the License, or 8 | * (at your option) any later version. 9 | * 10 | * This program is distributed in the hope that it will be useful, 11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | * GNU General Public License for more details. 14 | * 15 | * You should have received a copy of the GNU General Public License 16 | * along with this program. If not, see . 17 | */ 18 | 19 | package util 20 | 21 | import ( 22 | "os" 23 | ) 24 | 25 | const ( 26 | // UserSignal is a unique identifier for the signal that is sent through the 27 | // notification channel when a user event occurs. 28 | UserSignal internalSignal = internalSignal("USR") 29 | ) 30 | 31 | // RegisterUserSignalHandler registers a process signal handler that reacts to 32 | // and notifies on external user events, like SIGUSR1 on Unix. 33 | // NOTE: Unsupported and ignored on Microsoft Windows. 34 | func RegisterUserSignalHandler(notify chan os.Signal) { 35 | // Unsupported on MS Windows 36 | } 37 | -------------------------------------------------------------------------------- /util/util.goconvey: -------------------------------------------------------------------------------- 1 | // Timeout to avoid failing tests that block forever 2 | -timeout=3s 3 | --------------------------------------------------------------------------------