├── .dockerignore ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── command ├── allocations │ └── app.go ├── deployments │ └── app.go ├── evaluations │ └── app.go ├── jobs │ ├── base.go │ ├── job.go │ └── jobsliststub.go └── nodes │ └── app.go ├── go.mod ├── go.sum ├── helper └── manager.go ├── main.go └── sink ├── event_bridge.go ├── helper.go ├── http.go ├── kafka.go ├── kinesis.go ├── mongodb.go ├── nsq.go ├── rabbitmq.go ├── redis.go ├── sqs.go ├── stdout.go ├── structs.go ├── syslog_unix.go └── syslog_windows.go /.dockerignore: -------------------------------------------------------------------------------- 1 | build/ 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor/ 2 | !vendor/vendor.json 3 | build/ 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | 3 | language: go 4 | 5 | go: 6 | - 1.14.1 7 | 8 | services: 9 | - docker 10 | 11 | cache: 12 | directories: 13 | - /root/go/pkg 14 | 15 | env: 16 | global: 17 | - CGO_ENABLED=0 18 | - GOBUILD="linux-amd64 windows-amd64 darwin-amd64" 19 | 20 | script: 21 | - GOBUILD=${GOBUILD} make -j build 22 | - ls -la build 23 | 24 | - export PR=https://api.github.com/repos/$TRAVIS_REPO_SLUG/pulls/$TRAVIS_PULL_REQUEST 25 | - export BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; else echo `curl -s $PR | jq -r .head.ref`; fi) 26 | 27 | - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin 28 | - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then make -j docker-release TAG=pr-$TRAVIS_PULL_REQUEST COMMIT=$TRAVIS_COMMIT; fi 29 | - if [[ "$TRAVIS_TAG" =~ ^v.*$ ]]; then make -j docker-release TAG=$TRAVIS_TAG COMMIT=$TRAVIS_COMMIT; fi 30 | - if [[ "$BRANCH" == "master" ]]; then make -j docker-release TAG=latest COMMIT=$TRAVIS_COMMIT; fi 31 | 32 | deploy: 33 | provider: releases 34 | api_key: 35 | secure: inCWsnZFjpCOk3SCEgOb/Ffr8gTR1ELprchGl3tvO9V21Zz9Lv8EVPVEbPxf8sjD7dZ48Jd+4+EcSOJRVcVfQac4ZId0hi1xpBFjcpQwVK1gXbQ4U4GY+1oGxdlRpKysUofiFdmFRZcty/eOHVahRzBEd2Ysp28y2QCI0xkp3YFuRXed876hmeUHUlmUjxhqjk9/dfHu8btAMVyI/A2SA7zetxt6QZ//PjP+qjjtlKRl/J2ZsClHbBJ7pV46x/8KRIIwsp45t5OcXd69OLBbEM2Hae7aLPfcSd4h5LmD0yKmXC+498TxtnLP1mw1V1BiFmAKf3nhuBiuSZNJeJ4XwqSzuRf/YXO3RA/U7H9kKn7t2va3a7i50CWObwI/K6Y4rpGpa74YgYRhuJBW5PNakhPGfst5iEQyjFwYmAACOgiwHqBcNH7gaUZoGzoJnlijHWm98v+8Qr4xRx96DNQomI08Y7qz1/n1LihEMHRHy+oxcjf6P4F7A3NQqbmNqo946A1xreKUW4NO918aJSM2Cc+1KNNlf8OjHbEFd3OHhqi7GkC4BBuBzkZ/5P12imVyR/MrT7xpD1uImz2wRtiiujhbWn20aOslDxRRiLxOMsqnXMP4Yf7IUa06y1L7xNwlCqHuaRordP9/MWQo1ZAnd4jj3x/UvtjCYWj9AMN6/gs= 36 | file: 37 | - build/nomad-firehose-linux-amd64 38 | - build/nomad-firehose-windows-amd64 39 | - build/nomad-firehose-darwin-amd64 40 | skip_cleanup: true 41 | overwrite: true 42 | on: 43 | tags: true 44 | repo: seatgeek/nomad-firehose 45 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.14.1 as go-builder 2 | WORKDIR /go/src/app 3 | ENV CGO_ENABLED=0 4 | COPY go.mod go.sum ./ 5 | RUN go mod download 6 | COPY . . 7 | RUN CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "-X main.GitCommit=$(git describe --tags)" 8 | 9 | FROM debian:buster 10 | RUN apt-get update && apt-get install -y ca-certificates && apt-get clean 11 | COPY --from=go-builder /go/src/app/nomad-firehose /bin/ 12 | CMD [ "/bin/nomad-firehose" ] 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, SeatGeek 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # build config 2 | BUILD_DIR ?= $(abspath build) 3 | GET_GOARCH = $(word 2,$(subst -, ,$1)) 4 | GET_GOOS = $(word 1,$(subst -, ,$1)) 5 | GOBUILD ?= $(shell go env GOOS)-$(shell go env GOARCH) 6 | GOFILES_NOVENDOR = $(shell find . -type f -name '*.go' -not -path "./vendor/*") 7 | VETARGS? =-all 8 | GIT_COMMIT := $(shell git describe --tags) 9 | GIT_DIRTY := $(if $(shell git status --porcelain),+CHANGES) 10 | GO_LDFLAGS := "-X main.GitCommit=$(GIT_COMMIT)$(GIT_DIRTY)" 11 | 12 | $(BUILD_DIR): 13 | mkdir -p $@ 14 | 15 | # Install all go dependencies via "go get". Will use the "go.mod / go.sum" files automatically 16 | .PHONY: dependencies 17 | dependencies: 18 | @echo "==> go mod download" 19 | @go mod download 20 | 21 | # Create pseudo Make target for all cmd/ (or GOBUILD) provided 22 | # allow for "make cache_primer" to work automatically 23 | BINARIES = $(addprefix $(BUILD_DIR)/nomad-firehose-, $(GOBUILD)) 24 | $(BINARIES): $(BUILD_DIR)/nomad-firehose-%: $(BUILD_DIR) dependencies 25 | @echo "==> building $@ ..." 26 | GOOS=$(call GET_GOOS,$*) GOARCH=$(call GET_GOARCH,$*) CGO_ENABLED=0 go build -o $@ -ldflags $(GO_LDFLAGS) 27 | 28 | # Build all binaries (or APP_SERVER_NAME) and write to the BIN_DIR/ 29 | .PHONY: build 30 | build: 31 | @$(MAKE) -j $(BINARIES) 32 | 33 | # Format go source code 34 | .PHONY: fmt 35 | fmt: 36 | gofmt -w . 37 | 38 | # vet go source code 39 | .PHONY: vet 40 | vet: 41 | go vet ./... 42 | 43 | # Run full test suite 44 | .PHONY: test 45 | test: dependencies 46 | @echo "==> go test" 47 | @go test -v -covermode=count ./... 48 | 49 | # Build local Dockerfile 50 | .PHONY: docker-build 51 | docker-build: 52 | @echo "==> Docker build" 53 | docker build -t nomad-firehose-local . 54 | 55 | # Start docker shell 56 | .PHONY: docker-shell 57 | docker-shell: docker-build 58 | @echo "==> Docker run" 59 | @docker run --rm -it nomad-firehose-local bash 60 | 61 | .PHONY: docker-release 62 | docker-release: docker-build 63 | @echo "=> build and push Docker image ..." 64 | docker tag nomad-firehose-local seatgeek/nomad-firehose:$(TAG) 65 | docker push seatgeek/nomad-firehose:$(TAG) 66 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nomad-firehose 2 | 3 | `nomad-firehose` is a tool meant to enable teams to quickly build logic around nomad task events without hooking into Nomad API. 4 | 5 | ## Running 6 | 7 | The project has build artifacts for Linux, Darwin and Windows in the [GitHub releases tab](https://github.com/seatgeek/nomad-firehose/releases). 8 | 9 | A Docker container is also provided at [seatgeek/nomad-firehose](https://hub.docker.com/r/seatgeek/nomad-firehose/tags/) 10 | 11 | ## Requirements 12 | 13 | - Go 1.11 14 | 15 | ## Building 16 | 17 | To build a binary, run the following 18 | 19 | ```shell 20 | # get this repo 21 | go get github.com/seatgeek/nomad-firehose 22 | 23 | # go to the repo directory 24 | cd $GOPATH/src/github.com/seatgeek/nomad-firehose 25 | 26 | # build the `nomad-firehose` binary 27 | make build 28 | ``` 29 | 30 | This will create a `nomad-firehose` binary in your `$GOPATH/bin` directory. 31 | 32 | ## Configuration 33 | 34 | Any `NOMAD_*` env that the native `nomad` CLI tool supports are supported by this tool. 35 | 36 | Any `CONSUL_*` env that the native `consul` CLI tool supports are supported by this tool. 37 | 38 | The most basic requirement is `export NOMAD_ADDR=http://:4646` and `export CONSUL_HTTP_ADDR=:8500`. 39 | 40 | ### Consul 41 | 42 | `nomad-firehose` will use Consul to maintain leader-ship and store last event time processed (saved on quit or every 10s). 43 | 44 | This mean you can run more than 1 process of each firehose, and only one will actually do any work. 45 | 46 | Saving the last event time mean that restarting the process won't firehose all old changes to your sink, reducing duplicated events. 47 | 48 | By default, the Consul lock is maintained in KV at `nomad-firehose/${type}.lock` and the last event time is stored in KV at `nomad-firehose/${type}.value`. You can change the prefix from `nomad-firehose` by setting `NOMAD_FIREHOSE_CONSUL_PREFIX` to your desired prefix. 49 | 50 | #### Consul ACL Token Permissions 51 | 52 | If the Consul cluster being used is running ACLs, the following ACL policy will allow the required access: 53 | 54 | ```hcl 55 | key "nomad-firehose" { 56 | policy = "write" 57 | } 58 | session "" { 59 | policy = "write" 60 | } 61 | ``` 62 | 63 | If you've set a custom prefix, specify that in the `key` ACL entry instead. 64 | 65 | ### Kafka 66 | 67 | To connect to Kafka with TLS, set the SINK_KAFKA_CA_CERT_PATH to the path to your CA cert file. 68 | To use SASL/PLAIN authentication, set `$SINK_KAFKA_USER` and `$SINK_KAFKA_PASSWORD` environment variables. 69 | 70 | 71 | ## Usage 72 | 73 | The `nomad-firehose` binary has several helper subcommands. 74 | 75 | The sink type is configured using `$SINK_TYPE` environment variable. Valid values are: 76 | - `amqp` 77 | - `kinesis` 78 | - `nsq` 79 | - `redis` 80 | - `kafka` 81 | - `mongo` 82 | - `sqs` 83 | - `eventbridge` 84 | - `stdout` 85 | - `syslog` 86 | 87 | The `amqp` and `rabbitmq` sinks are configured using `$SINK_AMQP_CONNECTION` (`amqp://guest:guest@127.0.0.1:5672/`), `$SINK_AMQP_EXCHANGE`, `$SINK_AMQP_ROUTING_KEY`, and `$SINK_AMQP_WORKERS` (default: `1`) environment variables. 88 | 89 | The `http` sink is configured using `$SINK_HTTP_ADDRESS` (`localhost:8080/allocations`)` environment variable. 90 | 91 | The `kafka` sink is configured using `$SINK_KAFKA_BROKERS` (`kafka1:9092,kafka2:9092,kafka3:9092`), and `$SINK_KAFKA_TOPIC` environment variables. 92 | 93 | The `kinesis` sink is configured using `$SINK_KINESIS_STREAM_NAME` and `$SINK_KINESIS_PARTITION_KEY` environment variables. 94 | 95 | The `mongo` sink is configured using `$SINK_MONGODB_CONNECTION` (`mongodb://localhost:27017/`), `$SINK_MONGODB_DATABASE` and `$SINK_MONGODB_COLLECTION` environment variables. 96 | 97 | The `nsq` sink is configured using `$SINK_NSQ_ADDR` and `$SINK_NSQ_TOPIC_NAME` environment variables. 98 | 99 | The `redis` sink is configured using `$SINK_REDIS_URL` (`redis://[user]:[password]@127.0.0.1[:5672]/0`) and `$SINK_REDIS_KEY` environment variables. 100 | 101 | The `stdout` sink does not have any configuration, it will simply output the JSON to stdout for debugging. 102 | 103 | The `syslog` sink is configured using `$SINK_SYSLOG_PROTO` (e.g. `tcp`, `udp` - leave empty if logging to a local syslog socket), `$SINK_SYSLOG_ADDR` (e.g. `127.0.0.1:514` - leave empty if logging to a local syslog socket), and `$SINK_SYSLOG_TAG` (default: `nomad-firehose`). 104 | 105 | The `sqs` sink is configured using `$SINK_SQS_QUEUE_NAME` which is the name of the queue in SQS. This queue is expected to be a FIFO queue. 106 | The URL of the queue is inferred by the presence of the `AWS_REGION` and `AWS_ACCOUNT_ID` env variables. 107 | 108 | The `eventbridge` sink is configured using `$SINK_EVENT_BUS_NAME` which is the name of the bus in Event Bridge. The environment variables `SINK_EVENT_BUS_DETAIL_TYPE` and `SINK_EVENT_BUS_SOURCE` are used to configure the schema when creating Event Bus rules. 109 | 110 | ### `allocations` 111 | 112 | `nomad-firehose allocations` will monitor all allocation changes in the Nomad cluster and emit each task state as a new firehose event to the configured sink. 113 | 114 | The allocation output is different from the [default API response](https://www.nomadproject.io/api/allocations.html), as the tool will emit an event per new [TaskStates](https://www.nomadproject.io/docs/http/allocs.html), rather than all the previous events. 115 | 116 | ```json 117 | { 118 | "Name": "job.task[0]", 119 | "AllocationID": "1ef2eba2-00e4-3828-96d4-8e58b1447aaf", 120 | "DesiredStatus": "run", 121 | "DesiredDescription": "", 122 | "ClientStatus": "running", 123 | "ClientDescription": "", 124 | "JobID": "logrotate", 125 | "GroupName": "cron", 126 | "TaskName": "logrotate", 127 | "EvalID": "bf926150-ed30-6c13-c597-34d7a3165fdc", 128 | "TaskState": "running", 129 | "TaskFailed": false, 130 | "TaskStartedAt": "2017-06-30T19:58:28.325895579Z", 131 | "TaskFinishedAt": "0001-01-01T00:00:00Z", 132 | "TaskEvent": { 133 | "Type": "Task Setup", 134 | "Time": 1498852707712617200, 135 | "FailsTask": false, 136 | "RestartReason": "", 137 | "SetupError": "", 138 | "DriverError": "", 139 | "DriverMessage": "", 140 | "ExitCode": 0, 141 | "Signal": 0, 142 | "Message": "Building Task Directory", 143 | "KillReason": "", 144 | "KillTimeout": 0, 145 | "KillError": "", 146 | "StartDelay": 0, 147 | "DownloadError": "", 148 | "ValidationError": "", 149 | "DiskLimit": 0, 150 | "DiskSize": 0, 151 | "FailedSibling": "", 152 | "VaultError": "", 153 | "TaskSignalReason": "", 154 | "TaskSignal": "" 155 | } 156 | } 157 | ``` 158 | 159 | ### `nodes` 160 | 161 | `nomad-firehose nodes` will monitor all node changes in the Nomad cluster and emit a firehose event per change to the configured sink. 162 | 163 | The output will be equal to the [Nomad Node API structure](https://www.nomadproject.io/api/nodes.html) 164 | 165 | ### `evaluations` 166 | 167 | `nomad-firehose evaluations` will monitor all evaluation changes in the Nomad cluster and emit a firehose event per change to the configured sink. 168 | 169 | The output will be equal to the [Nomad Evaluation API structure](https://www.nomadproject.io/api/evaluations.html) 170 | 171 | ### `jobs` 172 | 173 | `nomad-firehose jobs` will monitor all job changes in the Nomad cluster and emit a firehose event per change to the configured sink. 174 | 175 | The output will be equal to the *full* [Nomad Job API structure](https://www.nomadproject.io/api/jobs.html#read-job) 176 | 177 | ### `jobliststubs` 178 | 179 | `nomad-firehose jobliststubs` will monitor all job changes in the Nomad cluster and emit a firehose event per change to the configured sink. 180 | 181 | The output will be equal to the job list [Nomad Job API structure](https://www.nomadproject.io/api/jobs.html#list-jobs) 182 | 183 | ### `deployments` 184 | 185 | `nomad-firehose deployments` will monitor all deployment changes in the Nomad cluster and emit a firehose event per change to the configured sink. 186 | 187 | The output will be equal to the *full* [Nomad Deployment API structure](https://www.nomadproject.io/api/deployments.html) 188 | -------------------------------------------------------------------------------- /command/allocations/app.go: -------------------------------------------------------------------------------- 1 | package allocations 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "time" 7 | 8 | nomad "github.com/hashicorp/nomad/api" 9 | "github.com/seatgeek/nomad-firehose/sink" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | // Firehose ... 14 | type Firehose struct { 15 | lastChangeTime int64 16 | lastChangeTimeCh chan interface{} 17 | nomadClient *nomad.Client 18 | sink sink.Sink 19 | stopCh chan struct{} 20 | } 21 | 22 | // AllocationUpdate ... 23 | type AllocationUpdate struct { 24 | Name string 25 | NodeID string 26 | AllocationID string 27 | DesiredStatus string 28 | DesiredDescription string 29 | ClientStatus string 30 | ClientDescription string 31 | JobID string 32 | GroupName string 33 | TaskName string 34 | EvalID string 35 | TaskState string 36 | TaskFailed bool 37 | TaskStartedAt *time.Time 38 | TaskFinishedAt *time.Time 39 | TaskEvent *nomad.TaskEvent 40 | } 41 | 42 | // NewFirehose ... 43 | func NewFirehose() (*Firehose, error) { 44 | nomadClient, err := nomad.NewClient(nomad.DefaultConfig()) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | sink, err := sink.GetSink("allocations") 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | return &Firehose{ 55 | nomadClient: nomadClient, 56 | sink: sink, 57 | stopCh: make(chan struct{}, 1), 58 | lastChangeTimeCh: make(chan interface{}, 1), 59 | }, nil 60 | } 61 | 62 | func (f *Firehose) Name() string { 63 | return "allocations" 64 | } 65 | 66 | func (f *Firehose) UpdateCh() <-chan interface{} { 67 | return f.lastChangeTimeCh 68 | } 69 | 70 | func (f *Firehose) SetRestoreValue(restoreValue interface{}) error { 71 | switch restoreValue.(type) { 72 | case int: 73 | f.lastChangeTime = int64(restoreValue.(int)) 74 | case int64: 75 | f.lastChangeTime = restoreValue.(int64) 76 | default: 77 | return fmt.Errorf("Unknown restore type '%T' with value '%+v'", restoreValue, restoreValue) 78 | } 79 | return nil 80 | } 81 | 82 | // Start the firehose 83 | func (f *Firehose) Start() { 84 | go f.sink.Start() 85 | 86 | // Stop chan for all tasks to depend on 87 | f.stopCh = make(chan struct{}) 88 | 89 | // watch for allocation changes 90 | go f.watch() 91 | 92 | // Save the last event time every 5s 93 | go f.persistLastChangeTime(5 * time.Second) 94 | 95 | // wait forever for a stop signal to happen 96 | for { 97 | select { 98 | case <-f.stopCh: 99 | return 100 | } 101 | } 102 | } 103 | 104 | // Stop the firehose 105 | func (f *Firehose) Stop() { 106 | close(f.stopCh) 107 | f.sink.Stop() 108 | } 109 | 110 | // Write the Last Change Time to Consul so if the process restarts, 111 | // it will try to resume from where it left off, not emitting tons of double events for 112 | // old events 113 | func (f *Firehose) persistLastChangeTime(interval time.Duration) { 114 | ticker := time.NewTicker(interval) 115 | 116 | for { 117 | select { 118 | case <-f.stopCh: 119 | f.lastChangeTimeCh <- f.lastChangeTime 120 | break 121 | case <-ticker.C: 122 | f.lastChangeTimeCh <- f.lastChangeTime 123 | } 124 | } 125 | } 126 | 127 | // publish an update from the firehose 128 | func (f *Firehose) publish(update *AllocationUpdate) { 129 | b, err := json.Marshal(update) 130 | if err != nil { 131 | log.Error(err) 132 | } 133 | 134 | f.sink.Put(b) 135 | } 136 | 137 | // Continously watch for changes to the allocation list and publish it as updates 138 | func (f *Firehose) watch() { 139 | q := &nomad.QueryOptions{ 140 | WaitIndex: 1, 141 | WaitTime: 5 * time.Minute, 142 | AllowStale: true, 143 | } 144 | 145 | newMax := f.lastChangeTime 146 | 147 | for { 148 | allocations, meta, err := f.nomadClient.Allocations().List(q) 149 | if err != nil { 150 | log.Errorf("Unable to fetch allocations: %s", err) 151 | time.Sleep(10 * time.Second) 152 | continue 153 | } 154 | 155 | remoteWaitIndex := meta.LastIndex 156 | localWaitIndex := q.WaitIndex 157 | 158 | // Only work if the WaitIndex have changed 159 | if remoteWaitIndex == localWaitIndex { 160 | log.Debugf("Allocations index is unchanged (%d == %d)", remoteWaitIndex, localWaitIndex) 161 | continue 162 | } 163 | 164 | log.Debugf("Allocations index is changed (%d <> %d)", remoteWaitIndex, localWaitIndex) 165 | 166 | // Iterate allocations and find events that have changed since last run 167 | for _, allocation := range allocations { 168 | for taskName, taskInfo := range allocation.TaskStates { 169 | for _, taskEvent := range taskInfo.Events { 170 | if taskEvent.Time <= f.lastChangeTime { 171 | continue 172 | } 173 | 174 | if taskEvent.Time > newMax { 175 | newMax = taskEvent.Time 176 | } 177 | 178 | payload := &AllocationUpdate{ 179 | Name: allocation.Name, 180 | NodeID: allocation.NodeID, 181 | AllocationID: allocation.ID, 182 | EvalID: allocation.EvalID, 183 | DesiredStatus: allocation.DesiredStatus, 184 | DesiredDescription: allocation.DesiredDescription, 185 | ClientStatus: allocation.ClientStatus, 186 | ClientDescription: allocation.ClientDescription, 187 | JobID: allocation.JobID, 188 | GroupName: allocation.TaskGroup, 189 | TaskName: taskName, 190 | TaskEvent: taskEvent, 191 | TaskState: taskInfo.State, 192 | TaskFailed: taskInfo.Failed, 193 | TaskStartedAt: &taskInfo.StartedAt, 194 | TaskFinishedAt: &taskInfo.FinishedAt, 195 | } 196 | 197 | f.publish(payload) 198 | } 199 | } 200 | } 201 | 202 | // Update WaitIndex and Last Change Time for next iteration 203 | q.WaitIndex = meta.LastIndex 204 | f.lastChangeTime = newMax 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /command/deployments/app.go: -------------------------------------------------------------------------------- 1 | package deployments 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "strconv" 8 | "time" 9 | 10 | nomad "github.com/hashicorp/nomad/api" 11 | "github.com/seatgeek/nomad-firehose/sink" 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | // Firehose ... 16 | type Firehose struct { 17 | lastChangeTime uint64 18 | lastChangeTimeCh chan interface{} 19 | nomadClient *nomad.Client 20 | sink sink.Sink 21 | stopCh chan struct{} 22 | } 23 | 24 | // NewFirehose ... 25 | func NewFirehose() (*Firehose, error) { 26 | nomadClient, err := nomad.NewClient(nomad.DefaultConfig()) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | sink, err := sink.GetSink("deployments") 32 | if err != nil { 33 | log.Fatal(err) 34 | os.Exit(1) 35 | } 36 | 37 | return &Firehose{ 38 | nomadClient: nomadClient, 39 | sink: sink, 40 | lastChangeTimeCh: make(chan interface{}, 1), 41 | }, nil 42 | } 43 | 44 | func (f *Firehose) Name() string { 45 | return "deployments" 46 | } 47 | 48 | func (f *Firehose) UpdateCh() <-chan interface{} { 49 | return f.lastChangeTimeCh 50 | } 51 | 52 | func (f *Firehose) SetRestoreValue(restoreValue interface{}) error { 53 | switch restoreValue.(type) { 54 | case int: 55 | f.lastChangeTime = uint64(restoreValue.(int)) 56 | case int64: 57 | f.lastChangeTime = uint64(restoreValue.(int64)) 58 | case string: 59 | restoreValueInt, _ := strconv.Atoi(restoreValue.(string)) 60 | f.lastChangeTime = uint64(restoreValueInt) 61 | default: 62 | return fmt.Errorf("Unable to compute restore time, not int or string (%T)", restoreValue) 63 | } 64 | 65 | return nil 66 | } 67 | 68 | // Start the firehose 69 | func (f *Firehose) Start() { 70 | go f.sink.Start() 71 | 72 | // Stop chan for all tasks to depend on 73 | f.stopCh = make(chan struct{}) 74 | 75 | // watch for deployment changes 76 | go f.watch() 77 | 78 | // Save the last event time every 5s 79 | go f.persistLastChangeTime(5 * time.Second) 80 | 81 | // wait forever for a stop signal to happen 82 | for { 83 | select { 84 | case <-f.stopCh: 85 | return 86 | } 87 | } 88 | } 89 | 90 | // Stop the firehose 91 | func (f *Firehose) Stop() { 92 | close(f.stopCh) 93 | f.sink.Stop() 94 | } 95 | 96 | // Write the Last Change Time to Consul so if the process restarts, 97 | // it will try to resume from where it left off, not emitting tons of double events for 98 | // old events 99 | func (f *Firehose) persistLastChangeTime(interval time.Duration) { 100 | ticker := time.NewTicker(interval) 101 | 102 | for { 103 | select { 104 | case <-f.stopCh: 105 | f.lastChangeTimeCh <- f.lastChangeTime 106 | break 107 | case <-ticker.C: 108 | f.lastChangeTimeCh <- f.lastChangeTime 109 | } 110 | } 111 | } 112 | 113 | // Publish an update from the firehose 114 | func (f *Firehose) Publish(update *nomad.Deployment) { 115 | b, err := json.Marshal(update) 116 | if err != nil { 117 | log.Error(err) 118 | } 119 | 120 | f.sink.Put(b) 121 | } 122 | 123 | // Continously watch for changes to the deployment list and publish it as updates 124 | func (f *Firehose) watch() { 125 | q := &nomad.QueryOptions{ 126 | WaitIndex: uint64(f.lastChangeTime), 127 | WaitTime: 5 * time.Minute, 128 | AllowStale: true, 129 | } 130 | 131 | newMax := uint64(f.lastChangeTime) 132 | 133 | for { 134 | deployments, meta, err := f.nomadClient.Deployments().List(q) 135 | if err != nil { 136 | log.Errorf("Unable to fetch deployments: %s", err) 137 | time.Sleep(10 * time.Second) 138 | continue 139 | } 140 | 141 | remoteWaitIndex := meta.LastIndex 142 | localWaitIndex := q.WaitIndex 143 | 144 | // Only work if the WaitIndex have changed 145 | if remoteWaitIndex == localWaitIndex { 146 | log.Debugf("Deployments index is unchanged (%d == %d)", remoteWaitIndex, localWaitIndex) 147 | continue 148 | } 149 | 150 | log.Debugf("Deployments index is changed (%d <> %d)", remoteWaitIndex, localWaitIndex) 151 | 152 | // Iterate deployments and find events that have changed since last run 153 | for _, deployment := range deployments { 154 | if deployment.ModifyIndex <= f.lastChangeTime { 155 | continue 156 | } 157 | 158 | if deployment.ModifyIndex > newMax { 159 | newMax = deployment.ModifyIndex 160 | } 161 | 162 | go func(DeploymentID string) { 163 | fullDeployment, _, err := f.nomadClient.Deployments().Info(DeploymentID, &nomad.QueryOptions{}) 164 | if err != nil { 165 | log.Errorf("Could not read deployment %s: %s", DeploymentID, err) 166 | return 167 | } 168 | 169 | f.Publish(fullDeployment) 170 | }(deployment.ID) 171 | } 172 | 173 | // Update WaitIndex and Last Change Time for next iteration 174 | q.WaitIndex = meta.LastIndex 175 | f.lastChangeTime = newMax 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /command/evaluations/app.go: -------------------------------------------------------------------------------- 1 | package evaluations 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "os" 7 | "time" 8 | 9 | nomad "github.com/hashicorp/nomad/api" 10 | "github.com/seatgeek/nomad-firehose/sink" 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | // Firehose ... 15 | type Firehose struct { 16 | lastChangeIndex uint64 17 | lastChangeTimeCh chan interface{} 18 | nomadClient *nomad.Client 19 | sink sink.Sink 20 | stopCh chan struct{} 21 | } 22 | 23 | // NewFirehose ... 24 | func NewFirehose() (*Firehose, error) { 25 | nomadClient, err := nomad.NewClient(nomad.DefaultConfig()) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | sink, err := sink.GetSink("evaluations") 31 | if err != nil { 32 | log.Fatal(err) 33 | os.Exit(1) 34 | } 35 | 36 | return &Firehose{ 37 | nomadClient: nomadClient, 38 | sink: sink, 39 | stopCh: make(chan struct{}, 1), 40 | lastChangeTimeCh: make(chan interface{}, 1), 41 | }, nil 42 | } 43 | 44 | func (f *Firehose) Name() string { 45 | return "evaluations" 46 | } 47 | 48 | func (f *Firehose) UpdateCh() <-chan interface{} { 49 | return f.lastChangeTimeCh 50 | } 51 | 52 | func (f *Firehose) SetRestoreValue(restoreValue interface{}) error { 53 | switch restoreValue.(type) { 54 | case int: 55 | f.lastChangeIndex = uint64(restoreValue.(int)) 56 | case int64: 57 | f.lastChangeIndex = uint64(restoreValue.(int64)) 58 | default: 59 | return fmt.Errorf("Unknown restore type '%T' with value '%+v'", restoreValue, restoreValue) 60 | } 61 | return nil 62 | } 63 | 64 | // Start the firehose 65 | func (f *Firehose) Start() { 66 | go f.sink.Start() 67 | 68 | // Stop chan for all tasks to depend on 69 | f.stopCh = make(chan struct{}) 70 | 71 | // watch for allocation changes 72 | go f.watch() 73 | 74 | // Save the last event time every 5s 75 | go f.persistLastChangeTime(5 * time.Second) 76 | 77 | // wait forever for a stop signal to happen 78 | select { 79 | case <-f.stopCh: 80 | return 81 | } 82 | } 83 | 84 | // Stop the firehose 85 | func (f *Firehose) Stop() { 86 | close(f.stopCh) 87 | f.sink.Stop() 88 | } 89 | 90 | // Write the Last Change Time to Consul so if the process restarts, 91 | // it will try to resume from where it left off, not emitting tons of double events for 92 | // old events 93 | func (f *Firehose) persistLastChangeTime(interval time.Duration) { 94 | ticker := time.NewTicker(interval) 95 | 96 | for { 97 | select { 98 | case <-f.stopCh: 99 | f.lastChangeTimeCh <- f.lastChangeIndex 100 | break 101 | case <-ticker.C: 102 | f.lastChangeTimeCh <- f.lastChangeIndex 103 | } 104 | } 105 | } 106 | 107 | // Publish an update from the firehose 108 | func (f *Firehose) Publish(update *nomad.Evaluation) { 109 | b, err := json.Marshal(update) 110 | if err != nil { 111 | log.Error(err) 112 | } 113 | 114 | f.sink.Put(b) 115 | } 116 | 117 | // Continously watch for changes to the allocation list and publish it as updates 118 | func (f *Firehose) watch() { 119 | q := &nomad.QueryOptions{ 120 | WaitIndex: f.lastChangeIndex, 121 | WaitTime: 5 * time.Minute, 122 | AllowStale: true, 123 | } 124 | 125 | for { 126 | log.Infof("Fetching evaluations from Nomad: %+v", q) 127 | 128 | evaluations, meta, err := f.nomadClient.Evaluations().List(q) 129 | if err != nil { 130 | log.Errorf("Unable to fetch evaluations: %s", err) 131 | time.Sleep(10 * time.Second) 132 | continue 133 | } 134 | 135 | // Only work if the WaitIndex have changed 136 | if meta.LastIndex == f.lastChangeIndex { 137 | log.Infof("Evaluations index is unchanged (%d == %d)", meta.LastIndex, f.lastChangeIndex) 138 | continue 139 | } 140 | 141 | log.Infof("Evaluations index is changed (%d <> %d)", meta.LastIndex, f.lastChangeIndex) 142 | 143 | // Iterate clients and find events that have changed since last run 144 | for _, evaluation := range evaluations { 145 | if evaluation.ModifyIndex != f.lastChangeIndex { 146 | continue 147 | } 148 | 149 | f.Publish(evaluation) 150 | evaluation = nil 151 | } 152 | 153 | evaluations = nil 154 | 155 | // Update WaitIndex and Last Change Time for next iteration 156 | f.lastChangeIndex = meta.LastIndex 157 | q.WaitIndex = meta.LastIndex 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /command/jobs/base.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | nomad "github.com/hashicorp/nomad/api" 8 | "github.com/seatgeek/nomad-firehose/sink" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | type WatchJobListFunc func(job *nomad.JobListStub) 13 | 14 | // Firehose ... 15 | type FirehoseBase struct { 16 | lastChangeIndex uint64 17 | lastChangeTimeCh chan interface{} 18 | nomadClient *nomad.Client 19 | sink sink.Sink 20 | stopCh chan struct{} 21 | } 22 | 23 | // NewFirehose ... 24 | func NewFirehoseBase() (*FirehoseBase, error) { 25 | nomadClient, err := nomad.NewClient(nomad.DefaultConfig()) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | sink, err := sink.GetSink("jobs") 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | return &FirehoseBase{ 36 | nomadClient: nomadClient, 37 | sink: sink, 38 | stopCh: make(chan struct{}, 1), 39 | lastChangeTimeCh: make(chan interface{}, 1), 40 | }, nil 41 | } 42 | 43 | func (f *FirehoseBase) UpdateCh() <-chan interface{} { 44 | return f.lastChangeTimeCh 45 | } 46 | 47 | func (f *FirehoseBase) SetRestoreValue(restoreValue interface{}) error { 48 | switch restoreValue.(type) { 49 | case int: 50 | f.lastChangeIndex = uint64(restoreValue.(int)) 51 | case int64: 52 | f.lastChangeIndex = uint64(restoreValue.(int64)) 53 | default: 54 | return fmt.Errorf("Unknown restore type '%T' with value '%+v'", restoreValue, restoreValue) 55 | } 56 | return nil 57 | } 58 | 59 | // Start the firehose 60 | func (f *FirehoseBase) Start(w WatchJobListFunc) { 61 | go f.sink.Start() 62 | 63 | // watch for allocation changes 64 | go f.watch(w) 65 | 66 | // Save the last event time every 5s 67 | go f.persistLastChangeTime(5 * time.Second) 68 | 69 | // wait forever for a stop signal to happen 70 | select { 71 | case <-f.stopCh: 72 | return 73 | } 74 | } 75 | 76 | // Stop the firehose 77 | func (f *FirehoseBase) Stop() { 78 | close(f.stopCh) 79 | f.sink.Stop() 80 | } 81 | 82 | // Write the Last Change Time to Consul so if the process restarts, 83 | // it will try to resume from where it left off, not emitting tons of double events for 84 | // old events 85 | func (f *FirehoseBase) persistLastChangeTime(interval time.Duration) { 86 | ticker := time.NewTicker(interval) 87 | 88 | for { 89 | select { 90 | case <-f.stopCh: 91 | f.lastChangeTimeCh <- f.lastChangeIndex 92 | break 93 | case <-ticker.C: 94 | f.lastChangeTimeCh <- f.lastChangeIndex 95 | } 96 | } 97 | } 98 | 99 | // Continously watch for changes to the allocation list and publish it as updates 100 | func (f *FirehoseBase) watch(w WatchJobListFunc) { 101 | q := &nomad.QueryOptions{ 102 | WaitIndex: f.lastChangeIndex, 103 | WaitTime: 5 * time.Minute, 104 | AllowStale: true, 105 | } 106 | 107 | newMax := f.lastChangeIndex 108 | 109 | for { 110 | jobs, meta, err := f.nomadClient.Jobs().List(q) 111 | if err != nil { 112 | log.Errorf("Unable to fetch jobs: %s", err) 113 | time.Sleep(10 * time.Second) 114 | continue 115 | } 116 | 117 | remoteWaitIndex := meta.LastIndex 118 | localWaitIndex := q.WaitIndex 119 | 120 | // Only work if the WaitIndex have changed 121 | if remoteWaitIndex == localWaitIndex { 122 | log.Debugf("Jobs index is unchanged (%d == %d)", remoteWaitIndex, localWaitIndex) 123 | continue 124 | } 125 | 126 | log.Debugf("Jobs index is changed (%d <> %d)", remoteWaitIndex, localWaitIndex) 127 | 128 | // Iterate jobs and find events that have changed since last run 129 | for _, job := range jobs { 130 | if job.ModifyIndex <= f.lastChangeIndex { 131 | continue 132 | } 133 | 134 | if job.ModifyIndex > newMax { 135 | newMax = job.ModifyIndex 136 | } 137 | 138 | w(job) 139 | } 140 | 141 | // Update WaitIndex and Last Change Time for next iteration 142 | q.WaitIndex = meta.LastIndex 143 | f.lastChangeIndex = newMax 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /command/jobs/job.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | nomad "github.com/hashicorp/nomad/api" 7 | log "github.com/sirupsen/logrus" 8 | ) 9 | 10 | // Firehose ... 11 | type JobFirehose struct { 12 | FirehoseBase 13 | } 14 | 15 | // NewFirehose ... 16 | func NewJobFirehose() (*JobFirehose, error) { 17 | base, err := NewFirehoseBase() 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | return &JobFirehose{FirehoseBase: *base}, nil 23 | } 24 | 25 | func (f *JobFirehose) Name() string { 26 | return "jobs" 27 | } 28 | 29 | // Publish an update from the firehose 30 | func (f *JobFirehose) Publish(update *nomad.Job) { 31 | b, err := json.Marshal(update) 32 | if err != nil { 33 | log.Error(err) 34 | } 35 | 36 | f.sink.Put(b) 37 | } 38 | 39 | func (f *JobFirehose) Start() { 40 | f.FirehoseBase.Start(f.watchJobList) 41 | } 42 | 43 | func (f *JobFirehose) watchJobList(job *nomad.JobListStub) { 44 | go func(jobID string) { 45 | fullJob, _, err := f.nomadClient.Jobs().Info(jobID, &nomad.QueryOptions{}) 46 | if err != nil { 47 | log.Errorf("Could not read job %s: %s", jobID, err) 48 | return 49 | } 50 | 51 | f.Publish(fullJob) 52 | }(job.ID) 53 | } 54 | -------------------------------------------------------------------------------- /command/jobs/jobsliststub.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | nomad "github.com/hashicorp/nomad/api" 7 | log "github.com/sirupsen/logrus" 8 | ) 9 | 10 | // Firehose ... 11 | type JobListStubFirehose struct { 12 | FirehoseBase 13 | } 14 | 15 | // NewFirehose ... 16 | func NewJobListStubFirehose() (*JobListStubFirehose, error) { 17 | base, err := NewFirehoseBase() 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | return &JobListStubFirehose{FirehoseBase: *base}, nil 23 | } 24 | 25 | func (f *JobListStubFirehose) Name() string { 26 | return "jobliststub" 27 | } 28 | 29 | func (f *JobListStubFirehose) Start() { 30 | f.FirehoseBase.Start(f.watchJobList) 31 | } 32 | 33 | // Publish an update from the firehose 34 | func (f *JobListStubFirehose) Publish(update *nomad.JobListStub) { 35 | b, err := json.Marshal(update) 36 | if err != nil { 37 | log.Error(err) 38 | } 39 | 40 | f.sink.Put(b) 41 | } 42 | 43 | func (f *JobListStubFirehose) watchJobList(job *nomad.JobListStub) { 44 | f.Publish(job) 45 | } 46 | -------------------------------------------------------------------------------- /command/nodes/app.go: -------------------------------------------------------------------------------- 1 | package nodes 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "time" 7 | 8 | nomad "github.com/hashicorp/nomad/api" 9 | "github.com/seatgeek/nomad-firehose/sink" 10 | log "github.com/sirupsen/logrus" 11 | ) 12 | 13 | // Firehose ... 14 | type Firehose struct { 15 | lastChangeIndex uint64 16 | lastChangeIndexCh chan interface{} 17 | nomadClient *nomad.Client 18 | sink sink.Sink 19 | stopCh chan struct{} 20 | } 21 | 22 | // NewFirehose ... 23 | func NewFirehose() (*Firehose, error) { 24 | nomadClient, err := nomad.NewClient(nomad.DefaultConfig()) 25 | if err != nil { 26 | return nil, err 27 | } 28 | 29 | sink, err := sink.GetSink("nodes") 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | return &Firehose{ 35 | nomadClient: nomadClient, 36 | sink: sink, 37 | stopCh: make(chan struct{}, 1), 38 | lastChangeIndexCh: make(chan interface{}, 1), 39 | }, nil 40 | } 41 | 42 | func (f *Firehose) Name() string { 43 | return "nodes" 44 | } 45 | 46 | func (f *Firehose) UpdateCh() <-chan interface{} { 47 | return f.lastChangeIndexCh 48 | } 49 | 50 | func (f *Firehose) SetRestoreValue(restoreValue interface{}) error { 51 | switch restoreValue.(type) { 52 | case int: 53 | f.lastChangeIndex = uint64(restoreValue.(int)) 54 | case int64: 55 | f.lastChangeIndex = uint64(restoreValue.(int64)) 56 | default: 57 | return fmt.Errorf("Unknown restore type '%T' with value '%+v'", restoreValue, restoreValue) 58 | } 59 | return nil 60 | } 61 | 62 | // Start the firehose 63 | func (f *Firehose) Start() { 64 | go f.sink.Start() 65 | 66 | // Stop chan for all tasks to depend on 67 | f.stopCh = make(chan struct{}) 68 | 69 | // watch for allocation changes 70 | go f.watch() 71 | 72 | // Save the last event time every 5s 73 | go f.persistLastChangeTime(5 * time.Second) 74 | 75 | // wait forever for a stop signal to happen 76 | select { 77 | case <-f.stopCh: 78 | return 79 | } 80 | } 81 | 82 | // Stop the firehose 83 | func (f *Firehose) Stop() { 84 | close(f.stopCh) 85 | f.sink.Stop() 86 | } 87 | 88 | // Write the Last Change Time to Consul so if the process restarts, 89 | // it will try to resume from where it left off, not emitting tons of double events for 90 | // old events 91 | func (f *Firehose) persistLastChangeTime(interval time.Duration) { 92 | ticker := time.NewTicker(interval) 93 | 94 | for { 95 | select { 96 | case <-f.stopCh: 97 | f.lastChangeIndexCh <- f.lastChangeIndex 98 | break 99 | case <-ticker.C: 100 | f.lastChangeIndexCh <- f.lastChangeIndex 101 | } 102 | } 103 | } 104 | 105 | // Publish an update from the firehose 106 | func (f *Firehose) Publish(update *nomad.Node) { 107 | b, err := json.Marshal(update) 108 | if err != nil { 109 | log.Error(err) 110 | } 111 | 112 | f.sink.Put(b) 113 | } 114 | 115 | // Continously watch for changes to the allocation list and publish it as updates 116 | func (f *Firehose) watch() { 117 | q := &nomad.QueryOptions{ 118 | WaitIndex: f.lastChangeIndex, 119 | WaitTime: 5 * time.Minute, 120 | AllowStale: true, 121 | } 122 | 123 | newMax := f.lastChangeIndex 124 | 125 | for { 126 | clients, meta, err := f.nomadClient.Nodes().List(q) 127 | if err != nil { 128 | log.Errorf("Unable to fetch clients: %s", err) 129 | time.Sleep(10 * time.Second) 130 | continue 131 | } 132 | 133 | remoteWaitIndex := meta.LastIndex 134 | localWaitIndex := q.WaitIndex 135 | 136 | // Only work if the WaitIndex have changed 137 | if remoteWaitIndex == localWaitIndex { 138 | log.Debugf("Clients index is unchanged (%d == %d)", remoteWaitIndex, localWaitIndex) 139 | continue 140 | } 141 | 142 | log.Debugf("Clients index is changed (%d <> %d)", remoteWaitIndex, localWaitIndex) 143 | 144 | // Iterate clients and find events that have changed since last run 145 | for _, client := range clients { 146 | if client.ModifyIndex <= f.lastChangeIndex { 147 | continue 148 | } 149 | 150 | if client.ModifyIndex > newMax { 151 | newMax = client.ModifyIndex 152 | } 153 | 154 | go func(clientId string) { 155 | fullClient, _, err := f.nomadClient.Nodes().Info(clientId, &nomad.QueryOptions{}) 156 | if err != nil { 157 | log.Errorf("Could not read client %s: %s", clientId, err) 158 | return 159 | } 160 | 161 | f.Publish(fullClient) 162 | }(client.ID) 163 | } 164 | 165 | // Update WaitIndex and Last Change Time for next iteration 166 | q.WaitIndex = meta.LastIndex 167 | f.lastChangeIndex = newMax 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/seatgeek/nomad-firehose 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/Shopify/sarama v1.19.0 7 | github.com/Shopify/toxiproxy v2.1.4+incompatible // indirect 8 | github.com/aws/aws-sdk-go v1.25.41 9 | github.com/docker/go-units v0.4.0 // indirect 10 | github.com/eapache/go-resiliency v1.1.0 // indirect 11 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect 12 | github.com/eapache/queue v1.1.0 // indirect 13 | github.com/frankban/quicktest v1.8.1 // indirect 14 | github.com/garyburd/redigo v1.6.0 15 | github.com/go-stack/stack v1.7.0 // indirect 16 | github.com/gorhill/cronexpr v0.0.0-20140423231348-a557574d6c02 // indirect 17 | github.com/hashicorp/consul v1.3.0 18 | github.com/hashicorp/go-uuid v1.0.2 // indirect 19 | github.com/hashicorp/memberlist v0.2.0 // indirect 20 | github.com/hashicorp/nomad v0.8.6 21 | github.com/hashicorp/raft v1.0.1-0.20180117202925-077966dbc90f // indirect 22 | github.com/hashicorp/serf v0.8.2-0.20180809141758-19bbd39e421b // indirect 23 | github.com/hashicorp/vault/api v1.0.4 // indirect 24 | github.com/mitchellh/hashstructure v0.0.0-20160118175604-1ef5c71b025a // indirect 25 | github.com/mongodb/mongo-go-driver v0.0.17 26 | github.com/nsqio/go-nsq v1.0.7 27 | github.com/pierrec/lz4 v2.4.1+incompatible // indirect 28 | github.com/pkg/errors v0.9.1 // indirect 29 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a // indirect 30 | github.com/seatgeek/logrus-gelf-formatter v0.0.0-20180829220724-ce23ecb3f367 31 | github.com/sirupsen/logrus v1.0.2-0.20170719154753-00386b3fbd63 32 | github.com/streadway/amqp v0.0.0-20181107104731-27835f1a64e9 33 | github.com/stretchr/testify v1.5.1 // indirect 34 | github.com/tidwall/pretty v1.0.1 // indirect 35 | github.com/ugorji/go v0.0.0-20170620060102-0053ebfd9d0e // indirect 36 | github.com/xdg/scram v0.0.0-20180714160537-b32d4bd2c91c // indirect 37 | github.com/xdg/stringprep v1.0.1-0.20180714160509-73f8eece6fdc // indirect 38 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2 // indirect 39 | gopkg.in/urfave/cli.v1 v1.20.0 40 | ) 41 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/Shopify/sarama v1.19.0 h1:9oksLxC6uxVPHPVYUmq6xhr1BOF/hHobWH2UzO67z1s= 4 | github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= 5 | github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= 6 | github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= 7 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= 8 | github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= 9 | github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 10 | github.com/aws/aws-sdk-go v1.25.41 h1:/hj7nZ0586wFqpwjNpzWiUTwtaMgxAZNZKHay80MdXw= 11 | github.com/aws/aws-sdk-go v1.25.41/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 12 | github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= 13 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 14 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 16 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 17 | github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= 18 | github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 19 | github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= 20 | github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= 21 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= 22 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= 23 | github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= 24 | github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= 25 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 26 | github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= 27 | github.com/frankban/quicktest v1.8.1 h1:PvpJR0Uq8SdX+zagCMsarBMlhz6ysGTf1+pRmCsRXqY= 28 | github.com/frankban/quicktest v1.8.1/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= 29 | github.com/garyburd/redigo v1.6.0 h1:0VruCpn7yAIIu7pWVClQC8wxCJEcG3nyzpMSHKi1PQc= 30 | github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= 31 | github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= 32 | github.com/go-stack/stack v1.7.0 h1:S04+lLfST9FvL8dl4R31wVUC/paZp/WQZbLmUgWboGw= 33 | github.com/go-stack/stack v1.7.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 34 | github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= 35 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 36 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 37 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 38 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 39 | github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= 40 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 41 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= 42 | github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 43 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 44 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 45 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 46 | github.com/gorhill/cronexpr v0.0.0-20140423231348-a557574d6c02 h1:Spo+4PFAGDqULAsZ7J69MOxq4/fwgZ0zvmDTBqpq7yU= 47 | github.com/gorhill/cronexpr v0.0.0-20140423231348-a557574d6c02/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= 48 | github.com/hashicorp/consul v1.3.0 h1:0ihJs1J8ejURfAbwhwv+USnf4oyqfAddv/3xXXv4ltg= 49 | github.com/hashicorp/consul v1.3.0/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI= 50 | github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= 51 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 52 | github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 53 | github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= 54 | github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= 55 | github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= 56 | github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= 57 | github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= 58 | github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= 59 | github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= 60 | github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= 61 | github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= 62 | github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= 63 | github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= 64 | github.com/hashicorp/go-retryablehttp v0.5.4 h1:1BZvpawXoJCWX6pNtow9+rpEj+3itIlutiqnntI6jOE= 65 | github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= 66 | github.com/hashicorp/go-rootcerts v1.0.1 h1:DMo4fmknnz0E0evoNYnV48RjWndOsmd6OW+09R3cEP8= 67 | github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= 68 | github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= 69 | github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= 70 | github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= 71 | github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 72 | github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 73 | github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= 74 | github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 75 | github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0= 76 | github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 77 | github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 78 | github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= 79 | github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= 80 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 81 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 82 | github.com/hashicorp/memberlist v0.2.0 h1:WeeNspppWi5s1OFefTviPQueC/Bq8dONfvNjPhiEQKE= 83 | github.com/hashicorp/memberlist v0.2.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= 84 | github.com/hashicorp/nomad v0.8.6 h1:z+gocir324zUa88k9bIXkf0RpSgjVa9Izut+iV8T2qg= 85 | github.com/hashicorp/nomad v0.8.6/go.mod h1:WRaKjdO1G2iqi86TvTjIYtKTyxg4pl7NLr9InxtWaI0= 86 | github.com/hashicorp/raft v1.0.1-0.20180117202925-077966dbc90f h1:Th3i4O3JJSY0Etvip3rpQaIxdaJspH3DsDIPPSz1HZQ= 87 | github.com/hashicorp/raft v1.0.1-0.20180117202925-077966dbc90f/go.mod h1:DVSAWItjLjTOkVbSpWQ0j0kUADIvDaCtBxIcbNAQLkI= 88 | github.com/hashicorp/serf v0.8.2-0.20180809141758-19bbd39e421b h1:Psch+6GuKqy4IaGo7xyBgXzOeDY71JROjJgsNVnqnfA= 89 | github.com/hashicorp/serf v0.8.2-0.20180809141758-19bbd39e421b/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE= 90 | github.com/hashicorp/vault/api v1.0.4 h1:j08Or/wryXT4AcHj1oCbMd7IijXcKzYUGw59LGu9onU= 91 | github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= 92 | github.com/hashicorp/vault/sdk v0.1.13 h1:mOEPeOhT7jl0J4AMl1E705+BcmeRs1VmKNb9F0sMLy8= 93 | github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= 94 | github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= 95 | github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= 96 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= 97 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 98 | github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= 99 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 100 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 101 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 102 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 103 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 104 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 105 | github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= 106 | github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= 107 | github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= 108 | github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= 109 | github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= 110 | github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= 111 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 112 | github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= 113 | github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= 114 | github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= 115 | github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= 116 | github.com/mitchellh/hashstructure v0.0.0-20160118175604-1ef5c71b025a h1:2p2+qxgbcNvMPt/Q/Nfd0mEHJokUf7rHAYDVQH1dKpM= 117 | github.com/mitchellh/hashstructure v0.0.0-20160118175604-1ef5c71b025a/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= 118 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 119 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 120 | github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= 121 | github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= 122 | github.com/mongodb/mongo-go-driver v0.0.17 h1:z59HzYE2ACIsX/xnURjlUGQCSEYXDYp0WiLzd+il8KI= 123 | github.com/mongodb/mongo-go-driver v0.0.17/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU= 124 | github.com/nsqio/go-nsq v1.0.7 h1:O0pIZJYTf+x7cZBA0UMY8WxFG79lYTURmWzAAh48ljY= 125 | github.com/nsqio/go-nsq v1.0.7/go.mod h1:XP5zaUs3pqf+Q71EqUJs3HYfBIqfK6G83WQMdNN+Ito= 126 | github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= 127 | github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 128 | github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= 129 | github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= 130 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 131 | github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= 132 | github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 133 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 134 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 135 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 136 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 137 | github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= 138 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= 139 | github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= 140 | github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 141 | github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= 142 | github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= 143 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= 144 | github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= 145 | github.com/seatgeek/logrus-gelf-formatter v0.0.0-20180829220724-ce23ecb3f367 h1:kjJVIlsZ+29dcgTtHVJDtK7ykrf7e1tW/63UvYSCEJw= 146 | github.com/seatgeek/logrus-gelf-formatter v0.0.0-20180829220724-ce23ecb3f367/go.mod h1:/THDZYi7F/BsVEcYzYPqdcWFQ+1C2InkawTKfLOAnzg= 147 | github.com/sirupsen/logrus v1.0.2-0.20170719154753-00386b3fbd63 h1:e1TzkPHDy+DvCa3TPVGHDFxuvwLtY35WZ/FBTCjGb5E= 148 | github.com/sirupsen/logrus v1.0.2-0.20170719154753-00386b3fbd63/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= 149 | github.com/streadway/amqp v0.0.0-20181107104731-27835f1a64e9 h1:xBuwuVDG/vbGv1b0Dn/06flcq0R6MITax8244EZYaKE= 150 | github.com/streadway/amqp v0.0.0-20181107104731-27835f1a64e9/go.mod h1:1WNBiOZtZQLpVAyu0iTduoJL9hEsMloAK5XWrtW0xdY= 151 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 152 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 153 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 154 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 155 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 156 | github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= 157 | github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= 158 | github.com/ugorji/go v0.0.0-20170620060102-0053ebfd9d0e h1:Eurc/1QbldPTy6eU1WSHOH1vuFc7BAUdKDWSsZd4ieo= 159 | github.com/ugorji/go v0.0.0-20170620060102-0053ebfd9d0e/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= 160 | github.com/xdg/scram v0.0.0-20180714160537-b32d4bd2c91c h1:xf5wYj0G/qPQrVU7H4AHS2l3FBtM1xUxmkVTegKUqd8= 161 | github.com/xdg/scram v0.0.0-20180714160537-b32d4bd2c91c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= 162 | github.com/xdg/stringprep v1.0.1-0.20180714160509-73f8eece6fdc h1:vIp1tjhVogU0yBy7w96P027ewvNPeH6gzuNcoc+NReU= 163 | github.com/xdg/stringprep v1.0.1-0.20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= 164 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 165 | golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 h1:ACG4HJsFiNMf47Y4PeRoebLNy/2lXT9EtprMuTFWt1M= 166 | golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= 167 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 168 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 169 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 170 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 171 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 172 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 173 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 174 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 175 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 176 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 177 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= 178 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 179 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= 180 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 181 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 182 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 183 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 184 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 185 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 186 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= 187 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 188 | golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 189 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 190 | golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 191 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 192 | golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 193 | golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 194 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M= 195 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 196 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 197 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 198 | golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 199 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 200 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 201 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= 202 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 203 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 204 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 205 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 206 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 207 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 208 | golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 209 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 210 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 211 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 212 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 213 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 214 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 215 | google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 216 | google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= 217 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 218 | google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 219 | gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= 220 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 221 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 222 | gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= 223 | gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= 224 | gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= 225 | gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= 226 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 227 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 228 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 229 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 230 | -------------------------------------------------------------------------------- /helper/manager.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/signal" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | consulapi "github.com/hashicorp/consul/api" 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | type Runner interface { 16 | Name() string 17 | SetRestoreValue(restoreTime interface{}) error 18 | Start() 19 | Stop() 20 | UpdateCh() <-chan interface{} 21 | } 22 | 23 | func NewManager(r Runner) *Manager { 24 | consulPrefix := "nomad-firehose" 25 | if v, ok := os.LookupEnv("NOMAD_FIREHOSE_CONSUL_PREFIX"); ok { 26 | consulPrefix = strings.TrimSuffix(v, "/") 27 | } 28 | return &Manager{ 29 | runner: r, 30 | logger: log.WithField("type", r.Name()), 31 | stopCh: make(chan interface{}), 32 | voluntarilyReleaseLockCh: make(chan interface{}), 33 | prefix: consulPrefix, 34 | } 35 | } 36 | 37 | type Manager struct { 38 | runner Runner 39 | client *consulapi.Client 40 | lock *consulapi.Lock 41 | lockCh <-chan struct{} // lock channel used by Consul SDK to notify about changes 42 | lockErrorCh <-chan struct{} // lock error channel used by Consul SDK to notify about errors related to the lock 43 | logger *log.Entry // logger for the consul connection struct 44 | prefix string // Consul KV prefix to write state to 45 | stopCh chan interface{} // internal channel used to stop all go-routines when gracefully shutting down 46 | voluntarilyReleaseLockCh chan interface{} 47 | } 48 | 49 | // cleanup will do cleanup tasks when the reconciler is shutting down 50 | func (m *Manager) cleanup() { 51 | m.logger.Debug("Releasing lock") 52 | m.releaseConsulLock() 53 | 54 | m.logger.Debug("Closing stopCh") 55 | close(m.stopCh) 56 | 57 | m.logger.Debugf("Cleanup complete") 58 | } 59 | 60 | // continuouslyAcquireConsulLeadership waits to acquire the lock to the Consul KV key. 61 | // it will run until the stopCh is closed 62 | func (m *Manager) continuouslyAcquireConsulLeadership() error { 63 | m.logger.Info("Starting to continously acquire leadership") 64 | 65 | interval := 250 * time.Millisecond 66 | timer := time.NewTimer(interval) 67 | 68 | for { 69 | select { 70 | // if closed, we should stop working 71 | case <-m.stopCh: 72 | return nil 73 | 74 | // Periodically try to acquire the consul lock 75 | case <-timer.C: 76 | if err := m.acquireConsulLeadership(); err != nil { 77 | return err 78 | } 79 | timer.Reset(interval) 80 | } 81 | } 82 | } 83 | 84 | // Read the Last Change Time from Consul KV, so we don't re-process tasks over and over on restart 85 | func (m *Manager) restoreLastChangeTime() interface{} { 86 | kv, _, err := m.client.KV().Get(fmt.Sprintf("%s/%s.value", m.prefix, m.runner.Name()), nil) 87 | if err != nil { 88 | return 0 89 | } 90 | 91 | // Ensure we got 92 | if kv != nil && kv.Value != nil { 93 | sv := string(kv.Value) 94 | v, err := strconv.ParseInt(sv, 10, 64) 95 | if err != nil { 96 | return 0 97 | } 98 | 99 | log.Infof("Restoring Last Change Time to %s", sv) 100 | return v 101 | } 102 | 103 | log.Info("No Last Change Time restore point, starting from scratch") 104 | return 0 105 | } 106 | 107 | // acquireConsulLeadership will one-off try to acquire the consul lock needed to become 108 | // redis Master node 109 | func (m *Manager) acquireConsulLeadership() error { 110 | var err error 111 | m.lock, err = m.client.LockOpts(&consulapi.LockOptions{ 112 | Key: fmt.Sprintf("%s/%s.lock", m.prefix, m.runner.Name()), 113 | SessionName: fmt.Sprintf("nomad-firehose-%s", m.runner.Name()), 114 | MonitorRetries: 10, 115 | MonitorRetryTime: 5 * time.Second, 116 | }) 117 | if err != nil { 118 | return fmt.Errorf("Failed create lock options: %+v", err) 119 | } 120 | 121 | // try to acquire the lock 122 | m.logger.Infof("Trying to acquire consul lock") 123 | m.lockErrorCh, err = m.lock.Lock(m.lockCh) 124 | if err != nil { 125 | return err 126 | } 127 | 128 | m.logger.Info("Lock successfully acquired") 129 | 130 | // 131 | // start monitoring the consul lock for errors / changes 132 | // 133 | 134 | m.voluntarilyReleaseLockCh = make(chan interface{}) 135 | if err := m.runner.SetRestoreValue(m.restoreLastChangeTime()); err != nil { 136 | return err 137 | } 138 | 139 | go m.runner.Start() 140 | 141 | // At this point, if we return from this function, we need to make sure 142 | // we release the lock 143 | defer func() { 144 | m.runner.Stop() 145 | 146 | err := m.lock.Unlock() 147 | m.handleConsulError(err) 148 | if err != nil { 149 | m.logger.Errorf("Could not release Consul Lock: %v", err) 150 | } else { 151 | m.logger.Info("Consul Lock successfully released") 152 | } 153 | }() 154 | 155 | // Wait for changes to Consul Lock 156 | for { 157 | select { 158 | case v := <-m.runner.UpdateCh(): 159 | var r string 160 | switch v.(type) { 161 | case int: 162 | r = strconv.Itoa(v.(int)) 163 | case int64, uint64: 164 | r = fmt.Sprintf("%d", v) 165 | default: 166 | return fmt.Errorf("Unknown update type '%T' with value '%+v'", v, v) 167 | } 168 | 169 | m.logger.Debugf("Writing lastChangedTime to KV: %s", r) 170 | kv := &consulapi.KVPair{ 171 | Key: fmt.Sprintf("%s/%s.value", m.prefix, m.runner.Name()), 172 | Value: []byte(r), 173 | } 174 | _, err := m.client.KV().Put(kv, nil) 175 | if err != nil { 176 | log.Error(err) 177 | } 178 | 179 | // Global stop of all go-routines, reconciler is shutting down 180 | case <-m.stopCh: 181 | return nil 182 | 183 | // Changes on the lock error channel 184 | // if the channel is closed, it mean that we no longer hold the lock 185 | // if written to, we simply pass on the message 186 | case data, ok := <-m.lockErrorCh: 187 | if !ok { 188 | return fmt.Errorf("Consul Lock error channel was closed, we no longer hold the lock") 189 | } 190 | 191 | m.logger.Warnf("Something wrote to lock error channel %+v", data) 192 | 193 | // voluntarily release our claim on the lock 194 | case <-m.voluntarilyReleaseLockCh: 195 | m.logger.Warnf("Voluntarily releasing the Consul lock") 196 | return nil 197 | } 198 | } 199 | } 200 | 201 | // releaseConsulLock stops consul lock handler") 202 | func (m *Manager) releaseConsulLock() { 203 | m.logger.Info("Releasing Consul lock") 204 | close(m.voluntarilyReleaseLockCh) 205 | } 206 | 207 | // handleConsulError is the error handler 208 | func (m *Manager) handleConsulError(err error) { 209 | // if no error 210 | if err == nil { 211 | return 212 | } 213 | 214 | m.logger.Errorf("Consul error: %v", err) 215 | } 216 | 217 | func (m *Manager) Start() error { 218 | m.logger.Info("Starting manager") 219 | 220 | var err error 221 | m.client, err = consulapi.NewClient(consulapi.DefaultConfig()) 222 | if err != nil { 223 | return err 224 | } 225 | 226 | go m.signalHandler() 227 | return m.continuouslyAcquireConsulLeadership() 228 | } 229 | 230 | // Close the stopCh if we get a signal, so we can gracefully shut down 231 | func (m *Manager) signalHandler() { 232 | m.logger.Info("Starting signal handler") 233 | 234 | c := make(chan os.Signal, 1) 235 | signal.Notify(c, os.Interrupt) 236 | 237 | select { 238 | case <-c: 239 | fmt.Println() 240 | log.Info("Caught signal, releasing lock and stopping...") 241 | m.cleanup() 242 | case <-m.stopCh: 243 | break 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "sort" 6 | 7 | gelf "github.com/seatgeek/logrus-gelf-formatter" 8 | "github.com/seatgeek/nomad-firehose/command/allocations" 9 | "github.com/seatgeek/nomad-firehose/command/deployments" 10 | "github.com/seatgeek/nomad-firehose/command/evaluations" 11 | "github.com/seatgeek/nomad-firehose/command/jobs" 12 | "github.com/seatgeek/nomad-firehose/command/nodes" 13 | "github.com/seatgeek/nomad-firehose/helper" 14 | log "github.com/sirupsen/logrus" 15 | cli "gopkg.in/urfave/cli.v1" 16 | ) 17 | 18 | var GitCommit string 19 | 20 | func main() { 21 | app := cli.NewApp() 22 | app.Name = "nomad-firehose" 23 | app.Usage = "easily firehose nomad events to a event sink" 24 | 25 | app.Version = GitCommit 26 | 27 | app.Flags = []cli.Flag{ 28 | cli.StringFlag{ 29 | Name: "log-level", 30 | Value: "info", 31 | Usage: "Debug level (debug, info, warn/warning, error, fatal, panic)", 32 | EnvVar: "LOG_LEVEL", 33 | }, 34 | cli.StringFlag{ 35 | Name: "log-format", 36 | Value: "text", 37 | Usage: "json or text", 38 | EnvVar: "LOG_FORMAT", 39 | }, 40 | } 41 | app.Commands = []cli.Command{ 42 | { 43 | Name: "allocations", 44 | Usage: "Firehose nomad allocation changes", 45 | Action: func(c *cli.Context) error { 46 | firehose, err := allocations.NewFirehose() 47 | if err != nil { 48 | log.Fatal(err) 49 | } 50 | 51 | manager := helper.NewManager(firehose) 52 | if err := manager.Start(); err != nil { 53 | log.Fatal(err) 54 | } 55 | 56 | return nil 57 | }, 58 | }, 59 | { 60 | Name: "nodes", 61 | Usage: "Firehose nomad node changes", 62 | Action: func(c *cli.Context) error { 63 | firehose, err := nodes.NewFirehose() 64 | if err != nil { 65 | log.Fatal(err) 66 | } 67 | 68 | manager := helper.NewManager(firehose) 69 | if err := manager.Start(); err != nil { 70 | log.Fatal(err) 71 | } 72 | 73 | return nil 74 | }, 75 | }, 76 | { 77 | Name: "evaluations", 78 | Usage: "Firehose nomad evaluation changes", 79 | Action: func(c *cli.Context) error { 80 | firehose, err := evaluations.NewFirehose() 81 | if err != nil { 82 | log.Fatal(err) 83 | } 84 | 85 | manager := helper.NewManager(firehose) 86 | if err := manager.Start(); err != nil { 87 | log.Fatal(err) 88 | } 89 | 90 | return nil 91 | }, 92 | }, 93 | { 94 | Name: "jobs", 95 | Usage: "Firehose nomad job changes", 96 | Action: func(c *cli.Context) error { 97 | firehose, err := jobs.NewJobFirehose() 98 | if err != nil { 99 | log.Fatal(err) 100 | } 101 | 102 | manager := helper.NewManager(firehose) 103 | if err := manager.Start(); err != nil { 104 | log.Fatal(err) 105 | } 106 | 107 | return nil 108 | }, 109 | }, 110 | { 111 | Name: "jobliststubs", 112 | Usage: "Firehose nomad job info changes", 113 | Action: func(c *cli.Context) error { 114 | firehose, err := jobs.NewJobListStubFirehose() 115 | if err != nil { 116 | log.Fatal(err) 117 | } 118 | 119 | manager := helper.NewManager(firehose) 120 | if err := manager.Start(); err != nil { 121 | log.Fatal(err) 122 | } 123 | 124 | return nil 125 | }, 126 | }, 127 | { 128 | Name: "deployments", 129 | Usage: "Firehose nomad deployment changes", 130 | Action: func(c *cli.Context) error { 131 | firehose, err := deployments.NewFirehose() 132 | if err != nil { 133 | log.Fatal(err) 134 | } 135 | 136 | manager := helper.NewManager(firehose) 137 | if err := manager.Start(); err != nil { 138 | log.Fatal(err) 139 | } 140 | 141 | return nil 142 | }, 143 | }, 144 | } 145 | app.Before = func(c *cli.Context) error { 146 | // convert the human passed log level into logrus levels 147 | level, err := log.ParseLevel(c.String("log-level")) 148 | if err != nil { 149 | log.Fatal(err) 150 | } 151 | log.SetLevel(level) 152 | log.SetOutput(os.Stderr) 153 | 154 | if c.String("log-format") == "json" { 155 | log.SetFormatter(&log.JSONFormatter{}) 156 | } 157 | 158 | if c.String("log-format") == "seatgeek-json" { 159 | log.SetFormatter(&log.JSONFormatter{ 160 | FieldMap: log.FieldMap{ 161 | log.FieldKeyTime: "@timestamp", 162 | log.FieldKeyLevel: "@level", 163 | log.FieldKeyMsg: "@message", 164 | }, 165 | }) 166 | } 167 | 168 | if c.String("log-format") == "gelf" { 169 | log.SetFormatter(&gelf.GelfFormatter{}) 170 | } 171 | 172 | return nil 173 | } 174 | 175 | sort.Sort(cli.FlagsByName(app.Flags)) 176 | app.Run(os.Args) 177 | } 178 | -------------------------------------------------------------------------------- /sink/event_bridge.go: -------------------------------------------------------------------------------- 1 | package sink 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/eventbridge" 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | // EventBridge ... 15 | type EBSink struct { 16 | session *session.Session 17 | eventbridge *eventbridge.EventBridge 18 | busName string 19 | detailType string 20 | source string 21 | stopCh chan interface{} 22 | putCh chan []byte 23 | batchCh chan [][]byte 24 | } 25 | 26 | // New Event Bus ... 27 | func NewEventBus() (*EBSink, error) { 28 | busName := os.Getenv("SINK_EVENT_BUS_NAME") 29 | detailType := os.Getenv("SINK_EVENT_BUS_DETAIL_TYPE") 30 | ebSource := os.Getenv("SINK_EVENT_BUS_SOURCE") 31 | 32 | if busName == "" { 33 | return nil, fmt.Errorf("[sink/eventbridge] Missing SINK_EVENT_BUS_NAME") 34 | } 35 | 36 | sess := session.Must(session.NewSession()) 37 | svc := eventbridge.New(sess) 38 | 39 | req, _ := svc.DescribeEventBusRequest(&eventbridge.DescribeEventBusInput{ 40 | Name: aws.String(busName), 41 | }) 42 | 43 | err := req.Send() 44 | 45 | if err != nil { 46 | return nil, fmt.Errorf("Failed to find Event Bus: %s", err) 47 | } 48 | 49 | return &EBSink{ 50 | session: sess, 51 | eventbridge: svc, 52 | busName: busName, 53 | detailType: detailType, 54 | source: ebSource, 55 | stopCh: make(chan interface{}), 56 | putCh: make(chan []byte, 1000), 57 | batchCh: make(chan [][]byte, 100), 58 | }, nil 59 | } 60 | 61 | // Start ... 62 | func (s *EBSink) Start() error { 63 | // Stop chan for all tasks to depend on 64 | s.stopCh = make(chan interface{}) 65 | 66 | go s.batch() 67 | go s.write() 68 | 69 | // wait forever for a stop signal to happen 70 | for { 71 | select { 72 | case <-s.stopCh: 73 | break 74 | } 75 | break 76 | } 77 | 78 | return nil 79 | } 80 | 81 | // Stop ... 82 | func (s *EBSink) Stop() { 83 | log.Infof("[sink/eventbridge] ensure writer queue is empty (%d messages left)", len(s.putCh)) 84 | 85 | for len(s.putCh) > 0 { 86 | log.Infof("[sink/eventbridge] Waiting for queue to drain - (%d messages left)", len(s.putCh)) 87 | time.Sleep(1 * time.Second) 88 | } 89 | 90 | close(s.stopCh) 91 | } 92 | 93 | // Put .. 94 | func (s *EBSink) Put(data []byte) error { 95 | s.putCh <- data 96 | 97 | return nil 98 | } 99 | 100 | func (s *EBSink) batch() { 101 | buffer := make([][]byte, 0) 102 | ticker := time.NewTicker(1 * time.Second) 103 | 104 | for { 105 | select { 106 | case data := <-s.putCh: 107 | buffer = append(buffer, data) 108 | 109 | if len(buffer) == 10 { 110 | s.batchCh <- buffer 111 | buffer = make([][]byte, 0) 112 | } 113 | 114 | case _ = <-ticker.C: 115 | // If there is anything else in the putCh, wait a little longer 116 | if len(s.putCh) > 0 { 117 | continue 118 | } 119 | 120 | if len(buffer) > 0 { 121 | s.batchCh <- buffer 122 | buffer = make([][]byte, 0) 123 | } 124 | } 125 | } 126 | } 127 | 128 | func (s *EBSink) write() { 129 | log.Infof("[sink/eventbridge] Starting writer") 130 | 131 | for { 132 | select { 133 | case batch := <-s.batchCh: 134 | entries := make([]*eventbridge.PutEventsRequestEntry, 0) 135 | 136 | for _, data := range batch { 137 | entry := &eventbridge.PutEventsRequestEntry{ 138 | EventBusName: aws.String(s.busName), 139 | Detail: aws.String(string(data)), 140 | DetailType: aws.String(s.detailType), 141 | Source: aws.String(s.source), 142 | } 143 | 144 | entries = append(entries, entry) 145 | } 146 | 147 | err := s.sendBatch(entries) 148 | 149 | if err != nil { 150 | log.Errorf("[sink/eventbridge] %s", err) 151 | } else { 152 | log.Infof("[sink/eventbridge] queued %d messages", len(batch)) 153 | } 154 | } 155 | } 156 | } 157 | 158 | func (s *EBSink) sendBatch(entries []*eventbridge.PutEventsRequestEntry) error { 159 | req, _ := s.eventbridge.PutEventsRequest(&eventbridge.PutEventsInput{ 160 | Entries: entries, 161 | }) 162 | err := req.Send() 163 | if err == nil { 164 | return err 165 | } 166 | 167 | return err 168 | } 169 | -------------------------------------------------------------------------------- /sink/helper.go: -------------------------------------------------------------------------------- 1 | package sink 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | // GetSink ... 9 | func GetSink(resourceName string) (Sink, error) { 10 | sinkType := os.Getenv("SINK_TYPE") 11 | if sinkType == "" { 12 | return nil, fmt.Errorf("Missing SINK_TYPE: amqp, http, kafka, kinesis, mongodb, nsq, rabbitmq, redis, sqs, stdout, syslog, eventbridge") 13 | } 14 | 15 | switch sinkType { 16 | case "amqp": 17 | return NewRabbitmq() 18 | case "http": 19 | return NewHttp() 20 | case "kafka": 21 | return NewKafka() 22 | case "kinesis": 23 | return NewKinesis() 24 | case "mongodb": 25 | return NewMongodb() 26 | case "nsq": 27 | return NewNSQ() 28 | case "rabbitmq": 29 | return NewRabbitmq() 30 | case "redis": 31 | return NewRedis() 32 | case "stdout": 33 | return NewStdout() 34 | case "syslog": 35 | return NewSyslog() 36 | case "sqs": 37 | return NewSQS(resourceName) 38 | case "eventbridge": 39 | return NewEventBus() 40 | default: 41 | return nil, fmt.Errorf("Invalid SINK_TYPE: %s, Valid values: amqp, http, kafka, kinesis, mongodb, nsq, rabbitmq, redis, sqs, eventbridge, stdout, syslog", sinkType) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /sink/http.go: -------------------------------------------------------------------------------- 1 | package sink 2 | 3 | import ( 4 | "bytes" 5 | "strconv" 6 | "time" 7 | 8 | "net/http" 9 | 10 | "os" 11 | 12 | "fmt" 13 | 14 | log "github.com/sirupsen/logrus" 15 | ) 16 | 17 | // HttpSink ... 18 | type HttpSink struct { 19 | address string 20 | workerCount int 21 | stopCh chan interface{} 22 | putCh chan []byte 23 | } 24 | 25 | // NewHttp ... 26 | func NewHttp() (*HttpSink, error) { 27 | address := os.Getenv("SINK_HTTP_ADDRESS") 28 | if address == "" { 29 | return nil, fmt.Errorf("[sink/http] Missing SINK_HTTP_ADDRESS (example: http://miau.com:8080/biau)") 30 | } 31 | 32 | workerCountStr := os.Getenv("SINK_WORKER_COUNT") 33 | if workerCountStr == "" { 34 | workerCountStr = "1" 35 | } 36 | workerCount, err := strconv.Atoi(workerCountStr) 37 | if err != nil { 38 | return nil, fmt.Errorf("Invalid SINK_WORKER_COUNT, must be an integer") 39 | } 40 | 41 | return &HttpSink{ 42 | address: address, 43 | workerCount: workerCount, 44 | stopCh: make(chan interface{}), 45 | putCh: make(chan []byte, 1000), 46 | }, nil 47 | } 48 | 49 | // Start ... 50 | func (s *HttpSink) Start() error { 51 | // Stop chan for all tasks to depend on 52 | s.stopCh = make(chan interface{}) 53 | 54 | for i := 0; i < s.workerCount; i++ { 55 | go s.send(i) 56 | } 57 | 58 | // wait forever for a stop signal to happen 59 | for { 60 | select { 61 | case <-s.stopCh: 62 | break 63 | } 64 | break 65 | } 66 | 67 | return nil 68 | } 69 | 70 | // Stop ... 71 | func (s *HttpSink) Stop() { 72 | log.Infof("[sink/http] ensure writer queue is empty (%d messages left)", len(s.putCh)) 73 | 74 | for len(s.putCh) > 0 { 75 | log.Infof("[sink/http] Waiting for queue to drain - (%d messages left)", len(s.putCh)) 76 | time.Sleep(1 * time.Second) 77 | } 78 | 79 | close(s.stopCh) 80 | } 81 | 82 | // Put .. 83 | func (s *HttpSink) Put(data []byte) error { 84 | s.putCh <- data 85 | 86 | return nil 87 | } 88 | 89 | func (s *HttpSink) send(id int) { 90 | log.Infof("[sink/http/%d] Starting writer", id) 91 | 92 | for { 93 | select { 94 | case data := <-s.putCh: 95 | resp, err := http.Post(s.address, "application/json; charset=utf-8", bytes.NewBuffer(data[:])) 96 | if err != nil { 97 | log.Errorf("[sink/http/%d] %s", id, err) 98 | } else { 99 | defer resp.Body.Close() 100 | log.Debugf("[sink/http/%d] publish ok", id) 101 | } 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /sink/kafka.go: -------------------------------------------------------------------------------- 1 | package sink 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "strings" 10 | "time" 11 | 12 | "github.com/Shopify/sarama" 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | // KafkaSink ... 17 | type KafkaSink struct { 18 | // Kafka brokers to send metrics to 19 | Brokers []string 20 | // Kafka topic 21 | Topic string 22 | 23 | producer sarama.SyncProducer 24 | 25 | stopCh chan interface{} 26 | putCh chan []byte 27 | } 28 | 29 | func createTlsConfiguration() (t *tls.Config) { 30 | caFile := os.Getenv("SINK_KAFKA_CA_CERT_PATH") 31 | if caFile != "" { 32 | caCert, err := ioutil.ReadFile(caFile) 33 | if err != nil { 34 | log.Fatal(err) 35 | } 36 | 37 | caCertPool := x509.NewCertPool() 38 | caCertPool.AppendCertsFromPEM(caCert) 39 | 40 | t = &tls.Config{ 41 | RootCAs: caCertPool, 42 | } 43 | } 44 | 45 | return t 46 | } 47 | 48 | // NewKafka ... 49 | func NewKafka() (*KafkaSink, error) { 50 | brokers := os.Getenv("SINK_KAFKA_BROKERS") 51 | if brokers == "" { 52 | return nil, fmt.Errorf("[sink/kafka] Missing SINK_KAFKA_BROKERS") 53 | } 54 | 55 | brokerList := strings.Split(brokers, ",") 56 | log.Debugf("[sink/kafka] Kafka brokers: %s", strings.Join(brokerList, ", ")) 57 | 58 | topic := os.Getenv("SINK_KAFKA_TOPIC") 59 | if topic == "" { 60 | return nil, fmt.Errorf("[sink/kafka] Missing SINK_KAFKA_TOPIC") 61 | } 62 | log.Debugf("[sink/kafka] Kafka topic: %s", topic) 63 | 64 | config := sarama.NewConfig() 65 | config.Producer.Return.Successes = true 66 | 67 | tlsConfig := createTlsConfiguration() 68 | if tlsConfig != nil { 69 | config.Net.TLS.Config = tlsConfig 70 | config.Net.TLS.Enable = true 71 | } 72 | 73 | user := os.Getenv("SINK_KAFKA_USER") 74 | if user != "" { 75 | password := os.Getenv("SINK_KAFKA_PASSWORD") 76 | config.Net.SASL.Enable = true 77 | config.Net.SASL.User = user 78 | config.Net.SASL.Password = password 79 | } 80 | 81 | producer, err := sarama.NewSyncProducer(brokerList, config) 82 | if err != nil { 83 | log.Fatal(err) 84 | os.Exit(1) 85 | } 86 | 87 | return &KafkaSink{ 88 | Brokers: brokerList, 89 | Topic: topic, 90 | producer: producer, 91 | stopCh: make(chan interface{}), 92 | putCh: make(chan []byte, 1000), 93 | }, nil 94 | } 95 | 96 | // Start ... 97 | func (s *KafkaSink) Start() error { 98 | // Stop chan for all tasks to depend on 99 | s.stopCh = make(chan interface{}) 100 | 101 | go s.write() 102 | 103 | return nil 104 | } 105 | 106 | // Stop ... 107 | func (s *KafkaSink) Stop() { 108 | log.Debugf("[sink/kafka] ensure writer queue is empty (%d messages left)", len(s.putCh)) 109 | 110 | for len(s.putCh) > 0 { 111 | log.Debugf("[sink/kafka] Waiting for queue to drain - (%d messages left)", len(s.putCh)) 112 | time.Sleep(1 * time.Second) 113 | } 114 | 115 | close(s.stopCh) 116 | } 117 | 118 | // Put .. 119 | func (s *KafkaSink) Put(data []byte) error { 120 | s.putCh <- data 121 | 122 | return nil 123 | } 124 | 125 | func (s *KafkaSink) write() { 126 | log.Info("[sink/kafka] Starting writer") 127 | 128 | for { 129 | select { 130 | case data := <-s.putCh: 131 | message := &sarama.ProducerMessage{Topic: s.Topic} 132 | message.Value = sarama.StringEncoder(string(data)) 133 | partition, offset, err := s.producer.SendMessage(message) 134 | if err != nil { 135 | log.Errorf("Failed to produce message: %s", err) 136 | } else { 137 | log.Debugf("[sink/kafka] topic=%s\tpartition=%d\toffset=%d\n", s.Topic, partition, offset) 138 | } 139 | } 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /sink/kinesis.go: -------------------------------------------------------------------------------- 1 | package sink 2 | 3 | import ( 4 | "time" 5 | 6 | "os" 7 | 8 | "fmt" 9 | 10 | "github.com/aws/aws-sdk-go/aws" 11 | "github.com/aws/aws-sdk-go/aws/session" 12 | "github.com/aws/aws-sdk-go/service/kinesis" 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | // KinesisSink ... 17 | type KinesisSink struct { 18 | session *session.Session 19 | kinesis *kinesis.Kinesis 20 | streamName string 21 | partitionKey string 22 | stopCh chan interface{} 23 | putCh chan []byte 24 | } 25 | 26 | // NewKinesis ... 27 | func NewKinesis() (*KinesisSink, error) { 28 | streamName := os.Getenv("SINK_KINESIS_STREAM_NAME") 29 | if streamName == "" { 30 | return nil, fmt.Errorf("[sink/kinesis] Missing SINK_KINESIS_STREAM_NAME") 31 | } 32 | 33 | partitionKey := os.Getenv("SINK_KINESIS_PARTITION_KEY") 34 | if partitionKey == "" { 35 | return nil, fmt.Errorf("[sink/kinesis] Missing SINK_KINESIS_PARTITION_KEY") 36 | } 37 | 38 | sess := session.Must(session.NewSession()) 39 | svc := kinesis.New(sess) 40 | 41 | return &KinesisSink{ 42 | session: sess, 43 | kinesis: svc, 44 | streamName: streamName, 45 | partitionKey: partitionKey, 46 | stopCh: make(chan interface{}), 47 | putCh: make(chan []byte, 1000), 48 | }, nil 49 | } 50 | 51 | // Start ... 52 | func (s *KinesisSink) Start() error { 53 | // Stop chan for all tasks to depend on 54 | s.stopCh = make(chan interface{}) 55 | 56 | // have 3 writers to kinesis 57 | go s.write(1) 58 | go s.write(2) 59 | go s.write(3) 60 | 61 | // wait forever for a stop signal to happen 62 | for { 63 | select { 64 | case <-s.stopCh: 65 | break 66 | } 67 | break 68 | } 69 | 70 | return nil 71 | } 72 | 73 | // Stop ... 74 | func (s *KinesisSink) Stop() { 75 | log.Infof("[sink/kinesis] ensure writer queue is empty (%d messages left)", len(s.putCh)) 76 | 77 | for len(s.putCh) > 0 { 78 | log.Infof("[sink/kinesis] Waiting for queue to drain - (%d messages left)", len(s.putCh)) 79 | time.Sleep(1 * time.Second) 80 | } 81 | 82 | close(s.stopCh) 83 | } 84 | 85 | // Put .. 86 | func (s *KinesisSink) Put(data []byte) error { 87 | s.putCh <- data 88 | 89 | return nil 90 | } 91 | 92 | func (s *KinesisSink) write(id int) { 93 | log.Infof("[sink/kinesis/%d] Starting writer", id) 94 | 95 | streamName := aws.String(s.streamName) 96 | partitionKey := aws.String(s.partitionKey) 97 | 98 | for { 99 | select { 100 | case data := <-s.putCh: 101 | putOutput, err := s.kinesis.PutRecord(&kinesis.PutRecordInput{ 102 | Data: data, 103 | StreamName: streamName, 104 | PartitionKey: partitionKey, 105 | }) 106 | 107 | if err != nil { 108 | log.Errorf("[sink/kinesis/%d] %s", id, err) 109 | } else { 110 | log.Infof("[sink/kinesis/%d] %v", id, putOutput) 111 | } 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /sink/mongodb.go: -------------------------------------------------------------------------------- 1 | package sink 2 | 3 | import ( 4 | "context" 5 | "strconv" 6 | "time" 7 | 8 | "os" 9 | 10 | "encoding/json" 11 | "fmt" 12 | 13 | log "github.com/sirupsen/logrus" 14 | 15 | mongo "github.com/mongodb/mongo-go-driver/mongo" 16 | ) 17 | 18 | // MongodbSink ... 19 | type MongodbSink struct { 20 | conn *mongo.Client 21 | database string 22 | collection string 23 | workerCount int 24 | stopCh chan interface{} 25 | putCh chan []byte 26 | } 27 | 28 | // NewMongodb ... 29 | func NewMongodb() (*MongodbSink, error) { 30 | connStr := os.Getenv("SINK_MONGODB_CONNECTION") 31 | if connStr == "" { 32 | return nil, fmt.Errorf("[sink/mongodb] Missing SINK_MONGODB_CONNECTION (example: mongodb://foo:bar@localhost:27017)") 33 | } 34 | 35 | database := os.Getenv("SINK_MONGODB_DATABASE") 36 | if database == "" { 37 | return nil, fmt.Errorf("[sink/mongodb] Mising SINK_MONGODB_DATABASE") 38 | } 39 | 40 | collection := os.Getenv("SINK_MONGODB_COLLECTION") 41 | if collection == "" { 42 | return nil, fmt.Errorf("[sink/mongodb] Missing SINK_MONGODB_COLLECTION") 43 | } 44 | 45 | workerCountStr := os.Getenv("SINK_MONGODB_WORKERS") 46 | if workerCountStr == "" { 47 | workerCountStr = "1" 48 | } 49 | workerCount, err := strconv.Atoi(workerCountStr) 50 | if err != nil { 51 | return nil, fmt.Errorf("Invalid SINK_MONGODB_WORKERS, must be an integer") 52 | } 53 | 54 | conn, err := mongo.NewClient(connStr) 55 | if err != nil { 56 | return nil, fmt.Errorf("[sink/mongodb] Invalid to connect to string: %s", err) 57 | } 58 | 59 | err = conn.Connect(context.Background()) 60 | if err != nil { 61 | return nil, fmt.Errorf("[sink/mongodb] failed to connect to string: %s", err) 62 | } 63 | 64 | return &MongodbSink{ 65 | conn: conn, 66 | database: database, 67 | collection: collection, 68 | workerCount: workerCount, 69 | stopCh: make(chan interface{}), 70 | putCh: make(chan []byte, 1000), 71 | }, nil 72 | } 73 | 74 | // Start ... 75 | func (s *MongodbSink) Start() error { 76 | // Stop chan for all tasks to depend on 77 | s.stopCh = make(chan interface{}) 78 | 79 | for i := 0; i < s.workerCount; i++ { 80 | go s.write(i) 81 | } 82 | 83 | // wait forever for a stop signal to happen 84 | for { 85 | select { 86 | case <-s.stopCh: 87 | break 88 | } 89 | break 90 | } 91 | 92 | return nil 93 | } 94 | 95 | // Stop ... 96 | func (s *MongodbSink) Stop() { 97 | log.Infof("[sink/mongodb] ensure writer queue is empty (%d messages left)", len(s.putCh)) 98 | 99 | for len(s.putCh) > 0 { 100 | log.Infof("[sink/mongodb] Waiting for queue to drain - (%d messages left)", len(s.putCh)) 101 | time.Sleep(1 * time.Second) 102 | } 103 | 104 | close(s.stopCh) 105 | defer s.conn.Disconnect(context.Background()) 106 | } 107 | 108 | // Put .. 109 | func (s *MongodbSink) Put(data []byte) error { 110 | s.putCh <- data 111 | 112 | return nil 113 | } 114 | 115 | func (s *MongodbSink) write(id int) { 116 | log.Infof("[sink/mongodb/%d] Starting writer", id) 117 | 118 | collection := s.conn.Database(s.database).Collection(s.collection) 119 | 120 | for { 121 | select { 122 | case data := <-s.putCh: 123 | m := make(map[string]interface{}) 124 | err := json.Unmarshal(data, &m) 125 | 126 | if err != nil { 127 | log.Errorf("[sink/mongodb/%d] %s", id, err) 128 | continue 129 | } 130 | _, err = collection.InsertOne(context.Background(), m) 131 | if err != nil { 132 | log.Errorf("[sink/mongodb/%d] %s", id, err) 133 | } else { 134 | log.Debugf("[sink/mongodb/%d] publish ok", id) 135 | } 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /sink/nsq.go: -------------------------------------------------------------------------------- 1 | package sink 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/nsqio/go-nsq" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | type NSQSink struct { 13 | producer *nsq.Producer 14 | topicName string 15 | stopCh chan interface{} 16 | putCh chan []byte 17 | } 18 | 19 | func NewNSQ() (*NSQSink, error) { 20 | 21 | addrNSQ := os.Getenv("SINK_NSQ_ADDR") 22 | if addrNSQ == "" { 23 | return nil, fmt.Errorf("[sink/nsq] Missing SINK_NSQ_ADDR (example: 127.0.0.1:4150)") 24 | } 25 | log.Infof("[sink/nsq] SINK_NSQ_ADDR=%s", addrNSQ) 26 | 27 | topicName := os.Getenv("SINK_NSQ_TOPIC_NAME") 28 | if topicName == "" { 29 | return nil, fmt.Errorf("[sink/nsq] Missing SINK_NSQ_TOPIC_NAME (example: nomad-firehose)") 30 | } 31 | log.Infof("[sink/nsq] SINK_NSQ_TOPIC_NAME=%s", topicName) 32 | 33 | conf := nsq.NewConfig() 34 | producer, err := nsq.NewProducer(addrNSQ, conf) 35 | if err != nil { 36 | return nil, fmt.Errorf("[sink/nsq] Failed to connect to NSQ: %v", err) 37 | } 38 | 39 | return &NSQSink{ 40 | producer: producer, 41 | topicName: topicName, 42 | stopCh: make(chan interface{}), 43 | putCh: make(chan []byte, 1000), 44 | }, nil 45 | } 46 | 47 | func (s *NSQSink) Start() error { 48 | // Stop chan for all tasks to depend on 49 | s.stopCh = make(chan interface{}) 50 | 51 | // have 1 writer to NSQ 52 | go s.write(1) 53 | 54 | // wait forever for a stop signal to happen 55 | for { 56 | select { 57 | case <-s.stopCh: 58 | break 59 | } 60 | break 61 | } 62 | 63 | return nil 64 | } 65 | 66 | func (s *NSQSink) Stop() { 67 | log.Infof("[sink/nsq] ensure write queue is empty (%d messages left)", len(s.putCh)) 68 | 69 | for len(s.putCh) > 0 { 70 | log.Infof("[sink/nsq] Waiting for queue to drain - (%d messages left)", len(s.putCh)) 71 | time.Sleep(1 * time.Second) 72 | } 73 | 74 | close(s.stopCh) 75 | } 76 | 77 | func (s *NSQSink) Put(data []byte) error { 78 | s.putCh <- data 79 | 80 | return nil 81 | } 82 | 83 | func (s *NSQSink) write(id int) { 84 | log.Infof("[sink/nsq/%d] Starting writer", id) 85 | 86 | for { 87 | select { 88 | case data := <-s.putCh: 89 | if err := s.producer.Publish(s.topicName, data); err != nil { 90 | log.Infof("[sink/nsq/%d] %s", id, err) 91 | } else { 92 | log.Infof("[sink/nsq/%d] Publish OK", id) 93 | } 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /sink/rabbitmq.go: -------------------------------------------------------------------------------- 1 | package sink 2 | 3 | import ( 4 | "strconv" 5 | "time" 6 | 7 | "os" 8 | 9 | "fmt" 10 | 11 | log "github.com/sirupsen/logrus" 12 | "github.com/streadway/amqp" 13 | ) 14 | 15 | // RabbitmqSink ... 16 | type RabbitmqSink struct { 17 | conn *amqp.Connection 18 | exchange string 19 | routingKey string 20 | workerCount int 21 | stopCh chan interface{} 22 | putCh chan []byte 23 | } 24 | 25 | // NewRabbitmq ... 26 | func NewRabbitmq() (*RabbitmqSink, error) { 27 | connStr := os.Getenv("SINK_AMQP_CONNECTION") 28 | if connStr == "" { 29 | return nil, fmt.Errorf("[sink/amqp] Missing SINK_AMQP_CONNECTION (example: amqp://guest:guest@127.0.0.1:5672/)") 30 | } 31 | 32 | exchange := os.Getenv("SINK_AMQP_EXCHANGE") 33 | if exchange == "" { 34 | return nil, fmt.Errorf("[sink/amqp] Missing SINK_AMQP_EXCHANGE") 35 | } 36 | 37 | routingKey := os.Getenv("SINK_AMQP_ROUTING_KEY") 38 | if routingKey == "" { 39 | return nil, fmt.Errorf("[sink/amqp] Mising SINK_AMQP_ROUTING_KEY") 40 | } 41 | 42 | workerCountStr := os.Getenv("SINK_AMQP_WORKERS") 43 | if workerCountStr == "" { 44 | workerCountStr = "1" 45 | } 46 | workerCount, err := strconv.Atoi(workerCountStr) 47 | if err != nil { 48 | return nil, fmt.Errorf("Invalid SINK_AMQP_WORKERS value, must be an integer") 49 | } 50 | 51 | conn, err := amqp.Dial(connStr) 52 | if err != nil { 53 | return nil, fmt.Errorf("[sink/amqp] Failed to connect to AMQP: %s", err) 54 | } 55 | 56 | return &RabbitmqSink{ 57 | conn: conn, 58 | exchange: exchange, 59 | routingKey: routingKey, 60 | workerCount: workerCount, 61 | stopCh: make(chan interface{}), 62 | putCh: make(chan []byte, 1000), 63 | }, nil 64 | } 65 | 66 | // Start ... 67 | func (s *RabbitmqSink) Start() error { 68 | // Stop chan for all tasks to depend on 69 | s.stopCh = make(chan interface{}) 70 | 71 | for i := 0; i < s.workerCount; i++ { 72 | go s.write(i) 73 | } 74 | 75 | // wait forever for a stop signal to happen 76 | for { 77 | select { 78 | case <-s.stopCh: 79 | break 80 | } 81 | break 82 | } 83 | 84 | return nil 85 | } 86 | 87 | // Stop ... 88 | func (s *RabbitmqSink) Stop() { 89 | log.Infof("[sink/amqp] ensure writer queue is empty (%d messages left)", len(s.putCh)) 90 | 91 | for len(s.putCh) > 0 { 92 | log.Infof("[sink/amqp] Waiting for queue to drain - (%d messages left)", len(s.putCh)) 93 | time.Sleep(1 * time.Second) 94 | } 95 | 96 | close(s.stopCh) 97 | defer s.conn.Close() 98 | } 99 | 100 | // Put .. 101 | func (s *RabbitmqSink) Put(data []byte) error { 102 | s.putCh <- data 103 | 104 | return nil 105 | } 106 | 107 | func (s *RabbitmqSink) write(id int) { 108 | log.Infof("[sink/amqp/%d] Starting writer", id) 109 | 110 | ch, err := s.conn.Channel() 111 | if err != nil { 112 | log.Error(err) 113 | return 114 | } 115 | 116 | defer ch.Close() 117 | 118 | for { 119 | select { 120 | case data := <-s.putCh: 121 | err = ch.Publish( 122 | s.exchange, // exchange 123 | s.routingKey, // routing key 124 | false, // mandatory 125 | false, // immediate 126 | amqp.Publishing{ 127 | ContentType: "application/json", 128 | Body: data, 129 | }) 130 | 131 | if err != nil { 132 | log.Errorf("[sink/amqp/%d] %s", id, err) 133 | } else { 134 | log.Debugf("[sink/amqp/%d] publish ok", id) 135 | } 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /sink/redis.go: -------------------------------------------------------------------------------- 1 | package sink 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "time" 7 | 8 | "github.com/garyburd/redigo/redis" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | // RedisSink ... 13 | type RedisSink struct { 14 | pool *redis.Pool 15 | key string 16 | stopCh chan interface{} 17 | putCh chan []byte 18 | } 19 | 20 | // NewStdout ... 21 | func NewRedis() (*RedisSink, error) { 22 | redisURL := os.Getenv("SINK_REDIS_URL") 23 | if redisURL == "" { 24 | return nil, fmt.Errorf("[sink/redis] Missing SINK_REDIS_URL (example: redis://[user]:[password]@127.0.0.1[:5672]/0)") 25 | } 26 | 27 | redisKey := os.Getenv("SINK_REDIS_KEY") 28 | if redisKey == "" { 29 | return nil, fmt.Errorf("[sink/redis] Missing SINK_REDIS_KEY (example: my-key") 30 | } 31 | 32 | redisPool := redis.Pool{ 33 | MaxIdle: 2, 34 | MaxActive: 2, 35 | IdleTimeout: time.Minute, 36 | Dial: func() (redis.Conn, error) { return redis.DialURL(redisURL) }, 37 | TestOnBorrow: func(c redis.Conn, t time.Time) error { 38 | _, err := c.Do("PING") 39 | return err 40 | }, 41 | } 42 | 43 | return &RedisSink{ 44 | pool: &redisPool, 45 | key: redisKey, 46 | stopCh: make(chan interface{}), 47 | putCh: make(chan []byte, 1000), 48 | }, nil 49 | } 50 | 51 | // Start ... 52 | func (s *RedisSink) Start() error { 53 | // Stop chan for all tasks to depend on 54 | s.stopCh = make(chan interface{}) 55 | 56 | go s.write() 57 | 58 | // wait forever for a stop signal to happen 59 | for { 60 | select { 61 | case <-s.stopCh: 62 | break 63 | } 64 | break 65 | } 66 | 67 | return nil 68 | } 69 | 70 | // Stop ... 71 | func (s *RedisSink) Stop() { 72 | log.Debugf("[sink/redis] ensure writer queue is empty (%d messages left)", len(s.putCh)) 73 | 74 | for len(s.putCh) > 0 { 75 | log.Debugf("[sink/redis] Waiting for queue to drain - (%d messages left)", len(s.putCh)) 76 | time.Sleep(1 * time.Second) 77 | } 78 | 79 | close(s.stopCh) 80 | defer s.pool.Close() 81 | } 82 | 83 | // Put .. 84 | func (s *RedisSink) Put(data []byte) error { 85 | s.putCh <- data 86 | return nil 87 | } 88 | 89 | func (s *RedisSink) write() { 90 | log.Infof("[sink/redis] Starting writer to key '%s'", s.key) 91 | 92 | for { 93 | select { 94 | case data := <-s.putCh: 95 | conn := s.pool.Get() 96 | if _, err := conn.Do("RPUSH", s.key, data); err != nil { 97 | log.Infof("[sink/redis] %s", err) 98 | } else { 99 | log.Infof("[sink/redis] Published to key '%s'", s.key) 100 | } 101 | conn.Close() 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /sink/sqs.go: -------------------------------------------------------------------------------- 1 | package sink 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strconv" 7 | "strings" 8 | "time" 9 | 10 | "github.com/aws/aws-sdk-go/aws" 11 | "github.com/aws/aws-sdk-go/aws/session" 12 | "github.com/aws/aws-sdk-go/service/sqs" 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | // SQS ... 17 | type SQSSink struct { 18 | session *session.Session 19 | sqs *sqs.SQS 20 | queueName string 21 | groupId string 22 | stopCh chan interface{} 23 | putCh chan []byte 24 | batchCh chan [][]byte 25 | } 26 | 27 | // NNewSQS ... 28 | func NewSQS(groupId string) (*SQSSink, error) { 29 | queueName := os.Getenv("SINK_SQS_QUEUE_NAME") 30 | 31 | if queueName == "" { 32 | return nil, fmt.Errorf("[sink/sqs] Missing SINK_SQS_QUEUE_NAME") 33 | } 34 | 35 | sess := session.Must(session.NewSession()) 36 | svc := sqs.New(sess) 37 | 38 | output, err := svc.GetQueueUrl(&sqs.GetQueueUrlInput{ 39 | QueueName: aws.String(queueName), 40 | }) 41 | 42 | if err != nil { 43 | return nil, fmt.Errorf("Failed to find queue: %s", err) 44 | } 45 | 46 | return &SQSSink{ 47 | session: sess, 48 | sqs: svc, 49 | queueName: *output.QueueUrl, 50 | groupId: groupId, 51 | stopCh: make(chan interface{}), 52 | putCh: make(chan []byte, 1000), 53 | batchCh: make(chan [][]byte, 100), 54 | }, nil 55 | } 56 | 57 | // Start ... 58 | func (s *SQSSink) Start() error { 59 | // Stop chan for all tasks to depend on 60 | s.stopCh = make(chan interface{}) 61 | 62 | go s.batch() 63 | go s.write() 64 | 65 | // wait forever for a stop signal to happen 66 | for { 67 | select { 68 | case <-s.stopCh: 69 | break 70 | } 71 | break 72 | } 73 | 74 | return nil 75 | } 76 | 77 | // Stop ... 78 | func (s *SQSSink) Stop() { 79 | log.Infof("[sink/sqs] ensure writer queue is empty (%d messages left)", len(s.putCh)) 80 | 81 | for len(s.putCh) > 0 { 82 | log.Infof("[sink/sqs] Waiting for queue to drain - (%d messages left)", len(s.putCh)) 83 | time.Sleep(1 * time.Second) 84 | } 85 | 86 | close(s.stopCh) 87 | } 88 | 89 | // Put .. 90 | func (s *SQSSink) Put(data []byte) error { 91 | s.putCh <- data 92 | 93 | return nil 94 | } 95 | 96 | func (s *SQSSink) batch() { 97 | buffer := make([][]byte, 0) 98 | ticker := time.NewTicker(1 * time.Second) 99 | 100 | for { 101 | select { 102 | case data := <-s.putCh: 103 | buffer = append(buffer, data) 104 | 105 | if len(buffer) == 10 { 106 | s.batchCh <- buffer 107 | buffer = make([][]byte, 0) 108 | } 109 | 110 | case _ = <-ticker.C: 111 | // If there is anything else in the putCh, wait a little longer 112 | if len(s.putCh) > 0 { 113 | continue 114 | } 115 | 116 | if len(buffer) > 0 { 117 | s.batchCh <- buffer 118 | buffer = make([][]byte, 0) 119 | } 120 | } 121 | } 122 | } 123 | 124 | func (s *SQSSink) write() { 125 | log.Infof("[sink/sqs] Starting writer") 126 | 127 | var id int64 128 | 129 | for { 130 | select { 131 | case batch := <-s.batchCh: 132 | entries := make([]*sqs.SendMessageBatchRequestEntry, 0) 133 | 134 | for _, data := range batch { 135 | mID := aws.String(strconv.FormatInt(id, 10)) 136 | entry := &sqs.SendMessageBatchRequestEntry{ 137 | Id: mID, 138 | MessageBody: aws.String(string(data)), 139 | MessageGroupId: aws.String(s.groupId), 140 | MessageDeduplicationId: mID, 141 | } 142 | 143 | entries = append(entries, entry) 144 | id = id + 1 145 | } 146 | 147 | err := s.sendBatch(entries) 148 | if err != nil && strings.Contains(err.Error(), "AWS.SimpleQueueService.BatchRequestTooLong") { 149 | for i, el := range entries { 150 | err = s.sendBatch([]*sqs.SendMessageBatchRequestEntry{el}) 151 | if err != nil { 152 | log.Errorf("[sink/sqs] Retry failed for %d: %s", i, err) 153 | } else { 154 | log.Infof("[sink/sqs] Retry succeeded for %d", i) 155 | } 156 | } 157 | 158 | continue 159 | } 160 | 161 | if err != nil { 162 | log.Errorf("[sink/sqs] %s", err) 163 | } else { 164 | log.Infof("[sink/sqs] queued %d messages", len(batch)) 165 | } 166 | } 167 | } 168 | } 169 | 170 | func (s *SQSSink) sendBatch(entries []*sqs.SendMessageBatchRequestEntry) error { 171 | _, err := s.sqs.SendMessageBatch(&sqs.SendMessageBatchInput{ 172 | Entries: entries, 173 | QueueUrl: aws.String(s.queueName), 174 | }) 175 | 176 | return err 177 | } 178 | -------------------------------------------------------------------------------- /sink/stdout.go: -------------------------------------------------------------------------------- 1 | package sink 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | log "github.com/sirupsen/logrus" 8 | ) 9 | 10 | // StdoutSink ... 11 | type StdoutSink struct { 12 | stopCh chan interface{} 13 | putCh chan []byte 14 | } 15 | 16 | // NewStdout ... 17 | func NewStdout() (*StdoutSink, error) { 18 | return &StdoutSink{ 19 | stopCh: make(chan interface{}), 20 | putCh: make(chan []byte, 1000), 21 | }, nil 22 | } 23 | 24 | // Start ... 25 | func (s *StdoutSink) Start() error { 26 | // Stop chan for all tasks to depend on 27 | s.stopCh = make(chan interface{}) 28 | 29 | // wait forever for a stop signal to happen 30 | for { 31 | select { 32 | case <-s.stopCh: 33 | break 34 | } 35 | break 36 | } 37 | 38 | return nil 39 | } 40 | 41 | // Stop ... 42 | func (s *StdoutSink) Stop() { 43 | log.Infof("[sink/stdout] ensure writer queue is empty (%d messages left)", len(s.putCh)) 44 | 45 | for len(s.putCh) > 0 { 46 | log.Infof("[sink/stdout] Waiting for queue to drain - (%d messages left)", len(s.putCh)) 47 | time.Sleep(1 * time.Second) 48 | } 49 | 50 | close(s.stopCh) 51 | } 52 | 53 | // Put .. 54 | func (s *StdoutSink) Put(data []byte) error { 55 | fmt.Println(string(data)) 56 | return nil 57 | } 58 | -------------------------------------------------------------------------------- /sink/structs.go: -------------------------------------------------------------------------------- 1 | package sink 2 | 3 | // Sink ... 4 | type Sink interface { 5 | Start() error 6 | Stop() 7 | Put(data []byte) error 8 | } 9 | -------------------------------------------------------------------------------- /sink/syslog_unix.go: -------------------------------------------------------------------------------- 1 | // +build darwin linux 2 | 3 | package sink 4 | 5 | import ( 6 | "fmt" 7 | "log/syslog" 8 | "os" 9 | "time" 10 | 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | type SyslogSink struct { 15 | addr string 16 | proto string 17 | priority syslog.Priority 18 | tag string 19 | stopCh chan interface{} 20 | putCh chan []byte 21 | } 22 | 23 | // NewSyslog ... 24 | func NewSyslog() (*SyslogSink, error) { 25 | syslogProto := os.Getenv("SINK_SYSLOG_PROTO") 26 | // The log/syslog package has some interesting internal behaviours. If we 27 | // *do not* supply the network protocol, syslog.Dial assumes we are 28 | // connecting to a local syslog socket and configures itself for "local" 29 | // mode, which does not include the local hostname in messages written to 30 | // the socket (and avoids breaking a standard). 31 | // 32 | // See: https://github.com/golang/go/commit/87a6d75012986fb8867b746afcd42f742c119945 33 | if syslogProto == "" { 34 | log.Info("[sink/syslog] SINK_SYSLOG_PROTO not set - ignoring this and SINK_SYSLOG_ADDR, as syslog package will default to unixgram and an autodiscovered socket") 35 | } 36 | syslogAddr := os.Getenv("SINK_SYSLOG_ADDR") 37 | if syslogAddr == "" && syslogProto != "" { 38 | return nil, fmt.Errorf("[sink/syslog] Missing SINK_SYSLOG_ADDR (examples: 192.168.1.100:514") 39 | } 40 | syslogTag := os.Getenv("SINK_SYSLOG_TAG") 41 | if syslogTag == "" { 42 | log.Info("[sink/syslog] Missing SINK_SYSLOG_TAG - setting to default 'nomad-firehose'") 43 | syslogTag = "nomad-firehose" 44 | } 45 | 46 | return &SyslogSink{ 47 | addr: syslogAddr, 48 | proto: syslogProto, 49 | tag: syslogTag, 50 | priority: syslog.LOG_INFO, 51 | 52 | stopCh: make(chan interface{}), 53 | putCh: make(chan []byte, 1000), 54 | }, nil 55 | } 56 | 57 | // Start ... 58 | func (s *SyslogSink) Start() error { 59 | // Stop chan for all tasks to depend on 60 | s.stopCh = make(chan interface{}) 61 | 62 | go s.write() 63 | 64 | // wait forever for a stop signal to happen 65 | for { 66 | select { 67 | case <-s.stopCh: 68 | break 69 | } 70 | break 71 | } 72 | 73 | return nil 74 | } 75 | 76 | // Stop ... 77 | func (s *SyslogSink) Stop() { 78 | log.Infof("[sink/syslog] ensure writer queue is empty (%d messages left)", len(s.putCh)) 79 | 80 | for len(s.putCh) > 0 { 81 | log.Infof("[sink/syslog] Waiting for queue to drain - (%d messages left)", len(s.putCh)) 82 | time.Sleep(1 * time.Second) 83 | } 84 | 85 | close(s.stopCh) 86 | } 87 | 88 | // Put .. 89 | func (s *SyslogSink) Put(data []byte) error { 90 | s.putCh <- data 91 | return nil 92 | } 93 | 94 | func (s *SyslogSink) write() error { 95 | log.Infof("[sink/syslog] Starting writer - %s://%s - tag: %s", s.proto, s.addr, s.tag) 96 | writer, err := syslog.Dial(s.proto, s.addr, syslog.LOG_NOTICE, s.tag) 97 | if err != nil { 98 | log.Infof("[sink/syslog] ERROR initializing syslog writer: %q", err) 99 | return err 100 | } 101 | 102 | for { 103 | select { 104 | case data := <-s.putCh: 105 | // fmt.Fprint(writer, string(data)) 106 | _, err := writer.Write(data) 107 | if err != nil { 108 | log.Infof("[sink/syslog] ERROR writing to syslog: %q", err) 109 | } 110 | } 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /sink/syslog_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | package sink 4 | 5 | import ( 6 | "fmt" 7 | ) 8 | 9 | type SyslogSink struct{} 10 | 11 | // NewSyslog ... 12 | func NewSyslog() (*SyslogSink, error) { 13 | return &SyslogSink{}, fmt.Errorf("[sink/syslog] ERROR - not supported on Windows :(") 14 | } 15 | 16 | // Put ... 17 | func (s *SyslogSink) Put(_ []byte) error { 18 | return nil 19 | } 20 | 21 | // Start ... 22 | func (s *SyslogSink) Start() error { 23 | return nil 24 | } 25 | 26 | // Stop ... 27 | func (s *SyslogSink) Stop() { 28 | return 29 | } 30 | --------------------------------------------------------------------------------