├── .github └── workflows │ └── tests.yaml ├── .gitignore ├── .golangci.yaml ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── VERSION ├── bench.out ├── broker.go ├── broker_mock.go ├── broker_test.go ├── conn_mock.go ├── consumer.go ├── consumer_test.go ├── examples ├── bench_consume │ └── main.go ├── echo │ └── main.go ├── exponential_backoff │ └── main.go └── workers │ └── main.go ├── flushwriter.go ├── go.mod ├── go.sum ├── http.go ├── http_test.go ├── main.go ├── main_test.go ├── redis.go ├── redis_test.go ├── response.go ├── store.go ├── store_mock.go ├── store_test.go ├── testdata ├── cmd │ ├── redcli_darwin_arm64 │ └── redcli_linux_amd64 ├── localhost-key.pem └── localhost.pem ├── testutils_test.go └── value.go /.github/workflows/tests.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - master 5 | pull_request: 6 | name: Tests 7 | jobs: 8 | test: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Install Go 12 | uses: actions/setup-go@v3.5.0 13 | with: 14 | go-version: 1.19.x 15 | 16 | - name: Checkout code 17 | uses: actions/checkout@v3.3.0 18 | 19 | - name: Cache Go dependencies 20 | uses: actions/cache@v3.2.4 21 | with: 22 | path: | 23 | ~/go/pkg/mod # Module download cache 24 | ~/.cache/go-build # Build cache (Linux) 25 | ~/Library/Caches/go-build # Build cache (Mac) 26 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 27 | restore-keys: | 28 | ${{ runner.os }}-go- 29 | 30 | - name: Test 31 | # Temporarily disable the race detector until 32 | # https://github.com/tidwall/redcon/pull/62 33 | # run: go test -race ./... 34 | run: go test ./... 35 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | data 2 | certs 3 | miniqueue 4 | -------------------------------------------------------------------------------- /.golangci.yaml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 30s 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Builder image 2 | FROM golang:latest as builder 3 | 4 | COPY . /build 5 | 6 | WORKDIR /build 7 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o miniqueue . 8 | 9 | # Exec image 10 | FROM alpine:latest 11 | 12 | COPY --from=builder /build/miniqueue /miniqueue 13 | 14 | VOLUME /var/lib/miniqueue 15 | 16 | ENTRYPOINT ["/miniqueue"] 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Tom Arrell 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | DOCKER_TAG=$(shell [[ -z "$(shell git tag --points-at HEAD)" ]] && git rev-parse --short HEAD || git tag --points-at HEAD) 2 | 3 | .PHONY: run 4 | run: build ## Run miniqueue docker image built from HEAD 5 | docker run \ 6 | -v $(shell pwd)/testdata:/etc/miniqueue/certs \ 7 | -p 8080:8080 \ 8 | tomarrell/miniqueue:$(DOCKER_TAG) \ 9 | -cert /etc/miniqueue/certs/localhost.pem \ 10 | -key /etc/miniqueue/certs/localhost-key.pem \ 11 | -db /var/lib/miniqueue \ 12 | -human 13 | 14 | .PHONY: build 15 | build: ## Build a docker image with the git tag pointing to current HEAD or the current commit hash. 16 | docker build . -t tomarrell/miniqueue:$(DOCKER_TAG) 17 | 18 | .PHONY: bench 19 | bench: ## Run Go benchmarks 20 | go test -bench=. -run=$$^ 21 | 22 | 23 | ## Help display. 24 | ## Pulls comments from beside commands and prints a nicely formatted 25 | ## display with the commands and their usage information. 26 | 27 | .DEFAULT_GOAL := help 28 | 29 | help: ## Prints this help 30 | @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # miniqueue 2 | 3 | ![Tests](https://github.com/tomarrell/miniqueue/workflows/Tests/badge.svg) 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/tomarrell/miniqueue)](https://goreportcard.com/report/github.com/tomarrell/miniqueue) 5 | ![GitHub release (latest by date)](https://img.shields.io/github/v/release/tomarrell/miniqueue) 6 | 7 | A stupid simple, single binary message queue using HTTP/2 or Redis Protocol. 8 | 9 | Most messaging workloads don't require enormous amounts of data, endless 10 | features or infinite scaling. Instead, they'd probably be better off with 11 | something dead simple. 12 | 13 | miniqueue is just that. A ridiculously simple, high performance queue. You can 14 | publish bytes to topics and be sure that your consumers will receive what you 15 | published, nothing more. 16 | 17 | ## Features 18 | 19 | - Redis Protocol Support 20 | - Simple to run 21 | - Very fast, see [benchmarks](#benchmarks) 22 | - Not infinitely scalable 23 | - Multiple topics 24 | - HTTP/2 25 | - Publish 26 | - Subscribe 27 | - Acknowledgements 28 | - Persistence 29 | - Prometheus metrics [WIP] 30 | 31 | ## API 32 | 33 | ### Redis 34 | 35 | You can communicate with miniqueue using any major Redis library which supports 36 | custom commands. The command set is identical to the HTTP/2 implementation and 37 | listed under the [commands](#commands) heading. 38 | 39 | Examples of using the Redis interface can be found in the 40 | [redis_test.go](./redis_test.go) file. 41 | 42 | ### HTTP/2 43 | 44 | - POST `/publish/:topic`, where the body contains the bytes to publish to the topic. 45 | 46 | ```bash 47 | curl -X POST https://localhost:8080/publish/foo --data "helloworld" 48 | ``` 49 | 50 | - POST `/subscribe/:topic` - streams messages separated by `\n` 51 | 52 | - `client → server: "INIT"` 53 | - `server → client: { "msg": [base64], "error": "...", dackCount: 1 }` 54 | - `client → server: "ACK"` 55 | 56 | - DELETE `/:topic` - deletes the given topic, removing all messages. Note, this 57 | is an expensive operation for large topics. 58 | 59 | You can also find examples in the [`./examples/`](./examples/) directory. 60 | 61 | ## Usage 62 | 63 | miniqueue runs as a single binary, persisting the messages to the filesystem in 64 | a directory specified by the `-db` flag and exposes an HTTP/2 server on the port 65 | specified by the `-port` flag. 66 | 67 | **Note:** As the server uses HTTP/2, TLS is required. For testing, you can 68 | generate a certificate using [mkcert](https://github.com/FiloSottile/mkcert) and 69 | replace the ones in `./testdata` as these will not be trusted by your client, or 70 | specify your own certificate using the `-cert` and `-key` flags. 71 | 72 | ```bash 73 | Usage of ./miniqueue: 74 | -cert string 75 | path to TLS certificate (default "./testdata/localhost.pem") 76 | -db string 77 | path to the db file (default "./miniqueue") 78 | -human 79 | human readable logging output 80 | -key string 81 | path to TLS key (default "./testdata/localhost-key.pem") 82 | -level string 83 | (disabled|debug|info) (default "debug") 84 | -period duration 85 | period between runs to check and restore delayed messages (default 1s) 86 | -port int 87 | port used to run the server (default 8080) 88 | ``` 89 | 90 | Once running, miniqueue will expose an HTTP/2 server capable of bidirectional 91 | streaming between client and server. Subscribers will be delivered incoming 92 | messages and can send commands `ACK`, `NACK`, `BACK` [etc](#commands). Upon a 93 | subscriber disconnecting, any outstanding messages are automatically `NACK`'ed 94 | and returned to the front of the queue. 95 | 96 | Messages sent to subscribers are JSON encoded, containing additional information 97 | in some cases to enable certain features. The consumer payload looks like: 98 | 99 | ```js 100 | { 101 | "msg": "dGVzdA==", // base64 encoded msg 102 | "dackCount": 2, // number of times the msg has been DACK'ed 103 | } 104 | ``` 105 | 106 | In case of an error, the payload will be: 107 | ```js 108 | { 109 | "error": "uh oh, something went wrong" 110 | } 111 | ``` 112 | 113 | To get you started, here are some common ways to get up and running with `miniqueue`. 114 | 115 | ##### Start miniqueue with human readable logs 116 | 117 | ```bash 118 | λ ./miniqueue -human 119 | ``` 120 | 121 | ##### Start miniqueue with custom TLS certificate 122 | 123 | ```bash 124 | λ ./miniqueue -cert ./localhost.pem -key ./localhost-key.pem 125 | ``` 126 | 127 | ##### Start miniqueue on custom port 128 | 129 | ```bash 130 | λ ./miniqueue -port 8081 131 | ``` 132 | 133 | ## Docker 134 | 135 | As of `v0.7.0` there are published miniqueue docker images available in the 136 | Docker hub repository 137 | [`tomarrell/miniqueue`](https://hub.docker.com/repository/docker/tomarrell/miniqueue). 138 | 139 | It is recommended to use a tagged release build. The tag `latest` tracks the 140 | `master` branch. 141 | 142 | With the TLS certificate and key in a relative directory `./certs` (can be 143 | generated using [mkcert](https://github.com/FiloSottile/mkcert)). 144 | 145 | ```bash 146 | ./certs 147 | ├── localhost-key.pem 148 | └── localhost.pem 149 | ``` 150 | 151 | You can execute the following Docker command to run the image. 152 | 153 | ```bash 154 | $ docker run \ 155 | -v $(pwd)/certs:/etc/miniqueue/certs \ 156 | -p 8080:8080 \ 157 | tomarrell/miniqueue:v0.7.0 \ 158 | -cert /etc/miniqueue/certs/localhost.pem \ 159 | -key /etc/miniqueue/certs/localhost-key.pem \ 160 | -db /var/lib/miniqueue \ 161 | -human 162 | ``` 163 | 164 | ## Examples 165 | 166 | To take a look at some common usage, we have compiled some examples for 167 | reference in the [`./examples/`](./examples/) directory. Here you will find 168 | common patterns such as: 169 | 170 | - [Exponential backoff](./examples/exponential_backoff), `1s → 2s → 4s` etc 171 | - Failure resistant [workers](./examples/workers) 172 | - Simple [echo](./examples/echo) 173 | 174 | ## Commands 175 | 176 | A client may send commands to the server over a duplex connection. Commands are 177 | in the form of a **JSON string** to allow for simple encoding/decoding. 178 | 179 | Available commands are: 180 | 181 | - `"INIT"`: Establishes a new consumer on the topic. If you are consuming for 182 | the first time, this should be sent along with the request. 183 | 184 | - `"ACK"`: Acknowledges the current message, popping it from the topic and 185 | removing it. 186 | 187 | - `"NACK"`: Negatively acknowledges the current message, causing it to be 188 | returned to the *front* of the queue. If there is a ready consumer waiting 189 | for a message, it will immediately be delivered to this consumer. Otherwise 190 | it will be delivered as as one becomes available. 191 | 192 | - `"BACK"`: Negatively acknowledges the current message, causing it to be 193 | returned to the *back* of the queue. This will cause it to be processed 194 | again after the currently waiting messages. 195 | 196 | - `"DACK [seconds]"`: Negatively acknowledges the current message, placing it on 197 | a delay for a certain number of `seconds`. Once the delay expires, on the 198 | next tick given by the `-period` flag, the message will be returned to the 199 | front of the queue to be processed as soon as possible. 200 | 201 | DACK'ed messages will contain a `dackCount` key when consumed. This allows 202 | for doing exponential backoff for the same message if multiple failures 203 | occur. 204 | 205 | ## Benchmarks 206 | 207 | As miniqueue is still under development, take these benchmarks with a grain of 208 | salt. However, for those curious: 209 | 210 | **Publish** 211 | ```bash 212 | λ go-wrk -c 12 -d 10 -M POST -body "helloworld" https://localhost:8080/publish/test 213 | Running 10s test @ https://localhost:8080/publish/test 214 | 12 goroutine(s) running concurrently 215 | 142665 requests in 9.919498387s, 7.89MB read 216 | Requests/sec: 14382.28 217 | Transfer/sec: 814.62KB 218 | Avg Req Time: 834.36µs 219 | Fastest Request: 190µs 220 | Slowest Request: 141.091118ms 221 | Number of Errors: 0 222 | ``` 223 | 224 | **Consume + Ack** 225 | ```bash 226 | λ ./bench_consume -duration=10s 227 | consumed 42982 times in 10s 228 | 4298 (consume+ack)/second 229 | ``` 230 | 231 | Running on my MacBook Pro (15-inch, 2019), with a 2.6 GHz 6-Core Intel Core i7 232 | using Go `v1.15`. 233 | 234 | ## Contributing 235 | 236 | Contributors are more than welcome. Please feel free to open a PR to improve anything you don't like, or would like to add. No PR is too small! 237 | 238 | ## License 239 | 240 | This project is licensed under the MIT license. 241 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | v0.8.0 2 | -------------------------------------------------------------------------------- /bench.out: -------------------------------------------------------------------------------- 1 | go test -bench=. -run=$^ 2 | goos: darwin 3 | goarch: amd64 4 | pkg: github.com/tomarrell/miniqueue 5 | cpu: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz 6 | BenchmarkPublish-12 6488 175418 ns/op 7 | BenchmarkPurge-12 2214 909116 ns/op 8 | PASS 9 | ok github.com/tomarrell/miniqueue 3.569s 10 | -------------------------------------------------------------------------------- /broker.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | "github.com/rs/xid" 10 | "github.com/rs/zerolog/log" 11 | ) 12 | 13 | //go:generate mockgen -source=$GOFILE -destination=broker_mock.go -package=main 14 | type brokerer interface { 15 | Publish(topic string, value *value) error 16 | Subscribe(topic string) *consumer 17 | Unsubscribe(topic, id string) error 18 | Purge(topic string) error 19 | Topics() ([]string, error) 20 | } 21 | 22 | type broker struct { 23 | store storer 24 | consumers map[string][]*consumer 25 | sync.RWMutex 26 | } 27 | 28 | func newBroker(store storer) *broker { 29 | return &broker{ 30 | store: store, 31 | consumers: map[string][]*consumer{}, 32 | } 33 | } 34 | 35 | func (b *broker) Topics() ([]string, error) { 36 | meta, err := b.store.Meta() 37 | 38 | return meta.topics, err 39 | } 40 | 41 | // ProcessDelays is a blocking function which starts a loop to check and return 42 | // delayed messages which have completed their designated delay back to the main 43 | // queue. 44 | func (b *broker) ProcessDelays(ctx context.Context, period time.Duration) { 45 | log.Debug().Msg("starting delay queue processing") 46 | 47 | for { 48 | meta, err := b.store.Meta() 49 | if err != nil { 50 | continue 51 | } 52 | 53 | if err := processTopics(b, meta.topics); err != nil { 54 | log.Err(err).Msg("failed to process topics") 55 | } 56 | 57 | select { 58 | case <-time.After(period): 59 | case <-ctx.Done(): 60 | log.Debug().Msg("stopping delay queue processing, context cancelled") 61 | return 62 | } 63 | } 64 | } 65 | 66 | func processTopics(b *broker, topics []string) error { 67 | now := time.Now() 68 | 69 | for _, t := range topics { 70 | count, err := b.store.ReturnDelayed(t, now) 71 | if err != nil { 72 | log.Err(err).Msg("returning delayed messages to main queue") 73 | continue 74 | } 75 | 76 | if count >= 1 { 77 | log.Debug(). 78 | Str("topic", t). 79 | Int("count", count). 80 | Msg("returning delayed messages") 81 | 82 | b.NotifyConsumer(t, eventTypeMsgReturned) 83 | } 84 | } 85 | 86 | return nil 87 | } 88 | 89 | // Publish a message to a topic. 90 | func (b *broker) Publish(topic string, val *value) error { 91 | if err := b.store.Insert(topic, val); err != nil { 92 | return err 93 | } 94 | 95 | b.NotifyConsumer(topic, eventTypePublish) 96 | 97 | return nil 98 | } 99 | 100 | // Subscribe to a topic and return a consumer for the topic. 101 | func (b *broker) Subscribe(topic string) *consumer { 102 | cons := &consumer{ 103 | id: xid.New().String(), 104 | topic: topic, 105 | ackOffset: 0, 106 | store: b.store, 107 | eventChan: make(chan eventType), 108 | notifier: b, 109 | outstanding: false, 110 | } 111 | 112 | b.Lock() 113 | b.consumers[topic] = append(b.consumers[topic], cons) 114 | b.Unlock() 115 | 116 | return cons 117 | } 118 | 119 | // Unsubscribe removes the consumer from the available pool for the topic and 120 | // returns any messages with outstanding acknowledgements to the queue. 121 | func (b *broker) Unsubscribe(topic, id string) error { 122 | b.RLock() 123 | consumers := b.consumers[topic] 124 | b.RUnlock() 125 | 126 | for i, c := range consumers { 127 | if c.id == id { 128 | if c.outstanding { 129 | log.Debug().Str("id", c.id).Msg("nacking outstanding message") 130 | _ = c.Nack() 131 | } 132 | 133 | log.Debug().Str("id", c.id).Msg("unsubscribing consumer") 134 | 135 | b.Lock() 136 | length := len(b.consumers[topic]) 137 | b.consumers[topic][i] = b.consumers[topic][length-1] 138 | b.consumers[topic] = b.consumers[topic][:length-1] 139 | b.Unlock() 140 | 141 | return nil 142 | } 143 | } 144 | 145 | return fmt.Errorf("consumer ID %s not found for topic %s", id, topic) 146 | } 147 | 148 | // Purge removes the topic from the broker. 149 | func (b *broker) Purge(topic string) error { 150 | if err := b.store.Purge(topic); err != nil { 151 | return fmt.Errorf("purging topic in store: %v", err) 152 | } 153 | 154 | return nil 155 | } 156 | 157 | // Shutdown the broker. 158 | func (b *broker) Shutdown() error { 159 | return b.store.Close() 160 | } 161 | 162 | // NotifyConsumers notifies a waiting consumer of a topic that an event has 163 | // occurred. 164 | func (b *broker) NotifyConsumer(topic string, ev eventType) { 165 | b.RLock() 166 | defer b.RUnlock() 167 | 168 | for _, c := range b.consumers[topic] { 169 | select { 170 | case c.eventChan <- ev: 171 | return 172 | default: 173 | // TODO if it fails to send to the consumer, find another consumer to send 174 | // the message to and possibly remove this consumer from the pool. 175 | } 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /broker_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: broker.go 3 | 4 | // Package main is a generated GoMock package. 5 | package main 6 | 7 | import ( 8 | reflect "reflect" 9 | 10 | gomock "github.com/golang/mock/gomock" 11 | ) 12 | 13 | // Mockbrokerer is a mock of brokerer interface. 14 | type Mockbrokerer struct { 15 | ctrl *gomock.Controller 16 | recorder *MockbrokererMockRecorder 17 | } 18 | 19 | // MockbrokererMockRecorder is the mock recorder for Mockbrokerer. 20 | type MockbrokererMockRecorder struct { 21 | mock *Mockbrokerer 22 | } 23 | 24 | // NewMockbrokerer creates a new mock instance. 25 | func NewMockbrokerer(ctrl *gomock.Controller) *Mockbrokerer { 26 | mock := &Mockbrokerer{ctrl: ctrl} 27 | mock.recorder = &MockbrokererMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use. 32 | func (m *Mockbrokerer) EXPECT() *MockbrokererMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Publish mocks base method. 37 | func (m *Mockbrokerer) Publish(topic string, value *value) error { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "Publish", topic, value) 40 | ret0, _ := ret[0].(error) 41 | return ret0 42 | } 43 | 44 | // Publish indicates an expected call of Publish. 45 | func (mr *MockbrokererMockRecorder) Publish(topic, value interface{}) *gomock.Call { 46 | mr.mock.ctrl.T.Helper() 47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Publish", reflect.TypeOf((*Mockbrokerer)(nil).Publish), topic, value) 48 | } 49 | 50 | // Purge mocks base method. 51 | func (m *Mockbrokerer) Purge(topic string) error { 52 | m.ctrl.T.Helper() 53 | ret := m.ctrl.Call(m, "Purge", topic) 54 | ret0, _ := ret[0].(error) 55 | return ret0 56 | } 57 | 58 | // Purge indicates an expected call of Purge. 59 | func (mr *MockbrokererMockRecorder) Purge(topic interface{}) *gomock.Call { 60 | mr.mock.ctrl.T.Helper() 61 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Purge", reflect.TypeOf((*Mockbrokerer)(nil).Purge), topic) 62 | } 63 | 64 | // Subscribe mocks base method. 65 | func (m *Mockbrokerer) Subscribe(topic string) *consumer { 66 | m.ctrl.T.Helper() 67 | ret := m.ctrl.Call(m, "Subscribe", topic) 68 | ret0, _ := ret[0].(*consumer) 69 | return ret0 70 | } 71 | 72 | // Subscribe indicates an expected call of Subscribe. 73 | func (mr *MockbrokererMockRecorder) Subscribe(topic interface{}) *gomock.Call { 74 | mr.mock.ctrl.T.Helper() 75 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Subscribe", reflect.TypeOf((*Mockbrokerer)(nil).Subscribe), topic) 76 | } 77 | 78 | // Topics mocks base method. 79 | func (m *Mockbrokerer) Topics() ([]string, error) { 80 | m.ctrl.T.Helper() 81 | ret := m.ctrl.Call(m, "Topics") 82 | ret0, _ := ret[0].([]string) 83 | ret1, _ := ret[1].(error) 84 | return ret0, ret1 85 | } 86 | 87 | // Topics indicates an expected call of Topics. 88 | func (mr *MockbrokererMockRecorder) Topics() *gomock.Call { 89 | mr.mock.ctrl.T.Helper() 90 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Topics", reflect.TypeOf((*Mockbrokerer)(nil).Topics)) 91 | } 92 | 93 | // Unsubscribe mocks base method. 94 | func (m *Mockbrokerer) Unsubscribe(topic, id string) error { 95 | m.ctrl.T.Helper() 96 | ret := m.ctrl.Call(m, "Unsubscribe", topic, id) 97 | ret0, _ := ret[0].(error) 98 | return ret0 99 | } 100 | 101 | // Unsubscribe indicates an expected call of Unsubscribe. 102 | func (mr *MockbrokererMockRecorder) Unsubscribe(topic, id interface{}) *gomock.Call { 103 | mr.mock.ctrl.T.Helper() 104 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Unsubscribe", reflect.TypeOf((*Mockbrokerer)(nil).Unsubscribe), topic, id) 105 | } 106 | -------------------------------------------------------------------------------- /broker_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | gomock "github.com/golang/mock/gomock" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestBroker_Publish(t *testing.T) { 12 | ctrl := gomock.NewController(t) 13 | defer ctrl.Finish() 14 | 15 | var ( 16 | topic = "test_topic" 17 | value = newValue([]byte("test_value")) 18 | ) 19 | 20 | mockStore := NewMockstorer(ctrl) 21 | mockStore.EXPECT().Insert(topic, value) 22 | 23 | b := newBroker(mockStore) 24 | 25 | require.NoError(t, b.Publish(topic, value)) 26 | } 27 | 28 | func TestBroker_Subscribe(t *testing.T) { 29 | ctrl := gomock.NewController(t) 30 | defer ctrl.Finish() 31 | 32 | var ( 33 | topic = "test_topic" 34 | ) 35 | 36 | mockStore := NewMockstorer(ctrl) 37 | 38 | b := newBroker(mockStore) 39 | c := b.Subscribe(topic) 40 | 41 | require.IsType(t, &consumer{}, c) 42 | } 43 | 44 | func TestBroker_Unsubscribe(t *testing.T) { 45 | t.Run("removes consumer from the topic", func(t *testing.T) { 46 | b := broker{ 47 | consumers: map[string][]*consumer{}, 48 | } 49 | 50 | topic := "test_topic" 51 | 52 | c := b.Subscribe(topic) 53 | err := b.Unsubscribe(topic, c.id) 54 | require.NoError(t, err) 55 | require.Len(t, b.consumers[topic], 0) 56 | }) 57 | 58 | t.Run("removes correct consumer if there are multiple", func(t *testing.T) { 59 | b := broker{ 60 | consumers: map[string][]*consumer{}, 61 | } 62 | 63 | topic := "test_topic" 64 | 65 | c1 := b.Subscribe(topic) 66 | c2 := b.Subscribe(topic) 67 | err := b.Unsubscribe(topic, c1.id) 68 | require.NoError(t, err) 69 | require.Len(t, b.consumers[topic], 1) 70 | require.Equal(t, c2.id, b.consumers[topic][0].id) 71 | }) 72 | 73 | t.Run("returns an error if the consumer doesn't exist", func(t *testing.T) { 74 | b := broker{ 75 | consumers: map[string][]*consumer{}, 76 | } 77 | 78 | topic := "test_topic" 79 | 80 | err := b.Unsubscribe(topic, "test_id") 81 | require.Error(t, err) 82 | }) 83 | 84 | t.Run("nacks outstanding messages on consumer", func(t *testing.T) { 85 | ctrl := gomock.NewController(t) 86 | 87 | topic := "test_topic" 88 | 89 | mockStorer := NewMockstorer(ctrl) 90 | mockStorer.EXPECT().GetNext(topic).Return(nil, 0, nil) 91 | mockStorer.EXPECT().Nack(topic, 0).Return(nil) 92 | 93 | b := broker{ 94 | consumers: map[string][]*consumer{}, 95 | store: mockStorer, 96 | } 97 | 98 | c := b.Subscribe(topic) 99 | _, err := c.Next(context.Background()) 100 | require.NoError(t, err) 101 | 102 | err = b.Unsubscribe(topic, c.id) 103 | require.NoError(t, err) 104 | }) 105 | } 106 | -------------------------------------------------------------------------------- /conn_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: /Users/tom/go/pkg/mod/github.com/tidwall/redcon@v1.4.1/redcon.go 3 | 4 | // Package main is a generated GoMock package. 5 | package main 6 | 7 | import ( 8 | net "net" 9 | reflect "reflect" 10 | 11 | gomock "github.com/golang/mock/gomock" 12 | redcon "github.com/tidwall/redcon" 13 | ) 14 | 15 | // MockConn is a mock of Conn interface. 16 | type MockConn struct { 17 | ctrl *gomock.Controller 18 | recorder *MockConnMockRecorder 19 | } 20 | 21 | // MockConnMockRecorder is the mock recorder for MockConn. 22 | type MockConnMockRecorder struct { 23 | mock *MockConn 24 | } 25 | 26 | // NewMockConn creates a new mock instance. 27 | func NewMockConn(ctrl *gomock.Controller) *MockConn { 28 | mock := &MockConn{ctrl: ctrl} 29 | mock.recorder = &MockConnMockRecorder{mock} 30 | return mock 31 | } 32 | 33 | // EXPECT returns an object that allows the caller to indicate expected use. 34 | func (m *MockConn) EXPECT() *MockConnMockRecorder { 35 | return m.recorder 36 | } 37 | 38 | // Close mocks base method. 39 | func (m *MockConn) Close() error { 40 | m.ctrl.T.Helper() 41 | ret := m.ctrl.Call(m, "Close") 42 | ret0, _ := ret[0].(error) 43 | return ret0 44 | } 45 | 46 | // Close indicates an expected call of Close. 47 | func (mr *MockConnMockRecorder) Close() *gomock.Call { 48 | mr.mock.ctrl.T.Helper() 49 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockConn)(nil).Close)) 50 | } 51 | 52 | // Context mocks base method. 53 | func (m *MockConn) Context() interface{} { 54 | m.ctrl.T.Helper() 55 | ret := m.ctrl.Call(m, "Context") 56 | ret0, _ := ret[0].(interface{}) 57 | return ret0 58 | } 59 | 60 | // Context indicates an expected call of Context. 61 | func (mr *MockConnMockRecorder) Context() *gomock.Call { 62 | mr.mock.ctrl.T.Helper() 63 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Context", reflect.TypeOf((*MockConn)(nil).Context)) 64 | } 65 | 66 | // Detach mocks base method. 67 | func (m *MockConn) Detach() redcon.DetachedConn { 68 | m.ctrl.T.Helper() 69 | ret := m.ctrl.Call(m, "Detach") 70 | ret0, _ := ret[0].(redcon.DetachedConn) 71 | return ret0 72 | } 73 | 74 | // Detach indicates an expected call of Detach. 75 | func (mr *MockConnMockRecorder) Detach() *gomock.Call { 76 | mr.mock.ctrl.T.Helper() 77 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Detach", reflect.TypeOf((*MockConn)(nil).Detach)) 78 | } 79 | 80 | // NetConn mocks base method. 81 | func (m *MockConn) NetConn() net.Conn { 82 | m.ctrl.T.Helper() 83 | ret := m.ctrl.Call(m, "NetConn") 84 | ret0, _ := ret[0].(net.Conn) 85 | return ret0 86 | } 87 | 88 | // NetConn indicates an expected call of NetConn. 89 | func (mr *MockConnMockRecorder) NetConn() *gomock.Call { 90 | mr.mock.ctrl.T.Helper() 91 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetConn", reflect.TypeOf((*MockConn)(nil).NetConn)) 92 | } 93 | 94 | // PeekPipeline mocks base method. 95 | func (m *MockConn) PeekPipeline() []redcon.Command { 96 | m.ctrl.T.Helper() 97 | ret := m.ctrl.Call(m, "PeekPipeline") 98 | ret0, _ := ret[0].([]redcon.Command) 99 | return ret0 100 | } 101 | 102 | // PeekPipeline indicates an expected call of PeekPipeline. 103 | func (mr *MockConnMockRecorder) PeekPipeline() *gomock.Call { 104 | mr.mock.ctrl.T.Helper() 105 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PeekPipeline", reflect.TypeOf((*MockConn)(nil).PeekPipeline)) 106 | } 107 | 108 | // ReadPipeline mocks base method. 109 | func (m *MockConn) ReadPipeline() []redcon.Command { 110 | m.ctrl.T.Helper() 111 | ret := m.ctrl.Call(m, "ReadPipeline") 112 | ret0, _ := ret[0].([]redcon.Command) 113 | return ret0 114 | } 115 | 116 | // ReadPipeline indicates an expected call of ReadPipeline. 117 | func (mr *MockConnMockRecorder) ReadPipeline() *gomock.Call { 118 | mr.mock.ctrl.T.Helper() 119 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadPipeline", reflect.TypeOf((*MockConn)(nil).ReadPipeline)) 120 | } 121 | 122 | // RemoteAddr mocks base method. 123 | func (m *MockConn) RemoteAddr() string { 124 | m.ctrl.T.Helper() 125 | ret := m.ctrl.Call(m, "RemoteAddr") 126 | ret0, _ := ret[0].(string) 127 | return ret0 128 | } 129 | 130 | // RemoteAddr indicates an expected call of RemoteAddr. 131 | func (mr *MockConnMockRecorder) RemoteAddr() *gomock.Call { 132 | mr.mock.ctrl.T.Helper() 133 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoteAddr", reflect.TypeOf((*MockConn)(nil).RemoteAddr)) 134 | } 135 | 136 | // SetContext mocks base method. 137 | func (m *MockConn) SetContext(v interface{}) { 138 | m.ctrl.T.Helper() 139 | m.ctrl.Call(m, "SetContext", v) 140 | } 141 | 142 | // SetContext indicates an expected call of SetContext. 143 | func (mr *MockConnMockRecorder) SetContext(v interface{}) *gomock.Call { 144 | mr.mock.ctrl.T.Helper() 145 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetContext", reflect.TypeOf((*MockConn)(nil).SetContext), v) 146 | } 147 | 148 | // SetReadBuffer mocks base method. 149 | func (m *MockConn) SetReadBuffer(bytes int) { 150 | m.ctrl.T.Helper() 151 | m.ctrl.Call(m, "SetReadBuffer", bytes) 152 | } 153 | 154 | // SetReadBuffer indicates an expected call of SetReadBuffer. 155 | func (mr *MockConnMockRecorder) SetReadBuffer(bytes interface{}) *gomock.Call { 156 | mr.mock.ctrl.T.Helper() 157 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetReadBuffer", reflect.TypeOf((*MockConn)(nil).SetReadBuffer), bytes) 158 | } 159 | 160 | // WriteAny mocks base method. 161 | func (m *MockConn) WriteAny(any interface{}) { 162 | m.ctrl.T.Helper() 163 | m.ctrl.Call(m, "WriteAny", any) 164 | } 165 | 166 | // WriteAny indicates an expected call of WriteAny. 167 | func (mr *MockConnMockRecorder) WriteAny(any interface{}) *gomock.Call { 168 | mr.mock.ctrl.T.Helper() 169 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteAny", reflect.TypeOf((*MockConn)(nil).WriteAny), any) 170 | } 171 | 172 | // WriteArray mocks base method. 173 | func (m *MockConn) WriteArray(count int) { 174 | m.ctrl.T.Helper() 175 | m.ctrl.Call(m, "WriteArray", count) 176 | } 177 | 178 | // WriteArray indicates an expected call of WriteArray. 179 | func (mr *MockConnMockRecorder) WriteArray(count interface{}) *gomock.Call { 180 | mr.mock.ctrl.T.Helper() 181 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteArray", reflect.TypeOf((*MockConn)(nil).WriteArray), count) 182 | } 183 | 184 | // WriteBulk mocks base method. 185 | func (m *MockConn) WriteBulk(bulk []byte) { 186 | m.ctrl.T.Helper() 187 | m.ctrl.Call(m, "WriteBulk", bulk) 188 | } 189 | 190 | // WriteBulk indicates an expected call of WriteBulk. 191 | func (mr *MockConnMockRecorder) WriteBulk(bulk interface{}) *gomock.Call { 192 | mr.mock.ctrl.T.Helper() 193 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteBulk", reflect.TypeOf((*MockConn)(nil).WriteBulk), bulk) 194 | } 195 | 196 | // WriteBulkString mocks base method. 197 | func (m *MockConn) WriteBulkString(bulk string) { 198 | m.ctrl.T.Helper() 199 | m.ctrl.Call(m, "WriteBulkString", bulk) 200 | } 201 | 202 | // WriteBulkString indicates an expected call of WriteBulkString. 203 | func (mr *MockConnMockRecorder) WriteBulkString(bulk interface{}) *gomock.Call { 204 | mr.mock.ctrl.T.Helper() 205 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteBulkString", reflect.TypeOf((*MockConn)(nil).WriteBulkString), bulk) 206 | } 207 | 208 | // WriteError mocks base method. 209 | func (m *MockConn) WriteError(msg string) { 210 | m.ctrl.T.Helper() 211 | m.ctrl.Call(m, "WriteError", msg) 212 | } 213 | 214 | // WriteError indicates an expected call of WriteError. 215 | func (mr *MockConnMockRecorder) WriteError(msg interface{}) *gomock.Call { 216 | mr.mock.ctrl.T.Helper() 217 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteError", reflect.TypeOf((*MockConn)(nil).WriteError), msg) 218 | } 219 | 220 | // WriteInt mocks base method. 221 | func (m *MockConn) WriteInt(num int) { 222 | m.ctrl.T.Helper() 223 | m.ctrl.Call(m, "WriteInt", num) 224 | } 225 | 226 | // WriteInt indicates an expected call of WriteInt. 227 | func (mr *MockConnMockRecorder) WriteInt(num interface{}) *gomock.Call { 228 | mr.mock.ctrl.T.Helper() 229 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteInt", reflect.TypeOf((*MockConn)(nil).WriteInt), num) 230 | } 231 | 232 | // WriteInt64 mocks base method. 233 | func (m *MockConn) WriteInt64(num int64) { 234 | m.ctrl.T.Helper() 235 | m.ctrl.Call(m, "WriteInt64", num) 236 | } 237 | 238 | // WriteInt64 indicates an expected call of WriteInt64. 239 | func (mr *MockConnMockRecorder) WriteInt64(num interface{}) *gomock.Call { 240 | mr.mock.ctrl.T.Helper() 241 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteInt64", reflect.TypeOf((*MockConn)(nil).WriteInt64), num) 242 | } 243 | 244 | // WriteNull mocks base method. 245 | func (m *MockConn) WriteNull() { 246 | m.ctrl.T.Helper() 247 | m.ctrl.Call(m, "WriteNull") 248 | } 249 | 250 | // WriteNull indicates an expected call of WriteNull. 251 | func (mr *MockConnMockRecorder) WriteNull() *gomock.Call { 252 | mr.mock.ctrl.T.Helper() 253 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteNull", reflect.TypeOf((*MockConn)(nil).WriteNull)) 254 | } 255 | 256 | // WriteRaw mocks base method. 257 | func (m *MockConn) WriteRaw(data []byte) { 258 | m.ctrl.T.Helper() 259 | m.ctrl.Call(m, "WriteRaw", data) 260 | } 261 | 262 | // WriteRaw indicates an expected call of WriteRaw. 263 | func (mr *MockConnMockRecorder) WriteRaw(data interface{}) *gomock.Call { 264 | mr.mock.ctrl.T.Helper() 265 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteRaw", reflect.TypeOf((*MockConn)(nil).WriteRaw), data) 266 | } 267 | 268 | // WriteString mocks base method. 269 | func (m *MockConn) WriteString(str string) { 270 | m.ctrl.T.Helper() 271 | m.ctrl.Call(m, "WriteString", str) 272 | } 273 | 274 | // WriteString indicates an expected call of WriteString. 275 | func (mr *MockConnMockRecorder) WriteString(str interface{}) *gomock.Call { 276 | mr.mock.ctrl.T.Helper() 277 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteString", reflect.TypeOf((*MockConn)(nil).WriteString), str) 278 | } 279 | 280 | // WriteUint64 mocks base method. 281 | func (m *MockConn) WriteUint64(num uint64) { 282 | m.ctrl.T.Helper() 283 | m.ctrl.Call(m, "WriteUint64", num) 284 | } 285 | 286 | // WriteUint64 indicates an expected call of WriteUint64. 287 | func (mr *MockConnMockRecorder) WriteUint64(num interface{}) *gomock.Call { 288 | mr.mock.ctrl.T.Helper() 289 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteUint64", reflect.TypeOf((*MockConn)(nil).WriteUint64), num) 290 | } 291 | 292 | // MockDetachedConn is a mock of DetachedConn interface. 293 | type MockDetachedConn struct { 294 | ctrl *gomock.Controller 295 | recorder *MockDetachedConnMockRecorder 296 | } 297 | 298 | // MockDetachedConnMockRecorder is the mock recorder for MockDetachedConn. 299 | type MockDetachedConnMockRecorder struct { 300 | mock *MockDetachedConn 301 | } 302 | 303 | // NewMockDetachedConn creates a new mock instance. 304 | func NewMockDetachedConn(ctrl *gomock.Controller) *MockDetachedConn { 305 | mock := &MockDetachedConn{ctrl: ctrl} 306 | mock.recorder = &MockDetachedConnMockRecorder{mock} 307 | return mock 308 | } 309 | 310 | // EXPECT returns an object that allows the caller to indicate expected use. 311 | func (m *MockDetachedConn) EXPECT() *MockDetachedConnMockRecorder { 312 | return m.recorder 313 | } 314 | 315 | // Close mocks base method. 316 | func (m *MockDetachedConn) Close() error { 317 | m.ctrl.T.Helper() 318 | ret := m.ctrl.Call(m, "Close") 319 | ret0, _ := ret[0].(error) 320 | return ret0 321 | } 322 | 323 | // Close indicates an expected call of Close. 324 | func (mr *MockDetachedConnMockRecorder) Close() *gomock.Call { 325 | mr.mock.ctrl.T.Helper() 326 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockDetachedConn)(nil).Close)) 327 | } 328 | 329 | // Context mocks base method. 330 | func (m *MockDetachedConn) Context() interface{} { 331 | m.ctrl.T.Helper() 332 | ret := m.ctrl.Call(m, "Context") 333 | ret0, _ := ret[0].(interface{}) 334 | return ret0 335 | } 336 | 337 | // Context indicates an expected call of Context. 338 | func (mr *MockDetachedConnMockRecorder) Context() *gomock.Call { 339 | mr.mock.ctrl.T.Helper() 340 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Context", reflect.TypeOf((*MockDetachedConn)(nil).Context)) 341 | } 342 | 343 | // Detach mocks base method. 344 | func (m *MockDetachedConn) Detach() redcon.DetachedConn { 345 | m.ctrl.T.Helper() 346 | ret := m.ctrl.Call(m, "Detach") 347 | ret0, _ := ret[0].(redcon.DetachedConn) 348 | return ret0 349 | } 350 | 351 | // Detach indicates an expected call of Detach. 352 | func (mr *MockDetachedConnMockRecorder) Detach() *gomock.Call { 353 | mr.mock.ctrl.T.Helper() 354 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Detach", reflect.TypeOf((*MockDetachedConn)(nil).Detach)) 355 | } 356 | 357 | // Flush mocks base method. 358 | func (m *MockDetachedConn) Flush() error { 359 | m.ctrl.T.Helper() 360 | ret := m.ctrl.Call(m, "Flush") 361 | ret0, _ := ret[0].(error) 362 | return ret0 363 | } 364 | 365 | // Flush indicates an expected call of Flush. 366 | func (mr *MockDetachedConnMockRecorder) Flush() *gomock.Call { 367 | mr.mock.ctrl.T.Helper() 368 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Flush", reflect.TypeOf((*MockDetachedConn)(nil).Flush)) 369 | } 370 | 371 | // NetConn mocks base method. 372 | func (m *MockDetachedConn) NetConn() net.Conn { 373 | m.ctrl.T.Helper() 374 | ret := m.ctrl.Call(m, "NetConn") 375 | ret0, _ := ret[0].(net.Conn) 376 | return ret0 377 | } 378 | 379 | // NetConn indicates an expected call of NetConn. 380 | func (mr *MockDetachedConnMockRecorder) NetConn() *gomock.Call { 381 | mr.mock.ctrl.T.Helper() 382 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NetConn", reflect.TypeOf((*MockDetachedConn)(nil).NetConn)) 383 | } 384 | 385 | // PeekPipeline mocks base method. 386 | func (m *MockDetachedConn) PeekPipeline() []redcon.Command { 387 | m.ctrl.T.Helper() 388 | ret := m.ctrl.Call(m, "PeekPipeline") 389 | ret0, _ := ret[0].([]redcon.Command) 390 | return ret0 391 | } 392 | 393 | // PeekPipeline indicates an expected call of PeekPipeline. 394 | func (mr *MockDetachedConnMockRecorder) PeekPipeline() *gomock.Call { 395 | mr.mock.ctrl.T.Helper() 396 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PeekPipeline", reflect.TypeOf((*MockDetachedConn)(nil).PeekPipeline)) 397 | } 398 | 399 | // ReadCommand mocks base method. 400 | func (m *MockDetachedConn) ReadCommand() (redcon.Command, error) { 401 | m.ctrl.T.Helper() 402 | ret := m.ctrl.Call(m, "ReadCommand") 403 | ret0, _ := ret[0].(redcon.Command) 404 | ret1, _ := ret[1].(error) 405 | return ret0, ret1 406 | } 407 | 408 | // ReadCommand indicates an expected call of ReadCommand. 409 | func (mr *MockDetachedConnMockRecorder) ReadCommand() *gomock.Call { 410 | mr.mock.ctrl.T.Helper() 411 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadCommand", reflect.TypeOf((*MockDetachedConn)(nil).ReadCommand)) 412 | } 413 | 414 | // ReadPipeline mocks base method. 415 | func (m *MockDetachedConn) ReadPipeline() []redcon.Command { 416 | m.ctrl.T.Helper() 417 | ret := m.ctrl.Call(m, "ReadPipeline") 418 | ret0, _ := ret[0].([]redcon.Command) 419 | return ret0 420 | } 421 | 422 | // ReadPipeline indicates an expected call of ReadPipeline. 423 | func (mr *MockDetachedConnMockRecorder) ReadPipeline() *gomock.Call { 424 | mr.mock.ctrl.T.Helper() 425 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadPipeline", reflect.TypeOf((*MockDetachedConn)(nil).ReadPipeline)) 426 | } 427 | 428 | // RemoteAddr mocks base method. 429 | func (m *MockDetachedConn) RemoteAddr() string { 430 | m.ctrl.T.Helper() 431 | ret := m.ctrl.Call(m, "RemoteAddr") 432 | ret0, _ := ret[0].(string) 433 | return ret0 434 | } 435 | 436 | // RemoteAddr indicates an expected call of RemoteAddr. 437 | func (mr *MockDetachedConnMockRecorder) RemoteAddr() *gomock.Call { 438 | mr.mock.ctrl.T.Helper() 439 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoteAddr", reflect.TypeOf((*MockDetachedConn)(nil).RemoteAddr)) 440 | } 441 | 442 | // SetContext mocks base method. 443 | func (m *MockDetachedConn) SetContext(v interface{}) { 444 | m.ctrl.T.Helper() 445 | m.ctrl.Call(m, "SetContext", v) 446 | } 447 | 448 | // SetContext indicates an expected call of SetContext. 449 | func (mr *MockDetachedConnMockRecorder) SetContext(v interface{}) *gomock.Call { 450 | mr.mock.ctrl.T.Helper() 451 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetContext", reflect.TypeOf((*MockDetachedConn)(nil).SetContext), v) 452 | } 453 | 454 | // SetReadBuffer mocks base method. 455 | func (m *MockDetachedConn) SetReadBuffer(bytes int) { 456 | m.ctrl.T.Helper() 457 | m.ctrl.Call(m, "SetReadBuffer", bytes) 458 | } 459 | 460 | // SetReadBuffer indicates an expected call of SetReadBuffer. 461 | func (mr *MockDetachedConnMockRecorder) SetReadBuffer(bytes interface{}) *gomock.Call { 462 | mr.mock.ctrl.T.Helper() 463 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetReadBuffer", reflect.TypeOf((*MockDetachedConn)(nil).SetReadBuffer), bytes) 464 | } 465 | 466 | // WriteAny mocks base method. 467 | func (m *MockDetachedConn) WriteAny(any interface{}) { 468 | m.ctrl.T.Helper() 469 | m.ctrl.Call(m, "WriteAny", any) 470 | } 471 | 472 | // WriteAny indicates an expected call of WriteAny. 473 | func (mr *MockDetachedConnMockRecorder) WriteAny(any interface{}) *gomock.Call { 474 | mr.mock.ctrl.T.Helper() 475 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteAny", reflect.TypeOf((*MockDetachedConn)(nil).WriteAny), any) 476 | } 477 | 478 | // WriteArray mocks base method. 479 | func (m *MockDetachedConn) WriteArray(count int) { 480 | m.ctrl.T.Helper() 481 | m.ctrl.Call(m, "WriteArray", count) 482 | } 483 | 484 | // WriteArray indicates an expected call of WriteArray. 485 | func (mr *MockDetachedConnMockRecorder) WriteArray(count interface{}) *gomock.Call { 486 | mr.mock.ctrl.T.Helper() 487 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteArray", reflect.TypeOf((*MockDetachedConn)(nil).WriteArray), count) 488 | } 489 | 490 | // WriteBulk mocks base method. 491 | func (m *MockDetachedConn) WriteBulk(bulk []byte) { 492 | m.ctrl.T.Helper() 493 | m.ctrl.Call(m, "WriteBulk", bulk) 494 | } 495 | 496 | // WriteBulk indicates an expected call of WriteBulk. 497 | func (mr *MockDetachedConnMockRecorder) WriteBulk(bulk interface{}) *gomock.Call { 498 | mr.mock.ctrl.T.Helper() 499 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteBulk", reflect.TypeOf((*MockDetachedConn)(nil).WriteBulk), bulk) 500 | } 501 | 502 | // WriteBulkString mocks base method. 503 | func (m *MockDetachedConn) WriteBulkString(bulk string) { 504 | m.ctrl.T.Helper() 505 | m.ctrl.Call(m, "WriteBulkString", bulk) 506 | } 507 | 508 | // WriteBulkString indicates an expected call of WriteBulkString. 509 | func (mr *MockDetachedConnMockRecorder) WriteBulkString(bulk interface{}) *gomock.Call { 510 | mr.mock.ctrl.T.Helper() 511 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteBulkString", reflect.TypeOf((*MockDetachedConn)(nil).WriteBulkString), bulk) 512 | } 513 | 514 | // WriteError mocks base method. 515 | func (m *MockDetachedConn) WriteError(msg string) { 516 | m.ctrl.T.Helper() 517 | m.ctrl.Call(m, "WriteError", msg) 518 | } 519 | 520 | // WriteError indicates an expected call of WriteError. 521 | func (mr *MockDetachedConnMockRecorder) WriteError(msg interface{}) *gomock.Call { 522 | mr.mock.ctrl.T.Helper() 523 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteError", reflect.TypeOf((*MockDetachedConn)(nil).WriteError), msg) 524 | } 525 | 526 | // WriteInt mocks base method. 527 | func (m *MockDetachedConn) WriteInt(num int) { 528 | m.ctrl.T.Helper() 529 | m.ctrl.Call(m, "WriteInt", num) 530 | } 531 | 532 | // WriteInt indicates an expected call of WriteInt. 533 | func (mr *MockDetachedConnMockRecorder) WriteInt(num interface{}) *gomock.Call { 534 | mr.mock.ctrl.T.Helper() 535 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteInt", reflect.TypeOf((*MockDetachedConn)(nil).WriteInt), num) 536 | } 537 | 538 | // WriteInt64 mocks base method. 539 | func (m *MockDetachedConn) WriteInt64(num int64) { 540 | m.ctrl.T.Helper() 541 | m.ctrl.Call(m, "WriteInt64", num) 542 | } 543 | 544 | // WriteInt64 indicates an expected call of WriteInt64. 545 | func (mr *MockDetachedConnMockRecorder) WriteInt64(num interface{}) *gomock.Call { 546 | mr.mock.ctrl.T.Helper() 547 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteInt64", reflect.TypeOf((*MockDetachedConn)(nil).WriteInt64), num) 548 | } 549 | 550 | // WriteNull mocks base method. 551 | func (m *MockDetachedConn) WriteNull() { 552 | m.ctrl.T.Helper() 553 | m.ctrl.Call(m, "WriteNull") 554 | } 555 | 556 | // WriteNull indicates an expected call of WriteNull. 557 | func (mr *MockDetachedConnMockRecorder) WriteNull() *gomock.Call { 558 | mr.mock.ctrl.T.Helper() 559 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteNull", reflect.TypeOf((*MockDetachedConn)(nil).WriteNull)) 560 | } 561 | 562 | // WriteRaw mocks base method. 563 | func (m *MockDetachedConn) WriteRaw(data []byte) { 564 | m.ctrl.T.Helper() 565 | m.ctrl.Call(m, "WriteRaw", data) 566 | } 567 | 568 | // WriteRaw indicates an expected call of WriteRaw. 569 | func (mr *MockDetachedConnMockRecorder) WriteRaw(data interface{}) *gomock.Call { 570 | mr.mock.ctrl.T.Helper() 571 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteRaw", reflect.TypeOf((*MockDetachedConn)(nil).WriteRaw), data) 572 | } 573 | 574 | // WriteString mocks base method. 575 | func (m *MockDetachedConn) WriteString(str string) { 576 | m.ctrl.T.Helper() 577 | m.ctrl.Call(m, "WriteString", str) 578 | } 579 | 580 | // WriteString indicates an expected call of WriteString. 581 | func (mr *MockDetachedConnMockRecorder) WriteString(str interface{}) *gomock.Call { 582 | mr.mock.ctrl.T.Helper() 583 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteString", reflect.TypeOf((*MockDetachedConn)(nil).WriteString), str) 584 | } 585 | 586 | // WriteUint64 mocks base method. 587 | func (m *MockDetachedConn) WriteUint64(num uint64) { 588 | m.ctrl.T.Helper() 589 | m.ctrl.Call(m, "WriteUint64", num) 590 | } 591 | 592 | // WriteUint64 indicates an expected call of WriteUint64. 593 | func (mr *MockDetachedConnMockRecorder) WriteUint64(num interface{}) *gomock.Call { 594 | mr.mock.ctrl.T.Helper() 595 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteUint64", reflect.TypeOf((*MockDetachedConn)(nil).WriteUint64), num) 596 | } 597 | 598 | // MockHandler is a mock of Handler interface. 599 | type MockHandler struct { 600 | ctrl *gomock.Controller 601 | recorder *MockHandlerMockRecorder 602 | } 603 | 604 | // MockHandlerMockRecorder is the mock recorder for MockHandler. 605 | type MockHandlerMockRecorder struct { 606 | mock *MockHandler 607 | } 608 | 609 | // NewMockHandler creates a new mock instance. 610 | func NewMockHandler(ctrl *gomock.Controller) *MockHandler { 611 | mock := &MockHandler{ctrl: ctrl} 612 | mock.recorder = &MockHandlerMockRecorder{mock} 613 | return mock 614 | } 615 | 616 | // EXPECT returns an object that allows the caller to indicate expected use. 617 | func (m *MockHandler) EXPECT() *MockHandlerMockRecorder { 618 | return m.recorder 619 | } 620 | 621 | // ServeRESP mocks base method. 622 | func (m *MockHandler) ServeRESP(conn redcon.Conn, cmd redcon.Command) { 623 | m.ctrl.T.Helper() 624 | m.ctrl.Call(m, "ServeRESP", conn, cmd) 625 | } 626 | 627 | // ServeRESP indicates an expected call of ServeRESP. 628 | func (mr *MockHandlerMockRecorder) ServeRESP(conn, cmd interface{}) *gomock.Call { 629 | mr.mock.ctrl.T.Helper() 630 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ServeRESP", reflect.TypeOf((*MockHandler)(nil).ServeRESP), conn, cmd) 631 | } 632 | -------------------------------------------------------------------------------- /consumer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | ) 8 | 9 | const ( 10 | eventTypePublish eventType = iota 11 | eventTypeNack 12 | eventTypeBack 13 | eventTypeMsgReturned 14 | ) 15 | 16 | type eventType int 17 | 18 | type notifier interface { 19 | NotifyConsumer(topic string, ev eventType) 20 | } 21 | 22 | // consumer handles providing values iteratively to a single consumer. Methods 23 | // on a consumer are not thread safe as operations should occur serially. 24 | type consumer struct { 25 | id string 26 | topic string 27 | ackOffset int 28 | store storer 29 | eventChan chan eventType 30 | notifier notifier 31 | outstanding bool // indicates whether the consumer has an outstanding message to ack 32 | } 33 | 34 | func (c *consumer) String() string { 35 | return fmt.Sprintf("consumer{id: %s}", c.id) 36 | } 37 | 38 | // Next will attempt to retrieve the next value on the topic, or it will 39 | // block waiting for a msg indicating there is a new value available. 40 | func (c *consumer) Next(ctx context.Context) (val *value, err error) { 41 | // Prevent Next from being called if the consumer already has one outstanding 42 | // unacknowledged message. 43 | if c.outstanding { 44 | return nil, errors.New("unacknowledged message outstanding") 45 | } 46 | 47 | var ao int 48 | 49 | // Repeat trying to get the next value while the topic is either empty or not 50 | // created yet. It may exist sometime in the future. 51 | for { 52 | val, ao, err = c.store.GetNext(c.topic) 53 | if !errors.Is(err, errTopicEmpty) && !errors.Is(err, errTopicNotExist) { 54 | break 55 | } 56 | 57 | select { 58 | case <-c.eventChan: 59 | case <-ctx.Done(): 60 | return nil, errRequestCancelled 61 | } 62 | } 63 | if err != nil { 64 | return nil, fmt.Errorf("getting next from store: %v", err) 65 | } 66 | 67 | c.ackOffset = ao 68 | c.outstanding = true 69 | 70 | return val, err 71 | } 72 | 73 | // Ack acknowledges the previously consumed value. 74 | func (c *consumer) Ack() error { 75 | if err := c.store.Ack(c.topic, c.ackOffset); err != nil { 76 | return fmt.Errorf("acking topic %s with offset %d: %v", c.topic, c.ackOffset, err) 77 | } 78 | 79 | c.outstanding = false 80 | 81 | return nil 82 | } 83 | 84 | // Nack negatively acknowledges a message, returning it for consumption by other 85 | // consumers. 86 | func (c *consumer) Nack() error { 87 | if err := c.store.Nack(c.topic, c.ackOffset); err != nil { 88 | return fmt.Errorf("nacking topic %s with offset %d: %w", c.topic, c.ackOffset, err) 89 | } 90 | 91 | c.outstanding = false 92 | c.notifier.NotifyConsumer(c.topic, eventTypeNack) 93 | 94 | return nil 95 | } 96 | 97 | // Back negatively acknowledges a message, returning it to the back of the queue 98 | // for consumption. 99 | func (c *consumer) Back() error { 100 | if err := c.store.Back(c.topic, c.ackOffset); err != nil { 101 | return fmt.Errorf("backing topic %s with offset %d: %v", c.topic, c.ackOffset, err) 102 | } 103 | 104 | c.outstanding = false 105 | c.notifier.NotifyConsumer(c.topic, eventTypeBack) 106 | 107 | return nil 108 | } 109 | 110 | func (c *consumer) Dack(delaySeconds int) error { 111 | if err := c.store.Dack(c.topic, c.ackOffset, delaySeconds); err != nil { 112 | return fmt.Errorf("dacking topic %s with offset %d and delay %ds: %v", c.topic, c.ackOffset, delaySeconds, err) 113 | } 114 | 115 | c.outstanding = false 116 | 117 | return nil 118 | } 119 | 120 | // EventChan returns a channel to notify the consumer of events occurring on the 121 | // topic. 122 | func (c *consumer) EventChan() <-chan eventType { 123 | return c.eventChan 124 | } 125 | -------------------------------------------------------------------------------- /consumer_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | gomock "github.com/golang/mock/gomock" 8 | assert "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestConsumerNext(t *testing.T) { 12 | t.Run("next, ack, next", func(t *testing.T) { 13 | assert := assert.New(t) 14 | ctrl := gomock.NewController(t) 15 | defer ctrl.Finish() 16 | 17 | var ( 18 | topic = "test_topic" 19 | msg1 = newValue([]byte("message1")) 20 | msg2 = newValue([]byte("message2")) 21 | ) 22 | 23 | mockStore := NewMockstorer(ctrl) 24 | mockStore.EXPECT().GetNext(topic).Return(msg1, 0, nil) 25 | mockStore.EXPECT().Ack(topic, 0).Return(nil) 26 | mockStore.EXPECT().GetNext(topic).Return(msg2, 1, nil) 27 | 28 | b := newBroker(mockStore) 29 | c := b.Subscribe(topic) 30 | 31 | msg, err := c.Next(context.Background()) 32 | assert.NoError(err) 33 | assert.Equal(msg1, msg) 34 | assert.Equal(c.ackOffset, 0) 35 | 36 | assert.NoError(c.Ack()) 37 | 38 | msg, err = c.Next(context.Background()) 39 | assert.NoError(err) 40 | assert.Equal(msg2, msg) 41 | assert.Equal(c.ackOffset, 1) 42 | }) 43 | 44 | t.Run("next next, fails due to outstanding ack", func(t *testing.T) { 45 | assert := assert.New(t) 46 | ctrl := gomock.NewController(t) 47 | defer ctrl.Finish() 48 | 49 | var ( 50 | topic = "test_topic" 51 | msg1 = newValue([]byte("message1")) 52 | ) 53 | 54 | mockStore := NewMockstorer(ctrl) 55 | mockStore.EXPECT().GetNext(topic).Return(msg1, 0, nil) 56 | 57 | b := newBroker(mockStore) 58 | c := b.Subscribe(topic) 59 | 60 | msg, err := c.Next(context.Background()) 61 | assert.NoError(err) 62 | assert.Equal(msg1, msg) 63 | assert.Equal(c.ackOffset, 0) 64 | 65 | msg, err = c.Next(context.Background()) 66 | assert.Error(err) 67 | }) 68 | } 69 | -------------------------------------------------------------------------------- /examples/bench_consume/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net/http" 10 | "os" 11 | "time" 12 | ) 13 | 14 | var ( 15 | topic = "test" 16 | url = "https://localhost:8080" 17 | 18 | duration = flag.Duration("duration", 10*time.Second, "duration of the benchmark") 19 | ) 20 | 21 | func main() { 22 | flag.Parse() 23 | 24 | // Bail out if necessary 25 | go func() { 26 | <-time.After(*duration + time.Second) 27 | log.Fatal("blocked reading, maybe ran out of things to consume?") 28 | }() 29 | 30 | reader, writer := io.Pipe() 31 | enc := json.NewEncoder(writer) 32 | go func() { 33 | _ = enc.Encode("INIT") 34 | }() 35 | 36 | res, err := http.Post(fmt.Sprintf("%s/subscribe/%s", url, topic), "application/json", reader) 37 | if err != nil { 38 | log.Fatalf("failed to consume: %v", err) 39 | } 40 | if res.StatusCode != http.StatusOK { 41 | log.Fatalf("failed to consume, received status code: %d", res.StatusCode) 42 | } 43 | 44 | dec := json.NewDecoder(res.Body) 45 | 46 | timer := time.After(*duration) 47 | count := 0 48 | for { 49 | select { 50 | case <-timer: 51 | fmt.Printf("consumed %d times in %s\n", count, *duration) 52 | fmt.Printf("%d (consume+ack)/second\n", count/int(*duration/time.Second)) 53 | _ = writer.Close() 54 | os.Exit(0) 55 | default: 56 | } 57 | 58 | var out struct { 59 | Msg string `json:"msg"` 60 | Error string `json:"error"` 61 | } 62 | if _ = dec.Decode(&out); err != nil { 63 | log.Fatalf("failed decode response body: %v\n", err) 64 | } 65 | 66 | if out.Error != "" { 67 | log.Fatalf("received error: %s\n", out.Error) 68 | } 69 | 70 | if err := enc.Encode("ACK"); err != nil { 71 | log.Fatalf("ACK failed: %v\n", err) 72 | } 73 | 74 | count++ 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /examples/echo/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "encoding/base64" 7 | "encoding/json" 8 | "fmt" 9 | "log" 10 | "net/http" 11 | "os" 12 | "strings" 13 | ) 14 | 15 | var ( 16 | url = "https://localhost:8080" 17 | topic = "test_topic" 18 | ) 19 | 20 | func main() { 21 | sc := bufio.NewScanner(os.Stdin) 22 | 23 | for { 24 | fmt.Print("> ") 25 | sc.Scan() 26 | if err := sc.Err(); err != nil { 27 | log.Fatalf("an error occurred: %v", err) 28 | } 29 | 30 | input := sc.Text() 31 | 32 | if input == "q" { 33 | log.Println("bye!") 34 | os.Exit(0) 35 | } 36 | 37 | res, err := http.Post( 38 | fmt.Sprintf("%s/publish/%s", url, topic), 39 | "application/json", 40 | strings.NewReader(input), 41 | ) 42 | if err != nil { 43 | log.Printf("failed to publish: %v", err) 44 | continue 45 | } 46 | if res.StatusCode != http.StatusCreated { 47 | log.Printf("failed to publish, received status code: %d", res.StatusCode) 48 | } 49 | 50 | fmt.Printf("Published message %s to topic %s\n", input, topic) 51 | 52 | var buf bytes.Buffer 53 | enc := json.NewEncoder(&buf) 54 | enc.Encode("INIT") 55 | enc.Encode("ACK") 56 | 57 | res, err = http.Post(fmt.Sprintf("%s/subscribe/%s", url, topic), "application/json", &buf) 58 | if err != nil { 59 | log.Printf("failed to consume: %v", err) 60 | continue 61 | } 62 | if res.StatusCode != http.StatusOK { 63 | log.Printf("failed to consume, received status code: %d", res.StatusCode) 64 | continue 65 | } 66 | 67 | var subRes struct { 68 | Msg string `json:"msg"` 69 | Error string `json:"error"` 70 | } 71 | if err := json.NewDecoder(res.Body).Decode(&subRes); err != nil { 72 | log.Printf("failed decode response body: %v", err) 73 | } 74 | 75 | res.Body.Close() 76 | 77 | fmt.Printf("Consumed message: %s\n", mustBase64Decode(subRes.Msg)) 78 | } 79 | } 80 | 81 | func mustBase64Decode(b string) string { 82 | s, err := base64.StdEncoding.DecodeString(string(b)) 83 | if err != nil { 84 | log.Fatal(err) 85 | } 86 | 87 | return string(s) 88 | } 89 | -------------------------------------------------------------------------------- /examples/exponential_backoff/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "log" 10 | "math" 11 | "net/http" 12 | "os" 13 | "strings" 14 | ) 15 | 16 | var ( 17 | url = "https://localhost:8080" 18 | topic = "test_topic" 19 | ) 20 | 21 | func main() { 22 | flag.Parse() 23 | 24 | publish() 25 | consume(0) 26 | } 27 | 28 | func publish() { 29 | log := log.New(os.Stdout, "producer: ", 0) 30 | 31 | msg := "hello_world" 32 | res, err := http.Post(fmt.Sprintf("%s/publish/%s", url, topic), "application/json", strings.NewReader(msg)) 33 | if err != nil { 34 | log.Fatalf("failed to publish: %v\n", err) 35 | } 36 | if res.StatusCode != http.StatusCreated { 37 | log.Fatalf("failed to publish, received status code: %d\n", res.StatusCode) 38 | } 39 | 40 | log.Printf("published message %s\n", msg) 41 | } 42 | 43 | type subRes struct { 44 | Msg string `json:"msg"` 45 | Error string `json:"error"` 46 | DackCount int `json:"dackCount"` 47 | } 48 | 49 | func consume(id int) { 50 | for { 51 | log := log.New(os.Stdout, fmt.Sprintf("consumer-%d: ", id), 0) 52 | 53 | reader, writer := io.Pipe() 54 | enc := json.NewEncoder(writer) 55 | go func() { 56 | _ = enc.Encode("INIT") 57 | }() 58 | 59 | res, err := http.Post(fmt.Sprintf("%s/subscribe/%s", url, topic), "application/json", reader) 60 | if err != nil { 61 | log.Fatalf("failed to consume: %v", err) 62 | } 63 | if res.StatusCode != http.StatusOK { 64 | log.Fatalf("failed to consume, received status code: %d", res.StatusCode) 65 | } 66 | 67 | dec := json.NewDecoder(res.Body) 68 | 69 | for { 70 | var out subRes 71 | if _ = dec.Decode(&out); err != nil { 72 | log.Fatalf("failed decode response body: %v\n", err) 73 | } 74 | 75 | if out.Error != "" { 76 | log.Fatalf("received error: %s\n", out.Error) 77 | } 78 | 79 | delay := int(math.Pow(2, float64(out.DackCount))) 80 | 81 | strMsg := mustBase64Decode(out.Msg) 82 | log.Printf("consumed message: %s\n", strMsg) 83 | log.Printf("delaying message: %s by %ds\n", strMsg, delay) 84 | 85 | _ = enc.Encode(fmt.Sprintf("DACK %d", delay)) 86 | } 87 | } 88 | } 89 | 90 | func mustBase64Decode(b string) string { 91 | s, err := base64.StdEncoding.DecodeString(string(b)) 92 | if err != nil { 93 | log.Fatal(err) 94 | } 95 | 96 | return string(s) 97 | } 98 | -------------------------------------------------------------------------------- /examples/workers/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "log" 10 | "math/rand" 11 | "net/http" 12 | "os" 13 | "strconv" 14 | "strings" 15 | "time" 16 | ) 17 | 18 | var ( 19 | url = "https://localhost:8080" 20 | topic = "test_topic" 21 | 22 | consumers = flag.Int("consumers", 2, "number of consumers (minimum 1)") 23 | pubRate = flag.Duration("rate", time.Second, "the default rate at which to publish") 24 | maxSleepTime = flag.Int("sleep", 5, "upper bound for consumer random sleep seconds") 25 | validate = flag.Bool("validate", false, "run in validation mode, check for pub/sub consistency, must be run with only 1 consumer") 26 | nackChance = flag.Int("chance", 10, "1/n change to randomly send back a nack") 27 | ) 28 | 29 | func main() { 30 | flag.Parse() 31 | 32 | if *validate && *consumers > 1 { 33 | log.Fatal("validate can currently only be run with a single consumer") 34 | } 35 | 36 | log.SetFlags(0) 37 | 38 | go producer() 39 | 40 | // Give the producer time to create the topic 41 | time.Sleep(time.Second) 42 | 43 | for i := 0; i < *consumers-1; i++ { 44 | go consumer(i) 45 | } 46 | consumer(*consumers - 1) 47 | } 48 | 49 | func producer() { 50 | log := log.New(os.Stdout, "producer: ", 0) 51 | 52 | for n := 0; ; n++ { 53 | msg := fmt.Sprintf("%d", n) 54 | 55 | res, err := http.Post( 56 | fmt.Sprintf("%s/publish/%s", url, topic), 57 | "application/json", 58 | strings.NewReader(msg), 59 | ) 60 | if err != nil { 61 | log.Printf("failed to publish: %v\n", err) 62 | continue 63 | } 64 | if res.StatusCode != http.StatusCreated { 65 | log.Printf("failed to publish, received status code: %d\n", res.StatusCode) 66 | } 67 | 68 | log.Printf("published message %s\n", msg) 69 | 70 | time.Sleep(*pubRate) 71 | } 72 | } 73 | 74 | type subRes struct { 75 | Msg string `json:"msg"` 76 | Error string `json:"error"` 77 | } 78 | 79 | func consumer(id int) { 80 | restart: 81 | for { 82 | log := log.New(os.Stdout, fmt.Sprintf("consumer-%d: ", id), 0) 83 | 84 | reader, writer := io.Pipe() 85 | enc := json.NewEncoder(writer) 86 | go func() { 87 | _ = enc.Encode("INIT") 88 | }() 89 | 90 | res, err := http.Post(fmt.Sprintf("%s/subscribe/%s", url, topic), "application/json", reader) 91 | if err != nil { 92 | log.Fatalf("failed to consume: %v", err) 93 | } 94 | if res.StatusCode != http.StatusOK { 95 | log.Fatalf("failed to consume, received status code: %d", res.StatusCode) 96 | } 97 | 98 | dec := json.NewDecoder(res.Body) 99 | 100 | n := 0 101 | for { 102 | var out subRes 103 | if _ = dec.Decode(&out); err != nil { 104 | log.Printf("failed decode response body: %v\n", err) 105 | log.Printf("restarting consumer %d\n", id) 106 | continue restart 107 | } 108 | 109 | if out.Error != "" { 110 | res.Body.Close() 111 | log.Printf("received error: %s\n", out.Error) 112 | log.Printf("restarting consumer %d\n", id) 113 | continue restart 114 | } 115 | 116 | log.Printf("consumed message: %s\n", mustBase64Decode(out.Msg)) 117 | 118 | if !*validate { 119 | t := time.Duration(rand.Intn(5)) * time.Second 120 | log.Println("doing some work for", t) 121 | time.Sleep(t) 122 | } 123 | 124 | if *validate { 125 | c, _ := strconv.Atoi(mustBase64Decode(out.Msg)) 126 | if c != n { 127 | panic("uh oh") 128 | } 129 | 130 | // Randomly choose to ack or nack 131 | if rand.Intn(*nackChance) == 0 { 132 | _ = enc.Encode("NACK") 133 | continue 134 | } 135 | 136 | n++ 137 | } 138 | 139 | _ = enc.Encode("ACK") 140 | } 141 | } 142 | } 143 | 144 | func mustBase64Decode(b string) string { 145 | s, err := base64.StdEncoding.DecodeString(string(b)) 146 | if err != nil { 147 | log.Fatal(err) 148 | } 149 | 150 | return string(s) 151 | } 152 | -------------------------------------------------------------------------------- /flushwriter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | ) 7 | 8 | type flushWriter struct { 9 | f http.Flusher 10 | w io.Writer 11 | } 12 | 13 | func newFlushWriter(w io.Writer) flushWriter { 14 | fw := flushWriter{w: w} 15 | if f, ok := w.(http.Flusher); ok { 16 | fw.f = f 17 | } 18 | 19 | return fw 20 | } 21 | 22 | func (f flushWriter) Write(p []byte) (n int, err error) { 23 | n, err = f.w.Write(p) 24 | if f.f != nil { 25 | f.f.Flush() 26 | } 27 | 28 | return 29 | } 30 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tomarrell/miniqueue 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/golang/mock v1.6.0 7 | github.com/gorilla/mux v1.8.0 8 | github.com/rs/xid v1.4.0 9 | github.com/rs/zerolog v1.28.0 10 | github.com/stretchr/testify v1.6.1 11 | github.com/syndtr/goleveldb v1.0.0 12 | github.com/tidwall/redcon v1.6.2 13 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 14 | ) 15 | 16 | require ( 17 | github.com/davecgh/go-spew v1.1.1 // indirect 18 | github.com/golang/snappy v0.0.4 // indirect 19 | github.com/mattn/go-colorable v0.1.13 // indirect 20 | github.com/mattn/go-isatty v0.0.17 // indirect 21 | github.com/pmezard/go-difflib v1.0.0 // indirect 22 | github.com/tidwall/btree v1.6.0 // indirect 23 | github.com/tidwall/match v1.1.1 // indirect 24 | golang.org/x/sys v0.4.0 // indirect 25 | golang.org/x/text v0.3.3 // indirect 26 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect 27 | ) 28 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 6 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 7 | github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= 8 | github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= 9 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 10 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 11 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 12 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 13 | github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= 14 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 15 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 16 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 17 | github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= 18 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 19 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 20 | github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= 21 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 22 | github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= 23 | github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 24 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 25 | github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= 26 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 27 | github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= 28 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 29 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 30 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 31 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 32 | github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= 33 | github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= 34 | github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= 35 | github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= 36 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 37 | github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= 38 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 39 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 40 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 41 | github.com/tidwall/btree v1.1.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= 42 | github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= 43 | github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= 44 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 45 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 46 | github.com/tidwall/redcon v1.6.2 h1:5qfvrrybgtO85jnhSravmkZyC0D+7WstbfCs3MmPhow= 47 | github.com/tidwall/redcon v1.6.2/go.mod h1:p5Wbsgeyi2VSTBWOcA5vRXrOb9arFTcU2+ZzFjqV75Y= 48 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 49 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 50 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 51 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 52 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 53 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 54 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 55 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= 56 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 57 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 58 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 59 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 60 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 61 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 62 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 63 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 64 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 65 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 66 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 67 | golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 68 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 69 | golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= 70 | golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 71 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 72 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 73 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 74 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 75 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 76 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 77 | golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 78 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 79 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 80 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 81 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 82 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 83 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 84 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 85 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 86 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 87 | gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= 88 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 89 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 90 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 91 | -------------------------------------------------------------------------------- /http.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "strconv" 11 | "strings" 12 | 13 | "github.com/gorilla/mux" 14 | "github.com/rs/xid" 15 | "github.com/rs/zerolog" 16 | "github.com/rs/zerolog/log" 17 | ) 18 | 19 | const topicVarKey = "topic" 20 | 21 | const ( 22 | // CmdInit is the command to be sent with the initial subscribe request to 23 | // indicate a new consumer should be initialised. 24 | CmdInit = "INIT" 25 | // CmdAck notifies the server that the outstanding message was processed 26 | // successfully and can be removed from the queue. 27 | CmdAck = "ACK" 28 | // CmdNack notifies the server that the outstanding message was not processed 29 | // successfully and should be prepended to the queue to be processed again as 30 | // soon as possible. 31 | CmdNack = "NACK" 32 | // CmdBack notifies the server that the outstanding message was not processed 33 | // successfully and should be appended to the back of the queue to be 34 | // processed again after all the currently outstanding messages have been 35 | // processed. 36 | CmdBack = "BACK" 37 | // CmdDack notifies the server that the outstanding message was not processed 38 | // successfully and that it should be delayed by t seconds before being 39 | // prepended to the front of the queue for reprocessing. 40 | CmdDack = "DACK" 41 | ) 42 | 43 | const ( 44 | errInvalidTopicValue = serverError("invalid topic value") 45 | errReadBody = serverError("error reading the request body") 46 | errPublish = serverError("error publishing to broker") 47 | errNextValue = serverError("error getting next value for consumer") 48 | errAck = serverError("error ACKing message") 49 | errNack = serverError("error NACKing message") 50 | errBack = serverError("error BACKing message") 51 | errDecodingCmd = serverError("error decoding command") 52 | errRequestCancelled = serverError("request context cancelled") 53 | errPurge = serverError("failed to purge topic") 54 | ) 55 | 56 | type serverError string 57 | 58 | func (e serverError) Error() string { 59 | return string(e) 60 | } 61 | 62 | type httpServer struct { 63 | broker brokerer 64 | } 65 | 66 | func newHTTPServer(broker brokerer) *httpServer { 67 | return &httpServer{ 68 | broker: broker, 69 | } 70 | } 71 | 72 | func (s httpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { 73 | route := mux.NewRouter() 74 | 75 | route.HandleFunc("/{topic}", deleteHandler(s.broker)).Methods(http.MethodDelete) 76 | route.HandleFunc("/publish/{topic}", publishHandler(s.broker)).Methods(http.MethodPost) 77 | route.HandleFunc("/subscribe/{topic}", subscribeHandler(s.broker)).Methods(http.MethodPost) 78 | 79 | route.ServeHTTP(w, r) 80 | } 81 | 82 | func deleteHandler(broker brokerer) http.HandlerFunc { 83 | return func(w http.ResponseWriter, r *http.Request) { 84 | log := log.With(). 85 | Str("request_id", xid.New().String()). 86 | Str("handler", "delete"). 87 | Logger() 88 | 89 | // Read topic 90 | vars := mux.Vars(r) 91 | topic, ok := vars[topicVarKey] 92 | if !ok { 93 | log.Debug().Msg("invalid topic in path") 94 | 95 | w.WriteHeader(http.StatusBadRequest) 96 | respondError(log, json.NewEncoder(w), errInvalidTopicValue.Error()) 97 | 98 | return 99 | } 100 | 101 | log = log.With(). 102 | Str("topic", topic). 103 | Logger() 104 | 105 | log.Info().Msg("deleting topic") 106 | 107 | if err := broker.Purge(topic); err != nil { 108 | log.Err(err).Msg("failed purging topic") 109 | 110 | w.WriteHeader(http.StatusInternalServerError) 111 | respondError(log, json.NewEncoder(w), errPurge.Error()) 112 | 113 | return 114 | } 115 | 116 | log.Info().Msg("topic deleted") 117 | } 118 | } 119 | 120 | func publishHandler(broker brokerer) http.HandlerFunc { 121 | return func(w http.ResponseWriter, r *http.Request) { 122 | log := log.With(). 123 | Str("request_id", xid.New().String()). 124 | Str("handler", "publish"). 125 | Logger() 126 | 127 | // Read topic 128 | vars := mux.Vars(r) 129 | topic, ok := vars[topicVarKey] 130 | if !ok { 131 | log.Debug().Msg("invalid topic in path") 132 | 133 | w.WriteHeader(http.StatusBadRequest) 134 | respondError(log, json.NewEncoder(w), errInvalidTopicValue.Error()) 135 | 136 | return 137 | } 138 | 139 | log = log.With(). 140 | Str("topic", topic). 141 | Logger() 142 | 143 | log.Info().Msg("publishing to topic") 144 | 145 | b, err := ioutil.ReadAll(r.Body) 146 | if err != nil { 147 | log.Err(err).Msg("failed reading request body") 148 | 149 | w.WriteHeader(http.StatusInternalServerError) 150 | respondError(log, json.NewEncoder(w), errReadBody.Error()) 151 | 152 | return 153 | } 154 | defer r.Body.Close() 155 | 156 | newValue := newValue(b) 157 | 158 | if err := broker.Publish(topic, newValue); err != nil { 159 | log.Err(err).Msg("failed to publish to broker") 160 | 161 | w.WriteHeader(http.StatusInternalServerError) 162 | respondError(log, json.NewEncoder(w), errPublish.Error()) 163 | 164 | return 165 | } 166 | 167 | w.WriteHeader(http.StatusCreated) 168 | 169 | log.Debug(). 170 | Str("body", string(b)). 171 | Msg("successfully published to topic") 172 | } 173 | } 174 | 175 | func subscribeHandler(broker brokerer) http.HandlerFunc { 176 | return func(w http.ResponseWriter, r *http.Request) { 177 | ctx := r.Context() 178 | 179 | log := log.With(). 180 | Str("request_id", xid.New().String()). 181 | Str("handler", "subscribe"). 182 | Logger() 183 | 184 | // Read topic from URL 185 | vars := mux.Vars(r) 186 | topic, ok := vars[topicVarKey] 187 | if !ok { 188 | log.Debug().Msg("invalid topic in path") 189 | 190 | w.WriteHeader(http.StatusBadRequest) 191 | respondError(log, json.NewEncoder(w), errInvalidTopicValue.Error()) 192 | 193 | return 194 | } 195 | 196 | log = log.With().Str("topic", topic).Logger() 197 | 198 | log.Info(). 199 | Msg("subscribing to topic") 200 | 201 | // Wrap the writer in a flushWriter in order to immediately flush each write 202 | // to the client. 203 | cons := broker.Subscribe(topic) 204 | enc := json.NewEncoder(newFlushWriter(w)) 205 | dec := json.NewDecoder(r.Body) 206 | 207 | for { 208 | log := log 209 | 210 | var cmd string 211 | if err := dec.Decode(&cmd); isDisconnect(err) { 212 | log.Warn().Msg("client disconnected") 213 | 214 | if err := cons.Nack(); !errors.Is(err, errNackMsgNotExist) && err != nil { 215 | log.Err(err).Msg("nacking on disconnect") 216 | } 217 | 218 | if err := broker.Unsubscribe(cons.topic, cons.id); err != nil { 219 | log.Err(err).Msg("unsubscribing consumer") 220 | } 221 | 222 | return 223 | } else if err != nil { 224 | log.Err(err).Msg("failed decoding command") 225 | respondError(log, enc, errDecodingCmd.Error()) 226 | 227 | return 228 | } 229 | 230 | log = log.With().Str("cmd", cmd).Logger() 231 | 232 | cmdArgs := strings.Split(cmd, " ") 233 | 234 | switch cmdArgs[0] { 235 | case CmdInit: 236 | log.Debug().Msg("initialising consumer") 237 | 238 | handleConsumerNext(ctx, log, enc, cons) 239 | 240 | case CmdAck: 241 | log.Debug().Msg("ACKing message") 242 | 243 | if err := cons.Ack(); err != nil { 244 | log.Err(err).Msg("failed to ACK") 245 | respondError(log, enc, errAck.Error()) 246 | 247 | return 248 | } 249 | 250 | handleConsumerNext(ctx, log, enc, cons) 251 | 252 | case CmdNack: 253 | log.Debug().Msg("NACKing message") 254 | 255 | if err := cons.Nack(); err != nil { 256 | log.Err(err).Msg("failed to NACK") 257 | respondError(log, enc, errNack.Error()) 258 | 259 | return 260 | } 261 | 262 | handleConsumerNext(ctx, log, enc, cons) 263 | 264 | case CmdBack: 265 | log.Debug().Msg("BACKing message") 266 | 267 | if err := cons.Back(); err != nil { 268 | log.Err(err).Msg("failed to BACK") 269 | respondError(log, enc, errBack.Error()) 270 | 271 | return 272 | } 273 | 274 | handleConsumerNext(ctx, log, enc, cons) 275 | 276 | case CmdDack: 277 | log.Debug().Msg("DACKing message") 278 | 279 | if len(cmdArgs) < 2 { 280 | respondError(log, enc, "too few arguments provided to DACK") 281 | 282 | return 283 | } 284 | 285 | seconds, err := strconv.Atoi(cmdArgs[1]) 286 | if err != nil { 287 | respondError(log, enc, "invalid DACK duration argument at position [1]") 288 | 289 | return 290 | } 291 | 292 | if err := cons.Dack(seconds); err != nil { 293 | log.Err(err).Msg("failed to BACK") 294 | respondError(log, enc, errBack.Error()) 295 | 296 | return 297 | } 298 | 299 | handleConsumerNext(ctx, log, enc, cons) 300 | 301 | default: 302 | log.Warn().Msg("unrecognised command received") 303 | 304 | respondError(log, enc, "unrecognised command received") 305 | } 306 | } 307 | } 308 | } 309 | 310 | // handleConsumerNext attempts to retrieve the next value from the consumer, 311 | // handling any errors that may occur and responding to the client accordingly. 312 | func handleConsumerNext(ctx context.Context, log zerolog.Logger, enc *json.Encoder, cons *consumer) { 313 | val, err := cons.Next(ctx) 314 | switch { 315 | case errors.Is(err, errRequestCancelled): 316 | log.Info().Msg("client disconnected while waiting for message") 317 | 318 | return 319 | case err != nil: 320 | log.Err(err).Msg("failed to get next value for topic") 321 | respondError(log, enc, errNextValue.Error()) 322 | 323 | return 324 | default: 325 | respondMsg(log, enc, val) 326 | 327 | log.Debug(). 328 | Str("msg", string(val.Raw)). 329 | Msg("written message to client") 330 | } 331 | } 332 | 333 | func isDisconnect(err error) bool { 334 | return err != nil && (strings.Contains(err.Error(), "client disconnected") || 335 | strings.Contains(err.Error(), "; CANCEL") || 336 | errors.Is(err, io.EOF)) 337 | } 338 | -------------------------------------------------------------------------------- /http_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "net/http" 10 | "net/http/httptest" 11 | "strings" 12 | "testing" 13 | "time" 14 | 15 | "github.com/golang/mock/gomock" 16 | "github.com/gorilla/mux" 17 | "github.com/rs/zerolog" 18 | "github.com/stretchr/testify/assert" 19 | "github.com/syndtr/goleveldb/leveldb" 20 | "github.com/syndtr/goleveldb/leveldb/storage" 21 | ) 22 | 23 | const defaultTopic = "test_topic" 24 | 25 | func TestPublishSingleMessage(t *testing.T) { 26 | assert := assert.New(t) 27 | ctrl := gomock.NewController(t) 28 | defer ctrl.Finish() 29 | 30 | msg := newValue([]byte("test_value")) 31 | 32 | mockBroker := NewMockbrokerer(ctrl) 33 | mockBroker.EXPECT().Publish(defaultTopic, msg) 34 | 35 | rec := NewRecorder() 36 | req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/publish/%s", defaultTopic), bytes.NewReader(msg.Raw)) 37 | 38 | srv := newHTTPServer(mockBroker) 39 | srv.ServeHTTP(rec, req) 40 | 41 | assert.Equal(http.StatusCreated, rec.Code) 42 | } 43 | 44 | func TestSubscribeSingleMessage(t *testing.T) { 45 | assert := assert.New(t) 46 | 47 | db, err := leveldb.Open(storage.NewMemStorage(), nil) 48 | assert.NoError(err) 49 | 50 | b := newBroker(&store{db: db}) 51 | 52 | // Publish to the topic 53 | pubW := NewRecorder() 54 | msg := "test_message" 55 | r := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/publish/%s", defaultTopic), strings.NewReader(msg)) 56 | r = mux.SetURLVars(r, map[string]string{"topic": defaultTopic}) 57 | 58 | publishHandler(b)(pubW, r) 59 | assert.Equal(http.StatusCreated, pubW.Code) 60 | 61 | // Subscribe to the same topic 62 | subW := NewRecorder() 63 | r = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/subscribe/%s", defaultTopic), helperMustEncodeString(CmdInit)) 64 | r = mux.SetURLVars(r, map[string]string{"topic": defaultTopic}) 65 | 66 | go subscribeHandler(b)(subW, r) 67 | 68 | // Wait for the first message to be written 69 | decoder := NewDecodeWaiter(subW) 70 | 71 | // Read the first message 72 | var out subResponse 73 | assert.NoError(decoder.WaitAndDecode(&out)) 74 | assert.Equal(msg, string(out.Msg)) 75 | } 76 | 77 | func TestSubscribeAck(t *testing.T) { 78 | assert := assert.New(t) 79 | 80 | db, err := leveldb.Open(storage.NewMemStorage(), nil) 81 | assert.NoError(err) 82 | 83 | b := newBroker(&store{db: db}) 84 | 85 | // Publish to the topic 86 | msg1 := "test_message_1" 87 | r := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/publish/%s", defaultTopic), strings.NewReader(msg1)) 88 | r = mux.SetURLVars(r, map[string]string{"topic": defaultTopic}) 89 | 90 | // Publish twice 91 | pubW := NewRecorder() 92 | 93 | publishHandler(b)(pubW, r) 94 | assert.Equal(http.StatusCreated, pubW.Code) 95 | 96 | // Publish a second time to the topic with a different body 97 | msg2 := "test_message_2" 98 | r = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/publish/%s", defaultTopic), strings.NewReader(msg2)) 99 | r = mux.SetURLVars(r, map[string]string{"topic": defaultTopic}) 100 | 101 | publishHandler(b)(pubW, r) 102 | assert.Equal(http.StatusCreated, pubW.Code) 103 | 104 | // Subscribe to the same topic 105 | reader, writer := io.Pipe() 106 | encoder := json.NewEncoder(writer) 107 | go func() { 108 | assert.NoError(encoder.Encode(CmdInit)) 109 | }() 110 | 111 | subW := NewRecorder() 112 | r = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/subscribe/%s", defaultTopic), reader) 113 | r = mux.SetURLVars(r, map[string]string{"topic": defaultTopic}) 114 | 115 | go subscribeHandler(b)(subW, r) 116 | 117 | // Wait for the first message to be written 118 | decoder := NewDecodeWaiter(subW) 119 | 120 | // Read the first message, expect the first item published to the topic 121 | var out subResponse 122 | assert.NoError(decoder.WaitAndDecode(&out)) 123 | assert.Equal(msg1, string(out.Msg)) 124 | 125 | // Send an ACK back to the server, expect it to reply with next msg 126 | assert.NoError(encoder.Encode(CmdAck)) 127 | assert.NoError(decoder.WaitAndDecode(&out)) 128 | assert.Equal(msg2, string(out.Msg)) 129 | } 130 | 131 | func TestServerPublishSubscribeAck(t *testing.T) { 132 | assert := assert.New(t) 133 | 134 | srv, _, srvCloser := helperNewTestHTTPServer(t) 135 | defer srvCloser() 136 | 137 | // Publish 138 | msg1 := "test_msg_1" 139 | helperPublishMessage(t, srv, defaultTopic, msg1) 140 | 141 | // Setup a subscriber 142 | encoder, decoder, closeSub := helperSubscribeTopic(t, srv, defaultTopic) 143 | defer closeSub() 144 | 145 | // Consume message 146 | var out subResponse 147 | assert.NoError(decoder.Decode(&out)) 148 | assert.Equal(msg1, string(out.Msg)) 149 | 150 | // Send back and ACK 151 | assert.NoError(encoder.Encode(CmdAck)) 152 | 153 | // Simulate the next publish coming in slightly later 154 | // i.e. the next record may not be available already on the topic to 155 | // immediately send back 156 | time.Sleep(100 * time.Millisecond) 157 | 158 | // Publish a new message to the same topic 159 | msg2 := "test_msg_2" 160 | helperPublishMessage(t, srv, defaultTopic, msg2) 161 | 162 | // Read again from the queue, expect the new message 163 | assert.NoError(decoder.Decode(&out)) 164 | assert.Equal(msg2, string(out.Msg)) 165 | 166 | // Send back and ACK 167 | assert.NoError(encoder.Encode(CmdAck)) 168 | } 169 | 170 | func TestServerNack(t *testing.T) { 171 | assert := assert.New(t) 172 | 173 | srv, _, srvCloser := helperNewTestHTTPServer(t) 174 | defer srvCloser() 175 | 176 | msg1 := "test_msg_1" 177 | helperPublishMessage(t, srv, defaultTopic, msg1) 178 | 179 | enc, decoder, _ := helperSubscribeTopic(t, srv, defaultTopic) 180 | 181 | var out subResponse 182 | assert.NoError(decoder.Decode(&out)) 183 | assert.Equal(msg1, string(out.Msg)) 184 | 185 | assert.NoError(enc.Encode("NACK")) 186 | 187 | assert.NoError(decoder.Decode(&out)) 188 | assert.Equal(msg1, string(out.Msg)) 189 | } 190 | 191 | func TestServerBack(t *testing.T) { 192 | assert := assert.New(t) 193 | 194 | srv, _, srvCloser := helperNewTestHTTPServer(t) 195 | defer srvCloser() 196 | 197 | msg1 := "test_msg_1" 198 | helperPublishMessage(t, srv, defaultTopic, msg1) 199 | 200 | msg2 := "test_msg_2" 201 | helperPublishMessage(t, srv, defaultTopic, msg2) 202 | 203 | enc, decoder, _ := helperSubscribeTopic(t, srv, defaultTopic) 204 | 205 | var out subResponse 206 | assert.NoError(decoder.Decode(&out)) 207 | assert.Equal(msg1, string(out.Msg)) 208 | 209 | assert.NoError(enc.Encode("BACK")) 210 | 211 | assert.NoError(decoder.Decode(&out)) 212 | assert.Equal(msg2, string(out.Msg)) 213 | } 214 | 215 | func TestServerDack_MissingArg(t *testing.T) { 216 | assert := assert.New(t) 217 | 218 | srv, _, srvCloser := helperNewTestHTTPServer(t) 219 | defer srvCloser() 220 | 221 | msg1 := "test_msg_1" 222 | helperPublishMessage(t, srv, defaultTopic, msg1) 223 | 224 | enc, decoder, _ := helperSubscribeTopic(t, srv, defaultTopic) 225 | 226 | var out subResponse 227 | assert.NoError(decoder.Decode(&out)) 228 | assert.Equal(msg1, string(out.Msg)) 229 | 230 | assert.NoError(enc.Encode("DACK")) 231 | 232 | assert.NoError(decoder.Decode(&out)) 233 | assert.Contains(out.Error, "too few arguments") 234 | } 235 | 236 | func TestServerDack_InvalidArg(t *testing.T) { 237 | assert := assert.New(t) 238 | 239 | srv, _, srvCloser := helperNewTestHTTPServer(t) 240 | defer srvCloser() 241 | 242 | msg1 := "test_msg_1" 243 | helperPublishMessage(t, srv, defaultTopic, msg1) 244 | 245 | enc, decoder, _ := helperSubscribeTopic(t, srv, defaultTopic) 246 | 247 | var out subResponse 248 | assert.NoError(decoder.Decode(&out)) 249 | assert.Equal(msg1, string(out.Msg)) 250 | 251 | assert.NoError(enc.Encode("DACK oops")) 252 | 253 | assert.NoError(decoder.Decode(&out)) 254 | assert.Contains(out.Error, "invalid DACK duration argument") 255 | } 256 | 257 | func TestServerDack(t *testing.T) { 258 | assert := assert.New(t) 259 | 260 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 261 | defer cancel() 262 | 263 | srv, hooks, srvCloser := helperNewTestHTTPServer(t) 264 | defer srvCloser() 265 | go hooks.b.ProcessDelays(ctx, 100*time.Millisecond) 266 | 267 | msg1 := "test_msg_1" 268 | helperPublishMessage(t, srv, defaultTopic, msg1) 269 | 270 | enc, decoder, _ := helperSubscribeTopic(t, srv, defaultTopic) 271 | 272 | var out subResponse 273 | assert.NoError(decoder.Decode(&out)) 274 | assert.Equal(msg1, string(out.Msg)) 275 | assert.Equal(0, out.DackCount) 276 | 277 | assert.NoError(enc.Encode("DACK 1")) 278 | 279 | assert.NoError(decoder.Decode(&out)) 280 | assert.Equal(msg1, string(out.Msg)) 281 | assert.Equal(1, out.DackCount) 282 | } 283 | 284 | func TestServerConnectionLost(t *testing.T) { 285 | assert := assert.New(t) 286 | 287 | srv, _, srvCloser := helperNewTestHTTPServer(t) 288 | defer srvCloser() 289 | 290 | // Publish twice 291 | msg1 := "test_msg_1" 292 | helperPublishMessage(t, srv, defaultTopic, msg1) 293 | 294 | msg2 := "test_msg_2" 295 | helperPublishMessage(t, srv, defaultTopic, msg2) 296 | 297 | // Setup a subscriber 298 | _, decoder, closeSub := helperSubscribeTopic(t, srv, defaultTopic) 299 | 300 | // Consume message 301 | var out subResponse 302 | assert.NoError(decoder.Decode(&out)) 303 | assert.Equal(msg1, string(out.Msg)) 304 | 305 | // *Unexpectedly* close the connection 306 | closeSub() 307 | 308 | // We need to give it some time to Nack and be put back on the queue to 309 | // consume 310 | time.Sleep(100 * time.Millisecond) 311 | 312 | // Setup a new subscriber 313 | _, decoder, closeSub = helperSubscribeTopic(t, srv, defaultTopic) 314 | defer closeSub() 315 | 316 | // Expect the first message to be sent again as it was not acked 317 | out = subResponse{} 318 | assert.NoError(decoder.Decode(&out)) 319 | assert.Equal(msg1, string(out.Msg)) 320 | } 321 | 322 | func TestServerMultiConsumer(t *testing.T) { 323 | assert := assert.New(t) 324 | 325 | srv, _, srvCloser := helperNewTestHTTPServer(t) 326 | defer srvCloser() 327 | 328 | // Publish 329 | msg1 := "test_msg_1" 330 | helperPublishMessage(t, srv, defaultTopic, msg1) 331 | 332 | msg2 := "test_msg_2" 333 | helperPublishMessage(t, srv, defaultTopic, msg2) 334 | 335 | // Set up consumer 1 336 | encoder1, decoder1, closeSub := helperSubscribeTopic(t, srv, defaultTopic) 337 | defer closeSub() 338 | 339 | // Set up consumer 2 340 | encoder2, decoder2, closeSub := helperSubscribeTopic(t, srv, defaultTopic) 341 | defer closeSub() 342 | 343 | // Read from consumer 1 344 | var out1 subResponse 345 | assert.NoError(decoder1.Decode(&out1)) 346 | assert.Equal(msg1, string(out1.Msg)) 347 | 348 | // Read from consumer 2 349 | var out2 subResponse 350 | assert.NoError(decoder2.Decode(&out2)) 351 | assert.Equal(msg2, string(out2.Msg)) 352 | 353 | // Publish again 354 | msg3 := "test_msg_3" 355 | helperPublishMessage(t, srv, defaultTopic, msg3) 356 | 357 | msg4 := "test_msg_4" 358 | helperPublishMessage(t, srv, defaultTopic, msg4) 359 | 360 | // Consumer 1 sends back an ACK for its message 361 | assert.NoError(encoder1.Encode(CmdAck)) 362 | 363 | // It should receive the first message which has just been published 364 | var out3 subResponse 365 | assert.NoError(decoder1.Decode(&out3)) 366 | assert.Equal(msg3, string(out3.Msg)) 367 | 368 | // Consumer 2 sends back an ACK for its message 369 | assert.NoError(encoder2.Encode(CmdAck)) 370 | 371 | // It should receive the second message that was published 372 | var out4 subResponse 373 | assert.NoError(decoder2.Decode(&out4)) 374 | assert.Equal(msg4, string(out4.Msg)) 375 | } 376 | 377 | func TestServerMultiConsumerConnectionLost(t *testing.T) { 378 | assert := assert.New(t) 379 | 380 | srv, hooks, srvCloser := helperNewTestHTTPServer(t) 381 | defer srvCloser() 382 | 383 | // Publish once 384 | msg1 := "test_msg_1" 385 | helperPublishMessage(t, srv, defaultTopic, msg1) 386 | 387 | // Setup a subscriber 388 | _, decoder1, closeSub1 := helperSubscribeTopic(t, srv, defaultTopic) 389 | 390 | // Setup a second subscriber, which will be blocked until the message is 391 | // Nacked due to the first subscriber disconnecting. 392 | done := make(chan struct{}) 393 | go func() { 394 | _, decoder2, closeSub2 := helperSubscribeTopic(t, srv, defaultTopic) 395 | defer closeSub2() 396 | 397 | // Expect the first message to be sent again as it was not acked 398 | var out subResponse 399 | assert.NoError(decoder2.Decode(&out)) 400 | assert.Equal(msg1, string(out.Msg)) 401 | done <- struct{}{} 402 | }() 403 | 404 | // Consume message 405 | var out subResponse 406 | assert.NoError(decoder1.Decode(&out)) 407 | assert.Equal(msg1, string(out.Msg)) 408 | 409 | // *Unexpectedly* close the connection 410 | closeSub1() 411 | 412 | select { 413 | case <-time.After(time.Second): 414 | assert.FailNow("timed out waiting for second decode") 415 | case <-done: 416 | consumers := hooks.b.consumers[defaultTopic] 417 | assert.Len(consumers, 1) 418 | } 419 | } 420 | 421 | func TestServerDelete(t *testing.T) { 422 | assert := assert.New(t) 423 | 424 | srv, _, srvCloser := helperNewTestHTTPServer(t) 425 | t.Cleanup(srvCloser) 426 | 427 | // Publish twice 428 | msg1 := "test_msg_1" 429 | helperPublishMessage(t, srv, defaultTopic, msg1) 430 | 431 | msg2 := "test_msg_2" 432 | helperPublishMessage(t, srv, defaultTopic, msg2) 433 | 434 | // Setup a subscriber 435 | encoder, decoder, closeSub := helperSubscribeTopic(t, srv, defaultTopic) 436 | defer closeSub() 437 | 438 | var out subResponse 439 | assert.NoError(decoder.Decode(&out)) 440 | assert.Equal(out.Msg, []byte(msg1)) 441 | 442 | // Purge the topic 443 | req, _ := http.NewRequest(http.MethodDelete, fmt.Sprintf("%s/%s", srv.URL, defaultTopic), nil) 444 | res, err := srv.Client().Do(req) 445 | assert.NoError(err) 446 | res.Body.Close() 447 | assert.Equal(http.StatusOK, res.StatusCode) 448 | 449 | assert.NoError(encoder.Encode("ACK")) 450 | 451 | // Publish again after the purge 452 | msg3 := "test_msg_3" 453 | helperPublishMessage(t, srv, defaultTopic, msg3) 454 | 455 | // Expect that it consumes the most recently published message 456 | out = subResponse{} 457 | assert.NoError(decoder.Decode(&out)) 458 | assert.Equal("", out.Error) 459 | assert.Equal([]byte(msg3), out.Msg) 460 | 461 | time.Sleep(time.Second) 462 | } 463 | 464 | // Benchmarking 465 | 466 | func BenchmarkPublish(b *testing.B) { 467 | zerolog.SetGlobalLevel(zerolog.Disabled) 468 | 469 | const ( 470 | topic = "test_topic" 471 | msg = "test_value" 472 | ) 473 | 474 | db, err := leveldb.Open(storage.NewMemStorage(), nil) 475 | assert.NoError(b, err) 476 | 477 | srv := httptest.NewUnstartedServer(newHTTPServer(newBroker(&store{db: db}))) 478 | srv.EnableHTTP2 = true 479 | srv.StartTLS() 480 | 481 | var ( 482 | publishPath = fmt.Sprintf("%s/publish/%s", srv.URL, topic) 483 | ) 484 | 485 | b.ResetTimer() 486 | 487 | for n := 0; n < b.N; n++ { 488 | req, _ := http.NewRequest(http.MethodPost, publishPath, strings.NewReader(msg)) 489 | _, err := srv.Client().Do(req) 490 | assert.NoError(b, err) 491 | } 492 | } 493 | 494 | // 495 | // Helpers 496 | // 497 | 498 | type hooks struct { 499 | b *broker 500 | } 501 | 502 | // Returns a new, started, httptest server and a corresponding function which 503 | // will force close connections and close the server when called. 504 | func helperNewTestHTTPServer(t *testing.T) (*httptest.Server, hooks, func()) { 505 | t.Helper() 506 | 507 | db, err := leveldb.Open(storage.NewMemStorage(), nil) 508 | assert.NoError(t, err) 509 | 510 | b := newBroker(&store{path: "", db: db}) 511 | srv := httptest.NewUnstartedServer(newHTTPServer(b)) 512 | 513 | srv.EnableHTTP2 = true 514 | srv.StartTLS() 515 | 516 | return srv, hooks{b}, func() { 517 | srv.CloseClientConnections() 518 | srv.Close() 519 | } 520 | } 521 | 522 | func helperSubscribeTopic(t *testing.T, srv *httptest.Server, topicName string) (*json.Encoder, *json.Decoder, func()) { 523 | t.Helper() 524 | 525 | reader, writer := io.Pipe() 526 | encoder := json.NewEncoder(writer) 527 | go func() { 528 | assert.NoError(t, encoder.Encode(CmdInit)) 529 | }() 530 | 531 | req, err := http.NewRequest( 532 | http.MethodPost, 533 | fmt.Sprintf("%s/subscribe/%s", srv.URL, topicName), 534 | reader, 535 | ) 536 | assert.NoError(t, err) 537 | 538 | // Subscribe to topic 539 | res, err := srv.Client().Do(req) 540 | assert.NoError(t, err) 541 | assert.Equal(t, http.StatusOK, res.StatusCode) 542 | 543 | decoder := json.NewDecoder(res.Body) 544 | return encoder, decoder, func() { 545 | res.Body.Close() 546 | } 547 | } 548 | 549 | func helperPublishMessage(t *testing.T, srv *httptest.Server, topicName, msg string) *http.Response { 550 | t.Helper() 551 | 552 | publishPath := fmt.Sprintf("%s/publish/%s", srv.URL, topicName) 553 | req, err := http.NewRequest(http.MethodPost, publishPath, strings.NewReader(msg)) 554 | assert.NoError(t, err) 555 | 556 | res, err := srv.Client().Do(req) 557 | assert.NoError(t, err) 558 | assert.Equal(t, http.StatusCreated, res.StatusCode) 559 | 560 | t.Cleanup(func() { 561 | res.Body.Close() 562 | }) 563 | 564 | return res 565 | } 566 | 567 | func helperMustEncodeString(str string) io.Reader { 568 | var buf bytes.Buffer 569 | if err := json.NewEncoder(&buf).Encode(str); err != nil { 570 | panic(err) 571 | } 572 | 573 | return &buf 574 | } 575 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | _ "embed" 6 | "errors" 7 | "flag" 8 | "fmt" 9 | "net/http" 10 | "os" 11 | "time" 12 | 13 | "github.com/rs/zerolog" 14 | "github.com/rs/zerolog/log" 15 | "github.com/tidwall/redcon" 16 | ) 17 | 18 | var ( 19 | //go:embed VERSION 20 | version string 21 | ) 22 | 23 | const ( 24 | defaultHumanReadable = false 25 | defaultPort = 8080 26 | defaultCertPath = "./testdata/localhost.pem" 27 | defaultKeyPath = "./testdata/localhost-key.pem" 28 | defaultDBPath = "./data" 29 | defaultLogLevel = "debug" 30 | ) 31 | 32 | func main() { 33 | var ( 34 | humanReadable = flag.Bool("human", defaultHumanReadable, "human readable logging output") 35 | port = flag.Int("port", defaultPort, "port used to run the server") 36 | tlsCertPath = flag.String("cert", defaultCertPath, "path to TLS certificate") 37 | tlsKeyPath = flag.String("key", defaultKeyPath, "path to TLS key") 38 | dbPath = flag.String("db", defaultDBPath, "path to the db file") 39 | logLevel = flag.String("level", defaultLogLevel, "(disabled|debug|info)") 40 | delayPeriod = flag.Duration("period", time.Second, "period between runs to check and restore delayed messages") 41 | ) 42 | 43 | flag.Parse() 44 | 45 | if *humanReadable { 46 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 47 | } 48 | 49 | switch *logLevel { 50 | case "debug": 51 | log.Logger = log.Level(zerolog.DebugLevel) 52 | case "info": 53 | log.Logger = log.Level(zerolog.InfoLevel) 54 | case "disabled": 55 | log.Logger = log.Level(zerolog.Disabled) 56 | default: 57 | log.Fatal().Msg("invalid log level, see -h") 58 | } 59 | 60 | if *dbPath == defaultDBPath { 61 | log.Warn(). 62 | Msgf("no DB path specified, using default %s", defaultDBPath) 63 | } 64 | 65 | if *tlsCertPath == defaultCertPath { 66 | log.Warn(). 67 | Msgf("no TLS certificate path specified, using default %s", defaultCertPath) 68 | } 69 | 70 | if *tlsKeyPath == defaultKeyPath { 71 | log.Warn(). 72 | Msgf("no TLS key path specified, using default %s", defaultKeyPath) 73 | } 74 | 75 | ctx := context.Background() 76 | 77 | b := newBroker(newStore(*dbPath)) 78 | go b.ProcessDelays(ctx, *delayPeriod) 79 | 80 | switch { 81 | case false: 82 | runHTTP(b, port, tlsCertPath, tlsKeyPath) 83 | 84 | case true: 85 | runRedis(b, tlsCertPath, tlsKeyPath) 86 | } 87 | } 88 | 89 | func runRedis(b brokerer, tlsCertPath, tlsKeyPath *string) { 90 | log.Info(). 91 | Msg("starting miniqueue over redis") 92 | 93 | r := newRedis(b) 94 | 95 | err := redcon.ListenAndServe("localhost:6379", r.handleCmd, nil, nil) 96 | if err != nil { 97 | log.Err(err).Msg("closing server") 98 | } 99 | } 100 | 101 | func runHTTP(b brokerer, port *int, tlsCertPath, tlsKeyPath *string) { 102 | // Start the server 103 | p := fmt.Sprintf(":%d", *port) 104 | 105 | log.Info(). 106 | Str("port", p). 107 | Msg("starting miniqueue over HTTP") 108 | 109 | srv := newHTTPServer(b) 110 | 111 | if err := http.ListenAndServeTLS(p, *tlsCertPath, *tlsKeyPath, srv); !errors.Is(err, http.ErrServerClosed) { 112 | log.Fatal(). 113 | Err(err). 114 | Msg("server closed") 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/rs/zerolog" 7 | "github.com/rs/zerolog/log" 8 | ) 9 | 10 | func init() { 11 | log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}) 12 | } 13 | -------------------------------------------------------------------------------- /redis.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "strings" 9 | 10 | "github.com/rs/zerolog/log" 11 | "github.com/tidwall/redcon" 12 | ) 13 | 14 | const ( 15 | respOK = "OK" 16 | ) 17 | 18 | type redis struct { 19 | broker brokerer 20 | } 21 | 22 | func newRedis(b brokerer) *redis { 23 | return &redis{ 24 | broker: b, 25 | } 26 | } 27 | 28 | func (r *redis) handleCmd(conn redcon.Conn, rcmd redcon.Command) { 29 | cmd := string(rcmd.Args[0]) 30 | switch strings.ToLower(cmd) { 31 | default: 32 | conn.WriteError(fmt.Sprintf("unknown command '%s'", cmd)) 33 | 34 | case "info": 35 | conn.WriteString(fmt.Sprintf("redis_version:miniqueue_%s", version)) 36 | 37 | case "ping": 38 | conn.WriteString("pong") 39 | 40 | case "topics": 41 | handleRedisTopics(r.broker)(conn, rcmd) 42 | 43 | case "publish": 44 | handleRedisPublish(r.broker)(conn, rcmd) 45 | 46 | case "subscribe": 47 | handleRedisSubscribe(r.broker)(conn, rcmd) 48 | } 49 | } 50 | 51 | func handleRedisTopics(broker brokerer) redcon.HandlerFunc { 52 | return func(conn redcon.Conn, rcmd redcon.Command) { 53 | topics, err := broker.Topics() 54 | if err != nil { 55 | log.Err(err).Msg("failed to get topics") 56 | conn.WriteError("failed to get topics") 57 | return 58 | } 59 | 60 | conn.WriteString(fmt.Sprintf("%v", topics)) 61 | } 62 | } 63 | 64 | func handleRedisSubscribe(broker brokerer) redcon.HandlerFunc { 65 | return func(conn redcon.Conn, rcmd redcon.Command) { 66 | topic := string(rcmd.Args[1]) 67 | c := broker.Subscribe(topic) 68 | defer func() { 69 | if err := broker.Unsubscribe(topic, c.id); err != nil { 70 | log.Err(err).Msg("failed to unsubscribe") 71 | } 72 | }() 73 | 74 | log := log.With().Str("id", c.id).Logger() 75 | 76 | log.Debug().Str("topic", topic).Msg("new connection") 77 | 78 | ctx, cancel := context.WithCancel(context.Background()) 79 | defer cancel() 80 | 81 | // Detach the connection from the client so that we can control its 82 | // lifecycle independently. 83 | dconn := flushable{DetachedConn: conn.Detach()} 84 | dconn.SetContext(ctx) 85 | defer func() { 86 | log.Debug().Msg("closing connection") 87 | dconn.flush() 88 | dconn.Close() 89 | }() 90 | 91 | if len(rcmd.Args) != 2 { 92 | dconn.WriteError("invalid number of args, want: 2") 93 | return 94 | } 95 | 96 | for { 97 | select { 98 | case <-ctx.Done(): 99 | return 100 | default: 101 | } 102 | 103 | // Wait for a new value 104 | val, err := c.Next(ctx) 105 | if err != nil { 106 | log.Err(err).Msg("getting next value") 107 | dconn.WriteError("failed to get next value") 108 | return 109 | } 110 | 111 | log.Debug().Str("msg", string(val.Raw)).Msg("sending msg") 112 | 113 | dconn.WriteAny(val.Raw) 114 | if err := dconn.flush(); err != nil { 115 | log.Err(err).Msg("flushing msg") 116 | return 117 | } 118 | 119 | log.Debug().Msg("awaiting ack") 120 | 121 | cmd, err := dconn.ReadCommand() 122 | if errors.Is(err, io.EOF) { 123 | return 124 | } else if err != nil { 125 | log.Err(err).Msg("reading ack") 126 | dconn.WriteError("failed to get next value") 127 | return 128 | } 129 | 130 | if len(cmd.Args) != 1 { 131 | log.Error().Str("cmd", string(cmd.Raw)).Int("len", len(cmd.Args)).Msg("invalid cmd length") 132 | dconn.WriteError("invalid command") 133 | return 134 | } 135 | 136 | ackCmd := string(cmd.Args[0]) 137 | 138 | log.Debug().Str("cmd", ackCmd).Msg("received ack cmd") 139 | 140 | switch strings.ToUpper(ackCmd) { 141 | case CmdAck: 142 | if err := c.Ack(); err != nil { 143 | log.Err(err).Msg("acking") 144 | dconn.WriteError("failed to ack") 145 | return 146 | } 147 | case CmdBack: 148 | if err := c.Back(); err != nil { 149 | log.Err(err).Msg("backing") 150 | dconn.WriteError("failed to back") 151 | return 152 | } 153 | case CmdNack: 154 | if err := c.Nack(); err != nil { 155 | log.Err(err).Msg("Nacking") 156 | dconn.WriteError("failed to nack") 157 | return 158 | } 159 | case CmdDack: 160 | // TODO read extra arg 161 | if err := c.Dack(1); err != nil { 162 | log.Err(err).Msg("Nacking") 163 | dconn.WriteError("failed to nack") 164 | return 165 | } 166 | default: 167 | log.Error().Str("cmd", ackCmd).Msg("invalid ack command") 168 | dconn.WriteError("invalid ack command") 169 | return 170 | } 171 | 172 | dconn.WriteString(respOK) 173 | iferr(dconn.flush(), cancel) 174 | } 175 | } 176 | } 177 | 178 | func handleRedisPublish(broker brokerer) redcon.HandlerFunc { 179 | return func(conn redcon.Conn, rcmd redcon.Command) { 180 | if len(rcmd.Args) != 3 { 181 | conn.WriteError("invalid number of args, want: 3") 182 | return 183 | } 184 | 185 | var ( 186 | topic = string(rcmd.Args[1]) 187 | value = newValue(rcmd.Args[2]) 188 | ) 189 | 190 | if err := broker.Publish(topic, value); err != nil { 191 | log.Err(err).Msg("failed to publish") 192 | conn.WriteError("failed to publish") 193 | return 194 | } 195 | 196 | log.Debug(). 197 | Str("topic", topic). 198 | Str("msg", string(value.Raw)). 199 | Msg("msg published") 200 | 201 | conn.WriteString(respOK) 202 | } 203 | } 204 | 205 | type flushable struct { 206 | ctx context.Context 207 | 208 | redcon.DetachedConn 209 | } 210 | 211 | // Flush flushes pending messages to the client, handling any errors by 212 | // cancelling the receiver's context. 213 | func (f *flushable) flush() error { 214 | if err := f.DetachedConn.Flush(); err != nil { 215 | return err 216 | } 217 | 218 | return nil 219 | } 220 | 221 | func iferr(err error, fn func()) { 222 | if err != nil { 223 | fn() 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /redis_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "runtime" 8 | "strconv" 9 | "strings" 10 | "sync" 11 | "testing" 12 | "time" 13 | 14 | "github.com/stretchr/testify/require" 15 | redcon "github.com/tidwall/redcon" 16 | ) 17 | 18 | var redcliPath string 19 | 20 | func init() { 21 | if runtime.GOOS == "linux" && runtime.GOARCH == "amd64" { 22 | redcliPath = "./testdata/cmd/redcli_linux_amd64" 23 | } else if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" { 24 | redcliPath = "./testdata/cmd/redcli_darwin_arm64" 25 | } else { 26 | panic("unsupported test platform") 27 | } 28 | } 29 | 30 | func TestRedisPublish(t *testing.T) { 31 | t.Run("publish publishes to the respective queue", func(t *testing.T) { 32 | _ = helperNewTestRedisServer(t) 33 | 34 | var ( 35 | topic = "topic" 36 | val = "value" 37 | ) 38 | 39 | publishOne(t, topic, val) 40 | res := consumeOne(t, topic) 41 | require.Equal(t, val, res) 42 | }) 43 | } 44 | 45 | func TestRedisSubscribe(t *testing.T) { 46 | t.Run("subscriber waits for message", func(t *testing.T) { 47 | _ = helperNewTestRedisServer(t) 48 | 49 | var ( 50 | topic = "topic" 51 | val = "value" 52 | ) 53 | 54 | wg := sync.WaitGroup{} 55 | wg.Add(1) 56 | go func() { 57 | res := consumeOne(t, topic) 58 | require.Equal(t, val, res) 59 | wg.Done() 60 | }() 61 | 62 | publishOne(t, topic, val) 63 | wg.Wait() 64 | }) 65 | 66 | t.Run("subscriber waiting for two messages on same topic", func(t *testing.T) { 67 | _ = helperNewTestRedisServer(t) 68 | 69 | var ( 70 | topic = "topic" 71 | val1 = "value1" 72 | val2 = "value2" 73 | ) 74 | 75 | wg := sync.WaitGroup{} 76 | wg.Add(1) 77 | go func() { 78 | res := consumeOne(t, topic) 79 | require.Equal(t, val1, res) 80 | 81 | res = consumeOne(t, topic) 82 | require.Equal(t, val2, res) 83 | wg.Done() 84 | }() 85 | 86 | publishOne(t, topic, val1) 87 | publishOne(t, topic, val2) 88 | wg.Wait() 89 | }) 90 | 91 | t.Run("subscribers on seperate topics", func(t *testing.T) { 92 | _ = helperNewTestRedisServer(t) 93 | 94 | var ( 95 | topic1 = "topic1" 96 | topic2 = "topic2" 97 | val1 = "value1" 98 | val2 = "value2" 99 | ) 100 | 101 | wg := sync.WaitGroup{} 102 | wg.Add(1) 103 | go func() { 104 | res := consumeOne(t, topic1) 105 | require.Equal(t, val1, res) 106 | wg.Done() 107 | }() 108 | 109 | wg.Add(1) 110 | go func() { 111 | res := consumeOne(t, topic2) 112 | require.Equal(t, val2, res) 113 | wg.Done() 114 | }() 115 | 116 | publishOne(t, topic1, val1) 117 | publishOne(t, topic2, val2) 118 | wg.Wait() 119 | }) 120 | 121 | t.Run("large message quantity, same topic", func(t *testing.T) { 122 | _ = helperNewTestRedisServer(t) 123 | 124 | var ( 125 | topic = "topic" 126 | ) 127 | 128 | // Publish n messages 129 | n := 50 130 | for i := 0; i < n; i++ { 131 | publishOne(t, topic, strconv.Itoa(i)) 132 | } 133 | 134 | for i := 0; i < n; i++ { 135 | res := consumeOne(t, topic) 136 | require.Equal(t, strconv.Itoa(i), res) 137 | } 138 | }) 139 | } 140 | 141 | // Helpers 142 | 143 | func publishOne(t *testing.T, topic, value string) { 144 | t.Helper() 145 | cmd := exec.Command(redcliPath, "publish", topic, value) 146 | 147 | raw, err := cmd.CombinedOutput() 148 | if err != nil { 149 | t.Fatal(err, string(raw)) 150 | } 151 | 152 | out := strings.TrimSuffix(string(raw), "\n") 153 | 154 | fmt.Println(out) 155 | } 156 | 157 | func consumeOne(t *testing.T, topic string) string { 158 | t.Helper() 159 | cmd := exec.Command(redcliPath, "subscribe", "-c", "1", topic) 160 | 161 | raw, err := cmd.CombinedOutput() 162 | if err != nil { 163 | t.Fatal(err, string(raw)) 164 | } 165 | 166 | out := strings.TrimSuffix(string(raw), "\n") 167 | 168 | fmt.Println(out) 169 | 170 | return out 171 | } 172 | 173 | func helperNewTestRedisServer(t *testing.T) *redis { 174 | dir, err := os.MkdirTemp("", "miniqueue_") 175 | require.NoError(t, err) 176 | r := newRedis(newBroker(newStore(dir))) 177 | 178 | s := redcon.NewServer("localhost:6379", r.handleCmd, nil, nil) 179 | t.Cleanup(func() { 180 | s.Close() 181 | }) 182 | 183 | go func() { 184 | err := s.ListenAndServe() 185 | if err != nil { 186 | t.Error(err) 187 | } 188 | }() 189 | 190 | time.Sleep(10 * time.Millisecond) 191 | 192 | return r 193 | } 194 | -------------------------------------------------------------------------------- /response.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/rs/zerolog" 7 | ) 8 | 9 | type subResponse struct { 10 | Msg []byte `json:"msg,omitempty"` 11 | DackCount int `json:"dackCount,omitempty"` 12 | Error string `json:"error,omitempty"` 13 | } 14 | 15 | func respondMsg(log zerolog.Logger, e *json.Encoder, val *value) { 16 | res := subResponse{ 17 | Msg: val.Raw, 18 | DackCount: val.DackCount, 19 | } 20 | 21 | if err := e.Encode(res); err != nil { 22 | log.Err(err).Msg("failed to write response to client") 23 | } 24 | } 25 | 26 | func respondError(log zerolog.Logger, e *json.Encoder, errMsg string) { 27 | res := subResponse{ 28 | Error: errMsg, 29 | } 30 | 31 | if err := e.Encode(res); err != nil { 32 | log.Err(err).Msg("writing response to client") 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /store.go: -------------------------------------------------------------------------------- 1 | //go:generate mockgen -source=$GOFILE -destination=store_mock.go -package=main 2 | package main 3 | 4 | import ( 5 | "bytes" 6 | "encoding/binary" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "os" 11 | "strconv" 12 | "strings" 13 | "sync" 14 | "time" 15 | 16 | "github.com/rs/zerolog/log" 17 | "github.com/syndtr/goleveldb/leveldb" 18 | "github.com/syndtr/goleveldb/leveldb/iterator" 19 | "github.com/syndtr/goleveldb/leveldb/opt" 20 | "github.com/syndtr/goleveldb/leveldb/util" 21 | ) 22 | 23 | type metadata struct { 24 | topics []string 25 | } 26 | 27 | // storer should be safe for concurrent use. 28 | type storer interface { 29 | // Insert inserts a new record for a given topic. 30 | Insert(topic string, val *value) error 31 | 32 | // GetNext will retrieve the next value in the topic, as well as the AckKey 33 | // allowing future acking/nacking of the value. 34 | GetNext(topic string) (val *value, ackOffset int, err error) 35 | 36 | // Ack will acknowledge the processing of a message, removing it from the 37 | // topic entirely. 38 | Ack(topic string, ackOffset int) error 39 | 40 | // Nack will negatively acknowledge the message on a given topic, returning it 41 | // to the *front* of the consumption queue. 42 | Nack(topic string, ackOffset int) error 43 | 44 | // Back will negatively acknowledge the message on a given topic, returning it 45 | // to the *back* of the consumption queue. 46 | Back(topic string, ackOffset int) error 47 | 48 | // Dack will negatively acknowledge the message on a given topic, placing on 49 | // the delay queue with a given timestamp as part of the key for later 50 | // retrieval. 51 | Dack(topic string, ackOffset int, delaySeconds int) error 52 | 53 | // GetDelayed returns an iterator and a closer function allowing the caller to 54 | // iterate over the currently waiting messages for a given topic in 55 | // chronological delay order (those with soonest "done" time first). 56 | GetDelayed(topic string) (delayedIterator, func() error) 57 | 58 | // ReturnDelayed returns delayed messages with done times before the given 59 | // time back to the main queue returning the number of messages returned or an 60 | // error. 61 | ReturnDelayed(topic string, before time.Time) (count int, err error) 62 | 63 | // Meta returns the metadata of the database. 64 | Meta() (*metadata, error) 65 | 66 | // Close closes the store. 67 | Close() error 68 | 69 | // Purge deletes all data associated with a topic. 70 | Purge(topic string) error 71 | 72 | // Destroy removes the store from persistence. This is a destructive 73 | // operation. 74 | Destroy() 75 | } 76 | 77 | const ( 78 | errTopicEmpty = storeError("topic is empty") 79 | errTopicNotExist = storeError("topic does not exist") 80 | errNackMsgNotExist = storeError("msg to nack does not exist") 81 | errBackMsgNotExist = storeError("msg to back does not exist") 82 | errDackMsgNotExist = storeError("msg to dack does not exist") 83 | ) 84 | 85 | type storeError string 86 | 87 | func (s storeError) Error() string { 88 | return string(s) 89 | } 90 | 91 | const ( 92 | // metaTopics is a key which contains a JSON encoded slice 93 | metaTopics = "m-topics" 94 | 95 | // The topic queue is the primary queue containing the records to be 96 | // processed. We need to keep track of the head and the tail offsets of the 97 | // queue in their respective keys in order to quickly append/pop messages from 98 | // the queue. 99 | topicFmt = "t-%s-%d" // topic: [topic]-[offset] 100 | headPosKeyFmt = "t-%s-head" // key: [topic]-head 101 | tailPosKeyFmt = "t-%s-tail" // key: [topic]-tail 102 | 103 | // The ack topic queue is an auxiliary queue which allows keeping track of 104 | // outstanding messages which are waiting on a consumer acknowledgement 105 | // command. We only ever append to the end of this queue, delete records once 106 | // they have been ACK'ed or moved back to the primary queue for reprocessing. 107 | ackTopicFmt = "t-%s-ack-%d" // topic: [topic]-ack-[offset] 108 | ackTailPosKeyFmt = "t-%s-ack-tail" // key: [topic]-ack-tail 109 | 110 | // The delay topic contains messages in buckets with their designated return 111 | // time as a unix timestamp. This provides strict ordering, allowing iteration 112 | // over the items in prefixed byte-order. 113 | delayTopicPrefix = "t-%s-delay-" // topic: [topic]-delay- 114 | delayTopicFmt = delayTopicPrefix + "%d-%d" // topic: [topic]-delay-[until_unix_timestamp]-[local_index] 115 | ) 116 | 117 | // store handles the the underlying leveldb implementation. 118 | type store struct { 119 | path string 120 | db *leveldb.DB 121 | sync.Mutex 122 | } 123 | 124 | func newStore(dbPath string) storer { 125 | db, err := leveldb.OpenFile(dbPath, nil) 126 | if err != nil { 127 | log.Fatal().Err(err).Msg("failed to open levelDB") 128 | } 129 | 130 | return &store{ 131 | path: dbPath, 132 | db: db, 133 | } 134 | } 135 | 136 | // Ack will acknowledge the processing of a value, removing it from the topic 137 | // entirely. 138 | func (s *store) Ack(topic string, ackOffset int) error { 139 | s.Lock() 140 | defer s.Unlock() 141 | 142 | // Delete the used value 143 | key := fmt.Sprintf(ackTopicFmt, topic, ackOffset) 144 | if err := s.db.Delete([]byte(key), nil); err != nil { 145 | return fmt.Errorf("deleting from ack topic: %v", err) 146 | } 147 | 148 | return nil 149 | } 150 | 151 | // Nack will negatively acknowledge the value, on a given topic, returning it 152 | // to the front of the consumption queue. 153 | func (s *store) Nack(topic string, ackOffset int) error { 154 | s.Lock() 155 | defer s.Unlock() 156 | 157 | nackKey := []byte(fmt.Sprintf(ackTopicFmt, topic, ackOffset)) 158 | 159 | tx, err := s.db.OpenTransaction() 160 | if err != nil { 161 | return fmt.Errorf("opening transaction: %v", err) 162 | } 163 | 164 | exists, err := tx.Has(nackKey, nil) 165 | if err != nil { 166 | tx.Discard() 167 | return fmt.Errorf("checking has %s: %v", nackKey, err) 168 | } 169 | if !exists { 170 | tx.Discard() 171 | return errNackMsgNotExist 172 | } 173 | 174 | val, err := getOffset(tx, ackTopicFmt, topic, ackOffset) 175 | if err != nil { 176 | tx.Discard() 177 | return fmt.Errorf("getting ack msg from topic %s at offset %d: %v", topic, ackOffset, err) 178 | } 179 | 180 | if _, err := prependValue(tx, topicFmt, headPosKeyFmt, topic, val); err != nil { 181 | tx.Discard() 182 | return fmt.Errorf("prepending value to topic %s: %v", topic, err) 183 | } 184 | 185 | if err := tx.Delete(nackKey, nil); err != nil { 186 | tx.Discard() 187 | return fmt.Errorf("deleting ackKey %s: %v", nackKey, err) 188 | } 189 | 190 | if err := tx.Commit(); err != nil { 191 | tx.Discard() 192 | return fmt.Errorf("committing nack transaction: %v", err) 193 | } 194 | 195 | return nil 196 | } 197 | 198 | // Back will negatively acknowledge the value, on a given topic, returning it 199 | // to the back of the consumption queue. 200 | func (s *store) Back(topic string, ackOffset int) error { 201 | s.Lock() 202 | defer s.Unlock() 203 | 204 | backKey := []byte(fmt.Sprintf(ackTopicFmt, topic, ackOffset)) 205 | 206 | tx, err := s.db.OpenTransaction() 207 | if err != nil { 208 | return fmt.Errorf("opening transaction: %v", err) 209 | } 210 | 211 | exists, err := tx.Has(backKey, nil) 212 | if err != nil { 213 | tx.Discard() 214 | return fmt.Errorf("checking has %s: %v", backKey, err) 215 | } 216 | if !exists { 217 | tx.Discard() 218 | return errBackMsgNotExist 219 | } 220 | 221 | val, err := getOffset(tx, ackTopicFmt, topic, ackOffset) 222 | if err != nil { 223 | tx.Discard() 224 | return fmt.Errorf("getting ack msg from topic %s at offset %d: %v", topic, ackOffset, err) 225 | } 226 | 227 | if _, err := appendValue(tx, topicFmt, tailPosKeyFmt, topic, val); err != nil { 228 | tx.Discard() 229 | return fmt.Errorf("appending value to topic %s: %v", topic, err) 230 | } 231 | 232 | if err := tx.Delete(backKey, nil); err != nil { 233 | tx.Discard() 234 | return fmt.Errorf("deleting ackKey %s: %v", backKey, err) 235 | } 236 | 237 | if err := tx.Commit(); err != nil { 238 | tx.Discard() 239 | return fmt.Errorf("committing back transaction: %v", err) 240 | } 241 | 242 | return nil 243 | } 244 | 245 | // Insert creates a new record for a given topic, creating the topic in the 246 | // store if it doesn't already exist. If it does, the record is placed at the 247 | // end of the queue. 248 | func (s *store) Insert(topic string, val *value) error { 249 | s.Lock() 250 | defer s.Unlock() 251 | 252 | headPosKey := []byte(fmt.Sprintf(headPosKeyFmt, topic)) 253 | tailPosKey := []byte(fmt.Sprintf(tailPosKeyFmt, topic)) 254 | ackTailPosKey := []byte(fmt.Sprintf(ackTailPosKeyFmt, topic)) 255 | 256 | exists, err := s.db.Has(tailPosKey, nil) 257 | if err != nil { 258 | return fmt.Errorf("checking has %s: %v", tailPosKey, err) 259 | } 260 | 261 | // The key already exists 262 | if exists { 263 | if _, err := appendValue(s.db, topicFmt, tailPosKeyFmt, topic, val); err != nil { 264 | return err 265 | } 266 | 267 | return nil 268 | } 269 | 270 | // Add the topic to the list of topics 271 | if err := addTopicMeta(s.db, topic); err != nil { 272 | return fmt.Errorf("adding topic to meta: %v", err) 273 | } 274 | 275 | // Write initial head position 276 | headPos := make([]byte, 8) 277 | binary.PutVarint(headPos, 0) 278 | 279 | if err := s.db.Put(headPosKey, headPos, nil); err != nil { 280 | return fmt.Errorf("putting head position value: %v", err) 281 | } 282 | 283 | // Write initial ack topic head position 284 | ackTailPos := make([]byte, 8) 285 | binary.PutVarint(ackTailPos, 0) 286 | 287 | if err := s.db.Put(ackTailPosKey, ackTailPos, nil); err != nil { 288 | return fmt.Errorf("putting ack head position value: %v", err) 289 | } 290 | 291 | // Write initial tail position 292 | tailPos := make([]byte, 8) 293 | binary.PutVarint(tailPos, 1) 294 | 295 | if err := s.db.Put(tailPosKey, tailPos, nil); err != nil { 296 | return fmt.Errorf("putting tail position value: %v", err) 297 | } 298 | 299 | // Write new message to head 300 | newKey := []byte(fmt.Sprintf(topicFmt, topic, 0)) 301 | 302 | b, err := val.Encode() 303 | if err != nil { 304 | return fmt.Errorf("encoding value: %v", err) 305 | } 306 | 307 | if err := s.db.Put(newKey, b, nil); err != nil { 308 | return fmt.Errorf("putting first value for topic: %v", err) 309 | } 310 | 311 | return nil 312 | } 313 | 314 | // GetNext retrieves the first record for a topic, incrementing the head 315 | // position of the main array and pushing the value onto the ack array. 316 | func (s *store) GetNext(topic string) (*value, int, error) { 317 | s.Lock() 318 | defer s.Unlock() 319 | 320 | headOffset, err := getPos(s.db, headPosKeyFmt, topic) 321 | if err != nil { 322 | return nil, 0, err 323 | } 324 | 325 | val, err := getValue(s.db, topicFmt, topic, headOffset) 326 | if err != nil { 327 | return nil, 0, err 328 | } 329 | 330 | insertedOffset, err := appendValue(s.db, ackTopicFmt, ackTailPosKeyFmt, topic, val) 331 | if err != nil { 332 | return nil, 0, err 333 | } 334 | 335 | if _, _, err := addPos(s.db, headPosKeyFmt, topic, 1); err != nil { 336 | return nil, 0, err 337 | } 338 | 339 | return val, insertedOffset, nil 340 | } 341 | 342 | // Dack will negatively acknowledge the message on a given topic, placing on 343 | // the delay queue with a given timestamp as part of the key for later 344 | // retrieval. 345 | func (s *store) Dack(topic string, ackOffset int, delaySeconds int) error { 346 | s.Lock() 347 | defer s.Unlock() 348 | 349 | dackKey := []byte(fmt.Sprintf(ackTopicFmt, topic, ackOffset)) 350 | 351 | tx, err := s.db.OpenTransaction() 352 | if err != nil { 353 | return fmt.Errorf("opening transaction: %v", err) 354 | } 355 | 356 | exists, err := tx.Has(dackKey, nil) 357 | if err != nil { 358 | tx.Discard() 359 | return fmt.Errorf("checking has %s: %v", dackKey, err) 360 | } 361 | if !exists { 362 | tx.Discard() 363 | return errDackMsgNotExist 364 | } 365 | 366 | val, err := getOffset(tx, ackTopicFmt, topic, ackOffset) 367 | if err != nil { 368 | tx.Discard() 369 | return fmt.Errorf("getting ack msg from topic %s at offset %d: %v", topic, ackOffset, err) 370 | } 371 | 372 | val.DackCount++ 373 | 374 | if err := insertDelay(tx, topic, val, delaySeconds); err != nil { 375 | tx.Discard() 376 | return fmt.Errorf("inserting ack msg into delay topic from topic %s at offset %d: %v", topic, ackOffset, err) 377 | } 378 | 379 | if err := tx.Delete(dackKey, nil); err != nil { 380 | tx.Discard() 381 | return fmt.Errorf("deleting ackKey %s: %v", dackKey, err) 382 | } 383 | 384 | if err := tx.Commit(); err != nil { 385 | tx.Discard() 386 | return fmt.Errorf("committing dack transaction: %v", err) 387 | } 388 | 389 | return nil 390 | } 391 | 392 | // Requires version of mockgen with https://github.com/golang/mock/pull/405 due 393 | // to exposing a bug in previous versions. Unfortunately for now- 394 | // $ go get github.com/golang/mock/mockgen@HEAD 395 | // until a new release made. 396 | type delayedIterator interface { 397 | iterator.Iterator 398 | } 399 | 400 | // GetDelayed returns an iterator and a closer function allowing the caller to 401 | // iterate over the currently waiting messages for a given topic in 402 | // chronological delay order (those with soonest "done" time first). 403 | func (s *store) GetDelayed(topic string) (iterator delayedIterator, closer func() error) { 404 | s.Lock() 405 | defer s.Unlock() 406 | 407 | key := fmt.Sprintf(delayTopicPrefix, topic) 408 | prefix := util.BytesPrefix([]byte(key)) 409 | iter := s.db.NewIterator(prefix, nil) 410 | 411 | closer = func() error { 412 | iter.Release() 413 | return iter.Error() 414 | } 415 | 416 | return iter, closer 417 | } 418 | 419 | // ReturnDelayed returns delayed messages with done times before the given time 420 | // back to the main queue. 421 | func (s *store) ReturnDelayed(topic string, before time.Time) (int, error) { 422 | s.Lock() 423 | defer s.Unlock() 424 | 425 | key := fmt.Sprintf(delayTopicPrefix, topic) 426 | prefix := util.BytesPrefix([]byte(key)) 427 | iter := s.db.NewIterator(prefix, nil) 428 | defer iter.Release() // In case we return early, it is safe to call multiple times. 429 | 430 | tx, err := s.db.OpenTransaction() 431 | if err != nil { 432 | return 0, fmt.Errorf("opening transaction: %v", err) 433 | } 434 | 435 | count := 0 436 | 437 | // For each record, check if the timestamp is earlier than the given cutoff, 438 | // returning the record to the front of the main queue if it is. 439 | // 440 | // TODO fix returned messages being in reverse chronological order if there 441 | // are multiple being done at once 442 | for iter.Next() { 443 | key := iter.Key() 444 | delayTime, err := timeFromDelayKey(string(key)) 445 | if err != nil { 446 | tx.Discard() 447 | return 0, err 448 | } 449 | 450 | // The message has passed the delay point so should be returned to the front 451 | // of the main queue to be processed again. 452 | if delayTime.Before(before) { 453 | count++ 454 | val := iter.Value() 455 | 456 | v, err := decodeValue(val) 457 | if err != nil { 458 | return 0, err 459 | } 460 | 461 | if _, err := prependValue(tx, topicFmt, headPosKeyFmt, topic, v); err != nil { 462 | tx.Discard() 463 | return 0, err 464 | } 465 | if err := tx.Delete(key, nil); err != nil { 466 | tx.Discard() 467 | return 0, err 468 | } 469 | } else { 470 | // We've already reached a timestamp that is in the future, no need to 471 | // continue. 472 | break 473 | } 474 | } 475 | 476 | iter.Release() 477 | if err := iter.Error(); err != nil { 478 | tx.Discard() 479 | return 0, fmt.Errorf("iterating over delayed messages for topic %s: %v", topic, err) 480 | } 481 | 482 | if err := tx.Commit(); err != nil { 483 | tx.Discard() 484 | return 0, fmt.Errorf("committing nack transaction: %v", err) 485 | } 486 | 487 | return count, nil 488 | } 489 | 490 | func (s *store) Meta() (*metadata, error) { 491 | s.Lock() 492 | defer s.Unlock() 493 | 494 | topics, err := getTopicMeta(s.db) 495 | if err != nil { 496 | return nil, err 497 | } 498 | 499 | return &metadata{ 500 | topics: topics, 501 | }, nil 502 | } 503 | 504 | // Purge deletes all data associated with a topic. 505 | func (s *store) Purge(topic string) error { 506 | s.Lock() 507 | defer s.Unlock() 508 | 509 | batch := new(leveldb.Batch) 510 | 511 | prefix := util.BytesPrefix([]byte(fmt.Sprintf("t-%s", topic))) 512 | iter := s.db.NewIterator(prefix, nil) 513 | 514 | for iter.Next() { 515 | key := iter.Key() 516 | batch.Delete(key) 517 | } 518 | 519 | iter.Release() 520 | if err := iter.Error(); err != nil { 521 | return fmt.Errorf("iterating over purge prefix: %v", err) 522 | } 523 | 524 | if err := s.db.Write(batch, nil); err != nil { 525 | return fmt.Errorf("writing purge batch: %v", err) 526 | } 527 | 528 | // TODO measure performance impact of immediate compaction 529 | // if err := s.db.CompactRange(*prefix); err != nil { 530 | // return fmt.Errorf("compacting purged range: %v", err) 531 | // } 532 | 533 | return nil 534 | } 535 | 536 | // Close the store. 537 | func (s *store) Close() error { 538 | return s.db.Close() 539 | } 540 | 541 | // Destroy the underlying store. 542 | func (s *store) Destroy() { 543 | _ = s.Close() 544 | _ = os.RemoveAll(s.path) 545 | } 546 | 547 | // leveldber describes methods available on both a leveldb.DB and a 548 | // leveldb.Transaction. 549 | type leveldber interface { 550 | Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) 551 | Put(key, value []byte, wo *opt.WriteOptions) error 552 | Get(key []byte, ro *opt.ReadOptions) ([]byte, error) 553 | } 554 | 555 | func insertDelay(db leveldber, topic string, val *value, delaySeconds int) error { 556 | delayTo := time.Now().Unix() + int64(delaySeconds) 557 | 558 | var ( 559 | key string 560 | // localOffset to enable inserting multiple records at a given timestamp 561 | localOffset = 0 562 | ) 563 | 564 | for { 565 | key = fmt.Sprintf(delayTopicFmt, topic, delayTo, localOffset) 566 | exists, err := db.Has([]byte(key), nil) 567 | if err != nil { 568 | return fmt.Errorf("checking has %s: %v", key, err) 569 | } 570 | // A value in this position already exists, we need to combine them. 571 | if !exists { 572 | break 573 | } 574 | localOffset++ 575 | } 576 | 577 | b, err := val.Encode() 578 | if err != nil { 579 | return fmt.Errorf("encoding value: %v", err) 580 | } 581 | 582 | if err := db.Put([]byte(key), b, nil); err != nil { 583 | return fmt.Errorf("putting value: %v", err) 584 | } 585 | 586 | return nil 587 | } 588 | 589 | // getOffset retrieves a record for a topic with a specific offset. 590 | func getOffset(db leveldber, topicFmt string, topic string, offset int) (*value, error) { 591 | key := fmt.Sprintf(topicFmt, topic, offset) 592 | 593 | val, err := db.Get([]byte(key), nil) 594 | if err != nil { 595 | return nil, err 596 | } 597 | 598 | v, err := decodeValue(val) 599 | if err != nil { 600 | return nil, err 601 | } 602 | 603 | return v, nil 604 | } 605 | 606 | // getPos gets the integer position value (aka offset) for topic and key format. 607 | func getPos(db leveldber, topicFmt string, topic string) (int, error) { 608 | key := []byte(fmt.Sprintf(topicFmt, topic)) 609 | 610 | pos, err := db.Get(key, nil) 611 | if errors.Is(err, leveldb.ErrNotFound) { 612 | return 0, errTopicNotExist 613 | } 614 | if err != nil { 615 | return 0, fmt.Errorf("getting offset position position: %v", err) 616 | } 617 | 618 | i, err := binary.ReadVarint(bytes.NewReader(pos)) 619 | if err != nil { 620 | return 0, fmt.Errorf("reading offset position varint: %v", err) 621 | } 622 | 623 | return int(i), nil 624 | } 625 | 626 | // getValue returns the raw value stored given a key format, topic and offset. 627 | func getValue(db leveldber, topicFmt string, topic string, offset int) (*value, error) { 628 | key := fmt.Sprintf(topicFmt, topic, offset) 629 | 630 | val, err := db.Get([]byte(key), nil) 631 | if errors.Is(err, leveldb.ErrNotFound) { 632 | return nil, errTopicEmpty 633 | } 634 | if err != nil { 635 | return nil, fmt.Errorf("getting value with fmt [%s] from topic %s at offset %d: %v", topicFmt, topic, offset, err) 636 | } 637 | 638 | v, err := decodeValue(val) 639 | if err != nil { 640 | return nil, err 641 | } 642 | 643 | return v, nil 644 | } 645 | 646 | // appendValue returns inserts a new value to the end of a topic given, 647 | // returning the inserted offset. 648 | func appendValue(db leveldber, topicFmt, tailPosKeyFmt, topic string, val *value) (offset int, err error) { 649 | tailPosKey := []byte(fmt.Sprintf(tailPosKeyFmt, topic)) 650 | 651 | // Fetch the current tail position 652 | tailPosVal, err := db.Get(tailPosKey, nil) 653 | if err != nil { 654 | return 0, fmt.Errorf("getting tail position from db: %v", err) 655 | } 656 | 657 | origOffset, err := binary.ReadVarint(bytes.NewReader(tailPosVal)) 658 | if err != nil { 659 | return 0, fmt.Errorf("reading tail pos varint: %v", err) 660 | } 661 | 662 | // Write new record to next tail position 663 | newKey := []byte(fmt.Sprintf(topicFmt, topic, origOffset)) 664 | 665 | b, err := val.Encode() 666 | if err != nil { 667 | return 0, fmt.Errorf("encoding value: %v", err) 668 | } 669 | 670 | if err := db.Put(newKey, b, nil); err != nil { 671 | return 0, fmt.Errorf("putting value: %v", err) 672 | } 673 | 674 | // Update tail position 675 | tail := make([]byte, 8) 676 | binary.PutVarint(tail, origOffset+1) 677 | if err := db.Put(tailPosKey, tail, nil); err != nil { 678 | return 0, fmt.Errorf("putting new tail position: %v", err) 679 | } 680 | 681 | return int(origOffset), nil 682 | } 683 | 684 | // prependValue inserts a value to the head of a topic, decrementing the head 685 | // position and returning the offset of the prepended value. 686 | func prependValue(tx leveldber, topicFmt, headPosKeyFmt, topic string, val *value) (offset int, err error) { 687 | headPosKey := []byte(fmt.Sprintf(headPosKeyFmt, topic)) 688 | 689 | // Fetch the current head position 690 | headPosVal, err := tx.Get(headPosKey, nil) 691 | if err != nil { 692 | return 0, fmt.Errorf("getting head position from db: %v", err) 693 | } 694 | 695 | headOffset, err := binary.ReadVarint(bytes.NewReader(headPosVal)) 696 | if err != nil { 697 | return 0, fmt.Errorf("reading head pos varint: %v", err) 698 | } 699 | 700 | // Write new record to lower neighbouring position 701 | newHeadOffset := headOffset - 1 702 | newKey := []byte(fmt.Sprintf(topicFmt, topic, newHeadOffset)) 703 | 704 | b, err := val.Encode() 705 | if err != nil { 706 | return 0, fmt.Errorf("encoding value: %v", err) 707 | } 708 | 709 | if err := tx.Put(newKey, b, nil); err != nil { 710 | return 0, fmt.Errorf("putting value: %v", err) 711 | } 712 | 713 | // Update head position 714 | _, newPosition, err := addPos(tx, headPosKeyFmt, topic, -1) 715 | if err != nil { 716 | return 0, fmt.Errorf("decrementing head pos by 1: %v", err) 717 | } 718 | 719 | return int(newPosition), nil 720 | } 721 | 722 | // addPos adds the an integer to a given position pointer. 723 | func addPos(db leveldber, posKeyFmt string, topic string, sum int) (oldPosition, newPosition int, err error) { 724 | oldPos, err := getPos(db, posKeyFmt, topic) 725 | if err != nil { 726 | return 0, 0, err 727 | } 728 | 729 | newPos := oldPos + sum 730 | newPosBytes := make([]byte, 8) 731 | binary.PutVarint(newPosBytes, int64(newPos)) 732 | 733 | key := []byte(fmt.Sprintf(posKeyFmt, topic)) 734 | 735 | if err := db.Put(key, newPosBytes, nil); err != nil { 736 | return 0, 0, fmt.Errorf("putting new increment position: %v", err) 737 | } 738 | 739 | return oldPos, newPos, nil 740 | } 741 | 742 | // timeFromDelayKey returns the "done" timestamp from the key of a message in a 743 | // delay queue. 744 | func timeFromDelayKey(key string) (time.Time, error) { 745 | splitKey := strings.Split(key, "-") 746 | if len(splitKey) < 3 { 747 | return time.Time{}, fmt.Errorf("invalid delay key format: %s", key) 748 | } 749 | 750 | timestampInt, err := strconv.Atoi(splitKey[3]) 751 | if err != nil { 752 | return time.Time{}, fmt.Errorf("converting timestamp to int: %v", err) 753 | } 754 | 755 | timestampTime := time.Unix(int64(timestampInt), 0) 756 | 757 | return timestampTime, nil 758 | } 759 | 760 | func getTopicMeta(db leveldber) ([]string, error) { 761 | var topics []string 762 | 763 | key := []byte(metaTopics) 764 | exists, err := db.Has(key, nil) 765 | if err != nil { 766 | return nil, fmt.Errorf("checking has %s: %v", key, err) 767 | } 768 | if !exists { 769 | return []string{}, nil 770 | } 771 | 772 | val, err := db.Get(key, nil) 773 | if err != nil { 774 | return nil, fmt.Errorf("getting topics meta: %v", err) 775 | } 776 | 777 | if err := json.Unmarshal(val, &topics); err != nil { 778 | return nil, fmt.Errorf("unmarshalling topics meta: %v", err) 779 | } 780 | 781 | return topics, nil 782 | } 783 | 784 | func addTopicMeta(db leveldber, topic string) error { 785 | key := []byte(metaTopics) 786 | topics := []string{topic} 787 | 788 | exists, err := db.Has(key, nil) 789 | if err != nil { 790 | return fmt.Errorf("checking has %s: %v", key, err) 791 | } 792 | if exists { 793 | val, err := db.Get(key, nil) 794 | if err != nil { 795 | return fmt.Errorf("getting key %s: %v", key, err) 796 | } 797 | 798 | var existingTopics []string 799 | if err := json.Unmarshal(val, &existingTopics); err != nil { 800 | return fmt.Errorf("unmarshalling topics meta: %v", err) 801 | } 802 | 803 | topics = append(existingTopics, topic) 804 | } 805 | 806 | val, err := json.Marshal(topics) 807 | if err != nil { 808 | return fmt.Errorf("marshalling topics meta: %v", err) 809 | } 810 | 811 | if err := db.Put(key, val, nil); err != nil { 812 | return fmt.Errorf("putting topics meta %s: %v", key, err) 813 | } 814 | 815 | return nil 816 | } 817 | -------------------------------------------------------------------------------- /store_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: store.go 3 | 4 | // Package main is a generated GoMock package. 5 | package main 6 | 7 | import ( 8 | reflect "reflect" 9 | time "time" 10 | 11 | gomock "github.com/golang/mock/gomock" 12 | opt "github.com/syndtr/goleveldb/leveldb/opt" 13 | util "github.com/syndtr/goleveldb/leveldb/util" 14 | ) 15 | 16 | // Mockstorer is a mock of storer interface. 17 | type Mockstorer struct { 18 | ctrl *gomock.Controller 19 | recorder *MockstorerMockRecorder 20 | } 21 | 22 | // MockstorerMockRecorder is the mock recorder for Mockstorer. 23 | type MockstorerMockRecorder struct { 24 | mock *Mockstorer 25 | } 26 | 27 | // NewMockstorer creates a new mock instance. 28 | func NewMockstorer(ctrl *gomock.Controller) *Mockstorer { 29 | mock := &Mockstorer{ctrl: ctrl} 30 | mock.recorder = &MockstorerMockRecorder{mock} 31 | return mock 32 | } 33 | 34 | // EXPECT returns an object that allows the caller to indicate expected use. 35 | func (m *Mockstorer) EXPECT() *MockstorerMockRecorder { 36 | return m.recorder 37 | } 38 | 39 | // Ack mocks base method. 40 | func (m *Mockstorer) Ack(topic string, ackOffset int) error { 41 | m.ctrl.T.Helper() 42 | ret := m.ctrl.Call(m, "Ack", topic, ackOffset) 43 | ret0, _ := ret[0].(error) 44 | return ret0 45 | } 46 | 47 | // Ack indicates an expected call of Ack. 48 | func (mr *MockstorerMockRecorder) Ack(topic, ackOffset interface{}) *gomock.Call { 49 | mr.mock.ctrl.T.Helper() 50 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Ack", reflect.TypeOf((*Mockstorer)(nil).Ack), topic, ackOffset) 51 | } 52 | 53 | // Back mocks base method. 54 | func (m *Mockstorer) Back(topic string, ackOffset int) error { 55 | m.ctrl.T.Helper() 56 | ret := m.ctrl.Call(m, "Back", topic, ackOffset) 57 | ret0, _ := ret[0].(error) 58 | return ret0 59 | } 60 | 61 | // Back indicates an expected call of Back. 62 | func (mr *MockstorerMockRecorder) Back(topic, ackOffset interface{}) *gomock.Call { 63 | mr.mock.ctrl.T.Helper() 64 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Back", reflect.TypeOf((*Mockstorer)(nil).Back), topic, ackOffset) 65 | } 66 | 67 | // Close mocks base method. 68 | func (m *Mockstorer) Close() error { 69 | m.ctrl.T.Helper() 70 | ret := m.ctrl.Call(m, "Close") 71 | ret0, _ := ret[0].(error) 72 | return ret0 73 | } 74 | 75 | // Close indicates an expected call of Close. 76 | func (mr *MockstorerMockRecorder) Close() *gomock.Call { 77 | mr.mock.ctrl.T.Helper() 78 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*Mockstorer)(nil).Close)) 79 | } 80 | 81 | // Dack mocks base method. 82 | func (m *Mockstorer) Dack(topic string, ackOffset, delaySeconds int) error { 83 | m.ctrl.T.Helper() 84 | ret := m.ctrl.Call(m, "Dack", topic, ackOffset, delaySeconds) 85 | ret0, _ := ret[0].(error) 86 | return ret0 87 | } 88 | 89 | // Dack indicates an expected call of Dack. 90 | func (mr *MockstorerMockRecorder) Dack(topic, ackOffset, delaySeconds interface{}) *gomock.Call { 91 | mr.mock.ctrl.T.Helper() 92 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Dack", reflect.TypeOf((*Mockstorer)(nil).Dack), topic, ackOffset, delaySeconds) 93 | } 94 | 95 | // Destroy mocks base method. 96 | func (m *Mockstorer) Destroy() { 97 | m.ctrl.T.Helper() 98 | m.ctrl.Call(m, "Destroy") 99 | } 100 | 101 | // Destroy indicates an expected call of Destroy. 102 | func (mr *MockstorerMockRecorder) Destroy() *gomock.Call { 103 | mr.mock.ctrl.T.Helper() 104 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Destroy", reflect.TypeOf((*Mockstorer)(nil).Destroy)) 105 | } 106 | 107 | // GetDelayed mocks base method. 108 | func (m *Mockstorer) GetDelayed(topic string) (delayedIterator, func() error) { 109 | m.ctrl.T.Helper() 110 | ret := m.ctrl.Call(m, "GetDelayed", topic) 111 | ret0, _ := ret[0].(delayedIterator) 112 | ret1, _ := ret[1].(func() error) 113 | return ret0, ret1 114 | } 115 | 116 | // GetDelayed indicates an expected call of GetDelayed. 117 | func (mr *MockstorerMockRecorder) GetDelayed(topic interface{}) *gomock.Call { 118 | mr.mock.ctrl.T.Helper() 119 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDelayed", reflect.TypeOf((*Mockstorer)(nil).GetDelayed), topic) 120 | } 121 | 122 | // GetNext mocks base method. 123 | func (m *Mockstorer) GetNext(topic string) (*value, int, error) { 124 | m.ctrl.T.Helper() 125 | ret := m.ctrl.Call(m, "GetNext", topic) 126 | ret0, _ := ret[0].(*value) 127 | ret1, _ := ret[1].(int) 128 | ret2, _ := ret[2].(error) 129 | return ret0, ret1, ret2 130 | } 131 | 132 | // GetNext indicates an expected call of GetNext. 133 | func (mr *MockstorerMockRecorder) GetNext(topic interface{}) *gomock.Call { 134 | mr.mock.ctrl.T.Helper() 135 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNext", reflect.TypeOf((*Mockstorer)(nil).GetNext), topic) 136 | } 137 | 138 | // Insert mocks base method. 139 | func (m *Mockstorer) Insert(topic string, val *value) error { 140 | m.ctrl.T.Helper() 141 | ret := m.ctrl.Call(m, "Insert", topic, val) 142 | ret0, _ := ret[0].(error) 143 | return ret0 144 | } 145 | 146 | // Insert indicates an expected call of Insert. 147 | func (mr *MockstorerMockRecorder) Insert(topic, val interface{}) *gomock.Call { 148 | mr.mock.ctrl.T.Helper() 149 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Insert", reflect.TypeOf((*Mockstorer)(nil).Insert), topic, val) 150 | } 151 | 152 | // Meta mocks base method. 153 | func (m *Mockstorer) Meta() (*metadata, error) { 154 | m.ctrl.T.Helper() 155 | ret := m.ctrl.Call(m, "Meta") 156 | ret0, _ := ret[0].(*metadata) 157 | ret1, _ := ret[1].(error) 158 | return ret0, ret1 159 | } 160 | 161 | // Meta indicates an expected call of Meta. 162 | func (mr *MockstorerMockRecorder) Meta() *gomock.Call { 163 | mr.mock.ctrl.T.Helper() 164 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Meta", reflect.TypeOf((*Mockstorer)(nil).Meta)) 165 | } 166 | 167 | // Nack mocks base method. 168 | func (m *Mockstorer) Nack(topic string, ackOffset int) error { 169 | m.ctrl.T.Helper() 170 | ret := m.ctrl.Call(m, "Nack", topic, ackOffset) 171 | ret0, _ := ret[0].(error) 172 | return ret0 173 | } 174 | 175 | // Nack indicates an expected call of Nack. 176 | func (mr *MockstorerMockRecorder) Nack(topic, ackOffset interface{}) *gomock.Call { 177 | mr.mock.ctrl.T.Helper() 178 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Nack", reflect.TypeOf((*Mockstorer)(nil).Nack), topic, ackOffset) 179 | } 180 | 181 | // Purge mocks base method. 182 | func (m *Mockstorer) Purge(topic string) error { 183 | m.ctrl.T.Helper() 184 | ret := m.ctrl.Call(m, "Purge", topic) 185 | ret0, _ := ret[0].(error) 186 | return ret0 187 | } 188 | 189 | // Purge indicates an expected call of Purge. 190 | func (mr *MockstorerMockRecorder) Purge(topic interface{}) *gomock.Call { 191 | mr.mock.ctrl.T.Helper() 192 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Purge", reflect.TypeOf((*Mockstorer)(nil).Purge), topic) 193 | } 194 | 195 | // ReturnDelayed mocks base method. 196 | func (m *Mockstorer) ReturnDelayed(topic string, before time.Time) (int, error) { 197 | m.ctrl.T.Helper() 198 | ret := m.ctrl.Call(m, "ReturnDelayed", topic, before) 199 | ret0, _ := ret[0].(int) 200 | ret1, _ := ret[1].(error) 201 | return ret0, ret1 202 | } 203 | 204 | // ReturnDelayed indicates an expected call of ReturnDelayed. 205 | func (mr *MockstorerMockRecorder) ReturnDelayed(topic, before interface{}) *gomock.Call { 206 | mr.mock.ctrl.T.Helper() 207 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReturnDelayed", reflect.TypeOf((*Mockstorer)(nil).ReturnDelayed), topic, before) 208 | } 209 | 210 | // MockdelayedIterator is a mock of delayedIterator interface. 211 | type MockdelayedIterator struct { 212 | ctrl *gomock.Controller 213 | recorder *MockdelayedIteratorMockRecorder 214 | } 215 | 216 | // MockdelayedIteratorMockRecorder is the mock recorder for MockdelayedIterator. 217 | type MockdelayedIteratorMockRecorder struct { 218 | mock *MockdelayedIterator 219 | } 220 | 221 | // NewMockdelayedIterator creates a new mock instance. 222 | func NewMockdelayedIterator(ctrl *gomock.Controller) *MockdelayedIterator { 223 | mock := &MockdelayedIterator{ctrl: ctrl} 224 | mock.recorder = &MockdelayedIteratorMockRecorder{mock} 225 | return mock 226 | } 227 | 228 | // EXPECT returns an object that allows the caller to indicate expected use. 229 | func (m *MockdelayedIterator) EXPECT() *MockdelayedIteratorMockRecorder { 230 | return m.recorder 231 | } 232 | 233 | // Error mocks base method. 234 | func (m *MockdelayedIterator) Error() error { 235 | m.ctrl.T.Helper() 236 | ret := m.ctrl.Call(m, "Error") 237 | ret0, _ := ret[0].(error) 238 | return ret0 239 | } 240 | 241 | // Error indicates an expected call of Error. 242 | func (mr *MockdelayedIteratorMockRecorder) Error() *gomock.Call { 243 | mr.mock.ctrl.T.Helper() 244 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Error", reflect.TypeOf((*MockdelayedIterator)(nil).Error)) 245 | } 246 | 247 | // First mocks base method. 248 | func (m *MockdelayedIterator) First() bool { 249 | m.ctrl.T.Helper() 250 | ret := m.ctrl.Call(m, "First") 251 | ret0, _ := ret[0].(bool) 252 | return ret0 253 | } 254 | 255 | // First indicates an expected call of First. 256 | func (mr *MockdelayedIteratorMockRecorder) First() *gomock.Call { 257 | mr.mock.ctrl.T.Helper() 258 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "First", reflect.TypeOf((*MockdelayedIterator)(nil).First)) 259 | } 260 | 261 | // Key mocks base method. 262 | func (m *MockdelayedIterator) Key() []byte { 263 | m.ctrl.T.Helper() 264 | ret := m.ctrl.Call(m, "Key") 265 | ret0, _ := ret[0].([]byte) 266 | return ret0 267 | } 268 | 269 | // Key indicates an expected call of Key. 270 | func (mr *MockdelayedIteratorMockRecorder) Key() *gomock.Call { 271 | mr.mock.ctrl.T.Helper() 272 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Key", reflect.TypeOf((*MockdelayedIterator)(nil).Key)) 273 | } 274 | 275 | // Last mocks base method. 276 | func (m *MockdelayedIterator) Last() bool { 277 | m.ctrl.T.Helper() 278 | ret := m.ctrl.Call(m, "Last") 279 | ret0, _ := ret[0].(bool) 280 | return ret0 281 | } 282 | 283 | // Last indicates an expected call of Last. 284 | func (mr *MockdelayedIteratorMockRecorder) Last() *gomock.Call { 285 | mr.mock.ctrl.T.Helper() 286 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Last", reflect.TypeOf((*MockdelayedIterator)(nil).Last)) 287 | } 288 | 289 | // Next mocks base method. 290 | func (m *MockdelayedIterator) Next() bool { 291 | m.ctrl.T.Helper() 292 | ret := m.ctrl.Call(m, "Next") 293 | ret0, _ := ret[0].(bool) 294 | return ret0 295 | } 296 | 297 | // Next indicates an expected call of Next. 298 | func (mr *MockdelayedIteratorMockRecorder) Next() *gomock.Call { 299 | mr.mock.ctrl.T.Helper() 300 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Next", reflect.TypeOf((*MockdelayedIterator)(nil).Next)) 301 | } 302 | 303 | // Prev mocks base method. 304 | func (m *MockdelayedIterator) Prev() bool { 305 | m.ctrl.T.Helper() 306 | ret := m.ctrl.Call(m, "Prev") 307 | ret0, _ := ret[0].(bool) 308 | return ret0 309 | } 310 | 311 | // Prev indicates an expected call of Prev. 312 | func (mr *MockdelayedIteratorMockRecorder) Prev() *gomock.Call { 313 | mr.mock.ctrl.T.Helper() 314 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Prev", reflect.TypeOf((*MockdelayedIterator)(nil).Prev)) 315 | } 316 | 317 | // Release mocks base method. 318 | func (m *MockdelayedIterator) Release() { 319 | m.ctrl.T.Helper() 320 | m.ctrl.Call(m, "Release") 321 | } 322 | 323 | // Release indicates an expected call of Release. 324 | func (mr *MockdelayedIteratorMockRecorder) Release() *gomock.Call { 325 | mr.mock.ctrl.T.Helper() 326 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Release", reflect.TypeOf((*MockdelayedIterator)(nil).Release)) 327 | } 328 | 329 | // Seek mocks base method. 330 | func (m *MockdelayedIterator) Seek(key []byte) bool { 331 | m.ctrl.T.Helper() 332 | ret := m.ctrl.Call(m, "Seek", key) 333 | ret0, _ := ret[0].(bool) 334 | return ret0 335 | } 336 | 337 | // Seek indicates an expected call of Seek. 338 | func (mr *MockdelayedIteratorMockRecorder) Seek(key interface{}) *gomock.Call { 339 | mr.mock.ctrl.T.Helper() 340 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Seek", reflect.TypeOf((*MockdelayedIterator)(nil).Seek), key) 341 | } 342 | 343 | // SetReleaser mocks base method. 344 | func (m *MockdelayedIterator) SetReleaser(releaser util.Releaser) { 345 | m.ctrl.T.Helper() 346 | m.ctrl.Call(m, "SetReleaser", releaser) 347 | } 348 | 349 | // SetReleaser indicates an expected call of SetReleaser. 350 | func (mr *MockdelayedIteratorMockRecorder) SetReleaser(releaser interface{}) *gomock.Call { 351 | mr.mock.ctrl.T.Helper() 352 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetReleaser", reflect.TypeOf((*MockdelayedIterator)(nil).SetReleaser), releaser) 353 | } 354 | 355 | // Valid mocks base method. 356 | func (m *MockdelayedIterator) Valid() bool { 357 | m.ctrl.T.Helper() 358 | ret := m.ctrl.Call(m, "Valid") 359 | ret0, _ := ret[0].(bool) 360 | return ret0 361 | } 362 | 363 | // Valid indicates an expected call of Valid. 364 | func (mr *MockdelayedIteratorMockRecorder) Valid() *gomock.Call { 365 | mr.mock.ctrl.T.Helper() 366 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Valid", reflect.TypeOf((*MockdelayedIterator)(nil).Valid)) 367 | } 368 | 369 | // Value mocks base method. 370 | func (m *MockdelayedIterator) Value() []byte { 371 | m.ctrl.T.Helper() 372 | ret := m.ctrl.Call(m, "Value") 373 | ret0, _ := ret[0].([]byte) 374 | return ret0 375 | } 376 | 377 | // Value indicates an expected call of Value. 378 | func (mr *MockdelayedIteratorMockRecorder) Value() *gomock.Call { 379 | mr.mock.ctrl.T.Helper() 380 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Value", reflect.TypeOf((*MockdelayedIterator)(nil).Value)) 381 | } 382 | 383 | // Mockleveldber is a mock of leveldber interface. 384 | type Mockleveldber struct { 385 | ctrl *gomock.Controller 386 | recorder *MockleveldberMockRecorder 387 | } 388 | 389 | // MockleveldberMockRecorder is the mock recorder for Mockleveldber. 390 | type MockleveldberMockRecorder struct { 391 | mock *Mockleveldber 392 | } 393 | 394 | // NewMockleveldber creates a new mock instance. 395 | func NewMockleveldber(ctrl *gomock.Controller) *Mockleveldber { 396 | mock := &Mockleveldber{ctrl: ctrl} 397 | mock.recorder = &MockleveldberMockRecorder{mock} 398 | return mock 399 | } 400 | 401 | // EXPECT returns an object that allows the caller to indicate expected use. 402 | func (m *Mockleveldber) EXPECT() *MockleveldberMockRecorder { 403 | return m.recorder 404 | } 405 | 406 | // Get mocks base method. 407 | func (m *Mockleveldber) Get(key []byte, ro *opt.ReadOptions) ([]byte, error) { 408 | m.ctrl.T.Helper() 409 | ret := m.ctrl.Call(m, "Get", key, ro) 410 | ret0, _ := ret[0].([]byte) 411 | ret1, _ := ret[1].(error) 412 | return ret0, ret1 413 | } 414 | 415 | // Get indicates an expected call of Get. 416 | func (mr *MockleveldberMockRecorder) Get(key, ro interface{}) *gomock.Call { 417 | mr.mock.ctrl.T.Helper() 418 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*Mockleveldber)(nil).Get), key, ro) 419 | } 420 | 421 | // Has mocks base method. 422 | func (m *Mockleveldber) Has(key []byte, ro *opt.ReadOptions) (bool, error) { 423 | m.ctrl.T.Helper() 424 | ret := m.ctrl.Call(m, "Has", key, ro) 425 | ret0, _ := ret[0].(bool) 426 | ret1, _ := ret[1].(error) 427 | return ret0, ret1 428 | } 429 | 430 | // Has indicates an expected call of Has. 431 | func (mr *MockleveldberMockRecorder) Has(key, ro interface{}) *gomock.Call { 432 | mr.mock.ctrl.T.Helper() 433 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Has", reflect.TypeOf((*Mockleveldber)(nil).Has), key, ro) 434 | } 435 | 436 | // Put mocks base method. 437 | func (m *Mockleveldber) Put(key, value []byte, wo *opt.WriteOptions) error { 438 | m.ctrl.T.Helper() 439 | ret := m.ctrl.Call(m, "Put", key, value, wo) 440 | ret0, _ := ret[0].(error) 441 | return ret0 442 | } 443 | 444 | // Put indicates an expected call of Put. 445 | func (mr *MockleveldberMockRecorder) Put(key, value, wo interface{}) *gomock.Call { 446 | mr.mock.ctrl.T.Helper() 447 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*Mockleveldber)(nil).Put), key, value, wo) 448 | } 449 | -------------------------------------------------------------------------------- /store_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | "testing" 9 | "time" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | const tmpDBPath = "/tmp/miniqueue_test_db" 15 | 16 | // Insert 17 | func TestInsert_Single(t *testing.T) { 18 | s := newStore(tmpDBPath) 19 | t.Cleanup(s.Destroy) 20 | 21 | assert.NoError(t, s.Insert(defaultTopic, newValue([]byte("test_value")))) 22 | 23 | val, _, err := s.GetNext(defaultTopic) 24 | assert.NoError(t, err) 25 | assert.Equal(t, "test_value", string(val.Raw)) 26 | } 27 | 28 | func TestInsert_TopicMeta(t *testing.T) { 29 | s := newStore(tmpDBPath).(*store) 30 | t.Cleanup(s.Destroy) 31 | 32 | assert.NoError(t, s.Insert(defaultTopic, newValue([]byte("test_value_1")))) 33 | 34 | var topics []string 35 | val, err := s.db.Get([]byte(metaTopics), nil) 36 | assert.NoError(t, err) 37 | assert.NoError(t, json.Unmarshal(val, &topics)) 38 | assert.Contains(t, topics, defaultTopic) 39 | 40 | assert.NoError(t, s.Insert("other_topic", newValue([]byte("test_value_2")))) 41 | val, err = s.db.Get([]byte(metaTopics), nil) 42 | assert.NoError(t, err) 43 | assert.NoError(t, json.Unmarshal(val, &topics)) 44 | assert.Contains(t, topics, "other_topic") 45 | } 46 | 47 | func TestInsert_TwoSameTopic(t *testing.T) { 48 | s := newStore(tmpDBPath).(*store) 49 | t.Cleanup(s.Destroy) 50 | 51 | var ( 52 | msg1 = newValue([]byte("test_value_1")) 53 | msg2 = newValue([]byte("test_value_2")) 54 | ) 55 | 56 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 57 | assert.NoError(t, s.Insert(defaultTopic, msg2)) 58 | 59 | val, err := getOffset(s.db, topicFmt, defaultTopic, 0) 60 | assert.NoError(t, err) 61 | assert.Equal(t, msg1, val) 62 | 63 | val, err = getOffset(s.db, topicFmt, defaultTopic, 1) 64 | assert.NoError(t, err) 65 | assert.Equal(t, msg2, val) 66 | } 67 | 68 | func TestInsert_ThreeSameTopic(t *testing.T) { 69 | s := newStore(tmpDBPath).(*store) 70 | t.Cleanup(s.Destroy) 71 | 72 | var ( 73 | msg1 = newValue([]byte("test_value_1")) 74 | msg2 = newValue([]byte("test_value_2")) 75 | msg3 = newValue([]byte("test_value_3")) 76 | ) 77 | 78 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 79 | assert.NoError(t, s.Insert(defaultTopic, msg2)) 80 | assert.NoError(t, s.Insert(defaultTopic, msg3)) 81 | 82 | val, err := getOffset(s.db, topicFmt, defaultTopic, 0) 83 | assert.NoError(t, err) 84 | assert.Equal(t, msg1, val) 85 | 86 | val, err = getOffset(s.db, topicFmt, defaultTopic, 1) 87 | assert.NoError(t, err) 88 | assert.Equal(t, msg2, val) 89 | 90 | val, err = getOffset(s.db, topicFmt, defaultTopic, 2) 91 | assert.NoError(t, err) 92 | assert.Equal(t, msg3, val) 93 | } 94 | 95 | // GetNext 96 | func TestGetNext(t *testing.T) { 97 | s := newStore(tmpDBPath) 98 | t.Cleanup(s.Destroy) 99 | 100 | var ( 101 | msg1 = newValue([]byte("test_value_1")) 102 | msg2 = newValue([]byte("test_value_2")) 103 | msg3 = newValue([]byte("test_value_3")) 104 | msg4 = newValue([]byte("test_value_4")) 105 | ) 106 | 107 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 108 | assert.NoError(t, s.Insert(defaultTopic, msg2)) 109 | assert.NoError(t, s.Insert(defaultTopic, msg3)) 110 | 111 | val, offset, err := s.GetNext(defaultTopic) 112 | assert.NoError(t, err) 113 | assert.Equal(t, msg1, val) 114 | assert.Equal(t, 0, offset) 115 | 116 | val, offset, err = s.GetNext(defaultTopic) 117 | assert.NoError(t, err) 118 | assert.Equal(t, msg2, val) 119 | assert.Equal(t, 1, offset) 120 | 121 | assert.NoError(t, s.Insert(defaultTopic, msg4)) 122 | 123 | val, offset, err = s.GetNext(defaultTopic) 124 | assert.NoError(t, err) 125 | assert.Equal(t, msg3, val) 126 | assert.Equal(t, 2, offset) 127 | 128 | val, offset, err = s.GetNext(defaultTopic) 129 | assert.NoError(t, err) 130 | assert.Equal(t, msg4, val) 131 | assert.Equal(t, 3, offset) 132 | } 133 | 134 | func TestGetNext_TopicNotInitialised(t *testing.T) { 135 | s := newStore(tmpDBPath) 136 | t.Cleanup(s.Destroy) 137 | 138 | val, _, err := s.GetNext(defaultTopic) 139 | assert.Equal(t, errTopicNotExist, err) 140 | assert.Nil(t, val) 141 | } 142 | 143 | // Ack 144 | func TestAck(t *testing.T) { 145 | s := newStore(tmpDBPath).(*store) 146 | t.Cleanup(s.Destroy) 147 | 148 | ackOffset := 1 149 | key := []byte(fmt.Sprintf(ackTopicFmt, defaultTopic, ackOffset)) 150 | assert.NoError(t, s.db.Put(key, []byte("hello_world"), nil)) 151 | 152 | assert.NoError(t, s.Ack(defaultTopic, ackOffset)) 153 | 154 | has, err := s.db.Has(key, nil) 155 | assert.NoError(t, err) 156 | assert.False(t, has) 157 | } 158 | 159 | func TestAck_WithPos(t *testing.T) { 160 | s := newStore(tmpDBPath).(*store) 161 | t.Cleanup(s.Destroy) 162 | 163 | assert.NoError(t, s.Insert(defaultTopic, newValue([]byte("test_value_1")))) 164 | 165 | val, ackOffset, err := s.GetNext(defaultTopic) 166 | assert.NoError(t, err) 167 | assert.Equal(t, "test_value_1", string(val.Raw)) 168 | 169 | val, err = getOffset(s.db, ackTopicFmt, defaultTopic, ackOffset) 170 | assert.NoError(t, err) 171 | assert.Equal(t, "test_value_1", string(val.Raw)) 172 | 173 | assert.NoError(t, s.Ack(defaultTopic, ackOffset)) 174 | 175 | _, err = getOffset(s.db, ackTopicFmt, defaultTopic, ackOffset) 176 | assert.Error(t, err) 177 | } 178 | 179 | // Nack 180 | func TestNack(t *testing.T) { 181 | s := newStore(tmpDBPath) 182 | t.Cleanup(s.Destroy) 183 | 184 | assert.NoError(t, s.Insert(defaultTopic, newValue([]byte("test_value_1")))) 185 | 186 | _, offset, err := s.GetNext(defaultTopic) 187 | assert.NoError(t, err) 188 | 189 | assert.NoError(t, s.Nack(defaultTopic, offset)) 190 | } 191 | 192 | func TestNack_Twice(t *testing.T) { 193 | s := newStore(tmpDBPath) 194 | t.Cleanup(s.Destroy) 195 | 196 | assert.NoError(t, s.Insert(defaultTopic, newValue([]byte("test_value_1")))) 197 | 198 | _, offset, err := s.GetNext(defaultTopic) 199 | assert.NoError(t, err) 200 | 201 | // First Nack 202 | assert.NoError(t, s.Nack(defaultTopic, offset)) 203 | 204 | // Second Nack 205 | err = s.Nack(defaultTopic, offset) 206 | assert.Error(t, err) 207 | assert.Equal(t, err, errNackMsgNotExist) 208 | } 209 | 210 | func TestNack_AndGet(t *testing.T) { 211 | s := newStore(tmpDBPath) 212 | t.Cleanup(s.Destroy) 213 | 214 | var ( 215 | msg1 = newValue([]byte("test_value_1")) 216 | msg2 = newValue([]byte("test_value_2")) 217 | ) 218 | 219 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 220 | assert.NoError(t, s.Insert(defaultTopic, msg2)) 221 | 222 | val, offset, err := s.GetNext(defaultTopic) 223 | assert.NoError(t, err) 224 | assert.Equal(t, msg1, val) 225 | 226 | assert.NoError(t, s.Nack(defaultTopic, offset)) 227 | 228 | val, _, err = s.GetNext(defaultTopic) 229 | assert.NoError(t, err) 230 | assert.Equal(t, msg1, val) 231 | } 232 | 233 | // Back 234 | func TestBack(t *testing.T) { 235 | s := newStore(tmpDBPath) 236 | t.Cleanup(s.Destroy) 237 | 238 | assert.NoError(t, s.Insert(defaultTopic, newValue([]byte("test_value_1")))) 239 | 240 | _, offset, err := s.GetNext(defaultTopic) 241 | assert.NoError(t, err) 242 | 243 | assert.NoError(t, s.Back(defaultTopic, offset)) 244 | } 245 | 246 | func TestBack_Get(t *testing.T) { 247 | s := newStore(tmpDBPath) 248 | t.Cleanup(s.Destroy) 249 | 250 | var ( 251 | msg1 = newValue([]byte("test_value_1")) 252 | msg2 = newValue([]byte("test_value_2")) 253 | ) 254 | 255 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 256 | assert.NoError(t, s.Insert(defaultTopic, msg2)) 257 | 258 | _, offset, err := s.GetNext(defaultTopic) 259 | assert.NoError(t, err) 260 | 261 | assert.NoError(t, s.Back(defaultTopic, offset)) 262 | 263 | v, _, err := s.GetNext(defaultTopic) 264 | assert.NoError(t, err) 265 | assert.Equal(t, msg2, v) 266 | } 267 | 268 | // Dack 269 | func TestDack(t *testing.T) { 270 | s := newStore(tmpDBPath).(*store) 271 | t.Cleanup(s.Destroy) 272 | 273 | msg1 := newValue([]byte("test_value_1")) 274 | msg2 := newValue([]byte("test_value_2")) 275 | 276 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 277 | assert.NoError(t, s.Insert(defaultTopic, msg2)) 278 | 279 | _, offset, _ := s.GetNext(defaultTopic) 280 | assert.NoError(t, s.Dack(defaultTopic, offset, 1)) 281 | 282 | _, offset, _ = s.GetNext(defaultTopic) 283 | assert.NoError(t, s.Dack(defaultTopic, offset, 3)) 284 | } 285 | 286 | func TestDack_SameTime(t *testing.T) { 287 | s := newStore(tmpDBPath).(*store) 288 | t.Cleanup(s.Destroy) 289 | 290 | msg1 := newValue([]byte("test_value_1")) 291 | msg2 := newValue([]byte("test_value_2")) 292 | 293 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 294 | assert.NoError(t, s.Insert(defaultTopic, msg2)) 295 | 296 | _, offset1, _ := s.GetNext(defaultTopic) 297 | _, offset2, _ := s.GetNext(defaultTopic) 298 | assert.NoError(t, s.Dack(defaultTopic, offset1, 1)) 299 | assert.NoError(t, s.Dack(defaultTopic, offset2, 1)) 300 | } 301 | 302 | // GetDelayed 303 | func TestGetDelayed(t *testing.T) { 304 | s := newStore(tmpDBPath).(*store) 305 | t.Cleanup(s.Destroy) 306 | 307 | startTime := time.Now() 308 | 309 | msg1 := newValue([]byte("test_value_1")) 310 | msg2 := newValue([]byte("test_value_2")) 311 | 312 | assert.NoError(t, insertDelay(s.db, defaultTopic, msg1, 1)) 313 | assert.NoError(t, insertDelay(s.db, defaultTopic, msg2, 3)) 314 | 315 | iter, closer := s.GetDelayed(defaultTopic) 316 | 317 | assert.True(t, iter.Next()) 318 | timestamp, _ := strconv.Atoi(strings.Split(string(iter.Key()), "-")[3]) 319 | delayToTime := time.Unix(int64(timestamp), 0) 320 | assert.True(t, startTime.Before(delayToTime), "expected delay timestamp to be after now") 321 | 322 | assert.True(t, iter.Next()) 323 | timestamp, _ = strconv.Atoi(strings.Split(string(iter.Key()), "-")[3]) 324 | delayToTime = time.Unix(int64(timestamp), 0) 325 | assert.True(t, startTime.Before(delayToTime), "expected delay timestamp to be after now") 326 | 327 | assert.NoError(t, closer()) 328 | } 329 | 330 | func TestGetDelayed_SameTimestamp(t *testing.T) { 331 | s := newStore(tmpDBPath).(*store) 332 | t.Cleanup(s.Destroy) 333 | 334 | startTime := time.Now() 335 | 336 | msg1 := newValue([]byte("test_value_1")) 337 | msg2 := newValue([]byte("test_value_2")) 338 | 339 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 340 | assert.NoError(t, s.Insert(defaultTopic, msg2)) 341 | 342 | _, offset1, _ := s.GetNext(defaultTopic) 343 | _, offset2, _ := s.GetNext(defaultTopic) 344 | assert.NoError(t, s.Dack(defaultTopic, offset1, 1)) 345 | assert.NoError(t, s.Dack(defaultTopic, offset2, 1)) 346 | 347 | iter, closer := s.GetDelayed(defaultTopic) 348 | 349 | assert.True(t, iter.Next()) 350 | localOffset := strings.Split(string(iter.Key()), "-")[4] 351 | timestamp, _ := strconv.Atoi(strings.Split(string(iter.Key()), "-")[3]) 352 | delayToTime := time.Unix(int64(timestamp), 0) 353 | assert.True(t, startTime.Before(delayToTime), "expected delay timestamp to be after now") 354 | assert.Equal(t, "0", localOffset) 355 | 356 | assert.True(t, iter.Next()) 357 | localOffset = strings.Split(string(iter.Key()), "-")[4] 358 | timestamp, _ = strconv.Atoi(strings.Split(string(iter.Key()), "-")[3]) 359 | delayToTime = time.Unix(int64(timestamp), 0) 360 | assert.True(t, startTime.Before(delayToTime), "expected delay timestamp to be after now") 361 | assert.Equal(t, "1", localOffset) 362 | 363 | assert.NoError(t, closer()) 364 | } 365 | 366 | // ReturnDelayed 367 | func TestReturnDelayed(t *testing.T) { 368 | s := newStore(tmpDBPath).(*store) 369 | t.Cleanup(s.Destroy) 370 | 371 | msg1 := newValue([]byte("test_value_1")) 372 | msg2 := newValue([]byte("test_value_2")) 373 | 374 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 375 | assert.NoError(t, s.Insert(defaultTopic, msg2)) 376 | 377 | _, offset, _ := s.GetNext(defaultTopic) 378 | assert.NoError(t, s.Dack(defaultTopic, offset, 1)) 379 | _, offset, _ = s.GetNext(defaultTopic) 380 | assert.NoError(t, s.Dack(defaultTopic, offset, 3)) 381 | 382 | count, err := s.ReturnDelayed(defaultTopic, time.Now().Add(time.Minute)) 383 | assert.NoError(t, err) 384 | assert.Equal(t, 2, count) 385 | } 386 | 387 | func TestReturnDelayed_ReturnToMainQueue(t *testing.T) { 388 | s := newStore(tmpDBPath).(*store) 389 | t.Cleanup(s.Destroy) 390 | 391 | msg1 := newValue([]byte("test_value_1")) 392 | msg2 := newValue([]byte("test_value_2")) 393 | 394 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 395 | assert.NoError(t, s.Insert(defaultTopic, msg2)) 396 | 397 | _, offset, _ := s.GetNext(defaultTopic) 398 | assert.NoError(t, s.Dack(defaultTopic, offset, 1)) 399 | 400 | _, offset, _ = s.GetNext(defaultTopic) 401 | assert.NoError(t, s.Dack(defaultTopic, offset, 3)) 402 | 403 | count, err := s.ReturnDelayed(defaultTopic, time.Now().Add(time.Minute)) 404 | assert.NoError(t, err) 405 | assert.Equal(t, 2, count) 406 | 407 | b, _, err := s.GetNext(defaultTopic) 408 | assert.NoError(t, err) 409 | msg2.DackCount = 1 410 | assert.Equal(t, msg2, b) 411 | 412 | b, _, err = s.GetNext(defaultTopic) 413 | assert.NoError(t, err) 414 | msg1.DackCount = 1 415 | assert.Equal(t, msg1, b) 416 | } 417 | 418 | func TestReturnDelayed_ReturnSameTimeToMainQueue(t *testing.T) { 419 | s := newStore(tmpDBPath).(*store) 420 | t.Cleanup(s.Destroy) 421 | 422 | msg1 := newValue([]byte("test_value_1")) 423 | msg2 := newValue([]byte("test_value_2")) 424 | 425 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 426 | assert.NoError(t, s.Insert(defaultTopic, msg2)) 427 | 428 | _, offset1, _ := s.GetNext(defaultTopic) 429 | _, offset2, _ := s.GetNext(defaultTopic) 430 | assert.NoError(t, s.Dack(defaultTopic, offset1, 1)) 431 | assert.NoError(t, s.Dack(defaultTopic, offset2, 1)) 432 | 433 | count, err := s.ReturnDelayed(defaultTopic, time.Now().Add(time.Minute)) 434 | assert.NoError(t, err) 435 | assert.Equal(t, 2, count) 436 | 437 | b, _, err := s.GetNext(defaultTopic) 438 | assert.NoError(t, err) 439 | msg2.DackCount = 1 440 | assert.Equal(t, msg2, b) 441 | 442 | b, _, err = s.GetNext(defaultTopic) 443 | assert.NoError(t, err) 444 | msg1.DackCount = 1 445 | assert.Equal(t, msg1, b) 446 | } 447 | 448 | func TestReturnDelayed_ReturnedMultipleTimes(t *testing.T) { 449 | s := newStore(tmpDBPath).(*store) 450 | t.Cleanup(s.Destroy) 451 | 452 | msg1 := newValue([]byte("test_value_1")) 453 | 454 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 455 | 456 | _, offset, _ := s.GetNext(defaultTopic) 457 | assert.NoError(t, s.Dack(defaultTopic, offset, 1)) 458 | 459 | // Return the message to the main queue 460 | count, err := s.ReturnDelayed(defaultTopic, time.Now().Add(time.Minute)) 461 | assert.NoError(t, err) 462 | assert.Equal(t, 1, count) 463 | 464 | b, offset, err := s.GetNext(defaultTopic) 465 | assert.NoError(t, err) 466 | msg1.DackCount = 1 467 | assert.Equal(t, msg1, b) 468 | 469 | // DACK the same message again 470 | assert.NoError(t, s.Dack(defaultTopic, offset, 1)) 471 | 472 | // Return it, again 473 | count, err = s.ReturnDelayed(defaultTopic, time.Now().Add(time.Minute)) 474 | assert.NoError(t, err) 475 | assert.Equal(t, 1, count) 476 | 477 | b, _, err = s.GetNext(defaultTopic) 478 | assert.NoError(t, err) 479 | msg1.DackCount = 2 480 | assert.Equal(t, msg1, b) 481 | } 482 | 483 | // Purge 484 | func TestPurge(t *testing.T) { 485 | s := newStore(tmpDBPath) 486 | t.Cleanup(s.Destroy) 487 | 488 | msg1 := newValue([]byte("test_value_1")) 489 | assert.NoError(t, s.Insert(defaultTopic, msg1)) 490 | 491 | // Check the value was inserted successfully 492 | val, offset, err := s.GetNext(defaultTopic) 493 | assert.Equal(t, msg1, val) 494 | assert.Equal(t, 0, offset) 495 | assert.NoError(t, err) 496 | 497 | // Purge the topic 498 | assert.NoError(t, s.Purge(defaultTopic)) 499 | 500 | // Attempt to retrieve from non-existent topic 501 | _, _, err = s.GetNext(defaultTopic) 502 | assert.Error(t, err, "topic does not exist") 503 | 504 | // Insert a new value into the topic 505 | msg2 := newValue([]byte("test_value_2")) 506 | assert.NoError(t, s.Insert(defaultTopic, msg2)) 507 | 508 | // Check the correct value is read back 509 | val, offset, err = s.GetNext(defaultTopic) 510 | assert.NoError(t, err) 511 | assert.Equal(t, msg2, val) 512 | assert.Equal(t, 0, offset) 513 | } 514 | 515 | func BenchmarkPurge(b *testing.B) { 516 | s := newStore(b.TempDir()) 517 | b.Cleanup(s.Destroy) 518 | 519 | for n := 0; n < b.N; n++ { 520 | assert.NoError(b, s.Insert(defaultTopic, newValue([]byte("hello world")))) 521 | assert.NoError(b, s.Purge(defaultTopic)) 522 | } 523 | } 524 | 525 | // Close 526 | func TestClose(t *testing.T) { 527 | // TODO 528 | } 529 | -------------------------------------------------------------------------------- /testdata/cmd/redcli_darwin_arm64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomarrell/miniqueue/d7b57a7ae48f413440be55aa9ad96819408a5898/testdata/cmd/redcli_darwin_arm64 -------------------------------------------------------------------------------- /testdata/cmd/redcli_linux_amd64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tomarrell/miniqueue/d7b57a7ae48f413440be55aa9ad96819408a5898/testdata/cmd/redcli_linux_amd64 -------------------------------------------------------------------------------- /testdata/localhost-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDO55bPS/HIedNT 3 | A6lcpRn/zHHsyKy+HLGHGFigmUxd1gPyPaMlEzpgScDijFdysdhxtdZLmv7EAIgb 4 | YdGqPWM50RnbEwA9swCtVQGQGZP5WSiBEiBwXsErL33Q0dAtv1tqS7O92A4MCbSk 5 | LSKQRewZN4OuKYcNooBCrsLpTlSiNh4iA4Yrs1Y4cNlNi+p5UEiTkMvJJnETOHXQ 6 | dXMEnIl/Z5ywo/zYPnE58SOjf7TDzYXL7KGq6JLkpoS44sZonHfq5/FyU7npBoqt 7 | xYprOOsg86e1xBFKpZAACtcFNMWs3fjRn7jhtWhgdOqzNJe4f9MT1CnrCkJIBdMv 8 | /qwUT06fAgMBAAECggEAXCoLIpH4xM2Hld32rj8ZIrnmMYx+bj0H017D8931h4MS 9 | xPAx4Qz8nvGbiw7q0TtpZ9WQ/KKxQ4vdFR1wsL2hmpqLQuvm2pUHAy7vgEn0gUj2 10 | 4u+5JXT/5QXFrKQZcJdh2CqaiUFZmIEmR0+Xqt6Kufzhmk4DS86MBaglSJnM12go 11 | FKDauUI3EFDA0f5W2eb6h1d7Ok5t3I635kwpuUsnFl6TVvb5EHZU1XKKZpPq0GMa 12 | gFhfFKAQZTaUuJYdeJgHp8tqVzEzlVptIZtHaW+MJtu1g1CBePPlJ51hICfIcsb7 13 | 9KXW7D6MMB2+IiIfXT+5b2C/8jgBF8+H2o/2GnBQWQKBgQD7VpNq5HIqSbU8gcSJ 14 | T0B0S40bA898ITDBvaERDmADhV5tmUIhuM8Vy6Ky6asZj1s/iDV/9eeQaTiIAX0+ 15 | XeP+Pn3GvNBLJIybDjJUncF9x8zgQNQSdZ67OjXmwkcyKqR1LyfVkLuzrD+9gpc3 16 | +xiu4VkYOtAJtcR3vQY0I4bvHQKBgQDSvgfEaM5hLvOV06J4oyzTRSEWtwgDyHe3 17 | rXTybkwGJKPFOXlo6tesWvDQCZ5vcZqLMgmntakN3vXm5+aPsPIWP+LbQ2dLUUHj 18 | Pav7qkalxk/gklctDzmNKKgB3YfAXfyGzbYOh8LrtRyPImyEKflRgcL9zw96H3Xu 19 | ghlfozTb6wKBgQDvdwBDot6nLczwlk8T6B9n7ifF6m6APPtATBUus/yEvkhGsfOR 20 | P4yGnpsoTXvIgY6VzIf0n+z96VKEOq8CgeBc91tMw87NGUih3vfTKO8WkQvBSeME 21 | p24Rwpdigg3lXT2NrN0OHLTJrj6Yp9i97I4K6QfDDx3xcm57CuzjNko0fQKBgC0p 22 | A8kXHIK+6PwGah6n+QcdHUYc2t7UqrL1vMXm1OvMFjxBYL8W9Di/FDPAm+8NzSxf 23 | AKqrxxpt2QwuTb4lEPurnRWXkB8XvqLPqHc5ugH0SVG6imvhg1e4iqg7rMeQXHkW 24 | xBjBBwgzu3cAzXhU9lR9FigFoy9sZn1B6+YOt1kZAoGAALbzC8FaR5pX+SVpQuNO 25 | uXT8Ups2q8AivwPbiXrE0y9zhZmt8Fmve0lac8MyFYp3p0qE4u4zdE5pPEBCTtRP 26 | 74uIghQx987U1EeF8CjE/wKkf1TkMfv5UsxBPzaOQ7r20yPiTYl+DyQaD2fXXo9M 27 | EyGD+bfdHWXffMCyDGr6ApA= 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /testdata/localhost.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIETTCCArWgAwIBAgIRAPeuHd2urqZHVGt+QawLkD0wDQYJKoZIhvcNAQELBQAw 3 | gYkxHjAcBgNVBAoTFW1rY2VydCBkZXZlbG9wbWVudCBDQTEvMC0GA1UECwwmdG9t 4 | QFRvbS1NYWNib29rLVByby5sb2NhbCAoVG9tIEFycmVsbCkxNjA0BgNVBAMMLW1r 5 | Y2VydCB0b21AVG9tLU1hY2Jvb2stUHJvLmxvY2FsIChUb20gQXJyZWxsKTAeFw0y 6 | MTAxMTIxNzU4MzVaFw0yMzA0MTIxNjU4MzVaMFoxJzAlBgNVBAoTHm1rY2VydCBk 7 | ZXZlbG9wbWVudCBjZXJ0aWZpY2F0ZTEvMC0GA1UECwwmdG9tQFRvbS1NYWNib29r 8 | LVByby5sb2NhbCAoVG9tIEFycmVsbCkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw 9 | ggEKAoIBAQDO55bPS/HIedNTA6lcpRn/zHHsyKy+HLGHGFigmUxd1gPyPaMlEzpg 10 | ScDijFdysdhxtdZLmv7EAIgbYdGqPWM50RnbEwA9swCtVQGQGZP5WSiBEiBwXsEr 11 | L33Q0dAtv1tqS7O92A4MCbSkLSKQRewZN4OuKYcNooBCrsLpTlSiNh4iA4Yrs1Y4 12 | cNlNi+p5UEiTkMvJJnETOHXQdXMEnIl/Z5ywo/zYPnE58SOjf7TDzYXL7KGq6JLk 13 | poS44sZonHfq5/FyU7npBoqtxYprOOsg86e1xBFKpZAACtcFNMWs3fjRn7jhtWhg 14 | dOqzNJe4f9MT1CnrCkJIBdMv/qwUT06fAgMBAAGjXjBcMA4GA1UdDwEB/wQEAwIF 15 | oDATBgNVHSUEDDAKBggrBgEFBQcDATAfBgNVHSMEGDAWgBRA7zRQcXzUDsme9XVu 16 | YV9FP+98RDAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQADggGB 17 | ALAggMcxnjdGZ1Par2U0V29SLrv31LgBK2K4JQsj73aQSWP7JHXtkX79VpJTIH2J 18 | 2vEy4II2MOdUMjIknL7lcdvL2CM/iCrtWBe2EH8knvHo7/t7NRnKRN0zzqoU/+Rw 19 | fVxwC3QGb1QqhizB4UN3/QOEPRYylLTTz0WjbvVI1TfId0xk7MZ5RlNmNEWNRB// 20 | cFT8MQhoC9tPHEzxeYQNvGvn+QhI2AuUvLq5EPeGLwqy9foUu0XqTglg+ZQ9Z9ly 21 | 3MYppBJmBS5bflt5t702MZgNFh36vwT5lR/iNUUkNl6xzkEv+kD7MDTZRz24cehP 22 | Ai3emnMSJIkorHQj3edz64h0GElgNsuzOMGBdV7n271sc8mcVaI/aZak/SKzrP6r 23 | dKYCpczsEpZOFTbvSsJ0AnYuULvZjldkLFl9krTXC7tx49yYB607MUJ7Re/2Vfvf 24 | NAhBYcxIq19YpEfz6/zBFgysX92+99ugDcvw78jZcIBnbx9USHRAWXUBRZe0eWWp 25 | cw== 26 | -----END CERTIFICATE----- 27 | -------------------------------------------------------------------------------- /testutils_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "net/http" 11 | "net/textproto" 12 | "strconv" 13 | "strings" 14 | "sync" 15 | "time" 16 | 17 | "golang.org/x/net/http/httpguts" 18 | ) 19 | 20 | const timeout = time.Second 21 | 22 | // DecodeWaiter adds a method to wait a specific amount of time trying to decode 23 | // before returning a timeout. 24 | type DecodeWaiter struct { 25 | *json.Decoder 26 | } 27 | 28 | func NewDecodeWaiter(r io.Reader) *DecodeWaiter { 29 | return &DecodeWaiter{Decoder: json.NewDecoder(r)} 30 | } 31 | 32 | func (d *DecodeWaiter) WaitAndDecode(v interface{}) error { 33 | timer := time.After(timeout) 34 | 35 | for { 36 | select { 37 | case <-timer: 38 | return errors.New("timed out waiting for decode") 39 | default: 40 | if d.More() { 41 | return d.Decode(v) 42 | } 43 | } 44 | } 45 | } 46 | 47 | // ResponseRecorder is an implementation of http.ResponseWriter that 48 | // records its mutations for later inspection in tests. 49 | type ResponseRecorder struct { 50 | // Code is the HTTP response code set by WriteHeader. 51 | // 52 | // Note that if a Handler never calls WriteHeader or Write, 53 | // this might end up being 0, rather than the implicit 54 | // http.StatusOK. To get the implicit value, use the Result 55 | // method. 56 | Code int 57 | 58 | // HeaderMap contains the headers explicitly set by the Handler. 59 | // It is an internal detail. 60 | // 61 | // Deprecated: HeaderMap exists for historical compatibility 62 | // and should not be used. To access the headers returned by a handler, 63 | // use the Response.Header map as returned by the Result method. 64 | HeaderMap http.Header 65 | 66 | // Body is the buffer to which the Handler's Write calls are sent. 67 | // If nil, the Writes are silently discarded. 68 | Body *bytes.Buffer 69 | 70 | // Flushed is whether the Handler called Flush. 71 | Flushed bool 72 | 73 | result *http.Response // cache of Result's return value 74 | snapHeader http.Header // snapshot of HeaderMap at first Write 75 | wroteHeader bool 76 | 77 | sync.Mutex 78 | } 79 | 80 | // NewRecorder returns an initialized ResponseRecorder. 81 | func NewRecorder() *ResponseRecorder { 82 | return &ResponseRecorder{ 83 | HeaderMap: make(http.Header), 84 | Body: new(bytes.Buffer), 85 | Code: 200, 86 | } 87 | } 88 | 89 | // Header implements http.ResponseWriter. It returns the response 90 | // headers to mutate within a handler. To test the headers that were 91 | // written after a handler completes, use the Result method and see 92 | // the returned Response value's Header. 93 | func (rw *ResponseRecorder) Header() http.Header { 94 | m := rw.HeaderMap 95 | if m == nil { 96 | m = make(http.Header) 97 | rw.HeaderMap = m 98 | } 99 | return m 100 | } 101 | 102 | // writeHeader writes a header if it was not written yet and 103 | // detects Content-Type if needed. 104 | // 105 | // bytes or str are the beginning of the response body. 106 | // We pass both to avoid unnecessarily generate garbage 107 | // in rw.WriteString which was created for performance reasons. 108 | // Non-nil bytes win. 109 | func (rw *ResponseRecorder) writeHeader(b []byte, str string) { 110 | if rw.wroteHeader { 111 | return 112 | } 113 | if len(str) > 512 { 114 | str = str[:512] 115 | } 116 | 117 | m := rw.Header() 118 | 119 | _, hasType := m["Content-Type"] 120 | hasTE := m.Get("Transfer-Encoding") != "" 121 | if !hasType && !hasTE { 122 | if b == nil { 123 | b = []byte(str) 124 | } 125 | m.Set("Content-Type", http.DetectContentType(b)) 126 | } 127 | 128 | rw.WriteHeader(200) 129 | } 130 | 131 | // Write implements http.ResponseWriter. The data in buf is written to 132 | // rw.Body, if not nil. 133 | func (rw *ResponseRecorder) Write(buf []byte) (int, error) { 134 | rw.Lock() 135 | defer rw.Unlock() 136 | 137 | rw.writeHeader(buf, "") 138 | if rw.Body != nil { 139 | rw.Body.Write(buf) 140 | } 141 | return len(buf), nil 142 | } 143 | 144 | // WriteString implements io.StringWriter. The data in str is written 145 | // to rw.Body, if not nil. 146 | func (rw *ResponseRecorder) WriteString(str string) (int, error) { 147 | rw.Lock() 148 | defer rw.Unlock() 149 | 150 | rw.writeHeader(nil, str) 151 | if rw.Body != nil { 152 | rw.Body.WriteString(str) 153 | } 154 | return len(str), nil 155 | } 156 | 157 | // WriteHeader implements http.ResponseWriter. 158 | func (rw *ResponseRecorder) WriteHeader(code int) { 159 | if rw.wroteHeader { 160 | return 161 | } 162 | rw.Code = code 163 | rw.wroteHeader = true 164 | if rw.HeaderMap == nil { 165 | rw.HeaderMap = make(http.Header) 166 | } 167 | rw.snapHeader = rw.HeaderMap.Clone() 168 | } 169 | 170 | // Flush implements http.Flusher. To test whether Flush was 171 | // called, see rw.Flushed. 172 | func (rw *ResponseRecorder) Flush() { 173 | if !rw.wroteHeader { 174 | rw.WriteHeader(200) 175 | } 176 | rw.Flushed = true 177 | } 178 | 179 | // Read implements a thread safe reader for the body of the ResponseRecorder. 180 | func (rw *ResponseRecorder) Read(p []byte) (n int, err error) { 181 | rw.Lock() 182 | defer rw.Unlock() 183 | return rw.Body.Read(p) 184 | } 185 | 186 | // Result returns the response generated by the handler. 187 | // 188 | // The returned Response will have at least its StatusCode, 189 | // Header, Body, and optionally Trailer populated. 190 | // More fields may be populated in the future, so callers should 191 | // not DeepEqual the result in tests. 192 | // 193 | // The Response.Header is a snapshot of the headers at the time of the 194 | // first write call, or at the time of this call, if the handler never 195 | // did a write. 196 | // 197 | // The Response.Body is guaranteed to be non-nil and Body.Read call is 198 | // guaranteed to not return any error other than io.EOF. 199 | // 200 | // Result must only be called after the handler has finished running. 201 | func (rw *ResponseRecorder) Result() *http.Response { 202 | if rw.result != nil { 203 | return rw.result 204 | } 205 | if rw.snapHeader == nil { 206 | rw.snapHeader = rw.HeaderMap.Clone() 207 | } 208 | res := &http.Response{ 209 | Proto: "HTTP/1.1", 210 | ProtoMajor: 1, 211 | ProtoMinor: 1, 212 | StatusCode: rw.Code, 213 | Header: rw.snapHeader, 214 | } 215 | rw.result = res 216 | if res.StatusCode == 0 { 217 | res.StatusCode = 200 218 | } 219 | res.Status = fmt.Sprintf("%03d %s", res.StatusCode, http.StatusText(res.StatusCode)) 220 | if rw.Body != nil { 221 | res.Body = ioutil.NopCloser(bytes.NewReader(rw.Body.Bytes())) 222 | } else { 223 | res.Body = http.NoBody 224 | } 225 | res.ContentLength = parseContentLength(res.Header.Get("Content-Length")) 226 | 227 | if trailers, ok := rw.snapHeader["Trailer"]; ok { 228 | res.Trailer = make(http.Header, len(trailers)) 229 | for _, k := range trailers { 230 | k = http.CanonicalHeaderKey(k) 231 | if !httpguts.ValidTrailerHeader(k) { 232 | // Ignore since forbidden by RFC 7230, section 4.1.2. 233 | continue 234 | } 235 | vv, ok := rw.HeaderMap[k] 236 | if !ok { 237 | continue 238 | } 239 | vv2 := make([]string, len(vv)) 240 | copy(vv2, vv) 241 | res.Trailer[k] = vv2 242 | } 243 | } 244 | for k, vv := range rw.HeaderMap { 245 | if !strings.HasPrefix(k, http.TrailerPrefix) { 246 | continue 247 | } 248 | if res.Trailer == nil { 249 | res.Trailer = make(http.Header) 250 | } 251 | for _, v := range vv { 252 | res.Trailer.Add(strings.TrimPrefix(k, http.TrailerPrefix), v) 253 | } 254 | } 255 | return res 256 | } 257 | 258 | // parseContentLength trims whitespace from s and returns -1 if no value 259 | // is set, or the value if it's >= 0. 260 | // 261 | // This a modified version of same function found in net/http/transfer.go. This 262 | // one just ignores an invalid header. 263 | func parseContentLength(cl string) int64 { 264 | cl = textproto.TrimString(cl) 265 | if cl == "" { 266 | return -1 267 | } 268 | n, err := strconv.ParseUint(cl, 10, 63) 269 | if err != nil { 270 | return -1 271 | } 272 | return int64(n) 273 | } 274 | -------------------------------------------------------------------------------- /value.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | "fmt" 7 | ) 8 | 9 | type value struct { 10 | DackCount int 11 | Raw []byte 12 | } 13 | 14 | func newValue(b []byte) *value { 15 | return &value{ 16 | DackCount: 0, 17 | Raw: b, 18 | } 19 | } 20 | 21 | func (v *value) Encode() ([]byte, error) { 22 | var buf bytes.Buffer 23 | if err := gob.NewEncoder(&buf).Encode(v); err != nil { 24 | return nil, fmt.Errorf("gob encoding value: %v", err) 25 | } 26 | 27 | return buf.Bytes(), nil 28 | } 29 | 30 | func decodeValue(b []byte) (*value, error) { 31 | var v value 32 | if err := gob.NewDecoder(bytes.NewReader(b)).Decode(&v); err != nil { 33 | return nil, fmt.Errorf("gob decoding value: %v", err) 34 | } 35 | 36 | return &v, nil 37 | } 38 | --------------------------------------------------------------------------------