├── .github └── workflows │ ├── pr.yml │ └── tests.yml ├── Dockerfile ├── LICENSE ├── README.md ├── broker ├── broker.go ├── broker_test.go └── options.go ├── client ├── README.md ├── client.go ├── grpc │ └── grpc.go ├── http.go ├── http │ └── http.go ├── options.go ├── resolver │ ├── resolver.go │ └── resolver_test.go └── selector │ ├── selector.go │ └── selector_test.go ├── examples ├── pub │ └── pub.go └── sub │ └── sub.go ├── generate.go ├── go.mod ├── go.sum ├── main.go ├── mq.png ├── proto ├── mq.pb.go └── mq.proto └── server ├── grpc ├── grpc.go └── handler.go ├── http ├── handler.go ├── http.go └── writer.go ├── options.go ├── server.go └── util ├── address.go └── certificate.go /.github/workflows/pr.yml: -------------------------------------------------------------------------------- 1 | name: PR Sanity Check 2 | on: pull_request 3 | 4 | jobs: 5 | 6 | prtest: 7 | name: PR sanity check 8 | runs-on: ubuntu-latest 9 | steps: 10 | 11 | - name: Set up Go 1.18 12 | uses: actions/setup-go@v1 13 | with: 14 | go-version: 1.18 15 | id: go 16 | 17 | - name: Check out code into the Go module directory 18 | uses: actions/checkout@v2 19 | 20 | - name: Get dependencies 21 | run: | 22 | go get -v -t -d ./... 23 | 24 | - name: Run tests 25 | id: tests 26 | env: 27 | IN_TRAVIS_CI: yes 28 | run: go test -v ./... 29 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | on: [push] 3 | 4 | jobs: 5 | 6 | test: 7 | name: Test repo 8 | runs-on: ubuntu-latest 9 | steps: 10 | 11 | - name: Set up Go 1.18 12 | uses: actions/setup-go@v1 13 | with: 14 | go-version: 1.18 15 | id: go 16 | 17 | - name: Check out code into the Go module directory 18 | uses: actions/checkout@v2 19 | 20 | - name: Get dependencies 21 | run: | 22 | go get -v -t -d ./... 23 | 24 | - name: Run tests 25 | id: tests 26 | env: 27 | IN_TRAVIS_CI: yes 28 | run: go test -v ./... 29 | 30 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.2 2 | RUN apk add --update ca-certificates && \ 3 | rm -rf /var/cache/apk/* /tmp/* 4 | ADD mq /mq 5 | WORKDIR / 6 | ENTRYPOINT [ "/mq" ] 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | Copyright 2015 Asim Aslam. 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | http://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. 192 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MQ [![License](https://img.shields.io/:license-apache-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Go Report Card](https://goreportcard.com/badge/asim/mq)](https://goreportcard.com/report/github.com/asim/mq) 2 | 3 | MQ is an in-memory message broker 4 | 5 | ## Features 6 | 7 | - In-memory message broker 8 | - HTTP or gRPC transport 9 | - Clustering 10 | - Sharding 11 | - Proxying 12 | - Discovery 13 | - Auto retries 14 | - TLS support 15 | - Command line interface 16 | - Interactive prompt 17 | - Go client library 18 | 19 | Emque generates a self signed certificate by default if no TLS config is specified 20 | 21 | ## API 22 | 23 | Publish 24 | ``` 25 | /pub?topic=string publish payload as body 26 | ``` 27 | 28 | Subscribe 29 | ``` 30 | /sub?topic=string subscribe as websocket 31 | ``` 32 | 33 | ## Architecture 34 | 35 | - Emque servers are standalone servers with in-memory queues and provide a HTTP API 36 | - Emque clients shard or cluster Emque servers by publish/subscribing to one or all servers 37 | - Emque proxies use the go client to cluster Emque servers and provide a unified HTTP API 38 | 39 |

40 | 41 |

42 | 43 | Because of this simplistic architecture, proxies and servers can be chained to build message pipelines 44 | 45 | ## Usage 46 | 47 | ### Install 48 | 49 | ```shell 50 | go get github.com/asim/mq 51 | ``` 52 | 53 | ### Run Server 54 | 55 | Listens on `*:8081` 56 | ```shell 57 | mq 58 | ``` 59 | 60 | Set server address 61 | ```shell 62 | mq --address=localhost:9091 63 | ``` 64 | 65 | Enable TLS 66 | ```shell 67 | mq --cert_file=cert.pem --key_file=key.pem 68 | ``` 69 | 70 | Persist to file per topic 71 | ```shell 72 | mq --persist 73 | ``` 74 | 75 | Use gRPC transport 76 | ```shell 77 | mq --transport=grpc 78 | ``` 79 | 80 | ### Run Proxy 81 | 82 | Emque can be run as a proxy which includes clustering, sharding and auto retry features. 83 | 84 | Clustering: Publish and subscribe to all Emque servers 85 | 86 | ```shell 87 | mq --proxy --servers=10.0.0.1:8081,10.0.0.1:8082,10.0.0.1:8083 88 | ``` 89 | 90 | Sharding: Requests are sent to a single server based on topic 91 | 92 | ```shell 93 | mq --proxy --servers=10.0.0.1:8081,10.0.0.1:8082,10.0.0.1:8083 --select=shard 94 | ``` 95 | 96 | Resolver: Use a name resolver rather than specifying server ips 97 | 98 | ```shell 99 | mq --proxy --resolver=dns --servers=mq.proxy.dev 100 | ``` 101 | 102 | ### Run Client 103 | 104 | Publish 105 | 106 | ```shell 107 | echo "A completely arbitrary message" | mq --client --topic=foo --publish --servers=localhost:8081 108 | ``` 109 | 110 | Subscribe 111 | 112 | ```shell 113 | mq --client --topic=foo --subscribe --servers=localhost:8081 114 | ``` 115 | 116 | Interactive mode 117 | ```shell 118 | mq -i --topic=foo 119 | ``` 120 | 121 | ### Publish 122 | 123 | Publish via HTTP 124 | 125 | ``` 126 | curl -k -d "A completely arbitrary message" "https://localhost:8081/pub?topic=foo" 127 | ``` 128 | 129 | ### Subscribe 130 | 131 | Subscribe via websockets 132 | 133 | ``` 134 | curl -k -i -N -H "Connection: Upgrade" \ 135 | -H "Upgrade: websocket" \ 136 | -H "Host: localhost:8081" \ 137 | -H "Origin:http://localhost:8081" \ 138 | -H "Sec-Websocket-Version: 13" \ 139 | -H "Sec-Websocket-Key: Emque" \ 140 | "https://localhost:8081/sub?topic=foo" 141 | ``` 142 | 143 | ## Go Client [![GoDoc](https://godoc.org/github.com/asim/mq/client?status.svg)](https://godoc.org/github.com/asim/mq/client) 144 | 145 | Emque provides a simple go client 146 | 147 | ```go 148 | import "github.com/asim/mq/client" 149 | ``` 150 | 151 | ### Publish 152 | 153 | ```go 154 | // publish to topic foo 155 | err := client.Publish("foo", []byte(`bar`)) 156 | ``` 157 | 158 | ### Subscribe 159 | 160 | ```go 161 | // subscribe to topic foo 162 | ch, err := client.Subscribe("foo") 163 | if err != nil { 164 | return 165 | } 166 | 167 | data := <-ch 168 | ``` 169 | 170 | ### New Client 171 | 172 | ```go 173 | // defaults to Emque server localhost:8081 174 | c := client.New() 175 | ``` 176 | 177 | gRPC client 178 | 179 | ```go 180 | import "github.com/asim/mq/client/grpc" 181 | 182 | c := grpc.New() 183 | ``` 184 | 185 | ### Clustering 186 | 187 | Clustering is supported on the client side. Publish/Subscribe operations are performed against all servers. 188 | 189 | ```go 190 | c := client.New( 191 | client.WithServers("10.0.0.1:8081", "10.0.0.1:8082", "10.0.0.1:8083"), 192 | ) 193 | ``` 194 | 195 | ### Sharding 196 | 197 | Sharding is supported via client much like gomemcache. Publish/Subscribe operations are performed against a single server. 198 | 199 | ```go 200 | import "github.com/asim/mq/client/selector" 201 | 202 | c := client.New( 203 | client.WithServers("10.0.0.1:8081", "10.0.0.1:8082", "10.0.0.1:8083"), 204 | client.WithSelector(new(selector.Shard)), 205 | ) 206 | ``` 207 | ### Resolver 208 | 209 | A name resolver can be used to discover the ip addresses of Emque servers 210 | 211 | ```go 212 | import "github.com/asim/mq/client/resolver" 213 | 214 | c := client.New( 215 | // use the DNS resolver 216 | client.WithResolver(new(resolver.DNS)), 217 | // specify DNS name as server 218 | client.WithServers("mq.proxy.local"), 219 | ) 220 | ``` 221 | -------------------------------------------------------------------------------- /broker/broker.go: -------------------------------------------------------------------------------- 1 | package broker 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "os" 7 | "sync" 8 | "time" 9 | 10 | "github.com/asim/mq/client" 11 | ) 12 | 13 | var ( 14 | Default Broker = newBroker() 15 | ) 16 | 17 | // internal broker 18 | type broker struct { 19 | exit chan bool 20 | options *Options 21 | 22 | sync.RWMutex 23 | topics map[string][]chan []byte 24 | 25 | mtx sync.RWMutex 26 | persisted map[string]bool 27 | } 28 | 29 | // internal message for persistence 30 | type message struct { 31 | Timestamp int64 `json:"timestamp"` 32 | Topic string `json:"topic"` 33 | Payload []byte `json:"payload"` 34 | } 35 | 36 | // Broker is the message broker 37 | type Broker interface { 38 | Close() error 39 | Publish(topic string, payload []byte) error 40 | Subscribe(topic string) (<-chan []byte, error) 41 | Unsubscribe(topic string, sub <-chan []byte) error 42 | } 43 | 44 | func newBroker(opts ...Option) *broker { 45 | options := new(Options) 46 | for _, o := range opts { 47 | o(options) 48 | } 49 | 50 | if options.Client == nil { 51 | options.Client = client.New() 52 | } 53 | 54 | return &broker{ 55 | exit: make(chan bool), 56 | options: options, 57 | topics: make(map[string][]chan []byte), 58 | persisted: make(map[string]bool), 59 | } 60 | } 61 | 62 | func (b *broker) publish(payload []byte, subscribers []chan []byte) { 63 | n := len(subscribers) 64 | c := 1 65 | 66 | // increase concurrency if there are many subscribers 67 | switch { 68 | case n > 1000: 69 | c = 3 70 | case n > 100: 71 | c = 2 72 | } 73 | 74 | // publisher function 75 | pub := func(start int) { 76 | // iterate the subscribers 77 | for j := start; j < n; j += c { 78 | select { 79 | // push the payload to subscriber 80 | case subscribers[j] <- payload: 81 | // only wait 5 milliseconds for subscriber 82 | case <-time.After(time.Millisecond * 5): 83 | case <-b.exit: 84 | return 85 | } 86 | } 87 | } 88 | 89 | // concurrent publish 90 | for i := 0; i < c; i++ { 91 | go pub(i) 92 | } 93 | } 94 | 95 | func (b *broker) persist(topic string) error { 96 | b.mtx.Lock() 97 | defer b.mtx.Unlock() 98 | 99 | if b.persisted[topic] { 100 | return nil 101 | } 102 | 103 | ch, err := b.Subscribe(topic) 104 | if err != nil { 105 | return err 106 | } 107 | 108 | f, err := os.OpenFile(topic+".mq", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660) 109 | if err != nil { 110 | return err 111 | } 112 | 113 | go func() { 114 | var pending []byte 115 | newline := []byte{'\n'} 116 | t := time.NewTicker(time.Second) 117 | defer t.Stop() 118 | defer f.Close() 119 | 120 | for { 121 | select { 122 | case p := <-ch: 123 | b, err := json.Marshal(&message{ 124 | Timestamp: time.Now().UnixNano(), 125 | Topic: topic, 126 | Payload: p, 127 | }) 128 | if err != nil { 129 | continue 130 | } 131 | pending = append(pending, b...) 132 | pending = append(pending, newline...) 133 | case <-t.C: 134 | if len(pending) == 0 { 135 | continue 136 | } 137 | f.Write(pending) 138 | pending = nil 139 | case <-b.exit: 140 | return 141 | } 142 | } 143 | }() 144 | 145 | b.persisted[topic] = true 146 | 147 | return nil 148 | } 149 | 150 | func (b *broker) Close() error { 151 | select { 152 | case <-b.exit: 153 | return nil 154 | default: 155 | close(b.exit) 156 | b.Lock() 157 | b.topics = make(map[string][]chan []byte) 158 | b.Unlock() 159 | b.options.Client.Close() 160 | } 161 | return nil 162 | } 163 | 164 | func (b *broker) Publish(topic string, payload []byte) error { 165 | select { 166 | case <-b.exit: 167 | return errors.New("broker closed") 168 | default: 169 | } 170 | 171 | if b.options.Proxy { 172 | return b.options.Client.Publish(topic, payload) 173 | } 174 | 175 | b.RLock() 176 | subscribers, ok := b.topics[topic] 177 | b.RUnlock() 178 | if !ok { 179 | // persist? 180 | if !b.options.Persist { 181 | return nil 182 | } 183 | if err := b.persist(topic); err != nil { 184 | return err 185 | } 186 | } 187 | 188 | b.publish(payload, subscribers) 189 | return nil 190 | } 191 | 192 | func (b *broker) Subscribe(topic string) (<-chan []byte, error) { 193 | select { 194 | case <-b.exit: 195 | return nil, errors.New("broker closed") 196 | default: 197 | } 198 | 199 | if b.options.Proxy { 200 | return b.options.Client.Subscribe(topic) 201 | } 202 | 203 | ch := make(chan []byte, 100) 204 | b.Lock() 205 | b.topics[topic] = append(b.topics[topic], ch) 206 | b.Unlock() 207 | return ch, nil 208 | } 209 | 210 | func (b *broker) Unsubscribe(topic string, sub <-chan []byte) error { 211 | select { 212 | case <-b.exit: 213 | return errors.New("broker closed") 214 | default: 215 | } 216 | 217 | if b.options.Proxy { 218 | return b.options.Client.Unsubscribe(sub) 219 | } 220 | 221 | b.RLock() 222 | subscribers, ok := b.topics[topic] 223 | b.RUnlock() 224 | 225 | if !ok { 226 | return nil 227 | } 228 | 229 | var subs []chan []byte 230 | for _, subscriber := range subscribers { 231 | if subscriber == sub { 232 | continue 233 | } 234 | subs = append(subs, subscriber) 235 | } 236 | 237 | b.Lock() 238 | b.topics[topic] = subs 239 | b.Unlock() 240 | return nil 241 | } 242 | 243 | func Publish(topic string, payload []byte) error { 244 | return Default.Publish(topic, payload) 245 | } 246 | 247 | func Subscribe(topic string) (<-chan []byte, error) { 248 | return Default.Subscribe(topic) 249 | } 250 | 251 | func Unsubscribe(topic string, sub <-chan []byte) error { 252 | return Default.Unsubscribe(topic, sub) 253 | } 254 | 255 | func New(opts ...Option) *broker { 256 | return newBroker(opts...) 257 | } 258 | -------------------------------------------------------------------------------- /broker/broker_test.go: -------------------------------------------------------------------------------- 1 | package broker 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "testing" 7 | ) 8 | 9 | func TestBroker(t *testing.T) { 10 | b := New() 11 | 12 | var wg sync.WaitGroup 13 | 14 | for i := 0; i < 10; i++ { 15 | topic := fmt.Sprintf("test%d", i) 16 | payload := []byte(topic) 17 | 18 | ch, err := b.Subscribe(topic) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | wg.Add(1) 24 | 25 | go func() { 26 | e := <-ch 27 | if string(e) != string(payload) { 28 | t.Fatalf("%s expected %s got %s", topic, string(payload), string(e)) 29 | } 30 | if err := b.Unsubscribe(topic, ch); err != nil { 31 | t.Fatal(err) 32 | } 33 | wg.Done() 34 | }() 35 | 36 | if err := b.Publish(topic, payload); err != nil { 37 | t.Fatal(err) 38 | } 39 | } 40 | 41 | wg.Wait() 42 | } 43 | -------------------------------------------------------------------------------- /broker/options.go: -------------------------------------------------------------------------------- 1 | package broker 2 | 3 | import ( 4 | "github.com/asim/mq/client" 5 | ) 6 | 7 | type Options struct { 8 | Client client.Client 9 | Proxy bool 10 | Persist bool 11 | } 12 | 13 | type Option func(o *Options) 14 | 15 | func Client(c client.Client) Option { 16 | return func(o *Options) { 17 | o.Client = c 18 | } 19 | } 20 | 21 | func Proxy(b bool) Option { 22 | return func(o *Options) { 23 | o.Proxy = b 24 | } 25 | } 26 | 27 | func Persist(b bool) Option { 28 | return func(o *Options) { 29 | o.Persist = b 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /client/README.md: -------------------------------------------------------------------------------- 1 | # Go Client [![GoDoc](https://godoc.org/github.com/asim/mq/client?status.svg)](https://godoc.org/github.com/asim/mq/client) 2 | 3 | ## Usage 4 | 5 | ### Publish 6 | 7 | ```go 8 | package main 9 | 10 | import ( 11 | "log" 12 | "time" 13 | 14 | "github.com/asim/mq/client" 15 | ) 16 | 17 | func main() { 18 | tick := time.NewTicker(time.Second) 19 | 20 | for _ = range tick.C { 21 | if err := client.Publish("foo", []byte(`bar`)); err != nil { 22 | log.Println(err) 23 | break 24 | } 25 | } 26 | } 27 | ``` 28 | 29 | ### Subscribe 30 | ```go 31 | package main 32 | 33 | import ( 34 | "log" 35 | 36 | "github.com/asim/mq/client" 37 | ) 38 | 39 | func main() { 40 | ch, err := client.Subscribe("foo") 41 | if err != nil { 42 | log.Println(err) 43 | return 44 | } 45 | 46 | for e := range ch { 47 | log.Println(string(e)) 48 | } 49 | 50 | log.Println("channel closed") 51 | } 52 | ``` 53 | 54 | ### New Client 55 | 56 | ```go 57 | // defaults to MQ server localhost:8081 58 | c := client.New() 59 | ``` 60 | 61 | gRPC client 62 | 63 | ```go 64 | import "github.com/asim/mq/client/grpc" 65 | 66 | c := grpc.New() 67 | ``` 68 | 69 | ### Clustering 70 | 71 | Clustering is supported on the client side. Publish/Subscribe operations are performed against all servers. 72 | 73 | ```go 74 | c := client.New( 75 | client.WithServers("10.0.0.1:8081", "10.0.0.1:8082", "10.0.0.1:8083"), 76 | ) 77 | ``` 78 | 79 | ### Sharding 80 | 81 | Sharding is supported via client much like gomemcache. Publish/Subscribe operations are performed against a single server. 82 | 83 | ```go 84 | import "github.com/asim/mq/client/selector" 85 | 86 | c := client.New( 87 | client.WithServers("10.0.0.1:8081", "10.0.0.1:8082", "10.0.0.1:8083"), 88 | client.WithSelector(new(selector.Shard)), 89 | ) 90 | ``` 91 | 92 | ### Resolver 93 | 94 | A name resolver can be used to discover the ip addresses of MQ servers 95 | 96 | ```go 97 | import "github.com/asim/mq/client/resolver" 98 | 99 | c := client.New( 100 | // use the DNS resolver 101 | client.WithResolver(new(resolver.DNS)), 102 | // specify DNS name as server 103 | client.WithServers("mq.proxy.local"), 104 | ) 105 | ``` 106 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | // Client is the interface provided by this package 4 | type Client interface { 5 | Close() error 6 | Publish(topic string, payload []byte) error 7 | Subscribe(topic string) (<-chan []byte, error) 8 | Unsubscribe(<-chan []byte) error 9 | } 10 | 11 | // Resolver resolves a name to a list of servers 12 | type Resolver interface { 13 | Resolve(name string) ([]string, error) 14 | } 15 | 16 | // Selector provides a server list to publish/subscribe to 17 | type Selector interface { 18 | Get(topic string) ([]string, error) 19 | Set(servers ...string) error 20 | } 21 | 22 | var ( 23 | // The default client 24 | Default = New() 25 | // The default server list 26 | Servers = []string{"http://127.0.0.1:8081"} 27 | // The default number of retries 28 | Retries = 1 29 | ) 30 | 31 | // Publish via the default Client 32 | func Publish(topic string, payload []byte) error { 33 | return Default.Publish(topic, payload) 34 | } 35 | 36 | // Subscribe via the default Client 37 | func Subscribe(topic string) (<-chan []byte, error) { 38 | return Default.Subscribe(topic) 39 | } 40 | 41 | // Unsubscribe via the default Client 42 | func Unsubscribe(ch <-chan []byte) error { 43 | return Default.Unsubscribe(ch) 44 | } 45 | 46 | // New returns a new Client 47 | func New(opts ...Option) Client { 48 | return newHTTPClient(opts...) 49 | } 50 | -------------------------------------------------------------------------------- /client/grpc/grpc.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "crypto/tls" 5 | "errors" 6 | "sync" 7 | "time" 8 | 9 | "github.com/asim/mq/client" 10 | "github.com/asim/mq/client/selector" 11 | pb "github.com/asim/mq/proto" 12 | "golang.org/x/net/context" 13 | "google.golang.org/grpc" 14 | "google.golang.org/grpc/credentials" 15 | ) 16 | 17 | // internal grpcClient 18 | type grpcClient struct { 19 | exit chan bool 20 | options client.Options 21 | 22 | sync.RWMutex 23 | subscribers map[<-chan []byte]*subscriber 24 | } 25 | 26 | // internal subscriber 27 | type subscriber struct { 28 | wg sync.WaitGroup 29 | ch chan<- []byte 30 | exit chan bool 31 | topic string 32 | } 33 | 34 | func grpcPublish(addr, topic string, payload []byte) error { 35 | var dialOpts []grpc.DialOption 36 | 37 | creds := credentials.NewTLS(&tls.Config{ 38 | InsecureSkipVerify: true, 39 | }) 40 | 41 | dialOpts = append(dialOpts, grpc.WithTransportCredentials(creds)) 42 | 43 | conn, err := grpc.Dial(addr, dialOpts...) 44 | if err != nil { 45 | return err 46 | } 47 | defer conn.Close() 48 | 49 | c := pb.NewMQClient(conn) 50 | _, err = c.Pub(context.TODO(), &pb.PubRequest{ 51 | Topic: topic, 52 | Payload: payload, 53 | }) 54 | 55 | return err 56 | } 57 | 58 | func grpcSubscribe(addr string, s *subscriber) error { 59 | var dialOpts []grpc.DialOption 60 | 61 | creds := credentials.NewTLS(&tls.Config{ 62 | InsecureSkipVerify: true, 63 | }) 64 | 65 | dialOpts = append(dialOpts, grpc.WithTransportCredentials(creds)) 66 | 67 | conn, err := grpc.Dial(addr, dialOpts...) 68 | if err != nil { 69 | return err 70 | } 71 | 72 | c := pb.NewMQClient(conn) 73 | sub, err := c.Sub(context.TODO(), &pb.SubRequest{ 74 | Topic: s.topic, 75 | }) 76 | if err != nil { 77 | return err 78 | } 79 | 80 | go func() { 81 | select { 82 | case <-s.exit: 83 | conn.Close() 84 | } 85 | }() 86 | 87 | go func() { 88 | defer s.wg.Done() 89 | 90 | for { 91 | rsp, err := sub.Recv() 92 | if err != nil { 93 | conn.Close() 94 | return 95 | } 96 | 97 | select { 98 | case s.ch <- rsp.Payload: 99 | case <-s.exit: 100 | return 101 | } 102 | } 103 | }() 104 | 105 | return nil 106 | } 107 | 108 | func (c *grpcClient) run() { 109 | // is there a resolver? 110 | if c.options.Resolver == nil { 111 | return 112 | } 113 | 114 | t := time.NewTicker(time.Second * 30) 115 | defer t.Stop() 116 | 117 | for { 118 | select { 119 | case <-t.C: 120 | var servers []string 121 | 122 | // iterate names 123 | for _, server := range c.options.Servers { 124 | ips, err := c.options.Resolver.Resolve(server) 125 | if err != nil { 126 | continue 127 | } 128 | servers = append(servers, ips...) 129 | } 130 | 131 | // only set if we have servers 132 | if len(servers) > 0 { 133 | c.options.Selector.Set(servers...) 134 | } 135 | case <-c.exit: 136 | return 137 | } 138 | } 139 | } 140 | 141 | func (c *grpcClient) Close() error { 142 | select { 143 | case <-c.exit: 144 | return nil 145 | default: 146 | close(c.exit) 147 | c.Lock() 148 | for _, sub := range c.subscribers { 149 | sub.Close() 150 | } 151 | c.Unlock() 152 | } 153 | return nil 154 | } 155 | 156 | func (c *grpcClient) Publish(topic string, payload []byte) error { 157 | select { 158 | case <-c.exit: 159 | return errors.New("client closed") 160 | default: 161 | } 162 | 163 | servers, err := c.options.Selector.Get(topic) 164 | if err != nil { 165 | return err 166 | } 167 | 168 | var grr error 169 | for _, addr := range servers { 170 | for i := 0; i < 1+c.options.Retries; i++ { 171 | err := grpcPublish(addr, topic, payload) 172 | if err == nil { 173 | break 174 | } 175 | grr = err 176 | } 177 | } 178 | return grr 179 | } 180 | 181 | func (c *grpcClient) Subscribe(topic string) (<-chan []byte, error) { 182 | select { 183 | case <-c.exit: 184 | return nil, errors.New("client closed") 185 | default: 186 | } 187 | 188 | servers, err := c.options.Selector.Get(topic) 189 | if err != nil { 190 | return nil, err 191 | } 192 | 193 | ch := make(chan []byte, len(c.options.Servers)*256) 194 | 195 | s := &subscriber{ 196 | ch: ch, 197 | exit: make(chan bool), 198 | topic: topic, 199 | } 200 | 201 | var grr error 202 | for _, addr := range servers { 203 | for i := 0; i < 1+c.options.Retries; i++ { 204 | err := grpcSubscribe(addr, s) 205 | if err == nil { 206 | s.wg.Add(1) 207 | break 208 | } 209 | grr = err 210 | } 211 | } 212 | 213 | return ch, grr 214 | } 215 | 216 | func (c *grpcClient) Unsubscribe(ch <-chan []byte) error { 217 | select { 218 | case <-c.exit: 219 | return errors.New("client closed") 220 | default: 221 | } 222 | 223 | c.Lock() 224 | defer c.Unlock() 225 | if sub, ok := c.subscribers[ch]; ok { 226 | return sub.Close() 227 | } 228 | return nil 229 | } 230 | 231 | func (s *subscriber) Close() error { 232 | select { 233 | case <-s.exit: 234 | default: 235 | close(s.exit) 236 | s.wg.Wait() 237 | } 238 | return nil 239 | } 240 | 241 | // New returns a grpc Client 242 | func New(opts ...client.Option) *grpcClient { 243 | options := client.Options{ 244 | Selector: new(selector.All), 245 | Servers: client.Servers, 246 | Retries: client.Retries, 247 | } 248 | 249 | for _, o := range opts { 250 | o(&options) 251 | } 252 | 253 | // set servers 254 | options.Selector.Set(options.Servers...) 255 | 256 | c := &grpcClient{ 257 | exit: make(chan bool), 258 | options: options, 259 | subscribers: make(map[<-chan []byte]*subscriber), 260 | } 261 | go c.run() 262 | return c 263 | } 264 | -------------------------------------------------------------------------------- /client/http.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "bytes" 5 | "crypto/tls" 6 | "errors" 7 | "fmt" 8 | "net" 9 | "net/http" 10 | "strings" 11 | "sync" 12 | "time" 13 | 14 | "github.com/gorilla/websocket" 15 | ) 16 | 17 | // internal httpClient 18 | type httpClient struct { 19 | exit chan bool 20 | options Options 21 | 22 | sync.RWMutex 23 | subscribers map[<-chan []byte]*subscriber 24 | } 25 | 26 | // internal subscriber 27 | type subscriber struct { 28 | wg sync.WaitGroup 29 | ch chan<- []byte 30 | exit chan bool 31 | topic string 32 | } 33 | 34 | // internal select all 35 | type all struct { 36 | sync.RWMutex 37 | servers []string 38 | } 39 | 40 | var ( 41 | httpc = &http.Client{ 42 | Transport: &http.Transport{ 43 | Proxy: http.ProxyFromEnvironment, 44 | Dial: (&net.Dialer{ 45 | Timeout: 30 * time.Second, 46 | KeepAlive: 30 * time.Second, 47 | }).Dial, 48 | TLSHandshakeTimeout: 10 * time.Second, 49 | TLSClientConfig: &tls.Config{ 50 | InsecureSkipVerify: true, 51 | }, 52 | }, 53 | } 54 | 55 | wsd = &websocket.Dialer{ 56 | Proxy: http.ProxyFromEnvironment, 57 | TLSClientConfig: &tls.Config{ 58 | InsecureSkipVerify: true, 59 | }, 60 | } 61 | ) 62 | 63 | func publish(addr, topic string, payload []byte) error { 64 | url := fmt.Sprintf("%s/pub?topic=%s", addr, topic) 65 | rsp, err := httpc.Post(url, "application/json", bytes.NewBuffer(payload)) 66 | if err != nil { 67 | return err 68 | } 69 | rsp.Body.Close() 70 | if rsp.StatusCode != 200 { 71 | return fmt.Errorf("Non 200 response %d", rsp.StatusCode) 72 | } 73 | return nil 74 | } 75 | 76 | func subscribe(addr string, s *subscriber) error { 77 | if strings.HasPrefix(addr, "http") { 78 | addr = strings.TrimPrefix(addr, "http") 79 | addr = "ws" + addr 80 | } 81 | 82 | url := fmt.Sprintf("%s/sub?topic=%s", addr, s.topic) 83 | c, _, err := wsd.Dial(url, make(http.Header)) 84 | if err != nil { 85 | return err 86 | } 87 | 88 | go func() { 89 | select { 90 | case <-s.exit: 91 | c.Close() 92 | } 93 | }() 94 | 95 | go func() { 96 | defer s.wg.Done() 97 | 98 | for { 99 | t, p, err := c.ReadMessage() 100 | if err != nil || t == websocket.CloseMessage { 101 | c.Close() 102 | return 103 | } 104 | 105 | select { 106 | case s.ch <- p: 107 | case <-s.exit: 108 | c.Close() 109 | return 110 | } 111 | } 112 | }() 113 | 114 | return nil 115 | } 116 | 117 | func (sa *all) Get(topic string) ([]string, error) { 118 | sa.RLock() 119 | if len(sa.servers) == 0 { 120 | sa.RUnlock() 121 | return nil, errors.New("no servers") 122 | } 123 | servers := sa.servers 124 | sa.RUnlock() 125 | return servers, nil 126 | } 127 | 128 | func (sa *all) Set(servers ...string) error { 129 | sa.Lock() 130 | sa.servers = servers 131 | sa.Unlock() 132 | return nil 133 | } 134 | 135 | func (c *httpClient) run() { 136 | // is there a resolver? 137 | if c.options.Resolver == nil { 138 | return 139 | } 140 | 141 | t := time.NewTicker(time.Second * 30) 142 | defer t.Stop() 143 | 144 | for { 145 | select { 146 | case <-t.C: 147 | var servers []string 148 | 149 | // iterate names 150 | for _, server := range c.options.Servers { 151 | ips, err := c.options.Resolver.Resolve(server) 152 | if err != nil { 153 | continue 154 | } 155 | for i, ip := range ips { 156 | if !strings.HasPrefix(ip, "http") { 157 | ips[i] = fmt.Sprintf("https://%s", ip) 158 | } 159 | } 160 | servers = append(servers, ips...) 161 | } 162 | 163 | // only set if we have servers 164 | if len(servers) > 0 { 165 | c.options.Selector.Set(servers...) 166 | } 167 | case <-c.exit: 168 | return 169 | } 170 | } 171 | } 172 | 173 | func (c *httpClient) Close() error { 174 | select { 175 | case <-c.exit: 176 | return nil 177 | default: 178 | close(c.exit) 179 | c.Lock() 180 | for _, sub := range c.subscribers { 181 | sub.Close() 182 | } 183 | c.Unlock() 184 | } 185 | return nil 186 | } 187 | 188 | func (c *httpClient) Publish(topic string, payload []byte) error { 189 | select { 190 | case <-c.exit: 191 | return errors.New("client closed") 192 | default: 193 | } 194 | 195 | servers, err := c.options.Selector.Get(topic) 196 | if err != nil { 197 | return err 198 | } 199 | 200 | var grr error 201 | for _, addr := range servers { 202 | for i := 0; i < 1+c.options.Retries; i++ { 203 | err := publish(addr, topic, payload) 204 | if err == nil { 205 | break 206 | } 207 | grr = err 208 | } 209 | } 210 | return grr 211 | } 212 | 213 | func (c *httpClient) Subscribe(topic string) (<-chan []byte, error) { 214 | select { 215 | case <-c.exit: 216 | return nil, errors.New("client closed") 217 | default: 218 | } 219 | 220 | servers, err := c.options.Selector.Get(topic) 221 | if err != nil { 222 | return nil, err 223 | } 224 | 225 | ch := make(chan []byte, len(c.options.Servers)*256) 226 | 227 | s := &subscriber{ 228 | ch: ch, 229 | exit: make(chan bool), 230 | topic: topic, 231 | } 232 | 233 | var grr error 234 | for _, addr := range servers { 235 | for i := 0; i < 1+c.options.Retries; i++ { 236 | err := subscribe(addr, s) 237 | if err == nil { 238 | s.wg.Add(1) 239 | break 240 | } 241 | grr = err 242 | } 243 | } 244 | 245 | return ch, grr 246 | } 247 | 248 | func (c *httpClient) Unsubscribe(ch <-chan []byte) error { 249 | select { 250 | case <-c.exit: 251 | return errors.New("client closed") 252 | default: 253 | } 254 | 255 | c.Lock() 256 | defer c.Unlock() 257 | if sub, ok := c.subscribers[ch]; ok { 258 | return sub.Close() 259 | } 260 | return nil 261 | } 262 | 263 | func (s *subscriber) Close() error { 264 | select { 265 | case <-s.exit: 266 | default: 267 | close(s.exit) 268 | s.wg.Wait() 269 | } 270 | return nil 271 | } 272 | 273 | // newHTTPClient returns a http Client 274 | func newHTTPClient(opts ...Option) *httpClient { 275 | options := Options{ 276 | Selector: new(all), 277 | Servers: Servers, 278 | Retries: Retries, 279 | } 280 | 281 | for _, o := range opts { 282 | o(&options) 283 | } 284 | 285 | var servers []string 286 | 287 | for _, addr := range options.Servers { 288 | if !strings.HasPrefix(addr, "http") { 289 | addr = fmt.Sprintf("https://%s", addr) 290 | } 291 | servers = append(servers, addr) 292 | } 293 | 294 | // set servers 295 | WithServers(servers...)(&options) 296 | options.Selector.Set(options.Servers...) 297 | 298 | c := &httpClient{ 299 | exit: make(chan bool), 300 | options: options, 301 | subscribers: make(map[<-chan []byte]*subscriber), 302 | } 303 | go c.run() 304 | return c 305 | } 306 | -------------------------------------------------------------------------------- /client/http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "github.com/asim/mq/client" 5 | ) 6 | 7 | // New returns a http client 8 | func New(opts ...client.Option) client.Client { 9 | return client.New(opts...) 10 | } 11 | -------------------------------------------------------------------------------- /client/options.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | type Options struct { 4 | // Number of retry attempts 5 | Retries int 6 | // Resolver 7 | Resolver Resolver 8 | // Server list 9 | Servers []string 10 | // Selector 11 | Selector Selector 12 | } 13 | 14 | type Option func(o *Options) 15 | 16 | // WithRetries sets the number of retry attempts 17 | func WithRetries(i int) Option { 18 | return func(o *Options) { 19 | o.Retries = i 20 | } 21 | } 22 | 23 | // WithResolver sets the resolver used to get the server list 24 | func WithResolver(r Resolver) Option { 25 | return func(o *Options) { 26 | o.Resolver = r 27 | } 28 | } 29 | 30 | // WithSelector sets the server selector used by the client 31 | func WithSelector(s Selector) Option { 32 | return func(o *Options) { 33 | o.Selector = s 34 | } 35 | } 36 | 37 | // WithServers sets the servers used by the client 38 | func WithServers(addrs ...string) Option { 39 | return func(o *Options) { 40 | o.Servers = addrs 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /client/resolver/resolver.go: -------------------------------------------------------------------------------- 1 | package resolver 2 | 3 | import ( 4 | "net" 5 | ) 6 | 7 | // IP resolver returns the IP specified 8 | type IP struct{} 9 | 10 | // DNS resolver returns IPs based on a DNS name 11 | type DNS struct{} 12 | 13 | func (ip *IP) Resolve(name string) ([]string, error) { 14 | return []string{name}, nil 15 | } 16 | 17 | func (dns *DNS) Resolve(name string) ([]string, error) { 18 | ips, err := net.LookupIP(name) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | var addrs []string 24 | 25 | for _, ip := range ips { 26 | if ipv4 := ip.To4(); ipv4 != nil { 27 | addrs = append(addrs, ipv4.String()) 28 | } 29 | } 30 | 31 | return addrs, nil 32 | } 33 | -------------------------------------------------------------------------------- /client/resolver/resolver_test.go: -------------------------------------------------------------------------------- 1 | package resolver 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestIPResolver(t *testing.T) { 8 | ip := new(IP) 9 | ips, err := ip.Resolve("127.0.0.1") 10 | if err != nil { 11 | t.Fatal(err) 12 | } 13 | if len(ips) == 0 { 14 | t.Fatal("expected ip got", ips) 15 | } 16 | if ips[0] != "127.0.0.1" { 17 | t.Fatal("expected 127.0.0.1 got", ips[0]) 18 | } 19 | } 20 | 21 | func TestDNSResolver(t *testing.T) { 22 | dns := new(DNS) 23 | 24 | ips, err := dns.Resolve("localhost") 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | if len(ips) == 0 { 29 | t.Fatal("expected ip got", ips) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /client/selector/selector.go: -------------------------------------------------------------------------------- 1 | package selector 2 | 3 | import ( 4 | "errors" 5 | "hash/crc32" 6 | "sync" 7 | ) 8 | 9 | // All is a Selector that returns all servers 10 | type All struct { 11 | sync.RWMutex 12 | servers []string 13 | } 14 | 15 | // Shard is a Selector that shards to a single server 16 | type Shard struct { 17 | sync.RWMutex 18 | servers []string 19 | } 20 | 21 | func (sa *All) Get(topic string) ([]string, error) { 22 | sa.RLock() 23 | if len(sa.servers) == 0 { 24 | sa.RUnlock() 25 | return nil, errors.New("no servers") 26 | } 27 | servers := sa.servers 28 | sa.RUnlock() 29 | return servers, nil 30 | } 31 | 32 | func (sa *All) Set(servers ...string) error { 33 | sa.Lock() 34 | sa.servers = servers 35 | sa.Unlock() 36 | return nil 37 | } 38 | 39 | func (ss *Shard) Get(topic string) ([]string, error) { 40 | ss.RLock() 41 | length := len(ss.servers) 42 | if length == 0 { 43 | ss.RUnlock() 44 | return nil, errors.New("no servers") 45 | } 46 | if length == 1 { 47 | servers := ss.servers 48 | ss.RUnlock() 49 | return servers, nil 50 | } 51 | cs := crc32.ChecksumIEEE([]byte(topic)) 52 | server := ss.servers[cs%uint32(length)] 53 | ss.RUnlock() 54 | return []string{server}, nil 55 | } 56 | 57 | func (ss *Shard) Set(servers ...string) error { 58 | ss.Lock() 59 | ss.servers = servers 60 | ss.Unlock() 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /client/selector/selector_test.go: -------------------------------------------------------------------------------- 1 | package selector 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestSelectAll(t *testing.T) { 8 | testServers := []string{"a", "b", "c"} 9 | sa := new(All) 10 | if err := sa.Set(testServers...); err != nil { 11 | t.Fatal(err) 12 | } 13 | servers, err := sa.Get("test") 14 | if err != nil { 15 | t.Fatal(err) 16 | } 17 | for _, tserver := range testServers { 18 | var seen bool 19 | for _, server := range servers { 20 | if server == tserver { 21 | seen = true 22 | break 23 | } 24 | } 25 | if !seen { 26 | t.Fatal("did not find", tserver) 27 | } 28 | } 29 | } 30 | 31 | func TestSelectShard(t *testing.T) { 32 | testServers := []string{"a", "b", "c"} 33 | ss := new(Shard) 34 | if err := ss.Set(testServers...); err != nil { 35 | t.Fatal(err) 36 | } 37 | servers, err := ss.Get("test") 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | var seen bool 42 | for _, tserver := range testServers { 43 | for _, server := range servers { 44 | if server == tserver { 45 | seen = true 46 | break 47 | } 48 | } 49 | } 50 | if !seen { 51 | t.Fatal("did not find any test servers") 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /examples/pub/pub.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "strings" 7 | "time" 8 | 9 | "github.com/asim/mq/client" 10 | ) 11 | 12 | var ( 13 | servers = flag.String("servers", "localhost:8081", "Comma separated list of MQ servers") 14 | ) 15 | 16 | func main() { 17 | flag.Parse() 18 | 19 | c := client.New( 20 | client.WithServers(strings.Split(*servers, ",")...), 21 | ) 22 | tick := time.NewTicker(time.Second) 23 | 24 | for _ = range tick.C { 25 | if err := c.Publish("foo", []byte(`bar`)); err != nil { 26 | log.Println(err) 27 | break 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /examples/sub/sub.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "strings" 7 | 8 | "github.com/asim/mq/client" 9 | ) 10 | 11 | var ( 12 | servers = flag.String("servers", "localhost:8081", "Comma separated list of MQ servers") 13 | ) 14 | 15 | func main() { 16 | flag.Parse() 17 | 18 | c := client.New( 19 | client.WithServers(strings.Split(*servers, ",")...), 20 | ) 21 | 22 | ch, err := c.Subscribe("foo") 23 | if err != nil { 24 | log.Println(err) 25 | return 26 | } 27 | defer c.Unsubscribe(ch) 28 | 29 | for { 30 | select { 31 | case e := <-ch: 32 | log.Println(string(e)) 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /generate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | //go:generate protoc --go_out=plugins=grpc:. --go_opt=paths=source_relative proto/mq.proto 4 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/asim/mq 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/gorilla/handlers v1.5.1 7 | github.com/gorilla/websocket v1.4.2 8 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b 9 | google.golang.org/grpc v1.40.0 10 | google.golang.org/protobuf v1.25.0 11 | ) 12 | 13 | require ( 14 | github.com/felixge/httpsnoop v1.0.1 // indirect 15 | github.com/golang/protobuf v1.4.3 // indirect 16 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f // indirect 17 | golang.org/x/text v0.3.3 // indirect 18 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 4 | github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= 5 | github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= 6 | github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= 7 | github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= 8 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 9 | github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= 10 | github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= 11 | github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= 12 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 14 | github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= 15 | github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= 16 | github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= 17 | github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= 18 | github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= 19 | github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= 20 | github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 21 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 22 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 23 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 24 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 25 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 26 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 27 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 28 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 29 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 30 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 31 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 32 | github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= 33 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 34 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 35 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 36 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 37 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 38 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 39 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 40 | github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= 41 | github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 42 | github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 43 | github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= 44 | github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= 45 | github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= 46 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 47 | github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= 48 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 49 | github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 50 | github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= 51 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 52 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 53 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 54 | go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= 55 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 56 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 57 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 58 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 59 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 60 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 61 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 62 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 63 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 64 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 65 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 66 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 67 | golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 68 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= 69 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 70 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 71 | golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 72 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 73 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 74 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 75 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 76 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 77 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 78 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 79 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 80 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= 81 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 82 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 83 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 84 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 85 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 86 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 87 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 88 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 89 | golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= 90 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 91 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 92 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 93 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 94 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 95 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 96 | google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= 97 | google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= 98 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= 99 | google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= 100 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 101 | google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= 102 | google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= 103 | google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= 104 | google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= 105 | google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= 106 | google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= 107 | google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= 108 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 109 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 110 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 111 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 112 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 113 | google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 114 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 115 | google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 116 | google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= 117 | google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= 118 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 119 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 120 | gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 121 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 122 | honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 123 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | "github.com/asim/mq/broker" 14 | mqclient "github.com/asim/mq/client" 15 | mqgrpc "github.com/asim/mq/client/grpc" 16 | mqresolver "github.com/asim/mq/client/resolver" 17 | mqselector "github.com/asim/mq/client/selector" 18 | "github.com/asim/mq/server" 19 | grpcsrv "github.com/asim/mq/server/grpc" 20 | httpsrv "github.com/asim/mq/server/http" 21 | ) 22 | 23 | var ( 24 | address = flag.String("address", ":8081", "MQ server address") 25 | cert = flag.String("cert_file", "", "TLS certificate file") 26 | key = flag.String("key_file", "", "TLS key file") 27 | 28 | // server persist to file 29 | persist = flag.Bool("persist", false, "Persist messages to [topic].mq file per topic") 30 | 31 | // proxy flags 32 | proxy = flag.Bool("proxy", false, "Proxy for an MQ cluster") 33 | retries = flag.Int("retries", 1, "Number of retries for publish or subscribe") 34 | servers = flag.String("servers", "", "Comma separated MQ cluster list used by Proxy") 35 | 36 | // client flags 37 | interactive = flag.Bool("i", false, "Interactive client mode") 38 | client = flag.Bool("client", false, "Run the MQ client") 39 | publish = flag.Bool("publish", false, "Publish via the MQ client") 40 | subscribe = flag.Bool("subscribe", false, "Subscribe via the MQ client") 41 | topic = flag.String("topic", "", "Topic for client to publish or subscribe to") 42 | 43 | // select strategy 44 | selector = flag.String("select", "all", "Server select strategy. Supports all, shard") 45 | // resolver for discovery 46 | resolver = flag.String("resolver", "ip", "Server resolver for discovery. Supports ip, dns") 47 | // transport http or grpc 48 | transport = flag.String("transport", "http", "Transport for communication. Support http, grpc") 49 | ) 50 | 51 | func init() { 52 | flag.Parse() 53 | 54 | if *proxy && *client { 55 | log.Fatal("Client and proxy flags cannot be specified together") 56 | } 57 | 58 | if *proxy && len(*servers) == 0 { 59 | log.Fatal("Proxy enabled without MQ server list") 60 | } 61 | 62 | if *client && len(*topic) == 0 { 63 | log.Fatal("Topic not specified") 64 | } 65 | 66 | if *client && !*publish && !*subscribe { 67 | log.Fatal("Specify whether to publish or subscribe") 68 | } 69 | 70 | if (*client || *interactive) && len(*servers) == 0 { 71 | *servers = "localhost:8081" 72 | } 73 | 74 | var bclient mqclient.Client 75 | var selecter mqclient.Selector 76 | var resolvor mqclient.Resolver 77 | 78 | switch *selector { 79 | case "shard": 80 | selecter = new(mqselector.Shard) 81 | default: 82 | selecter = new(mqselector.All) 83 | } 84 | 85 | switch *resolver { 86 | case "dns": 87 | resolvor = new(mqresolver.DNS) 88 | default: 89 | } 90 | 91 | options := []mqclient.Option{ 92 | mqclient.WithResolver(resolvor), 93 | mqclient.WithSelector(selecter), 94 | mqclient.WithServers(strings.Split(*servers, ",")...), 95 | mqclient.WithRetries(*retries), 96 | } 97 | 98 | switch *transport { 99 | case "grpc": 100 | bclient = mqgrpc.New(options...) 101 | default: 102 | bclient = mqclient.New(options...) 103 | } 104 | 105 | broker.Default = broker.New( 106 | broker.Client(bclient), 107 | broker.Persist(*persist), 108 | broker.Proxy(*client || *proxy || *interactive), 109 | ) 110 | } 111 | 112 | func cli() { 113 | wg := sync.WaitGroup{} 114 | p := make(chan []byte, 1000) 115 | d := map[string]time.Time{} 116 | ttl := time.Millisecond * 10 117 | tick := time.NewTicker(time.Second * 5) 118 | 119 | // process publish 120 | if *publish || *interactive { 121 | wg.Add(1) 122 | go func() { 123 | scanner := bufio.NewScanner(os.Stdin) 124 | for scanner.Scan() { 125 | if *interactive { 126 | p <- scanner.Bytes() 127 | } 128 | broker.Publish(*topic, scanner.Bytes()) 129 | } 130 | wg.Done() 131 | }() 132 | } 133 | 134 | // subscribe? 135 | if !(*subscribe || *interactive) { 136 | wg.Wait() 137 | return 138 | } 139 | 140 | // process subscribe 141 | ch, err := broker.Subscribe(*topic) 142 | if err != nil { 143 | fmt.Println(err) 144 | return 145 | } 146 | defer broker.Unsubscribe(*topic, ch) 147 | 148 | for { 149 | select { 150 | // process sub event 151 | case b := <-ch: 152 | // skip if deduped 153 | if t, ok := d[string(b)]; ok && time.Since(t) < ttl { 154 | continue 155 | } 156 | d[string(b)] = time.Now() 157 | fmt.Println(string(b)) 158 | // add dedupe entry 159 | case b := <-p: 160 | d[string(b)] = time.Now() 161 | // flush deduper 162 | case <-tick.C: 163 | d = map[string]time.Time{} 164 | } 165 | } 166 | 167 | wg.Wait() 168 | } 169 | 170 | func main() { 171 | // handle client 172 | if *client || *interactive { 173 | cli() 174 | return 175 | } 176 | 177 | // cleanup broker 178 | defer broker.Default.Close() 179 | 180 | options := []server.Option{ 181 | server.WithAddress(*address), 182 | } 183 | 184 | // proxy enabled 185 | if *proxy { 186 | log.Println("Proxy enabled") 187 | } 188 | 189 | // tls enabled 190 | if len(*cert) > 0 && len(*key) > 0 { 191 | log.Println("TLS Enabled") 192 | options = append(options, server.WithTLS(*cert, *key)) 193 | } 194 | 195 | var server server.Server 196 | 197 | // now serve the transport 198 | switch *transport { 199 | case "grpc": 200 | log.Println("GRPC transport enabled") 201 | server = grpcsrv.New(options...) 202 | default: 203 | log.Println("HTTP transport enabled") 204 | server = httpsrv.New(options...) 205 | } 206 | 207 | log.Println("MQ listening on", *address) 208 | if err := server.Run(); err != nil { 209 | log.Fatal(err) 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /mq.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/asim/mq/c89d5180a6ca6b1aef1e7c0c8091b59859013edd/mq.png -------------------------------------------------------------------------------- /proto/mq.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.27.1 4 | // protoc v3.15.6 5 | // source: proto/mq.proto 6 | 7 | package mq 8 | 9 | import ( 10 | context "context" 11 | grpc "google.golang.org/grpc" 12 | codes "google.golang.org/grpc/codes" 13 | status "google.golang.org/grpc/status" 14 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 15 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 16 | reflect "reflect" 17 | sync "sync" 18 | ) 19 | 20 | const ( 21 | // Verify that this generated code is sufficiently up-to-date. 22 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 23 | // Verify that runtime/protoimpl is sufficiently up-to-date. 24 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 25 | ) 26 | 27 | type PubRequest struct { 28 | state protoimpl.MessageState 29 | sizeCache protoimpl.SizeCache 30 | unknownFields protoimpl.UnknownFields 31 | 32 | Topic string `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` 33 | Payload []byte `protobuf:"bytes,2,opt,name=payload,proto3" json:"payload,omitempty"` 34 | } 35 | 36 | func (x *PubRequest) Reset() { 37 | *x = PubRequest{} 38 | if protoimpl.UnsafeEnabled { 39 | mi := &file_proto_mq_proto_msgTypes[0] 40 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 41 | ms.StoreMessageInfo(mi) 42 | } 43 | } 44 | 45 | func (x *PubRequest) String() string { 46 | return protoimpl.X.MessageStringOf(x) 47 | } 48 | 49 | func (*PubRequest) ProtoMessage() {} 50 | 51 | func (x *PubRequest) ProtoReflect() protoreflect.Message { 52 | mi := &file_proto_mq_proto_msgTypes[0] 53 | if protoimpl.UnsafeEnabled && x != nil { 54 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 55 | if ms.LoadMessageInfo() == nil { 56 | ms.StoreMessageInfo(mi) 57 | } 58 | return ms 59 | } 60 | return mi.MessageOf(x) 61 | } 62 | 63 | // Deprecated: Use PubRequest.ProtoReflect.Descriptor instead. 64 | func (*PubRequest) Descriptor() ([]byte, []int) { 65 | return file_proto_mq_proto_rawDescGZIP(), []int{0} 66 | } 67 | 68 | func (x *PubRequest) GetTopic() string { 69 | if x != nil { 70 | return x.Topic 71 | } 72 | return "" 73 | } 74 | 75 | func (x *PubRequest) GetPayload() []byte { 76 | if x != nil { 77 | return x.Payload 78 | } 79 | return nil 80 | } 81 | 82 | type PubResponse struct { 83 | state protoimpl.MessageState 84 | sizeCache protoimpl.SizeCache 85 | unknownFields protoimpl.UnknownFields 86 | } 87 | 88 | func (x *PubResponse) Reset() { 89 | *x = PubResponse{} 90 | if protoimpl.UnsafeEnabled { 91 | mi := &file_proto_mq_proto_msgTypes[1] 92 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 93 | ms.StoreMessageInfo(mi) 94 | } 95 | } 96 | 97 | func (x *PubResponse) String() string { 98 | return protoimpl.X.MessageStringOf(x) 99 | } 100 | 101 | func (*PubResponse) ProtoMessage() {} 102 | 103 | func (x *PubResponse) ProtoReflect() protoreflect.Message { 104 | mi := &file_proto_mq_proto_msgTypes[1] 105 | if protoimpl.UnsafeEnabled && x != nil { 106 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 107 | if ms.LoadMessageInfo() == nil { 108 | ms.StoreMessageInfo(mi) 109 | } 110 | return ms 111 | } 112 | return mi.MessageOf(x) 113 | } 114 | 115 | // Deprecated: Use PubResponse.ProtoReflect.Descriptor instead. 116 | func (*PubResponse) Descriptor() ([]byte, []int) { 117 | return file_proto_mq_proto_rawDescGZIP(), []int{1} 118 | } 119 | 120 | type SubRequest struct { 121 | state protoimpl.MessageState 122 | sizeCache protoimpl.SizeCache 123 | unknownFields protoimpl.UnknownFields 124 | 125 | Topic string `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` 126 | } 127 | 128 | func (x *SubRequest) Reset() { 129 | *x = SubRequest{} 130 | if protoimpl.UnsafeEnabled { 131 | mi := &file_proto_mq_proto_msgTypes[2] 132 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 133 | ms.StoreMessageInfo(mi) 134 | } 135 | } 136 | 137 | func (x *SubRequest) String() string { 138 | return protoimpl.X.MessageStringOf(x) 139 | } 140 | 141 | func (*SubRequest) ProtoMessage() {} 142 | 143 | func (x *SubRequest) ProtoReflect() protoreflect.Message { 144 | mi := &file_proto_mq_proto_msgTypes[2] 145 | if protoimpl.UnsafeEnabled && x != nil { 146 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 147 | if ms.LoadMessageInfo() == nil { 148 | ms.StoreMessageInfo(mi) 149 | } 150 | return ms 151 | } 152 | return mi.MessageOf(x) 153 | } 154 | 155 | // Deprecated: Use SubRequest.ProtoReflect.Descriptor instead. 156 | func (*SubRequest) Descriptor() ([]byte, []int) { 157 | return file_proto_mq_proto_rawDescGZIP(), []int{2} 158 | } 159 | 160 | func (x *SubRequest) GetTopic() string { 161 | if x != nil { 162 | return x.Topic 163 | } 164 | return "" 165 | } 166 | 167 | type SubResponse struct { 168 | state protoimpl.MessageState 169 | sizeCache protoimpl.SizeCache 170 | unknownFields protoimpl.UnknownFields 171 | 172 | Payload []byte `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` 173 | } 174 | 175 | func (x *SubResponse) Reset() { 176 | *x = SubResponse{} 177 | if protoimpl.UnsafeEnabled { 178 | mi := &file_proto_mq_proto_msgTypes[3] 179 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 180 | ms.StoreMessageInfo(mi) 181 | } 182 | } 183 | 184 | func (x *SubResponse) String() string { 185 | return protoimpl.X.MessageStringOf(x) 186 | } 187 | 188 | func (*SubResponse) ProtoMessage() {} 189 | 190 | func (x *SubResponse) ProtoReflect() protoreflect.Message { 191 | mi := &file_proto_mq_proto_msgTypes[3] 192 | if protoimpl.UnsafeEnabled && x != nil { 193 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 194 | if ms.LoadMessageInfo() == nil { 195 | ms.StoreMessageInfo(mi) 196 | } 197 | return ms 198 | } 199 | return mi.MessageOf(x) 200 | } 201 | 202 | // Deprecated: Use SubResponse.ProtoReflect.Descriptor instead. 203 | func (*SubResponse) Descriptor() ([]byte, []int) { 204 | return file_proto_mq_proto_rawDescGZIP(), []int{3} 205 | } 206 | 207 | func (x *SubResponse) GetPayload() []byte { 208 | if x != nil { 209 | return x.Payload 210 | } 211 | return nil 212 | } 213 | 214 | var File_proto_mq_proto protoreflect.FileDescriptor 215 | 216 | var file_proto_mq_proto_rawDesc = []byte{ 217 | 0x0a, 0x0e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x71, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 218 | 0x12, 0x02, 0x6d, 0x71, 0x22, 0x3c, 0x0a, 0x0a, 0x50, 0x75, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 219 | 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 220 | 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 221 | 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 222 | 0x61, 0x64, 0x22, 0x0d, 0x0a, 0x0b, 0x50, 0x75, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 223 | 0x65, 0x22, 0x22, 0x0a, 0x0a, 0x53, 0x75, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 224 | 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 225 | 0x74, 0x6f, 0x70, 0x69, 0x63, 0x22, 0x27, 0x0a, 0x0b, 0x53, 0x75, 0x62, 0x52, 0x65, 0x73, 0x70, 226 | 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 227 | 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x32, 0x5a, 228 | 0x0a, 0x02, 0x4d, 0x51, 0x12, 0x28, 0x0a, 0x03, 0x50, 0x75, 0x62, 0x12, 0x0e, 0x2e, 0x6d, 0x71, 229 | 0x2e, 0x50, 0x75, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0f, 0x2e, 0x6d, 0x71, 230 | 0x2e, 0x50, 0x75, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2a, 231 | 0x0a, 0x03, 0x53, 0x75, 0x62, 0x12, 0x0e, 0x2e, 0x6d, 0x71, 0x2e, 0x53, 0x75, 0x62, 0x52, 0x65, 232 | 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0f, 0x2e, 0x6d, 0x71, 0x2e, 0x53, 0x75, 0x62, 0x52, 0x65, 233 | 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x42, 0x0c, 0x5a, 0x0a, 0x2e, 0x2f, 234 | 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x6d, 0x71, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 235 | } 236 | 237 | var ( 238 | file_proto_mq_proto_rawDescOnce sync.Once 239 | file_proto_mq_proto_rawDescData = file_proto_mq_proto_rawDesc 240 | ) 241 | 242 | func file_proto_mq_proto_rawDescGZIP() []byte { 243 | file_proto_mq_proto_rawDescOnce.Do(func() { 244 | file_proto_mq_proto_rawDescData = protoimpl.X.CompressGZIP(file_proto_mq_proto_rawDescData) 245 | }) 246 | return file_proto_mq_proto_rawDescData 247 | } 248 | 249 | var file_proto_mq_proto_msgTypes = make([]protoimpl.MessageInfo, 4) 250 | var file_proto_mq_proto_goTypes = []interface{}{ 251 | (*PubRequest)(nil), // 0: mq.PubRequest 252 | (*PubResponse)(nil), // 1: mq.PubResponse 253 | (*SubRequest)(nil), // 2: mq.SubRequest 254 | (*SubResponse)(nil), // 3: mq.SubResponse 255 | } 256 | var file_proto_mq_proto_depIdxs = []int32{ 257 | 0, // 0: mq.MQ.Pub:input_type -> mq.PubRequest 258 | 2, // 1: mq.MQ.Sub:input_type -> mq.SubRequest 259 | 1, // 2: mq.MQ.Pub:output_type -> mq.PubResponse 260 | 3, // 3: mq.MQ.Sub:output_type -> mq.SubResponse 261 | 2, // [2:4] is the sub-list for method output_type 262 | 0, // [0:2] is the sub-list for method input_type 263 | 0, // [0:0] is the sub-list for extension type_name 264 | 0, // [0:0] is the sub-list for extension extendee 265 | 0, // [0:0] is the sub-list for field type_name 266 | } 267 | 268 | func init() { file_proto_mq_proto_init() } 269 | func file_proto_mq_proto_init() { 270 | if File_proto_mq_proto != nil { 271 | return 272 | } 273 | if !protoimpl.UnsafeEnabled { 274 | file_proto_mq_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 275 | switch v := v.(*PubRequest); i { 276 | case 0: 277 | return &v.state 278 | case 1: 279 | return &v.sizeCache 280 | case 2: 281 | return &v.unknownFields 282 | default: 283 | return nil 284 | } 285 | } 286 | file_proto_mq_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 287 | switch v := v.(*PubResponse); i { 288 | case 0: 289 | return &v.state 290 | case 1: 291 | return &v.sizeCache 292 | case 2: 293 | return &v.unknownFields 294 | default: 295 | return nil 296 | } 297 | } 298 | file_proto_mq_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { 299 | switch v := v.(*SubRequest); i { 300 | case 0: 301 | return &v.state 302 | case 1: 303 | return &v.sizeCache 304 | case 2: 305 | return &v.unknownFields 306 | default: 307 | return nil 308 | } 309 | } 310 | file_proto_mq_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { 311 | switch v := v.(*SubResponse); i { 312 | case 0: 313 | return &v.state 314 | case 1: 315 | return &v.sizeCache 316 | case 2: 317 | return &v.unknownFields 318 | default: 319 | return nil 320 | } 321 | } 322 | } 323 | type x struct{} 324 | out := protoimpl.TypeBuilder{ 325 | File: protoimpl.DescBuilder{ 326 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 327 | RawDescriptor: file_proto_mq_proto_rawDesc, 328 | NumEnums: 0, 329 | NumMessages: 4, 330 | NumExtensions: 0, 331 | NumServices: 1, 332 | }, 333 | GoTypes: file_proto_mq_proto_goTypes, 334 | DependencyIndexes: file_proto_mq_proto_depIdxs, 335 | MessageInfos: file_proto_mq_proto_msgTypes, 336 | }.Build() 337 | File_proto_mq_proto = out.File 338 | file_proto_mq_proto_rawDesc = nil 339 | file_proto_mq_proto_goTypes = nil 340 | file_proto_mq_proto_depIdxs = nil 341 | } 342 | 343 | // Reference imports to suppress errors if they are not otherwise used. 344 | var _ context.Context 345 | var _ grpc.ClientConnInterface 346 | 347 | // This is a compile-time assertion to ensure that this generated file 348 | // is compatible with the grpc package it is being compiled against. 349 | const _ = grpc.SupportPackageIsVersion6 350 | 351 | // MQClient is the client API for MQ service. 352 | // 353 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. 354 | type MQClient interface { 355 | Pub(ctx context.Context, in *PubRequest, opts ...grpc.CallOption) (*PubResponse, error) 356 | Sub(ctx context.Context, in *SubRequest, opts ...grpc.CallOption) (MQ_SubClient, error) 357 | } 358 | 359 | type mQClient struct { 360 | cc grpc.ClientConnInterface 361 | } 362 | 363 | func NewMQClient(cc grpc.ClientConnInterface) MQClient { 364 | return &mQClient{cc} 365 | } 366 | 367 | func (c *mQClient) Pub(ctx context.Context, in *PubRequest, opts ...grpc.CallOption) (*PubResponse, error) { 368 | out := new(PubResponse) 369 | err := c.cc.Invoke(ctx, "/mq.MQ/Pub", in, out, opts...) 370 | if err != nil { 371 | return nil, err 372 | } 373 | return out, nil 374 | } 375 | 376 | func (c *mQClient) Sub(ctx context.Context, in *SubRequest, opts ...grpc.CallOption) (MQ_SubClient, error) { 377 | stream, err := c.cc.NewStream(ctx, &_MQ_serviceDesc.Streams[0], "/mq.MQ/Sub", opts...) 378 | if err != nil { 379 | return nil, err 380 | } 381 | x := &mQSubClient{stream} 382 | if err := x.ClientStream.SendMsg(in); err != nil { 383 | return nil, err 384 | } 385 | if err := x.ClientStream.CloseSend(); err != nil { 386 | return nil, err 387 | } 388 | return x, nil 389 | } 390 | 391 | type MQ_SubClient interface { 392 | Recv() (*SubResponse, error) 393 | grpc.ClientStream 394 | } 395 | 396 | type mQSubClient struct { 397 | grpc.ClientStream 398 | } 399 | 400 | func (x *mQSubClient) Recv() (*SubResponse, error) { 401 | m := new(SubResponse) 402 | if err := x.ClientStream.RecvMsg(m); err != nil { 403 | return nil, err 404 | } 405 | return m, nil 406 | } 407 | 408 | // MQServer is the server API for MQ service. 409 | type MQServer interface { 410 | Pub(context.Context, *PubRequest) (*PubResponse, error) 411 | Sub(*SubRequest, MQ_SubServer) error 412 | } 413 | 414 | // UnimplementedMQServer can be embedded to have forward compatible implementations. 415 | type UnimplementedMQServer struct { 416 | } 417 | 418 | func (*UnimplementedMQServer) Pub(context.Context, *PubRequest) (*PubResponse, error) { 419 | return nil, status.Errorf(codes.Unimplemented, "method Pub not implemented") 420 | } 421 | func (*UnimplementedMQServer) Sub(*SubRequest, MQ_SubServer) error { 422 | return status.Errorf(codes.Unimplemented, "method Sub not implemented") 423 | } 424 | 425 | func RegisterMQServer(s *grpc.Server, srv MQServer) { 426 | s.RegisterService(&_MQ_serviceDesc, srv) 427 | } 428 | 429 | func _MQ_Pub_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 430 | in := new(PubRequest) 431 | if err := dec(in); err != nil { 432 | return nil, err 433 | } 434 | if interceptor == nil { 435 | return srv.(MQServer).Pub(ctx, in) 436 | } 437 | info := &grpc.UnaryServerInfo{ 438 | Server: srv, 439 | FullMethod: "/mq.MQ/Pub", 440 | } 441 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 442 | return srv.(MQServer).Pub(ctx, req.(*PubRequest)) 443 | } 444 | return interceptor(ctx, in, info, handler) 445 | } 446 | 447 | func _MQ_Sub_Handler(srv interface{}, stream grpc.ServerStream) error { 448 | m := new(SubRequest) 449 | if err := stream.RecvMsg(m); err != nil { 450 | return err 451 | } 452 | return srv.(MQServer).Sub(m, &mQSubServer{stream}) 453 | } 454 | 455 | type MQ_SubServer interface { 456 | Send(*SubResponse) error 457 | grpc.ServerStream 458 | } 459 | 460 | type mQSubServer struct { 461 | grpc.ServerStream 462 | } 463 | 464 | func (x *mQSubServer) Send(m *SubResponse) error { 465 | return x.ServerStream.SendMsg(m) 466 | } 467 | 468 | var _MQ_serviceDesc = grpc.ServiceDesc{ 469 | ServiceName: "mq.MQ", 470 | HandlerType: (*MQServer)(nil), 471 | Methods: []grpc.MethodDesc{ 472 | { 473 | MethodName: "Pub", 474 | Handler: _MQ_Pub_Handler, 475 | }, 476 | }, 477 | Streams: []grpc.StreamDesc{ 478 | { 479 | StreamName: "Sub", 480 | Handler: _MQ_Sub_Handler, 481 | ServerStreams: true, 482 | }, 483 | }, 484 | Metadata: "proto/mq.proto", 485 | } 486 | -------------------------------------------------------------------------------- /proto/mq.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package mq; 4 | 5 | option go_package = "./proto;mq"; 6 | 7 | service MQ { 8 | rpc Pub(PubRequest) returns (PubResponse) {} 9 | rpc Sub(SubRequest) returns (stream SubResponse) {} 10 | } 11 | 12 | message PubRequest { 13 | string topic = 1; 14 | bytes payload = 2; 15 | } 16 | 17 | message PubResponse { 18 | } 19 | 20 | message SubRequest { 21 | string topic = 1; 22 | } 23 | 24 | message SubResponse { 25 | bytes payload = 1; 26 | } 27 | -------------------------------------------------------------------------------- /server/grpc/grpc.go: -------------------------------------------------------------------------------- 1 | package grpc 2 | 3 | import ( 4 | "net" 5 | 6 | mq "github.com/asim/mq/proto" 7 | "github.com/asim/mq/server" 8 | "github.com/asim/mq/server/util" 9 | "google.golang.org/grpc" 10 | "google.golang.org/grpc/credentials" 11 | ) 12 | 13 | type grpcServer struct { 14 | options *server.Options 15 | srv *grpc.Server 16 | } 17 | 18 | func (g *grpcServer) Run() error { 19 | l, err := net.Listen("tcp", g.options.Address) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | var opts []grpc.ServerOption 25 | 26 | // tls enabled 27 | if g.options.TLS != nil { 28 | creds, err := credentials.NewServerTLSFromFile( 29 | g.options.TLS.CertFile, 30 | g.options.TLS.KeyFile, 31 | ) 32 | if err != nil { 33 | return err 34 | } 35 | opts = append(opts, grpc.Creds(creds)) 36 | } else { 37 | // generate tls config 38 | addr, err := util.Address(g.options.Address) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | cert, err := util.Certificate(addr) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | creds := credentials.NewServerTLSFromCert(&cert) 49 | opts = append(opts, grpc.Creds(creds)) 50 | } 51 | 52 | // new grpc server 53 | srv := grpc.NewServer(opts...) 54 | g.srv = srv 55 | 56 | // register MQ server 57 | mq.RegisterMQServer(srv, new(handler)) 58 | 59 | // serve 60 | return srv.Serve(l) 61 | } 62 | 63 | func (g *grpcServer) Stop() error { 64 | g.srv.GracefulStop() 65 | return nil 66 | 67 | } 68 | 69 | func New(opts ...server.Option) *grpcServer { 70 | options := new(server.Options) 71 | for _, o := range opts { 72 | o(options) 73 | } 74 | return &grpcServer{ 75 | options: options, 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /server/grpc/handler.go: -------------------------------------------------------------------------------- 1 | package grpc 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/asim/mq/broker" 7 | "github.com/asim/mq/proto" 8 | "golang.org/x/net/context" 9 | ) 10 | 11 | type handler struct{} 12 | 13 | func (h *handler) Pub(ctx context.Context, req *mq.PubRequest) (*mq.PubResponse, error) { 14 | if err := broker.Publish(req.Topic, req.Payload); err != nil { 15 | return nil, fmt.Errorf("pub error: %v", err) 16 | } 17 | return new(mq.PubResponse), nil 18 | } 19 | 20 | func (h *handler) Sub(req *mq.SubRequest, stream mq.MQ_SubServer) error { 21 | ch, err := broker.Subscribe(req.Topic) 22 | if err != nil { 23 | return fmt.Errorf("could not subscribe: %v", err) 24 | } 25 | defer broker.Unsubscribe(req.Topic, ch) 26 | 27 | for p := range ch { 28 | if err := stream.Send(&mq.SubResponse{Payload: p}); err != nil { 29 | return fmt.Errorf("failed to send payload: %v", err) 30 | } 31 | } 32 | 33 | return nil 34 | } 35 | -------------------------------------------------------------------------------- /server/http/handler.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | 8 | "github.com/asim/mq/broker" 9 | "github.com/gorilla/websocket" 10 | ) 11 | 12 | var upgrader = websocket.Upgrader{ 13 | ReadBufferSize: 1024, 14 | WriteBufferSize: 1024, 15 | CheckOrigin: func(r *http.Request) bool { 16 | return true 17 | }, 18 | } 19 | 20 | func pub(w http.ResponseWriter, r *http.Request) { 21 | topic := r.URL.Query().Get("topic") 22 | 23 | if websocket.IsWebSocketUpgrade(r) { 24 | conn, err := upgrader.Upgrade(w, r, w.Header()) 25 | if err != nil { 26 | return 27 | } 28 | for { 29 | messageType, b, err := conn.ReadMessage() 30 | if messageType == -1 { 31 | return 32 | } 33 | if err != nil { 34 | continue 35 | } 36 | broker.Publish(topic, b) 37 | } 38 | } else { 39 | b, err := ioutil.ReadAll(r.Body) 40 | if err != nil { 41 | http.Error(w, "Pub error", http.StatusInternalServerError) 42 | return 43 | } 44 | r.Body.Close() 45 | if err := broker.Publish(topic, b); err != nil { 46 | http.Error(w, "Pub error", http.StatusInternalServerError) 47 | } 48 | } 49 | } 50 | 51 | func sub(w http.ResponseWriter, r *http.Request) { 52 | var wr writer 53 | 54 | if websocket.IsWebSocketUpgrade(r) { 55 | conn, err := upgrader.Upgrade(w, r, w.Header()) 56 | if err != nil { 57 | return 58 | } 59 | // Drain the websocket so that we handle pings and connection close 60 | go func(c *websocket.Conn) { 61 | for { 62 | if _, _, err := c.NextReader(); err != nil { 63 | c.Close() 64 | break 65 | } 66 | } 67 | }(conn) 68 | wr = &wsWriter{conn} 69 | } else { 70 | wr = &httpWriter{w} 71 | } 72 | 73 | topic := r.URL.Query().Get("topic") 74 | 75 | ch, err := broker.Subscribe(topic) 76 | if err != nil { 77 | http.Error(w, fmt.Sprintf("Could not retrieve events: %v", err), http.StatusInternalServerError) 78 | return 79 | } 80 | defer broker.Unsubscribe(topic, ch) 81 | 82 | for { 83 | select { 84 | case e := <-ch: 85 | if err = wr.Write(e); err != nil { 86 | return 87 | } 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /server/http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "net/http" 7 | "os" 8 | 9 | "github.com/asim/mq/server" 10 | "github.com/asim/mq/server/util" 11 | "github.com/gorilla/handlers" 12 | ) 13 | 14 | type httpServer struct { 15 | options *server.Options 16 | srv *http.Server 17 | } 18 | 19 | func (h *httpServer) Run() error { 20 | // MQ Handlers 21 | http.HandleFunc("/pub", pub) 22 | http.HandleFunc("/sub", sub) 23 | 24 | // logging handler 25 | handler := handlers.LoggingHandler(os.Stdout, http.DefaultServeMux) 26 | address := h.options.Address 27 | 28 | // tls cert and key specified 29 | if h.options.TLS != nil { 30 | cert := h.options.TLS.CertFile 31 | key := h.options.TLS.KeyFile 32 | return http.ListenAndServeTLS(address, cert, key, handler) 33 | } 34 | 35 | // generate tls config 36 | addr, err := util.Address(address) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | cert, err := util.Certificate(addr) 42 | if err != nil { 43 | return err 44 | } 45 | 46 | config := &tls.Config{ 47 | Certificates: []tls.Certificate{cert}, 48 | NextProtos: []string{"h2", "http/1.1"}, 49 | } 50 | 51 | l, err := tls.Listen("tcp", address, config) 52 | if err != nil { 53 | return err 54 | } 55 | defer l.Close() 56 | 57 | srv := &http.Server{ 58 | Addr: address, 59 | Handler: handler, 60 | TLSConfig: config, 61 | } 62 | 63 | h.srv = srv 64 | 65 | return srv.Serve(l) 66 | } 67 | 68 | func (h *httpServer) Stop() error { 69 | return h.srv.Shutdown(context.TODO()) 70 | } 71 | 72 | func New(opts ...server.Option) *httpServer { 73 | options := new(server.Options) 74 | for _, o := range opts { 75 | o(options) 76 | } 77 | return &httpServer{ 78 | options: options, 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /server/http/writer.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/gorilla/websocket" 7 | ) 8 | 9 | type writer interface { 10 | Write(b []byte) error 11 | } 12 | 13 | type httpWriter struct { 14 | w http.ResponseWriter 15 | } 16 | 17 | type wsWriter struct { 18 | conn *websocket.Conn 19 | } 20 | 21 | func (w *httpWriter) Write(b []byte) error { 22 | if _, err := w.w.Write(b); err != nil { 23 | return err 24 | } 25 | if f, ok := w.w.(http.Flusher); ok { 26 | f.Flush() 27 | } 28 | return nil 29 | } 30 | 31 | func (w *wsWriter) Write(b []byte) error { 32 | return w.conn.WriteMessage(websocket.BinaryMessage, b) 33 | } 34 | -------------------------------------------------------------------------------- /server/options.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | type Options struct { 4 | Address string 5 | TLS *TLS 6 | } 7 | 8 | type TLS struct { 9 | CertFile string 10 | KeyFile string 11 | } 12 | 13 | type Option func(o *Options) 14 | 15 | func WithAddress(addr string) Option { 16 | return func(o *Options) { 17 | o.Address = addr 18 | } 19 | } 20 | 21 | func WithTLS(certFile, keyFile string) Option { 22 | return func(o *Options) { 23 | o.TLS = &TLS{ 24 | CertFile: certFile, 25 | KeyFile: keyFile, 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | type Server interface { 4 | Run() error 5 | Stop() error 6 | } 7 | -------------------------------------------------------------------------------- /server/util/address.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | ) 7 | 8 | var ( 9 | privateBlocks []*net.IPNet 10 | ) 11 | 12 | func init() { 13 | for _, b := range []string{"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"} { 14 | if _, block, err := net.ParseCIDR(b); err == nil { 15 | privateBlocks = append(privateBlocks, block) 16 | } 17 | } 18 | } 19 | 20 | func Address(addr string) (string, error) { 21 | if len(addr) > 0 && (addr != "0.0.0.0" && addr != "[::]") { 22 | return addr, nil 23 | } 24 | 25 | addrs, err := net.InterfaceAddrs() 26 | if err != nil { 27 | return "", fmt.Errorf("Failed to get interface addresses! Err: %v", err) 28 | } 29 | 30 | var ipAddr []byte 31 | 32 | for _, rawAddr := range addrs { 33 | var ip net.IP 34 | switch addr := rawAddr.(type) { 35 | case *net.IPAddr: 36 | ip = addr.IP 37 | case *net.IPNet: 38 | ip = addr.IP 39 | default: 40 | continue 41 | } 42 | 43 | if ip.To4() == nil { 44 | continue 45 | } 46 | 47 | if !isPrivateIP(ip.String()) { 48 | continue 49 | } 50 | 51 | ipAddr = ip 52 | break 53 | } 54 | 55 | if ipAddr == nil { 56 | return "", fmt.Errorf("No private IP address found, and explicit IP not provided") 57 | } 58 | 59 | return net.IP(ipAddr).String(), nil 60 | } 61 | 62 | func isPrivateIP(ipAddr string) bool { 63 | ip := net.ParseIP(ipAddr) 64 | for _, priv := range privateBlocks { 65 | if priv.Contains(ip) { 66 | return true 67 | } 68 | } 69 | return false 70 | } 71 | -------------------------------------------------------------------------------- /server/util/certificate.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bytes" 5 | "crypto/ecdsa" 6 | "crypto/elliptic" 7 | "crypto/rand" 8 | "crypto/tls" 9 | "crypto/x509" 10 | "crypto/x509/pkix" 11 | "encoding/pem" 12 | "math/big" 13 | "net" 14 | "time" 15 | ) 16 | 17 | func Certificate(host ...string) (tls.Certificate, error) { 18 | priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) 19 | if err != nil { 20 | return tls.Certificate{}, err 21 | } 22 | 23 | notBefore := time.Now() 24 | notAfter := notBefore.Add(time.Hour * 24 * 365) 25 | 26 | serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) 27 | serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) 28 | if err != nil { 29 | return tls.Certificate{}, err 30 | } 31 | 32 | template := x509.Certificate{ 33 | SerialNumber: serialNumber, 34 | Subject: pkix.Name{ 35 | Organization: []string{"Acme Co"}, 36 | }, 37 | NotBefore: notBefore, 38 | NotAfter: notAfter, 39 | 40 | KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, 41 | ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, 42 | BasicConstraintsValid: true, 43 | } 44 | 45 | for _, h := range host { 46 | if ip := net.ParseIP(h); ip != nil { 47 | template.IPAddresses = append(template.IPAddresses, ip) 48 | } else { 49 | template.DNSNames = append(template.DNSNames, h) 50 | } 51 | } 52 | 53 | template.IsCA = true 54 | template.KeyUsage |= x509.KeyUsageCertSign 55 | 56 | derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) 57 | if err != nil { 58 | return tls.Certificate{}, err 59 | } 60 | 61 | // create public key 62 | certOut := bytes.NewBuffer(nil) 63 | pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) 64 | 65 | // create private key 66 | keyOut := bytes.NewBuffer(nil) 67 | b, err := x509.MarshalECPrivateKey(priv) 68 | if err != nil { 69 | return tls.Certificate{}, err 70 | } 71 | pem.Encode(keyOut, &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}) 72 | 73 | return tls.X509KeyPair(certOut.Bytes(), keyOut.Bytes()) 74 | } 75 | --------------------------------------------------------------------------------