├── .idea
├── codeStyles
│ └── codeStyleConfig.xml
├── dictionaries
│ └── Hung.xml
├── dqueue.iml
├── misc.xml
├── modules.xml
├── vcs.xml
└── workspace.xml
├── README.md
├── conf
└── config.json
├── database
├── buntdb.go
├── database.go
└── mysql.go
├── diagram
└── diagram.jpg
├── example_client.go
├── handler
├── message.go
└── service.go
├── helper
├── common.go
└── server.go
├── main.go
├── models
├── data.go
├── memory.go
├── message.go
└── mysql.go
├── protobuf
├── build.sh
├── message.micro.go
├── message.pb.go
├── message.proto
├── service.micro.go
├── service.pb.go
└── service.proto
├── queue
├── kafka.go
├── nats.go
├── nats_streaming.go
├── queue.go
└── stan.go
├── worker
└── worker.go
└── wrapper
└── client.go
/.idea/codeStyles/codeStyleConfig.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
--------------------------------------------------------------------------------
/.idea/dictionaries/Hung.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/.idea/dqueue.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 | evn
124 |
125 |
126 | $PROJECT_DIR$/protobuf
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
153 |
154 |
155 |
156 |
157 | true
158 | DEFINITION_ORDER
159 |
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
204 |
205 |
206 |
207 |
208 |
209 |
210 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
223 |
224 |
225 |
226 |
227 |
228 |
229 |
230 |
231 |
232 |
233 |
234 |
235 |
236 |
237 |
238 |
239 |
240 |
241 |
242 |
243 |
244 |
245 |
246 |
247 |
248 |
249 |
250 |
251 |
252 |
253 |
254 |
255 |
256 |
257 |
258 |
259 |
260 |
261 |
262 |
263 |
264 |
265 |
266 |
267 |
268 |
269 |
270 |
271 |
272 |
273 |
274 |
275 |
276 |
277 |
278 |
279 |
280 |
281 |
282 |
283 |
284 |
285 |
286 |
287 |
288 |
289 |
290 |
291 |
292 |
293 |
294 |
295 |
296 |
297 |
298 |
299 |
300 |
301 |
302 |
303 |
304 |
305 |
306 |
307 |
308 |
309 |
310 |
311 |
312 |
313 |
314 |
315 |
316 |
317 |
318 |
319 |
320 |
321 |
322 |
323 |
324 |
325 |
326 |
327 |
328 |
329 |
330 |
331 |
332 |
333 |
334 |
335 |
336 |
337 |
338 |
339 |
340 |
341 |
342 |
343 |
344 |
345 |
346 |
347 |
348 |
349 |
350 |
351 |
352 |
353 |
354 |
355 |
356 |
357 |
358 |
359 |
360 |
361 |
362 |
363 |
364 |
365 |
366 |
367 |
368 |
369 |
370 |
371 |
372 |
373 |
374 |
375 |
376 |
377 |
378 |
379 |
380 |
381 |
382 |
383 |
384 |
385 |
386 |
387 |
388 |
389 |
390 |
391 |
392 |
393 |
394 |
395 |
396 |
397 |
398 |
399 |
400 |
401 |
402 |
403 |
404 |
405 |
406 |
407 |
408 |
409 |
410 |
411 |
412 |
413 |
414 |
415 |
416 |
417 |
418 |
419 |
420 |
421 |
422 |
423 |
424 |
425 |
426 |
427 |
428 |
429 |
430 |
431 |
432 |
433 |
434 |
435 |
436 |
437 |
438 |
439 |
440 |
441 |
442 |
443 |
444 |
445 |
446 |
447 |
448 |
449 |
450 |
451 |
452 |
453 |
454 |
455 |
456 |
457 |
458 |
459 |
460 |
461 |
462 |
463 |
464 |
465 |
466 |
467 |
468 |
469 |
470 |
471 |
472 |
473 |
474 |
475 |
476 |
477 |
478 |
479 |
480 |
481 |
482 |
483 |
484 |
485 |
486 |
487 |
488 |
489 |
490 |
491 |
492 |
493 |
494 |
495 |
496 |
497 |
498 |
499 |
500 |
501 |
502 |
503 |
504 |
505 |
506 |
507 |
508 |
509 |
510 |
511 |
512 |
513 |
514 |
515 |
516 |
517 |
518 |
519 |
520 |
521 |
522 |
523 |
524 |
525 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # dqueue
2 |
3 | dqueue is a delay queue written in Golang. dqueue was born because the need of a simple queuing layer which support delay before processing message.
4 | A message pushed into dqueue will be stay in dqueue (which stored in database - currently mysql) until delay time reached. Delay time is client defined, in second. Message has following format:
5 |
6 | ```json
7 | {
8 | "delay": integer,
9 | "data": string,
10 | "retryCount": integer
11 | }
12 | ```
13 |
14 | For example:
15 |
16 | ```json
17 | {
18 | "delay": 1,
19 | "data": "hello world",
20 | "retryCount": 1
21 | }
22 | ```
23 |
24 | dqueue will delay message for `1 * 1 = 1 second`, then push to message queue backend defined in configuration file, directive `queueType`. Then if subscriber fail to process message, eq: fail to store to database, fail to call or update an API, subscriber can then push back message into dqueue with `retryCount = currentRetryCount + 1`, message will be delay `1 * 2 = 2 seconds` before published to queue backend, and so on
25 |
26 | ```json
27 | {
28 | "delay": 1,
29 | "data": "hello world",
30 | "retryCount": 2
31 | }
32 | ```
33 |
34 | # Architecture
35 |
36 | 
37 |
38 | dqueue is written in golang, using go-micro framework, you can start as much as possible instance to scale dqueue. Client can also use go-micro client to work with dqueue server (include in this project)
39 |
40 | Currently dqueue uses mysql as intermediate layer, and support multiple message queue system: nats, nats streaming, kafka
41 |
42 | One can extend dqueue by adding other message queue system by implement `queue` interface:
43 |
44 | ```go
45 | package queue
46 |
47 | type Queue interface {
48 | PublishMessage([]byte) error
49 | }
50 | ```
51 |
52 | see nats for example.
53 |
54 | One can also extend dqueue to support other intermediate database by implement `database` interface:
55 |
56 | ```go
57 | type Database interface {
58 | Init() error
59 | }
60 | ```
61 |
62 | dqueue also has apis to support listing, monitoring, and forcing a message to be in queue immediately using `force` method
63 |
64 |
65 | # usage
66 |
67 | Install dqueue
68 |
69 | ```bash
70 | go get github.com/whatvn/dqueue
71 | cd $GOPATH/src/github.com/whatvn/dqueue
72 | go build main.go
73 | ```
74 |
75 | change configuration file according to your system:
76 |
77 | ```json
78 | {
79 | "queueType": "NATS",
80 | "dbType": "mysql",
81 | "hosts": {
82 | "nats": {
83 | "address": "0.0.0.0",
84 | "port": "4222"
85 | },
86 | "stan": {
87 | "address": "0.0.0.0",
88 | "port": "4222",
89 | "clusterID": "test-cluster",
90 | "clientID": "retry-worker"
91 | },
92 | "mysql": {
93 | "address": "127.0.0.1",
94 | "port": "3306",
95 | "user": "root",
96 | "password": "123456",
97 | "database": "delay_queue"
98 | }
99 | }
100 | }
101 | ```
102 | where:
103 |
104 | - **queueType** is queue server backend
105 | - **dbType** is intermediate database server, currenly `mysql` and `memory` support
106 |
107 |
108 | To publish a message into dqueue
109 | ```go
110 |
111 | package main
112 |
113 | import (
114 | "context"
115 | "log"
116 | "time"
117 |
118 | "github.com/whatvn/dqueue/protobuf"
119 | "github.com/whatvn/dqueue/wrapper"
120 | )
121 |
122 | func main() {
123 | client := wrapper.NewDelayQueueClient()
124 | message := &delayQueue.QueueRequest{
125 | Messsage: "hello world",
126 | RetryCount: 3,
127 | Delay: 1,
128 | }
129 | ctx := context.Background()
130 | for {
131 | resp, err := client.Publish(ctx, message)
132 | time.Sleep(1 * time.Second)
133 | log.Println(resp, err)
134 | }
135 | }
136 |
137 | ```
138 |
139 | To subscribe a message from your queue backend, use queue backend client
140 | # License
141 |
142 | BSD License
--------------------------------------------------------------------------------
/conf/config.json:
--------------------------------------------------------------------------------
1 | {
2 | "queueType": "NATS",
3 | "dbType": "mysql",
4 | "hosts": {
5 | "nats": {
6 | "address": "0.0.0.0",
7 | "port": "4222"
8 | },
9 | "stan": {
10 | "address": "0.0.0.0",
11 | "port": "4222",
12 | "clusterID": "test-cluster",
13 | "clientID": "retry-worker"
14 | },
15 | "mysql": {
16 | "address": "127.0.0.1",
17 | "port": "3306",
18 | "user": "root",
19 | "password": "123456",
20 | "database": "delay_queue"
21 | }
22 | }
23 | }
--------------------------------------------------------------------------------
/database/buntdb.go:
--------------------------------------------------------------------------------
1 | package database
2 |
3 | import (
4 | "github.com/tidwall/buntdb"
5 | "fmt"
6 | )
7 |
8 | type MemoryDb struct {
9 | Engine *buntdb.DB
10 | }
11 |
12 | func NewMemDb() *MemoryDb {
13 | mdb := &MemoryDb{}
14 | err := mdb.Init()
15 | if err != nil {
16 | panic(err)
17 | }
18 | return mdb
19 | }
20 |
21 | func (db *MemoryDb) Init() error {
22 |
23 | bdb, err := buntdb.Open(":memory:")
24 | if err != nil {
25 | return err
26 | }
27 | err = bdb.Update(func(tx *buntdb.Tx) error {
28 | return tx.CreateIndex("ts", "*", buntdb.IndexJSON("timestamp"))
29 | })
30 | db.Engine = bdb
31 | return err
32 | }
33 |
34 | func (db *MemoryDb) Insert(id int64, data string) error {
35 | ids := fmt.Sprintf("%d", id)
36 | return db.Engine.Update(func(tx *buntdb.Tx) error {
37 | _, _, err := tx.Set(ids, string(data), nil)
38 | return err
39 | })
40 | }
41 |
42 | func (db *MemoryDb) Delete(id int64) error {
43 | fmt.Println("delete id", id)
44 | ids := fmt.Sprintf("%d", id)
45 | return db.Engine.Update(func(tx *buntdb.Tx) error {
46 | fmt.Println("delete id", id)
47 | _, err := tx.Delete(ids)
48 | fmt.Println("delete error: ", err)
49 | return err
50 | })
51 | }
52 |
--------------------------------------------------------------------------------
/database/database.go:
--------------------------------------------------------------------------------
1 | package database
2 |
3 | import (
4 | "github.com/whatvn/dqueue/helper"
5 | "errors"
6 | )
7 |
8 | type Database interface {
9 | Init() error
10 | }
11 |
12 | func NewDatabase() Database {
13 | dbType := helper.GetDbType()
14 | switch dbType {
15 | case "mysql":
16 | return newMySqlDatabase()
17 | default:
18 | panic(errors.New("not implement"))
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/database/mysql.go:
--------------------------------------------------------------------------------
1 | package database
2 |
3 | import (
4 | "github.com/astaxie/beego/orm"
5 | "time"
6 | "os"
7 | "strconv"
8 | "github.com/whatvn/dqueue/helper"
9 | _ "github.com/go-sql-driver/mysql"
10 | log "github.com/golang/glog"
11 | "fmt"
12 | )
13 |
14 | type mysql struct {}
15 |
16 | func (database *mysql) Init() error {
17 | var mysql DbConfig
18 | helper.Config(&mysql, "hosts", "mysql")
19 | return initialize(&DbConfig{
20 | User: mysql.User,
21 | Password: mysql.Password,
22 | Address: mysql.Address,
23 | Database: mysql.Database,
24 | })
25 | }
26 |
27 | func newMySqlDatabase() Database {
28 | return &mysql{}
29 | }
30 |
31 | type DbConfig struct {
32 | User string
33 | Password string
34 | Address string
35 | Database string
36 | }
37 |
38 | var (
39 | maxIdle = 100
40 | maxConn = 200
41 | lifeTime = 300
42 | )
43 |
44 |
45 | func initialize(conf *DbConfig) error {
46 | var connectString = fmt.Sprintf(
47 | "%s:%s@tcp(%s)/%s?charset=utf8&collation=utf8mb4_general_ci&loc=%s",
48 | conf.User, conf.Password, conf.Address, conf.Database, "Asia%2fBangkok")
49 |
50 | orm.RegisterDriver("mysql", orm.DRMySQL)
51 | log.Info("connect database with connect string: ", connectString)
52 |
53 | if err := orm.RegisterDataBase("default", "mysql",
54 | connectString, maxIdle, maxConn); err != nil {
55 | log.Info("error when RegisterDataBase: ", err)
56 | return err
57 | }
58 | //
59 | db, err := orm.GetDB("default")
60 | if err != nil {
61 | log.Fatal("get default DB error:" + err.Error())
62 | return err
63 | }
64 | db.SetConnMaxLifetime(time.Duration(lifeTime) * time.Second)
65 | orm.Debug, orm.DebugLog = false, orm.NewLog(os.Stdout)
66 | if dbMigrate := os.Getenv("DB_MIGRATE"); len(dbMigrate) == 0 {
67 | log.Info("init database without migrate, config: ", connectString)
68 | } else {
69 | if isMigrate, _ := strconv.ParseBool(dbMigrate); isMigrate {
70 | if err := orm.RunSyncdb("default", false, false); err != nil {
71 | log.Info("error when migrate db", err)
72 | return err
73 | }
74 | log.Info("init database migrate mode, config: ", connectString)
75 | }
76 | }
77 | return nil
78 | }
79 |
--------------------------------------------------------------------------------
/diagram/diagram.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/whatvn/dqueue/de61991f4300408540d5c5794601d79d89596f1e/diagram/diagram.jpg
--------------------------------------------------------------------------------
/example_client.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "context"
5 | "log"
6 | "time"
7 |
8 | "github.com/whatvn/dqueue/protobuf"
9 | "github.com/whatvn/dqueue/wrapper"
10 | "flag"
11 | "github.com/spf13/cast"
12 | )
13 |
14 | func main() {
15 | flag.Set("logtostderr", "true")
16 | flag.Set("v", "2")
17 | flag.Parse()
18 | client := wrapper.NewDelayQueueClient()
19 |
20 | i := 1
21 | for {
22 | message := &delayQueue.QueueRequest{
23 | Messsage: "hello world" + cast.ToString(i),
24 | RetryCount: 3,
25 | Delay: 3,
26 | }
27 | ctx := context.Background()
28 | resp, err := client.Publish(ctx, message)
29 | i += 1
30 | time.Sleep(100 * time.Millisecond)
31 | log.Println(resp, err)
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/handler/message.go:
--------------------------------------------------------------------------------
1 | package handler
2 |
3 | import (
4 | "strconv"
5 |
6 | log "github.com/golang/glog"
7 | "github.com/whatvn/dqueue/models"
8 | "github.com/whatvn/dqueue/protobuf"
9 | "github.com/gin-gonic/gin"
10 | )
11 |
12 | type MessageHandler struct{}
13 |
14 |
15 | func (md *MessageHandler) GetAllMessages(c *gin.Context) {
16 | log.Info("Received Get all messages request")
17 | var (
18 | response = &delayQueue.QueryListMessagesResp{}
19 | )
20 |
21 | msgList, err := message.All()
22 | if err != nil {
23 | log.Error("cannot get all message, error: ", err)
24 | response.ReturnCode = message.Fail
25 | c.JSON(200, response)
26 | }
27 |
28 | log.Info("message list", msgList)
29 |
30 | for _, msg := range msgList {
31 | response.MsgList = append(response.MsgList, &delayQueue.MessageData{
32 | TimeStamp: msg.TimeStamp,
33 | Delay: int32(msg.Delay),
34 | RetryCount: int32(msg.RetryCount),
35 | Data: msg.Data,
36 | })
37 | }
38 |
39 | response.ReturnCode = message.Success
40 | response.Message = message.ErrorMessage(message.Success)
41 |
42 |
43 | log.Info("response", response.ReturnCode)
44 |
45 | c.JSON(200, response)
46 | }
47 |
48 | func (md *MessageHandler) GetListMessage(c *gin.Context) {
49 | log.Info("Received Get list message request")
50 |
51 | offset, _ := strconv.Atoi(c.Param("offset"))
52 | limit, _ := strconv.Atoi(c.Param("limit"))
53 |
54 | var (
55 | response = &delayQueue.QueryListMessagesResp{}
56 | )
57 |
58 | msgList, err := message.List(offset, limit)
59 | if err != nil {
60 | log.Error("cannot get message list, error: ", err)
61 | response.ReturnCode = message.Fail
62 | c.JSON(200, response)
63 | }
64 |
65 | for _, msg := range msgList {
66 | response.MsgList = append(response.MsgList, &delayQueue.MessageData{
67 | TimeStamp: msg.TimeStamp,
68 | Delay: int32(msg.Delay),
69 | RetryCount: int32(msg.RetryCount),
70 | Data: msg.Data,
71 | })
72 | }
73 |
74 | response.ReturnCode = message.Success
75 |
76 | c.JSON(200, response)
77 | }
78 |
79 | func (md *MessageHandler) GetListMessageByData(c *gin.Context) {
80 | log.Info("Received Get list message by data request")
81 | data := c.Param("data")
82 |
83 | log.Info("data", data)
84 |
85 | var (
86 | response = &delayQueue.QueryListMessagesResp{}
87 | )
88 |
89 | msgList, err := message.SearchBy(data)
90 | if err != nil {
91 | log.Error("cannot get message by pattern: ", data, "error: ", err)
92 | response.ReturnCode = message.Fail
93 | c.JSON(200, response)
94 | return
95 | }
96 |
97 | for _, msg := range msgList {
98 | response.MsgList = append(response.MsgList, &delayQueue.MessageData{
99 | TimeStamp: msg.TimeStamp,
100 | Delay: int32(msg.Delay),
101 | RetryCount: int32(msg.RetryCount),
102 | Data: msg.Data,
103 | })
104 | }
105 |
106 | response.ReturnCode = message.Success
107 |
108 | c.JSON(200, response)
109 | }
110 |
111 | func (md *MessageHandler) ForceMessage(c *gin.Context) {
112 | log.Info("Received Get force message request")
113 |
114 | var (
115 | response = &delayQueue.ReturnCommon{}
116 | msg message.Message
117 | )
118 |
119 | msgId, err := strconv.Atoi(c.Param("id"))
120 | if err != nil {
121 | response.ReturnCode = message.Fail
122 | goto Done
123 | }
124 | msg = message.NewMessageById(int64(msgId))
125 | err = msg.Force()
126 | if err != nil {
127 | log.Error("cannot update message timestamp, error: ", err)
128 | response.ReturnCode = message.Fail
129 | goto Done
130 | }
131 | response.ReturnCode = message.Success
132 |
133 | Done:
134 | c.JSON(200, response)
135 | }
136 |
--------------------------------------------------------------------------------
/handler/service.go:
--------------------------------------------------------------------------------
1 | package handler
2 |
3 | import (
4 | "context"
5 |
6 | "github.com/whatvn/dqueue/models"
7 | "github.com/whatvn/dqueue/protobuf"
8 | log "github.com/golang/glog"
9 | )
10 |
11 | type MicroHandler struct {
12 | }
13 |
14 | // NewMicroServiceHandler
15 | func NewMicroServiceHandler() delayQueue.DelayQueueHandler {
16 | return &MicroHandler{}
17 | }
18 |
19 | func (handler *MicroHandler) Publish(ctx context.Context, request *delayQueue.QueueRequest, response *delayQueue.QueueResponse) error {
20 | msg := message.NewMessage(request)
21 | _, err := msg.Save()
22 | if err != nil {
23 | log.Error("cannot add message to database, message: ", msg, "error: ", err)
24 | response.ReturnCode = message.Fail
25 | response.Message = message.ErrorMessage(message.Fail)
26 | return nil
27 | }
28 |
29 | response.ReturnCode = message.Success
30 | response.Message = message.ErrorMessage(message.Success)
31 | return nil
32 | }
33 |
--------------------------------------------------------------------------------
/helper/common.go:
--------------------------------------------------------------------------------
1 | package helper
2 |
3 | import (
4 | "time"
5 |
6 | config "github.com/micro/go-config"
7 | "github.com/micro/go-config/source/file"
8 | log "github.com/golang/glog"
9 | )
10 |
11 |
12 |
13 | func Now() int64 {
14 | loc, err := time.LoadLocation("Asia/Bangkok")
15 | if err != nil {
16 | panic(err)
17 | }
18 | return time.Now().In(loc).Unix()
19 | }
20 |
21 |
22 | func NowNano() int64 {
23 | loc, err := time.LoadLocation("Asia/Bangkok")
24 | if err != nil {
25 | panic(err)
26 | }
27 | return time.Now().In(loc).UnixNano()
28 | }
29 |
30 | func NowPlus(extraSecond int32) int64 {
31 | return Now() + int64(extraSecond)
32 | }
33 |
34 |
35 | func Config(v interface{}, params ...string) {
36 | config.Load(file.NewSource(
37 | file.WithPath("conf/config.json"),
38 | ))
39 | err := config.Get(params...).Scan(v)
40 | if err != nil {
41 | log.Error("configuration error: ", err)
42 | panic(err)
43 | }
44 | }
45 |
46 | func GetQueueType() string {
47 | config.Load(file.NewSource(
48 | file.WithPath("conf/config.json"),
49 | ))
50 | queueType := config.Get("queueType").String("")
51 | return queueType
52 | }
53 |
54 |
55 | func GetDbType() string {
56 | config.Load(file.NewSource(
57 | file.WithPath("conf/config.json"),
58 | ))
59 | dbType := config.Get("dbType").String("")
60 | return dbType
61 | }
62 |
--------------------------------------------------------------------------------
/helper/server.go:
--------------------------------------------------------------------------------
1 | package helper
2 |
3 | import (
4 | "time"
5 |
6 | "context"
7 |
8 | "github.com/micro/go-grpc"
9 | "github.com/micro/go-micro"
10 | "github.com/micro/go-micro/registry"
11 | micro_server "github.com/micro/go-micro/server"
12 | opWrapper "github.com/micro/go-plugins/wrapper/trace/opentracing"
13 | "github.com/opentracing/opentracing-go"
14 | opConfig "github.com/uber/jaeger-client-go/config"
15 | "github.com/uber/jaeger-client-go/rpcmetrics"
16 | "github.com/uber/jaeger-lib/metrics/expvar"
17 | log "github.com/golang/glog"
18 | )
19 |
20 | type server struct {
21 | tracer opentracing.Tracer
22 | service micro.Service
23 | }
24 |
25 | //NewServer return server wrapper for go-micro ecosystem
26 | func NewServer(domain string, consulAddr string, opAddress string) *server {
27 | s := server{}
28 | s.initTracer(domain, opAddress)
29 | s.initService(domain, consulAddr)
30 | return &s
31 | }
32 |
33 | func (s *server) GetService() micro.Service { return s.service }
34 |
35 | func (s *server) initService(domain string, consulAddr string) {
36 | s.service = grpc.NewService(
37 | micro.WrapHandler(LogHandlerWrapper()),
38 | micro.WrapHandler(opWrapper.NewHandlerWrapper(opentracing.GlobalTracer())),
39 | micro.RegisterTTL(time.Second*30),
40 | micro.RegisterInterval(time.Second*15),
41 | micro.Name(domain), micro.Version("latest"),
42 | micro.Registry(registry.NewRegistry(registry.Addrs(consulAddr))),
43 | //micro.WrapHandler(ValidateHandlerWrapper()),
44 | )
45 | s.service.Server().Init(micro_server.Wait(true))
46 | s.service.Init()
47 | }
48 |
49 | func (s *server) initTracer(domain string, opAddress string) {
50 | s.tracer, _, _ = opConfig.Configuration{
51 | ServiceName: domain,
52 | Sampler: &opConfig.SamplerConfig{
53 | Type: "const",
54 | Param: 1,
55 | },
56 | Reporter: &opConfig.ReporterConfig{
57 | LogSpans: false,
58 | BufferFlushInterval: 1 * time.Second,
59 | LocalAgentHostPort: opAddress,
60 | }}.
61 | NewTracer(
62 | opConfig.Observer(rpcmetrics.NewObserver(
63 | expvar.NewFactory(10).Namespace(domain, nil),
64 | rpcmetrics.DefaultNameNormalizer)))
65 | log.Info("init tracer done")
66 | opentracing.SetGlobalTracer(s.tracer)
67 | }
68 |
69 | //LogHandlerWrapper log middleware wrapper for server
70 | func LogHandlerWrapper() micro_server.HandlerWrapper {
71 | return func(h micro_server.HandlerFunc) micro_server.HandlerFunc {
72 | return func(ctx context.Context, req micro_server.Request, rsp interface{}) error {
73 | log.Infof("request: %v\n", req.Method())
74 | return h(ctx, req, rsp)
75 | }
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 |
6 | "github.com/whatvn/dqueue/handler"
7 | "github.com/whatvn/dqueue/helper"
8 | "github.com/whatvn/dqueue/protobuf"
9 | "github.com/whatvn/dqueue/worker"
10 |
11 | "github.com/gin-gonic/gin"
12 | "github.com/micro/go-web"
13 | "flag"
14 | "os"
15 | "os/signal"
16 | "syscall"
17 | "fmt"
18 | "time"
19 | "github.com/whatvn/dqueue/database"
20 | "github.com/pkg/profile"
21 | )
22 |
23 | const (
24 | Domain = "DelayQueue"
25 | Consul = "127.0.0.1:8500"
26 | OpenTracing = "127.0.0.1:8200"
27 | )
28 |
29 | func init() {
30 | if helper.GetDbType() == "mysql" {
31 | db := database.NewDatabase()
32 | db.Init()
33 | }
34 |
35 | flag.Set("logtostderr", "true")
36 | flag.Set("v", "2")
37 | flag.Parse()
38 | }
39 |
40 | func WebServer() {
41 | // Create service
42 | apiService := web.NewService(
43 | web.Name("go.micro.api.message"),
44 | )
45 |
46 | apiService.Init()
47 |
48 | // Create RESTful handler (using Gin)
49 | messageHandler := new(handler.MessageHandler)
50 | router := gin.Default()
51 | router.POST("/message/getall", messageHandler.GetAllMessages)
52 | router.POST("/message/getlistbydata/:data", messageHandler.GetListMessageByData)
53 | router.POST("/message/force/:data", messageHandler.ForceMessage)
54 | router.POST("/message/getlist/:offset/:limit", messageHandler.GetListMessage)
55 |
56 | // Register Handler
57 | apiService.Handle("/", router)
58 |
59 | // Run server
60 | if err := apiService.Run(); err != nil {
61 | log.Fatal(err)
62 | }
63 | }
64 |
65 | func main() {
66 | defer profile.Start().Stop()
67 |
68 | service := helper.NewServer(Domain, Consul, OpenTracing).GetService()
69 | delayQueue.RegisterDelayQueueHandler(service.Server(), handler.NewMicroServiceHandler())
70 | queueWorker := worker.NewWorker(helper.GetQueueType())
71 |
72 | var gracefulStop = make(chan os.Signal)
73 | signal.Notify(gracefulStop, syscall.SIGTERM)
74 | signal.Notify(gracefulStop, syscall.SIGINT)
75 | go func() {
76 | sig := <-gracefulStop
77 | fmt.Printf("caught sig: %+v", sig)
78 | fmt.Println("Wait for 2 second to finish processing")
79 | time.Sleep(2*time.Second)
80 | os.Exit(0)
81 | }()
82 |
83 | go WebServer()
84 |
85 | go queueWorker.Run()
86 |
87 | err := service.Run()
88 | if err != nil {
89 | log.Println("server cannot start, error: ", err)
90 | }
91 | }
92 |
--------------------------------------------------------------------------------
/models/data.go:
--------------------------------------------------------------------------------
1 | package message
2 |
3 | import (
4 | "errors"
5 | "encoding/json"
6 | )
7 |
8 | const (
9 | Success = iota + 1
10 | Fail
11 | )
12 |
13 | const (
14 | MYSQL = "mysql"
15 | MEMORY = "memory"
16 | )
17 |
18 | const (
19 | LIMIT_RANGE_MSG = 1000
20 | )
21 |
22 |
23 | var NotImplementError = errors.New("have not implemented")
24 | var errorText = map[int32]string{
25 | Success: "operation success",
26 | Fail: "fail to process request",
27 | }
28 |
29 | func ErrorMessage(code int32) string {
30 | return errorText[code]
31 | }
32 |
33 |
34 | type TsRange struct {
35 | Timestamp int64 `json:"timestamp"`
36 | }
37 |
38 | func NewTsRange(ts int64) TsRange {
39 | return TsRange{
40 | Timestamp: ts,
41 | }
42 | }
43 |
44 | func (t TsRange) ToJson() string {
45 | b, _ := json.Marshal(&t)
46 | return string(b)
47 | }
--------------------------------------------------------------------------------
/models/memory.go:
--------------------------------------------------------------------------------
1 | package message
2 |
3 | import (
4 | "github.com/whatvn/dqueue/helper"
5 |
6 | "github.com/whatvn/dqueue/database"
7 | "sync/atomic"
8 | "encoding/json"
9 | )
10 |
11 | var (
12 | db *database.MemoryDb
13 | id int64 = 0
14 | )
15 |
16 | type MemoryMessage struct {
17 | MySqlMessage
18 | }
19 |
20 | func GetMemDb() *database.MemoryDb {
21 | return db
22 | }
23 | func init() {
24 | if helper.GetDbType() == MEMORY {
25 | db = database.NewMemDb()
26 | }
27 | }
28 |
29 | func (m *MemoryMessage) Save() (int64, error) {
30 | atomic.AddInt64(&id, 1)
31 | return id, db.Insert(id, m.json())
32 | }
33 |
34 | func (m *MemoryMessage) json() string {
35 | b, _ := json.Marshal(m)
36 | return string(b)
37 | }
38 |
39 | func (m *MemoryMessage) byte() []byte {
40 | b, _ := json.Marshal(m)
41 | return b
42 | }
43 |
44 | func (m *MemoryMessage) Delete() error {
45 | return db.Delete(m.Id)
46 | }
47 |
48 | func (m *MemoryMessage) Force() error {
49 | m.TimeStamp = helper.NowPlus(-100)
50 | return db.Insert(m.Id, m.json())
51 | }
52 |
53 |
--------------------------------------------------------------------------------
/models/message.go:
--------------------------------------------------------------------------------
1 | package message
2 |
3 | import (
4 | "github.com/whatvn/dqueue/protobuf"
5 | "github.com/whatvn/dqueue/helper"
6 | "github.com/astaxie/beego/orm"
7 | "github.com/whatvn/dqueue/queue"
8 | "github.com/tidwall/buntdb"
9 | log "github.com/golang/glog"
10 | "strconv"
11 | )
12 |
13 | type Message interface {
14 | Save() (int64, error)
15 | Force() error
16 | Delete() error
17 | }
18 |
19 |
20 |
21 | func NewMessage(queueRequest *delayQueue.QueueRequest) Message {
22 |
23 | msg := MySqlMessage{
24 | TimeStamp: helper.NowPlus(queueRequest.Delay * queueRequest.RetryCount),
25 | Data: queueRequest.Messsage,
26 | RetryCount: int(queueRequest.RetryCount),
27 | Delay: int(queueRequest.Delay),
28 | }
29 | switch helper.GetDbType() {
30 | case MYSQL:
31 | return &msg
32 | case MEMORY:
33 | return &MemoryMessage{
34 | MySqlMessage: msg,
35 | }
36 | default:
37 | panic(NotImplementError)
38 |
39 | }
40 | }
41 |
42 | func NewMessageById(id int64) Message {
43 | msg := MySqlMessage{
44 | Id: id,
45 | }
46 | switch helper.GetDbType() {
47 | case MYSQL:
48 | return &msg
49 | case MEMORY:
50 | return &MemoryMessage{
51 | MySqlMessage: msg,
52 | }
53 | default:
54 | panic(NotImplementError)
55 |
56 | }
57 | }
58 |
59 | func Publish(q queue.Queue) error {
60 | now := helper.Now()
61 | switch helper.GetDbType() {
62 | case MYSQL:
63 | var ml = make([]*MySqlMessage, 0)
64 | o := orm.NewOrm()
65 | cond := orm.NewCondition().And("time_stamp__lt", now)
66 |
67 | qs := o.QueryTable("MySqlMessage")
68 | qs = qs.SetCond(cond).Limit(LIMIT_RANGE_MSG)
69 | _, err := qs.All(&ml)
70 | if err != nil && err != orm.ErrNoRows {
71 | return err
72 | }
73 |
74 | for _, m := range ml {
75 | err = q.PublishMessage(m.byte())
76 | if err == nil {
77 | m.Delete()
78 | }
79 | }
80 | return nil
81 |
82 | case MEMORY:
83 | deleteChan := make(chan *MemoryMessage, 1)
84 | err := GetMemDb().Engine.View(func(tx *buntdb.Tx) error {
85 | err := tx.AscendRange("ts", NewTsRange(0).ToJson(), NewTsRange(now).ToJson(), func(key, value string) bool {
86 | bs := []byte(value)
87 |
88 | err := q.PublishMessage(bs)
89 | log.Error(err)
90 | if err == nil {
91 | m := &MemoryMessage{}
92 | mId, _ := strconv.ParseInt(key, 10, 0)
93 | log.Info("push message id to delete chan ", mId)
94 | m.Id = mId
95 | deleteChan <- m
96 | }
97 |
98 | return true
99 | })
100 | close(deleteChan)
101 | return err
102 | })
103 | log.Error("AscendRange error: ", err)
104 | if err == nil {
105 | for m := range deleteChan {
106 | log.Info("m: ", m)
107 | if err = m.Delete(); err != nil {
108 | return err
109 | }
110 | }
111 | return nil
112 | }
113 | return err
114 |
115 | default:
116 | panic(NotImplementError)
117 | }
118 | }
119 |
--------------------------------------------------------------------------------
/models/mysql.go:
--------------------------------------------------------------------------------
1 | package message
2 |
3 | import (
4 | log "github.com/golang/glog"
5 |
6 | "git.zapa.cloud/evn-gateway/DelayQueue/helper"
7 | "github.com/astaxie/beego/orm"
8 | "encoding/json"
9 | )
10 |
11 | type MySqlMessage struct {
12 | Id int64 `json:"-" orm:"auto"`
13 | TimeStamp int64 `json:"timestamp"`
14 | Data string `json:"Message" orm:"type(text)"`
15 | RetryCount int `json:"retry_count"`
16 | Delay int `json:"delay"`
17 | }
18 |
19 | func init() {
20 | orm.RegisterModel(new(MySqlMessage))
21 | }
22 |
23 | func (m *MySqlMessage) Save() (id int64, err error) {
24 | o := orm.NewOrm()
25 | id, err = o.Insert(m)
26 | return
27 | }
28 |
29 | func (m *MySqlMessage) Force() (err error) {
30 | o := orm.NewOrm()
31 | if err = o.Read(m, "id"); err == nil {
32 | m.TimeStamp = helper.Now() - 2
33 |
34 | var num int64
35 | if num, err = o.Update(m, "TimeStamp"); err == nil {
36 | log.Info("Number of records updated in database:", num)
37 | }
38 | }
39 | return
40 | }
41 |
42 | func (m *MySqlMessage) json() string {
43 | b, _ := json.Marshal(m)
44 | return string(b)
45 | }
46 |
47 | func (m *MySqlMessage) byte() []byte {
48 | b, _ := json.Marshal(m)
49 | return b
50 | }
51 |
52 | func (m *MySqlMessage) Delete() error {
53 | o := orm.NewOrm()
54 | num, err := o.Delete(m)
55 | if err == nil {
56 | log.Info("Number of records deleted in database:", num)
57 | }
58 | return err
59 | }
60 |
61 | func All() (ml []*MySqlMessage, err error) {
62 | o := orm.NewOrm()
63 | _, err = o.QueryTable("MySqlMessage").All(&ml)
64 | if err != nil && err != orm.ErrNoRows {
65 | return nil, err
66 | }
67 |
68 | return ml, nil
69 | }
70 |
71 | func List(offset int, limit int) (ml []*MySqlMessage, err error) {
72 | o := orm.NewOrm()
73 | _, err = o.QueryTable("MySqlMessage").Offset(offset).Limit(limit).All(&ml)
74 | if err != nil && err != orm.ErrNoRows {
75 | return nil, err
76 | }
77 |
78 | return ml, nil
79 | }
80 |
81 | func SearchBy(data string) ([]*MySqlMessage, error) {
82 | o := orm.NewOrm()
83 | var msgList []*MySqlMessage
84 |
85 | num, err := o.Raw("select * from `Message` where data like ?", "%"+data+"%").QueryRows(&msgList)
86 | if err != nil {
87 | return nil, err
88 | }
89 | log.Info("Message nums: ", num)
90 | return msgList, nil
91 | }
92 |
--------------------------------------------------------------------------------
/protobuf/build.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | protoc -I/usr/local/include -I. \
4 | -I$GOPATH/src \
5 | --go_out=plugins=grpc:. --micro_out=.\
6 | *.proto
7 |
--------------------------------------------------------------------------------
/protobuf/message.micro.go:
--------------------------------------------------------------------------------
1 | // Code generated by protoc-gen-micro. DO NOT EDIT.
2 | // source: message.proto
3 |
4 | /*
5 | Package delayQueue is a generated protocol buffer package.
6 |
7 | It is generated from these files:
8 | message.proto
9 | service.proto
10 |
11 | It has these top-level messages:
12 | MessageReq
13 | MessageData
14 | ReturnCommon
15 | QueryListMessagesResp
16 | QueueRequest
17 | QueueResponse
18 | */
19 | package delayQueue
20 |
21 | import proto "github.com/golang/protobuf/proto"
22 | import fmt "fmt"
23 | import math "math"
24 |
25 | import (
26 | client "github.com/micro/go-micro/client"
27 | server "github.com/micro/go-micro/server"
28 | context "context"
29 | )
30 |
31 | // Reference imports to suppress errors if they are not otherwise used.
32 | var _ = proto.Marshal
33 | var _ = fmt.Errorf
34 | var _ = math.Inf
35 |
36 | // This is a compile-time assertion to ensure that this generated file
37 | // is compatible with the proto package it is being compiled against.
38 | // A compilation error at this line likely means your copy of the
39 | // proto package needs to be updated.
40 | const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
41 |
42 | // Reference imports to suppress errors if they are not otherwise used.
43 | var _ context.Context
44 | var _ client.Option
45 | var _ server.Option
46 |
47 | // Client API for Message service
48 |
49 | type MessageService interface {
50 | GetAllMessages(ctx context.Context, in *MessageReq, opts ...client.CallOption) (*QueryListMessagesResp, error)
51 | ForceMessage(ctx context.Context, in *MessageReq, opts ...client.CallOption) (*ReturnCommon, error)
52 | GetListMessage(ctx context.Context, in *MessageReq, opts ...client.CallOption) (*QueryListMessagesResp, error)
53 | }
54 |
55 | type messageService struct {
56 | c client.Client
57 | name string
58 | }
59 |
60 | func NewMessageService(name string, c client.Client) MessageService {
61 | if c == nil {
62 | c = client.NewClient()
63 | }
64 | if len(name) == 0 {
65 | name = "delayQueue"
66 | }
67 | return &messageService{
68 | c: c,
69 | name: name,
70 | }
71 | }
72 |
73 | func (c *messageService) GetAllMessages(ctx context.Context, in *MessageReq, opts ...client.CallOption) (*QueryListMessagesResp, error) {
74 | req := c.c.NewRequest(c.name, "Message.GetAllMessages", in)
75 | out := new(QueryListMessagesResp)
76 | err := c.c.Call(ctx, req, out, opts...)
77 | if err != nil {
78 | return nil, err
79 | }
80 | return out, nil
81 | }
82 |
83 | func (c *messageService) ForceMessage(ctx context.Context, in *MessageReq, opts ...client.CallOption) (*ReturnCommon, error) {
84 | req := c.c.NewRequest(c.name, "Message.ForceMessage", in)
85 | out := new(ReturnCommon)
86 | err := c.c.Call(ctx, req, out, opts...)
87 | if err != nil {
88 | return nil, err
89 | }
90 | return out, nil
91 | }
92 |
93 | func (c *messageService) GetListMessage(ctx context.Context, in *MessageReq, opts ...client.CallOption) (*QueryListMessagesResp, error) {
94 | req := c.c.NewRequest(c.name, "Message.GetListMessage", in)
95 | out := new(QueryListMessagesResp)
96 | err := c.c.Call(ctx, req, out, opts...)
97 | if err != nil {
98 | return nil, err
99 | }
100 | return out, nil
101 | }
102 |
103 | // Server API for Message service
104 |
105 | type MessageHandler interface {
106 | GetAllMessages(context.Context, *MessageReq, *QueryListMessagesResp) error
107 | ForceMessage(context.Context, *MessageReq, *ReturnCommon) error
108 | GetListMessage(context.Context, *MessageReq, *QueryListMessagesResp) error
109 | }
110 |
111 | func RegisterMessageHandler(s server.Server, hdlr MessageHandler, opts ...server.HandlerOption) error {
112 | type message interface {
113 | GetAllMessages(ctx context.Context, in *MessageReq, out *QueryListMessagesResp) error
114 | ForceMessage(ctx context.Context, in *MessageReq, out *ReturnCommon) error
115 | GetListMessage(ctx context.Context, in *MessageReq, out *QueryListMessagesResp) error
116 | }
117 | type Message struct {
118 | message
119 | }
120 | h := &messageHandler{hdlr}
121 | return s.Handle(s.NewHandler(&Message{h}, opts...))
122 | }
123 |
124 | type messageHandler struct {
125 | MessageHandler
126 | }
127 |
128 | func (h *messageHandler) GetAllMessages(ctx context.Context, in *MessageReq, out *QueryListMessagesResp) error {
129 | return h.MessageHandler.GetAllMessages(ctx, in, out)
130 | }
131 |
132 | func (h *messageHandler) ForceMessage(ctx context.Context, in *MessageReq, out *ReturnCommon) error {
133 | return h.MessageHandler.ForceMessage(ctx, in, out)
134 | }
135 |
136 | func (h *messageHandler) GetListMessage(ctx context.Context, in *MessageReq, out *QueryListMessagesResp) error {
137 | return h.MessageHandler.GetListMessage(ctx, in, out)
138 | }
139 |
--------------------------------------------------------------------------------
/protobuf/message.pb.go:
--------------------------------------------------------------------------------
1 | // Code generated by protoc-gen-go. DO NOT EDIT.
2 | // source: message.proto
3 |
4 | package delayQueue
5 |
6 | import (
7 | context "context"
8 | fmt "fmt"
9 | proto "github.com/golang/protobuf/proto"
10 | grpc "google.golang.org/grpc"
11 | math "math"
12 | )
13 |
14 | // Reference imports to suppress errors if they are not otherwise used.
15 | var _ = proto.Marshal
16 | var _ = fmt.Errorf
17 | var _ = math.Inf
18 |
19 | // This is a compile-time assertion to ensure that this generated file
20 | // is compatible with the proto package it is being compiled against.
21 | // A compilation error at this line likely means your copy of the
22 | // proto package needs to be updated.
23 | const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
24 |
25 | type MessageReq struct {
26 | Messsage string `protobuf:"bytes,1,opt,name=messsage,proto3" json:"messsage,omitempty"`
27 | XXX_NoUnkeyedLiteral struct{} `json:"-"`
28 | XXX_unrecognized []byte `json:"-"`
29 | XXX_sizecache int32 `json:"-"`
30 | }
31 |
32 | func (m *MessageReq) Reset() { *m = MessageReq{} }
33 | func (m *MessageReq) String() string { return proto.CompactTextString(m) }
34 | func (*MessageReq) ProtoMessage() {}
35 | func (*MessageReq) Descriptor() ([]byte, []int) {
36 | return fileDescriptor_33c57e4bae7b9afd, []int{0}
37 | }
38 |
39 | func (m *MessageReq) XXX_Unmarshal(b []byte) error {
40 | return xxx_messageInfo_MessageReq.Unmarshal(m, b)
41 | }
42 | func (m *MessageReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
43 | return xxx_messageInfo_MessageReq.Marshal(b, m, deterministic)
44 | }
45 | func (m *MessageReq) XXX_Merge(src proto.Message) {
46 | xxx_messageInfo_MessageReq.Merge(m, src)
47 | }
48 | func (m *MessageReq) XXX_Size() int {
49 | return xxx_messageInfo_MessageReq.Size(m)
50 | }
51 | func (m *MessageReq) XXX_DiscardUnknown() {
52 | xxx_messageInfo_MessageReq.DiscardUnknown(m)
53 | }
54 |
55 | var xxx_messageInfo_MessageReq proto.InternalMessageInfo
56 |
57 | func (m *MessageReq) GetMesssage() string {
58 | if m != nil {
59 | return m.Messsage
60 | }
61 | return ""
62 | }
63 |
64 | type MessageData struct {
65 | TimeStamp int64 `protobuf:"varint,1,opt,name=timeStamp,proto3" json:"timeStamp,omitempty"`
66 | Delay int32 `protobuf:"varint,2,opt,name=delay,proto3" json:"delay,omitempty"`
67 | RetryCount int32 `protobuf:"varint,3,opt,name=retryCount,proto3" json:"retryCount,omitempty"`
68 | Data string `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"`
69 | XXX_NoUnkeyedLiteral struct{} `json:"-"`
70 | XXX_unrecognized []byte `json:"-"`
71 | XXX_sizecache int32 `json:"-"`
72 | }
73 |
74 | func (m *MessageData) Reset() { *m = MessageData{} }
75 | func (m *MessageData) String() string { return proto.CompactTextString(m) }
76 | func (*MessageData) ProtoMessage() {}
77 | func (*MessageData) Descriptor() ([]byte, []int) {
78 | return fileDescriptor_33c57e4bae7b9afd, []int{1}
79 | }
80 |
81 | func (m *MessageData) XXX_Unmarshal(b []byte) error {
82 | return xxx_messageInfo_MessageData.Unmarshal(m, b)
83 | }
84 | func (m *MessageData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
85 | return xxx_messageInfo_MessageData.Marshal(b, m, deterministic)
86 | }
87 | func (m *MessageData) XXX_Merge(src proto.Message) {
88 | xxx_messageInfo_MessageData.Merge(m, src)
89 | }
90 | func (m *MessageData) XXX_Size() int {
91 | return xxx_messageInfo_MessageData.Size(m)
92 | }
93 | func (m *MessageData) XXX_DiscardUnknown() {
94 | xxx_messageInfo_MessageData.DiscardUnknown(m)
95 | }
96 |
97 | var xxx_messageInfo_MessageData proto.InternalMessageInfo
98 |
99 | func (m *MessageData) GetTimeStamp() int64 {
100 | if m != nil {
101 | return m.TimeStamp
102 | }
103 | return 0
104 | }
105 |
106 | func (m *MessageData) GetDelay() int32 {
107 | if m != nil {
108 | return m.Delay
109 | }
110 | return 0
111 | }
112 |
113 | func (m *MessageData) GetRetryCount() int32 {
114 | if m != nil {
115 | return m.RetryCount
116 | }
117 | return 0
118 | }
119 |
120 | func (m *MessageData) GetData() string {
121 | if m != nil {
122 | return m.Data
123 | }
124 | return ""
125 | }
126 |
127 | type ReturnCommon struct {
128 | ReturnCode int32 `protobuf:"varint,1,opt,name=returnCode,proto3" json:"returnCode,omitempty"`
129 | Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
130 | XXX_NoUnkeyedLiteral struct{} `json:"-"`
131 | XXX_unrecognized []byte `json:"-"`
132 | XXX_sizecache int32 `json:"-"`
133 | }
134 |
135 | func (m *ReturnCommon) Reset() { *m = ReturnCommon{} }
136 | func (m *ReturnCommon) String() string { return proto.CompactTextString(m) }
137 | func (*ReturnCommon) ProtoMessage() {}
138 | func (*ReturnCommon) Descriptor() ([]byte, []int) {
139 | return fileDescriptor_33c57e4bae7b9afd, []int{2}
140 | }
141 |
142 | func (m *ReturnCommon) XXX_Unmarshal(b []byte) error {
143 | return xxx_messageInfo_ReturnCommon.Unmarshal(m, b)
144 | }
145 | func (m *ReturnCommon) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
146 | return xxx_messageInfo_ReturnCommon.Marshal(b, m, deterministic)
147 | }
148 | func (m *ReturnCommon) XXX_Merge(src proto.Message) {
149 | xxx_messageInfo_ReturnCommon.Merge(m, src)
150 | }
151 | func (m *ReturnCommon) XXX_Size() int {
152 | return xxx_messageInfo_ReturnCommon.Size(m)
153 | }
154 | func (m *ReturnCommon) XXX_DiscardUnknown() {
155 | xxx_messageInfo_ReturnCommon.DiscardUnknown(m)
156 | }
157 |
158 | var xxx_messageInfo_ReturnCommon proto.InternalMessageInfo
159 |
160 | func (m *ReturnCommon) GetReturnCode() int32 {
161 | if m != nil {
162 | return m.ReturnCode
163 | }
164 | return 0
165 | }
166 |
167 | func (m *ReturnCommon) GetMessage() string {
168 | if m != nil {
169 | return m.Message
170 | }
171 | return ""
172 | }
173 |
174 | type QueryListMessagesResp struct {
175 | ReturnCode int32 `protobuf:"varint,1,opt,name=returnCode,proto3" json:"returnCode,omitempty"`
176 | Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
177 | MsgList []*MessageData `protobuf:"bytes,3,rep,name=msgList,proto3" json:"msgList,omitempty"`
178 | XXX_NoUnkeyedLiteral struct{} `json:"-"`
179 | XXX_unrecognized []byte `json:"-"`
180 | XXX_sizecache int32 `json:"-"`
181 | }
182 |
183 | func (m *QueryListMessagesResp) Reset() { *m = QueryListMessagesResp{} }
184 | func (m *QueryListMessagesResp) String() string { return proto.CompactTextString(m) }
185 | func (*QueryListMessagesResp) ProtoMessage() {}
186 | func (*QueryListMessagesResp) Descriptor() ([]byte, []int) {
187 | return fileDescriptor_33c57e4bae7b9afd, []int{3}
188 | }
189 |
190 | func (m *QueryListMessagesResp) XXX_Unmarshal(b []byte) error {
191 | return xxx_messageInfo_QueryListMessagesResp.Unmarshal(m, b)
192 | }
193 | func (m *QueryListMessagesResp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
194 | return xxx_messageInfo_QueryListMessagesResp.Marshal(b, m, deterministic)
195 | }
196 | func (m *QueryListMessagesResp) XXX_Merge(src proto.Message) {
197 | xxx_messageInfo_QueryListMessagesResp.Merge(m, src)
198 | }
199 | func (m *QueryListMessagesResp) XXX_Size() int {
200 | return xxx_messageInfo_QueryListMessagesResp.Size(m)
201 | }
202 | func (m *QueryListMessagesResp) XXX_DiscardUnknown() {
203 | xxx_messageInfo_QueryListMessagesResp.DiscardUnknown(m)
204 | }
205 |
206 | var xxx_messageInfo_QueryListMessagesResp proto.InternalMessageInfo
207 |
208 | func (m *QueryListMessagesResp) GetReturnCode() int32 {
209 | if m != nil {
210 | return m.ReturnCode
211 | }
212 | return 0
213 | }
214 |
215 | func (m *QueryListMessagesResp) GetMessage() string {
216 | if m != nil {
217 | return m.Message
218 | }
219 | return ""
220 | }
221 |
222 | func (m *QueryListMessagesResp) GetMsgList() []*MessageData {
223 | if m != nil {
224 | return m.MsgList
225 | }
226 | return nil
227 | }
228 |
229 | func init() {
230 | proto.RegisterType((*MessageReq)(nil), "delayQueue.MessageReq")
231 | proto.RegisterType((*MessageData)(nil), "delayQueue.MessageData")
232 | proto.RegisterType((*ReturnCommon)(nil), "delayQueue.ReturnCommon")
233 | proto.RegisterType((*QueryListMessagesResp)(nil), "delayQueue.QueryListMessagesResp")
234 | }
235 |
236 | func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) }
237 |
238 | var fileDescriptor_33c57e4bae7b9afd = []byte{
239 | // 298 bytes of a gzipped FileDescriptorProto
240 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x92, 0xcf, 0x4a, 0xc3, 0x40,
241 | 0x10, 0xc6, 0x89, 0x69, 0x8d, 0x99, 0x56, 0x0f, 0x8b, 0x7f, 0x96, 0x22, 0x12, 0x73, 0xca, 0x29,
242 | 0x60, 0x7d, 0x01, 0xa5, 0xa2, 0x82, 0x7a, 0xe8, 0xfa, 0x04, 0xab, 0x19, 0x4a, 0x21, 0x9b, 0x4d,
243 | 0xf7, 0xcf, 0x21, 0x77, 0x5f, 0xd5, 0xf7, 0x90, 0x6c, 0x36, 0x36, 0x42, 0xbd, 0xe8, 0x2d, 0xdf,
244 | 0x37, 0x93, 0xdf, 0x7c, 0x33, 0x2c, 0x1c, 0x0a, 0xd4, 0x9a, 0xaf, 0x30, 0xaf, 0x95, 0x34, 0x92,
245 | 0x40, 0x81, 0x25, 0x6f, 0x96, 0x16, 0x2d, 0xa6, 0x19, 0xc0, 0x4b, 0x57, 0x64, 0xb8, 0x21, 0x33,
246 | 0x38, 0x68, 0x5b, 0x5b, 0x49, 0x83, 0x24, 0xc8, 0x62, 0xf6, 0xad, 0x53, 0x0b, 0x13, 0xdf, 0x79,
247 | 0xc7, 0x0d, 0x27, 0xe7, 0x10, 0x9b, 0xb5, 0xc0, 0x57, 0xc3, 0x45, 0xed, 0x7a, 0x43, 0xb6, 0x35,
248 | 0xc8, 0x31, 0x8c, 0xdd, 0x10, 0xba, 0x97, 0x04, 0xd9, 0x98, 0x75, 0x82, 0x5c, 0x00, 0x28, 0x34,
249 | 0xaa, 0x59, 0x48, 0x5b, 0x19, 0x1a, 0xba, 0xd2, 0xc0, 0x21, 0x04, 0x46, 0x05, 0x37, 0x9c, 0x8e,
250 | 0xdc, 0x68, 0xf7, 0x9d, 0x3e, 0xc2, 0x94, 0xa1, 0xb1, 0xaa, 0x5a, 0x48, 0x21, 0x64, 0xe5, 0x19,
251 | 0x4e, 0x17, 0x5d, 0xc8, 0x8e, 0xe1, 0x1d, 0x42, 0x21, 0xf2, 0xdb, 0xba, 0xd9, 0x31, 0xeb, 0x65,
252 | 0xfa, 0x11, 0xc0, 0xc9, 0xd2, 0xa2, 0x6a, 0x9e, 0xd7, 0xda, 0xf8, 0x55, 0x34, 0x43, 0x5d, 0xff,
253 | 0x9d, 0x49, 0xae, 0x20, 0x12, 0x7a, 0xd5, 0x02, 0x69, 0x98, 0x84, 0xd9, 0x64, 0x7e, 0x96, 0x6f,
254 | 0x8f, 0x9b, 0x0f, 0xee, 0xc5, 0xfa, 0xbe, 0xf9, 0x67, 0x00, 0x91, 0x2f, 0x90, 0x27, 0x38, 0x7a,
255 | 0x40, 0x73, 0x5b, 0x96, 0x7d, 0x1c, 0x72, 0xba, 0xe3, 0x7f, 0x86, 0x9b, 0xd9, 0xe5, 0xd0, 0xdf,
256 | 0xbd, 0xc5, 0x0d, 0x4c, 0xef, 0xa5, 0x7a, 0xc7, 0x1e, 0xfe, 0x1b, 0x8a, 0x0e, 0xfd, 0x1f, 0xb7,
257 | 0xed, 0xe2, 0x0c, 0xc0, 0xff, 0x88, 0xf3, 0xb6, 0xef, 0x1e, 0xdb, 0xf5, 0x57, 0x00, 0x00, 0x00,
258 | 0xff, 0xff, 0x4d, 0xac, 0x5a, 0x79, 0x7d, 0x02, 0x00, 0x00,
259 | }
260 |
261 | // Reference imports to suppress errors if they are not otherwise used.
262 | var _ context.Context
263 | var _ grpc.ClientConn
264 |
265 | // This is a compile-time assertion to ensure that this generated file
266 | // is compatible with the grpc package it is being compiled against.
267 | const _ = grpc.SupportPackageIsVersion4
268 |
269 | // MessageClient is the client API for Message service.
270 | //
271 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
272 | type MessageClient interface {
273 | GetAllMessages(ctx context.Context, in *MessageReq, opts ...grpc.CallOption) (*QueryListMessagesResp, error)
274 | ForceMessage(ctx context.Context, in *MessageReq, opts ...grpc.CallOption) (*ReturnCommon, error)
275 | GetListMessage(ctx context.Context, in *MessageReq, opts ...grpc.CallOption) (*QueryListMessagesResp, error)
276 | }
277 |
278 | type messageClient struct {
279 | cc *grpc.ClientConn
280 | }
281 |
282 | func NewMessageClient(cc *grpc.ClientConn) MessageClient {
283 | return &messageClient{cc}
284 | }
285 |
286 | func (c *messageClient) GetAllMessages(ctx context.Context, in *MessageReq, opts ...grpc.CallOption) (*QueryListMessagesResp, error) {
287 | out := new(QueryListMessagesResp)
288 | err := c.cc.Invoke(ctx, "/delayQueue.Message/GetAllMessages", in, out, opts...)
289 | if err != nil {
290 | return nil, err
291 | }
292 | return out, nil
293 | }
294 |
295 | func (c *messageClient) ForceMessage(ctx context.Context, in *MessageReq, opts ...grpc.CallOption) (*ReturnCommon, error) {
296 | out := new(ReturnCommon)
297 | err := c.cc.Invoke(ctx, "/delayQueue.Message/ForceMessage", in, out, opts...)
298 | if err != nil {
299 | return nil, err
300 | }
301 | return out, nil
302 | }
303 |
304 | func (c *messageClient) GetListMessage(ctx context.Context, in *MessageReq, opts ...grpc.CallOption) (*QueryListMessagesResp, error) {
305 | out := new(QueryListMessagesResp)
306 | err := c.cc.Invoke(ctx, "/delayQueue.Message/GetListMessage", in, out, opts...)
307 | if err != nil {
308 | return nil, err
309 | }
310 | return out, nil
311 | }
312 |
313 | // MessageServer is the server API for Message service.
314 | type MessageServer interface {
315 | GetAllMessages(context.Context, *MessageReq) (*QueryListMessagesResp, error)
316 | ForceMessage(context.Context, *MessageReq) (*ReturnCommon, error)
317 | GetListMessage(context.Context, *MessageReq) (*QueryListMessagesResp, error)
318 | }
319 |
320 | func RegisterMessageServer(s *grpc.Server, srv MessageServer) {
321 | s.RegisterService(&_Message_serviceDesc, srv)
322 | }
323 |
324 | func _Message_GetAllMessages_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
325 | in := new(MessageReq)
326 | if err := dec(in); err != nil {
327 | return nil, err
328 | }
329 | if interceptor == nil {
330 | return srv.(MessageServer).GetAllMessages(ctx, in)
331 | }
332 | info := &grpc.UnaryServerInfo{
333 | Server: srv,
334 | FullMethod: "/delayQueue.Message/GetAllMessages",
335 | }
336 | handler := func(ctx context.Context, req interface{}) (interface{}, error) {
337 | return srv.(MessageServer).GetAllMessages(ctx, req.(*MessageReq))
338 | }
339 | return interceptor(ctx, in, info, handler)
340 | }
341 |
342 | func _Message_ForceMessage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
343 | in := new(MessageReq)
344 | if err := dec(in); err != nil {
345 | return nil, err
346 | }
347 | if interceptor == nil {
348 | return srv.(MessageServer).ForceMessage(ctx, in)
349 | }
350 | info := &grpc.UnaryServerInfo{
351 | Server: srv,
352 | FullMethod: "/delayQueue.Message/ForceMessage",
353 | }
354 | handler := func(ctx context.Context, req interface{}) (interface{}, error) {
355 | return srv.(MessageServer).ForceMessage(ctx, req.(*MessageReq))
356 | }
357 | return interceptor(ctx, in, info, handler)
358 | }
359 |
360 | func _Message_GetListMessage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
361 | in := new(MessageReq)
362 | if err := dec(in); err != nil {
363 | return nil, err
364 | }
365 | if interceptor == nil {
366 | return srv.(MessageServer).GetListMessage(ctx, in)
367 | }
368 | info := &grpc.UnaryServerInfo{
369 | Server: srv,
370 | FullMethod: "/delayQueue.Message/GetListMessage",
371 | }
372 | handler := func(ctx context.Context, req interface{}) (interface{}, error) {
373 | return srv.(MessageServer).GetListMessage(ctx, req.(*MessageReq))
374 | }
375 | return interceptor(ctx, in, info, handler)
376 | }
377 |
378 | var _Message_serviceDesc = grpc.ServiceDesc{
379 | ServiceName: "delayQueue.Message",
380 | HandlerType: (*MessageServer)(nil),
381 | Methods: []grpc.MethodDesc{
382 | {
383 | MethodName: "GetAllMessages",
384 | Handler: _Message_GetAllMessages_Handler,
385 | },
386 | {
387 | MethodName: "ForceMessage",
388 | Handler: _Message_ForceMessage_Handler,
389 | },
390 | {
391 | MethodName: "GetListMessage",
392 | Handler: _Message_GetListMessage_Handler,
393 | },
394 | },
395 | Streams: []grpc.StreamDesc{},
396 | Metadata: "message.proto",
397 | }
398 |
--------------------------------------------------------------------------------
/protobuf/message.proto:
--------------------------------------------------------------------------------
1 | syntax = "proto3";
2 | package delayQueue;
3 |
4 | message MessageReq {
5 | string messsage = 1;
6 | }
7 |
8 | message MessageData {
9 | int64 timeStamp = 1;
10 | int32 delay = 2;
11 | int32 retryCount = 3;
12 | string data = 4;
13 | }
14 |
15 | message ReturnCommon {
16 | int32 returnCode = 1;
17 | string message = 2;
18 | }
19 |
20 | message QueryListMessagesResp {
21 | int32 returnCode = 1;
22 | string message = 2;
23 | repeated MessageData msgList = 3;
24 | }
25 |
26 |
27 |
28 | service Message {
29 | rpc GetAllMessages (MessageReq) returns (QueryListMessagesResp);
30 | }
31 |
--------------------------------------------------------------------------------
/protobuf/service.micro.go:
--------------------------------------------------------------------------------
1 | // Code generated by protoc-gen-micro. DO NOT EDIT.
2 | // source: service.proto
3 |
4 | package delayQueue
5 |
6 | import proto "github.com/golang/protobuf/proto"
7 | import fmt "fmt"
8 | import math "math"
9 |
10 | import (
11 | client "github.com/micro/go-micro/client"
12 | server "github.com/micro/go-micro/server"
13 | context "context"
14 | )
15 |
16 | // Reference imports to suppress errors if they are not otherwise used.
17 | var _ = proto.Marshal
18 | var _ = fmt.Errorf
19 | var _ = math.Inf
20 |
21 | // Reference imports to suppress errors if they are not otherwise used.
22 | var _ context.Context
23 | var _ client.Option
24 | var _ server.Option
25 |
26 | // Client API for DelayQueue service
27 |
28 | type DelayQueueService interface {
29 | Publish(ctx context.Context, in *QueueRequest, opts ...client.CallOption) (*QueueResponse, error)
30 | }
31 |
32 | type delayQueueService struct {
33 | c client.Client
34 | name string
35 | }
36 |
37 | func NewDelayQueueService(name string, c client.Client) DelayQueueService {
38 | if c == nil {
39 | c = client.NewClient()
40 | }
41 | if len(name) == 0 {
42 | name = "delayQueue"
43 | }
44 | return &delayQueueService{
45 | c: c,
46 | name: name,
47 | }
48 | }
49 |
50 | func (c *delayQueueService) Publish(ctx context.Context, in *QueueRequest, opts ...client.CallOption) (*QueueResponse, error) {
51 | req := c.c.NewRequest(c.name, "DelayQueue.Publish", in)
52 | out := new(QueueResponse)
53 | err := c.c.Call(ctx, req, out, opts...)
54 | if err != nil {
55 | return nil, err
56 | }
57 | return out, nil
58 | }
59 |
60 | // Server API for DelayQueue service
61 |
62 | type DelayQueueHandler interface {
63 | Publish(context.Context, *QueueRequest, *QueueResponse) error
64 | }
65 |
66 | func RegisterDelayQueueHandler(s server.Server, hdlr DelayQueueHandler, opts ...server.HandlerOption) error {
67 | type delayQueue interface {
68 | Publish(ctx context.Context, in *QueueRequest, out *QueueResponse) error
69 | }
70 | type DelayQueue struct {
71 | delayQueue
72 | }
73 | h := &delayQueueHandler{hdlr}
74 | return s.Handle(s.NewHandler(&DelayQueue{h}, opts...))
75 | }
76 |
77 | type delayQueueHandler struct {
78 | DelayQueueHandler
79 | }
80 |
81 | func (h *delayQueueHandler) Publish(ctx context.Context, in *QueueRequest, out *QueueResponse) error {
82 | return h.DelayQueueHandler.Publish(ctx, in, out)
83 | }
84 |
--------------------------------------------------------------------------------
/protobuf/service.pb.go:
--------------------------------------------------------------------------------
1 | // Code generated by protoc-gen-go. DO NOT EDIT.
2 | // source: service.proto
3 |
4 | package delayQueue
5 |
6 | import (
7 | context "context"
8 | fmt "fmt"
9 | proto "github.com/golang/protobuf/proto"
10 | grpc "google.golang.org/grpc"
11 | math "math"
12 | )
13 |
14 | // Reference imports to suppress errors if they are not otherwise used.
15 | var _ = proto.Marshal
16 | var _ = fmt.Errorf
17 | var _ = math.Inf
18 |
19 | // This is a compile-time assertion to ensure that this generated file
20 | // is compatible with the proto package it is being compiled against.
21 | // A compilation error at this line likely means your copy of the
22 | // proto package needs to be updated.
23 | const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
24 |
25 | type QueueRequest struct {
26 | Messsage string `protobuf:"bytes,1,opt,name=messsage,proto3" json:"messsage,omitempty"`
27 | RetryCount int32 `protobuf:"varint,2,opt,name=retryCount,proto3" json:"retryCount,omitempty"`
28 | Delay int32 `protobuf:"varint,3,opt,name=delay,proto3" json:"delay,omitempty"`
29 | XXX_NoUnkeyedLiteral struct{} `json:"-"`
30 | XXX_unrecognized []byte `json:"-"`
31 | XXX_sizecache int32 `json:"-"`
32 | }
33 |
34 | func (m *QueueRequest) Reset() { *m = QueueRequest{} }
35 | func (m *QueueRequest) String() string { return proto.CompactTextString(m) }
36 | func (*QueueRequest) ProtoMessage() {}
37 | func (*QueueRequest) Descriptor() ([]byte, []int) {
38 | return fileDescriptor_a0b84a42fa06f626, []int{0}
39 | }
40 |
41 | func (m *QueueRequest) XXX_Unmarshal(b []byte) error {
42 | return xxx_messageInfo_QueueRequest.Unmarshal(m, b)
43 | }
44 | func (m *QueueRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
45 | return xxx_messageInfo_QueueRequest.Marshal(b, m, deterministic)
46 | }
47 | func (m *QueueRequest) XXX_Merge(src proto.Message) {
48 | xxx_messageInfo_QueueRequest.Merge(m, src)
49 | }
50 | func (m *QueueRequest) XXX_Size() int {
51 | return xxx_messageInfo_QueueRequest.Size(m)
52 | }
53 | func (m *QueueRequest) XXX_DiscardUnknown() {
54 | xxx_messageInfo_QueueRequest.DiscardUnknown(m)
55 | }
56 |
57 | var xxx_messageInfo_QueueRequest proto.InternalMessageInfo
58 |
59 | func (m *QueueRequest) GetMesssage() string {
60 | if m != nil {
61 | return m.Messsage
62 | }
63 | return ""
64 | }
65 |
66 | func (m *QueueRequest) GetRetryCount() int32 {
67 | if m != nil {
68 | return m.RetryCount
69 | }
70 | return 0
71 | }
72 |
73 | func (m *QueueRequest) GetDelay() int32 {
74 | if m != nil {
75 | return m.Delay
76 | }
77 | return 0
78 | }
79 |
80 | type QueueResponse struct {
81 | ReturnCode int64 `protobuf:"varint,1,opt,name=returnCode,proto3" json:"returnCode,omitempty"`
82 | Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
83 | XXX_NoUnkeyedLiteral struct{} `json:"-"`
84 | XXX_unrecognized []byte `json:"-"`
85 | XXX_sizecache int32 `json:"-"`
86 | }
87 |
88 | func (m *QueueResponse) Reset() { *m = QueueResponse{} }
89 | func (m *QueueResponse) String() string { return proto.CompactTextString(m) }
90 | func (*QueueResponse) ProtoMessage() {}
91 | func (*QueueResponse) Descriptor() ([]byte, []int) {
92 | return fileDescriptor_a0b84a42fa06f626, []int{1}
93 | }
94 |
95 | func (m *QueueResponse) XXX_Unmarshal(b []byte) error {
96 | return xxx_messageInfo_QueueResponse.Unmarshal(m, b)
97 | }
98 | func (m *QueueResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
99 | return xxx_messageInfo_QueueResponse.Marshal(b, m, deterministic)
100 | }
101 | func (m *QueueResponse) XXX_Merge(src proto.Message) {
102 | xxx_messageInfo_QueueResponse.Merge(m, src)
103 | }
104 | func (m *QueueResponse) XXX_Size() int {
105 | return xxx_messageInfo_QueueResponse.Size(m)
106 | }
107 | func (m *QueueResponse) XXX_DiscardUnknown() {
108 | xxx_messageInfo_QueueResponse.DiscardUnknown(m)
109 | }
110 |
111 | var xxx_messageInfo_QueueResponse proto.InternalMessageInfo
112 |
113 | func (m *QueueResponse) GetReturnCode() int64 {
114 | if m != nil {
115 | return m.ReturnCode
116 | }
117 | return 0
118 | }
119 |
120 | func (m *QueueResponse) GetMessage() string {
121 | if m != nil {
122 | return m.Message
123 | }
124 | return ""
125 | }
126 |
127 | func init() {
128 | proto.RegisterType((*QueueRequest)(nil), "delayQueue.QueueRequest")
129 | proto.RegisterType((*QueueResponse)(nil), "delayQueue.QueueResponse")
130 | }
131 |
132 | func init() { proto.RegisterFile("service.proto", fileDescriptor_a0b84a42fa06f626) }
133 |
134 | var fileDescriptor_a0b84a42fa06f626 = []byte{
135 | // 197 bytes of a gzipped FileDescriptorProto
136 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2d, 0x4e, 0x2d, 0x2a,
137 | 0xcb, 0x4c, 0x4e, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x4a, 0x49, 0xcd, 0x49, 0xac,
138 | 0x0c, 0x2c, 0x4d, 0x2d, 0x4d, 0x55, 0x4a, 0xe0, 0xe2, 0x01, 0x33, 0x82, 0x52, 0x0b, 0x4b, 0x53,
139 | 0x8b, 0x4b, 0x84, 0xa4, 0xb8, 0x38, 0x72, 0x53, 0x8b, 0x8b, 0x8b, 0x13, 0xd3, 0x53, 0x25, 0x18,
140 | 0x15, 0x18, 0x35, 0x38, 0x83, 0xe0, 0x7c, 0x21, 0x39, 0x2e, 0xae, 0xa2, 0xd4, 0x92, 0xa2, 0x4a,
141 | 0xe7, 0xfc, 0xd2, 0xbc, 0x12, 0x09, 0x26, 0x05, 0x46, 0x0d, 0xd6, 0x20, 0x24, 0x11, 0x21, 0x11,
142 | 0x2e, 0x56, 0xb0, 0xc9, 0x12, 0xcc, 0x60, 0x29, 0x08, 0x47, 0xc9, 0x93, 0x8b, 0x17, 0x6a, 0x43,
143 | 0x71, 0x41, 0x7e, 0x5e, 0x31, 0xcc, 0x98, 0xd2, 0xa2, 0x3c, 0xe7, 0xfc, 0x14, 0x88, 0x25, 0xcc,
144 | 0x41, 0x48, 0x22, 0x42, 0x12, 0x5c, 0xec, 0x20, 0x2b, 0x41, 0x2e, 0x60, 0x02, 0xbb, 0x00, 0xc6,
145 | 0x35, 0xf2, 0xe1, 0xe2, 0x72, 0x81, 0x3b, 0x5d, 0xc8, 0x8e, 0x8b, 0x3d, 0xa0, 0x34, 0x29, 0x27,
146 | 0xb3, 0x38, 0x43, 0x48, 0x42, 0x0f, 0xe1, 0x25, 0x3d, 0x64, 0xff, 0x48, 0x49, 0x62, 0x91, 0x81,
147 | 0xb8, 0x23, 0x89, 0x0d, 0x1c, 0x1a, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x42, 0x5e, 0xcd,
148 | 0x18, 0x1e, 0x01, 0x00, 0x00,
149 | }
150 |
151 | // Reference imports to suppress errors if they are not otherwise used.
152 | var _ context.Context
153 | var _ grpc.ClientConn
154 |
155 | // This is a compile-time assertion to ensure that this generated file
156 | // is compatible with the grpc package it is being compiled against.
157 | const _ = grpc.SupportPackageIsVersion4
158 |
159 | // DelayQueueClient is the client API for DelayQueue service.
160 | //
161 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
162 | type DelayQueueClient interface {
163 | Publish(ctx context.Context, in *QueueRequest, opts ...grpc.CallOption) (*QueueResponse, error)
164 | }
165 |
166 | type delayQueueClient struct {
167 | cc *grpc.ClientConn
168 | }
169 |
170 | func NewDelayQueueClient(cc *grpc.ClientConn) DelayQueueClient {
171 | return &delayQueueClient{cc}
172 | }
173 |
174 | func (c *delayQueueClient) Publish(ctx context.Context, in *QueueRequest, opts ...grpc.CallOption) (*QueueResponse, error) {
175 | out := new(QueueResponse)
176 | err := c.cc.Invoke(ctx, "/delayQueue.DelayQueue/Publish", in, out, opts...)
177 | if err != nil {
178 | return nil, err
179 | }
180 | return out, nil
181 | }
182 |
183 | // DelayQueueServer is the server API for DelayQueue service.
184 | type DelayQueueServer interface {
185 | Publish(context.Context, *QueueRequest) (*QueueResponse, error)
186 | }
187 |
188 | func RegisterDelayQueueServer(s *grpc.Server, srv DelayQueueServer) {
189 | s.RegisterService(&_DelayQueue_serviceDesc, srv)
190 | }
191 |
192 | func _DelayQueue_Publish_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
193 | in := new(QueueRequest)
194 | if err := dec(in); err != nil {
195 | return nil, err
196 | }
197 | if interceptor == nil {
198 | return srv.(DelayQueueServer).Publish(ctx, in)
199 | }
200 | info := &grpc.UnaryServerInfo{
201 | Server: srv,
202 | FullMethod: "/delayQueue.DelayQueue/Publish",
203 | }
204 | handler := func(ctx context.Context, req interface{}) (interface{}, error) {
205 | return srv.(DelayQueueServer).Publish(ctx, req.(*QueueRequest))
206 | }
207 | return interceptor(ctx, in, info, handler)
208 | }
209 |
210 | var _DelayQueue_serviceDesc = grpc.ServiceDesc{
211 | ServiceName: "delayQueue.DelayQueue",
212 | HandlerType: (*DelayQueueServer)(nil),
213 | Methods: []grpc.MethodDesc{
214 | {
215 | MethodName: "Publish",
216 | Handler: _DelayQueue_Publish_Handler,
217 | },
218 | },
219 | Streams: []grpc.StreamDesc{},
220 | Metadata: "service.proto",
221 | }
222 |
--------------------------------------------------------------------------------
/protobuf/service.proto:
--------------------------------------------------------------------------------
1 | syntax = "proto3";
2 | package delayQueue;
3 |
4 | message QueueRequest {
5 | string messsage = 1;
6 | int32 retryCount = 2;
7 | int32 delay = 3;
8 | }
9 |
10 | message QueueResponse {
11 | int64 returnCode = 1;
12 | string message = 2;
13 | }
14 |
15 |
16 | service DelayQueue {
17 | rpc Publish (QueueRequest) returns (QueueResponse);
18 | }
19 |
--------------------------------------------------------------------------------
/queue/kafka.go:
--------------------------------------------------------------------------------
1 | package queue
2 |
3 | import (
4 |
5 | "time"
6 | log "github.com/golang/glog"
7 | "github.com/Shopify/sarama"
8 | kafka "github.com/Shopify/sarama"
9 | "github.com/whatvn/dqueue/helper"
10 | )
11 |
12 | type kafkaConfig struct {
13 | Host, Port, Topic string
14 | }
15 | type kafkaQueue struct {
16 | client Writer
17 | }
18 |
19 | func (q *kafkaQueue) PublishMessage(message []byte) error {
20 | return q.client.WriteRaw(message)
21 | }
22 |
23 | func NewKafkaQueue() *kafkaQueue {
24 | var conf kafkaConfig
25 | helper.Config(&conf, "hosts", "kafka")
26 | writer := createWriter(conf.Host+":"+conf.Port, conf.Topic)
27 | return &kafkaQueue{
28 | client: writer,
29 | }
30 | }
31 |
32 | //Writer ...
33 | type Writer interface {
34 | WriteRaw([]byte) error
35 | Write(kafka.Encoder)
36 | }
37 | type writer struct {
38 | topic string
39 | producer kafka.SyncProducer
40 | }
41 |
42 | //CreateWriter ....
43 | func createWriter(addr, topic string) Writer {
44 | cfg := kafka.NewConfig()
45 | cfg.Producer.RequiredAcks = kafka.WaitForLocal
46 | cfg.Producer.Flush.Frequency = 50 * time.Millisecond
47 | producer, err := kafka.NewSyncProducer([]string{addr}, cfg)
48 | if err != nil {
49 | log.Error("kafka connection error: ", err)
50 | panic(err)
51 | }
52 |
53 | return &writer{topic: topic,
54 | producer: producer}
55 |
56 | }
57 | func (w *writer) Write(v kafka.Encoder) {
58 | msg := &kafka.ProducerMessage{
59 | Topic: w.topic,
60 | Value: v,
61 | }
62 | w.producer.SendMessage(msg)
63 | }
64 | func (w *writer) WriteRaw(v []byte) error {
65 | msg := &kafka.ProducerMessage{
66 | Topic: w.topic,
67 | Value: sarama.ByteEncoder(v),
68 | }
69 | _, _, err := w.producer.SendMessage(msg)
70 | return err
71 | }
72 |
--------------------------------------------------------------------------------
/queue/nats.go:
--------------------------------------------------------------------------------
1 | package queue
2 |
3 | import (
4 | "fmt"
5 | "time"
6 | log "github.com/golang/glog"
7 | "github.com/nats-io/go-nats"
8 | "github.com/whatvn/dqueue/helper"
9 | )
10 |
11 | type natsQueue struct {
12 | client *nats.Conn
13 | }
14 |
15 | type natsConfig struct {
16 | Address, Port string
17 | }
18 |
19 | func NewNatsQueue() *natsQueue {
20 | var conf natsConfig
21 | helper.Config(&conf, "hosts", "nats")
22 | server := fmt.Sprintf("%s:%s", conf.Address, conf.Port)
23 | log.Info("nats config: ", conf)
24 | opts := nats.Options{
25 | Url: server,
26 | // Servers: servers,
27 | AllowReconnect: true,
28 | MaxReconnect: 10,
29 | ReconnectWait: 5 * time.Second,
30 | Timeout: 1 * time.Second,
31 | }
32 |
33 | nc, err := opts.Connect()
34 | if err != nil {
35 | log.Error("nats connection error: ", err)
36 | panic(err)
37 | }
38 |
39 | return &natsQueue{
40 | client: nc,
41 | }
42 | }
43 |
44 | func (queue *natsQueue) PublishMessage(message []byte) error {
45 | err := queue.client.Publish("pendingqueue", message)
46 | log.Info("publish message: ", string(message))
47 | if err != nil {
48 | log.Error("cannot publish message", string(message), "error:", err)
49 | return err
50 | }
51 | return nil
52 | }
53 |
--------------------------------------------------------------------------------
/queue/nats_streaming.go:
--------------------------------------------------------------------------------
1 | package queue
2 |
3 | import "github.com/nats-io/go-nats-streaming"
4 |
5 | type NatsStreaming struct {
6 | client stan.Conn
7 | queueName string
8 | }
9 |
10 | func (n *NatsStreaming) Publish(message []byte) {
11 | n.client.Publish(n.queueName, message)
12 | }
--------------------------------------------------------------------------------
/queue/queue.go:
--------------------------------------------------------------------------------
1 | package queue
2 |
3 | type Queue interface {
4 | PublishMessage([]byte) error
5 | }
6 |
--------------------------------------------------------------------------------
/queue/stan.go:
--------------------------------------------------------------------------------
1 | package queue
2 |
3 | import (
4 | "fmt"
5 |
6 | log "github.com/golang/glog"
7 | "github.com/nats-io/go-nats-streaming"
8 | "github.com/whatvn/dqueue/helper"
9 | )
10 |
11 | type stanQueue struct {
12 | client stan.Conn
13 | }
14 |
15 | type stanConfig struct {
16 | Address, Port, ClusterID, ClientID string
17 | }
18 |
19 | func NewStanQueue() *stanQueue {
20 | var conf stanConfig
21 | helper.Config(&conf, "hosts", "stan")
22 | server := fmt.Sprintf("%s:%s", conf.Address, conf.Port)
23 | natsUrl := "nats://" + server
24 | log.Info("connecting to nats streaming : ", natsUrl)
25 |
26 | ns, err := stan.Connect(conf.ClusterID, conf.ClientID, stan.NatsURL(natsUrl))
27 | if err != nil {
28 | log.Error("cannot cannot to nats streaming , error: ", err)
29 | panic(err)
30 | }
31 | return &stanQueue{
32 | client: ns,
33 | }
34 | }
35 |
36 | func (queue *stanQueue) PublishMessage(message []byte) error {
37 | err := queue.client.Publish("pendingqueue", message)
38 | if err != nil {
39 | log.Error("cannot publish message", string(message), "error:", err)
40 | return err
41 | }
42 | return nil
43 | }
44 |
--------------------------------------------------------------------------------
/worker/worker.go:
--------------------------------------------------------------------------------
1 | package worker
2 |
3 | import (
4 | log "github.com/golang/glog"
5 | "time"
6 |
7 | "github.com/whatvn/dqueue/queue"
8 | "github.com/whatvn/dqueue/models"
9 | )
10 |
11 | type Worker struct {
12 | queue queue.Queue
13 | }
14 |
15 | func NewWorker(queueType string) *Worker {
16 | log.Infof("worker type: %s started", queueType)
17 | w := &Worker{}
18 |
19 | switch queueType {
20 | case "NATS":
21 | w.queue = queue.NewNatsQueue()
22 | break
23 | case "STAN":
24 | w.queue = queue.NewStanQueue()
25 | break
26 | case "KAFKA":
27 | w.queue = queue.NewKafkaQueue()
28 | break
29 | default:
30 | panic(message.NotImplementError)
31 | }
32 | return w
33 | }
34 |
35 | func (w *Worker) Run() {
36 | for {
37 | err := message.Publish(w.queue)
38 | if err != nil {
39 | log.Info("cannot get message from database, error: ", err)
40 | time.Sleep(2 * time.Second)
41 | continue
42 | }
43 | time.Sleep(500 * time.Millisecond)
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/wrapper/client.go:
--------------------------------------------------------------------------------
1 | package wrapper
2 |
3 | import (
4 | "context"
5 | "time"
6 |
7 | "github.com/whatvn/dqueue/protobuf"
8 |
9 | log "github.com/golang/glog"
10 | "github.com/micro/go-grpc"
11 | "github.com/micro/go-micro"
12 | "github.com/micro/go-micro/client"
13 | "github.com/micro/go-micro/registry"
14 | "github.com/micro/go-micro/selector"
15 | op "github.com/micro/go-plugins/wrapper/trace/opentracing"
16 | "github.com/opentracing/opentracing-go"
17 | )
18 |
19 | const (
20 | Domain = "DelayQueue"
21 | Consul = "127.0.0.1:8500"
22 | OpenTracing = "127.0.0.1:8200"
23 | )
24 |
25 | type logWrapper struct {
26 | client.Client
27 | }
28 |
29 | func (c *logWrapper) Call(ctx context.Context, req client.Request,
30 | rsp interface{}, opts ...client.CallOption) error {
31 | log.Infof("[client][%v] request: %v; options: %v", time.Now().Format(time.StampMilli), req.Method(), opts)
32 | return c.Client.Call(ctx, req, rsp, opts...)
33 | }
34 |
35 | func logWrap(c client.Client) client.Client {
36 | return &logWrapper{c}
37 | }
38 |
39 | type clientWrapper struct {
40 | tracer opentracing.Tracer
41 | service micro.Service
42 | }
43 |
44 | // NewClient return client wrapper for go-micro ecosystem
45 | func NewClient(domain string, consulAddr string, opts ...client.Option) *clientWrapper {
46 | log.Info("[Start InitClient]", domain, consulAddr)
47 | defer log.Info("[End InitClient]")
48 |
49 | // add options to client
50 | opts = append(opts, client.Retries(3))
51 | opts = append(opts, client.PoolSize(20))
52 |
53 | c := clientWrapper{}
54 | c.tracer = opentracing.GlobalTracer()
55 | c.initService(domain, consulAddr, opts...)
56 | log.Info("Init %s client done!", domain)
57 | return &c
58 | }
59 |
60 | func (c *clientWrapper) GetClient() client.Client { return c.service.Client() }
61 |
62 | func (c *clientWrapper) GetService() micro.Service { return c.service }
63 |
64 | func (c *clientWrapper) initService(domain string, consulAddr string, opts ...client.Option) {
65 |
66 | r := registry.NewRegistry(registry.Addrs(consulAddr))
67 | c.service = grpc.NewService(
68 | micro.Name(domain),
69 | micro.Registry(r),
70 | micro.Selector(selector.NewSelector(selector.Registry(r))),
71 | micro.WrapClient(
72 | logWrap,
73 | op.NewClientWrapper(opentracing.GlobalTracer()),
74 | ),
75 | )
76 | c.GetClient().Init(opts...)
77 | }
78 |
79 | func NewDelayQueueClient() delayQueue.DelayQueueService {
80 | service := NewClient(Domain, Consul).GetService()
81 | return delayQueue.NewDelayQueueService(Domain, service.Client())
82 | }
83 |
--------------------------------------------------------------------------------