├── Makefile ├── .travis.yml ├── glide.yaml ├── .gitignore ├── .editorconfig ├── msq ├── models_test.go ├── payload.go ├── payload_test.go ├── models.go ├── global_test.go ├── connection_test.go ├── queue.go ├── listener.go ├── connection.go ├── listener_test.go └── queue_test.go ├── LICENSE ├── glide.lock └── README.md /Makefile: -------------------------------------------------------------------------------- 1 | test: 2 | go test -v ./msq 3 | 4 | test-race: 5 | go test -v -race ./msq 6 | 7 | link: 8 | ln -s $(PWD) ~/go/src 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - "1.x" 4 | - "1.8" 5 | - "1.10.x" 6 | - master 7 | install: 8 | - go get -v github.com/Masterminds/glide 9 | - cd $GOPATH/src/github.com/Masterminds/glide && git checkout tags/0.10.2 && go install && cd - 10 | - glide install 11 | script: go test -v ./msq 12 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: msq 2 | import: 3 | - package: github.com/jinzhu/gorm 4 | version: ^1.9.1 5 | - package: github.com/stretchr/testify 6 | version: ~1.2.2 7 | - package: github.com/go-sql-driver/mysql 8 | version: ~1.4.0 9 | - package: github.com/mattn/go-sqlite3 10 | version: ~1.9.0 11 | - package: github.com/twinj/uuid 12 | version: ~1.0.0 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Don't need to commit databases 15 | *.db 16 | 17 | # Some other files we don't need to commit. 18 | .DS_Store 19 | vendor/ -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | indent_size = 4 3 | indent_style = space 4 | end_of_line = lf 5 | charset = utf-8 6 | trim_trailing_whitespace = true 7 | insert_final_newline = true 8 | 9 | [*.md] 10 | trim_trailing_whitespace = false 11 | 12 | [Makefile] 13 | indent_style = tab 14 | 15 | [*.go] 16 | indent_style = tab 17 | indent_size = 4 18 | 19 | [*.yml] 20 | indent_style = space 21 | indent_size = 2 22 | -------------------------------------------------------------------------------- /msq/models_test.go: -------------------------------------------------------------------------------- 1 | package msq 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestEventPayload(t *testing.T) { 10 | event := Event{ 11 | Payload: `{"example":"json", "payload":"4tests"}`, 12 | } 13 | 14 | assert.NotEmpty(t, event.Payload) 15 | 16 | payload, err := event.GetPayload() 17 | 18 | if assert.Nil(t, err) { 19 | assert.Equal(t, payload["example"], "json") 20 | assert.Equal(t, payload["payload"], "4tests") 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /msq/payload.go: -------------------------------------------------------------------------------- 1 | package msq 2 | 3 | import "encoding/json" 4 | 5 | type Payload map[string]interface{} 6 | 7 | func (p *Payload) Marshal() ([]byte, error) { 8 | 9 | marshalledData, err := json.Marshal(p) 10 | 11 | if err != nil { 12 | return []byte{}, err 13 | } 14 | 15 | return marshalledData, nil 16 | } 17 | 18 | func (p *Payload) UnMarshal(data []byte) (*Payload, error) { 19 | 20 | err := json.Unmarshal(data, p) 21 | 22 | if err != nil { 23 | return p, err 24 | } 25 | 26 | return p, nil 27 | } 28 | -------------------------------------------------------------------------------- /msq/payload_test.go: -------------------------------------------------------------------------------- 1 | package msq 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestMarshalPayload(t *testing.T) { 10 | setup() 11 | defer teardown() 12 | 13 | assert.Equal(t, payload["example"].(string), "data") 14 | 15 | bytes, err := payload.Marshal() 16 | 17 | assert.Nil(t, err) 18 | assert.NotEqual(t, bytes, []byte{}) 19 | } 20 | 21 | func TestUnMarshalPayload(t *testing.T) { 22 | setup() 23 | defer teardown() 24 | 25 | payload, err := payload.UnMarshal([]byte(`{"unmarshal": "testing", "numbers": [1,2,3,4]}`)) 26 | 27 | assert.Nil(t, err) 28 | 29 | data, ok := (*payload)["unmarshal"] 30 | 31 | assert.True(t, ok) 32 | 33 | assert.Equal(t, data.(string), "testing") 34 | } 35 | -------------------------------------------------------------------------------- /msq/models.go: -------------------------------------------------------------------------------- 1 | package msq 2 | 3 | import ( 4 | "github.com/jinzhu/gorm" 5 | "github.com/twinj/uuid" 6 | ) 7 | 8 | type Event struct { 9 | gorm.Model 10 | UID string `gorm:"type:varchar(255);index:uid"` 11 | Namespace string `gorm:"type:varchar(255);index:namespace;not null"` 12 | Payload string `gorm:"type:text"` 13 | Retries int `gorm:"size:1;index:retries;default:0"` 14 | } 15 | 16 | func (e *Event) BeforeCreate(scope *gorm.Scope) error { 17 | scope.SetColumn("UID", uuid.NewV4().String()) 18 | return nil 19 | } 20 | 21 | func (e *Event) GetPayload() (Payload, error) { 22 | p := Payload{} 23 | returnPayload, err := p.UnMarshal([]byte(e.Payload)) 24 | 25 | if err != nil { 26 | return Payload{}, err 27 | } 28 | 29 | return *returnPayload, nil 30 | } 31 | -------------------------------------------------------------------------------- /msq/global_test.go: -------------------------------------------------------------------------------- 1 | package msq 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | var connection *Connection 8 | var connectionConfig *ConnectionConfig 9 | var payload Payload 10 | var queueConfig *QueueConfig 11 | var listenerConfig ListenerConfig 12 | 13 | func setup() { 14 | connectionConfig = &ConnectionConfig{ 15 | Type: "sqlite", 16 | Database: "../test.db", 17 | Charset: "utf8", 18 | } 19 | 20 | payload = Payload{ 21 | "example": "data", 22 | "is": map[string]string{ 23 | "being": "shown", 24 | }, 25 | "here": []int{1, 2, 3, 4}, 26 | } 27 | 28 | queueConfig = &QueueConfig{ 29 | Name: "testing", 30 | MaxRetries: 4, 31 | MessageTTL: 5 * time.Minute, 32 | } 33 | 34 | listenerConfig = ListenerConfig{ 35 | Interval: 100 * time.Millisecond, 36 | Timeout: 250 * time.Millisecond, 37 | } 38 | } 39 | 40 | func teardown() { 41 | connection.Database().DropTable(&Event{}) 42 | 43 | connection.Close() 44 | } 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Analog Republic 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: 7e30d94c2cdb0ff4ab87e9d37e321d2b6eb3590048092d1d7c1d1671dd19b6b6 2 | updated: 2018-08-03T11:20:24.309805607+01:00 3 | imports: 4 | - name: github.com/go-sql-driver/mysql 5 | version: d523deb1b23d913de5bdada721a6071e71283618 6 | - name: github.com/jinzhu/gorm 7 | version: 6ed508ec6a4ecb3531899a69cbc746ccf65a4166 8 | subpackages: 9 | - dialects/mysql 10 | - dialects/sqlite 11 | - name: github.com/jinzhu/inflection 12 | version: 04140366298a54a039076d798123ffa108fff46c 13 | - name: github.com/mattn/go-sqlite3 14 | version: 25ecb14adfc7543176f7d85291ec7dba82c6f7e4 15 | - name: github.com/stretchr/testify 16 | version: f35b8ab0b5a2cef36673838d662e249dd9c94686 17 | subpackages: 18 | - assert 19 | - name: github.com/twinj/uuid 20 | version: 835a10bbd6bce40820349a68b1368a62c3c5617c 21 | - name: google.golang.org/appengine 22 | version: d1d14a4f7b9509a25ea8dc059e6820bbfb387beb 23 | subpackages: 24 | - cloudsql 25 | testImports: 26 | - name: github.com/davecgh/go-spew 27 | version: 8991bc29aa16c548c550c7ff78260e27b9ab7c73 28 | subpackages: 29 | - spew 30 | - name: github.com/pmezard/go-difflib 31 | version: 792786c7400a136282c1664665ae0a8db921c6c2 32 | subpackages: 33 | - difflib 34 | -------------------------------------------------------------------------------- /msq/connection_test.go: -------------------------------------------------------------------------------- 1 | package msq 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestConnectionConnect(t *testing.T) { 10 | setup() 11 | defer teardown() 12 | 13 | connection = &Connection{ 14 | Config: *connectionConfig, 15 | } 16 | 17 | err := connection.Attempt() 18 | 19 | assert.Nil(t, err) 20 | } 21 | 22 | func TestConnectionMigrateDatabase(t *testing.T) { 23 | setup() 24 | defer teardown() 25 | 26 | connection = &Connection{ 27 | Config: *connectionConfig, 28 | } 29 | 30 | err := connection.Attempt() 31 | 32 | assert.Nil(t, err) 33 | 34 | err = connection.SetupDatabase() 35 | 36 | assert.Nil(t, err) 37 | } 38 | 39 | func TestConnectionClose(t *testing.T) { 40 | setup() 41 | 42 | connection = &Connection{ 43 | Config: *connectionConfig, 44 | } 45 | 46 | err := connection.Attempt() 47 | 48 | assert.Nil(t, err) 49 | 50 | err = connection.Close() 51 | 52 | assert.Nil(t, err) 53 | } 54 | 55 | func TestConnect(t *testing.T) { 56 | setup() 57 | defer teardown() 58 | 59 | config := *connectionConfig 60 | queue, err := Connect(config) 61 | 62 | assert.Nil(t, err) 63 | 64 | if assert.NotNil(t, queue) { 65 | assert.Equal(t, queue.Connection.Config.Type, config.Type) 66 | assert.Equal(t, queue.Connection.Config.Database, config.Database) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /msq/queue.go: -------------------------------------------------------------------------------- 1 | package msq 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type QueueConfig struct { 8 | Name string 9 | MaxRetries int64 10 | MessageTTL time.Duration 11 | } 12 | 13 | type Queue struct { 14 | Connection *Connection 15 | Config *QueueConfig 16 | } 17 | 18 | func (q *Queue) Configure(config *QueueConfig) { 19 | q.Config = config 20 | } 21 | 22 | func (q *Queue) Done(event *Event) error { 23 | return q.Connection.Database().Unscoped().Delete(event).Error 24 | } 25 | 26 | func (q *Queue) ReQueue(event *Event) error { 27 | 28 | now := time.Now() 29 | pushback := time.Now().Add(time.Millisecond * (time.Duration(event.Retries) * 100)) 30 | retries := event.Retries + 1 31 | 32 | return q.Connection.Database(). 33 | Unscoped(). 34 | Model(event). 35 | Updates(map[string]interface{}{ 36 | "deleted_at": nil, 37 | "created_at": pushback, 38 | "updated_at": now, 39 | "retries": retries, 40 | }).Error 41 | } 42 | 43 | func (q *Queue) Pop() (*Event, error) { 44 | event := &Event{} 45 | 46 | db := q.Connection.Database() 47 | 48 | err := db.Order("created_at asc"). 49 | Where("created_at <= ?", time.Now()). 50 | Where("retries <= ?", q.Config.MaxRetries). 51 | Where("namespace = ?", q.Config.Name). 52 | First(event).Error 53 | 54 | if err != nil { 55 | return event, err 56 | } 57 | 58 | db.Delete(event) 59 | 60 | return event, nil 61 | } 62 | 63 | func (q *Queue) Failed() ([]*Event, error) { 64 | events := []*Event{} 65 | 66 | db := q.Connection.Database() 67 | 68 | err := db.Unscoped().Order("created_at desc"). 69 | Where("namespace = ?", q.Config.Name). 70 | Find(&events). 71 | Error 72 | 73 | if err != nil { 74 | return events, err 75 | } 76 | 77 | return events, nil 78 | } 79 | 80 | func (q *Queue) Push(payload Payload) (*Event, error) { 81 | encodedPayload, err := payload.Marshal() 82 | 83 | if err != nil { 84 | return &Event{}, err 85 | } 86 | 87 | event := &Event{ 88 | Namespace: q.Config.Name, 89 | Payload: string(encodedPayload), 90 | Retries: 1, 91 | } 92 | 93 | err = q.Connection.Database().Create(event).Error 94 | 95 | if err != nil { 96 | return &Event{}, err 97 | } 98 | 99 | return event, nil 100 | } 101 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-mysql-queue 2 | 3 | [![Build Status](https://travis-ci.org/AnalogRepublic/go-mysql-queue.svg?branch=master)](https://travis-ci.org/AnalogRepublic/go-mysql-queue) 4 | [![GitHub release](https://img.shields.io/github/release/analogrepublic/go-mysql-queue.svg)](https://github.com/AnalogRepublic/go-mysql-queue) 5 | 6 | 7 | 8 | A Very Basic Queue / Job implementation which uses MySQL for underlying storage 9 | 10 | ## Example Usage 11 | 12 | ``` 13 | import ( 14 | "fmt" 15 | "time" 16 | 17 | msq "github.com/AnalogRepublic/go-mysql-queue" 18 | ) 19 | 20 | // Connect to our backend database 21 | queue, err := msq.Connect(msq.ConnectionConfig{ 22 | Type: "mysql", // or could use "sqlite", where the "database" field is the full path, e.g. "./test.db" 23 | Host: "localhost", 24 | Username: "root", 25 | Password: "root", 26 | Database: "queue", 27 | Locale: "UTC", 28 | }) 29 | 30 | if err != nil { 31 | panic(err) 32 | } 33 | 34 | queue.Configure(&msq.QueueConfig{ 35 | Name: "my-queue", // The namespace for the Queue 36 | MaxRetries: 3, // The maximum number of times the message can be retried. 37 | }) 38 | 39 | if err != nil { 40 | panic(err) 41 | } 42 | 43 | // Using a listener 44 | listener := &Listener{ 45 | Queue: *queue, 46 | Config: listenerConfig, 47 | } 48 | 49 | ctx := listener.Context() 50 | 51 | // Define how many you want to fetch on each tick 52 | numToFetch := 2 53 | 54 | // Start the listener 55 | listener.Start(func(events []Event) bool { 56 | for _, event := range events { 57 | fmt.Println("Received event " + event.UID) 58 | } 59 | 60 | return true 61 | }, numToFetch) 62 | 63 | fmt.Println("Listener started") 64 | 65 | select { 66 | case <-ctx.Done(): 67 | fmt.Println("Listener stopped") 68 | } 69 | 70 | // or manually pull an item off the queue 71 | event, err := queue.Pop() 72 | 73 | if err == nil { 74 | err := doSomethingWithMessage(event) 75 | 76 | // If we have an error we can requeue it 77 | if err != nil { 78 | queue.ReQueue(event) 79 | } else { 80 | // or say we're happy with it 81 | queue.Done(event) 82 | } 83 | } 84 | 85 | time.AfterFunc(5 * time.Second, func() { 86 | // Push a new item onto the Queue 87 | queue.Push(msq.Payload{ 88 | "example": "data", 89 | "testing": []string{ 90 | "a", 91 | "b", 92 | }, 93 | "oh-look": map[string]string{ 94 | "maps": "here", 95 | }, 96 | }) 97 | }) 98 | 99 | ``` 100 | -------------------------------------------------------------------------------- /msq/listener.go: -------------------------------------------------------------------------------- 1 | package msq 2 | 3 | import ( 4 | "context" 5 | "time" 6 | ) 7 | 8 | type ListenerConfig struct { 9 | Interval time.Duration 10 | Timeout time.Duration 11 | } 12 | 13 | type Listener struct { 14 | Running bool 15 | Queue Queue 16 | Config ListenerConfig 17 | interval <-chan time.Time 18 | stop chan bool 19 | ctx context.Context 20 | cancel func() 21 | } 22 | 23 | func (l *Listener) Context() context.Context { 24 | if l.ctx == nil { 25 | l.ctx, l.cancel = context.WithCancel(context.Background()) 26 | } 27 | 28 | return l.ctx 29 | } 30 | 31 | func (l *Listener) Start(handle func([]Event) bool, num int) { 32 | started := make(chan bool) 33 | 34 | if num < 1 { 35 | num = 1 36 | } 37 | 38 | go func() { 39 | if l.Running { 40 | panic("Cannot start the listener whilst it is already running") 41 | } 42 | 43 | defer l.cancel() 44 | 45 | firstTick := true 46 | 47 | l.interval = time.NewTicker(l.Config.Interval).C 48 | l.stop = make(chan bool) 49 | 50 | for { 51 | select { 52 | case <-l.interval: 53 | if !firstTick && !l.Running { 54 | return 55 | } 56 | 57 | if firstTick { 58 | l.Running = true 59 | started <- true 60 | firstTick = false 61 | } 62 | 63 | timeout := time.NewTimer(l.Config.Timeout).C 64 | 65 | // Go off and actually pull the events 66 | go func() { 67 | var resultValue bool 68 | result := make(chan bool) 69 | events := []Event{} 70 | 71 | // Depending on how many we want, that's what 72 | // we will pop off the queue 73 | for i := 0; i < num; i++ { 74 | event, err := l.Queue.Pop() 75 | 76 | if err == nil { 77 | events = append(events, *event) 78 | continue 79 | } 80 | } 81 | 82 | // Go off and handle those events 83 | go func(events []Event, handle func([]Event) bool, result chan bool) { 84 | result <- handle(events) 85 | }(events, handle, result) 86 | 87 | // Block on either a timeout on the handle 88 | // or a result from the handle. 89 | select { 90 | case <-timeout: 91 | for _, event := range events { 92 | l.Queue.ReQueue(&event) 93 | } 94 | case resultValue = <-result: 95 | for _, event := range events { 96 | if resultValue { 97 | l.Queue.Done(&event) 98 | } else { 99 | l.Queue.ReQueue(&event) 100 | } 101 | } 102 | 103 | break 104 | } 105 | }() 106 | case <-l.stop: 107 | l.Running = false 108 | break 109 | } 110 | } 111 | }() 112 | 113 | <-started 114 | } 115 | 116 | func (l *Listener) Stop() { 117 | l.stop <- true 118 | } 119 | -------------------------------------------------------------------------------- /msq/connection.go: -------------------------------------------------------------------------------- 1 | package msq 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strconv" 7 | 8 | "github.com/jinzhu/gorm" 9 | _ "github.com/jinzhu/gorm/dialects/mysql" 10 | _ "github.com/jinzhu/gorm/dialects/sqlite" 11 | ) 12 | 13 | type ConnectionConfig struct { 14 | Type string 15 | Host string 16 | Proto string 17 | Port int 18 | Username string 19 | Password string 20 | Database string 21 | Charset string 22 | Locale string 23 | Logging bool 24 | } 25 | 26 | type Connection struct { 27 | db *gorm.DB 28 | Config ConnectionConfig 29 | } 30 | 31 | func (c *Connection) Attempt() error { 32 | if c.Config.Locale == "" { 33 | c.Config.Locale = "Local" 34 | } 35 | 36 | if c.Config.Charset == "" { 37 | c.Config.Charset = "utf8mb4" 38 | } 39 | 40 | if c.Config.Proto == "" { 41 | c.Config.Proto = "tcp" 42 | } 43 | 44 | connectionString := c.getConnectionString() 45 | 46 | if c.Config.Logging { 47 | fmt.Println("Connecting to " + connectionString) 48 | } 49 | 50 | db, err := gorm.Open(c.getType(), connectionString) 51 | 52 | if err != nil { 53 | return err 54 | } 55 | 56 | db.LogMode(c.Config.Logging) 57 | 58 | c.db = db 59 | 60 | return nil 61 | } 62 | 63 | func (c *Connection) Close() error { 64 | return c.db.Close() 65 | } 66 | 67 | func (c *Connection) Database() *gorm.DB { 68 | return c.db 69 | } 70 | 71 | func (c *Connection) SetupDatabase() error { 72 | dbScope := c.db 73 | 74 | if c.getType() != "sqlite3" { 75 | tableOptions := fmt.Sprintf("ENGINE=InnoDB DEFAULT CHARSET=%s", c.Config.Charset) 76 | dbScope = dbScope.Set("gorm:table_options", tableOptions) 77 | } 78 | 79 | dbScope = dbScope.AutoMigrate(&Event{}) 80 | 81 | hasTable := dbScope.HasTable(&Event{}) 82 | 83 | if !hasTable { 84 | return errors.New("Events table was not created") 85 | } 86 | 87 | return nil 88 | } 89 | 90 | func (c *Connection) getType() string { 91 | if c.Config.Type == "sqlite" { 92 | return "sqlite3" 93 | } 94 | 95 | return c.Config.Type 96 | } 97 | 98 | func (c *Connection) getConnectionString() string { 99 | dbType := c.getType() 100 | 101 | port := strconv.Itoa(c.Config.Port) 102 | hostname := c.Config.Host 103 | 104 | if port != "" && port != "0" { 105 | hostname = fmt.Sprintf("%s:%s", c.Config.Host, port) 106 | } 107 | 108 | if dbType == "mysql" { 109 | return fmt.Sprintf( 110 | "%s:%s@%s(%s)/%s?charset=%s&parseTime=True&loc=%s", 111 | c.Config.Username, 112 | c.Config.Password, 113 | c.Config.Proto, 114 | hostname, 115 | c.Config.Database, 116 | c.Config.Charset, 117 | c.Config.Locale, 118 | ) 119 | } else if dbType == "sqlite3" { 120 | return c.Config.Database 121 | } 122 | 123 | panic("Invalid database type provided, must be 'myqsl' or 'sqlite3'/'sqlite'") 124 | } 125 | 126 | func Connect(config ConnectionConfig) (*Queue, error) { 127 | connection := &Connection{ 128 | Config: config, 129 | } 130 | 131 | err := connection.Attempt() 132 | 133 | if err != nil { 134 | return &Queue{}, err 135 | } 136 | 137 | connection.SetupDatabase() 138 | 139 | queue := &Queue{ 140 | Connection: connection, 141 | } 142 | 143 | return queue, nil 144 | } 145 | -------------------------------------------------------------------------------- /msq/listener_test.go: -------------------------------------------------------------------------------- 1 | package msq 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestStartStop(t *testing.T) { 11 | setup() 12 | defer teardown() 13 | 14 | config := *connectionConfig 15 | queue, err := Connect(config) 16 | 17 | queue.Configure(queueConfig) 18 | 19 | queuedEvent, err := queue.Push(payload) 20 | 21 | if assert.Nil(t, err) { 22 | listener := &Listener{ 23 | Queue: *queue, 24 | Config: listenerConfig, 25 | } 26 | 27 | assert.Equal(t, listener.Config.Interval, listenerConfig.Interval) 28 | assert.Equal(t, listener.Config.Timeout, listenerConfig.Timeout) 29 | 30 | ctx := listener.Context() 31 | 32 | listener.Start(func(events []Event) bool { 33 | if len(events) > 0 { 34 | assert.Equal(t, queuedEvent.UID, events[0].UID) 35 | } 36 | 37 | return true 38 | }, 1) 39 | 40 | go func() { 41 | assert.True(t, listener.Running, "The listener should be running") 42 | 43 | time.Sleep(time.Second) 44 | listener.Stop() 45 | }() 46 | 47 | select { 48 | case <-ctx.Done(): 49 | assert.False(t, listener.Running, "The listener should no longer be running") 50 | } 51 | } 52 | } 53 | 54 | func TestHandleFail(t *testing.T) { 55 | setup() 56 | defer teardown() 57 | 58 | config := *connectionConfig 59 | queue, err := Connect(config) 60 | 61 | queue.Configure(queueConfig) 62 | 63 | queuedEvent, err := queue.Push(payload) 64 | 65 | if assert.Nil(t, err) { 66 | listener := &Listener{ 67 | Queue: *queue, 68 | Config: listenerConfig, 69 | } 70 | 71 | assert.Equal(t, listener.Config.Interval, listenerConfig.Interval) 72 | assert.Equal(t, listener.Config.Timeout, listenerConfig.Timeout) 73 | 74 | ctx := listener.Context() 75 | 76 | listener.Start(func(events []Event) bool { 77 | if len(events) > 0 { 78 | assert.Equal(t, queuedEvent.UID, events[0].UID) 79 | } 80 | return false 81 | }, 1) 82 | 83 | go func() { 84 | assert.True(t, listener.Running, "The listener should be started") 85 | time.Sleep(2 * listenerConfig.Interval) 86 | 87 | failedEvents, err := queue.Failed() 88 | 89 | if assert.Nil(t, err, "We should get a list of failed events back") { 90 | assert.Equal(t, queuedEvent.UID, failedEvents[0].UID) 91 | queue.Done(failedEvents[0]) 92 | } 93 | 94 | listener.Stop() 95 | }() 96 | 97 | select { 98 | case <-ctx.Done(): 99 | assert.False(t, listener.Running, "The listener should no longer be running") 100 | } 101 | } 102 | } 103 | 104 | func TestHandleTimeout(t *testing.T) { 105 | setup() 106 | defer teardown() 107 | 108 | config := *connectionConfig 109 | queue, err := Connect(config) 110 | 111 | queue.Configure(queueConfig) 112 | 113 | queuedEvent, err := queue.Push(payload) 114 | 115 | if assert.Nil(t, err) { 116 | listener := &Listener{ 117 | Queue: *queue, 118 | Config: listenerConfig, 119 | } 120 | 121 | assert.Equal(t, listener.Config.Interval, listenerConfig.Interval) 122 | assert.Equal(t, listener.Config.Timeout, listenerConfig.Timeout) 123 | 124 | ctx := listener.Context() 125 | 126 | listener.Start(func(events []Event) bool { 127 | time.Sleep(2 * listenerConfig.Timeout) 128 | return false 129 | }, 1) 130 | 131 | go func() { 132 | assert.True(t, listener.Running, "The listener should be started") 133 | time.Sleep(2 * listenerConfig.Interval) 134 | 135 | failedEvents, err := queue.Failed() 136 | 137 | if assert.Nil(t, err, "We should get a list of failed events back") { 138 | assert.Equal(t, queuedEvent.UID, failedEvents[0].UID) 139 | queue.Done(failedEvents[0]) 140 | } 141 | 142 | listener.Stop() 143 | }() 144 | 145 | select { 146 | case <-ctx.Done(): 147 | assert.False(t, listener.Running, "The listener should no longer be running") 148 | } 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /msq/queue_test.go: -------------------------------------------------------------------------------- 1 | package msq 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestPush(t *testing.T) { 11 | setup() 12 | defer teardown() 13 | 14 | config := *connectionConfig 15 | queue, err := Connect(config) 16 | 17 | queue.Configure(queueConfig) 18 | 19 | event, err := queue.Push(payload) 20 | 21 | if assert.Nil(t, err) { 22 | assert.NotNil(t, event) 23 | 24 | assert.Equal(t, event.Namespace, queue.Config.Name) 25 | 26 | encodedPayload, err := payload.Marshal() 27 | 28 | if assert.Nil(t, err) { 29 | assert.Equal(t, event.Payload, string(encodedPayload)) 30 | } 31 | } 32 | } 33 | 34 | func TestPop(t *testing.T) { 35 | setup() 36 | defer teardown() 37 | 38 | config := *connectionConfig 39 | queue, err := Connect(config) 40 | 41 | queue.Configure(queueConfig) 42 | 43 | _, err = queue.Push(payload) 44 | 45 | if assert.Nil(t, err) { 46 | event, err := queue.Pop() 47 | 48 | if assert.Nil(t, err) { 49 | if assert.NotEqual(t, event.UID, "", "UID should not be empty") { 50 | encodedPayload, err := payload.Marshal() 51 | 52 | if assert.Nil(t, err) { 53 | assert.Equal(t, event.Payload, string(encodedPayload), "Payload should match") 54 | } 55 | } 56 | } 57 | } 58 | } 59 | 60 | func TestDone(t *testing.T) { 61 | setup() 62 | defer teardown() 63 | 64 | config := *connectionConfig 65 | queue, err := Connect(config) 66 | 67 | queue.Configure(queueConfig) 68 | 69 | event, err := queue.Pop() 70 | 71 | if assert.Nil(t, err, "There should be an event in the queue") { 72 | err := queue.Done(event) 73 | 74 | assert.Nil(t, err, "We should be able to remove the record") 75 | 76 | err = connection.Database().Where("uid = ?", event.UID).First(&Event{}).Error 77 | 78 | assert.NotNil(t, err, "We want the record to be missing as it should be removed") 79 | } 80 | } 81 | 82 | func TestReQueue(t *testing.T) { 83 | setup() 84 | defer teardown() 85 | 86 | config := *connectionConfig 87 | queue, err := Connect(config) 88 | 89 | queue.Configure(queueConfig) 90 | 91 | originalEvent, err := queue.Push(payload) 92 | 93 | if assert.Nil(t, err) { 94 | event, err := queue.Pop() 95 | 96 | if assert.Nil(t, err, "There should be an event in the queue") { 97 | // Simulate waiting 98 | time.Sleep(listenerConfig.Interval) 99 | 100 | err := queue.ReQueue(event) 101 | assert.Nil(t, err, "We should have no problem re-queuing the event") 102 | 103 | // Simulate waiting 104 | time.Sleep(listenerConfig.Interval) 105 | 106 | newEvent, err := queue.Pop() 107 | 108 | if assert.Nil(t, err, "We should find a requeued event in the queue") { 109 | assert.Equal(t, event.UID, newEvent.UID, "We should get back the same event") 110 | assert.NotEqual(t, originalEvent.Retries, newEvent.Retries, "The retries should have increased") 111 | assert.NotEqual(t, originalEvent.UpdatedAt.UnixNano(), newEvent.UpdatedAt.UnixNano(), "The updated at timestamps should be different") 112 | assert.NotEqual(t, originalEvent.CreatedAt.UnixNano(), newEvent.CreatedAt.UnixNano(), "The created at timestamps should be different") 113 | } 114 | } 115 | } 116 | } 117 | 118 | func TestFailed(t *testing.T) { 119 | setup() 120 | defer teardown() 121 | 122 | config := *connectionConfig 123 | queue, err := Connect(config) 124 | 125 | queue.Configure(queueConfig) 126 | 127 | originalEvent, err := queue.Push(payload) 128 | 129 | if assert.Nil(t, err) { 130 | event, err := queue.Pop() 131 | 132 | if assert.Nil(t, err, "There should be an event in the queue") { 133 | // Simulate waiting 134 | time.Sleep(listenerConfig.Interval) 135 | 136 | event.Retries = 3 137 | 138 | err := queue.ReQueue(event) 139 | assert.Nil(t, err, "We should have no problem re-queuing the event") 140 | 141 | // Simulate waiting 142 | time.Sleep(listenerConfig.Interval) 143 | 144 | failedEvents, err := queue.Failed() 145 | 146 | if assert.Nil(t, err, "We should get a list of a failed events") { 147 | assert.Equal(t, originalEvent.UID, failedEvents[0].UID) 148 | queue.Done(failedEvents[0]) 149 | } 150 | } 151 | } 152 | } 153 | --------------------------------------------------------------------------------