├── LICENSE.md ├── README.md ├── example ├── go.mod ├── go.sum ├── jobs │ └── emailUser.go └── main.go ├── go.mod ├── go.sum ├── queue.go └── queue_test.go /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Robin Verton 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pgjobs, a dead simple postgres job queueing mechanism 2 | 3 | This project aims to be a blueprint for your own job queue solution with Go and 4 | PostgreSQL. It is recommended to fork this project and adjust the job queue with 5 | features to your own needs. 6 | 7 | By using Postgres `SKIP LOCKED` feature, this allows to make a performant, 8 | non-blocking and independent queue. The technique is described 9 | [here](https://robinverton.de/blog/queueing-with-postgresql-and-go) and [here](https://www.crunchydata.com/blog/message-queuing-using-native-postgresql). 10 | 11 | * Performat: Non-blocking queue mechanism 12 | * Robust: Jobs will be 'freed' again when a worker crashes 13 | * Failed jobs will be retried until `MAX_RETRIES`, current attempt is passed as argument 14 | * Schedule jobs for later execution with `EnqueueAt(ctx, job, "queuename", timeAt)` 15 | * Support for multiple queues 16 | * Zero dependency 17 | 18 | ## Example usage 19 | 20 | A complete, runnable examples can be found under `./example/`. 21 | 22 | First define a `pgjobs.Job`, for example in `./jobs/emailUser.go`: 23 | 24 | ```go 25 | package jobs 26 | 27 | import ( 28 | "encoding/json" 29 | "log" 30 | 31 | "github.com/rverton/pgjobs" 32 | ) 33 | 34 | type EmailUser struct { 35 | Email string 36 | } 37 | 38 | func NewEmailUser(email string) *EmailUser { 39 | return &EmailUser{ 40 | Email: email, 41 | } 42 | } 43 | 44 | // the action which should be executed 45 | func (e EmailUser) Perform(attempt int32) error { 46 | log.Printf("emailing %v, attempt=%v", e.Email, attempt) 47 | return nil 48 | } 49 | 50 | // this is boilerplate code and does not need to be modified 51 | func (e EmailUser) Load(data string) (pgjobs.Job, error) { 52 | var n EmailUser 53 | err := json.Unmarshal([]byte(data), &n) 54 | return n, err 55 | } 56 | ``` 57 | 58 | You can then setup a queue, (optionally) enforce the jobs table schema, and work on queued jobs. 59 | 60 | ```go 61 | package main 62 | 63 | import ( 64 | "context" 65 | "database/sql" 66 | "example/jobs" 67 | "log" 68 | "os" 69 | 70 | _ "github.com/jackc/pgx/v5/stdlib" 71 | "github.com/rverton/pgjobs" 72 | ) 73 | 74 | func main() { 75 | db, err := sql.Open("postgres", os.Getenv("DB_URL")) 76 | if err != nil { 77 | log.Fatal(err) 78 | } 79 | 80 | ctx := context.Background() 81 | 82 | // initiate queue with postgres connection 83 | queue := pgjobs.NewQueue(db) 84 | if err := queue.SetupSchema(ctx); err != nil { 85 | panic(err) 86 | } 87 | 88 | job := jobs.NewEmailUser("foo@example.com") 89 | 90 | // enequeue an example job for immediate execution 91 | if err = queue.Enqueue(context.Background(), job, "default"); err != nil { 92 | log.Printf("error enqueueing: %v", err) 93 | } 94 | 95 | // enequeue an example job for execution in 10s+ 96 | if err = queue.EnqueueAt(context.Background(), job, "default", time.Now().Add(10*time.Second)); err != nil { 97 | log.Printf("error enqueueing: %v", err) 98 | } 99 | 100 | // start worker and pass all processable jobs 101 | // note: this worker will only process the passed jobs 102 | queues := []string{"default"} 103 | if err := queue.Worker(ctx, queues, &jobs.EmailUser{}); err != nil { 104 | log.Println(err) 105 | } 106 | } 107 | ``` 108 | 109 | ## Configuration 110 | 111 | * The polling interval for workers can be adjusted via `pgjobs.PollInterval`. 112 | * For all other configuration, it is currently recommended to create a fork and adjust as needed. 113 | 114 | ## ToDo 115 | 116 | * [X] Make job processing more robust by using a transaction 117 | * [X] Implement `attempt` handling 118 | * [X] Add error handling and retries? 119 | * [X] Add scheduled execution 120 | * [X] Remove `github.com/lib/pq` dependency 121 | * [ ] Add more tests (dequeing, reflection) 122 | * [ ] Add priority queuing 123 | -------------------------------------------------------------------------------- /example/go.mod: -------------------------------------------------------------------------------- 1 | module example 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/jackc/pgx/v5 v5.0.1 7 | github.com/rverton/pgjobs v0.0.0-20221006071552-3f6a75d7291a 8 | ) 9 | 10 | require ( 11 | github.com/jackc/pgpassfile v1.0.0 // indirect 12 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect 13 | github.com/lib/pq v1.10.7 // indirect 14 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect 15 | golang.org/x/text v0.3.7 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /example/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 4 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 5 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= 6 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= 7 | github.com/jackc/pgx/v5 v5.0.1 h1:JZu9othr7l8so2JMDAGeDUMXqERAuZpovyfl4H50tdg= 8 | github.com/jackc/pgx/v5 v5.0.1/go.mod h1:JBbvW3Hdw77jKl9uJrEDATUZIFM2VFPzRq4RWIhkF4o= 9 | github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= 10 | github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 11 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 12 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 13 | github.com/rverton/pgjobs v0.0.0-20221006071552-3f6a75d7291a h1:KV9zEExx/bWIdUSyM0ySupDLE50sNartLlY3tR9hJIg= 14 | github.com/rverton/pgjobs v0.0.0-20221006071552-3f6a75d7291a/go.mod h1:rFPncFNNZel9A4iDBmEDB4BoHi5dnggq2nWW7J+qiUE= 15 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 16 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 17 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 18 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 19 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= 20 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 21 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 22 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 23 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 24 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 25 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 26 | -------------------------------------------------------------------------------- /example/jobs/emailUser.go: -------------------------------------------------------------------------------- 1 | package jobs 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | 7 | "github.com/rverton/pgjobs" 8 | ) 9 | 10 | type EmailUser struct { 11 | Email string 12 | } 13 | 14 | func NewEmailUser(email string) *EmailUser { 15 | return &EmailUser{ 16 | Email: email, 17 | } 18 | } 19 | 20 | func (e EmailUser) Perform(attempt int32) error { 21 | log.Printf("emailing %v, attempt=%v", e.Email, attempt) 22 | return nil 23 | } 24 | 25 | func (e EmailUser) Load(data string) (pgjobs.Job, error) { 26 | var n EmailUser 27 | err := json.Unmarshal([]byte(data), &n) 28 | return n, err 29 | } 30 | -------------------------------------------------------------------------------- /example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "example/jobs" 7 | "log" 8 | "os" 9 | 10 | _ "github.com/jackc/pgx/v5/stdlib" 11 | "github.com/rverton/pgjobs" 12 | ) 13 | 14 | func main() { 15 | db, err := sql.Open("postgres", os.Getenv("DB_URL")) 16 | if err != nil { 17 | log.Fatal(err) 18 | } 19 | 20 | ctx := context.Background() 21 | 22 | // initiate queue with postgres connection 23 | queue := pgjobs.NewQueue(db) 24 | if err := queue.SetupSchema(ctx); err != nil { 25 | panic(err) 26 | } 27 | 28 | // example to enqueue a job 29 | job := jobs.NewEmailUser("foo@example.com") 30 | if err = queue.Enqueue(context.Background(), job, "default"); err != nil { 31 | log.Printf("error enqueueing: %v", err) 32 | } 33 | 34 | // start worker and pass all processable jobs 35 | queues := []string{"default"} 36 | if err := queue.Worker(ctx, queues, &jobs.EmailUser{}); err != nil { 37 | log.Println(err) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/rverton/pgjobs 2 | 3 | go 1.19 4 | 5 | require github.com/lib/pq v1.10.7 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= 2 | github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 3 | -------------------------------------------------------------------------------- /queue.go: -------------------------------------------------------------------------------- 1 | package pgjobs 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "database/sql" 7 | "encoding/json" 8 | "fmt" 9 | "log" 10 | "reflect" 11 | "strings" 12 | "sync" 13 | "time" 14 | ) 15 | 16 | const ( 17 | JOB_STATUS_SCHEDULED = "new" 18 | JOB_STATUS_FINISHED = "finished" 19 | JOB_STATUS_FAILED = "failed" 20 | 21 | MAX_RETRY = 3 22 | ) 23 | 24 | var PollInterval = 1 * time.Second 25 | var JobsTableName = "_jobs" 26 | 27 | type Job interface { 28 | Perform(attempt int32) error 29 | Load(data string) (Job, error) 30 | } 31 | 32 | type jobRaw struct { 33 | Id int64 34 | TypeName string 35 | Status string 36 | Queue string 37 | Data string 38 | Error string 39 | Attempt int32 40 | 41 | CreatedAt time.Time 42 | StartedAt time.Time 43 | FinishedAt time.Time 44 | } 45 | 46 | type JobQueue struct { 47 | db *sql.DB 48 | 49 | mutex sync.Mutex 50 | typeRegistry map[string]reflect.Type 51 | } 52 | 53 | func NewQueue(db *sql.DB) *JobQueue { 54 | return &JobQueue{ 55 | db: db, 56 | typeRegistry: make(map[string]reflect.Type), 57 | } 58 | } 59 | 60 | func (j *JobQueue) SetupSchema(ctx context.Context) error { 61 | schema := fmt.Sprintf(` 62 | CREATE TABLE IF NOT EXISTS %v ( 63 | id serial primary key, 64 | 65 | type_name text NOT NULL, 66 | status text NOT NULL, 67 | queue text NOT NULL, 68 | data text NOT NULL, 69 | 70 | error text, 71 | attempt int default 0, 72 | 73 | created_at timestamp not null default now(), 74 | scheduled_at timestamp, 75 | started_at timestamp, 76 | finished_at timestamp 77 | ); 78 | 79 | CREATE INDEX IF NOT EXISTS idx_jobs_status ON %v(status); 80 | CREATE INDEX IF NOT EXISTS idx_jobs_queue ON %v(queue);`, JobsTableName, JobsTableName, JobsTableName) 81 | _, err := j.db.ExecContext(ctx, schema) 82 | return err 83 | } 84 | 85 | func (j *JobQueue) EnqueueAt(ctx context.Context, job Job, queue string, at time.Time) error { 86 | typeName := j.typeName(job) 87 | 88 | data, err := json.Marshal(job) 89 | if err != nil { 90 | return fmt.Errorf("queue: failed marshaling: %v", err) 91 | } 92 | 93 | if _, err = j.db.ExecContext( 94 | ctx, 95 | `INSERT INTO `+JobsTableName+` (type_name, status, queue, data, scheduled_at) VALUES ($1, $2, $3, $4, $5)`, 96 | typeName, 97 | JOB_STATUS_SCHEDULED, 98 | queue, 99 | data, 100 | at, 101 | ); err != nil { 102 | return fmt.Errorf("queue: failed inserting job: %w", err) 103 | } 104 | 105 | return nil 106 | } 107 | 108 | func (j *JobQueue) Enqueue(ctx context.Context, job Job, queue string) error { 109 | log.Printf("queue: enqueing queue=%v job=%+v", queue, job) 110 | 111 | typeName := j.typeName(job) 112 | 113 | data, err := json.Marshal(job) 114 | if err != nil { 115 | return fmt.Errorf("queue: failed marshaling: %v", err) 116 | } 117 | 118 | if _, err = j.db.ExecContext( 119 | ctx, 120 | `INSERT INTO `+JobsTableName+` (type_name, status, queue, data) VALUES ($1, $2, $3, $4)`, 121 | typeName, 122 | JOB_STATUS_SCHEDULED, 123 | queue, 124 | data, 125 | ); err != nil { 126 | return fmt.Errorf("queue: failed inserting job: %w", err) 127 | } 128 | 129 | return nil 130 | } 131 | 132 | func (j *JobQueue) Dequeue(ctx context.Context, queues []string) error { 133 | var job jobRaw 134 | 135 | sqlStmt := ` 136 | UPDATE 137 | ` + JobsTableName + ` 138 | SET 139 | status = $1, 140 | started_at = now(), 141 | attempt = attempt + 1 142 | WHERE 143 | id IN ( 144 | SELECT 145 | id FROM ` + JobsTableName + ` j 146 | WHERE 147 | (j.status = $2 or (j.status = $3 and j.attempt < $4)) 148 | AND j.queue = any($5) 149 | AND j.type_name = any($6) 150 | AND (j.scheduled_at is null or (j.scheduled_at <= now())) 151 | ORDER BY 152 | j.scheduled_at, j.created_at 153 | FOR UPDATE SKIP LOCKED 154 | LIMIT 1) 155 | RETURNING id, type_name, data, attempt 156 | ` 157 | 158 | tx, err := j.db.BeginTx(ctx, nil) 159 | if err != nil { 160 | return err 161 | } 162 | defer tx.Rollback() 163 | 164 | queueArray, err := pqArray(queues) 165 | if err != nil { 166 | return err 167 | } 168 | 169 | typesArray, err := pqArray(mapKeys(j.typeRegistry)) 170 | if err != nil { 171 | return err 172 | } 173 | 174 | row := tx.QueryRowContext( 175 | ctx, 176 | sqlStmt, 177 | JOB_STATUS_FINISHED, 178 | JOB_STATUS_SCHEDULED, 179 | JOB_STATUS_FAILED, 180 | MAX_RETRY, 181 | queueArray, 182 | typesArray, 183 | ) 184 | err = row.Scan(&job.Id, &job.TypeName, &job.Data, &job.Attempt) 185 | if err == sql.ErrNoRows { 186 | return nil 187 | } else if err != nil { 188 | return err 189 | } 190 | 191 | if err != nil { 192 | return err 193 | } 194 | 195 | // get original go type based on type name 196 | jobType, err := j.getType(job.TypeName) 197 | if err != nil { 198 | _, err = tx.ExecContext(ctx, `UPDATE `+JobsTableName+` SET status = $1, finished_at = NOW(), error = $3 WHERE id = $2`, JOB_STATUS_FAILED, job.Id, err.Error()) 199 | if err != nil { 200 | return fmt.Errorf("unable to exec error for failed job", err) 201 | } 202 | 203 | if err = tx.Commit(); err != nil { 204 | return fmt.Errorf("unable to commit error for failed job", err) 205 | } 206 | 207 | return fmt.Errorf("unable to find related job '%v': %v", job.TypeName, err) 208 | } 209 | 210 | // create a new object by unmarshaling the job data 211 | loadedJob, err := jobType.Load(job.Data) 212 | if err != nil { 213 | return err 214 | } 215 | 216 | // execute job 217 | err = loadedJob.Perform(int32(job.Attempt)) 218 | if err != nil { 219 | // TODO: add retry handling and save error to job row 220 | _, err = tx.ExecContext(ctx, `UPDATE `+JobsTableName+` SET status = $1, finished_at = NOW(), error = $3 WHERE id = $2`, JOB_STATUS_FAILED, job.Id, err.Error()) 221 | if err != nil { 222 | return err 223 | } 224 | return tx.Commit() 225 | } 226 | 227 | _, err = tx.ExecContext(ctx, `UPDATE `+JobsTableName+` SET status = $1, finished_at = NOW() WHERE id = $2`, JOB_STATUS_FINISHED, job.Id) 228 | if err != nil { 229 | return fmt.Errorf("failed updating job status: %w", err) 230 | } 231 | 232 | return tx.Commit() 233 | } 234 | 235 | func (j *JobQueue) Worker(ctx context.Context, queues []string, types ...interface{}) error { 236 | // register all passed types in a type type registry. 237 | // this allows to map job types back to their corresponding go type 238 | // to execute the Perform() action. 239 | for _, t := range types { 240 | j.registerType(t) 241 | } 242 | 243 | tm := time.NewTicker(PollInterval) 244 | defer tm.Stop() 245 | for { 246 | select { 247 | case <-ctx.Done(): 248 | return ctx.Err() 249 | case <-tm.C: 250 | if err := j.Dequeue(ctx, queues); err != nil { 251 | log.Println("queue: dequeue failed", err) 252 | } 253 | } 254 | } 255 | } 256 | 257 | func (j *JobQueue) typeName(typedNil interface{}) string { 258 | name := reflect.TypeOf(typedNil).String() 259 | if strings.HasPrefix(name, "*") { 260 | name = name[1:] 261 | } 262 | 263 | return name 264 | } 265 | 266 | func (j *JobQueue) registerType(typedNil interface{}) { 267 | t := reflect.TypeOf(typedNil).Elem() 268 | name := j.typeName(typedNil) 269 | 270 | j.mutex.Lock() 271 | defer j.mutex.Unlock() 272 | j.typeRegistry[name] = t 273 | } 274 | 275 | func (j *JobQueue) getType(name string) (Job, error) { 276 | item, ok := j.typeRegistry[name] 277 | 278 | if !ok { 279 | return nil, fmt.Errorf("type not found in type registry. did you register the job?") 280 | } 281 | 282 | t := reflect.New(item).Elem().Interface().(Job) 283 | 284 | return t, nil 285 | } 286 | 287 | // pqArray and appendArrayQuotedBytes func extracted from https://github.com/lib/pq 288 | // to remove dependency on lib/pq 289 | func pqArray(a []string) (string, error) { 290 | if n := len(a); n > 0 { 291 | // There will be at least two curly brackets, 2*N bytes of quotes, 292 | // and N-1 bytes of delimiters. 293 | b := make([]byte, 1, 1+3*n) 294 | b[0] = '{' 295 | 296 | b = appendArrayQuotedBytes(b, []byte(a[0])) 297 | for i := 1; i < n; i++ { 298 | b = append(b, ',') 299 | b = appendArrayQuotedBytes(b, []byte(a[i])) 300 | } 301 | 302 | return string(append(b, '}')), nil 303 | } 304 | 305 | return "{}", nil 306 | } 307 | 308 | func appendArrayQuotedBytes(b, v []byte) []byte { 309 | b = append(b, '"') 310 | for { 311 | i := bytes.IndexAny(v, `"\`) 312 | if i < 0 { 313 | b = append(b, v...) 314 | break 315 | } 316 | if i > 0 { 317 | b = append(b, v[:i]...) 318 | } 319 | b = append(b, '\\', v[i]) 320 | v = v[i+1:] 321 | } 322 | return append(b, '"') 323 | } 324 | 325 | func mapKeys[K comparable, V any](m map[K]V) []K { 326 | keys := make([]K, 0, len(m)) 327 | for k := range m { 328 | keys = append(keys, k) 329 | } 330 | return keys 331 | } 332 | -------------------------------------------------------------------------------- /queue_test.go: -------------------------------------------------------------------------------- 1 | package pgjobs 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "encoding/json" 7 | "fmt" 8 | "os" 9 | "testing" 10 | 11 | _ "github.com/lib/pq" 12 | ) 13 | 14 | var db *sql.DB 15 | 16 | func TestMain(m *testing.M) { 17 | var err error 18 | dbUrl := os.Getenv("DB_URL_TEST") 19 | if dbUrl == "" { 20 | fmt.Fprintf(os.Stderr, "DB_URL_TEST not set") 21 | os.Exit(1) 22 | } 23 | 24 | db, err = sql.Open("postgres", dbUrl) 25 | if err != nil { 26 | fmt.Fprintf(os.Stderr, "unable to connect to test DB: %v", err) 27 | os.Exit(1) 28 | } 29 | 30 | // ensure clean state 31 | _, err = db.Exec("DROP TABLE IF EXISTS " + JobsTableName) 32 | if err != nil { 33 | fmt.Fprintf(os.Stderr, "unable to remove previous jobs table: %v", err) 34 | os.Exit(1) 35 | } 36 | 37 | // create initial schema 38 | if err := NewQueue(db).SetupSchema(context.Background()); err != nil { 39 | fmt.Fprintf(os.Stderr, "unable to create schema: %v", err) 40 | os.Exit(1) 41 | } 42 | 43 | code := m.Run() 44 | os.Exit(code) 45 | } 46 | 47 | func TestEnqueue(t *testing.T) { 48 | queue := NewQueue(db) 49 | job := NewEmailUser("foo@example.com") 50 | 51 | // enequeue an example job for immediate execution 52 | if err := queue.Enqueue(context.Background(), job, "default"); err != nil { 53 | t.Errorf("failed enqueueing: %v", err) 54 | } 55 | 56 | var cnt int 57 | if err := db.QueryRow("SELECT COUNT(*) FROM " + JobsTableName + "").Scan(&cnt); err != nil { 58 | t.Fatalf("error retrieving job: %v", err) 59 | } 60 | 61 | if cnt != 1 { 62 | t.Fatal("no job enqueued") 63 | } 64 | } 65 | 66 | // dummy job for testing 67 | type EmailUser struct { 68 | Email string 69 | } 70 | 71 | func NewEmailUser(email string) *EmailUser { 72 | return &EmailUser{ 73 | Email: email, 74 | } 75 | } 76 | 77 | // the action which should be executed 78 | func (e EmailUser) Perform(attempt int32) error { 79 | fmt.Printf("emailing %v, attempt=%v", e.Email, attempt) 80 | return nil 81 | } 82 | 83 | // this is boilerplate code and does not need to be modified 84 | func (e EmailUser) Load(data string) (Job, error) { 85 | var n EmailUser 86 | err := json.Unmarshal([]byte(data), &n) 87 | return n, err 88 | } 89 | --------------------------------------------------------------------------------