├── .gitignore ├── .travis.yml ├── Makefile ├── README.md ├── broker.go ├── broker_test.go ├── examples ├── consumer │ └── main.go └── producer │ └── main.go ├── go.mod ├── go.sum ├── gotasks.go ├── gotasks_test.go ├── handler.go ├── json.go ├── loop ├── loop.go └── loop_test.go ├── metrics └── metrics.go ├── pool ├── pool.go └── pool_test.go ├── queue.go ├── redis_broker.go ├── task_job.go └── task_job_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | /gotasks 2 | /coverage.out 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.14.x 4 | 5 | services: 6 | - redis-server 7 | 8 | before_install: 9 | - go get ./... 10 | 11 | script: 12 | - go test -race -coverprofile=coverage.txt -covermode=atomic 13 | 14 | after_success: 15 | - bash <(curl -s https://codecov.io/bash) 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | go fmt ./... 3 | go vet ./... 4 | go test -cover ./... -race 5 | 6 | cover: 7 | go test -cover -coverprofile=coverage.out -race ./... 8 | go tool cover -html=coverage.out 9 | rm -f coverage.out 10 | 11 | build: 12 | go build -o gotasks 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gotasks 2 | 3 | [![Build Status](https://travis-ci.org/jiajunhuang/gotasks.svg?branch=master)](https://travis-ci.org/jiajunhuang/gotasks) 4 | [![codecov](https://codecov.io/gh/jiajunhuang/gotasks/branch/master/graph/badge.svg)](https://codecov.io/gh/jiajunhuang/gotasks) 5 | 6 | gotasks is a task/job queue framework for Golang. Currently we support use Redis as broker, but you can replace it 7 | easily by implement `Broker` interface in `broker.go`. 8 | 9 | In gotasks, we encourage developer to split tasks into smaller pieces(see the demo bellow) so we can: 10 | 11 | - maintain tasks easily 12 | - split code into reentrant and un-reentrant pieces, so when reentrant part failed, framework will retry it automatically 13 | - concurrency control 14 | 15 | As handlers are chained, ArgsMap we give will be arguments to the first handler, and ArgsMap in it's return value will 16 | be arguments to the second handler. If any handler return a error, the execution chain will stop, and record where it 17 | is now, with the error string inside the task's `result_log` property. 18 | 19 | # Usage 20 | 21 | - producer generate tasks to broker: 22 | 23 | ```go 24 | package main 25 | 26 | import ( 27 | "github.com/jiajunhuang/gotasks" 28 | //"github.com/jiajunhuang/gotasks/metrics" 29 | ) 30 | 31 | const ( 32 | uniqueJobName = "a-unique-job-name" 33 | redisURL = "redis://127.0.0.1:6379/0" 34 | queueName = "job-queue-name" 35 | ) 36 | 37 | func main() { 38 | // set broker 39 | gotasks.UseRedisBroker(redisURL, gotasks.WithRedisTaskTTL(1000)) 40 | 41 | // enqueue 42 | // or you can use a queue: 43 | queue := gotasks.NewQueue(queueName, gotasks.WithMaxLimit(10)) 44 | queue.Enqueue(uniqueJobName, gotasks.MapToArgsMap(map[string]interface{}{})) // or gotasks.StructToArgsMap 45 | } 46 | ``` 47 | 48 | - consumer consumes tasks from broker: 49 | 50 | ```go 51 | package main 52 | 53 | import ( 54 | "context" 55 | "time" 56 | 57 | "github.com/jiajunhuang/gotasks" 58 | //"github.com/jiajunhuang/gotasks/metrics" 59 | ) 60 | 61 | const ( 62 | uniqueJobName = "a-unique-job-name" 63 | redisURL = "redis://127.0.0.1:6379/0" 64 | queueName = "job-queue-name" 65 | ) 66 | 67 | func Handler1(args gotasks.ArgsMap) (gotasks.ArgsMap, error) { 68 | time.Sleep(time.Duration(1) * time.Second) 69 | return args, nil 70 | } 71 | func Handler2(args gotasks.ArgsMap) (gotasks.ArgsMap, error) { 72 | time.Sleep(time.Duration(1) * time.Second) 73 | return args, nil 74 | } 75 | 76 | func register() { 77 | // if handler1 failed, the task will stop, but if handler2 failed(return a non-nil error) 78 | // handler2 will be retry 3 times, and sleep 100 ms each time 79 | gotasks.Register(uniqueJobName, Handler1, gotasks.Reentrant(Handler2, gotasks.WithMaxTimes(3), gotasks.WithSleepyMS(10))) 80 | } 81 | 82 | func worker() { 83 | ctx, cancel := context.WithCancel(context.Background()) 84 | defer cancel() 85 | 86 | gotasks.Run(ctx, queueName) 87 | } 88 | 89 | func main() { 90 | // set broker 91 | gotasks.UseRedisBroker(redisURL, gotasks.WithRedisTaskTTL(1000)) 92 | 93 | register() 94 | worker() 95 | } 96 | ``` 97 | 98 | ## License 99 | 100 | NOTE that from the first commit, we use a GPL-v3 open source license. 101 | -------------------------------------------------------------------------------- /broker.go: -------------------------------------------------------------------------------- 1 | package gotasks 2 | 3 | // NOTE: remember that functions in this file is not thread-safe(in Go, goroutine-safe), because we don't add a lock 4 | // to prevent functions call to UseRedisBroker. 5 | // But it is *safe* if you just call it once, in your initial code, it's unsafe if you change broker in serveral 6 | // goroutines. 7 | 8 | var ( 9 | broker Broker 10 | ) 11 | 12 | type Broker interface { 13 | Acquire(string) *Task 14 | Ack(*Task) bool 15 | Update(*Task) 16 | Enqueue(*Task) string 17 | QueueLen(string) int64 18 | } 19 | -------------------------------------------------------------------------------- /broker_test.go: -------------------------------------------------------------------------------- 1 | package gotasks 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestUseBadRedisBroker(t *testing.T) { 10 | assert.Panics(t, func() { UseRedisBroker("abcd", WithRedisTaskTTL(1)) }) 11 | assert.NotPanics(t, func() { UseRedisBroker(testRedisURL) }) 12 | } 13 | -------------------------------------------------------------------------------- /examples/consumer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/jiajunhuang/gotasks" 8 | //"github.com/jiajunhuang/gotasks/metrics" 9 | ) 10 | 11 | const ( 12 | uniqueJobName = "a-unique-job-name" 13 | redisURL = "redis://127.0.0.1:6379/0" 14 | queueName = "job-queue-name" 15 | ) 16 | 17 | func Handler1(args gotasks.ArgsMap) (gotasks.ArgsMap, error) { 18 | time.Sleep(time.Duration(1) * time.Second) 19 | return args, nil 20 | } 21 | func Handler2(args gotasks.ArgsMap) (gotasks.ArgsMap, error) { 22 | time.Sleep(time.Duration(1) * time.Second) 23 | return args, nil 24 | } 25 | 26 | func register() { 27 | // if handler1 failed, the task will stop, but if handler2 failed(return a non-nil error) 28 | // handler2 will be retry 3 times, and sleep 100 ms each time 29 | gotasks.Register(uniqueJobName, Handler1, gotasks.Reentrant(Handler2, gotasks.WithMaxTimes(3), gotasks.WithSleepyMS(10))) 30 | } 31 | 32 | func worker() { 33 | ctx, cancel := context.WithCancel(context.Background()) 34 | defer cancel() 35 | 36 | gotasks.Run(ctx, queueName) 37 | } 38 | 39 | func main() { 40 | // set broker 41 | gotasks.UseRedisBroker(redisURL, gotasks.WithRedisTaskTTL(1000)) 42 | 43 | register() 44 | worker() 45 | } 46 | -------------------------------------------------------------------------------- /examples/producer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/jiajunhuang/gotasks" 5 | //"github.com/jiajunhuang/gotasks/metrics" 6 | ) 7 | 8 | const ( 9 | uniqueJobName = "a-unique-job-name" 10 | redisURL = "redis://127.0.0.1:6379/0" 11 | queueName = "job-queue-name" 12 | ) 13 | 14 | func main() { 15 | // set broker 16 | gotasks.UseRedisBroker(redisURL, gotasks.WithRedisTaskTTL(1000)) 17 | 18 | // enqueue 19 | // or you can use a queue: 20 | queue := gotasks.NewQueue(queueName, gotasks.WithMaxLimit(10)) 21 | queue.Enqueue(uniqueJobName, gotasks.MapToArgsMap(map[string]interface{}{})) // or gotasks.StructToArgsMap 22 | } 23 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/jiajunhuang/gotasks 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/go-redis/redis/v7 v7.2.0 7 | github.com/google/uuid v1.1.1 8 | github.com/json-iterator/go v1.1.9 9 | github.com/prometheus/client_golang v1.5.1 10 | github.com/streadway/amqp v1.0.0 // indirect 11 | github.com/stretchr/testify v1.5.1 12 | ) 13 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 2 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 3 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 4 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 5 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 6 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 7 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 8 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 9 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= 10 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 15 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 16 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 17 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 18 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 19 | github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U= 20 | github.com/go-redis/redis/v7 v7.2.0 h1:CrCexy/jYWZjW0AyVoHlcJUeZN19VWlbepTh1Vq6dJs= 21 | github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= 22 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 23 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 24 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 25 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 26 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 27 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 28 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 29 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 30 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 31 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 32 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 33 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 34 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 35 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 36 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 37 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 38 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 39 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 40 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 41 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 42 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 43 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 44 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 45 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 46 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 47 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 48 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 49 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 50 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 51 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 52 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 53 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 54 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 55 | github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 56 | github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 57 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 58 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 59 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 60 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 61 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 62 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 63 | github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA= 64 | github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= 65 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 66 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 67 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= 68 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 69 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 70 | github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= 71 | github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= 72 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 73 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 74 | github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= 75 | github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= 76 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 77 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 78 | github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= 79 | github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= 80 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 81 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 82 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 83 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 84 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 85 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 86 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 87 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 88 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 89 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 90 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 91 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 92 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 93 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 94 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 95 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 96 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 97 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 98 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 99 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 100 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 101 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 102 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 103 | golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 104 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= 105 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 106 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 107 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 108 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 109 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 110 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 111 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 112 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 113 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 114 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 115 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 116 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 117 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 118 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 119 | gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= 120 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 121 | -------------------------------------------------------------------------------- /gotasks.go: -------------------------------------------------------------------------------- 1 | package gotasks 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "runtime/debug" 7 | "sync" 8 | "time" 9 | 10 | "github.com/jiajunhuang/gotasks/loop" 11 | "github.com/jiajunhuang/gotasks/pool" 12 | "github.com/prometheus/client_golang/prometheus" 13 | ) 14 | 15 | // gotasks is a job/task framework for Golang. 16 | // 17 | // Note that job will be executed in register order, and every job handle function 18 | // must have a signature which match gotasks.JobHandler, which receives a ArgsMap and 19 | // return a ArgsMap which will be arguments input for next handler. 20 | type AckWhenStatus int 21 | 22 | const ( 23 | AckWhenAcquired AckWhenStatus = iota 24 | AckWhenSucceed 25 | ) 26 | 27 | var ( 28 | jobMap = map[string][]JobHandler{} 29 | jobMapLock sync.RWMutex 30 | 31 | ackWhen = AckWhenSucceed 32 | ackWhenLock sync.Mutex 33 | 34 | // gotasks builtin queue 35 | FatalQueueName = "fatal" 36 | 37 | // prometheus 38 | taskHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ 39 | Name: "task_execution_stats", 40 | Help: "task execution duration and status(success/fail)", 41 | }, []string{"queue_name", "job_name", "status"}) 42 | taskGuage = prometheus.NewGaugeVec(prometheus.GaugeOpts{ 43 | Name: "task_queue_stats", 44 | Help: "task stats in queue", 45 | }, []string{"queue_name"}) 46 | ) 47 | 48 | func init() { 49 | prometheus.MustRegister(taskHistogram) 50 | prometheus.MustRegister(taskGuage) 51 | } 52 | 53 | // AckWhen set when will the ack be sent to broker 54 | func AckWhen(i AckWhenStatus) { 55 | ackWhenLock.Lock() 56 | defer ackWhenLock.Unlock() 57 | 58 | ackWhen = i 59 | } 60 | 61 | func Register(jobName string, handlers ...JobHandler) { 62 | jobMapLock.Lock() 63 | defer jobMapLock.Unlock() 64 | 65 | if _, ok := jobMap[jobName]; ok { 66 | log.Panicf("job name %s already exist, check your code", jobName) 67 | return // never executed here 68 | } 69 | 70 | jobMap[jobName] = handlers 71 | } 72 | 73 | func runHandlers(task *Task) { 74 | jobMapLock.RLock() 75 | defer jobMapLock.RUnlock() 76 | 77 | handlers, ok := jobMap[task.JobName] 78 | if !ok { 79 | log.Panicf("can't find job handlers of %s", task.JobName) 80 | return 81 | } 82 | 83 | var ( 84 | err error 85 | args = task.ArgsMap 86 | ) 87 | for i, handler := range handlers { 88 | if task.CurrentHandlerIndex > i { 89 | log.Printf("skip step %d of task %s because it was executed successfully", i, task.ID) 90 | continue 91 | } 92 | 93 | task.CurrentHandlerIndex = i 94 | handlerName := getHandlerName(handler) 95 | log.Printf("task %s is executing step %d with handler %s", task.ID, task.CurrentHandlerIndex, handlerName) 96 | 97 | reentrantMapLock.RLock() 98 | reentrantOptions, ok := reentrantMap[handlerName] 99 | reentrantMapLock.RUnlock() 100 | 101 | if ok { // check if the handler can retry 102 | for j := 0; j < reentrantOptions.MaxTimes; j++ { 103 | args, err = handler(args) 104 | if err == nil { 105 | break 106 | } 107 | time.Sleep(time.Microsecond * time.Duration(reentrantOptions.SleepyMS)) 108 | log.Printf("retry step %d of task %s the %d rd time", task.CurrentHandlerIndex, task.ID, j) 109 | } 110 | } else { 111 | args, err = handler(args) 112 | } 113 | 114 | // error occurred 115 | if err != nil { 116 | log.Panicf("failed to execute handler %s: %s", handlerName, err) 117 | } 118 | task.ArgsMap = args 119 | broker.Update(task) 120 | } 121 | } 122 | 123 | func handleTask(task *Task, queueName string) { 124 | defer func() { 125 | r := recover() 126 | status := "success" 127 | 128 | if r != nil { 129 | status = "fail" 130 | 131 | task.ResultLog = string(debug.Stack()) 132 | broker.Update(task) 133 | log.Printf("recovered from queue %s and task %+v with recover info %+v", queueName, task, r) 134 | } 135 | 136 | taskHistogram.WithLabelValues(task.QueueName, task.JobName, status).Observe(task.UpdatedAt.Sub(task.CreatedAt).Seconds()) 137 | 138 | if r != nil { 139 | // save to fatal queue 140 | task.QueueName = FatalQueueName 141 | broker.Enqueue(task) 142 | } 143 | }() 144 | 145 | runHandlers(task) 146 | } 147 | 148 | func run(ctx context.Context, wg *sync.WaitGroup, queue *Queue) { 149 | defer wg.Done() 150 | 151 | gopool := pool.NewGoPool(pool.WithMaxLimit(queue.MaxLimit)) 152 | defer gopool.Wait() 153 | 154 | err := loop.Execute(ctx, func() { 155 | fn := func() { 156 | task := broker.Acquire(queue.Name) 157 | 158 | if ackWhen == AckWhenAcquired { 159 | ok := broker.Ack(task) 160 | log.Printf("ack broker of task %+v with status %t", task.ID, ok) 161 | } 162 | handleTask(task, queue.Name) 163 | 164 | if ackWhen == AckWhenSucceed { 165 | ok := broker.Ack(task) 166 | log.Printf("ack broker of task %+v with status %t", task.ID, ok) 167 | } 168 | } 169 | 170 | if queue.Async { 171 | gopool.Submit(fn) 172 | } else { 173 | fn() 174 | } 175 | }) 176 | log.Printf("worker quit for queue %s: %s", queue.Name, err) 177 | } 178 | 179 | func monitorQueue(ctx context.Context, wg *sync.WaitGroup, queue *Queue) { 180 | defer wg.Done() 181 | 182 | err := loop.Execute(ctx, func() { 183 | taskGuage.WithLabelValues(queue.Name).Set(float64(broker.QueueLen(queue.Name))) 184 | time.Sleep(time.Second * time.Duration(queue.MonitorInterval)) 185 | }) 186 | log.Printf("monitor quit for queue %s: %s", queue.Name, err) 187 | } 188 | 189 | // Run a worker that listen on queues 190 | func Run(ctx context.Context, queueNames ...string) { 191 | wg := sync.WaitGroup{} 192 | 193 | wg.Add(1) 194 | go monitorQueue(ctx, &wg, NewQueue(FatalQueueName)) 195 | for _, queueName := range queueNames { 196 | wg.Add(2) 197 | queue := NewQueue(queueName) 198 | go run(ctx, &wg, queue) 199 | go monitorQueue(ctx, &wg, queue) 200 | } 201 | 202 | wg.Wait() 203 | } 204 | -------------------------------------------------------------------------------- /gotasks_test.go: -------------------------------------------------------------------------------- 1 | package gotasks 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "log" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | const ( 14 | testJobName = "test_job" 15 | testPanicJobName = "test_panic_job" 16 | testArgsPassJobName = "test_args_pass_job" 17 | testReentrantJobName = "test_reentrant_job" 18 | testHandlerNotFoundJobName = "test_handler_not_found" 19 | 20 | testQueueName = "test_queue" 21 | testRedisURL = "redis://127.0.0.1:6379/0" 22 | ) 23 | 24 | func TestAckWhen(t *testing.T) { 25 | AckWhen(AckWhenAcquired) 26 | AckWhen(AckWhenSucceed) 27 | } 28 | 29 | func TestGenFunctions(t *testing.T) { 30 | assert.Equal(t, "gt:task:abcd", genTaskName("abcd")) 31 | assert.Equal(t, "gt:queue:abcd", genQueueName("abcd")) 32 | } 33 | 34 | func TestRedisBroker(t *testing.T) { 35 | // register tasks 36 | handler1 := func(args ArgsMap) (ArgsMap, error) { 37 | time.Sleep(time.Duration(1) * time.Microsecond) 38 | return args, nil 39 | } 40 | handler2 := func(args ArgsMap) (ArgsMap, error) { 41 | time.Sleep(time.Duration(1) * time.Microsecond) 42 | return args, nil 43 | } 44 | Register(testJobName, handler1, handler2) 45 | 46 | // set broker 47 | UseRedisBroker(testRedisURL, WithRedisTaskTTL(100)) 48 | 49 | // enqueue 50 | log.Printf("current jobMap: %+v", jobMap) 51 | queue := NewQueue(testQueueName, WithMaxLimit(20), WithMonitorInterval(5)) 52 | taskID := queue.Enqueue(testJobName, MapToArgsMap(map[string]interface{}{})) 53 | defer rc.Del(genTaskName(taskID)) 54 | 55 | ctx, cancel := context.WithCancel(context.Background()) 56 | go Run(ctx, testQueueName) // it will blocking until the first job is executed 57 | time.Sleep(time.Second * time.Duration(1)) 58 | cancel() 59 | log.Printf("Run function returned, ctx: %+v", ctx) 60 | } 61 | 62 | func TestPanicHandler(t *testing.T) { 63 | // register tasks 64 | handler1 := func(args ArgsMap) (ArgsMap, error) { 65 | time.Sleep(time.Duration(1) * time.Microsecond) 66 | return args, nil 67 | } 68 | handler2 := func(args ArgsMap) (ArgsMap, error) { 69 | time.Sleep(time.Duration(1) * time.Microsecond) 70 | panic("whoops") 71 | //return args, nil 72 | } 73 | handler3 := func(args ArgsMap) (ArgsMap, error) { 74 | time.Sleep(time.Duration(1) * time.Microsecond) 75 | return args, nil 76 | } 77 | Register(testPanicJobName, handler1, handler2, handler3) 78 | 79 | // set broker 80 | UseRedisBroker(testRedisURL) 81 | 82 | // enqueue 83 | log.Printf("current jobMap: %+v", jobMap) 84 | queue := NewQueue(testQueueName) 85 | taskID := queue.Enqueue(testPanicJobName, MapToArgsMap(map[string]interface{}{})) 86 | defer rc.Del(genTaskName(taskID)) 87 | 88 | ctx, cancel := context.WithCancel(context.Background()) 89 | go Run(ctx, testQueueName) // it will blocking until the first job is executed 90 | time.Sleep(time.Second * time.Duration(1)) 91 | cancel() 92 | log.Printf("Run function returned, ctx: %+v", ctx) 93 | 94 | // check result 95 | taskBytes := []byte{} 96 | if err := rc.Get(genTaskName(taskID)).Scan(&taskBytes); err != nil { 97 | t.Logf("failed to get task %s: %s", taskID, err) 98 | t.FailNow() 99 | } 100 | task := Task{} 101 | if err := json.Unmarshal(taskBytes, &task); err != nil { 102 | t.Logf("failed to get task %s: %s", taskID, err) 103 | t.FailNow() 104 | } 105 | assert.Equal(t, 1, task.CurrentHandlerIndex) 106 | 107 | // check result ttl 108 | duration, err := rc.TTL(genTaskName(taskID)).Result() 109 | if err != nil { 110 | t.Logf("task %s should have ttl with err %s", taskID, err) 111 | t.FailNow() 112 | } 113 | if duration.Seconds() == 0 { 114 | t.Logf("task %s should have ttl but not", taskID) 115 | t.FailNow() 116 | } 117 | } 118 | 119 | func TestArgsPass(t *testing.T) { 120 | // register tasks 121 | handler1 := func(args ArgsMap) (ArgsMap, error) { 122 | time.Sleep(time.Duration(1) * time.Microsecond) 123 | args["hello"] = "world" 124 | return args, nil 125 | } 126 | handler2 := func(args ArgsMap) (ArgsMap, error) { 127 | time.Sleep(time.Duration(1) * time.Microsecond) 128 | assert.Equal(t, "world", args["hello"]) 129 | return args, nil 130 | } 131 | Register(testArgsPassJobName, handler1, handler2) 132 | 133 | // set broker 134 | UseRedisBroker(testRedisURL) 135 | 136 | // enqueue 137 | log.Printf("current jobMap: %+v", jobMap) 138 | queue := NewQueue(testQueueName) 139 | taskID := queue.Enqueue(testArgsPassJobName, MapToArgsMap(map[string]interface{}{})) 140 | defer rc.Del(genTaskName(taskID)) 141 | 142 | ctx, cancel := context.WithCancel(context.Background()) 143 | go Run(ctx, testQueueName) // it will blocking until the first job is executed 144 | time.Sleep(time.Second * time.Duration(1)) 145 | cancel() 146 | log.Printf("Run function returned, ctx: %+v", ctx) 147 | } 148 | 149 | func TestReentrant(t *testing.T) { 150 | // register tasks 151 | handler1 := func(args ArgsMap) (ArgsMap, error) { 152 | time.Sleep(time.Duration(1) * time.Microsecond) 153 | args["hello"] = "world" 154 | return args, nil 155 | } 156 | handler2 := func(args ArgsMap) (ArgsMap, error) { 157 | return args, errors.New("hello world error") 158 | } 159 | 160 | Register(testReentrantJobName, handler1, Reentrant(handler2, WithMaxTimes(3), WithSleepyMS(10))) 161 | 162 | // set broker 163 | UseRedisBroker(testRedisURL) 164 | 165 | // enqueue 166 | log.Printf("current jobMap: %+v", jobMap) 167 | queue := NewQueue(testQueueName) 168 | taskID := queue.Enqueue(testReentrantJobName, MapToArgsMap(map[string]interface{}{})) 169 | defer rc.Del(genTaskName(taskID)) 170 | 171 | ctx, cancel := context.WithCancel(context.Background()) 172 | go Run(ctx, testQueueName) // it will blocking until the first job is executed 173 | time.Sleep(time.Second * time.Duration(1)) 174 | cancel() 175 | log.Printf("Run function returned, ctx: %+v", ctx) 176 | } 177 | 178 | func TestJobHandlerNotFound(t *testing.T) { 179 | // set broker 180 | UseRedisBroker(testRedisURL) 181 | 182 | // enqueue 183 | log.Printf("current jobMap: %+v", jobMap) 184 | queue := NewQueue(testQueueName) 185 | taskID := queue.Enqueue(testHandlerNotFoundJobName, MapToArgsMap(map[string]interface{}{})) 186 | defer rc.Del(genTaskName(taskID)) 187 | 188 | ctx, cancel := context.WithCancel(context.Background()) 189 | go Run(ctx, testQueueName) // it will blocking until the first job is executed 190 | time.Sleep(time.Second * time.Duration(1)) 191 | cancel() 192 | log.Printf("Run function returned, ctx: %+v", ctx) 193 | } 194 | -------------------------------------------------------------------------------- /handler.go: -------------------------------------------------------------------------------- 1 | package gotasks 2 | 3 | import ( 4 | "log" 5 | "reflect" 6 | "runtime" 7 | "sync" 8 | ) 9 | 10 | type JobHandler func(ArgsMap) (ArgsMap, error) 11 | 12 | type ReentrantOptions struct { 13 | MaxTimes int 14 | SleepyMS int 15 | } 16 | 17 | type ReentrantOption func(*ReentrantOptions) 18 | 19 | func WithMaxTimes(max int) ReentrantOption { 20 | return func(ro *ReentrantOptions) { 21 | ro.MaxTimes = max 22 | } 23 | } 24 | 25 | func WithSleepyMS(ms int) ReentrantOption { 26 | return func(ro *ReentrantOptions) { 27 | ro.SleepyMS = ms 28 | } 29 | } 30 | 31 | var ( 32 | reentrantMap = map[string]ReentrantOptions{} 33 | reentrantMapLock sync.RWMutex 34 | ) 35 | 36 | func getHandlerName(handler JobHandler) string { 37 | return runtime.FuncForPC(reflect.ValueOf(handler).Pointer()).Name() 38 | } 39 | 40 | func Reentrant(handler JobHandler, options ...ReentrantOption) JobHandler { 41 | handlerName := getHandlerName(handler) 42 | reentrantMapLock.Lock() 43 | defer reentrantMapLock.Unlock() 44 | 45 | if _, ok := reentrantMap[handlerName]; ok { 46 | log.Panicf("reentrant options of %s already exists!", handlerName) 47 | return nil // never executed here 48 | } 49 | 50 | reentrantOptions := &ReentrantOptions{0, 0} 51 | for _, o := range options { 52 | o(reentrantOptions) 53 | } 54 | reentrantMap[handlerName] = *reentrantOptions 55 | 56 | return handler 57 | } 58 | -------------------------------------------------------------------------------- /json.go: -------------------------------------------------------------------------------- 1 | package gotasks 2 | 3 | import ( 4 | "github.com/json-iterator/go" 5 | ) 6 | 7 | var ( 8 | json = jsoniter.ConfigCompatibleWithStandardLibrary 9 | ) 10 | -------------------------------------------------------------------------------- /loop/loop.go: -------------------------------------------------------------------------------- 1 | package loop 2 | 3 | import ( 4 | "context" 5 | "log" 6 | ) 7 | 8 | // Execute loop execute fn until ctx.Done() is received 9 | func Execute(ctx context.Context, fn func()) error { 10 | for { 11 | select { 12 | case <-ctx.Done(): 13 | err := ctx.Err() 14 | log.Printf("ctx is done for %s", err) 15 | return err 16 | default: 17 | fn() 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /loop/loop_test.go: -------------------------------------------------------------------------------- 1 | package loop 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestLoop(t *testing.T) { 10 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(100)*time.Microsecond) 11 | Execute(ctx, func() { time.Sleep(time.Microsecond * time.Duration(10)) }) 12 | cancel() 13 | } 14 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/prometheus/client_golang/prometheus/promhttp" 7 | ) 8 | 9 | // RunServer start a http server that print metrics in /metrics 10 | func RunServer(addr string) { 11 | http.Handle("/metrics", promhttp.Handler()) 12 | http.ListenAndServe(addr, nil) 13 | } 14 | -------------------------------------------------------------------------------- /pool/pool.go: -------------------------------------------------------------------------------- 1 | package pool 2 | 3 | import ( 4 | "log" 5 | ) 6 | 7 | type GoPool struct { 8 | MaxLimit int 9 | 10 | tokenChan chan struct{} 11 | } 12 | 13 | type GoPoolOption func(*GoPool) 14 | 15 | func WithMaxLimit(max int) GoPoolOption { 16 | return func(gp *GoPool) { 17 | gp.MaxLimit = max 18 | gp.tokenChan = make(chan struct{}, gp.MaxLimit) 19 | 20 | for i := 0; i < gp.MaxLimit; i++ { 21 | gp.tokenChan <- struct{}{} 22 | } 23 | } 24 | } 25 | 26 | func NewGoPool(options ...GoPoolOption) *GoPool { 27 | p := &GoPool{} 28 | for _, o := range options { 29 | o(p) 30 | } 31 | 32 | return p 33 | } 34 | 35 | // Submit will wait a token, and then execute fn 36 | func (gp *GoPool) Submit(fn func()) { 37 | token := <-gp.tokenChan // if there are no tokens, we'll block here 38 | 39 | go func() { 40 | defer func() { 41 | gp.tokenChan <- token 42 | if r := recover(); r != nil { 43 | log.Printf("fn paniced: %s", r) 44 | } 45 | }() 46 | fn() 47 | }() 48 | } 49 | 50 | // Wait will wait all the tasks executed, and then return 51 | func (gp *GoPool) Wait() { 52 | for i := 0; i < gp.MaxLimit; i++ { 53 | <-gp.tokenChan 54 | } 55 | 56 | close(gp.tokenChan) 57 | } 58 | 59 | func (gp *GoPool) size() int { 60 | return len(gp.tokenChan) 61 | } 62 | -------------------------------------------------------------------------------- /pool/pool_test.go: -------------------------------------------------------------------------------- 1 | package pool 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestGoPool(t *testing.T) { 11 | gopool := NewGoPool(WithMaxLimit(3)) 12 | for i := 0; i < 10; i++ { 13 | go gopool.Submit(func() { time.Sleep(time.Hour * time.Duration(1)) }) 14 | } 15 | 16 | assert.Equal(t, 0, gopool.size()) 17 | } 18 | 19 | func TestGoPoolWithPanicFn(t *testing.T) { 20 | gopool := NewGoPool(WithMaxLimit(1)) 21 | for i := 0; i < 10; i++ { 22 | go gopool.Submit(func() { panic("whoops") }) 23 | } 24 | 25 | assert.Equal(t, 0, gopool.size()) 26 | } 27 | -------------------------------------------------------------------------------- /queue.go: -------------------------------------------------------------------------------- 1 | package gotasks 2 | 3 | import ( 4 | "log" 5 | ) 6 | 7 | type Queue struct { 8 | Name string 9 | MaxLimit int 10 | Async bool 11 | 12 | // monitor 13 | MonitorInterval int 14 | } 15 | 16 | type QueueOption func(*Queue) 17 | 18 | func WithMaxLimit(max int) QueueOption { 19 | return func(q *Queue) { 20 | q.MaxLimit = max 21 | } 22 | } 23 | 24 | func WithMonitorInterval(seconds int) QueueOption { 25 | return func(q *Queue) { 26 | q.MonitorInterval = seconds 27 | } 28 | } 29 | 30 | func WithAsyncHandleTask(async bool) QueueOption { 31 | return func(q *Queue) { 32 | q.Async = async 33 | } 34 | } 35 | 36 | func NewQueue(name string, options ...QueueOption) *Queue { 37 | queue := &Queue{name, 10, false, 5} 38 | 39 | for _, o := range options { 40 | o(queue) 41 | } 42 | 43 | return queue 44 | } 45 | 46 | func (q *Queue) Enqueue(jobName string, argsMap ArgsMap) string { 47 | return enqueue(q.Name, jobName, argsMap) 48 | } 49 | 50 | // enqueue a job(which will be wrapped in task) into queue 51 | func enqueue(queueName, jobName string, argsMap ArgsMap) string { 52 | taskID := broker.Enqueue(NewTask(queueName, jobName, argsMap)) 53 | log.Printf("job %s enqueued to %s, taskID is %s", jobName, queueName, taskID) 54 | return taskID 55 | } 56 | -------------------------------------------------------------------------------- /redis_broker.go: -------------------------------------------------------------------------------- 1 | package gotasks 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | redis "github.com/go-redis/redis/v7" 8 | ) 9 | 10 | var ( 11 | _ Broker = &RedisBroker{} 12 | 13 | // rc: RedisClient 14 | rc *redis.Client 15 | ) 16 | 17 | func genTaskName(taskID string) string { 18 | return "gt:task:" + taskID 19 | } 20 | 21 | func genQueueName(queueName string) string { 22 | return "gt:queue:" + queueName 23 | } 24 | 25 | type RedisBroker struct { 26 | TaskTTL int 27 | } 28 | 29 | type RedisBrokerOption func(rb *RedisBroker) 30 | 31 | func WithRedisTaskTTL(ttl int) RedisBrokerOption { 32 | return func(rb *RedisBroker) { 33 | rb.TaskTTL = ttl 34 | } 35 | } 36 | 37 | func UseRedisBroker(redisURL string, brokerOptions ...RedisBrokerOption) { 38 | options, err := redis.ParseURL(redisURL) 39 | if err != nil { 40 | log.Panicf("failed to parse redis URL %s: %s", redisURL, err) 41 | } 42 | 43 | rc = redis.NewClient(options) 44 | rb := &RedisBroker{} 45 | for _, o := range brokerOptions { 46 | o(rb) 47 | } 48 | 49 | broker = rb 50 | } 51 | 52 | func (r *RedisBroker) Acquire(queueName string) *Task { 53 | task := Task{} 54 | vs, err := rc.BRPop(time.Duration(0), genQueueName(queueName)).Result() 55 | if err != nil { 56 | log.Panicf("failed to get task from redis: %s", err) 57 | return nil // never executed 58 | } 59 | v := []byte(vs[1]) 60 | 61 | if err := json.Unmarshal(v, &task); err != nil { 62 | log.Panicf("failed to get task from redis: %s", err) 63 | return nil // never executed 64 | } 65 | 66 | return &task 67 | } 68 | 69 | func (r *RedisBroker) Ack(task *Task) bool { 70 | // redis doesn't support ACK 71 | return true 72 | } 73 | 74 | func (r *RedisBroker) Update(task *Task) { 75 | task.UpdatedAt = time.Now() 76 | taskBytes, err := json.Marshal(task) 77 | if err != nil { 78 | log.Panicf("failed to enquue task %+v: %s", task, err) 79 | return // never executed here 80 | } 81 | rc.Set(genTaskName(task.ID), taskBytes, time.Duration(r.TaskTTL)*time.Second) 82 | } 83 | 84 | func (r *RedisBroker) Enqueue(task *Task) string { 85 | taskBytes, err := json.Marshal(task) 86 | if err != nil { 87 | log.Panicf("failed to enquue task %+v: %s", task, err) 88 | return "" // never executed here 89 | } 90 | 91 | rc.Set(genTaskName(task.ID), taskBytes, time.Duration(r.TaskTTL)*time.Second) 92 | rc.LPush(genQueueName(task.QueueName), taskBytes) 93 | return task.ID 94 | } 95 | 96 | func (r *RedisBroker) QueueLen(queueName string) int64 { 97 | l, _ := rc.LLen(genQueueName(queueName)).Result() 98 | return l 99 | } 100 | -------------------------------------------------------------------------------- /task_job.go: -------------------------------------------------------------------------------- 1 | package gotasks 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/google/uuid" 8 | ) 9 | 10 | type ArgsMap map[string]interface{} 11 | 12 | // StructToArgsMap Convert struct to ArgsMap, e.g. am := StructToArgsMap(yourStruct) 13 | func StructToArgsMap(v interface{}) ArgsMap { 14 | v_bytes, err := json.Marshal(v) 15 | if err != nil { 16 | log.Panicf("failed to convert %+v to ArgsMap: %s", v, err) 17 | } 18 | argsMap := ArgsMap{} 19 | err = json.Unmarshal(v_bytes, &argsMap) 20 | if err != nil { 21 | log.Panicf("failed to convert %+v to ArgsMap: %s", v, err) 22 | } 23 | 24 | return argsMap 25 | } 26 | 27 | // MapToArgsMap Convert golang map to ArgsMap, e.g. am := MapToArgsMap(yourStruct) 28 | func MapToArgsMap(v interface{}) ArgsMap { 29 | return StructToArgsMap(v) 30 | } 31 | 32 | // ArgsMapToStruct Convert ArgsMap to struct, e.g. err := ArgsMapToStruct(am, &yourStruct) 33 | func ArgsMapToStruct(am ArgsMap, s interface{}) error { 34 | v_bytes, err := json.Marshal(am) 35 | if err != nil { 36 | return err 37 | } 38 | 39 | return json.Unmarshal(v_bytes, s) 40 | } 41 | 42 | type Task struct { 43 | ID string `json:"task_id"` 44 | CreatedAt time.Time `json:"created_at"` 45 | UpdatedAt time.Time `json:"updated_at"` 46 | QueueName string `json:"queue_name"` 47 | JobName string `json:"job_name"` 48 | ArgsMap ArgsMap `json:"args_map"` 49 | CurrentHandlerIndex int `json:"current_handler_index"` 50 | OriginalArgsMap ArgsMap `json:"original_args_map"` 51 | ResultLog string `json:"result_log"` 52 | } 53 | 54 | func NewTask(queueName, jobName string, argsMap ArgsMap) *Task { 55 | u, _ := uuid.NewUUID() 56 | now := time.Now() 57 | 58 | task := &Task{u.String(), now, now, queueName, jobName, argsMap, 0, argsMap, ""} 59 | 60 | return task 61 | } 62 | -------------------------------------------------------------------------------- /task_job_test.go: -------------------------------------------------------------------------------- 1 | package gotasks 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestArgsMapToStruct(t *testing.T) { 10 | type Case struct { 11 | Name string `json:"name"` 12 | This int `json:"this"` 13 | } 14 | 15 | s := Case{"hello", 1} 16 | 17 | am := StructToArgsMap(s) 18 | 19 | s2 := Case{} 20 | err := ArgsMapToStruct(am, &s2) 21 | if assert.NoError(t, err) { 22 | assert.Equal(t, s, s2) 23 | } 24 | } 25 | --------------------------------------------------------------------------------