├── worker_func.go ├── job.go ├── payload.go ├── .travis.yml ├── work.go ├── signal_stop_1.0.go ├── signal_stop_1.1.go ├── failure.go ├── interval_flag.go ├── process_test.go ├── workers.go ├── queues_flag.go ├── LICENSE ├── redis.go ├── signals.go ├── worker_test.go ├── interval_flag_test.go ├── process.go ├── queues_flag_test.go ├── doc.go ├── poller.go ├── goworker.go ├── worker.go ├── flags.go └── README.md /worker_func.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | type workerFunc func(string, ...interface{}) error 4 | -------------------------------------------------------------------------------- /job.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | type Job struct { 4 | Queue string 5 | Payload Payload 6 | } 7 | -------------------------------------------------------------------------------- /payload.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | type Payload struct { 4 | Class string `json:"class"` 5 | Args []interface{} `json:"args"` 6 | } 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.1 5 | - 1.2 6 | - 1.5.2 7 | - tip 8 | 9 | services: 10 | - redis-server 11 | 12 | matrix: 13 | allowed_failures: 14 | go: 15 | - tip 16 | -------------------------------------------------------------------------------- /work.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type work struct { 8 | Queue string `json:"queue"` 9 | RunAt time.Time `json:"run_at"` 10 | Payload Payload `json:"payload"` 11 | } 12 | -------------------------------------------------------------------------------- /signal_stop_1.0.go: -------------------------------------------------------------------------------- 1 | // +build !go1.1 2 | 3 | package goworker 4 | 5 | import ( 6 | "os" 7 | ) 8 | 9 | // Stops signals channel. This does not exist in 10 | // Go less than 1.1. 11 | func signalStop(c chan<- os.Signal) { 12 | } 13 | -------------------------------------------------------------------------------- /signal_stop_1.1.go: -------------------------------------------------------------------------------- 1 | // +build go1.1 2 | 3 | package goworker 4 | 5 | import ( 6 | "os" 7 | "os/signal" 8 | ) 9 | 10 | // Stops signals channel. This function exists 11 | // in Go greater or equal to 1.1. 12 | func signalStop(c chan<- os.Signal) { 13 | signal.Stop(c) 14 | } 15 | -------------------------------------------------------------------------------- /failure.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | type failure struct { 8 | FailedAt time.Time `json:"failed_at"` 9 | Payload Payload `json:"payload"` 10 | Exception string `json:"exception"` 11 | Error string `json:"error"` 12 | Backtrace []string `json:"backtrace"` 13 | Worker *worker `json:"worker"` 14 | Queue string `json:"queue"` 15 | } 16 | -------------------------------------------------------------------------------- /interval_flag.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "time" 7 | ) 8 | 9 | type intervalFlag time.Duration 10 | 11 | func (i *intervalFlag) Set(value string) error { 12 | f, err := strconv.ParseFloat(value, 64) 13 | if err != nil { 14 | return err 15 | } 16 | i.SetFloat(f) 17 | return nil 18 | } 19 | 20 | func (i *intervalFlag) SetFloat(value float64) error { 21 | *i = intervalFlag(time.Duration(value * float64(time.Second))) 22 | return nil 23 | } 24 | 25 | func (i *intervalFlag) String() string { 26 | return fmt.Sprint(*i) 27 | } 28 | -------------------------------------------------------------------------------- /process_test.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | var processStringTests = []struct { 8 | p process 9 | expected string 10 | }{ 11 | { 12 | process{}, 13 | ":0-:", 14 | }, 15 | { 16 | process{ 17 | Hostname: "hostname", 18 | Pid: 12345, 19 | ID: "123", 20 | Queues: []string{"high", "low"}, 21 | }, 22 | "hostname:12345-123:high,low", 23 | }, 24 | } 25 | 26 | func TestProcessString(t *testing.T) { 27 | for _, tt := range processStringTests { 28 | actual := tt.p.String() 29 | if actual != tt.expected { 30 | t.Errorf("Process(%#v): expected %s, actual %s", tt.p, tt.expected, actual) 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /workers.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | ) 7 | 8 | var ( 9 | workers map[string]workerFunc 10 | ) 11 | 12 | func init() { 13 | workers = make(map[string]workerFunc) 14 | } 15 | 16 | // Register registers a goworker worker function. Class 17 | // refers to the Ruby name of the class which enqueues the 18 | // job. Worker is a function which accepts a queue and an 19 | // arbitrary array of interfaces as arguments. 20 | func Register(class string, worker workerFunc) { 21 | workers[class] = worker 22 | } 23 | 24 | func Enqueue(job *Job) error { 25 | err := Init() 26 | if err != nil { 27 | return err 28 | } 29 | 30 | conn, err := GetConn() 31 | if err != nil { 32 | logger.Criticalf("Error on getting connection on enqueue") 33 | return err 34 | } 35 | defer PutConn(conn) 36 | 37 | buffer, err := json.Marshal(job.Payload) 38 | if err != nil { 39 | logger.Criticalf("Cant marshal payload on enqueue") 40 | return err 41 | } 42 | err = conn.Send("RPUSH", fmt.Sprintf("%squeue:%s", workerSettings.Namespace, job.Queue), buffer) 43 | if err != nil { 44 | logger.Criticalf("Cant push to queue") 45 | return err 46 | } 47 | 48 | return conn.Flush() 49 | } 50 | -------------------------------------------------------------------------------- /queues_flag.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | var ( 11 | errorEmptyQueues = errors.New("you must specify at least one queue") 12 | errorNonNumericWeight = errors.New("the weight must be a numeric value") 13 | ) 14 | 15 | type queuesFlag []string 16 | 17 | func (q *queuesFlag) Set(value string) error { 18 | for _, queueAndWeight := range strings.Split(value, ",") { 19 | if queueAndWeight == "" { 20 | continue 21 | } 22 | 23 | queue, weight, err := parseQueueAndWeight(queueAndWeight) 24 | if err != nil { 25 | return err 26 | } 27 | 28 | for i := 0; i < weight; i++ { 29 | *q = append(*q, queue) 30 | } 31 | } 32 | if len(*q) == 0 { 33 | return errorEmptyQueues 34 | } 35 | return nil 36 | } 37 | 38 | func (q *queuesFlag) String() string { 39 | return fmt.Sprint(*q) 40 | } 41 | 42 | func parseQueueAndWeight(queueAndWeight string) (queue string, weight int, err error) { 43 | parts := strings.SplitN(queueAndWeight, "=", 2) 44 | queue = parts[0] 45 | 46 | if queue == "" { 47 | return 48 | } 49 | 50 | if len(parts) == 1 { 51 | weight = 1 52 | } else { 53 | weight, err = strconv.Atoi(parts[1]) 54 | if err != nil { 55 | err = errorNonNumericWeight 56 | } 57 | } 58 | return 59 | } 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 Benjamin Manns 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | 22 | The goworker Logo 23 | 24 | The goworker logo is a work by Rachel Falwell combining the Go mascot by Renée 25 | French (CC-BY) and the Ruby logo by the Ruby Visual Identity Team (CC-BY-SA). 26 | The logo is released under a Creative Commons Attribution-ShareAlike 4.0 27 | International License in keeping with the restrictions of the works from which 28 | it is derived. 29 | -------------------------------------------------------------------------------- /redis.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "errors" 5 | "net/url" 6 | "time" 7 | 8 | "github.com/garyburd/redigo/redis" 9 | "github.com/youtube/vitess/go/pools" 10 | ) 11 | 12 | var ( 13 | errorInvalidScheme = errors.New("invalid Redis database URI scheme") 14 | ) 15 | 16 | type RedisConn struct { 17 | redis.Conn 18 | } 19 | 20 | func (r *RedisConn) Close() { 21 | _ = r.Conn.Close() 22 | } 23 | 24 | func newRedisFactory(uri string) pools.Factory { 25 | return func() (pools.Resource, error) { 26 | return redisConnFromURI(uri) 27 | } 28 | } 29 | 30 | func newRedisPool(uri string, capacity int, maxCapacity int, idleTimout time.Duration) *pools.ResourcePool { 31 | return pools.NewResourcePool(newRedisFactory(uri), capacity, maxCapacity, idleTimout) 32 | } 33 | 34 | func redisConnFromURI(uriString string) (*RedisConn, error) { 35 | uri, err := url.Parse(uriString) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | var network string 41 | var host string 42 | var password string 43 | var db string 44 | 45 | switch uri.Scheme { 46 | case "redis": 47 | network = "tcp" 48 | host = uri.Host 49 | if uri.User != nil { 50 | password, _ = uri.User.Password() 51 | } 52 | if len(uri.Path) > 1 { 53 | db = uri.Path[1:] 54 | } 55 | case "unix": 56 | network = "unix" 57 | host = uri.Path 58 | default: 59 | return nil, errorInvalidScheme 60 | } 61 | 62 | conn, err := redis.Dial(network, host) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | if password != "" { 68 | _, err := conn.Do("AUTH", password) 69 | if err != nil { 70 | conn.Close() 71 | return nil, err 72 | } 73 | } 74 | 75 | if db != "" { 76 | _, err := conn.Do("SELECT", db) 77 | if err != nil { 78 | conn.Close() 79 | return nil, err 80 | } 81 | } 82 | 83 | return &RedisConn{Conn: conn}, nil 84 | } 85 | -------------------------------------------------------------------------------- /signals.go: -------------------------------------------------------------------------------- 1 | // Signal Handling in goworker 2 | // 3 | // To stop goworker, send a QUIT, TERM, or INT 4 | // signal to the process. This will immediately 5 | // stop job polling. There can be up to 6 | // $CONCURRENCY jobs currently running, which 7 | // will continue to run until they are finished. 8 | // 9 | // Failure Modes 10 | // 11 | // Like Resque, goworker makes no guarantees 12 | // about the safety of jobs in the event of 13 | // process shutdown. Workers must be both 14 | // idempotent and tolerant to loss of the job in 15 | // the event of failure. 16 | // 17 | // If the process is killed with a KILL or by a 18 | // system failure, there may be one job that is 19 | // currently in the poller's buffer that will be 20 | // lost without any representation in either the 21 | // queue or the worker variable. 22 | // 23 | // If you are running Goworker on a system like 24 | // Heroku, which sends a TERM to signal a process 25 | // that it needs to stop, ten seconds later sends 26 | // a KILL to force the process to stop, your jobs 27 | // must finish within 10 seconds or they may be 28 | // lost. Jobs will be recoverable from the Redis 29 | // database under 30 | // 31 | // resque:worker::-: 32 | // 33 | // as a JSON object with keys queue, run_at, and 34 | // payload, but the process is manual. 35 | // Additionally, there is no guarantee that the 36 | // job in Redis under the worker key has not 37 | // finished, if the process is killed before 38 | // goworker can flush the update to Redis. 39 | package goworker 40 | 41 | import ( 42 | "os" 43 | "os/signal" 44 | "syscall" 45 | ) 46 | 47 | func signals() <-chan bool { 48 | quit := make(chan bool) 49 | 50 | go func() { 51 | signals := make(chan os.Signal) 52 | defer close(signals) 53 | 54 | signal.Notify(signals, syscall.SIGQUIT, syscall.SIGTERM, os.Interrupt) 55 | defer signalStop(signals) 56 | 57 | <-signals 58 | quit <- true 59 | }() 60 | 61 | return quit 62 | } 63 | -------------------------------------------------------------------------------- /worker_test.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | var workerMarshalJSONTests = []struct { 9 | w worker 10 | expected []byte 11 | }{ 12 | { 13 | worker{}, 14 | []byte(`":0-:"`), 15 | }, 16 | { 17 | worker{ 18 | process: process{ 19 | Hostname: "hostname", 20 | Pid: 12345, 21 | ID: "123", 22 | Queues: []string{"high", "low"}, 23 | }, 24 | }, 25 | []byte(`"hostname:12345-123:high,low"`), 26 | }, 27 | } 28 | 29 | func TestWorkerMarshalJSON(t *testing.T) { 30 | for _, tt := range workerMarshalJSONTests { 31 | actual, err := tt.w.MarshalJSON() 32 | if err != nil { 33 | t.Errorf("Worker(%#v): error %s", tt.w, err) 34 | } else { 35 | if string(actual) != string(tt.expected) { 36 | t.Errorf("Worker(%#v): expected %s, actual %s", tt.w, tt.expected, actual) 37 | } 38 | } 39 | } 40 | } 41 | 42 | func TestEnqueue(t *testing.T) { 43 | expectedArgs := []interface{}{"a", "lot", "of", "params"} 44 | jobName := "SomethingCool" 45 | queueName := "testQueue" 46 | expectedJob := &Job{ 47 | Queue: queueName, 48 | Payload: Payload{ 49 | Class: jobName, 50 | Args: expectedArgs, 51 | }, 52 | } 53 | 54 | workerSettings.Queues = []string{queueName} 55 | workerSettings.UseNumber = true 56 | workerSettings.ExitOnComplete = true 57 | 58 | err := Enqueue(expectedJob) 59 | if err != nil { 60 | t.Errorf("Error while enqueue %s", err) 61 | } 62 | 63 | actualArgs := []interface{}{} 64 | actualQueueName := "" 65 | Register(jobName, func(queue string, args ...interface{}) error { 66 | actualArgs = args 67 | actualQueueName = queue 68 | return nil 69 | }) 70 | if err := Work(); err != nil { 71 | t.Errorf("(Enqueue) Failed on work %s", err) 72 | } 73 | if !reflect.DeepEqual(actualArgs, expectedArgs) { 74 | t.Errorf("(Enqueue) Expected %v, actual %v", actualArgs, expectedArgs) 75 | } 76 | if !reflect.DeepEqual(actualQueueName, queueName) { 77 | t.Errorf("(Enqueue) Expected %v, actual %v", actualQueueName, queueName) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /interval_flag_test.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | var intervalFlagSetTests = []struct { 9 | v string 10 | expected intervalFlag 11 | }{ 12 | { 13 | "0", 14 | intervalFlag(0), 15 | }, 16 | { 17 | "1", 18 | intervalFlag(1 * time.Second), 19 | }, 20 | { 21 | "1.5", 22 | intervalFlag(1500 * time.Millisecond), 23 | }, 24 | } 25 | 26 | func TestIntervalFlagSet(t *testing.T) { 27 | for _, tt := range intervalFlagSetTests { 28 | actual := new(intervalFlag) 29 | if err := actual.Set(tt.v); err != nil { 30 | t.Errorf("IntervalFlag(%#v): set to %s error %s", actual, tt.v, err) 31 | } else { 32 | if *actual != tt.expected { 33 | t.Errorf("IntervalFlag: set to %s expected %v, actual %v", tt.v, tt.expected, actual) 34 | } 35 | } 36 | } 37 | } 38 | 39 | var intervalFlagSetFloatTests = []struct { 40 | v float64 41 | expected intervalFlag 42 | }{ 43 | { 44 | 0.0, 45 | intervalFlag(0), 46 | }, 47 | { 48 | 1.0, 49 | intervalFlag(1 * time.Second), 50 | }, 51 | { 52 | 1.5, 53 | intervalFlag(1500 * time.Millisecond), 54 | }, 55 | } 56 | 57 | func TestIntervalFlagSetFloat(t *testing.T) { 58 | for _, tt := range intervalFlagSetFloatTests { 59 | actual := new(intervalFlag) 60 | if err := actual.SetFloat(tt.v); err != nil { 61 | t.Errorf("IntervalFlag(%#v): set to %f error %s", actual, tt.v, err) 62 | } else { 63 | if *actual != tt.expected { 64 | t.Errorf("IntervalFlag: set to %f expected %v, actual %v", tt.v, tt.expected, actual) 65 | } 66 | } 67 | } 68 | } 69 | 70 | var intervalFlagStringTests = []struct { 71 | i intervalFlag 72 | expected string 73 | }{ 74 | { 75 | intervalFlag(0), 76 | "0", 77 | }, 78 | { 79 | intervalFlag(1 * time.Second), 80 | "1000000000", 81 | }, 82 | { 83 | intervalFlag(1500 * time.Millisecond), 84 | "1500000000", 85 | }, 86 | } 87 | 88 | func TestIntervalFlagString(t *testing.T) { 89 | for _, tt := range intervalFlagStringTests { 90 | actual := tt.i.String() 91 | if actual != tt.expected { 92 | t.Errorf("IntervalFlag(%#v): expected %s, actual %s", tt.i, tt.expected, actual) 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /process.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "strings" 8 | "time" 9 | ) 10 | 11 | type process struct { 12 | Hostname string 13 | Pid int 14 | ID string 15 | Queues []string 16 | } 17 | 18 | func newProcess(id string, queues []string) (*process, error) { 19 | hostname, err := os.Hostname() 20 | if err != nil { 21 | return nil, err 22 | } 23 | 24 | return &process{ 25 | Hostname: hostname, 26 | Pid: os.Getpid(), 27 | ID: id, 28 | Queues: queues, 29 | }, nil 30 | } 31 | 32 | func (p *process) String() string { 33 | return fmt.Sprintf("%s:%d-%s:%s", p.Hostname, p.Pid, p.ID, strings.Join(p.Queues, ",")) 34 | } 35 | 36 | func (p *process) open(conn *RedisConn) error { 37 | conn.Send("SADD", fmt.Sprintf("%sworkers", workerSettings.Namespace), p) 38 | conn.Send("SET", fmt.Sprintf("%sstat:processed:%v", workerSettings.Namespace, p), "0") 39 | conn.Send("SET", fmt.Sprintf("%sstat:failed:%v", workerSettings.Namespace, p), "0") 40 | conn.Flush() 41 | 42 | return nil 43 | } 44 | 45 | func (p *process) close(conn *RedisConn) error { 46 | logger.Infof("%v shutdown", p) 47 | conn.Send("SREM", fmt.Sprintf("%sworkers", workerSettings.Namespace), p) 48 | conn.Send("DEL", fmt.Sprintf("%sstat:processed:%s", workerSettings.Namespace, p)) 49 | conn.Send("DEL", fmt.Sprintf("%sstat:failed:%s", workerSettings.Namespace, p)) 50 | conn.Flush() 51 | 52 | return nil 53 | } 54 | 55 | func (p *process) start(conn *RedisConn) error { 56 | conn.Send("SET", fmt.Sprintf("%sworker:%s:started", workerSettings.Namespace, p), time.Now().String()) 57 | conn.Flush() 58 | 59 | return nil 60 | } 61 | 62 | func (p *process) finish(conn *RedisConn) error { 63 | conn.Send("DEL", fmt.Sprintf("%sworker:%s", workerSettings.Namespace, p)) 64 | conn.Send("DEL", fmt.Sprintf("%sworker:%s:started", workerSettings.Namespace, p)) 65 | conn.Flush() 66 | 67 | return nil 68 | } 69 | 70 | func (p *process) fail(conn *RedisConn) error { 71 | conn.Send("INCR", fmt.Sprintf("%sstat:failed", workerSettings.Namespace)) 72 | conn.Send("INCR", fmt.Sprintf("%sstat:failed:%s", workerSettings.Namespace, p)) 73 | conn.Flush() 74 | 75 | return nil 76 | } 77 | 78 | func (p *process) queues(strict bool) []string { 79 | // If the queues order is strict then just return them. 80 | if strict { 81 | return p.Queues 82 | } 83 | 84 | // If not then we want to to shuffle the queues before returning them. 85 | queues := make([]string, len(p.Queues)) 86 | for i, v := range rand.Perm(len(p.Queues)) { 87 | queues[i] = p.Queues[v] 88 | } 89 | return queues 90 | } 91 | -------------------------------------------------------------------------------- /queues_flag_test.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "testing" 7 | ) 8 | 9 | var queuesFlagSetTests = []struct { 10 | v string 11 | expected queuesFlag 12 | err error 13 | }{ 14 | { 15 | "", 16 | nil, 17 | errors.New("you must specify at least one queue"), 18 | }, 19 | { 20 | "high", 21 | queuesFlag([]string{"high"}), 22 | nil, 23 | }, 24 | { 25 | "high,low", 26 | queuesFlag([]string{"high", "low"}), 27 | nil, 28 | }, 29 | { 30 | "high=2,low=1", 31 | queuesFlag([]string{"high", "high", "low"}), 32 | nil, 33 | }, 34 | { 35 | "high=2,low", 36 | queuesFlag([]string{"high", "high", "low"}), 37 | nil, 38 | }, 39 | { 40 | "low=1,high=2", 41 | queuesFlag([]string{"low", "high", "high"}), 42 | nil, 43 | }, 44 | { 45 | "low=,high=2", 46 | nil, 47 | errors.New("the weight must be a numeric value"), 48 | }, 49 | { 50 | "low=a,high=2", 51 | nil, 52 | errors.New("the weight must be a numeric value"), 53 | }, 54 | { 55 | "low=", 56 | nil, 57 | errors.New("the weight must be a numeric value"), 58 | }, 59 | { 60 | "low=a", 61 | nil, 62 | errors.New("the weight must be a numeric value"), 63 | }, 64 | { 65 | "high=2,,,=1", 66 | queuesFlag([]string{"high", "high"}), 67 | nil, 68 | }, 69 | { 70 | ",,,", 71 | nil, 72 | errors.New("you must specify at least one queue"), 73 | }, 74 | { 75 | "=1", 76 | nil, 77 | errors.New("you must specify at least one queue"), 78 | }, 79 | } 80 | 81 | func TestQueuesFlagSet(t *testing.T) { 82 | for _, tt := range queuesFlagSetTests { 83 | actual := new(queuesFlag) 84 | err := actual.Set(tt.v) 85 | if fmt.Sprint(actual) != fmt.Sprint(tt.expected) { 86 | t.Errorf("QueuesFlag: set to %s expected %v, actual %v", tt.v, tt.expected, actual) 87 | } 88 | if (err != nil && tt.err == nil) || 89 | (err == nil && tt.err != nil) || 90 | (err != nil && tt.err != nil && err.Error() != tt.err.Error()) { 91 | t.Errorf("QueuesFlag: set to %s expected err %v, actual err %v", tt.v, tt.err, err) 92 | } 93 | } 94 | } 95 | 96 | var queuesFlagStringTests = []struct { 97 | q queuesFlag 98 | expected string 99 | }{ 100 | { 101 | queuesFlag([]string{"high"}), 102 | "[high]", 103 | }, 104 | { 105 | queuesFlag([]string{"high", "low"}), 106 | "[high low]", 107 | }, 108 | } 109 | 110 | func TestQueuesFlagString(t *testing.T) { 111 | for _, tt := range queuesFlagStringTests { 112 | actual := tt.q.String() 113 | if actual != tt.expected { 114 | t.Errorf("QueuesFlag(%#v): expected %s, actual %s", tt.q, tt.expected, actual) 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package goworker is a Resque-compatible, Go-based 2 | // background worker. It allows you to push jobs into a 3 | // queue using an expressive language like Ruby while 4 | // harnessing the efficiency and concurrency of Go to 5 | // minimize job latency and cost. 6 | // 7 | // goworker workers can run alongside Ruby Resque clients 8 | // so that you can keep all but your most 9 | // resource-intensive jobs in Ruby. 10 | // 11 | // To create a worker, write a function matching the 12 | // signature 13 | // 14 | // func(string, ...interface{}) error 15 | // 16 | // and register it using 17 | // 18 | // goworker.Register("MyClass", myFunc) 19 | // 20 | // Here is a simple worker that prints its arguments: 21 | // 22 | // package main 23 | // 24 | // import ( 25 | // "fmt" 26 | // "github.com/benmanns/goworker" 27 | // ) 28 | // 29 | // func myFunc(queue string, args ...interface{}) error { 30 | // fmt.Printf("From %s, %v\n", queue, args) 31 | // return nil 32 | // } 33 | // 34 | // func init() { 35 | // goworker.Register("MyClass", myFunc) 36 | // } 37 | // 38 | // func main() { 39 | // if err := goworker.Work(); err != nil { 40 | // fmt.Println("Error:", err) 41 | // } 42 | // } 43 | // 44 | // To create workers that share a database pool or other 45 | // resources, use a closure to share variables. 46 | // 47 | // package main 48 | // 49 | // import ( 50 | // "fmt" 51 | // "github.com/benmanns/goworker" 52 | // ) 53 | // 54 | // func newMyFunc(uri string) (func(queue string, args ...interface{}) error) { 55 | // foo := NewFoo(uri) 56 | // return func(queue string, args ...interface{}) error { 57 | // foo.Bar(args) 58 | // return nil 59 | // } 60 | // } 61 | // 62 | // func init() { 63 | // goworker.Register("MyClass", newMyFunc("http://www.example.com/")) 64 | // } 65 | // 66 | // func main() { 67 | // if err := goworker.Work(); err != nil { 68 | // fmt.Println("Error:", err) 69 | // } 70 | // } 71 | // 72 | // goworker worker functions receive the queue they are 73 | // serving and a slice of interfaces. To use them as 74 | // parameters to other functions, use Go type assertions 75 | // to convert them into usable types. 76 | // 77 | // // Expecting (int, string, float64) 78 | // func myFunc(queue, args ...interface{}) error { 79 | // idNum, ok := args[0].(json.Number) 80 | // if !ok { 81 | // return errorInvalidParam 82 | // } 83 | // id, err := idNum.Int64() 84 | // if err != nil { 85 | // return errorInvalidParam 86 | // } 87 | // name, ok := args[1].(string) 88 | // if !ok { 89 | // return errorInvalidParam 90 | // } 91 | // weightNum, ok := args[2].(json.Number) 92 | // if !ok { 93 | // return errorInvalidParam 94 | // } 95 | // weight, err := weightNum.Float64() 96 | // if err != nil { 97 | // return errorInvalidParam 98 | // } 99 | // doSomething(id, name, weight) 100 | // return nil 101 | // } 102 | // 103 | // For testing, it is helpful to use the redis-cli program 104 | // to insert jobs onto the Redis queue: 105 | // 106 | // redis-cli -r 100 RPUSH resque:queue:myqueue '{"class":"MyClass","args":["hi","there"]}' 107 | // 108 | // will insert 100 jobs for the MyClass worker onto the 109 | // myqueue queue. It is equivalent to: 110 | // 111 | // class MyClass 112 | // @queue = :myqueue 113 | // end 114 | // 115 | // 100.times do 116 | // Resque.enqueue MyClass, ['hi', 'there'] 117 | // end 118 | package goworker 119 | -------------------------------------------------------------------------------- /poller.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "time" 8 | ) 9 | 10 | type poller struct { 11 | process 12 | isStrict bool 13 | } 14 | 15 | func newPoller(queues []string, isStrict bool) (*poller, error) { 16 | process, err := newProcess("poller", queues) 17 | if err != nil { 18 | return nil, err 19 | } 20 | return &poller{ 21 | process: *process, 22 | isStrict: isStrict, 23 | }, nil 24 | } 25 | 26 | func (p *poller) getJob(conn *RedisConn) (*Job, error) { 27 | for _, queue := range p.queues(p.isStrict) { 28 | logger.Debugf("Checking %s", queue) 29 | 30 | reply, err := conn.Do("LPOP", fmt.Sprintf("%squeue:%s", workerSettings.Namespace, queue)) 31 | if err != nil { 32 | return nil, err 33 | } 34 | if reply != nil { 35 | logger.Debugf("Found job on %s", queue) 36 | 37 | job := &Job{Queue: queue} 38 | 39 | decoder := json.NewDecoder(bytes.NewReader(reply.([]byte))) 40 | if workerSettings.UseNumber { 41 | decoder.UseNumber() 42 | } 43 | 44 | if err := decoder.Decode(&job.Payload); err != nil { 45 | return nil, err 46 | } 47 | return job, nil 48 | } 49 | } 50 | 51 | return nil, nil 52 | } 53 | 54 | func (p *poller) poll(interval time.Duration, quit <-chan bool) <-chan *Job { 55 | jobs := make(chan *Job) 56 | 57 | conn, err := GetConn() 58 | if err != nil { 59 | logger.Criticalf("Error on getting connection in poller %s", p) 60 | close(jobs) 61 | return jobs 62 | } else { 63 | p.open(conn) 64 | p.start(conn) 65 | PutConn(conn) 66 | } 67 | 68 | go func() { 69 | defer func() { 70 | close(jobs) 71 | 72 | conn, err := GetConn() 73 | if err != nil { 74 | logger.Criticalf("Error on getting connection in poller %s", p) 75 | return 76 | } else { 77 | p.finish(conn) 78 | p.close(conn) 79 | PutConn(conn) 80 | } 81 | }() 82 | 83 | for { 84 | select { 85 | case <-quit: 86 | return 87 | default: 88 | conn, err := GetConn() 89 | if err != nil { 90 | logger.Criticalf("Error on getting connection in poller %s", p) 91 | return 92 | } 93 | 94 | job, err := p.getJob(conn) 95 | if err != nil { 96 | logger.Criticalf("Error on %v getting job from %v: %v", p, p.Queues, err) 97 | PutConn(conn) 98 | return 99 | } 100 | if job != nil { 101 | conn.Send("INCR", fmt.Sprintf("%sstat:processed:%v", workerSettings.Namespace, p)) 102 | conn.Flush() 103 | PutConn(conn) 104 | select { 105 | case jobs <- job: 106 | case <-quit: 107 | buf, err := json.Marshal(job.Payload) 108 | if err != nil { 109 | logger.Criticalf("Error requeueing %v: %v", job, err) 110 | return 111 | } 112 | conn, err := GetConn() 113 | if err != nil { 114 | logger.Criticalf("Error on getting connection in poller %s", p) 115 | return 116 | } 117 | 118 | conn.Send("LPUSH", fmt.Sprintf("%squeue:%s", workerSettings.Namespace, job.Queue), buf) 119 | conn.Flush() 120 | return 121 | } 122 | } else { 123 | PutConn(conn) 124 | if workerSettings.ExitOnComplete { 125 | return 126 | } 127 | logger.Debugf("Sleeping for %v", interval) 128 | logger.Debugf("Waiting for %v", p.Queues) 129 | 130 | timeout := time.After(interval) 131 | select { 132 | case <-quit: 133 | return 134 | case <-timeout: 135 | } 136 | } 137 | } 138 | } 139 | }() 140 | 141 | return jobs 142 | } 143 | -------------------------------------------------------------------------------- /goworker.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | "sync" 7 | "time" 8 | 9 | "golang.org/x/net/context" 10 | 11 | "github.com/cihub/seelog" 12 | "github.com/youtube/vitess/go/pools" 13 | ) 14 | 15 | var ( 16 | logger seelog.LoggerInterface 17 | pool *pools.ResourcePool 18 | ctx context.Context 19 | initMutex sync.Mutex 20 | initialized bool 21 | ) 22 | 23 | var workerSettings WorkerSettings 24 | 25 | type WorkerSettings struct { 26 | QueuesString string 27 | Queues queuesFlag 28 | IntervalFloat float64 29 | Interval intervalFlag 30 | Concurrency int 31 | Connections int 32 | URI string 33 | Namespace string 34 | ExitOnComplete bool 35 | IsStrict bool 36 | UseNumber bool 37 | } 38 | 39 | func SetSettings(settings WorkerSettings) { 40 | workerSettings = settings 41 | } 42 | 43 | // Init initializes the goworker process. This will be 44 | // called by the Work function, but may be used by programs 45 | // that wish to access goworker functions and configuration 46 | // without actually processing jobs. 47 | func Init() error { 48 | initMutex.Lock() 49 | defer initMutex.Unlock() 50 | if !initialized { 51 | var err error 52 | logger, err = seelog.LoggerFromWriterWithMinLevel(os.Stdout, seelog.InfoLvl) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | if err := flags(); err != nil { 58 | return err 59 | } 60 | ctx = context.Background() 61 | 62 | pool = newRedisPool(workerSettings.URI, workerSettings.Connections, workerSettings.Connections, time.Minute) 63 | 64 | initialized = true 65 | } 66 | return nil 67 | } 68 | 69 | // GetConn returns a connection from the goworker Redis 70 | // connection pool. When using the pool, check in 71 | // connections as quickly as possible, because holding a 72 | // connection will cause concurrent worker functions to lock 73 | // while they wait for an available connection. Expect this 74 | // API to change drastically. 75 | func GetConn() (*RedisConn, error) { 76 | resource, err := pool.Get(ctx) 77 | 78 | if err != nil { 79 | return nil, err 80 | } 81 | return resource.(*RedisConn), nil 82 | } 83 | 84 | // PutConn puts a connection back into the connection pool. 85 | // Run this as soon as you finish using a connection that 86 | // you got from GetConn. Expect this API to change 87 | // drastically. 88 | func PutConn(conn *RedisConn) { 89 | pool.Put(conn) 90 | } 91 | 92 | // Close cleans up resources initialized by goworker. This 93 | // will be called by Work when cleaning up. However, if you 94 | // are using the Init function to access goworker functions 95 | // and configuration without processing jobs by calling 96 | // Work, you should run this function when cleaning up. For 97 | // example, 98 | // 99 | // if err := goworker.Init(); err != nil { 100 | // fmt.Println("Error:", err) 101 | // } 102 | // defer goworker.Close() 103 | func Close() { 104 | initMutex.Lock() 105 | defer initMutex.Unlock() 106 | if initialized { 107 | pool.Close() 108 | initialized = false 109 | } 110 | } 111 | 112 | // Work starts the goworker process. Check for errors in 113 | // the return value. Work will take over the Go executable 114 | // and will run until a QUIT, INT, or TERM signal is 115 | // received, or until the queues are empty if the 116 | // -exit-on-complete flag is set. 117 | func Work() error { 118 | err := Init() 119 | if err != nil { 120 | return err 121 | } 122 | defer Close() 123 | 124 | quit := signals() 125 | 126 | poller, err := newPoller(workerSettings.Queues, workerSettings.IsStrict) 127 | if err != nil { 128 | return err 129 | } 130 | jobs := poller.poll(time.Duration(workerSettings.Interval), quit) 131 | 132 | var monitor sync.WaitGroup 133 | 134 | for id := 0; id < workerSettings.Concurrency; id++ { 135 | worker, err := newWorker(strconv.Itoa(id), workerSettings.Queues) 136 | if err != nil { 137 | return err 138 | } 139 | worker.work(jobs, &monitor) 140 | } 141 | 142 | monitor.Wait() 143 | 144 | return nil 145 | } 146 | -------------------------------------------------------------------------------- /worker.go: -------------------------------------------------------------------------------- 1 | package goworker 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | type worker struct { 12 | process 13 | } 14 | 15 | func newWorker(id string, queues []string) (*worker, error) { 16 | process, err := newProcess(id, queues) 17 | if err != nil { 18 | return nil, err 19 | } 20 | return &worker{ 21 | process: *process, 22 | }, nil 23 | } 24 | 25 | func (w *worker) MarshalJSON() ([]byte, error) { 26 | return json.Marshal(w.String()) 27 | } 28 | 29 | func (w *worker) start(conn *RedisConn, job *Job) error { 30 | work := &work{ 31 | Queue: job.Queue, 32 | RunAt: time.Now(), 33 | Payload: job.Payload, 34 | } 35 | 36 | buffer, err := json.Marshal(work) 37 | if err != nil { 38 | return err 39 | } 40 | 41 | conn.Send("SET", fmt.Sprintf("%sworker:%s", workerSettings.Namespace, w), buffer) 42 | logger.Debugf("Processing %s since %s [%v]", work.Queue, work.RunAt, work.Payload.Class) 43 | 44 | return w.process.start(conn) 45 | } 46 | 47 | func (w *worker) fail(conn *RedisConn, job *Job, err error) error { 48 | failure := &failure{ 49 | FailedAt: time.Now(), 50 | Payload: job.Payload, 51 | Exception: "Error", 52 | Error: err.Error(), 53 | Worker: w, 54 | Queue: job.Queue, 55 | } 56 | buffer, err := json.Marshal(failure) 57 | if err != nil { 58 | return err 59 | } 60 | conn.Send("RPUSH", fmt.Sprintf("%sfailed", workerSettings.Namespace), buffer) 61 | 62 | return w.process.fail(conn) 63 | } 64 | 65 | func (w *worker) succeed(conn *RedisConn, job *Job) error { 66 | conn.Send("INCR", fmt.Sprintf("%sstat:processed", workerSettings.Namespace)) 67 | conn.Send("INCR", fmt.Sprintf("%sstat:processed:%s", workerSettings.Namespace, w)) 68 | 69 | return nil 70 | } 71 | 72 | func (w *worker) finish(conn *RedisConn, job *Job, err error) error { 73 | if err != nil { 74 | w.fail(conn, job, err) 75 | } else { 76 | w.succeed(conn, job) 77 | } 78 | return w.process.finish(conn) 79 | } 80 | 81 | func (w *worker) work(jobs <-chan *Job, monitor *sync.WaitGroup) { 82 | conn, err := GetConn() 83 | if err != nil { 84 | logger.Criticalf("Error on getting connection in worker %v", w) 85 | return 86 | } else { 87 | w.open(conn) 88 | PutConn(conn) 89 | } 90 | 91 | monitor.Add(1) 92 | 93 | go func() { 94 | defer func() { 95 | defer monitor.Done() 96 | 97 | conn, err := GetConn() 98 | if err != nil { 99 | logger.Criticalf("Error on getting connection in worker %v", w) 100 | return 101 | } else { 102 | w.close(conn) 103 | PutConn(conn) 104 | } 105 | }() 106 | for job := range jobs { 107 | if workerFunc, ok := workers[job.Payload.Class]; ok { 108 | w.run(job, workerFunc) 109 | 110 | logger.Debugf("done: (Job{%s} | %s | %v)", job.Queue, job.Payload.Class, job.Payload.Args) 111 | } else { 112 | errorLog := fmt.Sprintf("No worker for %s in queue %s with args %v", job.Payload.Class, job.Queue, job.Payload.Args) 113 | logger.Critical(errorLog) 114 | 115 | conn, err := GetConn() 116 | if err != nil { 117 | logger.Criticalf("Error on getting connection in worker %v", w) 118 | return 119 | } else { 120 | w.finish(conn, job, errors.New(errorLog)) 121 | PutConn(conn) 122 | } 123 | } 124 | } 125 | }() 126 | } 127 | 128 | func (w *worker) run(job *Job, workerFunc workerFunc) { 129 | var err error 130 | defer func() { 131 | conn, errCon := GetConn() 132 | if errCon != nil { 133 | logger.Criticalf("Error on getting connection in worker %v", w) 134 | return 135 | } else { 136 | w.finish(conn, job, err) 137 | PutConn(conn) 138 | } 139 | }() 140 | defer func() { 141 | if r := recover(); r != nil { 142 | err = errors.New(fmt.Sprint(r)) 143 | } 144 | }() 145 | 146 | conn, err := GetConn() 147 | if err != nil { 148 | logger.Criticalf("Error on getting connection in worker %v", w) 149 | return 150 | } else { 151 | w.start(conn, job) 152 | PutConn(conn) 153 | } 154 | err = workerFunc(job.Queue, job.Payload.Args...) 155 | } 156 | -------------------------------------------------------------------------------- /flags.go: -------------------------------------------------------------------------------- 1 | // Running goworker 2 | // 3 | // After building your workers, you will have an 4 | // executable that you can run which will 5 | // automatically poll a Redis server and call 6 | // your workers as jobs arrive. 7 | // 8 | // Flags 9 | // 10 | // There are several flags which control the 11 | // operation of the goworker client. 12 | // 13 | // -queues="comma,delimited,queues" 14 | // — This is the only required flag. The 15 | // recommended practice is to separate your 16 | // Resque workers from your goworkers with 17 | // different queues. Otherwise, Resque worker 18 | // classes that have no goworker analog will 19 | // cause the goworker process to fail the jobs. 20 | // Because of this, there is no default queue, 21 | // nor is there a way to select all queues (à la 22 | // Resque's * queue). Queues are processed in 23 | // the order they are specififed. 24 | // If you have multiple queues you can assign 25 | // them weights. A queue with a weight of 2 will 26 | // be checked twice as often as a queue with a 27 | // weight of 1: -queues='high=2,low=1'. 28 | // 29 | // -interval=5.0 30 | // — Specifies the wait period between polling if 31 | // no job was in the queue the last time one was 32 | // requested. 33 | // 34 | // -concurrency=25 35 | // — Specifies the number of concurrently 36 | // executing workers. This number can be as low 37 | // as 1 or rather comfortably as high as 100,000, 38 | // and should be tuned to your workflow and the 39 | // availability of outside resources. 40 | // 41 | // -connections=2 42 | // — Specifies the maximum number of Redis 43 | // connections that goworker will consume between 44 | // the poller and all workers. There is not much 45 | // performance gain over two and a slight penalty 46 | // when using only one. This is configurable in 47 | // case you need to keep connection counts low 48 | // for cloud Redis providers who limit plans on 49 | // maxclients. 50 | // 51 | // -uri=redis://localhost:6379/ 52 | // — Specifies the URI of the Redis database from 53 | // which goworker polls for jobs. Accepts URIs of 54 | // the format redis://user:pass@host:port/db or 55 | // unix:///path/to/redis.sock. The flag may also 56 | // be set by the environment variable 57 | // $($REDIS_PROVIDER) or $REDIS_URL. E.g. set 58 | // $REDIS_PROVIDER to REDISTOGO_URL on Heroku to 59 | // let the Redis To Go add-on configure the Redis 60 | // database. 61 | // 62 | // -namespace=resque: 63 | // — Specifies the namespace from which goworker 64 | // retrieves jobs and stores stats on workers. 65 | // 66 | // -exit-on-complete=false 67 | // — Exits goworker when there are no jobs left 68 | // in the queue. This is helpful in conjunction 69 | // with the time command to benchmark different 70 | // configurations. 71 | // 72 | // -use-number=false 73 | // — Uses json.Number when decoding numbers in the 74 | // job payloads. This will avoid issues that 75 | // occur when goworker and the json package decode 76 | // large numbers as floats, which then get 77 | // encoded in scientific notation, losing 78 | // pecision. This will default to true soon. 79 | // 80 | // You can also configure your own flags for use 81 | // within your workers. Be sure to set them 82 | // before calling goworker.Main(). It is okay to 83 | // call flags.Parse() before calling 84 | // goworker.Main() if you need to do additional 85 | // processing on your flags. 86 | package goworker 87 | 88 | import ( 89 | "flag" 90 | "os" 91 | "strings" 92 | ) 93 | 94 | // Namespace returns the namespace flag for goworker. You 95 | // can use this with the GetConn and PutConn functions to 96 | // operate on the same namespace that goworker uses. 97 | func Namespace() string { 98 | return workerSettings.Namespace 99 | } 100 | 101 | func init() { 102 | flag.StringVar(&workerSettings.QueuesString, "queues", "", "a comma-separated list of Resque queues") 103 | 104 | flag.Float64Var(&workerSettings.IntervalFloat, "interval", 5.0, "sleep interval when no jobs are found") 105 | 106 | flag.IntVar(&workerSettings.Concurrency, "concurrency", 25, "the maximum number of concurrently executing jobs") 107 | 108 | flag.IntVar(&workerSettings.Connections, "connections", 2, "the maximum number of connections to the Redis database") 109 | 110 | redisProvider := os.Getenv("REDIS_PROVIDER") 111 | var redisEnvURI string 112 | if redisProvider != "" { 113 | redisEnvURI = os.Getenv(redisProvider) 114 | } else { 115 | redisEnvURI = os.Getenv("REDIS_URL") 116 | } 117 | if redisEnvURI == "" { 118 | redisEnvURI = "redis://localhost:6379/" 119 | } 120 | flag.StringVar(&workerSettings.URI, "uri", redisEnvURI, "the URI of the Redis server") 121 | 122 | flag.StringVar(&workerSettings.Namespace, "namespace", "resque:", "the Redis namespace") 123 | 124 | flag.BoolVar(&workerSettings.ExitOnComplete, "exit-on-complete", false, "exit when the queue is empty") 125 | 126 | flag.BoolVar(&workerSettings.UseNumber, "use-number", false, "use json.Number instead of float64 when decoding numbers in JSON. will default to true soon") 127 | } 128 | 129 | func flags() error { 130 | if !flag.Parsed() { 131 | flag.Parse() 132 | } 133 | if err := workerSettings.Queues.Set(workerSettings.QueuesString); err != nil { 134 | return err 135 | } 136 | if err := workerSettings.Interval.SetFloat(workerSettings.IntervalFloat); err != nil { 137 | return err 138 | } 139 | workerSettings.IsStrict = strings.IndexRune(workerSettings.QueuesString, '=') == -1 140 | 141 | if !workerSettings.UseNumber { 142 | logger.Warn("== DEPRECATION WARNING ==") 143 | logger.Warn(" Currently, encoding/json decodes numbers as float64.") 144 | logger.Warn(" This can cause numbers to lose precision as they are read from the Resque queue.") 145 | logger.Warn(" Set the -use-number flag to use json.Number when decoding numbers and remove this warning.") 146 | } 147 | 148 | return nil 149 | } 150 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # goworker 2 | 3 | [![Build Status](https://travis-ci.org/benmanns/goworker.png?branch=master)](https://travis-ci.org/benmanns/goworker) 4 | 5 | goworker is a Resque-compatible, Go-based background worker. It allows you to push jobs into a queue using an expressive language like Ruby while harnessing the efficiency and concurrency of Go to minimize job latency and cost. 6 | 7 | goworker workers can run alongside Ruby Resque clients so that you can keep all but your most resource-intensive jobs in Ruby. 8 | 9 | ## Installation 10 | 11 | To install goworker, use 12 | 13 | ```sh 14 | go get github.com/benmanns/goworker 15 | ``` 16 | 17 | to install the package, and then from your worker 18 | 19 | ```go 20 | import "github.com/benmanns/goworker" 21 | ``` 22 | 23 | ## Getting Started 24 | 25 | To create a worker, write a function matching the signature 26 | 27 | ```go 28 | func(string, ...interface{}) error 29 | ``` 30 | 31 | and register it using 32 | 33 | ```go 34 | goworker.Register("MyClass", myFunc) 35 | ``` 36 | 37 | Here is a simple worker that prints its arguments: 38 | 39 | ```go 40 | package main 41 | 42 | import ( 43 | "fmt" 44 | "github.com/benmanns/goworker" 45 | ) 46 | 47 | func myFunc(queue string, args ...interface{}) error { 48 | fmt.Printf("From %s, %v\n", queue, args) 49 | return nil 50 | } 51 | 52 | func init() { 53 | goworker.Register("MyClass", myFunc) 54 | } 55 | 56 | func main() { 57 | if err := goworker.Work(); err != nil { 58 | fmt.Println("Error:", err) 59 | } 60 | } 61 | ``` 62 | 63 | To create workers that share a database pool or other resources, use a closure to share variables. 64 | 65 | ```go 66 | package main 67 | 68 | import ( 69 | "fmt" 70 | "github.com/benmanns/goworker" 71 | ) 72 | 73 | func newMyFunc(uri string) (func(queue string, args ...interface{}) error) { 74 | foo := NewFoo(uri) 75 | return func(queue string, args ...interface{}) error { 76 | foo.Bar(args) 77 | return nil 78 | } 79 | } 80 | 81 | func init() { 82 | goworker.Register("MyClass", newMyFunc("http://www.example.com/")) 83 | } 84 | 85 | func main() { 86 | if err := goworker.Work(); err != nil { 87 | fmt.Println("Error:", err) 88 | } 89 | } 90 | ``` 91 | 92 | Here is a simple worker with settings: 93 | 94 | ```go 95 | package main 96 | 97 | import ( 98 | "fmt" 99 | "github.com/benmanns/goworker" 100 | ) 101 | 102 | func myFunc(queue string, args ...interface{}) error { 103 | fmt.Printf("From %s, %v\n", queue, args) 104 | return nil 105 | } 106 | 107 | func init() { 108 | settings := goworker.WorkerSettings{ 109 | URI: "redis://localhost:6379/", 110 | Connections: 100, 111 | Queues: []string{"myqueue", "delimited", "queues"}, 112 | UseNumber: true, 113 | ExitOnComplete: false, 114 | Concurrency: 2, 115 | Namespace: "resque:", 116 | Interval: 5.0, 117 | } 118 | goworker.SetSettings(settings) 119 | goworker.Register("MyClass", myFunc) 120 | } 121 | 122 | func main() { 123 | if err := goworker.Work(); err != nil { 124 | fmt.Println("Error:", err) 125 | } 126 | } 127 | ``` 128 | 129 | goworker worker functions receive the queue they are serving and a slice of interfaces. To use them as parameters to other functions, use Go type assertions to convert them into usable types. 130 | 131 | ```go 132 | // Expecting (int, string, float64) 133 | func myFunc(queue, args ...interface{}) error { 134 | idNum, ok := args[0].(json.Number) 135 | if !ok { 136 | return errorInvalidParam 137 | } 138 | id, err := idNum.Int64() 139 | if err != nil { 140 | return errorInvalidParam 141 | } 142 | name, ok := args[1].(string) 143 | if !ok { 144 | return errorInvalidParam 145 | } 146 | weightNum, ok := args[2].(json.Number) 147 | if !ok { 148 | return errorInvalidParam 149 | } 150 | weight, err := weightNum.Float64() 151 | if err != nil { 152 | return errorInvalidParam 153 | } 154 | doSomething(id, name, weight) 155 | return nil 156 | } 157 | ``` 158 | 159 | For testing, it is helpful to use the `redis-cli` program to insert jobs onto the Redis queue: 160 | 161 | ```sh 162 | redis-cli -r 100 RPUSH resque:queue:myqueue '{"class":"MyClass","args":["hi","there"]}' 163 | ``` 164 | 165 | will insert 100 jobs for the `MyClass` worker onto the `myqueue` queue. It is equivalent to: 166 | 167 | ```ruby 168 | class MyClass 169 | @queue = :myqueue 170 | end 171 | 172 | 100.times do 173 | Resque.enqueue MyClass, ['hi', 'there'] 174 | end 175 | ``` 176 | 177 | or 178 | 179 | ```golang 180 | goworker.Enqueue(&goworker.Job{ 181 | Queue: "myqueue", 182 | Payload: goworker.Payload{ 183 | Class: "MyClass", 184 | Args: []interface{}{"hi", "there"}, 185 | }, 186 | }) 187 | ``` 188 | 189 | ## Flags 190 | 191 | There are several flags which control the operation of the goworker client. 192 | 193 | * `-queues="comma,delimited,queues"` — This is the only required flag. The recommended practice is to separate your Resque workers from your goworkers with different queues. Otherwise, Resque worker classes that have no goworker analog will cause the goworker process to fail the jobs. Because of this, there is no default queue, nor is there a way to select all queues (à la Resque's `*` queue). If you have multiple queues you can assign them weights. A queue with a weight of 2 will be checked twice as often as a queue with a weight of 1: `-queues='high=2,low=1'`. 194 | * `-interval=5.0` — Specifies the wait period between polling if no job was in the queue the last time one was requested. 195 | * `-concurrency=25` — Specifies the number of concurrently executing workers. This number can be as low as 1 or rather comfortably as high as 100,000, and should be tuned to your workflow and the availability of outside resources. 196 | * `-connections=2` — Specifies the maximum number of Redis connections that goworker will consume between the poller and all workers. There is not much performance gain over two and a slight penalty when using only one. This is configurable in case you need to keep connection counts low for cloud Redis providers who limit plans on `maxclients`. 197 | * `-uri=redis://localhost:6379/` — Specifies the URI of the Redis database from which goworker polls for jobs. Accepts URIs of the format `redis://user:pass@host:port/db` or `unix:///path/to/redis.sock`. The flag may also be set by the environment variable `$($REDIS_PROVIDER)` or `$REDIS_URL`. E.g. set `$REDIS_PROVIDER` to `REDISTOGO_URL` on Heroku to let the Redis To Go add-on configure the Redis database. 198 | * `-namespace=resque:` — Specifies the namespace from which goworker retrieves jobs and stores stats on workers. 199 | * `-exit-on-complete=false` — Exits goworker when there are no jobs left in the queue. This is helpful in conjunction with the `time` command to benchmark different configurations. 200 | 201 | You can also configure your own flags for use within your workers. Be sure to set them before calling `goworker.Main()`. It is okay to call `flags.Parse()` before calling `goworker.Main()` if you need to do additional processing on your flags. 202 | 203 | ## Signal Handling in goworker 204 | 205 | To stop goworker, send a `QUIT`, `TERM`, or `INT` signal to the process. This will immediately stop job polling. There can be up to `$CONCURRENCY` jobs currently running, which will continue to run until they are finished. 206 | 207 | ## Failure Modes 208 | 209 | Like Resque, goworker makes no guarantees about the safety of jobs in the event of process shutdown. Workers must be both idempotent and tolerant to loss of the job in the event of failure. 210 | 211 | If the process is killed with a `KILL` or by a system failure, there may be one job that is currently in the poller's buffer that will be lost without any representation in either the queue or the worker variable. 212 | 213 | If you are running goworker on a system like Heroku, which sends a `TERM` to signal a process that it needs to stop, ten seconds later sends a `KILL` to force the process to stop, your jobs must finish within 10 seconds or they may be lost. Jobs will be recoverable from the Redis database under 214 | 215 | ``` 216 | resque:worker::-: 217 | ``` 218 | 219 | as a JSON object with keys `queue`, `run_at`, and `payload`, but the process is manual. Additionally, there is no guarantee that the job in Redis under the worker key has not finished, if the process is killed before goworker can flush the update to Redis. 220 | 221 | ## Contributing 222 | 223 | 1. [Fork it](https://github.com/benmanns/goworker/fork) 224 | 2. Create your feature branch (`git checkout -b my-new-feature`) 225 | 3. Commit your changes (`git commit -am 'Add some feature'`) 226 | 4. Push to the branch (`git push origin my-new-feature`) 227 | 5. Create new Pull Request 228 | --------------------------------------------------------------------------------