├── .gitignore ├── LICENCE ├── README.md ├── broker ├── broker.go ├── broker_dispatcher.go └── broker_test.go ├── bs ├── bs.go └── job.go ├── cli └── options.go ├── cmd └── cmd.go ├── cmdstalk.go ├── go.mod └── go.sum /.gitignore: -------------------------------------------------------------------------------- 1 | # Binary output of local `go build` 2 | /cmdstalk 3 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 99designs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | cmdstalk 2 | ======== 3 | 4 | Cmdstalk is a unix-process-based [beanstalkd][beanstalkd] queue broker. 5 | 6 | Written in [Go][golang], cmdstalk uses the [kr/beanstalk][beanstalk] 7 | library to interact with the [beanstalkd][beanstalkd] queue daemon. 8 | 9 | Each job is passed as stdin to a new instance of the configured worker command. 10 | On `exit(0)` the job is deleted. On `exit(1)` (or any non-zero status) the job 11 | is released with an exponential-backoff delay (releases^4), up to 10 times. 12 | 13 | If the worker has not finished by the time the job TTR is reached, the worker 14 | is killed (SIGTERM, SIGKILL) and the job is allowed to time out. When the 15 | job is subsequently reserved, the `timeouts: 1` will cause it to be buried. 16 | 17 | In this way, job workers can be arbitrary commands, and queue semantics are 18 | reduced down to basic unix concepts of exit status and signals. 19 | 20 | 21 | Install 22 | ------- 23 | 24 | From source: 25 | 26 | ```sh 27 | # Make sure you have a sane $GOPATH 28 | go install github.com/99designs/cmdstalk@latest 29 | ``` 30 | 31 | From binary: 32 | 33 | https://github.com/99designs/cmdstalk/releases 34 | 35 | 36 | Usage 37 | ----- 38 | 39 | ```sh 40 | cmdstalk -help 41 | # Usage of ./cmdstalk: 42 | # -address="127.0.0.1:11300": beanstalkd TCP address. 43 | # -all=false: Listen to all tubes, instead of -tubes=... 44 | # -cmd="": Command to run in worker. 45 | # -per-tube=1: Number of workers per tube. 46 | # -tubes=[default]: Comma separated list of tubes. 47 | 48 | # Watch three specific tubes. 49 | cmdstalk -cmd="/path/to/your/worker --your=flags --here" -tubes="one,two,three" 50 | 51 | # Watch all current and future tubes, four workers per tube. 52 | cmdstalk -all -cmd="cat" -per-tube=4 53 | ``` 54 | 55 | 56 | Dev 57 | --- 58 | 59 | ```sh 60 | # Run all tests, with minimal/buffered output. 61 | go test ./... 62 | 63 | # Run tests in the broker package with steaming output. 64 | (cd broker && go test -v) 65 | 66 | # Run cmdstalk from source. 67 | go run cmdstalk.go -cmd='hexdump -C' -tubes="default,another" 68 | 69 | # Build and run a binary. 70 | go build 71 | file cmdstalk # cmdstalk: Mach-O 64-bit executable x86_64 72 | ``` 73 | 74 | 75 | Release 76 | ------- 77 | 78 | ```sh 79 | # Set up cross-compiling tool. 80 | go install github.com/mitchellh/gox@latest 81 | gox -build-toolchain -os="darwin linux" -arch="amd64" 82 | 83 | # Compile for various systems. 84 | gox -os="darwin linux" -arch="amd64" 85 | gzip cmdstalk_*_* 86 | 87 | # Create a release. 88 | open https://github.com/99designs/cmdstalk/releases/new 89 | ``` 90 | 91 | 92 | TODO 93 | ---- 94 | 95 | * Graceful shutdown. 96 | * SIGKILL recalcitrant worker processes. 97 | * Logging improvements; stdout/stderr, concurrency-safety. 98 | * Interactive mode; single-concurrency, prompt for action for each job. 99 | 100 | 101 | --- 102 | 103 | Created by [Paul Annesley][pda] and [Lachlan Donald][lox]. 104 | 105 | © Copyright 2014 99designs Inc. 106 | 107 | [beanstalkd]: http://kr.github.io/beanstalkd/ 108 | [beanstalk]: http://godoc.org/github.com/kr/beanstalk 109 | [golang]: http://golang.org/ 110 | [pda]: https://twitter.com/pda 111 | [lox]: https://twitter.com/lox 112 | -------------------------------------------------------------------------------- /broker/broker.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package broker reserves jobs from beanstalkd, spawns worker processes, 3 | and manages the interaction between the two. 4 | */ 5 | package broker 6 | 7 | import ( 8 | "fmt" 9 | "log" 10 | "os" 11 | "time" 12 | 13 | "github.com/99designs/cmdstalk/bs" 14 | "github.com/99designs/cmdstalk/cmd" 15 | "github.com/kr/beanstalk" 16 | ) 17 | 18 | const ( 19 | // ttrMargin compensates for beanstalkd's integer precision. 20 | // e.g. reserving a TTR=1 job will show time-left=0. 21 | // We need to set our SIGTERM timer to time-left + ttrMargin. 22 | ttrMargin = 1 * time.Second 23 | 24 | // TimeoutTries is the number of timeouts a job must reach before it is 25 | // buried. Zero means never execute. 26 | TimeoutTries = 1 27 | 28 | // ReleaseTries is the number of releases a job must reach before it is 29 | // buried. Zero means never execute. 30 | ReleaseTries = 10 31 | ) 32 | 33 | type Broker struct { 34 | 35 | // Address of the beanstalkd server. 36 | Address string 37 | 38 | // The shell command to execute for each job. 39 | Cmd string 40 | 41 | // Tube name this broker will service. 42 | Tube string 43 | 44 | log *log.Logger 45 | results chan<- *JobResult 46 | } 47 | 48 | type JobResult struct { 49 | 50 | // Buried is true if the job was buried. 51 | Buried bool 52 | 53 | // Executed is true if the job command was executed (or attempted). 54 | Executed bool 55 | 56 | // ExitStatus of the command; 0 for success. 57 | ExitStatus int 58 | 59 | // JobId from beanstalkd. 60 | JobId uint64 61 | 62 | // Stdout of the command. 63 | Stdout []byte 64 | 65 | // TimedOut indicates the worker exceeded TTR for the job. 66 | // Note this is tracked by a timer, separately to beanstalkd. 67 | TimedOut bool 68 | 69 | // Error raised while attempting to handle the job. 70 | Error error 71 | } 72 | 73 | // New broker instance. 74 | func New(address, tube string, slot uint64, cmd string, results chan<- *JobResult) (b Broker) { 75 | b.Address = address 76 | b.Tube = tube 77 | b.Cmd = cmd 78 | 79 | b.log = log.New(os.Stdout, fmt.Sprintf("[%s:%d] ", tube, slot), log.LstdFlags) 80 | b.results = results 81 | return 82 | } 83 | 84 | // Run connects to beanstalkd and starts broking. 85 | // If ticks channel is present, one job is processed per tick. 86 | func (b *Broker) Run(ticks chan bool) { 87 | b.log.Println("command:", b.Cmd) 88 | b.log.Println("connecting to", b.Address) 89 | conn, err := beanstalk.Dial("tcp", b.Address) 90 | if err != nil { 91 | panic(err) 92 | } 93 | 94 | b.log.Println("watching", b.Tube) 95 | ts := beanstalk.NewTubeSet(conn, b.Tube) 96 | 97 | for { 98 | if ticks != nil { 99 | if _, ok := <-ticks; !ok { 100 | break 101 | } 102 | } 103 | 104 | b.log.Println("reserve (waiting for job)") 105 | id, body := bs.MustReserveWithoutTimeout(ts) 106 | job := bs.NewJob(id, body, conn) 107 | 108 | t, err := job.Timeouts() 109 | if err != nil { 110 | b.log.Panic(err) 111 | } 112 | if t >= TimeoutTries { 113 | b.log.Printf("job %d has %d timeouts, burying", job.Id, t) 114 | job.Bury() 115 | if b.results != nil { 116 | b.results <- &JobResult{JobId: job.Id, Buried: true} 117 | } 118 | continue 119 | } 120 | 121 | releases, err := job.Releases() 122 | if err != nil { 123 | b.log.Panic(err) 124 | } 125 | if releases >= ReleaseTries { 126 | b.log.Printf("job %d has %d releases, burying", job.Id, releases) 127 | job.Bury() 128 | if b.results != nil { 129 | b.results <- &JobResult{JobId: job.Id, Buried: true} 130 | } 131 | continue 132 | } 133 | 134 | b.log.Printf("executing job %d", job.Id) 135 | result, err := b.executeJob(job, b.Cmd) 136 | if err != nil { 137 | log.Panic(err) 138 | } 139 | 140 | err = b.handleResult(job, result) 141 | if err != nil { 142 | log.Panic(err) 143 | } 144 | 145 | if result.Error != nil { 146 | b.log.Println("result had error:", result.Error) 147 | } 148 | 149 | if b.results != nil { 150 | b.results <- result 151 | } 152 | } 153 | 154 | b.log.Println("broker finished") 155 | } 156 | 157 | func (b *Broker) executeJob(job bs.Job, shellCmd string) (result *JobResult, err error) { 158 | result = &JobResult{JobId: job.Id, Executed: true} 159 | 160 | ttr, err := job.TimeLeft() 161 | timer := time.NewTimer(ttr + ttrMargin) 162 | if err != nil { 163 | return 164 | } 165 | 166 | cmd, out, err := cmd.NewCommand(shellCmd) 167 | if err != nil { 168 | return 169 | } 170 | 171 | if err = cmd.StartWithStdin(job.Body); err != nil { 172 | return 173 | } 174 | 175 | // TODO: end loop when stdout closes 176 | stdoutReader: 177 | for { 178 | select { 179 | case <-timer.C: 180 | if err = cmd.Terminate(); err != nil { 181 | return 182 | } 183 | result.TimedOut = true 184 | case data, ok := <-out: 185 | if !ok { 186 | break stdoutReader 187 | } 188 | b.log.Printf("stdout: %s", data) 189 | result.Stdout = append(result.Stdout, data...) 190 | } 191 | } 192 | 193 | waitC := cmd.WaitChan() 194 | 195 | waitLoop: 196 | for { 197 | select { 198 | case wr := <-waitC: 199 | timer.Stop() 200 | if wr.Err == nil { 201 | err = wr.Err 202 | } 203 | result.ExitStatus = wr.Status 204 | break waitLoop 205 | case <-timer.C: 206 | cmd.Terminate() 207 | result.TimedOut = true 208 | } 209 | } 210 | 211 | return 212 | } 213 | 214 | func (b *Broker) handleResult(job bs.Job, result *JobResult) (err error) { 215 | if result.TimedOut { 216 | b.log.Printf("job %d timed out", job.Id) 217 | return 218 | } 219 | b.log.Printf("job %d finished with exit(%d)", job.Id, result.ExitStatus) 220 | switch result.ExitStatus { 221 | case 0: 222 | b.log.Printf("deleting job %d", job.Id) 223 | err = job.Delete() 224 | default: 225 | r, err := job.Releases() 226 | if err != nil { 227 | r = ReleaseTries 228 | } 229 | // r*r*r*r means final of 10 tries has 1h49m21s delay, 4h15m33s total. 230 | // See: http://play.golang.org/p/I15lUWoabI 231 | delay := time.Duration(r*r*r*r) * time.Second 232 | b.log.Printf("releasing job %d with %v delay (%d retries)", job.Id, delay, r) 233 | err = job.Release(delay) 234 | } 235 | return 236 | } 237 | -------------------------------------------------------------------------------- /broker/broker_dispatcher.go: -------------------------------------------------------------------------------- 1 | package broker 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/kr/beanstalk" 8 | ) 9 | 10 | const ( 11 | // ListTubeDelay is the time between sending list-tube to beanstalkd 12 | // to discover and watch newly created tubes. 13 | ListTubeDelay = 10 * time.Second 14 | ) 15 | 16 | // BrokerDispatcher manages the running of Broker instances for tubes. It can 17 | // be manually told tubes to start, or it can poll for tubes as they are 18 | // created. The `perTube` option determines how many brokers are started for 19 | // each tube. 20 | type BrokerDispatcher struct { 21 | address string 22 | cmd string 23 | conn *beanstalk.Conn 24 | perTube uint64 25 | tubeSet map[string]bool 26 | } 27 | 28 | func NewBrokerDispatcher(address, cmd string, perTube uint64) *BrokerDispatcher { 29 | return &BrokerDispatcher{ 30 | address: address, 31 | cmd: cmd, 32 | perTube: perTube, 33 | tubeSet: make(map[string]bool), 34 | } 35 | } 36 | 37 | // RunTube runs broker(s) for the specified tube. 38 | // The number of brokers started is determined by the perTube argument to 39 | // NewBrokerDispatcher. 40 | func (bd *BrokerDispatcher) RunTube(tube string) { 41 | bd.tubeSet[tube] = true 42 | for i := uint64(0); i < bd.perTube; i++ { 43 | bd.runBroker(tube, i) 44 | } 45 | } 46 | 47 | // RunTube runs brokers for the specified tubes. 48 | func (bd *BrokerDispatcher) RunTubes(tubes []string) { 49 | for _, tube := range tubes { 50 | bd.RunTube(tube) 51 | } 52 | } 53 | 54 | // RunAllTubes polls beanstalkd, running broker as new tubes are created. 55 | func (bd *BrokerDispatcher) RunAllTubes() (err error) { 56 | conn, err := beanstalk.Dial("tcp", bd.address) 57 | if err == nil { 58 | bd.conn = conn 59 | } else { 60 | return 61 | } 62 | 63 | go func() { 64 | ticker := instantTicker(ListTubeDelay) 65 | for _ = range ticker { 66 | if e := bd.watchNewTubes(); e != nil { 67 | log.Println(e) 68 | } 69 | } 70 | }() 71 | 72 | return 73 | } 74 | 75 | func (bd *BrokerDispatcher) runBroker(tube string, slot uint64) { 76 | go func() { 77 | b := New(bd.address, tube, slot, bd.cmd, nil) 78 | b.Run(nil) 79 | }() 80 | } 81 | 82 | func (bd *BrokerDispatcher) watchNewTubes() (err error) { 83 | tubes, err := bd.conn.ListTubes() 84 | if err != nil { 85 | return 86 | } 87 | 88 | for _, tube := range tubes { 89 | if !bd.tubeSet[tube] { 90 | bd.RunTube(tube) 91 | } 92 | } 93 | 94 | return 95 | } 96 | 97 | // Like time.Tick() but also fires immediately. 98 | func instantTicker(t time.Duration) <-chan time.Time { 99 | c := make(chan time.Time) 100 | ticker := time.NewTicker(t) 101 | go func() { 102 | c <- time.Now() 103 | for t := range ticker.C { 104 | c <- t 105 | } 106 | }() 107 | return c 108 | } 109 | -------------------------------------------------------------------------------- /broker/broker_test.go: -------------------------------------------------------------------------------- 1 | package broker 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | "math/rand" 7 | "strconv" 8 | "testing" 9 | "time" 10 | 11 | "github.com/kr/beanstalk" 12 | ) 13 | 14 | const ( 15 | address = "127.0.0.1:11300" 16 | defaultTtr = 10 * time.Second 17 | ) 18 | 19 | // TestWorkerSuccess demonstrates a successful exit(0) task (delete). 20 | func TestWorkerSuccess(t *testing.T) { 21 | tube, id := queueJob("hello world", 10, defaultTtr) 22 | expectStdout := []byte("HELLO WORLD") 23 | 24 | cmd := "tr [a-z] [A-Z]" 25 | results := make(chan *JobResult) 26 | b := New(address, tube, 0, cmd, results) 27 | 28 | ticks := make(chan bool) 29 | defer close(ticks) 30 | go b.Run(ticks) 31 | ticks <- true // handle a single job 32 | 33 | result := <-results 34 | 35 | if result.JobId != id { 36 | t.Fatalf("result.JobId %d != queueJob id %d", result.JobId, id) 37 | } 38 | if !bytes.Equal(result.Stdout, expectStdout) { 39 | t.Fatalf("Stdout mismatch: '%s' != '%s'\n", result.Stdout, expectStdout) 40 | } 41 | if result.ExitStatus != 0 { 42 | t.Fatalf("Unexpected exit status %d", result.ExitStatus) 43 | } 44 | 45 | assertTubeEmpty(tube) 46 | } 47 | 48 | // TestWorkerFailure demonstrates a failed exit(1) task (release). 49 | func TestWorkerFailure(t *testing.T) { 50 | tube, id := queueJob("hello world", 10, defaultTtr) 51 | 52 | cmd := "false" 53 | results := make(chan *JobResult) 54 | b := New(address, tube, 0, cmd, results) 55 | 56 | ticks := make(chan bool) 57 | defer close(ticks) 58 | go b.Run(ticks) 59 | ticks <- true // handle a single job 60 | 61 | result := <-results 62 | 63 | if result.JobId != id { 64 | t.Fatalf("result.JobId %d != queueJob id %d", result.JobId, id) 65 | } 66 | 67 | if result.ExitStatus != 1 { 68 | t.Fatalf("result.ExitStatus %d, expected 1", result.ExitStatus) 69 | } 70 | 71 | assertJobStat(t, id, "state", "ready") 72 | assertJobStat(t, id, "releases", "1") 73 | assertJobStat(t, id, "pri", "10") 74 | } 75 | 76 | func TestWorkerTimeout(t *testing.T) { 77 | ttr := 1 * time.Second 78 | tube, id := queueJob("TestWorkerTimeout", 10, ttr) 79 | 80 | cmd := "sleep 4" 81 | results := make(chan *JobResult) 82 | b := New(address, tube, 0, cmd, results) 83 | 84 | ticks := make(chan bool) 85 | defer close(ticks) 86 | go b.Run(ticks) 87 | 88 | start := time.Now() 89 | ticks <- true // handle a single job 90 | result := <-results 91 | duration := time.Since(start) 92 | 93 | if duration < 1*time.Second { 94 | t.Fatalf("%v too short to have timed out correctly", duration) 95 | } 96 | 97 | if !result.TimedOut { 98 | t.Fatalf("Expected job %d JobResult.TimedOut to be true", id) 99 | } 100 | 101 | assertJobStat(t, id, "state", "ready") 102 | assertJobStat(t, id, "timeouts", "1") 103 | 104 | ticks <- true // handle another job 105 | result = <-results 106 | if !result.Buried { 107 | t.Fatalf("Expected job %d JobResult.Buried", id) 108 | } 109 | assertJobStat(t, id, "state", "buried") 110 | assertJobStat(t, id, "timeouts", "1") 111 | } 112 | 113 | func queueJob(body string, priority uint32, ttr time.Duration) (string, uint64) { 114 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 115 | tubeName := "cmdstalk-test-" + strconv.FormatInt(r.Int63(), 16) 116 | assertTubeEmpty(tubeName) 117 | 118 | c, err := beanstalk.Dial("tcp", address) 119 | if err != nil { 120 | log.Fatal(err) 121 | } 122 | 123 | tube := beanstalk.Tube{Conn: c, Name: tubeName} 124 | 125 | id, err := tube.Put([]byte(body), priority, 0, ttr) 126 | if err != nil { 127 | log.Fatal(err) 128 | } 129 | 130 | return tubeName, id 131 | } 132 | 133 | func assertTubeEmpty(tubeName string) { 134 | // TODO 135 | } 136 | 137 | func assertJobStat(t *testing.T, id uint64, key, value string) { 138 | c, err := beanstalk.Dial("tcp", address) 139 | if err != nil { 140 | t.Fatal(err) 141 | } 142 | stats, err := c.StatsJob(id) 143 | if err != nil { 144 | t.Fatal(err) 145 | } 146 | if stats[key] != value { 147 | t.Fatalf("job %d %s = %s, expected %s", id, key, stats[key], value) 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /bs/bs.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package bs provides a richer and/or more domain-specific layer over 3 | github.com/kr/beanstalk, including active-record style Job type. 4 | */ 5 | package bs 6 | 7 | import ( 8 | "time" 9 | 10 | "github.com/kr/beanstalk" 11 | ) 12 | 13 | const ( 14 | // deadlineSoonDelay defines a period to sleep between receiving 15 | // DEADLINE_SOON in response to reserve, and re-attempting the reserve. 16 | DeadlineSoonDelay = 1 * time.Second 17 | ) 18 | 19 | // reserve-with-timeout until there's a job or something panic-worthy. 20 | // Handles beanstalk.ErrTimeout by retrying immediately. 21 | // Handles beanstalk.ErrDeadline by sleeping DeadlineSoonDelay before retry. 22 | // panics for other errors. 23 | func MustReserveWithoutTimeout(ts *beanstalk.TubeSet) (id uint64, body []byte) { 24 | var err error 25 | for { 26 | id, body, err = ts.Reserve(1 * time.Hour) 27 | if err == nil { 28 | return 29 | } else if err.(beanstalk.ConnError).Err == beanstalk.ErrTimeout { 30 | continue 31 | } else if err.(beanstalk.ConnError).Err == beanstalk.ErrDeadline { 32 | time.Sleep(DeadlineSoonDelay) 33 | continue 34 | } else { 35 | panic(err) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bs/job.go: -------------------------------------------------------------------------------- 1 | package bs 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/kr/beanstalk" 9 | ) 10 | 11 | // Job represents a beanstalkd job, and holds a reference to the connection so 12 | // that server actions can be taken as methods on the job. 13 | type Job struct { 14 | 15 | // The numeric beanstalkd-assigned job ID. 16 | Id uint64 17 | 18 | // The job payload data. 19 | Body []byte 20 | 21 | conn *beanstalk.Conn 22 | } 23 | 24 | // Create a Job instance. 25 | func NewJob(id uint64, body []byte, conn *beanstalk.Conn) Job { 26 | return Job{ 27 | Body: body, 28 | Id: id, 29 | conn: conn, 30 | } 31 | } 32 | 33 | // Bury the job, with its original priority. 34 | func (j Job) Bury() error { 35 | pri, err := j.Priority() 36 | if err != nil { 37 | return err 38 | } 39 | return j.conn.Bury(j.Id, pri) 40 | } 41 | 42 | // Delete the job. 43 | func (j Job) Delete() error { 44 | return j.conn.Delete(j.Id) 45 | } 46 | 47 | // Priority of the job, zero is most urgent, 4,294,967,295 is least. 48 | func (j Job) Priority() (uint32, error) { 49 | pri64, err := j.uint64Stat("pri") 50 | return uint32(pri64), err 51 | } 52 | 53 | // Release the job, with its original priority and no delay. 54 | func (j Job) Release(delay time.Duration) error { 55 | pri, err := j.Priority() 56 | if err != nil { 57 | return err 58 | } 59 | return j.conn.Release(j.Id, pri, delay) 60 | } 61 | 62 | // Releases counts how many times the job has been released back to the tube. 63 | func (j Job) Releases() (uint64, error) { 64 | return j.uint64Stat("releases") 65 | } 66 | 67 | func (j Job) String() string { 68 | stats, err := j.conn.StatsJob(j.Id) 69 | if err == nil { 70 | return fmt.Sprintf("Job %d %#v", j.Id, stats) 71 | } else { 72 | return fmt.Sprintf("Job %d (stats-job failed: %s)", j.Id, err) 73 | } 74 | } 75 | 76 | // TimeLeft as reported by beanstalkd, as a time.Duration. 77 | // beanstalkd reports as int(seconds), which defines the (low) precision. 78 | // Less than 1.0 seconds remaining will be reported as zero. 79 | func (j Job) TimeLeft() (time.Duration, error) { 80 | stats, err := j.conn.StatsJob(j.Id) 81 | if err != nil { 82 | return 0, err 83 | } 84 | return time.ParseDuration(stats["time-left"] + "s") 85 | } 86 | 87 | // Timeouts counts how many times the job has been reserved and reached TTR. 88 | func (j Job) Timeouts() (uint64, error) { 89 | return j.uint64Stat("timeouts") 90 | } 91 | 92 | func (j Job) uint64Stat(key string) (uint64, error) { 93 | stats, err := j.conn.StatsJob(j.Id) 94 | if err != nil { 95 | return 0, err 96 | } 97 | return strconv.ParseUint(stats[key], 10, 64) 98 | } 99 | -------------------------------------------------------------------------------- /cli/options.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package cli provides command line support for cmdstalk. 3 | */ 4 | package cli 5 | 6 | import ( 7 | "errors" 8 | "flag" 9 | "fmt" 10 | "os" 11 | "strings" 12 | ) 13 | 14 | // Options contains runtime configuration, and is generally the result of 15 | // parsing command line flags. 16 | type Options struct { 17 | 18 | // The beanstalkd TCP address. 19 | Address string 20 | 21 | // All == true means all tubes will be watched. 22 | All bool 23 | 24 | // The shell command to execute for each job. 25 | Cmd string 26 | 27 | // PerTube is the number of workers servicing each tube concurrently. 28 | PerTube uint64 29 | 30 | // The beanstalkd tubes to watch. 31 | Tubes TubeList 32 | } 33 | 34 | // TubeList is a list of beanstalkd tube names. 35 | type TubeList []string 36 | 37 | // Calls ParseFlags(), os.Exit(1) on error. 38 | func MustParseFlags() (o Options) { 39 | o, err := ParseFlags() 40 | if err != nil { 41 | flag.PrintDefaults() 42 | fmt.Println() 43 | fmt.Println(err) 44 | os.Exit(1) 45 | } 46 | return 47 | } 48 | 49 | // ParseFlags parses and validates CLI flags into an Options struct. 50 | func ParseFlags() (o Options, err error) { 51 | o.Tubes = TubeList{"default"} 52 | 53 | flag.StringVar(&o.Address, "address", "127.0.0.1:11300", "beanstalkd TCP address.") 54 | flag.BoolVar(&o.All, "all", false, "Listen to all tubes, instead of -tubes=...") 55 | flag.StringVar(&o.Cmd, "cmd", "", "Command to run in worker.") 56 | flag.Uint64Var(&o.PerTube, "per-tube", 1, "Number of workers per tube.") 57 | flag.Var(&o.Tubes, "tubes", "Comma separated list of tubes.") 58 | flag.Parse() 59 | 60 | err = validateOptions(o) 61 | 62 | return 63 | } 64 | 65 | func validateOptions(o Options) error { 66 | msgs := make([]string, 0) 67 | 68 | if o.Cmd == "" { 69 | msgs = append(msgs, "Command must not be empty.") 70 | } 71 | 72 | if o.Address == "" { 73 | msgs = append(msgs, "Address must not be empty.") 74 | } 75 | 76 | if len(msgs) == 0 { 77 | return nil 78 | } else { 79 | return errors.New(strings.Join(msgs, "\n")) 80 | } 81 | } 82 | 83 | // Set replaces the TubeList by parsing the comma-separated value string. 84 | func (t *TubeList) Set(value string) error { 85 | list := strings.Split(value, ",") 86 | for i, value := range list { 87 | list[i] = value 88 | } 89 | *t = list 90 | return nil 91 | } 92 | 93 | func (t *TubeList) String() string { 94 | return fmt.Sprint(*t) 95 | } 96 | -------------------------------------------------------------------------------- /cmd/cmd.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package cmd provides a more domain-specific layer over exec.Cmd. 3 | */ 4 | package cmd 5 | 6 | import ( 7 | "io" 8 | "os" 9 | "os/exec" 10 | "syscall" 11 | ) 12 | 13 | const ( 14 | // Shell will have the command line passed to its `-c` option. 15 | Shell = "/bin/bash" 16 | ) 17 | 18 | type Cmd struct { 19 | cmd *exec.Cmd 20 | stderrPipe io.ReadCloser 21 | stdinPipe io.WriteCloser 22 | stdoutPipe io.ReadCloser 23 | } 24 | 25 | // WaitResult is sent to the channel returned by WaitChan(). 26 | // It indicates the exit status, or a non-exit-status error e.g. IO error. 27 | // In the case of a non-exit-status, Status is -1 28 | type WaitResult struct { 29 | Status int 30 | Err error 31 | } 32 | 33 | // NewCommand returns a Cmd with IO configured, but not started. 34 | func NewCommand(shellCmd string) (cmd *Cmd, out <-chan []byte, err error) { 35 | cmd = &Cmd{} 36 | cmd.cmd = exec.Command(Shell, "-c", shellCmd) 37 | 38 | stdin, err := cmd.cmd.StdinPipe() 39 | if err == nil { 40 | cmd.stdinPipe = stdin 41 | } else { 42 | return 43 | } 44 | 45 | stdout, err := cmd.cmd.StdoutPipe() 46 | if err == nil { 47 | cmd.stdoutPipe = stdout 48 | } else { 49 | return 50 | } 51 | 52 | cmd.cmd.Stderr = os.Stderr 53 | cmd.stderrPipe = os.Stderr 54 | 55 | out = readerToChannel(cmd.stdoutPipe) 56 | return 57 | } 58 | 59 | // Start the process, write input to stdin, then close stdin. 60 | func (c *Cmd) StartWithStdin(input []byte) (err error) { 61 | err = c.cmd.Start() 62 | if err != nil { 63 | return 64 | } 65 | _, err = c.stdinPipe.Write(input) 66 | if err != nil { 67 | return 68 | } 69 | c.stdinPipe.Close() 70 | return nil 71 | } 72 | 73 | // Terminate the process with SIGTERM. 74 | // TODO: follow up with SIGKILL if still running. 75 | func (c *Cmd) Terminate() (err error) { 76 | return c.cmd.Process.Signal(syscall.SIGTERM) 77 | } 78 | 79 | // WaitChan starts a goroutine to wait for the command to exit, and returns 80 | // a channel over which will be sent the WaitResult, containing either the 81 | // exit status (0 for success) or a non-exit error, e.g. IO error. 82 | func (cmd *Cmd) WaitChan() <-chan WaitResult { 83 | ch := make(chan WaitResult) 84 | go func() { 85 | err := cmd.cmd.Wait() 86 | if err == nil { 87 | ch <- WaitResult{0, nil} 88 | } else if e1, ok := err.(*exec.ExitError); ok { 89 | status := e1.Sys().(syscall.WaitStatus).ExitStatus() 90 | ch <- WaitResult{status, nil} 91 | } else { 92 | ch <- WaitResult{-1, err} 93 | } 94 | }() 95 | return ch 96 | } 97 | 98 | func readerToChannel(reader io.Reader) <-chan []byte { 99 | c := make(chan []byte) 100 | go func() { 101 | buf := make([]byte, 1024) 102 | for { 103 | n, err := reader.Read(buf) 104 | if n > 0 { 105 | res := make([]byte, n) 106 | copy(res, buf[:n]) 107 | c <- res 108 | } 109 | if err != nil { 110 | close(c) 111 | break 112 | } 113 | } 114 | }() 115 | return c 116 | } 117 | -------------------------------------------------------------------------------- /cmdstalk.go: -------------------------------------------------------------------------------- 1 | /* 2 | Cmdstalk is a unix-process-based [beanstalkd][beanstalkd] queue broker. 3 | 4 | Written in [Go][golang], cmdstalk uses the [kr/beanstalk][beanstalk] 5 | library to interact with the [beanstalkd][beanstalkd] queue daemon. 6 | 7 | Each job is passed as stdin to a new instance of the configured worker 8 | command. On `exit(0)` the job is deleted. On `exit(1)` (or any non-zero 9 | status) the job is released with an exponential-backoff delay (releases^4), 10 | up to 10 times. 11 | 12 | If the worker has not finished by the time the job TTR is reached, the 13 | worker is killed (SIGTERM, SIGKILL) and the job is allowed to time out. 14 | When the job is subsequently reserved, the `timeouts: 1` will cause it to 15 | be buried. 16 | 17 | In this way, job workers can be arbitrary commands, and queue semantics are 18 | reduced down to basic unix concepts of exit status and signals. 19 | */ 20 | package main 21 | 22 | import ( 23 | "github.com/99designs/cmdstalk/broker" 24 | "github.com/99designs/cmdstalk/cli" 25 | ) 26 | 27 | func main() { 28 | opts := cli.MustParseFlags() 29 | 30 | bd := broker.NewBrokerDispatcher(opts.Address, opts.Cmd, opts.PerTube) 31 | 32 | if opts.All { 33 | bd.RunAllTubes() 34 | } else { 35 | bd.RunTubes(opts.Tubes) 36 | } 37 | 38 | // TODO: wire up to SIGTERM handler etc. 39 | exitChan := make(chan bool) 40 | <-exitChan 41 | } 42 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/99designs/cmdstalk 2 | 3 | go 1.17 4 | 5 | require github.com/kr/beanstalk v0.0.0-20130218090021-4011ceaa3cee 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/kr/beanstalk v0.0.0-20130218090021-4011ceaa3cee h1:WJEhmLuV1JSqpi2Wu2EUO4JhBWPyvtHU620wlF5E0LU= 2 | github.com/kr/beanstalk v0.0.0-20130218090021-4011ceaa3cee/go.mod h1:S640fId9Ag4k2hh6Hwwj62pMSZqfMtg/kfKPeAOhET8= 3 | --------------------------------------------------------------------------------