├── .gitignore ├── LICENSE ├── README.md ├── client.go ├── config.go ├── integration_test.py ├── jobqueue.go ├── jobqueue_test.go ├── listener.go ├── listener_test.go ├── mail.go ├── main.go ├── message.go ├── message_test.go ├── runner.go ├── runner_test.go ├── sleepthenecho ├── tube.go └── tube_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | _go_.6 2 | _obj/ 3 | glow 4 | fill.rb 5 | *.pyc 6 | README.html 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012, Glow Developers 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the distribution. 14 | 15 | * The names of the contributors may not be used to endorse or promote 16 | products derived from this software without specific prior written 17 | permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY 23 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 24 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 26 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # glow 2 | 3 | Distributed parallelization of tasks 4 | 5 | ## Setup 6 | 7 | - Install [beanstalkd](http://kr.github.com/beanstalkd/download.html) 8 | - [Download a glow binary](https://github.com/nutrun/glow/downloads) and add it to `$PATH` 9 | 10 | ### Building from source 11 | 12 | - Install [Go](http://golang.org/doc/install) 13 | - Install the [lentil](https://github.com/nutrun/lentil) beanstalkd client library 14 | - `go get github.com/nutrun/glow` or `cd && go install` 15 | 16 | ## Quickstart 17 | 18 | Start beanstalkd: 19 | 20 | ``` 21 | $ beanstalkd 22 | ``` 23 | 24 | Start a glow listener: 25 | 26 | ``` 27 | $ glow -listen 28 | ``` 29 | 30 | Submit a job: 31 | 32 | ``` 33 | $ glow -tube=test -out=/dev/stdout ls 34 | ``` 35 | 36 | The job's output should appear on the terminal running the glow listener. Invoke `glow -h` to list all available options. 37 | 38 | ## Listen 39 | 40 | A listener connects to the beanstalk queue specified by the environment variable `GLOW_QUEUE` (it defaults to `0.0.0.0:11300` if `GLOW_QUEUE` isn't specified), waits for jobs and executes them as they become available. In order to achieve parallelism, a glow system will have many hosts and a number of listeners on each host. The number of listeners per host should depend on the type of job and number of available cores. 41 | 42 | Listen options: 43 | 44 | ``` 45 | $ glow -h 2>&1 | grep listen 46 | ``` 47 | 48 | Start a listener: 49 | 50 | ``` 51 | $ GLOW_QUEUE=10.0.0.4:11300 glow -listen 52 | ``` 53 | 54 | Log not only errors: 55 | 56 | ``` 57 | $ glow -listen -v 58 | ``` 59 | 60 | 61 | ### Tube Dependencies 62 | 63 | A [beanstalk tube](https://github.com/kr/beanstalkd/blob/master/doc/protocol.txt#L105) is a priority based fifo queue of jobs. In glow, a tube can depend on one or more other tubes. Tube dependencies are specified in a JSON file: 64 | 65 | ``` 66 | $ cat > glow-deps.json 67 | { 68 | "foo": ["bar"], 69 | "baz": ["foo", "bar"] 70 | } 71 | 72 | $ glow -listen -deps=glow-deps.json 73 | ``` 74 | 75 | - Tube `foo` depends on tube bar: no jobs from `foo` will run while there are ready/delayed/reserved jobs in `bar` 76 | - Tube `bar` does not have any dependencies. Jobs from `bar` will run whenever there are free listeners available 77 | - Tube `baz` depends on tube `bar` and `foo`. It will block until `bar` and `foo` are done 78 | - Dependencies are not transitive. If `foo` depends on `bar` and `baz` depends on `foo`, `baz` doesn't depend on `bar` 79 | 80 | ### Excluding tubes 81 | 82 | A listener will not reserve jobs from any of the tubes specified by the `exclude` flag: 83 | 84 | ``` 85 | $ glow -listen -exclude=foo,bar 86 | ``` 87 | 88 | ### Email 89 | 90 | The SMTP server and email `FROM` field can be configured for glow's job failure email notifications: 91 | 92 | ``` 93 | $ glow -listen -SMTP-server=SMTP.example.com -mail-from=glow@example.com 94 | ``` 95 | 96 | Emails will only be sent when a list of recipients has been specified at job submission. 97 | 98 | ### Signals 99 | 100 | `SIGTERM` kills a listener and its running job immediatly: 101 | 102 | ``` 103 | $ killall glow 104 | ``` 105 | 106 | Shut down gracefully (wait for job to finish) with `SIGINT`: 107 | 108 | ``` 109 | $ killall -SIGINT glow 110 | ``` 111 | 112 | 113 | ## Submit 114 | 115 | Submit options: 116 | 117 | ``` 118 | $ glow -h 2>&1 | grep submit 119 | ``` 120 | 121 | Send a job to a tube on the beanstalkd queue to be executed by a listener (`-tube` is required): 122 | 123 | ``` 124 | $ glow -tube=mytube mycmd arg1 arg2 # [...argn] 125 | ``` 126 | 127 | ### Job delay 128 | 129 | [Delay](https://github.com/kr/beanstalkd/blob/master/doc/protocol.txt#L136) is an integer number of seconds to wait before making the job avaible to run: 130 | 131 | ``` 132 | $ glow -tube=mytube -delay=60 mycmd arg1 arg2 133 | ``` 134 | 135 | ### Failure emails 136 | 137 | ``` 138 | $ glow -tube=mytube -mailto=bob@example.com,alice@example.com mycmd arg1 arg2 139 | ``` 140 | 141 | ### Job output 142 | 143 | Job `stdout` and `stderr` can be redirected to a file: 144 | 145 | ``` 146 | $ glow -tube=mytube -stdout=/tmp/mycmd.out -stderr=/tmp/mycmd.err mycmd arg1 arg2 147 | ``` 148 | 149 | By default, a job's `stdout` and `stderr` are sent to `/dev/null` 150 | 151 | ### Job priority 152 | 153 | [Priority](https://github.com/kr/beanstalkd/blob/master/doc/protocol.txt#L132) is an integer < 2**32. Jobs with smaller priority values will be scheduled before jobs with larger priorities: 154 | 155 | ``` 156 | $ glow -tube=mytube -pri=177 mycmd arg1 arg2 157 | ``` 158 | 159 | ### Job working directory 160 | 161 | Where to run the job from. Defaults to `/tmp`. The listener will `chdir` to `workdir` before executing the job's command: 162 | 163 | ``` 164 | $ glow -tube=mytube -workdir=/home/bob/scripts mycmd arg1 arg2 165 | ``` 166 | 167 | ### Batch job submit 168 | For improved performance when queueng up a lot of jobs at once, a JSON list of jobs can be piped to glow's stdin: 169 | 170 | ``` 171 | $ echo '[{"cmd":"ls","arguments":["-l", "-a"],"pri":0,"tube":"foo","delay":0,"mailto":"example@example.com","out":"/tmp/glow.out","workdir":"/tmp/glow"},{"cmd":"ps","pri":1,"tube":"bar","delay":0,"mailto":"example@example.com","out":"/tmp/glow.out","workdir":"/tmp/glow"}]' | glow 172 | ``` 173 | 174 | ## Errors 175 | 176 | Every time a job exits with a non 0 exit status, glow sends a message to a tube on `GLOW_QUEUE` called `GLOW_ERRORS`. beanstalkd clients can listen on `GLOW_ERRORS` to implement custom error handling. 177 | 178 | If a listener was started with the `-smtp-server` flag set, failure emails will be sent to the list of recipients specified by the `-mailto` submit flag. 179 | 180 | ``` 181 | $ glow -listen -smtp-server=smtp.example.com:25 182 | ``` 183 | 184 | ``` 185 | $ glow -tube=mytube -mailto=foo@example.com mycmd arg1 arg2 186 | ``` 187 | 188 | ## Utilities 189 | 190 | ``` 191 | $ glow -h 2>&1 | grep -v 'submit\|listen' 192 | ``` 193 | 194 | ### Drain tubes 195 | 196 | Delete all jobs from a list of tubes, subsequently killing the tubes: 197 | 198 | ``` 199 | $ glow -drain=tube1,tube2 200 | ``` 201 | 202 | The output of `drain` is JSON that can be used to requeue the jobs by piping to `glow`. 203 | 204 | ### Pause tubes 205 | 206 | A list of tubes can be paused for a period of seconds specified by the `-pause-delay` int flag, during which jobs on those tubes will not be available to be reserved by listeners: 207 | 208 | ``` 209 | $ glow -pause=tube1,tube2 -pause-delay=600 210 | ``` 211 | 212 | ### Queue stats 213 | 214 | Show per tube beanstalkd queue statistics: 215 | 216 | ``` 217 | $ glow -stats 218 | ``` 219 | 220 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/nutrun/lentil" 8 | "os" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | type Client struct { 14 | q *lentil.Beanstalkd 15 | verbose bool 16 | } 17 | 18 | func NewClient(verbose bool) (*Client, error) { 19 | this := new(Client) 20 | q, err := lentil.Dial(Config.QueueAddr) 21 | if err != nil { 22 | return nil, err 23 | } 24 | this.q = q 25 | this.verbose = verbose 26 | return this, nil 27 | } 28 | 29 | func (this *Client) put(msg *Message) error { 30 | message, e := json.Marshal(msg) 31 | if this.verbose { 32 | fmt.Fprintf(os.Stderr, "QUEUEING UP: %s\n", message) 33 | } 34 | if e != nil { 35 | return e 36 | } 37 | if msg.Tube != "default" { 38 | e = this.q.Use(msg.Tube) 39 | if e != nil { 40 | return e 41 | } 42 | } 43 | _, e = this.q.Put(msg.Priority, msg.Delay, 60*60, message) // An hour TTR? 44 | return e 45 | } 46 | 47 | func (this *Client) putMany(input []byte) error { 48 | jobs, e := MessagesFromJSON(input) 49 | if e != nil { 50 | return e 51 | } 52 | for _, job := range jobs { 53 | e = this.put(job) 54 | if e != nil { 55 | return e 56 | } 57 | } 58 | return nil 59 | } 60 | 61 | func (this *Client) stats() error { 62 | q := NewJobQueue(this.q, false, make([]string, 0)) 63 | stats, err := json.Marshal(q) 64 | if err != nil { 65 | return err 66 | } 67 | buffer := bytes.NewBufferString("") 68 | err = json.Indent(buffer, stats, "", "\t") 69 | if err != nil { 70 | return err 71 | } 72 | fmt.Printf("%s\n", buffer.String()) 73 | return nil 74 | } 75 | 76 | func (this *Client) drain(tubes string) error { 77 | drainedJobs := make([]string, 0) 78 | for _, tube := range strings.Split(tubes, ",") { 79 | 80 | // Before draining a tube, first kick the delayed jobs 81 | 82 | stats, err := this.q.StatsTube(tube) 83 | if err != nil { 84 | return err 85 | } 86 | delayedJobs, err := strconv.ParseInt(stats["current-jobs-delayed"], 10, 32) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | err = this.q.Use(tube) 92 | if err != nil { 93 | return err 94 | } 95 | 96 | this.q.Kick(int(delayedJobs)) 97 | 98 | err = this.q.Use("default") 99 | if err != nil { 100 | return err 101 | } 102 | 103 | // Now drain jobs until there are none left 104 | 105 | _, err = this.q.Watch(tube) 106 | if err != nil { 107 | return err 108 | } 109 | _, err = this.q.Ignore("default") 110 | if err != nil { 111 | return err 112 | } 113 | for { 114 | job, err := this.q.ReserveWithTimeout(0) 115 | if err != nil && strings.HasPrefix(err.Error(), "TIMED_OUT") { 116 | break 117 | } 118 | if err != nil { 119 | return err 120 | } 121 | err = this.q.Delete(job.Id) 122 | if err != nil { 123 | return err 124 | } 125 | drainedJobs = append(drainedJobs, string(job.Body)) 126 | } 127 | } 128 | fmt.Fprintf(os.Stderr, "[%s]", strings.Join(drainedJobs, ",")) 129 | return nil 130 | } 131 | 132 | func (this *Client) pause(tubes string, delay int) error { 133 | for _, tube := range strings.Split(tubes, ",") { 134 | e := this.q.PauseTube(tube, delay) 135 | if e != nil { 136 | return e 137 | } 138 | fmt.Fprintf(os.Stderr, "Paused %s for %d seconds", tubes, delay) 139 | } 140 | return nil 141 | } 142 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "os" 7 | ) 8 | 9 | const ( 10 | DEFAULT_QUEUE_ADDR = "0.0.0.0:11300" 11 | DEFAULT_FROM_EMAIL = "glow@example.com" 12 | ) 13 | 14 | type Configuration struct { 15 | QueueAddr string 16 | SmtpServer string 17 | MailFrom string 18 | depsPath string 19 | deps map[string][]string 20 | errorQueue string 21 | } 22 | 23 | func NewConfig(deps, smtpserver, mailfrom string) *Configuration { 24 | config := new(Configuration) 25 | config.QueueAddr = os.Getenv("GLOW_QUEUE") 26 | if config.QueueAddr == "" { 27 | config.QueueAddr = DEFAULT_QUEUE_ADDR 28 | } 29 | config.SmtpServer = smtpserver 30 | config.MailFrom = mailfrom 31 | config.depsPath = deps 32 | if config.MailFrom == "" { 33 | config.MailFrom = DEFAULT_FROM_EMAIL 34 | } 35 | config.errorQueue = "GLOW_ERRORS" 36 | return config 37 | } 38 | 39 | func (this *Configuration) LoadDeps() error { 40 | if *deps == "" { 41 | return nil 42 | } 43 | this.deps = make(map[string][]string) 44 | dependencies, err := ioutil.ReadFile(this.depsPath) 45 | if err != nil { 46 | return err 47 | } 48 | return json.Unmarshal(dependencies, &this.deps) 49 | } 50 | -------------------------------------------------------------------------------- /integration_test.py: -------------------------------------------------------------------------------- 1 | import cjson 2 | import os 3 | import subprocess 4 | import tempfile 5 | import unittest 6 | 7 | from select import select 8 | 9 | HERE = os.path.dirname(os.path.abspath(__file__)) 10 | debug = False 11 | 12 | class TestGlowIntegration(unittest.TestCase): 13 | 14 | def setUp(self): 15 | for tube in tubes(): 16 | drain(tube) 17 | self.listener = Listener() 18 | 19 | def tearDown(self): 20 | self.listener.kill() 21 | 22 | def test_listener_runs_job(self): 23 | tmpfilename = temporary_file_name() 24 | self.listener.start() 25 | subprocess.check_call([glow_executable(), '-tube', 'listener_runs_job', '-stdout', tmpfilename, '/bin/echo', 'listener_runs_job']) 26 | self.listener.wait_for_job_completion({'tube': 'listener_runs_job', 'stdout': tmpfilename}) 27 | with open(tmpfilename, 'r') as outfile: 28 | self.assertEqual('listener_runs_job\n', outfile.read()) 29 | self.listener.interrupt() 30 | 31 | def test_local_runs_job(self): 32 | tmpfilename = temporary_file_name() 33 | subprocess.check_call([glow_executable(), '-local', '-stdout', tmpfilename, '/bin/echo', 'local_runs_job']) 34 | with open(tmpfilename, 'r') as outfile: 35 | self.assertEqual('local_runs_job\n', outfile.read()) 36 | 37 | def test_submit_many_jobs(self): 38 | tmpfilename1 = temporary_file_name() 39 | tmpfilename2 = temporary_file_name() 40 | self.listener.start() 41 | 42 | glow = subprocess.Popen([glow_executable()], stdin=subprocess.PIPE) 43 | print >>glow.stdin, cjson.encode([ 44 | {'cmd': 'echo', 'args': ['submit_many_jobs'], 'tube': 'submit_many_jobs', 'stdout': tmpfilename1 }, 45 | {'cmd': 'echo', 'args': ['submit_many_jobs'], 'tube': 'submit_many_jobs', 'stdout': tmpfilename2 } 46 | ]) 47 | glow.stdin.close() 48 | self.listener.wait_for_job_completion({'tube': 'submit_many_jobs', 'stdout': tmpfilename1}) 49 | self.listener.wait_for_job_completion({'tube': 'submit_many_jobs', 'stdout': tmpfilename2}) 50 | 51 | with open(tmpfilename1, 'r') as outfile: 52 | self.assertEqual('submit_many_jobs\n', outfile.read()) 53 | with open(tmpfilename2, 'r') as outfile: 54 | self.assertEqual('submit_many_jobs\n', outfile.read()) 55 | self.listener.interrupt() 56 | 57 | def test_listener_finishes_job_on_interrupt(self): 58 | tmpfilename = temporary_file_name() 59 | self.listener.start() 60 | 61 | subprocess.check_call([glow_executable(), '-tube', 'listener_finishes_job_on_interrupt', '-stdout', tmpfilename, '%s/sleepthenecho' % HERE, '3', 'listener_finishes_job_on_interrupt']) 62 | self.listener.wait_for_job_start({'tube': 'listener_finishes_job_on_interrupt', 'stdout': tmpfilename}) 63 | 64 | self.listener.interrupt() 65 | self.listener.wait_for_job_completion({'tube': 'listener_finishes_job_on_interrupt', 'stdout': tmpfilename}, seconds=10) 66 | 67 | with open(tmpfilename, 'r') as outfile: 68 | self.assertEqual('listener_finishes_job_on_interrupt\n', outfile.read()) 69 | 70 | def test_listener_kills_job_on_kill(self): 71 | tmpfilename = temporary_file_name() 72 | self.listener.start() 73 | 74 | subprocess.check_call([glow_executable(), '-tube', 'listener_kills_job_on_kill', '-stdout', tmpfilename, '%s/sleepthenecho' % HERE, '5', 'listener_kills_job_on_kill']) 75 | self.listener.wait_for_job_start({'tube': 'listener_kills_job_on_kill', 'stdout': tmpfilename}) 76 | 77 | self.listener.kill() 78 | self.listener.wait_for_shutdown() 79 | 80 | try: 81 | with open(tmpfilename, 'r') as outfile: 82 | self.assertNotEqual('listener_kills_job_on_kill\n', outfile.read()) 83 | except IOError as e: 84 | # ignore if file was never created 85 | if e.errno != 2: raise 86 | 87 | def test_unexecable_job_fails_with_error(self): 88 | self.assertFalse('GLOW_ERRORS' in tubes()) 89 | self.listener.start() 90 | subprocess.check_call([glow_executable(), '-tube', 'unexecable_job_fails_with_error', '/nonexistent/executable']) 91 | self.listener.wait_for_job_failure({'tube': 'unexecable_job_fails_with_error', 'cmd': '/nonexistent/executable'}) 92 | self.assertEqual(1, tubes()['GLOW_ERRORS']['jobs-ready']) 93 | 94 | def test_nonzero_exitstatus_fails_with_error(self): 95 | self.assertFalse('GLOW_ERRORS' in tubes()) 96 | self.listener.start() 97 | subprocess.check_call([glow_executable(), '-tube', 'nonzero_exitstatus_fails_with_error', 'cat', '/nonexistent/file']) 98 | self.listener.wait_for_job_failure({'tube': 'nonzero_exitstatus_fails_with_error', 'cmd': 'cat', 99 | 'args': ['/nonexistent/file']}) 100 | self.assertEqual(1, tubes()['GLOW_ERRORS']['jobs-ready']) 101 | 102 | def test_local_job_failure(self): 103 | self.assertFalse('GLOW_ERRORS' in tubes()) 104 | returncode = subprocess.call([glow_executable(), '-local', 'cat', '/nonexistent/file'], stderr=open('/dev/null', 'w')) 105 | self.assertNotEqual(0, returncode) 106 | self.assertEqual(1, tubes()['GLOW_ERRORS']['jobs-ready']) 107 | 108 | 109 | def test_unexecable_local_job_failure(self): 110 | self.assertFalse('GLOW_ERRORS' in tubes()) 111 | returncode = subprocess.call([glow_executable(), '-local', '/nonexistent/executable'], stderr=open('/dev/null', 'w')) 112 | self.assertNotEqual(0, returncode) 113 | self.assertEqual(1, tubes()['GLOW_ERRORS']['jobs-ready']) 114 | 115 | def test_create_output_file_if_not_exists(self): 116 | tmpfilename = temporary_file_name() 117 | self.assertFalse(os.path.exists(tmpfilename)) 118 | self.listener.start() 119 | subprocess.check_call([glow_executable(), '-tube', 'job', '-stdout', tmpfilename, '/bin/echo', 'job']) 120 | self.listener.wait_for_job_completion({'tube': 'job', 'stdout': tmpfilename}) 121 | with open(tmpfilename, 'r') as outfile: 122 | self.assertEqual('job\n', outfile.read()) 123 | self.listener.interrupt() 124 | 125 | def test_append_to_output_file_if_exists(self): 126 | tmpfilename = temporary_file_name() 127 | self.listener.start() 128 | subprocess.check_call([glow_executable(), '-tube', 'job1', '-stdout', tmpfilename, '/bin/echo', 'job1']) 129 | subprocess.check_call([glow_executable(), '-tube', 'job2', '-stdout', tmpfilename, '/bin/echo', 'job2']) 130 | self.listener.wait_for_job_completion({'tube': 'job2', 'stdout': tmpfilename}) 131 | with open(tmpfilename, 'r') as outfile: 132 | self.assertEqual('job1\njob2\n', outfile.read()) 133 | self.listener.interrupt() 134 | 135 | 136 | 137 | class Listener: 138 | def __init__(self): 139 | self.process = None 140 | 141 | def start(self): 142 | self.process = subprocess.Popen([glow_executable(), '-listen', '-v'], stderr=subprocess.PIPE) 143 | 144 | def interrupt(self): 145 | # Send SIGINT 146 | self.process.send_signal(2) 147 | 148 | def kill(self): 149 | if self.process: 150 | try: 151 | self.process.terminate() 152 | except OSError as e: 153 | # ignore if 'No such process' (already killed) 154 | if e.errno != 3: 155 | raise 156 | 157 | def wait_for_shutdown(self): 158 | self.process.wait() 159 | 160 | def wait_for_job_start(self, job_desc, seconds=3): 161 | self._wait_for_job_update(job_desc, 'RUNNING:', seconds) 162 | 163 | def wait_for_job_completion(self, job_desc, seconds=3): 164 | self._wait_for_job_update(job_desc, 'COMPLETE:', seconds) 165 | 166 | def wait_for_job_failure(self, job_desc, seconds=3): 167 | self._wait_for_job_update(job_desc, 'FAILED:', seconds) 168 | 169 | def _wait_for_job_update(self, job_desc, status, seconds, max_num_non_matching_events=10): 170 | num_events = 0 171 | while num_events < max_num_non_matching_events: 172 | fds, _, _ = select([self.process.stderr], [], [], seconds) 173 | if fds != [self.process.stderr]: 174 | raise Exception('timed out waiting for {0} {1}'.format(status, job_desc)) 175 | line = self.process.stderr.readline().split(' ')[2:] # get rid of log date/time 176 | line = " ".join(line) 177 | if debug: print line 178 | if line.startswith(status): 179 | job = cjson.decode(line[len(status):]) 180 | if all([job[k] == job_desc[k] for k in job_desc]): 181 | return job 182 | num_events += 1 183 | 184 | 185 | def temporary_file_name(): 186 | if debug: 187 | _, tmpfilename = tempfile.mkstemp() 188 | print 'temporary file:', tmpfilename 189 | return tmpfilename 190 | else: 191 | return tempfile.NamedTemporaryFile().name 192 | 193 | def glow_executable(): 194 | return '%s/glow' % HERE 195 | 196 | def tubes(): 197 | return cjson.decode(subprocess.check_output([glow_executable(), '-stats'])) 198 | 199 | def drain(tube): 200 | subprocess.check_call([glow_executable(), '-drain', tube]) 201 | 202 | 203 | 204 | if __name__ == '__main__': 205 | # this works, but nose is better 206 | unittest.main() 207 | -------------------------------------------------------------------------------- /jobqueue.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/nutrun/lentil" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | type JobQueue struct { 11 | q *lentil.Beanstalkd 12 | tubes map[string]*Tube 13 | inclusive bool 14 | filter []string 15 | } 16 | 17 | func NewJobQueue(q *lentil.Beanstalkd, inclusive bool, filter []string) *JobQueue { 18 | this := new(JobQueue) 19 | this.q = q 20 | this.inclusive = inclusive 21 | this.filter = filter 22 | return this 23 | } 24 | 25 | func (this *JobQueue) ReadyTubes() []*Tube { 26 | ready := make([]*Tube, 0) 27 | for _, tube := range this.tubes { 28 | if tube.Ready(this.tubes) { 29 | ready = append(ready, tube) 30 | } 31 | } 32 | return ready 33 | } 34 | 35 | func (this *JobQueue) ReserveFromTubes(tubes []*Tube) (*lentil.Job, error) { 36 | for _, tube := range tubes { 37 | if this.Include(tube.name) { 38 | _, e := this.q.Watch(tube.name) 39 | if e != nil { 40 | return nil, e 41 | } 42 | } 43 | } 44 | job, err := this.q.ReserveWithTimeout(0) 45 | for _, ignored := range tubes { 46 | if this.Include(ignored.name) { 47 | _, e := this.q.Ignore(ignored.name) 48 | if e != nil { 49 | return nil, e 50 | } 51 | } 52 | } 53 | return job, err 54 | } 55 | 56 | func (this *JobQueue) Next() (*lentil.Job, error) { 57 | this.refreshTubes() 58 | return this.ReserveFromTubes(this.ReadyTubes()) 59 | } 60 | 61 | func (this *JobQueue) Delete(id uint64) error { 62 | return this.q.Delete(id) 63 | } 64 | 65 | func (this *JobQueue) MarshalJSON() ([]byte, error) { 66 | tubes, err := queryTubes(this.q) 67 | if err != nil { 68 | return nil, err 69 | } 70 | return json.Marshal(tubes) 71 | } 72 | 73 | func (this *JobQueue) Include(tube string) bool { 74 | for _, filter := range this.filter { 75 | if strings.Contains(tube, filter) { 76 | return this.inclusive 77 | } 78 | } 79 | return !this.inclusive 80 | } 81 | 82 | func (this *JobQueue) refreshTubes() error { 83 | tubes, err := queryTubes(this.q) 84 | if err == nil { 85 | delete(tubes, Config.errorQueue) 86 | this.tubes = tubes 87 | } 88 | return err 89 | } 90 | 91 | func queryTubes(q *lentil.Beanstalkd) (map[string]*Tube, error) { 92 | tubes := make(map[string]*Tube) 93 | names, e := q.ListTubes() 94 | if e != nil { 95 | return nil, e 96 | } 97 | for _, tube := range names { 98 | if tube == "default" { 99 | continue 100 | } 101 | tubestats, e := q.StatsTube(tube) 102 | if e != nil { 103 | continue 104 | } 105 | ready, _ := strconv.Atoi(tubestats["current-jobs-ready"]) 106 | reserved, _ := strconv.Atoi(tubestats["current-jobs-reserved"]) 107 | delayed, _ := strconv.Atoi(tubestats["current-jobs-delayed"]) 108 | pause, _ := strconv.Atoi(tubestats["pause"]) 109 | tubes[tube] = NewTube(tube, uint(reserved), uint(ready), uint(delayed), uint(pause)) 110 | } 111 | return tubes, nil 112 | } 113 | -------------------------------------------------------------------------------- /jobqueue_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/nutrun/lentil" 7 | "testing" 8 | "os" 9 | ) 10 | 11 | func resetConfig() { 12 | Config = NewConfig("", "", "") 13 | Config.deps = make(map[string][]string) 14 | } 15 | 16 | func TestPriority(t *testing.T) { 17 | q := connect(t) 18 | resetConfig() 19 | Config.deps["tube1"] = []string{"tube2"} 20 | put(t, "job1", "tube1", 0, q) 21 | put(t, "job2", "tube2", 0, q) 22 | jobs := NewJobQueue(q, false, make([]string, 0)) 23 | assertNextJob(t, jobs, "job2") 24 | assertNextJob(t, jobs, "job1") 25 | } 26 | 27 | func TestIncludeExclude(t *testing.T) { 28 | q := connect(t) 29 | resetConfig() 30 | all := NewJobQueue(q, false, make([]string, 0)) 31 | if !all.Include("tube") { 32 | t.Errorf("Should include tube") 33 | } 34 | if !all.Include("another") { 35 | t.Errorf("Should include another") 36 | } 37 | none := NewJobQueue(q, true, make([]string, 0)) 38 | if none.Include("none") { 39 | t.Errorf("Should not include tube none") 40 | } 41 | include := NewJobQueue(q, true, []string{"in"}) 42 | if !include.Include("in") { 43 | t.Errorf("Should include tube in") 44 | } 45 | if include.Include("out") { 46 | t.Errorf("Should not include tube out") 47 | } 48 | exclude := NewJobQueue(q, false, []string{"out"}) 49 | if !exclude.Include("in") { 50 | t.Errorf("Should not include tube in") 51 | } 52 | if exclude.Include("out") { 53 | t.Errorf("Should not include tube out") 54 | } 55 | 56 | } 57 | 58 | func TestMoarPriorities(t *testing.T) { 59 | q := connect(t) 60 | resetConfig() 61 | Config.deps["tube3"] = []string{"tube2"} 62 | Config.deps["tube1"] = []string{"tube3"} 63 | put(t, "job11", "tube1", 0, q) 64 | put(t, "job21", "tube2", 0, q) 65 | put(t, "job31", "tube3", 0, q) 66 | put(t, "job22", "tube2", 0, q) 67 | put(t, "job32", "tube3", 0, q) 68 | put(t, "job12", "tube1", 0, q) 69 | jobs := NewJobQueue(q, false, make([]string, 0)) 70 | assertNextJob(t, jobs, "job21") 71 | assertNextJob(t, jobs, "job22") 72 | assertNextJob(t, jobs, "job31") 73 | assertNextJob(t, jobs, "job32") 74 | assertNextJob(t, jobs, "job11") 75 | assertNextJob(t, jobs, "job12") 76 | } 77 | 78 | func TestSleepWhenNoJobs(t *testing.T) { 79 | q := connect(t) 80 | resetConfig() 81 | jobs := NewJobQueue(q, false, make([]string, 0)) 82 | no_job, err := reserveNextJob(t, jobs, "job11") 83 | 84 | if no_job != nil { 85 | t.Error(fmt.Sprintf("Reserved %v when should not have", no_job)) 86 | } 87 | if err == nil { 88 | t.Error(fmt.Sprintf("Should have thrown a TIME_OUT, threw %v instead", err)) 89 | } 90 | 91 | } 92 | 93 | func TestBlockOnReserved(t *testing.T) { 94 | q := connect(t) 95 | resetConfig() 96 | Config.deps["tube1"] = []string{"tube2"} 97 | put(t, "job1", "tube1", 0, q) 98 | put(t, "job2", "tube2", 0, q) 99 | jobs := NewJobQueue(q, false, make([]string, 0)) 100 | job, err := reserveNextJob(t, jobs, "job2") 101 | if err != nil { 102 | t.Error(fmt.Sprintf("Could not reserve job %s", job)) 103 | } 104 | no_job, err := reserveNextJob(t, jobs, "job1") 105 | if no_job != nil { 106 | t.Error(fmt.Sprintf("Reserved %v when should not have", no_job)) 107 | } 108 | if err == nil { 109 | t.Error(fmt.Sprintf("Should have thrown a TIME_OUT, threw %v instead", err)) 110 | } 111 | 112 | } 113 | 114 | func TestBlockOnIgnored(t *testing.T) { 115 | q := connect(t) 116 | resetConfig() 117 | Config.deps["another"] = []string{"block_on"} 118 | put(t, "job", "block_on", 0, q) 119 | put(t, "another", "another", 0, q) 120 | jobs := NewJobQueue(q, false, []string{"block_on"}) 121 | no_job, err := reserveNextJob(t, jobs, "job") 122 | if no_job != nil { 123 | t.Error(fmt.Sprintf("Reserved %v when should not have", no_job)) 124 | } 125 | if err == nil { 126 | t.Error(fmt.Sprintf("Should have thrown a TIME_OUT, threw %v instead", err)) 127 | } 128 | 129 | } 130 | 131 | func assertNextJob(t *testing.T, jobqueue *JobQueue, expected string) { 132 | jobinfo := make(map[string]string) 133 | job, e := jobqueue.Next() 134 | if e != nil { 135 | t.Error(fmt.Sprintf("%v on [%v]", e, expected)) 136 | return 137 | } 138 | json.Unmarshal(job.Body, &jobinfo) 139 | if jobinfo["name"] != expected { 140 | t.Errorf("%s != %s\n", expected, jobinfo["name"]) 141 | } 142 | jobqueue.Delete(job.Id) 143 | } 144 | 145 | func reserveNextJob(t *testing.T, jobqueue *JobQueue, expected string) (*lentil.Job, error) { 146 | job, e := jobqueue.Next() 147 | if e != nil { 148 | return nil, e 149 | } 150 | return job, e 151 | } 152 | 153 | func put(t *testing.T, jobName, tube string, delay int, q *lentil.Beanstalkd) { 154 | job := make(map[string]string) 155 | job["tube"] = tube 156 | job["name"] = jobName 157 | jobjson, _ := json.Marshal(job) 158 | e := q.Use(tube) 159 | if e != nil { 160 | t.Fatal(e) 161 | } 162 | _, e = q.Put(0, delay, 60, jobjson) 163 | if e != nil { 164 | t.Error(e) 165 | } 166 | } 167 | 168 | func connect(t *testing.T) *lentil.Beanstalkd { 169 | gq := os.Getenv("GLOW_QUEUE") 170 | if gq == "" { 171 | gq = "localhost:11300" 172 | } 173 | q, e := lentil.Dial(gq) 174 | if e != nil { 175 | t.Fatal(e) 176 | } 177 | // Clear beanstalkd 178 | tubes, e := q.ListTubes() 179 | if e != nil { 180 | t.Fatal(e) 181 | } 182 | for _, tube := range tubes { 183 | if tube == "default" { 184 | continue 185 | } 186 | _, e = q.Watch(tube) 187 | if e != nil { 188 | t.Fatal(e) 189 | } 190 | for { 191 | job, e := q.ReserveWithTimeout(0) 192 | if e != nil { 193 | break 194 | } 195 | q.Delete(job.Id) 196 | } 197 | _, e := q.Ignore(tube) 198 | if e != nil { 199 | t.Fatal(e) 200 | } 201 | } 202 | return q 203 | } 204 | -------------------------------------------------------------------------------- /listener.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/nutrun/lentil" 6 | "log" 7 | "os" 8 | "os/signal" 9 | "strings" 10 | "syscall" 11 | "time" 12 | ) 13 | 14 | type Listener struct { 15 | Runner 16 | verbose bool 17 | jobqueue *JobQueue 18 | sig os.Signal 19 | logpath string 20 | logfile *os.File 21 | } 22 | 23 | func NewListener(verbose, inclusive bool, filter []string, logpath string) (*Listener, error) { 24 | this := new(Listener) 25 | this.logpath = logpath 26 | err := this.resetLog() 27 | if err != nil { 28 | return nil, err 29 | } 30 | q, err := lentil.Dial(Config.QueueAddr) 31 | if err != nil { 32 | return nil, err 33 | } 34 | this.q = q 35 | this.verbose = verbose 36 | this.jobqueue = NewJobQueue(this.q, inclusive, filter) 37 | return this, nil 38 | } 39 | 40 | func (this *Listener) run() { 41 | go this.trap() 42 | 43 | listenerloop: 44 | for { 45 | if this.sig != nil { 46 | os.Exit(0) 47 | } 48 | e := Config.LoadDeps() 49 | if e != nil { 50 | this.logger.Fatalf("Error loading dependency config: %s\n", e) 51 | } 52 | job, e := this.jobqueue.Next() 53 | if e != nil { 54 | if strings.Contains(e.Error(), "TIMED_OUT") { 55 | time.Sleep(time.Second) 56 | goto listenerloop 57 | } 58 | this.logger.Fatal(e) 59 | } 60 | if this.verbose { 61 | this.logger.Printf("RUNNING: %s", job.Body) 62 | } 63 | msg := new(Message) 64 | e = json.Unmarshal([]byte(job.Body), &msg) 65 | if e != nil { 66 | this.catch(msg, e) 67 | } 68 | e = this.execute(msg) 69 | if e == nil { 70 | if this.verbose { 71 | this.logger.Printf("COMPLETE: %s", job.Body) 72 | } 73 | } else { 74 | this.logger.Printf("FAILED: %s", job.Body) 75 | } 76 | e = this.jobqueue.Delete(job.Id) 77 | if e != nil { 78 | this.catch(msg, e) 79 | } 80 | } 81 | } 82 | 83 | // Wait for currently running job to finish before exiting on SIGTERM and SIGINT 84 | func (this *Listener) trap() { 85 | receivedSignal := make(chan os.Signal) 86 | signal.Notify(receivedSignal, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) 87 | this.sig = <-receivedSignal 88 | if this.sig.String() == syscall.SIGTERM.String() { 89 | this.logger.Printf("Got signal %d. Killing current job.\n", this.sig) 90 | if this.proc != nil { 91 | this.proc.Kill() 92 | } 93 | os.Exit(1) 94 | } else if this.sig.String() == syscall.SIGHUP.String() { 95 | this.logger.Printf("Got signal %d. Reopening log.\n", this.sig) 96 | e := this.resetLog() 97 | if e != nil { 98 | panic(e) 99 | } 100 | this.sig = nil 101 | } else if this.sig.String() == syscall.SIGINT.String() { 102 | this.logger.Printf("Got signal %d. Waiting for current job to complete.\n", this.sig) 103 | } 104 | go this.trap() 105 | } 106 | 107 | func (this *Listener) resetLog() error { 108 | if this.logfile != nil { 109 | this.logfile.Close() // Ignoring this error... 110 | } 111 | logfile, e := os.OpenFile(this.logpath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 112 | if e != nil { 113 | return e 114 | } 115 | this.logfile = logfile 116 | this.logger = log.New(this.logfile, "", log.LstdFlags) 117 | return nil 118 | } 119 | -------------------------------------------------------------------------------- /listener_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "os" 7 | "strings" 8 | "testing" 9 | ) 10 | 11 | func createTestMessage(cmd, out, workdir string) (*Message, error) { 12 | tokens := strings.Split(cmd, " ") 13 | return NewMessage(tokens[0], tokens[1:len(tokens)], "", workdir, out, out, "testtube", 0, 0) 14 | } 15 | 16 | func TestOutput(t *testing.T) { 17 | listener := new(Listener) 18 | msg, e := createTestMessage("echo you suck", "test.out", ".") 19 | if e != nil { 20 | t.Fatal(e) 21 | } 22 | listener.execute(msg) 23 | out, e := ioutil.ReadFile("test.out") 24 | if e != nil { 25 | t.Fatal(e) 26 | } 27 | if string(out) != "you suck\n" { 28 | t.Errorf("[%s] isn't you suck", out) 29 | } 30 | e = os.Remove("test.out") 31 | if e != nil { 32 | t.Fatal(e) 33 | } 34 | } 35 | 36 | func TestPutErrorOnBeanstalk(t *testing.T) { 37 | listener, err := NewListener(false, false, []string{}, "/dev/null") 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | msg, e := createTestMessage("lsdonmybrain", "test.out", ".") 42 | if e != nil { 43 | t.Fatal(e) 44 | } 45 | listener.execute(msg) 46 | listener.q.Watch(Config.errorQueue) 47 | failed, err := listener.q.ReserveWithTimeout(0) 48 | if err != nil { 49 | t.Fatal(err) 50 | } 51 | result := new(ErrMessage) 52 | err = json.Unmarshal(failed.Body, result) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | if result.Cmd != "lsdonmybrain" { 57 | t.Errorf("Recieved Unexpected Msg [%v]", string(failed.Body)) 58 | } 59 | listener.q.Delete(failed.Id) 60 | err = os.Remove("test.out") 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /mail.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "net/smtp" 4 | 5 | 6 | func SendMail(addr string, a smtp.Auth, from string, to []string, msg []byte) error { 7 | if (a != nil) { 8 | return smtp.SendMail(addr, a, from, to, msg) 9 | } 10 | c, err := smtp.Dial(addr) 11 | if err != nil { 12 | return err 13 | } 14 | defer c.Text.Close() 15 | if err = c.Hello("localhost"); err != nil { 16 | return err 17 | } 18 | if err = c.Mail(from); err != nil { 19 | return err 20 | } 21 | for _, addr := range to { 22 | if err = c.Rcpt(addr); err != nil { 23 | return err 24 | } 25 | } 26 | w, err := c.Data() 27 | if err != nil { 28 | return err 29 | } 30 | _, err = w.Write(msg) 31 | if err != nil { 32 | return err 33 | } 34 | err = w.Close() 35 | if err != nil { 36 | return err 37 | } 38 | return c.Quit() 39 | } -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "github.com/nutrun/lentil" 8 | "log" 9 | "os" 10 | "strings" 11 | ) 12 | 13 | var listener *bool = flag.Bool("listen", false, "Start listener") 14 | var help *bool = flag.Bool("help", false, "Show help") 15 | var mailto *string = flag.String("mailto", "", "Who to email on failure (comma separated) [submit]") 16 | var workdir *string = flag.String("workdir", "/tmp", "Directory to run job from [submit]") 17 | var stdout *string = flag.String("stdout", "/dev/null", "File to send job's stdout [submit]") 18 | var stderr *string = flag.String("stderr", "/dev/null", "File to send job's stderr [submit]") 19 | var tube *string = flag.String("tube", "", "Beanstalkd tube to send the job to [submit]") 20 | var stats *bool = flag.Bool("stats", false, "Show queue stats") 21 | var drain *string = flag.String("drain", "", "Empty tubes (comma separated)") 22 | var verbose *bool = flag.Bool("v", false, "Increase verbosity") 23 | var exclude *string = flag.String("exclude", "", "Tubes to ignore (comma separated) [listen]") 24 | var priority *int = flag.Int("pri", 0, "Job priority (smaller runs first) [submit]") 25 | var delay *int = flag.Int("delay", 0, "Job delay in seconds [submit]") 26 | var local *bool = flag.Bool("local", false, "Run locally, reporting errors to the configured beanstalk") 27 | var pause *string = flag.String("pause", "", "Pause tubes (comma separated)") 28 | var pausedelay *int = flag.Int("pause-delay", 0, "How many seconds to pause tubes for") 29 | var mailfrom *string = flag.String("mail-from", "glow@example.com", "Email 'from' field [listen]") 30 | var smtpserver *string = flag.String("smtp-server", "", "Server to use for sending emails [listen]") 31 | var deps *string = flag.String("deps", "", "Path to tube dependency config file [listen]") 32 | var logpath *string = flag.String("log", "/dev/stderr", "Path to log file [listen]") 33 | var mlen *int = flag.Int("mlen", 65536, "Max length of messeges sent to beanstalk in bytes [submit]") 34 | var Config *Configuration 35 | 36 | func main() { 37 | flag.Parse() 38 | Config = NewConfig(*deps, *smtpserver, *mailfrom) 39 | 40 | lentil.ReaderSize = *mlen 41 | 42 | if *listener { 43 | include := false 44 | filter := make([]string, 0) 45 | if *exclude != "" { 46 | filter = strings.Split(*exclude, ",") 47 | } 48 | l, e := NewListener(*verbose, include, filter, *logpath) 49 | if e != nil { 50 | fmt.Fprintln(os.Stderr, e) 51 | os.Exit(1) 52 | } 53 | l.run() 54 | return 55 | } else if *help { 56 | flag.Usage() 57 | os.Exit(1) 58 | } 59 | 60 | if *local { 61 | executable, arguments := parseCommand() 62 | // hack: local doesn't need tube, defaulting it to respect the Message API 63 | msg, e := NewMessage(executable, arguments, *mailto, *workdir, *stdout, *stderr, "localignore", 0, 0) 64 | if e != nil { 65 | fmt.Fprintln(os.Stderr, e) 66 | os.Exit(1) 67 | } 68 | logfile, e := os.OpenFile(*logpath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 69 | if e != nil { 70 | fmt.Fprintln(os.Stderr, e) 71 | os.Exit(1) 72 | } 73 | runner, e := NewRunner(*verbose, log.New(logfile, "", log.LstdFlags)) 74 | if e != nil { 75 | fmt.Fprintln(os.Stderr, e) 76 | os.Exit(1) 77 | } 78 | e = runner.execute(msg) 79 | if e != nil { 80 | fmt.Fprintln(os.Stderr, e) 81 | os.Exit(1) 82 | } 83 | return 84 | } 85 | 86 | c, e := NewClient(*verbose) 87 | if e != nil { 88 | fmt.Fprintln(os.Stderr, e) 89 | os.Exit(1) 90 | } 91 | 92 | if *drain != "" { 93 | e = c.drain(*drain) 94 | if e != nil { 95 | fmt.Fprintln(os.Stderr, e) 96 | os.Exit(1) 97 | } 98 | } else if *pause != "" { 99 | if *pausedelay == 0 { 100 | fmt.Fprintln(os.Stderr, "Usage: glow -pause= -pause-delay=") 101 | os.Exit(1) 102 | } 103 | e = c.pause(*pause, *pausedelay) 104 | } else if *stats { 105 | e = c.stats() 106 | if e != nil { 107 | fmt.Fprintln(os.Stderr, e) 108 | os.Exit(1) 109 | } 110 | } else if len(flag.Args()) == 0 { // Queue up many jobs from STDIN 111 | in := bufio.NewReaderSize(os.Stdin, 1024*1024) 112 | input := make([]byte, 0) 113 | for { 114 | line, e := in.ReadSlice('\n') 115 | if e != nil { 116 | if e.Error() == "EOF" { 117 | break 118 | } 119 | fmt.Fprintln(os.Stderr, e) 120 | os.Exit(1) 121 | } 122 | input = append(input, line...) 123 | } 124 | e = c.putMany(input) 125 | if e != nil { 126 | fmt.Fprintln(os.Stderr, e) 127 | os.Exit(1) 128 | } 129 | } else { // Queue up one job 130 | executable, arguments := parseCommand() 131 | msg, e := NewMessage(executable, arguments, *mailto, *workdir, *stdout, *stderr, *tube, *priority, *delay) 132 | if e != nil { 133 | fmt.Fprintln(os.Stderr, e) 134 | os.Exit(1) 135 | } 136 | e = c.put(msg) 137 | if e != nil { 138 | fmt.Fprintln(os.Stderr, e) 139 | os.Exit(1) 140 | } 141 | } 142 | } 143 | 144 | func parseCommand() (string, []string) { 145 | return flag.Args()[0], flag.Args()[1:len(flag.Args())] 146 | } 147 | -------------------------------------------------------------------------------- /message.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "os" 10 | "os/user" 11 | "path/filepath" 12 | "strings" 13 | ) 14 | 15 | const MAX_OUTFILE_READ_LEN = 16 * 1024 16 | 17 | type Message struct { 18 | Executable string `json:"cmd"` 19 | Arguments []string `json:"args"` 20 | Mailto string `json:"mailto"` 21 | Workdir string `json:"workdir"` 22 | Stdout string `json:"stdout"` 23 | Stderr string `json:"stderr"` 24 | Tube string `json:"tube"` 25 | Priority int `json:"pri"` 26 | Delay int `json:"delay"` 27 | User string `json:user` 28 | Env []string `json:env` 29 | } 30 | 31 | func NewMessage(executable string, args []string, mailto, workdir, stdout, stderr, tube string, pri, delay int) (*Message, error) { 32 | if tube == "" { 33 | return nil, errors.New("Missing required param -tube") 34 | } 35 | if workdir == "" { 36 | workdir = "/tmp" 37 | } 38 | absoluteWorkdir, e := filepath.Abs(workdir) 39 | if e != nil { 40 | return nil, e 41 | } 42 | if stdout == "" { 43 | stdout = "/dev/null" 44 | } 45 | if stderr == "" { 46 | stderr = "/dev/null" 47 | } 48 | u, e := user.Current() 49 | if e != nil { 50 | return nil, e 51 | } 52 | return &Message{executable, args, mailto, absoluteWorkdir, stdout, stderr, tube, pri, delay, u.Username, os.Environ()}, nil 53 | } 54 | 55 | func MessagesFromJSON(jsonstr []byte) ([]*Message, error) { 56 | vals := make([]*Message, 0) 57 | e := json.Unmarshal(jsonstr, &vals) 58 | if e != nil { 59 | return nil, e 60 | } 61 | messages := make([]*Message, len(vals)) 62 | for i, m := range vals { 63 | msg, e := NewMessage(m.Executable, m.Arguments, m.Mailto, m.Workdir, m.Stdout, m.Stderr, m.Tube, m.Priority, m.Delay) 64 | if e != nil { 65 | return nil, e 66 | } 67 | messages[i] = msg 68 | } 69 | return messages, nil 70 | } 71 | 72 | func (this *Message) getCommand() string { 73 | cmd := this.Executable 74 | if len(this.Arguments) > 0 { 75 | cmd += " " + strings.Join(this.Arguments, " ") 76 | } 77 | return cmd 78 | } 79 | 80 | // Read up to MAX_OUTFILE_READ_LEN from the files we send stdout or stderr to 81 | func (this *Message) readOutputFile(path string) ([]byte, error) { 82 | if path == "/dev/stdout" || path == "/dev/stderr" { 83 | return []byte{}, nil 84 | } 85 | f, e := os.Open(path) 86 | if e != nil { 87 | return nil, e 88 | } 89 | br := bufio.NewReader(f) 90 | lr := &io.LimitedReader{br, MAX_OUTFILE_READ_LEN} 91 | buf := make([]byte, MAX_OUTFILE_READ_LEN) 92 | n, e := lr.Read(buf) 93 | if e != nil { 94 | return nil, e 95 | } 96 | return buf[:n], nil 97 | } 98 | 99 | func (this *Message) readOut() string { 100 | hostname, _ := os.Hostname() 101 | content := make([]byte, 0) 102 | content = append(content, []byte(fmt.Sprintf("hostname: %v\nstdout: %v\nstderr: %v\n", hostname, this.Stdout, this.Stderr))...) 103 | stdout, e := this.readOutputFile(this.Stdout) 104 | if e != nil { 105 | content = append(content, []byte( 106 | fmt.Sprintf("Could not read stdout output from [%s]. %s\n", this.Stdout, e))...) 107 | } else { 108 | content = append(content, []byte("STDOUT:\n")...) 109 | content = append(content, stdout...) 110 | content = append(content, []byte("\n")...) 111 | } 112 | stderr, e := this.readOutputFile(this.Stderr) 113 | if e != nil { 114 | content = append(content, []byte( 115 | fmt.Sprintf("Could not read stderr output from [%s]. %s\n", this.Stderr, e))...) 116 | } else { 117 | content = append(content, []byte("STDERR:\n")...) 118 | content = append(content, stderr...) 119 | } 120 | return string(content) 121 | } 122 | 123 | type ErrMessage struct { 124 | Cmd string `json:"cmd"` 125 | Error string `json:"error"` 126 | Log string `json:"log"` 127 | } 128 | 129 | func NewErrMessage(msg *Message, e error) *ErrMessage { 130 | return &ErrMessage{msg.getCommand(), e.Error(), msg.readOut()} 131 | } 132 | -------------------------------------------------------------------------------- /message_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | func TestTubeRequired(t *testing.T) { 11 | _, e := NewMessage("executable", []string{"arg"}, "", "", "", "", "", 0, 0) 12 | if e == nil { 13 | t.Errorf("Should have missing tube error") 14 | } 15 | if e.Error() != "Missing required param -tube" { 16 | t.Errorf("[%s] isn't [%s]", e.Error(), "Missing required param -tube") 17 | } 18 | // Test same thing from json 19 | jsonstr := `[{"cmd": "echo", "args": ["arg1", "arg2"], "out": "out.txt", "pri": 15, "delay": 120}]` 20 | _, e = MessagesFromJSON([]byte(jsonstr)) 21 | if e == nil { 22 | t.Errorf("Should have missing tube error") 23 | } 24 | if e.Error() != "Missing required param -tube" { 25 | t.Errorf("[%s] isn't [%s]", e.Error(), "Missing required param -tube") 26 | } 27 | } 28 | 29 | func TestValidJSONUnmarshall(t *testing.T) { 30 | jsonstr := `[{"tube": "testtube", "cmd": "echo", "args": ["arg1", "arg2"], "stdout": "out.txt", "stderr": "err.txt", "pri": 15, "delay": 120}]` 31 | messages, e := MessagesFromJSON([]byte(jsonstr)) 32 | if e != nil { 33 | t.Fatal(e) 34 | } 35 | if len(messages) != 1 { 36 | t.Errorf("[%d] isn't [%d]", len(messages), 1) 37 | } 38 | msg := messages[0] 39 | if msg.Tube != "testtube" { 40 | t.Errorf("[%s] isn't [%s]", msg.Tube, "testtube") 41 | } 42 | if msg.Executable != "echo" { 43 | t.Errorf("[%s] isn't [%s]", msg.Executable, "echo") 44 | } 45 | if msg.Arguments[0] != "arg1" { 46 | t.Errorf("[%s] isn't [%s]", msg.Arguments[0], "arg1") 47 | } 48 | if msg.Arguments[1] != "arg2" { 49 | t.Errorf("[%s] isn't [%s]", msg.Arguments[1], "arg2") 50 | } 51 | if msg.Stdout != "out.txt" { 52 | t.Errorf("[%s] isn't [%s]", msg.Stdout, "out.txt") 53 | } 54 | if msg.Stderr != "err.txt" { 55 | t.Errorf("[%s] isn't [%s]", msg.Stderr, "err.txt") 56 | } 57 | if msg.Priority != 15 { 58 | t.Errorf("[%d] isn't [%d]", msg.Priority, 15) 59 | } 60 | if msg.Delay != 120 { 61 | t.Errorf("[%d] isn't [%d]", msg.Delay, 120) 62 | } 63 | } 64 | 65 | func TestDefaultWorkdir(t *testing.T) { 66 | msg, e := NewMessage("executable", []string{"arg"}, "", "", "", "", "testtube", 0, 0) 67 | if e != nil { 68 | t.Fatal(e) 69 | } 70 | if msg.Workdir != "/tmp" { 71 | t.Errorf("[%s] isn't [%s]", msg.Workdir, "/tmp") 72 | } 73 | } 74 | 75 | func TestReadOut(t *testing.T) { 76 | logfile := "glowtestreadout.log" 77 | logdata := "log data whatevs" 78 | e := ioutil.WriteFile(logfile, []byte(logdata), os.ModePerm) 79 | if e != nil { 80 | t.Fatal(e) 81 | } 82 | defer func(t *testing.T, logfile string) { 83 | e := os.Remove(logfile) 84 | if e != nil { 85 | t.Fatal(e) 86 | } 87 | }(t, logfile) 88 | msg, e := NewMessage("executable", []string{"arg"}, "email", "workdir", logfile, logfile, "testtube", 0, 0) 89 | hostname, e := os.Hostname() 90 | if e != nil { 91 | t.Fatal(e) 92 | } 93 | expectedOutput := fmt.Sprintf(`hostname: %s 94 | stdout: %s 95 | stderr: %s 96 | STDOUT: 97 | log data whatevs 98 | STDERR: 99 | log data whatevs`, hostname, logfile, logfile) 100 | actualOutput := msg.readOut() 101 | if expectedOutput != actualOutput { 102 | t.Errorf("[%s] isn't [%s]", expectedOutput, actualOutput) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /runner.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/nutrun/lentil" 7 | "log" 8 | "os" 9 | "os/exec" 10 | "os/user" 11 | "path/filepath" 12 | "strings" 13 | ) 14 | 15 | type Runner struct { 16 | q *lentil.Beanstalkd 17 | proc *os.Process 18 | verbose bool 19 | logger *log.Logger 20 | } 21 | 22 | func NewRunner(verbose bool, logger *log.Logger) (*Runner, error) { 23 | this := new(Runner) 24 | this.logger = logger 25 | q, err := lentil.Dial(Config.QueueAddr) 26 | 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | this.q = q 32 | this.verbose = verbose 33 | return this, nil 34 | } 35 | 36 | func (this *Runner) execute(msg *Message) error { 37 | workdir := msg.Workdir 38 | e := os.Chdir(workdir) 39 | if e != nil { 40 | this.catch(msg, e) 41 | return e 42 | } 43 | 44 | u, e := user.Current() 45 | if e != nil { 46 | this.catch(msg, e) 47 | return e 48 | } 49 | 50 | var cmd *exec.Cmd 51 | 52 | if u.Username != msg.User { 53 | cmd = exec.Command("su", msg.User, "-c", msg.getCommand()) 54 | cmd.Env = msg.Env 55 | } else { 56 | cmd = exec.Command(msg.Executable, msg.Arguments...) 57 | } 58 | 59 | stdoutDir := filepath.Dir(msg.Stdout) 60 | os.MkdirAll(stdoutDir, 0755) 61 | stderrDir := filepath.Dir(msg.Stderr) 62 | os.MkdirAll(stderrDir, 0755) 63 | stdoutF, e := os.OpenFile(msg.Stdout, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 64 | if e != nil { 65 | this.catch(msg, e) 66 | return e 67 | } 68 | defer stdoutF.Close() 69 | stderrF, e := os.OpenFile(msg.Stderr, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 70 | if e != nil { 71 | this.catch(msg, e) 72 | return e 73 | } 74 | defer stderrF.Close() 75 | cmd.Stdout = stdoutF 76 | cmd.Stderr = stderrF 77 | if this.verbose { 78 | this.logger.Printf("INFO: Running command '%s %s'\n", msg.Executable, strings.Join(msg.Arguments, " ")) 79 | this.logger.Printf("INFO: STDOUT to %s\n", msg.Stdout) 80 | this.logger.Printf("INFO: STDERR to %s\n", msg.Stderr) 81 | } 82 | e = cmd.Start() 83 | if e != nil { 84 | this.catch(msg, e) 85 | return e 86 | } 87 | this.proc = cmd.Process 88 | e = cmd.Wait() 89 | if e != nil { 90 | this.catch(msg, e) 91 | } 92 | this.proc = nil 93 | return e 94 | } 95 | 96 | // Log and email errors 97 | func (this *Runner) catch(msg *Message, e error) { 98 | this.logger.Printf("ERROR: %s\n", e) 99 | this.mail(msg, e) 100 | this.publishError(msg, e) 101 | } 102 | 103 | func (this *Runner) publishError(msg *Message, e error) { 104 | err := this.q.Use(Config.errorQueue) 105 | if err != nil { 106 | this.logger.Printf("ERROR: %s\n", err) 107 | return 108 | } 109 | payload, err := json.Marshal(NewErrMessage(msg, e)) 110 | if err != nil { 111 | this.logger.Printf("ERROR: %s\n", err) 112 | return 113 | } 114 | _, err = this.q.Put(0, 0, 60*60, payload) 115 | if err != nil { 116 | this.logger.Printf("ERROR: %s\n", err) 117 | } 118 | } 119 | 120 | func (this *Runner) mail(msg *Message, e error) { 121 | if Config.SmtpServer == "" { 122 | return 123 | } 124 | if len(msg.Mailto) < 1 { //no email addresses 125 | return 126 | } 127 | to := strings.Split(msg.Mailto, ",") 128 | subject := fmt.Sprintf("Subject: FAILED: %s\r\n\r\n", msg.getCommand()) 129 | hostname, _ := os.Hostname() 130 | mail := fmt.Sprintf("%s%s", subject, fmt.Sprintf("Ran on [%s]\n%s\n%s\n%s", hostname, subject, e, msg.readOut())) 131 | e = SendMail(Config.SmtpServer, nil, Config.MailFrom, to, []byte(mail)) 132 | if e != nil { 133 | this.logger.Printf("ERROR: %s\n", e) 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /runner_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "testing" 10 | ) 11 | 12 | func TestRunnerOutput(t *testing.T) { 13 | devnull, e := os.OpenFile(os.DevNull, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 14 | if e != nil { 15 | t.Fatal(e) 16 | } 17 | l := log.New(devnull, "", log.LstdFlags) 18 | runner, e := NewRunner(false, l) 19 | if e != nil { 20 | t.Fatal(e) 21 | } 22 | msg, e := createTestMessage("echo you suck", "test.out", ".") 23 | if e != nil { 24 | t.Fatal(e) 25 | } 26 | runner.execute(msg) 27 | out, e := ioutil.ReadFile("test.out") 28 | if e != nil { 29 | t.Fatal(e) 30 | } 31 | if string(out) != "you suck\n" { 32 | t.Errorf("[%s] isn't you suck", out) 33 | } 34 | e = os.Remove("test.out") 35 | if e != nil { 36 | t.Fatal(e) 37 | } 38 | } 39 | 40 | func TestRunnerShouldPutErrorOnBeanstalk(t *testing.T) { 41 | devnull, err := os.OpenFile(os.DevNull, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | l := log.New(devnull, "", log.LstdFlags) 46 | runner, err := NewRunner(false, l) 47 | if err != nil { 48 | t.Fatal(err) 49 | } 50 | log.SetOutput(bytes.NewBufferString("")) 51 | msg, e := createTestMessage("lsdonmybrain", "test.out", ".") 52 | if e != nil { 53 | t.Fatal(e) 54 | } 55 | runner.execute(msg) 56 | runner.q.Watch(Config.errorQueue) 57 | failed, err := runner.q.ReserveWithTimeout(0) 58 | if err != nil { 59 | t.Fatal(err) 60 | } 61 | result := new(ErrMessage) 62 | err = json.Unmarshal(failed.Body, result) 63 | if err != nil { 64 | t.Fatal(err) 65 | } 66 | if result.Cmd != "lsdonmybrain" { 67 | t.Errorf("Recieved Unexpected Msg [%v]", string(failed.Body)) 68 | } 69 | runner.q.Delete(failed.Id) 70 | err = os.Remove("test.out") 71 | if err != nil { 72 | t.Fatal(err) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /sleepthenecho: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | seconds=$1 4 | shift 5 | sleep $seconds 6 | echo $* 7 | -------------------------------------------------------------------------------- /tube.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | ) 6 | 7 | type Tube struct { 8 | deps []string 9 | name string 10 | reserved uint 11 | ready uint 12 | delayed uint 13 | pause uint 14 | } 15 | 16 | func NewTube(name string, reserved, ready, delayed, pause uint) *Tube { 17 | this := new(Tube) 18 | this.name = name 19 | this.reserved = reserved 20 | this.ready = ready 21 | this.delayed = delayed 22 | this.pause = pause 23 | this.deps = make([]string, 0) 24 | if deps, found := Config.deps[name]; found { 25 | this.deps = deps[:] 26 | } 27 | return this 28 | } 29 | 30 | func (this *Tube) Jobs() uint { 31 | return this.delayed + this.ready + this.reserved 32 | } 33 | 34 | func (this *Tube) Ready(queue map[string]*Tube) bool { 35 | for _, dep := range this.deps { 36 | if tube, found := queue[dep]; found { 37 | if tube.Jobs() > 0 { 38 | return false 39 | } 40 | } 41 | } 42 | return true 43 | } 44 | 45 | func (this *Tube) MarshalJSON() ([]byte, error) { 46 | stats := make(map[string]interface{}) 47 | stats["jobs-ready"] = this.ready 48 | stats["jobs-reserved"] = this.reserved 49 | stats["jobs-delayed"] = this.delayed 50 | if this.pause != 0 { 51 | stats["pause"] = this.pause 52 | } 53 | return json.Marshal(stats) 54 | } 55 | -------------------------------------------------------------------------------- /tube_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "testing" 4 | 5 | func TestDependencies(t *testing.T) { 6 | resetConfig() // Implemented in jobqueue_test.go 7 | Config.deps["tube1"] = []string{"tube2"} 8 | queue := make(map[string]*Tube) 9 | queue["tube1"] = NewTube("tube1", 0, 1, 0, 0) 10 | queue["tube2"] = NewTube("tube2", 0, 0, 0, 0) 11 | if !queue["tube1"].Ready(queue) { 12 | t.Error("y u no redi?") 13 | } 14 | queue["tube2"].ready = 1 15 | if queue["tube1"].Ready(queue) { 16 | t.Error("y u no redi?") 17 | } 18 | queue["tube2"].ready = 0 19 | queue["tube2"].delayed = 1 20 | if queue["tube1"].Ready(queue) { 21 | t.Error("y u no redi?") 22 | } 23 | } 24 | --------------------------------------------------------------------------------