├── .gitignore ├── LICENSE ├── README.md ├── curler ├── __init__.py ├── service.py └── twisted_gears │ ├── __init__.py │ ├── client.py │ ├── constants.py │ └── test_client.py ├── setup.py ├── test ├── run.sh └── webserver.py └── twisted └── plugins └── curler_plugin.py /.gitignore: -------------------------------------------------------------------------------- 1 | dropin.cache 2 | *.pid 3 | *.log 4 | *.pyc 5 | MANIFEST 6 | dist/ 7 | build/ 8 | *.iml 9 | *.ipr 10 | _trial_temp/ 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) Garret Heaton 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | curler 2 | ======= 3 | 4 | A [Gearman][gm] worker which hits a web service to do its work. Why? Because you already have a bunch of solid code in your web framework and don't want to port it to a standalone worker service. 5 | 6 | Basic flow: 7 | 8 | 1. User #10 on your service submits a form requesting some time-intensive task. 9 | 1. You insert a job into the `curler` Gearman queue with the data `{"method": "do_thing", "data": 10}` 10 | 1. curler receives the job and POSTs to `http://localhost/jobs/do_thing` with `data=10`. 11 | 1. You pull `10` from `post['data']` and do the thing. 12 | 13 | Installation & Usage 14 | -------------------- 15 | curler runs as a [twistd](http://linux.die.net/man/1/twistd) service. To install & run: 16 | 17 | $ git clone http://github.com/powdahound/curler.git 18 | $ cd curler/ 19 | $ sudo python setup.py install 20 | $ twistd --nodaemon curler --base-urls=http://localhost/jobs 21 | 22 | There are a few arguments to curler: 23 | 24 | * `--base-urls` - Base URLs which the `method` property is appended to. You can specify multiple URLs by separating them with commas and one will be chosen at random. 25 | * `--job-queue` - The Gearman job queue to monitor (defaults to 'curler'). 26 | * `--gearmand-server` - Gearman job servers to get jobs from (defaults to 'localhost:4730'). Separate multiple with commas. 27 | * `--num-workers` - Number of workers to run per server (# of jobs you can process in parallel). Uses nonblocking Twisted APIs instead of spawning extra processes or threads. Defaults to 5. 28 | * `--verbose` - Enables verbose logging (includes full request/response data). 29 | 30 | Run `twistd --help` to see how to run as a daemon. 31 | 32 | Job data 33 | ------- 34 | 35 | Jobs inserted into the curler queue must contain two properties: 36 | 37 | * `method` - Relative path of the URL to hit. 38 | * `data` - Arbitrary data string. POSTed as the `data` property. Use JSON if you need structure. 39 | 40 | Dependencies 41 | ------------- 42 | * Python 2.6+ 43 | * [Twisted](http://twistedmatrix.com/trac/) 44 | 45 | [gm]: http://gearman.org 46 | -------------------------------------------------------------------------------- /curler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hipchat/curler/b22bf79ecc4c1985038e0ba183ca7125be4b8ac0/curler/__init__.py -------------------------------------------------------------------------------- /curler/service.py: -------------------------------------------------------------------------------- 1 | import json 2 | import random 3 | import traceback 4 | import urllib 5 | from twisted_gears import client 6 | from time import time 7 | from twisted.application.service import Service 8 | from twisted.internet import defer, protocol, reactor, task 9 | from twisted.python import log 10 | from twisted.web.client import getPage, HTTPClientFactory 11 | from twisted.web.error import Error 12 | 13 | # we don't want to hear about each web request we make 14 | HTTPClientFactory.noisy = False 15 | 16 | 17 | # By default, verbose logging is disabled. This function is redefined 18 | # when the service starts if verbose logging is enabled. 19 | log.verbose = lambda x: None 20 | 21 | 22 | class CurlerClient(client.GearmanProtocol): 23 | 24 | def __init__(self, service, server, base_urls, job_queue, num_workers): 25 | self.service = service 26 | self.server = server 27 | self.base_urls = base_urls 28 | self.job_queue = job_queue 29 | self.num_workers = num_workers 30 | 31 | def connectionLost(self, reason): 32 | log.msg('CurlerClient lost connection to %s: %s' 33 | % (self.server, reason)) 34 | client.GearmanProtocol.connectionLost(self, reason) 35 | 36 | def connectionMade(self): 37 | log.msg('CurlerClient made connection to %s' % self.server) 38 | self.start_work() 39 | 40 | def start_work(self): 41 | worker = client.GearmanWorker(self) 42 | worker.registerFunction(self.job_queue, self.handle_job) 43 | 44 | log.msg('Firing up %d workers...' % self.num_workers) 45 | coop = task.Cooperator() 46 | for i in range(self.num_workers): 47 | reactor.callLater(0.1 * i, lambda: coop.coiterate(worker.doJobs())) 48 | 49 | @defer.inlineCallbacks 50 | def handle_job(self, job): 51 | time_start = time() 52 | try: 53 | log.msg('Got job: %s' % job.handle) 54 | log.verbose('data=%r' % job.data) 55 | response = yield self._make_request(job.handle, job.data) 56 | except Exception, e: 57 | log.msg('ERROR: Unhandled exception: %r' % e) 58 | # Log full traceback on multiple lines 59 | for line in traceback.format_exc().split('\n'): 60 | log.msg(line) 61 | response = {"error": "Internal curler error. Check the logs."} 62 | 63 | # always include handle in response 64 | response['job_handle'] = job.handle 65 | 66 | # log error if we're returning one 67 | if 'error' in response: 68 | log.msg('ERROR: %s' % response['error']) 69 | response['job_data'] = job.data 70 | 71 | # format response nicely 72 | response_json = json.dumps(response, sort_keys=True, indent=2) 73 | 74 | time_taken = int((time() - time_start) * 1000 + 0.5) 75 | log.msg('Completed job: %s, method=%s, time=%sms, status=%d' 76 | % (job.handle, response['url'], time_taken, 77 | response.get('status'))) 78 | defer.returnValue(response_json) 79 | 80 | @defer.inlineCallbacks 81 | def _make_request(self, handle, data): 82 | # make sure job arg is valid json 83 | try: 84 | job_data = json.loads(data, encoding='UTF-8') 85 | except ValueError, e: 86 | defer.returnValue({"error": "Job data is not valid JSON"}) 87 | 88 | # make sure it contains a method 89 | if 'method' not in job_data: 90 | defer.returnValue({"error": 91 | "Missing \"method\" property in job data"}) 92 | 93 | # make sure it contains data 94 | if 'data' not in job_data: 95 | defer.returnValue({"error": 96 | "Missing \"data\" property in job data"}) 97 | 98 | headers = self.build_headers(job_data) 99 | 100 | # we'll post the data as JSON, so convert it back 101 | data = json.dumps(job_data['data']) 102 | 103 | # select random base URL to hit 104 | path = random.choice(self.base_urls) 105 | url = str("%s/%s" % (path, job_data['method'])) 106 | 107 | try: 108 | log.verbose('POSTing to %s, data=%r' % (url, data)) 109 | postdata = urllib.urlencode({ 110 | "job_handle": handle, 111 | "data": data}) 112 | 113 | try: 114 | # despite our name, we're not actually using curl :) 115 | response = yield getPage(url, method='POST', postdata=postdata, 116 | headers=headers) 117 | status = 200 118 | except Error, e: 119 | status = int(e.status) 120 | response = e.response 121 | log.verbose('POST complete: status=%d, response=%r' 122 | % (status, response)) 123 | defer.returnValue({'url': url, 124 | 'status': status, 125 | 'response': response}) 126 | except Exception, e: 127 | defer.returnValue({"error": "POST failed: %r - %s" % (e, e)}) 128 | 129 | @staticmethod 130 | def build_headers(job_data): 131 | # default headers - can be overridden by job_data['headers'] 132 | headers = {'Content-Type': 'application/x-www-form-urlencoded'} 133 | if 'headers' in job_data: 134 | # headers can't be unicode but json.loads makes all the string unicode 135 | for key, value in job_data['headers'].iteritems(): 136 | if isinstance(key, unicode): 137 | key = key.encode('utf-8') 138 | if isinstance(value, unicode): 139 | value = value.encode('utf-8') 140 | headers[key] = value 141 | 142 | return headers 143 | 144 | 145 | class CurlerClientFactory(protocol.ReconnectingClientFactory): 146 | noisy = True 147 | protocol = CurlerClient 148 | 149 | # retry every 5 seconds for up to 10 minutes 150 | initialDelay = 5 151 | maxDelay = 5 152 | maxRetries = 120 153 | 154 | def __init__(self, service, server, base_urls, job_queue, num_workers): 155 | self.service = service 156 | self.server = server 157 | self.base_urls = base_urls 158 | self.job_queue = job_queue 159 | self.num_workers = num_workers 160 | 161 | def buildProtocol(self, addr): 162 | p = self.protocol(self.service, self.server, self.base_urls, 163 | self.job_queue, self.num_workers) 164 | p.factory = self 165 | return p 166 | 167 | 168 | class CurlerService(Service): 169 | 170 | def __init__(self, base_urls, gearmand_servers, job_queue, num_workers, 171 | verbose=False): 172 | self.base_urls = base_urls 173 | self.gearmand_servers = gearmand_servers 174 | self.job_queue = job_queue 175 | self.num_workers = num_workers 176 | 177 | # define verbose logging function 178 | if verbose: 179 | log.verbose = lambda x: log.msg('VERBOSE: %s' % x) 180 | 181 | @defer.inlineCallbacks 182 | def startService(self): 183 | Service.startService(self) 184 | log.msg('Service starting. servers=%r, job queue=%s, base urls=%r' 185 | % (self.gearmand_servers, self.job_queue, self.base_urls)) 186 | log.verbose('Verbose logging is enabled') 187 | 188 | for server in self.gearmand_servers: 189 | host, port = server.split(':') 190 | f = CurlerClientFactory(self, server, self.base_urls, 191 | self.job_queue, self.num_workers) 192 | proto = yield reactor.connectTCP(host, int(port), f) 193 | 194 | def stopService(self): 195 | Service.stopService(self) 196 | log.msg('Service stopping') 197 | -------------------------------------------------------------------------------- /curler/twisted_gears/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hipchat/curler/b22bf79ecc4c1985038e0ba183ca7125be4b8ac0/curler/twisted_gears/__init__.py -------------------------------------------------------------------------------- /curler/twisted_gears/client.py: -------------------------------------------------------------------------------- 1 | """ 2 | Gearman client implementation. 3 | """ 4 | 5 | import sys 6 | import struct 7 | 8 | from collections import deque 9 | 10 | from twisted.internet import defer 11 | from twisted.protocols import stateful 12 | from twisted.python import log 13 | 14 | from constants import * 15 | 16 | __all__ = ['GearmanProtocol', 'GearmanWorker', 'GearmanClient'] 17 | 18 | class GearmanProtocol(stateful.StatefulProtocol): 19 | """Base protocol for handling gearman connections.""" 20 | 21 | unsolicited = [ WORK_COMPLETE, WORK_FAIL, NOOP, 22 | WORK_DATA, WORK_WARNING, WORK_EXCEPTION ] 23 | 24 | def makeConnection(self, transport): 25 | self.receivingCommand = 0 26 | self.deferreds = deque() 27 | self.unsolicited_handlers = set() 28 | # curler: moved this to the end so that vars are initialized when 29 | # connectionMade() gets called 30 | stateful.StatefulProtocol.makeConnection(self, transport) 31 | 32 | def send_raw(self, cmd, data=''): 33 | """Send a command with the given data with no response.""" 34 | 35 | self.transport.writeSequence([REQ_MAGIC, 36 | struct.pack(">II", cmd, len(data)), 37 | data]) 38 | 39 | def send(self, cmd, data=''): 40 | """Send a command and get a deferred waiting for the response.""" 41 | self.send_raw(cmd, data) 42 | d = defer.Deferred() 43 | self.deferreds.append(d) 44 | return d 45 | 46 | def getInitialState(self): 47 | return self._headerReceived, HEADER_LEN 48 | 49 | def connectionLost(self, reason): 50 | for d in list(self.deferreds): 51 | d.errback(reason) 52 | self.deferreds.clear() 53 | 54 | def _headerReceived(self, header): 55 | if header[:4] != RES_MAGIC: 56 | log.msg("Invalid header magic returned, failing.") 57 | self.transport.loseConnection() 58 | return 59 | cmd, size = struct.unpack(">II", header[4:]) 60 | 61 | self.receivingCommand = cmd 62 | return self._completed, size 63 | 64 | def _completed(self, data): 65 | if self.receivingCommand in self.unsolicited: 66 | self._unsolicited(self.receivingCommand, data) 67 | else: 68 | d = self.deferreds.popleft() 69 | d.callback((self.receivingCommand, data)) 70 | self.receivingCommand = 0 71 | 72 | return self._headerReceived, HEADER_LEN 73 | 74 | def _unsolicited(self, cmd, data): 75 | for cb in self.unsolicited_handlers: 76 | cb(cmd, data) 77 | 78 | def register_unsolicited(self, cb): 79 | self.unsolicited_handlers.add(cb) 80 | 81 | def unregister_unsolicited(self, cb): 82 | self.unsolicited_handlers.discard(cb) 83 | 84 | def echo(self, data="hello"): 85 | """Send an echo request.""" 86 | 87 | return self.send(ECHO_REQ, data) 88 | 89 | class _GearmanJob(object): 90 | """A gearman job.""" 91 | 92 | def __init__(self, raw_data): 93 | self.handle, self.function, self.data = raw_data.split("\0", 2) 94 | 95 | def __repr__(self): 96 | return "" % (self.handle, 97 | self.function, 98 | len(self.data)) 99 | 100 | class GearmanWorker(object): 101 | """A gearman worker.""" 102 | 103 | def __init__(self, protocol): 104 | self.protocol = protocol 105 | self.functions = {} 106 | self.sleeping = None 107 | self.protocol.register_unsolicited(self._unsolicited) 108 | 109 | def setId(self, client_id): 110 | """Set the client ID for monitoring and what-not.""" 111 | self.protocol.send_raw(SET_CLIENT_ID, client_id) 112 | 113 | def registerFunction(self, name, func): 114 | """Register the ability to perform a function.""" 115 | 116 | self.functions[name] = func 117 | self.protocol.send_raw(CAN_DO, name) 118 | 119 | def _send_job_res(self, cmd, job, data=''): 120 | self.protocol.send_raw(cmd, job.handle + "\0" + data) 121 | 122 | def _sleep(self): 123 | if not self.sleeping: 124 | self.sleeping = defer.Deferred() 125 | self.protocol.send_raw(PRE_SLEEP) 126 | return self.sleeping 127 | 128 | def _unsolicited(self, cmd, data): 129 | assert cmd == NOOP 130 | if self.sleeping: 131 | self.sleeping.callback(None) 132 | self.sleeping = None 133 | 134 | @defer.inlineCallbacks 135 | def getJob(self): 136 | """Get the next job.""" 137 | 138 | # If we're currently sleeping, attach to the existing sleep. 139 | if self.sleeping: 140 | yield self._sleep() 141 | 142 | stuff = yield self.protocol.send(GRAB_JOB) 143 | while stuff[0] == NO_JOB: 144 | yield self._sleep() 145 | stuff = yield self.protocol.send(GRAB_JOB) 146 | defer.returnValue(_GearmanJob(stuff[1])) 147 | 148 | @defer.inlineCallbacks 149 | def _finishJob(self, job): 150 | assert job 151 | f = self.functions[job.function] 152 | assert f 153 | try: 154 | rv = yield f(job) 155 | if rv is None: 156 | rv = "" 157 | self._send_job_res(WORK_COMPLETE, job, rv) 158 | except: 159 | etype, emsg, bt = sys.exc_info() 160 | self._send_job_res(WORK_EXCEPTION, job, "%s(%s)" 161 | % (etype.__name__, emsg)) 162 | self._send_job_res(WORK_FAIL, job) 163 | 164 | def doJob(self): 165 | """Do a single job""" 166 | return self.getJob().addCallback(self._finishJob) 167 | 168 | def doJobs(self, keepGoing=lambda: True): 169 | """Do jobs forever (or until the given function returns False)""" 170 | while keepGoing(): 171 | yield self.doJob() 172 | 173 | class _GearmanJobHandle(object): 174 | 175 | def __init__(self, deferred): 176 | self._deferred = deferred 177 | self._work_data = [] 178 | self._work_warning = [] 179 | 180 | @property 181 | def work_data(self): 182 | return ''.join(self._work_data) 183 | 184 | @property 185 | def work_warning(self): 186 | return ''.join(self._work_warning) 187 | 188 | class GearmanJobFailed(Exception): 189 | """Exception thrown when a job fails.""" 190 | pass 191 | 192 | class GearmanClient(object): 193 | """A gearman client. 194 | 195 | Submits jobs and stuff.""" 196 | 197 | def __init__(self, protocol): 198 | self.protocol = protocol 199 | self.protocol.register_unsolicited(self._unsolicited) 200 | self.jobs = {} 201 | 202 | def _register(self, job_handle, job): 203 | self.jobs[job_handle] = job 204 | 205 | def _unsolicited(self, cmd, data): 206 | if cmd in [ WORK_COMPLETE, WORK_FAIL, 207 | WORK_DATA, WORK_WARNING ]: 208 | pos = data.find("\0") 209 | if pos == -1: 210 | handle = data 211 | else: 212 | handle = data[:pos] 213 | data = data[pos+1:] 214 | 215 | j = self.jobs[handle] 216 | 217 | if cmd in [ WORK_COMPLETE, WORK_FAIL]: 218 | self._jobFinished(cmd, j, handle, data) 219 | 220 | def _jobFinished(self, cmd, job, handle, data): 221 | # Delete the job if it's finished 222 | del self.jobs[handle] 223 | 224 | if cmd == WORK_COMPLETE: 225 | job._deferred.callback(data) 226 | elif cmd == WORK_FAIL: 227 | job._deferred.errback(GearmanJobFailed()) 228 | 229 | def _submit(self, cmd, function, data, unique_id): 230 | 231 | def _submitted(x, d): 232 | self._register(x[1], _GearmanJobHandle(d)) 233 | 234 | d = self.protocol.send(cmd, 235 | function + "\0" + unique_id + "\0" + data) 236 | 237 | rv = defer.Deferred() 238 | d.addCallback(_submitted, rv) 239 | 240 | return rv 241 | 242 | def submit(self, function, data, unique_id=''): 243 | """Submit a job with the given function name and data.""" 244 | return self._submit(SUBMIT_JOB, function, data, unique_id) 245 | 246 | def submitHigh(self, function, data, unique_id=''): 247 | """Submit a high priority job with the given function name and data.""" 248 | return self._submit(SUBMIT_JOB_HIGH, function, data, unique_id) 249 | 250 | def submitLow(self, function, data, unique_id=''): 251 | """Submit a low priority job with the given function name and data.""" 252 | return self._submit(SUBMIT_JOB_LOW, function, data, unique_id) 253 | 254 | def _submitBg(self, cmd, function, data, unique_id): 255 | return self.protocol.send(cmd, 256 | function + "\0" + unique_id + "\0" + data) 257 | 258 | def submitBackground(self, function, data, unique_id=''): 259 | """Submit a job for background execution.""" 260 | return self._submitBg(SUBMIT_JOB_BG, function, data, unique_id) 261 | 262 | def submitBackgroundLow(self, function, data, unique_id=''): 263 | """Submit a job for background execution at low priority.""" 264 | return self._submitBg(SUBMIT_JOB_LOW_BG, function, data, unique_id) 265 | 266 | def submitBackgroundHigh(self, function, data, unique_id=''): 267 | """Submit a job for background execution at high priority.""" 268 | return self._submitBg(SUBMIT_JOB_HIGH_BG, function, data, unique_id) 269 | -------------------------------------------------------------------------------- /curler/twisted_gears/constants.py: -------------------------------------------------------------------------------- 1 | """Protocol definitions.""" 2 | 3 | import struct 4 | 5 | REQ_MAGIC = "\0REQ" 6 | RES_MAGIC = "\0RES" 7 | 8 | CAN_DO = 1 9 | CANT_DO = 2 10 | RESET_ABILITIES = 3 11 | PRE_SLEEP = 4 12 | NOOP = 6 13 | SUBMIT_JOB = 7 14 | JOB_CREATED = 8 15 | GRAB_JOB = 9 16 | NO_JOB = 10 17 | JOB_ASSIGN = 11 18 | WORK_STATUS = 12 19 | WORK_COMPLETE = 13 20 | WORK_FAIL = 14 21 | GET_STATUS = 15 22 | ECHO_REQ = 16 23 | ECHO_RES = 17 24 | SUBMIT_JOB_BG = 18 25 | ERROR = 19 26 | STATUS_RES = 20 27 | SUBMIT_JOB_HIGH = 21 28 | SET_CLIENT_ID = 22 29 | CAN_DO_TIMEOUT = 23 30 | ALL_YOURS = 24 31 | WORK_EXCEPTION = 25 32 | OPTION_REQ = 26 33 | OPTION_RES = 27 34 | WORK_DATA = 28 35 | WORK_WARNING = 29 36 | GRAB_JOB_UNIQ = 30 37 | JOB_ASSIGN_UNIQ = 31 38 | SUBMIT_JOB_HIGH_BG = 32 39 | SUBMIT_JOB_LOW = 33 40 | SUBMIT_JOB_LOW_BG = 34 41 | SUBMIT_JOB_SCHED = 35 42 | SUBMIT_JOB_EPOCH = 36 43 | 44 | # Magic, type, data size 45 | PKT_FMT = ">III" 46 | HEADER_LEN = struct.calcsize(PKT_FMT) 47 | -------------------------------------------------------------------------------- /curler/twisted_gears/test_client.py: -------------------------------------------------------------------------------- 1 | import struct 2 | from collections import deque 3 | 4 | from zope.interface import implements 5 | 6 | from twisted.trial import unittest 7 | from twisted.internet import interfaces, reactor, defer 8 | 9 | import client, constants 10 | 11 | class TestTransport(object): 12 | 13 | implements(interfaces.ITransport) 14 | 15 | disconnecting = False 16 | 17 | def __init__(self): 18 | self.received = [] 19 | self.disconnected = 0 20 | 21 | def write(self, data): 22 | self.received.append(data) 23 | 24 | def writeSequence(self, data): 25 | self.received.extend(data) 26 | 27 | def loseConnection(self): 28 | self.disconnected += 1 29 | 30 | def getPeer(self): 31 | return None 32 | 33 | def getHost(self): 34 | return None 35 | 36 | class ExpectedFailure(Exception): 37 | pass 38 | 39 | class ProtocolTestCase(unittest.TestCase): 40 | 41 | def setUp(self): 42 | self.trans = TestTransport() 43 | self.gp = client.GearmanProtocol() 44 | self.gp.makeConnection(self.trans) 45 | 46 | def assertReceived(self, cmd, data): 47 | self.assertEquals(["\0REQ", 48 | struct.pack(">II", cmd, len(data)), 49 | data], 50 | self.trans.received[:3]) 51 | self.trans.received = self.trans.received[3:] 52 | 53 | def write_response(self, cmd, data): 54 | self.gp.dataReceived("\0RES") 55 | self.gp.dataReceived(struct.pack(">II", cmd, len(data))) 56 | self.gp.dataReceived(data) 57 | 58 | class GearmanProtocolTest(ProtocolTestCase): 59 | 60 | def test_makeConnection(self): 61 | self.assertEquals(0, self.gp.receivingCommand) 62 | self.assertEquals([], list(self.gp.deferreds)) 63 | self.assertEquals([], list(self.gp.unsolicited_handlers)) 64 | 65 | def test_send_raw(self): 66 | self.gp.send_raw(11, "some data") 67 | self.assertReceived(11, "some data") 68 | self.assertEquals(0, len(self.gp.deferreds)) 69 | 70 | def test_send(self): 71 | self.gp.send(11, "some data") 72 | self.assertReceived(11, "some data") 73 | self.assertEquals(1, len(self.gp.deferreds)) 74 | 75 | def test_connectionLost(self): 76 | d = self.gp.send(11, "test") 77 | d.addCallback(lambda x: unittest.FailTest()) 78 | d.addErrback(lambda x: x.trap(ExpectedFailure)) 79 | self.gp.connectionLost(ExpectedFailure()) 80 | return d 81 | 82 | def test_badResponse(self): 83 | self.assertEquals(0, self.trans.disconnected) 84 | self.trans.shouldLoseConnection = True 85 | self.gp.dataReceived("X" * constants.HEADER_LEN) 86 | reactor.callLater(0, self.assertEquals, 1, self.trans.disconnected) 87 | 88 | def test_send_echo(self): 89 | d = self.gp.echo() 90 | self.assertReceived(constants.ECHO_REQ, "hello") 91 | 92 | def test_echoRt(self): 93 | """Test an echo round trip.""" 94 | d = self.gp.echo() 95 | d.addCallback(lambda x: 96 | self.assertEquals(x, 97 | (constants.ECHO_RES, "hello"))) 98 | self.write_response(constants.ECHO_RES, "hello") 99 | return d 100 | 101 | def test_register_unsolicited(self): 102 | def cb(cmd, data): 103 | pass 104 | self.gp.register_unsolicited(cb) 105 | self.assertEquals(1, len(self.gp.unsolicited_handlers)) 106 | self.gp.register_unsolicited(cb) 107 | self.assertEquals(1, len(self.gp.unsolicited_handlers)) 108 | self.gp.register_unsolicited(lambda a,b: True) 109 | self.assertEquals(2, len(self.gp.unsolicited_handlers)) 110 | 111 | def test_unregister_unsolicited(self): 112 | def cb(cmd, data): 113 | pass 114 | self.gp.register_unsolicited(cb) 115 | self.assertEquals(1, len(self.gp.unsolicited_handlers)) 116 | self.gp.unregister_unsolicited(cb) 117 | self.assertEquals(0, len(self.gp.unsolicited_handlers)) 118 | 119 | def test_unsolicitedCallbackHandling(self): 120 | d = defer.Deferred() 121 | self.gp.register_unsolicited(lambda cmd, data: d.callback(True)) 122 | self.write_response(constants.WORK_COMPLETE, "test\0") 123 | return d 124 | 125 | class GearmanJobTest(unittest.TestCase): 126 | 127 | def test_constructor(self): 128 | gj = client._GearmanJob("footdle\0dys\0some data") 129 | self.assertEquals("footdle", gj.handle) 130 | self.assertEquals("dys", gj.function) 131 | self.assertEquals("some data", gj.data) 132 | 133 | self.assertEquals("", 134 | repr(gj)) 135 | 136 | class GearmanWorkerTest(ProtocolTestCase): 137 | 138 | def setUp(self): 139 | super(GearmanWorkerTest, self).setUp() 140 | self.gw = client.GearmanWorker(self.gp) 141 | 142 | def test_registerFunction(self): 143 | self.gw.registerFunction("awesomeness", lambda x: True) 144 | self.assertReceived(constants.CAN_DO, "awesomeness") 145 | 146 | def test_sendingJobResponse(self): 147 | job = client._GearmanJob("test\0blah\0junk") 148 | self.gw._send_job_res(constants.WORK_COMPLETE, job, "the value") 149 | self.assertReceived(constants.WORK_COMPLETE, "test\0the value") 150 | 151 | def test_sleep(self): 152 | a = [] 153 | for i in range(5): 154 | a.append(self.gw._sleep()) 155 | 156 | self.write_response(constants.NOOP, "") 157 | return defer.DeferredList(a) 158 | 159 | def test_getJob(self): 160 | d = self.gw.getJob() 161 | self.write_response(constants.JOB_ASSIGN, 162 | "footdle\0funk\0args and stuff") 163 | def _handleJob(j): 164 | self.assertEquals("footdle", j.handle) 165 | self.assertEquals("funk", j.function) 166 | self.assertEquals("args and stuff", j.data) 167 | 168 | d.addCallback(_handleJob) 169 | return d 170 | 171 | def test_getJobWithWaiting(self): 172 | d = self.gw.getJob() 173 | self.write_response(constants.NO_JOB, "") 174 | self.write_response(constants.NOOP, "") 175 | self.write_response(constants.JOB_ASSIGN, 176 | "footdle\0funk\0args and stuff") 177 | def _handleJob(j): 178 | self.assertEquals("footdle", j.handle) 179 | self.assertEquals("funk", j.function) 180 | self.assertEquals("args and stuff", j.data) 181 | 182 | d.addCallback(_handleJob) 183 | return d 184 | 185 | def test_getJobWithWaitingMultiNOOP(self): 186 | d = self.gw.getJob() 187 | self.write_response(constants.NO_JOB, "") 188 | self.write_response(constants.NOOP, "") 189 | self.write_response(constants.NOOP, "") 190 | self.write_response(constants.NOOP, "") 191 | self.write_response(constants.JOB_ASSIGN, 192 | "footdle\0funk\0args and stuff") 193 | def _handleJob(j): 194 | self.assertEquals("footdle", j.handle) 195 | self.assertEquals("funk", j.function) 196 | self.assertEquals("args and stuff", j.data) 197 | 198 | d.addCallback(_handleJob) 199 | return d 200 | 201 | def test_getJobWhileAlreadyWaiting(self): 202 | sd = self.gw._sleep() 203 | d = self.gw.getJob() 204 | self.write_response(constants.NOOP, "") 205 | self.write_response(constants.JOB_ASSIGN, 206 | "footdle\0funk\0args and stuff") 207 | def _handleJob(j): 208 | self.assertEquals("footdle", j.handle) 209 | self.assertEquals("funk", j.function) 210 | self.assertEquals("args and stuff", j.data) 211 | 212 | d.addCallback(_handleJob) 213 | return defer.DeferredList([sd, d]) 214 | 215 | def test_finishJob(self): 216 | self.gw.functions['blah'] = lambda x: x.data.upper() 217 | job = client._GearmanJob("test\0blah\0junk") 218 | d = self.gw._finishJob(job) 219 | 220 | d.addCallback(lambda x: 221 | self.assertReceived(constants.WORK_COMPLETE, 222 | "test\0JUNK")) 223 | 224 | def test_finishJobNull(self): 225 | self.gw.functions['blah'] = lambda x: None 226 | job = client._GearmanJob("test\0blah\0junk") 227 | d = self.gw._finishJob(job) 228 | 229 | d.addCallback(lambda x: 230 | self.assertReceived(constants.WORK_COMPLETE, 231 | "test\0")) 232 | 233 | def test_finishJobException(self): 234 | def _failing(x): 235 | raise Exception("failed") 236 | self.gw.functions['blah'] = _failing 237 | job = client._GearmanJob("test\0blah\0junk") 238 | d = self.gw._finishJob(job) 239 | 240 | def _checkReceived(x): 241 | self.assertReceived(constants.WORK_EXCEPTION, 242 | "test\0" + 'Exception(failed)') 243 | self.assertReceived(constants.WORK_FAIL, "test\0") 244 | 245 | d.addCallback(_checkReceived) 246 | 247 | def test_doJob(self): 248 | self.gw.functions['blah'] = lambda x: x.data.upper() 249 | d = self.gw.doJob() 250 | self.write_response(constants.JOB_ASSIGN, 251 | "footdle\0blah\0args and stuff") 252 | 253 | def _verify(x): 254 | self.assertReceived(constants.GRAB_JOB, "") 255 | self.assertReceived(constants.WORK_COMPLETE, 256 | "footdle\0ARGS AND STUFF") 257 | 258 | d.addCallback(_verify) 259 | return d 260 | 261 | def test_doJobs(self): 262 | self.gw.functions['blah'] = lambda x: x.data.upper() 263 | d = self.gw.doJobs().next() 264 | self.write_response(constants.JOB_ASSIGN, 265 | "footdle\0blah\0args and stuff") 266 | 267 | def _verify(x): 268 | self.assertReceived(constants.GRAB_JOB, "") 269 | self.assertReceived(constants.WORK_COMPLETE, 270 | "footdle\0ARGS AND STUFF") 271 | 272 | d.addCallback(_verify) 273 | return d 274 | 275 | def test_doJobsNoLoop(self): 276 | try: 277 | d = self.gw.doJobs(lambda: False).next() 278 | except StopIteration: 279 | pass 280 | 281 | def test_setId(self): 282 | self.gw.setId("my id") 283 | self.assertReceived(constants.SET_CLIENT_ID, "my id") 284 | 285 | class GearmanJobHandleTest(unittest.TestCase): 286 | 287 | def test_workData(self): 288 | gjh = client._GearmanJobHandle(None) 289 | gjh._work_data.extend(['test', 'ing']) 290 | self.assertEquals('testing', gjh.work_data) 291 | 292 | def test_workWarning(self): 293 | gjh = client._GearmanJobHandle(None) 294 | gjh._work_warning.extend(['test', 'ing']) 295 | self.assertEquals('testing', gjh.work_warning) 296 | 297 | class GearmanClientTest(ProtocolTestCase): 298 | 299 | def setUp(self): 300 | super(GearmanClientTest, self).setUp() 301 | self.gc = client.GearmanClient(self.gp) 302 | 303 | def test_unsolicitedUnused(self): 304 | self.gc._register('x', client._GearmanJobHandle(None)) 305 | self.gc._unsolicited(constants.WORK_DATA, "x\0some data") 306 | 307 | def test_unsolicitedUnusedNoData(self): 308 | self.gc._register('x', client._GearmanJobHandle(None)) 309 | self.gc._unsolicited(constants.WORK_DATA, "x") 310 | 311 | def test_finishJob(self): 312 | d = defer.Deferred() 313 | self.gc._register('x', client._GearmanJobHandle(d)) 314 | self.gc._unsolicited(constants.WORK_COMPLETE, "x\0some data") 315 | 316 | d.addCallback(lambda x: self.assertEquals("some data", x)) 317 | return d 318 | 319 | def test_failJob(self): 320 | d = defer.Deferred() 321 | self.gc._register('x', client._GearmanJobHandle(d)) 322 | self.gc._unsolicited(constants.WORK_FAIL, "x\0some data") 323 | 324 | d.addErrback(lambda x: x.trap(client.GearmanJobFailed)) 325 | return d 326 | 327 | def test_submit(self): 328 | d = self.gc.submit('test', 'test data') 329 | self.assertReceived(constants.SUBMIT_JOB, 'test\0\0test data') 330 | self.write_response(constants.JOB_CREATED, 'test_submit') 331 | self.write_response(constants.WORK_COMPLETE, 332 | 'test_submit\0done') 333 | d.addCallback(lambda x: self.assertEquals("done", x)) 334 | return d 335 | 336 | def test_submitHigh(self): 337 | d = self.gc.submitHigh('test', 'test data') 338 | self.assertReceived(constants.SUBMIT_JOB_HIGH, 'test\0\0test data') 339 | self.write_response(constants.JOB_CREATED, 'test_submit') 340 | self.write_response(constants.WORK_COMPLETE, 341 | 'test_submit\0done') 342 | d.addCallback(lambda x: self.assertEquals("done", x)) 343 | return d 344 | 345 | def test_submitLow(self): 346 | d = self.gc.submitLow('test', 'test data', 'uniqid') 347 | self.assertReceived(constants.SUBMIT_JOB_LOW, 'test\0uniqid\0test data') 348 | self.write_response(constants.JOB_CREATED, 'test_submit') 349 | self.write_response(constants.WORK_COMPLETE, 350 | 'test_submit\0done') 351 | d.addCallback(lambda x: self.assertEquals("done", x)) 352 | return d 353 | 354 | def test_submitBackground(self): 355 | d = self.gc.submitBackground('test', 'test data') 356 | self.assertReceived(constants.SUBMIT_JOB_BG, 'test\0\0test data') 357 | self.write_response(constants.JOB_CREATED, 'test_submit') 358 | return d 359 | 360 | def test_submitBackgroundLow(self): 361 | d = self.gc.submitBackgroundLow('test', 'test data') 362 | self.assertReceived(constants.SUBMIT_JOB_LOW_BG, 'test\0\0test data') 363 | self.write_response(constants.JOB_CREATED, 'test_submit') 364 | return d 365 | 366 | def test_submitBackgroundHigh(self): 367 | d = self.gc.submitBackgroundHigh('test', 'test data') 368 | self.assertReceived(constants.SUBMIT_JOB_HIGH_BG, 'test\0\0test data') 369 | self.write_response(constants.JOB_CREATED, 'test_submit') 370 | return d 371 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Based off http://chrismiles.livejournal.com/23399.html 4 | 5 | import sys 6 | 7 | try: 8 | import twisted 9 | except ImportError: 10 | raise SystemExit("Twisted not found. Make sure you " 11 | "have installed the Twisted core package.") 12 | 13 | from distutils.core import setup 14 | 15 | 16 | def refresh_plugin_cache(): 17 | from twisted.plugin import IPlugin, getPlugins 18 | list(getPlugins(IPlugin)) 19 | 20 | if __name__ == "__main__": 21 | setup(name='curler', 22 | version='1.1', 23 | description='Gearman worker that hits a web service to do work.', 24 | author='Garret Heaton', 25 | author_email='powdahound@gmail.com', 26 | url='http://github.com/powdahound/curler', 27 | packages=['curler', 28 | 'curler.twisted_gears', 29 | 'twisted.plugins'], 30 | package_data={ 31 | 'twisted': ['plugins/curler_plugin.py']} 32 | ) 33 | 34 | refresh_plugin_cache() 35 | -------------------------------------------------------------------------------- /test/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This is a simple test script which creates a gearman server, curler worker, 4 | # and basic web service and creates a few jobs. There must be a better way to 5 | # test this stuff... 6 | 7 | GEARMAND_PORT=4731 8 | GEARMAN_QUEUE=curler_test 9 | WEBSERVER_PORT=8047 10 | 11 | function clean_up { 12 | # kill services 13 | echo -e "\n" 14 | kill $CURLER_PID 15 | kill $WEBSERVER_PID 16 | kill $GEARMAND_PID 17 | exit 18 | } 19 | 20 | trap clean_up SIGHUP SIGINT SIGTERM SIGKILL 21 | 22 | # start gearmand 23 | echo "Running gearmand on port $GEARMAND_PORT" 24 | gearmand --port $GEARMAND_PORT & 25 | GEARMAND_PID=$! 26 | 27 | # start webserver 28 | echo "Running webserver on port $WEBSERVER_PORT" 29 | python webserver.py $WEBSERVER_PORT & 30 | WEBSERVER_PID=$! 31 | 32 | # start curler 33 | echo "Running curler" 34 | cd .. 35 | twistd -n curler \ 36 | --base-urls=http://localhost:$WEBSERVER_PORT \ 37 | --job-queue=$GEARMAN_QUEUE \ 38 | --gearmand-server=localhost:$GEARMAND_PORT & 39 | CURLER_PID=$! 40 | 41 | # let services fully start 42 | sleep 1 43 | 44 | echo "Running jobs..." 45 | echo -e "\n********** Should get 200 - OK **********" 46 | gearman \ 47 | -p $GEARMAND_PORT \ 48 | -f $GEARMAN_QUEUE '{"method": "success", "data": {}, "headers": {"X-TEST-HEADER": "(yey)"}}' 49 | 50 | echo -e "\n\n********** Should get 500 - FAIL **********" 51 | gearman \ 52 | -p $GEARMAND_PORT \ 53 | -f $GEARMAN_QUEUE '{"method": "fail", "data": {}}' 54 | 55 | clean_up -------------------------------------------------------------------------------- /test/webserver.py: -------------------------------------------------------------------------------- 1 | from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer 2 | import cgi 3 | import json 4 | import sys 5 | import time 6 | 7 | 8 | class TestHandler(BaseHTTPRequestHandler): 9 | 10 | def do_POST(self): 11 | # parse POST data 12 | env = {'REQUEST_METHOD': 'POST', 13 | 'CONTENT_TYPE': self.headers['Content-Type']} 14 | form = cgi.FieldStorage(self.rfile, self.headers, environ=env) 15 | 16 | content = 'OK' 17 | code = 200 18 | 19 | # special behaviors 20 | if self.path == '/sleep': 21 | data = json.loads(form['data'].value) 22 | time.sleep(int(data['secs'])) 23 | elif self.path == '/fail': 24 | content = 'FAIL' 25 | code = 500 26 | 27 | # print out the headers so we can verify custom headers are sent 28 | print self.headers 29 | 30 | self.send_response(code) 31 | self.send_header('Content-type', 'text/plain') 32 | self.end_headers() 33 | self.wfile.write(content) 34 | self.wfile.write("\nPOST data: %r" % form['data'].value) 35 | return 36 | 37 | 38 | def main(): 39 | try: 40 | port = int(sys.argv[1]) if len(sys.argv) > 1 else 8080 41 | server = HTTPServer(('', port), TestHandler) 42 | print 'Listening on port %d...' % port 43 | server.serve_forever() 44 | except KeyboardInterrupt, SystemExit: 45 | print 'Stopping server.' 46 | server.socket.close() 47 | 48 | if __name__ == '__main__': 49 | main() 50 | -------------------------------------------------------------------------------- /twisted/plugins/curler_plugin.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from curler.service import CurlerService 3 | from twisted.application.service import IServiceMaker 4 | from twisted.plugin import IPlugin 5 | from twisted.python import usage 6 | from zope.interface import implements 7 | 8 | 9 | class Options(usage.Options): 10 | optFlags = [ 11 | ["verbose", "v", "Verbose logging"]] 12 | 13 | optParameters = [ 14 | ["base-urls", "u", None, 15 | "Base paths to web services. Separate multiple with commas."], 16 | ["job-queue", "q", "curler", 17 | "Job queue to get jobs from."], 18 | ["gearmand-server", "g", "localhost:4730", 19 | "Gearman job servers. Separate multiple with commas."], 20 | ["num-workers", "n", 5, 21 | "Number of workers (max parallel jobs)."]] 22 | 23 | longdesc = 'curler is a Gearman worker service which does work by hitting \ 24 | a web service. \nPlease see http://github.com/powdahound/curler to \ 25 | report issues or get help.' 26 | 27 | 28 | class CurlerServiceMaker(object): 29 | implements(IServiceMaker, IPlugin) 30 | tapname = "curler" 31 | description = "A Gearman worker that hits a web service to do work." 32 | options = Options 33 | 34 | def makeService(self, options): 35 | if not options['base-urls'] or not options['gearmand-server'] \ 36 | or not options['job-queue'] or not options['num-workers']: 37 | print options 38 | sys.exit(1) 39 | 40 | base_urls = options['base-urls'].split(',') 41 | gearmand_servers = options['gearmand-server'].split(',') 42 | job_queue = options['job-queue'] 43 | num_workers = int(options['num-workers']) 44 | verbose = bool(options['verbose']) 45 | return CurlerService(base_urls, gearmand_servers, job_queue, 46 | num_workers, verbose) 47 | 48 | 49 | serviceMaker = CurlerServiceMaker() 50 | --------------------------------------------------------------------------------