├── src ├── __init__.py ├── restmq │ ├── __init__.py │ ├── assets │ │ ├── __init__.py │ │ └── acl.conf │ ├── static │ │ ├── img │ │ │ ├── glyphicons-halflings.png │ │ │ └── glyphicons-halflings-white.png │ │ ├── help.html │ │ ├── index.html │ │ ├── css │ │ │ ├── bootstrap-responsive.min.css │ │ │ └── bootstrap-responsive.css │ │ └── js │ │ │ └── bootstrap.min.js │ ├── templates │ │ ├── list_queues.html │ │ ├── jobs.html │ │ └── basic_routes.html │ ├── setup_extras.py │ ├── syslogd.py │ ├── dispatch.py │ ├── collectd.py │ ├── core.py │ └── web.py └── twisted │ └── plugins │ ├── syslogd_plugin.py │ ├── restmq_plugin.py │ └── collectd_plugin.py ├── dockerfiles └── supervisor │ ├── acl.conf │ ├── redis.conf │ └── restmq.conf ├── tests ├── README ├── post.txt ├── ab.txt ├── grinder.properties ├── grinderAnalyzer.diff ├── grinder.txt ├── restmq-test.py ├── bigmessage.py └── restmq_engine.py ├── .gitignore ├── examples ├── test_request.py ├── test_comet.py ├── mapreduce │ ├── README │ ├── map.py │ ├── reduce.py │ ├── map_keyfreq.py │ └── reduce_keyfreq.py ├── test_collectd.py ├── test_rest.sh ├── test_comet_em.rb ├── twitter_trends.py └── test_comet_curl.py ├── requirements.txt ├── contrib ├── system-monitor │ ├── lib │ │ ├── dummyfuncs.py │ │ ├── myloader.py │ │ ├── dispatcher.py │ │ ├── simplemonitor.py │ │ └── myconfig.py │ ├── config │ │ └── main.cfg │ ├── README │ ├── qfeeder.py │ └── qconsumer.py ├── websocket_proxy │ ├── README │ └── proxy.js ├── ruby_restmq_endpoint │ └── restmq.rb ├── restmq_init.sh └── interact.html ├── docs ├── QUICKSTART ├── README.syslogd ├── README.websocket ├── README.Protocol ├── README.nginx ├── README.qp ├── README.RestMQ ├── HTTP_CLIENTS.rst └── README.collectd ├── Vagrantfile ├── setup.py ├── TODO ├── Dockerfile ├── start_scripts ├── collectd_restmq_server ├── syslogd_restmq_server └── restmq_server ├── restmq └── assets │ └── acl.conf ├── README.rst └── ez_setup.py /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/restmq/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dockerfiles/supervisor/acl.conf: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/restmq/assets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/README: -------------------------------------------------------------------------------- 1 | misc performance tests. 2 | TODO: dtrace probes. 3 | 4 | 5 | -------------------------------------------------------------------------------- /tests/post.txt: -------------------------------------------------------------------------------- 1 | queue=reducer&value=%7B%22count%22%3A+302%2C+%22filename%22%3A+%22files%2Fbookfrag-aa%22%7D 2 | -------------------------------------------------------------------------------- /dockerfiles/supervisor/redis.conf: -------------------------------------------------------------------------------- 1 | [program:redis] 2 | command=/usr/bin/redis-server 3 | autostart=true 4 | autorestart=true -------------------------------------------------------------------------------- /src/restmq/static/img/glyphicons-halflings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gleicon/restmq/HEAD/src/restmq/static/img/glyphicons-halflings.png -------------------------------------------------------------------------------- /src/restmq/static/img/glyphicons-halflings-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gleicon/restmq/HEAD/src/restmq/static/img/glyphicons-halflings-white.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | *~ 4 | dropin.cache 5 | node/redis-node-client/* 6 | twistd.pid 7 | *.pid 8 | build/ 9 | dist/ 10 | *.log 11 | src/RestMQ.egg-info/ 12 | -------------------------------------------------------------------------------- /examples/test_request.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | payload = {'value': 'el requesto',} 4 | r = requests.post("http://localhost:8888/q/test", data=payload) 5 | print r.text 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Twisted==12.2.0 2 | cyclone==1.0-rc12 3 | distribute==0.6.28 4 | pyOpenSSL==0.13 5 | pycrypto==2.6 6 | ssh==1.7.14 7 | wsgiref==0.1.2 8 | zope.interface==4.0.1 9 | -------------------------------------------------------------------------------- /dockerfiles/supervisor/restmq.conf: -------------------------------------------------------------------------------- 1 | [program:restmq] 2 | command=/srv/restmq/start_scripts/restmq_server --acl=/etc/restmq/acl.conf --listen=0.0.0.0 3 | autostart=true 4 | autorestart=true -------------------------------------------------------------------------------- /contrib/system-monitor/lib/dummyfuncs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding: utf-8 3 | # author: Eduardo S. Scarpellini, 4 | 5 | def generic(data): 6 | for key in data.keys(): 7 | print " - [dummy] Value for key \"%s\": %s" % (key, data[key]) 8 | -------------------------------------------------------------------------------- /contrib/system-monitor/config/main.cfg: -------------------------------------------------------------------------------- 1 | [RESTMQ] 2 | host = localhost 3 | port = 8888 4 | queuename = monitor 5 | 6 | [DISPATCHER] 7 | cpu = dummyfuncs.generic 8 | mem = dummyfuncs.generic 9 | load = dummyfuncs.generic 10 | swap = dummyfuncs.generic 11 | -------------------------------------------------------------------------------- /contrib/websocket_proxy/README: -------------------------------------------------------------------------------- 1 | Node.js proxy for restmq 2 | Useful for deploying restmq's websocket endpoint, as of now nginx and other webservers do not offer proxies for it. 3 | http://gist.github.com/488873 4 | older node.js versions: http://gist.github.com/488845 5 | 6 | -------------------------------------------------------------------------------- /docs/QUICKSTART: -------------------------------------------------------------------------------- 1 | - Install cyclone (http://github.com/fiorix/cyclone) 2 | - run restmq_server (or collectd or syslogd, whichever you want) 3 | - mess with the startup script if you want it running in brackground 4 | - use supervisord to control start/stop (or god if you are into ruby) 5 | 6 | -------------------------------------------------------------------------------- /tests/ab.txt: -------------------------------------------------------------------------------- 1 | To execute tests using ab, use the bundled post.txt file, along with these command lines: 2 | 3 | Producer: 4 | ab -r -p post.txt -n 20000 -c 50 -T application/x-www-form-urlencoded http://127.0.0.1:8888/ 5 | 6 | Consumer: 7 | ab -n 1000 -c 20 http://127.0.0.1:8888/q/reducer 8 | 9 | -------------------------------------------------------------------------------- /tests/grinder.properties: -------------------------------------------------------------------------------- 1 | grinder.script = restmq-test.py 2 | grinder.processes = 1 3 | grinder.threads = 1000 4 | grinder.runs = 10 5 | grinder.logDirectory = log 6 | grinder.numberOfOldLogs = 5 7 | grinder.jvm.arguments = -Dpython.cachedir=/tmp -Dpython.home=/opt/jython2.5.1 -Dpython.path=/opt/jython2.5.1/Lib 8 | -------------------------------------------------------------------------------- /src/restmq/templates/list_queues.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | avaliable queue for {{extended_route}} 4 | 5 | 6 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /tests/grinderAnalyzer.diff: -------------------------------------------------------------------------------- 1 | 801c801 2 | < if grinderVersion >= 3.2: 3 | --- 4 | > if grinderVersion == 3.2: 5 | 803c803 6 | < logger.fatal("Unsupported grinder version: " + str(grinderVersion)) 7 | --- 8 | > logger.fatal("Unsupported grinder version: " + grinderVersion) 9 | 895c895 10 | < SUPPORTED_JYTHON_VERSIONS=["2.2.1", "2.5.0", "2.5.1"] 11 | --- 12 | > SUPPORTED_JYTHON_VERSIONS=["2.2.1", "2.5.0"] 13 | -------------------------------------------------------------------------------- /src/restmq/templates/jobs.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | jobs in queue {{ queue }} 4 | 5 | 6 | queue name: {{ queue }}
7 | queue objects: {{ queue_size }}
8 | job count: {{ job_count }}
9 |

latest 10 jobs

10 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /contrib/system-monitor/lib/myloader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding: utf-8 3 | # author: Eduardo S. Scarpellini, 4 | 5 | def myimport(name): 6 | components = name.split(".") 7 | 8 | try: 9 | attrref = __import__(components[0]) 10 | 11 | for component in components[1:]: 12 | attrref = getattr(attrref, component) 13 | except Exception, err: 14 | print "Error: %s" % err 15 | else: 16 | return attrref 17 | -------------------------------------------------------------------------------- /contrib/system-monitor/lib/dispatcher.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding: utf-8 3 | # author: Eduardo S. Scarpellini, 4 | 5 | import myconfig 6 | import myloader 7 | 8 | 9 | def exe(name, params): 10 | cfg = myconfig.read() 11 | cfgvalues = cfg.dispatcher() 12 | 13 | if cfgvalues.has_key(name): 14 | try: 15 | func = myloader.myimport(cfgvalues[name]) 16 | func(params) 17 | except Exception, err: 18 | print "Error: %s" % err 19 | else: 20 | print "Error: No function for key \"%s\"" % name 21 | -------------------------------------------------------------------------------- /tests/grinder.txt: -------------------------------------------------------------------------------- 1 | Testing with grinder and generating reports with Grinder Analyzer got easy thanks to Eduardo Scarpellini. 2 | There is a patch to GrinderAnalyzer's analyzer.py if you use cutting edge jython stuff. 3 | The basic files are: 4 | 5 | restmq-test.py 6 | grinder.properties 7 | 8 | To run grinder: java -Xms1024m -Xmx1024m -cp lib/grinder.jar net.grinder.Grinder grinder.properties 9 | To run the reports: $ /opt/jython2.5.1/bin/jython run.py ../grinder-3.3/log/data_parvati-0.log ../grinder-3.3/log/out_parvati-0.log 10 | 11 | 12 | -------------------------------------------------------------------------------- /contrib/system-monitor/README: -------------------------------------------------------------------------------- 1 | Monitoring job producer/consumer using restmq 2 | Author: Eduardo S. Scarpellini, 3 | Date: 23/12/2009 4 | 5 | Depends on statgrab and python-statgrab to collect system information. 6 | The dummyfunc packages can be easily converted to feed Cacti and check agains stored thresholds to generate alarms. 7 | 8 | Open restmq-client.py, configure RESTMQ_HOST and _PORT, and run (no need to be root): 9 | 10 | $ python restmq-client.py 11 | 12 | In another terminal/machine, configure and run the consumer: 13 | 14 | $ python qconsumer.py 15 | 16 | 17 | -------------------------------------------------------------------------------- /docs/README.syslogd: -------------------------------------------------------------------------------- 1 | RestMQ based syslogd daemon - Experimental protocol adapter for receiveing high volume syslog messages 2 | 3 | - TCP only 4 | - each posting host has a queue named syslogd: 5 | - run it in a standalone restmq, for N servers 6 | - consume your results of a restmq instance bound to the same redis database 7 | - default port 25000 8 | - performs lookup of severity and facility 9 | 10 | 11 | Use syslogd_restmq_server to run the frontend. 12 | 13 | Bind on /c/syslogd: for receiving thru the comet endpoint: 14 | 15 | - curl http://localhost:8888/c/syslogd:127.0.0.1 16 | 17 | 18 | Gleicon - 2010 19 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | VAGRANTFILE_API_VERSION = "2" 2 | 3 | 4 | $script = < 4 | 46 | 47 | 48 | 49 | 50 |

Interaction experiment

51 | 52 |

Debug

53 |
54 | 55 |
56 | Last message time: 57 |
58 |
59 | 60 |
61 | queue: test - last element 62 |
Output should appear here
63 |
64 | 65 |

Enter something in the entry below, 66 | the server will send it back to the out region above 67 |

68 | 69 |
70 | entry 71 |

Enter:

72 |
73 | 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /examples/twitter_trends.py: -------------------------------------------------------------------------------- 1 | # simple twitter producer for restmq. point your browser to http://localhost:8888/c/twitter and 2 | # execute it with python twitter_trends.py 3 | 4 | import json 5 | import pickle, re, os, urllib, urllib2 6 | 7 | 8 | def get_url(url): 9 | try: 10 | f = urllib2.urlopen(url) 11 | data = f.read() 12 | f.close() 13 | except urllib2.URLError, e: 14 | print e.code 15 | print e.read() 16 | return data 17 | 18 | def post_in_queue(subject, author, text): 19 | try: 20 | msg={'subject': subject, 'author':author, 'text':text} 21 | data = urllib.urlencode({'queue':'twitter', 'value':json.dumps(msg)}) 22 | r = urllib2.Request('http://localhost:8888/', data) 23 | f = urllib2.urlopen(r) 24 | data = f.read() 25 | f.close() 26 | except urllib2.URLError, e: 27 | print e.code 28 | print e.read() 29 | print data 30 | 31 | filename = "last_topic_ids.db" 32 | 33 | if os.path.exists(filename): 34 | last_topic_ids = pickle.load(file(filename, 'r+b')) 35 | else: 36 | last_topic_ids = {} 37 | 38 | 39 | 40 | trends_current = json.loads(get_url("http://search.twitter.com/trends/current.json")) 41 | c = trends_current["trends"] 42 | 43 | for a in c[c.keys()[0]]: 44 | if a['query'] not in last_topic_ids.keys(): 45 | url = "http://search.twitter.com/search.json?q=%s" % (urllib.quote_plus(a['query'])) 46 | else: 47 | url = "http://search.twitter.com/search.json?q=%s&since_id=%s" % (urllib.quote_plus(a['query']), last_topic_ids[a['query']]) 48 | print "--------------------------------------" 49 | print "%s: %s" % (a['name'], url) 50 | statuses = json.loads(get_url(url)) 51 | for s in statuses['results']: 52 | print repr(s) 53 | print "%s: %s" %(s['from_user'], s['text']) 54 | post_in_queue(a, s['from_user'], s['text']) 55 | last_topic_ids[a['query']] = statuses['max_id'] 56 | print "--------------------------------------" 57 | 58 | print "Last topic and posts ids: %s" % last_topic_ids 59 | pickle.dump(last_topic_ids, file(filename, 'w+b')) 60 | 61 | -------------------------------------------------------------------------------- /src/restmq/syslogd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | # launchctl unload /System/Library/LaunchDaemons/com.apple.syslogd.plist 5 | # launchctl load /System/Library/LaunchDaemons/com.apple.syslogd.plist 6 | 7 | from twisted.internet import reactor, defer 8 | from twisted.internet.protocol import Protocol, ServerFactory 9 | from twisted.protocols.basic import LineReceiver 10 | import time, re, math, json, os 11 | from restmq import core 12 | import cyclone.redis 13 | 14 | 15 | #<22>Nov 1 00:12:04 gleicon-vm1 postfix/smtpd[4880]: connect from localhost[127.0.0.1] 16 | severity = ['emerg', 'alert', 'crit', 'err', 'warn', 'notice', 'info', 'debug', ] 17 | 18 | facility = ['kern', 'user', 'mail', 'daemon', 'auth', 'syslog', 'lpr', 'news', 19 | 'uucp', 'cron', 'authpriv', 'ftp', 'ntp', 'audit', 'alert', 'at', 'local0', 20 | 'local1', 'local2', 'local3', 'local4', 'local5', 'local6', 'local7',] 21 | 22 | fs_match = re.compile("<(.+)>(.*)", re.I) 23 | 24 | class SyslogdProtocol(LineReceiver): 25 | delimiter = '\n' 26 | def connectionMade(self): 27 | print 'Connection from %r' % self.transport 28 | 29 | def lineReceived(self, line): 30 | host = self.transport.getHost().host 31 | queue_name = "syslogd:%s" % host 32 | k = {} 33 | k['line'] = line.strip() 34 | (fac, sev) = self._calc_lvl(k['line']) 35 | k['host'] = host 36 | k['tstamp'] = time.time() 37 | k['facility'] = fac 38 | k['severity'] = sev 39 | self.factory.oper.queue_add(queue_name, json.dumps(k)) 40 | 41 | def _calc_lvl(self, line): 42 | lvl = fs_match.split(line) 43 | if lvl and len(lvl) > 1: 44 | i = int(lvl[1]) 45 | fac = int(math.floor(i / 8)) 46 | sev = i - (fac * 8) 47 | return (facility[fac], severity[sev]) 48 | return (None, None) 49 | 50 | class SyslogdFactory(ServerFactory): 51 | protocol = SyslogdProtocol 52 | 53 | def __init__ (self, redis_host, redis_port, redis_pool, redis_db ): 54 | 55 | db = cyclone.redis.lazyConnectionPool( 56 | redis_host, redis_port, 57 | poolsize=redis_pool, dbid=redis_db) 58 | 59 | self.oper = core.RedisOperations(db) 60 | 61 | -------------------------------------------------------------------------------- /docs/README.RestMQ: -------------------------------------------------------------------------------- 1 | RestMQ is a queue broker which presents different ways to access its data. 2 | It is based on Redis, python, twisted and the cyclone webserver/framework. 3 | Most of its features were drawn around Redis, which can deal with different types of data. 4 | 5 | 6 | There goes an sketch of the basic queue algorithm: 7 | 8 | The most important method is queue_add: 9 | Given a queue named QUEUE 10 | INCR a key named QUEUE:UUID 11 | This UUID then is used to create a key QUEUE:UUID 12 | Check if this queue name is alread in the QUEUESET set. If not, add it. For now it just checks if 13 | its the first time the name is used. 14 | SET QUEUE:UUID with the data 15 | PUSH QUEUE:UUID in the list QUEUE:queue. 16 | 17 | All these data structures are accounted to give how many and which queue exists in the system, ensure that they keys 18 | wont repeat and that there is an order in which they are retrieved. 19 | 20 | GET will just POP a value from QUEUE:queue and GET the value under this key. 21 | 22 | Check engine.py for all the logic and examples. 23 | 24 | There are a few http routes: 25 | 26 | /q/queuename - entry for a simple REST queue manipulation. POST inserts, GET removes and returns, DELETE disconnects all comet consumers and purgue queue 27 | /queue - entry point for the JSON protocol (see PROTOCOL). Point your browser to this location for some examples 28 | /c/queuename - entry point for a experimental COMET light consumer. 29 | /p/queuename - entry point for queue policy management - see README.qp for details 30 | /stats/[queuename] - misc statuses and queue list - if queuename is given, shows len and status (paused|started) 31 | /j/queuename - basic job checking, for those implementing job schedulers. Point your browser here to check data about a queue 32 | /control/queuename - start/stop consumers (as in pausing them) 33 | 34 | The CometQueueHandler is a permanent consumer for objects in a queue. 35 | It must only feed new objects to a permanent http connection. Deletion is not handled here for now. 36 | As each queue object has its own key, it can be done thru /queue interface. 37 | It also can be changed for a hard consumer, just commenting out the softget inside the method. 38 | 39 | There is a Map/Reduce example under /examples which shows implements the cannonical wordcount and wordfreq examples. 40 | 41 | The policy management allows for broadcast or roundrobin data distribution between comet subscribers (the consumers using comet route only). see REAME.qp 42 | 43 | -------------------------------------------------------------------------------- /tests/bigmessage.py: -------------------------------------------------------------------------------- 1 | #!/user/bin/python 2 | # 3 | # functional test for medium/big messages 4 | # queue, check size both on restmq and redis 5 | # make sure the queue is empty 6 | 7 | import sys, json 8 | import urllib, urllib2 9 | import difflib 10 | import redis 11 | import hashlib 12 | import time 13 | 14 | QUEUENAME = 'bigmessages_%s' % time.time() 15 | 16 | def str_checksum(str_m): 17 | return hashlib.md5(str_m).hexdigest() 18 | 19 | def read_msg(file): 20 | try: 21 | f = open(file, 'r') 22 | words = f.read() 23 | f.close() 24 | except Exception, e: 25 | print "Exception: %s" % e 26 | return words 27 | 28 | def enqueue(filename, content): 29 | try: 30 | msg={'filename': filename, 'len':len(content), 'content':content} 31 | data = urllib.urlencode({'queue':QUEUENAME, 'value':json.dumps(msg)}) 32 | r = urllib2.Request('http://localhost:8888/', data) 33 | f = urllib2.urlopen(r) 34 | data = f.read() 35 | f.close() 36 | except urllib2.URLError, e: 37 | print e 38 | data = data.rstrip() 39 | ck_c = str_checksum(content) 40 | l_c = len(content) 41 | print '[Queued message] Key name: %s' % data 42 | print "[Queued message] checksum: %s size: %d bytes" % (ck_c, l_c) 43 | return (l_c, ck_c, data) 44 | 45 | def dequeue(): 46 | try: 47 | r = urllib2.Request('http://localhost:8888/q/%s' % QUEUENAME) 48 | f = urllib2.urlopen(r) 49 | data = f.read() 50 | f.close() 51 | except urllib2.URLError, e: 52 | print e 53 | data_dic = json.loads(data) 54 | dd2 = json.loads(data_dic['value']) 55 | content = dd2['content'] 56 | ck_c = str_checksum(content) 57 | l_c = len(content) 58 | print "[Dequeued message] checksum: %s size: %d bytes" %(ck_c, l_c) 59 | return (l_c, ck_c, content) 60 | 61 | def check_redis(redis_key): 62 | r = redis.Redis(host='localhost', port=6379, db=0) 63 | meh = r.get(redis_key) 64 | data_dic = json.loads(meh) 65 | content = data_dic['content'] 66 | ck_c = str_checksum(content) 67 | l_c = len(content) 68 | print '[Redis] checksum: %s size for key %s: %d' % (ck_c, redis_key, l_c) 69 | return (l_c, ck_c) 70 | 71 | if __name__ == '__main__': 72 | if len(sys.argv) < 2: 73 | print "big message testing" 74 | print "Usage: bigmessage.py " 75 | sys.exit(-1) 76 | fname = sys.argv[1] 77 | content = read_msg(fname) 78 | (len_queued, ck_queued, redis_key) = enqueue(fname, content) 79 | (len_dequeued, ck_dequeued, ddq) = dequeue() 80 | (len_redis, ck_redis) = check_redis(redis_key) 81 | if (len_queued != len_dequeued): print 'Invalid dequeued message size' 82 | if (len_queued != len_redis): print 'Invalid redis message size' 83 | 84 | # d = difflib.Differ() 85 | # diff = d.compare(content.split(','), ddq.split(',')) 86 | # print '\n'.join(list(diff)) 87 | 88 | -------------------------------------------------------------------------------- /examples/test_comet_curl.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # 5 | # Usage: python test-comet-curl.py [No of simultaneous connections] 6 | # Example: python test-comet-curl.py http://localhost:8888/c/test 100 7 | # Adapted from retrieve-multi.py from PyCurl library 8 | # 9 | 10 | import sys 11 | import pycurl 12 | 13 | def pretty_printer(who):#, buf): 14 | # print "%s: %s" %(who)#, buf) 15 | print "-> %s" %(who) 16 | 17 | try: 18 | import signal 19 | from signal import SIGPIPE, SIG_IGN 20 | signal.signal(signal.SIGPIPE, signal.SIG_IGN) 21 | except ImportError: 22 | pass 23 | 24 | 25 | # Get args 26 | num_conn = 10 27 | 28 | try: 29 | if len(sys.argv) < 2: 30 | print "Needs a URL" 31 | sys.exit(-1) 32 | url = sys.argv[1] 33 | print url 34 | if len(sys.argv) >= 3: 35 | num_conn = int(sys.argv[2]) 36 | except Exception, e: 37 | print e 38 | print "Usage: %s [No of simultaneous connections]" % sys.argv[0] 39 | raise SystemExit 40 | 41 | 42 | 43 | assert 1 <= num_conn <= 10000, "invalid number of concurrent connections" 44 | print "PycURL %s (compiled against 0x%x)" % (pycurl.version, pycurl.COMPILE_LIBCURL_VERSION_NUM) 45 | print "----- Getting", url, "URLs using", num_conn, "connections -----" 46 | 47 | 48 | # Pre-allocate a list of curl objects 49 | m = pycurl.CurlMulti() 50 | m.handles = [] 51 | for i in range(num_conn): 52 | c = pycurl.Curl() 53 | c.fp = None 54 | c.setopt(pycurl.FOLLOWLOCATION, 1) 55 | c.setopt(pycurl.MAXREDIRS, 5) 56 | c.setopt(pycurl.CONNECTTIMEOUT, 30) 57 | c.setopt(pycurl.TIMEOUT, 300) 58 | c.setopt(pycurl.NOSIGNAL, 1) 59 | m.handles.append(c) 60 | 61 | 62 | # Main loop 63 | freelist = m.handles[:] 64 | num_processed = 0 65 | while num_processed < num_conn: 66 | while freelist: 67 | c = freelist.pop() 68 | c.setopt(pycurl.URL, url) 69 | # c.setopt(pycurl.WRITEFUNCTION, pretty_printer) 70 | m.add_handle(c) 71 | 72 | while 1: 73 | ret, num_handles = m.perform() 74 | if ret != pycurl.E_CALL_MULTI_PERFORM: 75 | break 76 | 77 | while 1: 78 | num_q, ok_list, err_list = m.info_read() 79 | for c in ok_list: 80 | c.fp.close() 81 | c.fp = None 82 | m.remove_handle(c) 83 | print "Success:", c.filename, c.url, c.getinfo(pycurl.EFFECTIVE_URL) 84 | freelist.append(c) 85 | for c, errno, errmsg in err_list: 86 | c.fp.close() 87 | c.fp = None 88 | m.remove_handle(c) 89 | print "Failed: ", c.filename, c.url, errno, errmsg 90 | freelist.append(c) 91 | num_processed = num_processed + len(ok_list) + len(err_list) 92 | if num_q == 0: 93 | break 94 | m.select(1.0) 95 | 96 | 97 | # Cleanup 98 | for c in m.handles: 99 | if c.fp is not None: 100 | c.fp.close() 101 | c.fp = None 102 | c.close() 103 | m.close() 104 | 105 | -------------------------------------------------------------------------------- /src/restmq/dispatch.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from twisted.internet import defer 4 | 5 | class CommandDispatch: 6 | def __init__(self, ro): 7 | self.ro = ro 8 | 9 | @defer.inlineCallbacks 10 | def _add(self, jsonbody): 11 | """ 12 | add an object to the queue 13 | """ 14 | r={} 15 | try: 16 | e = yield self.ro.queue_add(jsonbody['queue'].encode("utf-8"), jsonbody['value'].encode("utf-8")) 17 | r['queue'] = jsonbody['queue'] 18 | r['value'] = jsonbody['value'] 19 | r['key'] = str(e) 20 | defer.returnValue(r) 21 | except Exception, e: 22 | defer.returnValue({"error":str(e)}) 23 | 24 | @defer.inlineCallbacks 25 | def _get(self, jsonbody): 26 | """ 27 | get an object from the queue. get is not destructive (wont pop the object out of the queue). 28 | check the 'count' attribute to see how many times a give object was requested 29 | """ 30 | r={} 31 | try: 32 | p, e = yield self.ro.queue_get(jsonbody['queue'].encode("utf-8"), softget=True) 33 | if e is None: 34 | defer.returnValue({'error': 'empty queue'}) 35 | else: 36 | r['queue'] = jsonbody['queue'] 37 | r['value'] = e['value'] 38 | r['key'] = e['key'] 39 | r['count'] = e['count'] 40 | defer.returnValue(r) 41 | except Exception, e: 42 | defer.returnValue({"error":str(e)}) 43 | 44 | 45 | @defer.inlineCallbacks 46 | def _take(self, jsonbody): 47 | """ 48 | get and delete an object from the queue. it is not really necessary as GET takes it out of the queue list, 49 | so it will be basically hanging around redis with no reference. For now its a two pass operation, with no 50 | guarantees and the same semantics as del 51 | """ 52 | r={} 53 | try: 54 | p, e = yield self.ro.queue_getdel(jsonbody['queue'].encode("utf-8")) 55 | if e == False: 56 | defer.returnValue({"error":"empty queue"}) 57 | 58 | if e == None: 59 | defer.returnValue({"error":"getdel error"}) 60 | r['queue'] = jsonbody['queue'] 61 | r['value'] = e['value'] 62 | r['key'] = e['key'] 63 | 64 | defer.returnValue(r) 65 | except Exception, e: 66 | defer.returnValue({"error":str(e)}) 67 | 68 | @defer.inlineCallbacks 69 | def _del(self, jsonbody): 70 | """ 71 | delete an object from the storage. returns key and a deleted attribute, false if the key doesn't exists' 72 | """ 73 | r={} 74 | try: 75 | e = yield self.ro.queue_del(jsonbody['queue'].encode("utf-8"), jsonbody['key']) 76 | r['queue'] = jsonbody['queue'] 77 | r['deleted'] = True if e['value'] > 0 else False 78 | r['key'] = e['key'] 79 | defer.returnValue(r) 80 | except Exception, e: 81 | defer.returnValue({"error":str(e)}) 82 | 83 | 84 | def execute(self, command, jsonbody): 85 | c = "_"+command 86 | if hasattr(self, c): 87 | m=getattr(self, c) 88 | return m(jsonbody) 89 | else: 90 | return None 91 | -------------------------------------------------------------------------------- /docs/HTTP_CLIENTS.rst: -------------------------------------------------------------------------------- 1 | On HTTP clients 2 | =============== 3 | 4 | Most of the examples for RestMQ are using either curl or urllib. The reason is that I wanted to show how simple is to interact with the broker. 5 | 6 | Real life is that you might find out clients that will be more restrict or lax regarding content type or headers. 7 | 8 | 9 | POSTing data 10 | ==== 11 | 12 | In general, methods that receive data via POST need to specify the 'application/x-www-form-urlencoded' content-type. 13 | 14 | On examples that use curl, this is already set in as you use the -d key (highlight is mine): 15 | 16 | 17 | $ curl -vvv -X POST -d "value=foobar" http://localhost:8888/q/test 18 | * About to connect() to localhost port 8888 (#0) 19 | * Trying ::1... Connection refused 20 | * Trying 127.0.0.1... connected 21 | * Connected to localhost (127.0.0.1) port 8888 (#0) 22 | > POST /q/test HTTP/1.1 23 | > User-Agent: curl/7.21.4 (universal-apple-darwin11.0) libcurl/7.21.4 OpenSSL/0.9.8r zlib/1.2.5 24 | > Host: localhost:8888 25 | > Accept: */* 26 | > Content-Length: 12 27 | *> Content-Type: application/x-www-form-urlencoded* 28 | > 29 | < HTTP/1.1 200 OK 30 | < Content-Length: 11 31 | < Content-Type: text/html; charset=UTF-8 32 | < Server: cyclone/1.0-rc6 33 | < 34 | test:1003 35 | * Connection #0 to host localhost left intact 36 | * Closing connection #0 37 | 38 | For urllib usually we construct the request like this: 39 | 40 | def post_in_queue(subject, author, text): 41 | try: 42 | msg={'subject': subject, 'author':author, 'text':text} 43 | *data = urllib.urlencode({'queue':'twitter', 'value':json.dumps(msg)})* 44 | r = urllib2.Request('http://localhost:8888/', data) 45 | f = urllib2.urlopen(r) 46 | data = f.read() 47 | f.close() 48 | except urllib2.URLError, e: 49 | print e.code 50 | print e.read() 51 | print data 52 | 53 | Using Python Requests (http://docs.python-requests.org/): 54 | 55 | import requests 56 | 57 | payload = {'value': 'el requesto',} 58 | r = requests.post("http://localhost:8888/q/test", data=payload) 59 | print r.text 60 | 61 | If you dont pass a dict to data, it will behave a bit like a PUT request. 62 | 63 | GETting data 64 | ============ 65 | 66 | GET methods will bind to URIs like single messages ('/q/queuename') or COMET stream ('/c/queuename'). 67 | If you client support connection pools, you should be probably be using them for long term connections. 68 | Check if they recicle and if your queue has a broadcast or roundrobin policy set. 69 | You can set roundrobin and receive a new message per connection. 70 | No special header is needed but take care of the return codes. 71 | 72 | 73 | -------------------------------------------------------------------------------- /docs/README.collectd: -------------------------------------------------------------------------------- 1 | collectd REST Endpoint 2 | ====================== 3 | 4 | Now RestMQ has a collectd rest endpoint to receive and queue messages from collectd and collectd_web 5 | There is an example at examples/test_collectd.py. 6 | Run with: collectd_restmq_server 7 | 8 | This is an specialization of RestMQ, that shows how flexible and easy the internals are. The main daemon is not restmq/web.py but restmq/collectd.py, which is using core and dispatch from RestMQ to implement only the necessary frontend using cyclone. 9 | 10 | The collectd entrypoint does a vaery simple and basic validation of the data send by the collectd daemon. 11 | 12 | * collectd Configuration: 13 | 14 | ** Data collection: 15 | 16 | RestMQ collectd entrypoint will decide which format to parse based on the 'Content-type' sent by the client. 17 | 18 | * Plain Text 19 | 20 | Use the write_http(http://collectd.org/documentation/manpages/collectd.conf.5.shtml#plugin_write_http) bundled with collectd to pass 21 | data to the RestMQ server using the Plain Text Protocol (http://collectd.org/wiki/index.php/Plain_text_protocol), RestMQ will 22 | generated a JSON with the data to be inserted on the RestMQ Queue: 23 | 24 | collectd.conf: 25 | ... 26 | 27 | /collectd/data"> 28 | Format Command 29 | # User "collectd" 30 | # Password "weCh3ik0" 31 | 32 | 33 | 34 | Example JSON inserted in the RestMQ: 35 | 36 | [{"host": "collectd_restmq_server", "type_instance": "df-boot", "plugin_instance": "df", "value": "1288548767:116497408.000000:391246848.000000", "interval": "10"}] 37 | 38 | * JSON Generated by collectd 39 | 40 | Use the write_http(http://collectd.org/documentation/manpages/collectd.conf.5.shtml#plugin_write_http) bundled with collectd to pass 41 | data to the RestMQ server using the the JSON generated by collectd, the collectd endpoint just insert the data in the RestMQ Queue: 42 | 43 | collectd.conf: 44 | ... 45 | 46 | /collectd/data"> 47 | Format JSON 48 | # User "collectd" 49 | # Password "yerpass" 50 | 51 | 52 | 53 | Example JSON inserted in the RestMQ: 54 | 55 | [{"values":[1.16497e+08,3.91247e+08],"time":1288625884,"interval":10,"host":"collectd_restmq_server","plugin":"df","plugin_instance":"","type":"df","type_instance":"boot"}] 56 | 57 | ** Event collection: 58 | 59 | Since the write_http is not yet used for events, you need to create a simple script using CURL to send events to the RestMQ server 60 | using the Plain Text Protocol (http://collectd.org/wiki/index.php/Plain_text_protocol): 61 | 62 | collectd.conf: 63 | ... 64 | 65 | # Exec "user:group" "/path/to/exec" 66 | NotificationExec "rgaiser:rgaiser" "/home/rgaiser/event.sh" 67 | 68 | 69 | 70 | event.sh: 71 | #!/bin/bash 72 | 73 | EVENT=`cat` 74 | /usr/bin/curl -X POST -d "$EVENT" http:///collectd/event 75 | 76 | Example JSON inserted in the RestMQ: 77 | 78 | [{"severity": "WARNING"}, {"time": "1288548727"}, {"host": "collectd_restmq_server"}, {"plugin": "df"}, {"type": "df"}, {"typeinstance": "boot"}, {"datasource": "used"}, {"currentvalue": "1.164974e+08"}, {"warningmin": "nan"}, {"warningmax": "1.000000e+01"}, {"failuremin": "nan"}, {"failuremax": "nan"}, {"event_text": "Host collectd_restmq_server, plugin df type df (instance boot): Data source \\"used\\" is currently 116497408.000000. That is above the warning threshold of 10.000000%."}] 79 | 80 | * RestMQ Example 81 | 82 | Use the examples/test_collectd.py to get the data from the RestMQ using the Comet consumer. 83 | 84 | Author: Roberto Gaiser (http://github.com/rgaiser) 85 | -------------------------------------------------------------------------------- /src/restmq/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | RestMQ 6 | 7 | 8 | 9 | 10 | 11 | 12 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 |
34 |
35 |
36 | 37 | 38 | 39 | 40 | 41 | RestMQ 42 |
43 |
44 |
45 | 46 |
47 |
48 |

49 |
50 |
51 |
52 |
53 |
54 |

Queues:

55 |
56 |
57 |

58 |
59 |

60 |

61 |

62 |

63 |
64 |
65 |
66 |
67 |
68 | 69 | 71 | 72 | 73 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /src/restmq/collectd.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import os.path 4 | import cyclone.web 5 | import cyclone.redis 6 | 7 | import pkg_resources as pkg 8 | 9 | from twisted.python import log 10 | from twisted.internet import defer 11 | 12 | from restmq import core 13 | 14 | import json 15 | import web 16 | 17 | class CollectdRestQueueHandler(web.RestQueueHandler): 18 | 19 | @web.authorize("rest_producer") 20 | @defer.inlineCallbacks 21 | def post(self, queue): 22 | value = self.request.body 23 | if value is None: 24 | raise cyclone.web.HTTPError(400) 25 | if queue == 'data': 26 | content_type = self.request.headers.get('Content-Type') 27 | queue = 'collectd_data' 28 | if content_type == 'text/plain': 29 | try: 30 | value = value.splitlines() 31 | value = self.collectd_plaintext_parser(value) 32 | value = json.dumps(value) 33 | except Exception, e: 34 | log.msg("ERROR: %s" % e) 35 | raise cyclone.web.HTTPError(503) 36 | elif content_type == 'application/json': 37 | pass 38 | else: 39 | log.msg("ERROR: Content-Type not expected %s" % content_type) 40 | raise cyclone.web.HTTPError(503) 41 | elif queue == 'event': 42 | queue = 'collectd_event' 43 | try: 44 | value = value.splitlines() 45 | event = value.pop() 46 | value = value[:-1] 47 | value = self.collectd_plaintext_parser(value) 48 | value.append({'event_text': event}) 49 | value = json.dumps(value) 50 | except Exception, e: 51 | log.msg("ERROR: %s" % e) 52 | raise cyclone.web.HTTPError(503) 53 | else: 54 | raise cyclone.web.HTTPError(400) 55 | callback = self.get_argument("callback", None) 56 | 57 | try: 58 | result = yield self.settings.oper.queue_add(queue, value) 59 | except Exception, e: 60 | log.msg("ERROR: oper.queue_add('%s', '%s') failed: %s" % (queue, value, e)) 61 | raise cyclone.web.HTTPError(503) 62 | 63 | if result: 64 | self.settings.comet.queue.put(queue) 65 | web.CustomHandler(self, callback).finish(result) 66 | else: 67 | raise cyclone.web.HTTPError(400) 68 | 69 | def collectd_plaintext_parser(self,lines): 70 | event_protocol = {'Severity': None, 71 | 'Time': None, 72 | 'Host': None, 73 | 'Plugin': None, 74 | 'Type': None, 75 | 'TypeInstance': None, 76 | 'DataSource': None, 77 | 'CurrentValue': None, 78 | 'WarningMin': None, 79 | 'WarningMax': None, 80 | 'FailureMin': None, 81 | 'FailureMax': None,} 82 | collectd_data = [] 83 | for line in lines: 84 | line = line.split(' ') 85 | if line[0] == 'PUTVAL': 86 | (host,plugin_instance,type_instance) = line[1].split('/') 87 | interval = line[2].split('=')[1] 88 | value = line[3] 89 | collectd_data.append({'host':host, 90 | 'plugin_instance':plugin_instance, 91 | 'type_instance':type_instance, 92 | 'interval':interval,'value':value}) 93 | elif line[0].rstrip(':') in event_protocol: 94 | key = line[0].rstrip(':').lower() 95 | value = line[1] 96 | collectd_data.append({key: value}) 97 | return collectd_data 98 | 99 | class Collectd(web.Application): 100 | 101 | def __init__(self, acl_file, redis_host, redis_port, redis_pool, redis_db): 102 | handlers = [ 103 | (r"/", web.IndexHandler), 104 | (r"/q/(.*)", web.RestQueueHandler), 105 | (r"/c/(.*)", web.CometQueueHandler), 106 | (r"/p/(.*)", web.PolicyQueueHandler), 107 | (r"/j/(.*)", web.JobQueueInfoHandler), 108 | (r"/stats/(.*)", web.StatusHandler), 109 | (r"/queue", web.QueueHandler), 110 | (r"/control/(.*)", web.QueueControlHandler), 111 | (r"/ws/(.*)", web.WebSocketQueueHandler), 112 | ] 113 | 114 | handlers.append((r"/collectd/(.*)", CollectdRestQueueHandler)) 115 | 116 | try: 117 | acl = web.ACL(acl_file) 118 | except Exception, e: 119 | log.msg("ERROR: Cannot load ACL file: %s" % e) 120 | raise RuntimeError("Cannot load ACL file: %s" % e) 121 | 122 | db = cyclone.redis.lazyConnectionPool( 123 | redis_host, redis_port, 124 | poolsize=redis_pool, dbid=redis_db) 125 | 126 | oper = core.RedisOperations(db) 127 | 128 | settings = { 129 | "db": db, 130 | "acl": acl, 131 | "oper": oper, 132 | "comet": web.CometDispatcher(oper), 133 | "static_path": pkg.resource_filename('restmq', 'static'), 134 | "template_path": pkg.resource_filename('restmq', 'templates'), 135 | } 136 | 137 | cyclone.web.Application.__init__(self, handlers, **settings) 138 | -------------------------------------------------------------------------------- /tests/restmq_engine.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import sys 5 | import json 6 | import cyclone.redis 7 | from restmq import core 8 | from optparse import OptionParser 9 | from twisted.internet import defer, reactor 10 | 11 | QUEUENAME = 'test' 12 | 13 | @defer.inlineCallbacks 14 | def test_operations(opt, args): 15 | """ 16 | test and docs for a redis based queue engine in python 17 | based on tx-redis 18 | """ 19 | try: 20 | rd = yield cyclone.redis.RedisConnectionPool() 21 | except Exception, e: 22 | print "Error creating redis pool %s" % e 23 | defer.returnValue(None) 24 | 25 | ro = core.RedisOperations(rd) 26 | 27 | if opt.producer == True: 28 | print "Running as producer" 29 | uuid = yield ro.queue_add(QUEUENAME, json.dumps({'value':'a value'})) 30 | print 'uuid: %s' % uuid 31 | 32 | if opt.consumer == True: 33 | print "Running as consumer" 34 | (policy, ret) = yield ro.queue_get( QUEUENAME) 35 | if ret != None: 36 | print "value: %s" % ret['value'] #json.loads(ret['value']) 37 | print "policy: %s" % policy 38 | else: 39 | print 'empty queue' 40 | 41 | if opt.stats == True: 42 | ll = yield ro.queue_stats(QUEUENAME) 43 | print "list len: %s" % ll 44 | 45 | sm = yield ro.queue_all() 46 | print "all queues: %s" % sm 47 | 48 | if opt.non_consumer == True: 49 | print "Running as consumer" 50 | (policy, ret) = yield ro.queue_get( QUEUENAME, softget=True) 51 | if ret != None: 52 | print "value: %s" % ret['value'] #json.loads(ret['value']) 53 | print "policy: %s" % policy 54 | else: 55 | print 'empty queue' 56 | 57 | if opt.get_policy == True: 58 | print "GET queue policy" 59 | ret = yield ro.queue_policy_get(QUEUENAME) 60 | print repr(ret) 61 | 62 | if ret != None: 63 | print "value: %s" % ret['value'] #json.loads(ret['value']) 64 | else: 65 | print 'empty queue policy' 66 | 67 | if opt.set_policy == True: 68 | print "SET queue policy" 69 | resp = yield ro.queue_policy_set(QUEUENAME, "roundrobin") 70 | print 'resp: %s' % resp 71 | 72 | if opt.get_del == True: 73 | print "Running as getdel consumer" 74 | (policy, ret) = yield ro.queue_getdel(QUEUENAME) 75 | if ret != None and ret != False: 76 | print "value: %s" % ret['value'] #json.loads(ret['value']) 77 | print "policy: %s" % policy 78 | else: 79 | print 'empty queue' 80 | 81 | if opt.tail_mget == True: 82 | print "Running as tail multiget" 83 | (policy, ret) = yield ro.queue_tail(QUEUENAME) 84 | if ret != None and ret != False: 85 | print "value: %s" % repr(ret) #json.loads(ret['value']) 86 | print "policy: %s" % policy 87 | else: 88 | print 'empty queue' 89 | 90 | if opt.count_objects == True: 91 | print "Running as count object" 92 | ret = yield ro.queue_count_elements(QUEUENAME) 93 | if ret != None and ret != False: 94 | print "value: %s" % repr(ret) #json.loads(ret['value']) 95 | else: 96 | print 'empty queue' 97 | 98 | if opt.queue_last_items == True: 99 | print "Running as count object" 100 | ret = yield ro.queue_last_items(QUEUENAME) 101 | if ret != None and ret != False: 102 | print "value: %s" % repr(ret) #json.loads(ret['value']) 103 | else: 104 | print 'empty queue' 105 | 106 | if opt.authorize == True: 107 | print "Running authorization" 108 | ret = yield ro.authorize(QUEUENAME, 'aaa123') 109 | print ret 110 | 111 | if opt.create_auth == True: 112 | print "Creating auth record" 113 | ret = yield ro._create_auth_record('aaa123', [QUEUENAME], ['create']) 114 | print ret 115 | 116 | 117 | def main(): 118 | p = OptionParser() 119 | p.add_option("-p", "--producer", action="store_true", dest="producer", help="Run as producer") 120 | p.add_option("-c", "--consumer", action="store_true", dest="consumer", help="Run as consumer") 121 | p.add_option("-g", "--non-consumer", action="store_true", dest="non_consumer", help="Run as a non destructive consumer") 122 | p.add_option("-s", "--stats", action="store_true", dest="stats", help="Stats") 123 | p.add_option("-q", "--get_policy", action="store_true", dest="get_policy", help="Get queue policy") 124 | p.add_option("-j", "--set_policy", action="store_true", dest="set_policy", help="Set queue policy") 125 | p.add_option("-k", "--get_delete", action="store_true", dest="get_del", help="Consumer get del") 126 | p.add_option("-t", "--tail_multiget", action="store_true", dest="tail_mget", help="Multi get 10 keys") 127 | p.add_option("-u", "--count_objects", action="store_true", dest="count_objects", help="Count objects of a given queue") 128 | p.add_option("-i", "--list_last_items", action="store_true", dest="queue_last_items", help="List the latest queue items") 129 | # authorization tests 130 | p.add_option("-a", "--authorize", action="store_true", dest="authorize", help="authorize a key for queues/privileges") 131 | p.add_option("-r", "--create_auth", action="store_true", dest="create_auth", help="Create an authorization record") 132 | 133 | 134 | (opt, args)=p.parse_args(sys.argv[1:]) 135 | 136 | test_operations(opt, args).addCallback(lambda ign: reactor.stop()) 137 | 138 | 139 | if __name__ == "__main__": 140 | main() 141 | reactor.run() 142 | -------------------------------------------------------------------------------- /src/restmq/static/css/bootstrap-responsive.min.css: -------------------------------------------------------------------------------- 1 | .clearfix{*zoom:1;}.clearfix:before,.clearfix:after{display:table;content:"";} 2 | .clearfix:after{clear:both;} 3 | .hidden{display:none;visibility:hidden;} 4 | @media (max-width:480px){.nav-collapse{-webkit-transform:translate3d(0, 0, 0);} .page-header h1 small{display:block;line-height:18px;} input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input{display:block;width:100%;min-height:28px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;-ms-box-sizing:border-box;box-sizing:border-box;} .input-prepend input[class*="span"],.input-append input[class*="span"]{width:auto;} input[type="checkbox"],input[type="radio"]{border:1px solid #ccc;} .form-horizontal .control-group>label{float:none;width:auto;padding-top:0;text-align:left;} .form-horizontal .controls{margin-left:0;} .form-horizontal .control-list{padding-top:0;} .form-horizontal .form-actions{padding-left:10px;padding-right:10px;} .modal{position:absolute;top:10px;left:10px;right:10px;width:auto;margin:0;}.modal.fade.in{top:auto;} .modal-header .close{padding:10px;margin:-10px;} .carousel-caption{position:static;}}@media (max-width:767px){.container{width:auto;padding:0 20px;} .row-fluid{width:100%;} .row{margin-left:0;} .row>[class*="span"],.row-fluid>[class*="span"]{float:none;display:block;width:auto;margin:0;}}@media (min-width:768px) and (max-width:979px){.row{margin-left:-20px;*zoom:1;}.row:before,.row:after{display:table;content:"";} .row:after{clear:both;} [class*="span"]{float:left;margin-left:20px;} .span1{width:42px;} .span2{width:104px;} .span3{width:166px;} .span4{width:228px;} .span5{width:290px;} .span6{width:352px;} .span7{width:414px;} .span8{width:476px;} .span9{width:538px;} .span10{width:600px;} .span11{width:662px;} .span12,.container{width:724px;} .offset1{margin-left:82px;} .offset2{margin-left:144px;} .offset3{margin-left:206px;} .offset4{margin-left:268px;} .offset5{margin-left:330px;} .offset6{margin-left:392px;} .offset7{margin-left:454px;} .offset8{margin-left:516px;} .offset9{margin-left:578px;} .offset10{margin-left:640px;} .offset11{margin-left:702px;} .row-fluid{width:100%;*zoom:1;}.row-fluid:before,.row-fluid:after{display:table;content:"";} .row-fluid:after{clear:both;} .row-fluid>[class*="span"]{float:left;margin-left:2.762430939%;} .row-fluid>[class*="span"]:first-child{margin-left:0;} .row-fluid>.span1{width:5.801104972%;} .row-fluid>.span2{width:14.364640883%;} .row-fluid>.span3{width:22.928176794%;} .row-fluid>.span4{width:31.491712705%;} .row-fluid>.span5{width:40.055248616%;} .row-fluid>.span6{width:48.618784527%;} .row-fluid>.span7{width:57.182320438000005%;} .row-fluid>.span8{width:65.74585634900001%;} .row-fluid>.span9{width:74.30939226%;} .row-fluid>.span10{width:82.87292817100001%;} .row-fluid>.span11{width:91.436464082%;} .row-fluid>.span12{width:99.999999993%;} input.span1,textarea.span1,.uneditable-input.span1{width:32px;} input.span2,textarea.span2,.uneditable-input.span2{width:94px;} input.span3,textarea.span3,.uneditable-input.span3{width:156px;} input.span4,textarea.span4,.uneditable-input.span4{width:218px;} input.span5,textarea.span5,.uneditable-input.span5{width:280px;} input.span6,textarea.span6,.uneditable-input.span6{width:342px;} input.span7,textarea.span7,.uneditable-input.span7{width:404px;} input.span8,textarea.span8,.uneditable-input.span8{width:466px;} input.span9,textarea.span9,.uneditable-input.span9{width:528px;} input.span10,textarea.span10,.uneditable-input.span10{width:590px;} input.span11,textarea.span11,.uneditable-input.span11{width:652px;} input.span12,textarea.span12,.uneditable-input.span12{width:714px;}}@media (max-width:979px){body{padding-top:0;} .navbar-fixed-top{position:static;margin-bottom:18px;} .navbar-fixed-top .navbar-inner{padding:5px;} .navbar .container{width:auto;padding:0;} .navbar .brand{padding-left:10px;padding-right:10px;margin:0 0 0 -5px;} .navbar .nav-collapse{clear:left;} .navbar .nav{float:none;margin:0 0 9px;} .navbar .nav>li{float:none;} .navbar .nav>li>a{margin-bottom:2px;} .navbar .nav>.divider-vertical{display:none;} .navbar .nav .nav-header{color:#999999;text-shadow:none;} .navbar .nav>li>a,.navbar .dropdown-menu a{padding:6px 15px;font-weight:bold;color:#999999;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;} .navbar .dropdown-menu li+li a{margin-bottom:2px;} .navbar .nav>li>a:hover,.navbar .dropdown-menu a:hover{background-color:#222222;} .navbar .dropdown-menu{position:static;top:auto;left:auto;float:none;display:block;max-width:none;margin:0 15px;padding:0;background-color:transparent;border:none;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none;} .navbar .dropdown-menu:before,.navbar .dropdown-menu:after{display:none;} .navbar .dropdown-menu .divider{display:none;} .navbar-form,.navbar-search{float:none;padding:9px 15px;margin:9px 0;border-top:1px solid #222222;border-bottom:1px solid #222222;-webkit-box-shadow:inset 0 1px 0 rgba(255, 255, 255, 0.1),0 1px 0 rgba(255, 255, 255, 0.1);-moz-box-shadow:inset 0 1px 0 rgba(255, 255, 255, 0.1),0 1px 0 rgba(255, 255, 255, 0.1);box-shadow:inset 0 1px 0 rgba(255, 255, 255, 0.1),0 1px 0 rgba(255, 255, 255, 0.1);} .navbar .nav.pull-right{float:none;margin-left:0;} .navbar-static .navbar-inner{padding-left:10px;padding-right:10px;} .btn-navbar{display:block;} .nav-collapse{overflow:hidden;height:0;}}@media (min-width:980px){.nav-collapse.collapse{height:auto !important;}}@media (min-width:1200px){.row{margin-left:-30px;*zoom:1;}.row:before,.row:after{display:table;content:"";} .row:after{clear:both;} [class*="span"]{float:left;margin-left:30px;} .span1{width:70px;} .span2{width:170px;} .span3{width:270px;} .span4{width:370px;} .span5{width:470px;} .span6{width:570px;} .span7{width:670px;} .span8{width:770px;} .span9{width:870px;} .span10{width:970px;} .span11{width:1070px;} .span12,.container{width:1170px;} .offset1{margin-left:130px;} .offset2{margin-left:230px;} .offset3{margin-left:330px;} .offset4{margin-left:430px;} .offset5{margin-left:530px;} .offset6{margin-left:630px;} .offset7{margin-left:730px;} .offset8{margin-left:830px;} .offset9{margin-left:930px;} .offset10{margin-left:1030px;} .offset11{margin-left:1130px;} .row-fluid{width:100%;*zoom:1;}.row-fluid:before,.row-fluid:after{display:table;content:"";} .row-fluid:after{clear:both;} .row-fluid>[class*="span"]{float:left;margin-left:2.564102564%;} .row-fluid>[class*="span"]:first-child{margin-left:0;} .row-fluid>.span1{width:5.982905983%;} .row-fluid>.span2{width:14.529914530000001%;} .row-fluid>.span3{width:23.076923077%;} .row-fluid>.span4{width:31.623931624%;} .row-fluid>.span5{width:40.170940171000005%;} .row-fluid>.span6{width:48.717948718%;} .row-fluid>.span7{width:57.264957265%;} .row-fluid>.span8{width:65.81196581200001%;} .row-fluid>.span9{width:74.358974359%;} .row-fluid>.span10{width:82.905982906%;} .row-fluid>.span11{width:91.45299145300001%;} .row-fluid>.span12{width:100%;} input.span1,textarea.span1,.uneditable-input.span1{width:60px;} input.span2,textarea.span2,.uneditable-input.span2{width:160px;} input.span3,textarea.span3,.uneditable-input.span3{width:260px;} input.span4,textarea.span4,.uneditable-input.span4{width:360px;} input.span5,textarea.span5,.uneditable-input.span5{width:460px;} input.span6,textarea.span6,.uneditable-input.span6{width:560px;} input.span7,textarea.span7,.uneditable-input.span7{width:660px;} input.span8,textarea.span8,.uneditable-input.span8{width:760px;} input.span9,textarea.span9,.uneditable-input.span9{width:860px;} input.span10,textarea.span10,.uneditable-input.span10{width:960px;} input.span11,textarea.span11,.uneditable-input.span11{width:1060px;} input.span12,textarea.span12,.uneditable-input.span12{width:1160px;} .thumbnails{margin-left:-30px;} .thumbnails>li{margin-left:30px;}} 5 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ====== 2 | RESTMQ 3 | ====== 4 | 5 | Redis based message queue. 6 | -------------------------- 7 | 8 | :Info: See `RestMQ website `_ 9 | :Info: See `my blog ` for more information. 10 | :Author: `Gleicon Moraes ` 11 | :Author: `Alexandre Fiori `_ 12 | 13 | 14 | About 15 | ===== 16 | 17 | RestMQ is a message queue which uses HTTP as transport, JSON to format a minimalist protocol and is organized as REST 18 | resources. It stands on the shoulder of giants, built over Python, Twisted, `Cyclone `_ (a Tornado implementation over twisted) and Redis. 19 | 20 | Redis is more than just a key/value db, and its data types provided support for this project. 21 | 22 | Queues are created on the fly, when a message is sent to them. They are simple to use as a curl request can be. HTTP Comet, websockets and server sent events may be used for data streaming. 23 | 24 | This release is a cleanup of the original code, that can also be found on github. 25 | 26 | Install 27 | ======= 28 | 29 | $ python setup.py install 30 | run with bash start_scripts/restmq_server or taylor your own script. Note that currently restmq is presented as a twisted plugin. 31 | 32 | Alternatively you can utilize [Vagrant](https://www.vagrantup.com/) and our [Vagrantfile](Vagrantfile) which will handle installation and configuration of redis and RestMQ. 33 | 34 | Another option is to run the project inside a docker container: 35 | 36 | ``` 37 | docker build -t restmq . 38 | docker run --rm -p 6379:6379 -p 8888:8888 restmq 39 | ``` 40 | 41 | It will run restmq, exposing both 8888 (for its http port) and 6379 (for its redis server) to the host environment. 42 | 43 | Queue Policy 44 | ============ 45 | 46 | Every queue is created with a default policy, which is `broadcast`. It means that each message 47 | pushed to a queue will be forwarded to all comet and websocket consumers. 48 | 49 | The alternative policy is `roundrobin`, which will distribute these messages in a round robin 50 | fashion to all comet and websocket consumers. 51 | 52 | The queue policy won't affect regular GET commands to pop a single message from a queue. 53 | 54 | See `README.qp `_ for details. 55 | 56 | 57 | Queue Flow Control 58 | ================== 59 | 60 | Yes, it does support start and stop. We just need to document it properly. 61 | 62 | 63 | Example 64 | ======== 65 | A http client (curl) post to /queue: 66 | 67 | Point your browser to http://localhost:8888/c/test 68 | 69 | Run $ curl -d "queue=test&value=foobar" http://localhost:8888/ 70 | 71 | Your browser is acting as a consumer to the queue. Using json encoded data it's easy to fit the data into a js based app. 72 | 73 | Aside from the COMET consumer, there are xmlrpc methods, rest routes and the JSON protocol to manipulate queue items. 74 | 75 | 76 | COMET consumer 77 | ============== 78 | 79 | There is a COMET based consumer, which will bind even if the queue doesn't already exists. 80 | 81 | The main route is thru /c/. It can be tested using curl: 82 | 83 | $ curl http://localhost:8888/c/test 84 | 85 | In another terminal, run $ curl -d "value=foobar" http://localhost:8888/q/test 86 | 87 | This is the basic usage pattern for map/reduce (see examples). 88 | 89 | See below on how to purge and disconnect all consumers from a queue, using DELETE. 90 | 91 | 92 | WebSocket consumer 93 | ================== 94 | 95 | Now that cyclone has websockets support, check README.websocket to test it. 96 | 97 | If you are using a browser or library which already supports websockets, you may take advantage of this interface. 98 | 99 | 100 | REST services 101 | ============= 102 | 103 | A queue can be accessed as /q/. 104 | 105 | GET requests will dequeue an object. 106 | 107 | POST requests inserts an object in the queue 108 | 109 | DELETE requests will purgue the queue. 110 | 111 | The usual pattern is listen in the COMET consumer (/c/) and insert new stuff at the REST route (POST /q/ uses the JSON protocol. Its the same protocol I've implemented for http://jsonqueue.appspot.com. 118 | 119 | :: 120 | 121 | { 122 | "cmd": "add", 123 | "queue": "genesis", 124 | "value": "abacab" 125 | } 126 | 127 | Creates the queue named *"genesis"* and inserts the string *"abacab"* as the message. 128 | 129 | If we want to take that out of the queue, the payload would be like that: 130 | 131 | :: 132 | 133 | { 134 | "cmd": "take", 135 | "queue": "genesis" 136 | } 137 | 138 | 139 | The message can be formatted as a json object, so more complex data can be sent. 140 | It really mimics some of `Amazon SQS `_ workings, because it's a simple queue. 141 | 142 | For the first release it has: 143 | 144 | - Select, EPoll or KQueue concurrency (depends on twisted) 145 | - Persistent storage using Redis 146 | - Can work on pools, N daemons consuming from the same queues. 147 | - Small codebase 148 | - Lightweight 149 | - Cute ? 150 | 151 | 152 | Dependencies 153 | ============ 154 | - `cyclone `_: 155 | git clone git://github.com/fiorix/cyclone.git 156 | 157 | 158 | Running 159 | ======= 160 | 161 | The `redis_server `_ script will start the service. It's a bash script used to both configure and run RestMQ. The default version of the wrapper script will run the server in foreground, and log messages will be written to the standard output. 162 | 163 | Editing the script is mandatory for configuring RestMQ for production. 164 | 165 | :: 166 | 167 | $ ./restmq_server --help 168 | Usage: twistd [options] restmq [options] 169 | Options: 170 | --acl= acl configuration file for endpoints [default: acl.conf] 171 | --redis-host= hostname or ip address of the redis server [default: 127.0.0.1] 172 | --redis-port= port number of the redis server [default: 6379] 173 | --redis-pool= connection pool size [default: 10] 174 | --port= port number to listen on [default: 8888] 175 | --listen= interface to listen on [default: 127.0.0.1] 176 | --version 177 | --help Display this help and exit. 178 | 179 | 180 | Tests 181 | ===== 182 | 183 | :: 184 | 185 | examples/test_rest.sh 186 | examples/test_xmlrpc.py 187 | python examples/test_comet.py 188 | python examples/twitter_trends.py 189 | python examples/test_comet_curl.py 190 | python restmq_engine.py -h 191 | 192 | 193 | Files 194 | ===== 195 | 196 | If you're a developer looking for extending RestMQ's functionality, have a look at these files: 197 | 198 | - `restmq/web.py `_: the web service code 199 | - `restmq/core.py `_: redis/queue operations logic 200 | - `restmq/dispatch.py `_: a simple command dispatcher 201 | - `restmq_engine.py `_: the redis abstraction layer to the queue algorithm (command line tool) 202 | 203 | 204 | Credits 205 | ======= 206 | Thanks to (in no particular order): 207 | 208 | - Salvatore Sanfilippo for redis and for NoSQL patterns discussion. 209 | - Alexandre Fiori for the redis client enhancement and patches. 210 | - Roberto Gaiser for the collectd daemon 211 | - Vitor Pellegrino for dockerizing restmq 212 | -------------------------------------------------------------------------------- /ez_setup.py: -------------------------------------------------------------------------------- 1 | #!python 2 | """Bootstrap setuptools installation 3 | 4 | If you want to use setuptools in your package's setup.py, just include this 5 | file in the same directory with it, and add this to the top of your setup.py:: 6 | 7 | from ez_setup import use_setuptools 8 | use_setuptools() 9 | 10 | If you want to require a specific version of setuptools, set a download 11 | mirror, or use an alternate download directory, you can do so by supplying 12 | the appropriate options to ``use_setuptools()``. 13 | 14 | This file can also be run as a script to install or upgrade setuptools. 15 | """ 16 | import sys 17 | DEFAULT_VERSION = "0.6c11" 18 | MINIMUM_VERSION = "0.6c9" 19 | DEFAULT_URL = "http://pypi.python.org/packages/%s/s/setuptools/" % sys.version[:3] 20 | 21 | md5_data = { 22 | 'setuptools-0.6b1-py2.3.egg': '8822caf901250d848b996b7f25c6e6ca', 23 | 'setuptools-0.6b1-py2.4.egg': 'b79a8a403e4502fbb85ee3f1941735cb', 24 | 'setuptools-0.6b2-py2.3.egg': '5657759d8a6d8fc44070a9d07272d99b', 25 | 'setuptools-0.6b2-py2.4.egg': '4996a8d169d2be661fa32a6e52e4f82a', 26 | 'setuptools-0.6b3-py2.3.egg': 'bb31c0fc7399a63579975cad9f5a0618', 27 | 'setuptools-0.6b3-py2.4.egg': '38a8c6b3d6ecd22247f179f7da669fac', 28 | 'setuptools-0.6b4-py2.3.egg': '62045a24ed4e1ebc77fe039aa4e6f7e5', 29 | 'setuptools-0.6b4-py2.4.egg': '4cb2a185d228dacffb2d17f103b3b1c4', 30 | 'setuptools-0.6c1-py2.3.egg': 'b3f2b5539d65cb7f74ad79127f1a908c', 31 | 'setuptools-0.6c1-py2.4.egg': 'b45adeda0667d2d2ffe14009364f2a4b', 32 | 'setuptools-0.6c10-py2.3.egg': 'ce1e2ab5d3a0256456d9fc13800a7090', 33 | 'setuptools-0.6c10-py2.4.egg': '57d6d9d6e9b80772c59a53a8433a5dd4', 34 | 'setuptools-0.6c10-py2.5.egg': 'de46ac8b1c97c895572e5e8596aeb8c7', 35 | 'setuptools-0.6c10-py2.6.egg': '58ea40aef06da02ce641495523a0b7f5', 36 | 'setuptools-0.6c11-py2.3.egg': '2baeac6e13d414a9d28e7ba5b5a596de', 37 | 'setuptools-0.6c11-py2.4.egg': 'bd639f9b0eac4c42497034dec2ec0c2b', 38 | 'setuptools-0.6c11-py2.5.egg': '64c94f3bf7a72a13ec83e0b24f2749b2', 39 | 'setuptools-0.6c11-py2.6.egg': 'bfa92100bd772d5a213eedd356d64086', 40 | 'setuptools-0.6c2-py2.3.egg': 'f0064bf6aa2b7d0f3ba0b43f20817c27', 41 | 'setuptools-0.6c2-py2.4.egg': '616192eec35f47e8ea16cd6a122b7277', 42 | 'setuptools-0.6c3-py2.3.egg': 'f181fa125dfe85a259c9cd6f1d7b78fa', 43 | 'setuptools-0.6c3-py2.4.egg': 'e0ed74682c998bfb73bf803a50e7b71e', 44 | 'setuptools-0.6c3-py2.5.egg': 'abef16fdd61955514841c7c6bd98965e', 45 | 'setuptools-0.6c4-py2.3.egg': 'b0b9131acab32022bfac7f44c5d7971f', 46 | 'setuptools-0.6c4-py2.4.egg': '2a1f9656d4fbf3c97bf946c0a124e6e2', 47 | 'setuptools-0.6c4-py2.5.egg': '8f5a052e32cdb9c72bcf4b5526f28afc', 48 | 'setuptools-0.6c5-py2.3.egg': 'ee9fd80965da04f2f3e6b3576e9d8167', 49 | 'setuptools-0.6c5-py2.4.egg': 'afe2adf1c01701ee841761f5bcd8aa64', 50 | 'setuptools-0.6c5-py2.5.egg': 'a8d3f61494ccaa8714dfed37bccd3d5d', 51 | 'setuptools-0.6c6-py2.3.egg': '35686b78116a668847237b69d549ec20', 52 | 'setuptools-0.6c6-py2.4.egg': '3c56af57be3225019260a644430065ab', 53 | 'setuptools-0.6c6-py2.5.egg': 'b2f8a7520709a5b34f80946de5f02f53', 54 | 'setuptools-0.6c7-py2.3.egg': '209fdf9adc3a615e5115b725658e13e2', 55 | 'setuptools-0.6c7-py2.4.egg': '5a8f954807d46a0fb67cf1f26c55a82e', 56 | 'setuptools-0.6c7-py2.5.egg': '45d2ad28f9750e7434111fde831e8372', 57 | 'setuptools-0.6c8-py2.3.egg': '50759d29b349db8cfd807ba8303f1902', 58 | 'setuptools-0.6c8-py2.4.egg': 'cba38d74f7d483c06e9daa6070cce6de', 59 | 'setuptools-0.6c8-py2.5.egg': '1721747ee329dc150590a58b3e1ac95b', 60 | 'setuptools-0.6c9-py2.3.egg': 'a83c4020414807b496e4cfbe08507c03', 61 | 'setuptools-0.6c9-py2.4.egg': '260a2be2e5388d66bdaee06abec6342a', 62 | 'setuptools-0.6c9-py2.5.egg': 'fe67c3e5a17b12c0e7c541b7ea43a8e6', 63 | 'setuptools-0.6c9-py2.6.egg': 'ca37b1ff16fa2ede6e19383e7b59245a', 64 | } 65 | 66 | import sys, os 67 | try: from hashlib import md5 68 | except ImportError: from md5 import md5 69 | 70 | def _validate_md5(egg_name, data): 71 | if egg_name in md5_data: 72 | digest = md5(data).hexdigest() 73 | if digest != md5_data[egg_name]: 74 | print >>sys.stderr, ( 75 | "md5 validation of %s failed! (Possible download problem?)" 76 | % egg_name 77 | ) 78 | sys.exit(2) 79 | return data 80 | 81 | def use_setuptools( 82 | version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, 83 | download_delay=15, min_version=MINIMUM_VERSION 84 | ): 85 | """Automatically find/download setuptools and make it available on sys.path 86 | 87 | `version` should be a valid setuptools version number that is available 88 | as an egg for download under the `download_base` URL (which should end with 89 | a '/'). `to_dir` is the directory where setuptools will be downloaded, if 90 | it is not already available. If `download_delay` is specified, it should 91 | be the number of seconds that will be paused before initiating a download, 92 | should one be required. If an older version of setuptools is installed, 93 | this routine will print a message to ``sys.stderr`` and raise SystemExit in 94 | an attempt to abort the calling script. 95 | """ 96 | was_imported = 'pkg_resources' in sys.modules or 'setuptools' in sys.modules 97 | def do_download(): 98 | egg = download_setuptools(version, download_base, to_dir, download_delay) 99 | sys.path.insert(0, egg) 100 | import setuptools; setuptools.bootstrap_install_from = egg 101 | try: 102 | import pkg_resources 103 | except ImportError: 104 | return do_download() 105 | try: 106 | if not min_version: 107 | min_version = version 108 | pkg_resources.require("setuptools>="+min_version) 109 | return 110 | except pkg_resources.VersionConflict, e: 111 | if was_imported: 112 | print >>sys.stderr, ( 113 | "The required version of setuptools (>=%s) is not available, and\n" 114 | "can't be installed while this script is running. Please install\n" 115 | " a more recent version first, using 'easy_install -U setuptools'." 116 | "\n\n(Currently using %r)" 117 | ) % (version, e.args[0]) 118 | sys.exit(2) 119 | else: 120 | del pkg_resources, sys.modules['pkg_resources'] # reload ok 121 | return do_download() 122 | except pkg_resources.DistributionNotFound: 123 | return do_download() 124 | 125 | def download_setuptools( 126 | version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, 127 | delay = 15 128 | ): 129 | """Download setuptools from a specified location and return its filename 130 | 131 | `version` should be a valid setuptools version number that is available 132 | as an egg for download under the `download_base` URL (which should end 133 | with a '/'). `to_dir` is the directory where the egg will be downloaded. 134 | `delay` is the number of seconds to pause before an actual download attempt. 135 | """ 136 | import urllib2, shutil 137 | egg_name = "setuptools-%s-py%s.egg" % (version,sys.version[:3]) 138 | url = download_base + egg_name 139 | saveto = os.path.join(to_dir, egg_name) 140 | src = dst = None 141 | if not os.path.exists(saveto): # Avoid repeated downloads 142 | try: 143 | from distutils import log 144 | if delay: 145 | log.warn(""" 146 | --------------------------------------------------------------------------- 147 | This script requires setuptools version %s to run (even to display 148 | help). I will attempt to download it for you (from 149 | %s), but 150 | you may need to enable firewall access for this script first. 151 | I will start the download in %d seconds. 152 | 153 | (Note: if this machine does not have network access, please obtain the file 154 | 155 | %s 156 | 157 | and place it in this directory before rerunning this script.) 158 | ---------------------------------------------------------------------------""", 159 | version, download_base, delay, url 160 | ); from time import sleep; sleep(delay) 161 | log.warn("Downloading %s", url) 162 | src = urllib2.urlopen(url) 163 | # Read/write all in one block, so we don't create a corrupt file 164 | # if the download is interrupted. 165 | data = _validate_md5(egg_name, src.read()) 166 | dst = open(saveto,"wb"); dst.write(data) 167 | finally: 168 | if src: src.close() 169 | if dst: dst.close() 170 | return os.path.realpath(saveto) 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | def main(argv, version=DEFAULT_VERSION): 208 | """Install or upgrade setuptools and EasyInstall""" 209 | try: 210 | import setuptools 211 | except ImportError: 212 | egg = None 213 | try: 214 | egg = download_setuptools(version, delay=0) 215 | sys.path.insert(0,egg) 216 | from setuptools.command.easy_install import main 217 | return main(list(argv)+[egg]) # we're done here 218 | finally: 219 | if egg and os.path.exists(egg): 220 | os.unlink(egg) 221 | else: 222 | if setuptools.__version__ == '0.0.1': 223 | print >>sys.stderr, ( 224 | "You have an obsolete version of setuptools installed. Please\n" 225 | "remove it from your system entirely before rerunning this script." 226 | ) 227 | sys.exit(2) 228 | 229 | req = "setuptools>="+version 230 | import pkg_resources 231 | try: 232 | pkg_resources.require(req) 233 | except pkg_resources.VersionConflict: 234 | try: 235 | from setuptools.command.easy_install import main 236 | except ImportError: 237 | from easy_install import main 238 | main(list(argv)+[download_setuptools(delay=0)]) 239 | sys.exit(0) # try to force an exit 240 | else: 241 | if argv: 242 | from setuptools.command.easy_install import main 243 | main(argv) 244 | else: 245 | print "Setuptools version",version,"or greater has been installed." 246 | print '(Run "ez_setup.py -U setuptools" to reinstall or upgrade.)' 247 | 248 | def update_md5(filenames): 249 | """Update our built-in md5 registry""" 250 | 251 | import re 252 | 253 | for name in filenames: 254 | base = os.path.basename(name) 255 | f = open(name,'rb') 256 | md5_data[base] = md5(f.read()).hexdigest() 257 | f.close() 258 | 259 | data = [" %r: %r,\n" % it for it in md5_data.items()] 260 | data.sort() 261 | repl = "".join(data) 262 | 263 | import inspect 264 | srcfile = inspect.getsourcefile(sys.modules[__name__]) 265 | f = open(srcfile, 'rb'); src = f.read(); f.close() 266 | 267 | match = re.search("\nmd5_data = {\n([^}]+)}", src) 268 | if not match: 269 | print >>sys.stderr, "Internal error!" 270 | sys.exit(2) 271 | 272 | src = src[:match.start(1)] + repl + src[match.end(1):] 273 | f = open(srcfile,'w') 274 | f.write(src) 275 | f.close() 276 | 277 | 278 | if __name__=='__main__': 279 | if len(sys.argv)>2 and sys.argv[1]=='--md5update': 280 | update_md5(sys.argv[2:]) 281 | else: 282 | main(sys.argv[1:]) 283 | 284 | 285 | 286 | 287 | 288 | 289 | -------------------------------------------------------------------------------- /src/restmq/static/css/bootstrap-responsive.css: -------------------------------------------------------------------------------- 1 | /*! 2 | * Bootstrap Responsive v2.0.1 3 | * 4 | * Copyright 2012 Twitter, Inc 5 | * Licensed under the Apache License v2.0 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Designed and built with all the love in the world @twitter by @mdo and @fat. 9 | */ 10 | .clearfix { 11 | *zoom: 1; 12 | } 13 | .clearfix:before, .clearfix:after { 14 | display: table; 15 | content: ""; 16 | } 17 | .clearfix:after { 18 | clear: both; 19 | } 20 | .hidden { 21 | display: none; 22 | visibility: hidden; 23 | } 24 | @media (max-width: 480px) { 25 | .nav-collapse { 26 | -webkit-transform: translate3d(0, 0, 0); 27 | } 28 | .page-header h1 small { 29 | display: block; 30 | line-height: 18px; 31 | } 32 | input[class*="span"], 33 | select[class*="span"], 34 | textarea[class*="span"], 35 | .uneditable-input { 36 | display: block; 37 | width: 100%; 38 | min-height: 28px; 39 | /* Make inputs at least the height of their button counterpart */ 40 | 41 | /* Makes inputs behave like true block-level elements */ 42 | 43 | -webkit-box-sizing: border-box; 44 | /* Older Webkit */ 45 | 46 | -moz-box-sizing: border-box; 47 | /* Older FF */ 48 | 49 | -ms-box-sizing: border-box; 50 | /* IE8 */ 51 | 52 | box-sizing: border-box; 53 | /* CSS3 spec*/ 54 | 55 | } 56 | .input-prepend input[class*="span"], .input-append input[class*="span"] { 57 | width: auto; 58 | } 59 | input[type="checkbox"], input[type="radio"] { 60 | border: 1px solid #ccc; 61 | } 62 | .form-horizontal .control-group > label { 63 | float: none; 64 | width: auto; 65 | padding-top: 0; 66 | text-align: left; 67 | } 68 | .form-horizontal .controls { 69 | margin-left: 0; 70 | } 71 | .form-horizontal .control-list { 72 | padding-top: 0; 73 | } 74 | .form-horizontal .form-actions { 75 | padding-left: 10px; 76 | padding-right: 10px; 77 | } 78 | .modal { 79 | position: absolute; 80 | top: 10px; 81 | left: 10px; 82 | right: 10px; 83 | width: auto; 84 | margin: 0; 85 | } 86 | .modal.fade.in { 87 | top: auto; 88 | } 89 | .modal-header .close { 90 | padding: 10px; 91 | margin: -10px; 92 | } 93 | .carousel-caption { 94 | position: static; 95 | } 96 | } 97 | @media (max-width: 767px) { 98 | .container { 99 | width: auto; 100 | padding: 0 20px; 101 | } 102 | .row-fluid { 103 | width: 100%; 104 | } 105 | .row { 106 | margin-left: 0; 107 | } 108 | .row > [class*="span"], .row-fluid > [class*="span"] { 109 | float: none; 110 | display: block; 111 | width: auto; 112 | margin: 0; 113 | } 114 | } 115 | @media (min-width: 768px) and (max-width: 979px) { 116 | .row { 117 | margin-left: -20px; 118 | *zoom: 1; 119 | } 120 | .row:before, .row:after { 121 | display: table; 122 | content: ""; 123 | } 124 | .row:after { 125 | clear: both; 126 | } 127 | [class*="span"] { 128 | float: left; 129 | margin-left: 20px; 130 | } 131 | .span1 { 132 | width: 42px; 133 | } 134 | .span2 { 135 | width: 104px; 136 | } 137 | .span3 { 138 | width: 166px; 139 | } 140 | .span4 { 141 | width: 228px; 142 | } 143 | .span5 { 144 | width: 290px; 145 | } 146 | .span6 { 147 | width: 352px; 148 | } 149 | .span7 { 150 | width: 414px; 151 | } 152 | .span8 { 153 | width: 476px; 154 | } 155 | .span9 { 156 | width: 538px; 157 | } 158 | .span10 { 159 | width: 600px; 160 | } 161 | .span11 { 162 | width: 662px; 163 | } 164 | .span12, .container { 165 | width: 724px; 166 | } 167 | .offset1 { 168 | margin-left: 82px; 169 | } 170 | .offset2 { 171 | margin-left: 144px; 172 | } 173 | .offset3 { 174 | margin-left: 206px; 175 | } 176 | .offset4 { 177 | margin-left: 268px; 178 | } 179 | .offset5 { 180 | margin-left: 330px; 181 | } 182 | .offset6 { 183 | margin-left: 392px; 184 | } 185 | .offset7 { 186 | margin-left: 454px; 187 | } 188 | .offset8 { 189 | margin-left: 516px; 190 | } 191 | .offset9 { 192 | margin-left: 578px; 193 | } 194 | .offset10 { 195 | margin-left: 640px; 196 | } 197 | .offset11 { 198 | margin-left: 702px; 199 | } 200 | .row-fluid { 201 | width: 100%; 202 | *zoom: 1; 203 | } 204 | .row-fluid:before, .row-fluid:after { 205 | display: table; 206 | content: ""; 207 | } 208 | .row-fluid:after { 209 | clear: both; 210 | } 211 | .row-fluid > [class*="span"] { 212 | float: left; 213 | margin-left: 2.762430939%; 214 | } 215 | .row-fluid > [class*="span"]:first-child { 216 | margin-left: 0; 217 | } 218 | .row-fluid > .span1 { 219 | width: 5.801104972%; 220 | } 221 | .row-fluid > .span2 { 222 | width: 14.364640883%; 223 | } 224 | .row-fluid > .span3 { 225 | width: 22.928176794%; 226 | } 227 | .row-fluid > .span4 { 228 | width: 31.491712705%; 229 | } 230 | .row-fluid > .span5 { 231 | width: 40.055248616%; 232 | } 233 | .row-fluid > .span6 { 234 | width: 48.618784527%; 235 | } 236 | .row-fluid > .span7 { 237 | width: 57.182320438000005%; 238 | } 239 | .row-fluid > .span8 { 240 | width: 65.74585634900001%; 241 | } 242 | .row-fluid > .span9 { 243 | width: 74.30939226%; 244 | } 245 | .row-fluid > .span10 { 246 | width: 82.87292817100001%; 247 | } 248 | .row-fluid > .span11 { 249 | width: 91.436464082%; 250 | } 251 | .row-fluid > .span12 { 252 | width: 99.999999993%; 253 | } 254 | input.span1, textarea.span1, .uneditable-input.span1 { 255 | width: 32px; 256 | } 257 | input.span2, textarea.span2, .uneditable-input.span2 { 258 | width: 94px; 259 | } 260 | input.span3, textarea.span3, .uneditable-input.span3 { 261 | width: 156px; 262 | } 263 | input.span4, textarea.span4, .uneditable-input.span4 { 264 | width: 218px; 265 | } 266 | input.span5, textarea.span5, .uneditable-input.span5 { 267 | width: 280px; 268 | } 269 | input.span6, textarea.span6, .uneditable-input.span6 { 270 | width: 342px; 271 | } 272 | input.span7, textarea.span7, .uneditable-input.span7 { 273 | width: 404px; 274 | } 275 | input.span8, textarea.span8, .uneditable-input.span8 { 276 | width: 466px; 277 | } 278 | input.span9, textarea.span9, .uneditable-input.span9 { 279 | width: 528px; 280 | } 281 | input.span10, textarea.span10, .uneditable-input.span10 { 282 | width: 590px; 283 | } 284 | input.span11, textarea.span11, .uneditable-input.span11 { 285 | width: 652px; 286 | } 287 | input.span12, textarea.span12, .uneditable-input.span12 { 288 | width: 714px; 289 | } 290 | } 291 | @media (max-width: 979px) { 292 | body { 293 | padding-top: 0; 294 | } 295 | .navbar-fixed-top { 296 | position: static; 297 | margin-bottom: 18px; 298 | } 299 | .navbar-fixed-top .navbar-inner { 300 | padding: 5px; 301 | } 302 | .navbar .container { 303 | width: auto; 304 | padding: 0; 305 | } 306 | .navbar .brand { 307 | padding-left: 10px; 308 | padding-right: 10px; 309 | margin: 0 0 0 -5px; 310 | } 311 | .navbar .nav-collapse { 312 | clear: left; 313 | } 314 | .navbar .nav { 315 | float: none; 316 | margin: 0 0 9px; 317 | } 318 | .navbar .nav > li { 319 | float: none; 320 | } 321 | .navbar .nav > li > a { 322 | margin-bottom: 2px; 323 | } 324 | .navbar .nav > .divider-vertical { 325 | display: none; 326 | } 327 | .navbar .nav .nav-header { 328 | color: #999999; 329 | text-shadow: none; 330 | } 331 | .navbar .nav > li > a, .navbar .dropdown-menu a { 332 | padding: 6px 15px; 333 | font-weight: bold; 334 | color: #999999; 335 | -webkit-border-radius: 3px; 336 | -moz-border-radius: 3px; 337 | border-radius: 3px; 338 | } 339 | .navbar .dropdown-menu li + li a { 340 | margin-bottom: 2px; 341 | } 342 | .navbar .nav > li > a:hover, .navbar .dropdown-menu a:hover { 343 | background-color: #222222; 344 | } 345 | .navbar .dropdown-menu { 346 | position: static; 347 | top: auto; 348 | left: auto; 349 | float: none; 350 | display: block; 351 | max-width: none; 352 | margin: 0 15px; 353 | padding: 0; 354 | background-color: transparent; 355 | border: none; 356 | -webkit-border-radius: 0; 357 | -moz-border-radius: 0; 358 | border-radius: 0; 359 | -webkit-box-shadow: none; 360 | -moz-box-shadow: none; 361 | box-shadow: none; 362 | } 363 | .navbar .dropdown-menu:before, .navbar .dropdown-menu:after { 364 | display: none; 365 | } 366 | .navbar .dropdown-menu .divider { 367 | display: none; 368 | } 369 | .navbar-form, .navbar-search { 370 | float: none; 371 | padding: 9px 15px; 372 | margin: 9px 0; 373 | border-top: 1px solid #222222; 374 | border-bottom: 1px solid #222222; 375 | -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); 376 | -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); 377 | box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); 378 | } 379 | .navbar .nav.pull-right { 380 | float: none; 381 | margin-left: 0; 382 | } 383 | .navbar-static .navbar-inner { 384 | padding-left: 10px; 385 | padding-right: 10px; 386 | } 387 | .btn-navbar { 388 | display: block; 389 | } 390 | .nav-collapse { 391 | overflow: hidden; 392 | height: 0; 393 | } 394 | } 395 | @media (min-width: 980px) { 396 | .nav-collapse.collapse { 397 | height: auto !important; 398 | } 399 | } 400 | @media (min-width: 1200px) { 401 | .row { 402 | margin-left: -30px; 403 | *zoom: 1; 404 | } 405 | .row:before, .row:after { 406 | display: table; 407 | content: ""; 408 | } 409 | .row:after { 410 | clear: both; 411 | } 412 | [class*="span"] { 413 | float: left; 414 | margin-left: 30px; 415 | } 416 | .span1 { 417 | width: 70px; 418 | } 419 | .span2 { 420 | width: 170px; 421 | } 422 | .span3 { 423 | width: 270px; 424 | } 425 | .span4 { 426 | width: 370px; 427 | } 428 | .span5 { 429 | width: 470px; 430 | } 431 | .span6 { 432 | width: 570px; 433 | } 434 | .span7 { 435 | width: 670px; 436 | } 437 | .span8 { 438 | width: 770px; 439 | } 440 | .span9 { 441 | width: 870px; 442 | } 443 | .span10 { 444 | width: 970px; 445 | } 446 | .span11 { 447 | width: 1070px; 448 | } 449 | .span12, .container { 450 | width: 1170px; 451 | } 452 | .offset1 { 453 | margin-left: 130px; 454 | } 455 | .offset2 { 456 | margin-left: 230px; 457 | } 458 | .offset3 { 459 | margin-left: 330px; 460 | } 461 | .offset4 { 462 | margin-left: 430px; 463 | } 464 | .offset5 { 465 | margin-left: 530px; 466 | } 467 | .offset6 { 468 | margin-left: 630px; 469 | } 470 | .offset7 { 471 | margin-left: 730px; 472 | } 473 | .offset8 { 474 | margin-left: 830px; 475 | } 476 | .offset9 { 477 | margin-left: 930px; 478 | } 479 | .offset10 { 480 | margin-left: 1030px; 481 | } 482 | .offset11 { 483 | margin-left: 1130px; 484 | } 485 | .row-fluid { 486 | width: 100%; 487 | *zoom: 1; 488 | } 489 | .row-fluid:before, .row-fluid:after { 490 | display: table; 491 | content: ""; 492 | } 493 | .row-fluid:after { 494 | clear: both; 495 | } 496 | .row-fluid > [class*="span"] { 497 | float: left; 498 | margin-left: 2.564102564%; 499 | } 500 | .row-fluid > [class*="span"]:first-child { 501 | margin-left: 0; 502 | } 503 | .row-fluid > .span1 { 504 | width: 5.982905983%; 505 | } 506 | .row-fluid > .span2 { 507 | width: 14.529914530000001%; 508 | } 509 | .row-fluid > .span3 { 510 | width: 23.076923077%; 511 | } 512 | .row-fluid > .span4 { 513 | width: 31.623931624%; 514 | } 515 | .row-fluid > .span5 { 516 | width: 40.170940171000005%; 517 | } 518 | .row-fluid > .span6 { 519 | width: 48.717948718%; 520 | } 521 | .row-fluid > .span7 { 522 | width: 57.264957265%; 523 | } 524 | .row-fluid > .span8 { 525 | width: 65.81196581200001%; 526 | } 527 | .row-fluid > .span9 { 528 | width: 74.358974359%; 529 | } 530 | .row-fluid > .span10 { 531 | width: 82.905982906%; 532 | } 533 | .row-fluid > .span11 { 534 | width: 91.45299145300001%; 535 | } 536 | .row-fluid > .span12 { 537 | width: 100%; 538 | } 539 | input.span1, textarea.span1, .uneditable-input.span1 { 540 | width: 60px; 541 | } 542 | input.span2, textarea.span2, .uneditable-input.span2 { 543 | width: 160px; 544 | } 545 | input.span3, textarea.span3, .uneditable-input.span3 { 546 | width: 260px; 547 | } 548 | input.span4, textarea.span4, .uneditable-input.span4 { 549 | width: 360px; 550 | } 551 | input.span5, textarea.span5, .uneditable-input.span5 { 552 | width: 460px; 553 | } 554 | input.span6, textarea.span6, .uneditable-input.span6 { 555 | width: 560px; 556 | } 557 | input.span7, textarea.span7, .uneditable-input.span7 { 558 | width: 660px; 559 | } 560 | input.span8, textarea.span8, .uneditable-input.span8 { 561 | width: 760px; 562 | } 563 | input.span9, textarea.span9, .uneditable-input.span9 { 564 | width: 860px; 565 | } 566 | input.span10, textarea.span10, .uneditable-input.span10 { 567 | width: 960px; 568 | } 569 | input.span11, textarea.span11, .uneditable-input.span11 { 570 | width: 1060px; 571 | } 572 | input.span12, textarea.span12, .uneditable-input.span12 { 573 | width: 1160px; 574 | } 575 | .thumbnails { 576 | margin-left: -30px; 577 | } 578 | .thumbnails > li { 579 | margin-left: 30px; 580 | } 581 | } 582 | -------------------------------------------------------------------------------- /src/restmq/core.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import types 4 | import cyclone.escape 5 | from twisted.internet import defer 6 | import itertools 7 | 8 | POLICY_BROADCAST = 1 9 | POLICY_ROUNDROBIN = 2 10 | QUEUE_STATUS = 'queuestat:' 11 | QUEUE_POLICY = "%s:queuepolicy" 12 | QUEUE_NAME = '%s:queue' 13 | 14 | class RedisOperations: 15 | """ 16 | add element to the queue: 17 | - increments a UUID record 18 | - store the object using a key as :uuid 19 | - push this key into a list named :queue 20 | - push this list name into the general QUEUESET 21 | get element from queue: 22 | - pop a key from the list 23 | - get and return, along with its key 24 | 25 | del element from the queue: 26 | - tricky part. there must be a queue_get() before. The object is out of the queue already. delete it. 27 | 28 | - TODO: the object may have an expiration instead of straight deletion 29 | - TODO: RPOPLPUSH can be used to put it in another queue as a backlog 30 | - TODO: persistence management (on/off/status) 31 | """ 32 | 33 | def __init__(self, redis): 34 | self.STOPQUEUE = 0 35 | self.STARTQUEUE = 1 36 | self.redis = redis 37 | self.policies = { 38 | "broadcast": POLICY_BROADCAST, 39 | "roundrobin": POLICY_ROUNDROBIN, 40 | } 41 | self.inverted_policies = dict([[v, k] for k, v in self.policies.items()]) 42 | self.QUEUESET = 'QUEUESET' # the set which holds all queues 43 | self.PUBSUB_SUFIX = 'PUBSUB' 44 | 45 | def normalize(self, item): 46 | if isinstance(item, types.StringType): 47 | return item 48 | elif isinstance(item, types.UnicodeType): 49 | try: 50 | return item.encode("utf-8") 51 | except: 52 | raise ValueError("strings must be utf-8") 53 | else: 54 | raise ValueError("data must be utf-8") 55 | 56 | @defer.inlineCallbacks 57 | def authorize(self, queue, authkey): 58 | """ Authorize an operation for a given queue using an authentication key 59 | The basic mechanism is a check against Redis to see if key named AUTHKEY: exists 60 | If it exists, check against its content to see wheter the queue is authorized. 61 | Authorization is either read/write a queue and create new queues 62 | queues and priv are lists in the authorization record 63 | returns boolean 64 | """ 65 | queue, authkey = self.normalize(queue), self.normalize(authkey) 66 | # get key and analyze {'queues': ['q1','q2','q3'], 'privs': ['create']} 67 | avkey = "AUTHKEY:%s" % authkey 68 | authval = yield self.redis.get(avkey.encode('utf-8')) 69 | if authval == None: 70 | defer.returnValue(False) 71 | try: 72 | adata = cyclone.escape.json_decode(authval) 73 | except Exception, e: 74 | defer.returnValue(None) 75 | if queue in adata['queues']: 76 | defer.returnValue(True) 77 | elif 'create' in adata['privs']: 78 | defer.returnValue(True) 79 | 80 | defer.returnValue(False) 81 | 82 | @defer.inlineCallbacks 83 | def _create_auth_record(self, authkey, queues=[], privs=[]): 84 | """ create a authorization record. queues and privs are lists """ 85 | authkey = self.normalize(authkey) 86 | avkey = "AUTHKEY:%s" % authkey 87 | avkey = self.normalize(avkey) 88 | authrecord = {'queues': queues, 'privs':privs} 89 | 90 | res = yield self.redis.set(avkey, cyclone.escape.json_encode(authrecord)) 91 | defer.returnValue(res) 92 | 93 | @defer.inlineCallbacks 94 | def queue_add(self, queue, value, ttl=None): 95 | queue, value = self.normalize(queue), self.normalize(value) 96 | 97 | uuid = yield self.redis.incr("%s:UUID" % queue) 98 | key = '%s:%d' % (queue, uuid) 99 | res = yield self.redis.set(key, value) 100 | if ttl is not None: 101 | res = yield self.redis.expire(key, ttl) 102 | internal_queue_name = QUEUE_NAME % self.normalize(queue) 103 | if uuid == 1: # TODO: use ismember() 104 | # either by checking uuid or by ismember, this is where you must know if the queue is a new one. 105 | # add to queues set 106 | res = yield self.redis.sadd(self.QUEUESET, queue) 107 | 108 | ckey = '%s:%s' % (QUEUE_STATUS, queue) 109 | res = yield self.redis.set(ckey, self.STARTQUEUE) 110 | 111 | res = yield self.redis.lpush(internal_queue_name, key) 112 | defer.returnValue(key) 113 | 114 | @defer.inlineCallbacks 115 | def queue_get(self, queue, softget=False): 116 | """ 117 | GET can be either soft or hard. 118 | SOFTGET means that the object is not POP'ed from its queue list. It only gets a refcounter which is incremente for each GET 119 | HARDGET is the default behaviour. It POPs the key from its queue list. 120 | NoSQL dbs as mongodb would have other ways to deal with it. May be an interesting port. 121 | The reasoning behing refcounters is that they are important in some job scheduler patterns. 122 | To really cleanup the queue, one would have to issue a DEL after a hard GET. 123 | """ 124 | policy = None 125 | queue = self.normalize(queue) 126 | lkey = QUEUE_NAME % self.normalize(queue) 127 | if softget == False: 128 | okey = yield self.redis.rpop(lkey) 129 | else: 130 | okey = yield self.redis.lindex(lkey, "-1") 131 | 132 | if okey == None: 133 | defer.returnValue((None, None)) 134 | 135 | qpkey = QUEUE_POLICY % queue 136 | (policy, val) = yield self.redis.mget([qpkey, okey.encode('utf-8')]) 137 | c=0 138 | if softget == True: 139 | c = yield self.redis.incr('%s:refcount' % okey.encode('utf-8')) 140 | 141 | defer.returnValue((policy or POLICY_BROADCAST, {'key':okey, 'value':val, 'count':c})) 142 | 143 | @defer.inlineCallbacks 144 | def queue_del(self, queue, okey): 145 | """ 146 | DELetes an element from redis (not from the queue). 147 | Its important to make sure a GET was issued before a DEL. Its a kinda hard to guess the direct object key w/o a GET tho. 148 | the return value contains the key and value, which is a del return code from Redis. > 1 success and N keys where deleted, 0 == failure 149 | """ 150 | queue, okey = self.normalize(queue), self.normalize(okey) 151 | val = yield self.redis.delete(okey) 152 | defer.returnValue({'key':okey, 'value':val}) 153 | 154 | @defer.inlineCallbacks 155 | def queue_len(self, queue): 156 | lkey = QUEUE_NAME % self.normalize(queue) 157 | ll = yield self.redis.llen(lkey) 158 | defer.returnValue(ll) 159 | 160 | @defer.inlineCallbacks 161 | def queue_all(self): 162 | sm = yield self.redis.smembers(self.QUEUESET) 163 | defer.returnValue({'queues': sm}) 164 | 165 | @defer.inlineCallbacks 166 | def queue_getdel(self, queue): 167 | policy = None 168 | queue = self.normalize(queue) 169 | lkey = QUEUE_NAME % self.normalize(queue) 170 | 171 | okey = yield self.redis.rpop(lkey) # take from queue's list 172 | if okey == None: 173 | defer.returnValue((None, False)) 174 | 175 | okey = self.normalize(okey) 176 | nkey = '%s:lock' % okey 177 | ren = yield self.redis.rename(okey, nkey) # rename key 178 | 179 | if ren == None: 180 | defer.returnValue((None,None)) 181 | 182 | qpkey = QUEUE_POLICY % queue 183 | (policy, val) = yield self.redis.mget(qpkey, nkey) 184 | delk = yield self.redis.delete(nkey) 185 | if delk == 0: 186 | defer.returnValue((None, None)) 187 | else: 188 | defer.returnValue((policy, {'key':okey, 'value':val})) 189 | 190 | @defer.inlineCallbacks 191 | def queue_policy_set(self, queue, policy): 192 | queue, policy = self.normalize(queue), self.normalize(policy) 193 | if policy in ("broadcast", "roundrobin"): 194 | policy_id = self.policies[policy] 195 | qpkey = QUEUE_POLICY % queue 196 | res = yield self.redis.set(qpkey, policy_id) 197 | defer.returnValue(res) 198 | defer.returnValue({'queue': queue, 'response': res}) 199 | else: 200 | defer.returnValue(None) 201 | 202 | @defer.inlineCallbacks 203 | def queue_policy_get(self, queue): 204 | queue = self.normalize(queue) 205 | qpkey = QUEUE_POLICY % queue 206 | val = yield self.redis.get(qpkey) 207 | name = self.inverted_policies.get(val, "unknown") 208 | defer.returnValue([val, name]) 209 | 210 | @defer.inlineCallbacks 211 | def queue_tail(self, queue, keyno=10, delete_obj=False): 212 | """ 213 | TAIL follows on GET, but returns keyno keys instead of only one key. 214 | keyno could be a LLEN function over the queue list, but it lends almost the same effect. 215 | LRANGE could too fetch the latest keys, even if there was less than keyno keys. MGET could be used too. 216 | TODO: does DELete belongs here ? 217 | returns a tuple (policy, returnvalues[]) 218 | """ 219 | policy = None 220 | queue = self.normalize(queue) 221 | lkey = QUEUE_NAME % self.normalize(queue) 222 | multivalue = [] 223 | for a in range(keyno): 224 | nk = yield self.redis.rpop(lkey) 225 | if nk != None: 226 | t = nk.encode('utf-8') 227 | else: 228 | continue 229 | 230 | if delete_obj == True: 231 | okey = self.normalize(t) 232 | t = '%s:lock' % okey 233 | ren = yield self.redis.rename(okey, t) 234 | if ren == None: continue 235 | 236 | v = yield self.redis.get(t) 237 | delk = yield self.redis.delete(t) 238 | if delk == 0: continue 239 | else: 240 | v = yield self.redis.get(t) 241 | 242 | multivalue.append({'key': okey, 'value':v.encode('utf-8')}) 243 | 244 | qpkey = QUEUE_POLICY % queue 245 | policy = yield self.redis.get(qpkey) 246 | defer.returnValue((policy or POLICY_BROADCAST, multivalue)) 247 | 248 | @defer.inlineCallbacks 249 | def queue_count_elements(self, queue): 250 | # this is necessary to evaluate how many objects still undeleted on redis. 251 | # seems like it triggers a condition which the client disconnects from redis 252 | try: 253 | lkey = '%s*' % self.normalize(queue) 254 | ll = yield self.redis.keys(lkey) 255 | defer.returnValue({"objects":len(ll)}) 256 | except Exception, e: 257 | defer.returnValue({"error":str(e)}) 258 | 259 | @defer.inlineCallbacks 260 | def queue_last_items(self, queue, count=10): 261 | """ 262 | returns a list with the last count items in the queue 263 | """ 264 | queue = self.normalize(queue) 265 | lkey = QUEUE_NAME % self.normalize(queue) 266 | multivalue = yield self.redis.lrange(lkey, 0, count-1) 267 | 268 | defer.returnValue( multivalue) 269 | 270 | @defer.inlineCallbacks 271 | def queue_changestatus(self, queue, status): 272 | """Statuses: core.STOPQUEUE/core.STARTQUEUE""" 273 | if status != self.STOPQUEUE and status != self.STARTQUEUE: 274 | defer.returnValue(None) 275 | 276 | key = '%s:%s' % (QUEUE_STATUS, queue) 277 | res = yield self.redis.set(key, status) 278 | defer.returnValue({'queue':queue, 'status':status}) 279 | 280 | @defer.inlineCallbacks 281 | def queue_status(self, queue): 282 | key = '%s:%s' % (QUEUE_STATUS, queue) 283 | res = yield self.redis.get(key) 284 | defer.returnValue({'queue':queue, 'status':res}) 285 | 286 | @defer.inlineCallbacks 287 | def queue_purge(self, queue): 288 | #TODO Must del all keys (or set expire) 289 | #it could rename the queue list, add to a deletion SET and use a task to clean it 290 | 291 | lkey = QUEUE_NAME % self.normalize(queue) 292 | res = yield self.redis.delete(lkey) 293 | defer.returnValue({'queue':queue, 'status':res}) 294 | 295 | @defer.inlineCallbacks 296 | def pubsub(self, queue_name, content): 297 | key = "%s:%s" % (queue_name, self.PUBSUB_SUFIX) 298 | r = yield self.redis.publish(key, content) 299 | 300 | 301 | 302 | @defer.inlineCallbacks 303 | def queue_block_multi_get(self, queue_list): 304 | """ 305 | waits on a list of queues, get back with the first queue that 306 | received data. 307 | this makes the redis locallity very important as if there are other 308 | instances doing the same the policy wont be respected. OTOH it makes 309 | it fast by not polling lists and waiting x seconds 310 | """ 311 | ql = [QUEUE_NAME % self.normalize(queue) for queue in queue_list] 312 | res = yield self.redis.brpop(ql) 313 | if res is not None: 314 | q = self.normalize(res[1]) 315 | qpkey = QUEUE_POLICY % q 316 | (p, v) = yield self.redis.mget([qpkey, q]) 317 | defer.returnValue((q, p, {'key':q, 'value':v})) 318 | else: 319 | defer.returnValue(None) 320 | 321 | @defer.inlineCallbacks 322 | def multi_queue_by_status(self, queue_list, filter_by=None): 323 | if filter_by is None: filter_by = self.STARTQUEUE 324 | ql = ["%s:%s" % (QUEUE_STATUS, self.normalize(queue)) for queue in queue_list] 325 | res = yield self.redis.mget(ql) 326 | qs = [True if r != filter_by else False for r in res] 327 | r = itertools.compress(ql, qs) 328 | defer.returnValue(list(r)) 329 | -------------------------------------------------------------------------------- /src/restmq/static/js/bootstrap.min.js: -------------------------------------------------------------------------------- 1 | !function(a){a(function(){"use strict",a.support.transition=function(){var b=document.body||document.documentElement,c=b.style,d=c.transition!==undefined||c.WebkitTransition!==undefined||c.MozTransition!==undefined||c.MsTransition!==undefined||c.OTransition!==undefined;return d&&{end:function(){var b="TransitionEnd";return a.browser.webkit?b="webkitTransitionEnd":a.browser.mozilla?b="transitionend":a.browser.opera&&(b="oTransitionEnd"),b}()}}()})}(window.jQuery),!function(a){"use strict";var b='[data-dismiss="alert"]',c=function(c){a(c).on("click",b,this.close)};c.prototype={constructor:c,close:function(b){function f(){e.trigger("closed").remove()}var c=a(this),d=c.attr("data-target"),e;d||(d=c.attr("href"),d=d&&d.replace(/.*(?=#[^\s]*$)/,"")),e=a(d),e.trigger("close"),b&&b.preventDefault(),e.length||(e=c.hasClass("alert")?c:c.parent()),e.trigger("close").removeClass("in"),a.support.transition&&e.hasClass("fade")?e.on(a.support.transition.end,f):f()}},a.fn.alert=function(b){return this.each(function(){var d=a(this),e=d.data("alert");e||d.data("alert",e=new c(this)),typeof b=="string"&&e[b].call(d)})},a.fn.alert.Constructor=c,a(function(){a("body").on("click.alert.data-api",b,c.prototype.close)})}(window.jQuery),!function(a){"use strict";var b=function(b,c){this.$element=a(b),this.options=a.extend({},a.fn.button.defaults,c)};b.prototype={constructor:b,setState:function(a){var b="disabled",c=this.$element,d=c.data(),e=c.is("input")?"val":"html";a+="Text",d.resetText||c.data("resetText",c[e]()),c[e](d[a]||this.options[a]),setTimeout(function(){a=="loadingText"?c.addClass(b).attr(b,b):c.removeClass(b).removeAttr(b)},0)},toggle:function(){var a=this.$element.parent('[data-toggle="buttons-radio"]');a&&a.find(".active").removeClass("active"),this.$element.toggleClass("active")}},a.fn.button=function(c){return this.each(function(){var d=a(this),e=d.data("button"),f=typeof c=="object"&&c;e||d.data("button",e=new b(this,f)),c=="toggle"?e.toggle():c&&e.setState(c)})},a.fn.button.defaults={loadingText:"loading..."},a.fn.button.Constructor=b,a(function(){a("body").on("click.button.data-api","[data-toggle^=button]",function(b){a(b.currentTarget).button("toggle")})})}(window.jQuery),!function(a){"use strict";var b=function(b,c){this.$element=a(b),this.options=a.extend({},a.fn.carousel.defaults,c),this.options.slide&&this.slide(this.options.slide)};b.prototype={cycle:function(){return this.interval=setInterval(a.proxy(this.next,this),this.options.interval),this},to:function(b){var c=this.$element.find(".active"),d=c.parent().children(),e=d.index(c),f=this;if(b>d.length-1||b<0)return;return this.sliding?this.$element.one("slid",function(){f.to(b)}):e==b?this.pause().cycle():this.slide(b>e?"next":"prev",a(d[b]))},pause:function(){return clearInterval(this.interval),this.interval=null,this},next:function(){if(this.sliding)return;return this.slide("next")},prev:function(){if(this.sliding)return;return this.slide("prev")},slide:function(b,c){var d=this.$element.find(".active"),e=c||d[b](),f=this.interval,g=b=="next"?"left":"right",h=b=="next"?"first":"last",i=this;if(!e.length)return;return this.sliding=!0,f&&this.pause(),e=e.length?e:this.$element.find(".item")[h](),!a.support.transition&&this.$element.hasClass("slide")?(this.$element.trigger("slide"),d.removeClass("active"),e.addClass("active"),this.sliding=!1,this.$element.trigger("slid")):(e.addClass(b),e[0].offsetWidth,d.addClass(g),e.addClass(g),this.$element.trigger("slide"),this.$element.one(a.support.transition.end,function(){e.removeClass([b,g].join(" ")).addClass("active"),d.removeClass(["active",g].join(" ")),i.sliding=!1,setTimeout(function(){i.$element.trigger("slid")},0)})),f&&this.cycle(),this}},a.fn.carousel=function(c){return this.each(function(){var d=a(this),e=d.data("carousel"),f=typeof c=="object"&&c;e||d.data("carousel",e=new b(this,f)),typeof c=="number"?e.to(c):typeof c=="string"||(c=f.slide)?e[c]():e.cycle()})},a.fn.carousel.defaults={interval:5e3},a.fn.carousel.Constructor=b,a(function(){a("body").on("click.carousel.data-api","[data-slide]",function(b){var c=a(this),d,e=a(c.attr("data-target")||(d=c.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,"")),f=!e.data("modal")&&a.extend({},e.data(),c.data());e.carousel(f),b.preventDefault()})})}(window.jQuery),!function(a){"use strict";var b=function(b,c){this.$element=a(b),this.options=a.extend({},a.fn.collapse.defaults,c),this.options.parent&&(this.$parent=a(this.options.parent)),this.options.toggle&&this.toggle()};b.prototype={constructor:b,dimension:function(){var a=this.$element.hasClass("width");return a?"width":"height"},show:function(){var b=this.dimension(),c=a.camelCase(["scroll",b].join("-")),d=this.$parent&&this.$parent.find(".in"),e;d&&d.length&&(e=d.data("collapse"),d.collapse("hide"),e||d.data("collapse",null)),this.$element[b](0),this.transition("addClass","show","shown"),this.$element[b](this.$element[0][c])},hide:function(){var a=this.dimension();this.reset(this.$element[a]()),this.transition("removeClass","hide","hidden"),this.$element[a](0)},reset:function(a){var b=this.dimension();this.$element.removeClass("collapse")[b](a||"auto")[0].offsetWidth,this.$element.addClass("collapse")},transition:function(b,c,d){var e=this,f=function(){c=="show"&&e.reset(),e.$element.trigger(d)};this.$element.trigger(c)[b]("in"),a.support.transition&&this.$element.hasClass("collapse")?this.$element.one(a.support.transition.end,f):f()},toggle:function(){this[this.$element.hasClass("in")?"hide":"show"]()}},a.fn.collapse=function(c){return this.each(function(){var d=a(this),e=d.data("collapse"),f=typeof c=="object"&&c;e||d.data("collapse",e=new b(this,f)),typeof c=="string"&&e[c]()})},a.fn.collapse.defaults={toggle:!0},a.fn.collapse.Constructor=b,a(function(){a("body").on("click.collapse.data-api","[data-toggle=collapse]",function(b){var c=a(this),d,e=c.attr("data-target")||b.preventDefault()||(d=c.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,""),f=a(e).data("collapse")?"toggle":c.data();a(e).collapse(f)})})}(window.jQuery),!function(a){function d(){a(b).parent().removeClass("open")}"use strict";var b='[data-toggle="dropdown"]',c=function(b){var c=a(b).on("click.dropdown.data-api",this.toggle);a("html").on("click.dropdown.data-api",function(){c.parent().removeClass("open")})};c.prototype={constructor:c,toggle:function(b){var c=a(this),e=c.attr("data-target"),f,g;return e||(e=c.attr("href"),e=e&&e.replace(/.*(?=#[^\s]*$)/,"")),f=a(e),f.length||(f=c.parent()),g=f.hasClass("open"),d(),!g&&f.toggleClass("open"),!1}},a.fn.dropdown=function(b){return this.each(function(){var d=a(this),e=d.data("dropdown");e||d.data("dropdown",e=new c(this)),typeof b=="string"&&e[b].call(d)})},a.fn.dropdown.Constructor=c,a(function(){a("html").on("click.dropdown.data-api",d),a("body").on("click.dropdown.data-api",b,c.prototype.toggle)})}(window.jQuery),!function(a){function c(){var b=this,c=setTimeout(function(){b.$element.off(a.support.transition.end),d.call(b)},500);this.$element.one(a.support.transition.end,function(){clearTimeout(c),d.call(b)})}function d(a){this.$element.hide().trigger("hidden"),e.call(this)}function e(b){var c=this,d=this.$element.hasClass("fade")?"fade":"";if(this.isShown&&this.options.backdrop){var e=a.support.transition&&d;this.$backdrop=a('
').appendTo(document.body),this.options.backdrop!="static"&&this.$backdrop.click(a.proxy(this.hide,this)),e&&this.$backdrop[0].offsetWidth,this.$backdrop.addClass("in"),e?this.$backdrop.one(a.support.transition.end,b):b()}else!this.isShown&&this.$backdrop?(this.$backdrop.removeClass("in"),a.support.transition&&this.$element.hasClass("fade")?this.$backdrop.one(a.support.transition.end,a.proxy(f,this)):f.call(this)):b&&b()}function f(){this.$backdrop.remove(),this.$backdrop=null}function g(){var b=this;this.isShown&&this.options.keyboard?a(document).on("keyup.dismiss.modal",function(a){a.which==27&&b.hide()}):this.isShown||a(document).off("keyup.dismiss.modal")}"use strict";var b=function(b,c){this.options=c,this.$element=a(b).delegate('[data-dismiss="modal"]',"click.dismiss.modal",a.proxy(this.hide,this))};b.prototype={constructor:b,toggle:function(){return this[this.isShown?"hide":"show"]()},show:function(){var b=this;if(this.isShown)return;a("body").addClass("modal-open"),this.isShown=!0,this.$element.trigger("show"),g.call(this),e.call(this,function(){var c=a.support.transition&&b.$element.hasClass("fade");!b.$element.parent().length&&b.$element.appendTo(document.body),b.$element.show(),c&&b.$element[0].offsetWidth,b.$element.addClass("in"),c?b.$element.one(a.support.transition.end,function(){b.$element.trigger("shown")}):b.$element.trigger("shown")})},hide:function(b){b&&b.preventDefault();if(!this.isShown)return;var e=this;this.isShown=!1,a("body").removeClass("modal-open"),g.call(this),this.$element.trigger("hide").removeClass("in"),a.support.transition&&this.$element.hasClass("fade")?c.call(this):d.call(this)}},a.fn.modal=function(c){return this.each(function(){var d=a(this),e=d.data("modal"),f=a.extend({},a.fn.modal.defaults,d.data(),typeof c=="object"&&c);e||d.data("modal",e=new b(this,f)),typeof c=="string"?e[c]():f.show&&e.show()})},a.fn.modal.defaults={backdrop:!0,keyboard:!0,show:!0},a.fn.modal.Constructor=b,a(function(){a("body").on("click.modal.data-api",'[data-toggle="modal"]',function(b){var c=a(this),d,e=a(c.attr("data-target")||(d=c.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,"")),f=e.data("modal")?"toggle":a.extend({},e.data(),c.data());b.preventDefault(),e.modal(f)})})}(window.jQuery),!function(a){"use strict";var b=function(a,b){this.init("tooltip",a,b)};b.prototype={constructor:b,init:function(b,c,d){var e,f;this.type=b,this.$element=a(c),this.options=this.getOptions(d),this.enabled=!0,this.options.trigger!="manual"&&(e=this.options.trigger=="hover"?"mouseenter":"focus",f=this.options.trigger=="hover"?"mouseleave":"blur",this.$element.on(e,this.options.selector,a.proxy(this.enter,this)),this.$element.on(f,this.options.selector,a.proxy(this.leave,this))),this.options.selector?this._options=a.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},getOptions:function(b){return b=a.extend({},a.fn[this.type].defaults,b,this.$element.data()),b.delay&&typeof b.delay=="number"&&(b.delay={show:b.delay,hide:b.delay}),b},enter:function(b){var c=a(b.currentTarget)[this.type](this._options).data(this.type);!c.options.delay||!c.options.delay.show?c.show():(c.hoverState="in",setTimeout(function(){c.hoverState=="in"&&c.show()},c.options.delay.show))},leave:function(b){var c=a(b.currentTarget)[this.type](this._options).data(this.type);!c.options.delay||!c.options.delay.hide?c.hide():(c.hoverState="out",setTimeout(function(){c.hoverState=="out"&&c.hide()},c.options.delay.hide))},show:function(){var a,b,c,d,e,f,g;if(this.hasContent()&&this.enabled){a=this.tip(),this.setContent(),this.options.animation&&a.addClass("fade"),f=typeof this.options.placement=="function"?this.options.placement.call(this,a[0],this.$element[0]):this.options.placement,b=/in/.test(f),a.remove().css({top:0,left:0,display:"block"}).appendTo(b?this.$element:document.body),c=this.getPosition(b),d=a[0].offsetWidth,e=a[0].offsetHeight;switch(b?f.split(" ")[1]:f){case"bottom":g={top:c.top+c.height,left:c.left+c.width/2-d/2};break;case"top":g={top:c.top-e,left:c.left+c.width/2-d/2};break;case"left":g={top:c.top+c.height/2-e/2,left:c.left-d};break;case"right":g={top:c.top+c.height/2-e/2,left:c.left+c.width}}a.css(g).addClass(f).addClass("in")}},setContent:function(){var a=this.tip();a.find(".tooltip-inner").html(this.getTitle()),a.removeClass("fade in top bottom left right")},hide:function(){function d(){var b=setTimeout(function(){c.off(a.support.transition.end).remove()},500);c.one(a.support.transition.end,function(){clearTimeout(b),c.remove()})}var b=this,c=this.tip();c.removeClass("in"),a.support.transition&&this.$tip.hasClass("fade")?d():c.remove()},fixTitle:function(){var a=this.$element;(a.attr("title")||typeof a.attr("data-original-title")!="string")&&a.attr("data-original-title",a.attr("title")||"").removeAttr("title")},hasContent:function(){return this.getTitle()},getPosition:function(b){return a.extend({},b?{top:0,left:0}:this.$element.offset(),{width:this.$element[0].offsetWidth,height:this.$element[0].offsetHeight})},getTitle:function(){var a,b=this.$element,c=this.options;return a=b.attr("data-original-title")||(typeof c.title=="function"?c.title.call(b[0]):c.title),a=a.toString().replace(/(^\s*|\s*$)/,""),a},tip:function(){return this.$tip=this.$tip||a(this.options.template)},validate:function(){this.$element[0].parentNode||(this.hide(),this.$element=null,this.options=null)},enable:function(){this.enabled=!0},disable:function(){this.enabled=!1},toggleEnabled:function(){this.enabled=!this.enabled},toggle:function(){this[this.tip().hasClass("in")?"hide":"show"]()}},a.fn.tooltip=function(c){return this.each(function(){var d=a(this),e=d.data("tooltip"),f=typeof c=="object"&&c;e||d.data("tooltip",e=new b(this,f)),typeof c=="string"&&e[c]()})},a.fn.tooltip.Constructor=b,a.fn.tooltip.defaults={animation:!0,delay:0,selector:!1,placement:"top",trigger:"hover",title:"",template:'
'}}(window.jQuery),!function(a){"use strict";var b=function(a,b){this.init("popover",a,b)};b.prototype=a.extend({},a.fn.tooltip.Constructor.prototype,{constructor:b,setContent:function(){var b=this.tip(),c=this.getTitle(),d=this.getContent();b.find(".popover-title")[a.type(c)=="object"?"append":"html"](c),b.find(".popover-content > *")[a.type(d)=="object"?"append":"html"](d),b.removeClass("fade top bottom left right in")},hasContent:function(){return this.getTitle()||this.getContent()},getContent:function(){var a,b=this.$element,c=this.options;return a=b.attr("data-content")||(typeof c.content=="function"?c.content.call(b[0]):c.content),a=a.toString().replace(/(^\s*|\s*$)/,""),a},tip:function(){return this.$tip||(this.$tip=a(this.options.template)),this.$tip}}),a.fn.popover=function(c){return this.each(function(){var d=a(this),e=d.data("popover"),f=typeof c=="object"&&c;e||d.data("popover",e=new b(this,f)),typeof c=="string"&&e[c]()})},a.fn.popover.Constructor=b,a.fn.popover.defaults=a.extend({},a.fn.tooltip.defaults,{placement:"right",content:"",template:'

'})}(window.jQuery),!function(a){function b(b,c){var d=a.proxy(this.process,this),e=a(b).is("body")?a(window):a(b),f;this.options=a.extend({},a.fn.scrollspy.defaults,c),this.$scrollElement=e.on("scroll.scroll.data-api",d),this.selector=(this.options.target||(f=a(b).attr("href"))&&f.replace(/.*(?=#[^\s]+$)/,"")||"")+" .nav li > a",this.$body=a("body").on("click.scroll.data-api",this.selector,d),this.refresh(),this.process()}"use strict",b.prototype={constructor:b,refresh:function(){this.targets=this.$body.find(this.selector).map(function(){var b=a(this).attr("href");return/^#\w/.test(b)&&a(b).length?b:null}),this.offsets=a.map(this.targets,function(b){return a(b).position().top})},process:function(){var a=this.$scrollElement.scrollTop()+this.options.offset,b=this.offsets,c=this.targets,d=this.activeTarget,e;for(e=b.length;e--;)d!=c[e]&&a>=b[e]&&(!b[e+1]||a<=b[e+1])&&this.activate(c[e])},activate:function(a){var b;this.activeTarget=a,this.$body.find(this.selector).parent(".active").removeClass("active"),b=this.$body.find(this.selector+'[href="'+a+'"]').parent("li").addClass("active"),b.parent(".dropdown-menu")&&b.closest("li.dropdown").addClass("active")}},a.fn.scrollspy=function(c){return this.each(function(){var d=a(this),e=d.data("scrollspy"),f=typeof c=="object"&&c;e||d.data("scrollspy",e=new b(this,f)),typeof c=="string"&&e[c]()})},a.fn.scrollspy.Constructor=b,a.fn.scrollspy.defaults={offset:10},a(function(){a('[data-spy="scroll"]').each(function(){var b=a(this);b.scrollspy(b.data())})})}(window.jQuery),!function(a){"use strict";var b=function(b){this.element=a(b)};b.prototype={constructor:b,show:function(){var b=this.element,c=b.closest("ul:not(.dropdown-menu)"),d=b.attr("data-target"),e,f;d||(d=b.attr("href"),d=d&&d.replace(/.*(?=#[^\s]*$)/,""));if(b.parent("li").hasClass("active"))return;e=c.find(".active a").last()[0],b.trigger({type:"show",relatedTarget:e}),f=a(d),this.activate(b.parent("li"),c),this.activate(f,f.parent(),function(){b.trigger({type:"shown",relatedTarget:e})})},activate:function(b,c,d){function g(){e.removeClass("active").find("> .dropdown-menu > .active").removeClass("active"),b.addClass("active"),f?(b[0].offsetWidth,b.addClass("in")):b.removeClass("fade"),b.parent(".dropdown-menu")&&b.closest("li.dropdown").addClass("active"),d&&d()}var e=c.find("> .active"),f=d&&a.support.transition&&e.hasClass("fade");f?e.one(a.support.transition.end,g):g(),e.removeClass("in")}},a.fn.tab=function(c){return this.each(function(){var d=a(this),e=d.data("tab");e||d.data("tab",e=new b(this)),typeof c=="string"&&e[c]()})},a.fn.tab.Constructor=b,a(function(){a("body").on("click.tab.data-api",'[data-toggle="tab"], [data-toggle="pill"]',function(b){b.preventDefault(),a(this).tab("show")})})}(window.jQuery),!function(a){"use strict";var b=function(b,c){this.$element=a(b),this.options=a.extend({},a.fn.typeahead.defaults,c),this.matcher=this.options.matcher||this.matcher,this.sorter=this.options.sorter||this.sorter,this.highlighter=this.options.highlighter||this.highlighter,this.$menu=a(this.options.menu).appendTo("body"),this.source=this.options.source,this.shown=!1,this.listen()};b.prototype={constructor:b,select:function(){var a=this.$menu.find(".active").attr("data-value");return this.$element.val(a),this.hide()},show:function(){var b=a.extend({},this.$element.offset(),{height:this.$element[0].offsetHeight});return this.$menu.css({top:b.top+b.height,left:b.left}),this.$menu.show(),this.shown=!0,this},hide:function(){return this.$menu.hide(),this.shown=!1,this},lookup:function(b){var c=this,d,e;return this.query=this.$element.val(),this.query?(d=a.grep(this.source,function(a){if(c.matcher(a))return a}),d=this.sorter(d),d.length?this.render(d.slice(0,this.options.items)).show():this.shown?this.hide():this):this.shown?this.hide():this},matcher:function(a){return~a.toLowerCase().indexOf(this.query.toLowerCase())},sorter:function(a){var b=[],c=[],d=[],e;while(e=a.shift())e.toLowerCase().indexOf(this.query.toLowerCase())?~e.indexOf(this.query)?c.push(e):d.push(e):b.push(e);return b.concat(c,d)},highlighter:function(a){return a.replace(new RegExp("("+this.query+")","ig"),function(a,b){return""+b+""})},render:function(b){var c=this;return b=a(b).map(function(b,d){return b=a(c.options.item).attr("data-value",d),b.find("a").html(c.highlighter(d)),b[0]}),b.first().addClass("active"),this.$menu.html(b),this},next:function(b){var c=this.$menu.find(".active").removeClass("active"),d=c.next();d.length||(d=a(this.$menu.find("li")[0])),d.addClass("active")},prev:function(a){var b=this.$menu.find(".active").removeClass("active"),c=b.prev();c.length||(c=this.$menu.find("li").last()),c.addClass("active")},listen:function(){this.$element.on("blur",a.proxy(this.blur,this)).on("keypress",a.proxy(this.keypress,this)).on("keyup",a.proxy(this.keyup,this)),(a.browser.webkit||a.browser.msie)&&this.$element.on("keydown",a.proxy(this.keypress,this)),this.$menu.on("click",a.proxy(this.click,this)).on("mouseenter","li",a.proxy(this.mouseenter,this))},keyup:function(a){a.stopPropagation(),a.preventDefault();switch(a.keyCode){case 40:case 38:break;case 9:case 13:if(!this.shown)return;this.select();break;case 27:this.hide();break;default:this.lookup()}},keypress:function(a){a.stopPropagation();if(!this.shown)return;switch(a.keyCode){case 9:case 13:case 27:a.preventDefault();break;case 38:a.preventDefault(),this.prev();break;case 40:a.preventDefault(),this.next()}},blur:function(a){var b=this;a.stopPropagation(),a.preventDefault(),setTimeout(function(){b.hide()},150)},click:function(a){a.stopPropagation(),a.preventDefault(),this.select()},mouseenter:function(b){this.$menu.find(".active").removeClass("active"),a(b.currentTarget).addClass("active")}},a.fn.typeahead=function(c){return this.each(function(){var d=a(this),e=d.data("typeahead"),f=typeof c=="object"&&c;e||d.data("typeahead",e=new b(this,f)),typeof c=="string"&&e[c]()})},a.fn.typeahead.defaults={source:[],items:8,menu:'
    ',item:'
    • '},a.fn.typeahead.Constructor=b,a(function(){a("body").on("focus.typeahead.data-api",'[data-provide="typeahead"]',function(b){var c=a(this);if(c.data("typeahead"))return;b.preventDefault(),c.typeahead(c.data())})})}(window.jQuery); -------------------------------------------------------------------------------- /src/restmq/web.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from os.path import dirname, abspath, join 4 | import types 5 | import base64 6 | import hashlib 7 | import os.path 8 | import functools 9 | import cyclone.web 10 | import cyclone.redis 11 | import cyclone.escape 12 | import cyclone.websocket 13 | 14 | from collections import defaultdict 15 | from ConfigParser import ConfigParser 16 | 17 | from twisted.python import log 18 | from twisted.internet import task, defer, reactor 19 | 20 | import pkg_resources as pkg 21 | 22 | from restmq import core 23 | from restmq import dispatch 24 | 25 | class InvalidAddress(Exception): 26 | pass 27 | 28 | class InvalidPassword(Exception): 29 | pass 30 | 31 | def authorize(category): 32 | def decorator(method): 33 | @functools.wraps(method) 34 | def wrapper(self, *args, **kwargs): 35 | try: 36 | self.settings.acl.apply(self, category) 37 | except InvalidAddress: 38 | raise cyclone.web.HTTPError(401) 39 | except InvalidPassword: 40 | raise cyclone.web.HTTPAuthenticationRequired("Basic", "RestMQ Restricted Access") 41 | else: 42 | return method(self, *args, **kwargs) 43 | return wrapper 44 | return decorator 45 | 46 | 47 | class CustomHandler(object): 48 | def __init__(self, handler, json_callback=None): 49 | self.handler = handler 50 | self.json_callback = json_callback 51 | 52 | def write(self, text): 53 | if not isinstance(text, types.StringType): 54 | text = cyclone.escape.json_encode(text) 55 | 56 | if isinstance(self.handler, cyclone.websocket.WebSocketHandler): 57 | self.handler.sendMessage(text) 58 | else: 59 | if self.json_callback: 60 | self.handler.write("%s(%s);\r\n" % (self.json_callback, text)) 61 | else: 62 | self.handler.write(text+"\r\n") 63 | 64 | def flush(self): 65 | self.handler.flush() 66 | 67 | def finish(self, text=None): 68 | if buffer: 69 | self.write(text) 70 | self.handler.finish() 71 | 72 | 73 | class IndexHandler(cyclone.web.RequestHandler): 74 | @authorize("rest_consumer") 75 | @defer.inlineCallbacks 76 | def get(self): 77 | try: 78 | queue = self.get_argument("queue", None) 79 | callback = self.get_argument("callback", None) 80 | if queue is None: 81 | self.redirect("/static/index.html") 82 | defer.returnValue(None) 83 | except Exception, e: 84 | print e 85 | 86 | try: 87 | policy, value = yield self.settings.oper.queue_get(queue) 88 | except Exception, e: 89 | log.msg("ERROR: oper.queue_get('%s') failed: %s" % (queue, e)) 90 | cyclone.web.HTTPError(503) 91 | 92 | if value: 93 | CustomHandler(self, callback).finish(value) 94 | else: 95 | raise cyclone.web.HTTPError(404) 96 | 97 | @authorize("rest_producer") 98 | @defer.inlineCallbacks 99 | def post(self): 100 | queue = self.get_argument("queue") 101 | msg = self.get_argument("msg", None) 102 | value = self.get_argument("value", None) 103 | ttl = self.get_argument("ttl", None) 104 | 105 | if msg is None and value is None: 106 | raise cyclone.web.HTTPError(400) 107 | callback = self.get_argument("callback", None) 108 | 109 | try: 110 | result = yield self.settings.oper.queue_add(queue, value, ttl=ttl) 111 | except Exception, e: 112 | log.msg("ERROR: oper.queue_add('%s', '%s') failed: %s" % (queue, value, e)) 113 | raise cyclone.web.HTTPError(503) 114 | 115 | if result: 116 | self.settings.comet.queue.put(queue) 117 | CustomHandler(self, callback).finish(result) 118 | else: 119 | raise cyclone.web.HTTPError(400) 120 | 121 | 122 | class RestQueueHandler(cyclone.web.RequestHandler): 123 | """ 124 | RestQueueHandler applies HTTP Methods to a given queue. 125 | GET /q/queuename gets an object out of the queue. 126 | POST /q/queuename inserts an object in the queue (I know, it could be PUT). The payload comes in the parameter body 127 | DELETE method purge and delete the queue. It will close all comet connections. 128 | """ 129 | @authorize("rest_consumer") 130 | @defer.inlineCallbacks 131 | def get(self, queue): 132 | callback = self.get_argument("callback", None) 133 | 134 | if queue: 135 | try: 136 | policy, value = yield self.settings.oper.queue_get(queue) 137 | except Exception, e: 138 | log.msg("ERROR: oper.queue_get('%s') failed: %s" % (queue, e)) 139 | raise cyclone.web.HTTPError(503) 140 | 141 | if value is None: raise cyclone.web.HTTPError(204) 142 | CustomHandler(self, callback).finish(value) 143 | else: 144 | try: 145 | allqueues = yield self.settings.oper.queue_all() 146 | except Exception, e: 147 | log.msg("ERROR: oper.queue_all() failed: %s" % e) 148 | raise cyclone.web.HTTPError(503) 149 | 150 | self.render("list_queues.html", route="q", extended_route="REST", allqueues=allqueues["queues"]) 151 | 152 | @authorize("rest_producer") 153 | @defer.inlineCallbacks 154 | def post(self, queue): 155 | msg = self.get_argument("msg", None) 156 | value = self.get_argument("value", None) 157 | ttl = self.get_argument("ttl", None) 158 | if msg is None and value is None: 159 | raise cyclone.web.HTTPError(400) 160 | callback = self.get_argument("callback", None) 161 | 162 | try: 163 | result = yield self.settings.oper.queue_add(queue, msg or value, ttl=ttl) 164 | except Exception, e: 165 | log.msg("ERROR: oper.queue_add('%s', '%s') failed: %s" % (queue, msg or value, e)) 166 | raise cyclone.web.HTTPError(503) 167 | 168 | if result: 169 | self.settings.comet.queue.put(queue) 170 | CustomHandler(self, callback).finish(result) 171 | else: 172 | raise cyclone.web.HTTPError(400) 173 | 174 | @authorize("rest_producer") 175 | @defer.inlineCallbacks 176 | def delete(self, queue): 177 | callback = self.get_argument("callback", None) 178 | clients = self.settings.comet.presence.get(queue, []) 179 | 180 | for conn in clients: 181 | try: 182 | conn.finish() 183 | except Exception, e: 184 | log.msg("ERROR: cannot close client connection: %s = %s" % (conn, e)) 185 | 186 | try: 187 | result = yield self.settings.oper.queue_purge(queue) 188 | except Exception, e: 189 | log.msg("ERROR: oper.queue_purge('%s') failed: %s" % (queue, e)) 190 | raise cyclone.web.HTTPError(503) 191 | 192 | CustomHandler(self, callback).finish(result) 193 | 194 | 195 | class QueueHandler(cyclone.web.RequestHandler): 196 | """ QueueHandler deals with the json protocol""" 197 | 198 | def get(self): 199 | self.redirect("/static/help.html") 200 | 201 | @authorize("rest_producer") 202 | @defer.inlineCallbacks 203 | def post(self): 204 | msg = self.get_argument("msg", None) 205 | body = self.get_argument("body", None) 206 | 207 | if msg is None and body is None: 208 | raise cyclone.web.HTTPError(400) 209 | 210 | try: 211 | jsonbody = cyclone.escape.json_decode(msg or body) 212 | cmd = jsonbody["cmd"] 213 | assert cmd 214 | except: 215 | raise cyclone.web.HTTPError(400, "Malformed JSON. Invalid format.") 216 | 217 | try: 218 | result = yield dispatch.CommandDispatch(self.settings.oper).execute(cmd, jsonbody) 219 | except Exception, e: 220 | log.msg("ERROR: CommandDispatch/oper.%s('%s') failed: %s" % (cmd, jsonbody, e)) 221 | raise cyclone.web.HTTPError(503) 222 | 223 | if result: 224 | self.finish(result) 225 | else: 226 | self.finish(cyclone.escape.json_encode({"error":"null resultset"})) 227 | 228 | 229 | class CometQueueHandler(cyclone.web.RequestHandler): 230 | """ 231 | CometQueueHandler is a permanent consumer for objects in a queue. 232 | it must only feed new objects to a permanent http connection 233 | deletion is not handled here for now. 234 | As each queue object has its own key, it can be done thru /queue interface 235 | """ 236 | def _on_disconnect(self, why, handler, queue_name): 237 | try: 238 | self.settings.comet.presence[queue_name].remove(handler) 239 | if not len(self.settings.comet.presence[queue_name]): 240 | self.settings.comet.presence.pop(queue_name) 241 | except: 242 | pass 243 | 244 | @authorize("comet_consumer") 245 | @cyclone.web.asynchronous 246 | def get(self, queue): 247 | """ 248 | this method is meant to build light http consumers emulating a subscription 249 | simple test: point the browser to /c/test 250 | execute python engine.py -p, check the browser to see if the object appears, 251 | then execute engine.py -c again, to make another object appear in the browser. 252 | Not it deletes objects from redis. To change it, set getdel to False on CometDispatcher 253 | """ 254 | 255 | self.set_header("Content-Type", "text/plain") 256 | callback = self.get_argument("callback", None) 257 | 258 | handler = CustomHandler(self, callback) 259 | 260 | try: 261 | queue_name = queue.encode("utf-8") 262 | except: 263 | raise cyclone.web.HTTPError(400, "Invalid Queue Name") 264 | 265 | self.settings.comet.presence[queue_name].append(handler) 266 | self.notifyFinish().addCallback(self._on_disconnect, handler, queue_name) 267 | 268 | 269 | class PolicyQueueHandler(cyclone.web.RequestHandler): 270 | @authorize("rest_consumer") 271 | @defer.inlineCallbacks 272 | def get(self, queue): 273 | callback = self.get_argument("callback", None) 274 | 275 | try: 276 | policy, policy_name = yield self.settings.oper.queue_policy_get(queue) 277 | r = {'queue':queue, 'value': policy_name} 278 | CustomHandler(self, callback).finish(r) 279 | except Exception, e: 280 | log.msg("ERROR: oper.queue_policy_get('%s') failed: %s" % (queue, e)) 281 | raise cyclone.web.HTTPError(503) 282 | 283 | @authorize("rest_producer") 284 | @defer.inlineCallbacks 285 | def post(self, queue): 286 | policy = self.get_argument("policy") 287 | callback = self.get_argument("callback", None) 288 | 289 | try: 290 | result = yield self.settings.oper.queue_policy_set(queue, policy) 291 | except Exception, e: 292 | log.msg("ERROR: oper.queue_policy_set('%s', '%s') failed: %s" % (queue, policy, e)) 293 | raise cyclone.web.HTTPError(503) 294 | 295 | CustomHandler(self, callback).finish(result) 296 | 297 | 298 | class JobQueueInfoHandler(cyclone.web.RequestHandler): 299 | @defer.inlineCallbacks 300 | def get(self, queue): 301 | try: 302 | jobs = yield self.settings.oper.queue_last_items(queue) 303 | job_count = yield self.settings.oper.queue_len(queue) 304 | queue_obj_count = yield self.settings.oper.queue_count_elements(queue) 305 | except Exception, e: 306 | log.msg("ERROR: Cannot get JOB data: queue=%s, %s" % (queue, e)) 307 | raise cyclone.web.HTTPError(503) 308 | 309 | self.render("jobs.html", queue=queue, jobs=jobs, job_count=job_count, queue_size=queue_obj_count) 310 | 311 | 312 | class StatusHandler(cyclone.web.RequestHandler): 313 | @authorize("rest_consumer") 314 | @defer.inlineCallbacks 315 | def get(self, queue): 316 | self.set_header("Content-Type", "application/json") 317 | jsoncallback = self.get_argument("callback", None) 318 | res = {} 319 | 320 | if queue is None or len(queue) < 1: 321 | try: 322 | allqueues = yield self.settings.oper.queue_all() 323 | except Exception, e: 324 | log.msg("ERROR: oper.queue_all() failed: %s" % e) 325 | raise cyclone.web.HTTPError(503) 326 | ql = list(allqueues["queues"]) 327 | res["queues"]={} 328 | for q in ql: 329 | qn = str(q) 330 | res["queues"][qn]={} 331 | res["queues"][qn]["name"]=qn 332 | res["queues"][qn]['len'] = yield self.settings.oper.queue_len(q) 333 | st = yield self.settings.oper.queue_status(q) 334 | res["queues"][qn]['status'] = st["status"] if st['status'] else "" 335 | pl, pl_name = yield self.settings.oper.queue_policy_get(q) 336 | res["queues"][qn]['policy'] = pl if pl else "" 337 | 338 | res['redis'] = repr(self.settings.db) 339 | res['count'] = len(ql) 340 | if jsoncallback is not None: res = "%s(%s)" % (jsoncallback, res) 341 | self.finish(res) 342 | 343 | else: 344 | try: 345 | qlen = yield self.settings.oper.queue_len(queue) 346 | except Exception, e: 347 | log.msg("ERROR: oper.queue_len('%s') failed: %s" % (queue, e)) 348 | raise cyclone.web.HTTPError(503) 349 | 350 | resp = "%s" % cyclone.escape.json_encode({ 351 | "redis": repr(self.settings.db), 352 | "queue": queue, "len": qlen}) 353 | 354 | if jsoncallback is not None: resp = "%s(%s)" % (jsoncallback, resp) 355 | self.finish(resp) 356 | 357 | 358 | class CometDispatcher(object): 359 | def __init__(self, oper, del_obj=True): 360 | self.oper = oper 361 | self.queue = defer.DeferredQueue() 362 | self.presence = defaultdict(lambda: []) 363 | self.qcounter = defaultdict(lambda: 0) 364 | self.queue.get().addCallback(self._new_data) 365 | task.LoopingCall(self._auto_dispatch).start(1) # secs between checkings 366 | task.LoopingCall(self._counters_cleanup).start(30) # presence maintenance 367 | self.delete_objects = del_obj 368 | 369 | def _new_data(self, queue_name): 370 | self.dispatch(queue_name) 371 | self.queue.get().addCallback(self._new_data) 372 | 373 | def _auto_dispatch(self): 374 | for queue_name, handlers in self.presence.items(): 375 | self.dispatch(queue_name, handlers) 376 | 377 | def _counters_cleanup(self): 378 | keys = self.qcounter.keys() 379 | for queue_name in keys: 380 | if not self.presence.has_key(queue_name): 381 | self.qcounter.pop(queue_name) 382 | 383 | @defer.inlineCallbacks 384 | def dispatch(self, queue_name, handlers=None): 385 | try: 386 | qstat = yield self.oper.queue_status(queue_name) 387 | except Exception, e: 388 | log.msg("ERROR: oper.queue_status('%s') failed: %s" % (queue_name, e)) 389 | defer.returnValue(None) 390 | 391 | if qstat["status"] != self.oper.STARTQUEUE: 392 | defer.returnValue(None) 393 | 394 | handlers = handlers or self.presence.get(queue_name) 395 | if handlers: 396 | try: 397 | policy, contents = yield self.oper.queue_tail(queue_name, delete_obj = self.delete_objects) 398 | assert policy and contents and isinstance(contents, types.ListType) 399 | except: 400 | defer.returnValue(None) 401 | 402 | self._dump(handlers, contents, policy) 403 | 404 | def _dump(self, handlers, contents, policy = None): 405 | if policy is None: 406 | policy == core.POLICY_BROADCAST 407 | 408 | size = len(handlers) 409 | 410 | if policy == core.POLICY_BROADCAST: 411 | self._send(handlers, contents) 412 | 413 | elif policy == core.POLICY_ROUNDROBIN: 414 | idx = self.qcounter[queue_name] % size 415 | self._send((handlers[idx],), contents) 416 | self.qcounter[queue_name] += 1 417 | 418 | def _send(self, handlers, contents): 419 | for handler in handlers: 420 | for content in contents: 421 | try: 422 | handler.write(cyclone.escape.json_encode(content)) 423 | handler.flush() 424 | except Exception, e: 425 | log.msg("ERROR: Cannot write to comet client: %s = %s" % (handler, e)) 426 | 427 | 428 | class QueueControlHandler(cyclone.web.RequestHandler): 429 | """ QueueControlHandler stops/starts a queue (pause consumers)""" 430 | 431 | @authorize("rest_consumer") 432 | @defer.inlineCallbacks 433 | def get(self, queue): 434 | self.set_header("Content-Type", "application/json") 435 | 436 | stats={} 437 | if queue: 438 | try: 439 | qstat = yield self.settings.oper.queue_status(queue) 440 | except Exception, e: 441 | log.msg("ERROR: oper.queue_status('%s') failed: %s" % (queue, e)) 442 | raise cyclone.web.HTTPError(503) 443 | 444 | stats={"redis": repr(self.settings.db), "queue": queue, "status": qstat} 445 | 446 | else: 447 | try: 448 | allqueues = yield self.settings.oper.queue_all() 449 | except Exception, e: 450 | log.msg("ERROR: oper.queue_all() failed: %s" % e) 451 | raise cyclone.web.HTTPError(503) 452 | 453 | aq={} 454 | for q in allqueues: 455 | try: 456 | aq[q] = yield self.settings.oper.queue_status(q) 457 | except Exception, e: 458 | log.msg("ERROR: oper.queue_status('%s') failed: %s" % (q, e)) 459 | raise cyclone.web.HTTPError(503) 460 | 461 | stats={"redis": repr(self.settings.db), "queues": aq, "count": len(aq)} 462 | 463 | self.finish("%s\r\n" % cyclone.escape.json_encode(stats)) 464 | 465 | @authorize("rest_producer") 466 | @defer.inlineCallbacks 467 | def post(self, queue): 468 | status = self.get_argument("status", None) 469 | jsoncallback = self.get_argument("callback", None) 470 | 471 | if status == "start": 472 | try: 473 | qstat = yield self.settings.oper.queue_changestatus(queue, self.settings.oper.STARTQUEUE) 474 | except Exception, e: 475 | log.msg("ERROR: oper.queue_changestatus('%s', STARTQUEUE) failed: %s" % (queue, e)) 476 | raise cyclone.web.HTTPError(503) 477 | 478 | elif status == "stop": 479 | try: 480 | qstat = yield self.settings.oper.queue_changestatus(queue, self.settings.oper.STOPQUEUE) 481 | except Exception, e: 482 | log.msg("ERROR: oper.queue_changestatus('%s', STOPQUEUE) failed: %s" % (queue, e)) 483 | raise cyclone.web.HTTPError(503) 484 | 485 | else: 486 | qstat = "invalid status: %s" % status 487 | 488 | resp = "%s" % cyclone.escape.json_encode({"stat": qstat }) 489 | 490 | if jsoncallback is not None: resp = "%s(%s)" % (jsoncallback, resp) 491 | self.finish("%s\r\n" % resp) 492 | 493 | 494 | class WebSocketQueueHandler(cyclone.websocket.WebSocketHandler): 495 | """ 496 | Guess what, I had a fever, and the only prescription is websocket 497 | """ 498 | def _disconnected(self, why, handler, queue_name): 499 | try: 500 | self.settings.comet.presence[queue_name].remove(handler) 501 | if not len(self.settings.comet.presence[queue_name]): 502 | self.settings.comet.presence.pop(queue_name) 503 | except: 504 | pass 505 | 506 | def headersReceived(self): 507 | # for authenticated websocket clientes, the browser must set a 508 | # cookie like this: 509 | # document.cookie = "auth=user:password" 510 | # 511 | # see ACL.check_password for details 512 | try: 513 | self.settings.acl.apply(self, "websocket_consumer", websocket=True) 514 | except: 515 | raise cyclone.web.HTTPError(401) 516 | 517 | def connectionMade(self, queue): 518 | self.queue = queue 519 | handler = CustomHandler(self, None) 520 | 521 | try: 522 | queue_name = queue.encode("utf-8") 523 | except: 524 | raise cyclone.web.HTTPError(400, "Invalid Queue Name") 525 | 526 | self.settings.comet.presence[queue_name].append(handler) 527 | self.notifyFinish().addCallback(self._disconnected, handler, queue_name) 528 | 529 | def messageReceived(self, message): 530 | """ 531 | same idea as COMET consumer, but using websockets. how cool is that ? 532 | """ 533 | self.sendMessage(message) 534 | 535 | 536 | class ACL(object): 537 | def __init__(self, filename): 538 | self.md5 = None 539 | self.filename = filename 540 | 541 | self.rest_producer = {} 542 | self.rest_consumer = {} 543 | self.comet_consumer = {} 544 | self.websocket_consumer = {} 545 | 546 | self.parse(True) 547 | 548 | def parse(self, firstRun=False): 549 | try: 550 | if(self.filename.startswith('/') or os.path.exists(self.filename)): 551 | fp = open(self.filename) 552 | else: 553 | import pkg_resources as pkg 554 | fp = pkg.resource_stream('restmq.assets', self.filename) 555 | 556 | md5 = hashlib.md5(fp.read()).hexdigest() 557 | 558 | if self.md5 is None: 559 | self.md5 = md5 560 | else: 561 | if self.md5 == md5: 562 | return fp.close() 563 | 564 | fp.seek(0) 565 | cfg = ConfigParser() 566 | cfg.readfp(fp) 567 | fp.close() 568 | 569 | except Exception, e: 570 | if firstRun: 571 | raise e 572 | else: 573 | log.msg("ERROR: Could not reload configuration: %s" % e) 574 | return 575 | else: 576 | if not firstRun: 577 | log.msg("Reloading ACL configuration") 578 | 579 | for section in ("rest:producer", "rest:consumer", "comet:consumer", "websocket:consumer"): 580 | d = getattr(self, section.replace(":", "_")) 581 | 582 | try: 583 | hosts_allow = cfg.get(section, "hosts_allow") 584 | d["hosts_allow"] = hosts_allow != "all" and hosts_allow.split() or None 585 | except: 586 | d["hosts_allow"] = None 587 | 588 | try: 589 | hosts_deny = cfg.get(section, "hosts_deny") 590 | d["hosts_deny"] = hosts_deny != "all" and hosts_deny.split() or None 591 | except: 592 | d["hosts_deny"] = None 593 | 594 | try: 595 | username = cfg.get(section, "username") 596 | d["username"] = username 597 | except: 598 | d["username"] = None 599 | 600 | try: 601 | password = cfg.get(section, "password") 602 | d["password"] = password 603 | except: 604 | d["password"] = None 605 | 606 | reactor.callLater(60, self.parse) 607 | 608 | def check_password(self, client, username, password, websocket): 609 | try: 610 | if websocket is True: 611 | rusername, rpassword = client.get_cookie("auth").split(":", 1) 612 | assert username == rusername and password == rpassword 613 | 614 | else: 615 | authtype, authdata = client.request.headers["Authorization"].split() 616 | assert authtype == "Basic" 617 | rusername, rpassword = base64.b64decode(authdata).split(":", 1) 618 | assert username == rusername and password == rpassword 619 | except: 620 | raise InvalidPassword 621 | 622 | def apply(self, client, category, websocket=False): 623 | acl = getattr(self, category) 624 | require_password = acl["username"] and acl["password"] 625 | 626 | if acl["hosts_allow"]: 627 | for ip in acl["hosts_allow"]: 628 | if client.request.remote_ip.startswith(ip): 629 | if require_password: 630 | self.check_password(client, 631 | acl["username"], acl["password"], websocket) 632 | return 633 | else: 634 | return 635 | 636 | if acl["hosts_deny"] is None: 637 | raise InvalidAddress("ip address %s not allowed" % client.request.remote_ip) 638 | 639 | if acl["hosts_deny"]: 640 | for ip in acl["hosts_deny"]: 641 | if client.request.remote_ip.startswith(ip): 642 | raise InvalidAddress("ip address %s not allowed" % client.request.remote_ip) 643 | 644 | if require_password: 645 | self.check_password(client, acl["username"], acl["password"], websocket) 646 | 647 | 648 | class Application(cyclone.web.Application): 649 | def __init__(self, acl_file, redis_host, redis_port, redis_pool, redis_db): 650 | handlers = [ 651 | (r"/", IndexHandler), 652 | (r"/q/(.*)", RestQueueHandler), 653 | (r"/c/(.*)", CometQueueHandler), 654 | (r"/p/(.*)", PolicyQueueHandler), 655 | (r"/j/(.*)", JobQueueInfoHandler), 656 | (r"/stats/(.*)", StatusHandler), 657 | (r"/queue", QueueHandler), 658 | (r"/control/(.*)", QueueControlHandler), 659 | (r"/ws/(.*)", WebSocketQueueHandler), 660 | ] 661 | 662 | try: 663 | acl = ACL(acl_file) 664 | except Exception, e: 665 | log.msg("ERROR: Cannot load ACL file: %s" % e) 666 | raise RuntimeError("Cannot load ACL file: %s" % e) 667 | 668 | db = cyclone.redis.lazyConnectionPool( 669 | redis_host, redis_port, 670 | poolsize=redis_pool, dbid=redis_db) 671 | 672 | oper = core.RedisOperations(db) 673 | 674 | settings = { 675 | "db": db, 676 | "acl": acl, 677 | "oper": oper, 678 | "comet": CometDispatcher(oper), 679 | "static_path": pkg.resource_filename('restmq', 'static'), 680 | "template_path": pkg.resource_filename('restmq', 'templates'), 681 | } 682 | 683 | cyclone.web.Application.__init__(self, handlers, **settings) 684 | 685 | --------------------------------------------------------------------------------