├── README ├── carbon-agent-rabbitmq.py ├── examples ├── eth0_traf.py └── rabbit_queues.py ├── graphite_rabbitmq_config.py └── graphite_rabbitmq_publish.py /README: -------------------------------------------------------------------------------- 1 | 2 | Graphite-RabbitMQ Integration 3 | ============================= 4 | 5 | Graphite - http://graphite.wikidot.com 6 | RabbitMQ - http://www.rabbitmq.com 7 | Blog - http://somic.org/2009/05/21/graphite-rabbitmq-integration/ 8 | 9 | INSTALLATION 10 | ------------ 11 | 12 | Requires py-amqplib http://barryp.org/software/py-amqplib/ 13 | 14 | Drop contents of this repository into GRAPHITE_HOME/carbon 15 | directory, edit graphite_rabbitmq_config.py, run it 16 | to create queues, then run carbon-agent-rabbitmq.py with 17 | same options you run carbon-agent.py with. 18 | 19 | Watch out for multiple instances of carbon-persister.py 20 | writing to the same whisper files! 21 | 22 | -------------------------------------------------------------------------------- /carbon-agent-rabbitmq.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Copyright 2008 Orbitz WorldWide 3 | Copyright 2009 Dmitriy Samovskiy 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License.""" 16 | 17 | import sys 18 | if sys.version_info[0] != 2 or sys.version_info[1] < 4: 19 | print 'Python version >= 2.4 and < 3.0 is required' 20 | sys.exit(1) 21 | 22 | try: 23 | import graphite 24 | except: 25 | print "Failed to import the graphite package. Please verify that this package" 26 | print "was properly installed and that your PYTHONPATH environment variable" 27 | print "includes the directory in which it is installed." 28 | print "\nFor example, you may need to run the following command:\n" 29 | print "export PYTHONPATH=\"/home/myusername/lib/python/:$PYTHONPATH\"\n" 30 | sys.exit(1) 31 | 32 | import os, socket, time, traceback 33 | from getopt import getopt 34 | from signal import signal, SIGTERM 35 | from subprocess import * 36 | from select import select 37 | from schemalib import loadStorageSchemas 38 | from utils import daemonize, dropprivs, logify 39 | import amqplib.client_0_8 as amqp 40 | from graphite_rabbitmq_config import RABBITMQ_BROKER_DATA, GRAPHITE_QUEUE 41 | 42 | debug = False 43 | user = 'apache' 44 | 45 | try: 46 | (opts,args) = getopt(sys.argv[1:],"du:h") 47 | assert ('-h','') not in opts 48 | except: 49 | print """Usage: %s [options] 50 | 51 | Options: 52 | -d Debug mode 53 | -u user Drop privileges to run as user 54 | -h Display this help message 55 | """ % os.path.basename(sys.argv[0]) 56 | sys.exit(1) 57 | 58 | for opt,val in opts: 59 | if opt == '-d': 60 | debug = True 61 | elif opt == '-u': 62 | user = val 63 | 64 | if debug: 65 | logify() 66 | else: 67 | daemonize() 68 | logify('log/agent.log') 69 | pf = open('pid/agent.pid','w') 70 | pf.write( str(os.getpid()) ) 71 | pf.close() 72 | try: dropprivs(user) 73 | except: pass 74 | print 'carbon-agent started (pid=%d)' % os.getpid() 75 | 76 | def handleDeath(signum,frame): 77 | print 'Received SIGTERM, killing children' 78 | try: 79 | os.kill( persisterProcess.pid, SIGTERM ) 80 | print 'Sent SIGTERM to carbon-persister' 81 | os.wait() 82 | print 'wait() complete, exitting' 83 | except OSError: 84 | print 'carbon-persister appears to already be dead' 85 | sys.exit(0) 86 | 87 | signal(SIGTERM,handleDeath) 88 | 89 | devnullr = open('/dev/null','r') 90 | devnullw = open('/dev/null','w') 91 | 92 | persisterPipe = map( str, os.pipe() ) 93 | print 'created persister pipe, fds=%s' % str(persisterPipe) 94 | 95 | args = ['./carbon-persister.py',persisterPipe[0]] 96 | persisterProcess = Popen(args,stdin=devnullr,stdout=devnullw,stderr=devnullw) 97 | print 'carbon-persister started with pid %d' % persisterProcess.pid 98 | pf = open('pid/persister.pid','w') 99 | pf.write( str(persisterProcess.pid) ) 100 | pf.close() 101 | 102 | writeFD = int(persisterPipe[1]) 103 | 104 | def write_to_persister(msg): 105 | data = "%s" % msg.body 106 | if not data.endswith("\n"): data += "\n" 107 | written = os.write(writeFD, data) 108 | msg.channel.basic_ack(msg.delivery_tag) 109 | assert written == len(data), "write_to_persister: wrote only %d of %d" \ 110 | % (written, len(data)) 111 | 112 | while True: 113 | try: 114 | conn = amqp.Connection(**RABBITMQ_BROKER_DATA) 115 | ch = conn.channel() 116 | ch.basic_consume(GRAPHITE_QUEUE, callback=write_to_persister) 117 | while ch.callbacks: ch.wait() 118 | except Exception, e: 119 | print '%s Got exception in loop: %s' % (time.asctime(), str(e)) 120 | try: conn.close() 121 | except: pass 122 | time.sleep(1) 123 | 124 | 125 | -------------------------------------------------------------------------------- /examples/eth0_traf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | 5 | Parses Linux /proc/net/dev to get RX and TX bytes on interface IFACE 6 | 7 | """ 8 | 9 | import sys, os, time 10 | sys.path += [ os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ] 11 | from graphite_rabbitmq_publish import GraphiteRabbitMQPublisher 12 | 13 | try: 14 | iface = sys.argv[1] 15 | metric_prefix = sys.argv[2] 16 | except: 17 | print "Usage: %s iface metric_prefix" % sys.argv[0] 18 | sys.exit(1) 19 | 20 | def parse_proc_net_dev(iface): 21 | f = open('/proc/net/dev') 22 | r = 0 23 | t = 0 24 | for l in f: 25 | if l.find("%s:" % iface) == -1: continue 26 | spl = l.split() 27 | r, t = int(spl[0].split(':')[1]), int(spl[8]) 28 | f.close() 29 | return r, t 30 | 31 | rx = 0 32 | tx = 0 33 | first_sample = True 34 | while True: 35 | try: 36 | pub = GraphiteRabbitMQPublisher() 37 | while True: 38 | new_rx, new_tx = parse_proc_net_dev(iface) 39 | if not first_sample: 40 | pub.publish([ 41 | "%s.rx %d" % (metric_prefix, new_rx-rx), 42 | "%s.tx %d" % (metric_prefix, new_tx-tx) 43 | ]) 44 | print "%s rx:%d tx:%d" % (time.asctime(), new_rx-rx, new_tx-tx) 45 | rx, tx = new_rx, new_tx 46 | first_sample = False 47 | time.sleep(60) 48 | except Exception, e: 49 | print e 50 | time.sleep(20) 51 | 52 | -------------------------------------------------------------------------------- /examples/rabbit_queues.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # graphite publisher for some basic info on rabbitmq queues 4 | # 5 | # requires py-interface: 6 | # http://www.lysator.liu.se/~tab/erlang/py_interface/ 7 | # 8 | 9 | COOKIE = 'PQDZQNHHATKKMBYYBHSS' 10 | METRIC_PREFIX = 'rabbitmq.homeserver.queues' 11 | RABBIT_NODE = 'rabbit@home' 12 | 13 | metrics = [ "messages_ready", "messages_unacknowledged", 14 | "messages_uncommitted", "messages", "consumers", "memory" ] 15 | 16 | import sys, os, time 17 | sys.path += [ os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ] 18 | from graphite_rabbitmq_publish import GraphiteRabbitMQPublisher 19 | 20 | from py_interface import erl_node, erl_eventhandler 21 | from py_interface.erl_opts import ErlNodeOpts 22 | from py_interface.erl_term import ErlAtom, ErlBinary 23 | 24 | #from py_interface import erl_common 25 | #erl_common.DebugOnAll() 26 | 27 | rpc_args = [ ErlAtom('name') ] + [ ErlAtom(x) for x in metrics ] 28 | 29 | def __msg_handler_list_queues(msg): 30 | data_lines = [ ] 31 | for q in msg: 32 | name = q[0][1][-1].contents 33 | for atoms_tuple in q[1:]: 34 | data_lines.append("%s.%s.%s %d" % \ 35 | (METRIC_PREFIX, name, atoms_tuple[0].atomText, 36 | atoms_tuple[1])) 37 | print data_lines 38 | pub.publish(data_lines) 39 | erl_eventhandler.GetEventHandler().AddTimerEvent(60, 40 | rpc_list_queues, mbox=mbox) 41 | 42 | def start_pyrabbitmqctl_node(): 43 | node = erl_node.ErlNode("pyrabbitmqctl%d" % os.getpid(), 44 | ErlNodeOpts(cookie=COOKIE)) 45 | mbox = node.CreateMBox() 46 | return node, mbox 47 | 48 | def rpc_list_queues(mbox, vhost="/"): 49 | mbox.SendRPC( 50 | ErlAtom(RABBIT_NODE), 51 | ErlAtom('rabbit_amqqueue'), 52 | ErlAtom('info_all'), 53 | [ ErlBinary(vhost), rpc_args ], 54 | __msg_handler_list_queues 55 | ) 56 | 57 | global pub 58 | pub = GraphiteRabbitMQPublisher() 59 | 60 | node, mbox = start_pyrabbitmqctl_node() 61 | rpc_list_queues(mbox) 62 | erl_eventhandler.GetEventHandler().Loop() 63 | 64 | -------------------------------------------------------------------------------- /graphite_rabbitmq_config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | 5 | Defines common configuration data + tool to create graphite queue. 6 | 7 | """ 8 | 9 | 10 | RABBITMQ_BROKER_DATA = { 11 | 'host': 'localhost:5672', 12 | 'userid': 'guest', 13 | 'password': 'guest' 14 | } 15 | 16 | GRAPHITE_EXCHANGE = 'amq.direct' 17 | GRAPHITE_ROUTING_KEY = 'graphite' 18 | GRAPHITE_QUEUE = 'graphite_data' 19 | 20 | if __name__ == '__main__': 21 | import sys 22 | import amqplib.client_0_8 as amqp 23 | 24 | conn = amqp.Connection(**RABBITMQ_BROKER_DATA) 25 | ch = conn.channel() 26 | try: 27 | ch.queue_declare(queue=GRAPHITE_QUEUE, passive=True) 28 | print "Queue %s already exists." % GRAPHITE_QUEUE 29 | except: 30 | ch = conn.channel() 31 | ch.queue_declare(queue=GRAPHITE_QUEUE, durable=True, auto_delete=False) 32 | ch.queue_bind(GRAPHITE_QUEUE, GRAPHITE_EXCHANGE, GRAPHITE_ROUTING_KEY) 33 | ch.close() 34 | conn.close() 35 | print "Queue %s created." % GRAPHITE_QUEUE 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /graphite_rabbitmq_publish.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import time 4 | import sys 5 | import amqplib.client_0_8 as amqp 6 | from graphite_rabbitmq_config import RABBITMQ_BROKER_DATA, \ 7 | GRAPHITE_EXCHANGE, GRAPHITE_ROUTING_KEY 8 | 9 | class GraphiteRabbitMQPublisher: 10 | def __init__(self, rabbitmq_broker_data=RABBITMQ_BROKER_DATA, 11 | exchange=GRAPHITE_EXCHANGE, 12 | routing_key=GRAPHITE_ROUTING_KEY): 13 | self.rabbitmq_broker_data = rabbitmq_broker_data 14 | self.exchange = exchange 15 | self.routing_key=routing_key 16 | self.__channel = None 17 | 18 | def channel(self): 19 | if self.__channel is None: 20 | self.conn = amqp.Connection(**self.rabbitmq_broker_data) 21 | self.__channel = self.conn.channel() 22 | return self.__channel 23 | 24 | def publish(self, data, **defaults): 25 | """ 26 | data can be a dict {metric:value, ...} 27 | data can be a list ["metric value", "value", "metric value timestamp", ...] 28 | 29 | defaults can include timestamp and metric 30 | 31 | """ 32 | try: defaults['timestamp'] 33 | except KeyError: defaults['timestamp'] = int(time.time()) 34 | 35 | payload_lines = [ ] 36 | if type(data) == dict: 37 | for k in data: 38 | payload_lines.append("%s %s %d" % (k, str(data[k]), 39 | defaults['timestamp'])) 40 | elif type(data) == list: 41 | for k in data: 42 | parts = str(k).strip().split() 43 | m = None # metric 44 | v = None # value 45 | t = None # timestamp 46 | if len(parts) == 1: 47 | m = defaults['metric'] 48 | v = k 49 | t = defaults['timestamp'] 50 | elif len(parts) == 2: 51 | m,v = parts[:2] 52 | t = defaults['timestamp'] 53 | elif len(parts) == 3: 54 | m, v, t = parts 55 | else: 56 | raise ArgumentError, "bad line: %s" % k 57 | 58 | payload_lines.append("%s %s %d" % (m, v, t)) 59 | elif type(data) == str: 60 | if not len(data.strip().split()) == 3: 61 | raise ArgumentError, "bad line %s" % data 62 | payload_lines.append(data) 63 | 64 | self.channel().basic_publish( 65 | amqp.Message("\n".join(payload_lines), delivery_mode=2), 66 | exchange=self.exchange, routing_key=self.routing_key) 67 | 68 | 69 | if __name__ == '__main__': 70 | try: 71 | assert len(sys.argv[1:]) > 0, "Nothing to send" 72 | GraphiteRabbitMQPublisher().publish(sys.argv[1:]) 73 | except Exception, e: 74 | print "Error: %s" % str(e) 75 | print "Usage: %s 'metric value timestamp' ..." % sys.argv[0] 76 | 77 | 78 | --------------------------------------------------------------------------------