├── .github └── workflows │ └── lint_python.yml ├── .gitignore ├── COPYING ├── MANIFEST.in ├── Makefile ├── NEWS ├── README.md ├── bin └── pystatsd-server ├── debian ├── changelog ├── compat ├── control ├── copyright ├── pycompat ├── python-statsd.docs ├── rules └── watch ├── init ├── pystatsd.conf.upstart ├── pystatsd.default └── pystatsd.init ├── pystatsd ├── __init__.py ├── daemon.py ├── gmetric.py ├── server.py └── statsd.py ├── redhat ├── pystatsd-python2.4.patch └── pystatsd.spec ├── requirements.txt ├── setup.py ├── statsd_test.py └── tests ├── __init__.py ├── client.py └── server.py /.github/workflows/lint_python.yml: -------------------------------------------------------------------------------- 1 | name: lint_python 2 | on: 3 | pull_request: 4 | push: 5 | # branches: [master] 6 | jobs: 7 | lint_python: 8 | runs-on: ubuntu-latest 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | python-version: [2.7, 3.8] 13 | steps: 14 | - uses: actions/checkout@master 15 | - uses: actions/setup-python@master 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | - run: pip install codespell flake8 isort 19 | - if: matrix.python-version != 2.7 20 | run: | 21 | pip install --quiet black 22 | black . --diff --skip-string-normalization || true 23 | - run: codespell --ignore-words-list="gonna,process'" --quiet-level=2 # --skip="" 24 | - run: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 25 | - run: isort --recursive . || true 26 | - run: pip install -r requirements.txt 27 | - run: nosetests tests 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | build 3 | dist 4 | *.egg-info 5 | *.log 6 | venv 7 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Steve Ivy 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | pypi: 3 | python setup.py sdist upload 4 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sivy/pystatsd/f88e340e389e50f7a85ea8d2c7bfdd5c1ac97e2c/NEWS -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Introduction 2 | ------------ 3 | 4 | **pystatsd** is a client and server implementation of Etsy's brilliant statsd 5 | server, a front end/proxy for the Graphite stats collection and graphing server. 6 | 7 | * Graphite 8 | - http://graphite.wikidot.com 9 | * Statsd 10 | - code: https://github.com/etsy/statsd 11 | - blog post: http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/ 12 | 13 | **pystatsd** is [tested on](https://github.com/sivy/pystatsd/actions) Python 2.7 and 3.8. 14 | 15 | Status 16 | ------------- 17 | 18 | Reviewing and merging pull requests, bringing stuff up to date, with tests! 19 | 20 | [![lint_python](https://github.com/sivy/pystatsd/workflows/lint_python/badge.svg)](https://github.com/sivy/pystatsd/actions) 21 | 22 | Usage 23 | ------------- 24 | 25 | See statsd_test for sample usage: 26 | 27 | from pystatsd import Client, Server 28 | 29 | srvr = Server(debug=True) 30 | srvr.serve() 31 | 32 | sc = Client('example.org',8125) 33 | 34 | sc.timing('python_test.time',500) 35 | sc.increment('python_test.inc_int') # or sc.incr() 36 | sc.decrement('python_test.decr_int') # or sc.decr() 37 | sc.gauge('python_test.gauge', 42) 38 | 39 | Building a Debian Package 40 | ------------- 41 | 42 | To build a debian package, run `dpkg-buildpackage -rfakeroot` 43 | 44 | Upstart init Script 45 | ------------- 46 | Upstart is the daemon management system for Ubuntu. 47 | 48 | A basic upstart script has been included for the pystatsd server. It's located 49 | under init/, and will be installed to /usr/share/doc if you build/install a 50 | .deb file. The upstart script should be copied to /etc/init/pystatsd.conf and 51 | will read configuration variables from /etc/default/pystatsd. By default the 52 | pystatsd daemon runs as user 'nobody' which is a good thing from a security 53 | perspective. 54 | 55 | Troubleshooting 56 | ------------- 57 | 58 | You can see the raw values received by pystatsd by packet sniffing: 59 | 60 | $ sudo ngrep -qd any . udp dst port 8125 61 | 62 | You can see the raw values dispatched to carbon by packet sniffing: 63 | 64 | $ sudo ngrep -qd any stats tcp dst port 2003 65 | -------------------------------------------------------------------------------- /bin/pystatsd-server: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from pystatsd.server import run_server 3 | 4 | if __name__ == '__main__': 5 | run_server() -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | python-statsd (1.1) testing; urgency=low 2 | 3 | * Fix to truncate long metrics names before sending to ganglia 4 | 5 | -- seph Fri, 15 Sep 2017 20:18:57 +0000 6 | 7 | python-statsd (1.0-4) testing; urgency=low 8 | 9 | * update to 36a59d3b126ded4658aff25bce94e844a1c6413e 10 | * Fix path to README file 11 | 12 | -- Bruno Clermont Wed, 15 Aug 2012 15:01:00 +0200 13 | 14 | python-statsd (1.0-2) UNRELEASED; urgency=low 15 | 16 | [ Rob Terhaar ] 17 | * add requirement module for python-argparse 18 | * added upstart script to /usr/share/docs/python-statsd/ 19 | 20 | [ Gábor Farkas ] 21 | * do not override the threshold 22 | * fixed stats-name generation 23 | 24 | [ Rob Terhaar ] 25 | * updated README documentation with upstart information 26 | * manual pidfile mgmt is no longer needed since we're not daemonizing 27 | via pystatd's -D option 28 | * don't run daemon as root, user nobody works fine 29 | * argparse needs to be a install dep, not a build dep. 30 | 31 | -- Rob Terhaar Thu, 14 Jul 2011 20:05:09 +0000 32 | 33 | python-statsd (1.0) UNRELEASED; urgency=low 34 | 35 | * First public package. 36 | 37 | -- Rob Terhaar Tue, 12 Jul 2011 18:02:24 +0000 38 | -------------------------------------------------------------------------------- /debian/compat: -------------------------------------------------------------------------------- 1 | 7 2 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: python-statsd 2 | Maintainer: Rob Terhaar 3 | Section: python 4 | Priority: optional 5 | Standards-Version: 3.8.3 6 | Build-Depends: debhelper (>= 7.3), python-support (>= 1.0.3), python, python-setuptools 7 | XS-Python-Version: >= 2.6 8 | Vcs-Git: https://github.com/sivy/py-statsd 9 | Vcs-Browser: https://github.com/sivy/py-statsd 10 | Homepage: https://github.com/sivy/py-statsd 11 | 12 | Package: python-statsd 13 | Architecture: all 14 | Depends: ${python:Depends}, ${misc:Depends}, python-argparse 15 | XB-Python-Version: ${python:Versions} 16 | Description: Statsd, in python 17 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | Format-Specification: http://svn.debian.org/wsvn/dep/web/deps/dep5.mdwn?rev=59 2 | Source: https://github.com/sivy/py-statsd 3 | Maintainer: Rob Terhaar 4 | 5 | Files: * 6 | Copyright: unknown 7 | License: unknown 8 | 9 | Files: debian/* 10 | Copyright: © 2011 Atlantic Dynamic 11 | License: GPL-3 12 | 13 | License: GPL-3 14 | This program is free software: you can redistribute it and/or modify 15 | it under the terms of the GNU General Public License as published by 16 | the Free Software Foundation, version 3 of the License. 17 | . 18 | On a Debian system, you can find a copy of the GPL version 3 in 19 | /usr/share/common-licenses/GPL-3. 20 | -------------------------------------------------------------------------------- /debian/pycompat: -------------------------------------------------------------------------------- 1 | 2 2 | -------------------------------------------------------------------------------- /debian/python-statsd.docs: -------------------------------------------------------------------------------- 1 | README.md 2 | init/pystatsd.conf.upstart 3 | init/pystatsd.default 4 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | %: 3 | dh --buildsystem=python_distutils $@ 4 | 5 | override_dh_installchangelogs: 6 | dh_installchangelogs NEWS 7 | -------------------------------------------------------------------------------- /debian/watch: -------------------------------------------------------------------------------- 1 | version=3 2 | http://githubredir.debian.net/github/sivy/py-statsd (.*).tar.gz 3 | -------------------------------------------------------------------------------- /init/pystatsd.conf.upstart: -------------------------------------------------------------------------------- 1 | # pystatsd upstart script 2 | # 2011 - rob@atlanticdynamic.com 3 | # copy this to /etc/init/pystatsd.conf 4 | # 5 | 6 | description "start and stop the py-statsd server" 7 | version "1.1" 8 | author "Rob Terhaar" 9 | 10 | description "py-statsd server" 11 | respawn limit 15 5 12 | #oom never 13 | 14 | start on (local-filesystems 15 | and net-device-up IFACE!=lo) 16 | stop on shutdown 17 | 18 | respawn 19 | 20 | pre-start script 21 | . /etc/default/pystatsd 22 | end script 23 | 24 | script 25 | . /etc/default/pystatsd 26 | exec su -s /bin/sh -c "/usr/bin/pystatsd-server \ 27 | --port $LOCAL_PYSTATD_PORT \ 28 | --pct $PCT \ 29 | --flush-interval $FLUSH_INTERVAL \ 30 | --counters-prefix $COUNTERS_PREFIX \ 31 | --timers-prefix $TIMERS_PREFIX \ 32 | --graphite-host $GRAPHITE_HOST \ 33 | --graphite-port $GRAPHITE_PORT" $USER 34 | end script 35 | 36 | post-stop script 37 | end script 38 | -------------------------------------------------------------------------------- /init/pystatsd.default: -------------------------------------------------------------------------------- 1 | # config file template for pystatd. 2 | # copy this to /etc/default/pystatsd and the included upstart or init.d script, 3 | # will read daemon settings from this file 4 | 5 | # Local port pystatsd listens on 6 | LOCAL_PYSTATD_PORT=8125 7 | # stats pct threshold 8 | PCT=90 9 | # how often to send data to graphite in millisecnds 10 | FLUSH_INTERVAL=10000 11 | # prefix to append before sending counter data to graphite 12 | COUNTERS_PREFIX=stats 13 | # prefix to append before sending timing data to graphite 14 | TIMERS_PREFIX=stats.timers 15 | # host to connect to graphite on 16 | GRAPHITE_HOST=localhost 17 | # port to connect to graphite on 18 | GRAPHITE_PORT=2003 19 | # user to run pystatsd as 20 | USER=nobody 21 | -------------------------------------------------------------------------------- /init/pystatsd.init: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # pystatsd This shell script takes care of starting and stopping pystatsd. 4 | # 5 | # chkconfig: 2345 80 30 6 | # description: Pystatsd is a front end/proxy for the Graphite stats collection and graphing server. 7 | # 8 | # processname: pystatsd 9 | # config: /etc/sysconfig/pystatsd 10 | # pidfile: /var/run/pystatsd.pid 11 | 12 | ### BEGIN INIT INFO 13 | # Provides: pystatsd 14 | # Required-Start: $local_fs $network 15 | # Required-Stop: $local_fs $network 16 | # Default-Start: 2 3 4 5 17 | # Default-Stop: 0 1 6 18 | # Short-Description: start and stop pystatsd 19 | # Description: Pystatsd is a front end/proxy for the Graphite stats collection and graphing server. 20 | ### END INIT INFO 21 | 22 | # Source function library. 23 | . /etc/rc.d/init.d/functions 24 | 25 | # Default settings 26 | LOCAL_PYSTATD_PORT=8125 27 | PCT=90 28 | FLUSH_INTERVAL=10000 29 | COUNTERS_PREFIX=stats 30 | TIMERS_PREFIX=stats.timers 31 | GRAPHITE_HOST=localhost 32 | GRAPHITE_PORT=2003 33 | USER=nobody 34 | 35 | # Load settings file if it exists 36 | if [ -e /etc/default/pystatsd ]; then 37 | . /etc/default/pystatsd 38 | fi 39 | 40 | PIDFILE=/var/run/pystatsd.pid 41 | 42 | # Check that networking is up. 43 | [ "${NETWORKING}" = "no" ] && exit 1 44 | 45 | [ -x /usr/bin/pystatsd-server ] || exit 5 46 | 47 | prog="pystatsd" 48 | 49 | start() { 50 | # Start daemons. 51 | ret=0 52 | echo -n $"Starting $prog: " 53 | daemon /usr/bin/pystatsd-server --port $LOCAL_PYSTATD_PORT --pct $PCT --flush-interval $FLUSH_INTERVAL --counters-prefix $COUNTERS_PREFIX --timers-prefix $TIMERS_PREFIX --graphite-host $GRAPHITE_HOST --graphite-port $GRAPHITE_PORT --daemon --pidfile $PIDFILE 54 | RETVAL=$? 55 | echo 56 | [ $RETVAL -eq 0 ] && touch /var/lock/subsys/pystatsd 57 | let ret+=$RETVAL 58 | 59 | 60 | [ $ret -eq 0 ] && return 0 || return 1 61 | } 62 | 63 | stop() { 64 | # Stop daemons. 65 | echo -n $"Shutting down $prog: " 66 | /usr/bin/pystatsd-server --stop --pidfile $PIDFILE 67 | RETVAL=$? 68 | echo 69 | [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/pystatsd 70 | return $RETVAL 71 | } 72 | 73 | status -p $PIDFILE >/dev/null 74 | running=$? 75 | 76 | # See how we were called. 77 | case "$1" in 78 | start) 79 | [ $running -eq 0 ] && exit 0 80 | start 81 | RETVAL=$? 82 | ;; 83 | stop) 84 | [ $running -eq 0 ] || exit 0 85 | stop 86 | RETVAL=$? 87 | ;; 88 | restart|force-reload) 89 | stop 90 | start 91 | RETVAL=$? 92 | ;; 93 | condrestart|try-restart) 94 | [ $running -eq 0 ] || exit 0 95 | stop 96 | start 97 | RETVAL=$? 98 | ;; 99 | status) 100 | echo -n pystatsd; status -p $PIDFILE -l pystatsd 101 | RETVAL=$? 102 | ;; 103 | *) 104 | echo $"Usage: $0 {start|stop|restart|condrestart|status}" 105 | RETVAL=2 106 | esac 107 | 108 | exit $RETVAL 109 | -------------------------------------------------------------------------------- /pystatsd/__init__.py: -------------------------------------------------------------------------------- 1 | from .statsd import Client 2 | from .server import Server 3 | 4 | VERSION = (0, 1, 10) 5 | -------------------------------------------------------------------------------- /pystatsd/daemon.py: -------------------------------------------------------------------------------- 1 | """A generic daemon class. Subclass and override the run() method. 2 | 3 | Based on http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/ 4 | """ 5 | 6 | import atexit 7 | import os 8 | from signal import SIGTERM 9 | import sys 10 | import time 11 | 12 | 13 | class Daemon(object): 14 | def __init__(self, pidfile, stdin='/dev/null', stdout='/dev/null', 15 | stderr='/dev/null'): 16 | self.stdin = stdin 17 | self.stdout = stdout 18 | self.stderr = stderr 19 | self.pidfile = pidfile 20 | 21 | def daemonize(self): 22 | """UNIX double-fork magic.""" 23 | try: 24 | pid = os.fork() 25 | if pid > 0: 26 | # First parent; exit. 27 | sys.exit(0) 28 | except OSError as e: 29 | sys.stderr.write('Could not fork! %d (%s)\n' % 30 | (e.errno, e.strerror)) 31 | sys.exit(1) 32 | 33 | # Disconnect from parent environment. 34 | os.chdir('/') 35 | os.setsid() 36 | os.umask(0o022) 37 | 38 | # Fork again. 39 | try: 40 | pid = os.fork() 41 | if pid > 0: 42 | # Second parent; exit. 43 | sys.exit(0) 44 | except OSError as e: 45 | sys.stderr.write('Could not fork (2nd)! %d (%s)\n' % 46 | (e.errno, e.strerror)) 47 | sys.exit(1) 48 | 49 | # Redirect file descriptors. 50 | sys.stdout.flush() 51 | sys.stderr.flush() 52 | si = open(self.stdin, 'r') 53 | so = open(self.stdout, 'a+') 54 | se = open(self.stderr, 'a+', 0) 55 | os.dup2(si.fileno(), sys.stdin.fileno()) 56 | os.dup2(so.fileno(), sys.stdout.fileno()) 57 | os.dup2(se.fileno(), sys.stderr.fileno()) 58 | 59 | # Write the pidfile. 60 | atexit.register(self.delpid) 61 | pid = str(os.getpid()) 62 | with open(self.pidfile, 'w+') as fp: 63 | fp.write('%s\n' % pid) 64 | 65 | def delpid(self): 66 | os.remove(self.pidfile) 67 | 68 | def start(self, *args, **kw): 69 | """Start the daemon.""" 70 | pid = None 71 | if os.path.exists(self.pidfile): 72 | with open(self.pidfile, 'r') as fp: 73 | pid = int(fp.read().strip()) 74 | 75 | if pid: 76 | msg = 'pidfile (%s) exists. Daemon already running?\n' 77 | sys.stderr.write(msg % self.pidfile) 78 | sys.exit(1) 79 | 80 | self.daemonize() 81 | self.run(*args, **kw) 82 | 83 | def stop(self): 84 | """Stop the daemon.""" 85 | pid = None 86 | if os.path.exists(self.pidfile): 87 | with open(self.pidfile, 'r') as fp: 88 | pid = int(fp.read().strip()) 89 | 90 | if not pid: 91 | msg = 'pidfile (%s) does not exist. Daemon not running?\n' 92 | sys.stderr.write(msg % self.pidfile) 93 | return 94 | 95 | try: 96 | while 1: 97 | os.kill(pid, SIGTERM) 98 | time.sleep(0.1) 99 | except OSError as e: 100 | e = str(e) 101 | if e.find('No such process') > 0: 102 | if os.path.exists(self.pidfile): 103 | os.remove(self.pidfile) 104 | else: 105 | print(e) 106 | sys.exit(1) 107 | 108 | def restart(self, *args, **kw): 109 | """Restart the daemon.""" 110 | self.stop() 111 | self.start(*args, **kw) 112 | 113 | def run(self, *args, **kw): 114 | """Override this method.""" 115 | -------------------------------------------------------------------------------- /pystatsd/gmetric.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # This is the MIT License 4 | # http://www.opensource.org/licenses/mit-license.php 5 | # 6 | # Copyright (c) 2007,2008 Nick Galbreath 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy 9 | # of this software and associated documentation files (the "Software"), to deal 10 | # in the Software without restriction, including without limitation the rights 11 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | # copies of the Software, and to permit persons to whom the Software is 13 | # furnished to do so, subject to the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be included in 16 | # all copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 24 | # THE SOFTWARE. 25 | # 26 | 27 | # 28 | # Version 1.0 - 21-Apr-2007 29 | # initial 30 | # Version 2.0 - 16-Nov-2008 31 | # made class Gmetric thread safe 32 | # made gmetrix xdr writers _and readers_ 33 | # Now this only works for gmond 2.X packets, not tested with 3.X 34 | # 35 | # Version 3.0 - 09-Jan-2011 Author: Vladimir Vuksan 36 | # Made it work with the Ganglia 3.1 data format 37 | # 38 | # Version 3.1 - 30-Apr-2011 Author: Adam Tygart 39 | # Added Spoofing support 40 | 41 | 42 | from xdrlib import Packer, Unpacker 43 | import socket 44 | 45 | slope_str2int = {'zero':0, 46 | 'positive':1, 47 | 'negative':2, 48 | 'both':3, 49 | 'unspecified':4} 50 | 51 | # could be autogenerated from previous but whatever 52 | slope_int2str = {0: 'zero', 53 | 1: 'positive', 54 | 2: 'negative', 55 | 3: 'both', 56 | 4: 'unspecified'} 57 | 58 | 59 | class Gmetric: 60 | """ 61 | Class to send gmetric/gmond 2.X packets 62 | 63 | Thread safe 64 | """ 65 | 66 | type = ('', 'string', 'uint16', 'int16', 'uint32', 'int32', 'float', 67 | 'double', 'timestamp') 68 | protocol = ('udp', 'multicast') 69 | 70 | def __init__(self, host, port, protocol): 71 | if protocol not in self.protocol: 72 | raise ValueError("Protocol must be one of: " + str(self.protocol)) 73 | 74 | self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 75 | if protocol == 'multicast': 76 | self.socket.setsockopt(socket.IPPROTO_IP, 77 | socket.IP_MULTICAST_TTL, 20) 78 | self.hostport = (host, int(port)) 79 | #self.socket.connect(self.hostport) 80 | 81 | def send(self, NAME, VAL, TYPE='', UNITS='', SLOPE='both', 82 | TMAX=60, DMAX=0, GROUP="", SPOOF=""): 83 | if SLOPE not in slope_str2int: 84 | raise ValueError("Slope must be one of: " + str(list(self.slope.keys()))) 85 | if TYPE not in self.type: 86 | raise ValueError("Type must be one of: " + str(self.type)) 87 | if len(NAME) == 0: 88 | raise ValueError("Name must be non-empty") 89 | 90 | ( meta_msg, data_msg ) = gmetric_write(NAME, VAL, TYPE, UNITS, SLOPE, TMAX, DMAX, GROUP, SPOOF) 91 | # print msg 92 | 93 | self.socket.sendto(bytes(bytearray(meta_msg, "utf-8")), self.hostport) 94 | self.socket.sendto(bytes(bytearray(data_msg, "utf-8")), self.hostport) 95 | 96 | def gmetric_write(NAME, VAL, TYPE, UNITS, SLOPE, TMAX, DMAX, GROUP, SPOOF): 97 | """ 98 | Arguments are in all upper-case to match XML 99 | """ 100 | packer = Packer() 101 | HOSTNAME="test" 102 | if SPOOF == "": 103 | SPOOFENABLED=0 104 | else : 105 | SPOOFENABLED=1 106 | # Meta data about a metric 107 | packer.pack_int(128) 108 | if SPOOFENABLED == 1: 109 | packer.pack_string(SPOOF) 110 | else: 111 | packer.pack_string(HOSTNAME) 112 | packer.pack_string(NAME) 113 | packer.pack_int(SPOOFENABLED) 114 | packer.pack_string(TYPE) 115 | packer.pack_string(NAME) 116 | packer.pack_string(UNITS) 117 | packer.pack_int(slope_str2int[SLOPE]) # map slope string to int 118 | packer.pack_uint(int(TMAX)) 119 | packer.pack_uint(int(DMAX)) 120 | # Magic number. Indicates number of entries to follow. Put in 1 for GROUP 121 | if GROUP == "": 122 | packer.pack_int(0) 123 | else: 124 | packer.pack_int(1) 125 | packer.pack_string("GROUP") 126 | packer.pack_string(GROUP) 127 | 128 | # Actual data sent in a separate packet 129 | data = Packer() 130 | data.pack_int(128+5) 131 | if SPOOFENABLED == 1: 132 | data.pack_string(SPOOF) 133 | else: 134 | data.pack_string(HOSTNAME) 135 | data.pack_string(NAME) 136 | data.pack_int(SPOOFENABLED) 137 | data.pack_string("%s") 138 | data.pack_string(str(VAL)) 139 | 140 | return ( packer.get_buffer() , data.get_buffer() ) 141 | 142 | def gmetric_read(msg): 143 | unpacker = Unpacker(msg) 144 | values = dict() 145 | unpacker.unpack_int() 146 | values['TYPE'] = unpacker.unpack_string() 147 | values['NAME'] = unpacker.unpack_string() 148 | values['VAL'] = unpacker.unpack_string() 149 | values['UNITS'] = unpacker.unpack_string() 150 | values['SLOPE'] = slope_int2str[unpacker.unpack_int()] 151 | values['TMAX'] = unpacker.unpack_uint() 152 | values['DMAX'] = unpacker.unpack_uint() 153 | unpacker.done() 154 | return values 155 | 156 | 157 | if __name__ == '__main__': 158 | import optparse 159 | parser = optparse.OptionParser() 160 | parser.add_option("", "--protocol", dest="protocol", default="udp", 161 | help="The gmetric internet protocol, either udp or multicast, default udp") 162 | parser.add_option("", "--host", dest="host", default="127.0.0.1", 163 | help="GMond aggregator hostname to send data to") 164 | parser.add_option("", "--port", dest="port", default="8649", 165 | help="GMond aggregator port to send data to") 166 | parser.add_option("", "--name", dest="name", default="", 167 | help="The name of the metric") 168 | parser.add_option("", "--value", dest="value", default="", 169 | help="The value of the metric") 170 | parser.add_option("", "--units", dest="units", default="", 171 | help="The units for the value, e.g. 'kb/sec'") 172 | parser.add_option("", "--slope", dest="slope", default="both", 173 | help="The sign of the derivative of the value over time, one of zero, positive, negative, both, default both") 174 | parser.add_option("", "--type", dest="type", default="", 175 | help="The value data type, one of string, int8, uint8, int16, uint16, int32, uint32, float, double") 176 | parser.add_option("", "--tmax", dest="tmax", default="60", 177 | help="The maximum time in seconds between gmetric calls, default 60") 178 | parser.add_option("", "--dmax", dest="dmax", default="0", 179 | help="The lifetime in seconds of this metric, default=0, meaning unlimited") 180 | parser.add_option("", "--group", dest="group", default="", 181 | help="Group metric belongs to. If not specified Ganglia will show it as no_group") 182 | parser.add_option("", "--spoof", dest="spoof", default="", 183 | help="the address to spoof (ip:host). If not specified the metric will not be spoofed") 184 | (options,args) = parser.parse_args() 185 | 186 | g = Gmetric(options.host, options.port, options.protocol) 187 | g.send(options.name, options.value, options.type, options.units, 188 | options.slope, options.tmax, options.dmax, options.group, options.spoof) 189 | -------------------------------------------------------------------------------- /pystatsd/server.py: -------------------------------------------------------------------------------- 1 | import re 2 | import socket 3 | import threading 4 | import time 5 | import types 6 | import logging 7 | from subprocess import call 8 | from warnings import warn 9 | # from xdrlib import Packer, Unpacker 10 | 11 | try: 12 | from setproctitle import setproctitle 13 | except ImportError: 14 | setproctitle = None 15 | 16 | # Messily get the import for things we're distributing. This is in a 17 | # try block, since we seem to need different syntax based on some set 18 | # of Python versions and whether or not we're in a library. 19 | try: 20 | from . import gmetric 21 | from .daemon import Daemon 22 | except ValueError: 23 | import gmetric 24 | from daemon import Daemon 25 | 26 | 27 | __all__ = ['Server'] 28 | 29 | 30 | def _clean_key(k): 31 | return re.sub( 32 | r'[^a-zA-Z_\-0-9\.]', 33 | '', 34 | re.sub( 35 | r'\s+', 36 | '_', 37 | k.replace('/', '-').replace(' ', '_') 38 | ) 39 | ) 40 | 41 | TIMER_MSG = '''%(prefix)s.%(key)s.lower %(min)s %(ts)s 42 | %(prefix)s.%(key)s.count %(count)s %(ts)s 43 | %(prefix)s.%(key)s.mean %(mean)s %(ts)s 44 | %(prefix)s.%(key)s.upper %(max)s %(ts)s 45 | %(prefix)s.%(key)s.upper_%(pct_threshold)s %(max_threshold)s %(ts)s 46 | ''' 47 | 48 | 49 | class Server(object): 50 | 51 | def __init__(self, pct_threshold=90, debug=False, transport='graphite', 52 | ganglia_host='localhost', ganglia_port=8649, 53 | ganglia_spoof_host='statsd:statsd', ganglia_max_length=100, 54 | gmetric_exec='/usr/bin/gmetric', gmetric_options = '-d', 55 | graphite_host='localhost', graphite_port=2003, global_prefix=None, 56 | flush_interval=10000, 57 | no_aggregate_counters=False, counters_prefix='stats', 58 | timers_prefix='stats.timers', expire=0): 59 | self.buf = 8192 60 | self.flush_interval = flush_interval 61 | self.pct_threshold = pct_threshold 62 | self.transport = transport 63 | # Embedded Ganglia library options specific settings 64 | self.ganglia_host = ganglia_host 65 | self.ganglia_port = ganglia_port 66 | self.ganglia_protocol = "udp" 67 | # Use gmetric 68 | self.gmetric_exec = gmetric_exec 69 | self.gmetric_options = gmetric_options 70 | # Common Ganglia 71 | self.ganglia_max_length = ganglia_max_length 72 | # Set DMAX to flush interval plus 20%. That should avoid metrics to prematurely expire if there is 73 | # some type of a delay when flushing 74 | self.dmax = int(self.flush_interval * 1.2) 75 | # What hostname should these metrics be attached to. 76 | self.ganglia_spoof_host = ganglia_spoof_host 77 | 78 | # Graphite specific settings 79 | self.graphite_host = graphite_host 80 | self.graphite_port = graphite_port 81 | self.no_aggregate_counters = no_aggregate_counters 82 | self.counters_prefix = counters_prefix 83 | self.timers_prefix = timers_prefix 84 | self.debug = debug 85 | self.expire = expire 86 | 87 | # For services like Hosted Graphite, etc. 88 | self.global_prefix = global_prefix 89 | 90 | self.counters = {} 91 | self.timers = {} 92 | self.gauges = {} 93 | self.flusher = 0 94 | 95 | def send_to_ganglia_using_gmetric(self,k,v,group, units): 96 | if len(k) >= self.ganglia_max_length: 97 | log.debug("Ganglia metric too long. Ignoring: %s" % k) 98 | else: 99 | call([self.gmetric_exec, self.gmetric_options, "-u", units, "-g", group, "-t", "double", "-n", k, "-v", str(v) ]) 100 | 101 | 102 | def process(self, data): 103 | # the data is a sequence of newline-delimited metrics 104 | # a metric is in the form "name:value|rest" (rest may have more pipes) 105 | data.rstrip('\n') 106 | 107 | for metric in data.split('\n'): 108 | match = re.match('\A([^:]+):([^|]+)\|(.+)', metric) 109 | 110 | if match == None: 111 | warn("Skipping malformed metric: <%s>" % (metric)) 112 | continue 113 | 114 | key = _clean_key( match.group(1) ) 115 | value = match.group(2) 116 | rest = match.group(3).split('|') 117 | mtype = rest.pop(0) 118 | 119 | if (mtype == 'ms'): self.__record_timer(key, value, rest) 120 | elif (mtype == 'g' ): self.__record_gauge(key, value, rest) 121 | elif (mtype == 'c' ): self.__record_counter(key, value, rest) 122 | else: 123 | warn("Encountered unknown metric type in <%s>" % (metric)) 124 | 125 | def __record_timer(self, key, value, rest): 126 | ts = int(time.time()) 127 | timer = self.timers.setdefault(key, [ [], ts ]) 128 | timer[0].append(float(value or 0)) 129 | timer[1] = ts 130 | 131 | def __record_gauge(self, key, value, rest): 132 | ts = int(time.time()) 133 | self.gauges[key] = [ float(value), ts ] 134 | 135 | def __record_counter(self, key, value, rest): 136 | ts = int(time.time()) 137 | sample_rate = 1.0 138 | if len(rest) == 1: 139 | sample_rate = float(re.match('^@([\d\.]+)', rest[0]).group(1)) 140 | if sample_rate == 0: 141 | warn("Ignoring counter with sample rate of zero: <%s>" % (key)) 142 | return 143 | 144 | counter = self.counters.setdefault(key, [ 0, ts ]) 145 | counter[0] += float(value or 1) * (1 / sample_rate) 146 | counter[1] = ts 147 | 148 | def on_timer(self): 149 | """Executes flush(). Ignores any errors to make sure one exception 150 | doesn't halt the whole flushing process. 151 | """ 152 | try: 153 | self.flush() 154 | except Exception as e: 155 | log.exception('Error while flushing: %s', e) 156 | self._set_timer() 157 | 158 | def flush(self): 159 | ts = int(time.time()) 160 | stats = 0 161 | 162 | if self.transport == 'graphite': 163 | stat_string = '' 164 | elif self.transport == 'ganglia': 165 | g = gmetric.Gmetric(self.ganglia_host, self.ganglia_port, self.ganglia_protocol) 166 | 167 | for k, (v, t) in self.counters.items(): 168 | if self.expire > 0 and t + self.expire < ts: 169 | if self.debug: 170 | print("Expiring counter %s (age: %s)" % (k, ts -t)) 171 | del(self.counters[k]) 172 | continue 173 | v = float(v) 174 | v = v if self.no_aggregate_counters else v / (self.flush_interval / 1000) 175 | 176 | if self.debug: 177 | print("Sending %s => count=%s" % (k, v)) 178 | 179 | if self.transport == 'graphite': 180 | msg = '%s.%s %s %s\n' % (self.counters_prefix, k, v, ts) 181 | stat_string += msg 182 | elif self.transport == 'ganglia': 183 | # We put counters in _counters group. Underscore is to make sure counters show up 184 | # first in the GUI. Change below if you disagree 185 | if len(k) >= self.ganglia_max_length: 186 | log.debug("Ganglia metric too long. Ignoring: %s" % k) 187 | else: 188 | g.send(k, v, "double", "count", "both", 60, self.dmax, "_counters", self.ganglia_spoof_host) 189 | elif self.transport == 'ganglia-gmetric': 190 | self.send_to_ganglia_using_gmetric(k,v, "_counters", "count") 191 | 192 | # Clear the counter once the data is sent 193 | del(self.counters[k]) 194 | stats += 1 195 | 196 | for k, (v, t) in self.gauges.items(): 197 | if self.expire > 0 and t + self.expire < ts: 198 | if self.debug: 199 | print("Expiring gauge %s (age: %s)" % (k, ts - t)) 200 | del(self.gauges[k]) 201 | continue 202 | v = float(v) 203 | 204 | if self.debug: 205 | print("Sending %s => value=%s" % (k, v)) 206 | 207 | if self.transport == 'graphite': 208 | # note: counters and gauges implicitly end up in the same namespace 209 | msg = '%s.%s %s %s\n' % (self.counters_prefix, k, v, ts) 210 | stat_string += msg 211 | elif self.transport == 'ganglia': 212 | if len(k) >= self.ganglia_max_length: 213 | log.debug("Ganglia metric too long. Ignoring: %s" % k) 214 | else: 215 | g.send(k, v, "double", "count", "both", 60, self.dmax, "_gauges", self.ganglia_spoof_host) 216 | elif self.transport == 'ganglia-gmetric': 217 | self.send_to_ganglia_using_gmetric(k,v, "_gauges", "gauge") 218 | 219 | stats += 1 220 | 221 | for k, (v, t) in self.timers.items(): 222 | if self.expire > 0 and t + self.expire < ts: 223 | if self.debug: 224 | print("Expiring timer %s (age: %s)" % (k, ts - t)) 225 | del(self.timers[k]) 226 | continue 227 | if len(v) > 0: 228 | # Sort all the received values. We need it to extract percentiles 229 | v.sort() 230 | count = len(v) 231 | min = v[0] 232 | max = v[-1] 233 | 234 | mean = min 235 | max_threshold = max 236 | 237 | if count > 1: 238 | thresh_index = int((self.pct_threshold / 100.0) * count) 239 | max_threshold = v[thresh_index - 1] 240 | total = sum(v) 241 | mean = total / count 242 | 243 | del(self.timers[k]) 244 | 245 | if self.debug: 246 | print("Sending %s ====> lower=%s, mean=%s, upper=%s, %dpct=%s, count=%s" \ 247 | % (k, min, mean, max, self.pct_threshold, max_threshold, count)) 248 | 249 | if self.transport == 'graphite': 250 | 251 | stat_string += TIMER_MSG % { 252 | 'prefix': self.timers_prefix, 253 | 'key': k, 254 | 'mean': mean, 255 | 'max': max, 256 | 'min': min, 257 | 'count': count, 258 | 'max_threshold': max_threshold, 259 | 'pct_threshold': self.pct_threshold, 260 | 'ts': ts, 261 | } 262 | 263 | elif self.transport == 'ganglia': 264 | # We are gonna convert all times into seconds, then let rrdtool add proper SI unit. This avoids things like 265 | # 3521 k ms which is 3.521 seconds 266 | # What group should these metrics be in. For the time being we'll set it to the name of the key 267 | if len(k) >= self.ganglia_max_length: 268 | log.debug("Ganglia metric too long. Ignoring: %s" % k) 269 | else: 270 | group = k 271 | g.send(k + "_min", min / 1000, "double", "seconds", "both", 60, self.dmax, group, self.ganglia_spoof_host) 272 | g.send(k + "_mean", mean / 1000, "double", "seconds", "both", 60, self.dmax, group, self.ganglia_spoof_host) 273 | g.send(k + "_max", max / 1000, "double", "seconds", "both", 60, self.dmax, group, self.ganglia_spoof_host) 274 | g.send(k + "_count", count, "double", "count", "both", 60, self.dmax, group, self.ganglia_spoof_host) 275 | g.send(k + "_" + str(self.pct_threshold) + "pct", max_threshold / 1000, "double", "seconds", "both", 60, self.dmax, group, self.ganglia_spoof_host) 276 | elif self.transport == 'ganglia-gmetric': 277 | # We are gonna convert all times into seconds, then let rrdtool add proper SI unit. This avoids things like 278 | # 3521 k ms which is 3.521 seconds 279 | group = k 280 | self.send_to_ganglia_using_gmetric(k + "_mean", mean / 1000, group, "seconds") 281 | self.send_to_ganglia_using_gmetric(k + "_min", min / 1000 , group, "seconds") 282 | self.send_to_ganglia_using_gmetric(k + "_max", max / 1000, group, "seconds") 283 | self.send_to_ganglia_using_gmetric(k + "_count", count , group, "count") 284 | self.send_to_ganglia_using_gmetric(k + "_" + str(self.pct_threshold) + "pct", max_threshold / 1000, group, "seconds") 285 | 286 | stats += 1 287 | 288 | if self.transport == 'graphite': 289 | 290 | stat_string += "statsd.numStats %s %d\n" % (stats, ts) 291 | 292 | # Prepend stats with Hosted Graphite API key if necessary 293 | if self.global_prefix: 294 | stat_string = '\n'.join([ 295 | '%s.%s' % (self.global_prefix, s) for s in stat_string.split('\n')[:-1] 296 | ]) 297 | 298 | graphite = socket.socket() 299 | try: 300 | graphite.connect((self.graphite_host, self.graphite_port)) 301 | graphite.sendall(bytes(bytearray(stat_string, "utf-8"))) 302 | graphite.close() 303 | except socket.error as e: 304 | log.error("Error communicating with Graphite: %s" % e) 305 | if self.debug: 306 | print("Error communicating with Graphite: %s" % e) 307 | 308 | if self.debug: 309 | print("\n================== Flush completed. Waiting until next flush. Sent out %d metrics =======" \ 310 | % (stats)) 311 | 312 | def _set_timer(self): 313 | self._timer = threading.Timer(self.flush_interval / 1000, self.on_timer) 314 | self._timer.daemon = True 315 | self._timer.start() 316 | 317 | def serve(self, hostname='', port=8125): 318 | assert type(port) is int, 'port is not an integer: %s' % (port) 319 | addr = (hostname, port) 320 | self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 321 | self._sock.bind(addr) 322 | 323 | import signal 324 | 325 | def signal_handler(signal, frame): 326 | self.stop() 327 | signal.signal(signal.SIGINT, signal_handler) 328 | 329 | self._set_timer() 330 | while 1: 331 | data, addr = self._sock.recvfrom(self.buf) 332 | try: 333 | self.process(data) 334 | except Exception as error: 335 | log.error("Bad data from %s: %s",addr,error) 336 | 337 | 338 | def stop(self): 339 | self._timer.cancel() 340 | self._sock.close() 341 | 342 | 343 | class ServerDaemon(Daemon): 344 | def run(self, options): 345 | if setproctitle: 346 | setproctitle('pystatsd') 347 | server = Server(pct_threshold=options.pct, 348 | debug=options.debug, 349 | transport=options.transport, 350 | graphite_host=options.graphite_host, 351 | graphite_port=options.graphite_port, 352 | global_prefix=options.global_prefix, 353 | ganglia_host=options.ganglia_host, 354 | ganglia_spoof_host=options.ganglia_spoof_host, 355 | ganglia_port=options.ganglia_port, 356 | gmetric_exec=options.gmetric_exec, 357 | gmetric_options=options.gmetric_options, 358 | flush_interval=options.flush_interval, 359 | no_aggregate_counters=options.no_aggregate_counters, 360 | counters_prefix=options.counters_prefix, 361 | timers_prefix=options.timers_prefix, 362 | expire=options.expire) 363 | 364 | server.serve(options.name, options.port) 365 | 366 | 367 | def run_server(): 368 | import sys 369 | import argparse 370 | parser = argparse.ArgumentParser() 371 | parser.add_argument('-d', '--debug', dest='debug', action='store_true', help='debug mode', default=False) 372 | parser.add_argument('-n', '--name', dest='name', help='hostname to run on ', default='') 373 | parser.add_argument('-p', '--port', dest='port', help='port to run on (default: 8125)', type=int, default=8125) 374 | parser.add_argument('-r', '--transport', dest='transport', help='transport to use graphite, ganglia (uses embedded library) or ganglia-gmetric (uses gmetric)', type=str, default="graphite") 375 | parser.add_argument('--graphite-port', dest='graphite_port', help='port to connect to graphite on (default: 2003)', type=int, default=2003) 376 | parser.add_argument('--graphite-host', dest='graphite_host', help='host to connect to graphite on (default: localhost)', type=str, default='localhost') 377 | # Uses embedded Ganglia Library 378 | parser.add_argument('--ganglia-port', dest='ganglia_port', help='Unicast port to connect to ganglia on', type=int, default=8649) 379 | parser.add_argument('--ganglia-host', dest='ganglia_host', help='Unicast host to connect to ganglia on', type=str, default='localhost') 380 | parser.add_argument('--ganglia-spoof-host', dest='ganglia_spoof_host', help='host to report metrics as to ganglia', type=str, default='statsd:statsd') 381 | # Use gmetric 382 | parser.add_argument('--ganglia-gmetric-exec', dest='gmetric_exec', help='Use gmetric executable. Defaults to /usr/bin/gmetric', type=str, default="/usr/bin/gmetric") 383 | parser.add_argument('--ganglia-gmetric-options', dest='gmetric_options', help='Options to pass to gmetric. Defaults to -d 60', type=str, default="-d 60") 384 | # Common for ganglia 385 | parser.add_argument('--ganglia-max-length', dest='ganglia_max_length', help='Maximum length of metric names for ganglia. Defaults to 100 characters', type=str, default=100) 386 | # 387 | parser.add_argument('--flush-interval', dest='flush_interval', help='how often to send data to graphite in millis (default: 10000)', type=int, default=10000) 388 | parser.add_argument('--no-aggregate-counters', dest='no_aggregate_counters', help='should statsd report counters as absolute instead of count/sec', action='store_true') 389 | parser.add_argument('--global-prefix', dest='global_prefix', help='prefix to append to all stats sent to graphite. Useful for hosted services (ex: Hosted Graphite) or stats namespacing (default: None)', type=str, default=None) 390 | parser.add_argument('--counters-prefix', dest='counters_prefix', help='prefix to append before sending counter data to graphite (default: stats)', type=str, default='stats') 391 | parser.add_argument('--timers-prefix', dest='timers_prefix', help='prefix to append before sending timing data to graphite (default: stats.timers)', type=str, default='stats.timers') 392 | parser.add_argument('-t', '--pct', dest='pct', help='stats pct threshold (default: 90)', type=int, default=90) 393 | parser.add_argument('-D', '--daemon', dest='daemonize', action='store_true', help='daemonize', default=False) 394 | parser.add_argument('--pidfile', dest='pidfile', action='store', help='pid file', default='/var/run/pystatsd.pid') 395 | parser.add_argument('--restart', dest='restart', action='store_true', help='restart a running daemon', default=False) 396 | parser.add_argument('--stop', dest='stop', action='store_true', help='stop a running daemon', default=False) 397 | parser.add_argument('--expire', dest='expire', help='time-to-live for old stats (in secs)', type=int, default=0) 398 | options = parser.parse_args(sys.argv[1:]) 399 | 400 | log_level = logging.DEBUG if options.debug else logging.INFO 401 | logging.basicConfig(level=log_level,format='%(asctime)s [%(levelname)s] %(message)s') 402 | 403 | log.info("Starting up on %s" % options.port) 404 | daemon = ServerDaemon(options.pidfile) 405 | if options.daemonize: 406 | daemon.start(options) 407 | elif options.restart: 408 | daemon.restart(options) 409 | elif options.stop: 410 | daemon.stop() 411 | else: 412 | daemon.run(options) 413 | 414 | 415 | log = logging.getLogger(__name__) 416 | 417 | if __name__ == '__main__': 418 | run_server() 419 | -------------------------------------------------------------------------------- /pystatsd/statsd.py: -------------------------------------------------------------------------------- 1 | # statsd.py 2 | 3 | # Steve Ivy 4 | # http://monkinetic.com 5 | 6 | import logging 7 | import socket 8 | import random 9 | import time 10 | 11 | 12 | # Sends statistics to the stats daemon over UDP 13 | class Client(object): 14 | 15 | def __init__(self, host='localhost', port=8125, prefix=None): 16 | """ 17 | Create a new Statsd client. 18 | * host: the host where statsd is listening, defaults to localhost 19 | * port: the port where statsd is listening, defaults to 8125 20 | 21 | >>> from pystatsd import statsd 22 | >>> stats_client = statsd.Statsd(host, port) 23 | """ 24 | self.host = host 25 | self.port = int(port) 26 | self.addr = (socket.gethostbyname(self.host), self.port) 27 | self.prefix = prefix 28 | self.log = logging.getLogger("pystatsd.client") 29 | self.log.addHandler(logging.StreamHandler()) 30 | self.udp_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 31 | 32 | def timing_since(self, stat, start, sample_rate=1): 33 | """ 34 | Log timing information as the number of microseconds since the provided time float 35 | >>> start = time.time() 36 | >>> # do stuff 37 | >>> statsd_client.timing_since('some.time', start) 38 | """ 39 | self.timing(stat, int((time.time() - start) * 1000000), sample_rate) 40 | 41 | def timing(self, stat, time, sample_rate=1): 42 | """ 43 | Log timing information for a single stat 44 | >>> statsd_client.timing('some.time',500) 45 | """ 46 | stats = {stat: "%f|ms" % time} 47 | self.send(stats, sample_rate) 48 | 49 | def gauge(self, stat, value, sample_rate=1): 50 | """ 51 | Log gauge information for a single stat 52 | >>> statsd_client.gauge('some.gauge',42) 53 | """ 54 | stats = {stat: "%f|g" % value} 55 | self.send(stats, sample_rate) 56 | 57 | def increment(self, stats, sample_rate=1): 58 | """ 59 | Increments one or more stats counters 60 | >>> statsd_client.increment('some.int') 61 | >>> statsd_client.increment('some.int',0.5) 62 | """ 63 | self.update_stats(stats, 1, sample_rate=sample_rate) 64 | 65 | # alias 66 | incr = increment 67 | 68 | def decrement(self, stats, sample_rate=1): 69 | """ 70 | Decrements one or more stats counters 71 | >>> statsd_client.decrement('some.int') 72 | """ 73 | self.update_stats(stats, -1, sample_rate=sample_rate) 74 | 75 | # alias 76 | decr = decrement 77 | 78 | def update_stats(self, stats, delta, sample_rate=1): 79 | """ 80 | Updates one or more stats counters by arbitrary amounts 81 | >>> statsd_client.update_stats('some.int',10) 82 | """ 83 | if not isinstance(stats, list): 84 | stats = [stats] 85 | 86 | data = dict((stat, "%s|c" % delta) for stat in stats) 87 | self.send(data, sample_rate) 88 | 89 | def send(self, data, sample_rate=1): 90 | """ 91 | Squirt the metrics over UDP 92 | """ 93 | 94 | if self.prefix: 95 | data = dict((".".join((self.prefix, stat)), value) for stat, value in data.items()) 96 | 97 | if sample_rate < 1: 98 | if random.random() > sample_rate: 99 | return 100 | sampled_data = dict((stat, "%s|@%s" % (value, sample_rate)) 101 | for stat, value in data.items()) 102 | else: 103 | sampled_data = data 104 | 105 | try: 106 | [self.udp_sock.sendto(bytes(bytearray("%s:%s" % (stat, value), 107 | "utf-8")), self.addr) 108 | for stat, value in sampled_data.items()] 109 | except: 110 | self.log.exception("unexpected error") 111 | 112 | def __repr__(self): 113 | return "" % (self.addr, self.prefix) 114 | -------------------------------------------------------------------------------- /redhat/pystatsd-python2.4.patch: -------------------------------------------------------------------------------- 1 | diff -Naur pystatsd-0.1.7/pystatsd/daemon.py pystatsd-0.1.7.new/pystatsd/daemon.py 2 | --- pystatsd-0.1.7/pystatsd/daemon.py 2011-10-10 11:12:48.000000000 -0700 3 | +++ pystatsd-0.1.7.new/pystatsd/daemon.py 2011-10-10 11:07:40.000000000 -0700 4 | @@ -59,8 +59,11 @@ 5 | # Write the pidfile. 6 | atexit.register(self.delpid) 7 | pid = str(os.getpid()) 8 | - with open(self.pidfile, 'w+') as fp: 9 | + fp = open(self.pidfile, 'w+') 10 | + try: 11 | fp.write('%s\n' % pid) 12 | + finally: 13 | + fp.close() 14 | 15 | def delpid(self): 16 | os.remove(self.pidfile) 17 | @@ -69,8 +72,11 @@ 18 | """Start the daemon.""" 19 | pid = None 20 | if os.path.exists(self.pidfile): 21 | - with open(self.pidfile, 'r') as fp: 22 | + fp = open(self.pidfile, 'r') 23 | + try: 24 | pid = int(fp.read().strip()) 25 | + finally: 26 | + fp.close() 27 | 28 | if pid: 29 | msg = 'pidfile (%s) exists. Daemon already running?\n' 30 | @@ -84,8 +90,11 @@ 31 | """Stop the daemon.""" 32 | pid = None 33 | if os.path.exists(self.pidfile): 34 | - with open(self.pidfile, 'r') as fp: 35 | + fp = open(self.pidfile, 'r') 36 | + try: 37 | pid = int(fp.read().strip()) 38 | + finally: 39 | + fp.close() 40 | 41 | if not pid: 42 | msg = 'pidfile (%s) does not exist. Daemon not running?\n' 43 | diff -Naur pystatsd-0.1.7/pystatsd/server.py pystatsd-0.1.7.new/pystatsd/server.py 44 | --- pystatsd-0.1.7/pystatsd/server.py 2011-10-06 17:13:39.000000000 -0700 45 | +++ pystatsd-0.1.7.new/pystatsd/server.py 2011-10-10 11:59:36.000000000 -0700 46 | @@ -107,7 +107,10 @@ 47 | 48 | for k, v in self.counters.items(): 49 | v = float(v) 50 | - v = v if self.no_aggregate_counters else v / (self.flush_interval / 1000) 51 | + if self.no_aggregate_counters: 52 | + v = v 53 | + else: 54 | + v / (self.flush_interval / 1000) 55 | 56 | if self.debug: 57 | print "Sending %s => count=%s" % ( k, v ) 58 | -------------------------------------------------------------------------------- /redhat/pystatsd.spec: -------------------------------------------------------------------------------- 1 | %if 0%{?rhel} < 6 2 | %define needs_python24_patching 1 3 | %else 4 | %define needs_python24_patching 0 5 | %endif 6 | 7 | %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} 8 | 9 | Name: pystatsd 10 | Version: 0.1.7 11 | Release: 4%{?dist} 12 | Summary: Python implementation of the Statsd client/server 13 | Group: Applications/Internet 14 | License: Unknown 15 | URL: http://pypi.python.org/pypi/pystatsd/ 16 | Source0 http://pypi.python.org/packages/source/p/pystatsd/pystatsd-%{version}.tar.gz 17 | Patch0: pystatsd-python2.4.patch 18 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) 19 | BuildArch: noarch 20 | 21 | BuildRequires: python-devel 22 | %if 0%{?fedora} && 0%{?fedora} < 13 23 | BuildRequires: python-setuptools-devel 24 | %else 25 | BuildRequires: python-setuptools 26 | %endif 27 | 28 | Requires: python-argparse >= 1.2 29 | 30 | %description 31 | pystatsd is a client and server implementation of Etsy's brilliant statsd 32 | server, a front end/proxy for the Graphite stats collection and graphing server. 33 | 34 | * Graphite 35 | - http://graphite.wikidot.com 36 | * Statsd 37 | - code: https://github.com/etsy/statsd 38 | - blog post: http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/ 39 | 40 | %prep 41 | %setup -q 42 | 43 | %if %{needs_python24_patching} 44 | %patch0 -p1 45 | %endif 46 | 47 | %build 48 | %{__python} setup.py build 49 | 50 | %install 51 | rm -rf %{buildroot} 52 | %{__python} setup.py install --skip-build --root %{buildroot} 53 | mkdir -p %{buildroot}/etc/init.d 54 | install -m0755 init/pystatsd.init %{buildroot}/etc/init.d/pystatsd 55 | mkdir -p %{buildroot}/etc/default 56 | install -m0644 init/pystatsd.default %{buildroot}%{_sysconfdir}/default/pystatsd 57 | 58 | %clean 59 | rm -rf %{buildroot} 60 | 61 | %files 62 | %defattr(-,root,root,-) 63 | %doc README.md 64 | %config %{_sysconfdir}/default/pystatsd 65 | %{python_sitelib}/* 66 | /usr/bin/pystatsd-server 67 | /etc/init.d/pystatsd 68 | 69 | %changelog 70 | * Mon Apr 07 2014 Stefan Richter - 0.1.7-4 71 | - update to 4a60cbb2d8152925fa0d18b1666be3bad2e2884b 72 | - also use/install /etc/default/pystatsd 73 | * Wed Aug 15 2012 Bruno Clermont - 0.1.7-3 74 | - update to 36a59d3b126ded4658aff25bce94e844a1c6413e 75 | - Fix path to README file 76 | * Fri Mar 02 2012 Justin Burnham - 0.1.7-2 77 | - Add python-argparse requires. 78 | - Add init file to MANIFEST.in for setup.py sdist. 79 | * Thu Oct 07 2011 Sharif Nassar - 0.1.7-1 80 | - Initial package 81 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | nose 2 | mock 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup 3 | from pystatsd import VERSION 4 | 5 | def read(fname): 6 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 7 | 8 | setup( 9 | name = "pystatsd", 10 | version=".".join(map(str, VERSION)), 11 | author = "Steve Ivy", 12 | author_email = "steveivy@gmail.com", 13 | description = ("pystatsd is a client for Etsy's statsd server, a front end/proxy for the Graphite stats collection and graphing server."), 14 | url='https://github.com/sivy/py-statsd', 15 | license = "BSD", 16 | packages=['pystatsd'], 17 | long_description=read('README.md'), 18 | classifiers=[ 19 | "License :: OSI Approved :: BSD License", 20 | "Programming Language :: Python :: 2.7", 21 | "Programming Language :: Python :: 3.5", 22 | "Programming Language :: Python :: 3.6", 23 | "Programming Language :: Python :: 3.7", 24 | "Programming Language :: Python :: 3.8", 25 | ], 26 | scripts=['bin/pystatsd-server'] 27 | ) 28 | -------------------------------------------------------------------------------- /statsd_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from pystatsd import Client, Server 4 | 5 | sc = Client('localhost', 8125) 6 | 7 | sc.timing('python_test.time', 500) 8 | sc.increment('python_test.inc_int') 9 | sc.decrement('python_test.decr_int') 10 | sc.gauge('python_test.gauge', 42) 11 | 12 | srvr = Server(debug=True) 13 | srvr.serve() 14 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | from .client import * 2 | from .server import * 3 | -------------------------------------------------------------------------------- /tests/client.py: -------------------------------------------------------------------------------- 1 | import time 2 | import unittest 3 | import mock 4 | import socket 5 | import sys 6 | 7 | from pystatsd.statsd import Client 8 | 9 | 10 | if sys.version_info[0] < 3: 11 | def bytes(s, encode): 12 | return s 13 | 14 | 15 | class ClientBasicsTestCase(unittest.TestCase): 16 | """ 17 | Tests the basic operations of the client 18 | """ 19 | def setUp(self): 20 | self.patchers = [] 21 | 22 | socket_patcher = mock.patch('pystatsd.statsd.socket.socket') 23 | self.mock_socket = socket_patcher.start() 24 | self.patchers.append(socket_patcher) 25 | 26 | self.client = Client() 27 | self.addr = (socket.gethostbyname(self.client.host), self.client.port) 28 | 29 | def test_client_create(self): 30 | host, port = ('example.com', 8888) 31 | 32 | client = Client( 33 | host=host, 34 | port=port, 35 | prefix='pystatsd.tests') 36 | self.assertEqual(client.host, host) 37 | self.assertEqual(client.port, port) 38 | self.assertEqual(client.prefix, 'pystatsd.tests') 39 | self.assertEqual(client.addr, (socket.gethostbyname(host), port)) 40 | 41 | def test_basic_client_incr(self): 42 | stat = 'pystatsd.unittests.test_basic_client_incr' 43 | stat_str = stat + ':1|c' 44 | 45 | self.client.increment(stat) 46 | 47 | # thanks tos9 in #python for 'splaining the return_value bit. 48 | self.mock_socket.return_value.sendto.assert_called_with( 49 | bytes(stat_str, 'utf-8'), self.addr) 50 | 51 | def test_basic_client_decr(self): 52 | stat = 'pystatsd.unittests.test_basic_client_decr' 53 | stat_str = stat + ':-1|c' 54 | 55 | self.client.decrement(stat) 56 | 57 | # thanks tos9 in #python for 'splaining the return_value bit. 58 | self.mock_socket.return_value.sendto.assert_called_with( 59 | bytes(stat_str, 'utf-8'), self.addr) 60 | 61 | def test_basic_client_update_stats(self): 62 | stat = 'pystatsd.unittests.test_basic_client_update_stats' 63 | stat_str = stat + ':5|c' 64 | 65 | self.client.update_stats(stat, 5) 66 | 67 | # thanks tos9 in #python for 'splaining the return_value bit. 68 | self.mock_socket.return_value.sendto.assert_called_with( 69 | bytes(stat_str, 'utf-8'), self.addr) 70 | 71 | def test_basic_client_update_stats_multi(self): 72 | stats = [ 73 | 'pystatsd.unittests.test_basic_client_update_stats', 74 | 'pystatsd.unittests.test_basic_client_update_stats_multi' 75 | ] 76 | 77 | data = dict((stat, ":%s|c" % '5') for stat in stats) 78 | 79 | self.client.update_stats(stats, 5) 80 | 81 | for stat, value in data.items(): 82 | stat_str = stat + value 83 | # thanks tos9 in #python for 'splaining the return_value bit. 84 | self.mock_socket.return_value.sendto.assert_any_call( 85 | bytes(stat_str, 'utf-8'), self.addr) 86 | 87 | def test_basic_client_timing(self): 88 | stat = 'pystatsd.unittests.test_basic_client_timing.time' 89 | stat_str = stat + ':5.000000|ms' 90 | 91 | self.client.timing(stat, 5) 92 | 93 | # thanks tos9 in #python for 'splaining the return_value bit. 94 | self.mock_socket.return_value.sendto.assert_called_with( 95 | bytes(stat_str, 'utf-8'), self.addr) 96 | 97 | def test_basic_client_timing_since(self): 98 | ts = (1971, 6, 29, 4, 13, 0, 0, 0, -1) 99 | now = time.mktime(ts) 100 | # add 5 seconds 101 | ts = (1971, 6, 29, 4, 13, 5, 0, 0, -1) 102 | then = time.mktime(ts) 103 | mock_time_patcher = mock.patch('time.time', return_value=now) 104 | mock_time_patcher.start() 105 | 106 | stat = 'pystatsd.unittests.test_basic_client_timing_since.time' 107 | stat_str = stat + ':-5000000.000000|ms' 108 | 109 | self.client.timing_since(stat, then) 110 | 111 | # thanks tos9 in #python for 'splaining the return_value bit. 112 | self.mock_socket.return_value.sendto.assert_called_with( 113 | bytes(stat_str, 'utf-8'), self.addr) 114 | 115 | mock_time_patcher.stop() 116 | 117 | def tearDown(self): 118 | for patcher in self.patchers: 119 | patcher.stop() 120 | -------------------------------------------------------------------------------- /tests/server.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import mock 3 | 4 | # from pystatsd.statsd import Client 5 | from pystatsd.server import Server 6 | 7 | 8 | class ServerBasicsTestCase(unittest.TestCase): 9 | """ 10 | Tests the basic operations of the client 11 | """ 12 | def setUp(self): 13 | self.patchers = [] 14 | 15 | socket_patcher = mock.patch('pystatsd.statsd.socket.socket') 16 | self.mock_socket = socket_patcher.start() 17 | self.patchers.append(socket_patcher) 18 | 19 | def test_server_create(self): 20 | server = Server() 21 | 22 | if getattr(self, "assertIsNotNone", False): 23 | self.assertIsNotNone(server) 24 | else: 25 | assert server is not None 26 | --------------------------------------------------------------------------------