├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.rst ├── example1.py ├── example2.py ├── logstash ├── __init__.py ├── formatter.py ├── handler_amqp.py ├── handler_tcp.py └── handler_udp.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | MANIFEST 3 | .idea 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Packages 9 | *.egg 10 | *.egg-info 11 | dist 12 | build 13 | eggs 14 | parts 15 | bin 16 | var 17 | sdist 18 | develop-eggs 19 | .installed.cfg 20 | lib 21 | lib64 22 | 23 | # Installer logs 24 | pip-log.txt 25 | 26 | # Unit test / coverage reports 27 | .coverage 28 | .tox 29 | nosetests.xml 30 | 31 | # Translations 32 | *.mo 33 | 34 | # Mr Developer 35 | .mr.developer.cfg 36 | .project 37 | .pydevproject 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013, Volodymyr Klochan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include example1.py 4 | include example2.py 5 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | python-logstash 2 | =============== 3 | 4 | Python logging handler for Logstash. 5 | http://logstash.net/ 6 | 7 | Changelog 8 | ========= 9 | 0.4.8 10 | - Fixed Python 3 issues with JSON serialization. 11 | 0.4.7 12 | - Add couple of sensitive fields to the skip_list 13 | 0.4.6 14 | - Updated field names to match java counterparts supported by logstash crew 15 | 0.4.5 16 | - Allow passing exchange's routing key to AMQP handler 17 | 0.4.4 18 | - Fixed urllib import python3 compatibility. 19 | - Added long type to easy_types. 20 | 0.4.3 21 | - Added AMQP handler. 22 | 0.4.2 23 | - Updated README 24 | - Added ``tags`` parameter to handler 25 | 0.4.1 26 | - Added TCP handler. 27 | 0.3.1 28 | - Added support for Python 3 29 | 0.2.2 30 | - Split Handler into Handler and Formatter classes 31 | 0.2.1 32 | - Added support for the new JSON schema in Logstash 1.2.x. See details in 33 | http://tobrunet.ch/2013/09/logstash-1-2-0-upgrade-notes-included/ and 34 | https://logstash.jira.com/browse/LOGSTASH-675 35 | 36 | - Added ``version`` parameter. Available values: 1 (Logstash 1.2.x version format), 0 - default (previous version). 37 | 38 | 39 | Installation 40 | ============ 41 | 42 | Using pip:: 43 | 44 | pip install python-logstash 45 | 46 | Usage 47 | ===== 48 | 49 | ``LogstashHandler`` is a custom logging handler which sends Logstash messages using UDP. 50 | 51 | For example:: 52 | 53 | import logging 54 | import logstash 55 | import sys 56 | 57 | host = 'localhost' 58 | 59 | test_logger = logging.getLogger('python-logstash-logger') 60 | test_logger.setLevel(logging.INFO) 61 | test_logger.addHandler(logstash.LogstashHandler(host, 5959, version=1)) 62 | # test_logger.addHandler(logstash.TCPLogstashHandler(host, 5959, version=1)) 63 | 64 | test_logger.error('python-logstash: test logstash error message.') 65 | test_logger.info('python-logstash: test logstash info message.') 66 | test_logger.warning('python-logstash: test logstash warning message.') 67 | 68 | # add extra field to logstash message 69 | extra = { 70 | 'test_string': 'python version: ' + repr(sys.version_info), 71 | 'test_boolean': True, 72 | 'test_dict': {'a': 1, 'b': 'c'}, 73 | 'test_float': 1.23, 74 | 'test_integer': 123, 75 | 'test_list': [1, 2, '3'], 76 | } 77 | test_logger.info('python-logstash: test extra fields', extra=extra) 78 | 79 | When using ``extra`` field make sure you don't use reserved names. From `Python documentation `_. 80 | | "The keys in the dictionary passed in extra should not clash with the keys used by the logging system. (See the `Formatter `_ documentation for more information on which keys are used by the logging system.)" 81 | 82 | To use the AMQPLogstashHandler you will need to install pika first. 83 | 84 | pip install pika 85 | 86 | For example:: 87 | 88 | import logging 89 | import logstash 90 | 91 | test_logger = logging.getLogger('python-logstash-logger') 92 | test_logger.setLevel(logging.INFO) 93 | test_logger.addHandler(logstash.AMQPLogstashHandler(host='localhost', version=1)) 94 | 95 | test_logger.info('python-logstash: test logstash info message.') 96 | try: 97 | 1/0 98 | except: 99 | test_logger.exception('python-logstash-logger: Exception with stack trace!') 100 | 101 | 102 | 103 | Using with Django 104 | ================= 105 | 106 | Modify your ``settings.py`` to integrate ``python-logstash`` with Django's logging:: 107 | 108 | LOGGING = { 109 | ... 110 | 'handlers': { 111 | 'logstash': { 112 | 'level': 'DEBUG', 113 | 'class': 'logstash.LogstashHandler', 114 | 'host': 'localhost', 115 | 'port': 5959, # Default value: 5959 116 | 'version': 1, # Version of logstash event schema. Default value: 0 (for backward compatibility of the library) 117 | 'message_type': 'logstash', # 'type' field in logstash message. Default value: 'logstash'. 118 | 'fqdn': False, # Fully qualified domain name. Default value: false. 119 | 'tags': ['tag1', 'tag2'], # list of tags. Default: None. 120 | }, 121 | }, 122 | 'loggers': { 123 | 'django.request': { 124 | 'handlers': ['logstash'], 125 | 'level': 'DEBUG', 126 | 'propagate': True, 127 | }, 128 | }, 129 | ... 130 | } 131 | 132 | Example Logstash Configuration 133 | ============================== 134 | 135 | Example Logstash Configuration (``logstash.conf``) for Receiving Events from python-logstash is:: 136 | 137 | input { 138 | udp { 139 | port => 5959 140 | codec => json 141 | } 142 | } 143 | output { 144 | stdout { 145 | codec => rubydebug 146 | } 147 | } 148 | 149 | For TCP input you need to change the logstash's input to ``tcp`` and modify django log handler's class to ``logstash.TCPLogstashHandler`` 150 | -------------------------------------------------------------------------------- /example1.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import logstash 3 | import sys 4 | 5 | host = 'localhost' 6 | 7 | test_logger = logging.getLogger('python-logstash-logger') 8 | test_logger.setLevel(logging.INFO) 9 | test_logger.addHandler(logstash.LogstashHandler(host, 5959, version=1)) 10 | # test_logger.addHandler(logstash.TCPLogstashHandler(host, 5959, version=1)) 11 | 12 | test_logger.error('python-logstash: test logstash error message.') 13 | test_logger.info('python-logstash: test logstash info message.') 14 | test_logger.warning('python-logstash: test logstash warning message.') 15 | 16 | # add extra field to logstash message 17 | extra = { 18 | 'test_string': 'python version: ' + repr(sys.version_info), 19 | 'test_boolean': True, 20 | 'test_dict': {'a': 1, 'b': 'c'}, 21 | 'test_float': 1.23, 22 | 'test_integer': 123, 23 | 'test_list': [1, 2, '3'], 24 | } 25 | test_logger.info('python-logstash: test extra fields', extra=extra) 26 | -------------------------------------------------------------------------------- /example2.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import logstash 3 | 4 | # AMQP parameters 5 | host = 'localhost' 6 | username = 'guest' 7 | password= 'guest' 8 | exchange = 'logstash.py' 9 | 10 | # get a logger and set logging level 11 | test_logger = logging.getLogger('python-logstash-logger') 12 | test_logger.setLevel(logging.INFO) 13 | 14 | # add the handler 15 | test_logger.addHandler(logstash.AMQPLogstashHandler(version=1, 16 | host=host, 17 | durable=True, 18 | username=username, 19 | password=password, 20 | exchange=exchange)) 21 | 22 | # log 23 | test_logger.error('python-logstash: test logstash error message.') 24 | test_logger.info('python-logstash: test logstash info message.') 25 | test_logger.warning('python-logstash: test logstash warning message.') 26 | 27 | try: 28 | 1/0 29 | except: 30 | test_logger.exception('python-logstash: test logstash exception with stack trace') 31 | -------------------------------------------------------------------------------- /logstash/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from logstash.formatter import LogstashFormatterVersion0, LogstashFormatterVersion1 3 | 4 | from logstash.handler_tcp import TCPLogstashHandler 5 | from logstash.handler_udp import UDPLogstashHandler, LogstashHandler 6 | try: 7 | from logstash.handler_amqp import AMQPLogstashHandler 8 | except: 9 | # you need to install AMQP support to enable this handler. 10 | pass 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /logstash/formatter.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | import logging 3 | import socket 4 | import sys 5 | from datetime import datetime 6 | try: 7 | import json 8 | except ImportError: 9 | import simplejson as json 10 | 11 | 12 | class LogstashFormatterBase(logging.Formatter): 13 | 14 | def __init__(self, message_type='Logstash', tags=None, fqdn=False): 15 | self.message_type = message_type 16 | self.tags = tags if tags is not None else [] 17 | 18 | if fqdn: 19 | self.host = socket.getfqdn() 20 | else: 21 | self.host = socket.gethostname() 22 | 23 | def get_extra_fields(self, record): 24 | # The list contains all the attributes listed in 25 | # http://docs.python.org/library/logging.html#logrecord-attributes 26 | skip_list = ( 27 | 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename', 28 | 'funcName', 'id', 'levelname', 'levelno', 'lineno', 'module', 29 | 'msecs', 'msecs', 'message', 'msg', 'name', 'pathname', 'process', 30 | 'processName', 'relativeCreated', 'thread', 'threadName', 'extra', 31 | 'auth_token', 'password', 'stack_info') 32 | 33 | if sys.version_info < (3, 0): 34 | easy_types = (basestring, bool, dict, float, int, long, list, type(None)) 35 | else: 36 | easy_types = (str, bool, dict, float, int, list, type(None)) 37 | 38 | fields = {} 39 | 40 | for key, value in record.__dict__.items(): 41 | if key not in skip_list: 42 | if isinstance(value, easy_types): 43 | fields[key] = value 44 | else: 45 | fields[key] = repr(value) 46 | 47 | return fields 48 | 49 | def get_debug_fields(self, record): 50 | fields = { 51 | 'stack_trace': self.format_exception(record.exc_info), 52 | 'lineno': record.lineno, 53 | 'process': record.process, 54 | 'thread_name': record.threadName, 55 | } 56 | 57 | # funcName was added in 2.5 58 | if not getattr(record, 'funcName', None): 59 | fields['funcName'] = record.funcName 60 | 61 | # processName was added in 2.6 62 | if not getattr(record, 'processName', None): 63 | fields['processName'] = record.processName 64 | 65 | return fields 66 | 67 | @classmethod 68 | def format_source(cls, message_type, host, path): 69 | return "%s://%s/%s" % (message_type, host, path) 70 | 71 | @classmethod 72 | def format_timestamp(cls, time): 73 | tstamp = datetime.utcfromtimestamp(time) 74 | return tstamp.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % (tstamp.microsecond / 1000) + "Z" 75 | 76 | @classmethod 77 | def format_exception(cls, exc_info): 78 | return ''.join(traceback.format_exception(*exc_info)) if exc_info else '' 79 | 80 | @classmethod 81 | def serialize(cls, message): 82 | if sys.version_info < (3, 0): 83 | return json.dumps(message) 84 | else: 85 | return bytes(json.dumps(message, default=str), 'utf-8') 86 | 87 | class LogstashFormatterVersion0(LogstashFormatterBase): 88 | version = 0 89 | 90 | def format(self, record): 91 | # Create message dict 92 | message = { 93 | '@timestamp': self.format_timestamp(record.created), 94 | '@message': record.getMessage(), 95 | '@source': self.format_source(self.message_type, self.host, 96 | record.pathname), 97 | '@source_host': self.host, 98 | '@source_path': record.pathname, 99 | '@tags': self.tags, 100 | '@type': self.message_type, 101 | '@fields': { 102 | 'levelname': record.levelname, 103 | 'logger': record.name, 104 | }, 105 | } 106 | 107 | # Add extra fields 108 | message['@fields'].update(self.get_extra_fields(record)) 109 | 110 | # If exception, add debug info 111 | if record.exc_info: 112 | message['@fields'].update(self.get_debug_fields(record)) 113 | 114 | return self.serialize(message) 115 | 116 | 117 | class LogstashFormatterVersion1(LogstashFormatterBase): 118 | 119 | def format(self, record): 120 | # Create message dict 121 | message = { 122 | '@timestamp': self.format_timestamp(record.created), 123 | '@version': '1', 124 | 'message': record.getMessage(), 125 | 'host': self.host, 126 | 'path': record.pathname, 127 | 'tags': self.tags, 128 | 'type': self.message_type, 129 | 130 | # Extra Fields 131 | 'level': record.levelname, 132 | 'logger_name': record.name, 133 | } 134 | 135 | # Add extra fields 136 | message.update(self.get_extra_fields(record)) 137 | 138 | # If exception, add debug info 139 | if record.exc_info: 140 | message.update(self.get_debug_fields(record)) 141 | 142 | return self.serialize(message) 143 | -------------------------------------------------------------------------------- /logstash/handler_amqp.py: -------------------------------------------------------------------------------- 1 | import json 2 | try: 3 | from urllib import urlencode 4 | except ImportError: 5 | from urllib.parse import urlencode 6 | 7 | from logging import Filter 8 | from logging.handlers import SocketHandler 9 | 10 | import pika 11 | from logstash import formatter 12 | 13 | 14 | class AMQPLogstashHandler(SocketHandler, object): 15 | """AMQP Log Format handler 16 | 17 | :param host: AMQP host (default 'localhost') 18 | :param port: AMQP port (default 5672) 19 | :param username: AMQP user name (default 'guest', which is the default for 20 | RabbitMQ) 21 | :param password: AMQP password (default 'guest', which is the default for 22 | RabbitMQ) 23 | 24 | :param exchange: AMQP exchange. Default 'logging.gelf'. 25 | A queue binding must be defined on the server to prevent 26 | log messages from being dropped. 27 | :param exchange_type: AMQP exchange type (default 'fanout'). 28 | :param durable: AMQP exchange is durable (default False) 29 | :param virtual_host: AMQP virtual host (default '/'). 30 | :param passive: exchange is declared passively, meaning that an error is 31 | raised if the exchange does not exist, and succeeds otherwise. This is 32 | useful if the user does not have configure permission on the exchange. 33 | 34 | :param tags: list of tags for a logger (default is None). 35 | :param message_type: The type of the message (default logstash). 36 | :param version: version of logstash event schema (default is 0). 37 | 38 | :param extra_fields: Send extra fields on the log record to graylog 39 | if true (the default) 40 | :param fqdn: Use fully qualified domain name of localhost as source 41 | host (socket.getfqdn()). 42 | :param facility: Replace facility with specified value. If specified, 43 | record.name will be passed as `logger` parameter. 44 | """ 45 | 46 | def __init__(self, host='localhost', port=5672, username='guest', 47 | password='guest', exchange='logstash', exchange_type='fanout', 48 | virtual_host='/', message_type='logstash', tags=None, 49 | durable=False, passive=False, version=0, extra_fields=True, 50 | fqdn=False, facility=None, exchange_routing_key=''): 51 | 52 | 53 | # AMQP parameters 54 | self.host = host 55 | self.port = port 56 | self.username = username 57 | self.password = password 58 | self.exchange_type = exchange_type 59 | self.exchange = exchange 60 | self.exchange_is_durable = durable 61 | self.declare_exchange_passively = passive 62 | self.virtual_host = virtual_host 63 | self.routing_key = exchange_routing_key 64 | 65 | SocketHandler.__init__(self, host, port) 66 | 67 | # Extract Logstash paramaters 68 | self.tags = tags or [] 69 | fn = formatter.LogstashFormatterVersion1 if version == 1 \ 70 | else formatter.LogstashFormatterVersion0 71 | self.formatter = fn(message_type, tags, fqdn) 72 | 73 | # Standard logging parameters 74 | self.extra_fields = extra_fields 75 | self.fqdn = fqdn 76 | self.facility = facility 77 | 78 | def makeSocket(self, **kwargs): 79 | 80 | return PikaSocket(self.host, 81 | self.port, 82 | self.username, 83 | self.password, 84 | self.virtual_host, 85 | self.exchange, 86 | self.routing_key, 87 | self.exchange_is_durable, 88 | self.declare_exchange_passively, 89 | self.exchange_type) 90 | 91 | def makePickle(self, record): 92 | return self.formatter.format(record) 93 | 94 | 95 | class PikaSocket(object): 96 | 97 | def __init__(self, host, port, username, password, virtual_host, exchange, 98 | routing_key, durable, passive, exchange_type): 99 | 100 | # create connection parameters 101 | credentials = pika.PlainCredentials(username, password) 102 | parameters = pika.ConnectionParameters(host, port, virtual_host, 103 | credentials) 104 | 105 | # create connection & channel 106 | self.connection = pika.BlockingConnection(parameters) 107 | self.channel = self.connection.channel() 108 | 109 | # create an exchange, if needed 110 | self.channel.exchange_declare(exchange=exchange, 111 | exchange_type=exchange_type, 112 | passive=passive, 113 | durable=durable) 114 | 115 | # needed when publishing 116 | self.spec = pika.spec.BasicProperties(delivery_mode=2) 117 | self.routing_key = routing_key 118 | self.exchange = exchange 119 | 120 | 121 | def sendall(self, data): 122 | 123 | self.channel.basic_publish(self.exchange, 124 | self.routing_key, 125 | data, 126 | properties=self.spec) 127 | 128 | def close(self): 129 | try: 130 | self.connection.close() 131 | except Exception: 132 | pass 133 | -------------------------------------------------------------------------------- /logstash/handler_tcp.py: -------------------------------------------------------------------------------- 1 | from logging.handlers import DatagramHandler, SocketHandler 2 | from logstash import formatter 3 | 4 | 5 | # Derive from object to force a new-style class and thus allow super() to work 6 | # on Python 2.6 7 | class TCPLogstashHandler(SocketHandler, object): 8 | """Python logging handler for Logstash. Sends events over TCP. 9 | :param host: The host of the logstash server. 10 | :param port: The port of the logstash server (default 5959). 11 | :param message_type: The type of the message (default logstash). 12 | :param fqdn; Indicates whether to show fully qualified domain name or not (default False). 13 | :param version: version of logstash event schema (default is 0). 14 | :param tags: list of tags for a logger (default is None). 15 | """ 16 | 17 | def __init__(self, host, port=5959, message_type='logstash', tags=None, fqdn=False, version=0): 18 | super(TCPLogstashHandler, self).__init__(host, port) 19 | if version == 1: 20 | self.formatter = formatter.LogstashFormatterVersion1(message_type, tags, fqdn) 21 | else: 22 | self.formatter = formatter.LogstashFormatterVersion0(message_type, tags, fqdn) 23 | 24 | def makePickle(self, record): 25 | return self.formatter.format(record) + b'\n' 26 | -------------------------------------------------------------------------------- /logstash/handler_udp.py: -------------------------------------------------------------------------------- 1 | from logging.handlers import DatagramHandler, SocketHandler 2 | from logstash.handler_tcp import TCPLogstashHandler 3 | from logstash import formatter 4 | 5 | 6 | class UDPLogstashHandler(TCPLogstashHandler, DatagramHandler): 7 | """Python logging handler for Logstash. Sends events over UDP. 8 | :param host: The host of the logstash server. 9 | :param port: The port of the logstash server (default 5959). 10 | :param message_type: The type of the message (default logstash). 11 | :param fqdn; Indicates whether to show fully qualified domain name or not (default False). 12 | :param version: version of logstash event schema (default is 0). 13 | :param tags: list of tags for a logger (default is None). 14 | """ 15 | 16 | def makePickle(self, record): 17 | return self.formatter.format(record) 18 | 19 | 20 | # For backward compatibility 21 | LogstashHandler = UDPLogstashHandler 22 | 23 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | setup( 3 | name='python-logstash', 4 | packages=['logstash'], 5 | version='0.4.8', 6 | description='Python logging handler for Logstash.', 7 | long_description=open('README.rst').read(), 8 | license='MIT', 9 | author='Volodymyr Klochan', 10 | author_email='vklochan@gmail.com', 11 | url='https://github.com/vklochan/python-logstash', 12 | classifiers=[ 13 | 'Development Status :: 4 - Beta', 14 | 'Intended Audience :: Developers', 15 | 'License :: OSI Approved :: MIT License', 16 | 'Operating System :: OS Independent', 17 | 'Programming Language :: Python', 18 | 'Programming Language :: Python :: 2', 19 | 'Programming Language :: Python :: 3', 20 | 'Topic :: Internet :: WWW/HTTP', 21 | 'Topic :: Software Development :: Libraries :: Python Modules', 22 | 'Topic :: System :: Logging', 23 | ] 24 | ) 25 | --------------------------------------------------------------------------------