├── MANIFEST.in ├── tox.ini ├── .gitignore ├── LICENSE ├── tests └── test_log_message_positional_placeholder.py ├── setup.py ├── README.md └── logstash_formatter └── __init__.py /MANIFEST.in: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py33,py34,py35 3 | [testenv] 4 | changedir=tests 5 | commands=discover 6 | deps= 7 | discover 8 | six 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # See http://help.github.com/ignore-files/ for more about ignoring files. 2 | # 3 | # If you find yourself ignoring temporary files generated by your text editor 4 | # or operating system, you probably want to add a global ignore instead: 5 | # git config --global core.excludesfile ~/.gitignore_global 6 | 7 | # Ignore bundler config 8 | /.bundle 9 | 10 | # Ignore the build directory 11 | /build 12 | /dist 13 | *.egg-info 14 | *.deb 15 | 16 | # Ignore Sass' cache 17 | /.sass-cache 18 | 19 | # python compiled files 20 | *.pyc 21 | 22 | # ignore PyCharm files 23 | .idea/* 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Exoscale SA 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /tests/test_log_message_positional_placeholder.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from six import StringIO 3 | import unittest 4 | 5 | import logstash_formatter 6 | 7 | 8 | class LogMessagePositionalPlaceholderTest(unittest.TestCase): 9 | 10 | def setUp(self): 11 | self.stream = StringIO() 12 | handler = logging.StreamHandler(self.stream) 13 | handler.setFormatter(logstash_formatter.LogstashFormatterV1()) 14 | handler.setLevel(logging.DEBUG) 15 | self.log = logging.getLogger('test') 16 | self.log.addHandler(handler) 17 | self.log.setLevel(logging.DEBUG) 18 | 19 | def assertLogMessage(self, msg): 20 | self.assertTrue( 21 | '"message": "{}"'.format(msg) in self.stream.getvalue()) 22 | 23 | 24 | test_msgs = { 25 | 'normal_log_message': 'foo', 26 | 'implicit_positional_placeholder': '{}', 27 | 'explicit_positional_placeholder': '{0}', 28 | 'message_with_invalid_formatting': '{' 29 | } 30 | 31 | 32 | def make_method(msg): 33 | 34 | def test_msg(self): 35 | self.log.debug(msg) 36 | self.assertLogMessage(msg) 37 | 38 | return test_msg 39 | 40 | 41 | for name, msg in test_msgs.items(): 42 | name = 'test_{}'.format(name) 43 | setattr(LogMessagePositionalPlaceholderTest, name, make_method(msg)) 44 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | import unittest 3 | 4 | from setuptools import setup, find_packages 5 | from os import path 6 | 7 | def read(*parts): 8 | return codecs.open(path.join(path.dirname(__file__), *parts), 9 | encoding="utf-8").read() 10 | 11 | 12 | def tests_suite(): 13 | test_loader = unittest.TestLoader() 14 | test_suite = test_loader.discover('tests', pattern='test_*.py') 15 | return test_suite 16 | 17 | 18 | setup(name='logstash_formatter', 19 | version='0.5.17', 20 | description='JSON formatter meant for logstash', 21 | long_description=read('README.rst'), 22 | url='https://github.com/exoscale/python-logstash-formatter', 23 | author='Pierre-Yves Ritschard', 24 | author_email='pierre-yves.ritschard@exoscale.ch', 25 | license='MIT, see LICENSE file', 26 | packages=find_packages(), 27 | include_package_data=True, 28 | classifiers=[ 29 | 'Development Status :: 5 - Production/Stable', 30 | 'Intended Audience :: Developers', 31 | 'License :: OSI Approved :: MIT License', 32 | 'Natural Language :: English', 33 | 'Programming Language :: Python', 34 | 'Programming Language :: Python :: 3' 35 | ], 36 | zip_safe=False, 37 | test_suite='setup.tests_suite') 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # logstash_formatter: JSON logs for logstash 2 | This library is provided to allow standard python logging to output log data 3 | as json objects ready to be shipped out to logstash. 4 | 5 | This project has been originally open sourced by [exoscale](https://www.exoscale.ch/) (which is a great hosting service btw), 6 | thanks to them. 7 | 8 | ## Installing 9 | Using `pip` ([PyPI](https://pypi.python.org/pypi/logstash_formatter)): 10 | ~~~ 11 | pip install logstash_formatter 12 | ~~~ 13 | 14 | Manual: 15 | ~~~ 16 | git clone https://github.com/ulule/python-logstash-formatter.git 17 | cd python-logstash-formatter 18 | python setup.py install 19 | ~~~ 20 | 21 | ## Usage 22 | Json outputs are provided by the LogstashFormatter logging formatter. 23 | ~~~python 24 | import logging, sys 25 | from logstash_formatter import LogstashFormatterV1 26 | 27 | # configure logging 28 | handler = logging.StreamHandler(stream=sys.stdout) 29 | handler.setFormatter(LogstashFormatterV1()) 30 | logging.basicConfig(handlers=[handler], level=logging.INFO) 31 | 32 | # use it 33 | logging.info("my log") 34 | ~~~ 35 | 36 | The LogstashFormatter may take the following named parameters: 37 | * `fmt`: Config as a JSON string that supports: 38 | * `extra`: provide extra fields always present in logs. 39 | * `source_host`: override source host name. 40 | * `json_cls`: JSON encoder to forward to `json.dump`. 41 | * `json_default`: Default JSON representation for unknown types, 42 | by default coerce everything to a string. 43 | 44 | `LogstashFormatterV1` adheres to the more 1.2.0 schema and will not update 45 | fields, apart from a special handling of `msg` which will be updated to 46 | `message` when applicable. 47 | 48 | You can also add extra fields to your json output by specifying a dict in place of message, or by specifying 49 | the named argument `extra` as a dictionary. When supplying the `exc_info` named argument with a truthy value, 50 | and if an exception is found on the stack, its traceback will be attached to the payload as well. 51 | 52 | ~~~python 53 | logger.info({"account": 123, "ip": "172.20.19.18"}) 54 | logger.info("classic message for account: {account}", extra={"account": account}) 55 | 56 | try: 57 | h = {} 58 | h['key'] 59 | except: 60 | logger.info("something unexpected happened", exc_info=True) 61 | ~~~ 62 | 63 | ## Sample output for LogstashFormatter 64 | The following keys will be found in the output JSON: 65 | * `@source_host`: source hostname for the log 66 | * `@timestamp`: ISO 8601 timestamp 67 | * `@message`: short message for this log 68 | * `@fields`: all extra fields 69 | 70 | ~~~json 71 | { 72 | "@fields": { 73 | "account": "pyr", 74 | "args": [], 75 | "created": 1367480388.013037, 76 | "exception": [ 77 | "Traceback (most recent call last):\n", 78 | " File \"test.py\", line 16, in \n k['unknown']\n", 79 | "KeyError: 'unknown'\n" 80 | ], 81 | "filename": "test.py", 82 | "funcName": "", 83 | "levelname": "WARNING", 84 | "levelno": 30, 85 | "lineno": 18, 86 | "module": "test", 87 | "msecs": 13.036966323852539, 88 | "name": "root", 89 | "pathname": "test.py", 90 | "process": 1819, 91 | "processName": "MainProcess", 92 | "relativeCreated": 18.002986907958984, 93 | "thread": 140060726359808, 94 | "threadName": "MainThread" 95 | }, 96 | "@message": "TEST", 97 | "@source_host": "phoenix.spootnik.org", 98 | "@timestamp": "2013-05-02T09:39:48.013158" 99 | } 100 | ~~~ 101 | 102 | ## Sample output for LogstashFormatterV1 103 | The following keys will be found in the output JSON: 104 | * `@timestamp`: ISO 8601 timestamp 105 | * `@version`: Version of the schema 106 | 107 | ~~~json 108 | {"@version": 1, 109 | "account": "pyr", 110 | "lineno": 1, 111 | "levelno": 30, 112 | "filename": "test.py", 113 | "thread": 140566036444928, 114 | "@timestamp": "2015-03-30T09:46:23.000Z", 115 | "threadName": "MainThread", 116 | "relativeCreated": 51079.52117919922, 117 | "process": 10787, 118 | "source_host": "phoenix.spootnik.org", 119 | "processName": "MainProcess", 120 | "pathname": "test.py", 121 | "args": [], 122 | "module": "test", 123 | "msecs": 999.9005794525146, 124 | "created": 1427708782.9999006, 125 | "name": "root", 126 | "stack_info": null, 127 | "funcName": "", 128 | "levelname": "WARNING", 129 | "message": "foo"} 130 | ~~~ 131 | -------------------------------------------------------------------------------- /logstash_formatter/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This library is provided to allow standard python 3 | logging to output log data as JSON formatted strings 4 | ready to be shipped out to logstash. 5 | ''' 6 | import logging 7 | import socket 8 | import datetime 9 | import traceback as tb 10 | import json 11 | 12 | 13 | def _default_json_default(obj): 14 | """ 15 | Coerce everything to strings. 16 | All objects representing time get output as ISO8601. 17 | """ 18 | if isinstance(obj, (datetime.datetime, datetime.date, datetime.time)): 19 | return obj.isoformat() 20 | else: 21 | return str(obj) 22 | 23 | 24 | class LogstashFormatter(logging.Formatter): 25 | """ 26 | A custom formatter to prepare logs to be 27 | shipped out to logstash. 28 | """ 29 | 30 | def __init__(self, 31 | fmt=None, 32 | datefmt=None, 33 | style='%', 34 | json_cls=None, 35 | json_default=_default_json_default): 36 | """ 37 | :param fmt: Config as a JSON string, allowed fields; 38 | extra: provide extra fields always present in logs 39 | source_host: override source host name 40 | :param datefmt: Date format to use (required by logging.Formatter 41 | interface but not used) 42 | :param json_cls: JSON encoder to forward to json.dumps 43 | :param json_default: Default JSON representation for unknown types, 44 | by default coerce everything to a string 45 | """ 46 | 47 | if fmt is not None: 48 | self._fmt = json.loads(fmt) 49 | else: 50 | self._fmt = {} 51 | self.json_default = json_default 52 | self.json_cls = json_cls 53 | if 'extra' not in self._fmt: 54 | self.defaults = {} 55 | else: 56 | self.defaults = self._fmt['extra'] 57 | if 'source_host' in self._fmt: 58 | self.source_host = self._fmt['source_host'] 59 | else: 60 | try: 61 | self.source_host = socket.gethostname() 62 | except: 63 | self.source_host = "" 64 | 65 | def format(self, record): 66 | """ 67 | Format a log record to JSON, if the message is a dict 68 | assume an empty message and use the dict as additional 69 | fields. 70 | """ 71 | 72 | fields = record.__dict__.copy() 73 | 74 | if isinstance(record.msg, dict): 75 | fields.update(record.msg) 76 | fields.pop('msg') 77 | msg = "" 78 | else: 79 | msg = record.getMessage() 80 | 81 | try: 82 | msg = msg.format(**fields) 83 | except (KeyError, IndexError, ValueError): 84 | pass 85 | except: 86 | # in case we can not format the msg properly we log it as is instead of crashing 87 | msg = msg 88 | 89 | if 'msg' in fields: 90 | fields.pop('msg') 91 | 92 | if 'exc_info' in fields: 93 | if fields['exc_info']: 94 | formatted = tb.format_exception(*fields['exc_info']) 95 | fields['exception'] = formatted 96 | fields.pop('exc_info') 97 | 98 | if 'exc_text' in fields and not fields['exc_text']: 99 | fields.pop('exc_text') 100 | 101 | logr = self.defaults.copy() 102 | 103 | logr.update({'@message': msg, 104 | '@timestamp': datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ'), 105 | '@source_host': self.source_host, 106 | '@fields': self._build_fields(logr, fields)}) 107 | 108 | return json.dumps(logr, default=self.json_default, cls=self.json_cls) 109 | 110 | def _build_fields(self, defaults, fields): 111 | """Return provided fields including any in defaults 112 | 113 | >>> f = LogstashFormatter() 114 | # Verify that ``fields`` is used 115 | >>> f._build_fields({}, {'foo': 'one'}) == \ 116 | {'foo': 'one'} 117 | True 118 | # Verify that ``@fields`` in ``defaults`` is used 119 | >>> f._build_fields({'@fields': {'bar': 'two'}}, {'foo': 'one'}) == \ 120 | {'foo': 'one', 'bar': 'two'} 121 | True 122 | # Verify that ``fields`` takes precedence 123 | >>> f._build_fields({'@fields': {'foo': 'two'}}, {'foo': 'one'}) == \ 124 | {'foo': 'one'} 125 | True 126 | """ 127 | return dict(list(defaults.get('@fields', {}).items()) + list(fields.items())) 128 | 129 | 130 | class LogstashFormatterV1(LogstashFormatter): 131 | """ 132 | A custom formatter to prepare logs to be 133 | shipped out to logstash V1 format. 134 | """ 135 | 136 | def format(self, record): 137 | """ 138 | Format a log record to JSON, if the message is a dict 139 | assume an empty message and use the dict as additional 140 | fields. 141 | """ 142 | 143 | fields = record.__dict__.copy() 144 | 145 | if 'msg' in fields and isinstance(fields['msg'], dict): 146 | msg = fields.pop('msg') 147 | fields.update(msg) 148 | 149 | elif 'msg' in fields and 'message' not in fields: 150 | msg = record.getMessage() 151 | fields.pop('msg') 152 | 153 | try: 154 | msg = msg.format(**fields) 155 | except (KeyError, IndexError): 156 | pass 157 | except: 158 | # in case we can not format the msg properly we log it as is instead of crashing 159 | msg = msg 160 | fields['message'] = msg 161 | 162 | if 'exc_info' in fields: 163 | if fields['exc_info']: 164 | formatted = tb.format_exception(*fields['exc_info']) 165 | fields['exception'] = formatted 166 | fields.pop('exc_info') 167 | 168 | if 'exc_text' in fields and not fields['exc_text']: 169 | fields.pop('exc_text') 170 | 171 | now = datetime.datetime.utcnow() 172 | base_log = {'@timestamp': now.strftime("%Y-%m-%dT%H:%M:%S") + 173 | ".%03d" % (now.microsecond / 1000) + "Z", 174 | '@version': 1, 175 | 'source_host': self.source_host} 176 | base_log.update(fields) 177 | 178 | logr = self.defaults.copy() 179 | logr.update(base_log) 180 | 181 | return json.dumps(logr, default=self.json_default, cls=self.json_cls) 182 | --------------------------------------------------------------------------------