├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.rst ├── logfmt ├── __init__.py ├── formatter.py └── parser.py ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── test_formatter.py └── test_parser.py /.gitignore: -------------------------------------------------------------------------------- 1 | _trial_temp/ 2 | *.pyc 3 | build/ 4 | *.egg 5 | *.egg-info 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - 2.7 4 | - 3.2 5 | - 3.3 6 | script: python setup.py test 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2013 Jamshed Kakar, Timothée Peignier. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst LICENSE 2 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | This project is now managed by @wlonk at https://github.com/wlonk/logfmt-python. 2 | 3 | .. image:: https://secure.travis-ci.org/jkakar/logfmt-python.png?branch=master 4 | 5 | Logfmt 6 | ====== 7 | 8 | Python package for parsing log lines in the `logfmt` style. See the 9 | original project by Blake Mizerany and Keith Rarick for information 10 | about `logfmt` conventions and use: https://github.com/kr/logfmt 11 | 12 | 13 | Using logfmt 14 | ------------ 15 | 16 | Easily process lines from `logfmt` formatted input: :: 17 | 18 | from logfmt import parse 19 | 20 | input = StringIO('\n'.join(['key1=value1', 'key2=value2'])) 21 | for values in parse(input): 22 | print values 23 | 24 | This program produces this output: :: 25 | 26 | {'key1': 'value1'} 27 | {'key2': 'value2'} 28 | 29 | 30 | Easily generate lines in `logfmt` formatted output :: 31 | 32 | from logfmt import format 33 | 34 | for line in format({'key1': 'value1'}, {'key2': 'value2'}): 35 | print line 36 | 37 | 38 | This program produces this output: :: 39 | 40 | key1="value1" 41 | key2="value2" 42 | 43 | 44 | 45 | Installation 46 | ------------ 47 | 48 | To install it, simply: :: 49 | 50 | pip install logfmt 51 | 52 | -------------------------------------------------------------------------------- /logfmt/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 flake8:noqa -*- 2 | from logfmt.parser import parse_line 3 | from logfmt.formatter import format_line 4 | 5 | 6 | def parse(stream): 7 | for line in stream: 8 | values = parse_line(line) 9 | if values: 10 | yield values 11 | 12 | 13 | # Will take in a list of hashes. Each call will generate a string for 14 | # the next argument 15 | def format(*args): 16 | for hash in args: 17 | output = format_line(hash) 18 | if output: 19 | yield output 20 | -------------------------------------------------------------------------------- /logfmt/formatter.py: -------------------------------------------------------------------------------- 1 | import numbers 2 | import logging 3 | 4 | 5 | def format_line(extra): 6 | outarr = [] 7 | for k, v in extra.items(): 8 | if v is None: 9 | outarr.append("%s=" % k) 10 | continue 11 | 12 | if isinstance(v, bool): 13 | v = "true" if v else "false" 14 | elif isinstance(v, numbers.Number): 15 | pass 16 | else: 17 | if isinstance(v, (dict, object)): 18 | v = str(v) 19 | v = '"%s"' % v.replace('"', '\\"') 20 | outarr.append("%s=%s" % (k, v)) 21 | return " ".join(outarr) 22 | 23 | 24 | class LogfmtFormatter(logging.Formatter): 25 | def format(self, record): 26 | return ' '.join([ 27 | 'at=%' % record.levelname, 28 | 'msg="%"' % record.getMessage().replace('"', '\\"'), 29 | 'process=%' % record.processName, 30 | format_line(getattr(record, 'context', {})), 31 | ]) 32 | -------------------------------------------------------------------------------- /logfmt/parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | GARBAGE = 0 4 | KEY = 1 5 | EQUAL = 2 6 | IVALUE = 3 7 | QVALUE = 4 8 | 9 | 10 | def parse_line(line): 11 | output = {} 12 | key, value = (), () 13 | escaped = False 14 | state = GARBAGE 15 | for i, c in enumerate(line): 16 | i += 1 17 | if state == GARBAGE: 18 | if c > ' ' and c != '"' and c != '=': 19 | key = (c,) 20 | state = KEY 21 | continue 22 | if state == KEY: 23 | if c > ' ' and c != '"' and c != '=': 24 | state = KEY 25 | key += (c,) 26 | elif c == '=': 27 | output["".join(key).strip()] = True 28 | state = EQUAL 29 | else: 30 | output["".join(key).strip()] = True 31 | state = GARBAGE 32 | if i >= len(line): 33 | output["".join(key).strip()] = True 34 | continue 35 | if state == EQUAL: 36 | if c > ' ' and c != '"' and c != '=': 37 | value = (c,) 38 | state = IVALUE 39 | elif c == '"': 40 | value = () 41 | escaped = False 42 | state = QVALUE 43 | else: 44 | state = GARBAGE 45 | if i >= len(line): 46 | output["".join(key).strip()] = "".join(value) or True 47 | continue 48 | if state == IVALUE: 49 | if not (c > ' ' and c != '"' and c != '='): 50 | output["".join(key).strip()] = "".join(value) 51 | state = GARBAGE 52 | else: 53 | value += (c,) 54 | if i >= len(line): 55 | output["".join(key).strip()] = "".join(value) 56 | continue 57 | if state == QVALUE: 58 | if c == '\\': 59 | escaped = True 60 | elif c == '"': 61 | if escaped: 62 | escaped = False 63 | value += (c,) 64 | continue 65 | output["".join(key).strip()] = "".join(value) 66 | state = GARBAGE 67 | else: 68 | value += (c,) 69 | continue 70 | return output 71 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = logfmt 3 | version = 0.3 4 | author = Timothee Peignier 5 | author-email = timothee.peignier@tryphon.org 6 | summary = Parse log lines in the logfmt style. 7 | description-file = README.rst 8 | license = MIT 9 | classifier = 10 | Development Status :: 4 - Beta 11 | Intended Audience :: Developers 12 | License :: OSI Approved :: MIT License 13 | Operating System :: OS Independent 14 | Programming Language :: Python 15 | Programming Language :: Python :: 2.6 16 | Programming Language :: Python :: 2.7 17 | Programming Language :: Python :: 3.2 18 | Programming Language :: Python :: 3.3 19 | Topic :: Utilities 20 | 21 | [files] 22 | packages = 23 | logfmt 24 | extra_files = 25 | README.rst 26 | LICENSE 27 | 28 | [backwards_compat] 29 | zip_safe = false 30 | 31 | [wheel] 32 | universal = 1 33 | 34 | [test] 35 | test-module = tests 36 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from setuptools import setup 3 | 4 | setup(setup_requires=['d2to1'], d2to1=True) 5 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | 4 | from unittest import TestSuite, TestLoader 5 | 6 | 7 | def test_suite(): 8 | suite = TestSuite() 9 | tests = TestLoader().discover(start_dir=os.path.dirname(__file__)) 10 | suite.addTests(tests) 11 | return suite -------------------------------------------------------------------------------- /tests/test_formatter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from logfmt.formatter import format_line 3 | from unittest import TestCase 4 | from collections import OrderedDict 5 | import sys 6 | 7 | 8 | class FormatterTestCase(TestCase): 9 | def test_empty_log_line(self): 10 | data = format_line({}) 11 | self.assertEqual(data, "") 12 | 13 | def test_key_without_value(self): 14 | data = format_line( OrderedDict([("key1", None), ("key2", None)]) ) 15 | self.assertEqual(data, "key1= key2=") 16 | 17 | def test_boolean_values(self): 18 | data = format_line(OrderedDict([("key1", True), ("key2", False)])) 19 | self.assertEqual(data, "key1=true key2=false") 20 | 21 | def test_int_value(self): 22 | data = format_line( OrderedDict([("key1", -1), ("key2", 2342342)]) ) 23 | self.assertEqual(data, "key1=-1 key2=2342342") 24 | 25 | # In python 2.7, truncated to 12 digits total. 3.x does 16 26 | def test_float_value(self): 27 | data = format_line( OrderedDict([("key1", 342.23424), ("key2", -234234234.2342342)]) ) 28 | 29 | if sys.version_info < (3, 0): 30 | self.assertEqual(data, "key1=342.23424 key2=-234234234.234") 31 | else: 32 | self.assertEqual(data, "key1=342.23424 key2=-234234234.2342342") 33 | 34 | # Essentially, pre format your floats to strings 35 | def test_custom_float_format(self): 36 | data = format_line( OrderedDict([ 37 | ("key1", 342.23424), 38 | ("key2", "%.7f" % -234234234.2342342) 39 | ])) 40 | 41 | self.assertEqual(data, "key1=342.23424 key2=\"-234234234.2342342\"") 42 | 43 | def test_string_value(self): 44 | data = format_line( OrderedDict([ 45 | ("key1", """some random !@#$%^"&**_+-={}\\|;':,./<>?)"""), 46 | ("key2", """here's a line with 47 | more stuff on the next""") 48 | ])) 49 | 50 | self.assertEqual(data, '''key1="some random !@#$%^\\"&**_+-={}\\|;\':,./<>?)" key2="here\'s a line with\nmore stuff on the next"''') 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /tests/test_parser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from logfmt import parse_line 3 | from unittest import TestCase 4 | 5 | 6 | class ParserTestCase(TestCase): 7 | def test_empty_log_line(self): 8 | data = parse_line("") 9 | self.assertEqual(data, {}) 10 | 11 | def test_whitespace_only_log_line(self): 12 | data = parse_line("\t") 13 | self.assertEqual(data, {}) 14 | 15 | def test_key_without_value(self): 16 | data = parse_line("key") 17 | self.assertEqual(data, {'key': True}) 18 | 19 | def test_key_without_value_and_whitespace(self): 20 | data = parse_line(" key ") 21 | self.assertEqual(data, {'key': True}) 22 | 23 | def test_multiple_single_keys(self): 24 | data = parse_line("key1 key2") 25 | self.assertEqual(data, {'key1': True, 'key2': True}) 26 | 27 | def test_unquoted_value(self): 28 | data = parse_line("key=value") 29 | self.assertEqual(data, {'key': "value"}) 30 | 31 | def test_pairs(self): 32 | data = parse_line("key1=value1 key2=value2") 33 | self.assertEqual(data, {'key1': "value1", 'key2': "value2"}) 34 | 35 | def test_mixed_single_or_non_single_pairs(self): 36 | data = parse_line("key1=value1 key2") 37 | self.assertEqual(data, {'key1': "value1", 'key2': True}) 38 | 39 | def test_mixed_pairs_whatever_the_order(self): 40 | data = parse_line("key1 key2=value2") 41 | self.assertEqual(data, {'key1': True, 'key2': "value2"}) 42 | 43 | def test_quoted_value(self): 44 | data = parse_line('key="quoted value"') 45 | self.assertEqual(data, {'key': "quoted value"}) 46 | 47 | def test_escaped_quote_value(self): 48 | data = parse_line('key="quoted \\" value" r="esc\t"') 49 | self.assertEqual(data, {'key': 'quoted \" value', 'r': "esc\t"}) 50 | 51 | def test_mixed_pairs(self): 52 | data = parse_line('key1="quoted \\" value" key2 key3=value3') 53 | self.assertEqual(data, { 54 | 'key1': 'quoted \" value', 'key2': True, 'key3': "value3" 55 | }) 56 | 57 | def test_mixed_characters_pairs(self): 58 | data = parse_line('foo=bar a=14 baz="hello kitty" ƒ=2h3s cool%story=bro f %^asdf') 59 | self.assertEqual(data, { 60 | 'foo': "bar", 'a': "14", 'baz': "hello kitty", 'ƒ': "2h3s", 61 | "cool%story": "bro", 'f': True, "%^asdf": True 62 | }) 63 | 64 | def test_pair_with_empty_quote(self): 65 | data = parse_line('key=""') 66 | self.assertEqual(data, {'key': ""}) 67 | 68 | def test_single_character_value_at_end_of_string(self): 69 | data = parse_line('key=a') 70 | self.assertEqual(data, {'key': 'a'}) 71 | --------------------------------------------------------------------------------