├── .github └── workflows │ ├── build.yml │ └── release.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── requirements └── ci.txt ├── setup.cfg ├── setup.py ├── src └── pythonjsonlogger │ ├── __init__.py │ ├── jsonlogger.py │ └── py.typed ├── tests ├── __init__.py └── test_jsonlogger.py └── tox.ini /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Test python-json-logger 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | workflow_dispatch: 8 | inputs: 9 | logLevel: 10 | description: 'Log level' 11 | required: true 12 | default: 'warning' 13 | type: choice 14 | options: 15 | - info 16 | - warning 17 | - debug 18 | pull_request: 19 | types: [opened, reopened] 20 | 21 | jobs: 22 | test: 23 | runs-on: "ubuntu-20.04" #Moving down to 20.04 (latest is 22.04) because of python3.6 support 24 | strategy: 25 | fail-fast: false 26 | matrix: 27 | python-version: ["pypy-3.8", "pypy-3.9", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] 28 | 29 | steps: 30 | - uses: actions/checkout@v3 31 | 32 | - name: Set up Python ${{ matrix.python-version }} 33 | uses: actions/setup-python@v4 34 | with: 35 | python-version: ${{ matrix.python-version }} 36 | 37 | - name: Install dependencies 38 | run: | 39 | python -m pip install --upgrade pip 40 | pip install tox tox-gh-actions 41 | 42 | - name: Test with tox 43 | run: tox 44 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release python-json-logger build 2 | 3 | on: 4 | release: 5 | types: [ created ] 6 | 7 | workflow_dispatch: 8 | 9 | jobs: 10 | publish: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v3 15 | 16 | - name: Set up Python 3.11 17 | uses: actions/setup-python@v4 18 | with: 19 | python-version: 3.11 20 | 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | pip install setuptools wheel twine typing_extensions 25 | 26 | - name: Build and Upload to PyPi 27 | run: | 28 | python setup.py sdist bdist_wheel 29 | python -m twine upload dist/* 30 | env: 31 | TWINE_USERNAME: __token__ 32 | TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.swp 3 | build 4 | dist 5 | *.egg-info 6 | 7 | # Tests and validation 8 | .tox/ 9 | .mypy_cache 10 | 11 | # Python's venv 12 | .env 13 | .venv 14 | env 15 | 16 | # IDE 17 | .vscode 18 | .idea 19 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [2.0.7] - 2023-02-21 8 | ### Changed 9 | - Fix inclusion of py.typed in pip packages - @sth 10 | - Added pytest support with test file rename. Migrated to assertEqual 11 | 12 | ## [2.0.6] - 2023-02-14 13 | ### Changed 14 | - Parameter `rename_fields` in merge_record_extra is now optional - @afallou 15 | 16 | ## [2.0.5] - 2023-02-12 17 | ### Added 18 | - Allow reserved attrs to be renamed - @henkhogan 19 | - Support added for Python 3.11 20 | - Now verifying builds in Pypy 3.9 as well 21 | - Type annotations are now in the package - @louis-jaris 22 | ### Changed 23 | - Fix rename_fields for exc_info - @guilhermeferrari 24 | - Cleaned up test file for PEP8 - @lopagela 25 | - Cleaned up old Python 2 artifacts - @louis-jaris 26 | - Dropped Python 3.5 support - @idomozes 27 | - Moved type check via tox into 3.11 run only 28 | - Added test run in Python3.6 (will keep for a little while longer, but it's EOL so upgrade) 29 | 30 | ## [2.0.4] - 2022-07-11 31 | ### Changed 32 | - Fix too strict regex for percentage style logging - @aberres 33 | - 34 | 35 | ## [2.0.3] - 2022-07-08 36 | ### Added 37 | - Add PEP 561 marker/basic mypy configuration. - @bringhurst 38 | - Workaround logging.LogRecord.msg type of string. - @bringhurst 39 | ### Changed 40 | - Changed a link archive of the reference page in case it's down. - @ahonnecke 41 | - Removed unnecessary try-except around OrderedDict usage - @sozofaan 42 | - Update documentation link to json module + use https - @deronnax 43 | - Dropped 3.5 support. - @bringhurst 44 | 45 | ## [2.0.2] - 2021-07-27 46 | ### Added 47 | - Officially supporting 3.9 - @felixonmars. 48 | - You can now add static fields to log objects - @cosimomeli. 49 | ### Changed 50 | - Dropped 3.4 support. 51 | - Dropped Travis CI for Github Actions. 52 | - Wheel should build for python 3 instead of just 3.4 now. 53 | 54 | ## [2.0.1] - 2020-10-12 55 | ### Added 56 | - Support Pypi long descripton - @ereli-cb 57 | ### Changed 58 | - You can now rename output fields - @schlitzered 59 | 60 | ## [2.0.0] - 2020-09-26 61 | ### Added 62 | - New Changelog 63 | - Added timezone support to timestamps - @lalten 64 | - Refactored log record to function - @georgysavva 65 | - Add python 3.8 support - @tommilligan 66 | ### Removed 67 | - Support for Python 2.7 68 | - Debian directory 69 | 70 | ## [0.1.11] - 2019-03-29 71 | ### Added 72 | - Support for Python 3.7 73 | ### Changed 74 | - 'stack_info' flag in logging calls is now respected in JsonFormatter by [@ghShu](https://github.com/ghShu) 75 | 76 | 77 | [2.0.7]: https://github.com/madzak/python-json-logger/compare/v2.0.6...v2.0.7 78 | [2.0.6]: https://github.com/madzak/python-json-logger/compare/v2.0.5...v2.0.6 79 | [2.0.5]: https://github.com/madzak/python-json-logger/compare/v2.0.4...v2.0.5 80 | [2.0.4]: https://github.com/madzak/python-json-logger/compare/v2.0.3...v2.0.4 81 | [2.0.3]: https://github.com/madzak/python-json-logger/compare/v2.0.2...v2.0.3 82 | [2.0.2]: https://github.com/madzak/python-json-logger/compare/v2.0.1...v2.0.2 83 | [2.0.1]: https://github.com/madzak/python-json-logger/compare/v2.0.0...v2.0.1 84 | [2.0.0]: https://github.com/madzak/python-json-logger/compare/v0.1.11...v2.0.0 85 | [0.1.11]: https://github.com/madzak/python-json-logger/compare/v0.1.10...v0.1.11 86 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011, Zakaria Zajac 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 10 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.md 3 | recursive-include tests *.py 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # This project has been retired and is no longer actively maintained. We recommend transitioning to [nhairs/python-json-logger](https://github.com/nhairs/python-json-logger) for continued development, updates, and community support. Thank you for your understanding and for supporting this project over the years! 2 | 3 | ![Build Status](https://github.com/madzak/python-json-logger/actions/workflows/build.yml/badge.svg) 4 | [![License](https://img.shields.io/pypi/l/python-json-logger.svg)](https://pypi.python.org/pypi/python-json-logger/) 5 | [![Version](https://img.shields.io/pypi/v/python-json-logger.svg)](https://pypi.python.org/pypi/python-json-logger/) 6 | 7 | Overview 8 | ======= 9 | This library is provided to allow standard python logging to output log data as json objects. With JSON we can make our logs more readable by machines and we can stop writing custom parsers for syslog type records. 10 | 11 | News 12 | ======= 13 | Hi, I see this package is quiet alive and I am sorry for ignoring it so long. I will be stepping up my maintenance of this package so please allow me a week to get things back in order (and most likely a new minor version) and I'll post and update here once I am caught up. 14 | 15 | Installing 16 | ========== 17 | Pip: 18 | 19 | pip install python-json-logger 20 | 21 | Pypi: 22 | 23 | https://pypi.python.org/pypi/python-json-logger 24 | 25 | Manual: 26 | 27 | python setup.py install 28 | 29 | Usage 30 | ===== 31 | 32 | ## Integrating with Python's logging framework 33 | 34 | Json outputs are provided by the JsonFormatter logging formatter. You can add the custom formatter like below: 35 | 36 | **Please note: version 0.1.0 has changed the import structure, please update to the following example for proper importing** 37 | 38 | ```python 39 | import logging 40 | from pythonjsonlogger import jsonlogger 41 | 42 | logger = logging.getLogger() 43 | 44 | logHandler = logging.StreamHandler() 45 | formatter = jsonlogger.JsonFormatter() 46 | logHandler.setFormatter(formatter) 47 | logger.addHandler(logHandler) 48 | ``` 49 | 50 | ## Customizing fields 51 | 52 | The fmt parser can also be overidden if you want to have required fields that differ from the default of just `message`. 53 | 54 | These two invocations are equivalent: 55 | 56 | ```python 57 | class CustomJsonFormatter(jsonlogger.JsonFormatter): 58 | def parse(self): 59 | return self._fmt.split(';') 60 | 61 | formatter = CustomJsonFormatter('one;two') 62 | 63 | # is equivalent to: 64 | 65 | formatter = jsonlogger.JsonFormatter('%(one)s %(two)s') 66 | ``` 67 | 68 | You can also add extra fields to your json output by specifying a dict in place of message, as well as by specifying an `extra={}` argument. 69 | 70 | Contents of these dictionaries will be added at the root level of the entry and may override basic fields. 71 | 72 | You can also use the `add_fields` method to add to or generally normalize the set of default set of fields, it is called for every log event. For example, to unify default fields with those provided by [structlog](http://www.structlog.org/) you could do something like this: 73 | 74 | ```python 75 | class CustomJsonFormatter(jsonlogger.JsonFormatter): 76 | def add_fields(self, log_record, record, message_dict): 77 | super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict) 78 | if not log_record.get('timestamp'): 79 | # this doesn't use record.created, so it is slightly off 80 | now = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ') 81 | log_record['timestamp'] = now 82 | if log_record.get('level'): 83 | log_record['level'] = log_record['level'].upper() 84 | else: 85 | log_record['level'] = record.levelname 86 | 87 | formatter = CustomJsonFormatter('%(timestamp)s %(level)s %(name)s %(message)s') 88 | ``` 89 | 90 | Items added to the log record will be included in *every* log message, no matter what the format requires. 91 | 92 | ## Adding custom object serialization 93 | 94 | For custom handling of object serialization you can specify default json object translator or provide a custom encoder 95 | 96 | ```python 97 | def json_translate(obj): 98 | if isinstance(obj, MyClass): 99 | return {"special": obj.special} 100 | 101 | formatter = jsonlogger.JsonFormatter(json_default=json_translate, 102 | json_encoder=json.JSONEncoder) 103 | logHandler.setFormatter(formatter) 104 | 105 | logger.info({"special": "value", "run": 12}) 106 | logger.info("classic message", extra={"special": "value", "run": 12}) 107 | ``` 108 | 109 | ## Using a Config File 110 | 111 | To use the module with a config file using the [`fileConfig` function](https://docs.python.org/3/library/logging.config.html#logging.config.fileConfig), use the class `pythonjsonlogger.jsonlogger.JsonFormatter`. Here is a sample config file. 112 | 113 | ```ini 114 | [loggers] 115 | keys = root,custom 116 | 117 | [logger_root] 118 | handlers = 119 | 120 | [logger_custom] 121 | level = INFO 122 | handlers = custom 123 | qualname = custom 124 | 125 | [handlers] 126 | keys = custom 127 | 128 | [handler_custom] 129 | class = StreamHandler 130 | level = INFO 131 | formatter = json 132 | args = (sys.stdout,) 133 | 134 | [formatters] 135 | keys = json 136 | 137 | [formatter_json] 138 | format = %(message)s 139 | class = pythonjsonlogger.jsonlogger.JsonFormatter 140 | ``` 141 | 142 | Example Output 143 | ============== 144 | 145 | Sample JSON with a full formatter (basically the log message from the unit test). Every log message will appear on 1 line like a typical logger. 146 | 147 | ```json 148 | { 149 | "threadName": "MainThread", 150 | "name": "root", 151 | "thread": 140735202359648, 152 | "created": 1336281068.506248, 153 | "process": 41937, 154 | "processName": "MainProcess", 155 | "relativeCreated": 9.100914001464844, 156 | "module": "tests", 157 | "funcName": "testFormatKeys", 158 | "levelno": 20, 159 | "msecs": 506.24799728393555, 160 | "pathname": "tests/tests.py", 161 | "lineno": 60, 162 | "asctime": ["12-05-05 22:11:08,506248"], 163 | "message": "testing logging format", 164 | "filename": "tests.py", 165 | "levelname": "INFO", 166 | "special": "value", 167 | "run": 12 168 | } 169 | ``` 170 | 171 | External Examples 172 | ================= 173 | 174 | - [Wesley Tanaka - Structured log files in Python using python-json-logger](http://web.archive.org/web/20201130054012/https://wtanaka.com/node/8201) 175 | 176 | - [Archive](https://web.archive.org/web/20201130054012/https://wtanaka.com/node/8201) 177 | -------------------------------------------------------------------------------- /requirements/ci.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | pylint 3 | unittest-xml-reporting 4 | wheel 5 | setuptools 6 | tox 7 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [mypy] 2 | 3 | # For details on each flag, please see the mypy documentation at: 4 | # https://mypy.readthedocs.io/en/stable/config_file.html#config-file 5 | 6 | # Import Discovery 7 | mypy_path = src 8 | namespace_packages = true 9 | 10 | # Disallow dynamic typing 11 | disallow_any_unimported = true 12 | disallow_any_expr = false 13 | disallow_any_decorated = true 14 | disallow_any_explicit = false 15 | disallow_any_generics = false 16 | disallow_subclassing_any = true 17 | 18 | # Untyped definitions and calls 19 | disallow_untyped_calls = false 20 | disallow_untyped_defs = false 21 | disallow_incomplete_defs = true 22 | check_untyped_defs = true 23 | disallow_untyped_decorators = true 24 | 25 | # None and Optional handling 26 | no_implicit_optional = true 27 | 28 | # Configuring warnings 29 | warn_redundant_casts = true 30 | warn_unused_ignores = true 31 | warn_no_return = true 32 | warn_return_any = true 33 | warn_unreachable = true 34 | 35 | # Miscellaneous strictness flags 36 | implicit_reexport = true 37 | strict_equality = true 38 | 39 | # Configuring error messages 40 | show_error_context = true 41 | show_column_numbers = true 42 | show_error_codes = true 43 | pretty = true 44 | show_absolute_path = true 45 | 46 | # Miscellaneous 47 | warn_unused_configs = true 48 | verbosity = 0 49 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from os import path 2 | from setuptools import setup, find_packages 3 | 4 | # read the contents of your README file 5 | this_directory = path.abspath(path.dirname(__file__)) 6 | with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: 7 | long_description = f.read() 8 | 9 | setup( 10 | name="python-json-logger", 11 | version="2.0.7", 12 | url="http://github.com/madzak/python-json-logger", 13 | license="BSD", 14 | include_package_data=True, 15 | description="A python library adding a json log formatter", 16 | long_description=long_description, 17 | long_description_content_type='text/markdown', 18 | author="Zakaria Zajac", 19 | author_email="zak@madzak.com", 20 | package_dir={'': 'src'}, 21 | package_data={"pythonjsonlogger": ["py.typed"]}, 22 | packages=find_packages("src", exclude="tests"), 23 | # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires 24 | python_requires=">=3.6", 25 | test_suite="tests.tests", 26 | classifiers=[ 27 | 'Development Status :: 6 - Mature', 28 | 'Intended Audience :: Developers', 29 | 'License :: OSI Approved :: BSD License', 30 | 'Operating System :: OS Independent', 31 | 'Programming Language :: Python', 32 | 'Programming Language :: Python :: 3', 33 | 'Programming Language :: Python :: 3.6', 34 | 'Programming Language :: Python :: 3.7', 35 | 'Programming Language :: Python :: 3.8', 36 | 'Programming Language :: Python :: 3.9', 37 | 'Programming Language :: Python :: 3.10', 38 | 'Programming Language :: Python :: 3.11', 39 | 'Topic :: System :: Logging', 40 | ] 41 | ) 42 | -------------------------------------------------------------------------------- /src/pythonjsonlogger/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/madzak/python-json-logger/be42d82ebd1b3276b739a314545f7a3f518ff222/src/pythonjsonlogger/__init__.py -------------------------------------------------------------------------------- /src/pythonjsonlogger/jsonlogger.py: -------------------------------------------------------------------------------- 1 | """ 2 | This library is provided to allow standard python logging 3 | to output log data as JSON formatted strings 4 | """ 5 | import logging 6 | import json 7 | import re 8 | import traceback 9 | import importlib 10 | from datetime import date, datetime, time, timezone 11 | from typing import Any, Callable, Dict, List, Optional, Tuple, Union 12 | 13 | from inspect import istraceback 14 | 15 | from collections import OrderedDict 16 | 17 | # skip natural LogRecord attributes 18 | # http://docs.python.org/library/logging.html#logrecord-attributes 19 | RESERVED_ATTRS: Tuple[str, ...] = ( 20 | "args", 21 | "asctime", 22 | "created", 23 | "exc_info", 24 | "exc_text", 25 | "filename", 26 | "funcName", 27 | "levelname", 28 | "levelno", 29 | "lineno", 30 | "module", 31 | "msecs", 32 | "message", 33 | "msg", 34 | "name", 35 | "pathname", 36 | "process", 37 | "processName", 38 | "relativeCreated", 39 | "stack_info", 40 | "thread", 41 | "threadName", 42 | ) 43 | 44 | OptionalCallableOrStr = Optional[Union[Callable, str]] 45 | 46 | 47 | def merge_record_extra( 48 | record: logging.LogRecord, 49 | target: Dict, 50 | reserved: Union[Dict, List], 51 | rename_fields: Optional[Dict[str, str]] = None, 52 | ) -> Dict: 53 | """ 54 | Merges extra attributes from LogRecord object into target dictionary 55 | 56 | :param record: logging.LogRecord 57 | :param target: dict to update 58 | :param reserved: dict or list with reserved keys to skip 59 | :param rename_fields: an optional dict, used to rename field names in the output. 60 | Rename levelname to log.level: {'levelname': 'log.level'} 61 | """ 62 | if rename_fields is None: 63 | rename_fields = {} 64 | for key, value in record.__dict__.items(): 65 | # this allows to have numeric keys 66 | if key not in reserved and not ( 67 | hasattr(key, "startswith") and key.startswith("_") 68 | ): 69 | target[rename_fields.get(key, key)] = value 70 | return target 71 | 72 | 73 | class JsonEncoder(json.JSONEncoder): 74 | """ 75 | A custom encoder extending the default JSONEncoder 76 | """ 77 | 78 | def default(self, obj): 79 | if isinstance(obj, (date, datetime, time)): 80 | return self.format_datetime_obj(obj) 81 | 82 | elif istraceback(obj): 83 | return "".join(traceback.format_tb(obj)).strip() 84 | 85 | elif type(obj) == Exception or isinstance(obj, Exception) or type(obj) == type: 86 | return str(obj) 87 | 88 | try: 89 | return super(JsonEncoder, self).default(obj) 90 | 91 | except TypeError: 92 | try: 93 | return str(obj) 94 | 95 | except Exception: 96 | return None 97 | 98 | def format_datetime_obj(self, obj): 99 | return obj.isoformat() 100 | 101 | 102 | class JsonFormatter(logging.Formatter): 103 | """ 104 | A custom formatter to format logging records as json strings. 105 | Extra values will be formatted as str() if not supported by 106 | json default encoder 107 | """ 108 | 109 | def __init__( 110 | self, 111 | *args: Any, 112 | json_default: OptionalCallableOrStr = None, 113 | json_encoder: OptionalCallableOrStr = None, 114 | json_serialiser: Union[Callable, str] = json.dumps, 115 | json_indent: Optional[Union[int, str]] = None, 116 | json_ensure_ascii: bool = True, 117 | prefix: str = "", 118 | rename_fields: Optional[dict] = None, 119 | static_fields: Optional[dict] = None, 120 | reserved_attrs: Tuple[str, ...] = RESERVED_ATTRS, 121 | timestamp: Union[bool, str] = False, 122 | **kwargs: Any 123 | ): 124 | """ 125 | :param json_default: a function for encoding non-standard objects 126 | as outlined in https://docs.python.org/3/library/json.html 127 | :param json_encoder: optional custom encoder 128 | :param json_serializer: a :meth:`json.dumps`-compatible callable 129 | that will be used to serialize the log record. 130 | :param json_indent: indent parameter for json.dumps 131 | :param json_ensure_ascii: ensure_ascii parameter for json.dumps 132 | :param prefix: an optional string prefix added at the beginning of 133 | the formatted string 134 | :param rename_fields: an optional dict, used to rename field names in the output. 135 | Rename message to @message: {'message': '@message'} 136 | :param static_fields: an optional dict, used to add fields with static values to all logs 137 | :param reserved_attrs: an optional list of fields that will be skipped when 138 | outputting json log record. Defaults to all log record attributes: 139 | http://docs.python.org/library/logging.html#logrecord-attributes 140 | :param timestamp: an optional string/boolean field to add a timestamp when 141 | outputting the json log record. If string is passed, timestamp will be added 142 | to log record using string as key. If True boolean is passed, timestamp key 143 | will be "timestamp". Defaults to False/off. 144 | """ 145 | self.json_default = self._str_to_fn(json_default) 146 | self.json_encoder = self._str_to_fn(json_encoder) 147 | self.json_serializer = self._str_to_fn(json_serialiser) 148 | self.json_indent = json_indent 149 | self.json_ensure_ascii = json_ensure_ascii 150 | self.prefix = prefix 151 | self.rename_fields = rename_fields or {} 152 | self.static_fields = static_fields or {} 153 | self.reserved_attrs = dict(zip(reserved_attrs, reserved_attrs)) 154 | self.timestamp = timestamp 155 | 156 | # super(JsonFormatter, self).__init__(*args, **kwargs) 157 | logging.Formatter.__init__(self, *args, **kwargs) 158 | if not self.json_encoder and not self.json_default: 159 | self.json_encoder = JsonEncoder 160 | 161 | self._required_fields = self.parse() 162 | self._skip_fields = dict(zip(self._required_fields, self._required_fields)) 163 | self._skip_fields.update(self.reserved_attrs) 164 | 165 | def _str_to_fn(self, fn_as_str): 166 | """ 167 | If the argument is not a string, return whatever was passed in. 168 | Parses a string such as package.module.function, imports the module 169 | and returns the function. 170 | 171 | :param fn_as_str: The string to parse. If not a string, return it. 172 | """ 173 | if not isinstance(fn_as_str, str): 174 | return fn_as_str 175 | 176 | path, _, function = fn_as_str.rpartition(".") 177 | module = importlib.import_module(path) 178 | return getattr(module, function) 179 | 180 | def parse(self) -> List[str]: 181 | """ 182 | Parses format string looking for substitutions 183 | 184 | This method is responsible for returning a list of fields (as strings) 185 | to include in all log messages. 186 | """ 187 | if isinstance(self._style, logging.StringTemplateStyle): 188 | formatter_style_pattern = re.compile(r"\$\{(.+?)\}", re.IGNORECASE) 189 | elif isinstance(self._style, logging.StrFormatStyle): 190 | formatter_style_pattern = re.compile(r"\{(.+?)\}", re.IGNORECASE) 191 | # PercentStyle is parent class of StringTemplateStyle and StrFormatStyle so 192 | # it needs to be checked last. 193 | elif isinstance(self._style, logging.PercentStyle): 194 | formatter_style_pattern = re.compile(r"%\((.+?)\)", re.IGNORECASE) 195 | else: 196 | raise ValueError("Invalid format: %s" % self._fmt) 197 | 198 | if self._fmt: 199 | return formatter_style_pattern.findall(self._fmt) 200 | else: 201 | return [] 202 | 203 | def add_fields( 204 | self, 205 | log_record: Dict[str, Any], 206 | record: logging.LogRecord, 207 | message_dict: Dict[str, Any], 208 | ) -> None: 209 | """ 210 | Override this method to implement custom logic for adding fields. 211 | """ 212 | for field in self._required_fields: 213 | log_record[field] = record.__dict__.get(field) 214 | 215 | log_record.update(self.static_fields) 216 | log_record.update(message_dict) 217 | merge_record_extra( 218 | record, 219 | log_record, 220 | reserved=self._skip_fields, 221 | rename_fields=self.rename_fields, 222 | ) 223 | 224 | if self.timestamp: 225 | key = self.timestamp if type(self.timestamp) == str else "timestamp" 226 | log_record[key] = datetime.fromtimestamp(record.created, tz=timezone.utc) 227 | 228 | self._perform_rename_log_fields(log_record) 229 | 230 | def _perform_rename_log_fields(self, log_record): 231 | for old_field_name, new_field_name in self.rename_fields.items(): 232 | log_record[new_field_name] = log_record[old_field_name] 233 | del log_record[old_field_name] 234 | 235 | def process_log_record(self, log_record): 236 | """ 237 | Override this method to implement custom logic 238 | on the possibly ordered dictionary. 239 | """ 240 | return log_record 241 | 242 | def jsonify_log_record(self, log_record): 243 | """Returns a json string of the log record.""" 244 | return self.json_serializer( 245 | log_record, 246 | default=self.json_default, 247 | cls=self.json_encoder, 248 | indent=self.json_indent, 249 | ensure_ascii=self.json_ensure_ascii, 250 | ) 251 | 252 | def serialize_log_record(self, log_record: Dict[str, Any]) -> str: 253 | """Returns the final representation of the log record.""" 254 | return "%s%s" % (self.prefix, self.jsonify_log_record(log_record)) 255 | 256 | def format(self, record: logging.LogRecord) -> str: 257 | """Formats a log record and serializes to json""" 258 | message_dict: Dict[str, Any] = {} 259 | # FIXME: logging.LogRecord.msg and logging.LogRecord.message in typeshed 260 | # are always type of str. We shouldn't need to override that. 261 | if isinstance(record.msg, dict): 262 | message_dict = record.msg 263 | record.message = "" 264 | else: 265 | record.message = record.getMessage() 266 | # only format time if needed 267 | if "asctime" in self._required_fields: 268 | record.asctime = self.formatTime(record, self.datefmt) 269 | 270 | # Display formatted exception, but allow overriding it in the 271 | # user-supplied dict. 272 | if record.exc_info and not message_dict.get("exc_info"): 273 | message_dict["exc_info"] = self.formatException(record.exc_info) 274 | if not message_dict.get("exc_info") and record.exc_text: 275 | message_dict["exc_info"] = record.exc_text 276 | # Display formatted record of stack frames 277 | # default format is a string returned from :func:`traceback.print_stack` 278 | if record.stack_info and not message_dict.get("stack_info"): 279 | message_dict["stack_info"] = self.formatStack(record.stack_info) 280 | 281 | log_record: Dict[str, Any] = OrderedDict() 282 | self.add_fields(log_record, record, message_dict) 283 | log_record = self.process_log_record(log_record) 284 | 285 | return self.serialize_log_record(log_record) 286 | -------------------------------------------------------------------------------- /src/pythonjsonlogger/py.typed: -------------------------------------------------------------------------------- 1 | # PEP-561 marker. https://mypy.readthedocs.io/en/latest/installed_packages.html 2 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/madzak/python-json-logger/be42d82ebd1b3276b739a314545f7a3f518ff222/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_jsonlogger.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import unittest 3 | import unittest.mock 4 | import logging 5 | import json 6 | import sys 7 | import traceback 8 | import random 9 | 10 | try: 11 | import xmlrunner # noqa 12 | except ImportError: 13 | pass 14 | 15 | from io import StringIO 16 | 17 | sys.path.append('src/python-json-logger') 18 | from pythonjsonlogger import jsonlogger 19 | import datetime 20 | 21 | 22 | class TestJsonLogger(unittest.TestCase): 23 | def setUp(self): 24 | self.log = logging.getLogger("logging-test-{}".format(random.randint(1, 101))) 25 | self.log.setLevel(logging.DEBUG) 26 | self.buffer = StringIO() 27 | 28 | self.log_handler = logging.StreamHandler(self.buffer) 29 | self.log.addHandler(self.log_handler) 30 | 31 | def test_default_format(self): 32 | fr = jsonlogger.JsonFormatter() 33 | self.log_handler.setFormatter(fr) 34 | 35 | msg = "testing logging format" 36 | self.log.info(msg) 37 | log_json = json.loads(self.buffer.getvalue()) 38 | 39 | self.assertEqual(log_json["message"], msg) 40 | 41 | def test_percentage_format(self): 42 | fr = jsonlogger.JsonFormatter( 43 | # All kind of different styles to check the regex 44 | '[%(levelname)8s] %(message)s %(filename)s:%(lineno)d %(asctime)' 45 | ) 46 | self.log_handler.setFormatter(fr) 47 | 48 | msg = "testing logging format" 49 | self.log.info(msg) 50 | log_json = json.loads(self.buffer.getvalue()) 51 | 52 | self.assertEqual(log_json["message"], msg) 53 | self.assertEqual(log_json.keys(), {'levelname', 'message', 'filename', 'lineno', 'asctime'}) 54 | 55 | def test_rename_base_field(self): 56 | fr = jsonlogger.JsonFormatter(rename_fields={'message': '@message'}) 57 | self.log_handler.setFormatter(fr) 58 | 59 | msg = "testing logging format" 60 | self.log.info(msg) 61 | log_json = json.loads(self.buffer.getvalue()) 62 | 63 | self.assertEqual(log_json["@message"], msg) 64 | 65 | def test_rename_nonexistent_field(self): 66 | fr = jsonlogger.JsonFormatter(rename_fields={'nonexistent_key': 'new_name'}) 67 | self.log_handler.setFormatter(fr) 68 | 69 | stderr_watcher = StringIO() 70 | sys.stderr = stderr_watcher 71 | self.log.info("testing logging rename") 72 | 73 | self.assertTrue("KeyError: 'nonexistent_key'" in stderr_watcher.getvalue()) 74 | 75 | def test_add_static_fields(self): 76 | fr = jsonlogger.JsonFormatter(static_fields={'log_stream': 'kafka'}) 77 | 78 | self.log_handler.setFormatter(fr) 79 | 80 | msg = "testing static fields" 81 | self.log.info(msg) 82 | log_json = json.loads(self.buffer.getvalue()) 83 | 84 | self.assertEqual(log_json["log_stream"], "kafka") 85 | self.assertEqual(log_json["message"], msg) 86 | 87 | def test_format_keys(self): 88 | supported_keys = [ 89 | 'asctime', 90 | 'created', 91 | 'filename', 92 | 'funcName', 93 | 'levelname', 94 | 'levelno', 95 | 'lineno', 96 | 'module', 97 | 'msecs', 98 | 'message', 99 | 'name', 100 | 'pathname', 101 | 'process', 102 | 'processName', 103 | 'relativeCreated', 104 | 'thread', 105 | 'threadName' 106 | ] 107 | 108 | log_format = lambda x: ['%({0:s})s'.format(i) for i in x] 109 | custom_format = ' '.join(log_format(supported_keys)) 110 | 111 | fr = jsonlogger.JsonFormatter(custom_format) 112 | self.log_handler.setFormatter(fr) 113 | 114 | msg = "testing logging format" 115 | self.log.info(msg) 116 | log_msg = self.buffer.getvalue() 117 | log_json = json.loads(log_msg) 118 | 119 | for supported_key in supported_keys: 120 | if supported_key in log_json: 121 | self.assertTrue(True) 122 | 123 | def test_unknown_format_key(self): 124 | fr = jsonlogger.JsonFormatter('%(unknown_key)s %(message)s') 125 | 126 | self.log_handler.setFormatter(fr) 127 | msg = "testing unknown logging format" 128 | try: 129 | self.log.info(msg) 130 | except Exception: 131 | self.assertTrue(False, "Should succeed") 132 | 133 | def test_log_adict(self): 134 | fr = jsonlogger.JsonFormatter() 135 | self.log_handler.setFormatter(fr) 136 | 137 | msg = {"text": "testing logging", "num": 1, 5: "9", 138 | "nested": {"more": "data"}} 139 | 140 | self.log.info(msg) 141 | log_json = json.loads(self.buffer.getvalue()) 142 | self.assertEqual(log_json.get("text"), msg["text"]) 143 | self.assertEqual(log_json.get("num"), msg["num"]) 144 | self.assertEqual(log_json.get("5"), msg[5]) 145 | self.assertEqual(log_json.get("nested"), msg["nested"]) 146 | self.assertEqual(log_json["message"], "") 147 | 148 | def test_log_extra(self): 149 | fr = jsonlogger.JsonFormatter() 150 | self.log_handler.setFormatter(fr) 151 | 152 | extra = {"text": "testing logging", "num": 1, 5: "9", 153 | "nested": {"more": "data"}} 154 | self.log.info("hello", extra=extra) 155 | log_json = json.loads(self.buffer.getvalue()) 156 | self.assertEqual(log_json.get("text"), extra["text"]) 157 | self.assertEqual(log_json.get("num"), extra["num"]) 158 | self.assertEqual(log_json.get("5"), extra[5]) 159 | self.assertEqual(log_json.get("nested"), extra["nested"]) 160 | self.assertEqual(log_json["message"], "hello") 161 | 162 | def test_json_default_encoder(self): 163 | fr = jsonlogger.JsonFormatter() 164 | self.log_handler.setFormatter(fr) 165 | 166 | msg = {"adate": datetime.datetime(1999, 12, 31, 23, 59), 167 | "otherdate": datetime.date(1789, 7, 14), 168 | "otherdatetime": datetime.datetime(1789, 7, 14, 23, 59), 169 | "otherdatetimeagain": datetime.datetime(1900, 1, 1)} 170 | self.log.info(msg) 171 | log_json = json.loads(self.buffer.getvalue()) 172 | self.assertEqual(log_json.get("adate"), "1999-12-31T23:59:00") 173 | self.assertEqual(log_json.get("otherdate"), "1789-07-14") 174 | self.assertEqual(log_json.get("otherdatetime"), "1789-07-14T23:59:00") 175 | self.assertEqual(log_json.get("otherdatetimeagain"), 176 | "1900-01-01T00:00:00") 177 | 178 | @unittest.mock.patch('time.time', return_value=1500000000.0) 179 | def test_json_default_encoder_with_timestamp(self, time_mock): 180 | fr = jsonlogger.JsonFormatter(timestamp=True) 181 | self.log_handler.setFormatter(fr) 182 | 183 | self.log.info("Hello") 184 | 185 | self.assertTrue(time_mock.called) 186 | log_json = json.loads(self.buffer.getvalue()) 187 | self.assertEqual(log_json.get("timestamp"), "2017-07-14T02:40:00+00:00") 188 | 189 | def test_json_custom_default(self): 190 | def custom(o): 191 | return "very custom" 192 | fr = jsonlogger.JsonFormatter(json_default=custom) 193 | self.log_handler.setFormatter(fr) 194 | 195 | msg = {"adate": datetime.datetime(1999, 12, 31, 23, 59), 196 | "normal": "value"} 197 | self.log.info(msg) 198 | log_json = json.loads(self.buffer.getvalue()) 199 | self.assertEqual(log_json.get("adate"), "very custom") 200 | self.assertEqual(log_json.get("normal"), "value") 201 | 202 | def test_json_custom_logic_adds_field(self): 203 | class CustomJsonFormatter(jsonlogger.JsonFormatter): 204 | 205 | def process_log_record(self, log_record): 206 | log_record["custom"] = "value" 207 | # Old Style "super" since Python 2.6's logging.Formatter is old 208 | # style 209 | return jsonlogger.JsonFormatter.process_log_record(self, log_record) 210 | 211 | self.log_handler.setFormatter(CustomJsonFormatter()) 212 | self.log.info("message") 213 | log_json = json.loads(self.buffer.getvalue()) 214 | self.assertEqual(log_json.get("custom"), "value") 215 | 216 | def get_traceback_from_exception_followed_by_log_call(self) -> str: 217 | try: 218 | raise Exception('test') 219 | except Exception: 220 | self.log.exception("hello") 221 | str_traceback = traceback.format_exc() 222 | # Formatter removes trailing new line 223 | if str_traceback.endswith('\n'): 224 | str_traceback = str_traceback[:-1] 225 | 226 | return str_traceback 227 | 228 | def test_exc_info(self): 229 | fr = jsonlogger.JsonFormatter() 230 | self.log_handler.setFormatter(fr) 231 | expected_value = self.get_traceback_from_exception_followed_by_log_call() 232 | 233 | log_json = json.loads(self.buffer.getvalue()) 234 | self.assertEqual(log_json.get("exc_info"), expected_value) 235 | 236 | def test_exc_info_renamed(self): 237 | fr = jsonlogger.JsonFormatter("%(exc_info)s", rename_fields={"exc_info": "stack_trace"}) 238 | self.log_handler.setFormatter(fr) 239 | expected_value = self.get_traceback_from_exception_followed_by_log_call() 240 | 241 | log_json = json.loads(self.buffer.getvalue()) 242 | self.assertEqual(log_json.get("stack_trace"), expected_value) 243 | self.assertEqual(log_json.get("exc_info"), None) 244 | 245 | def test_ensure_ascii_true(self): 246 | fr = jsonlogger.JsonFormatter() 247 | self.log_handler.setFormatter(fr) 248 | self.log.info('Привет') 249 | msg = self.buffer.getvalue().split('"message": "', 1)[1].split('"', 1)[0] 250 | self.assertEqual(msg, r"\u041f\u0440\u0438\u0432\u0435\u0442") 251 | 252 | def test_ensure_ascii_false(self): 253 | fr = jsonlogger.JsonFormatter(json_ensure_ascii=False) 254 | self.log_handler.setFormatter(fr) 255 | self.log.info('Привет') 256 | msg = self.buffer.getvalue().split('"message": "', 1)[1].split('"', 1)[0] 257 | self.assertEqual(msg, "Привет") 258 | 259 | def test_custom_object_serialization(self): 260 | def encode_complex(z): 261 | if isinstance(z, complex): 262 | return (z.real, z.imag) 263 | else: 264 | type_name = z.__class__.__name__ 265 | raise TypeError("Object of type '{}' is no JSON serializable".format(type_name)) 266 | 267 | formatter = jsonlogger.JsonFormatter(json_default=encode_complex, 268 | json_encoder=json.JSONEncoder) 269 | self.log_handler.setFormatter(formatter) 270 | 271 | value = { 272 | "special": complex(3, 8), 273 | } 274 | 275 | self.log.info(" message", extra=value) 276 | msg = self.buffer.getvalue() 277 | self.assertEqual(msg, "{\"message\": \" message\", \"special\": [3.0, 8.0]}\n") 278 | 279 | def test_rename_reserved_attrs(self): 280 | log_format = lambda x: ['%({0:s})s'.format(i) for i in x] 281 | reserved_attrs_map = { 282 | 'exc_info': 'error.type', 283 | 'exc_text': 'error.message', 284 | 'funcName': 'log.origin.function', 285 | 'levelname': 'log.level', 286 | 'module': 'log.origin.file.name', 287 | 'processName': 'process.name', 288 | 'threadName': 'process.thread.name', 289 | 'msg': 'log.message' 290 | } 291 | 292 | custom_format = ' '.join(log_format(reserved_attrs_map.keys())) 293 | reserved_attrs = [_ for _ in jsonlogger.RESERVED_ATTRS if _ not in list(reserved_attrs_map.keys())] 294 | formatter = jsonlogger.JsonFormatter(custom_format, reserved_attrs=reserved_attrs, rename_fields=reserved_attrs_map) 295 | self.log_handler.setFormatter(formatter) 296 | self.log.info("message") 297 | 298 | msg = self.buffer.getvalue() 299 | self.assertEqual(msg, '{"error.type": null, "error.message": null, "log.origin.function": "test_rename_reserved_attrs", "log.level": "INFO", "log.origin.file.name": "test_jsonlogger", "process.name": "MainProcess", "process.thread.name": "MainThread", "log.message": "message"}\n') 300 | 301 | def test_merge_record_extra(self): 302 | record = logging.LogRecord("name", level=1, pathname="", lineno=1, msg="Some message", args=None, exc_info=None) 303 | output = jsonlogger.merge_record_extra(record, target=dict(foo="bar"), reserved=[]) 304 | self.assertIn("foo", output) 305 | self.assertIn("msg", output) 306 | self.assertEqual(output["foo"], "bar") 307 | self.assertEqual(output["msg"], "Some message") 308 | 309 | 310 | if __name__ == '__main__': 311 | if len(sys.argv[1:]) > 0: 312 | if sys.argv[1] == 'xml': 313 | testSuite = unittest.TestLoader().loadTestsFromTestCase( 314 | TestJsonLogger) 315 | xmlrunner.XMLTestRunner(output='reports').run(testSuite) 316 | else: 317 | unittest.main() 318 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | requires = tox>=3 3 | envlist = lint, type, pypy{38,39}, py{36,37,38,39,310,311} 4 | 5 | [gh-actions] 6 | python = 7 | pypy-3.8: pypy38 8 | pypy-3.9: pypy39 9 | 3.6: py36 10 | 3.7: py37 11 | 3.8: py38 12 | 3.9: py39 13 | 3.10: py310 14 | 3.11: py311, type 15 | 16 | [testenv] 17 | description = run unit tests 18 | commands = 19 | python -m unittest discover 20 | 21 | [testenv:lint] 22 | description = run linters 23 | skip_install = true 24 | deps = 25 | black>=22.12 26 | commands = 27 | black src 28 | 29 | [testenv:type] 30 | description = run type checks 31 | deps = 32 | mypy>=1.0 33 | commands = 34 | mypy src 35 | --------------------------------------------------------------------------------