├── .editorconfig ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── logging_loki ├── __init__.py ├── const.py ├── emitter.py └── handlers.py ├── pyproject.toml ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── test_emitter_v0.py └── test_emitter_v1.py └── tox.ini /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [Makefile] 4 | indent_style = tab 5 | 6 | [*.py] 7 | charset = utf-8 8 | indent_style = space 9 | indent_size = 4 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | 53 | # Translations 54 | *.mo 55 | *.pot 56 | 57 | # Django stuff: 58 | *.log 59 | local_settings.py 60 | db.sqlite3 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # IPython 79 | profile_default/ 80 | ipython_config.py 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # Environments 92 | .env 93 | .venv 94 | env/ 95 | venv/ 96 | ENV/ 97 | env.bak/ 98 | venv.bak/ 99 | 100 | # Spyder project settings 101 | .spyderproject 102 | .spyproject 103 | 104 | # Rope project settings 105 | .ropeproject 106 | 107 | # mkdocs documentation 108 | /site 109 | 110 | # mypy 111 | .mypy_cache/ 112 | .dmypy.json 113 | dmypy.json 114 | 115 | # Pyre type checker 116 | .pyre/ 117 | 118 | # JetBrains IDE 119 | .idea 120 | 121 | # MacOS 122 | .DS_Store 123 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.6" 4 | - "3.7" 5 | - "3.8" 6 | dist: xenial 7 | install: pip install tox tox-venv tox-travis 8 | script: tox -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Andrey Maslov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | python-logging-loki 2 | =================== 3 | 4 | [![PyPI version](https://img.shields.io/pypi/v/python-logging-loki.svg)](https://pypi.org/project/python-logging-loki/) 5 | [![Python version](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue.svg)](https://www.python.org/) 6 | [![License](https://img.shields.io/pypi/l/python-logging-loki.svg)](https://opensource.org/licenses/MIT) 7 | [![Build Status](https://travis-ci.org/GreyZmeem/python-logging-loki.svg?branch=master)](https://travis-ci.org/GreyZmeem/python-logging-loki) 8 | 9 | Python logging handler for Loki. 10 | https://grafana.com/loki 11 | 12 | Installation 13 | ============ 14 | ```bash 15 | pip install python-logging-loki 16 | ``` 17 | 18 | Usage 19 | ===== 20 | 21 | ```python 22 | import logging 23 | import logging_loki 24 | 25 | 26 | handler = logging_loki.LokiHandler( 27 | url="https://my-loki-instance/loki/api/v1/push", 28 | tags={"application": "my-app"}, 29 | auth=("username", "password"), 30 | version="1", 31 | ) 32 | 33 | logger = logging.getLogger("my-logger") 34 | logger.addHandler(handler) 35 | logger.error( 36 | "Something happened", 37 | extra={"tags": {"service": "my-service"}}, 38 | ) 39 | ``` 40 | 41 | Example above will send `Something happened` message along with these labels: 42 | - Default labels from handler 43 | - Message level as `serverity` 44 | - Logger's name as `logger` 45 | - Labels from `tags` item of `extra` dict 46 | 47 | The given example is blocking (i.e. each call will wait for the message to be sent). 48 | But you can use the built-in `QueueHandler` and` QueueListener` to send messages in a separate thread. 49 | 50 | ```python 51 | import logging.handlers 52 | import logging_loki 53 | from multiprocessing import Queue 54 | 55 | 56 | queue = Queue(-1) 57 | handler = logging.handlers.QueueHandler(queue) 58 | handler_loki = logging_loki.LokiHandler( 59 | url="https://my-loki-instance/loki/api/v1/push", 60 | tags={"application": "my-app"}, 61 | auth=("username", "password"), 62 | version="1", 63 | ) 64 | logging.handlers.QueueListener(queue, handler_loki) 65 | 66 | logger = logging.getLogger("my-logger") 67 | logger.addHandler(handler) 68 | logger.error(...) 69 | ``` 70 | 71 | Or you can use `LokiQueueHandler` shortcut, which will automatically create listener and handler. 72 | 73 | ```python 74 | import logging.handlers 75 | import logging_loki 76 | from multiprocessing import Queue 77 | 78 | 79 | handler = logging_loki.LokiQueueHandler( 80 | Queue(-1), 81 | url="https://my-loki-instance/loki/api/v1/push", 82 | tags={"application": "my-app"}, 83 | auth=("username", "password"), 84 | version="1", 85 | ) 86 | 87 | logger = logging.getLogger("my-logger") 88 | logger.addHandler(handler) 89 | logger.error(...) 90 | ``` 91 | -------------------------------------------------------------------------------- /logging_loki/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from logging_loki.handlers import LokiHandler 4 | from logging_loki.handlers import LokiQueueHandler 5 | 6 | __all__ = ["LokiHandler", "LokiQueueHandler"] 7 | __version__ = "0.3.1" 8 | name = "logging_loki" 9 | -------------------------------------------------------------------------------- /logging_loki/const.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import string 4 | from typing import Tuple 5 | 6 | #: Default Loki emitter version. 7 | emitter_ver: str = "0" 8 | #: Size of LRU cache for LogQL label formatting. 9 | format_label_lru_size: int = 256 10 | 11 | #: Success HTTP status code from Loki API. 12 | success_response_code: int = 204 13 | 14 | #: Label name indicating logging level. 15 | level_tag: str = "severity" 16 | #: Label name indicating logger name. 17 | logger_tag: str = "logger" 18 | 19 | #: String contains chars that can be used in label names in LogQL. 20 | label_allowed_chars: str = "".join((string.ascii_letters, string.digits, "_")) 21 | #: A list of pairs of characters to replace in the label name. 22 | label_replace_with: Tuple[Tuple[str, str], ...] = ( 23 | ("'", ""), 24 | ('"', ""), 25 | (" ", "_"), 26 | (".", "_"), 27 | ("-", "_"), 28 | ) 29 | -------------------------------------------------------------------------------- /logging_loki/emitter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import abc 4 | import copy 5 | import functools 6 | import logging 7 | import time 8 | from logging.config import ConvertingDict 9 | from typing import Any 10 | from typing import Dict 11 | from typing import List 12 | from typing import Optional 13 | from typing import Tuple 14 | 15 | import requests 16 | import rfc3339 17 | 18 | from logging_loki import const 19 | 20 | BasicAuth = Optional[Tuple[str, str]] 21 | 22 | 23 | class LokiEmitter(abc.ABC): 24 | """Base Loki emitter class.""" 25 | 26 | success_response_code = const.success_response_code 27 | level_tag = const.level_tag 28 | logger_tag = const.logger_tag 29 | label_allowed_chars = const.label_allowed_chars 30 | label_replace_with = const.label_replace_with 31 | session_class = requests.Session 32 | 33 | def __init__(self, url: str, tags: Optional[dict] = None, auth: BasicAuth = None): 34 | """ 35 | Create new Loki emitter. 36 | 37 | Arguments: 38 | url: Endpoint used to send log entries to Loki (e.g. `https://my-loki-instance/loki/api/v1/push`). 39 | tags: Default tags added to every log record. 40 | auth: Optional tuple with username and password for basic HTTP authentication. 41 | 42 | """ 43 | #: Tags that will be added to all records handled by this handler. 44 | self.tags = tags or {} 45 | #: Loki JSON push endpoint (e.g `http://127.0.0.1/loki/api/v1/push`) 46 | self.url = url 47 | #: Optional tuple with username and password for basic authentication. 48 | self.auth = auth 49 | 50 | self._session: Optional[requests.Session] = None 51 | 52 | def __call__(self, record: logging.LogRecord, line: str): 53 | """Send log record to Loki.""" 54 | payload = self.build_payload(record, line) 55 | resp = self.session.post(self.url, json=payload) 56 | if resp.status_code != self.success_response_code: 57 | raise ValueError("Unexpected Loki API response status code: {0}".format(resp.status_code)) 58 | 59 | @abc.abstractmethod 60 | def build_payload(self, record: logging.LogRecord, line) -> dict: 61 | """Build JSON payload with a log entry.""" 62 | raise NotImplementedError # pragma: no cover 63 | 64 | @property 65 | def session(self) -> requests.Session: 66 | """Create HTTP session.""" 67 | if self._session is None: 68 | self._session = self.session_class() 69 | self._session.auth = self.auth or None 70 | return self._session 71 | 72 | def close(self): 73 | """Close HTTP session.""" 74 | if self._session is not None: 75 | self._session.close() 76 | self._session = None 77 | 78 | @functools.lru_cache(const.format_label_lru_size) 79 | def format_label(self, label: str) -> str: 80 | """ 81 | Build label to match prometheus format. 82 | 83 | `Label format `_ 84 | """ 85 | for char_from, char_to in self.label_replace_with: 86 | label = label.replace(char_from, char_to) 87 | return "".join(char for char in label if char in self.label_allowed_chars) 88 | 89 | def build_tags(self, record: logging.LogRecord) -> Dict[str, Any]: 90 | """Return tags that must be send to Loki with a log record.""" 91 | tags = dict(self.tags) if isinstance(self.tags, ConvertingDict) else self.tags 92 | tags = copy.deepcopy(tags) 93 | tags[self.level_tag] = record.levelname.lower() 94 | tags[self.logger_tag] = record.name 95 | 96 | extra_tags = getattr(record, "tags", {}) 97 | if not isinstance(extra_tags, dict): 98 | return tags 99 | 100 | for tag_name, tag_value in extra_tags.items(): 101 | cleared_name = self.format_label(tag_name) 102 | if cleared_name: 103 | tags[cleared_name] = tag_value 104 | 105 | return tags 106 | 107 | 108 | class LokiEmitterV0(LokiEmitter): 109 | """Emitter for Loki < 0.4.0.""" 110 | 111 | def build_payload(self, record: logging.LogRecord, line) -> dict: 112 | """Build JSON payload with a log entry.""" 113 | labels = self.build_labels(record) 114 | ts = rfc3339.format_microsecond(record.created) 115 | stream = { 116 | "labels": labels, 117 | "entries": [{"ts": ts, "line": line}], 118 | } 119 | return {"streams": [stream]} 120 | 121 | def build_labels(self, record: logging.LogRecord) -> str: 122 | """Return Loki labels string.""" 123 | labels: List[str] = [] 124 | for label_name, label_value in self.build_tags(record).items(): 125 | cleared_name = self.format_label(str(label_name)) 126 | cleared_value = str(label_value).replace('"', r"\"") 127 | labels.append('{0}="{1}"'.format(cleared_name, cleared_value)) 128 | return "{{{0}}}".format(",".join(labels)) 129 | 130 | 131 | class LokiEmitterV1(LokiEmitter): 132 | """Emitter for Loki >= 0.4.0.""" 133 | 134 | def build_payload(self, record: logging.LogRecord, line) -> dict: 135 | """Build JSON payload with a log entry.""" 136 | labels = self.build_tags(record) 137 | ns = 1e9 138 | ts = str(int(time.time() * ns)) 139 | stream = { 140 | "stream": labels, 141 | "values": [[ts, line]], 142 | } 143 | return {"streams": [stream]} 144 | -------------------------------------------------------------------------------- /logging_loki/handlers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import logging 4 | import warnings 5 | from logging.handlers import QueueHandler 6 | from logging.handlers import QueueListener 7 | from queue import Queue 8 | from typing import Dict 9 | from typing import Optional 10 | from typing import Type 11 | 12 | from logging_loki import const 13 | from logging_loki import emitter 14 | 15 | 16 | class LokiQueueHandler(QueueHandler): 17 | """This handler automatically creates listener and `LokiHandler` to handle logs queue.""" 18 | 19 | def __init__(self, queue: Queue, **kwargs): 20 | """Create new logger handler with the specified queue and kwargs for the `LokiHandler`.""" 21 | super().__init__(queue) 22 | self.handler = LokiHandler(**kwargs) # noqa: WPS110 23 | self.listener = QueueListener(self.queue, self.handler) 24 | self.listener.start() 25 | 26 | 27 | class LokiHandler(logging.Handler): 28 | """ 29 | Log handler that sends log records to Loki. 30 | 31 | `Loki API `_ 32 | """ 33 | 34 | emitters: Dict[str, Type[emitter.LokiEmitter]] = { 35 | "0": emitter.LokiEmitterV0, 36 | "1": emitter.LokiEmitterV1, 37 | } 38 | 39 | def __init__( 40 | self, 41 | url: str, 42 | tags: Optional[dict] = None, 43 | auth: Optional[emitter.BasicAuth] = None, 44 | version: Optional[str] = None, 45 | ): 46 | """ 47 | Create new Loki logging handler. 48 | 49 | Arguments: 50 | url: Endpoint used to send log entries to Loki (e.g. `https://my-loki-instance/loki/api/v1/push`). 51 | tags: Default tags added to every log record. 52 | auth: Optional tuple with username and password for basic HTTP authentication. 53 | version: Version of Loki emitter to use. 54 | 55 | """ 56 | super().__init__() 57 | 58 | if version is None and const.emitter_ver == "0": 59 | msg = ( 60 | "Loki /api/prom/push endpoint is in the depreciation process starting from version 0.4.0.", 61 | "Explicitly set the emitter version to '0' if you want to use the old endpoint.", 62 | "Or specify '1' if you have Loki version> = 0.4.0.", 63 | "When the old API is removed from Loki, the handler will use the new version by default.", 64 | ) 65 | warnings.warn(" ".join(msg), DeprecationWarning) 66 | 67 | version = version or const.emitter_ver 68 | if version not in self.emitters: 69 | raise ValueError("Unknown emitter version: {0}".format(version)) 70 | self.emitter = self.emitters[version](url, tags, auth) 71 | 72 | def handleError(self, record): # noqa: N802 73 | """Close emitter and let default handler take actions on error.""" 74 | self.emitter.close() 75 | super().handleError(record) 76 | 77 | def emit(self, record: logging.LogRecord): 78 | """Send log record to Loki.""" 79 | # noinspection PyBroadException 80 | try: 81 | self.emitter(record, self.format(record)) 82 | except Exception: 83 | self.handleError(record) 84 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools >= 40.0.0"] 3 | build-backend = "setuptools.build_meta" 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [isort] 2 | multi_line_output = 3 3 | include_trailing_comma = true 4 | line_length = 119 5 | force_single_line = True 6 | forced_separate = logging_loki 7 | default_section = THIRDPARTY 8 | 9 | [flake8] 10 | max-line-length = 120 11 | inline-quotes = double 12 | max-imports = 15 13 | exclude = 14 | .tox 15 | logging_loki/__init__.py 16 | ignore = D100,D104,DAR 17 | per-file-ignores = 18 | logging_loki/const.py:WPS226 19 | tests/*:D,S101,WPS118,WPS202,WPS204,WPS210,WPS226,WPS442 20 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import setuptools 4 | 5 | with open("README.md", "r") as fh: 6 | long_description = fh.read() 7 | 8 | setuptools.setup( 9 | name="python-logging-loki", 10 | version="0.3.1", 11 | description="Python logging handler for Grafana Loki.", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | license="MIT", 15 | author="Andrey Maslov", 16 | author_email="greyzmeem@gmail.com", 17 | url="https://github.com/greyzmeem/python-logging-loki", 18 | packages=setuptools.find_packages(exclude=("tests",)), 19 | python_requires=">=3.6", 20 | install_requires=["rfc3339>=6.1", "requests"], 21 | classifiers=[ 22 | "Development Status :: 4 - Beta", 23 | "Intended Audience :: Developers", 24 | "License :: OSI Approved :: MIT License", 25 | "Operating System :: OS Independent", 26 | "Programming Language :: Python", 27 | "Programming Language :: Python :: 3", 28 | "Programming Language :: Python :: 3.6", 29 | "Programming Language :: Python :: 3.7", 30 | "Programming Language :: Python :: 3.8", 31 | "Topic :: Software Development :: Libraries :: Python Modules", 32 | "Topic :: System :: Logging", 33 | "Topic :: Internet :: WWW/HTTP", 34 | ], 35 | ) 36 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GreyZmeem/python-logging-loki/1dc706a11de14e0cc0122d9489d7b69e91b3c6be/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_emitter_v0.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import logging 4 | import time 5 | from logging.config import dictConfig as loggingDictConfig 6 | from queue import Queue 7 | from typing import Tuple 8 | from unittest.mock import MagicMock 9 | 10 | import pytest 11 | import rfc3339 12 | from freezegun import freeze_time 13 | 14 | from logging_loki.emitter import LokiEmitterV0 15 | 16 | emitter_url: str = "https://example.net/api/prom/push" 17 | record_kwargs = { 18 | "name": "test", 19 | "level": logging.WARNING, 20 | "fn": "", 21 | "lno": "", 22 | "msg": "Test", 23 | "args": None, 24 | "exc_info": None, 25 | } 26 | 27 | 28 | @pytest.fixture() 29 | def emitter_v0() -> Tuple[LokiEmitterV0, MagicMock]: 30 | """Create v1 emitter with mocked http session.""" 31 | response = MagicMock() 32 | response.status_code = LokiEmitterV0.success_response_code 33 | session = MagicMock() 34 | session().post = MagicMock(return_value=response) 35 | 36 | instance = LokiEmitterV0(url=emitter_url) 37 | instance.session_class = session 38 | 39 | return instance, session 40 | 41 | 42 | def create_record(**kwargs) -> logging.LogRecord: 43 | """Create test logging record.""" 44 | log = logging.Logger(__name__) 45 | return log.makeRecord(**{**record_kwargs, **kwargs}) 46 | 47 | 48 | def get_stream(session: MagicMock) -> dict: 49 | """Return first stream item from json payload.""" 50 | kwargs = session().post.call_args[1] 51 | streams = kwargs["json"]["streams"] 52 | return streams[0] 53 | 54 | 55 | def test_record_sent_to_emitter_url(emitter_v0): 56 | emitter, session = emitter_v0 57 | emitter(create_record(), "") 58 | 59 | got = session().post.call_args 60 | assert got[0][0] == emitter_url 61 | 62 | 63 | def test_default_tags_added_to_payload(emitter_v0): 64 | emitter, session = emitter_v0 65 | emitter.tags = {"app": "emitter"} 66 | emitter(create_record(), "") 67 | 68 | stream = get_stream(session) 69 | level = logging.getLevelName(record_kwargs["level"]).lower() 70 | expected_tags = ( 71 | 'app="emitter"', 72 | '{0}="{1}"'.format(emitter.level_tag, level), 73 | '{0}="{1}"'.format(emitter.logger_tag, record_kwargs["name"]), 74 | ) 75 | expected = ",".join(expected_tags) 76 | expected = "{{{0}}}".format(expected) 77 | assert stream["labels"] == expected 78 | 79 | 80 | def test_extra_tag_added(emitter_v0): 81 | emitter, session = emitter_v0 82 | record = create_record(extra={"tags": {"extra_tag": "extra_value"}}) 83 | emitter(record, "") 84 | 85 | stream = get_stream(session) 86 | assert 'extra_tag="extra_value"' in stream["labels"] 87 | 88 | 89 | @pytest.mark.parametrize( 90 | "emitter_v0, label", 91 | ( 92 | (emitter_v0, "test_'svc"), 93 | (emitter_v0, 'test_"svc'), 94 | (emitter_v0, "test svc"), 95 | (emitter_v0, "test-svc"), 96 | (emitter_v0, "test.svc"), 97 | (emitter_v0, "!test_svc?"), 98 | ), 99 | indirect=["emitter_v0"], 100 | ) 101 | def test_label_properly_formatted(emitter_v0, label: str): 102 | emitter, session = emitter_v0 103 | record = create_record(extra={"tags": {label: "extra_value"}}) 104 | emitter(record, "") 105 | 106 | stream = get_stream(session) 107 | assert ',test_svc="extra_value"' in stream["labels"] 108 | 109 | 110 | def test_empty_label_is_not_added_to_stream(emitter_v0): 111 | emitter, session = emitter_v0 112 | record = create_record(extra={"tags": {"!": "extra_value"}}) 113 | emitter(record, "") 114 | 115 | stream = get_stream(session) 116 | assert "!" not in stream["labels"] 117 | assert ",=" not in stream["labels"] 118 | 119 | 120 | def test_non_dict_extra_tag_is_not_added_to_stream(emitter_v0): 121 | emitter, session = emitter_v0 122 | record = create_record(extra={"tags": "invalid"}) 123 | emitter(record, "") 124 | 125 | stream = get_stream(session) 126 | assert "invalid" not in stream["labels"] 127 | 128 | 129 | def test_raises_value_error_on_non_successful_response(emitter_v0): 130 | emitter, session = emitter_v0 131 | session().post().status_code = None 132 | with pytest.raises(ValueError): 133 | emitter(create_record(), "") 134 | pytest.fail("Must raise ValueError on non-successful Loki response") # pragma: no cover 135 | 136 | 137 | def test_logged_messaged_added_to_values(emitter_v0): 138 | emitter, session = emitter_v0 139 | emitter(create_record(), "Test message") 140 | 141 | stream = get_stream(session) 142 | assert stream["entries"][0]["line"] == "Test message" 143 | 144 | 145 | @freeze_time("2019-11-04 00:25:08.123456") 146 | def test_timestamp_added_to_values(emitter_v0): 147 | emitter, session = emitter_v0 148 | emitter(create_record(), "") 149 | 150 | stream = get_stream(session) 151 | expected = rfc3339.format_microsecond(time.time()) 152 | assert stream["entries"][0]["ts"] == expected 153 | 154 | 155 | def test_session_is_closed(emitter_v0): 156 | emitter, session = emitter_v0 157 | emitter(create_record(), "") 158 | emitter.close() 159 | session().close.assert_called_once() 160 | assert emitter._session is None # noqa: WPS437 161 | 162 | 163 | def test_can_build_tags_from_converting_dict(emitter_v0): 164 | logger_name = "converting_dict_tags_v0" 165 | config = { 166 | "version": 1, 167 | "disable_existing_loggers": False, 168 | "handlers": { 169 | logger_name: { 170 | "class": "logging_loki.LokiQueueHandler", 171 | "queue": Queue(-1), 172 | "url": emitter_url, 173 | "tags": {"test": "test"}, 174 | "version": "0", 175 | }, 176 | }, 177 | "loggers": {logger_name: {"handlers": [logger_name], "level": "DEBUG"}}, 178 | } 179 | loggingDictConfig(config) 180 | 181 | logger = logging.getLogger(logger_name) 182 | emitter: LokiEmitterV0 = logger.handlers[0].handler.emitter 183 | emitter.build_tags(create_record()) 184 | -------------------------------------------------------------------------------- /tests/test_emitter_v1.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import logging 4 | from logging.config import dictConfig as loggingDictConfig 5 | from queue import Queue 6 | from typing import Tuple 7 | from unittest.mock import MagicMock 8 | 9 | import pytest 10 | from freezegun import freeze_time 11 | 12 | from logging_loki.emitter import LokiEmitterV1 13 | 14 | emitter_url: str = "https://example.net/loki/api/v1/push/" 15 | record_kwargs = { 16 | "name": "test", 17 | "level": logging.WARNING, 18 | "fn": "", 19 | "lno": "", 20 | "msg": "Test", 21 | "args": None, 22 | "exc_info": None, 23 | } 24 | 25 | 26 | @pytest.fixture() 27 | def emitter_v1() -> Tuple[LokiEmitterV1, MagicMock]: 28 | """Create v1 emitter with mocked http session.""" 29 | response = MagicMock() 30 | response.status_code = LokiEmitterV1.success_response_code 31 | session = MagicMock() 32 | session().post = MagicMock(return_value=response) 33 | 34 | instance = LokiEmitterV1(url=emitter_url) 35 | instance.session_class = session 36 | 37 | return instance, session 38 | 39 | 40 | def create_record(**kwargs) -> logging.LogRecord: 41 | """Create test logging record.""" 42 | log = logging.Logger(__name__) 43 | return log.makeRecord(**{**record_kwargs, **kwargs}) 44 | 45 | 46 | def get_stream(session: MagicMock) -> dict: 47 | """Return first stream item from json payload.""" 48 | kwargs = session().post.call_args[1] 49 | streams = kwargs["json"]["streams"] 50 | return streams[0] 51 | 52 | 53 | def test_record_sent_to_emitter_url(emitter_v1): 54 | emitter, session = emitter_v1 55 | emitter(create_record(), "") 56 | 57 | got = session().post.call_args 58 | assert got[0][0] == emitter_url 59 | 60 | 61 | def test_default_tags_added_to_payload(emitter_v1): 62 | emitter, session = emitter_v1 63 | emitter.tags = {"app": "emitter"} 64 | emitter(create_record(), "") 65 | 66 | stream = get_stream(session) 67 | level = logging.getLevelName(record_kwargs["level"]).lower() 68 | expected = { 69 | emitter.level_tag: level, 70 | emitter.logger_tag: record_kwargs["name"], 71 | "app": "emitter", 72 | } 73 | assert stream["stream"] == expected 74 | 75 | 76 | def test_extra_tag_added(emitter_v1): 77 | emitter, session = emitter_v1 78 | record = create_record(extra={"tags": {"extra_tag": "extra_value"}}) 79 | emitter(record, "") 80 | 81 | stream = get_stream(session) 82 | assert stream["stream"]["extra_tag"] == "extra_value" 83 | 84 | 85 | @pytest.mark.parametrize( 86 | "emitter_v1, label", 87 | ( 88 | (emitter_v1, "test_'svc"), 89 | (emitter_v1, 'test_"svc'), 90 | (emitter_v1, "test svc"), 91 | (emitter_v1, "test-svc"), 92 | (emitter_v1, "test.svc"), 93 | (emitter_v1, "!test_svc?"), 94 | ), 95 | indirect=["emitter_v1"], 96 | ) 97 | def test_label_properly_formatted(emitter_v1, label: str): 98 | emitter, session = emitter_v1 99 | record = create_record(extra={"tags": {label: "extra_value"}}) 100 | emitter(record, "") 101 | 102 | stream = get_stream(session) 103 | assert stream["stream"]["test_svc"] == "extra_value" 104 | 105 | 106 | def test_empty_label_is_not_added_to_stream(emitter_v1): 107 | emitter, session = emitter_v1 108 | record = create_record(extra={"tags": {"!": "extra_value"}}) 109 | emitter(record, "") 110 | 111 | stream = get_stream(session) 112 | assert set(stream["stream"]) == {emitter.logger_tag, emitter.level_tag} 113 | 114 | 115 | def test_non_dict_extra_tag_is_not_added_to_stream(emitter_v1): 116 | emitter, session = emitter_v1 117 | record = create_record(extra={"tags": "invalid"}) 118 | emitter(record, "") 119 | 120 | stream = get_stream(session) 121 | assert set(stream["stream"]) == {emitter.logger_tag, emitter.level_tag} 122 | 123 | 124 | def test_raises_value_error_on_non_successful_response(emitter_v1): 125 | emitter, session = emitter_v1 126 | session().post().status_code = None 127 | with pytest.raises(ValueError): 128 | emitter(create_record(), "") 129 | pytest.fail("Must raise ValueError on non-successful Loki response") # pragma: no cover 130 | 131 | 132 | def test_logged_messaged_added_to_values(emitter_v1): 133 | emitter, session = emitter_v1 134 | emitter(create_record(), "Test message") 135 | 136 | stream = get_stream(session) 137 | assert stream["values"][0][1] == "Test message" 138 | 139 | 140 | @freeze_time("2019-11-04 00:25:08.123456") 141 | def test_timestamp_added_to_values(emitter_v1): 142 | emitter, session = emitter_v1 143 | emitter(create_record(), "") 144 | 145 | stream = get_stream(session) 146 | expected = 1572827108123456000 147 | assert stream["values"][0][0] == str(expected) 148 | 149 | 150 | def test_session_is_closed(emitter_v1): 151 | emitter, session = emitter_v1 152 | emitter(create_record(), "") 153 | emitter.close() 154 | session().close.assert_called_once() 155 | assert emitter._session is None # noqa: WPS437 156 | 157 | 158 | def test_can_build_tags_from_converting_dict(emitter_v1): 159 | logger_name = "converting_dict_tags_v1" 160 | config = { 161 | "version": 1, 162 | "disable_existing_loggers": False, 163 | "handlers": { 164 | logger_name: { 165 | "class": "logging_loki.LokiQueueHandler", 166 | "queue": Queue(-1), 167 | "url": emitter_url, 168 | "tags": {"test": "test"}, 169 | "version": "1", 170 | }, 171 | }, 172 | "loggers": {logger_name: {"handlers": [logger_name], "level": "DEBUG"}}, 173 | } 174 | loggingDictConfig(config) 175 | 176 | logger = logging.getLogger(logger_name) 177 | emitter: LokiEmitterV1 = logger.handlers[0].handler.emitter 178 | emitter.build_tags(create_record()) 179 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | py{36,37,38}, 4 | flake8, 5 | black 6 | isolated_build = true 7 | 8 | [travis] 9 | python = 10 | 3.6: py36 11 | 3.7: py37, flake8, black 12 | 3.8: py38 13 | 14 | [testenv] 15 | setenv = 16 | LC_ALL = en_US.UTF-8 17 | LANG = en_US.UTF-8 18 | deps = 19 | pytest 20 | coverage 21 | freezegun 22 | commands = coverage run -m pytest [] 23 | 24 | [testenv:flake8] 25 | skip_install = true 26 | basepython = python3.7 27 | deps = wemake-python-styleguide 28 | commands = flake8 . 29 | 30 | [testenv:black] 31 | skip_install = true 32 | basepython = python3.7 33 | deps = black==19.10b0 34 | commands = black --check --diff -l 120 -t py36 . 35 | --------------------------------------------------------------------------------