├── tests ├── __init__.py ├── unit │ ├── __init__.py │ ├── core │ │ ├── __init__.py │ │ └── test_framework.py │ ├── formatters │ │ ├── __init__.py │ │ ├── test_stacktrace_formatter.py │ │ └── test_json_formatter.py │ ├── record │ │ ├── __init__.py │ │ ├── test_util.py │ │ └── test_request_log_record.py │ └── test_init.py ├── django_logging │ ├── __init__.py │ └── test_app │ │ ├── __init__.py │ │ ├── urls.py │ │ ├── settings.py │ │ ├── views.py │ │ └── test_django_logging.py ├── common_test_params.py ├── schema_util.py ├── util.py ├── log_schemas.py ├── test_flask_logging.py ├── test_sanic_logging.py ├── test_job_logging.py └── test_falcon_logging.py ├── sap ├── cf_logging │ ├── core │ │ ├── __init__.py │ │ ├── constants.py │ │ ├── response_reader.py │ │ ├── context.py │ │ ├── framework.py │ │ └── request_reader.py │ ├── record │ │ ├── __init__.py │ │ ├── application_info.py │ │ ├── util.py │ │ ├── simple_log_record.py │ │ └── request_log_record.py │ ├── formatters │ │ ├── __init__.py │ │ ├── json_formatter.py │ │ └── stacktrace_formatter.py │ ├── job_logging │ │ ├── __init__.py │ │ ├── context.py │ │ └── framework.py │ ├── defaults.py │ ├── flask_logging │ │ ├── response_reader.py │ │ ├── context.py │ │ ├── request_reader.py │ │ └── __init__.py │ ├── falcon_logging │ │ ├── response_reader.py │ │ ├── context.py │ │ ├── request_reader.py │ │ └── __init__.py │ ├── django_logging │ │ ├── response_reader.py │ │ ├── context.py │ │ ├── request_reader.py │ │ └── __init__.py │ ├── sanic_logging │ │ ├── response_reader.py │ │ ├── request_reader.py │ │ ├── context.py │ │ └── __init__.py │ └── __init__.py └── __init__.py ├── MANIFEST.in ├── conftest.py ├── setup.cfg ├── .travis.yml ├── tox.ini ├── test-requirements.txt ├── .coveragerc ├── .gitignore ├── .reuse └── dep5 ├── setup.py ├── CHANGELOG.md ├── LICENSES └── Apache-2.0.txt ├── LICENSE └── README.rst /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sap/cf_logging/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sap/cf_logging/record/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/django_logging/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/formatters/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/unit/record/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sap/cf_logging/formatters/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sap/cf_logging/job_logging/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/django_logging/test_app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CHANGELOG.md LICENSE README.rst 2 | -------------------------------------------------------------------------------- /sap/__init__.py: -------------------------------------------------------------------------------- 1 | ''' sap namespace ''' 2 | __path__ = __import__('pkgutil').extend_path(__path__, __name__) 3 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | if sys.version_info < (3, 5): 5 | collect_ignore = ['tests/test_sanic_logging.py'] 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal=1 3 | 4 | [metadata] 5 | license_file=LICENSE 6 | long_description=file: README.rst 7 | 8 | [aliases] 9 | test=pytest 10 | 11 | [tool:pytest] 12 | addopts = --verbose 13 | python_files = tests/* 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | matrix: 4 | fast_finish: true 5 | include: 6 | - python: 2.7 7 | env: TOXENV=py27 8 | - python: 3.8 9 | env: TOXENV=py38 10 | - python: 3.8 11 | env: TOXENV=lint 12 | install: 13 | - pip install tox 14 | 15 | script: tox 16 | -------------------------------------------------------------------------------- /sap/cf_logging/core/constants.py: -------------------------------------------------------------------------------- 1 | """ Logging module constants """ 2 | REQUEST_KEY = 'request' 3 | RESPONSE_KEY = 'response' 4 | 5 | LOG_SENSITIVE_CONNECTION_DATA = 'LOG_SENSITIVE_CONNECTION_DATA' 6 | LOG_REMOTE_USER = 'LOG_REMOTE_USER' 7 | LOG_REFERER = 'LOG_REFERER' 8 | 9 | STACKTRACE_MAX_SIZE = 55 * 1024 10 | -------------------------------------------------------------------------------- /tests/django_logging/test_app/urls.py: -------------------------------------------------------------------------------- 1 | """ Urls for example django test app """ 2 | from django.conf.urls import url 3 | 4 | from tests.django_logging.test_app.views import IndexView 5 | 6 | # pylint: disable=invalid-name 7 | urlpatterns = [ 8 | url("^test/path$", IndexView.as_view(), name='log-index') 9 | ] 10 | -------------------------------------------------------------------------------- /sap/cf_logging/defaults.py: -------------------------------------------------------------------------------- 1 | """ Default values used inside the framework """ 2 | import logging 3 | 4 | from datetime import datetime 5 | 6 | 7 | UNKNOWN = '-' 8 | REDACTED = 'redacted' 9 | RESPONSE_SIZE_B = -1 10 | RESPONSE_TIME_MS = 0 11 | REQUEST_SIZE_B = -1 12 | STATUS = -1 13 | 14 | DEFAULT_LOGGING_LEVEL = logging.INFO 15 | 16 | UNIX_EPOCH = datetime(1970, 1, 1) 17 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py38,lint 3 | 4 | [testenv] 5 | deps = -rtest-requirements.txt 6 | 7 | [testenv:py27] 8 | commands = py.test {posargs} 9 | 10 | [testenv:py38] 11 | setenv = SANIC_REGISTER = False 12 | commands = py.test --cov=sap tests {posargs} 13 | 14 | [testenv:lint] 15 | basepython=python3.8 16 | commands= 17 | pylint sap 18 | pylint --extension-pkg-whitelist=falcon tests 19 | -------------------------------------------------------------------------------- /tests/django_logging/test_app/settings.py: -------------------------------------------------------------------------------- 1 | """ Example django test app settings """ 2 | SECRET_KEY = 'fake-key' 3 | INSTALLED_APPS = [ 4 | "tests", 5 | ] 6 | 7 | DATABASES = { 8 | 'default': { 9 | 'ENGINE': 'django.db.backends.sqlite3' 10 | } 11 | } 12 | 13 | ROOT_URLCONF = 'tests.django_logging.test_app.urls' 14 | 15 | MIDDLEWARE = [ 16 | 'sap.cf_logging.django_logging.LoggingMiddleware', 17 | ] 18 | -------------------------------------------------------------------------------- /sap/cf_logging/flask_logging/response_reader.py: -------------------------------------------------------------------------------- 1 | """ Flask response reader """ 2 | from sap.cf_logging.core.response_reader import ResponseReader 3 | 4 | 5 | class FlaskResponseReader(ResponseReader): 6 | """ Implements Flask specific `ResponseReader` """ 7 | 8 | def get_status_code(self, response): 9 | return response.status_code 10 | 11 | def get_response_size(self, response): 12 | return response.calculate_content_length() 13 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | asyncio 2 | falcon 3 | falcon-auth==1.0.3; python_version == '2.7' 4 | falcon-auth; python_version >= '3.5' 5 | Flask 6 | sanic; python_version >= '3.5' 7 | sanic-testing==0.3.0; python_version >= '3.5' 8 | aiohttp; python_version >= '3.5' 9 | sonic182-json-validator 10 | pytest 11 | pytest-cov 12 | pytest-mock 13 | pylint==1.9.5; python_version == '2.7' 14 | pylint==2.5.3; python_version >= '3.5' 15 | tox 16 | django==1.11.29; python_version == '2.7' 17 | django==3.2.25; python_version >= '3.5' 18 | -------------------------------------------------------------------------------- /sap/cf_logging/falcon_logging/response_reader.py: -------------------------------------------------------------------------------- 1 | """ Falcon response reader """ 2 | from sap.cf_logging.core.response_reader import ResponseReader 3 | 4 | CONTENT_LENGTH = 'Content-Length' 5 | 6 | 7 | class FalconResponseReader(ResponseReader): 8 | """ Read log related properties out of falcon response """ 9 | 10 | def get_status_code(self, response): 11 | return response.status.split(' ', 1)[0] 12 | 13 | def get_response_size(self, response): 14 | return response.get_header('Content-Length') 15 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | # .coveragerc to control coverage.py 2 | [run] 3 | branch = True 4 | 5 | [report] 6 | # Regexes for lines to exclude from consideration 7 | exclude_lines = 8 | # Have to re-enable the standard pragma 9 | pragma: no cover 10 | 11 | # Don't complain about missing debug-only code: 12 | def __repr__ 13 | if self\.debug 14 | 15 | # Don't complain if tests don't hit defensive assertion code: 16 | raise AssertionError 17 | raise NotImplementedError 18 | 19 | 20 | [html] 21 | directory = htmlcov 22 | -------------------------------------------------------------------------------- /sap/cf_logging/django_logging/response_reader.py: -------------------------------------------------------------------------------- 1 | """ Django response reader """ 2 | from sap.cf_logging.core.response_reader import ResponseReader 3 | 4 | 5 | class DjangoResponseReader(ResponseReader): 6 | """ Read log related properties out of Django response """ 7 | 8 | def get_status_code(self, response): 9 | return response.status_code 10 | 11 | def get_response_size(self, response): 12 | return len(response.content) 13 | 14 | def get_content_type(self, response): 15 | return response.get('Content-Type') 16 | -------------------------------------------------------------------------------- /sap/cf_logging/sanic_logging/response_reader.py: -------------------------------------------------------------------------------- 1 | """ Sanic response reader """ 2 | from sap.cf_logging.core.response_reader import ResponseReader 3 | 4 | CONTENT_LENGTH = 'Content-Length' 5 | 6 | 7 | class SanicResponseReader(ResponseReader): 8 | """ Implements Sanic specific `ResponseReader` """ 9 | 10 | def get_status_code(self, response): 11 | return response.status 12 | 13 | def get_response_size(self, response): 14 | if CONTENT_LENGTH in response.headers: 15 | return response.headers.get(CONTENT_LENGTH) 16 | return None 17 | -------------------------------------------------------------------------------- /sap/cf_logging/job_logging/context.py: -------------------------------------------------------------------------------- 1 | """ Job logging context - used by the logging package to keep log data """ 2 | import threading 3 | from sap.cf_logging.core.context import Context 4 | 5 | 6 | class JobContext(Context, threading.local): 7 | """ Stores logging context in dict """ 8 | 9 | def __init__(self): 10 | super(JobContext, self).__init__() 11 | self._mem_store = {} 12 | 13 | def set(self, key, value, request): 14 | self._mem_store[key] = value 15 | 16 | def get(self, key, request): 17 | return self._mem_store[key] if key in self._mem_store else None 18 | -------------------------------------------------------------------------------- /sap/cf_logging/record/application_info.py: -------------------------------------------------------------------------------- 1 | """ Module for record constants """ 2 | import os 3 | 4 | from sap.cf_logging import defaults 5 | from sap.cf_logging.record import util 6 | 7 | 8 | LAYER = 'python' 9 | COMPONENT_ID = util.get_vcap_param('application_id', defaults.UNKNOWN) 10 | COMPONENT_NAME = util.get_vcap_param('name', defaults.UNKNOWN) 11 | COMPONENT_INSTANCE = int(os.getenv('CF_INSTANCE_INDEX', "0")) 12 | SPACE_ID = util.get_vcap_param('space_id', defaults.UNKNOWN) 13 | SPACE_NAME = util.get_vcap_param('space_name', defaults.UNKNOWN) 14 | CONTAINER_ID = os.getenv('CF_INSTANCE_IP', defaults.UNKNOWN) 15 | COMPONENT_TYPE = 'application' 16 | -------------------------------------------------------------------------------- /sap/cf_logging/flask_logging/context.py: -------------------------------------------------------------------------------- 1 | """ Flask logging context - used by the logging package to keep 2 | request specific data, needed for logging purposes. 3 | For example correlation_id needs to be stored during request processing, 4 | so all log entries contain it. 5 | """ 6 | from flask import g 7 | from sap.cf_logging.core.context import Context 8 | 9 | 10 | class FlaskContext(Context): 11 | """ Stores logging context in Flask's request scope """ 12 | 13 | def set(self, key, value, request): 14 | if g: 15 | setattr(g, key, value) 16 | 17 | def get(self, key, request): 18 | return getattr(g, key, None) if g else None 19 | -------------------------------------------------------------------------------- /sap/cf_logging/falcon_logging/context.py: -------------------------------------------------------------------------------- 1 | """ Falcon logging context - used by the logging package to keep 2 | request specific data, needed for logging purposes. 3 | For example correlation_id needs to be stored during request processing, 4 | so all log entries contain it. 5 | """ 6 | 7 | from sap.cf_logging.core.context import Context 8 | 9 | 10 | class FalconContext(Context): 11 | """ Stores logging context in Falcon's request object""" 12 | 13 | def set(self, key, value, request): 14 | if request: 15 | request.context[key] = value 16 | 17 | def get(self, key, request): 18 | return request.context.get(key) if request else None 19 | -------------------------------------------------------------------------------- /sap/cf_logging/sanic_logging/request_reader.py: -------------------------------------------------------------------------------- 1 | """ Sanic request reader """ 2 | from sap.cf_logging import defaults 3 | from sap.cf_logging.core.request_reader import RequestReader 4 | 5 | 6 | class SanicRequestReader(RequestReader): 7 | """ Reads data from Sanic request """ 8 | 9 | def get_remote_user(self, request): 10 | return defaults.UNKNOWN 11 | 12 | def get_protocol(self, request): 13 | return defaults.UNKNOWN 14 | 15 | def get_content_length(self, request): 16 | return defaults.UNKNOWN 17 | 18 | def get_remote_ip(self, request): 19 | return request.ip[0] 20 | 21 | def get_remote_port(self, request): 22 | return None 23 | -------------------------------------------------------------------------------- /sap/cf_logging/sanic_logging/context.py: -------------------------------------------------------------------------------- 1 | """ Sanic logging context - used by the logging package to keep 2 | request specific data during request processing for logging purposes. 3 | For example correlation_id needs to be stored so all log entries contain it. 4 | """ 5 | from sap.cf_logging.core.context import Context 6 | 7 | CONTEXT_NAME = 'cf_logger_context' 8 | 9 | 10 | class SanicContext(Context): 11 | """ Stores logging context in Sanic's request object """ 12 | 13 | def set(self, key, value, request): 14 | if request is not None: 15 | setattr(request.ctx, key, value) 16 | 17 | def get(self, key, request): 18 | if request is None: 19 | return None 20 | 21 | return getattr(request.ctx, key, None) 22 | -------------------------------------------------------------------------------- /sap/cf_logging/flask_logging/request_reader.py: -------------------------------------------------------------------------------- 1 | """ Flask request reader """ 2 | from sap.cf_logging import defaults 3 | from sap.cf_logging.core.request_reader import RequestReader 4 | 5 | 6 | class FlaskRequestReader(RequestReader): 7 | """ Reads data from Flask request """ 8 | 9 | def get_remote_user(self, request): 10 | if request.authorization is not None: 11 | return request.authorization.username 12 | 13 | return defaults.UNKNOWN 14 | 15 | def get_protocol(self, request): 16 | return request.environ.get('SERVER_PROTOCOL') 17 | 18 | def get_content_length(self, request): 19 | return request.content_length 20 | 21 | def get_remote_ip(self, request): 22 | return request.remote_addr 23 | 24 | def get_remote_port(self, request): 25 | return request.environ.get('REMOTE_PORT') 26 | -------------------------------------------------------------------------------- /sap/cf_logging/job_logging/framework.py: -------------------------------------------------------------------------------- 1 | """ Framework to be used by worker/job running applications """ 2 | from sap.cf_logging.core.framework import Framework 3 | from sap.cf_logging.job_logging.context import JobContext 4 | from sap.cf_logging.core.request_reader import RequestReader 5 | from sap.cf_logging.core.response_reader import ResponseReader 6 | 7 | JOB_FRAMEWORK_NAME = 'job.framework' 8 | 9 | 10 | class JobFramework(Framework): 11 | """ Simple framework using default request and response readers. 12 | Uses JobContext to keeping properties in memory """ 13 | 14 | def __init__(self, context=None, custom_fields=None): 15 | super(JobFramework, self).__init__( 16 | JOB_FRAMEWORK_NAME, 17 | context or JobContext(), 18 | RequestReader(), 19 | ResponseReader(), 20 | custom_fields=custom_fields 21 | ) 22 | -------------------------------------------------------------------------------- /sap/cf_logging/formatters/json_formatter.py: -------------------------------------------------------------------------------- 1 | """ Module for the JsonFormatter """ 2 | import json 3 | import logging 4 | import sys 5 | from sap.cf_logging.record.simple_log_record import SimpleLogRecord 6 | 7 | def _default_serializer(obj): 8 | return str(obj) 9 | 10 | if sys.version_info[0] == 3: 11 | def _encode(obj): 12 | return json.dumps(obj, default=_default_serializer) 13 | else: 14 | def _encode(obj): 15 | return unicode(json.dumps(obj, default=_default_serializer)) # pylint: disable=undefined-variable 16 | 17 | 18 | class JsonFormatter(logging.Formatter): 19 | """ 20 | Format application log in JSON format 21 | """ 22 | 23 | def format(self, record): 24 | """ Format the known log records in JSON format """ 25 | if isinstance(record, SimpleLogRecord): 26 | return _encode(record.format()) 27 | return super(JsonFormatter, self).format(record) 28 | -------------------------------------------------------------------------------- /sap/cf_logging/core/response_reader.py: -------------------------------------------------------------------------------- 1 | """ Module for the ResponseReader class """ 2 | 3 | 4 | class ResponseReader(object): # pylint: disable=useless-object-inheritance 5 | """ 6 | Helper class for extracting logging-relevant information from HTTP response object 7 | """ 8 | 9 | def get_status_code(self, response): 10 | """ 11 | get response's integer status code 12 | 13 | :param response: 14 | """ 15 | raise NotImplementedError 16 | 17 | def get_response_size(self, response): 18 | """ 19 | get response's size in bytes 20 | 21 | :param response: 22 | """ 23 | raise NotImplementedError 24 | 25 | # pylint: disable=no-self-use 26 | def get_content_type(self, response): 27 | """ 28 | get response's MIME/media type 29 | 30 | :param response: 31 | """ 32 | return response.content_type 33 | -------------------------------------------------------------------------------- /sap/cf_logging/django_logging/context.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django logging context - used by the logging package to keep 3 | request specific data, needed for logging purposes. 4 | For example correlation_id needs to be stored during request processing, 5 | so all log entries contain it. 6 | """ 7 | 8 | from sap.cf_logging.core.context import Context 9 | 10 | 11 | def _init_context(request): 12 | if not hasattr(request, 'context'): 13 | request.context = {} 14 | 15 | class DjangoContext(Context): 16 | """ Stores logging context in Django's request objecct """ 17 | 18 | def set(self, key, value, request): 19 | if request is None: 20 | return 21 | _init_context(request) 22 | request.context[key] = value 23 | 24 | def get(self, key, request): 25 | if request is None: 26 | return None 27 | _init_context(request) 28 | return request.context.get(key) if request else None 29 | -------------------------------------------------------------------------------- /sap/cf_logging/falcon_logging/request_reader.py: -------------------------------------------------------------------------------- 1 | """ Falcon request reader """ 2 | 3 | from sap.cf_logging import defaults 4 | from sap.cf_logging.core.request_reader import RequestReader 5 | 6 | 7 | class FalconRequestReader(RequestReader): 8 | """ Read log related properties out of falcon request """ 9 | 10 | def __init__(self, username_key): 11 | self._username_key = username_key 12 | 13 | def get_remote_user(self, request): 14 | user = request.context.get('user') 15 | if user and self._username_key: 16 | return user.get(self._username_key) or defaults.UNKNOWN 17 | 18 | return defaults.UNKNOWN 19 | 20 | def get_protocol(self, request): 21 | return request.scheme 22 | 23 | def get_content_length(self, request): 24 | # pylint: disable=duplicate-code 25 | return request.content_length 26 | 27 | def get_remote_ip(self, request): 28 | # pylint: disable=duplicate-code 29 | return request.remote_addr 30 | 31 | def get_remote_port(self, request): 32 | return defaults.UNKNOWN 33 | -------------------------------------------------------------------------------- /sap/cf_logging/core/context.py: -------------------------------------------------------------------------------- 1 | """ Module Context """ 2 | 3 | _CORRELATION_ID_KEY = 'correlation_id' 4 | 5 | class Context(object): # pylint: disable=useless-object-inheritance 6 | """ Class for getting and setting context variables """ 7 | 8 | def set(self, key, value, request): 9 | """ Store session variable """ 10 | raise NotImplementedError 11 | 12 | def get(self, key, request): 13 | """ Get session variable """ 14 | raise NotImplementedError 15 | 16 | def get_correlation_id(self, request=None): 17 | """ Gets the current correlation_id. Ensure that you are calling this method in the 18 | appropriate location of your code having in mind which framework you are using. """ 19 | 20 | return self.get(_CORRELATION_ID_KEY, request) 21 | 22 | def set_correlation_id(self, value, request=None): 23 | """ Sets the current correlation_id. Ensure that you are calling this method in the 24 | appropriate location of your code having in mind which framework you are using. """ 25 | 26 | return self.set(_CORRELATION_ID_KEY, value, request) 27 | -------------------------------------------------------------------------------- /tests/unit/formatters/test_stacktrace_formatter.py: -------------------------------------------------------------------------------- 1 | """ Module testing the functionality of the StacktraceFormatter class """ 2 | from sap.cf_logging.formatters.stacktrace_formatter import format_stacktrace 3 | 4 | 5 | STACKTRACE = ''.join(['Traceback (most recent call last):\n', 6 | 'File "nonexistent_file.py", line 100, in nonexistent_function\n', 7 | 'raise ValueError("Oh no")\n', 8 | 'ValueError: Oh no']) 9 | 10 | 11 | def test_stacktrace_not_truncated(): 12 | """ Test that stacktrace is not truncated when smaller than the stacktrace maximum size """ 13 | formatted = format_stacktrace(STACKTRACE) 14 | assert "TRUNCATED" not in formatted 15 | assert "OMITTED" not in formatted 16 | 17 | 18 | def test_stacktrace_truncated(monkeypatch): 19 | """ Test that stacktrace is truncated when bigger than the stacktrace maximum size """ 20 | monkeypatch.setattr('sap.cf_logging.core.constants.STACKTRACE_MAX_SIZE', 120) 21 | 22 | formatted = ''.join(format_stacktrace(STACKTRACE)) 23 | assert "TRUNCATED" in formatted 24 | assert "OMITTED" in formatted 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .vscode 3 | .DS_Store 4 | *.iml 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | .env 16 | env/ 17 | env3/ 18 | venv/ 19 | local_tests/ 20 | build/ 21 | develop-eggs/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | dist/ 27 | lib64/ 28 | parts/ 29 | .pytest_cache/ 30 | sdist/ 31 | var/ 32 | *.egg-info/ 33 | .installed.cfg 34 | *.egg 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *,cover 55 | .hypothesis/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | #Ipython Notebook 71 | .ipynb_checkpoints 72 | .project 73 | .pydevproject 74 | .settings/ 75 | 76 | -------------------------------------------------------------------------------- /sap/cf_logging/django_logging/request_reader.py: -------------------------------------------------------------------------------- 1 | """ Django request reader """ 2 | 3 | from sap.cf_logging import defaults 4 | from sap.cf_logging.core.request_reader import RequestReader 5 | 6 | 7 | class DjangoRequestReader(RequestReader): 8 | """ Read log related properties out of Django request """ 9 | 10 | def get_remote_user(self, request): 11 | return request.META.get('REMOTE_USER') or defaults.UNKNOWN 12 | 13 | def get_protocol(self, request): 14 | return request.scheme 15 | 16 | def get_content_length(self, request): 17 | return request.META.get('CONTENT_LENGTH') or defaults.UNKNOWN 18 | 19 | def get_remote_ip(self, request): 20 | return request.META.get('REMOTE_ADDR') 21 | 22 | def get_remote_port(self, request): 23 | return request.META.get('SERVER_PORT') or defaults.UNKNOWN 24 | 25 | def get_http_header(self, request, header_name, default=None): 26 | if request is None: 27 | return default 28 | 29 | if header_name in request.META: 30 | return request.META.get(header_name) 31 | if header_name.upper() in request.META: 32 | return request.META.get(header_name.upper()) 33 | 34 | return default 35 | -------------------------------------------------------------------------------- /tests/common_test_params.py: -------------------------------------------------------------------------------- 1 | """ Test parameters common for Flask and Sanic """ 2 | import base64 3 | import sys 4 | from tests.schema_util import string as v_str 5 | from tests.schema_util import num as v_num 6 | 7 | if sys.version_info[0] == 3: 8 | def _bytes(arg): 9 | return bytes(arg, 'ascii') 10 | else: 11 | def _bytes(arg): 12 | return bytes(arg) 13 | 14 | 15 | def get_web_record_header_fixtures(): 16 | """ returns: list of tuples (headers, expectation) """ 17 | test_cases = [ 18 | ({}, { 19 | 'remote_user': v_str('-'), 20 | 'method': v_str('GET'), 21 | 'response_status': v_num(val=200), 22 | 'response_content_type': v_str('text/plain'), 23 | 'request': v_str('/test/path')}) 24 | ] 25 | for header in ['X-Correlation-ID', 'X-CorrelationID', 'X-Request-ID', 'X-Vcap-Request-Id']: 26 | test_cases.append(({header: '298ebf9d-be1d-11e7-88ff-2c44fd152864'}, 27 | {'correlation_id': v_str('298ebf9d-be1d-11e7-88ff-2c44fd152864')})) 28 | 29 | return test_cases 30 | 31 | 32 | def auth_basic(user, passwd): 33 | """ generates basic authentication header content """ 34 | return 'Basic ' + base64.b64encode(_bytes(user + ':' + passwd)).decode('ascii') 35 | -------------------------------------------------------------------------------- /tests/unit/record/test_util.py: -------------------------------------------------------------------------------- 1 | """ Tests `record.util` package """ 2 | from datetime import datetime 3 | 4 | from sap.cf_logging.defaults import UNIX_EPOCH 5 | from sap.cf_logging.record import util 6 | 7 | 8 | def test_parse_int_default(): 9 | """ test util.parse_int will return default for invalid input""" 10 | assert util.parse_int('1a23', 3) == 3 11 | assert util.parse_int('{}', -1) == -1 12 | 13 | 14 | def test_parse_int(): 15 | """ test util.parse_int works correctly """ 16 | assert util.parse_int('194', 5) == 194 17 | assert util.parse_int(9811, -1) == 9811 18 | 19 | 20 | def test_epoch_nano_second(): 21 | """ test util.epoch_nano_second calculates correctly """ 22 | date = datetime(2017, 1, 1, 0, 0, 0, 12) 23 | nanoseconds = 1483228800000012000 24 | assert util.epoch_nano_second(date) == nanoseconds 25 | 26 | 27 | def test_iso_time_format(): 28 | """ test util.iso_time_format builds ISO date string """ 29 | date = datetime(2017, 1, 5, 1, 2, 3, 2000) 30 | assert util.iso_time_format(date) == '2017-01-05T01:02:03.002Z' 31 | 32 | 33 | def test_time_delta_ms(): 34 | """ test time_delta_ms calculates delta between date and unix epoch""" 35 | date = datetime(2017, 1, 1, 0, 0, 0, 12000) 36 | assert util.time_delta_ms(UNIX_EPOCH, date) == 1483228800012 37 | -------------------------------------------------------------------------------- /tests/django_logging/test_app/views.py: -------------------------------------------------------------------------------- 1 | """ Views for example django test app """ 2 | import logging 3 | 4 | from django.http import HttpResponse 5 | from django.views import generic 6 | 7 | from sap.cf_logging.core.constants import REQUEST_KEY 8 | from tests.util import config_logger, check_log_record 9 | from tests.log_schemas import JOB_LOG_SCHEMA 10 | 11 | # pylint: disable=unused-argument 12 | 13 | class IndexView(generic.View): 14 | """ View that is hit on the index route """ 15 | def get(self, request): # pylint: disable=no-self-use 16 | """ Return a basic http response """ 17 | return HttpResponse("Hello test!", content_type='text/plain') 18 | 19 | 20 | class UserLoggingView(generic.View): 21 | """ View that logs custom user information """ 22 | provide_request = False 23 | 24 | def __init__(self, *args, **kwargs): 25 | self.logger, self.stream = config_logger('user.logging') 26 | super(UserLoggingView, self).__init__(*args, **kwargs) 27 | 28 | def get(self, request, *args, **kwargs): 29 | """ Log a custom user message with the logger """ 30 | expected = kwargs.get('expected') or {} 31 | extra = kwargs.get('extra') or {} 32 | if self.provide_request: 33 | extra.update({REQUEST_KEY: request}) 34 | 35 | self.logger.log(logging.INFO, 'in route headers', extra=extra) 36 | assert check_log_record(self.stream, JOB_LOG_SCHEMA, expected) == {} 37 | return HttpResponse("ok", content_type='text/plain') 38 | -------------------------------------------------------------------------------- /tests/schema_util.py: -------------------------------------------------------------------------------- 1 | """ Common utils and defaults used in the tests""" 2 | 3 | WORD = r'.+' 4 | TEXT = r'.*' 5 | STRING_NUM = r'[\d+|-]' 6 | IP = r'[[0-9]+|.?]+\d$' 7 | HOST_NAME = r'[[0-9]+|.?]+\d$' 8 | 9 | LEVEL = ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET'] 10 | 11 | 12 | def num(val=None): 13 | """ Return validation schema for numbers """ 14 | if val is None: 15 | return {'type': int} 16 | return { 17 | 'type': int, 18 | 'gt': val - 1, 19 | 'lt': val + 1 20 | } 21 | 22 | 23 | def pos_num(): 24 | """ Return validation schema for positive numbers """ 25 | return { 26 | 'type': int, 27 | 'gt': -1, 28 | 'gt_error': 'Not a positive number' 29 | } 30 | 31 | 32 | def string(regex): 33 | """ Return validation schema for strings """ 34 | return { 35 | 'format': regex, 36 | 'format_error': 'Incorrect {value} string value' 37 | } 38 | 39 | 40 | def enum(lst): 41 | """ Return validation schema for lists """ 42 | return { 43 | 'in': lst, 44 | 'in_error': 'Value not in list of expected values' 45 | } 46 | 47 | 48 | def iso_datetime(): 49 | """ Return validation schema for datetime """ 50 | return { 51 | 'format': r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z', 52 | 'format_error': 'Invalid date format' 53 | } 54 | 55 | 56 | def extend(dict1, dict2): 57 | """ Extend dict1 with dict2 """ 58 | new_dict = dict1.copy() 59 | new_dict.update(dict2) 60 | return new_dict 61 | -------------------------------------------------------------------------------- /tests/util.py: -------------------------------------------------------------------------------- 1 | """ Module for common functions """ 2 | import io 3 | import logging 4 | import json 5 | import os 6 | from json_validator.validator import JsonValidator 7 | from sap.cf_logging.formatters.json_formatter import JsonFormatter 8 | from sap.cf_logging.core.constants import \ 9 | LOG_SENSITIVE_CONNECTION_DATA, LOG_REMOTE_USER, LOG_REFERER 10 | 11 | from tests.schema_util import extend 12 | 13 | 14 | def check_log_record(stream, schema, expected): 15 | """ Using the JsonValidator check that the data in the stream 16 | matches the expected output 17 | """ 18 | log_json = stream.getvalue() 19 | log_object = json.JSONDecoder().decode(log_json) 20 | expected_json = extend(schema, expected) 21 | _, error = JsonValidator(expected_json).validate(log_object) 22 | print('----------------------------------------------->') 23 | print(log_json) 24 | print('<-----------------------------------------------') 25 | return error 26 | 27 | 28 | def config_logger(logger_name): 29 | """ Function to configure a JSONLogger and print the output into a stream""" 30 | stream = io.StringIO() 31 | stream_handler = logging.StreamHandler(stream) 32 | stream_handler.setFormatter(JsonFormatter()) 33 | logger = logging.getLogger(logger_name) 34 | logger.addHandler(stream_handler) 35 | return logger, stream 36 | 37 | def enable_sensitive_fields_logging(): 38 | """ sets a few logging related env vars """ 39 | os.environ[LOG_SENSITIVE_CONNECTION_DATA] = 'true' 40 | os.environ[LOG_REMOTE_USER] = 'true' 41 | os.environ[LOG_REFERER] = 'true' 42 | -------------------------------------------------------------------------------- /sap/cf_logging/formatters/stacktrace_formatter.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module for formatting utilities for stacktrace 3 | generated by user logging.exception call 4 | """ 5 | import re 6 | 7 | from sap.cf_logging.core import constants 8 | 9 | 10 | def format_stacktrace(stacktrace): 11 | """ 12 | Removes tab characters 13 | Truncates stacktrace to maximum size 14 | 15 | :param stacktrace: string representation of a stacktrace 16 | """ 17 | if not isinstance(stacktrace, str): 18 | return '' 19 | 20 | stacktrace = re.sub('\t', ' ', stacktrace) 21 | 22 | if len(stacktrace) <= constants.STACKTRACE_MAX_SIZE: 23 | return stacktrace 24 | 25 | stacktrace_beginning = _stacktrace_beginning( 26 | stacktrace, constants.STACKTRACE_MAX_SIZE // 3 27 | ) 28 | 29 | stacktrace_end = _stacktrace_end( 30 | stacktrace, (constants.STACKTRACE_MAX_SIZE // 3) * 2 31 | ) 32 | 33 | new_stacktrace = "-------- STACK TRACE TRUNCATED --------\n" + stacktrace_beginning +\ 34 | "-------- OMITTED --------\n" + stacktrace_end 35 | 36 | return new_stacktrace 37 | 38 | def _stacktrace_beginning(stacktrace, size): 39 | """ Gets the first `size` bytes of the stacktrace """ 40 | if len(stacktrace) <= size: 41 | return stacktrace 42 | 43 | return stacktrace[:size] 44 | 45 | def _stacktrace_end(stacktrace, size): 46 | """ Gets the last `size` bytes of the stacktrace """ 47 | stacktrace_length = len(stacktrace) 48 | if stacktrace_length <= size: 49 | return stacktrace 50 | 51 | return stacktrace[:-(stacktrace_length-size)] 52 | -------------------------------------------------------------------------------- /sap/cf_logging/record/util.py: -------------------------------------------------------------------------------- 1 | """ Util module with helper functions """ 2 | import json 3 | import os 4 | 5 | from sap.cf_logging.defaults import UNIX_EPOCH 6 | 7 | VCAP_APPLICATION = json.loads(os.getenv('VCAP_APPLICATION', default='{}')) 8 | 9 | 10 | def get_vcap_param(param_name, default=None): 11 | """ 12 | Returns a parameter from the VCAP_APPLICATION environment variable 13 | :param param_name: the name of the parameter 14 | :param default: a default value if it cannot be found 15 | :return: The value of the parameter, the default if it cannot be found, 16 | or '-' if the default is not specified. 17 | """ 18 | return VCAP_APPLICATION.get(param_name, default) 19 | 20 | 21 | def epoch_nano_second(datetime_): 22 | """ Returns the nanoseconds since epoch time """ 23 | return int((datetime_ - UNIX_EPOCH).total_seconds()) * 1000000000 + datetime_.microsecond * 1000 24 | 25 | 26 | def iso_time_format(datetime_): 27 | """ Returns ISO time formatted string """ 28 | return '%04d-%02d-%02dT%02d:%02d:%02d.%03dZ' % ( 29 | datetime_.year, datetime_.month, datetime_.day, datetime_.hour, 30 | datetime_.minute, datetime_.second, int(datetime_.microsecond / 1000)) 31 | 32 | 33 | def time_delta_ms(start, end): 34 | """ Returns the delta time between to datetime objects """ 35 | time_delta = end - start 36 | return int(time_delta.total_seconds()) * 1000 + \ 37 | int(time_delta.microseconds / 1000) 38 | 39 | 40 | def parse_int(_int, default): 41 | """ Parses an int and returns the result 42 | A default can be provided in case the parse fails 43 | """ 44 | try: 45 | integer = int(_int) 46 | except: # pylint: disable-msg=bare-except 47 | integer = default 48 | return integer 49 | -------------------------------------------------------------------------------- /sap/cf_logging/core/framework.py: -------------------------------------------------------------------------------- 1 | """ Module framework """ 2 | import sys 3 | from sap.cf_logging.core.context import Context 4 | from sap.cf_logging.core.request_reader import RequestReader 5 | from sap.cf_logging.core.response_reader import ResponseReader 6 | 7 | 8 | STR_CLASS = str if sys.version_info[0] == 3 else basestring # pylint: disable=undefined-variable 9 | 10 | 11 | def _check_instance(obj, clazz): 12 | if not isinstance(obj, clazz): 13 | raise TypeError('Provided object is not valid {}'.format(clazz.__name__)) 14 | 15 | 16 | class Framework(object): # pylint: disable=useless-object-inheritance 17 | """ Framework class holds Context, RequestReader, ResponseReader """ 18 | 19 | # pylint: disable=too-many-arguments 20 | def __init__(self, name, context, request_reader, response_reader, custom_fields=None): 21 | if not name or not isinstance(name, STR_CLASS): 22 | raise TypeError('Provided name is not valid string') 23 | _check_instance(context, Context) 24 | _check_instance(request_reader, RequestReader) 25 | _check_instance(response_reader, ResponseReader) 26 | self._name = name 27 | self._context = context 28 | self._request_reader = request_reader 29 | self._response_reader = response_reader 30 | self._custom_fields = custom_fields or {} 31 | 32 | @property 33 | def custom_fields(self): 34 | """ Get the custom fields """ 35 | return self._custom_fields 36 | 37 | @property 38 | def context(self): 39 | """ Get Context """ 40 | return self._context 41 | 42 | @property 43 | def request_reader(self): 44 | """ Get RequestReader """ 45 | return self._request_reader 46 | 47 | @property 48 | def response_reader(self): 49 | """ Get Response Reader """ 50 | return self._response_reader 51 | -------------------------------------------------------------------------------- /.reuse/dep5: -------------------------------------------------------------------------------- 1 | Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: cf-python-logging-support 3 | Source: https://github.com/SAP/cf-python-logging-support 4 | Disclaimer: The code in this project may include calls to APIs (“API Calls”) of 5 | SAP or third-party products or services developed outside of this project 6 | (“External Products”). 7 | “APIs” means application programming interfaces, as well as their respective 8 | specifications and implementing code that allows software to communicate with 9 | other software. 10 | API Calls to External Products are not licensed under the open source license 11 | that governs this project. The use of such API Calls and related External 12 | Products are subject to applicable additional agreements with the relevant 13 | provider of the External Products. In no event shall the open source license 14 | that governs this project grant any rights in or to any External Products,or 15 | alter, expand or supersede any terms of the applicable additional agreements. 16 | If you have a valid license agreement with SAP for the use of a particular SAP 17 | External Product, then you may make use of any API Calls included in this 18 | project’s code for that SAP External Product, subject to the terms of such 19 | license agreement. If you do not have a valid license agreement for the use of 20 | a particular SAP External Product, then you may only make use of any API Calls 21 | in this project for that SAP External Product for your internal, non-productive 22 | and non-commercial test and evaluation of such API Calls. Nothing herein grants 23 | you any rights to use or access any SAP External Product, or provide any third 24 | parties the right to use of access any SAP External Product, through API Calls. 25 | 26 | Files: * 27 | Copyright: 2017-2021 SAP SE or an SAP affiliate company and cf-python-logging-support contributors 28 | License: Apache-2.0 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | ''' setup.py ''' 2 | import ast 3 | import re 4 | 5 | from setuptools import find_packages 6 | from setuptools import setup 7 | 8 | _VERSION_REGEX = re.compile(r'__version__\s+=\s+(.*)') 9 | with open('./sap/cf_logging/__init__.py', 'rb') as f: 10 | VERSION = str(ast.literal_eval(_VERSION_REGEX.search(f.read().decode('utf-8')).group(1))) 11 | 12 | setup( 13 | name='sap_cf_logging', 14 | version=VERSION, 15 | url='https://github.com/SAP/cf-python-logging-support', 16 | license='Apache License, Version 2.0', 17 | author='SAP', 18 | description='Python logging library to emit JSON logs in a SAP CloudFoundry environment', 19 | long_description_content_type='text/x-rst', 20 | packages=find_packages(include=['sap*']), 21 | include_package_data=True, 22 | zip_safe=False, 23 | platforms='any', 24 | install_requires=[], 25 | classifiers=[ # http://pypi.python.org/pypi?%3Aaction=list_classifiers 26 | 'Development Status :: 4 - Beta', 27 | 'Framework :: Flask', 28 | 'Intended Audience :: Developers', 29 | 'License :: OSI Approved :: Apache Software License', 30 | 'Operating System :: OS Independent', 31 | 'Programming Language :: Python', 32 | 'Programming Language :: Python :: 2', 33 | 'Programming Language :: Python :: 2.6', 34 | 'Programming Language :: Python :: 2.7', 35 | 'Programming Language :: Python :: 3', 36 | 'Programming Language :: Python :: 3.3', 37 | 'Programming Language :: Python :: 3.4', 38 | 'Programming Language :: Python :: 3.5', 39 | 'Programming Language :: Python :: 3.6', 40 | 'Programming Language :: Python :: 3.8', 41 | 'Programming Language :: Python :: 3.9', 42 | 'Programming Language :: Python :: 3.10', 43 | 'Programming Language :: Python :: 3.11', 44 | 'Topic :: Software Development :: Libraries :: Python Modules', 45 | 'Topic :: Software Development', 46 | 'Topic :: System :: Logging', 47 | ] 48 | ) 49 | -------------------------------------------------------------------------------- /tests/unit/core/test_framework.py: -------------------------------------------------------------------------------- 1 | """ Tests `core.framework` """ 2 | import pytest 3 | from sap.cf_logging.core.context import Context 4 | from sap.cf_logging.core.request_reader import RequestReader 5 | from sap.cf_logging.core.response_reader import ResponseReader 6 | from sap.cf_logging.core.framework import Framework 7 | 8 | # pylint: disable=abstract-method 9 | 10 | CONTEXT = Context() 11 | REQUEST_READER = RequestReader() 12 | RESPONSE_READER = ResponseReader() 13 | 14 | 15 | @pytest.mark.parametrize('initializers', [ 16 | {'context': None, 'request_reader': REQUEST_READER, 'response_reader': RESPONSE_READER}, 17 | {'context': {}, 'request_reader': REQUEST_READER, 'response_reader': RESPONSE_READER}, 18 | {'context': CONTEXT, 'request_reader': None, 'response_reader': RESPONSE_READER}, 19 | {'context': CONTEXT, 'request_reader': {}, 'response_reader': RESPONSE_READER}, 20 | {'context': CONTEXT, 'request_reader': REQUEST_READER, 'response_reader': None}, 21 | {'context': CONTEXT, 'request_reader': REQUEST_READER, 'response_reader': {}} 22 | ]) 23 | @pytest.mark.xfail(raises=TypeError, strict=True) 24 | def test_init(initializers): 25 | """ test constructor with invalid context, request_reader and response_reader """ 26 | Framework('django', **initializers) 27 | 28 | 29 | def test_init_accept_inherited(): 30 | """ test Framework::init accepts inherited classes arguments """ 31 | 32 | class MyContext(Context): # pylint: disable=missing-docstring 33 | pass 34 | 35 | class MyRequestReader(RequestReader): # pylint: disable=missing-docstring 36 | pass 37 | 38 | class MyResponseReader(ResponseReader): # pylint: disable=missing-docstring 39 | pass 40 | 41 | Framework('name', MyContext(), MyRequestReader(), MyResponseReader()) 42 | 43 | 44 | @pytest.mark.parametrize('name', [None, 123, '']) 45 | @pytest.mark.xfail(raises=TypeError, strict=True) 46 | def test_init_name(name): 47 | """ test invalid 'name' provided to constructor """ 48 | Framework(name, CONTEXT, REQUEST_READER, RESPONSE_READER) 49 | -------------------------------------------------------------------------------- /tests/unit/test_init.py: -------------------------------------------------------------------------------- 1 | """ Tests for cf_logging.init """ 2 | import logging 3 | import pytest 4 | from sap import cf_logging 5 | 6 | from sap.cf_logging.core.constants import REQUEST_KEY, RESPONSE_KEY 7 | from sap.cf_logging.core.framework import Framework 8 | from sap.cf_logging.record.request_log_record import RequestWebRecord 9 | from sap.cf_logging.record.simple_log_record import SimpleLogRecord 10 | 11 | 12 | # pylint: disable=protected-access 13 | 14 | @pytest.mark.xfail(raises=RuntimeError, reason='cf_logging.init can be called once', strict=True) 15 | def test_init_called_twice(mocker): 16 | """ test cf_logging.init can be called only once """ 17 | framework = mocker.Mock(Framework) 18 | cf_logging._SETUP_DONE = False 19 | cf_logging.init(framework, level=logging.DEBUG) 20 | cf_logging.init(framework, level=logging.DEBUG) 21 | 22 | 23 | @pytest.mark.xfail(raises=TypeError, reason='', strict=True) 24 | def test_init_incorrect_framework(): 25 | """ test cf_logging.init fails for invalid framework """ 26 | cf_logging._SETUP_DONE = False 27 | cf_logging.init({}) 28 | 29 | 30 | def _make_record(extra): 31 | cf_logger = cf_logging.CfLogger('mylogger') 32 | return cf_logger.makeRecord('', '', '', '', '', '', '', '', extra=extra) 33 | 34 | 35 | def test_init_cf_logger_simple_log(mocker): 36 | """ tests CfLogger creates SimpleLogRecord if extra is incomplete """ 37 | framework = mocker.Mock(Framework) 38 | mocker.patch.object(framework, 'custom_fields', return_value=None) 39 | cf_logging.init(framework) 40 | assert isinstance(_make_record(extra={}), SimpleLogRecord) 41 | assert isinstance(_make_record(extra={REQUEST_KEY: {}}), SimpleLogRecord) 42 | assert isinstance(_make_record(extra={RESPONSE_KEY: {}}), SimpleLogRecord) 43 | 44 | 45 | def test_init_cf_logger_web_log(mocker): 46 | """ tests CfLogger creates SimpleLogRecord if extra contains request and response """ 47 | mocker.patch.object(RequestWebRecord, '__init__', lambda *a, **kwa: None) 48 | record = _make_record(extra={REQUEST_KEY: {}, RESPONSE_KEY: {}}) 49 | assert isinstance(record, RequestWebRecord) 50 | -------------------------------------------------------------------------------- /tests/unit/formatters/test_json_formatter.py: -------------------------------------------------------------------------------- 1 | """ Tests json log formatting """ 2 | import json 3 | import logging 4 | from sap.cf_logging.job_logging.framework import JobFramework 5 | from sap.cf_logging.record.simple_log_record import SimpleLogRecord 6 | from sap.cf_logging.formatters.json_formatter import JsonFormatter 7 | 8 | 9 | LEVEL, FILE, LINE, EXC_INFO = logging.INFO, "(unknown file)", 0, None 10 | FORMATTER = JsonFormatter() 11 | 12 | 13 | def test_unknown_records_format(): 14 | """ test unknown log records will be delegated to logging.Formatter """ 15 | log_record = logging.LogRecord('name', LEVEL, FILE, LINE, 'msg', [], EXC_INFO) 16 | assert FORMATTER.format(log_record) == 'msg' 17 | 18 | 19 | def test_non_json_serializable(): 20 | """ test json formatter handles non JSON serializable object """ 21 | class _MyClass(object): # pylint: disable=too-few-public-methods,useless-object-inheritance 22 | pass 23 | 24 | extra = {'cls': _MyClass()} 25 | framework = JobFramework() 26 | log_record = SimpleLogRecord(extra, framework, 'name', LEVEL, FILE, LINE, 'msg', [], EXC_INFO) 27 | record_object = json.loads(FORMATTER.format(log_record)) 28 | assert record_object.get('cls') is not None 29 | assert 'MyClass' in record_object.get('cls') 30 | 31 | def test_stacktrace_is_added_to_msg_field(): 32 | """ 33 | Tests that JSONFormatter adds stracktrace to msg field. The stacktrace field 34 | is no longer rendered in Kibana, see https://github.com/SAP/cf-python-logging-support/issues/45 35 | for related report. 36 | """ 37 | # Generate exception for the test 38 | try: 39 | raise ValueError("Dummy Exception") 40 | except ValueError as e: 41 | exc_info = (type(e), e, e.__traceback__) 42 | 43 | framework = JobFramework() 44 | extra = {} 45 | 46 | log_record = SimpleLogRecord(extra, framework, 'name', logging.ERROR, FILE, LINE, 'Error found!', [], exc_info) 47 | record_object = json.loads(FORMATTER.format(log_record)) 48 | assert "Dummy Exception" in "".join(record_object["stacktrace"]) 49 | expected_msg = "Error found!" 50 | assert record_object["msg"] == expected_msg 51 | -------------------------------------------------------------------------------- /tests/log_schemas.py: -------------------------------------------------------------------------------- 1 | """ Log JSON schemas according to sonic182-json-validator 2 | to be used for validation of real log records """ 3 | import tests.schema_util as u 4 | 5 | CF_ATTRIBUTES_SCHEMA = { 6 | 'layer': u.string(r'^python$'), 7 | 'component_id': u.string(u.TEXT), 8 | 'component_name': u.string(u.TEXT), 9 | 'component_instance': u.pos_num(), 10 | 'space_id': u.string(u.TEXT), 11 | 'space_name': u.string(u.TEXT), 12 | 'container_id': u.string(u.TEXT), 13 | 'component_type': u.string(u.TEXT) 14 | } 15 | 16 | JOB_LOG_SCHEMA = u.extend(CF_ATTRIBUTES_SCHEMA, { 17 | 'type': { 18 | 'in': ['log'], 19 | 'in_error': 'Invalid "type" property' 20 | }, 21 | 'correlation_id': u.string(r'([a-z\d]+-?)*'), 22 | 'logger': u.string(u.WORD), 23 | 'thread': u.string(u.WORD), 24 | 'level': u.enum(u.LEVEL), 25 | 'written_at': u.iso_datetime(), 26 | 'written_ts': u.pos_num(), 27 | 'msg': u.string(u.WORD), 28 | 'component_type': u.string(r'^application$'), 29 | }) 30 | 31 | CUST_FIELD_SCHEMA = u.extend(JOB_LOG_SCHEMA, { 32 | '#cf': { 33 | 'type': dict, 34 | 'properties': { 35 | 'string': {'type' : list, 'items': { 36 | 'type': dict, 37 | 'properties': { 38 | 'v': u.string(u.WORD), 39 | 'k': u.string(u.WORD), 40 | 'i': u.pos_num() 41 | } 42 | }} 43 | } 44 | } 45 | }) 46 | 47 | WEB_LOG_SCHEMA = u.extend(CF_ATTRIBUTES_SCHEMA, { 48 | 'type': u.string('^request$'), 49 | 'written_at': u.iso_datetime(), 50 | 'written_ts': u.pos_num(), 51 | 'correlation_id': u.string(r'.*'), 52 | # 'correlation_id' : u.string(r'[a-z|\d+-|\d]+'), 53 | 'remote_user': u.string(u.TEXT), 54 | 'request': u.string(r'^/.*'), 55 | 'referer': u.string(u.TEXT), 56 | 'x_forwarded_for': u.string(u.TEXT), 57 | 'protocol': u.string(u.TEXT), 58 | 'method': u.string(r'^GET$'), 59 | 'remote_ip': u.string(u.IP), 60 | 'request_size_b': u.num(), 61 | 'remote_host': u.string(u.HOST_NAME), 62 | 'remote_port': u.string(u.STRING_NUM), 63 | 'request_received_at': u.iso_datetime(), 64 | 'direction': u.string(r'IN'), 65 | 'response_time_ms': u.pos_num(), 66 | 'response_status': u.pos_num(), 67 | 'response_size_b': u.num(), 68 | 'response_content_type': u.string(r'^text/.*'), 69 | 'response_sent_at': u.iso_datetime() 70 | }) 71 | -------------------------------------------------------------------------------- /sap/cf_logging/__init__.py: -------------------------------------------------------------------------------- 1 | """ Module that contains the CfLogger class """ 2 | import logging 3 | import sys 4 | 5 | from sap.cf_logging import defaults 6 | from sap.cf_logging.core.constants import REQUEST_KEY, RESPONSE_KEY 7 | from sap.cf_logging.core.framework import Framework 8 | from sap.cf_logging.formatters.json_formatter import JsonFormatter 9 | from sap.cf_logging.job_logging.framework import JobFramework 10 | from sap.cf_logging.record.request_log_record import RequestWebRecord 11 | from sap.cf_logging.record.simple_log_record import SimpleLogRecord 12 | 13 | __version__ = '4.2.7' 14 | 15 | _SETUP_DONE = False 16 | FRAMEWORK = None 17 | 18 | 19 | class CfLogger(logging.Logger): 20 | """ CfLogger class inherits from logging.Logger and makes custom 21 | log messages 22 | """ 23 | 24 | # pylint: disable=too-many-arguments,arguments-differ,keyword-arg-before-vararg 25 | def makeRecord(self, name, level, fn, lno, msg, msgargs, exc_info, 26 | func=None, extra=None, *args, **kwargs): 27 | """ Returns SimpleLogMessage or a RequestWebRecord depending on the extra variable """ 28 | # check what record type this is 29 | cls = None 30 | if extra is not None and REQUEST_KEY in extra and RESPONSE_KEY in extra: 31 | cls = RequestWebRecord 32 | else: 33 | cls = SimpleLogRecord 34 | 35 | return cls(extra, FRAMEWORK, name, level, fn, lno, msg, msgargs, exc_info, 36 | func, *args, **kwargs) 37 | 38 | def init(cfl_framework=None, level=defaults.DEFAULT_LOGGING_LEVEL, custom_fields=None): 39 | """ Initialize function. It sets up the logging library to output JSON 40 | formatted messages. 41 | 42 | Optional arguments framework to use and logging.level 43 | """ 44 | global FRAMEWORK # pylint: disable=global-statement 45 | global _SETUP_DONE # pylint: disable=global-statement 46 | if _SETUP_DONE: 47 | raise RuntimeError('cf_logging already initialized') 48 | 49 | if cfl_framework is not None and not isinstance(cfl_framework, Framework): 50 | raise TypeError('expecting framework of type {}'.format(Framework.__name__)) 51 | 52 | _SETUP_DONE = True 53 | FRAMEWORK = cfl_framework or JobFramework(custom_fields=custom_fields) 54 | 55 | logging.setLoggerClass(CfLogger) 56 | 57 | handler = logging.StreamHandler(sys.stdout) 58 | handler.setFormatter(JsonFormatter()) 59 | 60 | root = logging.getLogger() 61 | root.setLevel(level) 62 | root.addHandler(handler) 63 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. 3 | 4 | This project adheres to [Semantic Versioning](http://semver.org/). 5 | 6 | The format is based on [Keep a Changelog](http://keepachangelog.com/). 7 | 8 | 9 | ## 4.2.7 - 2024-06-27 10 | 11 | ### Fixed 12 | 13 | - Remove stacktrace from the message element of the log 14 | 15 | ## 4.2.6 - 2024-02-26 16 | 17 | ### Update 18 | 19 | - Update django version => 3.2.24 20 | 21 | ## 4.2.5 - 2023-07-28 22 | 23 | ### Fixed 24 | 25 | - Include stacktrace also for non-error log level if exc_info is present 26 | 27 | ## 4.2.4 - 2022-04-26 28 | 29 | ### Update 30 | 31 | - Update django version => 2.2.28 32 | 33 | ## 4.2.3 - 2022-03-23 34 | 35 | ### Fixed 36 | 37 | - Fix stacktrace not showing in kibana 38 | 39 | ## 4.2.2 - 2021-04-01 40 | 41 | ### Fixed 42 | 43 | - Fix stacktrace format 44 | 45 | ## 4.2.1 - 2021-02-23 46 | 47 | ### Added 48 | 49 | - Add support for custom fields 50 | 51 | ### Fixed 52 | 53 | - Fix context store for Sanic 54 | 55 | ## 4.1.1 - 2019-04-19 56 | 57 | ### Fixed 58 | 59 | - Fix logging not usable outside request 60 | 61 | ## 4.1.0 - 2018-09-13 62 | 63 | ### Added 64 | 65 | - Django support 66 | 67 | ## 4.0.1 - 2018-07-10 68 | 69 | ### Fixed 70 | 71 | - Log error throws an exception 72 | 73 | ## 4.0.0 - 2018-07-04 74 | 75 | ### Added 76 | - Log exception stacktrace 77 | 78 | ### Changed 79 | - Incompatible change: removed `log` function from request in falcon support 80 | 81 | ## 3.3.1 - 2018-06-18 82 | 83 | ### Fixed 84 | - Correlation ID should be thread safe 85 | 86 | ## 3.3.0 - 2018-06-07 87 | 88 | ### Added 89 | - Support set and get correlation ID 90 | 91 | ## 3.2.0 - 2018-05-31 92 | 93 | ### Added 94 | - Support for Falcon web framework 95 | 96 | ### Changed 97 | - Hide sensitive fields by default 98 | 99 | ### Fixed 100 | - Do not apply JSON formatting on unknown LogRecord 101 | 102 | ## 3.1.1 - 2018-03-06 103 | 104 | ### Fixed 105 | - Fix missing sap namespace in response readers 106 | 107 | ## 3.1.0 - 2018-01-04 108 | 109 | ### Changed 110 | - Introduced sap namespace 111 | 112 | ## 3.0.1 - 2017-12-11 113 | 114 | ### Changed 115 | - Improved documentation 116 | 117 | ## 3.0.0 - 2017-11-16 118 | 119 | ### Changed 120 | - Improved JSON logging 121 | - Simplified integration with Flask applications 122 | - Simplified integration with Sanic applications 123 | - Cleanup for open-sourcing 124 | -------------------------------------------------------------------------------- /sap/cf_logging/django_logging/__init__.py: -------------------------------------------------------------------------------- 1 | """ Logging support for Django based applications """ 2 | import logging 3 | from datetime import datetime 4 | 5 | from sap import cf_logging 6 | from sap.cf_logging import defaults 7 | from sap.cf_logging.core.constants import REQUEST_KEY, RESPONSE_KEY 8 | from sap.cf_logging.core.framework import Framework 9 | from sap.cf_logging.django_logging.context import DjangoContext 10 | from sap.cf_logging.django_logging.request_reader import DjangoRequestReader 11 | from sap.cf_logging.django_logging.response_reader import DjangoResponseReader 12 | 13 | DJANGO_FRAMEWORK_NAME = 'django.framework' 14 | 15 | 16 | class LoggingMiddleware(object): # pylint: disable=useless-object-inheritance 17 | """ Django logging middleware """ 18 | 19 | def __init__(self, get_response, logger_name='cf.django.logger'): 20 | self._logger_name = logger_name 21 | self._get_response = get_response 22 | 23 | def __call__(self, request): 24 | self.process_request(request) 25 | response = self._get_response(request) 26 | 27 | response = self.process_response(request, response) 28 | return response 29 | 30 | def process_request(self, request):# pylint: disable=no-self-use 31 | """ 32 | Process the request before routing it. 33 | 34 | :param request: - Django Request object 35 | """ 36 | framework = cf_logging.FRAMEWORK 37 | cid = framework.request_reader.get_correlation_id(request) 38 | framework.context.set_correlation_id(cid, request) 39 | framework.context.set('request_started_at', datetime.utcnow(), request) 40 | 41 | def process_response(self, request, response): 42 | """ 43 | Post-processing of the response (after routing). 44 | 45 | :param request: - Django Request object 46 | :param request: - Django Response object 47 | """ 48 | cf_logging.FRAMEWORK.context.set( 49 | 'response_sent_at', datetime.utcnow(), request) 50 | extra = {REQUEST_KEY: request, RESPONSE_KEY: response} 51 | logging.getLogger(self._logger_name).info('', extra=extra) 52 | return response 53 | 54 | 55 | def init(level=defaults.DEFAULT_LOGGING_LEVEL, custom_fields=None): 56 | """ 57 | Initializes logging in JSON format. 58 | 59 | :param level: - valid log level from standard logging package (optional) 60 | """ 61 | framework = Framework(DJANGO_FRAMEWORK_NAME, DjangoContext(), 62 | DjangoRequestReader(), DjangoResponseReader(), 63 | custom_fields=custom_fields) 64 | 65 | cf_logging.init(framework, level) 66 | -------------------------------------------------------------------------------- /sap/cf_logging/flask_logging/__init__.py: -------------------------------------------------------------------------------- 1 | """ Flask logging support package 2 | - Configures standard logging package to produce JSON 3 | - Produces info request log entry per request 4 | """ 5 | import logging 6 | from datetime import datetime 7 | from functools import wraps 8 | import flask 9 | from flask import request 10 | 11 | from sap import cf_logging 12 | from sap.cf_logging import defaults 13 | from sap.cf_logging.core.constants import REQUEST_KEY, RESPONSE_KEY 14 | from sap.cf_logging.core.framework import Framework 15 | from sap.cf_logging.flask_logging.context import FlaskContext 16 | from sap.cf_logging.flask_logging.request_reader import FlaskRequestReader 17 | from sap.cf_logging.flask_logging.response_reader import FlaskResponseReader 18 | 19 | FLASK_FRAMEWORK_NAME = 'flask.framework' 20 | 21 | 22 | def before_request(wrapped): 23 | """ Use as a decorator on Flask before_request handler 24 | Handles correlation_id by setting it in the context for log records 25 | """ 26 | @wraps(wrapped) 27 | def _wrapper(): 28 | framework = cf_logging.FRAMEWORK 29 | cid = framework.request_reader.get_correlation_id(request) 30 | framework.context.set_correlation_id(cid, request) 31 | framework.context.set('request_started_at', datetime.utcnow(), request) 32 | return wrapped() 33 | return _wrapper 34 | 35 | 36 | def after_request(wrapped): 37 | """ Use as a decorator on Flask after_request handler 38 | Creates info log record per request 39 | """ 40 | @wraps(wrapped) 41 | def _wrapper(response): 42 | cf_logging.FRAMEWORK.context.set( 43 | 'response_sent_at', datetime.utcnow(), request) 44 | extra = {REQUEST_KEY: request, RESPONSE_KEY: response} 45 | logging.getLogger('cf.flask.logger').info('', extra=extra) 46 | return wrapped(response) 47 | return _wrapper 48 | 49 | 50 | def init(app, level=defaults.DEFAULT_LOGGING_LEVEL, custom_fields=None): 51 | """ Initializes logging in JSON format. 52 | 53 | Adds before and after request handlers to `app` object to enable request info log. 54 | :param app: - Flask application object 55 | :param level: - valid log level from standard logging package (optional) 56 | """ 57 | if not isinstance(app, flask.Flask): 58 | raise TypeError('application should be instance of Flask') 59 | 60 | _init_framework(level, custom_fields=custom_fields) 61 | 62 | @app.before_request 63 | @before_request 64 | def _app_before_request(): 65 | pass 66 | 67 | @app.after_request 68 | @after_request 69 | def _app_after_request(response): 70 | return response 71 | 72 | 73 | def _init_framework(level, custom_fields): 74 | logging.getLogger('werkzeug').disabled = True 75 | 76 | framework = Framework(FLASK_FRAMEWORK_NAME, 77 | FlaskContext(), FlaskRequestReader(), FlaskResponseReader(), 78 | custom_fields=custom_fields) 79 | cf_logging.init(framework, level) 80 | -------------------------------------------------------------------------------- /sap/cf_logging/falcon_logging/__init__.py: -------------------------------------------------------------------------------- 1 | """ Logging support for Falcon https://falconframework.org/ based applications """ 2 | import logging 3 | from datetime import datetime 4 | 5 | import falcon 6 | 7 | from sap import cf_logging 8 | from sap.cf_logging import defaults 9 | from sap.cf_logging.core.constants import REQUEST_KEY, RESPONSE_KEY 10 | from sap.cf_logging.core.framework import Framework 11 | from sap.cf_logging.falcon_logging.context import FalconContext 12 | from sap.cf_logging.falcon_logging.request_reader import FalconRequestReader 13 | from sap.cf_logging.falcon_logging.response_reader import FalconResponseReader 14 | 15 | FALCON_FRAMEWORK_NAME = 'falcon.framework' 16 | 17 | 18 | class LoggingMiddleware(object): # pylint: disable=useless-object-inheritance 19 | """ Falcon logging middleware """ 20 | 21 | def __init__(self, logger_name='cf.falcon.logger'): 22 | self._logger_name = logger_name 23 | 24 | def process_request(self, request, response): # pylint: disable=unused-argument,no-self-use 25 | """Process the request before routing it. 26 | 27 | :param request: - Falcon Request object 28 | :param response: - Falcon Response object 29 | """ 30 | framework = cf_logging.FRAMEWORK 31 | cid = framework.request_reader.get_correlation_id(request) 32 | framework.context.set_correlation_id(cid, request) 33 | framework.context.set('request_started_at', datetime.utcnow(), request) 34 | 35 | def process_response(self, request, response, resource, req_succeeded): # pylint: disable=unused-argument 36 | """Post-processing of the response (after routing). 37 | 38 | :param request: - Falcon Request object 39 | :param response: - Falcon Response object 40 | :param resource: - Falcon Resource object to which the request was routed 41 | :param req_succeeded: - True if no exceptions were raised while 42 | the framework processed and routed the request 43 | """ 44 | cf_logging.FRAMEWORK.context.set( 45 | 'response_sent_at', datetime.utcnow(), request) 46 | extra = {REQUEST_KEY: request, RESPONSE_KEY: response} 47 | logging.getLogger(self._logger_name).info('', extra=extra) 48 | 49 | 50 | def init(app, level=defaults.DEFAULT_LOGGING_LEVEL, username_key='username', custom_fields=None): 51 | """ Initializes logging in JSON format. 52 | 53 | :param app: - Falcon application object 54 | :param level: - valid log level from standard logging package (optional) 55 | :param username_key: key used by the framework to get the username 56 | out of the request user, set in the request context, 57 | like `request.context.get('user').get(key)` 58 | """ 59 | if not isinstance(app, falcon.API): 60 | raise TypeError('application should be instance of Falcon API') 61 | 62 | framework = Framework(FALCON_FRAMEWORK_NAME, FalconContext(), 63 | FalconRequestReader(username_key), FalconResponseReader(), 64 | custom_fields=custom_fields) 65 | cf_logging.init(framework, level) 66 | -------------------------------------------------------------------------------- /sap/cf_logging/sanic_logging/__init__.py: -------------------------------------------------------------------------------- 1 | """ Flask logging support package 2 | - Configures standard logging package to produce JSON 3 | - Produces info request log entry per request 4 | """ 5 | import logging 6 | from datetime import datetime 7 | from functools import wraps 8 | from sanic import Sanic 9 | 10 | from sap import cf_logging 11 | from sap.cf_logging import defaults 12 | from sap.cf_logging.core.constants import REQUEST_KEY, RESPONSE_KEY 13 | from sap.cf_logging.core.framework import Framework 14 | from sap.cf_logging.sanic_logging.context import SanicContext 15 | from sap.cf_logging.sanic_logging.request_reader import SanicRequestReader 16 | from sap.cf_logging.sanic_logging.response_reader import SanicResponseReader 17 | 18 | SANIC_FRAMEWORK_NAME = 'sanic.framework' 19 | 20 | 21 | def before_request(wrapped): 22 | """ Use as decorator on Sanic's before_request handler 23 | Handles correlation_id by setting it in the context for log records 24 | """ 25 | 26 | @wraps(wrapped) 27 | def _wrapper(request): 28 | correlation_id = cf_logging.FRAMEWORK.request_reader.get_correlation_id(request) 29 | cf_logging.FRAMEWORK.context.set_correlation_id(correlation_id, request) 30 | cf_logging.FRAMEWORK.context.set('request_started_at', datetime.utcnow(), request) 31 | return wrapped(request) 32 | 33 | return _wrapper 34 | 35 | 36 | def after_request(wrapped): 37 | """ Use as decorator on Sanic after_request handler 38 | Creates info log record per request 39 | """ 40 | 41 | @wraps(wrapped) 42 | def _wrapper(request, response): 43 | cf_logging.FRAMEWORK.context.set('response_sent_at', datetime.utcnow(), request) 44 | extra = {REQUEST_KEY: request, RESPONSE_KEY: response} 45 | logging.getLogger('cf.sanic.logger').info('', extra=extra) 46 | return wrapped(request, response) 47 | 48 | return _wrapper 49 | 50 | 51 | def init(app, level=defaults.DEFAULT_LOGGING_LEVEL, custom_framework=None, custom_fields=None): 52 | """ Initializes logging in JSON format. 53 | 54 | Adds before and after request handlers to the `app` object to enable request info log. 55 | :param app: - Flask application object 56 | :param level: - valid log level from standard logging package (optional) 57 | :param custom_framework: - `Framework` instance - use in case you need 58 | to change request processing behaviour for example to customize context storage 59 | """ 60 | if not isinstance(app, Sanic): 61 | raise TypeError('application should be instance of Sanic') 62 | 63 | framework = custom_framework or \ 64 | Framework( 65 | SANIC_FRAMEWORK_NAME, 66 | SanicContext(), 67 | SanicRequestReader(), 68 | SanicResponseReader(), 69 | custom_fields=custom_fields 70 | ) 71 | 72 | cf_logging.init(framework, level) 73 | 74 | @app.middleware('request') 75 | @before_request 76 | def _before_request(request): # pylint: disable=unused-argument 77 | pass 78 | 79 | @app.middleware('response') 80 | @after_request 81 | def _after_request(request, response): # pylint: disable=unused-argument 82 | pass 83 | -------------------------------------------------------------------------------- /tests/django_logging/test_app/test_django_logging.py: -------------------------------------------------------------------------------- 1 | """ Module that tests the integration of cf_logging with Django """ 2 | import sys 3 | import os 4 | import pytest 5 | 6 | from django.test import Client 7 | from django.conf.urls import url 8 | from django.conf import settings 9 | 10 | from sap import cf_logging 11 | from sap.cf_logging import django_logging 12 | from tests.log_schemas import WEB_LOG_SCHEMA 13 | from tests.common_test_params import ( 14 | v_str, get_web_record_header_fixtures 15 | ) 16 | from tests.util import ( 17 | check_log_record, 18 | enable_sensitive_fields_logging, 19 | config_logger 20 | ) 21 | 22 | from tests.django_logging.test_app.views import UserLoggingView 23 | 24 | 25 | os.environ['DJANGO_SETTINGS_MODULE'] = 'tests.django_logging.test_app.settings' 26 | 27 | 28 | @pytest.fixture(autouse=True) 29 | def before_each(): 30 | """ enable all fields to be logged """ 31 | enable_sensitive_fields_logging() 32 | yield 33 | 34 | 35 | FIXTURE = get_web_record_header_fixtures() 36 | 37 | @pytest.mark.parametrize('headers, expected', FIXTURE) 38 | def test_django_request_log(headers, expected): 39 | """ That the expected records are logged by the logging library """ 40 | _set_up_django_logging() 41 | _check_django_request_log(headers, expected) 42 | 43 | 44 | def test_web_log(): 45 | """ That the custom properties are logged """ 46 | _user_logging({}, {'myproperty': 'myval'}, {'myproperty': v_str('myval')}) 47 | 48 | 49 | def test_correlation_id(): 50 | """ Test the correlation id is logged when coming from the headers """ 51 | _user_logging( 52 | {'X-CorrelationID': '298ebf9d-be1d-11e7-88ff-2c44fd152860'}, 53 | {}, 54 | {'correlation_id': v_str('298ebf9d-be1d-11e7-88ff-2c44fd152860')}, 55 | True 56 | ) 57 | 58 | 59 | def test_missing_request(): 60 | """ That the correlation id is missing when the request is missing """ 61 | _user_logging( 62 | {'X-CorrelationID': '298ebf9d-be1d-11e7-88ff-2c44fd152860'}, 63 | {}, 64 | {'correlation_id': v_str('-')}, 65 | False 66 | ) 67 | 68 | def test_custom_fields_set(): 69 | """ Test custom fields are set up """ 70 | _set_up_django_logging() 71 | assert 'cf1' in cf_logging.FRAMEWORK.custom_fields.keys() 72 | 73 | def _check_django_request_log(headers, expected): 74 | _, stream = config_logger('cf.django.logger') 75 | 76 | client = Client() 77 | _check_expected_response(client.get('/test/path', **headers), body='Hello test!') 78 | assert check_log_record(stream, WEB_LOG_SCHEMA, expected) == {} 79 | 80 | 81 | # Helper functions 82 | def _set_up_django_logging(): 83 | cf_logging._SETUP_DONE = False # pylint: disable=protected-access 84 | django_logging.init(custom_fields={'cf1': None}) 85 | 86 | 87 | def _check_expected_response(response, status_code=200, body='ok'): 88 | assert response.status_code == status_code 89 | if body is not None: 90 | assert response.content.decode() == body 91 | 92 | 93 | def _user_logging(headers, extra, expected, provide_request=False): 94 | sys.modules[settings.ROOT_URLCONF].urlpatterns.append( 95 | url('^test/user/logging$', UserLoggingView.as_view(provide_request=provide_request), 96 | {'extra': extra, 'expected': expected})) 97 | 98 | 99 | _set_up_django_logging() 100 | client = Client() 101 | _check_expected_response(client.get('/test/user/logging', **headers)) 102 | -------------------------------------------------------------------------------- /sap/cf_logging/core/request_reader.py: -------------------------------------------------------------------------------- 1 | """ request_reader provides an interface for the RequestReader class """ 2 | import uuid 3 | 4 | CORRELATION_ID_HEADERS = ['X-Correlation-ID', 5 | 'X-CorrelationID', 'X-Request-ID', 'X-Vcap-Request-Id'] 6 | 7 | 8 | class RequestReader(object): # pylint: disable=useless-object-inheritance 9 | """ 10 | Helper class for extracting logging-relevant information from HTTP request object 11 | """ 12 | 13 | def get_correlation_id(self, request): 14 | """ 15 | We assume that the request is a valid request object. 16 | 17 | :param request: 18 | """ 19 | if request is None: 20 | return None 21 | 22 | for header in CORRELATION_ID_HEADERS: 23 | value = self.get_http_header(request, header) 24 | if value is not None: 25 | return value 26 | return str(uuid.uuid1()) 27 | 28 | # pylint: disable=no-self-use 29 | def get_http_header(self, request, header_name, default=None): 30 | """ 31 | get the HTTP header's value given its name 32 | 33 | :param request: 34 | :param header_name: name of header 35 | :param default: default value if header is not found 36 | :return: 37 | """ 38 | if request is None or not hasattr(request, 'headers'): 39 | return default 40 | 41 | if header_name in request.headers: 42 | return request.headers.get(header_name) 43 | if header_name.upper() in request.headers: 44 | return request.headers.get(header_name.upper()) 45 | return default 46 | 47 | def get_remote_user(self, request): 48 | """ 49 | 50 | :param request: 51 | """ 52 | raise NotImplementedError 53 | 54 | def get_protocol(self, request): 55 | """ 56 | We assume that request is a valid request object. 57 | Gets the request protocol (e.g. HTTP/1.1). 58 | 59 | :return: The request protocol or None if it cannot be determined 60 | """ 61 | raise NotImplementedError 62 | 63 | # pylint: disable=no-self-use 64 | def get_path(self, request): 65 | """ 66 | We assume that request is a valid request object. 67 | Gets the request path. 68 | 69 | :return: the request path (e.g. /index.html) 70 | """ 71 | return request.path 72 | 73 | def get_content_length(self, request): 74 | """ 75 | We assume that request is a valid request object. 76 | The content length of the request. 77 | 78 | :return: the content length of the request or None if it cannot be determined 79 | """ 80 | raise NotImplementedError 81 | 82 | # pylint: disable=no-self-use 83 | def get_method(self, request): 84 | """ 85 | We assume that request is a valid request object. 86 | Gets the request method (e.g. GET, POST, etc.). 87 | 88 | :return: The request method or None if it cannot be determined 89 | """ 90 | return request.method 91 | 92 | def get_remote_ip(self, request): 93 | """ 94 | We assume that request is a valid request object. 95 | Gets the remote IP of the request initiator. 96 | 97 | :return: An ip address or None if it cannot be determined 98 | """ 99 | raise NotImplementedError 100 | 101 | def get_remote_port(self, request): 102 | """ 103 | We assume that request is a valid request object. 104 | Gets the remote port of the request initiator. 105 | 106 | :return: A port or None if it cannot be determined 107 | """ 108 | raise NotImplementedError 109 | -------------------------------------------------------------------------------- /tests/test_flask_logging.py: -------------------------------------------------------------------------------- 1 | """ Module that tests the integration of cf_logging with Flask """ 2 | import logging 3 | import pytest 4 | from flask import Flask 5 | from flask import Response 6 | from sap import cf_logging 7 | from sap.cf_logging import flask_logging 8 | from tests.log_schemas import WEB_LOG_SCHEMA, JOB_LOG_SCHEMA, CUST_FIELD_SCHEMA 9 | from tests.common_test_params import v_str, v_num, auth_basic, get_web_record_header_fixtures 10 | from tests.util import ( 11 | check_log_record, 12 | enable_sensitive_fields_logging, 13 | config_logger, 14 | ) 15 | 16 | 17 | # pylint: disable=protected-access 18 | 19 | @pytest.mark.xfail(raises=TypeError, strict=True) 20 | def test_flask_requires_valid_app(): 21 | """ Test the init api expects a valid app """ 22 | flask_logging.init({}) 23 | 24 | 25 | FIXTURE = get_web_record_header_fixtures() 26 | FIXTURE.append(({'Authorization': auth_basic('user', 'pass')}, 27 | {'remote_user': v_str('user')})) 28 | FIXTURE.append(({}, {'response_size_b': v_num(val=2)})) 29 | 30 | 31 | @pytest.fixture(autouse=True) 32 | def before_each(): 33 | # pylint: disable=duplicate-code 34 | """ enable all fields to be logged """ 35 | enable_sensitive_fields_logging() 36 | yield 37 | 38 | 39 | @pytest.mark.parametrize("headers, expected", FIXTURE) 40 | def test_flask_request_log(headers, expected): 41 | """ That the expected records are logged by the logging library """ 42 | 43 | app = Flask(__name__) 44 | 45 | @app.route('/test/path') 46 | def _root(): 47 | return Response('ok', mimetype='text/plain') 48 | 49 | _set_up_flask_logging(app) 50 | _, stream = config_logger('cf.flask.logger') 51 | 52 | client = app.test_client() 53 | _check_expected_response(client.get('/test/path', headers=headers)) 54 | assert check_log_record(stream, WEB_LOG_SCHEMA, expected) == {} 55 | 56 | 57 | def test_web_log(): 58 | """ That the custom properties are logged """ 59 | _user_logging({}, {'myprop': 'myval'}, {'myprop': v_str('myval')}) 60 | 61 | 62 | def test_correlation_id(): 63 | """ Test the correlation id is logged when coming from the headers """ 64 | _user_logging({'X-CorrelationID': '298ebf9d-be1d-11e7-88ff-2c44fd152860'}, 65 | {}, 66 | {'correlation_id': v_str('298ebf9d-be1d-11e7-88ff-2c44fd152860')}) 67 | 68 | 69 | def test_logging_without_request(): 70 | """ Test logger is usable in non request context """ 71 | app = Flask(__name__) 72 | _set_up_flask_logging(app) 73 | logger, stream = config_logger('main.logger') 74 | logger.info('works') 75 | assert check_log_record(stream, JOB_LOG_SCHEMA, {'msg': v_str('works')}) == {} 76 | 77 | def test_custom_field_loggin(): 78 | """ Test custom fields are generated """ 79 | app = Flask(__name__) 80 | _set_up_flask_logging(app) 81 | logger, stream = config_logger('main.logger') 82 | logger.info('works', extra={'cf1': 'yes'}) 83 | assert check_log_record(stream, CUST_FIELD_SCHEMA, {}) == {} 84 | 85 | # Helper functions 86 | def _set_up_flask_logging(app, level=logging.DEBUG): 87 | cf_logging._SETUP_DONE = False 88 | flask_logging.init(app, level, custom_fields={'cf1': None, 'cf2': None}) 89 | 90 | 91 | def _user_logging(headers, extra, expected): 92 | app = Flask(__name__) 93 | 94 | @app.route('/test/user/logging') 95 | def _logging_correlation_id_route(): 96 | logger, stream = config_logger('user.logging') 97 | logger.info('in route headers', extra=extra) 98 | assert check_log_record(stream, JOB_LOG_SCHEMA, expected) == {} 99 | return Response('ok') 100 | 101 | _set_up_flask_logging(app) 102 | client = app.test_client() 103 | _check_expected_response(client.get('/test/user/logging', headers=headers)) 104 | 105 | 106 | def _check_expected_response(response, status_code=200, body='ok'): 107 | assert response.status_code == status_code 108 | if body is not None: 109 | assert response.get_data().decode() == body 110 | -------------------------------------------------------------------------------- /sap/cf_logging/record/simple_log_record.py: -------------------------------------------------------------------------------- 1 | """ Module SimpleLogRecord """ 2 | import logging 3 | import traceback 4 | 5 | from datetime import datetime 6 | from sap.cf_logging import defaults 7 | from sap.cf_logging.core.constants import REQUEST_KEY, RESPONSE_KEY 8 | from sap.cf_logging.record import application_info 9 | from sap.cf_logging.record import util 10 | 11 | from sap.cf_logging.formatters.stacktrace_formatter import format_stacktrace 12 | 13 | _SKIP_ATTRIBUTES = ["type", "written_at", "written_ts", "correlation_id", "remote_user", "referer", 14 | "x_forwarded_for", "protocol", "method", "remote_ip", "request_size_b", 15 | "remote_host", "remote_port", "request_received_at", "direction", 16 | "response_time_ms", "response_status", "response_size_b", 17 | "response_content_type", "response_sent_at", REQUEST_KEY, RESPONSE_KEY] 18 | 19 | 20 | class SimpleLogRecord(logging.LogRecord): 21 | """ SimpleLogRecord class holds data for user logged messages """ 22 | 23 | # pylint: disable=too-many-arguments,too-many-locals 24 | 25 | def __init__(self, extra, framework, *args, **kwargs): 26 | super(SimpleLogRecord, self).__init__(*args, **kwargs) 27 | 28 | utcnow = datetime.utcnow() 29 | self.written_at = util.iso_time_format(utcnow) 30 | self.written_ts = util.epoch_nano_second(utcnow) 31 | 32 | request = extra[REQUEST_KEY] if extra and REQUEST_KEY in extra else None 33 | 34 | self.correlation_id = framework.context.get_correlation_id(request) or defaults.UNKNOWN 35 | 36 | self.custom_fields = {} 37 | for key, value in framework.custom_fields.items(): 38 | if extra and key in extra: 39 | if extra[key] is not None: 40 | self.custom_fields[key] = extra[key] 41 | elif value is not None: 42 | self.custom_fields[key] = value 43 | 44 | self.extra = dict((key, value) for key, value in extra.items() 45 | if key not in _SKIP_ATTRIBUTES and 46 | key not in framework.custom_fields.keys()) if extra else {} 47 | for key, value in self.extra.items(): 48 | setattr(self, key, value) 49 | 50 | def format_cf_attributes(self): 51 | """ Add common and Cloud Foundry environment specific attributes """ 52 | record = { 53 | 'component_id': application_info.COMPONENT_ID, 54 | 'component_name': application_info.COMPONENT_NAME, 55 | 'component_instance': application_info.COMPONENT_INSTANCE, 56 | 'space_id': application_info.SPACE_ID, 57 | 'space_name': application_info.SPACE_NAME, 58 | 'container_id': application_info.CONTAINER_ID, 59 | 'component_type': application_info.COMPONENT_TYPE, 60 | 'written_at': self.written_at, 61 | 'written_ts': self.written_ts, 62 | 'correlation_id': self.correlation_id, 63 | 'layer': application_info.LAYER 64 | } 65 | return record 66 | 67 | def format(self): 68 | """ Returns a dict record with properties to be logged """ 69 | record = self.format_cf_attributes() 70 | record.update({ 71 | 'type': 'log', 72 | 'logger': self.name, 73 | 'thread': self.threadName, 74 | 'level': self.levelname, 75 | 'msg': self.getMessage(), 76 | }) 77 | 78 | if self.exc_info: 79 | stacktrace = ''.join(traceback.format_exception(*self.exc_info)) 80 | stacktrace = format_stacktrace(stacktrace) 81 | record['stacktrace'] = stacktrace.split('\n') 82 | 83 | 84 | record.update(self.extra) 85 | if len(self.custom_fields) > 0: 86 | record.update(self._format_custom_fields()) 87 | return record 88 | 89 | def _format_custom_fields(self): 90 | res = {"#cf": {"string": []}} 91 | for i, (key, value) in enumerate(self.custom_fields.items()): 92 | res['#cf']['string'].append( 93 | {"k": str(key), "v": str(value), "i": i} 94 | ) 95 | return res 96 | -------------------------------------------------------------------------------- /tests/test_sanic_logging.py: -------------------------------------------------------------------------------- 1 | """ Module that tests the integration of cf_logging with Sanic """ 2 | import logging 3 | import pytest 4 | import sanic 5 | from sanic.response import text 6 | from sap import cf_logging 7 | 8 | from sap.cf_logging import sanic_logging 9 | from tests.log_schemas import WEB_LOG_SCHEMA, JOB_LOG_SCHEMA 10 | from tests.common_test_params import v_str, v_num, get_web_record_header_fixtures 11 | from tests.schema_util import extend 12 | from tests.util import ( 13 | check_log_record, 14 | config_logger, 15 | enable_sensitive_fields_logging, 16 | ) 17 | 18 | 19 | # pylint: disable=protected-access 20 | @pytest.mark.xfail(raises=TypeError, strict=True) 21 | def test_sanic_requires_valid_app(): 22 | """ Test that the api requires a valid app to be passed in """ 23 | sanic_logging.init({}) 24 | 25 | 26 | FIXTURE = get_web_record_header_fixtures() 27 | FIXTURE.append(({'no-content-length': '1'}, {'response_size_b': v_num(-1)})) 28 | 29 | 30 | @pytest.fixture(autouse=True) 31 | def before_each(): 32 | """ enable all fields to be logged """ 33 | enable_sensitive_fields_logging() 34 | yield 35 | 36 | 37 | @pytest.mark.parametrize("headers, expected", FIXTURE) 38 | def test_sanic_request_log(headers, expected): 39 | """ Test that the JSON logs contain the expected properties based on the 40 | input. 41 | """ 42 | app = sanic.Sanic('test cf_logging') 43 | 44 | @app.route('/test/path') 45 | async def _headers_route(request): 46 | if 'no-content-length' in request.headers: 47 | return text('ok', headers={'Content-Type': 'text/plain'}) 48 | return text('ok', headers={'Content-Length': 2, 'Content-Type': 'text/plain'}) 49 | 50 | _set_up_sanic_logging(app) 51 | _, stream = config_logger('cf.sanic.logger') 52 | 53 | client = app.test_client 54 | _check_expected_response(client.get( 55 | '/test/path', headers=headers)[1], 200, 'ok') 56 | assert check_log_record(stream, WEB_LOG_SCHEMA, expected) == {} 57 | 58 | 59 | def test_web_log(): 60 | """ Test that custom attributes are logged """ 61 | _user_logging({}, {'myprop': 'myval'}, {'myprop': v_str('myval')}, False) 62 | 63 | 64 | def test_missing_request(): 65 | """ That the correlation ID when the request is missing """ 66 | _user_logging({'X-CorrelationID': '298ebf9d-be1d-11e7-88ff-2c44fd152860'}, 67 | {}, 68 | {'correlation_id': v_str('-')}, 69 | False) 70 | 71 | 72 | def test_logs_correlation_id(): 73 | """ Test the setting of the correlation id based on the headers """ 74 | _user_logging({'X-CorrelationID': '298ebf9d-be1d-11e7-88ff-2c44fd152860'}, 75 | {}, 76 | {'correlation_id': v_str( 77 | '298ebf9d-be1d-11e7-88ff-2c44fd152860')}, 78 | True) 79 | 80 | def test_custom_fields_set(): 81 | """ Test custom fields are set up """ 82 | app = sanic.Sanic('test cf_logging') 83 | _set_up_sanic_logging(app) 84 | assert 'cf1' in cf_logging.FRAMEWORK.custom_fields.keys() 85 | 86 | 87 | # Helper functions 88 | def _set_up_sanic_logging(app, level=logging.DEBUG): 89 | cf_logging._SETUP_DONE = False 90 | sanic_logging.init(app, level, custom_fields={'cf1': None}) 91 | 92 | 93 | def _user_logging(headers, extra, expected, provide_request=False): 94 | app = sanic.Sanic(__name__) 95 | 96 | @app.route('/test/user/logging') 97 | async def _logging_correlation_id_route(request): 98 | logger, stream = config_logger('user.logging') 99 | new_extra = extend(extra, {'request': request}) if provide_request else extra 100 | logger.info('in route headers', extra=new_extra) 101 | assert check_log_record(stream, JOB_LOG_SCHEMA, expected) == {} 102 | return text('ok') 103 | 104 | _set_up_sanic_logging(app) 105 | client = app.test_client 106 | _check_expected_response(client.get('/test/user/logging', headers=headers)[1]) 107 | 108 | 109 | def _check_expected_response(response, status_code=200, body=None): 110 | print(response.text) 111 | assert response.status == status_code 112 | 113 | if body is not None: 114 | assert response.text == body 115 | -------------------------------------------------------------------------------- /sap/cf_logging/record/request_log_record.py: -------------------------------------------------------------------------------- 1 | """ Module that holds the RequestWebRecord class """ 2 | import os 3 | from sap.cf_logging import defaults 4 | from sap.cf_logging.core.constants import REQUEST_KEY, RESPONSE_KEY, \ 5 | LOG_SENSITIVE_CONNECTION_DATA, LOG_REMOTE_USER, LOG_REFERER 6 | from sap.cf_logging.record import util 7 | from sap.cf_logging.record.simple_log_record import SimpleLogRecord 8 | 9 | 10 | PROPS = ['type', 'direction', 'remote_user', 'request', 'referer', 11 | 'x_forwarded_for', 'protocol', 'method', 'remote_ip', 12 | 'request_size_b', 'remote_host', 'remote_port', 'request_received_at', 13 | 'response_sent_at', 'response_time_ms', 'response_status', 14 | 'response_size_b', 'response_content_type'] 15 | 16 | # pylint: disable=too-many-instance-attributes 17 | 18 | 19 | class RequestWebRecord(SimpleLogRecord): 20 | """ RequestWebRecord class holds request/response log data 21 | It is used in the formatter class 22 | """ 23 | 24 | # pylint: disable=too-many-locals 25 | def __init__(self, extra, framework, *args, **kwargs): 26 | 27 | super(RequestWebRecord, self).__init__(extra, framework, *args, **kwargs) 28 | 29 | context = framework.context 30 | request_reader = framework.request_reader 31 | response_reader = framework.response_reader 32 | 33 | request = extra[REQUEST_KEY] 34 | response = extra[RESPONSE_KEY] 35 | 36 | props = dict((key, value) for key, value in extra.items() 37 | if key not in [RESPONSE_KEY, REQUEST_KEY]) 38 | for key, value in props.items(): 39 | setattr(self, key, value) 40 | 41 | length = request_reader.get_content_length(request) 42 | remote_ip = request_reader.get_remote_ip(request) 43 | 44 | self.type = 'request' 45 | self.direction = 'IN' 46 | self.remote_user = request_reader.get_remote_user(request) 47 | self.request = request_reader.get_path(request) 48 | self.referer = request_reader.get_http_header( 49 | request, 'referer', defaults.UNKNOWN) 50 | self.x_forwarded_for = request_reader.get_http_header( 51 | request, 'x-forwarded-for', defaults.UNKNOWN) 52 | self.protocol = request_reader.get_protocol(request) 53 | self.method = request_reader.get_method(request) 54 | self.remote_ip = remote_ip 55 | self.request_size_b = util.parse_int(length, -1) 56 | self.remote_host = remote_ip 57 | self.remote_port = request_reader.get_remote_port( 58 | request) or defaults.UNKNOWN 59 | 60 | request_start = context.get( 61 | 'request_started_at', request) or defaults.UNIX_EPOCH 62 | self.request_received_at = util.iso_time_format(request_start) 63 | 64 | # response related 65 | response_sent_at = context.get( 66 | 'response_sent_at', request) or defaults.UNIX_EPOCH 67 | self.response_sent_at = util.iso_time_format(response_sent_at) 68 | self.response_time_ms = util.time_delta_ms( 69 | request_start, response_sent_at) 70 | 71 | self.response_status = util.parse_int( 72 | response_reader.get_status_code(response), defaults.STATUS) 73 | self.response_size_b = util.parse_int( 74 | response_reader.get_response_size(response), defaults.RESPONSE_SIZE_B) 75 | self.response_content_type = response_reader.get_content_type(response) 76 | 77 | self._hide_sensitive_fields() 78 | 79 | def format(self): 80 | record = super(RequestWebRecord, self).format_cf_attributes() 81 | request_properties = dict( 82 | (key, value) for key, value in self.__dict__.items() if key in PROPS) 83 | record.update(request_properties) 84 | return record 85 | 86 | def _hide_sensitive_fields(self): 87 | if os.environ.get(LOG_SENSITIVE_CONNECTION_DATA, 'false').lower() != 'true': 88 | self.remote_ip = defaults.REDACTED 89 | self.remote_host = defaults.REDACTED 90 | self.remote_port = defaults.REDACTED 91 | self.x_forwarded_for = defaults.REDACTED 92 | 93 | if os.environ.get(LOG_REMOTE_USER, 'false').lower() != 'true': 94 | self.remote_user = defaults.REDACTED 95 | 96 | if os.environ.get(LOG_REFERER, 'false').lower() != 'true': 97 | self.referer = defaults.REDACTED 98 | -------------------------------------------------------------------------------- /tests/test_job_logging.py: -------------------------------------------------------------------------------- 1 | """ Module to test the cf_logging library """ 2 | import uuid 3 | import logging 4 | import time 5 | import threading 6 | from json import JSONDecoder 7 | import pytest 8 | from json_validator.validator import JsonValidator 9 | from sap import cf_logging 10 | from tests.log_schemas import JOB_LOG_SCHEMA 11 | from tests.util import config_logger 12 | 13 | # pylint: disable=protected-access 14 | 15 | 16 | @pytest.fixture(autouse=True) 17 | def before_each(): 18 | """ reset logging framework """ 19 | cf_logging._SETUP_DONE = False 20 | 21 | 22 | @pytest.mark.parametrize('log_callback', [ 23 | lambda logger, msg: logger.debug('message: %s', msg), 24 | lambda logger, msg: logger.info('message: %s', msg), 25 | lambda logger, msg: logger.warning('message: %s', msg), 26 | lambda logger, msg: logger.error('message: %s', msg), 27 | lambda logger, msg: logger.critical('message: %s', msg) 28 | ]) 29 | def test_log_in_expected_format(log_callback): 30 | """ Test the cf_logger as a standalone """ 31 | cf_logging.init(level=logging.DEBUG) 32 | logger, stream = config_logger('cli.test') 33 | log_callback(logger, 'hi') 34 | log_json = JSONDecoder().decode(stream.getvalue()) 35 | _, error = JsonValidator(JOB_LOG_SCHEMA).validate(log_json) 36 | 37 | assert error == {} 38 | 39 | 40 | def test_set_correlation_id(): 41 | """ Test setting correlation_id """ 42 | correlation_id = '1234' 43 | cf_logging.init(level=logging.DEBUG) 44 | cf_logging.FRAMEWORK.context.set_correlation_id(correlation_id) 45 | 46 | logger, stream = config_logger('cli.test') 47 | logger.info('hi') 48 | 49 | log_json = JSONDecoder().decode(stream.getvalue()) 50 | _, error = JsonValidator(JOB_LOG_SCHEMA).validate(log_json) 51 | 52 | assert error == {} 53 | assert log_json['correlation_id'] == correlation_id 54 | assert cf_logging.FRAMEWORK.context.get_correlation_id() == correlation_id 55 | 56 | 57 | def test_exception_stacktrace(): 58 | """ Test exception stacktrace is logged """ 59 | cf_logging.init(level=logging.DEBUG) 60 | logger, stream = config_logger('cli.test') 61 | 62 | try: 63 | return 1 / 0 64 | except ZeroDivisionError: 65 | logger.exception('zero division error') 66 | log_json = JSONDecoder().decode(stream.getvalue()) 67 | _, error = JsonValidator(JOB_LOG_SCHEMA).validate(log_json) 68 | 69 | assert error == {} 70 | assert 'ZeroDivisionError' in str(log_json['stacktrace']) 71 | assert log_json["msg"] == 'zero division error' 72 | 73 | 74 | def test_exception_stacktrace_info_level(): 75 | """ Test exception stacktrace is logged """ 76 | cf_logging.init(level=logging.DEBUG) 77 | logger, stream = config_logger('cli.test') 78 | 79 | try: 80 | return 1 / 0 81 | except ZeroDivisionError as exc: 82 | logger.info('zero division error', exc_info=exc) 83 | log_json = JSONDecoder().decode(stream.getvalue()) 84 | _, error = JsonValidator(JOB_LOG_SCHEMA).validate(log_json) 85 | 86 | assert error == {} 87 | assert 'ZeroDivisionError' in str(log_json['stacktrace']) 88 | assert log_json["msg"] == 'zero division error' 89 | 90 | 91 | def test_custom_fields_set(): 92 | """ Test custom fields are set up """ 93 | cf_logging.init(level=logging.DEBUG, custom_fields={'cf1': None}) 94 | assert 'cf1' in cf_logging.FRAMEWORK.custom_fields.keys() 95 | 96 | def test_thread_safety(): 97 | """ test context keeps separate correlation ID per thread """ 98 | class _SampleThread(threading.Thread): 99 | def __init__(self): 100 | super(_SampleThread, self).__init__() 101 | self.correlation_id = str(uuid.uuid1()) 102 | self.read_correlation_id = '' 103 | 104 | def run(self): 105 | cf_logging.FRAMEWORK.context.set_correlation_id(self.correlation_id) 106 | time.sleep(0.1) 107 | self.read_correlation_id = cf_logging.FRAMEWORK.context.get_correlation_id() 108 | 109 | cf_logging.init(level=logging.DEBUG) 110 | 111 | thread_one = _SampleThread() 112 | thread_two = _SampleThread() 113 | 114 | thread_one.start() 115 | thread_two.start() 116 | 117 | thread_one.join() 118 | thread_two.join() 119 | 120 | assert thread_one.correlation_id == thread_one.read_correlation_id 121 | assert thread_two.correlation_id == thread_two.read_correlation_id 122 | -------------------------------------------------------------------------------- /tests/unit/record/test_request_log_record.py: -------------------------------------------------------------------------------- 1 | """ Test `RequestWebRecord` """ 2 | import logging 3 | import os 4 | import pytest 5 | from sap.cf_logging import defaults 6 | from sap.cf_logging.record.request_log_record import RequestWebRecord 7 | from sap.cf_logging.core.context import Context 8 | from sap.cf_logging.core.framework import Framework 9 | from sap.cf_logging.core.request_reader import RequestReader 10 | from sap.cf_logging.core.response_reader import ResponseReader 11 | from sap.cf_logging.core.constants import REQUEST_KEY, RESPONSE_KEY 12 | 13 | # pylint: disable=missing-docstring, invalid-name 14 | 15 | FRAMEWORK = None 16 | 17 | @pytest.fixture(autouse=True) 18 | def before_each_setup_mocks(mocker): 19 | context = Context() 20 | mocker.patch.object(context, 'get', return_value=None) 21 | 22 | request_reader = RequestReader() 23 | 24 | request_reader.get_http_header = lambda self, key, default: 'some.host' \ 25 | if key == 'x-forwarded-for' else 'referer' 26 | 27 | mocker.patch.object(request_reader, 'get_remote_user', return_value='user') 28 | mocker.patch.object(request_reader, 'get_protocol', return_value='http') 29 | mocker.patch.object(request_reader, 'get_content_length', return_value=0) 30 | mocker.patch.object(request_reader, 'get_remote_ip', 31 | return_value='1.2.3.4') 32 | mocker.patch.object(request_reader, 'get_remote_port', return_value='1234') 33 | mocker.patch.object(request_reader, 'get_path', return_value='/test/path') 34 | mocker.patch.object(request_reader, 'get_method', return_value='GET') 35 | 36 | response_reader = ResponseReader() 37 | mocker.patch.object(response_reader, 'get_status_code', return_value='200') 38 | mocker.patch.object(response_reader, 'get_response_size', return_value=0) 39 | mocker.patch.object(response_reader, 'get_content_type', 40 | return_value='text/plain') 41 | 42 | _clean_log_env_vars() 43 | 44 | global FRAMEWORK # pylint: disable=global-statement 45 | FRAMEWORK = Framework('name', context, request_reader, response_reader) 46 | yield 47 | 48 | 49 | def test_hiding_sensitive_fields_by_default(): 50 | log_record = RequestWebRecord({REQUEST_KEY: None, RESPONSE_KEY: None}, 51 | FRAMEWORK, 'name', logging.DEBUG, 'pathname', 1, 'msg', [], None) 52 | 53 | _assert_sensitive_fields_redacted(log_record) 54 | assert log_record.remote_user == defaults.REDACTED 55 | assert log_record.referer == defaults.REDACTED 56 | 57 | 58 | def test_logging_sensitive_connection_data(): 59 | os.environ['LOG_SENSITIVE_CONNECTION_DATA'] = 'true' 60 | 61 | log_record = RequestWebRecord({REQUEST_KEY: None, RESPONSE_KEY: None}, 62 | FRAMEWORK, 'name', logging.DEBUG, 'pathname', 1, 'msg', [], None) 63 | 64 | assert log_record.remote_ip == '1.2.3.4' 65 | assert log_record.remote_host == '1.2.3.4' 66 | assert log_record.remote_port == '1234' 67 | assert log_record.x_forwarded_for == 'some.host' 68 | assert log_record.remote_user == defaults.REDACTED 69 | assert log_record.referer == defaults.REDACTED 70 | 71 | 72 | def test_logging_remote_user(): 73 | os.environ['LOG_REMOTE_USER'] = 'true' 74 | 75 | log_record = RequestWebRecord({REQUEST_KEY: None, RESPONSE_KEY: None}, 76 | FRAMEWORK, 'name', logging.DEBUG, 'pathname', 1, 'msg', [], None) 77 | 78 | _assert_sensitive_fields_redacted(log_record) 79 | assert log_record.remote_user == 'user' 80 | assert log_record.referer == defaults.REDACTED 81 | 82 | 83 | def test_logging_referer(): 84 | os.environ['LOG_REFERER'] = 'true' 85 | 86 | log_record = RequestWebRecord({REQUEST_KEY: None, RESPONSE_KEY: None}, 87 | FRAMEWORK, 'name', logging.DEBUG, 'pathname', 1, 'msg', [], None) 88 | 89 | _assert_sensitive_fields_redacted(log_record) 90 | assert log_record.remote_user == defaults.REDACTED 91 | assert log_record.referer == 'referer' 92 | 93 | 94 | def test_incorrect_env_var_value(): 95 | os.environ['LOG_SENSITIVE_CONNECTION_DATA'] = 'false' 96 | os.environ['LOG_REMOTE_USER'] = 'some-string' 97 | os.environ['LOG_REFERER'] = '' 98 | 99 | log_record = RequestWebRecord({REQUEST_KEY: None, RESPONSE_KEY: None}, 100 | FRAMEWORK, 'name', logging.DEBUG, 'pathname', 1, 'msg', [], None) 101 | 102 | _assert_sensitive_fields_redacted(log_record) 103 | assert log_record.remote_user == defaults.REDACTED 104 | assert log_record.referer == defaults.REDACTED 105 | 106 | 107 | def _clean_log_env_vars(): 108 | for key in ['LOG_SENSITIVE_CONNECTION_DATA', 'LOG_REMOTE_USER', 'LOG_REFERER']: 109 | if os.environ.get(key): 110 | del os.environ[key] 111 | 112 | 113 | def _assert_sensitive_fields_redacted(log_record): 114 | assert log_record.remote_ip == defaults.REDACTED 115 | assert log_record.remote_host == defaults.REDACTED 116 | assert log_record.remote_port == defaults.REDACTED 117 | assert log_record.x_forwarded_for == defaults.REDACTED 118 | -------------------------------------------------------------------------------- /tests/test_falcon_logging.py: -------------------------------------------------------------------------------- 1 | """ Module that tests the integration of cf_logging with Falcon """ 2 | import logging 3 | import pytest 4 | import falcon 5 | from falcon import testing 6 | from falcon_auth import FalconAuthMiddleware, BasicAuthBackend 7 | from sap import cf_logging 8 | from sap.cf_logging import falcon_logging 9 | from sap.cf_logging.core.constants import REQUEST_KEY 10 | from tests.log_schemas import WEB_LOG_SCHEMA, JOB_LOG_SCHEMA 11 | from tests.common_test_params import ( 12 | v_str, auth_basic, get_web_record_header_fixtures 13 | ) 14 | from tests.util import ( 15 | check_log_record, 16 | config_logger, 17 | enable_sensitive_fields_logging 18 | ) 19 | 20 | 21 | # pylint: disable=protected-access, missing-docstring,too-few-public-methods 22 | 23 | @pytest.fixture(autouse=True) 24 | def before_each(): 25 | """ enable all fields to be logged """ 26 | enable_sensitive_fields_logging() 27 | yield 28 | 29 | 30 | @pytest.mark.xfail(raises=TypeError, strict=True) 31 | def test_falcon_requires_valid_app(): 32 | """ Test the init api expects a valid app """ 33 | falcon_logging.init({}) 34 | 35 | 36 | FIXTURE = get_web_record_header_fixtures() 37 | 38 | class TestResource: 39 | def on_get(self, req, resp): # pylint: disable=unused-argument,no-self-use 40 | resp.set_header('Content-Type', 'text/plain') 41 | resp.status = falcon.HTTP_200 42 | resp.body = 'ok' 43 | 44 | 45 | @pytest.mark.parametrize('headers, expected', FIXTURE) 46 | def test_falcon_request_log(headers, expected): 47 | """ That the expected records are logged by the logging library """ 48 | app = falcon.API(middleware=falcon_logging.LoggingMiddleware()) 49 | app.add_route('/test/path', TestResource()) 50 | 51 | _set_up_falcon_logging(app) 52 | _check_falcon_request_log(app, headers, expected) 53 | 54 | 55 | class User(object): # pylint: disable=useless-object-inheritance 56 | def __init__(self, key, name): 57 | self.key = key 58 | self.name = name 59 | 60 | @pytest.mark.parametrize('user', [ 61 | User(None, 'user'), 62 | User('custom_username_key', 'new_user') 63 | ]) 64 | def test_falcon_request_logs_user(user): 65 | user_dict = dict([(user.key or 'username', user.name)]) 66 | basic_auth = BasicAuthBackend(lambda username, password: user_dict) 67 | app = falcon.API(middleware=[ 68 | falcon_logging.LoggingMiddleware(), 69 | FalconAuthMiddleware(basic_auth) 70 | ]) 71 | app.add_route('/test/path', TestResource()) 72 | 73 | args = [app, user.key] if user.key else [app] 74 | _set_up_falcon_logging(*args) 75 | 76 | expected = {'remote_user': v_str(user.name)} 77 | _check_falcon_request_log(app, {'Authorization': str(auth_basic(user.name, 'pass'))}, expected) 78 | 79 | 80 | def _check_falcon_request_log(app, headers, expected): 81 | _, stream = config_logger('cf.falcon.logger') 82 | 83 | client = testing.TestClient(app) 84 | _check_expected_response( 85 | client.simulate_get('/test/path', headers=headers)) 86 | assert check_log_record(stream, WEB_LOG_SCHEMA, expected) == {} 87 | 88 | 89 | def test_web_log(): 90 | """ That the custom properties are logged """ 91 | _user_logging({}, {'myprop': 'myval'}, {'myprop': v_str('myval')}) 92 | 93 | def test_logging_without_request(): 94 | """ Test logger is usable in non request context """ 95 | app = falcon.API() 96 | _set_up_falcon_logging(app) 97 | cf_logging.FRAMEWORK.context.set_correlation_id('value') 98 | 99 | logger, stream = config_logger('main.logger') 100 | logger.info('works') 101 | assert check_log_record(stream, JOB_LOG_SCHEMA, {'msg': v_str('works')}) == {} 102 | 103 | 104 | def test_correlation_id(): 105 | """ Test the correlation id is logged when coming from the headers """ 106 | _user_logging( 107 | {'X-Correlation-ID': '298ebf9d-be1d-11e7-88ff-2c44fd152860'}, 108 | {}, 109 | {'correlation_id': v_str('298ebf9d-be1d-11e7-88ff-2c44fd152860')} 110 | ) 111 | 112 | def test_custom_fields_set(): 113 | """ Test custom fields are set up """ 114 | app = falcon.API() 115 | _set_up_falcon_logging(app) 116 | assert 'cf1' in cf_logging.FRAMEWORK.custom_fields.keys() 117 | 118 | # Helper functions 119 | def _set_up_falcon_logging(app, *args): 120 | cf_logging._SETUP_DONE = False 121 | falcon_logging.init(app, logging.DEBUG, *args, custom_fields={'cf1': None}) 122 | 123 | 124 | class UserResourceRoute(object): # pylint: disable=useless-object-inheritance 125 | def __init__(self, extra, expected): 126 | self.extra = extra 127 | self.expected = expected 128 | self.logger, self.stream = config_logger('user.logging') 129 | 130 | def on_get(self, req, resp): 131 | self.extra.update({REQUEST_KEY: req}) 132 | self.logger.log(logging.INFO, 'in route headers', extra=self.extra) 133 | assert check_log_record(self.stream, JOB_LOG_SCHEMA, self.expected) == {} 134 | 135 | resp.set_header('Content-Type', 'text/plain') 136 | resp.status = falcon.HTTP_200 137 | resp.body = 'ok' 138 | 139 | 140 | def _user_logging(headers, extra, expected): 141 | app = falcon.API(middleware=[ 142 | falcon_logging.LoggingMiddleware() 143 | ]) 144 | app.add_route('/test/user/logging', UserResourceRoute(extra, expected)) 145 | _set_up_falcon_logging(app) 146 | client = testing.TestClient(app) 147 | _check_expected_response(client.simulate_get('/test/user/logging', 148 | headers=headers)) 149 | 150 | 151 | def _check_expected_response(response, status_code=200, body='ok'): 152 | assert response.status_code == status_code 153 | if body is not None: 154 | assert response.text == body 155 | -------------------------------------------------------------------------------- /LICENSES/Apache-2.0.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 10 | 11 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 12 | 13 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 14 | 15 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 16 | 17 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 18 | 19 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 20 | 21 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 22 | 23 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 24 | 25 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 26 | 27 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 28 | 29 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 30 | 31 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 32 | 33 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 34 | 35 | (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and 36 | 37 | (b) You must cause any modified files to carry prominent notices stating that You changed the files; and 38 | 39 | (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 40 | 41 | (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 42 | 43 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 44 | 45 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 46 | 47 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 48 | 49 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 50 | 51 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 52 | 53 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 54 | 55 | END OF TERMS AND CONDITIONS 56 | 57 | APPENDIX: How to apply the Apache License to your work. 58 | 59 | To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. 60 | 61 | Copyright [yyyy] [name of copyright owner] 62 | 63 | Licensed under the Apache License, Version 2.0 (the "License"); 64 | you may not use this file except in compliance with the License. 65 | You may obtain a copy of the License at 66 | 67 | http://www.apache.org/licenses/LICENSE-2.0 68 | 69 | Unless required by applicable law or agreed to in writing, software 70 | distributed under the License is distributed on an "AS IS" BASIS, 71 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 72 | See the License for the specific language governing permissions and 73 | limitations under the License. 74 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | 2 | Python logging library to emit JSON logs in a SAP CloudFoundry environment. 3 | =========================================================================== 4 | 5 | .. image:: https://api.reuse.software/badge/github.com/SAP/cf-python-logging-support 6 | :target: https://api.reuse.software/info/github.com/SAP/cf-python-logging-support 7 | 8 | This is a collection of support libraries for Python applications running on Cloud Foundry that 9 | serve two main purposes: provide (a) means to emit structured application log messages and (b) 10 | instrument web applications of your application stack to collect request metrics. 11 | 12 | For details on the concepts and log formats, please look at the sibling project for `java logging 13 | support `__. 14 | 15 | 16 | Features 17 | ----------- 18 | 19 | 1. Lightweight, no dependencies. Support of Python 2.7 & 3.5. 20 | 2. Compatible with the Python **logging** module. Minimal configuration needed. 21 | 3. Emits JSON logs (`format 22 | details `__). 23 | 4. Supports **correlation-id**. 24 | 5. Supports request instrumentation. Built in support for: 25 | 26 | * `Flask 0.1x `__ 27 | * `Sanic 0.5.x `__ 28 | * `Falcon `__ 29 | * `Django `__ 30 | * Extensible to support others 31 | 32 | 6. Includes CF-specific information (space id, app id, etc.) to logs. 33 | 7. Supports adding extra properties to JSON log object. 34 | 35 | Installation 36 | ------------ 37 | 38 | Install the package with pip: 39 | 40 | :: 41 | 42 | pip install sap_cf_logging 43 | 44 | Usage 45 | ----- 46 | 47 | Setting up your application 48 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 49 | 50 | Logging library needs to be initialized. Depending on you application type, different initialization 51 | is used. You should usually do this in your application entrypoint. 52 | 53 | For CLI applications you just need to call ``cf_logging.init()`` *once* to configure the library. 54 | The library will try to configure future loggers to emit logs in JSON format. 55 | 56 | If you are using one of the supported frameworks, check the `Configuration <#configuration>`__ 57 | section to see how to configure it. 58 | 59 | **Setting up the CloudFoundry environment** 60 | 61 | In order for your logs to appear in the Kibana dashboard, you have to create an **application-logs** 62 | service instance and bind it to your application. 63 | 64 | 65 | Configuration 66 | ~~~~~~~~~~~~~ 67 | 68 | After installation use the following guide to configure the Python cf logging library. 69 | 70 | Flask 71 | ^^^^^ 72 | 73 | First import the ``cf_logging`` library and setup Flask logging on the application. 74 | 75 | .. code:: python 76 | 77 | from sap.cf_logging import flask_logging 78 | 79 | app = flask.Flask(__name__) 80 | flask_logging.init(app, logging.INFO) 81 | 82 | Next use Python’s logging library 83 | 84 | .. code:: python 85 | 86 | @app.route('/') 87 | def root_route(): 88 | logger = logging.getLogger('my.logger') 89 | logger.info('Hi') 90 | 91 | return 'ok' 92 | 93 | Note the logs generated by the application 94 | 95 | Sanic 96 | ^^^^^ 97 | 98 | .. code:: python 99 | 100 | import sanic 101 | import logging 102 | 103 | from sanic.response import HTTPResponse 104 | from sap.cf_logging import sanic_logging 105 | from sap.cf_logging.core.constants import REQUEST_KEY 106 | 107 | app = sanic.Sanic('test.cf_logging') 108 | sanic_logging.init(app) 109 | 110 | @app.route('/') 111 | async def two(request): 112 | extra = {REQUEST_KEY: request} 113 | logging.getLogger('my.logger').debug('Hi', extra = extra) 114 | return HTTPResponse(body='ok') 115 | 116 | **Note**: With Sanic you need to pass the request with an ``extra`` parameter in the logging API. 117 | This is needed in order to get the *correlation_id* generated at the beginning of the request or 118 | fetched from the HTTP headers. 119 | 120 | Falcon 121 | ^^^^^^ 122 | 123 | .. code:: python 124 | 125 | 126 | import falcon 127 | from sap.cf_logging import falcon_logging 128 | from sap.cf_logging.core.constants import REQUEST_KEY 129 | 130 | 131 | class Resource: 132 | def on_get(self, req, resp): 133 | extra = {REQUEST_KEY: req} 134 | logging.getLogger('my.logger').log('Resource requested', extra=extra) 135 | resp.media = {'name': 'Cloud Foundry'} 136 | 137 | 138 | app = falcon.API(middleware=[ 139 | falcon_logging.LoggingMiddleware() 140 | ]) 141 | app.add_route('/resource', Resource()) 142 | falcon_logging.init(app) 143 | 144 | Django 145 | ^^^^^^ 146 | 147 | .. code:: bash 148 | 149 | django-admin startproject example 150 | 151 | .. code:: python 152 | 153 | # example/settings.py 154 | 155 | MIDDLEWARES = [ 156 | # ..., 157 | 'sap.cf_logging.django_logging.LoggingMiddleware' 158 | ] 159 | 160 | # example/wsgi.py 161 | 162 | # ... 163 | from sap.cf_logging import django_logging 164 | 165 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sap_logtesting.settings") 166 | django_logging.init() 167 | 168 | # ... 169 | 170 | Create a new app 171 | 172 | .. code:: bash 173 | 174 | python manage.py startapp example_app 175 | 176 | .. code:: python 177 | 178 | # example_app/views.py 179 | 180 | import logging 181 | 182 | from django.http import HttpResponse 183 | from sap.cf_logging.core.constants import REQUEST_KEY 184 | 185 | def index(request): 186 | extra = {REQUEST_KEY: request} 187 | logger = logging.getLogger('my.logger') 188 | logger.info("Resource requested", extra=extra) 189 | return HttpResponse("ok") 190 | 191 | # example_app/urls.py 192 | 193 | from django.conf.urls import url 194 | 195 | from . import views 196 | 197 | urlpatterns = [ 198 | url('^$', views.index) 199 | ] 200 | 201 | # example/urls.py 202 | 203 | from django.contrib import admin 204 | from django.conf.urls import url, include 205 | 206 | urlpatterns = [ 207 | url('admin/', admin.site.urls), 208 | url('example/', include('example_app.urls')) 209 | ] 210 | 211 | General 212 | ^^^^^^^ 213 | 214 | .. code:: python 215 | 216 | import logging 217 | from sap import cf_logging 218 | 219 | cf_logging.init() 220 | 221 | logger = logging.getLogger("cli.logger") 222 | logger.info('hi') 223 | 224 | **Notes**: All loggers set up and created before the initialization of the Cloud Foundry logging library will 225 | be left untouched. When using Flask and Sanic with the logging library before and 226 | after request middleware is attached, and it will capture response times for each request. 227 | 228 | 229 | Custom Fields 230 | """"""""""""" 231 | 232 | To use custom fields. Pass a dictionary property custom_fields to the initialize method: 233 | 234 | .. code:: python 235 | 236 | import logging 237 | from sap import cf_logging 238 | cf_logging.init(custom_fields={"foo": "default", "bar": None}) 239 | 240 | Here we mark the two fields: foo and bar as custom_fields. Logging with: 241 | 242 | .. code:: python 243 | 244 | logging.getLogger('my.logger').debug('Hi') 245 | 246 | The property foo will be output as a custom field with a value "default". The property bar will not be logged, as it does not have a value. 247 | 248 | To log bar, provide a value when logging: 249 | 250 | .. code:: python 251 | 252 | logging.getLogger('my.logger').debug('Hi', extra={"bar": "new_value"}) 253 | 254 | It is also possible to log foo with a different value: 255 | 256 | .. code:: python 257 | 258 | logging.getLogger('my.logger').debug('Hi', extra={"foo": "hello"}) 259 | 260 | 261 | Setting and getting correlation ID 262 | """""""""""""""""""""""""""""""""" 263 | 264 | When using cf_logging in a web application you don't need to set the correlation ID, because the logging library will fetch it from the HTTP headers and set it. 265 | For non web applications you could set the correlation ID manually, so that the log entries can be filtered later on based on the ``correlation_id`` log property. 266 | In this case the correlation ID is kept in a thread local variable and each thread should set its own correlation ID. 267 | 268 | Setting and getting the correlation_id can be done via: 269 | 270 | .. code:: python 271 | 272 | cf_logging.FRAMEWORK.context.get_correlation_id() 273 | cf_logging.FRAMEWORK.context.set_correlation_id(value) 274 | 275 | If you need to get the correlation ID in a web application, take into account the framework you are using. 276 | In async frameworks like Sanic and Falcon the context is stored into the request object and you need to provide the request to the call: 277 | 278 | .. code:: python 279 | 280 | cf_logging.FRAMEWORK.context.get_correlation_id(request) 281 | 282 | 283 | Logging sensitive data 284 | ^^^^^^^^^^^^^^^^^^^^^^ 285 | 286 | The logging library does not log sensitive fields by default. Those fields are replaced with 'redacted' instead of their original content. 287 | The following fields are considered sensitive data: ``remote_ip``, ``remote_host``, ``remote_port``, ``x_forwarded_for``, ``remote_user``, ``referer``. 288 | Logging of all or some of these fields can be activated by setting the following environment variables: 289 | 290 | +-----------------------------------+-----------+------------------------------------------------------------------------+ 291 | | Environment variable | Value | Enables sensitive field | 292 | +===================================+===========+========================================================================+ 293 | | ``LOG_SENSITIVE_CONNECTION_DATA`` | true | ``remote_ip``, ``remote_host``, ``remote_port``, ``x_forwarded_for`` | 294 | +-----------------------------------+-----------+------------------------------------------------------------------------+ 295 | | ``LOG_REMOTE_USER`` | true | ``remote_user`` | 296 | +-----------------------------------+-----------+------------------------------------------------------------------------+ 297 | | ``LOG_REFERER`` | true | ``referer`` | 298 | +-----------------------------------+-----------+------------------------------------------------------------------------+ 299 | 300 | This behavior matches the corresponding mechanism in the `CF Java Logging Support library `__. 301 | 302 | Examples 303 | ~~~~~~~~ 304 | 305 | For more examples please see the tests within the ``./tests/`` directory. 306 | 307 | Requirements 308 | ------------ 309 | 310 | No external requirements are needed to run the package. 311 | 312 | Limitations 313 | ----------- 314 | 315 | NA 316 | 317 | Known Issues 318 | ------------ 319 | 320 | NA 321 | 322 | How to obtain support 323 | --------------------- 324 | 325 | Please open an issue on the github page. 326 | 327 | Contributing 328 | ------------ 329 | 330 | Please create a pull request and briefly describe the nature of the change. Please submit a test 331 | case along with your pull request. 332 | 333 | To-Do (upcoming changes) 334 | ------------------------ 335 | 336 | NA 337 | 338 | Changelog 339 | --------- 340 | 341 | See `CHANGELOG file `__. 342 | 343 | License 344 | ------- 345 | 346 | Copyright (c) 2017-2021 SAP SE or an SAP affiliate company and cf-python-logging-support contributors. Please see our `LICENSE file `__ for copyright and license information. Detailed information including third-party components and their licensing/copyright information is available `via the REUSE tool `__. 347 | 348 | --------------------------------------------------------------------------------