├── tests ├── __init__.py ├── README.md ├── test_uploader.py ├── test_frame.py ├── test_helpers.py ├── test_handler.py ├── test_flusher.py └── test_formatter.py ├── setup.cfg ├── requirements.txt ├── test-requirements.txt ├── MANIFEST.in ├── .gitignore ├── .travis.yml ├── timber ├── compat.py ├── __init__.py ├── uploader.py ├── formatter.py ├── helpers.py ├── frame.py ├── handler.py └── flusher.py ├── tox.ini ├── LICENSE.md ├── setup.py ├── README.md └── RELEASING.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests>=2.18.4 2 | msgpack>=0.5.6 3 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | coverage>=3.7.1 2 | httpretty>=0.9.4 3 | nose>=1.3.7 4 | unittest2>=0.8.0 5 | mock>=1.0.1 6 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE.md 2 | include README.md 3 | include PYPIREADME.rst 4 | include setup.py 5 | include *requirements.txt 6 | graft tests 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.pyc 3 | *.log 4 | .python-version 5 | htmlcov 6 | .coverage 7 | .tox 8 | *.egg-info 9 | build/* 10 | dist/* 11 | .DS_Store 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - "2.7" 5 | - "3.4" 6 | - "3.5" 7 | - "3.6" 8 | install: pip install tox-travis 9 | script: tox 10 | -------------------------------------------------------------------------------- /timber/compat.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | 4 | try: 5 | import queue 6 | except ImportError: 7 | import Queue as queue 8 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, py34, py35, py36 3 | 4 | [testenv] 5 | deps = 6 | -rtest-requirements.txt 7 | -rrequirements.txt 8 | commands = 9 | nosetests --with-coverage --cover-branches --cover-package=timber 10 | -------------------------------------------------------------------------------- /timber/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | 4 | from .handler import TimberHandler 5 | from .helpers import TimberContext, DEFAULT_CONTEXT 6 | from .formatter import TimberFormatter 7 | 8 | __version__ = '2.1.0' 9 | 10 | context = DEFAULT_CONTEXT 11 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # Testing 2 | 3 | 1. Install the packages required for development and testing: 4 | 5 | ```bash 6 | pip install -r requirements.txt 7 | pip install -r test-requirements.txt 8 | ``` 9 | 10 | 2. Run the tests: 11 | 12 | ```bash 13 | nosetests 14 | ``` 15 | 16 | To see test coverage, run the following and then open `./htmlcov/index.html` in your browser: 17 | 18 | ```bash 19 | nosetests --with-coverage --cover-branches --cover-package=timber tests 20 | coverage html --include='timber*' 21 | ``` -------------------------------------------------------------------------------- /timber/uploader.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | import msgpack 4 | import requests 5 | 6 | 7 | class Uploader(object): 8 | def __init__(self, api_key, source_id, host): 9 | self.api_key = api_key 10 | self.source_id = source_id 11 | self.host = host 12 | self.headers = { 13 | 'Authorization': 'Bearer %s' % api_key, 14 | 'Content-Type': 'application/msgpack', 15 | } 16 | 17 | def __call__(self, frame): 18 | endpoint = self.host + '/sources/' + self.source_id + '/frames' 19 | data = msgpack.packb(frame, use_bin_type=True) 20 | return requests.post(endpoint, data=data, headers=self.headers) 21 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # License 2 | 3 | Copyright (c) 2018, Timber Technologies, Inc. 4 | 5 | Permission to use, copy, modify, and/or distribute this software for any purpose 6 | with or without fee is hereby granted, provided that the above copyright notice 7 | and this permission notice appear in all copies. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND 11 | FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS 13 | OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 14 | TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF 15 | THIS SOFTWARE. 16 | -------------------------------------------------------------------------------- /timber/formatter.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | import logging 4 | import json 5 | 6 | from .helpers import DEFAULT_CONTEXT 7 | from .frame import create_frame 8 | 9 | 10 | class TimberFormatter(logging.Formatter): 11 | def __init__(self, 12 | context=DEFAULT_CONTEXT, 13 | json_default=None, 14 | json_encoder=None): 15 | self.context = context 16 | self.json_default = json_default 17 | self.json_encoder = json_encoder 18 | 19 | def format(self, record): 20 | # Because the formatter does not have an underlying format string for 21 | # which `extra` may be used to substitute arguments (see 22 | # https://docs.python.org/2/library/logging.html#logging.debug ), we 23 | # augment the log frame with all of the entries in extra. 24 | frame = create_frame(record, record.getMessage(), self.context, include_all_extra=True) 25 | return json.dumps(frame, default=self.json_default, cls=self.json_encoder) 26 | -------------------------------------------------------------------------------- /tests/test_uploader.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | import msgpack 4 | import mock 5 | import unittest2 6 | 7 | from timber.uploader import Uploader 8 | 9 | 10 | class TestUploader(unittest2.TestCase): 11 | host = 'https://timber.io' 12 | source_id = 'dummy_source_id' 13 | api_key = 'dummy_api_key' 14 | frame = [1, 2, 3] 15 | 16 | @mock.patch('timber.uploader.requests.post') 17 | def test_call(self, post): 18 | def mock_post(endpoint, data=None, headers=None): 19 | # Check that the data is sent to ther correct endpoint 20 | self.assertEqual(endpoint, self.host + '/sources/' + self.source_id + '/frames') 21 | # Check the content-type 22 | self.assertIsInstance(headers, dict) 23 | self.assertIn('Authorization', headers) 24 | self.assertEqual('application/msgpack', headers.get('Content-Type')) 25 | # Check the content was msgpacked correctly 26 | self.assertEqual(msgpack.unpackb(data, raw=False), self.frame) 27 | 28 | post.side_effect = mock_post 29 | u = Uploader(self.api_key, self.source_id, self.host) 30 | u(self.frame) 31 | 32 | self.assertTrue(post.called) 33 | -------------------------------------------------------------------------------- /timber/helpers.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | 4 | 5 | class TimberContext(object): 6 | def __init__(self): 7 | self.extras = [] 8 | 9 | def context(self, *args, **kwargs): 10 | if args: 11 | raise ValueError( 12 | 'All contexts must be passed by name as keyword arguments' 13 | ) 14 | for key, val in kwargs.items(): 15 | if not isinstance(val, dict): 16 | raise ValueError( 17 | 'All contexts must be dictionaries: %s' % key 18 | ) 19 | self.extras.append(kwargs) 20 | return self 21 | 22 | def __call__(self, *args, **kwargs): 23 | return self.context(*args, **kwargs) 24 | 25 | def __enter__(self): 26 | return self 27 | 28 | def __exit__(self, type_, value, traceback): 29 | if type_ is not None: 30 | return False 31 | self.extras.pop() 32 | return self 33 | 34 | def exists(self): 35 | return bool(self.extras) 36 | 37 | def collapse(self): 38 | x = {} 39 | for contexts in self.extras: 40 | for name, data in contexts.items(): 41 | x.setdefault(name, {}).update(data) 42 | return x 43 | 44 | 45 | DEFAULT_CONTEXT = TimberContext() 46 | -------------------------------------------------------------------------------- /tests/test_frame.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | from timber.frame import create_frame 4 | from timber.handler import TimberHandler 5 | from timber.helpers import TimberContext 6 | import unittest2 7 | import logging 8 | 9 | class TestTimberLogEntry(unittest2.TestCase): 10 | def test_create_frame_happy_path(self): 11 | handler = TimberHandler(api_key="some-api-key", source_id="some-source-id") 12 | log_record = logging.LogRecord("timber-test", 20, "/some/path", 10, "Some log message", [], None) 13 | frame = create_frame(log_record, log_record.getMessage(), TimberContext()) 14 | self.assertTrue(frame['level'] == 'info') 15 | 16 | def test_create_frame_with_extra(self): 17 | handler = TimberHandler(api_key="some-api-key", source_id="some-source-id") 18 | 19 | log_record = logging.LogRecord("timber-test", 20, "/some/path", 10, "Some log message", [], None) 20 | extra = {'non_dict_key': 'string_value', 'dict_key': {'name': 'Test Test'}} 21 | log_record.__dict__.update(extra) # This is how `extra` gets included in the LogRecord 22 | 23 | # By default, non-dict keys are excluded. 24 | frame = create_frame(log_record, log_record.getMessage(), TimberContext()) 25 | self.assertTrue(frame['level'] == 'info') 26 | self.assertIn('dict_key', frame) 27 | self.assertNotIn('non_dict_key', frame) 28 | 29 | frame = create_frame(log_record, log_record.getMessage(), TimberContext(), include_all_extra=True) 30 | self.assertIn('non_dict_key', frame) 31 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import os 3 | from setuptools import setup 4 | 5 | 6 | VERSION = '2.1.0' 7 | ROOT_DIR = os.path.dirname(__file__) 8 | 9 | REQUIREMENTS = [ 10 | line.strip() for line in 11 | open(os.path.join(ROOT_DIR, 'requirements.txt')).readlines() 12 | ] 13 | 14 | with open("README.md", "r") as fh: 15 | long_description = fh.read() 16 | 17 | setup( 18 | name='timber', 19 | version=VERSION, 20 | packages=['timber'], 21 | include_package_data=True, 22 | license='MIT', 23 | description='timber.io client API library', 24 | long_description=long_description, 25 | long_description_content_type='text/markdown', 26 | url='https://github.com/timberio/timber-python', 27 | download_url='https://github.com/timberio/timber-python/tarball/%s' % ( 28 | VERSION), 29 | keywords=['api', 'timber', 'logging', 'client'], 30 | install_requires=REQUIREMENTS, 31 | author='Timber Technologies, Inc.', 32 | author_email='help@timber.io', 33 | classifiers=[ 34 | 'Intended Audience :: Developers', 35 | 'License :: OSI Approved :: MIT License', 36 | 'Operating System :: OS Independent', 37 | 'Programming Language :: Python :: 2', 38 | 'Programming Language :: Python :: 2.7', 39 | 'Programming Language :: Python :: 3', 40 | 'Programming Language :: Python :: 3.4', 41 | 'Programming Language :: Python :: 3.5', 42 | 'Programming Language :: Python :: 3.6', 43 | 'Programming Language :: Python', 44 | 'Topic :: Software Development :: Libraries :: Python Modules', 45 | ], 46 | ) 47 | -------------------------------------------------------------------------------- /tests/test_helpers.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | import unittest2 4 | 5 | from timber import TimberContext 6 | 7 | 8 | class TestTimberContext(unittest2.TestCase): 9 | 10 | def test_exists(self): 11 | c = TimberContext() 12 | self.assertFalse(c.exists()) 13 | with c(user={'name': 'a'}): 14 | self.assertTrue(c.exists()) 15 | self.assertFalse(c.exists()) 16 | 17 | def test_only_accepts_keyword_argument_dicts(self): 18 | c = TimberContext() 19 | # Named context passes 20 | c(user={'name': 'a'}) 21 | # Non-named contexts fail, even if they're dicts 22 | for garbage in ['x', 1, [{'name': 'a'}], {'name': 'a'}]: 23 | with self.assertRaises(ValueError): 24 | c(garbage) 25 | # Named contexts fail if they are not dicts 26 | for garbage in [{'user': 1}, {'user': []}, {'user': tuple()}]: 27 | with self.assertRaises(ValueError): 28 | c(**garbage) 29 | 30 | def test_does_not_suppress_exceptions(self): 31 | c = TimberContext() 32 | with self.assertRaises(ValueError): 33 | with c(user={'name': 'a'}): 34 | raise ValueError('should be thrown') 35 | 36 | def test_nested_collapse(self): 37 | c = TimberContext() 38 | self.assertEqual(c.collapse(), {}) 39 | 40 | with c(user={'name': 'a', 'count': 1}): 41 | self.assertEqual( 42 | c.collapse(), 43 | {'user': {'name': 'a', 'count': 1}} 44 | ) 45 | 46 | with c(user={'name': 'b'}, other={'foo': 'bar'}): 47 | self.assertEqual( 48 | c.collapse(), 49 | {'user': {'name': 'b', 'count': 1}, 50 | 'other': {'foo': 'bar'}} 51 | ) 52 | 53 | self.assertEqual( 54 | c.collapse(), 55 | {'user': {'name': 'a', 'count': 1}} 56 | ) 57 | 58 | self.assertEqual(c.collapse(), {}) 59 | -------------------------------------------------------------------------------- /timber/frame.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | import msgpack 4 | from datetime import datetime 5 | 6 | 7 | def create_frame(record, message, context, include_all_extra=False): 8 | r = record.__dict__ 9 | frame = {} 10 | frame['dt'] = datetime.utcfromtimestamp(r['created']).isoformat() 11 | frame['level'] = level = _levelname(r['levelname']) 12 | frame['severity'] = int(r['levelno'] / 10) 13 | frame['message'] = message 14 | frame['context'] = ctx = {} 15 | 16 | # Runtime context 17 | ctx['runtime'] = runtime = {} 18 | runtime['function'] = r['funcName'] 19 | runtime['file'] = r['filename'] 20 | runtime['line'] = r['lineno'] 21 | runtime['thread_id'] = r['thread'] 22 | runtime['thread_name'] = r['threadName'] 23 | runtime['logger_name'] = r['name'] 24 | 25 | # Runtime context 26 | ctx['system'] = system = {} 27 | system['pid'] = r['process'] 28 | system['process_name'] = r['processName'] 29 | 30 | # Custom context 31 | if context.exists(): 32 | ctx.update(context.collapse()) 33 | 34 | events = _parse_custom_events(record, include_all_extra) 35 | if events: 36 | frame.update(events) 37 | 38 | return frame 39 | 40 | 41 | def _parse_custom_events(record, include_all_extra): 42 | default_keys = { 43 | 'args', 'asctime', 'created', 'exc_info', 'exc_text', 'filename', 44 | 'funcName', 'levelname', 'levelno', 'lineno', 'module', 'msecs', 45 | 'message', 'msg', 'name', 'pathname', 'process', 'processName', 46 | 'relativeCreated', 'thread', 'threadName' 47 | } 48 | events = {} 49 | for key, val in record.__dict__.items(): 50 | if key in default_keys: 51 | continue 52 | if not include_all_extra and not isinstance(val, dict): 53 | continue 54 | events[key] = val 55 | return events 56 | 57 | 58 | def _levelname(level): 59 | return { 60 | 'debug': 'debug', 61 | 'info': 'info', 62 | 'warning': 'warn', 63 | 'error': 'error', 64 | 'critical': 'critical', 65 | }[level.lower()] 66 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🌲 Timber - Great Python Logging Made Easy 2 | 3 |

4 | 5 | 6 | 7 |
8 |

9 | 10 | [![ISC License](https://img.shields.io/badge/license-ISC-ff69b4.svg)](LICENSE.md) 11 | [![Pypi](https://img.shields.io/pypi/v/timber.svg)](https://pypi.python.org/pypi/timber) 12 | [![Python Support](https://img.shields.io/pypi/pyversions/timber.svg)](https://pypi.python.org/pypi/django-analytical) 13 | [![Build Status](https://travis-ci.org/timberio/timber-python.svg?branch=master)](https://travis-ci.org/timberio/timber-python) 14 | 15 | [Timber.io](https://timber.io) is a hosted service for aggregating logs across your entire stack - 16 | [any language](https://docs.timber.io/setup/languages), 17 | [any platform](https://docs.timber.io/setup/platforms), 18 | [any data source](https://docs.timber.io/setup/log_forwarders). 19 | 20 | Unlike traditional logging tools, Timber integrates with language runtimes to automatically 21 | capture in-app context, turning your text-based logs into rich structured events. 22 | Timber integrates with Python through this library. And Timber's 23 | [rich free-form query tools](https://docs-new.timber.io/usage/live-tailing#query-syntax) and 24 | [real-time tailing](https://docs-new.timber.io/usage/live-tailing), make drilling down into 25 | important stats easier than ever. 26 | 27 | --- 28 | 29 | ### Features 30 | 31 | * Simple integration. Integrates with the Python `logging` library. 32 | * Support for structured logging and events. 33 | * Support for context. 34 | * Automatically captures useful context. 35 | * Performant, light weight, with a thoughtful design. 36 | 37 | --- 38 | 39 | ### Get Started 40 | 41 | * **[Installation](https://docs.timber.io/setup/languages/python#installation)** 42 | * **[Configuration](https://docs.timber.io/setup/languages/python#configuration)** 43 | * **[Usage](https://docs.timber.io/setup/languages/python#usage)** 44 | * **[Guides](https://docs.timber.io/setup/languages/python#guides)** 45 | * **[Automatic Context](https://docs.timber.io/setup/languages/python#automatic-context)** 46 | * **[Performance](https://docs.timber.io/setup/languages/python#performance)** 47 | 48 | --- 49 | 50 |

51 | Timber • 52 | Docs • 53 | Pricing • 54 | Security • 55 | Compliance 56 |

-------------------------------------------------------------------------------- /timber/handler.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | import logging 4 | import multiprocessing 5 | 6 | from .compat import queue 7 | from .helpers import DEFAULT_CONTEXT 8 | from .flusher import FlushWorker 9 | from .uploader import Uploader 10 | from .frame import create_frame 11 | 12 | DEFAULT_HOST = 'https://logs.timber.io' 13 | DEFAULT_BUFFER_CAPACITY = 1000 14 | DEFAULT_FLUSH_INTERVAL = 2 15 | DEFAULT_RAISE_EXCEPTIONS = False 16 | DEFAULT_DROP_EXTRA_EVENTS = True 17 | 18 | 19 | class TimberHandler(logging.Handler): 20 | def __init__(self, 21 | api_key, 22 | source_id, 23 | host=DEFAULT_HOST, 24 | buffer_capacity=DEFAULT_BUFFER_CAPACITY, 25 | flush_interval=DEFAULT_FLUSH_INTERVAL, 26 | raise_exceptions=DEFAULT_RAISE_EXCEPTIONS, 27 | drop_extra_events=DEFAULT_DROP_EXTRA_EVENTS, 28 | context=DEFAULT_CONTEXT, 29 | level=logging.NOTSET): 30 | super(TimberHandler, self).__init__(level=level) 31 | self.api_key = api_key 32 | self.source_id = source_id 33 | self.host = host 34 | self.context = context 35 | self.pipe = multiprocessing.JoinableQueue(maxsize=buffer_capacity) 36 | self.uploader = Uploader(self.api_key, self.source_id, self.host) 37 | self.drop_extra_events = drop_extra_events 38 | self.buffer_capacity = buffer_capacity 39 | self.flush_interval = flush_interval 40 | self.raise_exceptions = raise_exceptions 41 | self.dropcount = 0 42 | self.flush_thread = FlushWorker( 43 | self.uploader, 44 | self.pipe, 45 | self.buffer_capacity, 46 | self.flush_interval 47 | ) 48 | if self._is_main_process(): 49 | self.flush_thread.start() 50 | 51 | def _is_main_process(self): 52 | return multiprocessing.current_process()._parent_pid == None 53 | 54 | def emit(self, record): 55 | try: 56 | if self._is_main_process() and not self.flush_thread.is_alive(): 57 | self.flush_thread.start() 58 | message = self.format(record) 59 | frame = create_frame(record, message, self.context) 60 | try: 61 | self.pipe.put(frame, block=(not self.drop_extra_events)) 62 | except queue.Full: 63 | # Only raised when not blocking, which means that extra events 64 | # should be dropped. 65 | self.dropcount += 1 66 | pass 67 | except Exception as e: 68 | if self.raise_exceptions: 69 | raise e 70 | -------------------------------------------------------------------------------- /timber/flusher.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | import sys 4 | import time 5 | import threading 6 | 7 | from .compat import queue 8 | 9 | RETRY_SCHEDULE = (1, 10, 60) # seconds 10 | 11 | 12 | class FlushWorker(threading.Thread): 13 | def __init__(self, upload, pipe, buffer_capacity, flush_interval): 14 | threading.Thread.__init__(self) 15 | self.parent_thread = threading.currentThread() 16 | self.upload = upload 17 | self.pipe = pipe 18 | self.buffer_capacity = buffer_capacity 19 | self.flush_interval = flush_interval 20 | 21 | def run(self): 22 | while True: 23 | self.step() 24 | 25 | def step(self): 26 | last_flush = time.time() 27 | time_remaining = _initial_time_remaining(self.flush_interval) 28 | frame = [] 29 | 30 | # If the parent thread has exited but there are still outstanding 31 | # events, attempt to send them before exiting. 32 | shutdown = not self.parent_thread.is_alive() 33 | 34 | # Fill phase: take events out of the queue and group them for sending. 35 | # Takes up to `buffer_capacity` events out of the queue and groups them 36 | # for sending; may send fewer than `buffer_capacity` events if 37 | # `flush_interval` seconds have passed without sending any events. 38 | while len(frame) < self.buffer_capacity and time_remaining > 0: 39 | try: 40 | # Blocks for up to 1.0 seconds for each item to prevent 41 | # spinning and burning CPU unnecessarily. Could block for the 42 | # entire amount of `time_remaining` but then in the case that 43 | # the parent thread has exited, that entire amount of time 44 | # would be waited before this child worker thread exits. 45 | entry = self.pipe.get(block=(not shutdown), timeout=1.0) 46 | frame.append(entry) 47 | self.pipe.task_done() 48 | except queue.Empty: 49 | if shutdown: 50 | break 51 | shutdown = not self.parent_thread.is_alive() 52 | time_remaining = _calculate_time_remaining(last_flush, self.flush_interval) 53 | 54 | # Send phase: takes the outstanding events (up to `buffer_capacity` 55 | # count) and sends them to the Timber endpoint all at once. If the 56 | # request fails in a way that can be retried, it is retried with an 57 | # exponential backoff in between attempts. 58 | if frame: 59 | for delay in RETRY_SCHEDULE + (None, ): 60 | response = self.upload(frame) 61 | if not _should_retry(response.status_code): 62 | break 63 | if delay is not None: 64 | time.sleep(delay) 65 | 66 | if shutdown: 67 | sys.exit(0) 68 | 69 | 70 | def _initial_time_remaining(flush_interval): 71 | return flush_interval 72 | 73 | 74 | def _calculate_time_remaining(last_flush, flush_interval): 75 | elapsed = time.time() - last_flush 76 | time_remaining = max(flush_interval - elapsed, 0) 77 | return time_remaining 78 | 79 | 80 | def _should_retry(status_code): 81 | return 500 <= status_code < 600 82 | -------------------------------------------------------------------------------- /RELEASING.md: -------------------------------------------------------------------------------- 1 | # Publishing New Versions 2 | 3 | Please follow the steps below when releasing a new version. This helps ensure 4 | that the experience for users is consistent and well documented. 5 | 6 | Publishing releases occurs in two parts: tagging in the Git repository and 7 | publishing on Github; and publishing to pypi. 8 | 9 | ## Versioning 10 | 11 | The `timber-python` package uses [Semantic Versioning](http://semver.org/) for its 12 | releases. This means you should think critically about how the changes made in 13 | the library impact functionality. The Semantic Versioning summary has a good 14 | overview. When in doubt, prefer incrementing the MINOR version over the PATCH 15 | version. 16 | 17 | To check what changes have been made since the last release, you can use the 18 | `git log` function. Let's say the last release was `2.0.1`; the tag for that 19 | is `v2.0.1`, so we can see the changes between that release and the `HEAD` of 20 | `master` using the following command: 21 | 22 | ```bash 23 | git log --pretty=oneline v1.0.7..master 24 | ``` 25 | 26 | You can drop the `--pretty=oneline` to view the full commit messages, but the 27 | general size of the comparison should give you a good idea of how much change 28 | has occured. 29 | 30 | ## Git Tagging 31 | 32 | Git tags are powerful tools that can be used to reference specific commits in 33 | the repository. In addition to helping us manage which commits are included in a 34 | release, they are also used by the documentation system to build reference links 35 | back to the original source code. 36 | 37 | `timber-python` releases should _always_ use annotated Git tags. The tag should 38 | be of the form `v#{Version}`. So, if you're publishing version 5.0.9, the tag is 39 | `v5.0.9`. If you are publishing version 1.0.7-rc.1, the tag is `v1.0.7-rc.1`. 40 | Naming the tag improperly will result in the links from the documentation to the 41 | source code breaking. 42 | 43 | Annotated tags can be created using `git tag -a`. For example, to create tag 44 | `v1.7.1` at the current commit, you would use the following command: 45 | 46 | ```bash 47 | git tag -a v1.7.1 48 | ``` 49 | 50 | This opens the default editor from your shell profile where you are expected to 51 | provide the tag's message. When you save and quit the editor, the tag will be 52 | created. If you exit the editor before saving, the tag creation will be aborted. 53 | 54 | The tag message should take the following form: 55 | 56 | ``` 57 | #{Version} - #{dd MMMM YYYY} 58 | 59 | #{Changes} 60 | ``` 61 | 62 | The changes should be a list summarizing the changes that were made, possibly 63 | with links to the relevant issues and pull requests. You may also have a longer 64 | prose description at the top. The date should be the current date. The format is 65 | Here's a full example: 66 | 67 | ``` 68 | 2.9.0-rc.3 - 07 June 2017 69 | 70 | This is a release candidate for the new syslog logger backend fixing a number of 71 | reported issues. 72 | 73 | - The socket will now be released cleanly even if the backend encounters an 74 | error (see #1058, #1067, and #1069) 75 | - The output will now be truncated so that it is not divided up into different 76 | separate lines by the syslog daemon (see #1043 and #1075) 77 | - The destination syslog daemon's configuration can be determined at runtime 78 | now by calling a configuration function (see #1049 and #1052) 79 | ``` 80 | 81 | ## New Release Instructions 82 | 83 | To actually perform a release, follow these steps: 84 | 85 | 1. Increment the version number inside of `setup.py` and `timber/__init__.py` then commit it to `master`. 86 | 2. Review existing documentation. 87 | 3. Make sure that you have checked out `master` and have performed `git pull`. 88 | 4. Create an annotated tag for the release. 89 | 5. Push the new tags to GitHub using `git push --tags` 90 | 6. Create a new release on the [GitHub 91 | releases](https://github.com/timberio/timber-python/releases) page. Use the 92 | tag you just created, with the first line of the tag's message as the title 93 | and the rest of the message as the body. Make sure to check the pre-release 94 | box if this is a release candidate. 95 | 7. Run `python setup.py sdist bdist_wheel` 96 | 8. Run `twine upload dist/*` 97 | 98 | If you have any questions, ping @DavidAntaramian or @LucioFranco. 99 | -------------------------------------------------------------------------------- /tests/test_handler.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | import mock 4 | import time 5 | import threading 6 | import unittest2 7 | import logging 8 | 9 | from timber.handler import TimberHandler 10 | 11 | 12 | class TestTimberHandler(unittest2.TestCase): 13 | api_key = 'dummy_api_key' 14 | source_id = 'source_id' 15 | host = 'dummy_host' 16 | 17 | @mock.patch('timber.handler.FlushWorker') 18 | def test_handler_creates_uploader_from_args(self, MockWorker): 19 | handler = TimberHandler(api_key=self.api_key, source_id=self.source_id, host=self.host) 20 | self.assertEqual(handler.uploader.api_key, self.api_key) 21 | self.assertEqual(handler.uploader.host, self.host) 22 | 23 | @mock.patch('timber.handler.FlushWorker') 24 | def test_handler_creates_pipe_from_args(self, MockWorker): 25 | buffer_capacity = 9 26 | flush_interval = 1 27 | handler = TimberHandler( 28 | api_key=self.api_key, 29 | source_id=self.source_id, 30 | buffer_capacity=buffer_capacity, 31 | flush_interval=flush_interval 32 | ) 33 | self.assertEqual(handler.pipe._maxsize, buffer_capacity) 34 | 35 | @mock.patch('timber.handler.FlushWorker') 36 | def test_handler_creates_and_starts_worker_from_args(self, MockWorker): 37 | buffer_capacity = 9 38 | flush_interval = 9 39 | handler = TimberHandler(api_key=self.api_key, source_id=self.source_id, buffer_capacity=buffer_capacity, flush_interval=flush_interval) 40 | MockWorker.assert_called_with( 41 | handler.uploader, 42 | handler.pipe, 43 | buffer_capacity, 44 | flush_interval 45 | ) 46 | self.assertTrue(handler.flush_thread.start.called) 47 | 48 | @mock.patch('timber.handler.FlushWorker') 49 | def test_emit_starts_thread_if_not_alive(self, MockWorker): 50 | handler = TimberHandler(api_key=self.api_key, source_id=self.source_id) 51 | self.assertTrue(handler.flush_thread.start.call_count, 1) 52 | handler.flush_thread.is_alive = mock.Mock(return_value=False) 53 | 54 | logger = logging.getLogger(__name__) 55 | logger.handlers = [] 56 | logger.addHandler(handler) 57 | logger.critical('hello') 58 | 59 | self.assertEqual(handler.flush_thread.start.call_count, 2) 60 | 61 | @mock.patch('timber.handler.FlushWorker') 62 | def test_emit_drops_records_if_configured(self, MockWorker): 63 | buffer_capacity = 1 64 | handler = TimberHandler( 65 | api_key=self.api_key, 66 | source_id=self.source_id, 67 | buffer_capacity=buffer_capacity, 68 | drop_extra_events=True 69 | ) 70 | 71 | logger = logging.getLogger(__name__) 72 | logger.handlers = [] 73 | logger.addHandler(handler) 74 | logger.critical('hello') 75 | logger.critical('goodbye') 76 | 77 | log_entry = handler.pipe.get() 78 | self.assertEqual(log_entry['message'], 'hello') 79 | self.assertTrue(handler.pipe.empty()) 80 | self.assertEqual(handler.dropcount, 1) 81 | 82 | @mock.patch('timber.handler.FlushWorker') 83 | def test_emit_does_not_drop_records_if_configured(self, MockWorker): 84 | buffer_capacity = 1 85 | handler = TimberHandler( 86 | api_key=self.api_key, 87 | source_id=self.source_id, 88 | buffer_capacity=buffer_capacity, 89 | drop_extra_events=False 90 | ) 91 | 92 | def consumer(q): 93 | while True: 94 | if q.full(): 95 | while not q.empty(): 96 | _ = q.get(block=True) 97 | time.sleep(.2) 98 | 99 | t = threading.Thread(target=consumer, args=(handler.pipe,)) 100 | t.daemon = True 101 | 102 | logger = logging.getLogger(__name__) 103 | logger.handlers = [] 104 | logger.addHandler(handler) 105 | logger.critical('hello') 106 | 107 | self.assertTrue(handler.pipe.full()) 108 | t.start() 109 | logger.critical('goodbye') 110 | logger.critical('goodbye2') 111 | 112 | self.assertEqual(handler.dropcount, 0) 113 | 114 | @mock.patch('timber.handler.FlushWorker') 115 | def test_error_suppression(self, MockWorker): 116 | buffer_capacity = 1 117 | handler = TimberHandler( 118 | api_key=self.api_key, 119 | source_id=self.source_id, 120 | buffer_capacity=buffer_capacity, 121 | raise_exceptions=True 122 | ) 123 | 124 | handler.pipe = mock.MagicMock(put=mock.Mock(side_effect=ValueError)) 125 | 126 | logger = logging.getLogger(__name__) 127 | logger.handlers = [] 128 | logger.addHandler(handler) 129 | 130 | with self.assertRaises(ValueError): 131 | logger.critical('hello') 132 | 133 | handler.raise_exceptions = False 134 | logger.critical('hello') 135 | -------------------------------------------------------------------------------- /tests/test_flusher.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | import mock 4 | import time 5 | import threading 6 | import unittest2 7 | 8 | from timber.compat import queue 9 | from timber.flusher import RETRY_SCHEDULE 10 | from timber.flusher import FlushWorker 11 | from timber.uploader import Uploader 12 | 13 | 14 | class TestFlushWorker(unittest2.TestCase): 15 | host = 'https://timber.io' 16 | api_key = 'dummy_api_key' 17 | source_id = 'dummy_source_id' 18 | buffer_capacity = 5 19 | flush_interval = 2 20 | 21 | def _setup_worker(self, uploader=None): 22 | pipe = queue.Queue(maxsize=self.buffer_capacity) 23 | uploader = uploader or Uploader(self.api_key, self.source_id, self.host) 24 | fw = FlushWorker(uploader, pipe, self.buffer_capacity, self.flush_interval) 25 | return pipe, uploader, fw 26 | 27 | def test_is_thread(self): 28 | pipe, uploader, fw = self._setup_worker() 29 | self.assertIsInstance(fw, threading.Thread) 30 | 31 | def test_flushes_when_queue_is_full(self): 32 | first_frame = list(range(self.buffer_capacity)) 33 | second_frame = list(range(self.buffer_capacity, self.buffer_capacity * 2)) 34 | self.calls = 0 35 | self.flush_interval = 1000 36 | 37 | def uploader(frame): 38 | self.calls += 1 39 | self.assertEqual(frame, first_frame) 40 | return mock.MagicMock(status_code=202) 41 | 42 | pipe, _, fw = self._setup_worker(uploader) 43 | 44 | for log in first_frame: 45 | pipe.put(log, block=False) 46 | 47 | t1 = time.time() 48 | fw.step() 49 | t2 = time.time() 50 | self.assertLess(t2 - t1, self.flush_interval) 51 | 52 | self.assertEqual(self.calls, 1) 53 | 54 | @mock.patch('timber.flusher._calculate_time_remaining') 55 | def test_flushes_after_interval(self, calculate_time_remaining): 56 | self.buffer_capacity = 10 57 | num_items = 2 58 | first_frame = list(range(self.buffer_capacity)) 59 | self.assertLess(num_items, self.buffer_capacity) 60 | 61 | self.upload_calls = 0 62 | def uploader(frame): 63 | self.upload_calls += 1 64 | self.assertEqual(frame, first_frame[:num_items]) 65 | return mock.MagicMock(status_code=202) 66 | 67 | self.timeout_calls = 0 68 | def timeout(last_flush, interval): 69 | self.timeout_calls += 1 70 | # Until the last item has been retrieved from the pipe, the timeout 71 | # length doesn't matter. After the last item has been retrieved, 72 | # return a very small number so that the blocking get times out 73 | if self.timeout_calls < num_items: 74 | return 1000000 75 | return 0 76 | calculate_time_remaining.side_effect = timeout 77 | 78 | pipe, _, fw = self._setup_worker(uploader) 79 | for i in range(num_items): 80 | pipe.put(first_frame[i], block=False) 81 | 82 | fw.step() 83 | self.assertEqual(self.upload_calls, 1) 84 | self.assertEqual(self.timeout_calls, 2) 85 | 86 | @mock.patch('timber.flusher._calculate_time_remaining') 87 | @mock.patch('timber.flusher._initial_time_remaining') 88 | def test_does_nothing_without_any_items(self, initial_time_remaining, calculate_time_remaining): 89 | calculate_time_remaining.side_effect = lambda a,b: 0.0 90 | initial_time_remaining.side_effect = lambda a: 0.0001 91 | 92 | uploader = mock.MagicMock(side_effect=mock.MagicMock(status_code=202)) 93 | pipe, _, fw = self._setup_worker(uploader) 94 | 95 | self.assertEqual(pipe.qsize(), 0) 96 | fw.step() 97 | self.assertFalse(uploader.called) 98 | 99 | @mock.patch('timber.flusher.time.sleep') 100 | def test_retries_according_to_schedule(self, mock_sleep): 101 | first_frame = list(range(self.buffer_capacity)) 102 | 103 | self.uploader_calls = 0 104 | def uploader(frame): 105 | self.uploader_calls += 1 106 | self.assertEqual(frame, first_frame) 107 | return mock.MagicMock(status_code=500) 108 | 109 | self.sleep_calls = 0 110 | def sleep(time): 111 | self.assertEqual(time, RETRY_SCHEDULE[self.sleep_calls]) 112 | self.sleep_calls += 1 113 | mock_sleep.side_effect = sleep 114 | 115 | pipe, _, fw = self._setup_worker(uploader) 116 | 117 | for log in first_frame: 118 | pipe.put(log, block=False) 119 | 120 | fw.step() 121 | self.assertEqual(self.uploader_calls, len(RETRY_SCHEDULE) + 1) 122 | self.assertEqual(self.sleep_calls, len(RETRY_SCHEDULE)) 123 | 124 | @mock.patch('timber.flusher.sys.exit') 125 | def test_shutdown_condition_empties_queue_and_calls_exit(self, mock_exit): 126 | self.buffer_capacity = 10 127 | num_items = 5 128 | first_frame = list(range(self.buffer_capacity)) 129 | self.assertLess(num_items, self.buffer_capacity) 130 | 131 | self.upload_calls = 0 132 | def uploader(frame): 133 | self.upload_calls += 1 134 | self.assertEqual(frame, first_frame[:num_items]) 135 | return mock.MagicMock(status_code=202) 136 | 137 | pipe, _, fw = self._setup_worker(uploader) 138 | fw.parent_thread = mock.MagicMock(is_alive=lambda: False) 139 | 140 | for i in range(num_items): 141 | pipe.put(first_frame[i], block=False) 142 | 143 | fw.step() 144 | self.assertEqual(self.upload_calls, 1) 145 | self.assertEqual(mock_exit.call_count, 1) 146 | -------------------------------------------------------------------------------- /tests/test_formatter.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from __future__ import print_function, unicode_literals 3 | import mock 4 | import time 5 | import threading 6 | import json 7 | import unittest2 8 | import pdb 9 | import logging 10 | 11 | import timber 12 | from timber.formatter import TimberFormatter 13 | from timber.helpers import TimberContext 14 | 15 | 16 | class TestTimberFormatter(unittest2.TestCase): 17 | def setUp(self): 18 | self.context = TimberContext() 19 | self.customer = {'id': '1'} 20 | self.order = {'id': '1234', 'amount': 200, 'item': '#19849'} 21 | 22 | def _check_and_get_line(self, loglines): 23 | self.assertEqual(len(loglines), 1) 24 | return loglines[0] 25 | 26 | def test_format_emits_single_line(self): 27 | formatter = timber.TimberFormatter(context=self.context) 28 | logger, loglines = logger_and_lines(formatter) 29 | self.assertFalse(loglines) 30 | 31 | logger.info('Hello\n\n\n\n\n\nWorld') 32 | line = self._check_and_get_line(loglines) 33 | self.assertEqual(len(line.split('\n')), 1) 34 | 35 | def test_format_creates_json_serialized_frame_with_context(self): 36 | formatter = timber.TimberFormatter(context=self.context) 37 | logger, loglines = logger_and_lines(formatter) 38 | self.assertFalse(loglines) 39 | 40 | with self.context(customer=self.customer): 41 | logger.info('Received order id=%s', self.order['id'], extra={'order': self.order}) 42 | 43 | line = self._check_and_get_line(loglines) 44 | frame = json.loads(line) 45 | self.assertEqual(frame['message'], 'Received order id=%s' % self.order['id']) 46 | self.assertEqual(frame['order'], self.order) 47 | self.assertEqual(frame['context']['customer'], self.customer) 48 | 49 | def test_format_collapses_context(self): 50 | formatter = timber.TimberFormatter(context=self.context) 51 | logger, loglines = logger_and_lines(formatter) 52 | self.assertFalse(loglines) 53 | 54 | with self.context(customer=self.customer): 55 | with self.context(customer={'trusted': True}): 56 | logger.info('Received an order', extra={'order': self.order}) 57 | 58 | line = self._check_and_get_line(loglines) 59 | frame = json.loads(line) 60 | self.assertEqual(frame['message'], 'Received an order') 61 | self.assertEqual(frame['order'], self.order) 62 | self.assertEqual(frame['context']['customer'], {'id': self.customer['id'], 'trusted': True}) 63 | 64 | def test_format_with_custom_default_json_serializer(self): 65 | def suppress_encoding_errors(obj): 66 | return 'Could not encode type=%s' % type(obj).__name__ 67 | 68 | default_formatter = timber.TimberFormatter(context=self.context) 69 | default_logger, _ = logger_and_lines(default_formatter, 'default') 70 | 71 | suppress_formatter = timber.TimberFormatter(context=self.context, json_default=suppress_encoding_errors) 72 | suppress_logger, loglines = logger_and_lines(suppress_formatter, 'suppress') 73 | 74 | self.assertIsNot(default_logger, suppress_logger) 75 | 76 | with self.context(data={'not_encodable': Dummy()}): 77 | with self.assertRaises(TypeError): 78 | default_logger.info('hello') 79 | suppress_logger.info('goodbye') 80 | 81 | line = self._check_and_get_line(loglines) 82 | frame = json.loads(line) 83 | self.assertEqual(frame['message'], 'goodbye') 84 | self.assertEqual(frame['context']['data'], {'not_encodable': 'Could not encode type=Dummy'}) 85 | 86 | def test_format_with_custom_default_json_encoder(self): 87 | default_formatter = timber.TimberFormatter(context=self.context) 88 | default_logger, _ = logger_and_lines(default_formatter, 'default') 89 | 90 | dummy_capable_formatter = timber.TimberFormatter(context=self.context, json_encoder=DummyCapableEncoder) 91 | dummy_capable_logger, loglines = logger_and_lines(dummy_capable_formatter, 'dummy_capable') 92 | 93 | self.assertIsNot(default_logger, dummy_capable_logger) 94 | 95 | with self.context(data={'not_encodable': Dummy()}): 96 | with self.assertRaises(TypeError): 97 | default_logger.info('hello') 98 | dummy_capable_logger.info('goodbye') 99 | 100 | line = self._check_and_get_line(loglines) 101 | frame = json.loads(line) 102 | self.assertEqual(frame['message'], 'goodbye') 103 | self.assertEqual(frame['context']['data'], {'not_encodable': ''}) 104 | 105 | 106 | class Dummy(object): 107 | """ Because this is a custom class, it cannot be encoded by the default JSONEncoder. """ 108 | 109 | 110 | class DummyCapableEncoder(json.JSONEncoder): 111 | """ A JSONEncoder that can encode instances of the Dummy class. """ 112 | def default(self, obj): 113 | if isinstance(obj, Dummy): 114 | return '' 115 | return super(CustomEncoder, self).default(obj) 116 | 117 | 118 | class ListHandler(logging.Handler): 119 | """ Accumulates all log lines in a list for testing purposes. """ 120 | def __init__(self, *args, **kwargs): 121 | super(ListHandler, self).__init__(*args, **kwargs) 122 | self.lines = [] 123 | 124 | def emit(self, record): 125 | logline = self.format(record) 126 | self.lines.append(logline) 127 | 128 | def logger_and_lines(formatter, name=__name__): 129 | """ Helper for more easily writing formatter tests. """ 130 | logger = logging.getLogger(name) 131 | logger.setLevel(logging.DEBUG) 132 | logger.handlers = [] 133 | handler = ListHandler() 134 | handler.setFormatter(formatter) 135 | logger.addHandler(handler) 136 | return logger, handler.lines 137 | --------------------------------------------------------------------------------