├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.rst ├── logentries ├── __init__.py ├── helpers.py ├── metrics.py └── utils.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | Logentries.egg-info 3 | dist/*.egg 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2013 Logentries 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | 3 | recursive-exclude * __pycache__ 4 | recursive-exclude * *.py[co] 5 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Logentries Logger 2 | ================= 3 | 4 | *This plug-in is no officially supported or maintained by Logentries.* 5 | 6 | *We currently have a volunteer maintainer for this project, but if you have major issues with the plug-in, please contact support@logentries.com.* 7 | 8 | 9 | This is a plugin library to enable logging to Logentries from the Python Logger. 10 | Additionally this plugin allows the user to get an overview of methods being executed, 11 | their execution time, as well as CPU and Memory statistics. 12 | Logentries is a real-time log management service on the cloud. 13 | More info at https://logentries.com. Note that this plugin is 14 | **asynchronous**. 15 | 16 | Setup 17 | ----- 18 | 19 | To use this library, you must first create an account on Logentries. 20 | This will only take a few moments. 21 | 22 | Install 23 | ------- 24 | 25 | To install this library, use the following command: 26 | 27 | ``pip install logentries`` 28 | 29 | Usage 30 | ----- 31 | 32 | .. code-block:: python 33 | 34 | #!/usr/bin/env python 35 | 36 | import logging 37 | from logentries import LogentriesHandler 38 | 39 | 40 | log = logging.getLogger('logentries') 41 | log.setLevel(logging.INFO) 42 | test = LogentriesHandler(LOGENTRIES_TOKEN) 43 | 44 | log.addHandler(test) 45 | 46 | log.warn("Warning message") 47 | log.info("Info message") 48 | 49 | sleep(10) 50 | 51 | 52 | Usage with metric functionality 53 | ------------------------------- 54 | 55 | .. code-block:: python 56 | 57 | import time 58 | import logging 59 | from logentries import LogentriesHandler, metrics 60 | 61 | 62 | TEST = metrics.Metric(LOGENTRIES_METRIC_TOKEN) 63 | 64 | @TEST.metric() 65 | def function_one(t): 66 | """A dummy function that takes some time.""" 67 | time.sleep(t) 68 | 69 | if __name__ == '__main__': 70 | function_one(1) 71 | 72 | 73 | Metric.Time() 74 | ------------- 75 | 76 | This decorator function is used to log the execution time of given function. In the above example ``@TEST.time()`` will wrap ``function_one`` and send log message containing the name and execution time of this function. 77 | 78 | 79 | 80 | Configure 81 | --------- 82 | 83 | The parameter ``LOGENTRIES_TOKEN`` needs to be filled in to point to a 84 | file in your Logentries account. 85 | 86 | The parameter ``LOGENTRIES_METRIC_TOKEN`` needs to be filled in to point to a metric collection file in your Logentries account. However, please note that metric data can be send to LOGENTRIES_TOKEN and merged with other standard logs. 87 | 88 | In your Logentries account, create a logfile, selecting ``Token TCP`` as 89 | the source\_type. This will print a Token UUID. This 90 | is the value to use for ``LOGENTRIES_TOKEN`` or ``LOGENTRIES_METRIC_TOKEN``. 91 | 92 | The appender will attempt to send your log data over TLS over port 443, 93 | otherwise it will send over port 80. 94 | 95 | You are now ready to start logging 96 | -------------------------------------------------------------------------------- /logentries/__init__.py: -------------------------------------------------------------------------------- 1 | from .utils import LogentriesHandler 2 | -------------------------------------------------------------------------------- /logentries/helpers.py: -------------------------------------------------------------------------------- 1 | 2 | """ This file contains some helpers methods in both Python2 and 3 """ 3 | import sys 4 | import re 5 | 6 | if sys.version < '3': 7 | # Python2.x imports 8 | import Queue 9 | import codecs 10 | else: 11 | # Python 3.x imports 12 | import queue 13 | 14 | 15 | def check_token(token): 16 | """ Checks if the given token is a valid UUID.""" 17 | valid = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-" 18 | r"[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") 19 | 20 | return valid.match(token) 21 | 22 | # We need to do some things different pending if its Python 2.x or 3.x 23 | if sys.version < '3': 24 | def to_unicode(ch): 25 | return codecs.unicode_escape_decode(ch)[0] 26 | 27 | def is_unicode(ch): 28 | return isinstance(ch, unicode) 29 | 30 | def create_unicode(ch): 31 | try: 32 | return unicode(ch, 'utf-8') 33 | except UnicodeDecodeError as e: 34 | return str(e) 35 | 36 | def create_queue(max_size): 37 | return Queue.Queue(max_size) 38 | else: 39 | def to_unicode(ch): 40 | return ch 41 | 42 | def is_unicode(ch): 43 | return isinstance(ch, str) 44 | 45 | def create_unicode(ch): 46 | return str(ch) 47 | 48 | def create_queue(max_size): 49 | return queue.Queue(max_size) 50 | -------------------------------------------------------------------------------- /logentries/metrics.py: -------------------------------------------------------------------------------- 1 | from logentries import LogentriesHandler 2 | from threading import Lock 3 | from functools import wraps 4 | import logging 5 | import time 6 | import sys 7 | import psutil 8 | 9 | glob_time = 0 10 | glob_name = 0 11 | 12 | log = logging.getLogger('logentries') 13 | log.setLevel(logging.INFO) 14 | 15 | class Metric(object): 16 | 17 | def __init__(self, token): 18 | self._count = 0.0 19 | self._sum = 0.0 20 | self._lock = Lock() 21 | self.token = token 22 | handler = LogentriesHandler(token) 23 | log.addHandler(handler) 24 | 25 | def observe(self, amount): 26 | with self._lock: 27 | self._count += 1 28 | self._sum += amount 29 | 30 | def metric(self): 31 | '''Mesaure function execution time in seconds 32 | and forward it to Logentries''' 33 | 34 | class Timer(object): 35 | 36 | def __init__(self, summary): 37 | self._summary = summary 38 | 39 | def __enter__(self): 40 | self._start = time.time() 41 | 42 | def __exit__(self, typ, value, traceback): 43 | global glob_time 44 | self._summary.observe(max(time.time() - self._start, 0)) 45 | glob_time = time.time()- self._start 46 | log.info("function_name=" + glob_name + " " + "execution_time=" + str(glob_time) + " " + "cpu=" + str(psutil.cpu_percent(interval=None)) + " " + "cpu_count=" + str(psutil.cpu_count())+ " " + "memory=" + str(psutil.virtual_memory()) ) 47 | 48 | def __call__(self, f): 49 | @wraps(f) 50 | def wrapped(*args, **kwargs): 51 | with self: 52 | global glob_name 53 | glob_name = f.__name__ 54 | 55 | return f(*args, **kwargs) 56 | return wrapped 57 | return Timer(self) 58 | -------------------------------------------------------------------------------- /logentries/utils.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # vim: set ts=4 sw=4 et: 3 | """ This file contains some utils for connecting to Logentries 4 | as well as storing logs in a queue and sending them.""" 5 | 6 | VERSION = '2.0.7' 7 | 8 | from logentries import helpers as le_helpers 9 | 10 | import logging 11 | import threading 12 | import socket 13 | import random 14 | import time 15 | import sys 16 | 17 | import certifi 18 | 19 | 20 | # Size of the internal event queue 21 | QUEUE_SIZE = 32768 22 | # Logentries API server address 23 | LE_API_DEFAULT = "logentries.commadotai.com" 24 | # Port number for token logging to Logentries API server 25 | LE_PORT_DEFAULT = 80 26 | LE_TLS_PORT_DEFAULT = 443 27 | # Minimal delay between attempts to reconnect in seconds 28 | MIN_DELAY = 0.1 29 | # Maximal delay between attempts to recconect in seconds 30 | MAX_DELAY = 10 31 | # Unicode Line separator character \u2028 32 | LINE_SEP = le_helpers.to_unicode('\u2028') 33 | 34 | 35 | # LE appender signature - used for debugging messages 36 | LE = "LE: " 37 | # Error message displayed when an incorrect Token has been detected 38 | INVALID_TOKEN = ("\n\nIt appears the LOGENTRIES_TOKEN " 39 | "parameter you entered is incorrect!\n\n") 40 | 41 | 42 | def dbg(msg): 43 | print(LE + msg) 44 | 45 | 46 | class PlainTextSocketAppender(threading.Thread): 47 | def __init__(self, verbose=True, le_api=LE_API_DEFAULT, le_port=LE_PORT_DEFAULT, le_tls_port=LE_TLS_PORT_DEFAULT): 48 | threading.Thread.__init__(self) 49 | 50 | # Logentries API server address 51 | self.le_api = le_api 52 | 53 | # Port number for token logging to Logentries API server 54 | self.le_port = le_port 55 | self.le_tls_port = le_tls_port 56 | 57 | self.daemon = True 58 | self.verbose = verbose 59 | self._conn = None 60 | self._queue = le_helpers.create_queue(QUEUE_SIZE) 61 | 62 | def empty(self): 63 | return self._queue.empty() 64 | 65 | def open_connection(self): 66 | self._conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 67 | self._conn.connect((self.le_api, self.le_port)) 68 | 69 | def reopen_connection(self): 70 | self.close_connection() 71 | 72 | root_delay = MIN_DELAY 73 | while True: 74 | try: 75 | self.open_connection() 76 | return 77 | except Exception: 78 | if self.verbose: 79 | dbg("Unable to connect to Logentries") 80 | 81 | root_delay *= 2 82 | if(root_delay > MAX_DELAY): 83 | root_delay = MAX_DELAY 84 | 85 | wait_for = root_delay + random.uniform(0, root_delay) 86 | 87 | try: 88 | time.sleep(wait_for) 89 | except KeyboardInterrupt: 90 | raise 91 | 92 | def close_connection(self): 93 | if self._conn is not None: 94 | self._conn.close() 95 | 96 | def run(self): 97 | try: 98 | # Open connection 99 | self.reopen_connection() 100 | 101 | # Send data in queue 102 | while True: 103 | # Take data from queue 104 | data = self._queue.get(block=True) 105 | 106 | # Replace newlines with Unicode line separator 107 | # for multi-line events 108 | if not le_helpers.is_unicode(data): 109 | multiline = le_helpers.create_unicode(data).replace( 110 | '\n', LINE_SEP) 111 | else: 112 | multiline = data.replace('\n', LINE_SEP) 113 | multiline += "\n" 114 | # Send data, reconnect if needed 115 | while True: 116 | try: 117 | self._conn.send(multiline.encode('utf-8')) 118 | except socket.error: 119 | self.reopen_connection() 120 | continue 121 | break 122 | except KeyboardInterrupt: 123 | if self.verbose: 124 | dbg("Logentries asynchronous socket client interrupted") 125 | 126 | self.close_connection() 127 | 128 | SocketAppender = PlainTextSocketAppender 129 | 130 | try: 131 | import ssl 132 | ssl_enabled = True 133 | except ImportError: # for systems without TLS support. 134 | ssl_enabled = False 135 | dbg("Unable to import ssl module. Will send over port 80.") 136 | else: 137 | class TLSSocketAppender(PlainTextSocketAppender): 138 | 139 | def open_connection(self): 140 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 141 | sock = ssl.wrap_socket( 142 | sock=sock, 143 | keyfile=None, 144 | certfile=None, 145 | server_side=False, 146 | cert_reqs=ssl.CERT_REQUIRED, 147 | ssl_version=getattr( 148 | ssl, 149 | 'PROTOCOL_TLSv1_2', 150 | ssl.PROTOCOL_TLSv1 151 | ), 152 | ca_certs=certifi.where(), 153 | do_handshake_on_connect=True, 154 | suppress_ragged_eofs=True, 155 | ) 156 | 157 | sock.connect((self.le_api, self.le_tls_port)) 158 | self._conn = sock 159 | 160 | 161 | class LogentriesHandler(logging.Handler): 162 | def __init__(self, token, use_tls=True, verbose=True, format=None, le_api=LE_API_DEFAULT, le_port=LE_PORT_DEFAULT, le_tls_port=LE_TLS_PORT_DEFAULT): 163 | logging.Handler.__init__(self) 164 | self.token = token 165 | self.good_config = True 166 | self.verbose = verbose 167 | # give the socket 10 seconds to flush, 168 | # otherwise drop logs 169 | self.timeout = 10 170 | if not le_helpers.check_token(token): 171 | if self.verbose: 172 | dbg(INVALID_TOKEN) 173 | self.good_config = False 174 | if format is None: 175 | format = logging.Formatter('%(asctime)s : %(levelname)s, %(message)s', 176 | '%a %b %d %H:%M:%S %Z %Y') 177 | self.setFormatter(format) 178 | self.setLevel(logging.DEBUG) 179 | if use_tls and ssl_enabled: 180 | self._thread = TLSSocketAppender(verbose=verbose, le_api=le_api, le_port=le_port, le_tls_port=le_tls_port) 181 | else: 182 | self._thread = SocketAppender(verbose=verbose, le_api=le_api, le_port=le_port, le_tls_port=le_tls_port) 183 | 184 | def flush(self): 185 | # wait for all queued logs to be send 186 | now = time.time() 187 | while not self._thread.empty(): 188 | time.sleep(0.2) 189 | if time.time() - now > self.timeout: 190 | break 191 | 192 | def emit_raw(self, msg): 193 | if self.good_config and not self._thread.is_alive(): 194 | try: 195 | self._thread.start() 196 | if self.verbose: 197 | dbg("Starting Logentries Asynchronous Socket Appender") 198 | except RuntimeError: # It's already started. 199 | pass 200 | 201 | msg = self.token + msg 202 | try: 203 | self._thread._queue.put_nowait(msg) 204 | except Exception: 205 | # Queue is full, try to remove the oldest message and put again 206 | try: 207 | self._thread._queue.get_nowait() 208 | self._thread._queue.put_nowait(msg) 209 | except Exception: 210 | # Race condition, no need for any action here 211 | pass 212 | 213 | def emit(self, record): 214 | msg = self.format(record).rstrip('\n') 215 | self.emit_raw(msg) 216 | 217 | def close(self): 218 | logging.Handler.close(self) 219 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | with open('README.rst', 'r') as f: 5 | long_description = f.read() 6 | 7 | 8 | setup( 9 | name='Logentries', 10 | version='0.8', 11 | author='Mark Lacomber', 12 | author_email='marklacomber@gmail.com', 13 | packages=['logentries'], 14 | scripts=[], 15 | url='http://pypi.python.org/pypi/Logentries/', 16 | license='LICENSE.txt', 17 | description='Python Logger plugin to send logs to Logentries', 18 | long_description=long_description, 19 | install_requires=[ 20 | "certifi", 21 | ], 22 | classifiers=[ 23 | 'Intended Audience :: Developers', 24 | 'License :: OSI Approved :: MIT License', 25 | 'Operating System :: OS Independent', 26 | 'Programming Language :: Python :: 2', 27 | 'Programming Language :: Python :: 3', 28 | ] 29 | ) 30 | --------------------------------------------------------------------------------