├── MANIFEST.in ├── setup.cfg ├── segment └── analytics │ ├── version.py │ ├── test │ ├── module.py │ ├── request.py │ ├── utils.py │ ├── __init__.py │ ├── consumer.py │ └── client.py │ ├── __init__.py │ ├── request.py │ ├── utils.py │ ├── consumer.py │ └── client.py ├── e2e_test.sh ├── .gitignore ├── .github └── dependabot.yml ├── RELEASING.md ├── Makefile ├── .buildscripts └── e2e.sh ├── setup.py ├── simulator.py ├── .circleci └── config.yml ├── HISTORY.md ├── analytics ├── consumer.py └── test │ └── client.py ├── README.md └── .pylintrc /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /segment/analytics/version.py: -------------------------------------------------------------------------------- 1 | VERSION = '2.2.0' 2 | -------------------------------------------------------------------------------- /e2e_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | python ./simulator.py "$@" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **sublime** 2 | *.pyc 3 | dist 4 | *.egg-info 5 | dist 6 | MANIFEST 7 | build 8 | .eggs 9 | *.bat 10 | .vscode/ 11 | .idea/ 12 | .python-version 13 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 10 8 | reviewers: 9 | - heitorsampaio 10 | -------------------------------------------------------------------------------- /RELEASING.md: -------------------------------------------------------------------------------- 1 | Releasing 2 | ========= 3 | 4 | 1. Update `VERSION` in `analytics/version.py` to the new version. 5 | 2. Update the `HISTORY.md` for the impending release. 6 | 3. `git commit -am "Release X.Y.Z."` (where X.Y.Z is the new version) 7 | 4. `git tag -a X.Y.Z -m "Version X.Y.Z"` (where X.Y.Z is the new version). 8 | 5. `make release`. 9 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | install: 2 | pip install --edit .[test] 3 | 4 | test: 5 | pylint --rcfile=.pylintrc --reports=y --exit-zero analytics | tee pylint.out 6 | flake8 --max-complexity=10 --statistics analytics > flake8.out || true 7 | 8 | release: 9 | python setup.py sdist bdist_wheel 10 | twine upload dist/* 11 | 12 | e2e_test: 13 | .buildscripts/e2e.sh 14 | 15 | .PHONY: test release e2e_test 16 | -------------------------------------------------------------------------------- /.buildscripts/e2e.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | if [ "${RUN_E2E_TESTS}" != "true" ]; then 6 | echo "Skipping end to end tests." 7 | else 8 | echo "Running end to end tests..." 9 | wget https://github.com/segmentio/library-e2e-tester/releases/download/0.2.2/tester_linux_amd64 10 | chmod +x tester_linux_amd64 11 | chmod +x e2e_test.sh 12 | ./tester_linux_amd64 -segment-write-key="${SEGMENT_WRITE_KEY}" -webhook-auth-username="${WEBHOOK_AUTH_USERNAME}" -webhook-bucket="${WEBHOOK_BUCKET}" -path='./e2e_test.sh' 13 | echo "End to end tests completed!" 14 | fi 15 | -------------------------------------------------------------------------------- /segment/analytics/test/module.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import analytics 4 | 5 | 6 | class TestModule(unittest.TestCase): 7 | 8 | # def failed(self): 9 | # self.failed = True 10 | 11 | def setUp(self): 12 | self.failed = False 13 | analytics.write_key = 'testsecret' 14 | analytics.on_error = self.failed 15 | 16 | def test_no_write_key(self): 17 | analytics.write_key = None 18 | self.assertRaises(Exception, analytics.track) 19 | 20 | def test_no_host(self): 21 | analytics.host = None 22 | self.assertRaises(Exception, analytics.track) 23 | 24 | def test_track(self): 25 | analytics.track('userId', 'python module event') 26 | analytics.flush() 27 | 28 | def test_identify(self): 29 | analytics.identify('userId', {'email': 'user@email.com'}) 30 | analytics.flush() 31 | 32 | def test_group(self): 33 | analytics.group('userId', 'groupId') 34 | analytics.flush() 35 | 36 | def test_alias(self): 37 | analytics.alias('previousId', 'userId') 38 | analytics.flush() 39 | 40 | def test_page(self): 41 | analytics.page('userId') 42 | analytics.flush() 43 | 44 | def test_screen(self): 45 | analytics.screen('userId') 46 | analytics.flush() 47 | 48 | def test_flush(self): 49 | analytics.flush() 50 | -------------------------------------------------------------------------------- /segment/analytics/test/request.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, date 2 | import unittest 3 | import json 4 | import requests 5 | 6 | from analytics.request import post, DatetimeSerializer 7 | 8 | 9 | class TestRequests(unittest.TestCase): 10 | 11 | def test_valid_request(self): 12 | res = post('testsecret', batch=[{ 13 | 'userId': 'userId', 14 | 'event': 'python event', 15 | 'type': 'track' 16 | }]) 17 | self.assertEqual(res.status_code, 200) 18 | 19 | def test_invalid_request_error(self): 20 | self.assertRaises(Exception, post, 'testsecret', 21 | 'https://api.segment.io', False, '[{]') 22 | 23 | def test_invalid_host(self): 24 | self.assertRaises(Exception, post, 'testsecret', 25 | 'api.segment.io/', batch=[]) 26 | 27 | def test_datetime_serialization(self): 28 | data = {'created': datetime(2012, 3, 4, 5, 6, 7, 891011)} 29 | result = json.dumps(data, cls=DatetimeSerializer) 30 | self.assertEqual(result, '{"created": "2012-03-04T05:06:07.891011"}') 31 | 32 | def test_date_serialization(self): 33 | today = date.today() 34 | data = {'created': today} 35 | result = json.dumps(data, cls=DatetimeSerializer) 36 | expected = '{"created": "%s"}' % today.isoformat() 37 | self.assertEqual(result, expected) 38 | 39 | def test_should_not_timeout(self): 40 | res = post('testsecret', batch=[{ 41 | 'userId': 'userId', 42 | 'event': 'python event', 43 | 'type': 'track' 44 | }], timeout=15) 45 | self.assertEqual(res.status_code, 200) 46 | 47 | def test_should_timeout(self): 48 | with self.assertRaises(requests.ReadTimeout): 49 | post('testsecret', batch=[{ 50 | 'userId': 'userId', 51 | 'event': 'python event', 52 | 'type': 'track' 53 | }], timeout=0.0001) 54 | 55 | def test_proxies(self): 56 | res = post('testsecret', batch=[{ 57 | 'userId': 'userId', 58 | 'event': 'python event', 59 | 'type': 'track', 60 | 'proxies': '203.243.63.16:80' 61 | }]) 62 | self.assertEqual(res.status_code, 200) 63 | -------------------------------------------------------------------------------- /segment/analytics/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from segment.analytics.version import VERSION 3 | from segment.analytics.client import Client 4 | 5 | __version__ = VERSION 6 | 7 | """Settings.""" 8 | write_key = Client.DefaultConfig.write_key 9 | host = Client.DefaultConfig.host 10 | on_error = Client.DefaultConfig.on_error 11 | debug = Client.DefaultConfig.debug 12 | send = Client.DefaultConfig.send 13 | sync_mode = Client.DefaultConfig.sync_mode 14 | max_queue_size = Client.DefaultConfig.max_queue_size 15 | gzip = Client.DefaultConfig.gzip 16 | timeout = Client.DefaultConfig.timeout 17 | max_retries = Client.DefaultConfig.max_retries 18 | 19 | default_client = None 20 | 21 | 22 | def track(*args, **kwargs): 23 | """Send a track call.""" 24 | _proxy('track', *args, **kwargs) 25 | 26 | 27 | def identify(*args, **kwargs): 28 | """Send a identify call.""" 29 | _proxy('identify', *args, **kwargs) 30 | 31 | 32 | def group(*args, **kwargs): 33 | """Send a group call.""" 34 | _proxy('group', *args, **kwargs) 35 | 36 | 37 | def alias(*args, **kwargs): 38 | """Send a alias call.""" 39 | _proxy('alias', *args, **kwargs) 40 | 41 | 42 | def page(*args, **kwargs): 43 | """Send a page call.""" 44 | _proxy('page', *args, **kwargs) 45 | 46 | 47 | def screen(*args, **kwargs): 48 | """Send a screen call.""" 49 | _proxy('screen', *args, **kwargs) 50 | 51 | 52 | def flush(): 53 | """Tell the client to flush.""" 54 | _proxy('flush') 55 | 56 | 57 | def join(): 58 | """Block program until the client clears the queue""" 59 | _proxy('join') 60 | 61 | 62 | def shutdown(): 63 | """Flush all messages and cleanly shutdown the client""" 64 | _proxy('flush') 65 | _proxy('join') 66 | 67 | 68 | def _proxy(method, *args, **kwargs): 69 | """Create an analytics client if one doesn't exist and send to it.""" 70 | global default_client 71 | if not default_client: 72 | default_client = Client(write_key, host=host, debug=debug, 73 | max_queue_size=max_queue_size, 74 | send=send, on_error=on_error, 75 | gzip=gzip, max_retries=max_retries, 76 | sync_mode=sync_mode, timeout=timeout) 77 | 78 | fn = getattr(default_client, method) 79 | fn(*args, **kwargs) 80 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | try: 5 | from setuptools import setup 6 | except ImportError: 7 | from distutils.core import setup 8 | 9 | # Don't import analytics-python module here, since deps may not be installed 10 | sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'segment')) 11 | sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'analytics')) 12 | from analytics.version import VERSION 13 | 14 | long_description = ''' 15 | Segment is the simplest way to integrate analytics into your application. 16 | One API allows you to turn on any other analytics service. No more learning 17 | new APIs, repeated code, and wasted development time. 18 | 19 | This is the official python client that wraps the Segment REST API (https://segment.com). 20 | 21 | Documentation and more details at https://github.com/segmentio/analytics-python 22 | ''' 23 | 24 | install_requires = [ 25 | "requests~=2.7", 26 | "monotonic~=1.5", 27 | "backoff~=1.10", 28 | "python-dateutil~=2.2" 29 | ] 30 | 31 | tests_require = [ 32 | "mock==2.0.0", 33 | "pylint==2.8.0", 34 | "flake8==3.7.9", 35 | ] 36 | 37 | setup( 38 | name='segment-analytics-python', 39 | version=VERSION, 40 | url='https://github.com/segmentio/analytics-python', 41 | author='Segment', 42 | author_email='friends@segment.com', 43 | maintainer='Segment', 44 | maintainer_email='friends@segment.com', 45 | test_suite='analytics.test.all', 46 | packages=['segment.analytics', 'analytics.test'], 47 | python_requires='>=3.6.0', 48 | license='MIT License', 49 | install_requires=install_requires, 50 | extras_require={ 51 | 'test': tests_require 52 | }, 53 | description='The hassle-free way to integrate analytics into any python application.', 54 | long_description=long_description, 55 | classifiers=[ 56 | "Development Status :: 5 - Production/Stable", 57 | "Intended Audience :: Developers", 58 | "License :: OSI Approved :: MIT License", 59 | "Operating System :: OS Independent", 60 | "Programming Language :: Python", 61 | "Programming Language :: Python :: 3.6", 62 | "Programming Language :: Python :: 3.7", 63 | "Programming Language :: Python :: 3.8", 64 | "Programming Language :: Python :: 3.9", 65 | ], 66 | ) 67 | -------------------------------------------------------------------------------- /segment/analytics/test/utils.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime, timedelta 2 | from decimal import Decimal 3 | import unittest 4 | 5 | from dateutil.tz import tzutc 6 | 7 | from analytics import utils 8 | 9 | 10 | class TestUtils(unittest.TestCase): 11 | 12 | def test_timezone_utils(self): 13 | now = datetime.now() 14 | utcnow = datetime.now(tz=tzutc()) 15 | self.assertTrue(utils.is_naive(now)) 16 | self.assertFalse(utils.is_naive(utcnow)) 17 | 18 | fixed = utils.guess_timezone(now) 19 | self.assertFalse(utils.is_naive(fixed)) 20 | 21 | shouldnt_be_edited = utils.guess_timezone(utcnow) 22 | self.assertEqual(utcnow, shouldnt_be_edited) 23 | 24 | def test_clean(self): 25 | simple = { 26 | 'decimal': Decimal('0.142857'), 27 | 'unicode': 'woo', 28 | 'date': datetime.now(), 29 | 'long': 200000000, 30 | 'integer': 1, 31 | 'float': 2.0, 32 | 'bool': True, 33 | 'str': 'woo', 34 | 'none': None 35 | } 36 | 37 | complicated = { 38 | 'exception': Exception('This should show up'), 39 | 'timedelta': timedelta(microseconds=20), 40 | 'list': [1, 2, 3] 41 | } 42 | 43 | combined = dict(simple.items()) 44 | combined.update(complicated.items()) 45 | 46 | pre_clean_keys = combined.keys() 47 | 48 | utils.clean(combined) 49 | self.assertEqual(combined.keys(), pre_clean_keys) 50 | 51 | def test_clean_with_dates(self): 52 | dict_with_dates = { 53 | 'birthdate': date(1980, 1, 1), 54 | 'registration': datetime.utcnow(), 55 | } 56 | self.assertEqual(dict_with_dates, utils.clean(dict_with_dates)) 57 | 58 | @classmethod 59 | def test_bytes(cls): 60 | item = bytes(10) 61 | utils.clean(item) 62 | 63 | def test_clean_fn(self): 64 | cleaned = utils.clean({'fn': lambda x: x, 'number': 4}) 65 | self.assertEqual(cleaned['number'], 4) 66 | if 'fn' in cleaned: 67 | self.assertEqual(cleaned['fn'], None) 68 | 69 | def test_remove_slash(self): 70 | self.assertEqual('http://segment.io', 71 | utils.remove_trailing_slash('http://segment.io/')) 72 | self.assertEqual('http://segment.io', 73 | utils.remove_trailing_slash('http://segment.io')) 74 | -------------------------------------------------------------------------------- /segment/analytics/request.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime 2 | from io import BytesIO 3 | from gzip import GzipFile 4 | import logging 5 | import json 6 | from dateutil.tz import tzutc 7 | from requests.auth import HTTPBasicAuth 8 | from requests import sessions 9 | 10 | from segment.analytics.version import VERSION 11 | from segment.analytics.utils import remove_trailing_slash 12 | 13 | _session = sessions.Session() 14 | 15 | 16 | def post(write_key, host=None, gzip=False, timeout=15, proxies=None, **kwargs): 17 | """Post the `kwargs` to the API""" 18 | log = logging.getLogger('segment') 19 | body = kwargs 20 | body["sentAt"] = datetime.utcnow().replace(tzinfo=tzutc()).isoformat() 21 | url = remove_trailing_slash(host or 'https://api.segment.io') + '/v1/batch' 22 | auth = HTTPBasicAuth(write_key, '') 23 | data = json.dumps(body, cls=DatetimeSerializer) 24 | log.debug('making request: %s', data) 25 | headers = { 26 | 'Content-Type': 'application/json', 27 | 'User-Agent': 'analytics-python/' + VERSION 28 | } 29 | if gzip: 30 | headers['Content-Encoding'] = 'gzip' 31 | buf = BytesIO() 32 | with GzipFile(fileobj=buf, mode='w') as gz: 33 | # 'data' was produced by json.dumps(), 34 | # whose default encoding is utf-8. 35 | gz.write(data.encode('utf-8')) 36 | data = buf.getvalue() 37 | 38 | kwargs = { 39 | "data": data, 40 | "auth": auth, 41 | "headers": headers, 42 | "timeout": 15, 43 | } 44 | 45 | if proxies: 46 | kwargs['proxies'] = proxies 47 | 48 | res = _session.post(url, data=data, auth=auth, 49 | headers=headers, timeout=timeout) 50 | 51 | if res.status_code == 200: 52 | log.debug('data uploaded successfully') 53 | return res 54 | 55 | try: 56 | payload = res.json() 57 | log.debug('received response: %s', payload) 58 | raise APIError(res.status_code, payload['code'], payload['message']) 59 | except ValueError: 60 | raise APIError(res.status_code, 'unknown', res.text) 61 | 62 | 63 | class APIError(Exception): 64 | 65 | def __init__(self, status, code, message): 66 | self.message = message 67 | self.status = status 68 | self.code = code 69 | 70 | def __str__(self): 71 | msg = "[Segment] {0}: {1} ({2})" 72 | return msg.format(self.code, self.message, self.status) 73 | 74 | 75 | class DatetimeSerializer(json.JSONEncoder): 76 | def default(self, obj): 77 | if isinstance(obj, (date, datetime)): 78 | return obj.isoformat() 79 | 80 | return json.JSONEncoder.default(self, obj) 81 | -------------------------------------------------------------------------------- /segment/analytics/utils.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | import logging 3 | import numbers 4 | 5 | from decimal import Decimal 6 | from datetime import date, datetime 7 | from dateutil.tz import tzlocal, tzutc 8 | 9 | log = logging.getLogger('segment') 10 | 11 | 12 | def is_naive(dt): 13 | """Determines if a given datetime.datetime is naive.""" 14 | return dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None 15 | 16 | 17 | def total_seconds(delta): 18 | """Determines total seconds with python < 2.7 compat.""" 19 | # http://stackoverflow.com/questions/3694835/python-2-6-5-divide-timedelta-with-timedelta 20 | return (delta.microseconds 21 | + (delta.seconds + delta.days * 24 * 3600) * 1e6) / 1e6 22 | 23 | 24 | def guess_timezone(dt): 25 | """Attempts to convert a naive datetime to an aware datetime.""" 26 | if is_naive(dt): 27 | # attempts to guess the datetime.datetime.now() local timezone 28 | # case, and then defaults to utc 29 | delta = datetime.now() - dt 30 | if total_seconds(delta) < 5: 31 | # this was created using datetime.datetime.now() 32 | # so we are in the local timezone 33 | return dt.replace(tzinfo=tzlocal()) 34 | # at this point, the best we can do is guess UTC 35 | return dt.replace(tzinfo=tzutc()) 36 | 37 | return dt 38 | 39 | 40 | def remove_trailing_slash(host): 41 | if host.endswith('/'): 42 | return host[:-1] 43 | return host 44 | 45 | 46 | def clean(item): 47 | if isinstance(item, Decimal): 48 | return float(item) 49 | elif isinstance(item, (str, bool, numbers.Number, datetime, 50 | date, type(None))): 51 | return item 52 | elif isinstance(item, (set, list, tuple)): 53 | return _clean_list(item) 54 | elif isinstance(item, dict): 55 | return _clean_dict(item) 56 | elif isinstance(item, Enum): 57 | return clean(item.value) 58 | else: 59 | return _coerce_unicode(item) 60 | 61 | 62 | def _clean_list(list_): 63 | return [clean(item) for item in list_] 64 | 65 | 66 | def _clean_dict(dict_): 67 | data = {} 68 | for k, v in dict_.items(): 69 | try: 70 | data[k] = clean(v) 71 | except TypeError: 72 | log.warning( 73 | 'Dictionary values must be serializeable to ' 74 | 'JSON "%s" value %s of type %s is unsupported.', 75 | k, v, type(v), 76 | ) 77 | return data 78 | 79 | 80 | def _coerce_unicode(cmplx): 81 | try: 82 | item = cmplx.decode("utf-8", "strict") 83 | except AttributeError as exception: 84 | item = ":".join(exception) 85 | item.decode("utf-8", "strict") 86 | log.warning('Error decoding: %s', item) 87 | return None 88 | return item 89 | -------------------------------------------------------------------------------- /segment/analytics/test/__init__.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import pkgutil 3 | import logging 4 | import sys 5 | import analytics 6 | 7 | from analytics.client import Client 8 | 9 | 10 | def all_names(): 11 | for _, modname, _ in pkgutil.iter_modules(__path__): 12 | yield 'analytics.test.' + modname 13 | 14 | 15 | def all(): 16 | logging.basicConfig(stream=sys.stderr) 17 | return unittest.defaultTestLoader.loadTestsFromNames(all_names()) 18 | 19 | 20 | class TestInit(unittest.TestCase): 21 | def test_writeKey(self): 22 | self.assertIsNone(analytics.default_client) 23 | analytics.flush() 24 | self.assertEqual(analytics.default_client.write_key, 'test-init') 25 | 26 | def test_debug(self): 27 | self.assertIsNone(analytics.default_client) 28 | analytics.debug = True 29 | analytics.flush() 30 | self.assertTrue(analytics.default_client.debug) 31 | analytics.default_client = None 32 | analytics.debug = False 33 | analytics.flush() 34 | self.assertFalse(analytics.default_client.debug) 35 | 36 | def test_gzip(self): 37 | self.assertIsNone(analytics.default_client) 38 | analytics.gzip = True 39 | analytics.flush() 40 | self.assertTrue(analytics.default_client.gzip) 41 | analytics.default_client = None 42 | analytics.gzip = False 43 | analytics.flush() 44 | self.assertFalse(analytics.default_client.gzip) 45 | 46 | def test_host(self): 47 | self.assertIsNone(analytics.default_client) 48 | analytics.host = 'test-host' 49 | analytics.flush() 50 | self.assertEqual(analytics.default_client.host, 'test-host') 51 | 52 | def test_max_queue_size(self): 53 | self.assertIsNone(analytics.default_client) 54 | analytics.max_queue_size = 1337 55 | analytics.flush() 56 | self.assertEqual(analytics.default_client.queue.maxsize, 1337) 57 | 58 | def test_max_retries(self): 59 | self.assertIsNone(analytics.default_client) 60 | client = Client('testsecret', max_retries=42) 61 | for consumer in client.consumers: 62 | self.assertEqual(consumer.retries, 42) 63 | 64 | def test_sync_mode(self): 65 | self.assertIsNone(analytics.default_client) 66 | analytics.sync_mode = True 67 | analytics.flush() 68 | self.assertTrue(analytics.default_client.sync_mode) 69 | analytics.default_client = None 70 | analytics.sync_mode = False 71 | analytics.flush() 72 | self.assertFalse(analytics.default_client.sync_mode) 73 | 74 | def test_timeout(self): 75 | self.assertIsNone(analytics.default_client) 76 | analytics.timeout = 1.234 77 | analytics.flush() 78 | self.assertEqual(analytics.default_client.timeout, 1.234) 79 | 80 | def setUp(self): 81 | analytics.write_key = 'test-init' 82 | analytics.default_client = None 83 | -------------------------------------------------------------------------------- /simulator.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import argparse 3 | import json 4 | import segment.analytics as analytics 5 | 6 | __name__ = 'simulator.py' 7 | __version__ = '0.0.1' 8 | __description__ = 'scripting simulator' 9 | 10 | 11 | def json_hash(str): 12 | if str: 13 | return json.loads(str) 14 | 15 | # analytics -method= -segment-write-key= [options] 16 | 17 | 18 | parser = argparse.ArgumentParser(description='send a segment message') 19 | 20 | parser.add_argument('--writeKey', help='the Segment writeKey') 21 | parser.add_argument('--type', help='The Segment message type') 22 | 23 | parser.add_argument('--userId', help='the user id to send the event as') 24 | parser.add_argument( 25 | '--anonymousId', help='the anonymous user id to send the event as') 26 | parser.add_argument( 27 | '--context', help='additional context for the event (JSON-encoded)') 28 | 29 | parser.add_argument('--event', help='the event name to send with the event') 30 | parser.add_argument( 31 | '--properties', help='the event properties to send (JSON-encoded)') 32 | 33 | parser.add_argument( 34 | '--name', help='name of the screen or page to send with the message') 35 | 36 | parser.add_argument( 37 | '--traits', help='the identify/group traits to send (JSON-encoded)') 38 | 39 | parser.add_argument('--groupId', help='the group id') 40 | 41 | options = parser.parse_args() 42 | 43 | 44 | def failed(status, msg): 45 | raise Exception(msg) 46 | 47 | 48 | def track(): 49 | analytics.track(options.userId, options.event, anonymous_id=options.anonymousId, 50 | properties=json_hash(options.properties), context=json_hash(options.context)) 51 | 52 | 53 | def page(): 54 | analytics.page(options.userId, name=options.name, anonymous_id=options.anonymousId, 55 | properties=json_hash(options.properties), context=json_hash(options.context)) 56 | 57 | 58 | def screen(): 59 | analytics.screen(options.userId, name=options.name, anonymous_id=options.anonymousId, 60 | properties=json_hash(options.properties), context=json_hash(options.context)) 61 | 62 | 63 | def identify(): 64 | analytics.identify(options.userId, anonymous_id=options.anonymousId, 65 | traits=json_hash(options.traits), context=json_hash(options.context)) 66 | 67 | 68 | def group(): 69 | analytics.group(options.userId, options.groupId, json_hash(options.traits), 70 | json_hash(options.context), anonymous_id=options.anonymousId) 71 | 72 | 73 | def unknown(): 74 | print() 75 | 76 | 77 | analytics.write_key = options.writeKey 78 | analytics.on_error = failed 79 | analytics.debug = True 80 | 81 | log = logging.getLogger('segment') 82 | ch = logging.StreamHandler() 83 | ch.setLevel(logging.DEBUG) 84 | log.addHandler(ch) 85 | 86 | switcher = { 87 | "track": track, 88 | "page": page, 89 | "screen": screen, 90 | "identify": identify, 91 | "group": group 92 | } 93 | 94 | func = switcher.get(options.type) 95 | if func: 96 | func() 97 | analytics.shutdown() 98 | else: 99 | print("Invalid Message Type " + options.type) 100 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | defaults: 3 | taggedReleasesFilter: &taggedReleasesFilter 4 | tags: 5 | only: /^\d+\.\d+\.\d+((a|b|rc)\d)?$/ # matches 1.2.3, 1.2.3a1, 1.2.3b1, 1.2.3rc1 etc.. 6 | jobs: 7 | build: 8 | docker: 9 | - image: circleci/python:3.8 10 | steps: 11 | - checkout 12 | - run: pip3 install python-dateutil backoff monotonic 13 | - run: pip3 install --user . 14 | - run: sudo pip3 install pylint==2.8.0 flake8 mock==3.0.5 python-dateutil 15 | - run: make test 16 | - store_artifacts: 17 | path: pylint.out 18 | - store_artifacts: 19 | path: flake8.out 20 | 21 | snyk: 22 | docker: 23 | - image: circleci/python:3.9 24 | steps: 25 | - checkout 26 | - attach_workspace: { at: . } 27 | - run: pip3 install dephell 28 | - run: pip3 install --user appdirs 29 | - run: dephell deps convert --from=setup.py --to=requirements.txt 30 | - run: pip3 install --user -r requirements.txt 31 | - run: curl -sL https://raw.githubusercontent.com/segmentio/snyk_helpers/master/initialization/snyk.sh | sh 32 | 33 | test_36: &test 34 | docker: 35 | - image: circleci/python:3.6 36 | steps: 37 | - checkout 38 | - run: pip3 install python-dateutil backoff monotonic 39 | - run: pip3 install --user .[test] 40 | - run: 41 | name: Linting with Flake8 42 | command: | 43 | git diff origin/master..HEAD analytics | flake8 --diff --max-complexity=10 analytics 44 | - run: make test 45 | - run: make e2e_test 46 | 47 | test_37: 48 | <<: *test 49 | docker: 50 | - image: circleci/python:3.7 51 | 52 | test_38: 53 | <<: *test 54 | docker: 55 | - image: circleci/python:3.8 56 | 57 | test_39: 58 | <<: *test 59 | docker: 60 | - image: circleci/python:3.9 61 | 62 | publish: 63 | docker: 64 | - image: circleci/python:3.9 65 | steps: 66 | - checkout 67 | - run: sudo pip install twine 68 | - run: make release 69 | 70 | workflows: 71 | version: 2 72 | build_test_release: 73 | jobs: 74 | - build: 75 | filters: 76 | <<: *taggedReleasesFilter 77 | - test_36: 78 | filters: 79 | <<: *taggedReleasesFilter 80 | - test_37: 81 | filters: 82 | <<: *taggedReleasesFilter 83 | - test_38: 84 | filters: 85 | <<: *taggedReleasesFilter 86 | - test_39: 87 | filters: 88 | <<: *taggedReleasesFilter 89 | - publish: 90 | requires: 91 | - build 92 | - test_36 93 | - test_37 94 | - test_38 95 | - test_39 96 | filters: 97 | <<: *taggedReleasesFilter 98 | branches: 99 | ignore: /.*/ 100 | static_analysis: 101 | jobs: 102 | - build 103 | - snyk: 104 | context: snyk 105 | requires: 106 | - build 107 | scheduled_e2e_test: 108 | triggers: 109 | - schedule: 110 | cron: "0 * * * *" 111 | filters: 112 | branches: 113 | only: 114 | - master 115 | - scheduled_e2e_testing 116 | jobs: 117 | - test_36 118 | - test_37 119 | - test_38 120 | - test_39 121 | -------------------------------------------------------------------------------- /HISTORY.md: -------------------------------------------------------------------------------- 1 | # 2.2.0 / 2022-03-07 2 | - Remove Python 2 support 3 | - Remove six package 4 | 5 | # 2.1.0 / 2022-03-04 6 | 7 | - Raise exception on large message 8 | - Automatically coerce Enum values inside messages 9 | - Handle exceptions in the try catch and log them 10 | 11 | 12 | # 2.0.0 / 2021-10-01 13 | 14 | - Update package name and namespace name 15 | 16 | 17 | # 1.5.0 / 2021-09-23 18 | - Update tests with latest dependencies 19 | - Remove unsupported python versions 2.7 & 3.5 20 | 21 | # 1.4.0 / 2021-07-16 22 | - Fix the missing `upload_size` parameter 23 | 24 | # 1.3.1 / 2021-05-12 25 | 26 | - Fix linting code and readme heling basic things. 27 | - Add support for HTTP proxy 28 | - Allows more settings to be configured from singleton 29 | 30 | # 1.3.0-beta1 / 2019-04-27 31 | 32 | - Add `sync_mode` option ([#147](https://github.com/segmentio/analytics-python/pull/147)) 33 | 34 | # 1.3.0-beta0 / 2018-10-10 35 | 36 | - Add User-Agent header to messages 37 | - Don't retry sending on client errors except 429 38 | - Allow user-defined upload interval 39 | - Add `shutdown` function 40 | - Add gzip support 41 | - Add exponential backoff with jitter when retrying 42 | - Add a paramater in Client to configure max retries 43 | - Limit batch upload size to 500KB 44 | - Drop messages greater than 32kb 45 | - Allow user-defined upload size 46 | - Support custom messageId 47 | 48 | # 1.2.9 / 2017-11-28 49 | 50 | - [Fix](https://github.com/segmentio/analytics-python/pull/102): Stringify non-string userIds and anonymousIds. 51 | 52 | # 1.2.8 / 2017-09-20 53 | 54 | - [Fix](https://github.com/segmentio/analytics-python/issues/94): Date objects are removed from event properties. 55 | - [Fix](https://github.com/segmentio/analytics-python/pull/98): Fix for regression introduced in version 1.2.4. 56 | 57 | # 1.2.7 / 2017-01-31 58 | 59 | - [Fix](https://github.com/segmentio/analytics-python/pull/92): Correctly serialize date objects. 60 | 61 | # 1.2.6 / 2016-12-07 62 | 63 | - dont add messages to the queue if send is false 64 | - drop py32 support 65 | 66 | # 1.2.5 / 2016-07-02 67 | 68 | - Fix outdated python-dateutil<2 requirement for python2 - dateutil > 2.1 runs is python2 compatible 69 | - Fix a bug introduced in 1.2.4 where we could try to join a thread that was not yet started 70 | 71 | # 1.2.4 / 2016-06-06 72 | 73 | - Fix race conditions in overflow and flush tests 74 | - Join daemon thread on interpreter exit to prevent value errors 75 | - Capitalize HISTORY.md (#76) 76 | - Quick fix for Decimal to send as a float 77 | 78 | # 1.2.3 / 2016-03-23 79 | 80 | - relaxing requests dep 81 | 82 | # 1.2.2 / 2016-03-17 83 | 84 | - Fix environment markers definition 85 | - Use proper way for defining conditional dependencies 86 | 87 | # 1.2.1 / 2016-03-11 88 | 89 | - fixing requirements.txt 90 | 91 | # 1.2.0 / 2016-03-11 92 | 93 | - adding versioned requirements.txt file 94 | 95 | # 1.1.0 / 2015-06-23 96 | 97 | - Adding fixes for handling invalid json types 98 | - Fixing byte/bytearray handling 99 | - Adding `logging.DEBUG` fix for `setLevel` 100 | - Support HTTP keep-alive using a Session connection pool 101 | - Suppport universal wheels 102 | - adding .sentAt 103 | - make it really testable 104 | - fixing overflow test 105 | - removing .io's 106 | - Update README.md 107 | - spacing 108 | 109 | # 1.0.3 / 2014-09-30 110 | 111 | - adding top level send option 112 | 113 | # 1.0.2 / 2014-09-17 114 | 115 | - fixing debug logging levels 116 | 117 | # 1.0.1 / 2014-09-08 118 | 119 | - fixing unicode handling, for write_key and events 120 | - adding six to requirements.txt and install scripts 121 | 122 | # 1.0.0 / 2014-09-05 123 | 124 | - updating to spec 1.0 125 | - adding python3 support 126 | - moving to analytics.write_key API 127 | - moving consumer to a separate thread 128 | - adding request retries 129 | - making analytics.flush() syncrhonous 130 | - adding full travis tests 131 | 132 | # 0.4.4 / 2013-11-21 133 | 134 | - add < python 2.7 compatibility by removing `delta.total_seconds` 135 | 136 | # 0.4.3 / 2013-11-13 137 | 138 | - added datetime serialization fix (alexlouden) 139 | 140 | # 0.4.2 / 2013-06-26 141 | 142 | - Added history.d change log 143 | - Merging https://github.com/segmentio/analytics-python/pull/14 to add support for lists and PEP8 fixes. Thanks https://github.com/dfee! 144 | - Fixing #12, adding static public API to analytics.**init** 145 | -------------------------------------------------------------------------------- /analytics/consumer.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from threading import Thread 3 | import json 4 | import monotonic 5 | import backoff 6 | 7 | 8 | from analytics.request import post, APIError, DatetimeSerializer 9 | 10 | from queue import Empty 11 | 12 | MAX_MSG_SIZE = 32 << 10 13 | 14 | # Our servers only accept batches less than 500KB. Here limit is set slightly 15 | # lower to leave space for extra data that will be added later, eg. "sentAt". 16 | BATCH_SIZE_LIMIT = 475000 17 | 18 | 19 | class Consumer(Thread): 20 | """Consumes the messages from the client's queue.""" 21 | log = logging.getLogger('segment') 22 | 23 | def __init__(self, queue, write_key, upload_size=100, host=None, 24 | on_error=None, upload_interval=0.5, gzip=False, retries=10, 25 | timeout=15, proxies=None): 26 | """Create a consumer thread.""" 27 | Thread.__init__(self) 28 | # Make consumer a daemon thread so that it doesn't block program exit 29 | self.daemon = True 30 | self.upload_size = upload_size 31 | self.upload_interval = upload_interval 32 | self.write_key = write_key 33 | self.host = host 34 | self.on_error = on_error 35 | self.queue = queue 36 | self.gzip = gzip 37 | # It's important to set running in the constructor: if we are asked to 38 | # pause immediately after construction, we might set running to True in 39 | # run() *after* we set it to False in pause... and keep running 40 | # forever. 41 | self.running = True 42 | self.retries = retries 43 | self.timeout = timeout 44 | self.proxies = proxies 45 | 46 | def run(self): 47 | """Runs the consumer.""" 48 | self.log.debug('consumer is running...') 49 | while self.running: 50 | self.upload() 51 | 52 | self.log.debug('consumer exited.') 53 | 54 | def pause(self): 55 | """Pause the consumer.""" 56 | self.running = False 57 | 58 | def upload(self): 59 | """Upload the next batch of items, return whether successful.""" 60 | success = False 61 | batch = self.next() 62 | if len(batch) == 0: 63 | return False 64 | 65 | try: 66 | self.request(batch) 67 | success = True 68 | except Exception as e: 69 | self.log.error('error uploading: %s', e) 70 | success = False 71 | if self.on_error: 72 | self.on_error(e, batch) 73 | finally: 74 | # mark items as acknowledged from queue 75 | for _ in batch: 76 | self.queue.task_done() 77 | return success 78 | 79 | def next(self): 80 | """Return the next batch of items to upload.""" 81 | queue = self.queue 82 | items = [] 83 | 84 | start_time = monotonic.monotonic() 85 | total_size = 0 86 | 87 | while len(items) < self.upload_size: 88 | elapsed = monotonic.monotonic() - start_time 89 | if elapsed >= self.upload_interval: 90 | break 91 | try: 92 | item = queue.get( 93 | block=True, timeout=self.upload_interval - elapsed) 94 | item_size = len(json.dumps( 95 | item, cls=DatetimeSerializer).encode()) 96 | if item_size > MAX_MSG_SIZE: 97 | self.log.error( 98 | 'Item exceeds 32kb limit, dropping. (%s)', str(item)) 99 | continue 100 | items.append(item) 101 | total_size += item_size 102 | if total_size >= BATCH_SIZE_LIMIT: 103 | self.log.debug( 104 | 'hit batch size limit (size: %d)', total_size) 105 | break 106 | except Empty: 107 | break 108 | 109 | return items 110 | 111 | def request(self, batch): 112 | """Attempt to upload the batch and retry before raising an error """ 113 | 114 | def fatal_exception(exc): 115 | if isinstance(exc, APIError): 116 | # retry on server errors and client errors 117 | # with 429 status code (rate limited), 118 | # don't retry on other client errors 119 | return (400 <= exc.status < 500) and exc.status != 429 120 | else: 121 | # retry on all other errors (eg. network) 122 | return False 123 | 124 | @backoff.on_exception( 125 | backoff.expo, 126 | Exception, 127 | max_tries=self.retries + 1, 128 | giveup=fatal_exception) 129 | def send_request(): 130 | post(self.write_key, self.host, gzip=self.gzip, 131 | timeout=self.timeout, batch=batch, proxies=self.proxies) 132 | 133 | send_request() 134 | -------------------------------------------------------------------------------- /segment/analytics/consumer.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from threading import Thread 3 | import monotonic 4 | import backoff 5 | import json 6 | 7 | from segment.analytics.request import post, APIError, DatetimeSerializer 8 | 9 | from queue import Empty 10 | 11 | MAX_MSG_SIZE = 32 << 10 12 | 13 | # Our servers only accept batches less than 500KB. Here limit is set slightly 14 | # lower to leave space for extra data that will be added later, eg. "sentAt". 15 | BATCH_SIZE_LIMIT = 475000 16 | 17 | 18 | class Consumer(Thread): 19 | """Consumes the messages from the client's queue.""" 20 | log = logging.getLogger('segment') 21 | 22 | def __init__(self, queue, write_key, upload_size=100, host=None, 23 | on_error=None, upload_interval=0.5, gzip=False, retries=10, 24 | timeout=15, proxies=None): 25 | """Create a consumer thread.""" 26 | Thread.__init__(self) 27 | # Make consumer a daemon thread so that it doesn't block program exit 28 | self.daemon = True 29 | self.upload_size = upload_size 30 | self.upload_interval = upload_interval 31 | self.write_key = write_key 32 | self.host = host 33 | self.on_error = on_error 34 | self.queue = queue 35 | self.gzip = gzip 36 | # It's important to set running in the constructor: if we are asked to 37 | # pause immediately after construction, we might set running to True in 38 | # run() *after* we set it to False in pause... and keep running 39 | # forever. 40 | self.running = True 41 | self.retries = retries 42 | self.timeout = timeout 43 | self.proxies = proxies 44 | 45 | def run(self): 46 | """Runs the consumer.""" 47 | self.log.debug('consumer is running...') 48 | while self.running: 49 | self.upload() 50 | 51 | self.log.debug('consumer exited.') 52 | 53 | def pause(self): 54 | """Pause the consumer.""" 55 | self.running = False 56 | 57 | def upload(self): 58 | """Upload the next batch of items, return whether successful.""" 59 | success = False 60 | batch = self.next() 61 | if len(batch) == 0: 62 | return False 63 | 64 | try: 65 | self.request(batch) 66 | success = True 67 | except Exception as e: 68 | self.log.error('error uploading: %s', e) 69 | success = False 70 | if self.on_error: 71 | self.on_error(e, batch) 72 | finally: 73 | # mark items as acknowledged from queue 74 | for _ in batch: 75 | self.queue.task_done() 76 | return success 77 | 78 | def next(self): 79 | """Return the next batch of items to upload.""" 80 | queue = self.queue 81 | items = [] 82 | 83 | start_time = monotonic.monotonic() 84 | total_size = 0 85 | 86 | while len(items) < self.upload_size: 87 | elapsed = monotonic.monotonic() - start_time 88 | if elapsed >= self.upload_interval: 89 | break 90 | try: 91 | item = queue.get( 92 | block=True, timeout=self.upload_interval - elapsed) 93 | item_size = len(json.dumps( 94 | item, cls=DatetimeSerializer).encode()) 95 | if item_size > MAX_MSG_SIZE: 96 | self.log.error( 97 | 'Item exceeds 32kb limit, dropping. (%s)', str(item)) 98 | continue 99 | items.append(item) 100 | total_size += item_size 101 | if total_size >= BATCH_SIZE_LIMIT: 102 | self.log.debug( 103 | 'hit batch size limit (size: %d)', total_size) 104 | break 105 | except Exception as e: 106 | self.log.error('Exception: %s', e) 107 | except Empty: 108 | break 109 | 110 | return items 111 | 112 | def request(self, batch): 113 | """Attempt to upload the batch and retry before raising an error """ 114 | 115 | def fatal_exception(exc): 116 | if isinstance(exc, APIError): 117 | # retry on server errors and client errors 118 | # with 429 status code (rate limited), 119 | # don't retry on other client errors 120 | return (400 <= exc.status < 500) and exc.status != 429 121 | else: 122 | # retry on all other errors (eg. network) 123 | return False 124 | 125 | @backoff.on_exception( 126 | backoff.expo, 127 | Exception, 128 | max_tries=self.retries + 1, 129 | giveup=fatal_exception) 130 | def send_request(): 131 | post(self.write_key, self.host, gzip=self.gzip, 132 | timeout=self.timeout, batch=batch, proxies=self.proxies) 133 | 134 | send_request() 135 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | analytics-python 2 | ============== 3 | 4 | [![CircleCI](https://circleci.com/gh/segmentio/analytics-python/tree/master.svg?style=svg&circle-token=c0b411a3e21943918294714ad1d75a1cfc718f79)](https://circleci.com/gh/segmentio/analytics-python/tree/master) 5 | 6 | 7 | analytics-python is a python client for [Segment](https://segment.com) 8 | 9 |
10 | 11 |

You can't fix what you can't measure

12 |
13 | 14 | Analytics helps you measure your users, product, and business. It unlocks insights into your app's funnel, core business metrics, and whether you have a product-market fit. 15 | 16 | ## 🚀 How to get started 17 | 1. **Collect analytics data** from your app(s). 18 | - The top 200 Segment companies collect data from 5+ source types (web, mobile, server, CRM, etc.). 19 | 2. **Send the data to analytics tools** (for example, Google Analytics, Amplitude, Mixpanel). 20 | - Over 250+ Segment companies send data to eight categories of destinations such as analytics tools, warehouses, email marketing, and remarketing systems, session recording, and more. 21 | 3. **Explore your data** by creating metrics (for example, new signups, retention cohorts, and revenue generation). 22 | - The best Segment companies use retention cohorts to measure product-market fit. Netflix has 70% paid retention after 12 months, 30% after 7 years. 23 | 24 | [Segment](https://segment.com) collects analytics data and allows you to send it to more than 250 apps (such as Google Analytics, Mixpanel, Optimizely, Facebook Ads, Slack, Sentry) just by flipping a switch. You only need one Segment code snippet, and you can turn integrations on and off at will, with no additional code. [Sign up with Segment today](https://app.segment.com/signup). 25 | 26 | ### 🤔 Why? 27 | 1. **Power all your analytics apps with the same data**. Instead of writing code to integrate all of your tools individually, send data to Segment, once. 28 | 29 | 2. **Install tracking for the last time**. We're the last integration you'll ever need to write. You only need to instrument Segment once. Reduce all of your tracking code and advertising tags into a single set of API calls. 30 | 31 | 3. **Send data from anywhere**. Send Segment data from any device, and we'll transform and send it on to any tool. 32 | 33 | 4. **Query your data in SQL**. Slice, dice, and analyze your data in detail with Segment SQL. We'll transform and load your customer behavioral data directly from your apps into Amazon Redshift, Google BigQuery, or Postgres. Save weeks of engineering time by not having to invent your data warehouse and ETL pipeline. 34 | 35 | For example, you can capture data on any app: 36 | ```python 37 | analytics.track('Order Completed', { price: 99.84 }) 38 | ``` 39 | Then, query the resulting data in SQL: 40 | ```sql 41 | select * from app.order_completed 42 | order by price desc 43 | ``` 44 | 45 | ## 👨‍💻 Getting Started 46 | 47 | Install `analytics-python` using pip: 48 | 49 | ```bash 50 | pip3 install segment-analytics-python 51 | ``` 52 | 53 | or you can clone this repo: 54 | ```bash 55 | git clone https://github.com/segmentio/analytics-python.git 56 | 57 | cd analytics-python 58 | 59 | sudo python3 setup.py install 60 | ``` 61 | 62 | Now inside your app, you'll want to **set your** `write_key` before making any analytics calls: 63 | 64 | ```python 65 | import analytics 66 | 67 | analytics.write_key = 'YOUR_WRITE_KEY' 68 | ``` 69 | **Note** If you need to send data to multiple Segment sources, you can initialize a new Client for each `write_key` 70 | 71 | ### 🚀 Startup Program 72 |
73 | 74 |
75 | If you are part of a new startup (<$5M raised, <2 years since founding), we just launched a new startup program for you. You can get a Segment Team plan (up to $25,000 value in Segment credits) for free up to 2 years — apply here! 76 | 77 | ## Documentation 78 | 79 | Documentation is available at [https://segment.com/libraries/python](https://segment.com/libraries/python). 80 | 81 | ## License 82 | 83 | ``` 84 | WWWWWW||WWWWWW 85 | W W W||W W W 86 | || 87 | ( OO )__________ 88 | / | \ 89 | /o o| MIT \ 90 | \___/||_||__||_|| * 91 | || || || || 92 | _||_|| _||_|| 93 | (__|__|(__|__| 94 | ``` 95 | 96 | (The MIT License) 97 | 98 | Copyright (c) 2013 Segment Inc. 99 | 100 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 101 | 102 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 103 | 104 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 105 | 106 | [![forthebadge](https://forthebadge.com/images/badges/built-with-love.svg)](https://forthebadge.com) 107 | -------------------------------------------------------------------------------- /segment/analytics/test/consumer.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import mock 3 | import time 4 | import json 5 | 6 | try: 7 | from queue import Queue 8 | except ImportError: 9 | from Queue import Queue 10 | 11 | from analytics.consumer import Consumer, MAX_MSG_SIZE 12 | from analytics.request import APIError 13 | 14 | 15 | class TestConsumer(unittest.TestCase): 16 | 17 | def test_next(self): 18 | q = Queue() 19 | consumer = Consumer(q, '') 20 | q.put(1) 21 | next = consumer.next() 22 | self.assertEqual(next, [1]) 23 | 24 | def test_next_limit(self): 25 | q = Queue() 26 | upload_size = 50 27 | consumer = Consumer(q, '', upload_size) 28 | for i in range(10000): 29 | q.put(i) 30 | next = consumer.next() 31 | self.assertEqual(next, list(range(upload_size))) 32 | 33 | def test_dropping_oversize_msg(self): 34 | q = Queue() 35 | consumer = Consumer(q, '') 36 | oversize_msg = {'m': 'x' * MAX_MSG_SIZE} 37 | q.put(oversize_msg) 38 | next = consumer.next() 39 | self.assertEqual(next, []) 40 | self.assertTrue(q.empty()) 41 | 42 | def test_upload(self): 43 | q = Queue() 44 | consumer = Consumer(q, 'testsecret') 45 | track = { 46 | 'type': 'track', 47 | 'event': 'python event', 48 | 'userId': 'userId' 49 | } 50 | q.put(track) 51 | success = consumer.upload() 52 | self.assertTrue(success) 53 | 54 | def test_upload_interval(self): 55 | # Put _n_ items in the queue, pausing a little bit more than 56 | # _upload_interval_ after each one. 57 | # The consumer should upload _n_ times. 58 | q = Queue() 59 | upload_interval = 0.3 60 | consumer = Consumer(q, 'testsecret', upload_size=10, 61 | upload_interval=upload_interval) 62 | with mock.patch('analytics.consumer.post') as mock_post: 63 | consumer.start() 64 | for i in range(0, 3): 65 | track = { 66 | 'type': 'track', 67 | 'event': 'python event %d' % i, 68 | 'userId': 'userId' 69 | } 70 | q.put(track) 71 | time.sleep(upload_interval * 1.1) 72 | self.assertEqual(mock_post.call_count, 3) 73 | 74 | def test_multiple_uploads_per_interval(self): 75 | # Put _upload_size*2_ items in the queue at once, then pause for 76 | # _upload_interval_. The consumer should upload 2 times. 77 | q = Queue() 78 | upload_interval = 0.5 79 | upload_size = 10 80 | consumer = Consumer(q, 'testsecret', upload_size=upload_size, 81 | upload_interval=upload_interval) 82 | with mock.patch('analytics.consumer.post') as mock_post: 83 | consumer.start() 84 | for i in range(0, upload_size * 2): 85 | track = { 86 | 'type': 'track', 87 | 'event': 'python event %d' % i, 88 | 'userId': 'userId' 89 | } 90 | q.put(track) 91 | time.sleep(upload_interval * 1.1) 92 | self.assertEqual(mock_post.call_count, 2) 93 | 94 | @classmethod 95 | def test_request(cls): 96 | consumer = Consumer(None, 'testsecret') 97 | track = { 98 | 'type': 'track', 99 | 'event': 'python event', 100 | 'userId': 'userId' 101 | } 102 | consumer.request([track]) 103 | 104 | def _test_request_retry(self, consumer, 105 | expected_exception, exception_count): 106 | 107 | def mock_post(*args, **kwargs): 108 | mock_post.call_count += 1 109 | if mock_post.call_count <= exception_count: 110 | raise expected_exception 111 | mock_post.call_count = 0 112 | 113 | with mock.patch('analytics.consumer.post', 114 | mock.Mock(side_effect=mock_post)): 115 | track = { 116 | 'type': 'track', 117 | 'event': 'python event', 118 | 'userId': 'userId' 119 | } 120 | # request() should succeed if the number of exceptions raised is 121 | # less than the retries parameter. 122 | if exception_count <= consumer.retries: 123 | consumer.request([track]) 124 | else: 125 | # if exceptions are raised more times than the retries 126 | # parameter, we expect the exception to be returned to 127 | # the caller. 128 | try: 129 | consumer.request([track]) 130 | except type(expected_exception) as exc: 131 | self.assertEqual(exc, expected_exception) 132 | else: 133 | self.fail( 134 | "request() should raise an exception if still failing " 135 | "after %d retries" % consumer.retries) 136 | 137 | def test_request_retry(self): 138 | # we should retry on general errors 139 | consumer = Consumer(None, 'testsecret') 140 | self._test_request_retry(consumer, Exception('generic exception'), 2) 141 | 142 | # we should retry on server errors 143 | consumer = Consumer(None, 'testsecret') 144 | self._test_request_retry(consumer, APIError( 145 | 500, 'code', 'Internal Server Error'), 2) 146 | 147 | # we should retry on HTTP 429 errors 148 | consumer = Consumer(None, 'testsecret') 149 | self._test_request_retry(consumer, APIError( 150 | 429, 'code', 'Too Many Requests'), 2) 151 | 152 | # we should NOT retry on other client errors 153 | consumer = Consumer(None, 'testsecret') 154 | api_error = APIError(400, 'code', 'Client Errors') 155 | try: 156 | self._test_request_retry(consumer, api_error, 1) 157 | except APIError: 158 | pass 159 | else: 160 | self.fail('request() should not retry on client errors') 161 | 162 | # test for number of exceptions raise > retries value 163 | consumer = Consumer(None, 'testsecret', retries=3) 164 | self._test_request_retry(consumer, APIError( 165 | 500, 'code', 'Internal Server Error'), 3) 166 | 167 | def test_pause(self): 168 | consumer = Consumer(None, 'testsecret') 169 | consumer.pause() 170 | self.assertFalse(consumer.running) 171 | 172 | def test_max_batch_size(self): 173 | q = Queue() 174 | consumer = Consumer( 175 | q, 'testsecret', upload_size=100000, upload_interval=3) 176 | track = { 177 | 'type': 'track', 178 | 'event': 'python event', 179 | 'userId': 'userId' 180 | } 181 | msg_size = len(json.dumps(track).encode()) 182 | # number of messages in a maximum-size batch 183 | n_msgs = int(475000 / msg_size) 184 | 185 | def mock_post_fn(_, data, **kwargs): 186 | res = mock.Mock() 187 | res.status_code = 200 188 | self.assertTrue(len(data.encode()) < 500000, 189 | 'batch size (%d) exceeds 500KB limit' 190 | % len(data.encode())) 191 | return res 192 | 193 | with mock.patch('analytics.request._session.post', 194 | side_effect=mock_post_fn) as mock_post: 195 | consumer.start() 196 | for _ in range(0, n_msgs + 2): 197 | q.put(track) 198 | q.join() 199 | self.assertEqual(mock_post.call_count, 2) 200 | 201 | @classmethod 202 | def test_proxies(cls): 203 | consumer = Consumer(None, 'testsecret', proxies='203.243.63.16:80') 204 | track = { 205 | 'type': 'track', 206 | 'event': 'python event', 207 | 'userId': 'userId' 208 | } 209 | consumer.request([track]) 210 | -------------------------------------------------------------------------------- /segment/analytics/client.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from uuid import uuid4 3 | import logging 4 | import numbers 5 | import atexit 6 | import json 7 | 8 | from dateutil.tz import tzutc 9 | 10 | from segment.analytics.utils import guess_timezone, clean 11 | from segment.analytics.consumer import Consumer, MAX_MSG_SIZE 12 | from segment.analytics.request import post, DatetimeSerializer 13 | from segment.analytics.version import VERSION 14 | 15 | import queue 16 | 17 | ID_TYPES = (numbers.Number, str) 18 | 19 | 20 | class Client(object): 21 | class DefaultConfig(object): 22 | write_key = None 23 | host = None 24 | on_error = None 25 | debug = False 26 | send = True 27 | sync_mode = False 28 | max_queue_size = 10000 29 | gzip = False 30 | timeout = 15 31 | max_retries = 10 32 | proxies = None 33 | thread = 1 34 | upload_interval = 0.5 35 | upload_size = 100 36 | 37 | """Create a new Segment client.""" 38 | log = logging.getLogger('segment') 39 | 40 | def __init__(self, 41 | write_key=DefaultConfig.write_key, 42 | host=DefaultConfig.host, 43 | debug=DefaultConfig.debug, 44 | max_queue_size=DefaultConfig.max_queue_size, 45 | send=DefaultConfig.send, 46 | on_error=DefaultConfig.on_error, 47 | gzip=DefaultConfig.gzip, 48 | max_retries=DefaultConfig.max_retries, 49 | sync_mode=DefaultConfig.sync_mode, 50 | timeout=DefaultConfig.timeout, 51 | proxies=DefaultConfig.proxies, 52 | thread=DefaultConfig.thread, 53 | upload_size=DefaultConfig.upload_size, 54 | upload_interval=DefaultConfig.upload_interval,): 55 | require('write_key', write_key, str) 56 | 57 | self.queue = queue.Queue(max_queue_size) 58 | self.write_key = write_key 59 | self.on_error = on_error 60 | self.debug = debug 61 | self.send = send 62 | self.sync_mode = sync_mode 63 | self.host = host 64 | self.gzip = gzip 65 | self.timeout = timeout 66 | self.proxies = proxies 67 | 68 | if debug: 69 | self.log.setLevel(logging.DEBUG) 70 | 71 | if sync_mode: 72 | self.consumers = None 73 | else: 74 | # On program exit, allow the consumer thread to exit cleanly. 75 | # This prevents exceptions and a messy shutdown when the 76 | # interpreter is destroyed before the daemon thread finishes 77 | # execution. However, it is *not* the same as flushing the queue! 78 | # To guarantee all messages have been delivered, you'll still need 79 | # to call flush(). 80 | if send: 81 | atexit.register(self.join) 82 | for _ in range(thread): 83 | self.consumers = [] 84 | consumer = Consumer( 85 | self.queue, write_key, host=host, on_error=on_error, 86 | upload_size=upload_size, upload_interval=upload_interval, 87 | gzip=gzip, retries=max_retries, timeout=timeout, 88 | proxies=proxies, 89 | ) 90 | self.consumers.append(consumer) 91 | 92 | # if we've disabled sending, just don't start the consumer 93 | if send: 94 | consumer.start() 95 | 96 | def identify(self, user_id=None, traits=None, context=None, timestamp=None, 97 | anonymous_id=None, integrations=None, message_id=None): 98 | traits = traits or {} 99 | context = context or {} 100 | integrations = integrations or {} 101 | require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) 102 | require('traits', traits, dict) 103 | 104 | msg = { 105 | 'integrations': integrations, 106 | 'anonymousId': anonymous_id, 107 | 'timestamp': timestamp, 108 | 'context': context, 109 | 'type': 'identify', 110 | 'userId': user_id, 111 | 'traits': traits, 112 | 'messageId': message_id, 113 | } 114 | 115 | return self._enqueue(msg) 116 | 117 | def track(self, user_id=None, event=None, properties=None, context=None, 118 | timestamp=None, anonymous_id=None, integrations=None, 119 | message_id=None): 120 | properties = properties or {} 121 | context = context or {} 122 | integrations = integrations or {} 123 | require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) 124 | require('properties', properties, dict) 125 | require('event', event, str) 126 | 127 | msg = { 128 | 'integrations': integrations, 129 | 'anonymousId': anonymous_id, 130 | 'properties': properties, 131 | 'timestamp': timestamp, 132 | 'context': context, 133 | 'userId': user_id, 134 | 'type': 'track', 135 | 'event': event, 136 | 'messageId': message_id, 137 | } 138 | 139 | return self._enqueue(msg) 140 | 141 | def alias(self, previous_id=None, user_id=None, context=None, 142 | timestamp=None, integrations=None, message_id=None): 143 | context = context or {} 144 | integrations = integrations or {} 145 | require('previous_id', previous_id, ID_TYPES) 146 | require('user_id', user_id, ID_TYPES) 147 | 148 | msg = { 149 | 'integrations': integrations, 150 | 'previousId': previous_id, 151 | 'timestamp': timestamp, 152 | 'context': context, 153 | 'userId': user_id, 154 | 'type': 'alias', 155 | 'messageId': message_id, 156 | } 157 | 158 | return self._enqueue(msg) 159 | 160 | def group(self, user_id=None, group_id=None, traits=None, context=None, 161 | timestamp=None, anonymous_id=None, integrations=None, 162 | message_id=None): 163 | traits = traits or {} 164 | context = context or {} 165 | integrations = integrations or {} 166 | require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) 167 | require('group_id', group_id, ID_TYPES) 168 | require('traits', traits, dict) 169 | 170 | msg = { 171 | 'integrations': integrations, 172 | 'anonymousId': anonymous_id, 173 | 'timestamp': timestamp, 174 | 'groupId': group_id, 175 | 'context': context, 176 | 'userId': user_id, 177 | 'traits': traits, 178 | 'type': 'group', 179 | 'messageId': message_id, 180 | } 181 | 182 | return self._enqueue(msg) 183 | 184 | def page(self, user_id=None, category=None, name=None, properties=None, 185 | context=None, timestamp=None, anonymous_id=None, 186 | integrations=None, message_id=None): 187 | properties = properties or {} 188 | context = context or {} 189 | integrations = integrations or {} 190 | require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) 191 | require('properties', properties, dict) 192 | 193 | if name: 194 | require('name', name, str) 195 | if category: 196 | require('category', category, str) 197 | 198 | msg = { 199 | 'integrations': integrations, 200 | 'anonymousId': anonymous_id, 201 | 'properties': properties, 202 | 'timestamp': timestamp, 203 | 'category': category, 204 | 'context': context, 205 | 'userId': user_id, 206 | 'type': 'page', 207 | 'name': name, 208 | 'messageId': message_id, 209 | } 210 | 211 | return self._enqueue(msg) 212 | 213 | def screen(self, user_id=None, category=None, name=None, properties=None, 214 | context=None, timestamp=None, anonymous_id=None, 215 | integrations=None, message_id=None): 216 | properties = properties or {} 217 | context = context or {} 218 | integrations = integrations or {} 219 | require('user_id or anonymous_id', user_id or anonymous_id, ID_TYPES) 220 | require('properties', properties, dict) 221 | 222 | if name: 223 | require('name', name, str) 224 | if category: 225 | require('category', category, str) 226 | 227 | msg = { 228 | 'integrations': integrations, 229 | 'anonymousId': anonymous_id, 230 | 'properties': properties, 231 | 'timestamp': timestamp, 232 | 'category': category, 233 | 'context': context, 234 | 'userId': user_id, 235 | 'type': 'screen', 236 | 'name': name, 237 | 'messageId': message_id, 238 | } 239 | 240 | return self._enqueue(msg) 241 | 242 | def _enqueue(self, msg): 243 | """Push a new `msg` onto the queue, return `(success, msg)`""" 244 | timestamp = msg['timestamp'] 245 | if timestamp is None: 246 | timestamp = datetime.utcnow().replace(tzinfo=tzutc()) 247 | message_id = msg.get('messageId') 248 | if message_id is None: 249 | message_id = uuid4() 250 | 251 | require('integrations', msg['integrations'], dict) 252 | require('type', msg['type'], str) 253 | require('timestamp', timestamp, datetime) 254 | require('context', msg['context'], dict) 255 | 256 | # add common 257 | timestamp = guess_timezone(timestamp) 258 | msg['timestamp'] = timestamp.isoformat() 259 | msg['messageId'] = stringify_id(message_id) 260 | msg['context']['library'] = { 261 | 'name': 'analytics-python', 262 | 'version': VERSION 263 | } 264 | 265 | msg['userId'] = stringify_id(msg.get('userId', None)) 266 | msg['anonymousId'] = stringify_id(msg.get('anonymousId', None)) 267 | 268 | msg = clean(msg) 269 | self.log.debug('queueing: %s', msg) 270 | 271 | # Check message size. 272 | msg_size = len(json.dumps(msg, cls=DatetimeSerializer).encode()) 273 | if msg_size > MAX_MSG_SIZE: 274 | raise RuntimeError('Message exceeds %skb limit. (%s)', str(int(MAX_MSG_SIZE / 1024)), str(msg)) 275 | 276 | # if send is False, return msg as if it was successfully queued 277 | if not self.send: 278 | return True, msg 279 | 280 | if self.sync_mode: 281 | self.log.debug('enqueued with blocking %s.', msg['type']) 282 | post(self.write_key, self.host, gzip=self.gzip, 283 | timeout=self.timeout, proxies=self.proxies, batch=[msg]) 284 | 285 | return True, msg 286 | 287 | try: 288 | self.queue.put(msg, block=False) 289 | self.log.debug('enqueued %s.', msg['type']) 290 | return True, msg 291 | except queue.Full: 292 | self.log.warning('analytics-python queue is full') 293 | return False, msg 294 | 295 | def flush(self): 296 | """Forces a flush from the internal queue to the server""" 297 | queue = self.queue 298 | size = queue.qsize() 299 | queue.join() 300 | # Note that this message may not be precise, because of threading. 301 | self.log.debug('successfully flushed about %s items.', size) 302 | 303 | def join(self): 304 | """Ends the consumer thread once the queue is empty. 305 | Blocks execution until finished 306 | """ 307 | for consumer in self.consumers: 308 | consumer.pause() 309 | try: 310 | consumer.join() 311 | except RuntimeError: 312 | # consumer thread has not started 313 | pass 314 | 315 | def shutdown(self): 316 | """Flush all messages and cleanly shutdown the client""" 317 | self.flush() 318 | self.join() 319 | 320 | 321 | def require(name, field, data_type): 322 | """Require that the named `field` has the right `data_type`""" 323 | if not isinstance(field, data_type): 324 | msg = '{0} must have {1}, got: {2}'.format(name, data_type, field) 325 | raise AssertionError(msg) 326 | 327 | 328 | def stringify_id(val): 329 | if val is None: 330 | return None 331 | if isinstance(val, str): 332 | return val 333 | return str(val) 334 | -------------------------------------------------------------------------------- /analytics/test/client.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime 2 | import unittest 3 | import time 4 | import mock 5 | 6 | from analytics.version import VERSION 7 | from analytics.client import Client 8 | 9 | 10 | class TestClient(unittest.TestCase): 11 | 12 | def fail(self): 13 | """Mark the failure handler""" 14 | self.failed = True 15 | 16 | def setUp(self): 17 | self.failed = False 18 | self.client = Client('testsecret', on_error=self.fail) 19 | 20 | def test_requires_write_key(self): 21 | self.assertRaises(AssertionError, Client) 22 | 23 | def test_empty_flush(self): 24 | self.client.flush() 25 | 26 | def test_basic_track(self): 27 | client = self.client 28 | success, msg = client.track('userId', 'python test event') 29 | client.flush() 30 | self.assertTrue(success) 31 | self.assertFalse(self.failed) 32 | 33 | self.assertEqual(msg['event'], 'python test event') 34 | self.assertTrue(isinstance(msg['timestamp'], str)) 35 | self.assertTrue(isinstance(msg['messageId'], str)) 36 | self.assertEqual(msg['userId'], 'userId') 37 | self.assertEqual(msg['properties'], {}) 38 | self.assertEqual(msg['type'], 'track') 39 | 40 | def test_stringifies_user_id(self): 41 | # A large number that loses precision in node: 42 | # node -e "console.log(157963456373623802 + 1)" > 157963456373623800 43 | client = self.client 44 | success, msg = client.track( 45 | user_id=157963456373623802, event='python test event') 46 | client.flush() 47 | self.assertTrue(success) 48 | self.assertFalse(self.failed) 49 | 50 | self.assertEqual(msg['userId'], '157963456373623802') 51 | self.assertEqual(msg['anonymousId'], None) 52 | 53 | def test_stringifies_anonymous_id(self): 54 | # A large number that loses precision in node: 55 | # node -e "console.log(157963456373623803 + 1)" > 157963456373623800 56 | client = self.client 57 | success, msg = client.track( 58 | anonymous_id=157963456373623803, event='python test event') 59 | client.flush() 60 | self.assertTrue(success) 61 | self.assertFalse(self.failed) 62 | 63 | self.assertEqual(msg['userId'], None) 64 | self.assertEqual(msg['anonymousId'], '157963456373623803') 65 | 66 | def test_advanced_track(self): 67 | client = self.client 68 | success, msg = client.track( 69 | 'userId', 'python test event', {'property': 'value'}, 70 | {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', 71 | {'Amplitude': True}, 'messageId') 72 | 73 | self.assertTrue(success) 74 | 75 | self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') 76 | self.assertEqual(msg['properties'], {'property': 'value'}) 77 | self.assertEqual(msg['integrations'], {'Amplitude': True}) 78 | self.assertEqual(msg['context']['ip'], '192.168.0.1') 79 | self.assertEqual(msg['event'], 'python test event') 80 | self.assertEqual(msg['anonymousId'], 'anonymousId') 81 | self.assertEqual(msg['context']['library'], { 82 | 'name': 'analytics-python', 83 | 'version': VERSION 84 | }) 85 | self.assertEqual(msg['messageId'], 'messageId') 86 | self.assertEqual(msg['userId'], 'userId') 87 | self.assertEqual(msg['type'], 'track') 88 | 89 | def test_basic_identify(self): 90 | client = self.client 91 | success, msg = client.identify('userId', {'trait': 'value'}) 92 | client.flush() 93 | self.assertTrue(success) 94 | self.assertFalse(self.failed) 95 | 96 | self.assertEqual(msg['traits'], {'trait': 'value'}) 97 | self.assertTrue(isinstance(msg['timestamp'], str)) 98 | self.assertTrue(isinstance(msg['messageId'], str)) 99 | self.assertEqual(msg['userId'], 'userId') 100 | self.assertEqual(msg['type'], 'identify') 101 | 102 | def test_advanced_identify(self): 103 | client = self.client 104 | success, msg = client.identify( 105 | 'userId', {'trait': 'value'}, {'ip': '192.168.0.1'}, 106 | datetime(2014, 9, 3), 'anonymousId', {'Amplitude': True}, 107 | 'messageId') 108 | 109 | self.assertTrue(success) 110 | 111 | self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') 112 | self.assertEqual(msg['integrations'], {'Amplitude': True}) 113 | self.assertEqual(msg['context']['ip'], '192.168.0.1') 114 | self.assertEqual(msg['traits'], {'trait': 'value'}) 115 | self.assertEqual(msg['anonymousId'], 'anonymousId') 116 | self.assertEqual(msg['context']['library'], { 117 | 'name': 'analytics-python', 118 | 'version': VERSION 119 | }) 120 | self.assertTrue(isinstance(msg['timestamp'], str)) 121 | self.assertEqual(msg['messageId'], 'messageId') 122 | self.assertEqual(msg['userId'], 'userId') 123 | self.assertEqual(msg['type'], 'identify') 124 | 125 | def test_basic_group(self): 126 | client = self.client 127 | success, msg = client.group('userId', 'groupId') 128 | client.flush() 129 | self.assertTrue(success) 130 | self.assertFalse(self.failed) 131 | 132 | self.assertEqual(msg['groupId'], 'groupId') 133 | self.assertEqual(msg['userId'], 'userId') 134 | self.assertEqual(msg['type'], 'group') 135 | 136 | def test_advanced_group(self): 137 | client = self.client 138 | success, msg = client.group( 139 | 'userId', 'groupId', {'trait': 'value'}, {'ip': '192.168.0.1'}, 140 | datetime(2014, 9, 3), 'anonymousId', {'Amplitude': True}, 141 | 'messageId') 142 | 143 | self.assertTrue(success) 144 | 145 | self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') 146 | self.assertEqual(msg['integrations'], {'Amplitude': True}) 147 | self.assertEqual(msg['context']['ip'], '192.168.0.1') 148 | self.assertEqual(msg['traits'], {'trait': 'value'}) 149 | self.assertEqual(msg['anonymousId'], 'anonymousId') 150 | self.assertEqual(msg['context']['library'], { 151 | 'name': 'analytics-python', 152 | 'version': VERSION 153 | }) 154 | self.assertTrue(isinstance(msg['timestamp'], str)) 155 | self.assertEqual(msg['messageId'], 'messageId') 156 | self.assertEqual(msg['userId'], 'userId') 157 | self.assertEqual(msg['type'], 'group') 158 | 159 | def test_basic_alias(self): 160 | client = self.client 161 | success, msg = client.alias('previousId', 'userId') 162 | client.flush() 163 | self.assertTrue(success) 164 | self.assertFalse(self.failed) 165 | self.assertEqual(msg['previousId'], 'previousId') 166 | self.assertEqual(msg['userId'], 'userId') 167 | 168 | def test_basic_page(self): 169 | client = self.client 170 | success, msg = client.page('userId', name='name') 171 | self.assertFalse(self.failed) 172 | client.flush() 173 | self.assertTrue(success) 174 | self.assertEqual(msg['userId'], 'userId') 175 | self.assertEqual(msg['type'], 'page') 176 | self.assertEqual(msg['name'], 'name') 177 | 178 | def test_advanced_page(self): 179 | client = self.client 180 | success, msg = client.page( 181 | 'userId', 'category', 'name', {'property': 'value'}, 182 | {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', 183 | {'Amplitude': True}, 'messageId') 184 | 185 | self.assertTrue(success) 186 | 187 | self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') 188 | self.assertEqual(msg['integrations'], {'Amplitude': True}) 189 | self.assertEqual(msg['context']['ip'], '192.168.0.1') 190 | self.assertEqual(msg['properties'], {'property': 'value'}) 191 | self.assertEqual(msg['anonymousId'], 'anonymousId') 192 | self.assertEqual(msg['context']['library'], { 193 | 'name': 'analytics-python', 194 | 'version': VERSION 195 | }) 196 | self.assertEqual(msg['category'], 'category') 197 | self.assertTrue(isinstance(msg['timestamp'], str)) 198 | self.assertEqual(msg['messageId'], 'messageId') 199 | self.assertEqual(msg['userId'], 'userId') 200 | self.assertEqual(msg['type'], 'page') 201 | self.assertEqual(msg['name'], 'name') 202 | 203 | def test_basic_screen(self): 204 | client = self.client 205 | success, msg = client.screen('userId', name='name') 206 | client.flush() 207 | self.assertTrue(success) 208 | self.assertEqual(msg['userId'], 'userId') 209 | self.assertEqual(msg['type'], 'screen') 210 | self.assertEqual(msg['name'], 'name') 211 | 212 | def test_advanced_screen(self): 213 | client = self.client 214 | success, msg = client.screen( 215 | 'userId', 'category', 'name', {'property': 'value'}, 216 | {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', 217 | {'Amplitude': True}, 'messageId') 218 | 219 | self.assertTrue(success) 220 | 221 | self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') 222 | self.assertEqual(msg['integrations'], {'Amplitude': True}) 223 | self.assertEqual(msg['context']['ip'], '192.168.0.1') 224 | self.assertEqual(msg['properties'], {'property': 'value'}) 225 | self.assertEqual(msg['anonymousId'], 'anonymousId') 226 | self.assertEqual(msg['context']['library'], { 227 | 'name': 'analytics-python', 228 | 'version': VERSION 229 | }) 230 | self.assertTrue(isinstance(msg['timestamp'], str)) 231 | self.assertEqual(msg['messageId'], 'messageId') 232 | self.assertEqual(msg['category'], 'category') 233 | self.assertEqual(msg['userId'], 'userId') 234 | self.assertEqual(msg['type'], 'screen') 235 | self.assertEqual(msg['name'], 'name') 236 | 237 | def test_flush(self): 238 | client = self.client 239 | # set up the consumer with more requests than a single batch will allow 240 | for _ in range(1000): 241 | _, _ = client.identify('userId', {'trait': 'value'}) 242 | # We can't reliably assert that the queue is non-empty here; that's 243 | # a race condition. We do our best to load it up though. 244 | client.flush() 245 | # Make sure that the client queue is empty after flushing 246 | self.assertTrue(client.queue.empty()) 247 | 248 | def test_shutdown(self): 249 | client = self.client 250 | # set up the consumer with more requests than a single batch will allow 251 | for _ in range(1000): 252 | _, _ = client.identify('userId', {'trait': 'value'}) 253 | client.shutdown() 254 | # we expect two things after shutdown: 255 | # 1. client queue is empty 256 | # 2. consumer thread has stopped 257 | self.assertTrue(client.queue.empty()) 258 | for consumer in client.consumers: 259 | self.assertFalse(consumer.is_alive()) 260 | 261 | def test_synchronous(self): 262 | client = Client('testsecret', sync_mode=True) 263 | 264 | success, _ = client.identify('userId') 265 | self.assertFalse(client.consumers) 266 | self.assertTrue(client.queue.empty()) 267 | self.assertTrue(success) 268 | 269 | def test_overflow(self): 270 | client = Client('testsecret', max_queue_size=1) 271 | # Ensure consumer thread is no longer uploading 272 | client.join() 273 | 274 | for _ in range(10): 275 | client.identify('userId') 276 | 277 | success, _ = client.identify('userId') 278 | # Make sure we are informed that the queue is at capacity 279 | self.assertFalse(success) 280 | 281 | def test_success_on_invalid_write_key(self): 282 | client = Client('bad_key', on_error=self.fail) 283 | client.track('userId', 'event') 284 | client.flush() 285 | self.assertFalse(self.failed) 286 | 287 | def test_numeric_user_id(self): 288 | self.client.track(1234, 'python event') 289 | self.client.flush() 290 | self.assertFalse(self.failed) 291 | 292 | def test_identify_with_date_object(self): 293 | client = self.client 294 | success, msg = client.identify( 295 | 'userId', 296 | { 297 | 'birthdate': date(1981, 2, 2), 298 | }, 299 | ) 300 | client.flush() 301 | self.assertTrue(success) 302 | self.assertFalse(self.failed) 303 | 304 | self.assertEqual(msg['traits'], {'birthdate': date(1981, 2, 2)}) 305 | 306 | def test_gzip(self): 307 | client = Client('testsecret', on_error=self.fail, gzip=True) 308 | for _ in range(10): 309 | client.identify('userId', {'trait': 'value'}) 310 | client.flush() 311 | self.assertFalse(self.failed) 312 | 313 | def test_user_defined_upload_size(self): 314 | client = Client('testsecret', on_error=self.fail, 315 | upload_size=10, upload_interval=3) 316 | 317 | def mock_post_fn(**kwargs): 318 | self.assertEqual(len(kwargs['batch']), 10) 319 | 320 | # the post function should be called 2 times, with a batch size of 10 321 | # each time. 322 | with mock.patch('analytics.consumer.post', side_effect=mock_post_fn) \ 323 | as mock_post: 324 | for _ in range(20): 325 | client.identify('userId', {'trait': 'value'}) 326 | time.sleep(1) 327 | self.assertEqual(mock_post.call_count, 2) 328 | 329 | def test_user_defined_timeout(self): 330 | client = Client('testsecret', timeout=10) 331 | for consumer in client.consumers: 332 | self.assertEqual(consumer.timeout, 10) 333 | 334 | def test_default_timeout_15(self): 335 | client = Client('testsecret') 336 | for consumer in client.consumers: 337 | self.assertEqual(consumer.timeout, 15) 338 | 339 | def test_proxies(self): 340 | client = Client('testsecret', proxies='203.243.63.16:80') 341 | success, msg = client.identify('userId', {'trait': 'value'}) 342 | self.assertTrue(success) 343 | -------------------------------------------------------------------------------- /segment/analytics/test/client.py: -------------------------------------------------------------------------------- 1 | from datetime import date, datetime 2 | import unittest 3 | import time 4 | import mock 5 | 6 | from analytics.version import VERSION 7 | from analytics.client import Client 8 | 9 | 10 | class TestClient(unittest.TestCase): 11 | 12 | def fail(self, e, batch): 13 | """Mark the failure handler""" 14 | self.failed = True 15 | 16 | def setUp(self): 17 | self.failed = False 18 | self.client = Client('testsecret', on_error=self.fail) 19 | 20 | def test_requires_write_key(self): 21 | self.assertRaises(AssertionError, Client) 22 | 23 | def test_empty_flush(self): 24 | self.client.flush() 25 | 26 | def test_basic_track(self): 27 | client = self.client 28 | success, msg = client.track('userId', 'python test event') 29 | client.flush() 30 | self.assertTrue(success) 31 | self.assertFalse(self.failed) 32 | 33 | self.assertEqual(msg['event'], 'python test event') 34 | self.assertTrue(isinstance(msg['timestamp'], str)) 35 | self.assertTrue(isinstance(msg['messageId'], str)) 36 | self.assertEqual(msg['userId'], 'userId') 37 | self.assertEqual(msg['properties'], {}) 38 | self.assertEqual(msg['type'], 'track') 39 | 40 | def test_stringifies_user_id(self): 41 | # A large number that loses precision in node: 42 | # node -e "console.log(157963456373623802 + 1)" > 157963456373623800 43 | client = self.client 44 | success, msg = client.track( 45 | user_id=157963456373623802, event='python test event') 46 | client.flush() 47 | self.assertTrue(success) 48 | self.assertFalse(self.failed) 49 | 50 | self.assertEqual(msg['userId'], '157963456373623802') 51 | self.assertEqual(msg['anonymousId'], None) 52 | 53 | def test_stringifies_anonymous_id(self): 54 | # A large number that loses precision in node: 55 | # node -e "console.log(157963456373623803 + 1)" > 157963456373623800 56 | client = self.client 57 | success, msg = client.track( 58 | anonymous_id=157963456373623803, event='python test event') 59 | client.flush() 60 | self.assertTrue(success) 61 | self.assertFalse(self.failed) 62 | 63 | self.assertEqual(msg['userId'], None) 64 | self.assertEqual(msg['anonymousId'], '157963456373623803') 65 | 66 | def test_advanced_track(self): 67 | client = self.client 68 | success, msg = client.track( 69 | 'userId', 'python test event', {'property': 'value'}, 70 | {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', 71 | {'Amplitude': True}, 'messageId') 72 | 73 | self.assertTrue(success) 74 | 75 | self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') 76 | self.assertEqual(msg['properties'], {'property': 'value'}) 77 | self.assertEqual(msg['integrations'], {'Amplitude': True}) 78 | self.assertEqual(msg['context']['ip'], '192.168.0.1') 79 | self.assertEqual(msg['event'], 'python test event') 80 | self.assertEqual(msg['anonymousId'], 'anonymousId') 81 | self.assertEqual(msg['context']['library'], { 82 | 'name': 'analytics-python', 83 | 'version': VERSION 84 | }) 85 | self.assertEqual(msg['messageId'], 'messageId') 86 | self.assertEqual(msg['userId'], 'userId') 87 | self.assertEqual(msg['type'], 'track') 88 | 89 | def test_basic_identify(self): 90 | client = self.client 91 | success, msg = client.identify('userId', {'trait': 'value'}) 92 | client.flush() 93 | self.assertTrue(success) 94 | self.assertFalse(self.failed) 95 | 96 | self.assertEqual(msg['traits'], {'trait': 'value'}) 97 | self.assertTrue(isinstance(msg['timestamp'], str)) 98 | self.assertTrue(isinstance(msg['messageId'], str)) 99 | self.assertEqual(msg['userId'], 'userId') 100 | self.assertEqual(msg['type'], 'identify') 101 | 102 | def test_advanced_identify(self): 103 | client = self.client 104 | success, msg = client.identify( 105 | 'userId', {'trait': 'value'}, {'ip': '192.168.0.1'}, 106 | datetime(2014, 9, 3), 'anonymousId', {'Amplitude': True}, 107 | 'messageId') 108 | 109 | self.assertTrue(success) 110 | 111 | self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') 112 | self.assertEqual(msg['integrations'], {'Amplitude': True}) 113 | self.assertEqual(msg['context']['ip'], '192.168.0.1') 114 | self.assertEqual(msg['traits'], {'trait': 'value'}) 115 | self.assertEqual(msg['anonymousId'], 'anonymousId') 116 | self.assertEqual(msg['context']['library'], { 117 | 'name': 'analytics-python', 118 | 'version': VERSION 119 | }) 120 | self.assertTrue(isinstance(msg['timestamp'], str)) 121 | self.assertEqual(msg['messageId'], 'messageId') 122 | self.assertEqual(msg['userId'], 'userId') 123 | self.assertEqual(msg['type'], 'identify') 124 | 125 | def test_basic_group(self): 126 | client = self.client 127 | success, msg = client.group('userId', 'groupId') 128 | client.flush() 129 | self.assertTrue(success) 130 | self.assertFalse(self.failed) 131 | 132 | self.assertEqual(msg['groupId'], 'groupId') 133 | self.assertEqual(msg['userId'], 'userId') 134 | self.assertEqual(msg['type'], 'group') 135 | 136 | def test_advanced_group(self): 137 | client = self.client 138 | success, msg = client.group( 139 | 'userId', 'groupId', {'trait': 'value'}, {'ip': '192.168.0.1'}, 140 | datetime(2014, 9, 3), 'anonymousId', {'Amplitude': True}, 141 | 'messageId') 142 | 143 | self.assertTrue(success) 144 | 145 | self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') 146 | self.assertEqual(msg['integrations'], {'Amplitude': True}) 147 | self.assertEqual(msg['context']['ip'], '192.168.0.1') 148 | self.assertEqual(msg['traits'], {'trait': 'value'}) 149 | self.assertEqual(msg['anonymousId'], 'anonymousId') 150 | self.assertEqual(msg['context']['library'], { 151 | 'name': 'analytics-python', 152 | 'version': VERSION 153 | }) 154 | self.assertTrue(isinstance(msg['timestamp'], str)) 155 | self.assertEqual(msg['messageId'], 'messageId') 156 | self.assertEqual(msg['userId'], 'userId') 157 | self.assertEqual(msg['type'], 'group') 158 | 159 | def test_basic_alias(self): 160 | client = self.client 161 | success, msg = client.alias('previousId', 'userId') 162 | client.flush() 163 | self.assertTrue(success) 164 | self.assertFalse(self.failed) 165 | self.assertEqual(msg['previousId'], 'previousId') 166 | self.assertEqual(msg['userId'], 'userId') 167 | 168 | def test_basic_page(self): 169 | client = self.client 170 | success, msg = client.page('userId', name='name') 171 | self.assertFalse(self.failed) 172 | client.flush() 173 | self.assertTrue(success) 174 | self.assertEqual(msg['userId'], 'userId') 175 | self.assertEqual(msg['type'], 'page') 176 | self.assertEqual(msg['name'], 'name') 177 | 178 | def test_advanced_page(self): 179 | client = self.client 180 | success, msg = client.page( 181 | 'userId', 'category', 'name', {'property': 'value'}, 182 | {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', 183 | {'Amplitude': True}, 'messageId') 184 | 185 | self.assertTrue(success) 186 | 187 | self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') 188 | self.assertEqual(msg['integrations'], {'Amplitude': True}) 189 | self.assertEqual(msg['context']['ip'], '192.168.0.1') 190 | self.assertEqual(msg['properties'], {'property': 'value'}) 191 | self.assertEqual(msg['anonymousId'], 'anonymousId') 192 | self.assertEqual(msg['context']['library'], { 193 | 'name': 'analytics-python', 194 | 'version': VERSION 195 | }) 196 | self.assertEqual(msg['category'], 'category') 197 | self.assertTrue(isinstance(msg['timestamp'], str)) 198 | self.assertEqual(msg['messageId'], 'messageId') 199 | self.assertEqual(msg['userId'], 'userId') 200 | self.assertEqual(msg['type'], 'page') 201 | self.assertEqual(msg['name'], 'name') 202 | 203 | def test_basic_screen(self): 204 | client = self.client 205 | success, msg = client.screen('userId', name='name') 206 | client.flush() 207 | self.assertTrue(success) 208 | self.assertEqual(msg['userId'], 'userId') 209 | self.assertEqual(msg['type'], 'screen') 210 | self.assertEqual(msg['name'], 'name') 211 | 212 | def test_advanced_screen(self): 213 | client = self.client 214 | success, msg = client.screen( 215 | 'userId', 'category', 'name', {'property': 'value'}, 216 | {'ip': '192.168.0.1'}, datetime(2014, 9, 3), 'anonymousId', 217 | {'Amplitude': True}, 'messageId') 218 | 219 | self.assertTrue(success) 220 | 221 | self.assertEqual(msg['timestamp'], '2014-09-03T00:00:00+00:00') 222 | self.assertEqual(msg['integrations'], {'Amplitude': True}) 223 | self.assertEqual(msg['context']['ip'], '192.168.0.1') 224 | self.assertEqual(msg['properties'], {'property': 'value'}) 225 | self.assertEqual(msg['anonymousId'], 'anonymousId') 226 | self.assertEqual(msg['context']['library'], { 227 | 'name': 'analytics-python', 228 | 'version': VERSION 229 | }) 230 | self.assertTrue(isinstance(msg['timestamp'], str)) 231 | self.assertEqual(msg['messageId'], 'messageId') 232 | self.assertEqual(msg['category'], 'category') 233 | self.assertEqual(msg['userId'], 'userId') 234 | self.assertEqual(msg['type'], 'screen') 235 | self.assertEqual(msg['name'], 'name') 236 | 237 | def test_flush(self): 238 | client = self.client 239 | # set up the consumer with more requests than a single batch will allow 240 | for _ in range(1000): 241 | _, _ = client.identify('userId', {'trait': 'value'}) 242 | # We can't reliably assert that the queue is non-empty here; that's 243 | # a race condition. We do our best to load it up though. 244 | client.flush() 245 | # Make sure that the client queue is empty after flushing 246 | self.assertTrue(client.queue.empty()) 247 | 248 | def test_shutdown(self): 249 | client = self.client 250 | # set up the consumer with more requests than a single batch will allow 251 | for _ in range(1000): 252 | _, _ = client.identify('userId', {'trait': 'value'}) 253 | client.shutdown() 254 | # we expect two things after shutdown: 255 | # 1. client queue is empty 256 | # 2. consumer thread has stopped 257 | self.assertTrue(client.queue.empty()) 258 | for consumer in client.consumers: 259 | self.assertFalse(consumer.is_alive()) 260 | 261 | def test_synchronous(self): 262 | client = Client('testsecret', sync_mode=True) 263 | 264 | success, _ = client.identify('userId') 265 | self.assertFalse(client.consumers) 266 | self.assertTrue(client.queue.empty()) 267 | self.assertTrue(success) 268 | 269 | def test_overflow(self): 270 | client = Client('testsecret', max_queue_size=1) 271 | # Ensure consumer thread is no longer uploading 272 | client.join() 273 | 274 | for _ in range(10): 275 | client.identify('userId') 276 | 277 | success, _ = client.identify('userId') 278 | # Make sure we are informed that the queue is at capacity 279 | self.assertFalse(success) 280 | 281 | def test_success_on_invalid_write_key(self): 282 | client = Client('bad_key', on_error=self.fail) 283 | client.track('userId', 'event') 284 | client.flush() 285 | self.assertFalse(self.failed) 286 | 287 | def test_unicode(self): 288 | Client('unicode_key') 289 | 290 | def test_numeric_user_id(self): 291 | self.client.track(1234, 'python event') 292 | self.client.flush() 293 | self.assertFalse(self.failed) 294 | 295 | def test_debug(self): 296 | Client('bad_key', debug=True) 297 | 298 | def test_identify_with_date_object(self): 299 | client = self.client 300 | success, msg = client.identify( 301 | 'userId', 302 | { 303 | 'birthdate': date(1981, 2, 2), 304 | }, 305 | ) 306 | client.flush() 307 | self.assertTrue(success) 308 | self.assertFalse(self.failed) 309 | 310 | self.assertEqual(msg['traits'], {'birthdate': date(1981, 2, 2)}) 311 | 312 | def test_gzip(self): 313 | client = Client('testsecret', on_error=self.fail, gzip=True) 314 | for _ in range(10): 315 | client.identify('userId', {'trait': 'value'}) 316 | client.flush() 317 | self.assertFalse(self.failed) 318 | 319 | def test_user_defined_upload_size(self): 320 | client = Client('testsecret', on_error=self.fail, 321 | upload_size=10, upload_interval=3) 322 | 323 | def mock_post_fn(*args, **kwargs): 324 | self.assertEqual(len(kwargs['batch']), 10) 325 | 326 | # the post function should be called 2 times, with a batch size of 10 327 | # each time. 328 | with mock.patch('analytics.consumer.post', side_effect=mock_post_fn) \ 329 | as mock_post: 330 | for _ in range(20): 331 | client.identify('userId', {'trait': 'value'}) 332 | time.sleep(1) 333 | self.assertEqual(mock_post.call_count, 2) 334 | 335 | def test_user_defined_timeout(self): 336 | client = Client('testsecret', timeout=10) 337 | for consumer in client.consumers: 338 | self.assertEqual(consumer.timeout, 10) 339 | 340 | def test_default_timeout_15(self): 341 | client = Client('testsecret') 342 | for consumer in client.consumers: 343 | self.assertEqual(consumer.timeout, 15) 344 | 345 | def test_proxies(self): 346 | client = Client('testsecret', proxies='203.243.63.16:80') 347 | success, msg = client.identify('userId', {'trait': 'value'}) 348 | self.assertTrue(success) 349 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Add files or directories to the ignore list. They should be base names, not 4 | # paths. 5 | ignore=CVS 6 | 7 | # Add files or directories matching the regex patterns to the denylist. The 8 | # regex matches against base names, not paths. 9 | ignore-patterns= 10 | 11 | # Python code to execute, usually for sys.path manipulation such as 12 | # pygtk.require(). 13 | #init-hook= 14 | 15 | # Use multiple processes to speed up Pylint. 16 | jobs=1 17 | 18 | # List of plugins (as comma separated values of python modules names) to load, 19 | # usually to register additional checkers. 20 | load-plugins= 21 | 22 | # Pickle collected data for later comparisons. 23 | persistent=yes 24 | 25 | # Specify a configuration file. 26 | #rcfile= 27 | 28 | # When enabled, pylint would attempt to guess common misconfiguration and emit 29 | # user-friendly hints instead of false-positive error messages 30 | suggestion-mode=yes 31 | 32 | # Allow loading of arbitrary C extensions. Extensions are imported into the 33 | # active Python interpreter and may run arbitrary code. 34 | unsafe-load-any-extension=no 35 | 36 | 37 | [MESSAGES CONTROL] 38 | 39 | # Only show warnings with the listed confidence levels. Leave empty to show 40 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 41 | confidence= 42 | 43 | # Disable the message, report, category or checker with the given id(s). You 44 | # can either give multiple identifiers separated by comma (,) or put this 45 | # option multiple times (only on the command line, not in the configuration 46 | # file where it should appear only once).You can also use "--disable=all" to 47 | # disable everything first and then reenable specific checks. For example, if 48 | # you want to run only the similarities checker, you can use "--disable=all 49 | # --enable=similarities". If you want to run only the classes checker, but have 50 | # no Warning level messages displayed, use"--disable=all --enable=classes 51 | # --disable=W" 52 | disable=too-many-public-methods, 53 | no-else-return, 54 | print-statement, 55 | invalid-name, 56 | global-statement, 57 | too-many-arguments, 58 | missing-docstring, 59 | too-many-instance-attributes, 60 | parameter-unpacking, 61 | unpacking-in-except, 62 | old-raise-syntax, 63 | backtick, 64 | long-suffix, 65 | old-ne-operator, 66 | old-octal-literal, 67 | import-star-module-level, 68 | non-ascii-bytes-literal, 69 | invalid-unicode-literal, 70 | raw-checker-failed, 71 | bad-inline-option, 72 | locally-disabled, 73 | locally-enabled, 74 | file-ignored, 75 | suppressed-message, 76 | useless-suppression, 77 | deprecated-pragma, 78 | apply-builtin, 79 | basestring-builtin, 80 | buffer-builtin, 81 | cmp-builtin, 82 | coerce-builtin, 83 | execfile-builtin, 84 | file-builtin, 85 | long-builtin, 86 | raw_input-builtin, 87 | reduce-builtin, 88 | standarderror-builtin, 89 | unicode-builtin, 90 | xrange-builtin, 91 | coerce-method, 92 | delslice-method, 93 | getslice-method, 94 | setslice-method, 95 | no-absolute-import, 96 | old-division, 97 | dict-iter-method, 98 | dict-view-method, 99 | next-method-called, 100 | metaclass-assignment, 101 | indexing-exception, 102 | raising-string, 103 | reload-builtin, 104 | oct-method, 105 | hex-method, 106 | nonzero-method, 107 | cmp-method, 108 | input-builtin, 109 | round-builtin, 110 | intern-builtin, 111 | unichr-builtin, 112 | map-builtin-not-iterating, 113 | zip-builtin-not-iterating, 114 | range-builtin-not-iterating, 115 | filter-builtin-not-iterating, 116 | using-cmp-argument, 117 | eq-without-hash, 118 | div-method, 119 | idiv-method, 120 | rdiv-method, 121 | exception-message-attribute, 122 | invalid-str-codec, 123 | sys-max-int, 124 | bad-python3-import, 125 | deprecated-string-function, 126 | deprecated-str-translate-call, 127 | deprecated-itertools-function, 128 | deprecated-types-field, 129 | next-method-defined, 130 | dict-items-not-iterating, 131 | dict-keys-not-iterating, 132 | dict-values-not-iterating, 133 | deprecated-operator-function, 134 | deprecated-urllib-function, 135 | xreadlines-attribute, 136 | deprecated-sys-function, 137 | exception-escape, 138 | comprehension-escape 139 | 140 | # Enable the message, report, category or checker with the given id(s). You can 141 | # either give multiple identifier separated by comma (,) or put this option 142 | # multiple time (only on the command line, not in the configuration file where 143 | # it should appear only once). See also the "--disable" option for examples. 144 | enable=c-extension-no-member 145 | 146 | 147 | [REPORTS] 148 | 149 | # Python expression which should return a note less than 10 (10 is the highest 150 | # note). You have access to the variables errors warning, statement which 151 | # respectively contain the number of errors / warnings messages and the total 152 | # number of statements analyzed. This is used by the global evaluation report 153 | # (RP0004). 154 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 155 | 156 | # Template used to display messages. This is a python new-style format string 157 | # used to format the message information. See doc for all details 158 | msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg} 159 | 160 | # Set the output format. Available formats are text, parseable, colorized, json 161 | # and msvs (visual studio).You can also give a reporter class, eg 162 | # mypackage.mymodule.MyReporterClass. 163 | output-format=text 164 | 165 | # Tells whether to display a full report or only the messages 166 | reports=no 167 | 168 | # Activate the evaluation score. 169 | score=yes 170 | 171 | 172 | [REFACTORING] 173 | 174 | # Maximum number of nested blocks for function / method body 175 | max-nested-blocks=5 176 | 177 | # Complete name of functions that never returns. When checking for 178 | # inconsistent-return-statements if a never returning function is called then 179 | # it will be considered as an explicit return statement and no message will be 180 | # printed. 181 | never-returning-functions=optparse.Values,sys.exit 182 | 183 | 184 | [LOGGING] 185 | 186 | # Logging modules to check that the string format arguments are in logging 187 | # function parameter format 188 | logging-modules=logging 189 | 190 | 191 | [SPELLING] 192 | 193 | # Limits count of emitted suggestions for spelling mistakes 194 | max-spelling-suggestions=4 195 | 196 | # Spelling dictionary name. Available dictionaries: none. To make it working 197 | # install python-enchant package. 198 | spelling-dict= 199 | 200 | # List of comma separated words that should not be checked. 201 | spelling-ignore-words= 202 | 203 | # A path to a file that contains private dictionary; one word per line. 204 | spelling-private-dict-file= 205 | 206 | # Tells whether to store unknown words to indicated private dictionary in 207 | # --spelling-private-dict-file option instead of raising a message. 208 | spelling-store-unknown-words=no 209 | 210 | 211 | [MISCELLANEOUS] 212 | 213 | # List of note tags to take in consideration, separated by a comma. 214 | notes=FIXME, 215 | XXX, 216 | TODO 217 | 218 | 219 | [SIMILARITIES] 220 | 221 | # Ignore comments when computing similarities. 222 | ignore-comments=yes 223 | 224 | # Ignore docstrings when computing similarities. 225 | ignore-docstrings=yes 226 | 227 | # Ignore imports when computing similarities. 228 | ignore-imports=no 229 | 230 | # Minimum lines number of a similarity. 231 | min-similarity-lines=4 232 | 233 | 234 | [TYPECHECK] 235 | 236 | # List of decorators that produce context managers, such as 237 | # contextlib.contextmanager. Add to this list to register other decorators that 238 | # produce valid context managers. 239 | contextmanager-decorators=contextlib.contextmanager 240 | 241 | # List of members which are set dynamically and missed by pylint inference 242 | # system, and so shouldn't trigger E1101 when accessed. Python regular 243 | # expressions are accepted. 244 | generated-members= 245 | 246 | # Tells whether missing members accessed in mixin class should be ignored. A 247 | # mixin class is detected if its name ends with "mixin" (case insensitive). 248 | ignore-mixin-members=yes 249 | 250 | # This flag controls whether pylint should warn about no-member and similar 251 | # checks whenever an opaque object is returned when inferring. The inference 252 | # can return multiple potential results while evaluating a Python object, but 253 | # some branches might not be evaluated, which results in partial inference. In 254 | # that case, it might be useful to still emit no-member and other checks for 255 | # the rest of the inferred objects. 256 | ignore-on-opaque-inference=yes 257 | 258 | # List of class names for which member attributes should not be checked (useful 259 | # for classes with dynamically set attributes). This supports the use of 260 | # qualified names. 261 | ignored-classes=optparse.Values,thread._local,_thread._local 262 | 263 | # List of module names for which member attributes should not be checked 264 | # (useful for modules/projects where namespaces are manipulated during runtime 265 | # and thus existing member attributes cannot be deduced by static analysis. It 266 | # supports qualified module names, as well as Unix pattern matching. 267 | ignored-modules= 268 | 269 | # Show a hint with possible names when a member name was not found. The aspect 270 | # of finding the hint is based on edit distance. 271 | missing-member-hint=yes 272 | 273 | # The minimum edit distance a name should have in order to be considered a 274 | # similar match for a missing member name. 275 | missing-member-hint-distance=1 276 | 277 | # The total number of similar names that should be taken in consideration when 278 | # showing a hint for a missing member. 279 | missing-member-max-choices=1 280 | 281 | 282 | [VARIABLES] 283 | 284 | # List of additional names supposed to be defined in builtins. Remember that 285 | # you should avoid to define new builtins when possible. 286 | additional-builtins= 287 | 288 | # Tells whether unused global variables should be treated as a violation. 289 | allow-global-unused-variables=yes 290 | 291 | # List of strings which can identify a callback function by name. A callback 292 | # name must start or end with one of those strings. 293 | callbacks=cb_, 294 | _cb 295 | 296 | # A regular expression matching the name of dummy variables (i.e. expectedly 297 | # not used). 298 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 299 | 300 | # Argument names that match this expression will be ignored. Default to name 301 | # with leading underscore 302 | ignored-argument-names=_.*|^ignored_|^unused_ 303 | 304 | # Tells whether we should check for unused import in __init__ files. 305 | init-import=no 306 | 307 | # List of qualified module names which can have objects that can redefine 308 | # builtins. 309 | redefining-builtins-modules=past.builtins,future.builtins,io,builtins 310 | 311 | 312 | [FORMAT] 313 | 314 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 315 | expected-line-ending-format= 316 | 317 | # Regexp for a line that is allowed to be longer than the limit. 318 | ignore-long-lines=^\s*(# )??$ 319 | 320 | # Number of spaces of indent required inside a hanging or continued line. 321 | indent-after-paren=4 322 | 323 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 324 | # tab). 325 | indent-string=' ' 326 | 327 | # Maximum number of characters on a single line. 328 | max-line-length=100 329 | 330 | # Maximum number of lines in a module 331 | max-module-lines=1000 332 | 333 | # List of optional constructs for which whitespace checking is disabled. `dict- 334 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 335 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 336 | # `empty-line` allows space-only lines. 337 | no-space-check=trailing-comma, 338 | dict-separator 339 | 340 | # Allow the body of a class to be on the same line as the declaration if body 341 | # contains single statement. 342 | single-line-class-stmt=no 343 | 344 | # Allow the body of an if to be on the same line as the test if there is no 345 | # else. 346 | single-line-if-stmt=no 347 | 348 | 349 | [BASIC] 350 | 351 | # Naming style matching correct argument names 352 | argument-naming-style=snake_case 353 | 354 | # Regular expression matching correct argument names. Overrides argument- 355 | # naming-style 356 | #argument-rgx= 357 | 358 | # Naming style matching correct attribute names 359 | attr-naming-style=snake_case 360 | 361 | # Regular expression matching correct attribute names. Overrides attr-naming- 362 | # style 363 | #attr-rgx= 364 | 365 | # Bad variable names which should always be refused, separated by a comma 366 | bad-names=foo, 367 | bar, 368 | baz, 369 | toto, 370 | tutu, 371 | tata 372 | 373 | # Naming style matching correct class attribute names 374 | class-attribute-naming-style=any 375 | 376 | # Regular expression matching correct class attribute names. Overrides class- 377 | # attribute-naming-style 378 | #class-attribute-rgx= 379 | 380 | # Naming style matching correct class names 381 | class-naming-style=PascalCase 382 | 383 | # Regular expression matching correct class names. Overrides class-naming-style 384 | #class-rgx= 385 | 386 | # Naming style matching correct constant names 387 | const-naming-style=UPPER_CASE 388 | 389 | # Regular expression matching correct constant names. Overrides const-naming- 390 | # style 391 | #const-rgx= 392 | 393 | # Minimum line length for functions/classes that require docstrings, shorter 394 | # ones are exempt. 395 | docstring-min-length=-1 396 | 397 | # Naming style matching correct function names 398 | function-naming-style=snake_case 399 | 400 | # Regular expression matching correct function names. Overrides function- 401 | # naming-style 402 | #function-rgx= 403 | 404 | # Good variable names which should always be accepted, separated by a comma 405 | good-names=i, 406 | j, 407 | k, 408 | ex, 409 | Run, 410 | _ 411 | 412 | # Include a hint for the correct naming format with invalid-name 413 | include-naming-hint=no 414 | 415 | # Naming style matching correct inline iteration names 416 | inlinevar-naming-style=any 417 | 418 | # Regular expression matching correct inline iteration names. Overrides 419 | # inlinevar-naming-style 420 | #inlinevar-rgx= 421 | 422 | # Naming style matching correct method names 423 | method-naming-style=snake_case 424 | 425 | # Regular expression matching correct method names. Overrides method-naming- 426 | # style 427 | #method-rgx= 428 | 429 | # Naming style matching correct module names 430 | module-naming-style=snake_case 431 | 432 | # Regular expression matching correct module names. Overrides module-naming- 433 | # style 434 | #module-rgx= 435 | 436 | # Colon-delimited sets of names that determine each other's naming style when 437 | # the name regexes allow several styles. 438 | name-group= 439 | 440 | # Regular expression which should only match function or class names that do 441 | # not require a docstring. 442 | no-docstring-rgx=^_ 443 | 444 | # List of decorators that produce properties, such as abc.abstractproperty. Add 445 | # to this list to register other decorators that produce valid properties. 446 | property-classes=abc.abstractproperty 447 | 448 | # Naming style matching correct variable names 449 | variable-naming-style=snake_case 450 | 451 | # Regular expression matching correct variable names. Overrides variable- 452 | # naming-style 453 | #variable-rgx= 454 | 455 | 456 | [DESIGN] 457 | 458 | # Maximum number of arguments for function / method 459 | max-args=5 460 | 461 | # Maximum number of attributes for a class (see R0902). 462 | max-attributes=7 463 | 464 | # Maximum number of boolean expressions in a if statement 465 | max-bool-expr=5 466 | 467 | # Maximum number of branch for function / method body 468 | max-branches=12 469 | 470 | # Maximum number of locals for function / method body 471 | max-locals=20 472 | 473 | # Maximum number of parents for a class (see R0901). 474 | max-parents=7 475 | 476 | # Maximum number of public methods for a class (see R0904). 477 | max-public-methods=20 478 | 479 | # Maximum number of return / yield for function / method body 480 | max-returns=6 481 | 482 | # Maximum number of statements in function / method body 483 | max-statements=50 484 | 485 | # Minimum number of public methods for a class (see R0903). 486 | min-public-methods=2 487 | 488 | 489 | [CLASSES] 490 | 491 | # List of method names used to declare (i.e. assign) instance attributes. 492 | defining-attr-methods=__init__, 493 | __new__, 494 | setUp 495 | 496 | # List of member names, which should be excluded from the protected access 497 | # warning. 498 | exclude-protected=_asdict, 499 | _fields, 500 | _replace, 501 | _source, 502 | _make 503 | 504 | # List of valid names for the first argument in a class method. 505 | valid-classmethod-first-arg=cls 506 | 507 | # List of valid names for the first argument in a metaclass class method. 508 | valid-metaclass-classmethod-first-arg=mcs 509 | 510 | 511 | [IMPORTS] 512 | 513 | # Allow wildcard imports from modules that define __all__. 514 | allow-wildcard-with-all=no 515 | 516 | # Analyse import fallback blocks. This can be used to support both Python 2 and 517 | # 3 compatible code, which means that the block might have code that exists 518 | # only in one or another interpreter, leading to false positives when analysed. 519 | analyse-fallback-blocks=no 520 | 521 | # Deprecated modules which should not be used, separated by a comma 522 | deprecated-modules=regsub, 523 | TERMIOS, 524 | Bastion, 525 | rexec 526 | 527 | # Create a graph of external dependencies in the given file (report RP0402 must 528 | # not be disabled) 529 | ext-import-graph= 530 | 531 | # Create a graph of every (i.e. internal and external) dependencies in the 532 | # given file (report RP0402 must not be disabled) 533 | import-graph= 534 | 535 | # Create a graph of internal dependencies in the given file (report RP0402 must 536 | # not be disabled) 537 | int-import-graph= 538 | 539 | # Force import order to recognize a module as part of the standard 540 | # compatibility libraries. 541 | known-standard-library= 542 | 543 | # Force import order to recognize a module as part of a third party library. 544 | known-third-party=enchant 545 | 546 | 547 | [EXCEPTIONS] 548 | 549 | # Exceptions that will emit a warning when being caught. Defaults to 550 | # "Exception" 551 | overgeneral-exceptions=Exception 552 | --------------------------------------------------------------------------------