├── .gitignore ├── .travis.yml ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── cherami_client ├── __init__.py ├── ack_message_result.py ├── ack_thread.py ├── client.py ├── consumer.py ├── consumer_thread.py ├── idl │ └── cherami.thrift ├── lib │ ├── __init__.py │ ├── cherami.py │ ├── cherami_frontend.py │ ├── cherami_input.py │ ├── cherami_output.py │ └── util.py ├── publisher.py ├── publisher_thread.py └── reconfigure_thread.py ├── config ├── consumer.yaml ├── publisher.yaml └── test.yaml ├── demo ├── __init__.py ├── example_client.py ├── example_consumer.py └── example_publisher.py ├── requirements-test.txt ├── requirements.txt ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── test_consumer.py ├── test_publisher.py └── test_util.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[cod] 2 | *.orig 3 | 4 | # C extensions 5 | *.so 6 | 7 | # OS X Junk 8 | .DS_Store 9 | 10 | # Packages 11 | *.egg 12 | *.egg-info 13 | 14 | bin 15 | build 16 | develop-eggs 17 | dist 18 | eggs 19 | parts 20 | sdist 21 | var 22 | 23 | __pycache__ 24 | .cache/ 25 | 26 | # Installer logs 27 | pip-log.txt 28 | 29 | # Unit test / coverage reports 30 | .coverage 31 | .tox 32 | .noseids 33 | 34 | # Ignore python virtual environments 35 | env* 36 | thrift_env 37 | 38 | # Ignore local logs 39 | *.log 40 | logs/* 41 | !logs/.gitkeep 42 | 43 | # Ignore docs 44 | docs/_build/* 45 | 46 | # Ignore coverage output 47 | *.xml 48 | coverage/ 49 | htmlcov/ 50 | 51 | # Ignore benchmarks output 52 | perf.log 53 | perf.svg 54 | 55 | # IDE 56 | *.swp 57 | .idea/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | install: 4 | - pip install coveralls 5 | - make bootstrap 6 | 7 | env: 8 | - CLAY_CONFIG=./config/test.yaml 9 | 10 | script: 11 | - make test 12 | 13 | after_success: 14 | - coverage run --source cherami_client setup.py test 15 | - coveralls -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Contributing 3 | ============ 4 | 5 | We'd love your help in making Cherami great. If you find a bug or need a new feature, open an issue and we will respond as fast as we can. 6 | If you want to implement new feature(s) and/or fix bug(s) yourself, open a pull request with the appropriate unit tests and we will merge it after review. 7 | 8 | Note: All contributors also need to fill out the `Uber Contributor License Agreement `_ before we can merge in any of your changes. 9 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | HISTORY 3 | =========== 4 | 5 | 1.0.3 (2017-08-29) 6 | ------------------ 7 | - add Python 3 queue support by using Python Six package 8 | 9 | 1.0.2 (2017-07-20) 10 | ------------------ 11 | - Remove parse_requirements usage in setup.py 12 | 13 | 1.0.1 (2017-07-13) 14 | ------------------ 15 | - Add PyYAML dependency 16 | 17 | 1.0.0 (2017-07-11) 18 | ------------------ 19 | - Initial open source release 20 | 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 Uber Technologies, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include LICENSE 3 | recursive-include cherami_client/idl * 4 | recursive-include config *.yaml 5 | recursive-exclude * *.py[co] 6 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help bootstrap clean lint test coverage docs release install jenkins 2 | 3 | help: 4 | @echo "clean - remove all build, test, coverage and Python artifacts" 5 | @echo "clean-build - remove build artifacts" 6 | @echo "clean-pyc - remove Python file artifacts" 7 | @echo "clean-test - remove test and coverage artifacts" 8 | @echo "lint - check style with flake8" 9 | @echo "test - run tests quickly with the default Python" 10 | @echo "coverage - check code coverage quickly with the default Python" 11 | @echo "release - package and upload a release" 12 | @echo "install - install the package to the active Python's site-packages" 13 | 14 | bootstrap: 15 | virtualenv --setuptools env 16 | . env/bin/activate 17 | pip install --upgrade setuptools 18 | pip install --upgrade "pip>=7,<8" 19 | pip install -r requirements.txt 20 | pip install -r requirements-test.txt 21 | 22 | clean: clean-build clean-pyc clean-test 23 | 24 | clean-build: 25 | rm -fr build/ 26 | rm -fr dist/ 27 | rm -fr .eggs/ 28 | find . -name '*.egg-info' -exec rm -fr {} + 29 | find . -name '*.egg' -exec rm -f {} + 30 | 31 | clean-pyc: 32 | find . -name '*.pyc' -exec rm -f {} + 33 | find . -name '*.pyo' -exec rm -f {} + 34 | find . -name '*~' -exec rm -f {} + 35 | find . -name '__pycache__' -exec rm -fr {} + 36 | 37 | clean-test: 38 | rm -f .coverage 39 | rm -fr htmlcov/ 40 | 41 | lint: 42 | flake8 cherami_client tests 43 | 44 | test: 45 | python setup.py test $(TEST_ARGS) 46 | 47 | jenkins: test 48 | 49 | coverage: test 50 | coverage run --source cherami_client setup.py test 51 | coverage report -m 52 | coverage html 53 | open htmlcov/index.html 54 | 55 | release: clean 56 | fullrelease 57 | 58 | install: clean 59 | python setup.py install 60 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://travis-ci.org/uber/cherami-client-python.svg?branch=master 2 | :target: https://travis-ci.org/uber/cherami-client-python 3 | 4 | .. image:: https://coveralls.io/repos/github/uber/cherami-client-python/badge.svg?branch=master 5 | :target: https://coveralls.io/github/uber/cherami-client-python?branch=master 6 | 7 | .. image:: https://badge.fury.io/py/cherami-client.svg 8 | :target: https://badge.fury.io/py/cherami-client 9 | 10 | =============================== 11 | Cherami Client For Python 12 | =============================== 13 | 14 | (This project is deprecated and not maintained.) 15 | 16 | Python client library for publishing/consuming messages to/from `Cherami `_. 17 | 18 | Installation 19 | ------------ 20 | 21 | ``pip install cherami-client`` 22 | 23 | Usage 24 | ----- 25 | 26 | Create and edit the ``.yaml`` file under ``./config`` 27 | 28 | See Example: 29 | :: 30 | cat ./config/test.yaml 31 | 32 | Set the clay environment variable: 33 | :: 34 | export CLAY_CONFIG=./config/test.yaml 35 | 36 | Run the example client: 37 | :: 38 | python ./demo/example_publisher.py 39 | python ./demo/example_consumer.py 40 | 41 | Contributing 42 | ------------ 43 | We'd love your help in making Cherami great. If you find a bug or need a new feature, open an issue and we will respond as fast as we can. 44 | If you want to implement new feature(s) and/or fix bug(s) yourself, open a pull request with the appropriate unit tests and we will merge it after review. 45 | 46 | Note: All contributors also need to fill out the `Uber Contributor License Agreement `_ before we can merge in any of your changes. 47 | 48 | Documentation 49 | ------------- 50 | Interested in learning more about Cherami? Read the blog post: `eng.uber.com/cherami `_ 51 | 52 | License 53 | ------- 54 | MIT License, please see `LICENSE `_ for details. 55 | -------------------------------------------------------------------------------- /cherami_client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/cherami-client-python/a0b708afdc228611f8d041f5581163c5e5c7757e/cherami_client/__init__.py -------------------------------------------------------------------------------- /cherami_client/ack_message_result.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | from __future__ import absolute_import 22 | 23 | 24 | # A class to wrap all information needed to handle an ack message result 25 | class AckMessageResult(object): 26 | def __init__(self, call_success, is_ack, delivery_token, error_msg): 27 | self.call_success = call_success 28 | self.is_ack = is_ack 29 | self.delivery_token = delivery_token 30 | self.error_msg = error_msg 31 | -------------------------------------------------------------------------------- /cherami_client/ack_thread.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | from __future__ import absolute_import 22 | 23 | import traceback 24 | from threading import Thread, Event 25 | from six.moves.queue import Empty 26 | 27 | from cherami_client.lib import util, cherami 28 | from cherami_client.ack_message_result import AckMessageResult 29 | 30 | 31 | class AckThread(Thread): 32 | def __init__(self, tchannel, headers, logger, ack_queue, timeout_seconds): 33 | Thread.__init__(self) 34 | self.tchannel = tchannel 35 | self.headers = headers 36 | self.logger = logger 37 | self.ack_queue = ack_queue 38 | self.timeout_seconds = timeout_seconds 39 | self.stop_signal = Event() 40 | 41 | def stop(self): 42 | self.stop_signal.set() 43 | 44 | def run(self): 45 | while not self.stop_signal.is_set(): 46 | try: 47 | hostport = None 48 | try: 49 | is_ack, delivery_token, callback \ 50 | = self.ack_queue.get(block=True, timeout=self.timeout_seconds) 51 | hostport = util.get_hostport_from_delivery_token(delivery_token) 52 | util.stats_count(self.tchannel.name, 'consumer_ack_queue.dequeue', hostport, 1) 53 | except Empty: 54 | continue 55 | 56 | ack_id = util.get_ack_id_from_delivery_token(delivery_token) 57 | request = cherami.AckMessagesRequest(ackIds=[ack_id] if is_ack else [], 58 | nackIds=[ack_id] if not is_ack else []) 59 | 60 | util.execute_output_host(tchannel=self.tchannel, 61 | headers=self.headers, 62 | hostport=hostport, 63 | timeout=self.timeout_seconds, 64 | method_name='ackMessages', 65 | request=request) 66 | 67 | callback(AckMessageResult(call_success=True, 68 | is_ack=is_ack, 69 | delivery_token=delivery_token, 70 | error_msg=None)) 71 | 72 | except Exception as e: 73 | self.logger.info({ 74 | 'msg': 'error ack msg from output host', 75 | 'hostport': hostport, 76 | 'ack id': ack_id, 77 | 'is ack': is_ack, 78 | 'traceback': traceback.format_exc(), 79 | 'exception': str(e) 80 | }) 81 | callback(AckMessageResult(call_success=False, 82 | is_ack=is_ack, 83 | delivery_token=delivery_token, 84 | error_msg=str(e))) 85 | -------------------------------------------------------------------------------- /cherami_client/client.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import socket 22 | 23 | from tchannel.sync import TChannel as TChannelSyncClient 24 | from cherami_client.lib import util 25 | from cherami_client import publisher, consumer 26 | 27 | 28 | class Client(object): 29 | 30 | tchannel = None 31 | headers = None 32 | timeout_seconds = 30 33 | 34 | # reconfigure_interval_seconds: this parameter controls how frequent we try to get the latest hosts 35 | # that are serving the destination or consumer group in our background thread 36 | # 37 | # deployment_str: controls the deployment the client will connect to. 38 | # use 'dev' to connect to dev server 39 | # use 'prod' to connect to production 40 | # use 'staging' or 'staging2' to connect to staging/staging2 41 | # 42 | # hyperbahn_host: the path to the hyperbahn host file 43 | # 44 | # By default client will connect to cherami via hyperbahn. If you want to connect to cherami via a 45 | # specific ip/port(for example during local test), you can create a tchannel object with the ip/port 46 | # as known peers 47 | # For example: 48 | # tchannel = TChannelSyncClient(name='my_service', known_peers=['172.17.0.2:4922']) 49 | # client = Client(tchannel, logger) 50 | def __init__(self, 51 | tchannel, 52 | logger, 53 | client_name=None, 54 | headers={}, 55 | timeout_seconds=30, 56 | reconfigure_interval_seconds=10, 57 | deployment_str='prod', 58 | hyperbahn_host='', 59 | ): 60 | self.logger = logger 61 | self.headers = headers 62 | self.deployment_str = deployment_str 63 | self.headers['user-name'] = util.get_username() 64 | self.headers['host-name'] = socket.gethostname() 65 | self.timeout_seconds = timeout_seconds 66 | self.reconfigure_interval_seconds = reconfigure_interval_seconds 67 | 68 | if not tchannel: 69 | if not client_name: 70 | raise Exception("Client name is needed when tchannel not provided") 71 | elif not hyperbahn_host: 72 | raise Exception("Hyperbahn host is needed when tchannel not provided") 73 | else: 74 | self.tchannel = TChannelSyncClient(name=client_name) 75 | self.tchannel.advertise(router_file=hyperbahn_host) 76 | else: 77 | self.tchannel = tchannel 78 | 79 | # close the client connection 80 | def close(self): 81 | pass 82 | 83 | # create a consumer 84 | # Note consumer object should be a singleton 85 | # pre_fetch_count: This controls how many messages we can pre-fetch in total 86 | # ack_message_buffer_size: This controls the ack messages buffer size.i.e.count of pending ack messages 87 | # ack_message_thread_count: This controls how many threads we can have to send ack messages to Cherami. 88 | def create_consumer( 89 | self, 90 | path, 91 | consumer_group_name, 92 | pre_fetch_count=50, 93 | ack_message_buffer_size=50, 94 | ack_message_thread_count=4,): 95 | return consumer.Consumer( 96 | logger=self.logger, 97 | deployment_str=self.deployment_str, 98 | path=path, 99 | consumer_group_name=consumer_group_name, 100 | tchannel=self.tchannel, 101 | headers=self.headers, 102 | pre_fetch_count=pre_fetch_count, 103 | timeout_seconds=self.timeout_seconds, 104 | ack_message_buffer_size=ack_message_buffer_size, 105 | ack_message_thread_count=ack_message_thread_count, 106 | reconfigure_interval_seconds=self.reconfigure_interval_seconds, 107 | ) 108 | 109 | # create a publisher 110 | # Note publisher object should be a singleton 111 | def create_publisher(self, path): 112 | if not path: 113 | raise Exception("Path is needed") 114 | return publisher.Publisher( 115 | logger=self.logger, 116 | path=path, 117 | deployment_str=self.deployment_str, 118 | tchannel=self.tchannel, 119 | headers=self.headers, 120 | timeout_seconds=self.timeout_seconds, 121 | reconfigure_interval_seconds=self.reconfigure_interval_seconds 122 | ) 123 | 124 | def create_destination(self, create_destination_request): 125 | return util.execute_frontend( 126 | self.tchannel, 127 | self.deployment_str, 128 | self.headers, 129 | self.timeout_seconds, 130 | 'createDestination', 131 | create_destination_request) 132 | 133 | def read_destination(self, read_destination_request): 134 | return util.execute_frontend( 135 | self.tchannel, 136 | self.deployment_str, 137 | self.headers, 138 | self.timeout_seconds, 139 | 'readDestination', 140 | read_destination_request) 141 | 142 | def create_consumer_group(self, create_consumer_group_request): 143 | return util.execute_frontend( 144 | self.tchannel, 145 | self.deployment_str, 146 | self.headers, 147 | self.timeout_seconds, 148 | 'createConsumerGroup', 149 | create_consumer_group_request) 150 | 151 | def read_consumer_group(self, read_consumer_group_request): 152 | return util.execute_frontend( 153 | self.tchannel, 154 | self.deployment_str, 155 | self.headers, 156 | self.timeout_seconds, 157 | 'readConsumerGroup', 158 | read_consumer_group_request) 159 | 160 | def purge_DLQ_for_consumer_group(self, purge_DLQ_for_consumer_group_request): 161 | return util.execute_frontend( 162 | self.tchannel, 163 | self.deployment_str, 164 | self.headers, 165 | self.timeout_seconds, 166 | 'purgeDLQForConsumerGroup', 167 | purge_DLQ_for_consumer_group_request) 168 | 169 | def merge_DLQ_for_consumer_group(self, merge_DLQ_for_consumer_group_request): 170 | return util.execute_frontend( 171 | self.tchannel, 172 | self.deployment_str, 173 | self.headers, 174 | self.timeout_seconds, 175 | 'mergeDLQForConsumerGroup', 176 | merge_DLQ_for_consumer_group_request) 177 | -------------------------------------------------------------------------------- /cherami_client/consumer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import time 22 | 23 | from threading import Event 24 | from six.moves import queue 25 | 26 | from clay import stats 27 | from cherami_client.lib import cherami, util 28 | from cherami_client.consumer_thread import ConsumerThread 29 | from cherami_client.ack_thread import AckThread 30 | from cherami_client.reconfigure_thread import ReconfigureThread 31 | from cherami_client.ack_message_result import AckMessageResult 32 | 33 | 34 | class Consumer(object): 35 | def __init__(self, 36 | logger, 37 | deployment_str, 38 | path, 39 | consumer_group_name, 40 | tchannel, 41 | headers, 42 | pre_fetch_count, 43 | timeout_seconds, 44 | ack_message_buffer_size, 45 | ack_message_thread_count, 46 | reconfigure_interval_seconds, 47 | ): 48 | self.logger = logger 49 | self.deployment_str = deployment_str 50 | self.path = path 51 | self.consumer_group_name = consumer_group_name 52 | self.tchannel = tchannel 53 | self.headers = headers 54 | self.pre_fetch_count = pre_fetch_count 55 | self.msg_queue = queue.Queue(pre_fetch_count) 56 | self.msg_batch_size = max(pre_fetch_count / 10, 1) 57 | self.timeout_seconds = timeout_seconds 58 | self.consumer_threads = {} 59 | self.ack_queue = queue.Queue(ack_message_buffer_size) 60 | self.ack_threads_count = ack_message_thread_count 61 | self.ack_threads = [] 62 | 63 | self.reconfigure_signal = Event() 64 | self.reconfigure_interval_seconds = reconfigure_interval_seconds 65 | self.reconfigure_thread = None 66 | 67 | # whether to start the consumer thread. Only set to false in unit test 68 | self.start_consumer_thread = True 69 | 70 | def _do_not_start_consumer_thread(self): 71 | self.start_consumer_thread = False 72 | 73 | def _reconfigure(self): 74 | self.logger.info('consumer reconfiguration started') 75 | 76 | hosts = util.execute_frontend( 77 | self.tchannel, self.deployment_str, {}, self.timeout_seconds, 'readConsumerGroupHosts', 78 | cherami.ReadConsumerGroupHostsRequest( 79 | destinationPath=self.path, 80 | consumerGroupName=self.consumer_group_name 81 | )) 82 | 83 | host_connections = map(lambda h: util.get_connection_key(h), hosts.hostAddresses) \ 84 | if hosts.hostAddresses is not None else [] 85 | host_connection_set = set(host_connections) 86 | existing_connection_set = set(self.consumer_threads.keys()) 87 | missing_connection_set = host_connection_set - existing_connection_set 88 | extra_connection_set = existing_connection_set - host_connection_set 89 | 90 | # clean up 91 | for extra_conn in extra_connection_set: 92 | self.logger.info('cleaning up connection %s', extra_conn) 93 | self.consumer_threads[extra_conn].stop() 94 | del self.consumer_threads[extra_conn] 95 | 96 | # start up 97 | for missing_conn in missing_connection_set: 98 | self.logger.info('creating new connection %s', missing_conn) 99 | consumer_thread = ConsumerThread(tchannel=self.tchannel, 100 | headers=self.headers, 101 | logger=self.logger, 102 | msg_queue=self.msg_queue, 103 | hostport=missing_conn, 104 | path=self.path, 105 | consumer_group_name=self.consumer_group_name, 106 | timeout_seconds=self.timeout_seconds, 107 | msg_batch_size=self.msg_batch_size 108 | ) 109 | self.consumer_threads[missing_conn] = consumer_thread 110 | if self.start_consumer_thread: 111 | consumer_thread.start() 112 | 113 | self.logger.info('consumer reconfiguration succeeded') 114 | 115 | def _start_ack_threads(self): 116 | for i in range(0, self.ack_threads_count): 117 | ack_thread = AckThread(tchannel=self.tchannel, 118 | headers=self.headers, 119 | logger=self.logger, 120 | ack_queue=self.ack_queue, 121 | timeout_seconds=self.timeout_seconds) 122 | ack_thread.start() 123 | self.ack_threads.append(ack_thread) 124 | 125 | # open the consumer. If succeed, we can start to consume messages 126 | # Otherwise, we should retry opening (with backoff) 127 | def open(self): 128 | try: 129 | self._reconfigure() 130 | self.reconfigure_thread = ReconfigureThread( 131 | interval_seconds=self.reconfigure_interval_seconds, 132 | reconfigure_signal=self.reconfigure_signal, 133 | reconfigure_func=self._reconfigure, 134 | logger=self.logger, 135 | ) 136 | self.reconfigure_thread.start() 137 | 138 | self._start_ack_threads() 139 | 140 | self.logger.info('consumer opened') 141 | except Exception as e: 142 | self.logger.exception('Failed to open consumer: %s', e) 143 | self.close() 144 | raise e 145 | 146 | # close the consumer 147 | def close(self): 148 | if self.reconfigure_thread: 149 | self.reconfigure_thread.stop() 150 | 151 | for worker in self.consumer_threads.itervalues(): 152 | worker.stop() 153 | 154 | for ack_thread in self.ack_threads: 155 | ack_thread.stop() 156 | 157 | # Receive messages from cherami. This returns an array of tuple. First value of the tuple is a delivery_token, 158 | # which can be used to ack or nack the message. The second value of the tuple is the actual message, which is a 159 | # cherami.ConsumerMessage(in cherami.thrift) object 160 | def receive(self, num_msgs): 161 | start_time = time.time() 162 | timeout_stats = 'cherami_client_python.{}.receive.timeout'.format(self.tchannel.name) 163 | duration_stats = 'cherami_client_python.{}.receive.duration'.format(self.tchannel.name) 164 | 165 | msgs = [] 166 | end_time = time.time() + self.timeout_seconds 167 | while len(msgs) < num_msgs: 168 | seconds_remaining = end_time - time.time() 169 | if seconds_remaining <= 0: 170 | stats.count(timeout_stats, 1) 171 | stats.timing(duration_stats, util.time_diff_in_ms(start_time, time.time())) 172 | return msgs 173 | try: 174 | msgs.append(self.msg_queue.get(block=True, timeout=seconds_remaining)) 175 | self.msg_queue.task_done() 176 | 177 | util.stats_count(self.tchannel.name, 'consumer_msg_queue.dequeue', None, 1) 178 | except queue.Empty: 179 | pass 180 | stats.timing(duration_stats, util.time_diff_in_ms(start_time, time.time())) 181 | return msgs 182 | 183 | # verify checksum of the message received from cherami 184 | # return true if the data matches checksum. Otherwise return false 185 | # Consumer needs to perform this verification and decide what to do based on returned result 186 | def verify_checksum(self, consumer_message): 187 | if consumer_message.payload and consumer_message.payload.data: 188 | if consumer_message.payload.crc32IEEEDataChecksum: 189 | return util.calc_crc(consumer_message.payload.data, cherami.ChecksumOption.CRC32IEEE) \ 190 | == consumer_message.payload.crc32IEEEDataChecksum 191 | if consumer_message.payload.md5DataChecksum: 192 | return util.calc_crc(consumer_message.payload.data, cherami.ChecksumOption.MD5) \ 193 | == consumer_message.payload.md5DataChecksum 194 | return True 195 | 196 | # Ack can be used by application to Ack a message so it is not delivered to 197 | # any other consumer 198 | def ack(self, delivery_token): 199 | return self._respond(is_ack=True, delivery_token=delivery_token) 200 | 201 | def ack_async(self, delivery_token, callback): 202 | return self._respond_async(is_ack=True, delivery_token=delivery_token, callback=callback) 203 | 204 | # Nack can be used by application to Nack a message so it can be delivered to 205 | # another consumer immediately without waiting for the timeout to expire 206 | def nack(self, delivery_token): 207 | return self._respond(is_ack=False, delivery_token=delivery_token) 208 | 209 | def nack_async(self, delivery_token, callback): 210 | return self._respond_async(is_ack=False, delivery_token=delivery_token, callback=callback) 211 | 212 | def _respond(self, is_ack, delivery_token): 213 | if not delivery_token: 214 | return 215 | 216 | done_signal = Event() 217 | result = [] 218 | 219 | def callback(ack_result): 220 | result.append(ack_result) 221 | done_signal.set() 222 | 223 | self._respond_async(is_ack, delivery_token, callback) 224 | 225 | done = done_signal.wait(self.timeout_seconds) 226 | if not done or not result: 227 | self.logger.info({ 228 | 'msg': 'ack failure', 229 | 'delivery token': delivery_token, 230 | 'error msg': 'timed out' 231 | }) 232 | return False 233 | else: 234 | if result[0].call_success: 235 | return True 236 | else: 237 | self.logger.info({ 238 | 'msg': 'ack failure', 239 | 'delivery token': result[0].delivery_token, 240 | 'error msg': result[0].error_msg 241 | }) 242 | return False 243 | 244 | def _respond_async(self, is_ack, delivery_token, callback): 245 | if delivery_token is None or callback is None: 246 | return 247 | 248 | try: 249 | self.ack_queue.put((is_ack, delivery_token, callback), 250 | block=True, 251 | timeout=self.timeout_seconds) 252 | 253 | hostport = util.get_hostport_from_delivery_token(delivery_token) 254 | util.stats_count(self.tchannel.name, 'consumer_ack_queue.enqueue', hostport, 1) 255 | except queue.Full: 256 | callback(AckMessageResult(call_success=False, 257 | is_ack=True, 258 | delivery_token=delivery_token, 259 | error_msg='ack message buffer is full')) 260 | -------------------------------------------------------------------------------- /cherami_client/consumer_thread.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | from __future__ import absolute_import 22 | 23 | import traceback 24 | from threading import Thread, Event 25 | from six.moves.queue import Full 26 | 27 | from cherami_client.lib import util 28 | from cherami_client.lib import cherami 29 | 30 | 31 | class ConsumerThread(Thread): 32 | def __init__(self, 33 | tchannel, 34 | headers, 35 | logger, 36 | msg_queue, 37 | hostport, 38 | path, 39 | consumer_group_name, 40 | timeout_seconds, 41 | msg_batch_size): 42 | Thread.__init__(self) 43 | self.tchannel = tchannel 44 | self.headers = headers 45 | self.logger = logger 46 | self.msg_queue = msg_queue 47 | self.hostport = hostport 48 | self.path = path 49 | self.consumer_group_name = consumer_group_name 50 | self.timeout_seconds = timeout_seconds 51 | self.msg_batch_size = msg_batch_size 52 | self.stop_signal = Event() 53 | 54 | def stop(self): 55 | self.stop_signal.set() 56 | 57 | def run(self): 58 | request = cherami.ReceiveMessageBatchRequest(destinationPath=self.path, 59 | consumerGroupName=self.consumer_group_name, 60 | maxNumberOfMessages=self.msg_batch_size, 61 | receiveTimeout=max(1, self.timeout_seconds - 1) 62 | ) 63 | while not self.stop_signal.is_set(): 64 | # possible optimization: if we don't have enough capacity in the queue, 65 | # backoff for a bit before pulling from Cherami again 66 | try: 67 | result = util.execute_output_host(tchannel=self.tchannel, 68 | headers=self.headers, 69 | hostport=self.hostport, 70 | timeout=self.timeout_seconds, 71 | method_name='receiveMessageBatch', 72 | request=request) 73 | util.stats_count(self.tchannel.name, 74 | 'receiveMessageBatch.messages', 75 | self.hostport, 76 | len(result.messages)) 77 | 78 | for msg in result.messages: 79 | # if the queue is full, keep trying until there's free slot, or the thread has been shutdown 80 | while not self.stop_signal.is_set(): 81 | try: 82 | self.msg_queue.put((util.create_delivery_token(msg.ackId, self.hostport), msg), 83 | block=True, 84 | timeout=5) 85 | util.stats_count(self.tchannel.name, 86 | 'consumer_msg_queue.enqueue', 87 | self.hostport, 88 | 1) 89 | break 90 | except Full: 91 | pass 92 | except Exception as e: 93 | self.logger.info({ 94 | 'msg': 'error receiving msg from output host', 95 | 'hostport': self.hostport, 96 | 'traceback': traceback.format_exc(), 97 | 'exception': str(e) 98 | }) 99 | -------------------------------------------------------------------------------- /cherami_client/idl/cherami.thrift: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | namespace java com.uber.cherami 22 | 23 | exception EntityNotExistsError { 24 | 1: required string message 25 | } 26 | 27 | exception EntityAlreadyExistsError { 28 | 1: required string message 29 | } 30 | 31 | exception EntityDisabledError { 32 | 1: required string message 33 | } 34 | 35 | exception BadRequestError { 36 | 1: required string message 37 | } 38 | 39 | exception InvalidAddressError { 40 | 1: required string message 41 | } 42 | 43 | exception InternalServiceError { 44 | 1: required string message 45 | } 46 | 47 | exception TimeoutError { 48 | 1: required string message 49 | } 50 | 51 | exception QueueCacheMissError { 52 | 1: required string message 53 | } 54 | 55 | enum Protocol { 56 | TCHANNEL, 57 | WS, // websocket 58 | WSS // websocket secure 59 | } 60 | 61 | enum DestinationStatus { 62 | ENABLED, 63 | DISABLED, 64 | SENDONLY, 65 | RECEIVEONLY 66 | } 67 | 68 | enum ConsumerGroupStatus { 69 | ENABLED, 70 | DISABLED 71 | } 72 | 73 | /** 74 | * We support 2 type of Destinations, backed up by their own BStore implementations. 75 | * DestinationType needs to be specified at the time of creation and is immutable. 76 | **/ 77 | enum DestinationType { 78 | /** 79 | * This type of destination supports very fast throughput but does not support scheduling tasks 80 | * with a delay. It is backed by it's own BStore with WAL for writes and indexed reads on sequence numbers. 81 | * 82 | **/ 83 | PLAIN, 84 | 85 | /** 86 | * This type of destination is designed to support scheduling tasks with a delay. It is backed by a different 87 | * BStore with a WAL and indexed storage based on both Sequence numbers and Schedule Time. These are slightly 88 | * heavier than PLAIN queues and not optimal for scenario's which does not require timers. 89 | **/ 90 | TIMER, 91 | 92 | /** 93 | * !!!!! NOT YET SUPPORTED. !!!!!! 94 | * Limited throughput destination that guarantees fully consistent odered view of messages. 95 | * Each message has associated sequence number. Sequence numbers are guaranteed to be sequential. 96 | * Messages with incorrect sequence number provided by publisher are rejected. 97 | **/ 98 | LOG 99 | } 100 | 101 | enum ConsumerGroupType { 102 | /** 103 | * Each consumer receives disjoint set of messages and acks them individually. 104 | **/ 105 | COMPETING, 106 | 107 | /** 108 | * Each consumer receives all messages from a specified address. Valid for LOG destinations only. 109 | **/ 110 | STREAMING 111 | } 112 | 113 | enum ChecksumOption { 114 | /** 115 | * Use CRC32 IEEE checksum option. 116 | **/ 117 | CRC32IEEE, 118 | /** 119 | * Use md5 checksum option. 120 | **/ 121 | MD5 122 | } 123 | 124 | /** 125 | * This describes the entity and associated configuration, used by client application to send messages. 126 | * 127 | * @param path. Path which uniquely identifies the destination. 128 | * @param type. Type of destination (PLAIN, TIMER). 129 | * @param status. Status of destination. 130 | * @param consumedMessagesRetention. Time in seconds to keep consumed messages before deleting from storage. 131 | * @param unconsumedMessagesRetention. Time in seconds to keep messages that may not have been consumed, before deleting from storage. 132 | * @param createdAt. Time when destination was created. 133 | **/ 134 | struct DestinationDescription { 135 | 1: optional string path 136 | 2: optional DestinationType type 137 | 3: optional DestinationStatus status 138 | 4: optional i32 consumedMessagesRetention 139 | 5: optional i32 unconsumedMessagesRetention 140 | 6: optional string destinationUUID 141 | 7: optional string ownerEmail 142 | 8: optional ChecksumOption checksumOption = ChecksumOption.CRC32IEEE 143 | 10: optional bool isMultiZone 144 | 11: optional DestinationZoneConfigs zoneConfigs 145 | 20: optional SchemaInfo schemaInfo // Latest schema for this destination 146 | } 147 | 148 | struct SchemaInfo { 149 | 1: optional string type 150 | 2: optional i32 version 151 | 3: optional binary data // UTF-8 encoded byte array that stores schema data 152 | 4: optional string source // A URI that refers to the downloading source of schema 153 | 5: optional i64 (js.type = "Long") createdTimeUtc 154 | } 155 | 156 | struct DestinationZoneConfig { 157 | // 10: optional Zone deprecatedZoneName 158 | 11: optional string zone // zone name 159 | 20: optional bool allowPublish // whether we allow publishing from this zone 160 | 30: optional bool allowConsume // whether we allow consuming from this zone 161 | 40: optional bool alwaysReplicateTo // whether we need to replicate to this zone even if there’s no consumer group in that zone 162 | 50: optional i32 remoteExtentReplicaNum // the # of replica we need to have for remote extent 163 | } 164 | 165 | struct DestinationZoneConfigs { 166 | 10: optional list configs 167 | } 168 | 169 | struct CreateDestinationRequest { 170 | 1: optional string path 171 | 2: optional DestinationType type 172 | 3: optional i32 consumedMessagesRetention 173 | 4: optional i32 unconsumedMessagesRetention 174 | 5: optional string ownerEmail 175 | 6: optional ChecksumOption checksumOption = ChecksumOption.CRC32IEEE 176 | 10: optional bool isMultiZone 177 | 11: optional DestinationZoneConfigs zoneConfigs 178 | 20: optional SchemaInfo schemaInfo 179 | } 180 | 181 | struct ReadDestinationRequest { 182 | 1: optional string path 183 | } 184 | 185 | struct UpdateDestinationRequest { 186 | 1: optional string path 187 | 2: optional DestinationStatus status 188 | 3: optional i32 consumedMessagesRetention 189 | 4: optional i32 unconsumedMessagesRetention 190 | 5: optional string ownerEmail 191 | 6: optional ChecksumOption checksumOption 192 | 10: optional SchemaInfo schemaInfo 193 | } 194 | 195 | struct DeleteDestinationRequest { 196 | 1: optional string path 197 | } 198 | 199 | struct ListDestinationsRequest { 200 | 1: optional string prefix 201 | 2: optional binary pageToken 202 | 3: optional i64 (js.type = "Long") limit 203 | } 204 | 205 | struct ListDestinationsResult { 206 | 1: optional list destinations 207 | 2: optional binary nextPageToken 208 | } 209 | 210 | /** 211 | * This describes the entity and associated configuration, used by client application to consume messages from 212 | * a destination. 213 | * 214 | * @param destinationPath. Path which uniquely identifies the destination. 215 | * @param consumerGroupName. Unique identifier for each group of consumers. 216 | * @param startFrom. Timestamp used to start consuming messages from destination. This needs to be provided during 217 | * registration of the ConsumerGroup and cannot be updated later. 218 | * @param lockTimeoutInSeconds. Seconds to wait before redelivering message to another consumer. 219 | * @param maxDeliveryCount. Number of times trying to deliver the message without Ack before giving up and moving the 220 | * message to DLQ. 221 | * @param skipOlderMessagesInSeconds. This is useful for consumers who always wants to keep up and don't care about 222 | * backlog older than certain duration. 223 | * @param createdAt. Time when ConsumerGroup was registered. 224 | **/ 225 | struct ConsumerGroupDescription { 226 | 1: optional string destinationPath 227 | 2: optional string consumerGroupName 228 | 3: optional i64 (js.type = "Long") startFrom 229 | 4: optional ConsumerGroupStatus status 230 | 5: optional i32 lockTimeoutInSeconds 231 | 6: optional i32 maxDeliveryCount 232 | 7: optional i32 skipOlderMessagesInSeconds 233 | 8: optional string deadLetterQueueDestinationUUID 234 | 9: optional string destinationUUID 235 | 10: optional string consumerGroupUUID 236 | 11: optional string ownerEmail 237 | 12: optional ConsumerGroupType consumerGroupType 238 | 20: optional bool isMultiZone 239 | 21: optional ConsumerGroupZoneConfigs zoneConfigs 240 | } 241 | 242 | struct ConsumerGroupZoneConfig { 243 | // 10: optional Zone deprecatedZoneName 244 | 11: optional string zone // zone name 245 | 20: optional bool visible // whether the consumer group is visible in this zone 246 | } 247 | 248 | struct ConsumerGroupZoneConfigs { 249 | 10: optional list configs 250 | // 20: optional Zone deprecatedActiveZone 251 | 21: optional string activeZone 252 | } 253 | 254 | struct CreateConsumerGroupRequest { 255 | 1: optional string destinationPath 256 | 2: optional string consumerGroupName 257 | 3: optional i64 (js.type = "Long") startFrom 258 | 4: optional i32 lockTimeoutInSeconds 259 | 5: optional i32 maxDeliveryCount 260 | 6: optional i32 skipOlderMessagesInSeconds 261 | 7: optional string ownerEmail 262 | 8: optional ConsumerGroupType consumerGroupType // Default is COMPETING 263 | 10: optional bool isMultiZone 264 | 11: optional ConsumerGroupZoneConfigs zoneConfigs 265 | } 266 | 267 | struct ReadConsumerGroupRequest { 268 | 1: optional string destinationPath 269 | 2: optional string consumerGroupName 270 | } 271 | 272 | struct UpdateConsumerGroupRequest { 273 | 1: optional string destinationPath 274 | 2: optional string consumerGroupName 275 | 3: optional ConsumerGroupStatus status 276 | 4: optional i32 lockTimeoutInSeconds 277 | 5: optional i32 maxDeliveryCount 278 | 6: optional i32 skipOlderMessagesInSeconds 279 | 7: optional string ownerEmail 280 | } 281 | 282 | struct DeleteConsumerGroupRequest { 283 | 1: optional string destinationPath 284 | 2: optional string consumerGroupName 285 | } 286 | 287 | struct ListConsumerGroupRequest { 288 | 1: optional string destinationPath 289 | 2: optional string consumerGroupName 290 | 3: optional binary pageToken 291 | 4: optional i64 (js.type = "Long") limit 292 | } 293 | 294 | struct ListConsumerGroupResult { 295 | 1: optional list consumerGroups 296 | 2: optional binary nextPageToken 297 | } 298 | 299 | struct PurgeDLQForConsumerGroupRequest { 300 | 1: optional string destinationPath 301 | 2: optional string consumerGroupName 302 | } 303 | 304 | struct MergeDLQForConsumerGroupRequest { 305 | 1: optional string destinationPath 306 | 2: optional string consumerGroupName 307 | } 308 | 309 | /** 310 | * Address of BIn/BOut nodes used by client applications to open direct streams for publishing/consuming messages. 311 | * Publishers are expected to discover HostAddress for all BIn nodes serving a particular destination by calling 312 | * ReadDestinationHosts API on the Frontend and then use the address to directly open a stream to BIn node for 313 | * publishing messages. Similarly Consumers are expected to discover HostAddress for all BOut nodes serving a 314 | * particular pair of Destination-ConsumerGroup by calling the ReadConsumerGroupHosts API on the Frontend and then 315 | * use the address to directly open a stream to BOut node for consuming messages. 316 | **/ 317 | struct HostAddress { 318 | 1: optional string host 319 | 2: optional i32 port 320 | } 321 | 322 | struct HostProtocol { 323 | 10: optional list hostAddresses 324 | 20: optional Protocol protocol 325 | 30: optional bool deprecated 326 | } 327 | 328 | struct ReadDestinationHostsRequest { 329 | 1: optional string path 330 | } 331 | 332 | struct ReadDestinationHostsResult { 333 | 1: optional list hostAddresses // To be deprecated by hostProtocols 334 | 10: optional list hostProtocols 335 | } 336 | 337 | struct ReadPublisherOptionsRequest { 338 | 1: optional string path 339 | 2: optional i32 schema_version // The schema version publisher code is configured to 340 | } 341 | 342 | struct ReadPublisherOptionsResult { 343 | 1: optional list hostAddresses // To be deprecated by hostProtocols 344 | 10: optional list hostProtocols 345 | 20: optional ChecksumOption checksumOption 346 | 31: optional SchemaInfo schemaInfo // When publish, publisher can use any version not higher than this one 347 | } 348 | 349 | struct ReadConsumerGroupHostsRequest { 350 | 1: optional string destinationPath 351 | 2: optional string consumerGroupName 352 | } 353 | 354 | struct ReadConsumerGroupHostsResult { 355 | 1: optional list hostAddresses // To be deprecated by hostProtocols 356 | 10: optional list hostProtocols 357 | } 358 | 359 | enum Status { 360 | OK, 361 | FAILED, 362 | TIMEDOUT, 363 | THROTTLED 364 | } 365 | 366 | struct PutMessage { 367 | 1: optional string id // This is unique identifier of message 368 | 2: optional i32 delayMessageInSeconds // Applies to TIMER destinations only. 369 | 3: optional binary data 370 | 4: optional map userContext // This is user specified context to pass through 371 | // Put is rejected if previousMessageId doesn't match the id of the previous message. 372 | // Applies to LOG destinations only. 373 | 5: optional string previousMessageId 374 | 6: optional i64 (js.type = "Long") crc32IEEEDataChecksum // This is the crc32 checksum using IEEE polynomial 375 | // 7: optional i64 (js.type = "Long") crc32CastagnoliDataChecksum // This is the crc32 checksum using Castagnoli polynomial 376 | 8: optional binary md5DataChecksum // This is the md5 checksum for data field 377 | 10: optional i32 schemaVersion // Version of the schema that encodes binary data, default to 0 means no schema used 378 | } 379 | 380 | struct PutMessageAck { 381 | 1: optional string id // This is unique identifier of message 382 | 2: optional Status status 383 | 3: optional string message // This is for error message 384 | 4: optional string receipt 385 | 5: optional map userContext // This is user specified context to pass through 386 | 6: optional i64 (js.type = "Long") lsn // LOG destination Log Sequence Number. 387 | 7: optional i64 (js.type = "Long") address // LOG destination message address 388 | } 389 | 390 | exception InvalidAckIdError { 391 | 1: required string message 392 | 2: optional list ackIds 393 | 3: optional list nackIds 394 | } 395 | 396 | struct ConsumerMessage { 397 | 1: optional i64 (js.type = "Long") enqueueTimeUtc 398 | 2: optional string ackId // Global unique identifier to ack messages. Empty for STREAM consumer group. 399 | 3: optional PutMessage payload 400 | 4: optional i64 (js.type = "Long") lsn // LOG destination Log Sequence Number. 401 | 5: optional i64 (js.type = "Long") address // LOG destination message address 402 | } 403 | 404 | struct AckMessagesRequest { 405 | 1: optional list ackIds 406 | 2: optional list nackIds 407 | } 408 | 409 | struct SetConsumedMessagesRequest { 410 | 1: optional string destinationPath 411 | 2: optional string consumerGroupName 412 | 3: optional i64 (js.type = "Long") addressInclusive // Address of the last message to mark as consumed 413 | } 414 | 415 | struct PutMessageBatchRequest { 416 | 1: optional string destinationPath 417 | 2: optional list messages 418 | } 419 | 420 | struct PutMessageBatchResult { 421 | 1: optional list failedMessages 422 | 2: optional list successMessages 423 | } 424 | 425 | struct ReceiveMessageBatchRequest { 426 | 1: optional string destinationPath 427 | 2: optional string consumerGroupName 428 | 3: optional i32 maxNumberOfMessages 429 | 4: optional i32 receiveTimeout 430 | } 431 | 432 | struct ReceiveMessageBatchResult { 433 | 1: optional list messages 434 | } 435 | 436 | struct GetQueueDepthInfoRequest { 437 | 1: optional string key 438 | } 439 | 440 | struct GetQueueDepthInfoResult { 441 | 1: optional string value 442 | } 443 | 444 | service BFrontend { 445 | // returns the ip:port for this frontend (for discovery purpose) 446 | string HostPort() 447 | 448 | /*********************************************/ 449 | /***** Destination CRUD **********************/ 450 | DestinationDescription createDestination(1: CreateDestinationRequest createRequest) 451 | throws ( 452 | 1: EntityAlreadyExistsError entityExistsError, 453 | 2: BadRequestError requestError) 454 | 455 | DestinationDescription readDestination(1: ReadDestinationRequest getRequest) 456 | throws ( 457 | 1: EntityNotExistsError entityError, 458 | 2: BadRequestError requestError) 459 | 460 | DestinationDescription updateDestination(1: UpdateDestinationRequest updateRequest) 461 | throws ( 462 | 1: EntityNotExistsError entityError, 463 | 2: BadRequestError requestError) 464 | 465 | void deleteDestination(1: DeleteDestinationRequest deleteRequest) 466 | throws ( 467 | 1: EntityNotExistsError entityError, 468 | 2: BadRequestError requestError) 469 | 470 | ListDestinationsResult listDestinations(1: ListDestinationsRequest listRequest) 471 | throws ( 472 | 1: BadRequestError requestError) 473 | /*********************************************/ 474 | 475 | /*********************************************/ 476 | /***** ConsumerGroup CRUD ********************/ 477 | ConsumerGroupDescription createConsumerGroup(1: CreateConsumerGroupRequest registerRequest) 478 | throws ( 479 | 1: EntityAlreadyExistsError entityExistsError, 480 | 2: BadRequestError requestError) 481 | 482 | ConsumerGroupDescription readConsumerGroup(1: ReadConsumerGroupRequest getRequest) 483 | throws ( 484 | 1: EntityNotExistsError entityError, 485 | 2: BadRequestError requestError) 486 | 487 | ConsumerGroupDescription updateConsumerGroup(1: UpdateConsumerGroupRequest updateRequest) 488 | throws ( 489 | 1: EntityNotExistsError entityError, 490 | 2: BadRequestError requestError) 491 | 492 | void deleteConsumerGroup(1: DeleteConsumerGroupRequest deleteRequest) 493 | throws ( 494 | 1: EntityNotExistsError entityError, 495 | 2: BadRequestError requestError) 496 | 497 | ListConsumerGroupResult listConsumerGroups(1: ListConsumerGroupRequest listRequest) 498 | throws ( 499 | 1: BadRequestError requestError) 500 | 501 | /*********************************************/ 502 | 503 | /** 504 | * readDestinationHosts will be replaced by readPublisherOptions soon 505 | **/ 506 | ReadDestinationHostsResult readDestinationHosts(1: ReadDestinationHostsRequest getHostsRequest) 507 | throws ( 508 | 1: EntityNotExistsError entityError, 509 | 2: EntityDisabledError entityDisabled, 510 | 3: BadRequestError requestError) 511 | 512 | ReadPublisherOptionsResult readPublisherOptions(1: ReadPublisherOptionsRequest getPublisherOptionsRequest) 513 | throws ( 514 | 1: EntityNotExistsError entityError, 515 | 2: EntityDisabledError entityDisabled, 516 | 3: BadRequestError requestError) 517 | 518 | ReadConsumerGroupHostsResult readConsumerGroupHosts(1: ReadConsumerGroupHostsRequest getHostsRequest) 519 | throws ( 520 | 1: EntityNotExistsError entityError, 521 | 2: EntityDisabledError entityDisabled, 522 | 3: BadRequestError requestError) 523 | 524 | /*********************************************/ 525 | /***************** DLQ Management ************/ 526 | 527 | void purgeDLQForConsumerGroup(1: PurgeDLQForConsumerGroupRequest purgeRequest) 528 | throws ( 529 | 1: EntityNotExistsError entityError, 530 | 2: BadRequestError requestError) 531 | 532 | void mergeDLQForConsumerGroup(1: MergeDLQForConsumerGroupRequest mergeRequest) 533 | throws ( 534 | 1: EntityNotExistsError entityError, 535 | 2: BadRequestError requestError) 536 | 537 | /*************************************************************************/ 538 | /*********** Queue Information ******************************/ 539 | GetQueueDepthInfoResult getQueueDepthInfo(1: GetQueueDepthInfoRequest getQueueDepthInfoRequest) 540 | throws ( 541 | 1: QueueCacheMissError cacheMissError, 542 | 2: BadRequestError requestError) 543 | 544 | } 545 | 546 | struct PutMessageStream {} 547 | 548 | struct ReconfigureInfo { 549 | 1: optional string updateUUID 550 | } 551 | 552 | enum InputHostCommandType { 553 | ACK, 554 | RECONFIGURE 555 | } 556 | 557 | struct InputHostCommand { 558 | 1: optional InputHostCommandType type 559 | 2: optional PutMessageAck ack 560 | 3: optional ReconfigureInfo reconfigure 561 | } 562 | 563 | struct InputHostCommandStream {} 564 | 565 | service BIn { 566 | /** 567 | * Streaming support for Thrift only allows a single parameter if the request is streamed. 568 | * HACK: Use Thrift context to pass the following paramters: 569 | * 1) string path 570 | **/ 571 | InputHostCommandStream openPublisherStream( 572 | //1: string path, // TODO: this is not supported 573 | 1: PutMessageStream messages) 574 | throws ( 575 | 1: EntityNotExistsError entityError, 576 | 2: EntityDisabledError entityDisabled, 577 | 3: BadRequestError requestError) 578 | 579 | /** 580 | * Non-streaming publish API 581 | **/ 582 | PutMessageBatchResult putMessageBatch(1: PutMessageBatchRequest request) 583 | throws ( 584 | 1: EntityNotExistsError entityError, 585 | 2: EntityDisabledError entityDisabled, 586 | 3: BadRequestError requestError, 587 | 4: InternalServiceError internalServiceError) 588 | } 589 | 590 | 591 | enum OutputHostCommandType { 592 | MESSAGE, 593 | RECONFIGURE, 594 | // No more backlogged messages. Inserted into the sream if insertEndOfStreamMarker is true. 595 | END_OF_STREAM // STREAMING consumer only. 596 | } 597 | 598 | struct OutputHostCommand { 599 | 1: optional OutputHostCommandType type 600 | 2: optional ConsumerMessage message 601 | 3: optional ReconfigureInfo reconfigure 602 | } 603 | 604 | struct OutputHostCommandStream {} 605 | 606 | struct ControlFlow { 607 | 1: optional i32 credits 608 | } 609 | 610 | struct ControlFlowStream {} 611 | 612 | service BOut { 613 | /** 614 | * Streaming support for Thrift only allows a single parameter if the request is streamed. 615 | * HACK: Use Thrift context to pass the following paramters: 616 | * 1) string path 617 | * 2) string consumerGroupName 618 | **/ 619 | OutputHostCommandStream openConsumerStream( 620 | // TODO: Not Supported for tchannel streaming 621 | //1: string path, 622 | //2: string consumerGroupName, 623 | 1: ControlFlowStream controlFlows) 624 | throws ( 625 | 1: EntityNotExistsError entityError, 626 | 2: EntityDisabledError entityDisabled, 627 | 3: BadRequestError requestError) 628 | 629 | // Only applicable for STREAMING consumer group type 630 | OutputHostCommandStream openStreamingConsumerStream( 631 | // TODO: Not Supported for tchannel streaming 632 | // 1: string path, 633 | // 2: string consumerGroupName, 634 | // 3) i64 address to read from. Address is taken from ConsumerMessage.address. 635 | // If address is not specified then the stream is consumed from the last known message. 636 | // 4) bool insertEndOfStreamMarker when set to true END_OF_STREAM command is sent after the last backlogged message 637 | 1: ControlFlowStream controlFlows) 638 | throws ( 639 | 1: EntityNotExistsError entityError, 640 | 2: EntityDisabledError entityDisabled, 641 | 3: BadRequestError requestError) 642 | 643 | void ackMessages(1: AckMessagesRequest ackRequest) 644 | throws ( 645 | 1: InvalidAckIdError entityError, 646 | 2: BadRequestError requestError) 647 | 648 | // STREAMING consumer only. 649 | // Mark all messages up to specified address as consumed by the consumer group. 650 | void setConsumedMessages(1: SetConsumedMessagesRequest request) 651 | throws ( 652 | 1: InvalidAddressError entityError, 653 | 2: BadRequestError requestError) 654 | 655 | /** 656 | * Non-streaming consume API 657 | **/ 658 | ReceiveMessageBatchResult receiveMessageBatch(1: ReceiveMessageBatchRequest request) 659 | throws ( 660 | 1: EntityNotExistsError entityError, 661 | 2: EntityDisabledError entityDisabled, 662 | 3: BadRequestError requestError, 663 | 4: TimeoutError timeoutError) 664 | } 665 | -------------------------------------------------------------------------------- /cherami_client/lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/cherami-client-python/a0b708afdc228611f8d041f5581163c5e5c7757e/cherami_client/lib/__init__.py -------------------------------------------------------------------------------- /cherami_client/lib/cherami.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | from __future__ import absolute_import 22 | 23 | import os 24 | import sys 25 | 26 | from tchannel import thrift 27 | 28 | sys.modules[__name__] = thrift.load( 29 | path=os.path.join( 30 | os.path.dirname(__file__), 31 | '../idl/cherami.thrift', 32 | ), 33 | service='cherami-frontendhost', 34 | ) 35 | -------------------------------------------------------------------------------- /cherami_client/lib/cherami_frontend.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | from __future__ import absolute_import 22 | 23 | import os 24 | 25 | from tchannel import thrift 26 | 27 | thrift_file = '../idl/cherami.thrift' 28 | default_service_name = 'cherami-frontendhost' 29 | 30 | frontend_modules = {'': thrift.load( 31 | path=os.path.join( 32 | os.path.dirname(__file__), 33 | thrift_file, 34 | ), 35 | service=default_service_name, 36 | )} 37 | 38 | 39 | def load_frontend(env=''): 40 | if env is None or env.lower().startswith('prod') or env.lower().startswith('dev'): 41 | env = '' 42 | 43 | if env in frontend_modules: 44 | return frontend_modules[env] 45 | 46 | service_name = default_service_name 47 | if env: 48 | service_name += '_' 49 | service_name += env 50 | 51 | frontend_modules[env] = thrift.load( 52 | path=os.path.join( 53 | os.path.dirname(__file__), 54 | thrift_file, 55 | ), 56 | service=service_name, 57 | ) 58 | 59 | return frontend_modules[env] 60 | -------------------------------------------------------------------------------- /cherami_client/lib/cherami_input.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | from __future__ import absolute_import 22 | 23 | import os 24 | import sys 25 | 26 | from tchannel import thrift 27 | 28 | sys.modules[__name__] = thrift.load( 29 | path=os.path.join( 30 | os.path.dirname(__file__), 31 | '../idl/cherami.thrift', 32 | ), 33 | service='cherami-inputhost', 34 | ) 35 | -------------------------------------------------------------------------------- /cherami_client/lib/cherami_output.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | from __future__ import absolute_import 22 | 23 | import os 24 | import sys 25 | 26 | from tchannel import thrift 27 | 28 | sys.modules[__name__] = thrift.load( 29 | path=os.path.join( 30 | os.path.dirname(__file__), 31 | '../idl/cherami.thrift', 32 | ), 33 | service='cherami-outputhost', 34 | ) 35 | -------------------------------------------------------------------------------- /cherami_client/lib/util.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import os 22 | import pwd 23 | import time 24 | 25 | # crc libs 26 | import zlib 27 | import hashlib 28 | 29 | from cherami_client.lib import cherami, cherami_output, cherami_input, cherami_frontend 30 | from clay import stats 31 | 32 | 33 | # helper to execute thrift call 34 | def execute_frontend(tchannel, deployment_str, headers, timeout, method_name, request): 35 | frontend_module = cherami_frontend.load_frontend(deployment_str) 36 | method = getattr(frontend_module.BFrontend, method_name) 37 | if not callable(method): 38 | raise Exception("Not a valid callable method: " + method_name) 39 | 40 | start_time = time.time() 41 | try: 42 | stats_count(tchannel.name, '{}.calls'.format(method_name), None, 1) 43 | 44 | result = tchannel.thrift(method(request), headers=headers, timeout=timeout).result().body 45 | 46 | stats_count(tchannel.name, '{}.success'.format(method_name), None, 1) 47 | stats_timing(tchannel.name, '{}.duration.success'.format(method_name), start_time) 48 | 49 | return result 50 | except Exception: 51 | stats_count(tchannel.name, '{}.exception'.format(method_name), None, 1) 52 | stats_timing(tchannel.name, '{}.duration.exception'.format(method_name), start_time) 53 | raise 54 | 55 | 56 | def execute_input_host(tchannel, headers, hostport, timeout, method_name, request): 57 | method = getattr(cherami_input.BIn, method_name) 58 | if not callable(method): 59 | raise Exception("Not a valid callable method: " + method_name) 60 | 61 | start_time = time.time() 62 | try: 63 | stats_count(tchannel.name, '{}.calls'.format(method_name), hostport, 1) 64 | 65 | result = tchannel.thrift(method(request), headers=headers, timeout=timeout, hostport=hostport).result().body 66 | 67 | stats_count(tchannel.name, '{}.success'.format(method_name), hostport, 1) 68 | stats_timing(tchannel.name, '{}.duration.success'.format(method_name), start_time) 69 | 70 | return result 71 | except Exception: 72 | stats_count(tchannel.name, '{}.exception'.format(method_name), hostport, 1) 73 | stats_timing(tchannel.name, '{}.duration.exception'.format(method_name), start_time) 74 | raise 75 | 76 | 77 | def execute_output_host(tchannel, headers, hostport, timeout, method_name, request): 78 | method = getattr(cherami_output.BOut, method_name) 79 | if not callable(method): 80 | raise Exception("Not a valid callable method: " + method_name) 81 | 82 | start_time = time.time() 83 | try: 84 | stats_count(tchannel.name, '{}.calls'.format(method_name), hostport, 1) 85 | 86 | result = tchannel.thrift(method(request), headers=headers, timeout=timeout, hostport=hostport).result().body 87 | 88 | stats_count(tchannel.name, '{}.success'.format(method_name), hostport, 1) 89 | stats_timing(tchannel.name, '{}.duration.success'.format(method_name), start_time) 90 | 91 | return result 92 | except Exception: 93 | stats_count(tchannel.name, '{}.exception'.format(method_name), hostport, 1) 94 | stats_timing(tchannel.name, '{}.duration.exception'.format(method_name), start_time) 95 | raise 96 | 97 | 98 | def get_connection_key(host): 99 | return "{0}:{1}".format(host.host, host.port) 100 | 101 | 102 | def create_failed_message_ack(id, message): 103 | return cherami.PutMessageAck( 104 | id=id, 105 | status=cherami.Status.FAILED, 106 | message=message, 107 | ) 108 | 109 | 110 | def create_timeout_message_ack(id): 111 | return cherami.PutMessageAck( 112 | id=id, 113 | status=cherami.Status.TIMEDOUT, 114 | message='timeout', 115 | ) 116 | 117 | 118 | def create_delivery_token(ack_id, hostport): 119 | return (ack_id, hostport) 120 | 121 | 122 | def get_ack_id_from_delivery_token(delivery_token): 123 | return delivery_token[0] 124 | 125 | 126 | def get_hostport_from_delivery_token(delivery_token): 127 | return delivery_token[1] 128 | 129 | 130 | def stats_count(client_name, stats_name, hostport, count): 131 | overall_stats = 'cherami_client_python.{}.{}'.format(client_name, stats_name) 132 | stats.count(overall_stats, count) 133 | 134 | if hostport: 135 | hostport_stats = 'cherami_client_python.{}.{}.{}'\ 136 | .format(client_name, hostport.replace('.', '_').replace(':', '_'), stats_name) 137 | stats.count(hostport_stats, count) 138 | 139 | 140 | def stats_timing(client_name, stats_name, start_time): 141 | stat_name = 'cherami_client_python.{}.{}'.format(client_name, stats_name) 142 | stats.timing(stat_name, time_diff_in_ms(start_time, time.time())) 143 | 144 | 145 | def time_diff_in_ms(t1, t2): 146 | """Calculate the difference between two timestamps generated by time.time(). 147 | 148 | Returned value will be a float rounded to 2 digits after point, representing 149 | number of milliseconds between the two timestamps. 150 | 151 | :type t1: float 152 | :type t2: float 153 | :rtype: float 154 | """ 155 | return round((t2 - t1) * 1000.0, 2) 156 | 157 | 158 | def calc_crc(data, crc_type): 159 | if crc_type == cherami.ChecksumOption.CRC32IEEE: 160 | # Before python 3.0, the zlib.crc32() returns crc with range [-2**31, 2**31-1], 161 | # which is incompatible with python 3.0 and GoLang implementation 162 | # So we need to mask the return value with 0xffffffff. More on https://docs.python.org/2/library/zlib.html 163 | return zlib.crc32(data) & 0xffffffff 164 | if crc_type == cherami.ChecksumOption.MD5: 165 | return hashlib.md5(data).digest() 166 | return None 167 | 168 | 169 | def get_username(): 170 | """Gets the username of the user who owns the running process. 171 | 172 | Inspired by https://hg.python.org/cpython/file/3.5/Lib/getpass.py#l155. 173 | """ 174 | 175 | for environ_key in ('LOGNAME', 'USER', 'LNAME', 'USERNAME'): 176 | username = os.environ.get(environ_key) 177 | if username: 178 | return username 179 | 180 | user_id = os.getuid() 181 | return pwd.getpwuid(user_id)[0] 182 | -------------------------------------------------------------------------------- /cherami_client/publisher.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import threading 22 | 23 | from six.moves import queue 24 | from cherami_client.lib import cherami, cherami_input, util 25 | from cherami_client.publisher_thread import PublisherThread 26 | from cherami_client.reconfigure_thread import ReconfigureThread 27 | 28 | 29 | class Publisher(object): 30 | 31 | def __init__(self, 32 | logger, 33 | path, 34 | tchannel, 35 | deployment_str, 36 | headers, 37 | timeout_seconds, 38 | reconfigure_interval_seconds): 39 | self.logger = logger 40 | self.path = path 41 | self.tchannel = tchannel 42 | self.deployment_str = deployment_str 43 | self.headers = headers 44 | self.timeout_seconds = timeout_seconds 45 | self.task_queue = queue.Queue() 46 | self.workers = {} 47 | self.reconfigure_signal = threading.Event() 48 | self.reconfigure_interval_seconds = reconfigure_interval_seconds 49 | self.reconfigure_thread = None 50 | 51 | def _reconfigure(self): 52 | self.logger.info('publisher reconfiguration started') 53 | result = util.execute_frontend( 54 | self.tchannel, self.deployment_str, self.headers, self.timeout_seconds, 'readPublisherOptions', 55 | cherami.ReadPublisherOptionsRequest( 56 | path=self.path, 57 | )) 58 | 59 | hostAddresses = [] 60 | for host_protocol in result.hostProtocols: 61 | if host_protocol.protocol == cherami.Protocol.TCHANNEL: 62 | hostAddresses = host_protocol.hostAddresses 63 | break 64 | 65 | if not hostAddresses: 66 | raise Exception("tchannel protocol is not supported by cherami server") 67 | 68 | host_connection_set = set(map(lambda h: util.get_connection_key(h), hostAddresses)) 69 | existing_connection_set = set(self.workers.keys()) 70 | missing_connection_set = host_connection_set - existing_connection_set 71 | extra_connection_set = existing_connection_set - host_connection_set 72 | 73 | # clean up 74 | for extra_conn in extra_connection_set: 75 | self.logger.info('cleaning up connection %s', extra_conn) 76 | self.workers[extra_conn].stop() 77 | del self.workers[extra_conn] 78 | 79 | # start up 80 | for missing_conn in missing_connection_set: 81 | self.logger.info('creating new connection %s', missing_conn) 82 | worker = PublisherThread( 83 | path=self.path, 84 | task_queue=self.task_queue, 85 | tchannel=self.tchannel, 86 | hostport=missing_conn, 87 | headers=self.headers, 88 | timeout_seconds=self.timeout_seconds, 89 | checksum_option=result.checksumOption 90 | ) 91 | self.workers[missing_conn] = worker 92 | worker.start() 93 | 94 | self.logger.info('publisher reconfiguration succeeded') 95 | 96 | # open the publisher. If succeed, we can start to publish messages 97 | # Otherwise, we should retry opening (with backoff) 98 | def open(self): 99 | try: 100 | self._reconfigure() 101 | self.reconfigure_thread = ReconfigureThread( 102 | interval_seconds=self.reconfigure_interval_seconds, 103 | reconfigure_signal=self.reconfigure_signal, 104 | reconfigure_func=self._reconfigure, 105 | logger=self.logger, 106 | ) 107 | self.reconfigure_thread.start() 108 | except Exception as e: 109 | self.logger.exception('Failed to open publisher: %s', e) 110 | self.close() 111 | raise e 112 | 113 | # close the publisher 114 | def close(self): 115 | if self.reconfigure_thread: 116 | self.reconfigure_thread.stop() 117 | for worker in self.workers.itervalues(): 118 | worker.stop() 119 | 120 | # publish a message. Returns an ack(type is cherami.PutMessageAck) 121 | # the Status field of the ack indicates whether the publish was successful or not 122 | # id: an identifier client can use to identify messages \ 123 | # (cherami doesn't care about this field but just pass through) 124 | # data: message payload 125 | # user context: user specified context to pass through 126 | def publish(self, id, data, userContext={}): 127 | done_signal = threading.Event() 128 | result = [] 129 | 130 | def done_callback(r): 131 | result.append(r) 132 | done_signal.set() 133 | 134 | # publish and later on wait 135 | self.publish_async(id, data, done_callback, userContext) 136 | 137 | done = done_signal.wait(self.timeout_seconds) 138 | if not done: 139 | return util.create_timeout_message_ack(id) 140 | if len(result) == 0: 141 | return util.create_failed_message_ack(id, 'unexpected: callback does not carry result') 142 | return result[0] 143 | 144 | # asynchronously publish a message. 145 | # A callback function needs to be provided(it expects a cherami.PutMessageAck object as parameter) 146 | def publish_async(self, id, data, callback, userContext={}): 147 | msg = cherami_input.PutMessage( 148 | id=id, 149 | delayMessageInSeconds=0, 150 | data=data, 151 | userContext=userContext 152 | ) 153 | self.task_queue.put((msg, callback)) 154 | -------------------------------------------------------------------------------- /cherami_client/publisher_thread.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import threading 22 | import traceback 23 | from datetime import datetime 24 | from six.moves.queue import Empty 25 | 26 | from cherami_client.lib import cherami, cherami_input, util 27 | 28 | 29 | class PublisherThread(threading.Thread): 30 | def __init__(self, 31 | path, 32 | task_queue, 33 | tchannel, 34 | hostport, 35 | headers, 36 | timeout_seconds, 37 | checksum_option): 38 | threading.Thread.__init__(self) 39 | self.path = path 40 | self.task_queue = task_queue 41 | self.tchannel = tchannel 42 | self.hostport = hostport 43 | self.headers = headers 44 | self.timeout_seconds = timeout_seconds 45 | self.checksum_option = checksum_option 46 | self.stop_signal = threading.Event() 47 | self.thread_start_time = datetime.now() 48 | 49 | def stop(self): 50 | self.stop_signal.set() 51 | 52 | def run(self): 53 | while not self.stop_signal.is_set(): 54 | try: 55 | # remove from queue regardless 56 | msg, callback = self.task_queue.get(block=True, timeout=5) 57 | self.task_queue.task_done() 58 | 59 | if self.checksum_option == cherami.ChecksumOption.CRC32IEEE: 60 | msg.crc32IEEEDataChecksum = util.calc_crc(msg.data, self.checksum_option) 61 | elif self.checksum_option == cherami.ChecksumOption.MD5: 62 | msg.md5DataChecksum = util.calc_crc(msg.data, self.checksum_option) 63 | 64 | request = cherami_input.PutMessageBatchRequest( 65 | destinationPath=self.path, 66 | messages=[msg]) 67 | batch_result = util.execute_input_host(tchannel=self.tchannel, 68 | headers=self.headers, 69 | hostport=self.hostport, 70 | timeout=self.timeout_seconds, 71 | method_name='putMessageBatch', 72 | request=request) 73 | 74 | if not callable(callback): 75 | continue 76 | if batch_result and batch_result.successMessages and len(batch_result.successMessages) != 0: # noqa 77 | callback(batch_result.successMessages[0]) 78 | continue 79 | if batch_result and batch_result.failedMessages and len(batch_result.failedMessages) != 0: # noqa 80 | callback(batch_result.failedMessages[0]) 81 | continue 82 | 83 | # fallback: somehow no result received 84 | callback(util.create_failed_message_ack(msg.id, 'sender gets no result from input')) 85 | 86 | except Empty: 87 | pass 88 | 89 | except Exception: 90 | if msg and callable(callback): 91 | failure_msg = 'traceback:{0}, hostport:{1}, thread start time:{2}'\ 92 | .format(traceback.format_exc(), 93 | self.hostport, 94 | str(self.thread_start_time)) 95 | callback(util.create_failed_message_ack(msg.id, failure_msg)) 96 | -------------------------------------------------------------------------------- /cherami_client/reconfigure_thread.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import threading 22 | import traceback 23 | 24 | 25 | class ReconfigureThread(threading.Thread): 26 | def __init__(self, interval_seconds, reconfigure_signal, reconfigure_func, logger): 27 | threading.Thread.__init__(self) 28 | self.interval_seconds = interval_seconds 29 | self.reconfigure_signal = reconfigure_signal 30 | self.reconfigure_func = reconfigure_func 31 | self.logger = logger 32 | self.stop_signal = threading.Event() 33 | 34 | def stop(self): 35 | self.stop_signal.set() 36 | 37 | # set the reconfigure_signal so that we won't be blocked on 38 | # waiting for reconfig signal timeout 39 | self.reconfigure_signal.set() 40 | 41 | def run(self): 42 | while not self.stop_signal.is_set(): 43 | # trigger on either signal or wait timeout 44 | self.reconfigure_signal.wait(self.interval_seconds) 45 | 46 | if self.stop_signal.is_set(): 47 | return 48 | 49 | try: 50 | self.reconfigure_func() 51 | except Exception: 52 | self.logger.info('reconfiguration thread {0}, exception {1}' 53 | .format(threading.current_thread(), traceback.format_exc())) 54 | pass 55 | 56 | # done reconfigure, reset signal if needed 57 | if self.reconfigure_signal.is_set(): 58 | self.reconfigure_signal.clear() 59 | -------------------------------------------------------------------------------- /config/consumer.yaml: -------------------------------------------------------------------------------- 1 | # For client instance 2 | application_identifier: example 3 | known_peers: 127.0.0.1:4922 4 | deployment_str: prod 5 | reconnect_delay: 1 6 | timeout_seconds: 10 7 | reconfigure_interval_seconds: 4 8 | 9 | # For consumer instance 10 | cherami_consumer_group: 11 | consumer1: 12 | deployment_str: prod 13 | concurrency: 3 14 | reconnect_delay: 1 15 | prefetch_size: 50 16 | destination: /test/dest1 17 | consumer_group: /test/cg1 18 | nack_failed_message: false 19 | 20 | consumer2: 21 | deployment_str: prod 22 | concurrency: 3 23 | reconnect_delay: 1 24 | prefetch_size: 50 25 | destination: /test/dest1 26 | consumer_group: /test/cg2 27 | nack_failed_message: false 28 | 29 | -------------------------------------------------------------------------------- /config/publisher.yaml: -------------------------------------------------------------------------------- 1 | # For client instance 2 | application_identifier: example 3 | known_peers: 127.0.0.1:4922 4 | deployment_str: prod 5 | reconnect_delay: 1 6 | timeout_seconds: 10 7 | reconfigure_interval_seconds: 4 8 | 9 | # For publisher instance 10 | concurrency: 3 11 | reconnect_delay: 1 12 | destination: /test/dest1 13 | publisher: pub1 -------------------------------------------------------------------------------- /config/test.yaml: -------------------------------------------------------------------------------- 1 | application_identifier: test 2 | -------------------------------------------------------------------------------- /demo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/cherami-client-python/a0b708afdc228611f8d041f5581163c5e5c7757e/demo/__init__.py -------------------------------------------------------------------------------- /demo/example_client.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | """ 22 | A client instance that can be used to create publisher or consumer instances. 23 | """ 24 | 25 | from __future__ import absolute_import, print_function 26 | 27 | import time 28 | import logging 29 | 30 | from tchannel.sync import TChannel as TChannelSyncClient 31 | from cherami_client.client import Client 32 | 33 | 34 | tchannel = TChannelSyncClient(name='example_service', known_peers=['127.0.0.1:4922']) 35 | logger = logging.getLogger('example_client') 36 | 37 | while True: 38 | try: 39 | client = Client(tchannel, logger) 40 | print('Success: Created cherami client.') 41 | 42 | break 43 | except Exception as e: 44 | logger.exception('Failed to create to cherami: %s', e) 45 | time.sleep(2) 46 | -------------------------------------------------------------------------------- /demo/example_consumer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | from __future__ import absolute_import, print_function 22 | 23 | import time 24 | 25 | from demo.example_client import client 26 | 27 | 28 | destination = '/test/dest' 29 | consumer_group = '/test/cg' 30 | 31 | while True: 32 | try: 33 | consumer = client.create_consumer(destination, consumer_group) 34 | consumer.open() 35 | print('Consumer created.') 36 | break 37 | except Exception as e: 38 | print('Failed to create a consumer: %s', e) 39 | time.sleep(2) 40 | 41 | try: 42 | results = consumer.receive(num_msgs=2) 43 | for res in results: 44 | delivery_token = res[0] 45 | msg = res[1] 46 | try: 47 | print(msg.payload.data) 48 | consumer.ack(delivery_token) 49 | except Exception as e: 50 | consumer.nack(delivery_token) 51 | print('Failed to process a message: %s', e) 52 | pass 53 | except Exception as e: 54 | consumer.close() 55 | print('Failed to receive messages: ', e) 56 | 57 | consumer.close() 58 | -------------------------------------------------------------------------------- /demo/example_publisher.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | from __future__ import absolute_import, print_function 22 | 23 | import time 24 | 25 | from demo.example_client import client 26 | 27 | 28 | destination = '/test/dest' 29 | 30 | while True: 31 | try: 32 | publisher = client.create_publisher(destination) 33 | publisher.open() 34 | print('Publisher created.') 35 | break 36 | except Exception as e: 37 | print('Failed to connect to cherami: %s' % e) 38 | time.sleep(2) 39 | 40 | try: 41 | for i in range(2): 42 | publisher.publish(str(i), 'hello', {}) 43 | except Exception as e: 44 | print('Failed to publish to cherami: %s' % e) 45 | publisher.close() 46 | 47 | publisher.close() 48 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | # add dependencies in setup.py 2 | 3 | -r requirements.txt 4 | 5 | -e .[tests] 6 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # add dependencies in setup.py 2 | 3 | -e . 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E702,E712,E902,N802 3 | max-line-length = 120 4 | exclude = 5 | build/, 6 | .eggs/, 7 | .git/ 8 | 9 | [pytest] 10 | norecursedirs = build .eggs .git 11 | 12 | [zest.releaser] 13 | release = no 14 | history_file = HISTORY.rst 15 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from setuptools import setup, find_packages 4 | from pip.req import parse_requirements 5 | 6 | 7 | __version__ = '1.0.3' 8 | name = 'cherami-client' 9 | 10 | 11 | def read_long_description(filename="README.rst"): 12 | with open(filename) as f: 13 | return f.read().strip() 14 | 15 | 16 | setup( 17 | name=name, 18 | version=__version__, 19 | author='Wei Han', 20 | author_email='weihan@uber.com', 21 | url='https://github.com/uber/cherami-client-python', 22 | description='Cherami Python Client Library', 23 | packages=find_packages(exclude=['tests', 'demo', 'tests.*']), 24 | include_package_data=True, 25 | package_data={ 26 | name: [ 27 | 'idl/*.thrift', 28 | ], 29 | }, 30 | license='MIT', 31 | keywords='cherami python client', 32 | long_description=read_long_description(), 33 | install_requires=[ 34 | 'tchannel>=1.0.1', 35 | 'zest.releaser>=6.0,<7.0', 36 | 'crcmod', 37 | 'clay-flask', 38 | 'PyYAML', 39 | ], 40 | zip_safe=False, 41 | classifiers=[ 42 | 'Development Status :: 5 - Production/Stable', 43 | 'Intended Audience :: Developers', 44 | 'License :: OSI Approved :: MIT License', 45 | 'Natural Language :: English', 46 | 'Programming Language :: Python :: 2.7', 47 | ], 48 | test_suite='tests', 49 | extras_require={ 50 | 'tests': [ 51 | 'coverage==4.0.3', 52 | 'pytest==2.8.7', 53 | 'pytest-cov', 54 | 'pytest-ipdb==0.1-prerelease2', 55 | 'pytest-tornado', 56 | 'py==1.4.31', # via pytest 57 | 'flake8==2.5.4', 58 | 'pyflakes==1.0.0', # via flake8 59 | 'pep8==1.7.0', # via flake8 60 | 'mccabe==0.4.0', # via flake8 61 | 'mock==2.0.0', 62 | 'six==1.10.0', # via mock 63 | 'pbr==1.9.1', # via mock 64 | 'funcsigs==1.0.2', # via mock 65 | ], 66 | } 67 | ) 68 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber-archive/cherami-client-python/a0b708afdc228611f8d041f5581163c5e5c7757e/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_consumer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import unittest 22 | import mock 23 | from clay import config 24 | 25 | from cherami_client.lib import cherami, cherami_output 26 | from cherami_client.client import Client 27 | 28 | 29 | class TestConsumer(unittest.TestCase): 30 | 31 | def setUp(self): 32 | self.test_path = '/test/path' 33 | self.test_cg = 'test/cg' 34 | self.test_msg = cherami_output.PutMessage(data='msg') 35 | self.test_ack_id = 'test_ack_id' 36 | self.logger = config.get_logger('test') 37 | self.test_err_msg = 'test_err_msg' 38 | self.test_delivery_token = (self.test_ack_id, '0:0') 39 | 40 | self.output_hosts = mock.Mock(body=cherami.ReadConsumerGroupHostsResult( 41 | hostAddresses=map(lambda x: cherami.HostAddress(host=str(x), port=x), range(10)) 42 | )) 43 | 44 | self.received_msgs = mock.Mock(body=cherami_output.ReceiveMessageBatchResult( 45 | messages=[cherami_output.ConsumerMessage( 46 | ackId=self.test_ack_id, 47 | payload=self.test_msg 48 | )] 49 | )) 50 | 51 | self.no_msg = mock.Mock(body=cherami_output.ReceiveMessageBatchResult( 52 | messages=[] 53 | )) 54 | 55 | self.ack_ok_response = mock.Mock(body=None) 56 | 57 | self.mock_call = mock.Mock() 58 | self.mock_tchannel = mock.Mock() 59 | self.mock_tchannel.thrift.return_value = self.mock_call 60 | 61 | def test_consumer_create(self): 62 | self.mock_call.result.return_value = self.output_hosts 63 | 64 | client = Client(self.mock_tchannel, self.logger, timeout_seconds=1) 65 | 66 | consumer = client.create_consumer(self.test_path, self.test_cg) 67 | self.assertEquals(0, len(consumer.consumer_threads)) 68 | self.assertEquals(0, len(consumer.ack_threads)) 69 | self.assertIsNone(consumer.reconfigure_thread) 70 | 71 | consumer._do_not_start_consumer_thread() 72 | consumer.open() 73 | consumer.close() 74 | 75 | args, kwargs = self.mock_tchannel.thrift.call_args 76 | self.assertEquals('BFrontend::readConsumerGroupHosts', args[0].endpoint) 77 | self.assertEquals(self.test_path, args[0].call_args.getHostsRequest.destinationPath) 78 | self.assertEquals(self.test_cg, args[0].call_args.getHostsRequest.consumerGroupName) 79 | self.assertEquals(10, len(consumer.consumer_threads)) 80 | self.assertEquals(consumer.ack_threads_count, len(consumer.ack_threads)) 81 | self.assertTrue(consumer.reconfigure_thread.is_alive()) 82 | 83 | def test_consumer_consume(self): 84 | self.mock_call.result.return_value = self.output_hosts 85 | 86 | client = Client(self.mock_tchannel, self.logger, timeout_seconds=1) 87 | consumer = client.create_consumer(self.test_path, self.test_cg) 88 | consumer._do_not_start_consumer_thread() 89 | consumer.open() 90 | 91 | self.mock_call.result.return_value = self.received_msgs 92 | consumer.consumer_threads['0:0'].start() 93 | msgs = consumer.receive(1) 94 | consumer.close() 95 | 96 | args, kwargs = self.mock_tchannel.thrift.call_args 97 | self.assertEquals('BOut::receiveMessageBatch', args[0].endpoint) 98 | self.assertEquals(self.test_path, args[0].call_args.request.destinationPath) 99 | self.assertEquals(self.test_cg, args[0].call_args.request.consumerGroupName) 100 | self.assertEquals(1, len(msgs)) 101 | self.assertEquals(self.test_msg, msgs[0][1].payload) 102 | self.assertEquals(self.test_ack_id, msgs[0][1].ackId) 103 | 104 | def test_consumer_open_exception(self): 105 | self.mock_call.result.side_effect = Exception(self.test_err_msg) 106 | client = Client(self.mock_tchannel, self.logger) 107 | consumer = client.create_consumer(self.test_path, self.test_cg) 108 | self.assertRaises(Exception, consumer.open) 109 | self.assertTrue(consumer.reconfigure_thread is None) 110 | 111 | def test_consumer_no_message(self): 112 | self.mock_call.result.return_value = self.output_hosts 113 | 114 | client = Client(self.mock_tchannel, self.logger, timeout_seconds=1) 115 | consumer = client.create_consumer(self.test_path, self.test_cg) 116 | consumer._do_not_start_consumer_thread() 117 | consumer.open() 118 | 119 | self.mock_call.result.return_value = self.no_msg 120 | consumer.consumer_threads['0:0'].start() 121 | msgs = consumer.receive(1) 122 | consumer.close() 123 | 124 | args, kwargs = self.mock_tchannel.thrift.call_args 125 | self.assertEquals('BOut::receiveMessageBatch', args[0].endpoint) 126 | self.assertEquals(self.test_path, args[0].call_args.request.destinationPath) 127 | self.assertEquals(self.test_cg, args[0].call_args.request.consumerGroupName) 128 | self.assertEquals(0, len(msgs)) 129 | 130 | def test_consumer_exception(self): 131 | self.mock_call.result.return_value = self.output_hosts 132 | 133 | client = Client(self.mock_tchannel, self.logger, timeout_seconds=1) 134 | consumer = client.create_consumer(self.test_path, self.test_cg) 135 | consumer._do_not_start_consumer_thread() 136 | consumer.open() 137 | 138 | self.mock_call.result.side_effect = Exception('exception') 139 | consumer.consumer_threads['0:0'].start() 140 | msgs = consumer.receive(1) 141 | 142 | # verify the thread is still alive even though an excepton has been thrown by output host 143 | self.assertTrue(consumer.consumer_threads['0:0'].is_alive()) 144 | 145 | consumer.close() 146 | 147 | args, kwargs = self.mock_tchannel.thrift.call_args 148 | self.assertEquals('BOut::receiveMessageBatch', args[0].endpoint) 149 | self.assertEquals(self.test_path, args[0].call_args.request.destinationPath) 150 | self.assertEquals(self.test_cg, args[0].call_args.request.consumerGroupName) 151 | self.assertEquals(0, len(msgs)) 152 | 153 | def test_consumer_ack_ok(self): 154 | self.mock_call.result.return_value = self.output_hosts 155 | 156 | client = Client(self.mock_tchannel, self.logger, timeout_seconds=1) 157 | consumer = client.create_consumer(self.test_path, self.test_cg) 158 | consumer._do_not_start_consumer_thread() 159 | consumer.open() 160 | 161 | self.mock_call.result.return_value = self.ack_ok_response 162 | res = consumer.ack(self.test_delivery_token) 163 | consumer.close() 164 | 165 | args, kwargs = self.mock_tchannel.thrift.call_args 166 | self.assertEquals('BOut::ackMessages', args[0].endpoint) 167 | self.assertEquals(1, len(args[0].call_args.ackRequest.ackIds)) 168 | self.assertEquals(self.test_delivery_token[0], args[0].call_args.ackRequest.ackIds[0]) 169 | self.assertTrue(res) 170 | 171 | def test_consumer_ack_fail(self): 172 | self.mock_call.result.return_value = self.output_hosts 173 | 174 | client = Client(self.mock_tchannel, self.logger, timeout_seconds=1) 175 | consumer = client.create_consumer(self.test_path, self.test_cg) 176 | consumer._do_not_start_consumer_thread() 177 | consumer.open() 178 | 179 | self.mock_call.result.side_effect = Exception('exception') 180 | res = consumer.ack(self.test_delivery_token) 181 | consumer.close() 182 | 183 | args, kwargs = self.mock_tchannel.thrift.call_args 184 | self.assertEquals('BOut::ackMessages', args[0].endpoint) 185 | self.assertEquals(1, len(args[0].call_args.ackRequest.ackIds)) 186 | self.assertEquals(self.test_delivery_token[0], args[0].call_args.ackRequest.ackIds[0]) 187 | self.assertFalse(res) 188 | -------------------------------------------------------------------------------- /tests/test_publisher.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import unittest 22 | import mock 23 | import threading 24 | import time 25 | from clay import config 26 | 27 | from cherami_client.lib import cherami, cherami_input, util 28 | from cherami_client.client import Client 29 | 30 | 31 | class TestPublisher(unittest.TestCase): 32 | 33 | def setUp(self): 34 | self.test_path = '/test/path' 35 | self.test_msg = 'test_msg' 36 | self.test_msg_id = 'test_msg_id' 37 | self.test_receipt = 'test_ext_uuid:xxx' 38 | self.test_err_msg = 'test_err_msg' 39 | self.logger = config.get_logger('test') 40 | 41 | self.test_crc32 = util.calc_crc(self.test_msg, cherami.ChecksumOption.CRC32IEEE) 42 | self.test_md5 = util.calc_crc(self.test_msg, cherami.ChecksumOption.MD5) 43 | 44 | self.publisher_options = mock.Mock(body=cherami.ReadPublisherOptionsResult( 45 | hostProtocols=[cherami.HostProtocol( 46 | protocol=cherami.Protocol.TCHANNEL, 47 | hostAddresses=map(lambda x: cherami.HostAddress(host=str(x), port=x), range(10)))] 48 | )) 49 | 50 | self.publisher_options_crc32 = mock.Mock(body=cherami.ReadPublisherOptionsResult( 51 | hostProtocols=[cherami.HostProtocol( 52 | protocol=cherami.Protocol.TCHANNEL, 53 | hostAddresses=map(lambda x: cherami.HostAddress(host=str(x), port=x), range(10)))], 54 | checksumOption=cherami.ChecksumOption.CRC32IEEE 55 | )) 56 | 57 | self.publisher_options_md5 = mock.Mock(body=cherami.ReadPublisherOptionsResult( 58 | hostProtocols=[cherami.HostProtocol( 59 | protocol=cherami.Protocol.TCHANNEL, 60 | hostAddresses=map(lambda x: cherami.HostAddress(host=str(x), port=x), range(10)))], 61 | checksumOption=cherami.ChecksumOption.MD5 62 | )) 63 | 64 | self.publisher_options_diff = mock.Mock(body=cherami.ReadDestinationHostsResult( 65 | hostProtocols=[cherami.HostProtocol( 66 | protocol=cherami.Protocol.TCHANNEL, 67 | hostAddresses=map(lambda x: cherami.HostAddress(host=str(x), port=x), range(7, 15)))] 68 | )) 69 | 70 | self.send_ack_success = mock.Mock(body=cherami_input.PutMessageBatchResult( 71 | successMessages=[cherami_input.PutMessageAck( 72 | id=self.test_msg_id, 73 | status=cherami.Status.OK, 74 | receipt=self.test_receipt, 75 | )] 76 | )) 77 | self.send_ack_failed = mock.Mock(body=cherami_input.PutMessageBatchResult( 78 | failedMessages=[cherami_input.PutMessageAck( 79 | id=self.test_msg_id, 80 | status=cherami.Status.FAILED, 81 | message=self.test_err_msg, 82 | )] 83 | )) 84 | 85 | self.mock_call = mock.Mock() 86 | self.mock_tchannel = mock.Mock() 87 | self.mock_tchannel.thrift.return_value = self.mock_call 88 | 89 | def test_publisher_create(self): 90 | self.mock_call.result.return_value = self.publisher_options 91 | 92 | client = Client(self.mock_tchannel, self.logger) 93 | 94 | publisher = client.create_publisher(self.test_path) 95 | self.assertEquals(0, len(publisher.workers)) 96 | self.assertIsNone(publisher.reconfigure_thread) 97 | 98 | publisher.open() 99 | publisher.close() 100 | 101 | args, kwargs = self.mock_tchannel.thrift.call_args 102 | self.assertEquals('BFrontend::readPublisherOptions', args[0].endpoint) 103 | self.assertEquals(self.test_path, args[0].call_args.getPublisherOptionsRequest.path) 104 | self.assertEquals(10, len(publisher.workers)) 105 | self.assertTrue(publisher.reconfigure_thread.is_alive()) 106 | 107 | def test_publisher_publish(self): 108 | self.mock_call.result.return_value = self.publisher_options 109 | 110 | client = Client(self.mock_tchannel, self.logger) 111 | publisher = client.create_publisher(self.test_path) 112 | publisher.open() 113 | 114 | self.mock_call.result.return_value = self.send_ack_success 115 | ack = publisher.publish(self.test_msg_id, self.test_msg) 116 | publisher.close() 117 | 118 | args, kwargs = self.mock_tchannel.thrift.call_args 119 | self.assertEquals('BIn::putMessageBatch', args[0].endpoint) 120 | self.assertEquals(self.test_path, args[0].call_args.request.destinationPath) 121 | self.assertEquals(1, len(args[0].call_args.request.messages)) 122 | self.assertEquals(self.test_msg_id, args[0].call_args.request.messages[0].id) 123 | self.assertEquals(self.test_msg, args[0].call_args.request.messages[0].data) 124 | self.assertEquals(self.test_msg_id, ack.id) 125 | self.assertEquals(cherami.Status.OK, ack.status) 126 | self.assertEquals(self.test_receipt, ack.receipt) 127 | 128 | def test_publisher_publish_with_context(self): 129 | self.mock_call.result.return_value = self.publisher_options 130 | 131 | client = Client(self.mock_tchannel, self.logger) 132 | publisher = client.create_publisher(self.test_path) 133 | publisher.open() 134 | 135 | self.mock_call.result.return_value = self.send_ack_success 136 | context = {'mycontextkey': 'mycontextvalue'} 137 | ack = publisher.publish(self.test_msg_id, self.test_msg, context) 138 | publisher.close() 139 | 140 | args, kwargs = self.mock_tchannel.thrift.call_args 141 | self.assertEquals('BIn::putMessageBatch', args[0].endpoint) 142 | self.assertEquals(self.test_path, args[0].call_args.request.destinationPath) 143 | self.assertEquals(1, len(args[0].call_args.request.messages)) 144 | self.assertEquals(self.test_msg_id, args[0].call_args.request.messages[0].id) 145 | self.assertEquals(self.test_msg, args[0].call_args.request.messages[0].data) 146 | self.assertEquals(context, args[0].call_args.request.messages[0].userContext) 147 | self.assertEquals(self.test_msg_id, ack.id) 148 | self.assertEquals(cherami.Status.OK, ack.status) 149 | self.assertEquals(self.test_receipt, ack.receipt) 150 | 151 | def test_publisher_publish_failed(self): 152 | self.mock_call.result.return_value = self.publisher_options 153 | 154 | client = Client(self.mock_tchannel, self.logger) 155 | publisher = client.create_publisher(self.test_path) 156 | publisher.open() 157 | 158 | self.mock_call.result.return_value = self.send_ack_failed 159 | ack = publisher.publish(self.test_msg_id, self.test_msg) 160 | publisher.close() 161 | 162 | self.assertEquals(self.test_msg_id, ack.id) 163 | self.assertEquals(cherami.Status.FAILED, ack.status) 164 | self.assertEquals(self.test_err_msg, ack.message) 165 | 166 | def test_publisher_open_exception(self): 167 | self.mock_call.result.side_effect = Exception(self.test_err_msg) 168 | client = Client(self.mock_tchannel, self.logger) 169 | publisher = client.create_publisher(self.test_path) 170 | self.assertRaises(Exception, publisher.open) 171 | self.assertTrue(publisher.reconfigure_thread is None) 172 | 173 | def test_publisher_publish_exception(self): 174 | self.mock_call.result.return_value = self.publisher_options 175 | 176 | client = Client(self.mock_tchannel, self.logger) 177 | publisher = client.create_publisher(self.test_path) 178 | publisher.open() 179 | 180 | self.mock_call.result.side_effect = Exception(self.test_err_msg) 181 | ack = publisher.publish(self.test_msg_id, self.test_msg) 182 | publisher.close() 183 | 184 | self.assertEquals(self.test_msg_id, ack.id) 185 | self.assertEquals(cherami.Status.FAILED, ack.status) 186 | self.assertTrue(self.test_err_msg in ack.message) 187 | 188 | def test_publisher_publish_timeout(self): 189 | self.mock_call.result.return_value = self.publisher_options 190 | 191 | client = Client(self.mock_tchannel, self.logger, timeout_seconds=1) 192 | publisher = client.create_publisher(self.test_path) 193 | publisher.open() 194 | 195 | def side_effect(): 196 | time.sleep(3) 197 | return self.send_ack_success 198 | 199 | self.mock_call.result.side_effect = side_effect 200 | ack = publisher.publish(self.test_msg_id, self.test_msg) 201 | publisher.close() 202 | 203 | self.assertEquals(self.test_msg_id, ack.id) 204 | self.assertEquals(cherami.Status.TIMEDOUT, ack.status) 205 | 206 | def test_publisher_publish_async(self): 207 | self.mock_call.result.return_value = self.publisher_options 208 | done_signal = threading.Event() 209 | acks = [] 210 | 211 | client = Client(self.mock_tchannel, self.logger) 212 | publisher = client.create_publisher(self.test_path) 213 | publisher.open() 214 | 215 | def callback(ack): 216 | acks.append(ack) 217 | done_signal.set() 218 | 219 | self.mock_call.result.return_value = self.send_ack_success 220 | publisher.publish_async(self.test_msg_id, self.test_msg, callback) 221 | done_signal.wait(5) 222 | publisher.close() 223 | 224 | self.assertEquals(1, len(acks)) 225 | self.assertEquals(self.test_msg_id, acks[0].id) 226 | self.assertEquals(cherami.Status.OK, acks[0].status) 227 | self.assertEquals(self.test_receipt, acks[0].receipt) 228 | 229 | def test_publisher_publish_async_invalid_callback(self): 230 | self.mock_call.result.return_value = self.publisher_options 231 | 232 | client = Client(self.mock_tchannel, self.logger) 233 | publisher = client.create_publisher(self.test_path) 234 | publisher.open() 235 | 236 | # nothing happens if no callback provided (no exception) 237 | publisher.publish_async(self.test_msg_id, self.test_msg, None) 238 | publisher.close() 239 | 240 | def test_publisher_publish_crc32(self): 241 | self.mock_call.result.return_value = self.publisher_options_crc32 242 | 243 | client = Client(self.mock_tchannel, self.logger) 244 | publisher = client.create_publisher(self.test_path) 245 | publisher.open() 246 | 247 | self.mock_call.result.return_value = self.send_ack_success 248 | ack = publisher.publish(self.test_msg_id, self.test_msg) 249 | publisher.close() 250 | 251 | args, kwargs = self.mock_tchannel.thrift.call_args 252 | self.assertEquals('BIn::putMessageBatch', args[0].endpoint) 253 | self.assertEquals(self.test_path, args[0].call_args.request.destinationPath) 254 | self.assertEquals(1, len(args[0].call_args.request.messages)) 255 | self.assertEquals(self.test_msg_id, args[0].call_args.request.messages[0].id) 256 | self.assertEquals(self.test_msg, args[0].call_args.request.messages[0].data) 257 | self.assertEquals(self.test_crc32, args[0].call_args.request.messages[0].crc32IEEEDataChecksum) 258 | self.assertEquals(self.test_msg_id, ack.id) 259 | self.assertEquals(cherami.Status.OK, ack.status) 260 | self.assertEquals(self.test_receipt, ack.receipt) 261 | 262 | def test_publisher_publish_md5(self): 263 | self.mock_call.result.return_value = self.publisher_options_md5 264 | 265 | client = Client(self.mock_tchannel, self.logger) 266 | publisher = client.create_publisher(self.test_path) 267 | publisher.open() 268 | 269 | self.mock_call.result.return_value = self.send_ack_success 270 | ack = publisher.publish(self.test_msg_id, self.test_msg) 271 | publisher.close() 272 | 273 | args, kwargs = self.mock_tchannel.thrift.call_args 274 | self.assertEquals('BIn::putMessageBatch', args[0].endpoint) 275 | self.assertEquals(self.test_path, args[0].call_args.request.destinationPath) 276 | self.assertEquals(1, len(args[0].call_args.request.messages)) 277 | self.assertEquals(self.test_msg_id, args[0].call_args.request.messages[0].id) 278 | self.assertEquals(self.test_msg, args[0].call_args.request.messages[0].data) 279 | self.assertEquals(self.test_md5, args[0].call_args.request.messages[0].md5DataChecksum) 280 | self.assertEquals(self.test_msg_id, ack.id) 281 | self.assertEquals(cherami.Status.OK, ack.status) 282 | self.assertEquals(self.test_receipt, ack.receipt) 283 | 284 | def test_crc32(self): 285 | s = 'aaa' 286 | self.assertEquals(util.calc_crc(s, cherami.ChecksumOption.CRC32IEEE), 4027020077) 287 | 288 | # this is a timer based test, disable for now since it's time sensitive 289 | # def test_publisher_reconfigure(self): 290 | # self.mock_call.result.side_effect = [self.input_hosts, self.input_hosts_diff] 291 | # 292 | # client = NewClient(self.mock_tchannel, None) 293 | # publisher = client.create_publisher(self.test_path) 294 | # publisher.open() 295 | # self.assertEquals(10, len(publisher.workers)) 296 | # 297 | # time.sleep(15) 298 | # self.assertEquals(8, len(publisher.workers)) 299 | -------------------------------------------------------------------------------- /tests/test_util.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2017 Uber Technologies, Inc. 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | # THE SOFTWARE. 20 | 21 | import unittest 22 | 23 | from cherami_client.lib import cherami_frontend 24 | 25 | 26 | class TestUtil(unittest.TestCase): 27 | 28 | def setUp(self): 29 | pass 30 | 31 | def test_deployment_str(self): 32 | self.assertEquals('cherami-frontendhost', cherami_frontend.load_frontend().service) 33 | self.assertEquals('cherami-frontendhost', cherami_frontend.load_frontend(None).service) 34 | self.assertEquals('cherami-frontendhost', cherami_frontend.load_frontend('').service) 35 | self.assertEquals('cherami-frontendhost', cherami_frontend.load_frontend('prod').service) 36 | self.assertEquals('cherami-frontendhost', cherami_frontend.load_frontend('Prod').service) 37 | self.assertEquals('cherami-frontendhost', cherami_frontend.load_frontend('production').service) 38 | self.assertEquals('cherami-frontendhost', cherami_frontend.load_frontend('Production').service) 39 | self.assertEquals('cherami-frontendhost', cherami_frontend.load_frontend('dev').service) 40 | self.assertEquals('cherami-frontendhost', cherami_frontend.load_frontend('Dev').service) 41 | self.assertEquals('cherami-frontendhost', cherami_frontend.load_frontend('development').service) 42 | self.assertEquals('cherami-frontendhost', cherami_frontend.load_frontend('Development').service) 43 | self.assertEquals('cherami-frontendhost_staging', cherami_frontend.load_frontend('staging').service) 44 | self.assertEquals('cherami-frontendhost_staging2', cherami_frontend.load_frontend('staging2').service) 45 | --------------------------------------------------------------------------------