├── .gitignore ├── .travis.yml ├── LICENSE ├── README.rst ├── grpcresolver ├── __init__.py ├── address.py ├── channel.py ├── client.py ├── compat.py ├── registry.py └── resolver.py ├── setup.cfg ├── setup.py ├── tests ├── conftest.py ├── hello.proto ├── hello_pb2.py ├── hello_pb2_grpc.py ├── rpc.py ├── test_address.py ├── test_channel.py └── test_registry.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | 3 | *.pyc 4 | 5 | __pycache__/* 6 | 7 | .eggs/* 8 | 9 | *egg-info/* 10 | 11 | build/* 12 | 13 | dist/* 14 | 15 | .cache/* 16 | 17 | .tox/* 18 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - '3.4' 4 | - '3.5' 5 | - '3.6' 6 | - 3.7-dev 7 | - pypy 8 | 9 | matrix: 10 | fast_finish: true 11 | allow_failures: 12 | - python: 3.7-dev 13 | - python: pypy 14 | cache: python 15 | install: pip install tox 16 | script: tox -e py 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017 Yu Senbin 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | 2 | ============== 3 | grpc-resolver 4 | ============== 5 | 6 | .. image:: https://travis-ci.org/justdoit0823/grpc-resolver.svg?branch=master 7 | :target: https://travis-ci.org/justdoit0823/grpc-resolver 8 | 9 | 10 | A simple Python gRPC service registry and resolver, is compatible for Python 2 and 3. 11 | 12 | Now it only supports Etcd as configuration storage backend, but actually can support any storage backend. 13 | 14 | 15 | -------------- 16 | Requirements 17 | -------------- 18 | 19 | - grpcio 20 | 21 | - etcd3 22 | 23 | 24 | --------- 25 | Install 26 | --------- 27 | 28 | .. code-block:: shell 29 | 30 | $ pip install git+https://github.com/justdoit0823/grpc-resolver 31 | 32 | 33 | ----------- 34 | How to use 35 | ----------- 36 | 37 | Here we go: 38 | 39 | 40 | Service Resolve 41 | ================= 42 | 43 | .. code-block:: python 44 | 45 | >>> from grpcresolver import EtcdServiceResolver, RoundrobinChannel 46 | 47 | >>> import hello_pb2 48 | 49 | >>> import hello_pb2_grpc 50 | 51 | >>> resolver = EtcdServiceResolver(etcd_host='10.30.141.251', etcd_port=2376) 52 | 53 | >>> channel = RoundrobinChannel('your_service_name', resolver) 54 | 55 | >>> stub = hello_pb2_grpc.HelloStub(channel) 56 | 57 | >>> # do gRPC call as usual 58 | 59 | 60 | Service Registry 61 | ================== 62 | 63 | .. code-block:: python 64 | 65 | >>> from grpcresolver import EtcdServiceRegistry 66 | 67 | >>> registry = EtcdServiceResolver(etcd_host='10.30.33.11', etcd_port=2376) 68 | 69 | >>> registry.register(('your_grpc_service',), '192.168.10.20:11111', 360) 70 | 71 | 72 | -------------- 73 | How it works 74 | -------------- 75 | 76 | The ``grpcresolver.channel.LbChannel`` has implemented ``unary_unary``, ``unary_stream``, ``stream_unary`` and ``stream_stream`` four gRPC operations, but without ``subscribe`` and ``unsubscribe``. 77 | 78 | Meanwhile, also implementing the relative multi-callable operations. 79 | 80 | When calling gRPC service method, the channel will return a ``grpcresolver.channel.Channel`` object from it's local cache or create a new one after resolving from it's resolver. 81 | 82 | 83 | --------- 84 | Features 85 | --------- 86 | 87 | 88 | - Easy to integrate while using standard Python gRPC channel 89 | 90 | - Supporting gRPC service discovery with random and roundrobin algorithms 91 | 92 | - Automatic etcd node selection and retry when etcd client gRPC request failed 93 | -------------------------------------------------------------------------------- /grpcresolver/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | 3 | from .channel import RandomChannel, RoundrobinChannel 4 | from .client import EtcdClient 5 | from .registry import EtcdServiceRegistry 6 | from .resolver import EtcdServiceResolver 7 | 8 | 9 | __version__ = '0.1.0.alpha' 10 | 11 | 12 | __all__ = [ 13 | 'EtcdClient', 'RandomChannel', 'RoundrobinChannel', 14 | 'EtcdServiceRegistry', 'EtcdServiceResolver'] 15 | -------------------------------------------------------------------------------- /grpcresolver/address.py: -------------------------------------------------------------------------------- 1 | 2 | """gRPC service's address format module.""" 3 | 4 | import abc 5 | import json 6 | import six 7 | 8 | from grpcresolver.compat import b2str 9 | 10 | 11 | __all__ = ['Address', 'PlainAddress', 'JsonAddress'] 12 | 13 | 14 | class Address(six.with_metaclass(abc.ABCMeta)): 15 | """gRPC service address.""" 16 | 17 | @abc.abstractmethod 18 | def __init__(self, addr, metadata=None): 19 | raise NotImplementedError 20 | 21 | @abc.abstractmethod 22 | def add_value(self): 23 | raise NotImplementedError 24 | 25 | @abc.abstractmethod 26 | def delete_value(self): 27 | raise NotImplementedError 28 | 29 | @classmethod 30 | @abc.abstractmethod 31 | def from_value(cls, val, deserializer=None): 32 | raise NotImplementedError 33 | 34 | 35 | class PlainAddress(Address): 36 | """Plain text address.""" 37 | 38 | def __init__(self, addr, metadata=None): 39 | self._addr = addr 40 | 41 | def add_value(self): 42 | return self._addr 43 | 44 | def delete_value(self): 45 | return self._addr 46 | 47 | @classmethod 48 | def from_value(cls, val, deserializer=None): 49 | return b2str(val) 50 | 51 | 52 | class JsonAddress(Address): 53 | """Json address.""" 54 | 55 | add_op = 0 56 | delete_op = 1 57 | 58 | def __init__(self, addr, metadata=None, serializer=json.dumps): 59 | self._addr = addr 60 | self._metadata = metadata or {} 61 | self._serializer = serializer 62 | 63 | def add_value(self): 64 | return self._serializer({ 65 | 'Op': self.add_op, 'Addr': self._addr, 66 | 'Metadata': self._serializer(self._metadata)}) 67 | 68 | def delete_value(self): 69 | return self._serializer({ 70 | 'Op': self.delete_op, 'Addr': self._addr, 71 | 'Metadata': self._serializer(self._metadata)}) 72 | 73 | @classmethod 74 | def from_value(cls, val, deserializer=json.loads): 75 | addr_val = deserializer(b2str(val)) 76 | addr_val['Metadata'] = deserializer(addr_val['Metadata']) 77 | addr_op = addr_val['Op'] 78 | if addr_op == cls.add_op: 79 | return True, addr_val['Addr'] 80 | elif addr_op == cls.delete_op: 81 | return False, addr_val['Addr'] 82 | 83 | raise ValueError('invalid address value.') 84 | -------------------------------------------------------------------------------- /grpcresolver/channel.py: -------------------------------------------------------------------------------- 1 | 2 | """gRPC load balance channel module.""" 3 | 4 | import random 5 | 6 | import grpc 7 | from grpc import _common 8 | from grpc._channel import ( 9 | _UNARY_UNARY_INITIAL_DUE, _UNARY_STREAM_INITIAL_DUE, 10 | _STREAM_UNARY_INITIAL_DUE, _STREAM_STREAM_INITIAL_DUE, _EMPTY_FLAGS) 11 | from grpc._channel import ( 12 | _RPCState, _ChannelCallState, _ChannelConnectivityState, _Rendezvous) 13 | from grpc._channel import ( 14 | _start_unary_request, _end_unary_response_blocking, 15 | _consume_request_iterator, _channel_managed_call_management, 16 | _call_error_set_RPCstate, _handle_event, _event_handler, 17 | _check_call_error, _options, _deadline) 18 | from grpc._cython import cygrpc 19 | 20 | 21 | __all__ = ['RandomChannel', 'RoundrobinChannel'] 22 | 23 | 24 | class _UnaryUnaryMultiCallable(grpc.UnaryUnaryMultiCallable): 25 | 26 | def __init__( 27 | self, channel, method, request_serializer, 28 | response_deserializer): 29 | self._channel = channel 30 | self._method = method 31 | self._request_serializer = request_serializer 32 | self._response_deserializer = response_deserializer 33 | 34 | def _prepare(self, request, timeout, metadata): 35 | deadline, deadline_timespec, serialized_request, rendezvous = ( 36 | _start_unary_request(request, timeout, self._request_serializer)) 37 | if serialized_request is None: 38 | return None, None, None, None, rendezvous 39 | else: 40 | state = _RPCState(_UNARY_UNARY_INITIAL_DUE, None, None, None, None) 41 | operations = ( 42 | cygrpc.operation_send_initial_metadata( 43 | _common.to_cygrpc_metadata(metadata), _EMPTY_FLAGS), 44 | cygrpc.operation_send_message(serialized_request, _EMPTY_FLAGS), 45 | cygrpc.operation_send_close_from_client(_EMPTY_FLAGS), 46 | cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS), 47 | cygrpc.operation_receive_message(_EMPTY_FLAGS), 48 | cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) 49 | return state, operations, deadline, deadline_timespec, None 50 | 51 | def _blocking(self, channel, request, timeout, metadata, credentials): 52 | state, operations, deadline, deadline_timespec, rendezvous = self._prepare( 53 | request, timeout, metadata) 54 | if rendezvous: 55 | raise rendezvous 56 | else: 57 | completion_queue = cygrpc.CompletionQueue() 58 | call = channel.create_call(None, 0, completion_queue, 59 | self._method, None, 60 | deadline_timespec) 61 | if credentials is not None: 62 | call.set_credentials(credentials._credentials) 63 | call_error = call.start_client_batch( 64 | cygrpc.Operations(operations), None) 65 | _check_call_error(call_error, metadata) 66 | _handle_event(completion_queue.poll(), state, 67 | self._response_deserializer) 68 | return state, call, deadline 69 | 70 | def __call__(self, request, timeout=None, metadata=None, credentials=None): 71 | channel = self._channel.get_channel() 72 | state, call, deadline = self._blocking( 73 | channel.channel, request, timeout, metadata, credentials) 74 | return _end_unary_response_blocking(state, call, False, deadline) 75 | 76 | def with_call(self, request, timeout=None, metadata=None, credentials=None): 77 | channel = self._channel.get_channel() 78 | state, call, deadline = self._blocking( 79 | channel.channel, request, timeout, metadata, credentials) 80 | return _end_unary_response_blocking(state, call, True, deadline) 81 | 82 | def future(self, request, timeout=None, metadata=None, credentials=None): 83 | state, operations, deadline, deadline_timespec, rendezvous = self._prepare( 84 | request, timeout, metadata) 85 | if rendezvous: 86 | return rendezvous 87 | else: 88 | channel = self._channel.get_channel() 89 | call, drive_call = channel.managed_call( 90 | None, 0, self._method, None, deadline_timespec) 91 | 92 | if credentials is not None: 93 | call.set_credentials(credentials._credentials) 94 | 95 | event_handler = _event_handler( 96 | state, call, self._response_deserializer) 97 | with state.condition: 98 | call_error = call.start_client_batch( 99 | cygrpc.Operations(operations), event_handler) 100 | if call_error != cygrpc.CallError.ok: 101 | _call_error_set_RPCstate(state, call_error, metadata) 102 | return _Rendezvous(state, None, None, deadline) 103 | drive_call() 104 | return _Rendezvous(state, call, self._response_deserializer, 105 | deadline) 106 | 107 | 108 | class _UnaryStreamMultiCallable(grpc.UnaryStreamMultiCallable): 109 | 110 | def __init__(self, channel, method, request_serializer, 111 | response_deserializer): 112 | self._channel = channel 113 | self._method = method 114 | self._request_serializer = request_serializer 115 | self._response_deserializer = response_deserializer 116 | 117 | def __call__(self, request, timeout=None, metadata=None, credentials=None): 118 | deadline, deadline_timespec, serialized_request, rendezvous = ( 119 | _start_unary_request(request, timeout, self._request_serializer)) 120 | if serialized_request is None: 121 | raise rendezvous 122 | else: 123 | state = _RPCState(_UNARY_STREAM_INITIAL_DUE, None, None, None, None) 124 | channel = self._channel.get_channel() 125 | call, drive_call = channel.managed_call( 126 | None, 0, self._method, None, deadline_timespec) 127 | if credentials is not None: 128 | call.set_credentials(credentials._credentials) 129 | event_handler = _event_handler( 130 | state, call, self._response_deserializer) 131 | with state.condition: 132 | call.start_client_batch( 133 | cygrpc.Operations(( 134 | cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS), 135 | )), event_handler) 136 | operations = ( 137 | cygrpc.operation_send_initial_metadata( 138 | _common.to_cygrpc_metadata(metadata), 139 | _EMPTY_FLAGS), cygrpc.operation_send_message( 140 | serialized_request, _EMPTY_FLAGS), 141 | cygrpc.operation_send_close_from_client(_EMPTY_FLAGS), 142 | cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) 143 | call_error = call.start_client_batch( 144 | cygrpc.Operations(operations), event_handler) 145 | if call_error != cygrpc.CallError.ok: 146 | _call_error_set_RPCstate(state, call_error, metadata) 147 | return _Rendezvous(state, None, None, deadline) 148 | drive_call() 149 | return _Rendezvous(state, call, self._response_deserializer, 150 | deadline) 151 | 152 | 153 | class _StreamUnaryMultiCallable(grpc.StreamUnaryMultiCallable): 154 | 155 | def __init__( 156 | self, channel, method, request_serializer, response_deserializer): 157 | self._channel = channel 158 | self._method = method 159 | self._request_serializer = request_serializer 160 | self._response_deserializer = response_deserializer 161 | 162 | def _blocking( 163 | self, channel, request_iterator, timeout, metadata, credentials): 164 | deadline, deadline_timespec = _deadline(timeout) 165 | state = _RPCState(_STREAM_UNARY_INITIAL_DUE, None, None, None, None) 166 | completion_queue = cygrpc.CompletionQueue() 167 | call = channel.create_call( 168 | None, 0, completion_queue, self._method, None, deadline_timespec) 169 | if credentials is not None: 170 | call.set_credentials(credentials._credentials) 171 | with state.condition: 172 | call.start_client_batch( 173 | cygrpc.Operations( 174 | (cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),)), 175 | None) 176 | operations = ( 177 | cygrpc.operation_send_initial_metadata( 178 | _common.to_cygrpc_metadata(metadata), _EMPTY_FLAGS), 179 | cygrpc.operation_receive_message(_EMPTY_FLAGS), 180 | cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) 181 | call_error = call.start_client_batch( 182 | cygrpc.Operations(operations), None) 183 | _check_call_error(call_error, metadata) 184 | _consume_request_iterator(request_iterator, state, call, 185 | self._request_serializer) 186 | while True: 187 | event = completion_queue.poll() 188 | with state.condition: 189 | _handle_event(event, state, self._response_deserializer) 190 | state.condition.notify_all() 191 | if not state.due: 192 | break 193 | return state, call, deadline 194 | 195 | def __call__( 196 | self, request_iterator, timeout=None, metadata=None, 197 | credentials=None): 198 | channel = self._channel.get_channel() 199 | state, call, deadline = self._blocking( 200 | channel.channel, request_iterator, timeout, metadata, 201 | credentials) 202 | return _end_unary_response_blocking(state, call, False, deadline) 203 | 204 | def with_call( 205 | self, request_iterator, timeout=None, metadata=None, 206 | credentials=None): 207 | channel = self._channel.get_channel() 208 | state, call, deadline = self._blocking( 209 | channel.channel, request_iterator, timeout, metadata, credentials) 210 | return _end_unary_response_blocking(state, call, True, deadline) 211 | 212 | def future(self, 213 | request_iterator, 214 | timeout=None, 215 | metadata=None, 216 | credentials=None): 217 | deadline, deadline_timespec = _deadline(timeout) 218 | state = _RPCState(_STREAM_UNARY_INITIAL_DUE, None, None, None, None) 219 | channel = self._channel.get_channel() 220 | call, drive_call = channel.managed_call( 221 | None, 0, self._method, None, deadline_timespec) 222 | if credentials is not None: 223 | call.set_credentials(credentials._credentials) 224 | event_handler = _event_handler(state, call, self._response_deserializer) 225 | with state.condition: 226 | call.start_client_batch( 227 | cygrpc.Operations( 228 | (cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),)), 229 | event_handler) 230 | operations = ( 231 | cygrpc.operation_send_initial_metadata( 232 | _common.to_cygrpc_metadata(metadata), _EMPTY_FLAGS), 233 | cygrpc.operation_receive_message(_EMPTY_FLAGS), 234 | cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) 235 | call_error = call.start_client_batch( 236 | cygrpc.Operations(operations), event_handler) 237 | if call_error != cygrpc.CallError.ok: 238 | _call_error_set_RPCstate(state, call_error, metadata) 239 | return _Rendezvous(state, None, None, deadline) 240 | drive_call() 241 | _consume_request_iterator(request_iterator, state, call, 242 | self._request_serializer) 243 | return _Rendezvous(state, call, self._response_deserializer, deadline) 244 | 245 | 246 | class _StreamStreamMultiCallable(grpc.StreamStreamMultiCallable): 247 | 248 | def __init__( 249 | self, channel, method, request_serializer, response_deserializer): 250 | self._channel = channel 251 | self._method = method 252 | self._request_serializer = request_serializer 253 | self._response_deserializer = response_deserializer 254 | 255 | def __call__(self, 256 | request_iterator, 257 | timeout=None, 258 | metadata=None, 259 | credentials=None): 260 | deadline, deadline_timespec = _deadline(timeout) 261 | state = _RPCState(_STREAM_STREAM_INITIAL_DUE, None, None, None, None) 262 | channel = self._channel.get_channel() 263 | call, drive_call = channel.managed_call( 264 | None, 0, self._method, None, deadline_timespec) 265 | if credentials is not None: 266 | call.set_credentials(credentials._credentials) 267 | event_handler = _event_handler(state, call, self._response_deserializer) 268 | with state.condition: 269 | call.start_client_batch( 270 | cygrpc.Operations( 271 | (cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),)), 272 | event_handler) 273 | operations = ( 274 | cygrpc.operation_send_initial_metadata( 275 | _common.to_cygrpc_metadata(metadata), _EMPTY_FLAGS), 276 | cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),) 277 | call_error = call.start_client_batch( 278 | cygrpc.Operations(operations), event_handler) 279 | if call_error != cygrpc.CallError.ok: 280 | _call_error_set_RPCstate(state, call_error, metadata) 281 | return _Rendezvous(state, None, None, deadline) 282 | drive_call() 283 | _consume_request_iterator(request_iterator, state, call, 284 | self._request_serializer) 285 | return _Rendezvous(state, call, self._response_deserializer, deadline) 286 | 287 | 288 | class Channel(object): 289 | """An object communicates between `LbChannel` and gRPC request.""" 290 | 291 | __slots__ = ('target', 'channel', 'managed_call', 'connectivity_state') 292 | 293 | def __init__(self, target, options=None, credentials=None): 294 | options = options if options is not None else () 295 | self.target = target 296 | self.channel = channel = cygrpc.Channel( 297 | _common.encode(target), 298 | _common.channel_args(_options(options)), credentials) 299 | self.managed_call = _channel_managed_call_management( 300 | _ChannelCallState(channel)) 301 | self.connectivity_state = _ChannelConnectivityState(channel) 302 | 303 | 304 | class LbChannel(grpc.Channel): 305 | """A gRPC load balance channel.""" 306 | 307 | def __init__(self, service_name, resolver): 308 | self._service_name = service_name 309 | self._resolver = resolver 310 | self._channels = {} 311 | 312 | def select_target(self): 313 | raise NotImplementedError 314 | 315 | def get_channel(self): 316 | addr = self.select_target() 317 | try: 318 | return self._channels[addr] 319 | except KeyError: 320 | channel = Channel(addr) 321 | self._channels[addr] = channel 322 | return channel 323 | 324 | def release_channel(self, channel): 325 | name = self._service_name 326 | items = {name: ((), (channel.target,))} 327 | self._resolver.update(**items) 328 | 329 | def subscribe(self, callback, try_to_connect=False): 330 | raise NotImplementedError 331 | 332 | def unsubscribe(self, callback): 333 | raise NotImplementedError 334 | 335 | def unary_unary( 336 | self, method, request_serializer=None, response_deserializer=None): 337 | return _UnaryUnaryMultiCallable( 338 | self, _common.encode(method), request_serializer, 339 | response_deserializer) 340 | 341 | def unary_stream( 342 | self, method, request_serializer=None, response_deserializer=None): 343 | return _UnaryStreamMultiCallable( 344 | self, _common.encode(method), request_serializer, 345 | response_deserializer) 346 | 347 | def stream_unary( 348 | self, method, request_serializer=None, response_deserializer=None): 349 | return _StreamUnaryMultiCallable( 350 | self, _common.encode(method), request_serializer, 351 | response_deserializer) 352 | 353 | def stream_stream( 354 | self, method, request_serializer=None, response_deserializer=None): 355 | return _StreamStreamMultiCallable( 356 | self, _common.encode(method), request_serializer, 357 | response_deserializer) 358 | 359 | def __del__(self): 360 | del self._resolver 361 | 362 | 363 | class RandomChannel(LbChannel): 364 | """Random gRPC load balance channel.""" 365 | 366 | def select_target(self): 367 | addrs = self._resolver.resolve(self._service_name) 368 | addr_idx = random.randint(0, len(addrs) - 1) 369 | addr = addrs[addr_idx] 370 | 371 | return addr 372 | 373 | 374 | class RoundrobinChannel(LbChannel): 375 | """Roundrobin gRPC load balance channel.""" 376 | 377 | def __init__(self, service_name, resolver): 378 | super(RoundrobinChannel, self).__init__(service_name, resolver) 379 | self._cur_index = 0 380 | 381 | def select_target(self): 382 | addrs = self._resolver.resolve(self._service_name) 383 | addr_num = len(addrs) 384 | if addr_num == 0: 385 | raise ValueError('No channel.') 386 | 387 | addr = addrs[self._cur_index % addr_num] 388 | self._cur_index = (self._cur_index + 1) % addr_num 389 | 390 | return addr 391 | -------------------------------------------------------------------------------- /grpcresolver/client.py: -------------------------------------------------------------------------------- 1 | 2 | """Etcd client proxy module.""" 3 | 4 | import etcd3 5 | import grpc 6 | from grpc._channel import _Rendezvous 7 | 8 | 9 | __all__ = ['EtcdClient'] 10 | 11 | 12 | class EtcdClient(object): 13 | """A etcd client proxy.""" 14 | 15 | _suffer_status_code = ( 16 | grpc.StatusCode.UNAVAILABLE, grpc.StatusCode.ABORTED, 17 | grpc.StatusCode.RESOURCE_EXHAUSTED) 18 | 19 | def __init__(self, etcd_host, etcd_port): 20 | self._host = etcd_host 21 | self._port = etcd_port 22 | self._client_idx = 0 23 | self._cluster = None 24 | 25 | def call(self, method, *args, **kwargs): 26 | """Etcd operation gateway method.""" 27 | if self._cluster is None: 28 | # Lazy initialize etcd client 29 | client = etcd3.client(self._host, self._port) 30 | self._cluster = tuple( 31 | member._etcd_client for member in client.members) 32 | 33 | try_count = len(self._cluster) 34 | while try_count > 0: 35 | client = self._cluster[self._client_idx] 36 | try: 37 | ret = getattr(client, method)(*args, **kwargs) 38 | except _Rendezvous as e: 39 | if e.code() in self._suffer_status_code: 40 | self._client_idx = (self._client_idx + 1) % self._cluster 41 | try_count -= 1 42 | continue 43 | 44 | raise 45 | 46 | return ret 47 | 48 | def get(self, key): 49 | return self.call('get', key) 50 | 51 | def get_all(self): 52 | return self.call('get_all') 53 | 54 | def get_prefix(self, key_prefix, sort_order=None, sort_target='key'): 55 | return self.call( 56 | 'get_prefix', key_prefix, sort_order=sort_order, 57 | sort_target=sort_target) 58 | 59 | def put(self, key, value, lease=None): 60 | return self.call('put', key, value, lease=lease) 61 | 62 | def lease(self, ttl, lease_id=None): 63 | return self.call('lease', ttl, lease_id=lease_id) 64 | 65 | def delete(self, key): 66 | return self.call('delete', key) 67 | -------------------------------------------------------------------------------- /grpcresolver/compat.py: -------------------------------------------------------------------------------- 1 | 2 | """Compat methods module.""" 3 | 4 | import six 5 | 6 | 7 | __all__ = ['b2str'] 8 | 9 | 10 | def b2str(i_b): 11 | if issubclass(six.binary_type, six.string_types): 12 | return i_b 13 | 14 | return i_b.decode() 15 | -------------------------------------------------------------------------------- /grpcresolver/registry.py: -------------------------------------------------------------------------------- 1 | 2 | """gRPC service registry module.""" 3 | 4 | import abc 5 | import six 6 | 7 | from grpcresolver.address import PlainAddress, JsonAddress 8 | from grpcresolver.client import EtcdClient 9 | 10 | 11 | __all__ = ['EtcdServiceRegistry'] 12 | 13 | 14 | class ServiceRegistry(six.with_metaclass(abc.ABCMeta)): 15 | """A service registry.""" 16 | 17 | @abc.abstractmethod 18 | def register(self, service_names, service_addr, service_ttl): 19 | """Register services with the same address.""" 20 | raise NotImplementedError 21 | 22 | @abc.abstractmethod 23 | def heartbeat(self, service_addr=None): 24 | """Service registry heartbeat.""" 25 | raise NotImplementedError 26 | 27 | @abc.abstractmethod 28 | def unregister(self, service_names, service_addr): 29 | """Unregister services with the same address.""" 30 | raise NotImplementedError 31 | 32 | 33 | class EtcdServiceRegistry(ServiceRegistry): 34 | """gRPC service registry based on etcd.""" 35 | 36 | def __init__(self, etcd_host=None, etcd_port=None, etcd_client=None): 37 | """Initialize etcd service registry. 38 | 39 | :param etcd_host: (optional) etcd node host for :class:`client.EtcdClient`. 40 | :param etcd_port: (optional) etcd node port for :class:`client.EtcdClient`. 41 | :param etcd_client: (optional) A :class:`client.EtcdClient` object. 42 | 43 | """ 44 | self._client = etcd_client if etcd_client else EtcdClient( 45 | etcd_host, etcd_port) 46 | self._leases = {} 47 | self._services = {} 48 | 49 | def get_lease(self, service_addr, service_ttl): 50 | """Get a gRPC service lease from etcd. 51 | 52 | :param service_addr: gRPC service address. 53 | :param service_ttl: gRPC service lease ttl(seconds). 54 | :rtype `etcd3.lease.Lease` 55 | 56 | """ 57 | lease = self._leases.get(service_addr) 58 | if lease and lease.remaining_ttl > 0: 59 | return lease 60 | 61 | lease_id = hash(service_addr) 62 | lease = self._client.lease(service_ttl, lease_id) 63 | self._leases[service_addr] = lease 64 | return lease 65 | 66 | def _form_service_key(self, service_name, service_addr): 67 | """Return service's key in etcd.""" 68 | return '/'.join((service_name, service_addr)) 69 | 70 | def register( 71 | self, service_names, service_addr, service_ttl, addr_cls=None, 72 | metadata=None): 73 | """Register gRPC services with the same address. 74 | 75 | :param service_names: A collection of gRPC service name. 76 | :param service_addr: gRPC server address. 77 | :param service_ttl: gRPC service ttl(seconds). 78 | :param addr_cls: format class of gRPC service address. 79 | :param metadata: extra meta data for JsonAddress. 80 | 81 | """ 82 | lease = self.get_lease(service_addr, service_ttl) 83 | addr_cls = addr_cls or PlainAddress 84 | for service_name in service_names: 85 | key = self._form_service_key(service_name, service_addr) 86 | if addr_cls == JsonAddress: 87 | addr_obj = addr_cls(service_addr, metadata=metadata) 88 | else: 89 | addr_obj = addr_cls(service_addr) 90 | 91 | addr_val = addr_obj.add_value() 92 | self._client.put(key, addr_val, lease=lease) 93 | try: 94 | self._services[service_addr].add(service_name) 95 | except KeyError: 96 | self._services[service_addr] = {service_name} 97 | 98 | def heartbeat(self, service_addr=None): 99 | """gRPC service heartbeat.""" 100 | if service_addr: 101 | lease = self.get_lease(service_addr) 102 | leases = ((service_addr, lease),) 103 | else: 104 | leases = tuple(self._leases.items()) 105 | 106 | for service_addr, lease in leases: 107 | ret = lease.refresh()[0] 108 | if ret.TTL == 0: 109 | self.register( 110 | self._services[service_addr], service_addr, lease.ttl) 111 | 112 | def unregister(self, service_names, service_addr, addr_cls=None): 113 | """Unregister gRPC services with the same address. 114 | 115 | :param service_names: A collection of gRPC service name. 116 | :param service_addr: gRPC server address. 117 | 118 | """ 119 | addr_cls = addr_cls or PlainAddress 120 | etcd_delete = True 121 | if addr_cls != PlainAddress: 122 | etcd_delete = False 123 | 124 | for service_name in service_names: 125 | key = self._form_service_key(service_name, service_addr) 126 | if etcd_delete: 127 | self._client.delete(key) 128 | else: 129 | self._client.put(addr_cls(service_addr).delete_value()) 130 | 131 | self._services.get(service_addr, {}).discard(service_name) 132 | -------------------------------------------------------------------------------- /grpcresolver/resolver.py: -------------------------------------------------------------------------------- 1 | 2 | """Etcd resolver module.""" 3 | 4 | import abc 5 | import six 6 | import threading 7 | import time 8 | 9 | from grpcresolver.address import PlainAddress 10 | from grpcresolver.client import EtcdClient 11 | 12 | 13 | __all__ = ['EtcdServiceResolver'] 14 | 15 | 16 | class ServiceResolver(six.with_metaclass(abc.ABCMeta)): 17 | """gRPC service Resolver class.""" 18 | 19 | @abc.abstractmethod 20 | def resolve(self, name): 21 | raise NotADirectoryError 22 | 23 | @abc.abstractmethod 24 | def update(self, **kwargs): 25 | raise NotImplementedError 26 | 27 | @abc.abstractmethod 28 | def listen(self): 29 | raise NotImplementedError 30 | 31 | 32 | class EtcdServiceResolver(ServiceResolver): 33 | """gRPC service resolver based on Etcd.""" 34 | 35 | def __init__( 36 | self, etcd_host=None, etcd_port=None, etcd_client=None, 37 | start_listener=True, listen_timeout=5, addr_cls=None): 38 | """Initialize etcd service resolver. 39 | 40 | :param etcd_host: (optional) etcd node host for :class:`client.EtcdClient`. 41 | :param etcd_port: (optional) etcd node port for :class:`client.EtcdClient`. 42 | :param etcd_client: (optional) A :class:`client.EtcdClient` object. 43 | :param start_listener: (optional) Indicate whether starting the resolver listen thread. 44 | :param listen_timeout: (optional) Resolver thread listen timeout. 45 | :param addr_cls: (optional) address format class. 46 | 47 | """ 48 | self._listening = False 49 | self._stopped = False 50 | self._listen_thread = None 51 | self._listen_timeout = listen_timeout 52 | self._lock = threading.Lock() 53 | self._client = etcd_client if etcd_client else EtcdClient( 54 | etcd_host, etcd_port) 55 | self._names = {} 56 | self._addr_cls = addr_cls or PlainAddress 57 | 58 | if start_listener: 59 | self.start_listener() 60 | 61 | def resolve(self, name): 62 | """Resolve gRPC service name. 63 | 64 | :param name: gRPC service name. 65 | :rtype list: A collection gRPC server address. 66 | 67 | """ 68 | with self._lock: 69 | try: 70 | return self._names[name] 71 | except KeyError: 72 | addrs = self.get(name) 73 | self._names[name] = addrs 74 | return addrs 75 | 76 | def get(self, name): 77 | """Get values from Etcd. 78 | 79 | :param name: Etcd key prefix name. 80 | :rtype list: A collection of Etcd values. 81 | 82 | """ 83 | keys = self._client.get_prefix(name) 84 | vals = [] 85 | plain = True 86 | if self._addr_cls != PlainAddress: 87 | plain = False 88 | 89 | for val, metadata in keys: 90 | if plain: 91 | vals.append(self._addr_cls.from_value(val)) 92 | else: 93 | add, addr = self._addr_cls.from_value(val) 94 | if add: 95 | vals.append(addr) 96 | 97 | return vals 98 | 99 | def update(self, **kwargs): 100 | """Add or delete service address. 101 | 102 | :param kwargs: Dictionary of ``'service_name': ((add-address, delete-address)).`` 103 | 104 | """ 105 | with self._lock: 106 | for name, (add, delete) in kwargs.items(): 107 | try: 108 | self._names[name].extend(add) 109 | except KeyError: 110 | self._names[name] = add 111 | 112 | for del_item in delete: 113 | try: 114 | self._names[name].remove(del_item) 115 | except ValueError: 116 | continue 117 | 118 | def listen(self): 119 | """Listen for change about gRPC service address.""" 120 | while not self._stopped: 121 | for name in self._names: 122 | try: 123 | vals = self.get(name) 124 | except: 125 | continue 126 | else: 127 | with self._lock: 128 | self._names[name] = vals 129 | 130 | time.sleep(self._listen_timeout) 131 | 132 | def start_listener(self, daemon=True): 133 | """Start listen thread. 134 | 135 | :param daemon: Indicate whether start thread as a daemon. 136 | 137 | """ 138 | if self._listening: 139 | return 140 | 141 | thread_name = 'Thread-resolver-listener' 142 | self._listen_thread = threading.Thread( 143 | target=self.listen, name=thread_name) 144 | self._listen_thread.daemon = daemon 145 | self._listen_thread.start() 146 | self._listening = True 147 | 148 | def stop(self): 149 | """Stop service resolver.""" 150 | if self._stopped: 151 | return 152 | 153 | self._stopped = True 154 | 155 | def __del__(self): 156 | self.stop() 157 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | 2 | [egg_info] 3 | tag_build = dev 4 | tag_svn_revision = true 5 | 6 | [aliases] 7 | test=pytest 8 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | 2 | from setuptools import setup, find_packages 3 | import os 4 | 5 | version = '0.1.0.alpha' 6 | 7 | setup(name='grpc-resolver', 8 | version=version, 9 | description=( 10 | "A simple Python gRPC service resolver and registry based on Etcd."), 11 | long_description=open( 12 | os.path.join(os.path.dirname(__file__), 'README.rst')).read(), 13 | classifiers=[ 14 | 'Development Status :: 3 - Alpha', 15 | 'Intended Audience :: Developers', 16 | 'Programming Language :: Python', 17 | 'Programming Language :: Python :: 2', 18 | 'Programming Language :: Python :: 2.7', 19 | 'Programming Language :: Python :: 3', 20 | 'Programming Language :: Python :: 3.4', 21 | 'Programming Language :: Python :: 3.5', 22 | 'Programming Language :: Python :: 3.6', 23 | 'Topic :: Internet :: WWW/HTTP :: HTTP Servers', 24 | 'Topic :: Software Development :: Version Control :: Git', 25 | 'Topic :: System :: Networking', 26 | ], 27 | keywords='gRPC Etcd service resolver registry', 28 | author='SenbinYu', 29 | author_email='justdoit920823@gmail.com', 30 | url='https://github.com/justdoit0823/grpc-resolver', 31 | license='Apache2.0', 32 | packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), 33 | include_package_data=True, 34 | zip_safe=False, 35 | setup_requires=('pytest-runner',), 36 | install_requires=( 37 | # -*- Extra requirements: -*- 38 | 'grpcio>=1.3.0,<=1.8.0', 39 | 'etcd3>=0.6.2', 40 | 'six>=1.10.0' 41 | ), 42 | tests_require=( 43 | 'pytest', 44 | 'pytest-runner', 45 | 'pytest-mock>=1.6.0' 46 | ), 47 | entry_points=""" 48 | # -*- Entry points: -*- 49 | """ 50 | ) 51 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | 2 | from concurrent import futures 3 | 4 | import grpc 5 | import pytest 6 | 7 | from grpcresolver import EtcdServiceResolver, EtcdServiceRegistry 8 | import hello_pb2_grpc 9 | import rpc 10 | 11 | 12 | _TEST_GRPC_PORT = ( 13 | '127.0.0.1:50031', '127.0.0.1:50032', '127.0.0.1:50033', '127.0.0.1:50034') 14 | 15 | 16 | @pytest.fixture(scope='module') 17 | def grpc_addr(): 18 | return _TEST_GRPC_PORT 19 | 20 | 21 | @pytest.fixture(scope='module') 22 | def grpc_server(grpc_addr): 23 | server = grpc.server(futures.ThreadPoolExecutor(max_workers=2)) 24 | hello_pb2_grpc.add_HelloServicer_to_server(rpc.HelloGRpcServer(), server) 25 | for addr in grpc_addr: 26 | server.add_insecure_port(addr) 27 | 28 | server.start() 29 | 30 | yield server 31 | 32 | server.stop(0) 33 | 34 | 35 | @pytest.fixture(scope='function') 36 | def grpc_resolver(mocker, grpc_addr): 37 | client = mocker.Mock() 38 | resolver = EtcdServiceResolver(etcd_client=client, start_listener=False) 39 | 40 | def resolve(service_name): 41 | return grpc_addr 42 | 43 | old_resolve = resolver.resolve 44 | setattr(resolver, 'resolve', resolve) 45 | yield resolver 46 | 47 | setattr(resolver, 'resolve', old_resolve) 48 | 49 | 50 | @pytest.fixture(scope='function') 51 | def etcd_registry(mocker): 52 | client = mocker.Mock() 53 | registry = EtcdServiceRegistry(etcd_client=client) 54 | 55 | return registry 56 | -------------------------------------------------------------------------------- /tests/hello.proto: -------------------------------------------------------------------------------- 1 | 2 | syntax = "proto3"; 3 | 4 | package grpcresolver.hello; 5 | 6 | 7 | service Hello { 8 | 9 | rpc Greeter(HelloRequest) returns (HelloResponse) {} 10 | 11 | rpc GreeterResponseStream(HelloRequest) returns (stream HelloResponse) {} 12 | 13 | rpc StreamGreeter(stream HelloRequest) returns (HelloResponse) {} 14 | 15 | rpc StreamGreeterResponseStream(stream HelloRequest) returns (stream HelloResponse) {} 16 | 17 | } 18 | 19 | 20 | message HelloRequest { 21 | 22 | string message = 1; 23 | 24 | } 25 | 26 | 27 | message HelloResponse { 28 | 29 | string message = 1; 30 | 31 | } 32 | -------------------------------------------------------------------------------- /tests/hello_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: hello.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | from google.protobuf import descriptor_pb2 11 | # @@protoc_insertion_point(imports) 12 | 13 | _sym_db = _symbol_database.Default() 14 | 15 | 16 | 17 | 18 | DESCRIPTOR = _descriptor.FileDescriptor( 19 | name='hello.proto', 20 | package='grpcresolver.hello', 21 | syntax='proto3', 22 | serialized_pb=_b('\n\x0bhello.proto\x12\x12grpcresolver.hello\"\x1f\n\x0cHelloRequest\x12\x0f\n\x07message\x18\x01 \x01(\t\" \n\rHelloResponse\x12\x0f\n\x07message\x18\x01 \x01(\t2\xff\x02\n\x05Hello\x12P\n\x07Greeter\x12 .grpcresolver.hello.HelloRequest\x1a!.grpcresolver.hello.HelloResponse\"\x00\x12`\n\x15GreeterResponseStream\x12 .grpcresolver.hello.HelloRequest\x1a!.grpcresolver.hello.HelloResponse\"\x00\x30\x01\x12X\n\rStreamGreeter\x12 .grpcresolver.hello.HelloRequest\x1a!.grpcresolver.hello.HelloResponse\"\x00(\x01\x12h\n\x1bStreamGreeterResponseStream\x12 .grpcresolver.hello.HelloRequest\x1a!.grpcresolver.hello.HelloResponse\"\x00(\x01\x30\x01\x62\x06proto3') 23 | ) 24 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 25 | 26 | 27 | 28 | 29 | _HELLOREQUEST = _descriptor.Descriptor( 30 | name='HelloRequest', 31 | full_name='grpcresolver.hello.HelloRequest', 32 | filename=None, 33 | file=DESCRIPTOR, 34 | containing_type=None, 35 | fields=[ 36 | _descriptor.FieldDescriptor( 37 | name='message', full_name='grpcresolver.hello.HelloRequest.message', index=0, 38 | number=1, type=9, cpp_type=9, label=1, 39 | has_default_value=False, default_value=_b("").decode('utf-8'), 40 | message_type=None, enum_type=None, containing_type=None, 41 | is_extension=False, extension_scope=None, 42 | options=None), 43 | ], 44 | extensions=[ 45 | ], 46 | nested_types=[], 47 | enum_types=[ 48 | ], 49 | options=None, 50 | is_extendable=False, 51 | syntax='proto3', 52 | extension_ranges=[], 53 | oneofs=[ 54 | ], 55 | serialized_start=35, 56 | serialized_end=66, 57 | ) 58 | 59 | 60 | _HELLORESPONSE = _descriptor.Descriptor( 61 | name='HelloResponse', 62 | full_name='grpcresolver.hello.HelloResponse', 63 | filename=None, 64 | file=DESCRIPTOR, 65 | containing_type=None, 66 | fields=[ 67 | _descriptor.FieldDescriptor( 68 | name='message', full_name='grpcresolver.hello.HelloResponse.message', index=0, 69 | number=1, type=9, cpp_type=9, label=1, 70 | has_default_value=False, default_value=_b("").decode('utf-8'), 71 | message_type=None, enum_type=None, containing_type=None, 72 | is_extension=False, extension_scope=None, 73 | options=None), 74 | ], 75 | extensions=[ 76 | ], 77 | nested_types=[], 78 | enum_types=[ 79 | ], 80 | options=None, 81 | is_extendable=False, 82 | syntax='proto3', 83 | extension_ranges=[], 84 | oneofs=[ 85 | ], 86 | serialized_start=68, 87 | serialized_end=100, 88 | ) 89 | 90 | DESCRIPTOR.message_types_by_name['HelloRequest'] = _HELLOREQUEST 91 | DESCRIPTOR.message_types_by_name['HelloResponse'] = _HELLORESPONSE 92 | 93 | HelloRequest = _reflection.GeneratedProtocolMessageType('HelloRequest', (_message.Message,), dict( 94 | DESCRIPTOR = _HELLOREQUEST, 95 | __module__ = 'hello_pb2' 96 | # @@protoc_insertion_point(class_scope:grpcresolver.hello.HelloRequest) 97 | )) 98 | _sym_db.RegisterMessage(HelloRequest) 99 | 100 | HelloResponse = _reflection.GeneratedProtocolMessageType('HelloResponse', (_message.Message,), dict( 101 | DESCRIPTOR = _HELLORESPONSE, 102 | __module__ = 'hello_pb2' 103 | # @@protoc_insertion_point(class_scope:grpcresolver.hello.HelloResponse) 104 | )) 105 | _sym_db.RegisterMessage(HelloResponse) 106 | 107 | 108 | try: 109 | # THESE ELEMENTS WILL BE DEPRECATED. 110 | # Please use the generated *_pb2_grpc.py files instead. 111 | import grpc 112 | from grpc.beta import implementations as beta_implementations 113 | from grpc.beta import interfaces as beta_interfaces 114 | from grpc.framework.common import cardinality 115 | from grpc.framework.interfaces.face import utilities as face_utilities 116 | 117 | 118 | class HelloStub(object): 119 | 120 | def __init__(self, channel): 121 | """Constructor. 122 | 123 | Args: 124 | channel: A grpc.Channel. 125 | """ 126 | self.Greeter = channel.unary_unary( 127 | '/grpcresolver.hello.Hello/Greeter', 128 | request_serializer=HelloRequest.SerializeToString, 129 | response_deserializer=HelloResponse.FromString, 130 | ) 131 | self.GreeterResponseStream = channel.unary_stream( 132 | '/grpcresolver.hello.Hello/GreeterResponseStream', 133 | request_serializer=HelloRequest.SerializeToString, 134 | response_deserializer=HelloResponse.FromString, 135 | ) 136 | self.StreamGreeter = channel.stream_unary( 137 | '/grpcresolver.hello.Hello/StreamGreeter', 138 | request_serializer=HelloRequest.SerializeToString, 139 | response_deserializer=HelloResponse.FromString, 140 | ) 141 | self.StreamGreeterResponseStream = channel.stream_stream( 142 | '/grpcresolver.hello.Hello/StreamGreeterResponseStream', 143 | request_serializer=HelloRequest.SerializeToString, 144 | response_deserializer=HelloResponse.FromString, 145 | ) 146 | 147 | 148 | class HelloServicer(object): 149 | 150 | def Greeter(self, request, context): 151 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 152 | context.set_details('Method not implemented!') 153 | raise NotImplementedError('Method not implemented!') 154 | 155 | def GreeterResponseStream(self, request, context): 156 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 157 | context.set_details('Method not implemented!') 158 | raise NotImplementedError('Method not implemented!') 159 | 160 | def StreamGreeter(self, request_iterator, context): 161 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 162 | context.set_details('Method not implemented!') 163 | raise NotImplementedError('Method not implemented!') 164 | 165 | def StreamGreeterResponseStream(self, request_iterator, context): 166 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 167 | context.set_details('Method not implemented!') 168 | raise NotImplementedError('Method not implemented!') 169 | 170 | 171 | def add_HelloServicer_to_server(servicer, server): 172 | rpc_method_handlers = { 173 | 'Greeter': grpc.unary_unary_rpc_method_handler( 174 | servicer.Greeter, 175 | request_deserializer=HelloRequest.FromString, 176 | response_serializer=HelloResponse.SerializeToString, 177 | ), 178 | 'GreeterResponseStream': grpc.unary_stream_rpc_method_handler( 179 | servicer.GreeterResponseStream, 180 | request_deserializer=HelloRequest.FromString, 181 | response_serializer=HelloResponse.SerializeToString, 182 | ), 183 | 'StreamGreeter': grpc.stream_unary_rpc_method_handler( 184 | servicer.StreamGreeter, 185 | request_deserializer=HelloRequest.FromString, 186 | response_serializer=HelloResponse.SerializeToString, 187 | ), 188 | 'StreamGreeterResponseStream': grpc.stream_stream_rpc_method_handler( 189 | servicer.StreamGreeterResponseStream, 190 | request_deserializer=HelloRequest.FromString, 191 | response_serializer=HelloResponse.SerializeToString, 192 | ), 193 | } 194 | generic_handler = grpc.method_handlers_generic_handler( 195 | 'grpcresolver.hello.Hello', rpc_method_handlers) 196 | server.add_generic_rpc_handlers((generic_handler,)) 197 | 198 | 199 | class BetaHelloServicer(object): 200 | """The Beta API is deprecated for 0.15.0 and later. 201 | 202 | It is recommended to use the GA API (classes and functions in this 203 | file not marked beta) for all further purposes. This class was generated 204 | only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" 205 | def Greeter(self, request, context): 206 | context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) 207 | def GreeterResponseStream(self, request, context): 208 | context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) 209 | def StreamGreeter(self, request_iterator, context): 210 | context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) 211 | def StreamGreeterResponseStream(self, request_iterator, context): 212 | context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) 213 | 214 | 215 | class BetaHelloStub(object): 216 | """The Beta API is deprecated for 0.15.0 and later. 217 | 218 | It is recommended to use the GA API (classes and functions in this 219 | file not marked beta) for all further purposes. This class was generated 220 | only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" 221 | def Greeter(self, request, timeout, metadata=None, with_call=False, protocol_options=None): 222 | raise NotImplementedError() 223 | Greeter.future = None 224 | def GreeterResponseStream(self, request, timeout, metadata=None, with_call=False, protocol_options=None): 225 | raise NotImplementedError() 226 | def StreamGreeter(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None): 227 | raise NotImplementedError() 228 | StreamGreeter.future = None 229 | def StreamGreeterResponseStream(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None): 230 | raise NotImplementedError() 231 | 232 | 233 | def beta_create_Hello_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): 234 | """The Beta API is deprecated for 0.15.0 and later. 235 | 236 | It is recommended to use the GA API (classes and functions in this 237 | file not marked beta) for all further purposes. This function was 238 | generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" 239 | request_deserializers = { 240 | ('grpcresolver.hello.Hello', 'Greeter'): HelloRequest.FromString, 241 | ('grpcresolver.hello.Hello', 'GreeterResponseStream'): HelloRequest.FromString, 242 | ('grpcresolver.hello.Hello', 'StreamGreeter'): HelloRequest.FromString, 243 | ('grpcresolver.hello.Hello', 'StreamGreeterResponseStream'): HelloRequest.FromString, 244 | } 245 | response_serializers = { 246 | ('grpcresolver.hello.Hello', 'Greeter'): HelloResponse.SerializeToString, 247 | ('grpcresolver.hello.Hello', 'GreeterResponseStream'): HelloResponse.SerializeToString, 248 | ('grpcresolver.hello.Hello', 'StreamGreeter'): HelloResponse.SerializeToString, 249 | ('grpcresolver.hello.Hello', 'StreamGreeterResponseStream'): HelloResponse.SerializeToString, 250 | } 251 | method_implementations = { 252 | ('grpcresolver.hello.Hello', 'Greeter'): face_utilities.unary_unary_inline(servicer.Greeter), 253 | ('grpcresolver.hello.Hello', 'GreeterResponseStream'): face_utilities.unary_stream_inline(servicer.GreeterResponseStream), 254 | ('grpcresolver.hello.Hello', 'StreamGreeter'): face_utilities.stream_unary_inline(servicer.StreamGreeter), 255 | ('grpcresolver.hello.Hello', 'StreamGreeterResponseStream'): face_utilities.stream_stream_inline(servicer.StreamGreeterResponseStream), 256 | } 257 | server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) 258 | return beta_implementations.server(method_implementations, options=server_options) 259 | 260 | 261 | def beta_create_Hello_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): 262 | """The Beta API is deprecated for 0.15.0 and later. 263 | 264 | It is recommended to use the GA API (classes and functions in this 265 | file not marked beta) for all further purposes. This function was 266 | generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" 267 | request_serializers = { 268 | ('grpcresolver.hello.Hello', 'Greeter'): HelloRequest.SerializeToString, 269 | ('grpcresolver.hello.Hello', 'GreeterResponseStream'): HelloRequest.SerializeToString, 270 | ('grpcresolver.hello.Hello', 'StreamGreeter'): HelloRequest.SerializeToString, 271 | ('grpcresolver.hello.Hello', 'StreamGreeterResponseStream'): HelloRequest.SerializeToString, 272 | } 273 | response_deserializers = { 274 | ('grpcresolver.hello.Hello', 'Greeter'): HelloResponse.FromString, 275 | ('grpcresolver.hello.Hello', 'GreeterResponseStream'): HelloResponse.FromString, 276 | ('grpcresolver.hello.Hello', 'StreamGreeter'): HelloResponse.FromString, 277 | ('grpcresolver.hello.Hello', 'StreamGreeterResponseStream'): HelloResponse.FromString, 278 | } 279 | cardinalities = { 280 | 'Greeter': cardinality.Cardinality.UNARY_UNARY, 281 | 'GreeterResponseStream': cardinality.Cardinality.UNARY_STREAM, 282 | 'StreamGreeter': cardinality.Cardinality.STREAM_UNARY, 283 | 'StreamGreeterResponseStream': cardinality.Cardinality.STREAM_STREAM, 284 | } 285 | stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) 286 | return beta_implementations.dynamic_stub(channel, 'grpcresolver.hello.Hello', cardinalities, options=stub_options) 287 | except ImportError: 288 | pass 289 | # @@protoc_insertion_point(module_scope) 290 | -------------------------------------------------------------------------------- /tests/hello_pb2_grpc.py: -------------------------------------------------------------------------------- 1 | # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! 2 | import grpc 3 | 4 | import hello_pb2 as hello__pb2 5 | 6 | 7 | class HelloStub(object): 8 | 9 | def __init__(self, channel): 10 | """Constructor. 11 | 12 | Args: 13 | channel: A grpc.Channel. 14 | """ 15 | self.Greeter = channel.unary_unary( 16 | '/grpcresolver.hello.Hello/Greeter', 17 | request_serializer=hello__pb2.HelloRequest.SerializeToString, 18 | response_deserializer=hello__pb2.HelloResponse.FromString, 19 | ) 20 | self.GreeterResponseStream = channel.unary_stream( 21 | '/grpcresolver.hello.Hello/GreeterResponseStream', 22 | request_serializer=hello__pb2.HelloRequest.SerializeToString, 23 | response_deserializer=hello__pb2.HelloResponse.FromString, 24 | ) 25 | self.StreamGreeter = channel.stream_unary( 26 | '/grpcresolver.hello.Hello/StreamGreeter', 27 | request_serializer=hello__pb2.HelloRequest.SerializeToString, 28 | response_deserializer=hello__pb2.HelloResponse.FromString, 29 | ) 30 | self.StreamGreeterResponseStream = channel.stream_stream( 31 | '/grpcresolver.hello.Hello/StreamGreeterResponseStream', 32 | request_serializer=hello__pb2.HelloRequest.SerializeToString, 33 | response_deserializer=hello__pb2.HelloResponse.FromString, 34 | ) 35 | 36 | 37 | class HelloServicer(object): 38 | 39 | def Greeter(self, request, context): 40 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 41 | context.set_details('Method not implemented!') 42 | raise NotImplementedError('Method not implemented!') 43 | 44 | def GreeterResponseStream(self, request, context): 45 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 46 | context.set_details('Method not implemented!') 47 | raise NotImplementedError('Method not implemented!') 48 | 49 | def StreamGreeter(self, request_iterator, context): 50 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 51 | context.set_details('Method not implemented!') 52 | raise NotImplementedError('Method not implemented!') 53 | 54 | def StreamGreeterResponseStream(self, request_iterator, context): 55 | context.set_code(grpc.StatusCode.UNIMPLEMENTED) 56 | context.set_details('Method not implemented!') 57 | raise NotImplementedError('Method not implemented!') 58 | 59 | 60 | def add_HelloServicer_to_server(servicer, server): 61 | rpc_method_handlers = { 62 | 'Greeter': grpc.unary_unary_rpc_method_handler( 63 | servicer.Greeter, 64 | request_deserializer=hello__pb2.HelloRequest.FromString, 65 | response_serializer=hello__pb2.HelloResponse.SerializeToString, 66 | ), 67 | 'GreeterResponseStream': grpc.unary_stream_rpc_method_handler( 68 | servicer.GreeterResponseStream, 69 | request_deserializer=hello__pb2.HelloRequest.FromString, 70 | response_serializer=hello__pb2.HelloResponse.SerializeToString, 71 | ), 72 | 'StreamGreeter': grpc.stream_unary_rpc_method_handler( 73 | servicer.StreamGreeter, 74 | request_deserializer=hello__pb2.HelloRequest.FromString, 75 | response_serializer=hello__pb2.HelloResponse.SerializeToString, 76 | ), 77 | 'StreamGreeterResponseStream': grpc.stream_stream_rpc_method_handler( 78 | servicer.StreamGreeterResponseStream, 79 | request_deserializer=hello__pb2.HelloRequest.FromString, 80 | response_serializer=hello__pb2.HelloResponse.SerializeToString, 81 | ), 82 | } 83 | generic_handler = grpc.method_handlers_generic_handler( 84 | 'grpcresolver.hello.Hello', rpc_method_handlers) 85 | server.add_generic_rpc_handlers((generic_handler,)) 86 | -------------------------------------------------------------------------------- /tests/rpc.py: -------------------------------------------------------------------------------- 1 | 2 | import hello_pb2 3 | import hello_pb2_grpc 4 | 5 | 6 | __all__ = ['HelloGRpcServer'] 7 | 8 | 9 | class HelloGRpcServer(hello_pb2_grpc.HelloServicer): 10 | 11 | def Greeter(self, request, context): 12 | reply_msg = 'Hi, ' + request.message 13 | return hello_pb2.HelloResponse(message=reply_msg) 14 | 15 | def GreeterResponseStream(self, request, context): 16 | for msg in request.message: 17 | reply_msg = 'Hi, ' + msg 18 | yield hello_pb2.HelloResponse(message=reply_msg) 19 | 20 | def StreamGreeter(self, request_iterator, context): 21 | reply_msg = 'Hi, ' + ''.join( 22 | request.message for request in request_iterator) 23 | return hello_pb2.HelloResponse(message=reply_msg) 24 | 25 | def StreamGreeterResponseStream(self, request_iterator, context): 26 | for request in request_iterator: 27 | reply_msg = 'Hi, ' + request.message 28 | yield hello_pb2.HelloResponse(message=reply_msg) 29 | -------------------------------------------------------------------------------- /tests/test_address.py: -------------------------------------------------------------------------------- 1 | 2 | import json 3 | 4 | import pytest 5 | 6 | from grpcresolver.address import PlainAddress, JsonAddress 7 | 8 | 9 | @pytest.mark.parametrize( 10 | 'addr, exp_addr', ( 11 | ('1.2.3.4', '1.2.3.4'), 12 | ('5.6.7.8', '5.6.7.8'), 13 | ) 14 | ) 15 | def test_to_plain_address(addr, exp_addr): 16 | assert PlainAddress(addr).add_value() == exp_addr 17 | assert PlainAddress(addr).delete_value() == exp_addr 18 | 19 | 20 | @pytest.mark.parametrize( 21 | 'addr, exp_addr', ( 22 | (b'1.2.3.4', '1.2.3.4'), 23 | (b'5.6.7.8', '5.6.7.8'), 24 | (b'11.2.3.4', '11.2.3.4'), 25 | (b'55.6.7.8', '55.6.7.8'), 26 | ) 27 | ) 28 | def test_from_plain_address(addr, exp_addr): 29 | assert PlainAddress.from_value(addr) == exp_addr 30 | 31 | 32 | @pytest.mark.parametrize( 33 | 'val', ( 34 | '1.2.3.4', 35 | '5.6.7.8', 36 | ) 37 | ) 38 | def test_to_json_address(val): 39 | assert JsonAddress(val).add_value() == json.dumps({ 40 | 'Op': 0, 'Addr': val, 'Metadata': "{}"}) 41 | assert JsonAddress( 42 | val, metadata={'name': 'host1'}).delete_value() == json.dumps({ 43 | 'Op': 1, 'Addr': val, 'Metadata': json.dumps({'name': 'host1'})}) 44 | 45 | 46 | @pytest.mark.parametrize( 47 | 'val, op, addr', ( 48 | (b'{"Op": 1, "Addr": "1.2.3.4", "Metadata": "{}"}', False, '1.2.3.4'), 49 | (b'{"Op": 0, "Addr": "5.6.7.8", "Metadata": "{}"}', True, '5.6.7.8'), 50 | (b'{"Op": 1, "Addr": "11.2.3.4", "Metadata": "{}"}', False, '11.2.3.4'), 51 | (b'{"Op": 0, "Addr": "55.6.7.8", "Metadata": "{}"}', True, '55.6.7.8'), 52 | ) 53 | ) 54 | def test_from_json_address(val, op, addr): 55 | data = JsonAddress.from_value(val) 56 | assert (op, addr) == data 57 | -------------------------------------------------------------------------------- /tests/test_channel.py: -------------------------------------------------------------------------------- 1 | 2 | # -*- encoding: utf-8 -*- 3 | 4 | import pytest 5 | 6 | from grpcresolver import RoundrobinChannel 7 | import hello_pb2 8 | import hello_pb2_grpc 9 | 10 | 11 | @pytest.mark.parametrize( 12 | 'message, future', ( 13 | ('heslewle', False), 14 | ('sfwefewfwef', True), 15 | ('你好', False)) 16 | ) 17 | def test_unary_unary(grpc_server, grpc_resolver, message, future): 18 | channel = RoundrobinChannel('grpcresolver.hello.Hello', grpc_resolver) 19 | stub = hello_pb2_grpc.HelloStub(channel) 20 | 21 | request = hello_pb2.HelloRequest(message=message) 22 | if future: 23 | response = stub.Greeter.future(request).result() 24 | else: 25 | response = stub.Greeter(request) 26 | 27 | assert response.message 28 | 29 | 30 | @pytest.mark.parametrize( 31 | 'message', ( 32 | 'heslewle', 33 | 'sfwefewfwef', 34 | '你好') 35 | ) 36 | def test_unary_stream(grpc_server, grpc_resolver, message): 37 | channel = RoundrobinChannel('grpcresolver.hello.Hello', grpc_resolver) 38 | stub = hello_pb2_grpc.HelloStub(channel) 39 | 40 | request = hello_pb2.HelloRequest(message=message) 41 | response_iter = stub.GreeterResponseStream(request) 42 | 43 | for response in response_iter: 44 | assert response.message 45 | 46 | 47 | @pytest.mark.parametrize( 48 | 'messages, future', ( 49 | (('heslewle', 'sfwewef'), False), 50 | (('sfwefewfwef', 'sfwefwef'), True), 51 | (('shewfwefwef', '你好'), False)) 52 | ) 53 | def test_stream_unary(grpc_server, grpc_resolver, messages, future): 54 | channel = RoundrobinChannel('grpcresolver.hello.Hello', grpc_resolver) 55 | stub = hello_pb2_grpc.HelloStub(channel) 56 | 57 | request = (hello_pb2.HelloRequest(message=message) for message in messages) 58 | if future: 59 | response = stub.StreamGreeter.future(request).result() 60 | else: 61 | response = stub.StreamGreeter(request) 62 | 63 | assert response.message 64 | 65 | 66 | @pytest.mark.parametrize( 67 | 'messages', ( 68 | ('heslewle', 'sfwewef'), 69 | ('sfwefewfwef', 'sfwefwef'), 70 | ('shewfwefwef', '你好')) 71 | ) 72 | def test_stream_stream(grpc_server, grpc_resolver, messages): 73 | channel = RoundrobinChannel('grpcresolver.hello.Hello', grpc_resolver) 74 | stub = hello_pb2_grpc.HelloStub(channel) 75 | 76 | request = (hello_pb2.HelloRequest(message=message) for message in messages) 77 | response_iter = stub.StreamGreeterResponseStream(request) 78 | 79 | for response in response_iter: 80 | assert response.message 81 | 82 | 83 | def test_roundrobin_channel(grpc_resolver, grpc_addr): 84 | channel = RoundrobinChannel('grpcresolver.hello.Hello', grpc_resolver) 85 | addr_num = len(grpc_addr) 86 | loop_count = 10 87 | for addr_idx in range(addr_num * loop_count): 88 | assert channel.get_channel().target == grpc_addr[addr_idx % addr_num] 89 | -------------------------------------------------------------------------------- /tests/test_registry.py: -------------------------------------------------------------------------------- 1 | 2 | import pytest 3 | 4 | 5 | @pytest.mark.parametrize( 6 | 'service_names, service_addr, service_ttl', ( 7 | (('grpc.service_test', 'grpc.service_list'), '10.30.1.1.50011', 120), 8 | (('grpc.service_create', 'grpc.service_update'), '10.30.1.1.50011', 120), 9 | ) 10 | ) 11 | def test_service_registry( 12 | etcd_registry, service_names, service_addr, service_ttl): 13 | 14 | etcd_registry.register(service_names, service_addr, service_ttl) 15 | assert etcd_registry._services[service_addr] == set(service_names) 16 | assert service_addr in etcd_registry._leases 17 | 18 | etcd_registry.unregister((service_names[0],), service_addr) 19 | assert etcd_registry._services[service_addr] == {service_names[1]} 20 | assert service_addr in etcd_registry._leases 21 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | 2 | [tox] 3 | envlist = py27,py33,py34,py35,py36 4 | 5 | [testenv] 6 | 7 | deps = 8 | pytest-mock>=1.6.0 9 | pytest 10 | 11 | commands = pytest 12 | --------------------------------------------------------------------------------