├── .bumpversion.cfg ├── .gitignore ├── .travis.yml ├── CHANGES.rst ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.rst ├── conf ├── demo.yml ├── echo.yml ├── geocoder.yml ├── sample-node.yml ├── sentry.conf.py ├── tasks.yml └── web.yml ├── docs ├── Makefile ├── _diagrams │ └── serial_event.graffle ├── _static │ └── serial_event.svg ├── api │ ├── components_api.rst │ ├── config_api.rst │ ├── core_api.rst │ ├── index.rst │ ├── metrics_api.rst │ ├── patterns_api.rst │ ├── service_api.rst │ ├── testings_api.rst │ └── web_api.rst ├── cli.rst ├── conf.py ├── contributing.rst ├── faq.rst ├── glossary.rst ├── index.rst ├── installation.rst ├── internals │ └── index.rst ├── protocol.rst ├── topic_guides │ ├── configuration.rst │ ├── events.rst │ ├── http.rst │ ├── index.rst │ ├── rpc.rst │ ├── running_services.rst │ ├── serialization.rst │ ├── tasks.rst │ ├── testing.rst │ └── versioning.rst └── user_guide.rst ├── examples ├── demo.py ├── echo.py ├── geocoder.py ├── static │ ├── iris.svg │ ├── jquery.json.js │ ├── jquery.jsonrpcclient.js │ ├── jsonrpc.html │ └── monitor.html ├── tasks.py └── web.py ├── fabfile.py ├── lymph ├── __init__.py ├── autodoc.py ├── autoreload.py ├── cli │ ├── __init__.py │ ├── base.py │ ├── config.py │ ├── discover.py │ ├── emit.py │ ├── help.py │ ├── inspect.py │ ├── list.py │ ├── loglevel.py │ ├── main.py │ ├── request.py │ ├── service.py │ ├── shell.py │ ├── subscribe.py │ ├── tail.py │ ├── testing.py │ └── tests │ │ ├── __init__.py │ │ └── test_testing.py ├── client.py ├── config.py ├── core │ ├── __init__.py │ ├── channels.py │ ├── components.py │ ├── connection.py │ ├── container.py │ ├── declarations.py │ ├── decorators.py │ ├── events.py │ ├── interfaces.py │ ├── messages.py │ ├── monitoring │ │ ├── __init__.py │ │ ├── aggregator.py │ │ ├── global_metrics.py │ │ ├── metrics.py │ │ └── pusher.py │ ├── plugins.py │ ├── rpc.py │ ├── services.py │ ├── tests │ │ ├── __init__.py │ │ └── test_events.py │ ├── trace.py │ └── versioning.py ├── discovery │ ├── __init__.py │ ├── base.py │ ├── static.py │ └── zookeeper.py ├── events │ ├── __init__.py │ ├── base.py │ ├── kombu.py │ ├── local.py │ └── null.py ├── exceptions.py ├── monkey.py ├── patterns │ ├── __init__.py │ └── serial_events.py ├── plugins │ ├── __init__.py │ ├── newrelic.py │ └── sentry.py ├── serializers │ ├── __init__.py │ ├── base.py │ ├── kombu.py │ └── tests │ │ ├── __init__.py │ │ └── test_base.py ├── services │ ├── __init__.py │ ├── node.py │ └── scheduler.py ├── testing │ ├── __init__.py │ ├── mock_helpers.py │ ├── nose.py │ ├── nose2.py │ └── pytest.py ├── tests │ ├── __init__.py │ ├── integration │ │ ├── __init__.py │ │ ├── test_cli.py │ │ ├── test_kombu_events.py │ │ ├── test_web_interface.py │ │ └── test_zookeeper_discovery.py │ ├── monitoring │ │ ├── __init__.py │ │ ├── test_aggregator.py │ │ ├── test_global_metrics.py │ │ └── test_metrics.py │ ├── test_config.py │ ├── test_mock_helpers.py │ ├── test_mockcontainer.py │ ├── test_monkey.py │ ├── test_rpc_versioning.py │ └── test_utils.py ├── utils │ ├── __init__.py │ ├── event_indexing.py │ ├── gpool.py │ ├── logging.py │ ├── observables.py │ ├── ripdb.py │ ├── sockets.py │ └── tests │ │ ├── __init__.py │ │ ├── test_accumulator.py │ │ ├── test_event_indexing.py │ │ ├── test_logging.py │ │ ├── test_sockets.py │ │ └── test_utils.py └── web │ ├── __init__.py │ ├── handlers.py │ ├── interfaces.py │ ├── routing.py │ └── wsgi_server.py ├── nose2.cfg ├── requirements ├── dev.txt └── docs.txt ├── setup.cfg ├── setup.py └── tox.ini /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.16.0-dev 3 | parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?:-(?P\w+))? 4 | serialize = 5 | {major}.{minor}.{patch}-{release} 6 | {major}.{minor}.{patch} 7 | 8 | [bumpversion:part:release] 9 | optional_value = release 10 | values = 11 | dev 12 | release 13 | 14 | [bumpversion:file:setup.py] 15 | search = version='{current_version}' 16 | replace = version='{new_version}' 17 | 18 | [bumpversion:file:lymph/monkey.py] 19 | 20 | [bumpversion:file:docs/conf.py] 21 | 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .tox 2 | .coverage 3 | .coverage-report 4 | .lymph.yml 5 | *.pyc 6 | /env/ 7 | /lymph.egg-info/ 8 | /docs/_build/ 9 | /logs/ 10 | /lib/ 11 | /_*/ 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - 2.7 5 | - 3.4 6 | - 3.5 7 | 8 | services: 9 | - rabbitmq 10 | 11 | before_install: 12 | - sudo apt-get update 13 | 14 | install: 15 | - sudo apt-get install zookeeper 16 | - export MAJOR_TRAVIS_PYTHON_VERSION=$(python -c "import sys;sys.stdout.write(str(sys.version_info.major))") 17 | - pip install -r requirements/dev.txt 18 | 19 | before_script: 20 | - export ZOOKEEPER_PATH=/usr/share/java 21 | 22 | script: 23 | - mkdir -p tmp 24 | - pushd tmp 25 | - nosetests --with-lymph lymph 26 | - popd 27 | 28 | notifications: 29 | email: false 30 | irc: 31 | channels: 32 | - "chat.freenode.net#lymph" 33 | template: 34 | - "%{repository}#%{build_number} (%{branch} - %{commit} : %{author}): %{message}" 35 | - "Change view : %{compare_url}" 36 | - "Build details : %{build_url}" 37 | use_notice: true 38 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | 0.15.0 2 | ====== 3 | - Added nose2 plugin 4 | - Fixed access to dotted config keys 5 | 6 | 0.14.0 7 | ====== 8 | - Added RPC versioning support 9 | - Added --dump-headers option for ``lymph request`` 10 | 11 | 0.13.0 12 | ====== 13 | - Added ``--json`` option for ``lymph request`` 14 | - Added a generic health check endpoint for WebServiceInterface 15 | - Added `app_name` support for newrelic plugin 16 | 17 | 0.12.0 18 | ====== 19 | - Added monitoring for psutil metrics 20 | - Added RPC call entries in New Relic traces 21 | 22 | 0.11.0 23 | ====== 24 | - Made rpc connection properties configurable 25 | - Simplified collecting custom metrics 26 | 27 | 0.10.0 28 | ====== 29 | - Added retry support for event handlers. 30 | - Added event handler tracking for the newrelic plugin. 31 | - Fixed trace id propagation for deferred rpc calls. 32 | 33 | 0.9.0 34 | ===== 35 | - Added support for configurable metrics tags. 36 | 37 | 0.8.0 38 | ===== 39 | - Added lymph.task() decorator. 40 | 41 | 0.7.1 42 | ===== 43 | - Fixed a bug that prevented custom logging configuration from being used. 44 | 45 | 0.7.0 46 | ===== 47 | - Documentation cleanup 48 | - Monitoring data is now published on a random port. 49 | It is discoverable as `monitoring_endpoint`. 50 | - Changed datetime object serialization to always use ISO 8601 with timezone offset. 51 | Named timezones are no longer supported. This change is backwards incompatible. 52 | 53 | 0.6.0 54 | ===== 55 | - Added support for RabbitMQ failover for kombu events 56 | - Added `lymph config` command 57 | - Added option for `lymph request` to read request body from stdin 58 | 59 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include requirements/*.txt 3 | recursive-include lymph *.py 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: coverage docs flakes cloc 2 | 3 | coverage: 4 | -coverage run --timid --source=lymph -m py.test lymph 5 | coverage html 6 | 7 | docs: 8 | cd docs && make html 9 | 10 | clean-docs: 11 | cd docs && make clean html 12 | 13 | flakes: 14 | @flake8 lymph | cat 15 | 16 | cloc: 17 | @cloc --quiet lymph 18 | 19 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://travis-ci.org/deliveryhero/lymph.svg?branch=master 2 | :target: https://travis-ci.org/deliveryhero/lymph 3 | 4 | 5 | Lymph 6 | ===== 7 | 8 | lymph is an opinionated framework for Python services. Its features are 9 | 10 | * Discovery: pluggable service discovery (e.g. backed by ZooKeeper) 11 | * RPC: request-reply messaging (via ZeroMQ + MessagePack) 12 | * Events: pluggable and reliable pub-sub messaging (e.g. backed by RabbitMQ) 13 | * Process Management 14 | 15 | There's `documentation `_ on readthedocs.org. 16 | 17 | 18 | Installation (as a dependency) 19 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 20 | 21 | :: 22 | 23 | # py-monotime requires python headers, and gevent and cython require build-essential 24 | $ sudo apt-get install build-essential python-dev 25 | 26 | :: 27 | 28 | $ pip install https://github.com/deliveryhero/lymph.git#egg=lymph 29 | 30 | 31 | Development (of lymph itself) 32 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 33 | 34 | :: 35 | 36 | $ git clone https://github.com/deliveryhero/lymph.git 37 | $ cd lymph 38 | $ pip install -r requirements/dev.txt 39 | 40 | Run tests with ``tox``, build documentation with ``fab docs``. 41 | 42 | 43 | Running services 44 | ~~~~~~~~~~~~~~~~ 45 | 46 | To run the example services, you can use the example node config in 47 | ``conf/sample-node.yml``. You'll also need a local installation 48 | of `ZooKeeper`_ (with the configuration provided in the 49 | `Getting Started Guide`_) and `RabbitMQ`_:: 50 | 51 | $ export PYTHONPATH=examples 52 | $ cp conf/sample-node.yml .lymph.yml 53 | $ lymph node 54 | 55 | You can then discover running services:: 56 | 57 | $ lymph discover 58 | 59 | and send requests to them from the commandline:: 60 | 61 | $ lymph request echo.upper '{"text": "transform me"}' 62 | 63 | To see the log output of a running service, try:: 64 | 65 | $ lymph tail echo -l DEBUG 66 | 67 | 68 | .. _ZooKeeper: http://zookeeper.apache.org 69 | .. _Getting Started Guide: http://zookeeper.apache.org/doc/trunk/zookeeperStarted.html 70 | .. _RabbitMQ: http://www.rabbitmq.com/ 71 | -------------------------------------------------------------------------------- /conf/demo.yml: -------------------------------------------------------------------------------- 1 | interfaces: 2 | demo: 3 | class: demo:Client 4 | delay: 1 5 | -------------------------------------------------------------------------------- /conf/echo.yml: -------------------------------------------------------------------------------- 1 | interfaces: 2 | echo: 3 | class: echo:EchoService 4 | -------------------------------------------------------------------------------- /conf/geocoder.yml: -------------------------------------------------------------------------------- 1 | interfaces: 2 | geocoder: 3 | class: geocoder:Geocoder 4 | -------------------------------------------------------------------------------- /conf/sample-node.yml: -------------------------------------------------------------------------------- 1 | container: 2 | registry: 3 | class: lymph.discovery.zookeeper:ZookeeperServiceRegistry 4 | zkclient: 5 | class: kazoo.client:KazooClient 6 | hosts: 127.0.0.1:2181 7 | 8 | events: 9 | class: lymph.events.kombu:KombuEventSystem 10 | transport: amqp 11 | hostname: 127.0.0.1 12 | 13 | 14 | instances: 15 | echo: 16 | command: lymph instance --config=conf/echo.yml 17 | 18 | demo: 19 | command: lymph instance --config=conf/demo.yml 20 | -------------------------------------------------------------------------------- /conf/tasks.yml: -------------------------------------------------------------------------------- 1 | interfaces: 2 | job: 3 | class: tasks:TaskService 4 | -------------------------------------------------------------------------------- /conf/web.yml: -------------------------------------------------------------------------------- 1 | container: 2 | service_name: web 3 | 4 | interfaces: 5 | web: 6 | class: web:JsonrpcGateway 7 | port: 4080 8 | -------------------------------------------------------------------------------- /docs/api/components_api.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: lymph.core.components 2 | 3 | 4 | Components API 5 | ============== 6 | 7 | Components are objects that depend on a running service container. They are 8 | embedded in :class:`Componentized` objects. 9 | Since Componentized objects themselves are components, they form a tree of 10 | :class:`Component` instances with the container as the root. An example 11 | of a Component is :class:`lymph.core.interfaces.Interface`. 12 | 13 | 14 | .. class:: Component(error_hook=None, pool=None, metrics=None) 15 | 16 | .. attribute:: error_hook 17 | 18 | A Hook object that propagates exceptions for this component. 19 | Defaults to the ``error_hook`` of the parent component. 20 | 21 | .. attribute:: pool 22 | 23 | A pool that holds greenlets related to the component. 24 | Defaults to the ``pool`` of the parent component. 25 | 26 | .. attribute:: metrics 27 | 28 | An :class:`Aggregate ` of metrics for this component. 29 | Defaults to the ``metrics`` of the parent component. 30 | 31 | .. method:: on_start() 32 | 33 | Called when the container is started. 34 | 35 | .. method:: on_stop() 36 | 37 | Called when the container is stopped. 38 | 39 | .. method:: spawn(func, *args, **kwargs) 40 | 41 | Spawns a new greenlet in the greenlet pool of this component. 42 | If ``func`` exits with an exception, it is reported to the ``error_hook``. 43 | 44 | 45 | .. class:: Componentized() 46 | 47 | A collection of components; itself a component. 48 | 49 | .. method:: add_component(component) 50 | 51 | :param component: :class:`Component` 52 | 53 | Adds `component`. 54 | 55 | .. method:: on_start() 56 | 57 | Calls `on_start()` on all added components. 58 | 59 | .. method:: on_stop() 60 | 61 | Calls `on_stop()` on all added components. 62 | -------------------------------------------------------------------------------- /docs/api/config_api.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: lymph.config 2 | 3 | 4 | Config API 5 | ========== 6 | 7 | .. class:: ConfigView(config, prefix) 8 | 9 | A ConfigView allows access to a subtree of a :class:`Configuration` object. 10 | It implements the mapping protocol. Dotted path keys are translated into 11 | nested dictionary lookups, i.e. ``cv.get('a.b')`` is (roughly) equivalent to 12 | ``cv.get('a').get('b')``. 13 | 14 | If a value returned by :class:`ConfigView` methods is a dict, it will be 15 | wrapped in a :class:`ConfigView` itself. This – and getting dicts from a 16 | :class:`Configuration` object – are the preferred way to create new ConfigViews. 17 | 18 | 19 | .. attribute:: root 20 | 21 | A reference to the root :class:`Configuration` instance. 22 | 23 | 24 | .. class:: Configuration(values=None) 25 | 26 | :param values: an optional initial mapping 27 | 28 | Configuration implements the same interface as :class:`ConfigView` in addition 29 | to the methods described here. 30 | 31 | .. method:: load(file, sections=None) 32 | 33 | Reads yaml configuration from a file-like object. If sections is not 34 | None, only the keys given are imported 35 | 36 | .. method:: load_file(path, sections=None) 37 | 38 | Reads yaml configuration from the file at ``path``. 39 | 40 | .. method:: get_raw(key, default) 41 | 42 | Like ``get()``, but doesn't wrap dict values in :class:`ConfigView`. 43 | 44 | .. method:: create_instance(key, default_class=None, **kwargs) 45 | 46 | :param key: dotted config path (e.g. ``"container.rpc"``) 47 | :param default_class: class object or fully qualified name of a class 48 | :param kwargs: extra keyword arguments to be passed to the factory 49 | 50 | Creates an object from the config dict at ``key``. The instance is 51 | created by a factory that is specified by its fully qualified name in 52 | a ``class`` key of the config dict. 53 | 54 | If the factory has a ``from_config()`` method it is called with a :class:`ConfigView` 55 | of ``key``. Otherwise, the factory is called directly with the config values as keyword arguments. 56 | 57 | Extra keyword arguments to ``create_instance()`` are passed through to ``from_config()`` or mixed 58 | into the arguments if the factory is a plain callable. 59 | 60 | If the config doesn't have a ``class`` key the instance is create by ``default_class``, which can be 61 | either a fully qualifed name or a factory object. 62 | 63 | Given the following config file 64 | 65 | .. code-block:: yaml 66 | 67 | foo: 68 | class: pack.age:SomeClass 69 | extra_arg: 42 70 | 71 | 72 | you can create an instance of SomeClass 73 | 74 | .. code-block:: python 75 | 76 | # in pack/age.py 77 | class SomeClass(object): 78 | @classmethod 79 | def from_config(cls, config, **kwargs): 80 | assert config['extra_arg'] == 42 81 | assert kwargs['bar'] is True 82 | return cls(...) 83 | 84 | # in any module 85 | config = Configuration() 86 | config.load(...) 87 | config.create_instance('foo', bar=True) 88 | 89 | .. method:: get_instance(key, default_class, **kwargs) 90 | 91 | Like ``create_instance()``, but only creates a single instance for each 92 | key. 93 | 94 | 95 | -------------------------------------------------------------------------------- /docs/api/core_api.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: lymph.core.container 2 | 3 | 4 | Core API 5 | ======== 6 | 7 | 8 | .. class:: ServiceContainer 9 | 10 | .. classmethod:: from_config(config, **kwargs) 11 | 12 | .. method:: start() 13 | 14 | .. method:: stop() 15 | 16 | .. method:: send_message(address, msg) 17 | 18 | :param address: the address for this message; either a ZeroMQ endpoint a service name 19 | :param msg: the :class:`lymph.core.messages.Message` object that will be sent 20 | :return: :class:`lymph.core.channels.ReplyChannel` 21 | 22 | .. method:: lookup(address) 23 | 24 | :param address: an lymph address 25 | :return: :class:`lymph.core.services.Service` or :class:`lymph.core.services.ServiceInstance` 26 | 27 | 28 | .. currentmodule:: lymph.core.channels 29 | 30 | .. class:: ReplyChannel() 31 | 32 | .. method:: reply(body) 33 | 34 | :param body: a JSON serializable data structure 35 | 36 | .. method:: ack() 37 | 38 | acknowledges the request message 39 | 40 | 41 | .. class:: RequestChannel() 42 | 43 | .. method:: get(timeout=1) 44 | 45 | :return: :class:`lymph.core.messages.Message` 46 | 47 | returns the next reply message from this channel. Blocks until the reply 48 | is available. Raises :class:`Timeout ` after ``timeout`` seconds. 49 | 50 | 51 | .. currentmodule:: lymph.core.messages 52 | 53 | .. class:: Message 54 | 55 | .. attribute:: id 56 | 57 | .. attribute:: type 58 | 59 | .. attribute:: subject 60 | 61 | .. attribute:: body 62 | 63 | .. attribute:: packed_body 64 | 65 | 66 | .. currentmodule:: lymph.core.events 67 | 68 | .. class:: Event 69 | 70 | .. attribute:: type 71 | 72 | the event type / name 73 | 74 | .. attribute:: body 75 | 76 | dictionary with the payload of the message 77 | 78 | .. attribute:: source 79 | 80 | id of the event source service 81 | 82 | .. method:: __getitem__(name) 83 | 84 | gets an event parameter from the body 85 | 86 | 87 | .. currentmodule:: lymph.core.services 88 | 89 | 90 | .. class:: Service() 91 | 92 | Normally created by :meth:`ServiceContainer.lookup() `. 93 | Service objects represent lymph services. 94 | 95 | .. method:: __iter__() 96 | 97 | Yields all known :class:`instances ` of this service. 98 | 99 | .. method:: __len__() 100 | 101 | Returns the number of known instances of this service. 102 | 103 | 104 | .. class:: ServiceInstance() 105 | 106 | Describes a single service instance. 107 | Normally created by :meth:`ServiceContainer.lookup() ` 108 | 109 | .. attribute:: identity 110 | 111 | The identity string of this service instance 112 | 113 | .. attribute:: endpoint 114 | 115 | The rpc endpoint for this 116 | 117 | 118 | .. currentmodule:: lymph.core.connections 119 | 120 | .. class:: Connection 121 | 122 | You can attain a connection to an lymph service instance directly from :meth:`lymph.core.container.ServiceContainer.connect`, or 123 | from the higher-level API in :mod:`lymph.core.services`. 124 | For ZeroMQ endpoint addresses the following to statements are roughly equivalent:: 125 | 126 | container.connect(address) # only works for tcp://… addresses 127 | container.lookup(address).connect() # will also work for service names 128 | 129 | 130 | .. currentmodule:: lymph.core.interfaces 131 | 132 | .. class:: Proxy(container, address, namespace=None, timeout=1) 133 | 134 | .. method:: __getattr__(self, name) 135 | 136 | Returns a callable that will execute the RPC method with the given name. 137 | 138 | -------------------------------------------------------------------------------- /docs/api/index.rst: -------------------------------------------------------------------------------- 1 | 2 | API reference 3 | ============= 4 | 5 | Contents: 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | service_api 11 | core_api 12 | config_api 13 | web_api 14 | patterns_api 15 | components_api 16 | metrics_api 17 | testings_api 18 | -------------------------------------------------------------------------------- /docs/api/metrics_api.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: lymph.core.monitoring.metrics 2 | 3 | 4 | Metrics API 5 | =========== 6 | 7 | To follow the metrics protocol objects must be iterable repeatedly and yield 8 | ``(name, value, tags)``-triples, where ``name`` is a string, ``value`` is a float or int, 9 | and ``tags`` is a dict with string keys and values. 10 | 11 | 12 | .. class:: Metric(name, tags=None) 13 | 14 | An abstract base class for single series metrics, i.e. metric objects that 15 | only yield a single triple. 16 | 17 | .. method:: __iter__() 18 | 19 | **[abstract]** Yields metric values as a tuple in the form 20 | `(name, value, tags)`. 21 | 22 | 23 | .. class:: Gauge(name, value=0, tags=None) 24 | 25 | A gauge is a metric that represents a single numerical value that can 26 | arbitrarily go up and down. 27 | 28 | .. method:: set(value) 29 | 30 | 31 | .. class:: Callable(name, func, tags=None) 32 | 33 | Like a Gauge metric, but its value is determined by a callable. 34 | 35 | 36 | .. class:: Counter(name, tags=None) 37 | 38 | A counter is a cumulative metric that represents a single numerical 39 | value that only ever goes up. A counter is typically used to count 40 | requests served, tasks completed, errors occurred, etc. 41 | 42 | .. method:: __iadd__(value) 43 | 44 | Increment counter value. 45 | 46 | 47 | .. class:: TaggedCounter(name, tags=None) 48 | 49 | A tagged counter is a container metric that represents multiple 50 | counters per tags. A tagged counter is typically used to track a group 51 | of counters as one e.g. request served per function name, errors ocurred 52 | per exception name, etc. 53 | 54 | .. method:: incr(_by=1, **tags) 55 | 56 | Increment given counter ``type`` by ``_by``. 57 | 58 | 59 | .. class:: Aggregate(metrics=(), tags=None) 60 | 61 | :param metrics: iterable of metric objects 62 | :param tags: dict of tags to add to all metrics. 63 | 64 | Aggregates a collection of metrics into a single metrics object. 65 | 66 | .. method:: add(metric) 67 | 68 | :param metric: metric object 69 | 70 | Adds the given metric to collection. 71 | 72 | .. method:: add_tags(**tags) 73 | 74 | :param tags: string-valued dict 75 | 76 | Adds the given tags for all metrics. 77 | -------------------------------------------------------------------------------- /docs/api/patterns_api.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: lymph.patterns 2 | 3 | Pattern API 4 | =========== 5 | 6 | .. currentmodule:: lymph.patterns.serial_events 7 | 8 | .. decorator:: serial_event(*event_types, partition_count=12, key=None) 9 | 10 | :param event_types: event types that should be partitioned 11 | :param partition_count: number of queues that should be used to partition the events 12 | :param key: a function that maps :class:`Events ` to string keys. 13 | This function should have two arguments in its signature: the instance of 14 | current :class:`Interface ` and instance of the handled 15 | :class:`Event ` object. 16 | 17 | This event handler redistributes events into ``partition_count`` queues. 18 | These queues are then partitioned over all service instances and consumed sequentially, 19 | i.e. at most one event per queue at a time. 20 | 21 | .. image:: /../_static/serial_event.svg 22 | -------------------------------------------------------------------------------- /docs/api/web_api.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: lymph.web 2 | 3 | 4 | Web API 5 | ======== 6 | 7 | .. class:: WebServiceInterface 8 | 9 | .. attribute:: application 10 | 11 | WSGI application instance that this interface is running 12 | 13 | .. attribute:: url_map 14 | 15 | A `werkzeug.routing.Map`_ instance that is used to map requests to 16 | request handlers. Typically given as a class attribute. 17 | 18 | 19 | .. _werkzeug.routing.Map: http://werkzeug.pocoo.org/docs/0.10/routing/#maps-rules-and-adapters -------------------------------------------------------------------------------- /docs/cli.rst: -------------------------------------------------------------------------------- 1 | :tocdepth: 1 2 | 3 | Command Line Interface 4 | ====================== 5 | 6 | Lymph's cli lets you run, discover, inspect and interact with services. It is 7 | built to be your toolbelt when developing and running services. The cli is 8 | extensible. You can write custom lymph subcommands, e.g. `lymph top`_. 9 | 10 | .. contents:: 11 | :local: 12 | 13 | .. note:: 14 | 15 | Many of lymph's commands produce unicode output. Therefore, you'll have to 16 | set your locale (LC_ALL or LC_CTYPE) to UTF-8. 17 | 18 | If you want to pipe lymph commands with Python 2, you might have to set 19 | PYTHONIOENCODING to UTF-8 as well. 20 | 21 | Check the :ref:`FAQ `. 22 | 23 | 24 | This is an overview of lymph's cli. We don't document every command's 25 | arguments and parameters on purpose. Each is self-documenting: 26 | 27 | .. code:: bash 28 | 29 | $ lymph help # or 30 | $ lymph --help 31 | 32 | 33 | lymph list 34 | ------------ 35 | 36 | Prints a list of all available commands with their description. 37 | 38 | 39 | lymph instance 40 | ---------------- 41 | 42 | Runs a service instance. 43 | 44 | 45 | lymph discover 46 | ---------------- 47 | 48 | Discovers all available services and their instances, e.g.: 49 | 50 | 51 | lymph inspect 52 | --------------- 53 | 54 | Prints the RPC interface of a service with signature and docstrings. 55 | 56 | 57 | lymph request 58 | --------------- 59 | 60 | Invokes an RPC method of a service and prints the response. 61 | 62 | 63 | lymph emit 64 | ------------ 65 | 66 | Emits an event in the event system. 67 | 68 | 69 | lymph subscribe 70 | ----------------- 71 | 72 | Subscribes to an event type and prints every occurence. 73 | 74 | 75 | lymph node 76 | ------------ 77 | 78 | This is lymph's development server. It can run any number of services with any 79 | number of instances as well as any other dependency. 80 | 81 | 82 | lymph shell 83 | ------------- 84 | 85 | Starts an interactive Python shell for service instance, locally or remotely. 86 | 87 | 88 | lymph config 89 | -------------- 90 | 91 | Prints configuration for inspection 92 | 93 | 94 | 95 | .. _lymph top: http://github.com/mouadino/lymph-top 96 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | We try to follow `C4 (Collective Code Construction Contract)`_ for lymph development. 5 | Issues are tracked `on github `_. 6 | We accept code and documentation contributions via pull requests. 7 | 8 | 9 | .. _C4 (Collective Code Construction Contract): http://rfc.zeromq.org/spec:16 -------------------------------------------------------------------------------- /docs/faq.rst: -------------------------------------------------------------------------------- 1 | :tocdepth: 1 2 | .. _faq: 3 | 4 | FAQ 5 | === 6 | 7 | .. contents:: 8 | :local: 9 | 10 | Why does lymph crash with UnicodeDecodeError: 'ascii' codec can't encode character …? 11 | -------------------------------------------------------------------------------------- 12 | 13 | Since many lymph commands produce unicode output, you have to set your locale 14 | to UTF-8, e.g. with 15 | 16 | .. code:: bash 17 | 18 | $ export LC_ALL=en_US.UTF-8 19 | 20 | If you want to pipe lymph commands with Python 2, you might also have to set 21 | ``PYTHONIOENCODING`` 22 | 23 | .. code:: bash 24 | 25 | $ export PYTHONIOENCODING=UTF-8 26 | -------------------------------------------------------------------------------- /docs/glossary.rst: -------------------------------------------------------------------------------- 1 | Glossary 2 | ======== 3 | 4 | .. glossary:: 5 | 6 | service interface 7 | A collection of rpc methods and event listeners that are exposed by a service container. 8 | Interfaces are implemented as subclasses of :class:`lymph.Interface`. 9 | 10 | service container 11 | A service container manages rpc and event connections, service discovery, logging, and configuration 12 | for one or more service interfaces. There is one container per service instance. 13 | 14 | Containers are :class:`ServiceContainer ` objects. 15 | 16 | service instance 17 | A single process that runs a service container. 18 | It is usually created from the commandline with :ref:`lymph instance `. 19 | Each instance is assigned a unique identifier called *instances identity*. 20 | 21 | Instances are described by :class:`ServiceInstance ` objects. 22 | 23 | service 24 | A set of all service instances that exposes a common service interface is called a service. 25 | Though uncommon, instances may be part of more than one service. 26 | 27 | Services are described by :class:`Service ` objects. 28 | 29 | node 30 | A process monitor that runs service instances. You'd typically run one per machine. 31 | A node is started from the commandline with :ref:`lymph node `. 32 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | 2 | Welcome to lymph's documentation! 3 | ================================= 4 | 5 | lymph is a framework for Python services. lymph intends to be the glue between 6 | your services so you don't get sticky fingers. 7 | 8 | This is what a service looks like with lymph: 9 | 10 | .. code:: python 11 | 12 | import lymph 13 | 14 | 15 | class Greeting(lymph.interface): 16 | 17 | @lymph.rpc() 18 | def greet(self, name): 19 | ''' 20 | Returns a greeting for the given name 21 | ''' 22 | print(u'Saying to hi to %s' % name) 23 | self.emit(u'greeted', {'name': name}) 24 | return u'Hi, %s' % name 25 | 26 | Contents: 27 | 28 | .. toctree:: 29 | :maxdepth: 2 30 | 31 | installation 32 | user_guide 33 | cli 34 | topic_guides/index 35 | api/index 36 | protocol 37 | glossary 38 | faq 39 | contributing 40 | 41 | 42 | Indices and tables 43 | ================== 44 | 45 | * :ref:`genindex` 46 | * :ref:`modindex` 47 | * :ref:`search` 48 | 49 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | :tocdepth: 1 2 | 3 | Installation 4 | ============ 5 | 6 | Installing lymph itself (for Python 2.7 or 3.4) is as simple as: 7 | 8 | .. code:: bash 9 | 10 | pip install lymph 11 | 12 | Yet, in order to make full use of lymph you'll also need to install lymph's dependencies: 13 | `ZooKeeper`_ (for service discovery) and `RabbitMQ`_ (for events) and have them 14 | running. 15 | 16 | If these are already set up, you can skip straight and continue the next 17 | chapter. 18 | 19 | 20 | Installing dependencies 21 | ~~~~~~~~~~~~~~~~~~~~~~~ 22 | 23 | The RabbitMQ server's default configuration is enough for development and 24 | testing. For detailed information on how to configure ZooKeeper refer to the 25 | `ZooKeeper`_ webpage and the `Getting Started Guide`_. However, it's default 26 | configuration should also be enough. 27 | 28 | 29 | On Ubuntu 30 | --------- 31 | 32 | If you haven't already install Python essentials: 33 | 34 | .. code:: bash 35 | 36 | $ sudo apt-get install build-essential python-dev python-pip 37 | 38 | Install and start ZooKeeper using: 39 | 40 | .. code:: bash 41 | 42 | $ sudo apt-get install zookeeper zookeeperd 43 | $ sudo service zookeeper start 44 | 45 | ZooKeeper's configuration file is located at ``/etc/zookeeper/conf/zoo.cfg``. 46 | 47 | Install and start the RabbitMQ server: 48 | 49 | .. code:: bash 50 | 51 | $ sudo apt-get install rabbitmq-server 52 | $ sudo service rabbitmq-server start 53 | 54 | 55 | On OSX 56 | ------ 57 | 58 | Install RabbitMQ and ZooKeeper: 59 | 60 | .. code:: bash 61 | 62 | $ brew install rabbitmq zookeeper 63 | 64 | ZooKeeper's configuration file is located at 65 | ``/usr/local/etc/zookeeper/zoo.cfg``. 66 | 67 | 68 | .. _ZooKeeper: http://zookeeper.apache.org 69 | .. _RabbitMQ: http://www.rabbitmq.com/ 70 | .. _Getting Started Guide: http://zookeeper.apache.org/doc/trunk/zookeeperStarted.html 71 | .. _tox: https://testrun.org/tox/latest/ 72 | -------------------------------------------------------------------------------- /docs/internals/index.rst: -------------------------------------------------------------------------------- 1 | 2 | Internals 3 | ========= 4 | 5 | Contents: 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | protocol 11 | -------------------------------------------------------------------------------- /docs/protocol.rst: -------------------------------------------------------------------------------- 1 | The Lymph RPC Protocol 2 | ====================== 3 | 4 | Message format: 5 | 6 | ===== ======== =========================================================== 7 | Index Name Content 8 | ===== ======== =========================================================== 9 | 0 ID a random uuid 10 | 1 Type ``REQ``, ``REP``, ``ACK``, ``NACK``, or ``ERROR`` 11 | 2 Subject method name for "REQ" messages, else: 12 | message id of the corresponding request 13 | 3 Headers msgpack encoded header dict 14 | 4 Body msgpack encoded body 15 | ===== ======== =========================================================== 16 | -------------------------------------------------------------------------------- /docs/topic_guides/http.rst: -------------------------------------------------------------------------------- 1 | HTTP 2 | ==== 3 | 4 | .. code-block:: python 5 | 6 | from lymph.web.interfaces import WebServiceInterface 7 | from werkzeug.routing import Map, Rule 8 | from werkzeug.wrappers import Response 9 | 10 | 11 | class HttpHello(WebServiceInterface) 12 | url_map = Map([ 13 | Rule('/hello//', endpoint='hello'), 14 | ]) 15 | 16 | def hello(self, request, name): 17 | return Response('hello %s!' % name) 18 | 19 | 20 | .. class:: WebServiceInterface 21 | 22 | .. method:: is_healthy() 23 | 24 | 25 | Interface configuration 26 | ~~~~~~~~~~~~~~~~~~~~~~~~ 27 | 28 | .. describe:: interfaces..healthcheck.enabled 29 | 30 | Boolean: whether to respond to requests to ``interfaces..healthcheck.endpoint``. 31 | Defaults to ``True``. 32 | 33 | .. describe:: interfaces..healthcheck.endpoint 34 | 35 | Respond with 200 to requests for this path as long as :meth:`is_healthy() ` returns True, and 503 otherwise. 36 | Defaults to ``"/_health/"``. 37 | 38 | .. describe:: interfaces..port 39 | 40 | Listen on this port. Defaults to a random port. 41 | 42 | .. describe:: interfaces..wsgi_pool_size 43 | 44 | .. describe:: interfaces..tracing.request_header 45 | 46 | Name of an HTTP request header that may provide the trace id. 47 | Defaults to ``None``. 48 | 49 | .. describe:: interfaces..tracing.response_header 50 | 51 | Name of the HTTP response header that contains the trace id. 52 | Defaults to ``"X-Trace-Id"``. 53 | -------------------------------------------------------------------------------- /docs/topic_guides/index.rst: -------------------------------------------------------------------------------- 1 | :tocdepth: 2 2 | 3 | Topic guides 4 | ============ 5 | 6 | .. toctree:: 7 | :maxdepth: 1 8 | 9 | running_services 10 | configuration 11 | testing 12 | events 13 | rpc 14 | http 15 | serialization 16 | versioning 17 | -------------------------------------------------------------------------------- /docs/topic_guides/running_services.rst: -------------------------------------------------------------------------------- 1 | Running services 2 | ================ 3 | 4 | Overview 5 | ~~~~~~~~ 6 | 7 | There are two ways to start services with lymph. You can either start a lymph 8 | service directly from the command line using ``lymph instance`` or define 9 | all the services to start in a configuration file and start them all with 10 | lymph's development server ``lymph node``. 11 | 12 | 13 | lymph instance 14 | ~~~~~~~~~~~~~~ 15 | 16 | This command runs a single service instance given a config file with :ref:`interfaces ` 17 | 18 | .. code:: bash 19 | 20 | lymph instance --config=$PATH_TO_CONFIG_FILE 21 | 22 | 23 | 24 | Writing configuration files for ``lymph instance`` 25 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 26 | 27 | A configuration file of a lymph service requires the following sections: 28 | 29 | - container 30 | - interfaces 31 | 32 | You need to define a separate configuration file for each service or instance setup. If you have many services 33 | running, which would be the normal case in a productive lymph setup, the same information about ``container`` 34 | would be present in each file. In order to avoid having to copy the same information into every 35 | file and obtain a configuration mess, it is possible to set a default configuration file where lymph extracts the 36 | necessary information. This is usually the ``.lymph.yml`` file, which is also needed by ``lymph node`` (the standard 37 | way to start lymph services, see :doc: ``lymph node`` below). 38 | 39 | The default configuration file is set using the ``LYMPH_NODE_CONFIG`` environmental variable and is usually set by 40 | 41 | .. code:: bash 42 | 43 | $ export LYMPH_NODE_CONFIG="/path/to/lymph/config/.lymph.yml" 44 | 45 | .. describe:: interfaces 46 | 47 | Each service needs to have its ``interfaces`` defined in the respective service configuration file. The ``interfaces`` 48 | section defines which endpoints a service has (a service can have multiple endpoints) and the configuration of 49 | each endpoint (you can have multiple endpoints to the same service interface class, with different configurations). 50 | 51 | The interfaces section is made up of 52 | 53 | .. describe:: interfaces. 54 | 55 | Mapping from service name to instance configuration that will be passed to 56 | the implementation's :meth:`lymph.Service.apply_config()` method. 57 | 58 | which gives a name to a specific interface (i.e. the ``namespace`` part when referencing a service). If the interface 59 | has been named, it needs to be linked to a class that is a subclass of :class: `lymph.Interface`. 60 | 61 | .. describe:: interfaces..class 62 | 63 | The class that implements this interface, e.g. a subclass of :class:`lymph.Interface`. 64 | 65 | After the interface class has been defined, any additional configuration can be passed on to the interface class by 66 | defining any 67 | 68 | .. describe:: interfaces.. 69 | 70 | The whole ``interfaces.`` dict is available as configuration for the 71 | interface class. 72 | 73 | 74 | A simple example for an interface definition is: 75 | 76 | .. code:: yaml 77 | 78 | interfaces: 79 | echo: 80 | class: echo:EchoService 81 | 82 | and another example showing the use of additional interface options and the definition of multiple interfaces: 83 | 84 | .. code:: yaml 85 | 86 | interfaces: 87 | echo_small_valley: 88 | class: echo:EchoService 89 | delay: 1 90 | 91 | echo_large_valley: 92 | class: echo:EchoService 93 | delay: 10 94 | 95 | lymph node 96 | ----------- 97 | 98 | This command will start instances of services as defined in a configuration file. 99 | It will load as many instances as specified for each defined service. By default it will 100 | read the ``.lymph.yml`` file, but through the ``--config`` option, you can specify another 101 | configuration. You run this command by initiating: 102 | 103 | .. code:: bash 104 | 105 | $ lymph node 106 | 107 | 108 | Configuring ``lymph node`` 109 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 110 | 111 | .. describe:: instances. 112 | 113 | Besides the usual configuration sections for the ``container``, a 114 | section on ``instances`` needs to be added. In this section, each service is defined, 115 | together with the ``lymph instance`` command to start it, and the number of processes 116 | ``numprocesses`` each service should have. 117 | 118 | .. describe:: instances..command: 119 | 120 | A command (does not necessarily have to be a ``lymph instance`` command) that will 121 | be spawned by ``lymph node`` 122 | 123 | .. describe:: instances..numprocesses: 124 | 125 | Number of times the defined command is spawned 126 | 127 | 128 | An example of such an ``instances`` configuration block: 129 | 130 | .. code:: 131 | 132 | instances: 133 | echo: 134 | command: lymph instance --config=conf/echo.yml 135 | numprocesses: 10 136 | 137 | demo: 138 | command: lymph instance --config=conf/demo.yml 139 | 140 | -------------------------------------------------------------------------------- /docs/topic_guides/serialization.rst: -------------------------------------------------------------------------------- 1 | Serialization 2 | ============= 3 | 4 | Overview 5 | ~~~~~~~~ 6 | 7 | Lymph uses `msgpack`_ to serialize events and rpc arguments. 8 | In addition to the types supported directly by msgpack, the lymph serializer 9 | also handles the following basic Python types: 10 | ``set``, ``datetime.datetime``, ``datetime.date``, ``datetime.time``, ``uuid.UUID``, and ``decimal.Decimal``. 11 | 12 | 13 | .. _msgpack: www.msgpack.org 14 | 15 | 16 | Object level serialization 17 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 18 | 19 | Object level serialization can be defined by implementing ``_lymph_dump_`` method in classes subject to serialization. 20 | 21 | Object-level serialization can help to produce more concise code in certain situations, e.g.: 22 | 23 | .. code:: python 24 | 25 | class Process(object): 26 | ... 27 | 28 | def _lymph_dump_(self): 29 | return { 30 | 'pid': self.pid, 31 | 'name': self.name, 32 | } 33 | 34 | 35 | class Node(lymph.Interface): 36 | 37 | @lymph.rpc() 38 | def get_processes(self, service_type=None): 39 | procs = [] 40 | for proc in self._processes: 41 | if not service_type or proc.service_type == service_type: 42 | procs.append(proc) 43 | return procs 44 | 45 | @lymph.rpc() 46 | def stop(self, service_type=None): 47 | for proc in self.get_processes(service_type): 48 | proc.stop() 49 | 50 | In the example above by defining the ``_lymph_dump_`` in our Process class, we were able to reuse the rpc 51 | function ``get_processes``. 52 | -------------------------------------------------------------------------------- /docs/topic_guides/tasks.rst: -------------------------------------------------------------------------------- 1 | .. _topic-tasks: 2 | 3 | 4 | Tasks 5 | ===== 6 | 7 | .. code:: python 8 | 9 | import lymph 10 | import requests 11 | 12 | class BackgroundPush(lymph.Interface): 13 | @lymph.task() 14 | def push_to_3rd_party(self, data): 15 | requests.post("http://3rd-party.example.com/push", data) 16 | 17 | @lymph.rpc() 18 | def push(self, data): 19 | self.push_to_3rd_party.apply(data=data) 20 | 21 | 22 | Running worker instances: 23 | 24 | .. code:: 25 | 26 | $ lymph worker -c config.yml 27 | 28 | These instances will register as `{interface_name}.worker` and thus not respond 29 | to RPC requests sent to `{interface_name}`. -------------------------------------------------------------------------------- /docs/topic_guides/testing.rst: -------------------------------------------------------------------------------- 1 | Tests 2 | ~~~~~ 3 | 4 | You can test if your installation of lymph has been successful by running the 5 | unittests. You'll also have to set ``ZOOKEEPER_PATH`` to the directory that 6 | contains your ZooKeeper binaries (e.g. ``/usr/share/java`` on Ubuntu). 7 | 8 | You can then run the tests with either `tox`_ or ``nosetests`` directly. 9 | 10 | .. FIXME add unittesting coverage as well 11 | -------------------------------------------------------------------------------- /docs/topic_guides/versioning.rst: -------------------------------------------------------------------------------- 1 | Versioning interfaces 2 | ====================== 3 | 4 | 5 | .. code:: yaml 6 | 7 | interfaces: 8 | echo@1.5.0: 9 | class: echo:Echo 10 | 11 | echo@2.0.0: 12 | class: echo:Echo2 13 | 14 | 15 | 16 | 17 | Requesting Specific Versions 18 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 19 | 20 | from the command line: 21 | 22 | .. code:: console 23 | 24 | $ lymph request echo.upper@1.2 '{"text": "foo"}' 25 | 26 | 27 | from code: 28 | 29 | .. code:: python 30 | 31 | proxy = lymph.proxy('echo', version='1.1') 32 | 33 | -------------------------------------------------------------------------------- /docs/user_guide.rst: -------------------------------------------------------------------------------- 1 | .. _getting-started: 2 | 3 | 4 | User guide 5 | ========== 6 | 7 | You can find an introduction to lymph in Max Brauer's `import lymph`_ 8 | presentation. It attempts to get you up and running and covers most features of 9 | lymph. 10 | 11 | .. FIXME port the import lymph article to rst and move it here 12 | 13 | .. _import lymph: http://import-lymph.link 14 | -------------------------------------------------------------------------------- /examples/demo.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, unicode_literals 2 | 3 | import random 4 | import gevent 5 | import lymph 6 | from lymph.core import trace 7 | 8 | 9 | class Client(lymph.Interface): 10 | delay = .1 11 | 12 | echo = lymph.proxy('echo', timeout=2) 13 | 14 | def on_start(self): 15 | super(Client, self).on_start() 16 | gevent.spawn(self.loop) 17 | 18 | @lymph.event('uppercase_transform_finished') 19 | def on_uppercase(self, event): 20 | print(self.echo.echo(text="DONE"), event.body) 21 | 22 | def apply_config(self, config): 23 | super(Client, self).apply_config(config) 24 | self.delay = config.get('delay', .1) 25 | 26 | def loop(self): 27 | i = 0 28 | while True: 29 | gevent.sleep(self.delay) 30 | trace.set_id() 31 | try: 32 | result = self.echo.upper(text='foo_%s' % i) 33 | except lymph.RpcError: 34 | continue 35 | print("result = %s" % result) 36 | i += 1 37 | -------------------------------------------------------------------------------- /examples/echo.py: -------------------------------------------------------------------------------- 1 | import lymph 2 | 3 | 4 | class EchoService(lymph.Interface): 5 | 6 | @lymph.rpc() 7 | def echo(self, text=None): 8 | return text 9 | 10 | @lymph.rpc() 11 | def upper(self, text=None): 12 | self.emit('uppercase_transform_finished', {'text': text}) 13 | return text.upper() 14 | -------------------------------------------------------------------------------- /examples/geocoder.py: -------------------------------------------------------------------------------- 1 | import lymph 2 | from geopy.geocoders import GoogleV3 3 | 4 | 5 | class Geocoder(lymph.Interface): 6 | def on_start(self): 7 | self.geolocator = GoogleV3() 8 | 9 | @lymph.rpc() 10 | def geocode(self, address): 11 | matched_address, (lat, lng) = self.geolocator.geocode(address) 12 | return { 13 | 'address': matched_address, 14 | 'latitude': lat, 15 | 'longitude': lng, 16 | } 17 | -------------------------------------------------------------------------------- /examples/static/iris.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /examples/static/jquery.json.js: -------------------------------------------------------------------------------- 1 | /*! jQuery JSON plugin 2.4.0 | code.google.com/p/jquery-json */ 2 | (function($){'use strict';var escape=/["\\\x00-\x1f\x7f-\x9f]/g,meta={'\b':'\\b','\t':'\\t','\n':'\\n','\f':'\\f','\r':'\\r','"':'\\"','\\':'\\\\'},hasOwn=Object.prototype.hasOwnProperty;$.toJSON=typeof JSON==='object'&&JSON.stringify?JSON.stringify:function(o){if(o===null){return'null';} 3 | var pairs,k,name,val,type=$.type(o);if(type==='undefined'){return undefined;} 4 | if(type==='number'||type==='boolean'){return String(o);} 5 | if(type==='string'){return $.quoteString(o);} 6 | if(typeof o.toJSON==='function'){return $.toJSON(o.toJSON());} 7 | if(type==='date'){var month=o.getUTCMonth()+1,day=o.getUTCDate(),year=o.getUTCFullYear(),hours=o.getUTCHours(),minutes=o.getUTCMinutes(),seconds=o.getUTCSeconds(),milli=o.getUTCMilliseconds();if(month<10){month='0'+month;} 8 | if(day<10){day='0'+day;} 9 | if(hours<10){hours='0'+hours;} 10 | if(minutes<10){minutes='0'+minutes;} 11 | if(seconds<10){seconds='0'+seconds;} 12 | if(milli<100){milli='0'+milli;} 13 | if(milli<10){milli='0'+milli;} 14 | return'"'+year+'-'+month+'-'+day+'T'+ 15 | hours+':'+minutes+':'+seconds+'.'+milli+'Z"';} 16 | pairs=[];if($.isArray(o)){for(k=0;k 2 | 3 | 4 | 5 |

jsonrpc

6 |
7 |
8 |
9 | 10 | 11 |
12 | 13 |

14 | 15 | 16 | 17 | 18 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /examples/static/monitor.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | lymph monitor 5 | 6 | 11 | 12 | 13 |

monitor

14 |
15 |
16 | 17 | 18 | 19 | 20 | 80 | 81 | -------------------------------------------------------------------------------- /examples/tasks.py: -------------------------------------------------------------------------------- 1 | import lymph 2 | 3 | 4 | class TaskService(lymph.Interface): 5 | @lymph.task() 6 | def sum(self, numbers=None): 7 | print "got", numbers 8 | print "sum", sum(numbers) 9 | 10 | @lymph.rpc() 11 | def sum_numbers(self, numbers): 12 | self.sum.apply(numbers=numbers) 13 | -------------------------------------------------------------------------------- /examples/web.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from lymph.web.interfaces import WebServiceInterface 4 | 5 | from werkzeug.routing import Map, Rule 6 | from werkzeug.wrappers import Response 7 | 8 | 9 | class JsonrpcGateway(WebServiceInterface): 10 | 11 | url_map = Map([ 12 | Rule('/', endpoint='index'), 13 | Rule('/static/', endpoint='static_resource'), 14 | Rule('/api/jsonrpc//', endpoint='jsonrpc'), 15 | ]) 16 | 17 | def index(self, request): 18 | return self.static_resource(request, path='jsonrpc.html', content_type='text/html') 19 | 20 | def static_resource(self, request, path=None, content_type=None): 21 | with open('examples/static/%s' % path) as f: 22 | return Response(f.read(), content_type=content_type) 23 | 24 | def jsonrpc(self, request, service_type): 25 | req = json.load(request.stream) 26 | args = req['params'][0] 27 | result = self.request(service_type, str(req['method']), args) 28 | return Response(json.dumps({'result': {'result': result.body, 'gateway': self.container.endpoint}, 'error': None, 'id': req['id']}), content_type='application/json') 29 | -------------------------------------------------------------------------------- /fabfile.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | from fabric.api import local, lcd 3 | from fabric import state 4 | 5 | state.output['status'] = False 6 | 7 | 8 | def dependency_graph(): 9 | local('sfood -i -q --ignore=`pwd`/lymph/services lymph | grep -v /tests/ | grep -v /utils | grep -v exceptions.py | sfood-graph -p | dot -Tpdf -o deps.pdf') 10 | 11 | 12 | def docs(clean=False): 13 | with lcd('docs'): 14 | if clean: 15 | local('make clean') 16 | local('make html') 17 | 18 | 19 | def coverage(): 20 | local('coverage run --timid --source=lymph -m py.test lymph') 21 | local('coverage html') 22 | #local('open .coverage-report/index.html') 23 | 24 | 25 | def flakes(): 26 | import subprocess, yaml 27 | popen = subprocess.Popen(['cloc', 'lymph', '--yaml', '--quiet'], stdout=subprocess.PIPE) 28 | lines = int(yaml.load(popen.stdout)['Python']['code']) 29 | flakes = int(subprocess.check_output('flake8 lymph | wc -l', shell=True)) 30 | print '%s flakes, %s lines, %.5f flakes per kLOC, one flake every %.1f lines' % (flakes, lines, 1000 * flakes / lines, lines / flakes) 31 | 32 | 33 | def fixme(): 34 | local(r"egrep -rn '#\s*(FIXME|TODO|XXX)\b' lymph") 35 | 36 | -------------------------------------------------------------------------------- /lymph/__init__.py: -------------------------------------------------------------------------------- 1 | __import__('pkg_resources').declare_namespace(__name__) 2 | -------------------------------------------------------------------------------- /lymph/autodoc.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | 3 | import lymph 4 | from lymph.core.decorators import RPCBase 5 | 6 | import sphinx 7 | from sphinx.ext.autodoc import MethodDocumenter, ClassDocumenter, setup as autodoc_setup 8 | 9 | 10 | class RPCMethodDocumenter(MethodDocumenter): 11 | """Documenter for RPC methods.""" 12 | 13 | # Priority must be higher than AttributeDocumenter since the data 14 | # descriptor RPC decorator should treated in documentation as a 15 | # method and not an attribute. 16 | priority = 11 17 | 18 | @classmethod 19 | def can_document_member(cls, member, membername, isattr, parent): 20 | return isinstance(member, RPCBase) 21 | 22 | def format_args(self): 23 | """Override argument extraction by getting it from RPC decorator.""" 24 | args = inspect.formatargspec(*self.object.args) 25 | return args.replace('\\', '\\\\') 26 | 27 | def generate(self, *args, **kwargs): 28 | super(RPCMethodDocumenter, self).generate(*args, **kwargs) 29 | # If RPC decorator define exception to raise (e.g. _RPCDecorator), 30 | # include this laters in the documentation of the method. 31 | raises = getattr(self.object, 'raises', ()) 32 | if not isinstance(raises, tuple): 33 | raises = (raises, ) 34 | for ex in raises: 35 | self.add_line(u':raises %s: %s' % (ex.__name__, ex.__doc__), '') 36 | 37 | 38 | class RPCInterfaceDocumenter(ClassDocumenter): 39 | """Documenter for RPC Lymph Interfaces.""" 40 | 41 | @classmethod 42 | def can_document_member(cls, member, membername, isattr, parent): 43 | document = super(RPCInterfaceDocumenter, cls).can_document_member(member, membername, isattr, parent) 44 | return document and issubclass(member, lymph.Interface) 45 | 46 | def format_args(self): 47 | """Return empty since we don't want to document Interface __init__ argument 48 | since they are irrelevant for service usage. 49 | """ 50 | return '' 51 | 52 | def filter_members(self, members, want_all): 53 | """Filter interface attribute to only document RPC methods.""" 54 | members = super(RPCInterfaceDocumenter, self).filter_members(members, want_all) 55 | ret = [] 56 | for name, obj, isattr in members: 57 | if isinstance(obj, RPCBase): 58 | ret.append((name, obj, isattr)) 59 | return ret 60 | 61 | 62 | def setup(app): 63 | autodoc_setup(app) 64 | 65 | app.add_autodocumenter(RPCMethodDocumenter) 66 | app.add_autodocumenter(RPCInterfaceDocumenter) 67 | 68 | return sphinx.__version__ 69 | -------------------------------------------------------------------------------- /lymph/autoreload.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os 3 | import gevent 4 | 5 | 6 | def watch_modules(callback): 7 | modules = {} 8 | while True: 9 | for name, module in list(sys.modules.items()): 10 | if module is None or not hasattr(module, '__file__'): 11 | continue 12 | module_source_path = os.path.abspath(module.__file__).rstrip('c') 13 | try: 14 | stat = os.stat(module_source_path) 15 | except OSError: 16 | continue 17 | mtime = stat.st_mtime 18 | if name in modules and modules[name] != mtime: 19 | callback() 20 | modules[name] = mtime 21 | gevent.sleep(1) 22 | 23 | 24 | def set_source_change_callback(callback): 25 | gevent.spawn(watch_modules, callback) 26 | -------------------------------------------------------------------------------- /lymph/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/cli/__init__.py -------------------------------------------------------------------------------- /lymph/cli/base.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import functools 3 | import logging 4 | import pkg_resources 5 | import six 6 | import textwrap 7 | import traceback 8 | import sys 9 | 10 | from lymph.exceptions import Timeout, LookupFailure 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | docstring_format_vars = {k: textwrap.dedent(v).strip() for k, v in six.iteritems({ 16 | 'COMMON_OPTIONS': """ 17 | Common Options: 18 | --config=, -c Load configuration from the given path. 19 | --help, -h Print this help message and exit. 20 | --logfile= Redirect log output to the given file. 21 | --loglevel= Set the log level to one of DEBUG, INFO, WARNING, 22 | ERROR. [default: WARNING] 23 | --version Show the lymph version and exit. 24 | --color Force colored output. 25 | --no-color Disable colored output. 26 | --vars= Load environment variables from the given path. 27 | """, 28 | 'INSTANCE_OPTIONS': """ 29 | Instance Options: 30 | --isolated, -i Don't register this service. 31 | --port=, -p Use this port for the RPC endpoint. 32 | --ip=
Use this IP for all sockets. 33 | --guess-external-ip, -g Guess the public facing IP of this machine and 34 | use it instead of the provided address. 35 | --reload Automatically stop the service when imported 36 | python files in the current working directory 37 | change. The process will be restarted by the 38 | node. Do not use this in production. 39 | """, 40 | })} 41 | 42 | 43 | def format_docstring(doc): 44 | return textwrap.dedent(doc).format(**docstring_format_vars).strip() 45 | 46 | 47 | @six.add_metaclass(abc.ABCMeta) 48 | class Command(object): 49 | needs_config = True 50 | short_description = '' 51 | 52 | def __init__(self, args, config, terminal): 53 | self.args = args 54 | self.config = config 55 | self.terminal = terminal 56 | 57 | @classmethod 58 | def get_help(cls): 59 | return format_docstring(cls.__doc__) 60 | 61 | @abc.abstractmethod 62 | def run(self): 63 | raise NotImplementedError 64 | 65 | 66 | _command_class_cache = None 67 | 68 | 69 | def get_command_classes(): 70 | global _command_class_cache 71 | if _command_class_cache is None: 72 | _command_class_cache, entry_points = {}, {} 73 | for entry_point in pkg_resources.iter_entry_points('lymph.cli'): 74 | name = entry_point.name 75 | if name in entry_points: 76 | logger.error('ignoring duplicate command definition for %s (already installed: %s)', entry_point, entry_points[name]) 77 | continue 78 | entry_points[name] = entry_point 79 | try: 80 | cls = entry_point.load() 81 | cls.name = name 82 | _command_class_cache[name] = cls 83 | except ImportError: 84 | logger.exception('Import error for command entry point %s', entry_point) 85 | return _command_class_cache 86 | 87 | 88 | def get_command_class(name): 89 | return get_command_classes()[name] 90 | 91 | 92 | def handle_request_errors(func): 93 | @functools.wraps(func) 94 | def decorated(*args, **kwargs): 95 | try: 96 | func(*args, **kwargs) 97 | except LookupFailure as e: 98 | logger.error("The specified service name could not be found: %s: %s" % (type(e).__name__, e)) 99 | return 1 100 | except Timeout: 101 | logger.error("The request timed out. Either the service is not available or busy.") 102 | return 1 103 | return decorated 104 | -------------------------------------------------------------------------------- /lymph/cli/config.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import yaml 3 | from lymph.cli.base import Command 4 | 5 | 6 | class ConfigCommand(Command): 7 | """ 8 | Usage: lymph config [options] 9 | 10 | Prints configuration for inspection 11 | 12 | {COMMON_OPTIONS} 13 | """ 14 | 15 | short_description = 'Prints configuration for inspection' 16 | 17 | def run(self): 18 | print(yaml.safe_dump(self.config.values)) 19 | -------------------------------------------------------------------------------- /lymph/cli/discover.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function 3 | import json 4 | import sys 5 | 6 | from lymph.client import Client 7 | from lymph.cli.base import Command 8 | 9 | 10 | class DiscoverCommand(Command): 11 | """ 12 | Usage: lymph discover [] [--instances] [--ip=
| --guess-external-ip | -g] [--only-running] [options] 13 | 14 | Shows available services 15 | 16 | Options: 17 | 18 | --instances Show service instances. 19 | --json Output json. 20 | --full Show all published instance meta data. 21 | --all Show services without instances. 22 | --versions Show available versions. 23 | --ip=
Use this IP for all sockets. 24 | --guess-external-ip, -g Guess the public facing IP of this machine and 25 | use it instead of the provided address. 26 | --only-running DEPRECATED 27 | 28 | {COMMON_OPTIONS} 29 | """ 30 | 31 | short_description = 'Shows available services' 32 | 33 | def run(self): 34 | if self.args.get('--only-running'): 35 | sys.stderr.write("\n--only-running is deprecated (it's now the default)\n\n") 36 | client = Client.from_config(self.config) 37 | 38 | name = self.args.get('') 39 | if name: 40 | services = {name} 41 | self.args['--instances'] = True 42 | else: 43 | services = client.container.discover() 44 | 45 | instances = {} 46 | for interface_name in services: 47 | service = client.container.lookup(interface_name) 48 | if not service and not self.args.get('--all'): 49 | continue 50 | instances[interface_name] = service 51 | 52 | if self.args.get('--json'): 53 | print(json.dumps({ 54 | name: [instance.serialize() for instance in service] for name, service in instances.items() 55 | })) 56 | else: 57 | self.print_human_readable_output(instances) 58 | 59 | def print_service_label(self, label, instances): 60 | instance_count = len({instance.identity for instance in instances}) 61 | print(u"%s [%s]" % (self.terminal.red(label), instance_count)) 62 | 63 | def print_service_instances(self, instances): 64 | if not self.args.get('--instances'): 65 | return 66 | for d in sorted(instances, key=lambda d: d.identity): 67 | print(u'[%s] %-11s %s' % (d.identity[:10], d.version if d.version else u'–', d.endpoint)) 68 | if self.args.get('--full'): 69 | for k, v in sorted(d.serialize().items()): 70 | if k == 'endpoint': 71 | continue 72 | print(u' %s: %r' % (k, v)) 73 | print() 74 | 75 | def print_human_readable_output(self, instances): 76 | if instances: 77 | for interface_name, service in sorted(instances.items()): 78 | if self.args.get('--versions'): 79 | instances_by_version = {} 80 | for instance in service: 81 | instances_by_version.setdefault(instance.version, []).append(instance) 82 | for version in sorted(instances_by_version.keys()): 83 | self.print_service_label('%s@%s' % (service.name, version), instances_by_version[version]) 84 | self.print_service_instances(instances_by_version[version]) 85 | else: 86 | self.print_service_label(service.name, service) 87 | self.print_service_instances(service) 88 | else: 89 | print(u"No registered services found") 90 | -------------------------------------------------------------------------------- /lymph/cli/emit.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from lymph.client import Client 4 | from lymph.cli.base import Command 5 | from lymph.core import trace 6 | 7 | 8 | class EmitCommand(Command): 9 | """ 10 | Usage: lymph emit [] [options] 11 | 12 | Emits an event in the event system 13 | 14 | Options: 15 | --trace-id= Use the given trace_id. 16 | 17 | {COMMON_OPTIONS} 18 | """ 19 | 20 | short_description = 'Emits an event in the event system' 21 | 22 | def run(self): 23 | event_type = self.args.get('') 24 | body = json.loads(self.args.get('')) 25 | 26 | trace.set_id(self.args.get('--trace-id')) 27 | client = Client.from_config(self.config) 28 | client.emit(event_type, body) 29 | -------------------------------------------------------------------------------- /lymph/cli/help.py: -------------------------------------------------------------------------------- 1 | from lymph.cli.base import Command, get_command_classes, get_command_class, format_docstring 2 | 3 | 4 | HEADER = 'Usage: lymph [options] [...]' 5 | 6 | HELP = HEADER + """ 7 | lymph help display help overview 8 | lymph help display command documentation 9 | """ 10 | 11 | TEMPLATE = HEADER + """ 12 | 13 | {COMMON_OPTIONS} 14 | 15 | Commands: 16 | %s 17 | """ 18 | 19 | 20 | def _format_help(name, description, indent=' ', spaces=13, min_spaces=2): 21 | r"""Format ``name`` + ``description`` in an unified format 22 | that can be used to print beautiful help messages. 23 | 24 | If the name is too long (length is greater than ``spaces - min_spaces``) 25 | than the name and description will appear in different lines. 26 | 27 | Example: 28 | 29 | >>> print(_format_help('foo', 'foobar')) 30 | foo foobar 31 | >>> print(_format_help('foo', 'foobar', spaces=4)) 32 | foo 33 | foobar 34 | >>> print('\n'.join([ 35 | ... _format_help('help', 'Print help message'), 36 | ... _format_help('shell', 'Open an interactive Python shell.'), 37 | ... _format_help('storage-migration', 'One big name for a command option'), 38 | ... ])) 39 | ... 40 | help Print help message 41 | shell Open an interactive Python shell. 42 | storage-migration 43 | One big name for a command option 44 | 45 | 46 | """ 47 | if spaces - len(name) < min_spaces: 48 | return '\n'.join([ 49 | indent + name, 50 | indent + (' ' * spaces) + description 51 | ]) 52 | else: 53 | return indent + name + (' ' * (spaces - len(name))) + description 54 | 55 | 56 | class HelpCommand(Command): 57 | """ 58 | Usage: lymph help [] 59 | 60 | Displays help information about lymph commands 61 | """ 62 | 63 | short_description = 'Displays help information about lymph' 64 | needs_config = False 65 | _description = None 66 | 67 | @property 68 | def description(self): 69 | if self._description is None: 70 | classes = get_command_classes() 71 | cmds = [] 72 | for name, cls in classes.items(): 73 | cmds.append(_format_help(name, cls.short_description)) 74 | self._description = format_docstring(TEMPLATE % '\n'.join(cmds)) 75 | self._description += "\n\nlymph help to display command specific help" 76 | return self._description 77 | 78 | def run(self): 79 | name = self.args[''] 80 | if name: 81 | print(get_command_class(name).get_help()) 82 | else: 83 | print(self.description) 84 | 85 | -------------------------------------------------------------------------------- /lymph/cli/inspect.py: -------------------------------------------------------------------------------- 1 | import textwrap 2 | 3 | from lymph.client import Client 4 | from lymph.cli.base import Command, handle_request_errors 5 | 6 | 7 | class InspectCommand(Command): 8 | """ 9 | Usage: lymph inspect [--ip=
| --guess-external-ip | -g]
[options] 10 | 11 | Describes the RPC interface of a service 12 | 13 | Options: 14 | --ip=
Use this IP for all sockets. 15 | --guess-external-ip, -g Guess the public facing IP of this machine and 16 | use it instead of the provided address. 17 | 18 | {COMMON_OPTIONS} 19 | """ 20 | 21 | short_description = 'Describes the RPC interface of a service' 22 | 23 | @handle_request_errors 24 | def run(self): 25 | address = self.args['
'] 26 | client = Client.from_config(self.config) 27 | result = client.request(address, 'lymph.inspect', {}, timeout=5).body 28 | 29 | print('RPC interface of {}\n'.format(self.terminal.bold(address))) 30 | 31 | for method in sorted(result['methods'], key=lambda m: m['name']): 32 | print( 33 | "rpc {name}({params})\n\t {help}\n".format( 34 | name=self.terminal.red(method['name']), 35 | params=self.terminal.yellow(', '.join(method['params'])), 36 | help='\n '.join(textwrap.wrap(method['help'], 70)), 37 | ) 38 | ) 39 | 40 | -------------------------------------------------------------------------------- /lymph/cli/list.py: -------------------------------------------------------------------------------- 1 | from lymph.cli.base import Command, get_command_classes 2 | 3 | 4 | class ListCommand(Command): 5 | """ 6 | Usage: lymph list [options] 7 | 8 | Lists all available commands 9 | 10 | {COMMON_OPTIONS} 11 | """ 12 | 13 | short_description = 'Lists all available commands' 14 | needs_config = False 15 | 16 | def run(self): 17 | command_names = get_command_classes().keys() 18 | max_command_name = max(command_names, key=len) 19 | description_offset = len(max_command_name) + 2 20 | for name, cls in sorted(get_command_classes().items()): 21 | print(u'{t.bold}{name:<{offset}}{t.normal}{description}'.format( 22 | t=self.terminal, 23 | name=name, 24 | description=cls.short_description, 25 | offset=description_offset 26 | )) 27 | 28 | -------------------------------------------------------------------------------- /lymph/cli/loglevel.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function 3 | 4 | from lymph.client import Client 5 | from lymph.cli.base import Command, handle_request_errors 6 | 7 | 8 | class LogLevelCommand(Command): 9 | """ 10 | Usage: lymph change-loglevel
[options] 11 | 12 | Sets the log level of a service's logger (all instances) for a given amount of time and then resets. 13 | 14 | Options: 15 | --name=, -n Logger name to change. 16 | --level=, -l Logging level to use. 17 | --period=, -p Period where change will be effected, after 18 | the logging level will be reverted [default: 60] 19 | --guess-external-ip, -g Guess the public facing IP of this machine and 20 | use it instead of the provided address. 21 | 22 | {COMMON_OPTIONS} 23 | """ 24 | 25 | short_description = 'Set logging level of a service logger' 26 | 27 | @handle_request_errors 28 | def run(self): 29 | try: 30 | period = float(self.args.get('--period')) 31 | except ValueError: 32 | print("--period requires a number (e.g. --period=0.42)") 33 | return 1 34 | 35 | return self.change_loglevel( 36 | address=self.args['
'], 37 | logger=self.args['--name'], 38 | level=self.args['--level'], 39 | period=period, 40 | ) 41 | 42 | def change_loglevel(self, address, logger, level, period): 43 | client = Client.from_config(self.config) 44 | body = { 45 | 'qualname': logger, 46 | 'loglevel': level, 47 | 'period': period, 48 | } 49 | service = client.container.lookup(address) 50 | print("Changing logger '%s' of '%s' to '%s' for a period of %s seconds" % 51 | (logger, address, level, period)) 52 | for instance in service: 53 | client.request(instance.endpoint, 'lymph.change_loglevel', body) 54 | 55 | -------------------------------------------------------------------------------- /lymph/cli/main.py: -------------------------------------------------------------------------------- 1 | 2 | def setup_config(args): 3 | import os 4 | import sys 5 | 6 | from lymph.config import Configuration 7 | from lymph.utils.sockets import guess_external_ip 8 | 9 | vars_file = args.get('--vars') or os.environ.get('LYMPH_VARS') 10 | env_vars = Configuration(raw=True) 11 | if vars_file: 12 | env_vars.load_file(vars_file) 13 | os.environ['LYMPH_VARS'] = vars_file 14 | 15 | config = Configuration({'container': {}}, env=os.environ, var=env_vars) 16 | 17 | if 'LYMPH_NODE_CONFIG' in os.environ: 18 | config.load_file(os.environ['LYMPH_NODE_CONFIG'], sections=['container', 'registry', 'event_system', 'plugins', 'dependencies']) 19 | 20 | config_file = args.get('--config') or os.environ.get('LYMPH_CONFIG', '.lymph.yml') 21 | config.load_file(config_file) 22 | config.source = config_file 23 | 24 | config.setdefault('container.ip', os.environ.get('LYMPH_NODE_IP', '127.0.0.1')) 25 | ip = args.get('--ip') 26 | if args.get('--guess-external-ip'): 27 | if ip: 28 | sys.exit('Cannot combine --ip and --guess-external-ip') 29 | ip = guess_external_ip() 30 | if ip is None: 31 | sys.exit('Cannot guess external ip, aborting ...') 32 | if ip: 33 | config.set('container.ip', ip) 34 | 35 | port = args.get('--port') 36 | if port: 37 | config.set('container.port', port) 38 | 39 | return config 40 | 41 | 42 | def setup_terminal(args, config): 43 | import blessings 44 | 45 | force_color = args.get('--color', False) 46 | if args.get('--no-color', False): 47 | if force_color: 48 | raise ValueError("cannot combine --color and --no-color") 49 | force_color = None 50 | return blessings.Terminal(force_styling=force_color) 51 | 52 | 53 | def _excepthook(type, value, tb): 54 | import logging 55 | 56 | logger = logging.getLogger('lymph') 57 | logger.log(logging.CRITICAL, 'Uncaught exception', exc_info=(type, value, tb)) 58 | 59 | 60 | def main(argv=None): 61 | import lymph.monkey 62 | lymph.monkey.patch() 63 | 64 | import docopt 65 | import sys 66 | import logging 67 | 68 | from lymph import __version__ as VERSION 69 | from lymph.cli.help import HELP 70 | from lymph.cli.base import get_command_class 71 | from lymph.utils import logging as lymph_logging 72 | 73 | bootup_handler = logging.StreamHandler() 74 | logging.getLogger().addHandler(bootup_handler) 75 | 76 | args = docopt.docopt(HELP, argv, version=VERSION, options_first=True) 77 | name = args.pop('') 78 | argv = args.pop('') 79 | try: 80 | command_cls = get_command_class(name) 81 | except KeyError: 82 | print("'%s' is not a valid lymph command. See 'lymph list' or 'lymph help'." % name) 83 | return 1 84 | command_args = docopt.docopt(command_cls.get_help(), [name] + argv) 85 | args.update(command_args) 86 | 87 | config = setup_config(args) if command_cls.needs_config else None 88 | 89 | logging.getLogger().removeHandler(bootup_handler) 90 | if config: 91 | loglevel = args.get('--loglevel', 'ERROR') 92 | logfile = args.get('--logfile') 93 | 94 | lymph_logging.setup_logging(config, loglevel, logfile) 95 | else: 96 | logging.basicConfig() 97 | 98 | sys.excepthook = _excepthook 99 | 100 | terminal = setup_terminal(args, config) 101 | command = command_cls(args, config, terminal) 102 | return command.run() 103 | 104 | 105 | if __name__ == '__main__': 106 | main() 107 | -------------------------------------------------------------------------------- /lymph/cli/request.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function 3 | import json 4 | import logging 5 | import math 6 | import pprint 7 | import sys 8 | import time 9 | 10 | from gevent.pool import Pool 11 | 12 | from lymph.client import Client 13 | from lymph.exceptions import Timeout 14 | from lymph.cli.base import Command, handle_request_errors 15 | from lymph.core import trace 16 | from lymph.core.versioning import parse_versioned_name 17 | from lymph.serializers import json_serializer 18 | 19 | 20 | logger = logging.getLogger(__name__) 21 | 22 | 23 | class RequestCommand(Command): 24 | """ 25 | Usage: lymph request [options] [-] 26 | 27 | Sends a single RPC request to a service and outputs the response 28 | 29 | Parameters have to be JSON encoded 30 | 31 | Options: 32 | --ip=
Use this IP for all sockets. 33 | --guess-external-ip, -g Guess the public facing IP of this machine and 34 | use it instead of the provided address. 35 | --timeout= RPC timeout. [default: 2.0] 36 | --address= Send the request to the given instance. 37 | --trace-id= Use the given trace_id. 38 | --json Output JSON. 39 | -N Send a total of requests [default: 1]. 40 | -C Send requests from concurrent greenlets [default: 1]. 41 | --dump-headers, -D Show response headers. 42 | 43 | {COMMON_OPTIONS} 44 | """ 45 | 46 | short_description = 'Sends a single RPC request to a service and outputs the response' 47 | 48 | def _run_one_request(self, request): 49 | reply = request() 50 | if self.args.get('--dump-headers'): 51 | for key, value in reply.headers.items(): 52 | print('%s: %r' % (key, value)) 53 | print() 54 | if self.args.get('--json'): 55 | print(json_serializer.dumps(reply.body)) 56 | else: 57 | pprint.pprint(reply.body) 58 | 59 | def _run_many_requests(self, request, n, c): 60 | # one warm up request for lookup and connection creation 61 | request() 62 | 63 | timings = [] 64 | timeouts = [] 65 | 66 | def timed_request(i): 67 | start = time.time() 68 | try: 69 | request() 70 | except Timeout: 71 | timeouts.append(i) 72 | else: 73 | timings.append(1000 * (time.time() - start)) 74 | request_count = len(timings) + len(timeouts) 75 | if request_count % (n / 80) == 0: 76 | sys.stdout.write('.') 77 | sys.stdout.flush() 78 | 79 | pool = Pool(size=c) 80 | print("sending %i requests, concurrency %i" % (n, c)) 81 | start = time.time() 82 | pool.map(timed_request, range(n)) 83 | total_time = (time.time() - start) 84 | 85 | timings.sort() 86 | n_success = len(timings) 87 | n_timeout = len(timeouts) 88 | avg = sum(timings) / n_success 89 | stddev = math.sqrt(sum((t - avg) ** 2 for t in timings)) / n_success 90 | 91 | print() 92 | print('Requests per second: %8.2f Hz (#req=%s)' % (n_success / total_time, n_success)) 93 | print('Mean time per request: %8.2f ms (stddev=%.2f)' % (avg, stddev)) 94 | print('Timeout rate: %8.2f %% (#req=%s)' % (100 * n_timeout / float(n), n_timeout)) 95 | print('Total time: %8.2f s' % total_time) 96 | print() 97 | 98 | print('Percentiles:') 99 | print(' 0.0 %% %8.2f ms (min)' % timings[0]) 100 | for p in (50, 90, 95, 97, 98, 99, 99.5, 99.9): 101 | print('%5.1f %% %8.2f ms' % (p, timings[int(math.floor(0.01 * p * n_success))])) 102 | print('100.0 %% %8.2f ms (max)' % timings[-1]) 103 | 104 | @handle_request_errors 105 | def run(self): 106 | params = self.args.get('') 107 | if params == '-': 108 | params = sys.stdin.read() 109 | body = json.loads(params) 110 | try: 111 | timeout = float(self.args.get('--timeout')) 112 | except ValueError: 113 | print("--timeout requires a number (e.g. --timeout=0.42)") 114 | return 1 115 | subject, version = parse_versioned_name(self.args['']) 116 | address = self.args.get('--address') 117 | if not address: 118 | address = subject.rsplit('.', 1)[0] 119 | 120 | client = Client.from_config(self.config) 121 | 122 | def request(): 123 | trace.set_id(self.args.get('--trace-id')) 124 | return client.request(address, subject, body, timeout=timeout, version=version) 125 | 126 | N, C = int(self.args['-N']), int(self.args['-C']) 127 | 128 | if N == 1: 129 | return self._run_one_request(request) 130 | else: 131 | return self._run_many_requests(request, N, C) 132 | -------------------------------------------------------------------------------- /lymph/cli/shell.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import sys 3 | import telnetlib 4 | import logging 5 | 6 | from lymph.client import Client 7 | from lymph.cli.base import Command 8 | 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class ShellCommand(Command): 14 | """ 15 | Usage: lymph shell [options] 16 | 17 | Starts an interactive Python shell, locally or remotely 18 | 19 | Options: 20 | --remote= Service instance name and identity. 21 | --guess-external-ip, -g Guess the public facing IP of this machine and 22 | use it instead of the provided address. 23 | 24 | {COMMON_OPTIONS} 25 | 26 | Locally: 27 | 28 | In case the shell was open locally the following objects will be 29 | available in the global namespace: 30 | 31 | ``client`` 32 | a configured :class:`lymph.client.Client` instance 33 | 34 | ``config`` 35 | a loaded :class:`lymph.config.Configuration` instance 36 | 37 | Remotely: 38 | 39 | ``lymph shell --remote=`` can open a remote shell in a running 40 | service instance, but only if this service is run in ``--debug`` mode. 41 | 42 | In this shell you have access to the current container instance and 43 | helper functions for debugging purposes: 44 | 45 | ``container`` 46 | the :class:`lymph.core.container.Container` instance 47 | 48 | ``dump_stacks()`` 49 | dumps stack of all running greenlets and os threads 50 | 51 | Example: 52 | 53 | $ lymph shell --remote=echo:38428b071a6 --guess-external-ip 54 | """ 55 | 56 | short_description = 'Starts an interactive Python shell, locally or remotely' 57 | 58 | def run(self, **kwargs): 59 | self.client = Client.from_config(self.config) 60 | 61 | service_fullname = self.args.get('--remote') 62 | if service_fullname: 63 | return self._open_remote_shell(service_fullname) 64 | else: 65 | return self._open_local_shell() 66 | 67 | def get_imported_objects(self): 68 | return {'client': self.client, 'config': self.config} 69 | 70 | def _open_local_shell(self): 71 | imported_objects = self.get_imported_objects() 72 | try: 73 | import IPython 74 | except ImportError: 75 | IPython = None 76 | 77 | if IPython: 78 | IPython.start_ipython( 79 | argv=[], 80 | user_ns=imported_objects, 81 | banner1='Welcome to the lymph shell' 82 | ) 83 | else: 84 | import code 85 | code.interact(local=imported_objects) 86 | 87 | def _open_remote_shell(self, service_fullname): 88 | backdoor_endpoint = self._get_backdoor_endpoint(service_fullname) 89 | if not backdoor_endpoint: 90 | return "No backdoor setup for %s" % service_fullname 91 | 92 | host, port = backdoor_endpoint.split(':') 93 | 94 | self._open_telnet(host, port) 95 | 96 | def _get_backdoor_endpoint(self, service_fullname): 97 | try: 98 | name, identity_prefix = service_fullname.split(':') 99 | except ValueError: 100 | sys.exit("Malformed argument it should be in the format 'name:identity'") 101 | service = self.client.container.lookup(name) 102 | instance = service.get_instance(identity_prefix) 103 | if instance is None: 104 | sys.exit('Unkown instance %s' % service_fullname) 105 | return instance.backdoor_endpoint 106 | 107 | def _open_telnet(self, host, port): 108 | telnet = telnetlib.Telnet() 109 | telnet.open(host, port) 110 | 111 | telnet.interact() 112 | -------------------------------------------------------------------------------- /lymph/cli/subscribe.py: -------------------------------------------------------------------------------- 1 | import lymph 2 | from lymph.client import Client 3 | from lymph.cli.base import Command 4 | 5 | 6 | class SubscribeCommand(Command): 7 | """ 8 | Usage: lymph subscribe ... [options] 9 | 10 | Subscribes to event types and prints occurences on stdout 11 | 12 | {COMMON_OPTIONS} 13 | """ 14 | 15 | short_description = 'Subscribes to event types and prints occurences on stdout' 16 | 17 | def run(self): 18 | event_type = self.args.get('') 19 | 20 | class Subscriber(lymph.Interface): 21 | @lymph.event(*event_type) 22 | def on_event(self, event): 23 | print('%s: %r' % (event.evt_type, event.body)) 24 | 25 | client = Client.from_config(self.config, interface_cls=Subscriber) 26 | client.container.join() 27 | 28 | -------------------------------------------------------------------------------- /lymph/cli/tail.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import logging 4 | import collections 5 | 6 | import zmq.green as zmq 7 | 8 | from lymph.cli.base import Command 9 | from lymph.client import Client 10 | from lymph.core import services 11 | from lymph.utils.logging import get_loglevel 12 | 13 | 14 | class RemoteTail(collections.Iterator): 15 | """Tail remotely a stream of services published messages. 16 | 17 | Instance of this class implement the iterator protocol, which return 18 | messages as published by the remote services that this instance is register 19 | to. 20 | 21 | """ 22 | 23 | Entry = collections.namedtuple('Entry', 'topic instance msg') 24 | 25 | def __init__(self, ctx=None): 26 | if ctx is None: 27 | ctx = zmq.Context.instance() 28 | 29 | self._sock = ctx.socket(zmq.SUB) 30 | self._sock.setsockopt_string(zmq.SUBSCRIBE, u'') 31 | self._instances = {} 32 | 33 | @property 34 | def instances(self): 35 | return self._instances 36 | 37 | def _on_status_change(self, instance, action): 38 | """Connect to a given service instance.""" 39 | if action == services.ADDED: 40 | self._connect(instance) 41 | elif action == services.REMOVED: 42 | self._disconnect(instance) 43 | 44 | def _connect(self, instance): 45 | self._sock.connect(instance.log_endpoint) 46 | self._instances[instance.log_endpoint] = instance 47 | 48 | def _disconnect(self, instance): 49 | self._sock.disconnect(instance.log_endpoint) 50 | del self._instances[instance.log_endpoint] 51 | 52 | def subscribe_service(self, service): 53 | """Subscribe to a service stream. 54 | 55 | This is done by iterating over all instances in this service and 56 | connecting to them, while keeping tabs over this service to be able 57 | to connect and disconnect as instances get added or removed. 58 | 59 | Return: True if subscription worked else false. 60 | 61 | """ 62 | service.observe([services.ADDED, services.REMOVED], self._on_status_change) 63 | 64 | connected = False 65 | for instance in service: 66 | if instance.log_endpoint: 67 | self._connect(instance) 68 | connected = True 69 | return connected 70 | 71 | def next(self): 72 | """Return an instance of :class:`RemoteTail.Entry`.""" 73 | topic, endpoint, msg = self._sock.recv_multipart() 74 | return self.Entry(topic, self._instances[endpoint], msg) 75 | 76 | __next__ = next # For python3. 77 | 78 | 79 | class TailCommand(Command): 80 | """ 81 | Usage: lymph tail [options] [--level= | -l ]
... 82 | 83 | Streams the log output of services to stderr 84 | 85 | Options: 86 | --level=, -l Log level to subscribe to [default: INFO] 87 | 88 | {COMMON_OPTIONS} 89 | """ 90 | 91 | short_description = 'Streams the log output of services to stderr' 92 | 93 | def run(self): 94 | client = Client.from_config(self.config) 95 | tail = RemoteTail() 96 | 97 | for address in self.args['
']: 98 | connected = tail.subscribe_service(client.container.lookup(address)) 99 | if not connected: 100 | print("Couldn't connect to log endpoint of '%s'" % address) 101 | 102 | if not tail.instances: 103 | return 1 104 | 105 | level = get_loglevel(self.args['--level']) 106 | logger = logging.getLogger('lymph-tail-cli') 107 | logger.setLevel(level) 108 | 109 | console = logging.StreamHandler() 110 | console.setLevel(level) 111 | console.setFormatter(logging.Formatter('[%(service_type)s][%(identity)s] [%(levelname)s] %(message)s')) 112 | 113 | logger.addHandler(console) 114 | 115 | try: 116 | for topic, instance, msg in tail: 117 | level = getattr(logging, topic) 118 | extra = { 119 | 'identity': instance.identity[:10], 120 | 'service_type': instance.endpoint, 121 | } 122 | logger.log(level, msg, extra=extra) 123 | except KeyboardInterrupt: 124 | pass 125 | -------------------------------------------------------------------------------- /lymph/cli/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/cli/tests/__init__.py -------------------------------------------------------------------------------- /lymph/cli/tests/test_testing.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from lymph.cli import base 4 | from lymph.cli.testing import CommandFactory 5 | 6 | 7 | class CommandFactoryTest(unittest.TestCase): 8 | 9 | def test_returns_a_command_instance(self): 10 | class ExampleCommand(base.Command): 11 | def run(self): 12 | pass 13 | 14 | _create_command = CommandFactory(ExampleCommand) 15 | cmd = _create_command() 16 | self.assertIsInstance(cmd, ExampleCommand) 17 | -------------------------------------------------------------------------------- /lymph/client.py: -------------------------------------------------------------------------------- 1 | from lymph.core.container import create_container 2 | from lymph.core.interfaces import Interface 3 | 4 | 5 | class ClientInterface(Interface): 6 | def should_register(self): 7 | return False 8 | 9 | 10 | class Client(object): 11 | def __init__(self, container, interface=ClientInterface): 12 | self.container = container 13 | self.interface = container.install_interface(interface, name='_client') 14 | 15 | @classmethod 16 | def from_config(cls, config, **kwargs): 17 | interface_cls = kwargs.pop('interface_cls', ClientInterface) 18 | container = create_container(config) 19 | client = cls(container, interface_cls) 20 | container.start(register=False) 21 | return client 22 | 23 | def __getattr__(self, name): 24 | # FIXME: explicit is better than implicit 25 | return getattr(self.interface, name) 26 | 27 | 28 | -------------------------------------------------------------------------------- /lymph/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/core/__init__.py -------------------------------------------------------------------------------- /lymph/core/channels.py: -------------------------------------------------------------------------------- 1 | import gevent 2 | import gevent.queue 3 | 4 | from lymph.exceptions import Timeout, Nack, RemoteError 5 | from lymph.core.messages import Message 6 | 7 | 8 | class Channel(object): 9 | def __init__(self, request, server): 10 | self.request = request 11 | self.server = server 12 | 13 | 14 | class RequestChannel(Channel): 15 | def __init__(self, request, server): 16 | super(RequestChannel, self).__init__(request, server) 17 | self.queue = gevent.queue.Queue() 18 | 19 | def recv(self, msg): 20 | self.queue.put(msg) 21 | 22 | def get(self, timeout=1): 23 | try: 24 | msg = self.queue.get(timeout=timeout) 25 | if msg.type == Message.NACK: 26 | raise Nack(self.request) 27 | elif msg.type == Message.ERROR: 28 | raise RemoteError.from_reply(self.request, msg) 29 | return msg 30 | except gevent.queue.Empty: 31 | raise Timeout(self.request) 32 | finally: 33 | self.close() 34 | 35 | def close(self): 36 | del self.server.channels[self.request.id] 37 | 38 | 39 | class ReplyChannel(Channel): 40 | def __init__(self, request, server): 41 | super(ReplyChannel, self).__init__(request, server) 42 | self._sent_reply = False 43 | self._headers = {} 44 | 45 | def add_header(self, name, value): 46 | self._headers[name] = value 47 | 48 | def reply(self, body): 49 | self.server.send_reply(self.request, body, headers=self._headers) 50 | self._sent_reply = True 51 | 52 | def ack(self, unless_reply_sent=False): 53 | if unless_reply_sent and self._sent_reply: 54 | return 55 | self.server.send_reply(self.request, None, msg_type=Message.ACK, headers=self._headers) 56 | self._sent_reply = True 57 | 58 | def nack(self, unless_reply_sent=False): 59 | if unless_reply_sent and self._sent_reply: 60 | return 61 | self.server.send_reply(self.request, None, msg_type=Message.NACK, headers=self._headers) 62 | self._sent_reply = True 63 | 64 | def error(self, **body): 65 | self.server.send_reply(self.request, body, msg_type=Message.ERROR, headers=self._headers) 66 | 67 | def close(self): 68 | pass 69 | -------------------------------------------------------------------------------- /lymph/core/components.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import gevent 3 | import six 4 | 5 | 6 | class Component(object): 7 | def __init__(self, error_hook=None, pool=None, metrics=None): 8 | self._parent_component = None 9 | self.__error_hook = error_hook 10 | self.__pool = pool 11 | self.__metrics = metrics 12 | 13 | def set_parent(self, parent): 14 | self._parent_component = parent 15 | 16 | def on_start(self): 17 | pass 18 | 19 | def on_stop(self, **kwargs): 20 | pass 21 | 22 | @property 23 | def pool(self): 24 | if self.__pool is not None: 25 | return self.__pool 26 | if not self._parent_component: 27 | raise TypeError("root component without pool") 28 | return self._parent_component.pool 29 | 30 | @property 31 | def error_hook(self): 32 | if self.__error_hook: 33 | return self.__error_hook 34 | if not self._parent_component: 35 | raise TypeError("root component without error_hook") 36 | return self._parent_component.error_hook 37 | 38 | @property 39 | def metrics(self): 40 | if self.__metrics is not None: 41 | return self.__metrics 42 | if not self._parent_component: 43 | raise TypeError("root component without metrics") 44 | return self._parent_component.metrics 45 | 46 | def spawn(self, func, *args, **kwargs): 47 | def _inner(): 48 | try: 49 | return func(*args, **kwargs) 50 | except gevent.GreenletExit: 51 | raise 52 | except: 53 | self.error_hook(sys.exc_info()) 54 | raise 55 | return self.pool.spawn(_inner) 56 | 57 | 58 | class Declaration(object): 59 | def __init__(self, factory): 60 | self.factory = factory 61 | self._decorators = [] 62 | 63 | def __call__(self, *args, **kwargs): 64 | component = self.factory(*args, **kwargs) 65 | for decorator in self._decorators: 66 | component.func = decorator(component.func) 67 | return component 68 | 69 | def decorate(self, decorator): 70 | self._decorators.append(decorator) 71 | 72 | def __get__(self, componentized, cls): 73 | if componentized is None: 74 | return self 75 | try: 76 | return componentized._declared_components[self] 77 | except KeyError: 78 | return componentized.install(self) 79 | 80 | 81 | class ComponentizedBase(type): 82 | def __new__(cls, clsname, bases, attrs): 83 | declarations = set() 84 | for base in bases: 85 | if isinstance(base, ComponentizedBase): 86 | declarations.update(base.declarations) 87 | for name, value in six.iteritems(attrs): 88 | if isinstance(value, Declaration): 89 | value.name = name 90 | declarations.add(value) 91 | new_cls = super(ComponentizedBase, cls).__new__(cls, clsname, bases, attrs) 92 | new_cls.declarations = declarations 93 | return new_cls 94 | 95 | 96 | @six.add_metaclass(ComponentizedBase) 97 | class Componentized(Component): 98 | def __init__(self, **kwargs): 99 | super(Componentized, self).__init__(**kwargs) 100 | self._declared_components = {} 101 | self.__all_components = [] 102 | self.__started = False 103 | 104 | def add_component(self, component): 105 | component.set_parent(self) 106 | self.__all_components.append(component) 107 | if self.__started: 108 | component.on_start() 109 | 110 | def install(self, factory, **kwargs): 111 | if factory in self._declared_components: 112 | raise RuntimeError("already installed: %s" % factory) 113 | component = factory(self, **kwargs) 114 | self._declared_components[factory] = component 115 | self.add_component(component) 116 | return component 117 | 118 | def on_start(self): 119 | self.__started = True 120 | for declaration in self.declarations: 121 | # FIXME: is this the right place to force declaration resolution? 122 | declaration.__get__(self, type(self)) 123 | for component in self.__all_components: 124 | component.on_start() 125 | 126 | def on_stop(self, **kwargs): 127 | for component in reversed(self.__all_components): 128 | component.on_stop(**kwargs) 129 | -------------------------------------------------------------------------------- /lymph/core/declarations.py: -------------------------------------------------------------------------------- 1 | from lymph.core.components import Declaration 2 | 3 | 4 | def proxy(*args, **kwargs): 5 | def factory(interface): 6 | from lymph.core.interfaces import Proxy 7 | return Proxy(interface.container, *args, **kwargs) 8 | return Declaration(factory) 9 | -------------------------------------------------------------------------------- /lymph/core/decorators.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import collections 3 | import functools 4 | import inspect 5 | 6 | import six 7 | 8 | from lymph.core.declarations import Declaration 9 | 10 | 11 | @six.add_metaclass(abc.ABCMeta) 12 | class RPCBase(collections.Callable): 13 | """Base interface for RPC functions. 14 | 15 | Implementation of this interface can be used as decorator for 16 | functions/methods. 17 | 18 | Example :: 19 | 20 | >>> @Decorator 21 | ... def fibo(n): 22 | ... "Fibonacci number." 23 | ... if n <= 1: return n 24 | ... return fibo(n-1) + fibo(n-2) 25 | ... 26 | >>> fibo.__doc__ 27 | 'Fibonacci number.' 28 | >>> fibo.__name__ 29 | 'fibo' 30 | >>> fibo(3) 31 | 2 32 | """ 33 | 34 | def __init__(self, func, assigned=functools.WRAPPER_ASSIGNMENTS): 35 | self.original = func 36 | self._func = func 37 | 38 | functools.update_wrapper( 39 | self, func, assigned=functools.WRAPPER_ASSIGNMENTS) 40 | 41 | @property 42 | def args(self): 43 | """Return original function argument spec skipping self. 44 | 45 | Returns: 46 | ``inspect.ArgSpec``. 47 | """ 48 | spec = inspect.getargspec(self._func) 49 | return inspect.ArgSpec(spec.args[1:], *spec[1:]) 50 | 51 | def __get__(self, obj, obj_type=None): 52 | if obj is None: 53 | return self 54 | return functools.partial(self._func, obj) 55 | 56 | def __call__(self, *args, **kwargs): 57 | return self._func(*args, **kwargs) 58 | 59 | @abc.abstractmethod 60 | def rpc_call(self, interface, channel, *args, **kwargs): 61 | pass 62 | 63 | def decorate(self, decorator): 64 | self._func = decorator(self._func) 65 | 66 | 67 | class _RawRPCDecorator(RPCBase): 68 | 69 | @property 70 | def args(self): 71 | # Skip channel in the arguments spec. 72 | spec = super(_RawRPCDecorator, self).args 73 | return inspect.ArgSpec(spec.args[1:], *spec[1:]) 74 | 75 | def rpc_call(self, interface, channel, *args, **kwargs): 76 | return self._func(interface, channel, *args, **kwargs) 77 | 78 | 79 | class _RPCDecorator(RPCBase): 80 | 81 | def __init__(self, *args, **kwargs): 82 | self._raises = kwargs.pop('raises', ()) 83 | super(_RPCDecorator, self).__init__(*args, **kwargs) 84 | 85 | @property 86 | def raises(self): 87 | return self._raises 88 | 89 | def rpc_call(self, interface, channel, *args, **kwargs): 90 | try: 91 | ret = self._func(interface, *args, **kwargs) 92 | except self._raises as ex: 93 | channel.error(type=ex.__class__.__name__, message=str(ex)) 94 | else: 95 | channel.reply(ret) 96 | 97 | 98 | def raw_rpc(): 99 | return _RawRPCDecorator 100 | 101 | 102 | def rpc(raises=()): 103 | return functools.partial(_RPCDecorator, raises=raises) 104 | 105 | 106 | def event_handler(cls, *args, **kwargs): 107 | def decorator(func): 108 | from lymph.core.events import EventHandler 109 | if isinstance(func, EventHandler): 110 | raise TypeError('lymph.event() and lymph.task() decorators cannot be stacked') 111 | 112 | def factory(interface): 113 | return cls(interface, func, *args, **kwargs) 114 | declaration = Declaration(factory) 115 | # FIXME(emulbreh): we attach the class here to make TaskHandlers 116 | # identifyable in the Interface meta class. This isn't pretty and 117 | # should be cleaned up together with the whole Declaration mess. 118 | declaration.cls = cls 119 | return declaration 120 | return decorator 121 | 122 | 123 | def event(*event_types, **kwargs): 124 | from lymph.core.events import EventHandler 125 | return event_handler(EventHandler, event_types, **kwargs) 126 | 127 | 128 | def task(sequential=False): 129 | from lymph.core.events import TaskHandler 130 | return event_handler(TaskHandler, sequential=sequential) 131 | -------------------------------------------------------------------------------- /lymph/core/events.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import re 3 | import logging 4 | from uuid import uuid4 5 | from lymph.core.components import Component 6 | from lymph.core import trace 7 | 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | class Event(object): 13 | def __init__(self, evt_type, body, source=None, headers=None, event_id=None): 14 | self.event_id = event_id 15 | self.evt_type = evt_type 16 | self.body = body 17 | self.source = source 18 | self.headers = headers or {} 19 | 20 | def __getitem__(self, key): 21 | return self.body[key] 22 | 23 | def __iter__(self): 24 | return iter(self.body) 25 | 26 | def __repr__(self): 27 | return '' % (self.evt_type, self.body) 28 | 29 | def __str__(self): 30 | return '{type=%s id=%s}' % (self.evt_type, self.event_id) 31 | 32 | @classmethod 33 | def deserialize(cls, data): 34 | return cls(data.get('type'), data.get('body', {}), source=data.get('source'), headers=data.get('headers')) 35 | 36 | def serialize(self): 37 | return { 38 | 'type': self.evt_type, 39 | 'headers': self.headers, 40 | 'body': self.body, 41 | 'source': self.source, 42 | } 43 | 44 | 45 | class EventHandler(Component): 46 | def __init__(self, interface, func, event_types, sequential=False, queue_name=None, active=True, once=False, broadcast=False, retry=0): 47 | assert not (once and broadcast), "Once and broadcast cannot be enabled at the same time" 48 | super(EventHandler, self).__init__() 49 | self.func = func 50 | self.event_types = event_types 51 | self.sequential = sequential 52 | self.active = active 53 | self.interface = interface 54 | self.once = once 55 | self.broadcast = broadcast 56 | self.unique_key = str(uuid4()) if once or broadcast else None 57 | self.retry = retry 58 | self._queue_name = queue_name or func.__name__ 59 | 60 | @property 61 | def queue_name(self): 62 | if self.unique_key: 63 | return '%s-%s-%s' % (self.interface.name, self._queue_name, self.unique_key) 64 | else: 65 | return '%s-%s' % (self.interface.name, self._queue_name) 66 | 67 | @queue_name.setter 68 | def queue_name(self, value): 69 | self._queue_name = value 70 | 71 | def should_start(self): 72 | return not self.interface.container.worker 73 | 74 | def on_start(self): 75 | if self.should_start(): 76 | self.interface.container.subscribe(self, consume=self.active) 77 | 78 | def __call__(self, event, *args, **kwargs): 79 | trace.set_id(event.headers.get('trace_id')) 80 | logger.debug('' % ( 117 | self.id, 118 | self.type, 119 | self.subject, 120 | self.body, 121 | ) 122 | -------------------------------------------------------------------------------- /lymph/core/monitoring/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/core/monitoring/__init__.py -------------------------------------------------------------------------------- /lymph/core/monitoring/aggregator.py: -------------------------------------------------------------------------------- 1 | from lymph.core.monitoring.metrics import Aggregate 2 | from lymph.core.monitoring.global_metrics import RUsageMetrics, GeventMetrics, GarbageCollectionMetrics, ProcessMetrics 3 | 4 | 5 | class Aggregator(Aggregate): 6 | @classmethod 7 | def from_config(cls, config): 8 | tags = config.get_raw('tags', {}) 9 | # FIXME: move default metrics out 10 | return cls([ 11 | RUsageMetrics(), 12 | GarbageCollectionMetrics(), 13 | GeventMetrics(), 14 | ProcessMetrics(), 15 | ], tags=tags) 16 | -------------------------------------------------------------------------------- /lymph/core/monitoring/global_metrics.py: -------------------------------------------------------------------------------- 1 | import gc 2 | import resource 3 | 4 | import gevent 5 | import psutil 6 | 7 | 8 | RUSAGE_ATTRS = ( 9 | 'utime', 'stime', 10 | 'maxrss', 'ixrss', 'idrss', 'isrss', 11 | 'minflt', 'majflt', 'nswap', 12 | 'inblock', 'oublock', 13 | 'msgsnd', 'msgrcv', 14 | 'nsignals', 'nvcsw', 'nivcsw', 15 | ) 16 | 17 | 18 | class RUsageMetrics(object): 19 | def __init__(self, name='rusage'): 20 | self.attr_map = [('ru_{}'.format(attr), '{}.{}'.format(name, attr)) for attr in RUSAGE_ATTRS] 21 | 22 | def __iter__(self): 23 | ru = resource.getrusage(resource.RUSAGE_SELF) 24 | for ru_attr, series_name in self.attr_map: 25 | yield series_name, getattr(ru, ru_attr), {} 26 | 27 | 28 | class GarbageCollectionMetrics(object): 29 | def __init__(self, name='gc'): 30 | self.name = name 31 | 32 | def __iter__(self): 33 | yield '{}.garbage'.format(self.name), len(gc.garbage), {} 34 | for i, count in enumerate(gc.get_count()): 35 | yield '{}.count{}'.format(self.name, i), count, {} 36 | 37 | 38 | class GeventMetrics(object): 39 | def __init__(self, name='gevent'): 40 | self.name = name 41 | 42 | def __iter__(self): 43 | hub = gevent.get_hub() 44 | threadpool, loop = hub.threadpool, hub.loop 45 | yield 'gevent.threadpool.size', threadpool.size, {} 46 | yield 'gevent.threadpool.maxsize', threadpool.maxsize, {} 47 | yield 'gevent.active', loop.activecnt, {} 48 | yield 'gevent.pending', loop.pendingcnt, {} 49 | yield 'gevent.depth', loop.depth, {} 50 | 51 | 52 | class ProcessMetrics(object): 53 | def __init__(self): 54 | self.proc = psutil.Process() 55 | 56 | def __iter__(self): 57 | meminfo = self.proc.memory_info()._asdict() 58 | yield 'proc.mem.rss', meminfo['rss'], {} 59 | yield 'proc.mem.vms', meminfo['vms'], {} 60 | 61 | # Not available in OSX and solaris. 62 | if hasattr(self.proc, 'io_counters'): 63 | io_counts = self.proc.io_counters() 64 | yield 'proc.io.read_count', io_counts.read_count, {} 65 | yield 'proc.io.write_count', io_counts.write_count, {} 66 | yield 'proc.io.read_bytes', io_counts.read_bytes, {} 67 | yield 'proc.io.write_bytes', io_counts.write_bytes, {} 68 | 69 | ctxt_switches = self.proc.num_ctx_switches() 70 | yield 'proc.ctxt_switches.voluntary', ctxt_switches.voluntary, {} 71 | yield 'proc.ctxt_switches.involuntary', ctxt_switches.involuntary, {} 72 | 73 | cpu_times = self.proc.cpu_times() 74 | yield 'proc.cpu.user', cpu_times.user, {} 75 | yield 'proc.cpu.system', cpu_times.system, {} 76 | 77 | soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) 78 | yield 'proc.files.count', self.proc.num_fds(), {} 79 | yield 'proc.files.soft_limit', soft_limit, {} 80 | yield 'proc.files.hard_limit', hard_limit, {} 81 | 82 | yield 'proc.threads.count', self.proc.num_threads(), {} 83 | 84 | yield 'proc.sockets.count', len(self.proc.connections()), {} 85 | -------------------------------------------------------------------------------- /lymph/core/monitoring/metrics.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import collections 3 | import six 4 | 5 | 6 | @six.add_metaclass(abc.ABCMeta) 7 | class Metric(object): 8 | def __init__(self, name, tags=None): 9 | self._name = name 10 | self._tags = tags or {} 11 | 12 | @abc.abstractmethod 13 | def __iter__(self): 14 | raise NotImplementedError() 15 | 16 | def __repr__(self): 17 | return '%s(name=%r, tags=%r)' % ( 18 | self.__class__.__name__, 19 | self._name, 20 | self._tags, 21 | ) 22 | 23 | __str__ = __repr__ 24 | 25 | 26 | class Callable(Metric): 27 | def __init__(self, name, func, tags=None): 28 | super(Callable, self).__init__(name, tags) 29 | self.func = func 30 | 31 | def __iter__(self): 32 | yield (self._name, self.func(), self._tags) 33 | 34 | 35 | class Gauge(Metric): 36 | def __init__(self, name, value=0, tags=None): 37 | super(Gauge, self).__init__(name, tags) 38 | self.value = value 39 | 40 | def set(self, value): 41 | self.value = value 42 | 43 | def __iter__(self): 44 | yield self._name, self.value, self._tags 45 | 46 | 47 | class Generator(object): 48 | def __init__(self, func): 49 | self.func = func 50 | 51 | def __iter__(self): 52 | return self.func() 53 | 54 | 55 | class Aggregate(object): 56 | def __init__(self, metrics=(), tags=None): 57 | self._metrics = list(metrics) 58 | self._tags = tags or {} 59 | 60 | def add(self, metric): 61 | self._metrics.append(metric) 62 | return metric 63 | 64 | def add_tags(self, **tags): 65 | self._tags.update(tags) 66 | 67 | def __iter__(self): 68 | for metric in self._metrics: 69 | for name, value, tags in metric: 70 | tags.update(self._tags) 71 | yield name, value, tags 72 | 73 | 74 | class Counter(Metric): 75 | def __init__(self, name, tags=None): 76 | super(Counter, self).__init__(name, tags) 77 | self._value = 0 78 | 79 | def __iadd__(self, value): 80 | self._value += value 81 | return self 82 | 83 | def __iter__(self): 84 | yield self._name, self._value, self._tags 85 | 86 | 87 | class TaggedCounter(Metric): 88 | def __init__(self, name, tags=None): 89 | super(TaggedCounter, self).__init__(name, tags) 90 | self._values = collections.Counter() 91 | 92 | def incr(self, _by=1, **tags): 93 | tags.update(self._tags) 94 | self._values[frozenset(tags.items())] += _by 95 | 96 | def __iter__(self): 97 | for tags, count in six.iteritems(self._values): 98 | yield self._name, count, dict(tags) 99 | -------------------------------------------------------------------------------- /lymph/core/monitoring/pusher.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | 4 | import gevent 5 | import msgpack 6 | import zmq.green as zmq 7 | 8 | from lymph.core.components import Component 9 | from lymph.utils.sockets import bind_zmq_socket 10 | 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | class MonitorPusher(Component): 16 | def __init__(self, container, aggregator, endpoint='127.0.0.1', interval=2): 17 | super(MonitorPusher, self).__init__() 18 | self.container = container 19 | self.interval = interval 20 | ctx = zmq.Context.instance() 21 | self.socket = ctx.socket(zmq.PUB) 22 | self.endpoint, port = bind_zmq_socket(self.socket, endpoint) 23 | logger.info('binding monitoring endpoint %s', self.endpoint) 24 | self.aggregator = aggregator 25 | 26 | def on_start(self): 27 | self.loop_greenlet = self.container.spawn(self.loop) 28 | 29 | def on_stop(self, **kwargs): 30 | self.loop_greenlet.kill() 31 | 32 | def loop(self): 33 | last_stats = time.monotonic() 34 | while True: 35 | gevent.sleep(self.interval) 36 | dt = time.monotonic() - last_stats 37 | series = list(self.aggregator) 38 | stats = { 39 | 'time': time.time(), 40 | 'series': series, 41 | } 42 | last_stats += dt 43 | self.socket.send_multipart([b'stats', msgpack.dumps(stats)]) 44 | -------------------------------------------------------------------------------- /lymph/core/plugins.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from lymph.core.components import Component 4 | 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class Plugin(Component): 10 | def on_interface_installation(self, interface): 11 | pass 12 | 13 | 14 | class Hook(object): 15 | def __init__(self, name='hook'): 16 | self.name = name 17 | self.callbacks = [] 18 | 19 | def install(self, callback): 20 | self.callbacks.append(callback) 21 | 22 | def __call__(self, *args, **kwargs): 23 | for callback in self.callbacks: 24 | try: 25 | callback(*args, **kwargs) 26 | except: 27 | logger.exception('%s failure', self.name) 28 | -------------------------------------------------------------------------------- /lymph/core/services.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | import abc 3 | import logging 4 | 5 | import six 6 | import semantic_version 7 | 8 | from lymph.utils import observables, hash_id 9 | from lymph.core.versioning import compatible, serialize_version 10 | 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | # Event types propagated by Service when instances change. 15 | ADDED = 'ADDED' 16 | REMOVED = 'REMOVED' 17 | UPDATED = 'UPDATED' 18 | 19 | 20 | class ServiceInstance(object): 21 | def __init__(self, id=None, identity=None, **info): 22 | self.id = id 23 | self.identity = identity if identity else hash_id(info.get('endpoint')) 24 | self.info = {} 25 | self.update(**info) 26 | 27 | def update(self, **info): 28 | version = info.pop('version', None) 29 | if version: 30 | version = semantic_version.Version(version) 31 | self.version = version 32 | self.info.update(info) 33 | 34 | def __getattr__(self, name): 35 | try: 36 | return self.info[name] 37 | except KeyError: 38 | raise AttributeError(name) 39 | 40 | def serialize(self): 41 | d = { 42 | 'id': self.id, 43 | 'identity': self.identity, 44 | 'version': serialize_version(self.version), 45 | } 46 | d.update(self.info) 47 | return d 48 | 49 | 50 | @six.add_metaclass(abc.ABCMeta) 51 | class InstanceSet(observables.Observable): 52 | @abc.abstractmethod 53 | def __iter__(self): 54 | raise NotImplementedError() 55 | 56 | def match_version(self, version): 57 | return VersionedServiceView(self, version) 58 | 59 | 60 | class Service(InstanceSet): 61 | def __init__(self, name=None, instances=()): 62 | super(Service, self).__init__() 63 | self.name = name 64 | self.instances = {i.id: i for i in instances} 65 | self.version = None 66 | 67 | def __str__(self): 68 | return self.name 69 | 70 | def __iter__(self): 71 | return six.itervalues(self.instances) 72 | 73 | def __len__(self): 74 | return len(self.instances) 75 | 76 | def get_instance(self, prefix): 77 | for instance in six.itervalues(self.instances): 78 | if instance.id.startswith(prefix): 79 | return instance 80 | 81 | def identities(self): 82 | return list(self.instances.keys()) 83 | 84 | def remove(self, instance_id): 85 | try: 86 | instance = self.instances.pop(instance_id) 87 | except KeyError: 88 | pass 89 | else: 90 | self.notify_observers(REMOVED, instance) 91 | 92 | def update(self, instance_id, **info): 93 | try: 94 | instance = self.instances[instance_id] 95 | except KeyError: 96 | instance = self.instances[instance_id] = ServiceInstance(**info) 97 | self.notify_observers(ADDED, instance) 98 | else: 99 | instance.update(**info) 100 | self.notify_observers(UPDATED, instance) 101 | 102 | 103 | class VersionedServiceView(InstanceSet): 104 | def __init__(self, service, version): 105 | self.service = service 106 | self.spec = compatible(version) 107 | self.version = version 108 | 109 | def __str__(self): 110 | return '%s@%s' % (self.name, self.version) 111 | 112 | @property 113 | def name(self): 114 | return self.service.name 115 | 116 | def __iter__(self): 117 | for instance in self.service: 118 | if instance.version in self.spec: 119 | yield instance 120 | 121 | def observe(self, *args, **kwargs): 122 | return self.service.observe(*args, **kwargs) 123 | -------------------------------------------------------------------------------- /lymph/core/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/core/tests/__init__.py -------------------------------------------------------------------------------- /lymph/core/tests/test_events.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from lymph.core.events import EventDispatcher 4 | 5 | 6 | class EventDispatcherTest(unittest.TestCase): 7 | def setUp(self): 8 | self.dispatcher = EventDispatcher() 9 | self.handler_log = [] 10 | self.handlers = {} 11 | 12 | def make_handler(self, name): 13 | if name in self.handlers: 14 | return self.handlers[name] 15 | 16 | def handler(*args): 17 | self.handler_log.append((name, args)) 18 | 19 | handler.__name__ = name 20 | self.handlers[name] = handler 21 | return handler 22 | 23 | def assert_dispatched_patterns_equal(self, event_type, patterns): 24 | self.assertEqual( 25 | set(pattern for pattern, handler in self.dispatcher.dispatch(event_type)), 26 | set(patterns), 27 | ) 28 | 29 | def assert_dispatched_handlers_equal(self, event_type, handlers): 30 | self.assertEqual( 31 | set(handler.__name__ for pattern, handler in self.dispatcher.dispatch(event_type)), 32 | set(handlers), 33 | ) 34 | 35 | def test_basic_dispatch(self): 36 | self.dispatcher.register('foo', self.make_handler('foo')) 37 | self.dispatcher.register('bar', self.make_handler('bar')) 38 | self.dispatcher.register('foo.bar', self.make_handler('foo2')) 39 | 40 | self.assert_dispatched_patterns_equal('foo', {'foo'}) 41 | self.assert_dispatched_patterns_equal('bar', {'bar'}) 42 | self.assert_dispatched_patterns_equal('fooo', []) 43 | self.assert_dispatched_patterns_equal('foofoo', []) 44 | self.assert_dispatched_patterns_equal('', []) 45 | 46 | def test_wildcard_dispatch(self): 47 | self.dispatcher.register('foo', self.make_handler('foo')) 48 | self.dispatcher.register('#', self.make_handler('hash')) 49 | self.dispatcher.register('*', self.make_handler('star')) 50 | self.dispatcher.register('foo.*', self.make_handler('foo_star')) 51 | self.dispatcher.register('foo.#', self.make_handler('foo_hash')) 52 | 53 | self.assert_dispatched_patterns_equal('foo', {'foo', '*', '#'}) 54 | self.assert_dispatched_patterns_equal('foo.bar', {'#', 'foo.*', 'foo.#'}) 55 | self.assert_dispatched_patterns_equal('foo.bar.baz', {'#', 'foo.#'}) 56 | self.assert_dispatched_patterns_equal('', {'#'}) 57 | 58 | def test_multi_pattern_registration(self): 59 | self.dispatcher.register('foo', self.make_handler('foo')) 60 | self.dispatcher.register('#', self.make_handler('foo')) 61 | 62 | self.assert_dispatched_patterns_equal('foo', {'foo', '#'}) 63 | self.assert_dispatched_handlers_equal('foo', {'foo'}) 64 | 65 | def test_update(self): 66 | self.dispatcher.register('foo', self.make_handler('base_foo')) 67 | self.dispatcher.register('#', self.make_handler('hash')) 68 | 69 | ed = EventDispatcher() 70 | ed.register('foo', self.make_handler('foo')) 71 | ed.register('bar', self.make_handler('bar')) 72 | self.dispatcher.update(ed) 73 | 74 | self.assert_dispatched_handlers_equal('foo', {'foo', 'base_foo', 'hash'}) 75 | self.assert_dispatched_handlers_equal('bar', {'hash', 'bar'}) 76 | -------------------------------------------------------------------------------- /lymph/core/trace.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import uuid 3 | 4 | import gevent 5 | 6 | from lymph.utils.gpool import NonBlockingPool 7 | 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | def get_trace(greenlet=None): 13 | greenlet = greenlet or gevent.getcurrent() 14 | if not hasattr(greenlet, '_lymph_trace'): 15 | greenlet._lymph_trace = {} 16 | return greenlet._lymph_trace 17 | 18 | 19 | class GreenletWithTrace(gevent.Greenlet): 20 | def __init__(self, *args, **kwargs): 21 | super(GreenletWithTrace, self).__init__(*args, **kwargs) 22 | self._lymph_trace = get_trace().copy() 23 | 24 | 25 | class Group(NonBlockingPool): 26 | greenlet_class = GreenletWithTrace 27 | 28 | 29 | def trace(**kwargs): 30 | get_trace().update(kwargs) 31 | 32 | 33 | def set_id(trace_id=None): 34 | tid = trace_id or uuid.uuid4().hex 35 | trace(lymph_trace_id=tid) 36 | if trace_id is None: 37 | logger.debug('starting trace') 38 | return tid 39 | 40 | 41 | def get_id(): 42 | return get_trace().get('lymph_trace_id') 43 | 44 | 45 | class TraceFormatter(logging.Formatter): 46 | def format(self, record): 47 | record.trace_id = get_id() 48 | return super(TraceFormatter, self).format(record) 49 | -------------------------------------------------------------------------------- /lymph/core/versioning.py: -------------------------------------------------------------------------------- 1 | from semantic_version import Version, Spec 2 | import six 3 | 4 | 5 | def parse_versioned_name(name): 6 | if '@' not in name: 7 | return name, None 8 | name, version = name.split('@', 1) 9 | return name, Version.coerce(version) 10 | 11 | 12 | def compatible(v): 13 | return Spec('>=%s,<%s' % (v, v.next_major())) 14 | 15 | 16 | def get_lymph_version(): 17 | import lymph 18 | return lymph.__version__ 19 | 20 | 21 | def serialize_version(version): 22 | if not version: 23 | return None 24 | return six.text_type(version) 25 | -------------------------------------------------------------------------------- /lymph/discovery/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/discovery/__init__.py -------------------------------------------------------------------------------- /lymph/discovery/base.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import six 3 | 4 | from lymph.core.services import Service 5 | from lymph.core.components import Component 6 | 7 | 8 | SERVICE_NAMESPACE = 'services' 9 | 10 | 11 | @six.add_metaclass(abc.ABCMeta) 12 | class BaseServiceRegistry(Component): 13 | def __init__(self, **kwargs): 14 | super(BaseServiceRegistry, self).__init__(**kwargs) 15 | self.cache = {} 16 | 17 | def get(self, service_name, **kwargs): 18 | try: 19 | service = self.cache[service_name] 20 | except KeyError: 21 | service = Service(name=service_name) 22 | self.lookup(service, **kwargs) 23 | self.cache[service_name] = service 24 | return service 25 | 26 | @abc.abstractmethod 27 | def discover(self): 28 | raise NotImplementedError 29 | 30 | @abc.abstractmethod 31 | def lookup(self, service, timeout=1): 32 | raise NotImplementedError 33 | 34 | @abc.abstractmethod 35 | def register(self, service_name, instance, namespace=SERVICE_NAMESPACE): 36 | raise NotImplementedError 37 | 38 | @abc.abstractmethod 39 | def unregister(self, service_name, instance, namespace=SERVICE_NAMESPACE): 40 | raise NotImplementedError 41 | -------------------------------------------------------------------------------- /lymph/discovery/static.py: -------------------------------------------------------------------------------- 1 | from .base import BaseServiceRegistry, SERVICE_NAMESPACE 2 | from lymph.exceptions import LookupFailure 3 | 4 | 5 | class StaticServiceRegistryHub(object): 6 | def __init__(self): 7 | self.registry = {} 8 | 9 | def create_registry(self): 10 | return StaticServiceRegistry(self) 11 | 12 | def lookup(self, service, **kwargs): 13 | service_name = service.name 14 | try: 15 | instances = self.registry[service_name] 16 | for data in instances: 17 | service.update(data.get('id'), **data) 18 | except KeyError: 19 | raise LookupFailure() 20 | return service 21 | 22 | def register(self, service_name, instance, namespace=SERVICE_NAMESPACE): 23 | if namespace != SERVICE_NAMESPACE: 24 | return 25 | self.registry.setdefault(service_name, []).append(instance.serialize()) 26 | 27 | def unregister(self, service_name, instance, namespace=SERVICE_NAMESPACE): 28 | if namespace != SERVICE_NAMESPACE: 29 | return 30 | self.registry.get(service_name, []).remove(instance.serialize()) 31 | 32 | def discover(self): 33 | return list(self.registry.keys()) 34 | 35 | 36 | class StaticServiceRegistry(BaseServiceRegistry): 37 | def __init__(self, hub=None): 38 | super(StaticServiceRegistry, self).__init__() 39 | self.hub = hub or StaticServiceRegistryHub() 40 | 41 | def discover(self): 42 | return self.hub.discover() 43 | 44 | def lookup(self, service, **kwargs): 45 | return self.hub.lookup(service, **kwargs) 46 | 47 | def register(self, service_name, instance, **kwargs): 48 | return self.hub.register(service_name, instance, **kwargs) 49 | 50 | def unregister(self, service_name, instance, **kwargs): 51 | return self.hub.unregister(service_name, instance, **kwargs) 52 | -------------------------------------------------------------------------------- /lymph/events/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/events/__init__.py -------------------------------------------------------------------------------- /lymph/events/base.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import six 3 | 4 | from lymph.core.components import Component 5 | 6 | 7 | @six.add_metaclass(abc.ABCMeta) 8 | class BaseEventSystem(Component): 9 | @classmethod 10 | def from_config(cls, config, **kwargs): 11 | return cls(**kwargs) 12 | 13 | def install(self, container): 14 | self.container = container 15 | 16 | def on_start(self): 17 | pass 18 | 19 | def on_stop(self, **kwargs): 20 | pass 21 | 22 | def subscribe(self, handler): 23 | raise NotImplementedError 24 | 25 | def unsubscribe(self, handler): 26 | raise NotImplementedError 27 | 28 | @abc.abstractmethod 29 | def emit(self, event, delay=0): 30 | raise NotImplementedError 31 | -------------------------------------------------------------------------------- /lymph/events/local.py: -------------------------------------------------------------------------------- 1 | import gevent 2 | 3 | from lymph.core.events import EventDispatcher 4 | from lymph.events.base import BaseEventSystem 5 | 6 | 7 | class LocalEventSystem(BaseEventSystem): 8 | def __init__(self, **kwargs): 9 | super(LocalEventSystem, self).__init__(**kwargs) 10 | self.dispatcher = EventDispatcher() 11 | 12 | def subscribe(self, handler, **kwargs): 13 | for event_type in handler.event_types: 14 | self.dispatcher.register(event_type, handler) 15 | 16 | def unsubscribe(self, handler): 17 | raise NotImplementedError() 18 | 19 | def emit(self, event, delay=0): 20 | if delay: 21 | gevent.spawn_later(delay, self.dispatcher, event) 22 | else: 23 | self.dispatcher(event) 24 | -------------------------------------------------------------------------------- /lymph/events/null.py: -------------------------------------------------------------------------------- 1 | from lymph.events.base import BaseEventSystem 2 | 3 | 4 | class NullEventSystem(BaseEventSystem): 5 | def emit(self, event, delay=0): 6 | pass 7 | -------------------------------------------------------------------------------- /lymph/exceptions.py: -------------------------------------------------------------------------------- 1 | import six 2 | 3 | 4 | class RpcError(Exception): 5 | def __repr__(self): 6 | return '<%s: %s>' % (self.__class__.__name__, self) 7 | 8 | 9 | class RpcRequestError(RpcError): 10 | def __init__(self, request, *args, **kwargs): 11 | self.request = request 12 | super(RpcError, self).__init__(*args, **kwargs) 13 | 14 | 15 | class Timeout(RpcRequestError): 16 | pass 17 | 18 | 19 | class Nack(RpcRequestError): 20 | pass 21 | 22 | 23 | class LookupFailure(RpcError): 24 | pass 25 | 26 | 27 | class RegistrationFailure(RpcError): 28 | pass 29 | 30 | 31 | class EventHandlerTimeout(Exception): 32 | pass 33 | 34 | 35 | class _RemoteException(type): 36 | 37 | # Hold dynamically generated exception classes. 38 | __exclasses = {} 39 | 40 | def __getattr__(cls, errtype): 41 | return cls.__exclasses.setdefault(errtype, type(errtype, (cls,), {})) 42 | 43 | 44 | @six.add_metaclass(_RemoteException) 45 | class RemoteError(RpcRequestError): 46 | @classmethod 47 | def from_reply(cls, request, reply): 48 | errtype = reply.body.get('type', cls.__name__) 49 | subcls = getattr(cls, errtype) 50 | return subcls(request, reply.body.get('message', '')) 51 | 52 | 53 | class SocketNotCreated(Exception): 54 | pass 55 | 56 | 57 | class NoSharedSockets(Exception): 58 | pass 59 | 60 | 61 | class NotConnected(RpcError): 62 | pass 63 | 64 | 65 | class ResourceExhausted(Exception): 66 | pass 67 | 68 | 69 | class ConfigurationError(Exception): 70 | pass 71 | -------------------------------------------------------------------------------- /lymph/monkey.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | def patch(): 4 | if patch._initialized: 5 | return 6 | patch._initialized = True 7 | 8 | import gevent.monkey 9 | gevent.monkey.patch_all() 10 | 11 | import sys 12 | if sys.version_info.major < 3: 13 | _py2_patches() 14 | 15 | _export() 16 | patch._initialized = False 17 | 18 | 19 | def _export(): 20 | import lymph 21 | lymph.__version__ = '0.16.0-dev' 22 | 23 | from lymph.exceptions import RpcError, LookupFailure, Timeout 24 | from lymph.core.decorators import rpc, raw_rpc, event, task 25 | from lymph.core.interfaces import Interface 26 | from lymph.core.declarations import proxy 27 | 28 | for obj in (RpcError, LookupFailure, Timeout, rpc, raw_rpc, event, Interface, proxy, task): 29 | setattr(lymph, obj.__name__, obj) 30 | 31 | 32 | def _py2_patches(): 33 | import monotime # NOQA 34 | -------------------------------------------------------------------------------- /lymph/patterns/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/patterns/__init__.py -------------------------------------------------------------------------------- /lymph/patterns/serial_events.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import logging 3 | import collections 4 | 5 | import lymph 6 | import gevent 7 | from kazoo.handlers.gevent import SequentialGeventHandler 8 | 9 | from lymph.core.declarations import Declaration 10 | from lymph.core.events import Event 11 | from lymph.core.interfaces import Component 12 | 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | def serial_event(*event_types, **kwargs): 18 | if 'key' not in kwargs: 19 | raise TypeError('key argument is required') 20 | 21 | def decorator(func): 22 | def factory(interface): 23 | zkclient = interface.config.root.get_instance( 24 | 'components.SerialEventHandler.zkclient', 25 | handler=SequentialGeventHandler()) 26 | return SerialEventHandler(zkclient, interface, func, event_types, **kwargs) 27 | return Declaration(factory) 28 | return decorator 29 | 30 | 31 | class SerialEventHandler(Component): 32 | def __init__(self, zkclient, interface, func, event_types, key, partition_count=12): 33 | super(SerialEventHandler, self).__init__() 34 | self.zk = zkclient 35 | self.interface = interface 36 | self.partition_count = partition_count 37 | self.key = key 38 | self.consumer_func = func 39 | self.consumers = collections.OrderedDict() 40 | self.name = '%s.%s' % (interface.name, func.__name__) 41 | 42 | def _consume(interface, event): 43 | self.consumer_func(self.interface, Event.deserialize(event['event'])) 44 | 45 | for i in range(partition_count): 46 | queue = self.get_queue_name(i) 47 | e = lymph.event(queue, queue_name=queue, sequential=True, active=False)(_consume) 48 | handler = interface.install(e) 49 | self.consumers[handler] = interface.container.subscribe(handler, consume=False) 50 | self.partition = set() 51 | push_queue = self.get_queue_name('push') 52 | interface.install(lymph.event(*event_types, queue_name=push_queue)(self.push)) 53 | 54 | def on_start(self): 55 | super(SerialEventHandler, self).on_start() 56 | self.start() 57 | 58 | def on_stop(self, **kwargs): 59 | super(SerialEventHandler, self).on_stop(**kwargs) 60 | self.running = False 61 | 62 | def get_queue_name(self, index): 63 | return '%s.%s' % (self.consumer_func.__name__, index) 64 | 65 | def push(self, interface, event): 66 | key = str(self.key(interface, event)).encode('utf-8') 67 | index = int(hashlib.md5(key).hexdigest(), 16) % self.partition_count 68 | logger.debug('PUBLISH %s %s', self.get_queue_name(index), event) 69 | self.interface.emit(self.get_queue_name(index), {'event': event.serialize()}) 70 | 71 | def start(self): 72 | self.running = True 73 | self.interface.container.spawn(self.loop) 74 | 75 | def loop(self): 76 | while self.running: 77 | logger.info('starting partitioner') 78 | partitioner = self.zk.SetPartitioner( 79 | path='/lymph/serial_event_partitions/%s' % self.name, 80 | set=self.consumers.keys(), 81 | time_boundary=1, 82 | ) 83 | while True: 84 | if partitioner.failed: 85 | logger.error('partitioning failed') 86 | break 87 | elif partitioner.release: 88 | self.release_partition() 89 | partitioner.release_set() 90 | elif partitioner.acquired: 91 | self.update_partition(set(partitioner)) 92 | gevent.sleep(1) 93 | elif partitioner.allocating: 94 | partitioner.wait_for_acquire() 95 | 96 | def release_partition(self): 97 | for handler in self.partition: 98 | self.stop_consuming(handler) 99 | self.partition = set() 100 | 101 | def update_partition(self, partition): 102 | for queue in self.partition - partition: 103 | self.stop_consuming(queue) 104 | for queue in partition - self.partition: 105 | self.start_consuming(queue) 106 | if partition != self.partition: 107 | logger.info('partition: %s', ', '.join(h.queue_name for h in partition)) 108 | self.partition = partition 109 | 110 | def start_consuming(self, handler): 111 | self.consumers[handler].start() 112 | 113 | def stop_consuming(self, handler): 114 | self.consumers[handler].stop() 115 | -------------------------------------------------------------------------------- /lymph/plugins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/plugins/__init__.py -------------------------------------------------------------------------------- /lymph/plugins/newrelic.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, unicode_literals 2 | 3 | import functools 4 | 5 | import newrelic.agent 6 | import newrelic.config 7 | 8 | from lymph.core import trace 9 | from lymph.core.plugins import Plugin 10 | from lymph.core.container import ServiceContainer 11 | from lymph.core.channels import RequestChannel 12 | from lymph.core.interfaces import DeferredReply 13 | from lymph.web.interfaces import WebServiceInterface 14 | 15 | 16 | def with_trace_id(func): 17 | @functools.wraps(func) 18 | def wrapped(*args, **kwargs): 19 | newrelic.agent.add_custom_parameter('trace_id', trace.get_id()) 20 | return func(*args, **kwargs) 21 | return wrapped 22 | 23 | 24 | def trace_rpc_method(method, get_subject): 25 | @functools.wraps(method) 26 | def wrapped(self, *args, **kwargs): 27 | transaction = newrelic.agent.current_transaction() 28 | with newrelic.agent.FunctionTrace(transaction, name=get_subject(self), group='Python/RPC'): 29 | return method(self, *args, **kwargs) 30 | 31 | return wrapped 32 | 33 | 34 | class NewrelicPlugin(Plugin): 35 | def __init__(self, container, config_file=None, environment=None, app_name=None, **kwargs): 36 | super(NewrelicPlugin, self).__init__() 37 | self.container = container 38 | self.container.error_hook.install(self.on_error) 39 | self.container.http_request_hook.install(self.on_http_request) 40 | newrelic.agent.initialize(config_file, environment) 41 | 42 | settings = newrelic.agent.global_settings() 43 | if app_name: 44 | settings.app_name = app_name 45 | # `app_name` requires post-processing which is only triggered by 46 | # initialize(). We manually trigger it again with undocumented api: 47 | newrelic.config._process_app_name_setting() 48 | 49 | RequestChannel.get = trace_rpc_method(RequestChannel.get, lambda channel: channel.request.subject) 50 | DeferredReply.get = trace_rpc_method(DeferredReply.get, lambda deferred: deferred.subject) 51 | 52 | def on_interface_installation(self, interface): 53 | self._wrap_methods(interface.methods) 54 | self._wrap_methods(interface.event_handlers) 55 | if isinstance(interface, WebServiceInterface): 56 | interface.application = newrelic.agent.wsgi_application()(interface.application) 57 | 58 | def _wrap_methods(self, methods): 59 | for name, method in methods.items(): 60 | method.decorate(with_trace_id) 61 | method.decorate(newrelic.agent.background_task()) 62 | 63 | def on_error(self, exc_info, **kwargs): 64 | newrelic.agent.add_custom_parameter('trace_id', trace.get_id()) 65 | newrelic.agent.record_exception(exc_info) 66 | 67 | def on_http_request(self, request, rule, kwargs): 68 | newrelic.agent.set_transaction_name("%s %s" % (request.method, rule)) 69 | newrelic.agent.add_custom_parameter('trace_id', trace.get_id()) 70 | -------------------------------------------------------------------------------- /lymph/plugins/sentry.py: -------------------------------------------------------------------------------- 1 | from raven import Client 2 | 3 | from lymph.core.plugins import Plugin 4 | 5 | 6 | class SentryPlugin(Plugin): 7 | def __init__(self, container, dsn=None, **kwargs): 8 | super(SentryPlugin, self).__init__() 9 | self.container = container 10 | self.client = Client(dsn) 11 | self.container.error_hook.install(self.on_error) 12 | 13 | def on_error(self, exc_info, **kwargs): 14 | self.client.captureException(exc_info, **kwargs) 15 | -------------------------------------------------------------------------------- /lymph/serializers/__init__.py: -------------------------------------------------------------------------------- 1 | from lymph.serializers.base import msgpack_serializer, json_serializer, raw_embed # NOQA 2 | -------------------------------------------------------------------------------- /lymph/serializers/base.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import datetime 3 | import decimal 4 | import functools 5 | import json 6 | import uuid 7 | 8 | import msgpack 9 | import six 10 | import iso8601 11 | 12 | from lymph.utils import Undefined 13 | 14 | 15 | @six.add_metaclass(abc.ABCMeta) 16 | class ExtensionTypeSerializer(object): 17 | @abc.abstractmethod 18 | def serialize(self, obj): 19 | raise NotImplementedError 20 | 21 | @abc.abstractmethod 22 | def deserialize(self, obj): 23 | raise NotImplementedError 24 | 25 | 26 | class DatetimeSerializer(ExtensionTypeSerializer): 27 | format = '%Y-%m-%dT%H:%M:%S%z' 28 | 29 | def serialize(self, obj): 30 | return obj.strftime(self.format) 31 | 32 | def deserialize(self, obj): 33 | return iso8601.parse_date(obj, default_timezone=None) 34 | 35 | 36 | class DateSerializer(ExtensionTypeSerializer): 37 | format = '%Y-%m-%d' 38 | 39 | def serialize(self, obj): 40 | return obj.strftime(self.format) 41 | 42 | def deserialize(self, obj): 43 | return datetime.datetime.strptime(obj, self.format).date() 44 | 45 | 46 | class TimeSerializer(ExtensionTypeSerializer): 47 | format = '%H:%M:%SZ' 48 | 49 | def serialize(self, obj): 50 | return obj.strftime(self.format) 51 | 52 | def deserialize(self, obj): 53 | return datetime.datetime.strptime(obj, self.format).time() 54 | 55 | 56 | class StrSerializer(ExtensionTypeSerializer): 57 | def __init__(self, factory): 58 | self.factory = factory 59 | 60 | def serialize(self, obj): 61 | return str(obj) 62 | 63 | def deserialize(self, obj): 64 | return self.factory(obj) 65 | 66 | 67 | class SetSerializer(ExtensionTypeSerializer): 68 | def serialize(self, obj): 69 | return list(obj) 70 | 71 | def deserialize(self, obj): 72 | return set(obj) 73 | 74 | 75 | class UndefinedSerializer(ExtensionTypeSerializer): 76 | def serialize(self, obj): 77 | return '' 78 | 79 | def deserialize(self, obj): 80 | return Undefined 81 | 82 | 83 | _extension_type_serializers = { 84 | 'datetime': DatetimeSerializer(), 85 | 'date': DateSerializer(), 86 | 'time': TimeSerializer(), 87 | 'Decimal': StrSerializer(decimal.Decimal), 88 | 'UUID': StrSerializer(uuid.UUID), 89 | 'set': SetSerializer(), 90 | 'UndefinedType': UndefinedSerializer(), 91 | } 92 | 93 | 94 | class BaseSerializer(object): 95 | def __init__(self, dumps=None, loads=None, load=None, dump=None): 96 | self._dumps = dumps 97 | self._loads = loads 98 | self._load = load 99 | self._dump = dump 100 | 101 | def dump_object(self, obj): 102 | obj_type = type(obj) 103 | serializer = _extension_type_serializers.get(obj_type.__name__) 104 | if serializer: 105 | obj = { 106 | '__type__': obj_type.__name__, 107 | '_': serializer.serialize(obj), 108 | } 109 | elif hasattr(obj, '_lymph_dump_'): 110 | obj = obj._lymph_dump_() 111 | return obj 112 | 113 | def load_object(self, obj): 114 | obj_type = obj.get('__type__') 115 | if obj_type: 116 | serializer = _extension_type_serializers.get(obj_type) 117 | return serializer.deserialize(obj['_']) 118 | return obj 119 | 120 | def dumps(self, obj): 121 | return self._dumps(obj, default=self.dump_object) 122 | 123 | def loads(self, s): 124 | return self._loads(s, object_hook=self.load_object) 125 | 126 | def dump(self, obj, f): 127 | return self._dump(obj, f, default=self.dump_object) 128 | 129 | def load(self, f): 130 | return self._load(f, object_hook=self.load_object) 131 | 132 | 133 | EMBEDDED_MSGPACK_TYPE = 101 134 | 135 | 136 | def raw_embed(data): 137 | return msgpack.ExtType(EMBEDDED_MSGPACK_TYPE, data) 138 | 139 | 140 | def ext_hook(code, data): 141 | if code == EMBEDDED_MSGPACK_TYPE: 142 | return msgpack_serializer.loads(data) 143 | return msgpack.ExtType(code, data) 144 | 145 | 146 | def _msgpack_load(stream, *args, **kwargs): 147 | # temporary workaround for https://github.com/msgpack/msgpack-python/pull/143 148 | return msgpack.loads(stream.read(), *args, **kwargs) 149 | 150 | 151 | msgpack_serializer = BaseSerializer( 152 | dumps=functools.partial(msgpack.dumps, use_bin_type=True), 153 | loads=functools.partial(msgpack.loads, encoding='utf-8', ext_hook=ext_hook), 154 | dump=functools.partial(msgpack.dump, use_bin_type=True), 155 | load=functools.partial(_msgpack_load, encoding='utf-8', ext_hook=ext_hook), 156 | ) 157 | 158 | json_serializer = BaseSerializer(dumps=json.dumps, loads=json.loads, dump=json.dump, load=json.load) 159 | -------------------------------------------------------------------------------- /lymph/serializers/kombu.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from kombu.serialization import BytesIO 4 | 5 | from lymph.serializers.base import msgpack_serializer, json_serializer 6 | 7 | 8 | def _load_msgpack(s): 9 | return msgpack_serializer.load(BytesIO(s)) 10 | 11 | 12 | def _load_json(s): 13 | return json_serializer.load(BytesIO(s)) 14 | 15 | 16 | json_serializer_args = (json_serializer.dumps, _load_json, 'application/lymph+json', 'utf-8') 17 | msgpack_serializer_args = (msgpack_serializer.dumps, _load_msgpack, 'application/lymph+x-msgpack', 'binary') 18 | -------------------------------------------------------------------------------- /lymph/serializers/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/serializers/tests/__init__.py -------------------------------------------------------------------------------- /lymph/services/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/services/__init__.py -------------------------------------------------------------------------------- /lymph/services/node.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import gevent 4 | import os 5 | import psutil 6 | import six 7 | from gevent import subprocess 8 | from six.moves import range 9 | 10 | from lymph.core.interfaces import Interface 11 | from lymph.core.monitoring.metrics import Generator 12 | from lymph.utils.sockets import create_socket 13 | 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | class Process(object): 19 | def __init__(self, cmd, env=None, service_type='n/a'): 20 | self.cmd = cmd 21 | self.env = env 22 | self.service_type = service_type 23 | self._process = None 24 | self._popen = None 25 | 26 | def is_running(self): 27 | return self._process and self._process.is_running() 28 | 29 | def start(self): 30 | self._popen = subprocess.Popen( 31 | self.cmd, env=self.env, close_fds=False) 32 | self._process = psutil.Process(self._popen.pid) 33 | self._process.cpu_percent() 34 | 35 | def stop(self, **kwargs): 36 | signalnum = kwargs.get('signalnum') 37 | try: 38 | if signalnum: 39 | self._process.send_signal(signalnum) 40 | else: 41 | self._process.terminate() 42 | self._process.wait() 43 | except psutil.NoSuchProcess: 44 | pass 45 | 46 | def restart(self): 47 | print("restarting %s" % self) 48 | self.stop() 49 | self.start() 50 | 51 | def get_metrics(self): 52 | if not self.is_running(): 53 | return 54 | tags = {'service_type': self.service_type} 55 | try: 56 | memory = self._process.memory_info() 57 | yield 'node.process.memory.rss', memory.rss, tags 58 | yield 'node.process.memory.vms', memory.vms, tags 59 | yield 'node.process.cpu', self._process.cpu_percent(), tags 60 | except psutil.NoSuchProcess: 61 | pass 62 | 63 | 64 | class Node(Interface): 65 | def __init__(self, *args, **kwargs): 66 | super(Node, self).__init__(*args, **kwargs) 67 | self.sockets = {} 68 | self.processes = [] 69 | self.running = False 70 | self._sockets = [] 71 | self._services = [] 72 | 73 | def apply_config(self, config): 74 | for name, c in six.iteritems(config.get('instances', {})): 75 | self._services.append((name, c.get('command'), c.get('numprocesses', 1))) 76 | 77 | socket_config = config.get_raw('sockets', ()) 78 | if isinstance(socket_config, dict): 79 | socket_config = socket_config.values() 80 | for c in socket_config: 81 | self._sockets.append((c.get('host'), c.get('port'))) 82 | 83 | def on_start(self): 84 | self.create_shared_sockets() 85 | self.running = True 86 | shared_fds = json.dumps({port: s.fileno() for port, s in six.iteritems(self.sockets)}) 87 | for service_type, cmd, num in self._services: 88 | env = os.environ.copy() 89 | env.update({ 90 | 'LYMPH_NODE': self.container.endpoint, 91 | 'LYMPH_MONITOR': self.container.monitor.endpoint, 92 | 'LYMPH_NODE_IP': self.container.server.ip, 93 | 'LYMPH_SHARED_SOCKET_FDS': shared_fds, 94 | 'LYMPH_SERVICE_NAME': service_type, 95 | }) 96 | for i in range(num): 97 | p = Process(cmd.split(' '), env=env, service_type=service_type) 98 | self.processes.append(p) 99 | self.metrics.add(Generator(p.get_metrics)) 100 | logger.info('starting %s', cmd) 101 | p.start() 102 | self.container.spawn(self.watch_processes) 103 | 104 | def on_stop(self, **kwargs): 105 | logger.info("waiting for all service processes to die ...") 106 | self.running = False 107 | for p in self.processes: 108 | p.stop(**kwargs) 109 | super(Node, self).on_stop(**kwargs) 110 | 111 | def create_shared_sockets(self): 112 | for host, port in self._sockets: 113 | sock = create_socket( 114 | '%s:%s' % (host or self.container.server.ip, port), inheritable=True) 115 | self.sockets[port] = sock 116 | 117 | def watch_processes(self): 118 | while True: 119 | for process in self.processes: 120 | try: 121 | status = process._process.status() 122 | except psutil.NoSuchProcess: 123 | if self.running: 124 | process.start() 125 | continue 126 | if status in (psutil.STATUS_ZOMBIE, psutil.STATUS_DEAD): 127 | if self.running: 128 | process.restart() 129 | gevent.sleep(1) 130 | -------------------------------------------------------------------------------- /lymph/services/scheduler.py: -------------------------------------------------------------------------------- 1 | import gevent 2 | import msgpack 3 | import redis 4 | import time 5 | 6 | from lymph.core.interfaces import Interface 7 | from lymph.core.decorators import rpc 8 | from lymph.utils import make_id 9 | 10 | 11 | class Scheduler(Interface): 12 | service_type = 'scheduler' 13 | schedule_key = 'schedule' 14 | 15 | def __init__(self, *args, **kwargs): 16 | super(Scheduler, self).__init__(*args, **kwargs) 17 | self.redis = redis.StrictRedis() 18 | 19 | def on_start(self): 20 | self.container.spawn(self.loop) 21 | 22 | @rpc() 23 | def schedule(self, eta, event_type, payload): 24 | self.redis.zadd(self.schedule_key, eta, msgpack.dumps({ 25 | 'id': make_id(), 26 | 'event_type': event_type, 27 | 'payload': payload, 28 | })) 29 | 30 | def loop(self): 31 | while True: 32 | pipe = self.redis.pipeline() 33 | now = int(time.time()) 34 | pipe.zrangebyscore(self.schedule_key, 0, now) 35 | pipe.zremrangebyscore(self.schedule_key, 0, now) 36 | events, n = pipe.execute() 37 | for event in events: 38 | event = msgpack.loads(event, encoding='utf-8') 39 | self.emit(event['event_type'], event['payload']) 40 | gevent.sleep(1) 41 | -------------------------------------------------------------------------------- /lymph/testing/nose.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from nose.plugins import Plugin 6 | 7 | log = logging.getLogger('nose.plugins.lymph') 8 | 9 | 10 | class LymphPlugin(Plugin): 11 | """ 12 | Initializes the lymph framework before tests are run 13 | """ 14 | 15 | name = 'lymph' 16 | 17 | def begin(self): 18 | log.info("Initializing lymph framework") 19 | import lymph.monkey 20 | lymph.monkey.patch() 21 | -------------------------------------------------------------------------------- /lymph/testing/nose2.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | from nose2.events import Plugin 6 | 7 | 8 | log = logging.getLogger('nose2.plugins.lymph') 9 | 10 | 11 | class LymphPlugin(Plugin): 12 | """ 13 | Initializes the lymph framework before tests are run 14 | """ 15 | configSection = 'lymph' 16 | 17 | def createTests(self, event): 18 | log.info("Initializing lymph framework") 19 | import lymph.monkey 20 | lymph.monkey.patch() 21 | -------------------------------------------------------------------------------- /lymph/testing/pytest.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | 5 | log = logging.getLogger(__name__) 6 | 7 | 8 | def pytest_configure(config): 9 | log.info("Initializing the lymph framework") 10 | import lymph.monkey 11 | lymph.monkey.patch() 12 | -------------------------------------------------------------------------------- /lymph/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/tests/__init__.py -------------------------------------------------------------------------------- /lymph/tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/tests/integration/__init__.py -------------------------------------------------------------------------------- /lymph/tests/integration/test_cli.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import blessings 4 | import gevent 5 | import six 6 | 7 | from lymph.cli.testing import CliIntegrationTestCase, CliTestMixin 8 | from lymph.core.decorators import rpc 9 | from lymph.core.interfaces import Interface 10 | 11 | 12 | class Upper(Interface): 13 | 14 | @rpc() 15 | def upper(self, text=None): 16 | return text.upper() 17 | 18 | 19 | class RequestCommandTests(CliIntegrationTestCase): 20 | def setUp(self): 21 | super(RequestCommandTests, self).setUp() 22 | self.upper_container, interface = self.create_container(Upper, 'upper') 23 | 24 | def test_request(self): 25 | result = self.cli(['request', 'upper.upper', '{"text":"foo"}']) 26 | self.assertEqual(result.returncode, 0) 27 | output = "u'FOO'" if six.PY2 else "'FOO'" 28 | self.assertEqual(result.stdout.rstrip(), output) 29 | 30 | def test_negative_request(self): 31 | result = self.cli(['request', 'no_exiting_container', '{}']) 32 | self.assertEqual(result.returncode, 1) 33 | 34 | def test_inspect(self): 35 | # Use --no-color to facilitate string comparison. 36 | result = self.cli(['inspect', '--no-color', 'upper']) 37 | self.assertEqual(result.returncode, 0) 38 | self.assertIn('upper.upper(text)', result.stdout) 39 | 40 | def test_negative_inspect(self): 41 | result = self.cli(['inspect', 'no_existing_container']) 42 | self.assertEqual(result.returncode, 1) 43 | 44 | 45 | class ListCommandTests(CliTestMixin, unittest.TestCase): 46 | def test_list(self): 47 | self.assert_lines_equal(['list'], u""" 48 | {t.bold}config {t.normal}Prints configuration for inspection 49 | {t.bold}tail {t.normal}Streams the log output of services to stderr 50 | {t.bold}emit {t.normal}Emits an event in the event system 51 | {t.bold}request {t.normal}Sends a single RPC request to a service and outputs the response 52 | {t.bold}inspect {t.normal}Describes the RPC interface of a service 53 | {t.bold}discover {t.normal}Shows available services 54 | {t.bold}help {t.normal}Displays help information about lymph 55 | {t.bold}list {t.normal}Lists all available commands 56 | {t.bold}subscribe {t.normal}Subscribes to event types and prints occurences on stdout 57 | {t.bold}instance {t.normal}Runs a single service instance 58 | {t.bold}node {t.normal}Runs a node service that manages a group of processes on the same machine 59 | {t.bold}shell {t.normal}Starts an interactive Python shell, locally or remotely 60 | {t.bold}worker {t.normal}Runs a worker instance 61 | {t.bold}change-loglevel {t.normal}Set logging level of a service logger 62 | """.format(t=blessings.Terminal()), config=False) 63 | 64 | 65 | class HelpCommandTests(CliTestMixin, unittest.TestCase): 66 | def test_help(self): 67 | self.assert_first_line_equals(['help'], 'Usage: lymph [options] [...]', config=False) 68 | 69 | def test_help_list(self): 70 | self.assert_first_line_equals(['help', 'list'], 'Usage: lymph list [options]', config=False) 71 | 72 | 73 | class ServiceCommandTests(CliIntegrationTestCase): 74 | def test_instance(self): 75 | self.cli_config['interfaces'] = { 76 | 'echo': { 77 | 'class': 'lymph.tests.integration.test_cli:Upper', 78 | } 79 | } 80 | command_greenlet = gevent.spawn(self.cli, ['instance']) 81 | client = self.create_client() 82 | gevent.sleep(1) # FIXME: how can we wait for the instance to register? 83 | response = client.request('echo', 'echo.upper', {'text': 'hi'}, timeout=1) 84 | self.assertEqual(response.body, 'HI') 85 | command_greenlet.kill() 86 | command_greenlet.join() 87 | -------------------------------------------------------------------------------- /lymph/tests/integration/test_kombu_events.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | import kombu 3 | 4 | import lymph 5 | 6 | from lymph.events.kombu import KombuEventSystem 7 | from lymph.discovery.static import StaticServiceRegistryHub 8 | from lymph.testing import LymphIntegrationTestCase, AsyncTestsMixin 9 | 10 | 11 | class TestInterface(lymph.Interface): 12 | def __init__(self, *args, **kwargs): 13 | super(TestInterface, self).__init__(*args, **kwargs) 14 | self.collected_events = [] 15 | 16 | @lymph.event('foo') 17 | def on_foo(self, event): 18 | self.collected_events.append(event) 19 | 20 | @lymph.event('retryable_foo', retry=2) 21 | def on_retryable_foo(self, event): 22 | self.collected_events.append(event) 23 | raise Exception() 24 | 25 | @lymph.event('foo_broadcast', broadcast=True) 26 | def on_foo_broadcast(self, event): 27 | self.collected_events.append(event) 28 | 29 | 30 | class TestEventBroadcastInterface(lymph.Interface): 31 | def __init__(self, *args, **kwargs): 32 | super(TestEventBroadcastInterface, self).__init__(*args, **kwargs) 33 | self.collected_events = [] 34 | 35 | @lymph.event('foo_broadcast', broadcast=True) 36 | def on_foo_broadcast(self, event): 37 | self.collected_events.append(event) 38 | 39 | 40 | class KombuIntegrationTest(LymphIntegrationTestCase, AsyncTestsMixin): 41 | use_zookeeper = False 42 | 43 | def setUp(self): 44 | super(KombuIntegrationTest, self).setUp() 45 | self.exchange_name = 'test-%s' % uuid.uuid4() 46 | self.discovery_hub = StaticServiceRegistryHub() 47 | 48 | self.the_container, self.the_interface = self.create_container(TestInterface, 'test') 49 | self.the_container_broadcast, self.the_interface_broadcast = self.create_container(TestEventBroadcastInterface, 'test') 50 | self.lymph_client = self.create_client() 51 | 52 | def tearDown(self): 53 | super(KombuIntegrationTest, self).tearDown() 54 | connection = self.get_kombu_connection() 55 | exchange = kombu.Exchange(self.exchange_name) 56 | exchange(connection).delete() 57 | 58 | waiting_exchange = kombu.Exchange(self.the_container.events.waiting_exchange.name) 59 | waiting_exchange(connection).delete() 60 | 61 | retry_exchange = kombu.Exchange(self.the_container.events.retry_exchange.name) 62 | retry_exchange(connection).delete() 63 | 64 | for q in ('test-on_foo', 'test-on_retryable_foo'): 65 | self.delete_queue(q) 66 | 67 | def delete_queue(self, name): 68 | connection = self.get_kombu_connection() 69 | queue = kombu.Queue(name) 70 | queue(connection).delete() 71 | 72 | def get_kombu_connection(self): 73 | return kombu.Connection(transport='amqp', host='127.0.0.1') 74 | 75 | def create_event_system(self, **kwargs): 76 | return KombuEventSystem(self.get_kombu_connection(), self.exchange_name) 77 | 78 | def create_registry(self, **kwargs): 79 | return self.discovery_hub.create_registry(**kwargs) 80 | 81 | def received_check(self, n): 82 | def check(): 83 | return len(self.the_interface.collected_events) == n 84 | return check 85 | 86 | def received_broadcast_check(self, n): 87 | def check(): 88 | return (len(self.the_interface.collected_events) + len(self.the_interface_broadcast.collected_events)) == n 89 | return check 90 | 91 | def test_emit(self): 92 | self.lymph_client.emit('foo', {}) 93 | self.assert_eventually_true(self.received_check(1), timeout=10) 94 | self.assertEqual(self.the_interface.collected_events[0].evt_type, 'foo') 95 | 96 | def test_delayed_emit(self): 97 | self.lymph_client.emit('foo', {}, delay=.5) 98 | self.addCleanup(self.delete_queue, 'foo-wait_500') 99 | self.assert_temporarily_true(self.received_check(0), timeout=.2) 100 | self.assert_eventually_true(self.received_check(1), timeout=10) 101 | self.assertEqual(self.the_interface.collected_events[0].evt_type, 'foo') 102 | 103 | def test_broadcast_event(self): 104 | self.lymph_client.emit('foo_broadcast', {}) 105 | self.assert_eventually_true(self.received_broadcast_check(2), timeout=10) 106 | self.assertEqual(self.the_interface.collected_events[0].evt_type, 'foo_broadcast') 107 | self.assertEqual(self.the_interface_broadcast.collected_events[0].evt_type, 'foo_broadcast') 108 | 109 | def test_retryable_event(self): 110 | self.lymph_client.emit('retryable_foo', {}) 111 | self.assert_eventually_true(self.received_check(3), timeout=10) 112 | -------------------------------------------------------------------------------- /lymph/tests/integration/test_zookeeper_discovery.py: -------------------------------------------------------------------------------- 1 | import gevent 2 | 3 | from kazoo.client import KazooClient 4 | from kazoo.handlers.gevent import SequentialGeventHandler 5 | 6 | from lymph.core.decorators import rpc 7 | from lymph.core.interfaces import Interface 8 | from lymph.discovery.zookeeper import ZookeeperServiceRegistry 9 | from lymph.events.null import NullEventSystem 10 | from lymph.testing import LymphIntegrationTestCase 11 | 12 | 13 | class Upper(Interface): 14 | service_type = 'upper' 15 | 16 | @rpc() 17 | def upper(self, text=None): 18 | return text.upper() 19 | 20 | 21 | class ZookeeperIntegrationTest(LymphIntegrationTestCase): 22 | use_zookeeper = True 23 | 24 | def setUp(self): 25 | super(ZookeeperIntegrationTest, self).setUp() 26 | self.events = NullEventSystem() 27 | 28 | self.upper_container, interface = self.create_container(Upper, 'upper') 29 | self.lymph_client = self.create_client() 30 | 31 | def create_registry(self, **kwargs): 32 | zkclient = KazooClient(self.hosts, handler=SequentialGeventHandler()) 33 | return ZookeeperServiceRegistry(zkclient) 34 | 35 | def test_lookup(self): 36 | service = self.lymph_client.container.lookup('upper') 37 | self.assertEqual(len(service), 1) 38 | self.assertEqual(list(service)[0].endpoint, self.upper_container.endpoint) 39 | 40 | def test_upper(self): 41 | reply = self.lymph_client.request('upper', 'upper.upper', {'text': 'foo'}) 42 | self.assertEqual(reply.body, 'FOO') 43 | 44 | def test_ping(self): 45 | reply = self.lymph_client.request('upper', 'lymph.ping', {'payload': 42}) 46 | self.assertEqual(reply.body, 42) 47 | 48 | def test_status(self): 49 | reply = self.lymph_client.request('upper', 'lymph.status', {}) 50 | self.assertEqual(reply.body, { 51 | 'endpoint': self.upper_container.endpoint, 52 | 'identity': self.upper_container.identity, 53 | }) 54 | 55 | def test_get_metrics(self): 56 | reply = self.lymph_client.request('upper', 'lymph.get_metrics', {}) 57 | self.assertIsInstance(reply.body, list) 58 | 59 | def test_connection_loss(self): 60 | service = self.lymph_client.container.lookup('upper') 61 | self.assertEqual( 62 | [i.identity for i in service], 63 | [self.upper_container.identity], 64 | ) 65 | self.upper_container.service_registry.client.stop() 66 | self.upper_container.service_registry.client.start() 67 | gevent.sleep(.1) # XXX: give zk a chance to reconnect 68 | self.assertEqual( 69 | [i.identity for i in service], 70 | [self.upper_container.identity], 71 | ) 72 | -------------------------------------------------------------------------------- /lymph/tests/monitoring/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/tests/monitoring/__init__.py -------------------------------------------------------------------------------- /lymph/tests/monitoring/test_aggregator.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from lymph.core.monitoring.metrics import Gauge 4 | from lymph.core.monitoring.aggregator import Aggregator 5 | 6 | 7 | class AggregatorTestCase(unittest.TestCase): 8 | 9 | def test_aggregator_one_component(self): 10 | aggr = Aggregator([Gauge('dummy', 'one')]) 11 | 12 | self.assertIn(('dummy', 'one', {}), list(aggr)) 13 | 14 | def test_aggregator_multiple_components(self): 15 | aggr = Aggregator([ 16 | Gauge('dummy', 'one'), 17 | Gauge('dummy', 'two') 18 | ], tags={'name': 'test'}) 19 | 20 | self.assertIn(('dummy', 'one', {'name': 'test'}), list(aggr)) 21 | self.assertIn(('dummy', 'two', {'name': 'test'}), list(aggr)) 22 | 23 | def test_aggregator_add_multiple_tags(self): 24 | aggr = Aggregator([ 25 | Gauge('dummy', 'one') 26 | ], tags={'name': 'test'}) 27 | 28 | aggr.add_tags(origin='localhost', time='now') 29 | 30 | self.assertIn(('dummy', 'one', {'name': 'test', 'origin': 'localhost', 'time': 'now'}), list(aggr)) 31 | -------------------------------------------------------------------------------- /lymph/tests/monitoring/test_global_metrics.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from lymph.core.monitoring.global_metrics import ProcessMetrics 4 | 5 | 6 | class GlobalMetricsTests(unittest.TestCase): 7 | def setUp(self): 8 | self.process_metrics = ProcessMetrics() 9 | 10 | def test_process_metrics(self): 11 | metric_names = [m[0] for m in self.process_metrics] 12 | 13 | self.assertIn('proc.files.count', metric_names) 14 | self.assertIn('proc.threads.count', metric_names) 15 | self.assertIn('proc.mem.rss', metric_names) 16 | self.assertIn('proc.cpu.system', metric_names) 17 | -------------------------------------------------------------------------------- /lymph/tests/monitoring/test_metrics.py: -------------------------------------------------------------------------------- 1 | import operator 2 | import unittest 3 | 4 | from lymph.core.monitoring import metrics 5 | 6 | 7 | class CounterMetricsTest(unittest.TestCase): 8 | 9 | def test_get(self): 10 | counter = metrics.Counter('requests') 11 | 12 | self.assertEqual(list(counter), [('requests', 0, {})]) 13 | 14 | def test_repr(self): 15 | counter = metrics.Counter('requests') 16 | 17 | self.assertEqual(str(counter), repr(counter)) 18 | self.assertEqual(repr(counter), "Counter(name='requests', tags={})") 19 | 20 | def test_increment_by_one(self): 21 | counter = metrics.Counter('requests') 22 | counter += 1 23 | 24 | self.assertEqual(list(counter), [('requests', 1, {})]) 25 | 26 | def test_increment_by_many(self): 27 | counter = metrics.Counter('requests') 28 | counter += 66 29 | 30 | self.assertEqual(list(counter), [('requests', 66, {})]) 31 | 32 | 33 | class TaggedCounterMetricsTest(unittest.TestCase): 34 | 35 | def test_incr_one_type(self): 36 | error_counter = metrics.TaggedCounter('exception') 37 | 38 | error_counter.incr(type='ValueError') 39 | error_counter.incr(4, type='ValueError') 40 | 41 | self.assertEqual(list(error_counter), [ 42 | ('exception', 5, {'type': 'ValueError'})]) 43 | 44 | def test_incr_different_types(self): 45 | error_counter = metrics.TaggedCounter('exception') 46 | 47 | error_counter.incr(type='ValueError') 48 | error_counter.incr(type='ValueError') 49 | 50 | error_counter.incr(type='Nack') 51 | 52 | self.assertEqual( 53 | sorted(error_counter, key=operator.itemgetter(1)), [ 54 | ('exception', 1, {'type': 'Nack'}), 55 | ('exception', 2, {'type': 'ValueError'}), 56 | ]) 57 | 58 | 59 | class AggregateMetricsTest(unittest.TestCase): 60 | def test_aggregate(self): 61 | agg = metrics.Aggregate([ 62 | metrics.Gauge('a', 1, tags={'x': '1'}), 63 | metrics.Gauge('b', 2) 64 | ], tags={'y': '2'}) 65 | 66 | self.assertEqual(sorted(list(agg)), [ 67 | ('a', 1, {'x': '1', 'y': '2'}), 68 | ('b', 2, {'y': '2'}), 69 | ]) 70 | 71 | 72 | class GeneratorMetricsTest(unittest.TestCase): 73 | def test_generator(self): 74 | def get_metrics(): 75 | yield 'name', 42, {} 76 | yield 'name', 41, {'x': '1'} 77 | 78 | agg = metrics.Generator(get_metrics) 79 | expected = [ 80 | ('name', 42, {}), 81 | ('name', 41, {'x': '1'}), 82 | ] 83 | self.assertEqual(list(agg), expected) 84 | self.assertEqual(list(agg), expected) 85 | -------------------------------------------------------------------------------- /lymph/tests/test_mockcontainer.py: -------------------------------------------------------------------------------- 1 | import lymph 2 | from lymph.core import trace 3 | from lymph.core.interfaces import Interface 4 | from lymph.core.messages import Message 5 | from lymph.testing import RPCServiceTestCase 6 | from lymph.exceptions import RemoteError, Nack 7 | 8 | 9 | class Upper(Interface): 10 | service_type = 'upper' 11 | 12 | def __init__(self, *args, **kwargs): 13 | super(Upper, self).__init__(*args, **kwargs) 14 | self.eventlog = [] 15 | 16 | @lymph.rpc() 17 | def upper(self, text=None): 18 | return text.upper() 19 | 20 | @lymph.rpc() 21 | def indirect_upper(self, text=None): 22 | # Method to test that it's possible to call upper method as 23 | # you do normally with any method. 24 | return self.upper(text) 25 | 26 | @lymph.rpc(raises=(ValueError,)) 27 | def fail(self): 28 | raise ValueError('foobar') 29 | 30 | @lymph.raw_rpc() 31 | def just_ack(self, channel): 32 | channel.ack() 33 | 34 | @lymph.rpc() 35 | def auto_nack(self): 36 | raise ValueError('auto nack requested') 37 | 38 | @lymph.event('foo') 39 | def on_foo_event(self, event): 40 | self.eventlog.append((event.evt_type, event.body)) 41 | 42 | @lymph.rpc() 43 | def get_trace_id(self): 44 | return trace.get_id() 45 | 46 | 47 | class BasicMockTest(RPCServiceTestCase): 48 | 49 | service_class = Upper 50 | service_name = 'upper' 51 | 52 | def test_upper(self): 53 | reply = self.client.upper(text='foo') 54 | self.assertEqual(reply, 'FOO') 55 | 56 | def test_indirect_upper(self): 57 | reply = self.client.indirect_upper(text='foo') 58 | self.assertEqual(reply, 'FOO') 59 | 60 | def test_ping(self): 61 | reply = self.request('lymph.ping', {'payload': 42}) 62 | self.assertEqual(reply.body, 42) 63 | 64 | def test_status(self): 65 | reply = self.request('lymph.status', {}) 66 | self.assertEqual(reply.body, { 67 | 'endpoint': 'mock://127.0.0.1:1', 68 | 'identity': '52d967815c003dd3cd5c492971508657', 69 | }) 70 | 71 | def test_error(self): 72 | with self.assertRaisesRegexp(RemoteError, 'foobar'): 73 | self.client.fail() 74 | 75 | def test_exception_handling(self): 76 | with self.assertRaises(RemoteError.ValueError): 77 | self.client.fail() 78 | 79 | def test_ack(self): 80 | reply = self.request('upper.just_ack', {}) 81 | self.assertIsNone(reply.body) 82 | self.assertEqual(reply.type, Message.ACK) 83 | 84 | def test_auto_nack(self): 85 | with self.assertRaises(Nack): 86 | self.client.auto_nack() 87 | 88 | def test_events(self): 89 | log = self.service.eventlog 90 | self.assertEqual(log, []) 91 | self.emit('foo', {'arg': 42}) 92 | self.assertEqual(log, [('foo', {'arg': 42})]) 93 | self.emit('foo', {'arg': 43}) 94 | self.assertEqual(log, [('foo', {'arg': 42}), ('foo', {'arg': 43})]) 95 | 96 | def test_inspect(self): 97 | proxy = self.get_proxy(namespace='lymph') 98 | methods = proxy.inspect()['methods'] 99 | 100 | self.assertEqual(set(m['name'] for m in methods), set([ 101 | 'upper.fail', 'upper.upper', 'upper.auto_nack', 'upper.just_ack', 102 | 'lymph.status', 'lymph.inspect', 'lymph.ping', 'upper.indirect_upper', 103 | 'lymph.get_metrics', 'upper.get_trace_id', 'lymph.change_loglevel', 104 | ])) 105 | 106 | def test_trace_id_propagates_via_rpc(self): 107 | trace_id = trace.get_id() 108 | self.assertEqual(trace_id, self.client.get_trace_id()) 109 | 110 | def test_trace_id_propagates_via_deferred_rpc(self): 111 | trace_id = trace.get_id() 112 | self.assertEqual(trace_id, self.client.get_trace_id.defer().get()) 113 | -------------------------------------------------------------------------------- /lymph/tests/test_monkey.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class InitializeTest(unittest.TestCase): 5 | 6 | def test_assure_that_lymph_is_initialized_by_testrunner(self): 7 | import lymph.monkey 8 | self.assertTrue(lymph.monkey.patch._initialized) 9 | -------------------------------------------------------------------------------- /lymph/tests/test_rpc_versioning.py: -------------------------------------------------------------------------------- 1 | import lymph 2 | from lymph.core.interfaces import Interface 3 | from lymph.testing import MultiServiceRPCTestCase 4 | from lymph.exceptions import NotConnected 5 | 6 | 7 | class Foo(Interface): 8 | @lymph.rpc() 9 | def get_class_name(self): 10 | return self.__class__.__name__ 11 | 12 | 13 | class Foo11(Foo): 14 | pass 15 | 16 | 17 | class Foo12(Foo): 18 | pass 19 | 20 | 21 | class Foo15(Foo): 22 | pass 23 | 24 | 25 | class Foo21(Foo): 26 | pass 27 | 28 | 29 | class Foo33(Foo): 30 | pass 31 | 32 | 33 | class VersionedRpcTests(MultiServiceRPCTestCase): 34 | containers = [ 35 | { 36 | 'foo@1.1': {'class': Foo11}, 37 | 'foo@1.5': {'class': Foo15}, 38 | 'foo@2.1': {'class': Foo21}, 39 | }, 40 | { 41 | 'foo@1.2': {'class': Foo12}, 42 | 'foo@3.3': {'class': Foo33}, 43 | 'foo@4.0': {'class': Foo33}, # interface classes can be reused 44 | } 45 | ] 46 | 47 | def test_version_matching(self): 48 | proxy = self.client.proxy('foo', version='1.1') 49 | self.assertIn(proxy.get_class_name(), {'Foo11', 'Foo12', 'Foo15'}) 50 | 51 | proxy = self.client.proxy('foo', version='1.2') 52 | self.assertIn(proxy.get_class_name(), {'Foo12', 'Foo15'}) 53 | 54 | proxy = self.client.proxy('foo', version='1.7') 55 | self.assertRaises(NotConnected, proxy.get_class_name) 56 | 57 | proxy = self.client.proxy('foo', version='2.0') 58 | self.assertIn(proxy.get_class_name(), {'Foo21'}) 59 | 60 | proxy = self.client.proxy('foo', version='2.5') 61 | self.assertRaises(NotConnected, proxy.get_class_name) 62 | 63 | proxy = self.client.proxy('foo', version='4.0') 64 | self.assertIn(proxy.get_class_name(), {'Foo33'}) 65 | 66 | 67 | class UnversionedRpcTests(MultiServiceRPCTestCase): 68 | containers = [ 69 | { 70 | 'foo@1.5': {'class': Foo15}, 71 | 'foo': {'class': Foo11}, 72 | }, 73 | { 74 | 'foo@1.2': {'class': Foo12}, 75 | } 76 | ] 77 | 78 | def test_unversioned(self): 79 | proxy = self.client.proxy('foo') 80 | self.assertIn(proxy.get_class_name(), {'Foo11', 'Foo15', 'Foo12'}) 81 | 82 | -------------------------------------------------------------------------------- /lymph/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import gevent 4 | 5 | from lymph.utils import gpool 6 | 7 | 8 | def some_work(): 9 | pass 10 | 11 | 12 | class NonBlockingPoolTestCase(unittest.TestCase): 13 | 14 | def setUp(self): 15 | self.pool = gpool.NonBlockingPool(size=2) 16 | 17 | def test_full_pool_status(self): 18 | self.pool.spawn(some_work) 19 | self.pool.spawn(some_work) 20 | 21 | self.assertTrue(self.pool.full()) 22 | self.assertEqual(self.pool.free_count(), 0) 23 | 24 | def test_spawn_on_full_pool_should_fail(self): 25 | self.pool.spawn(some_work) 26 | self.pool.spawn(some_work) 27 | 28 | with self.assertRaises(gpool.RejectExcecutionError): 29 | self.pool.spawn(some_work) 30 | 31 | def test_finished_work_should_freeup_resources_from_pool(self): 32 | self.pool.spawn(some_work) 33 | 34 | gevent.wait(self.pool) 35 | 36 | self.assertEqual(self.pool.free_count(), 2) 37 | -------------------------------------------------------------------------------- /lymph/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division, print_function 2 | 3 | import collections 4 | import importlib 5 | import gc 6 | import gevent 7 | import hashlib 8 | import math 9 | import os 10 | import sys 11 | import threading 12 | import traceback 13 | import uuid 14 | 15 | import six 16 | 17 | 18 | class UndefinedType(object): 19 | def __repr__(self): 20 | return "Undefined" 21 | 22 | def __bool__(self): 23 | return False 24 | 25 | def __nonzero__(self): 26 | return False 27 | 28 | 29 | Undefined = UndefinedType() 30 | 31 | 32 | def import_object(module_name, object_path=None): 33 | if not object_path: 34 | if ':' not in module_name: 35 | raise ValueError("cannot import object %r" % module_name) 36 | module_name, object_path = module_name.split(':') 37 | mod = importlib.import_module(module_name) 38 | obj = mod 39 | for objname in object_path.split('.'): 40 | obj = getattr(obj, objname) 41 | return obj 42 | 43 | 44 | def make_id(): 45 | return uuid.uuid4().hex 46 | 47 | 48 | def hash_id(*bits): 49 | return hashlib.md5(six.text_type(bits).encode('utf-8')).hexdigest() 50 | 51 | 52 | _sqrt2 = math.sqrt(2) 53 | 54 | 55 | class Accumulator(object): 56 | def __init__(self): 57 | self.n = 0 58 | self.sum = 0 59 | self.square_sum = 0 60 | self._mean = None 61 | self._stddev = None 62 | 63 | def add(self, value): 64 | self.n += 1 65 | self.sum += value 66 | self.square_sum += value * value 67 | self._mean = None 68 | self._stddev = None 69 | 70 | def remove(self, value): 71 | self.n -= 1 72 | self.sum -= value 73 | self.square_sum -= value * value 74 | self._mean = None 75 | self._stddev = None 76 | 77 | @property 78 | def mean(self): 79 | if not self.n: 80 | return 0. 81 | if self._mean is None: 82 | self._mean = self.sum / self.n 83 | return self._mean 84 | 85 | @property 86 | def stddev(self): 87 | if not self.n: 88 | return 0. 89 | if self._stddev is None: 90 | mean = self.mean 91 | self._stddev = math.sqrt(self.square_sum / self.n - mean * mean) 92 | return self._stddev 93 | 94 | @property 95 | def stats(self): 96 | return {'mean': self.mean, 'stddev': self.stddev, 'n': self.n} 97 | 98 | 99 | class SampleWindow(Accumulator): 100 | def __init__(self, n=100, factor=1): 101 | super(SampleWindow, self).__init__() 102 | self.size = n 103 | self.factor = factor 104 | self.values = collections.deque([]) 105 | self.total = Accumulator() 106 | 107 | def __len__(self): 108 | return len(self.values) 109 | 110 | def is_full(self): 111 | return len(self.values) == self.size 112 | 113 | def add(self, value): 114 | value = value * self.factor 115 | super(SampleWindow, self).add(value) 116 | self.total.add(value) 117 | if self.is_full(): 118 | self.remove(self.values.popleft()) 119 | self.values.append(value) 120 | 121 | def p(self, value): 122 | """ 123 | returns the probability for samples greater than `value` given a normal 124 | distribution with mean and standard deviation derived from this window. 125 | """ 126 | if self.stddev == 0: 127 | return 1. if value == self.mean else 0. 128 | return 1 - math.erf(abs(value * self.factor - self.mean) / (self.stddev * _sqrt2)) 129 | 130 | 131 | def get_greenlets(): 132 | for object in gc.get_objects(): 133 | if isinstance(object, gevent.Greenlet): 134 | yield object 135 | 136 | 137 | def get_greenlets_frames(): 138 | for greenlet in get_greenlets(): 139 | yield str(greenlet), greenlet.gr_frame 140 | 141 | 142 | def get_threads_frames(): 143 | threads = {thread.ident: thread.name for thread in threading.enumerate()} 144 | for ident, frame in sys._current_frames().items(): 145 | name = threads.get(ident) 146 | if name: 147 | yield '%s:%s' % (ident, name), frame 148 | 149 | 150 | def format_stack(frame): 151 | tb = traceback.format_stack(frame) 152 | return ''.join(tb) 153 | 154 | 155 | def dump_stacks(output=print): 156 | output('PID: %s' % os.getpid()) 157 | output('Threads') 158 | for i, (name, frame) in enumerate(get_threads_frames()): 159 | output('Thread #%d: %s' % (i, name)) 160 | output(format_stack(frame)) 161 | output('Greenlets') 162 | for i, (name, frame) in enumerate(get_greenlets_frames()): 163 | output('Greenlet #%d: %s' % (i, name)) 164 | output(format_stack(frame)) 165 | -------------------------------------------------------------------------------- /lymph/utils/event_indexing.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from datetime import (date, datetime) 3 | import logging 4 | import six 5 | import uuid 6 | 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | class EventIndex(object): 12 | def __init__(self, es, index_name='events'): 13 | self.es = es 14 | self.index_name = index_name 15 | 16 | def prepare_object(self, data): 17 | return dict(self.prepare_value(key, value) 18 | for key, value in six.iteritems(data)) 19 | 20 | def prepare_value(self, key, value): 21 | if isinstance(value, bool): 22 | type_prefix = 'b' 23 | elif isinstance(value, six.integer_types): 24 | type_prefix = 'i' 25 | elif isinstance(value, six.string_types): 26 | type_prefix = 's' 27 | elif isinstance(value, float): 28 | type_prefix = 'f' 29 | elif isinstance(value, dict): 30 | type_prefix = 'o' 31 | value = self.prepare_object(value) 32 | elif isinstance(value, list): 33 | type_prefix = 'l' 34 | elif isinstance(value, (datetime, date)): 35 | type_prefix = 'd' 36 | elif isinstance(value, uuid.UUID): 37 | type_prefix = 'u' 38 | value = value.hex 39 | else: 40 | raise TypeError('cannot index values of type %s' % type(value)) 41 | return ('%s_%s' % (type_prefix, key)), value 42 | 43 | def index(self, event, index_name=None): 44 | event_id = uuid.uuid4().hex 45 | body = self.prepare_object(event.body) 46 | body.update({ 47 | 'type': event.evt_type, 48 | 'source': event.source, 49 | 'logged_at': datetime.utcnow(), 50 | }) 51 | self.es.index( 52 | index=index_name or self.index_name, 53 | doc_type='event', 54 | id=event_id, 55 | body=body, 56 | ) 57 | 58 | 59 | class DatedEventIndex(EventIndex): 60 | def create_index_alias(self): 61 | if self.es.indices.exists_alias(self.index_name): 62 | logger.info('index alias already exists') 63 | self.es.indices.put_alias( 64 | index='%s-*' % self.index_name, 65 | name=self.index_name, 66 | ) 67 | 68 | def get_index_name(self, dt): 69 | return '%s-%s' % (self.index_name, dt.strftime('%Y.%m.%d')) 70 | 71 | def index(self, event, index_name=None): 72 | index_name = self.get_index_name(datetime.now()) 73 | super(DatedEventIndex, self).index(event, index_name) 74 | 75 | -------------------------------------------------------------------------------- /lymph/utils/gpool.py: -------------------------------------------------------------------------------- 1 | from gevent.pool import Pool, Group 2 | 3 | from lymph.exceptions import ResourceExhausted 4 | 5 | 6 | class RejectExcecutionError(ResourceExhausted): 7 | pass 8 | 9 | 10 | class NonBlockingPool(Pool): 11 | """A gevent pool that when exhausted will wait for a given timeout 12 | for resources to be freed before rejecting the job by raising 13 | exc:``RejectedExcecutionError``. 14 | 15 | When the ``timeout`` is not given or set to None the pool will reject 16 | immediately without waiting when pool size reach max size. 17 | 18 | In case ``size`` is None this will create an unbound pool. 19 | 20 | """ 21 | 22 | def __init__(self, timeout=None, **kwargs): 23 | super(NonBlockingPool, self).__init__(**kwargs) 24 | self._timeout = timeout 25 | 26 | def add(self, greenlet): 27 | acquired = self._semaphore.acquire(blocking=False, timeout=self._timeout) 28 | # XXX(Mouad): Checking directly for False because DummySemaphore always 29 | # return None https://github.com/gevent/gevent/pull/544. 30 | if acquired is False: 31 | raise RejectExcecutionError('No more resource available to run %r' % greenlet) 32 | try: 33 | Group.add(self, greenlet) 34 | except: 35 | self._semaphore.release() 36 | raise 37 | -------------------------------------------------------------------------------- /lymph/utils/logging.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | from logging.config import dictConfig 5 | 6 | import six 7 | import copy 8 | import zmq.green as zmq 9 | 10 | from lymph.utils.sockets import bind_zmq_socket 11 | 12 | 13 | def get_loglevel(level_name): 14 | level = level_name.upper() 15 | if level not in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'): 16 | raise ValueError("unknown loglevel: %s" % level) 17 | return getattr(logging, level) 18 | 19 | 20 | def setup_logger(name): 21 | """Setup and return logger ``name`` with same settings as 'lymph' logger.""" 22 | lymph_logger = logging.getLogger('lymph') 23 | logger = logging.getLogger(name) 24 | for hdlr in lymph_logger.handlers: 25 | logger.addHandler(hdlr) 26 | logger.setLevel(lymph_logger.level) 27 | # Since we are using DictConfig all logger are disabled by default first, so 28 | # we are enabling any logger here ! 29 | logger.disabled = False 30 | return logger 31 | 32 | 33 | class PubLogHandler(logging.Handler): 34 | def __init__(self, endpoint, socket=None): 35 | super(PubLogHandler, self).__init__() 36 | self.socket = socket 37 | if self.socket is None: 38 | ctx = zmq.Context.instance() 39 | self.socket = ctx.socket(zmq.PUB) 40 | endpoint, port = bind_zmq_socket(self.socket, endpoint) 41 | self.endpoint = endpoint 42 | 43 | def emit(self, record): 44 | topic = record.levelname 45 | self.socket.send_multipart([ 46 | self._encode(topic), 47 | self._encode(self.endpoint), 48 | self._encode(self.format(record))]) 49 | 50 | @staticmethod 51 | def _encode(potentially_text, encoding='utf-8'): 52 | if isinstance(potentially_text, six.text_type): 53 | return potentially_text.encode(encoding) 54 | return potentially_text 55 | 56 | 57 | def setup_logging(config, loglevel, logfile): 58 | """Configure 'lymph' logger handlers and level. 59 | 60 | This function also set the container.log_endpoint in case it wasn't set. 61 | """ 62 | logconf = copy.deepcopy(config.get_raw('logging', {})) 63 | log_endpoint = config.get('container.log_endpoint') 64 | log_socket = None 65 | # Get log_endpoint in case it wasn't set in the config. 66 | if not log_endpoint: 67 | ctx = zmq.Context.instance() 68 | log_socket = ctx.socket(zmq.PUB) 69 | log_endpoint, port = bind_zmq_socket(log_socket, config.get('container.ip')) 70 | 71 | config.set('container.log_endpoint', log_endpoint) 72 | 73 | logconf.setdefault('version', 1) 74 | formatters = logconf.setdefault('formatters', {}) 75 | formatters.setdefault('_trace', { 76 | '()': 'lymph.core.trace.TraceFormatter', 77 | 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s - trace_id="%(trace_id)s"', 78 | }) 79 | handlers = logconf.setdefault('handlers', {}) 80 | handlers.setdefault('_zmqpub', { 81 | 'class': 'lymph.utils.logging.PubLogHandler', 82 | 'formatter': '_trace', 83 | 'endpoint': log_endpoint, 84 | 'socket': log_socket, 85 | }) 86 | console_logconf = { 87 | 'class': 'logging.StreamHandler', 88 | 'formatter': '_trace', 89 | 'level': 'DEBUG', 90 | } 91 | if logfile: 92 | console_logconf.update({ 93 | 'class': 'logging.FileHandler', 94 | 'filename': logfile 95 | }) 96 | handlers.setdefault('_console', console_logconf) 97 | loggers = logconf.setdefault('loggers', {}) 98 | loggers.setdefault('lymph', { 99 | 'handlers': ['_console', '_zmqpub'], 100 | 'level': loglevel.upper(), 101 | }) 102 | dictConfig(logconf) 103 | -------------------------------------------------------------------------------- /lymph/utils/observables.py: -------------------------------------------------------------------------------- 1 | 2 | class Observable(object): 3 | def __init__(self): 4 | self.observers = {} 5 | 6 | def notify_observers(self, action, *args, **kwargs): 7 | kwargs.setdefault('action', action) 8 | for callback in self.observers.get(action, ()): 9 | callback(*args, **kwargs) 10 | 11 | def observe(self, actions, callback): 12 | if not isinstance(actions, (tuple, list)): 13 | actions = (actions,) 14 | for action in actions: 15 | self.observers.setdefault(action, set()).add(callback) 16 | -------------------------------------------------------------------------------- /lymph/utils/ripdb.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import logging 4 | import pdb 5 | import six 6 | import socket 7 | import sys 8 | 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class Ripdb(pdb.Pdb): 14 | """ 15 | Based on 16 | * https://github.com/tamentis/rpdb 17 | * http://blog.ionelmc.ro/2013/06/05/python-debugging-tools/ 18 | 19 | """ 20 | def __init__(self, port=0): 21 | self.old_stdout = sys.stdout 22 | self.old_stdin = sys.stdin 23 | self.listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 24 | self.listen_socket.bind(('0.0.0.0', port)) 25 | if not port: 26 | logger.critical("PDB remote session open on: %s", self.listen_socket.getsockname()) 27 | six.print_("PDB remote session open on:", self.listen_socket.getsockname(), file=sys.__stderr__) 28 | sys.stderr.flush() 29 | self.listen_socket.listen(1) 30 | self.connected_socket, address = self.listen_socket.accept() 31 | self.handle = self.connected_socket.makefile('rw') 32 | pdb.Pdb.__init__(self, stdin=self.handle, stdout=self.handle) 33 | sys.stdout = sys.stdin = self.handle 34 | 35 | def do_continue(self, arg): 36 | sys.stdout = self.old_stdout 37 | sys.stdin = self.old_stdin 38 | self.handle.close() 39 | self.connected_socket.close() 40 | self.listen_socket.close() 41 | self.set_continue() 42 | return 1 43 | 44 | do_c = do_cont = do_continue 45 | 46 | 47 | def set_trace(): 48 | Ripdb().set_trace(sys._getframe().f_back) 49 | -------------------------------------------------------------------------------- /lymph/utils/sockets.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import os 3 | 4 | from six.moves import urllib 5 | import netifaces 6 | 7 | 8 | def guess_external_ip(): 9 | gateways = netifaces.gateways() 10 | try: 11 | ifnet = gateways['default'][netifaces.AF_INET][1] 12 | return netifaces.ifaddresses(ifnet)[netifaces.AF_INET][0]['addr'] 13 | except (KeyError, IndexError): 14 | return 15 | 16 | 17 | def bind_zmq_socket(sock, address, port=None): 18 | endpoint = address if '://' in address else 'tcp://%s' % address 19 | p = urllib.parse.urlparse(endpoint) 20 | if port and p.port and p.port != port: 21 | raise ValueError('two port numbers given: %s and %s' % (p.port, port)) 22 | 23 | if p.port: 24 | sock.bind(endpoint) 25 | port = p.port 26 | elif port: 27 | endpoint = '%s:%s' % (endpoint, port) 28 | sock.bind(endpoint) 29 | else: 30 | port = sock.bind_to_random_port(endpoint) 31 | endpoint = '%s:%s' % (endpoint, port) 32 | return endpoint, port 33 | 34 | 35 | # adapted from https://github.com/mozilla-services/chaussette/ 36 | def create_socket(host, family=socket.AF_INET, type=socket.SOCK_STREAM, 37 | backlog=2048, blocking=True, inheritable=False): 38 | if family == socket.AF_UNIX and not host.startswith('unix:'): 39 | raise ValueError('Your host needs to have the unix:/path form') 40 | if host.startswith('unix:'): 41 | family = socket.AF_UNIX 42 | 43 | if host.startswith('fd://'): 44 | fd = int(host[5:]) 45 | sock = socket.fromfd(fd, family, type) 46 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 47 | else: 48 | sock = socket.socket(family, type) 49 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 50 | if host.startswith('unix:'): 51 | filename = host[len('unix:'):] 52 | try: 53 | os.remove(filename) 54 | except OSError: 55 | pass 56 | sock.bind(filename) 57 | else: 58 | if ':' in host: 59 | host, port = host.rsplit(':', 1) 60 | port = int(port) 61 | else: 62 | host, port = '0.0.0.0', int(host) 63 | sock.bind((host, port)) 64 | sock.listen(backlog) 65 | 66 | if blocking: 67 | sock.setblocking(1) 68 | else: 69 | sock.setblocking(0) 70 | 71 | # Required since Python 3.4 to be able to share a socket with a child 72 | # process. 73 | if inheritable and hasattr(os, 'set_inheritable'): 74 | os.set_inheritable(sock.fileno(), True) 75 | 76 | return sock 77 | 78 | 79 | def get_unused_port(host="127.0.0.1", family=socket.AF_INET, socktype=socket.SOCK_STREAM): 80 | tempsock = socket.socket(family, socktype) 81 | tempsock.bind((host, 0)) 82 | port = tempsock.getsockname()[1] 83 | tempsock.close() 84 | del tempsock 85 | return port 86 | 87 | -------------------------------------------------------------------------------- /lymph/utils/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/utils/tests/__init__.py -------------------------------------------------------------------------------- /lymph/utils/tests/test_accumulator.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | from six.moves import range 4 | from unittest import TestCase 5 | 6 | from lymph.utils import Accumulator 7 | 8 | 9 | class AccumulatorTests(TestCase): 10 | def test_accumulator(self): 11 | acc = Accumulator() 12 | for i in range(1, 6): 13 | acc.add(i / 15.) 14 | self.assertEqual(acc.n, 5) 15 | self.assertEqual(acc.sum, 1) 16 | self.assertEqual(acc.mean, 0.2) 17 | self.assertEqual(acc.stddev, 0.09428090415820631) 18 | -------------------------------------------------------------------------------- /lymph/utils/tests/test_event_indexing.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import unittest 3 | from lymph.utils.event_indexing import EventIndex 4 | from lymph.core.events import Event 5 | from mock import patch, Mock 6 | 7 | 8 | class TestStore(unittest.TestCase): 9 | def setUp(self): 10 | def mockget(*args, **kwargs): 11 | return self.mocked 12 | 13 | def mockindex(*args, **kwargs): 14 | body = kwargs.get('body', {}) 15 | self.mocked = {'_source': body} 16 | self.es = Mock(get=mockget, index=mockindex) 17 | 18 | def test_stores_event(self): 19 | index = EventIndex(self.es, 'index_test_name') 20 | event = Event('test_event', {'number': 3, 21 | 'string': 'hi', 22 | 'float': 3.4, 23 | 'dict': {'one': 1, 'two': 'dos'}, 24 | 'date': datetime.date(2014, 5, 2), 25 | 'bool': True, 26 | 'list': [1, 2, 3]}) 27 | with patch('uuid.uuid4', Mock(hex='testuuid')): 28 | index.index(event) 29 | es_event = self.es.get(index='index_test_name', id='testuuid') 30 | self.assertEquals(es_event['_source']['s_string'], 'hi') 31 | self.assertEquals(es_event['_source']['i_number'], 3) 32 | self.assertEquals(es_event['_source']['f_float'], 3.4) 33 | self.assertEquals(es_event['_source']['o_dict'], {'s_two': 'dos', 34 | 'i_one': 1}) 35 | self.assertEquals(es_event['_source']['d_date'], datetime.date( 36 | 2014, 5, 2)) 37 | self.assertEquals(es_event['_source']['b_bool'], True) 38 | self.assertEquals(es_event['_source']['l_list'], [1, 2, 3]) 39 | self.assertEquals(es_event['_source']['type'], 'test_event') 40 | -------------------------------------------------------------------------------- /lymph/utils/tests/test_logging.py: -------------------------------------------------------------------------------- 1 | 2 | import logging 3 | from unittest import TestCase 4 | 5 | from lymph.utils.logging import get_loglevel 6 | 7 | 8 | class LoggingUtilsTests(TestCase): 9 | def test_get_loglevel(self): 10 | self.assertEqual(get_loglevel('DEBUG'), logging.DEBUG) 11 | self.assertEqual(get_loglevel('debug'), logging.DEBUG) 12 | self.assertEqual(get_loglevel('Debug'), logging.DEBUG) 13 | self.assertEqual(get_loglevel('INFO'), logging.INFO) 14 | self.assertEqual(get_loglevel('info'), logging.INFO) 15 | self.assertEqual(get_loglevel('ERROR'), logging.ERROR) 16 | self.assertEqual(get_loglevel('error'), logging.ERROR) 17 | self.assertEqual(get_loglevel('CRITICAL'), logging.CRITICAL) 18 | self.assertEqual(get_loglevel('critical'), logging.CRITICAL) 19 | 20 | self.assertRaises(ValueError, get_loglevel, 'FOO') 21 | self.assertRaises(ValueError, get_loglevel, '*') 22 | -------------------------------------------------------------------------------- /lymph/utils/tests/test_sockets.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from lymph.utils.sockets import guess_external_ip 4 | 5 | 6 | class SocketUtilsTests(TestCase): 7 | def test_guess_external_ip(self): 8 | self.assertNotEqual(guess_external_ip(), '127.0.0.1') 9 | -------------------------------------------------------------------------------- /lymph/utils/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from lymph.utils import import_object, Undefined 4 | 5 | 6 | class ImportTests(TestCase): 7 | 8 | def test_import_object(self): 9 | from lymph.core.container import ServiceContainer 10 | cls = import_object('lymph.core.container:ServiceContainer') 11 | self.assertIs(cls, ServiceContainer) 12 | 13 | def test_import_object_without_colon(self): 14 | self.assertRaises(ValueError, import_object, 'lymph.core.container.ServiceContainer') 15 | self.assertRaises(ValueError, import_object, 'lymph.core.container') 16 | 17 | 18 | class UndefinedTests(TestCase): 19 | def test_properties(self): 20 | self.assertNotEqual(Undefined, None) 21 | self.assertNotEqual(Undefined, False) 22 | self.assertFalse(bool(Undefined)) 23 | self.assertEqual(str(Undefined), 'Undefined') 24 | -------------------------------------------------------------------------------- /lymph/web/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/deliveryhero/lymph/62252135066fc80c22884354cddb990389627689/lymph/web/__init__.py -------------------------------------------------------------------------------- /lymph/web/handlers.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from werkzeug.exceptions import MethodNotAllowed 4 | 5 | 6 | http_methods = ('get', 'post', 'head', 'options', 'put', 'patch', 'delete') 7 | 8 | 9 | class RequestHandler(object): 10 | 11 | def __init__(self, interface, request): 12 | self.request = request 13 | self.interface = interface 14 | self._json = None 15 | 16 | def json(self): 17 | if not "application/json" == self.request.mimetype: 18 | raise ValueError("The request Content-Type is not JSON") 19 | 20 | if self._json is None: 21 | self._json = json.loads(self.request.get_data(as_text=True)) 22 | return self._json 23 | 24 | def dispatch(self, args): 25 | method = self.request.method.lower() 26 | if method not in http_methods: 27 | raise MethodNotAllowed() 28 | try: 29 | func = getattr(self, method) 30 | except AttributeError: 31 | raise MethodNotAllowed() 32 | return func(**args) 33 | -------------------------------------------------------------------------------- /lymph/web/routing.py: -------------------------------------------------------------------------------- 1 | from werkzeug.routing import Rule 2 | 3 | 4 | class HandledRule(Rule): 5 | 6 | def __init__(self, string, handler, **kwargs): 7 | self.handler = handler 8 | super(HandledRule, self).__init__(string, **kwargs) 9 | 10 | def get_empty_kwargs(self): 11 | empty_kwargs = super(HandledRule, self).get_empty_kwargs() 12 | empty_kwargs['handler'] = self.handler 13 | return empty_kwargs 14 | 15 | -------------------------------------------------------------------------------- /lymph/web/wsgi_server.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from gevent.pywsgi import WSGIServer, WSGIHandler 4 | 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class LymphWSGIHandler(WSGIHandler): 10 | 11 | def format_request(self): 12 | # XXX(Mouad): Copied shamessly from gevent.pywsgi.WSGIHandler's format_request 13 | # and removed only the datetime from the output, since it's already part of 14 | # lymph logger format. 15 | length = self.response_length or '-' 16 | if self.time_finish: 17 | delta = '%f' % (self.time_finish - self.time_start) 18 | else: 19 | delta = '-' 20 | client_address = self.client_address[0] if isinstance(self.client_address, tuple) else self.client_address 21 | return 'client=%s - - "%s" status=%s length=%s duration=%s (seconds)' % ( 22 | client_address or '-', 23 | getattr(self, 'requestline', ''), 24 | (getattr(self, 'status', None) or '000').split()[0], 25 | length, 26 | delta) 27 | 28 | def log_request(self): 29 | # XXX(Mouad): Workaround to log correctly in gevent wsgi. 30 | # https://github.com/gevent/gevent/issues/106 31 | logger.info(self.format_request()) 32 | 33 | 34 | class LymphWSGIServer(WSGIServer): 35 | handler_class = LymphWSGIHandler 36 | -------------------------------------------------------------------------------- /nose2.cfg: -------------------------------------------------------------------------------- 1 | [unittest] 2 | plugins = lymph.testing.nose2 3 | 4 | [lymph] 5 | always-on = yes 6 | -------------------------------------------------------------------------------- /requirements/dev.txt: -------------------------------------------------------------------------------- 1 | coverage==4.0a5 2 | pytest==2.5.2 3 | nose==1.3.4 4 | nose-progressive==1.5.1 5 | Sphinx==1.2.2 6 | sphinx-rtd-theme==0.1.5 7 | tox==1.7.1 8 | 9 | -e . 10 | -------------------------------------------------------------------------------- /requirements/docs.txt: -------------------------------------------------------------------------------- 1 | Sphinx==1.2b3 2 | sphinx-rtd-theme==0.1.5 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal=1 3 | 4 | [nosetests] 5 | with-lymph=1 6 | #with-progressive=1 7 | logging-clear-handlers=1 8 | 9 | [flake8] 10 | ignore = E501,W391 11 | 12 | [coverage:run] 13 | omit = 14 | 15 | [coverage:html] 16 | directory = .coverage-report 17 | 18 | [coverage:report] 19 | exclude_lines = 20 | raise NotImplementedError 21 | __import__\('pkg_resources'\).declare_namespace\(__name__\) 22 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from setuptools import setup, find_packages 3 | 4 | import sys 5 | 6 | 7 | with open('README.rst') as f: 8 | description = f.read() 9 | 10 | install_requires = [ 11 | 'docopt>=0.6.1', 12 | 'kazoo>=1.3.1', 13 | 'kombu>=3.0.16', 14 | 'gevent>=1.0.2', 15 | 'msgpack-python>=0.4.0', 16 | 'psutil>=2.1.1', 17 | 'PyYAML>=3.11', 18 | 'pyzmq>=14.3.0', 19 | 'redis>=2.9.1', 20 | 'setproctitle>=1.1.8', 21 | 'six>=1.6', 22 | 'Werkzeug>=0.10.4', 23 | 'blessings>=1.5.1', 24 | 'netifaces>=0.10.4', 25 | 'mock>=1.0.1', 26 | 'PyHamcrest>=1.8.2', 27 | 'pytz>=2015.4', 28 | 'iso8601>=0.1.10', 29 | 'semantic-version>=2.4.2', 30 | ] 31 | 32 | if sys.version_info.major == 2: 33 | install_requires.append('Monotime>=1.0') 34 | elif sys.version_info.major == 3: 35 | install_requires.remove('gevent>=1.0.2') 36 | install_requires.append('gevent==1.1b5') 37 | 38 | setup( 39 | name='lymph', 40 | url='http://github.com/deliveryhero/lymph/', 41 | version='0.16.0-dev', 42 | namespace_packages=['lymph'], 43 | packages=find_packages(), 44 | license=u'Apache License (2.0)', 45 | author=u'Delivery Hero Holding GmbH', 46 | maintainer=u'Johannes Dollinger', 47 | maintainer_email=u'johannes.dollinger@deliveryhero.com', 48 | description=u'a service framework', 49 | long_description=description, 50 | include_package_data=True, 51 | install_requires=install_requires, 52 | extras_require={ 53 | 'sentry': ['raven'], 54 | 'newrelic': ['newrelic'], 55 | }, 56 | entry_points={ 57 | 'console_scripts': ['lymph = lymph.cli.main:main'], 58 | 'lymph.cli': [ 59 | 'discover = lymph.cli.discover:DiscoverCommand', 60 | 'emit = lymph.cli.emit:EmitCommand', 61 | 'help = lymph.cli.help:HelpCommand', 62 | 'inspect = lymph.cli.inspect:InspectCommand', 63 | 'instance = lymph.cli.service:InstanceCommand', 64 | 'list = lymph.cli.list:ListCommand', 65 | 'node = lymph.cli.service:NodeCommand', 66 | 'request = lymph.cli.request:RequestCommand', 67 | 'shell = lymph.cli.shell:ShellCommand', 68 | 'subscribe = lymph.cli.subscribe:SubscribeCommand', 69 | 'tail = lymph.cli.tail:TailCommand', 70 | 'config = lymph.cli.config:ConfigCommand', 71 | 'worker = lymph.cli.service:WorkerCommand', 72 | 'change-loglevel = lymph.cli.loglevel:LogLevelCommand', 73 | ], 74 | 'nose.plugins.0.10': ['lymph = lymph.testing.nose:LymphPlugin'], 75 | 'pytest11': ['lymph = lymph.testing.pytest'], 76 | 'kombu.serializers': [ 77 | 'lymph-json = lymph.serializers.kombu:json_serializer_args', 78 | 'lymph-msgpack = lymph.serializers.kombu:msgpack_serializer_args', 79 | ], 80 | }, 81 | classifiers=[ 82 | 'Development Status :: 3 - Alpha', 83 | 'Intended Audience :: Developers', 84 | 'License :: OSI Approved :: Apache Software License', 85 | 'Operating System :: OS Independent', 86 | 'Programming Language :: Python :: 2.7', 87 | 'Programming Language :: Python :: 3' 88 | ] 89 | ) 90 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27,py34,py35,docs 3 | 4 | [testenv] 5 | usedevelop = True 6 | install_command = pip install {opts} {packages} 7 | # workaround for test discovery issues (nose + py.test): 8 | changedir = {envtmpdir} 9 | commands = nosetests --with-lymph --logging-clear-handlers --logging-level=ERROR --nocapture --with-xunit lymph [] 10 | deps = 11 | -rrequirements/dev.txt 12 | 13 | [testenv:docs] 14 | basepython = python 15 | changedir = . 16 | whitelist_externals = make 17 | deps = 18 | -rrequirements/docs.txt 19 | commands = 20 | make docs 21 | --------------------------------------------------------------------------------