├── tests ├── __init__.py ├── functional │ ├── __init__.py │ ├── test_jsonrpc_specs.py │ └── test_microservice.py ├── services │ ├── __init__.py │ ├── static_files │ │ ├── static1 │ │ └── static2 │ ├── template_files │ │ ├── template2.html │ │ └── template1.html │ ├── service_jsonrpc_specs.py │ ├── service_microservice.py │ ├── service_service_creation.py │ └── service_client.py ├── configuration │ ├── __init__.py │ ├── test_configurables.py │ └── test_cmd_configuration.py ├── test_utils.py ├── events │ ├── test_rabbitmq.py │ └── test_events.py ├── test_client.py ├── test_client_functional.py └── test_structs.py ├── examples ├── example_plugins │ ├── __init__.py │ └── service.py ├── example_modules │ ├── module_1.py │ ├── module_2.py │ └── service.py ├── example_stats │ └── service.py ├── example_publisher_subscriber │ ├── consumer.py │ └── producer.py ├── example_discovery │ ├── service2.py │ ├── service1.py │ └── registry.py ├── example_coroutine_method │ └── service.py ├── example_handler_ref │ └── service.py ├── example_events │ ├── service2.py │ └── service.py ├── example_client │ ├── service.py │ └── client.py ├── hello_world.py └── example_webapp_vuejs │ ├── service.py │ └── index.html ├── requirements.txt ├── .coveragerc ├── MANIFEST.in ├── pytest.ini ├── gemstone ├── __init__.py ├── config │ ├── __init__.py │ ├── configurable.py │ └── configurator.py ├── event │ ├── transport │ │ ├── __init__.py │ │ ├── redis_transport.py │ │ ├── base.py │ │ └── rabbitmq.py │ └── __init__.py ├── plugins │ ├── error.py │ ├── __init__.py │ └── base.py ├── core │ ├── __init__.py │ ├── container.py │ ├── decorators.py │ ├── structs.py │ └── handlers.py ├── client │ ├── __init__.py │ ├── structs.py │ └── remote_service.py ├── discovery │ ├── __init__.py │ ├── base.py │ ├── default.py │ ├── redis_strategy.py │ └── cache.py ├── errors.py ├── cli.py └── util.py ├── docs ├── reference │ ├── gemstone.util.rst │ ├── modules.rst │ ├── gemstone.event.rst │ ├── gemstone.config.rst │ ├── gemstone.plugins.rst │ ├── gemstone.discovery.rst │ ├── gemstone.client.rst │ └── gemstone.core.rst ├── topics │ ├── index.rst │ ├── service_discovery.rst │ ├── publisher_subscriber.rst │ ├── configuration.rst │ └── rpc.rst ├── index.rst ├── changes.rst ├── Makefile ├── make.bat └── conf.py ├── requirements_tests.txt ├── .travis.yml ├── TODO.md ├── appveyor.yml ├── scripts └── send_single_event.py ├── LICENSE ├── .gitignore ├── README.rst └── setup.py /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/functional/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/services/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/configuration/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/example_plugins/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tornado 2 | click 3 | simplejson -------------------------------------------------------------------------------- /tests/services/static_files/static1: -------------------------------------------------------------------------------- 1 | this is static 1 -------------------------------------------------------------------------------- /tests/services/static_files/static2: -------------------------------------------------------------------------------- 1 | this is static 2 -------------------------------------------------------------------------------- /tests/services/template_files/template2.html: -------------------------------------------------------------------------------- 1 | hello {{name}} -------------------------------------------------------------------------------- /tests/services/template_files/template1.html: -------------------------------------------------------------------------------- 1 | This is template 1 -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | 3 | exclude_lines = 4 | pragma: no cover 5 | pass -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include gemstone/data/* 2 | include LICENSE 3 | include requirements.txt 4 | include README.rst 5 | include CHANGES.rst -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | 3 | ; dont recurse the tests/services directory 4 | norecursedirs = services* 5 | 6 | ; ignore all classes 7 | python_classes = -------------------------------------------------------------------------------- /gemstone/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Build microservices with Python 3 | """ 4 | 5 | __author__ = "Vlad Calin" 6 | __email__ = "vlad.s.calin@gmail.com" 7 | 8 | __version__ = "0.12.0" 9 | -------------------------------------------------------------------------------- /docs/reference/gemstone.util.rst: -------------------------------------------------------------------------------- 1 | The gemstone.util module 2 | ======================== 3 | 4 | .. py:currentmodule:: gemstone.util 5 | .. automodule:: gemstone.util 6 | :members: 7 | 8 | -------------------------------------------------------------------------------- /requirements_tests.txt: -------------------------------------------------------------------------------- 1 | # standard requirements 2 | tornado 3 | click 4 | pika 5 | simplejson 6 | redis 7 | 8 | # testing 9 | pytest>=3 10 | pytest-tornado 11 | pytest-cov 12 | coveralls 13 | 14 | # documentation 15 | sphinx 16 | -------------------------------------------------------------------------------- /docs/topics/index.rst: -------------------------------------------------------------------------------- 1 | .. _topics-top: 2 | 3 | Topics 4 | ====== 5 | 6 | Various topics of interest 7 | 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | rpc 13 | publisher_subscriber 14 | service_discovery 15 | configuration -------------------------------------------------------------------------------- /examples/example_modules/module_1.py: -------------------------------------------------------------------------------- 1 | from gemstone.core.modules import Module 2 | import gemstone 3 | 4 | 5 | class FirstModule(Module): 6 | @gemstone.exposed_method("module1.say_hello") 7 | def say_hello(self): 8 | return "Hello from module 1!" 9 | -------------------------------------------------------------------------------- /examples/example_modules/module_2.py: -------------------------------------------------------------------------------- 1 | from gemstone.core.modules import Module 2 | import gemstone 3 | 4 | 5 | class SecondModule(Module): 6 | @gemstone.exposed_method("module2.say_hello") 7 | def say_hello(self): 8 | return "Hello from module 2!" 9 | -------------------------------------------------------------------------------- /gemstone/config/__init__.py: -------------------------------------------------------------------------------- 1 | from .configurator import BaseConfigurator, CommandLineConfigurator 2 | from .configurable import Configurable 3 | 4 | __all__ = [ 5 | 'BaseConfigurator', 6 | 'CommandLineConfigurator', 7 | 8 | 'Configurable' 9 | ] 10 | -------------------------------------------------------------------------------- /gemstone/event/transport/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import BaseEventTransport 2 | from .rabbitmq import RabbitMqEventTransport 3 | from .redis_transport import RedisEventTransport 4 | 5 | __all__ = [ 6 | 'BaseEventTransport', 7 | 8 | 'RabbitMqEventTransport', 9 | 'RedisEventTransport' 10 | ] 11 | -------------------------------------------------------------------------------- /gemstone/plugins/error.py: -------------------------------------------------------------------------------- 1 | class PluginError(Exception): 2 | """ 3 | Base class for plugin specific errors 4 | """ 5 | pass 6 | 7 | 8 | class MissingPluginNameError(PluginError): 9 | """ 10 | Raised when a plugin does not have a properly configured name 11 | """ 12 | pass 13 | -------------------------------------------------------------------------------- /gemstone/event/__init__.py: -------------------------------------------------------------------------------- 1 | from .transport.base import BaseEventTransport 2 | from .transport.redis_transport import RedisEventTransport 3 | from .transport.rabbitmq import RabbitMqEventTransport 4 | 5 | __all__ = [ 6 | 'BaseEventTransport', 7 | 'RedisEventTransport', 8 | 'RabbitMqEventTransport' 9 | ] 10 | -------------------------------------------------------------------------------- /docs/reference/modules.rst: -------------------------------------------------------------------------------- 1 | .. _reference-top: 2 | 3 | Modules 4 | ======= 5 | 6 | 7 | .. toctree:: 8 | :maxdepth: 2 9 | 10 | gemstone.core.rst 11 | gemstone.client.rst 12 | gemstone.config.rst 13 | gemstone.event.rst 14 | gemstone.discovery.rst 15 | gemstone.plugins.rst 16 | gemstone.util.rst -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.4" 4 | - "3.5" 5 | - "3.6" 6 | - "nightly" 7 | install: 8 | - "pip install ." 9 | - "pip install -r requirements_tests.txt" 10 | script: 11 | - "python -m pytest --cov=gemstone tests" 12 | after_success: 13 | - "coveralls -i" 14 | 15 | branches: 16 | only: 17 | - master -------------------------------------------------------------------------------- /examples/example_stats/service.py: -------------------------------------------------------------------------------- 1 | import gemstone 2 | 3 | 4 | class StatsService(gemstone.MicroService): 5 | name = "stats_example" 6 | port = 8000 7 | 8 | use_statistics = True 9 | 10 | @gemstone.exposed_method() 11 | def sum(self, a, b): 12 | return a + b 13 | 14 | 15 | if __name__ == '__main__': 16 | StatsService().start() 17 | -------------------------------------------------------------------------------- /gemstone/core/__init__.py: -------------------------------------------------------------------------------- 1 | from .microservice import MicroService 2 | from .container import Container 3 | from .decorators import event_handler, exposed_method 4 | from .handlers import GemstoneCustomHandler 5 | 6 | __all__ = [ 7 | 'MicroService', 8 | 9 | 'exposed_method', 10 | 'event_handler', 11 | 12 | 'GemstoneCustomHandler', 13 | 14 | 'Container' 15 | ] 16 | -------------------------------------------------------------------------------- /gemstone/client/__init__.py: -------------------------------------------------------------------------------- 1 | from .remote_service import RemoteService, AsyncMethodCall 2 | from .structs import AsyncMethodCall, Result, MethodCall, Notification, BatchResult 3 | 4 | __all__ = [ 5 | 'RemoteService', 6 | 'AsyncMethodCall' 7 | 8 | # structs 9 | 'AsyncMethodCall', 10 | 'Result', 11 | 'MethodCall', 12 | 'Notification', 13 | 'BatchResult' 14 | ] 15 | -------------------------------------------------------------------------------- /gemstone/discovery/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import BaseDiscoveryStrategy 2 | from .default import HttpDiscoveryStrategy 3 | from .redis_strategy import RedisDiscoveryStrategy 4 | 5 | from .cache import ServiceDiscoveryCache, DummyCache 6 | 7 | __all__ = [ 8 | 'BaseDiscoveryStrategy', 9 | 10 | 'HttpDiscoveryStrategy', 11 | 'RedisDiscoveryStrategy', 12 | 13 | 'ServiceDiscoveryCache', 14 | 'DummyCache' 15 | ] 16 | -------------------------------------------------------------------------------- /docs/reference/gemstone.event.rst: -------------------------------------------------------------------------------- 1 | The gemstone.event module 2 | ========================= 3 | 4 | .. py:currentmodule:: gemstone.event 5 | 6 | .. automodule:: gemstone.event 7 | 8 | Event transports 9 | ---------------- 10 | 11 | .. autoclass:: BaseEventTransport 12 | :members: 13 | 14 | .. autoclass:: RedisEventTransport 15 | :members: 16 | 17 | .. autoclass:: RabbitMqEventTransport 18 | :members: 19 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | - [ ] Add proper logging 2 | - [ ] Add more configurable access validation logic (through sessions?) 3 | - [ ] Add service discovery mechanism 4 | - [x] Add possibility to communicate with other ``pymicroservice`` based 5 | services. 6 | 7 | - [x] host documentation 8 | - [x] add CI + coveralls 9 | - [x] write better documentation 10 | - [x] assure a standard deployment lifecycle (commit + tag + push + tests + documentation + upload) 11 | -------------------------------------------------------------------------------- /examples/example_modules/service.py: -------------------------------------------------------------------------------- 1 | import gemstone 2 | 3 | from module_1 import FirstModule 4 | from module_2 import SecondModule 5 | 6 | 7 | class ModularizedMicroService(gemstone.MicroService): 8 | name = "module.example" 9 | host = "127.0.0.1" 10 | port = "8000" 11 | 12 | modules = [ 13 | FirstModule(), SecondModule() 14 | ] 15 | 16 | 17 | if __name__ == '__main__': 18 | ModularizedMicroService().start() 19 | -------------------------------------------------------------------------------- /docs/reference/gemstone.config.rst: -------------------------------------------------------------------------------- 1 | The gemstone.config module 2 | ========================== 3 | 4 | .. py:currentmodule:: gemstone.config 5 | 6 | .. automodule:: gemstone.config 7 | 8 | 9 | Configurables 10 | ------------- 11 | 12 | .. autoclass:: Configurable 13 | :members: 14 | 15 | Configurators 16 | ------------- 17 | 18 | .. autoclass:: BaseConfigurator 19 | :members: 20 | 21 | .. autoclass:: CommandLineConfigurator 22 | :members: 23 | -------------------------------------------------------------------------------- /docs/reference/gemstone.plugins.rst: -------------------------------------------------------------------------------- 1 | The gemstone.plugins module 2 | =========================== 3 | 4 | .. py:currentmodule:: gemstone.plugins 5 | .. automodule:: gemstone.plugins 6 | 7 | The gemstone.plugins.BasePlugin class 8 | ------------------------------------- 9 | 10 | .. autoclass:: BasePlugin 11 | :members: 12 | 13 | Exceptions 14 | ---------- 15 | 16 | .. autoexception:: PluginError 17 | :members: 18 | 19 | 20 | .. autoexception:: MissingPluginNameError 21 | :members: 22 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | matrix: 3 | - PYTHON: "C:\\Python34" 4 | - PYTHON: "C:\\Python34-x64" 5 | - PYTHON: "C:\\Python35" 6 | - PYTHON: "C:\\Python35-x64" 7 | - PYTHON: "C:\\Python36" 8 | - PYTHON: "C:\\Python36-x64" 9 | 10 | build: off 11 | 12 | install: 13 | - "%PYTHON%\\python.exe setup.py install" 14 | - "%PYTHON%\\python.exe -m pip install -r requirements_tests.txt" 15 | 16 | test_script: 17 | - "%PYTHON%\\python.exe -m pytest tests" 18 | 19 | branches: 20 | only: 21 | - master -------------------------------------------------------------------------------- /gemstone/errors.py: -------------------------------------------------------------------------------- 1 | class GemstoneError(Exception): 2 | pass 3 | 4 | 5 | class ServiceConfigurationError(GemstoneError): 6 | pass 7 | 8 | 9 | # RemoteService related exception 10 | 11 | class RemoteServiceError(GemstoneError): 12 | pass 13 | 14 | 15 | class CalledServiceError(RemoteServiceError): 16 | pass 17 | 18 | 19 | # Plugin specific 20 | 21 | class PluginDoesNotExistError(GemstoneError): 22 | """ 23 | Raised when a plugin is queried but no plugin with the 24 | specified name exists. 25 | """ 26 | pass 27 | -------------------------------------------------------------------------------- /examples/example_publisher_subscriber/consumer.py: -------------------------------------------------------------------------------- 1 | import gemstone 2 | 3 | from gemstone.event.transport import RabbitMqEventTransport 4 | 5 | 6 | class ConsumerService(gemstone.MicroService): 7 | name = "consumer" 8 | port = 8000 9 | 10 | event_transports = [ 11 | RabbitMqEventTransport("192.168.1.71", 5672, username="admin", password="X5f6rPmx1yYz") 12 | ] 13 | 14 | @gemstone.event_handler("test") 15 | def broadcast_msg(self, message): 16 | print(message) 17 | 18 | 19 | if __name__ == '__main__': 20 | ConsumerService().start() 21 | -------------------------------------------------------------------------------- /examples/example_discovery/service2.py: -------------------------------------------------------------------------------- 1 | import gemstone 2 | from gemstone.discovery.default import HttpDiscoveryStrategy 3 | from gemstone.discovery.redis_strategy import RedisDiscoveryStrategy 4 | 5 | 6 | class Service2(gemstone.MicroService): 7 | name = "service.2" 8 | 9 | port = 9000 10 | 11 | discovery_strategies = [ 12 | RedisDiscoveryStrategy("redis://localhost:6379/0") 13 | ] 14 | 15 | @gemstone.exposed_method() 16 | def say_hello(self, name): 17 | return "hello {}".format(name) 18 | 19 | 20 | if __name__ == '__main__': 21 | Service2().start() 22 | -------------------------------------------------------------------------------- /examples/example_coroutine_method/service.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from tornado.gen import coroutine 4 | import gemstone 5 | 6 | 7 | class CoroutineService(gemstone.MicroService): 8 | name = "coroutine_service" 9 | 10 | @gemstone.async_method 11 | @gemstone.public_method 12 | def get_secret(self): 13 | secret = yield self._executor.submit(self.blocking_method) 14 | return secret 15 | 16 | def blocking_method(self): 17 | time.sleep(2) 18 | return 10 19 | 20 | 21 | if __name__ == '__main__': 22 | service = CoroutineService() 23 | service.start() 24 | -------------------------------------------------------------------------------- /docs/reference/gemstone.discovery.rst: -------------------------------------------------------------------------------- 1 | The gemstone.discovery module 2 | ============================= 3 | 4 | .. py:currentmodule:: gemstone.discovery 5 | 6 | .. automodule:: gemstone.discovery 7 | 8 | Discovery strategies 9 | -------------------- 10 | 11 | .. autoclass:: BaseDiscoveryStrategy 12 | :members: 13 | 14 | .. autoclass:: HttpDiscoveryStrategy 15 | :members: 16 | 17 | .. autoclass:: RedisDiscoveryStrategy 18 | :members: 19 | 20 | Caches 21 | ------ 22 | 23 | .. autoclass:: ServiceDiscoveryCache 24 | :members: 25 | 26 | .. autoclass:: DummyCache 27 | :members: 28 | -------------------------------------------------------------------------------- /examples/example_handler_ref/service.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | import gemstone 3 | 4 | 5 | class TestMicroservice(gemstone.MicroService): 6 | name = "test" 7 | 8 | @gemstone.public_method 9 | @gemstone.requires_handler_reference 10 | def get_cookie(self, handler): 11 | return handler.get_cookie("Test", None) 12 | 13 | @gemstone.public_method 14 | @gemstone.requires_handler_reference 15 | def set_cookie(self, handler): 16 | handler.set_cookie("Test", str(uuid.uuid4())) 17 | return True 18 | 19 | 20 | if __name__ == '__main__': 21 | service = TestMicroservice() 22 | service.start() 23 | -------------------------------------------------------------------------------- /examples/example_events/service2.py: -------------------------------------------------------------------------------- 1 | from gemstone import MicroService, event_handler, exposed_method 2 | from gemstone.event.transport import rabbitmq, redis_transport 3 | 4 | 5 | class EventTestService2(MicroService): 6 | name = "event.test2" 7 | host = "127.0.0.1" 8 | port = 8000 9 | 10 | event_transports = [ 11 | redis_transport.RedisEventTransport("redis://127.0.0.1:6379/0") 12 | ] 13 | 14 | @exposed_method() 15 | def say_hello(self, name): 16 | self.emit_event("said_hello", {"name": name}) 17 | return "Hello {}".format(name) 18 | 19 | 20 | if __name__ == '__main__': 21 | service = EventTestService2() 22 | service.start() 23 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from gemstone.util import dynamic_load 4 | 5 | 6 | def test_dynamic_load(): 7 | import urllib.request 8 | import hashlib 9 | from urllib.parse import urlparse 10 | 11 | m = dynamic_load("urllib.request") 12 | assert m == urllib.request 13 | 14 | m = dynamic_load("hashlib") 15 | assert m == hashlib 16 | 17 | m = dynamic_load("urllib.parse.urlparse") 18 | assert m == urlparse 19 | 20 | 21 | def test_dynamic_load_errors(): 22 | with pytest.raises(ImportError): 23 | dynamic_load("not.existing") 24 | 25 | with pytest.raises(AttributeError): 26 | dynamic_load("gemstone.core.invalid_stuff") 27 | -------------------------------------------------------------------------------- /examples/example_client/service.py: -------------------------------------------------------------------------------- 1 | import random 2 | import time 3 | import gemstone 4 | 5 | from tornado.gen import sleep 6 | 7 | 8 | class TestMicroservice(gemstone.MicroService): 9 | name = "test" 10 | host = "127.0.0.1" 11 | port = 8000 12 | endpoint = "/api" 13 | 14 | @gemstone.exposed_method() 15 | def say_hello(self, name): 16 | return "hello {}".format(name) 17 | 18 | @gemstone.exposed_method(is_coroutine=True) 19 | def slow_method(self, seconds): 20 | yield sleep(seconds) 21 | return "finished sleeping for {} seconds".format(seconds) 22 | 23 | 24 | if __name__ == '__main__': 25 | service = TestMicroservice() 26 | service.start() 27 | -------------------------------------------------------------------------------- /examples/example_events/service.py: -------------------------------------------------------------------------------- 1 | from gemstone import MicroService, event_handler 2 | from gemstone.event.transport import rabbitmq, redis_transport 3 | 4 | 5 | class EventTestService(MicroService): 6 | name = "event.test" 7 | host = "127.0.0.1" 8 | port = 8080 9 | 10 | event_transports = [ 11 | redis_transport.RedisEventTransport("redis://127.0.0.1:6379/0") 12 | ] 13 | 14 | @event_handler("said_hello") 15 | def event_one_handler(self, body): 16 | self.logger.warning(body) 17 | self.logger.info("Somewhere, {} said hello :)".format(body["name"])) 18 | 19 | 20 | if __name__ == '__main__': 21 | service = EventTestService() 22 | service.start() 23 | -------------------------------------------------------------------------------- /docs/reference/gemstone.client.rst: -------------------------------------------------------------------------------- 1 | The gemstone.client module 2 | ========================== 3 | 4 | .. py:currentmodule:: gemstone.client 5 | .. automodule:: gemstone.client 6 | 7 | The gemstone.client.RemoteService class 8 | --------------------------------------- 9 | 10 | .. autoclass:: RemoteService 11 | :members: 12 | 13 | 14 | Various structures 15 | ------------------ 16 | 17 | .. autoclass:: AsyncMethodCall 18 | :members: 19 | 20 | .. autoclass:: MethodCall 21 | :members: 22 | 23 | .. autoclass:: Notification 24 | :members: 25 | 26 | .. autoclass:: Result 27 | :members: 28 | 29 | .. autoclass:: BatchResult 30 | :members: 31 | 32 | -------------------------------------------------------------------------------- /examples/example_discovery/service1.py: -------------------------------------------------------------------------------- 1 | import gemstone 2 | from gemstone.discovery.default import HttpDiscoveryStrategy 3 | from gemstone.discovery.redis_strategy import RedisDiscoveryStrategy 4 | 5 | 6 | class Service1(gemstone.MicroService): 7 | name = "service.1" 8 | 9 | port = 8000 10 | 11 | discovery_strategies = [ 12 | RedisDiscoveryStrategy("redis://localhost:6379/0") 13 | ] 14 | 15 | @gemstone.exposed_method() 16 | def say_hello(self, name): 17 | remote_service = self.get_service("service.2") 18 | print(remote_service) 19 | result = remote_service.call_method("say_hello", params=[name]) 20 | return result.result 21 | 22 | 23 | if __name__ == '__main__': 24 | Service1().start() 25 | -------------------------------------------------------------------------------- /examples/example_discovery/registry.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | import gemstone 4 | 5 | 6 | class ServiceRegistry(gemstone.MicroService): 7 | name = "service_registry" 8 | 9 | port = 8080 10 | 11 | services = {} 12 | lock = threading.Lock() 13 | 14 | @gemstone.exposed_method() 15 | def ping(self, name, url): 16 | with self.lock: 17 | self.services[name] = url 18 | 19 | @gemstone.exposed_method() 20 | def locate_service(self, name): 21 | with self.lock: 22 | location = self.services.get(name) 23 | if location: 24 | return [location] 25 | else: 26 | return [] 27 | 28 | 29 | if __name__ == '__main__': 30 | ServiceRegistry().start() 31 | -------------------------------------------------------------------------------- /examples/hello_world.py: -------------------------------------------------------------------------------- 1 | import gemstone 2 | from gemstone.config.configurable import Configurable 3 | 4 | class HelloWorldService(gemstone.MicroService): 5 | name = "hello_world_service" 6 | host = "127.0.0.1" 7 | port = 8000 8 | 9 | configurables = [ 10 | Configurable("a"), 11 | Configurable("b"), 12 | Configurable("c"), 13 | ] 14 | 15 | @gemstone.exposed_method() 16 | def say_hello(self, name): 17 | return "hello {}".format(name) 18 | 19 | def on_service_start(self): 20 | print("a = ", self.a) 21 | print("b = ", self.b) 22 | print("c = ", self.c) 23 | 24 | if __name__ == '__main__': 25 | service = HelloWorldService() 26 | service.configure() 27 | service.start() 28 | -------------------------------------------------------------------------------- /examples/example_publisher_subscriber/producer.py: -------------------------------------------------------------------------------- 1 | import gemstone 2 | 3 | from gemstone.event.transport import RabbitMqEventTransport 4 | 5 | 6 | class ProducerService(gemstone.MicroService): 7 | name = "producer" 8 | port = 8000 9 | 10 | event_transports = [ 11 | RabbitMqEventTransport("192.168.1.71", 5672, username="admin", password="X5f6rPmx1yYz") 12 | ] 13 | 14 | @gemstone.exposed_method() 15 | def broadcast_msg(self, message): 16 | self.emit_event("test", {"msg": message}, broadcast=True) 17 | return True 18 | 19 | @gemstone.exposed_method() 20 | def single_msg(self, message): 21 | self.emit_event("test", {"msg": message}, broadcast=False) 22 | return True 23 | 24 | 25 | if __name__ == '__main__': 26 | ProducerService().start() 27 | -------------------------------------------------------------------------------- /gemstone/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | This module provides various tools to create general use plugins for the microservice. 4 | 5 | A plugin can override specific attributes from the :py:class:`BasePlugin` class that will 6 | be called in specific situations. 7 | 8 | Also, a plugin can define extra methods that can be used later inside the method calls, 9 | as shown in the following example: 10 | 11 | :: 12 | 13 | @gemstone.core.exposed_method() 14 | def say_hello(self, name): 15 | self.get_plugin("remote_logging").log_info("Somebody said hello to {}".format(name)) 16 | return True 17 | 18 | """ 19 | 20 | from .base import BasePlugin 21 | from .error import MissingPluginNameError, PluginError 22 | 23 | __all__ = [ 24 | 'BasePlugin', 25 | 26 | 'MissingPluginNameError', 27 | 'PluginError' 28 | ] 29 | -------------------------------------------------------------------------------- /scripts/send_single_event.py: -------------------------------------------------------------------------------- 1 | import random 2 | import time 3 | 4 | import simplejson as json 5 | import pika 6 | 7 | 8 | def pack_message(msg): 9 | return json.dumps(msg) 10 | 11 | 12 | i = 0 13 | 14 | while True: 15 | connection = pika.BlockingConnection( 16 | pika.ConnectionParameters( 17 | host="192.168.1.71", 18 | credentials=pika.PlainCredentials(username="admin", password="X5f6rPmx1yYz") 19 | ) 20 | ) 21 | 22 | channel = connection.channel() 23 | 24 | channel.basic_publish( 25 | exchange="gemstone.events.event_{}".format(random.choice(["one", "two"])), 26 | routing_key='', 27 | body=pack_message( 28 | {"username": "vlad", "email": "vlad_calin@swag.ro", "tags": ["awesome", "programmer"], "age": 21}) 29 | ) 30 | connection.close() 31 | i += 1 32 | print(i) 33 | 34 | time.sleep(0.25) 35 | -------------------------------------------------------------------------------- /tests/services/service_jsonrpc_specs.py: -------------------------------------------------------------------------------- 1 | from gemstone.core import MicroService, exposed_method 2 | 3 | 4 | class ServiceJsonRpcSpecs(MicroService): 5 | name = "test.service" 6 | 7 | host = "127.0.0.1" 8 | port = 9999 9 | 10 | skip_configuration = True 11 | 12 | @exposed_method() 13 | def subtract(self, a, b): 14 | if not isinstance(a, int) or not isinstance(b, int): 15 | raise TypeError("Arguments must be integers") 16 | return a - b 17 | 18 | @exposed_method() 19 | def sum(self, *args): 20 | return sum(args) 21 | 22 | @exposed_method() 23 | def update(self, a): 24 | return str(a) 25 | 26 | @exposed_method() 27 | def get_data(self): 28 | return ["hello", 5] 29 | 30 | @exposed_method() 31 | def notify_hello(self, a): 32 | return a 33 | 34 | 35 | if __name__ == '__main__': 36 | service = ServiceJsonRpcSpecs() 37 | service.start() 38 | -------------------------------------------------------------------------------- /tests/services/service_microservice.py: -------------------------------------------------------------------------------- 1 | from gemstone.core import MicroService, exposed_method 2 | 3 | TEST_HOST, TEST_PORT = ("localhost", 65503) 4 | 5 | 6 | class TestService(MicroService): 7 | name = "service.test.2" 8 | 9 | host = TEST_HOST 10 | port = TEST_PORT 11 | 12 | @exposed_method() 13 | def say_hello(self): 14 | return "hello" 15 | 16 | @exposed_method() 17 | def subtract(self, a, b): 18 | return a - b 19 | 20 | @exposed_method() 21 | def sum(self, *args): 22 | return sum(args) 23 | 24 | @exposed_method() 25 | def divide(self, a, b): 26 | return a / b 27 | 28 | @exposed_method(private=True) 29 | def private_sum(self, a, b): 30 | return a + b 31 | 32 | @exposed_method() 33 | def test_raises(self): 34 | raise ValueError("This is a test") 35 | 36 | def authenticate_request(self, handler): 37 | api_token = handler.request.headers.get("x-testing-token") 38 | return api_token == "testing_token" 39 | -------------------------------------------------------------------------------- /tests/configuration/test_configurables.py: -------------------------------------------------------------------------------- 1 | from gemstone.config import Configurable 2 | 3 | 4 | def test_configurable_just_value(): 5 | configurable = Configurable("test") 6 | 7 | configurable.set_value("hello world") 8 | 9 | assert configurable.name == "test" 10 | assert configurable.get_final_value() == "hello world" 11 | 12 | 13 | def test_configurable_template(): 14 | configurable = Configurable("test", template=lambda x: x.split(",")) 15 | configurable.set_value("1,2,3,4") 16 | assert configurable.get_final_value() == ["1", "2", "3", "4"] 17 | 18 | configurable = Configurable("test", template=lambda x: [int(i) for i in x.split(",")]) 19 | configurable.set_value("1,2,3,4") 20 | assert configurable.get_final_value() == [1, 2, 3, 4] 21 | 22 | def sum_between_max_and_min(str_seq): 23 | items = [int(i) for i in str_seq.split(",")] 24 | return max(items) + min(items) 25 | 26 | configurable = Configurable("test_complex_template", template=sum_between_max_and_min) 27 | configurable.set_value("1,2,3,4,5") 28 | assert configurable.get_final_value() == 6 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Calin Vlad 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /gemstone/discovery/base.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class BaseDiscoveryStrategy(abc.ABC): 5 | """ 6 | Base class for service discovery strategies. 7 | """ 8 | @abc.abstractmethod 9 | def ping(self, name, location, **kwargs): 10 | """ 11 | Pings the service registry as defined by the implemented protocol with the 12 | necessary information about the service location. 13 | 14 | :param name: The name of the microservice 15 | :param location: The HTTP location of the microservice, where it can be accessed (multiple 16 | microservices might have the same location, for example when they 17 | are deployed behind a reverse proxy) 18 | :return: 19 | """ 20 | pass 21 | 22 | @abc.abstractmethod 23 | def locate(self, name): 24 | """ 25 | Attempts to locate a microservice with the given name. If no such service exists, 26 | must return ``None`` 27 | 28 | :param name: The name of the microservice to be located 29 | :return: a list of str with the URLs where a microservice with the given name can 30 | be found. 31 | """ 32 | pass 33 | -------------------------------------------------------------------------------- /examples/example_client/client.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import gemstone 4 | 5 | if __name__ == '__main__': 6 | service = gemstone.RemoteService("http://localhost:8000/test/v1/api") 7 | 8 | # test some blocking method calls 9 | print(service.methods.say_hello("world")) 10 | print(service.methods.say_hello(name="world")) 11 | 12 | # test an async call 13 | res = service.methods.say_hello("workd", __async=True) 14 | print(res) 15 | print(res.wait()) 16 | print(res.error()) 17 | print(res.result()) 18 | 19 | # # test a long async call 20 | # res = service.methods.slow_method(seconds=3, __async=True) 21 | # while not res.finished(): 22 | # print("still waiting") 23 | # time.sleep(0.5) 24 | # 25 | # print("Result is here!") 26 | # print(res.result()) 27 | 28 | # calling more stuff in parallel 29 | 30 | res1 = service.methods.slow_method(seconds=3, __async=True) 31 | res2 = service.methods.slow_method(seconds=4, __async=True) 32 | res3 = service.methods.slow_method(seconds=5, __async=True) 33 | 34 | gemstone.make_callbacks([res1, res2, res3], on_result=lambda x: print("[!] {}".format(x)), 35 | on_error=lambda x: print("[x] {}".format(x))) 36 | -------------------------------------------------------------------------------- /examples/example_webapp_vuejs/service.py: -------------------------------------------------------------------------------- 1 | import random 2 | import datetime 3 | 4 | from gemstone import MicroService, exposed_method, GemstoneCustomHandler 5 | 6 | 7 | class IndexHandler(GemstoneCustomHandler): 8 | def get(self): 9 | self.render("index.html") 10 | 11 | 12 | class VueJsExample(MicroService): 13 | name = "example.vue_js" 14 | 15 | host = "127.0.0.1" 16 | port = 8000 17 | 18 | template_dir = "." 19 | static_dirs = [("/static", ".")] 20 | 21 | extra_handlers = [ 22 | (r"/", IndexHandler) 23 | ] 24 | 25 | @exposed_method() 26 | def get_user_info(self): 27 | return random.choice([{ 28 | "username": "admin", 29 | "email": "admin@example.org", 30 | "last_seen": str(datetime.datetime.now()) 31 | }, { 32 | "username": "admin2", 33 | "email": "admin2@example.org", 34 | "last_seen": str(datetime.datetime.now()) 35 | }, { 36 | "username": "admin3", 37 | "email": "admin3@example.org", 38 | "last_seen": str(datetime.datetime.now()) 39 | }]) 40 | 41 | 42 | if __name__ == '__main__': 43 | service = VueJsExample() 44 | service.configure() 45 | 46 | print(service.port) 47 | print(service.host) 48 | service.start() 49 | -------------------------------------------------------------------------------- /docs/topics/service_discovery.rst: -------------------------------------------------------------------------------- 1 | .. _service-discovery: 2 | 3 | Service discovery 4 | ================= 5 | 6 | Enabling automatic service discovery 7 | ------------------------------------ 8 | 9 | In order to enable automatic service discovery, you need to define at least one 10 | discovery strategy. 11 | 12 | :: 13 | 14 | class ExampleService(gemstone.MicroService): 15 | # ... 16 | discovery_strategies = [ 17 | gemstone.discovery.RedisDiscoveryStrategy("redis://registry.example.com:6379/0"), 18 | # ... 19 | ] 20 | # ... 21 | 22 | 23 | Using service discovery 24 | ----------------------- 25 | 26 | You can use the :py:meth:`gemstone.MicroService.get_service` method to automatically 27 | discover other microservice that uses at least one discovery strategy as your service does, 28 | and interact with it directly. 29 | 30 | Example 31 | 32 | :: 33 | 34 | # ... 35 | remote_service = self.get_service("user_manager_service") 36 | if not remote_service: 37 | raise RuntimeError("Service could not be located") 38 | 39 | res = remote_service.call_method("find_user", {"username": "example"} 40 | # ... 41 | 42 | The :py:meth:`gemstone.MicroService.get_service` method returns a :py:meth:`gemstone.RemoteService` 43 | instance. See :ref:`gemstone_client` for more information on this topic. 44 | -------------------------------------------------------------------------------- /tests/events/test_rabbitmq.py: -------------------------------------------------------------------------------- 1 | import unittest.mock 2 | import pytest 3 | 4 | from gemstone.event.transport.rabbitmq import RabbitMqEventTransport 5 | 6 | 7 | def before_test_init(tr): 8 | tr.channel = unittest.mock.MagicMock() 9 | tr.channel.exchange_declare = unittest.mock.MagicMock() 10 | tr.channel.queue_declare = unittest.mock.MagicMock() 11 | tr.channel.queue_bind = unittest.mock.MagicMock() 12 | tr.channel.basic_consume = unittest.mock.MagicMock() 13 | tr.channel.start_consuming = unittest.mock.MagicMock() 14 | tr.channel.basic_reject = unittest.mock.MagicMock() 15 | tr.channel.basic_ack = unittest.mock.MagicMock() 16 | tr.channel.basic_publish = unittest.mock.MagicMock() 17 | tr.channel.close = unittest.mock.MagicMock() 18 | 19 | 20 | @unittest.mock.patch("pika.BlockingConnection") 21 | def test_add_handler(BlockingConnection): 22 | tr = RabbitMqEventTransport() 23 | before_test_init(tr) 24 | 25 | tr.register_event_handler(lambda x: print(x), "test") 26 | tr.register_event_handler(lambda x: print(x), "test2") 27 | tr.register_event_handler(lambda x: print(x), "test3") 28 | 29 | assert set(tr._handlers.keys()) == {"test", "test2", "test3"} 30 | 31 | 32 | @unittest.mock.patch("pika.BlockingConnection") 33 | def test_called_right_handler(BlockingConnection): 34 | tr = RabbitMqEventTransport() 35 | before_test_init(tr) 36 | -------------------------------------------------------------------------------- /tests/services/service_service_creation.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | from tornado.web import RequestHandler 4 | 5 | from gemstone.core import MicroService, exposed_method 6 | from gemstone.core.handlers import GemstoneCustomHandler 7 | 8 | 9 | class NameMissingService(MicroService): 10 | pass 11 | 12 | 13 | class BadMaxParallelBlockingTasksValueService(MicroService): 14 | name = "test.1" 15 | max_parallel_blocking_tasks = -3 16 | 17 | 18 | def get_static_dirs(): 19 | test_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static_files") 20 | return [("/static", test_dir)] 21 | 22 | 23 | HOST, PORT = "127.0.0.1", 14777 24 | 25 | 26 | def get_template_dir(): 27 | return os.path.join(os.path.dirname(os.path.abspath(__file__)), "template_files") 28 | 29 | 30 | class ExtraHandler1(GemstoneCustomHandler): 31 | def get(self): 32 | self.render("template1.html") 33 | 34 | 35 | class ExtraHandler2(GemstoneCustomHandler): 36 | def get(self): 37 | self.render("template2.html", name="world") 38 | 39 | 40 | class TestService2(MicroService): 41 | name = "test.2" 42 | host = HOST 43 | port = PORT 44 | 45 | static_dirs = get_static_dirs() 46 | template_dir = get_template_dir() 47 | 48 | extra_handlers = [ 49 | (r"/tmp1", ExtraHandler1), 50 | (r"/tmp2", ExtraHandler2) 51 | ] 52 | 53 | @exposed_method() 54 | def say_hello(self, who): 55 | return "hello " + who 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Misc 2 | .idea/ 3 | 4 | 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | env/ 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *,cover 50 | .hypothesis/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | 60 | # Flask stuff: 61 | instance/ 62 | .webassets-cache 63 | 64 | # Scrapy stuff: 65 | .scrapy 66 | 67 | # Sphinx documentation 68 | docs/_build/ 69 | docs/make.bat 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # IPython Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # dotenv 84 | .env 85 | 86 | # virtualenv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | 93 | # Rope project settings 94 | .ropeproject 95 | -------------------------------------------------------------------------------- /examples/example_plugins/service.py: -------------------------------------------------------------------------------- 1 | from gemstone.core.microservice import MicroService 2 | from gemstone.core.decorators import exposed_method 3 | from gemstone.plugins.base import BasePlugin 4 | 5 | 6 | class ExamplePlugin(BasePlugin): 7 | name = "example_plugin" 8 | 9 | def on_service_start(self): 10 | self.microservice.logger.info("Microservice started!") 11 | 12 | def on_service_stop(self): 13 | self.microservice.logger.info("Microservice stoped!") 14 | 15 | def on_internal_error(self, exc_instance): 16 | self.microservice.logger.error("Error!!!!! {}".format(exc_instance)) 17 | 18 | def on_method_call(self, jsonrpc_request): 19 | self.microservice.logger.error("Method called: name={} params={}".format( 20 | jsonrpc_request.method, jsonrpc_request.params 21 | )) 22 | 23 | self.microservice.logger.info("Extras: {}".format(jsonrpc_request.extra)) 24 | 25 | def custom_method(self): 26 | self.microservice.logger.warning("Custom method called!") 27 | 28 | 29 | class ExampleService(MicroService): 30 | name = "example_service" 31 | 32 | @exposed_method() 33 | def sum(self, a, b): 34 | return a + b 35 | 36 | @exposed_method() 37 | def product(self, a, b): 38 | self.get_plugin("example_plugin").custom_method() 39 | return a * b 40 | 41 | 42 | if __name__ == '__main__': 43 | service = ExampleService() 44 | service.register_plugin(ExamplePlugin()) 45 | service.start() 46 | -------------------------------------------------------------------------------- /gemstone/discovery/default.py: -------------------------------------------------------------------------------- 1 | import random 2 | import string 3 | 4 | from tornado.httpclient import HTTPClient, HTTPRequest 5 | import simplejson as json 6 | 7 | from gemstone.discovery.base import BaseDiscoveryStrategy 8 | 9 | 10 | class HttpDiscoveryStrategy(BaseDiscoveryStrategy): 11 | """ 12 | A discovery strategy that uses the HTTP protocol as transport. Each ``ping`` and ``locate`` 13 | calls translate to HTTP requests. 14 | 15 | """ 16 | def __init__(self, registry_location): 17 | self.registry = registry_location 18 | 19 | def make_jsonrpc_call(self, url, method, params): 20 | client = HTTPClient() 21 | body = json.dumps({ 22 | "jsonrpc": "2.0", 23 | "method": method, 24 | "params": params, 25 | "id": "".join([random.choice(string.ascii_letters) for _ in range(10)]) 26 | }) 27 | 28 | request = HTTPRequest(url, method="POST", headers={"content-type": "application/json"}, 29 | body=body) 30 | result = client.fetch(request) 31 | return result 32 | 33 | def ping(self, name, location, **kwargs): 34 | self.make_jsonrpc_call(self.registry, "ping", 35 | {"name": name, "url": location}) 36 | 37 | def locate(self, name): 38 | response = self.make_jsonrpc_call(self.registry, "locate_service", {"name": name}) 39 | resp_as_dict = json.loads(response.body.decode()) 40 | return resp_as_dict["result"] 41 | -------------------------------------------------------------------------------- /docs/topics/publisher_subscriber.rst: -------------------------------------------------------------------------------- 1 | .. _publisher-subscriber: 2 | 3 | Publisher-subscriber pattern 4 | ============================ 5 | 6 | In order to use the publisher-subscriber paradigm, you need to define at least one event transport 7 | 8 | The currently implemented event transports are 9 | 10 | - :py:class:`gemstone.event.transport.RabbitMqEventTransport` 11 | - :py:class:`gemstone.event.transport.RedisEventTransport` 12 | 13 | :: 14 | 15 | class ExampleService(gemstone.MicroService): 16 | # ... 17 | event_transports = [ 18 | gemstone.events.transport.RedisEventTransport("redis://127.0.0.1:6379/0"), 19 | gemstone.events.transport.RedisEventTransport("redis://redis.example.com:6379/0"), 20 | # ... 21 | ] 22 | # ... 23 | 24 | After that, for publishing an event, you must call the :py:meth:`gemstone.MicroService.emit_event` 25 | method 26 | 27 | :: 28 | 29 | @gemstone.exposed_method() 30 | def some_method(self): 31 | self.emit_event("test_event", {"message": "hello there"}) 32 | self.emit_event("method_calls", {"method": "some_method"}) 33 | # ... 34 | 35 | In order to subscribe to some kind of events, you need to designate a method 36 | as the event handler 37 | 38 | :: 39 | 40 | @gemstone.event_handler("test_event") 41 | def my_event_handler(self, event_body): 42 | self.logger.info("Received event: {}".format(event_body)) 43 | 44 | 45 | .. note:: 46 | Event handler methods will be executed on the main thread, so they should not be 47 | blocking. 48 | -------------------------------------------------------------------------------- /gemstone/discovery/redis_strategy.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import urllib.parse 3 | import hashlib 4 | 5 | import redis 6 | 7 | from gemstone.discovery.base import BaseDiscoveryStrategy 8 | 9 | _thread_local = threading.local() 10 | 11 | 12 | class RedisDiscoveryStrategy(BaseDiscoveryStrategy): 13 | def __init__(self, redis_url, time_to_live=180): 14 | parsed = urllib.parse.urlparse(redis_url) 15 | if parsed.scheme != "redis": 16 | raise ValueError("Invalid scheme: {}".format(parsed.scheme)) 17 | self.host = parsed.hostname 18 | self.port = parsed.port 19 | self.db = int(parsed.path[1:]) 20 | self.ttl = time_to_live 21 | 22 | self.connection = self._get_connection() 23 | 24 | def _get_connection(self): 25 | conn = getattr(_thread_local, "_redisconn", None) 26 | if conn: 27 | return conn 28 | 29 | conn = redis.StrictRedis(host=self.host, port=self.port, db=self.db) 30 | setattr(_thread_local, "_redisconn", conn) 31 | return conn 32 | 33 | def locate(self, name): 34 | values = [] 35 | keys = self.connection.keys(name + "#*") 36 | for key in keys: 37 | values.append(self.connection.get(key)) 38 | return [x.decode() for x in values] 39 | 40 | @staticmethod 41 | def make_hash(target): 42 | if isinstance(target, str): 43 | target = target.encode() 44 | return hashlib.md5(target).hexdigest() 45 | 46 | def ping(self, name, location, **kwargs): 47 | self.connection.setex(name + "#" + self.make_hash(location), 48 | self.ttl, location) 49 | -------------------------------------------------------------------------------- /tests/services/service_client.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from gemstone.core import MicroService, exposed_method 4 | 5 | HOST, PORT = "127.0.0.1", 6799 6 | PORT2 = PORT + 1 7 | 8 | 9 | class Service1(MicroService): 10 | name = "test.service.client.1" 11 | skip_configuration = True 12 | 13 | host = HOST 14 | port = PORT 15 | 16 | @exposed_method() 17 | def method1(self): 18 | return "hello there" 19 | 20 | @exposed_method() 21 | def method2(self, arg): 22 | return "hello there {}".format(arg) 23 | 24 | @exposed_method() 25 | def method3(self, a, b): 26 | if not isinstance(a, int) or not isinstance(b, int): 27 | raise ValueError("Bad type for a and b") 28 | return a + b 29 | 30 | @exposed_method() 31 | def sleep(self, seconds): 32 | time.sleep(seconds) 33 | return seconds 34 | 35 | @exposed_method() 36 | def sleep_with_error(self, seconds): 37 | time.sleep(seconds) 38 | raise ValueError(seconds) 39 | 40 | @exposed_method() 41 | def method4(self, arg1, arg2): 42 | return {"arg1": arg1, "arg2": arg2} 43 | 44 | @exposed_method(private=True) 45 | def method5(self, name): 46 | return "private {}".format(name) 47 | 48 | def authenticate_request(self, handler): 49 | api_token = handler.request.headers.get("x-api-token") 50 | return api_token == "test-token" 51 | 52 | 53 | class Service2(MicroService): 54 | name = "test.service.client.2" 55 | skip_configuration = True 56 | 57 | host = HOST 58 | port = PORT2 59 | 60 | @exposed_method(private=True) 61 | def test(self): 62 | return True 63 | 64 | def authenticate_request(self, handler): 65 | api_token = handler.request.headers.get("x-api-token") 66 | return api_token == "test-token" 67 | -------------------------------------------------------------------------------- /gemstone/plugins/base.py: -------------------------------------------------------------------------------- 1 | from gemstone.core.microservice import MicroService 2 | 3 | import abc 4 | 5 | from gemstone.plugins.error import MissingPluginNameError 6 | 7 | 8 | class BasePlugin(abc.ABC): 9 | """ 10 | Base class for creating a plugin. 11 | 12 | """ 13 | 14 | #: The name of the plugin. Must be unique per microservice. 15 | name = None 16 | 17 | def __init__(self): 18 | self.microservice = None 19 | 20 | if self.name is None: 21 | raise MissingPluginNameError("Instance {} does not have a name".format(self)) 22 | 23 | def set_microservice(self, microservice: MicroService): 24 | if not isinstance(microservice, MicroService): 25 | raise ValueError( 26 | "Expected gemstone.core.microservice.MicroService but got {} instead".format( 27 | microservice.__class__.__name__ 28 | )) 29 | 30 | self.microservice = microservice 31 | 32 | def on_service_start(self): 33 | """ 34 | Called once when the microservice starts, after it completed all the initialization 35 | steps 36 | """ 37 | pass 38 | 39 | def on_service_stop(self): 40 | """ 41 | Called once when the microservice stops. 42 | """ 43 | pass 44 | 45 | def on_method_call(self, jsonrpc_request): 46 | """ 47 | Called for every method call (even in batch requests, this method will be called 48 | for every request in batch). 49 | 50 | :param jsonrpc_request: a :py:class:`JsonRpcRequest` instance. 51 | """ 52 | pass 53 | 54 | def on_internal_error(self, exc_instance): 55 | """ 56 | Called when an internal error occurs when a method was called. 57 | 58 | :param exc_instance: The caught :py:class:`Exception` instance. 59 | """ 60 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | The **gemstone** framework 2 | ~~~~~~~~~~~~~~~~~~~~~~~~~~ 3 | 4 | .. image:: https://badge.fury.io/py/gemstone.svg 5 | :target: https://badge.fury.io/py/gemstone 6 | .. image:: https://travis-ci.org/vladcalin/gemstone.svg?branch=master 7 | :target: https://travis-ci.org/vladcalin/gemstone 8 | .. image :: https://ci.appveyor.com/api/projects/status/i6rep3022e7occ8e?svg=true 9 | :target: https://ci.appveyor.com/project/vladcalin/gemstone 10 | .. image:: https://readthedocs.org/projects/gemstone/badge/?version=latest 11 | :target: http://gemstone.readthedocs.io/en/latest/?badge=latest 12 | :alt: Documentation Status 13 | .. image:: https://coveralls.io/repos/github/vladcalin/gemstone/badge.svg?branch=master 14 | :target: https://coveralls.io/github/vladcalin/gemstone?branch=master 15 | .. image:: https://codeclimate.com/github/vladcalin/gemstone/badges/gpa.svg 16 | :target: https://codeclimate.com/github/vladcalin/gemstone 17 | :alt: Code Climate 18 | .. image:: https://landscape.io/github/vladcalin/gemstone/master/landscape.svg?style=flat 19 | :target: https://landscape.io/github/vladcalin/gemstone/master 20 | :alt: Code Health 21 | 22 | 23 | An extensible and simplistic library for writing microservices in Python. 24 | 25 | Core features: 26 | 27 | - JSON RPC 2.0 communication (request-response) 28 | - Event based communication (publisher-subscriber) 29 | - autodiscovery 30 | - dynamic configuration of the services 31 | - possibility to add web application functionality 32 | - API token based security 33 | 34 | See the documentation for more info: `documentation `_ 35 | 36 | Check out the issue tracker: `issue tracker `_ 37 | 38 | Changes: `Changelog `_ 39 | 40 | -------------------------------------------------------------------------------- /gemstone/cli.py: -------------------------------------------------------------------------------- 1 | import pprint 2 | import time 3 | import click 4 | 5 | from gemstone.client import RemoteService 6 | 7 | 8 | def format_params(params): 9 | parameters = {} 10 | for pair in params: 11 | k, *v = pair.split("=") 12 | parameters[k] = "=".join(v) 13 | return parameters 14 | 15 | 16 | @click.group() 17 | def cli(): 18 | pass 19 | 20 | 21 | @cli.command("call") 22 | @click.option("--registry", help="The service registry URL used for queries") 23 | @click.argument("name") 24 | @click.argument("method") 25 | @click.argument("params", nargs=-1) 26 | def call(registry, name, method, params): 27 | # TODO add proper validation and fail messages so that the user knows what is going on 28 | _start = time.time() 29 | service = RemoteService.get_service_by_name(registry, name) 30 | print("[!] Service identification: {:.5f} seconds".format(time.time() - _start)) 31 | print("[!] Service name: {}".format(service.name)) 32 | print("[!] Service URL : {}".format(service.url)) 33 | 34 | parameters = format_params(params) 35 | 36 | _start = time.time() 37 | result = getattr(service.methods, method)(**parameters) 38 | print("[!] Method call: {:.5f} seconds".format(time.time() - _start)) 39 | print("[!] Result:\n") 40 | pprint.pprint(result) 41 | 42 | 43 | @cli.command("call_raw") 44 | @click.argument("url") 45 | @click.argument("method") 46 | @click.argument("params", nargs=-1) 47 | def call_raw(url, method, params): 48 | _start = time.time() 49 | service = RemoteService(url) 50 | print("[!] Service identification: {:.5f} seconds".format(time.time() - _start)) 51 | parameters = format_params(params) 52 | 53 | _start = time.time() 54 | result = getattr(service.methods, method)(**parameters) 55 | print("[!] Method call: {:.5f} seconds".format(time.time() - _start)) 56 | 57 | print("[!] Result:\n") 58 | pprint.pprint(result) 59 | -------------------------------------------------------------------------------- /gemstone/core/container.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class Container(abc.ABC): 5 | """ 6 | A container for exposed methods and/or event handlers 7 | for a better modularization of the application. 8 | 9 | Example usage 10 | 11 | :: 12 | 13 | # in users.py 14 | 15 | class UsersModule(gemstone.Container): 16 | 17 | @gemstone.exposed_method("users.register") 18 | def users_register(self, username, password): 19 | pass 20 | 21 | @gemstone.exposed_method("users.login") 22 | def users_login(self) 23 | 24 | """ 25 | 26 | def __init__(self): 27 | self.microservice = None 28 | 29 | def set_microservice(self, microservice): 30 | self.microservice = microservice 31 | 32 | def get_executor(self): 33 | """ 34 | Returns the executor instance used by the microservice. 35 | """ 36 | return self.microservice.get_executor() 37 | 38 | def get_io_loop(self): 39 | """ 40 | Returns the current IOLoop used by the microservice. 41 | :return: 42 | """ 43 | return self.microservice.get_io_loop() 44 | 45 | def get_exposed_methods(self): 46 | exposed = [] 47 | for item in self._iter_methods(): 48 | if getattr(item, "_exposed_public", False) or \ 49 | getattr(item, "_exposed_private", False): 50 | exposed.append(item) 51 | return exposed 52 | 53 | def get_event_handlers(self): 54 | handlers = [] 55 | for item in self._iter_methods(): 56 | if getattr(item, "_event_handler", False): 57 | handlers.append(item) 58 | return handlers 59 | 60 | def _iter_methods(self): 61 | for item_name in dir(self): 62 | item = getattr(self, item_name) 63 | if callable(item): 64 | yield item 65 | -------------------------------------------------------------------------------- /gemstone/config/configurable.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | Configurable component of the configurable sub-framework. 4 | 5 | Example usage of configurables 6 | 7 | :: 8 | 9 | class MyMicroService(MicroService): 10 | 11 | name = "jadasd" 12 | port = 8000 13 | host = "127.0.0.1" 14 | accessible_at = "http://..." 15 | registry_urls = [...] 16 | 17 | configurables = [ 18 | Configurable("port", template=lambda x: int(x)), 19 | Configurable("host") 20 | ] 21 | configurators = [ 22 | CommandLineConfigurator() 23 | ] 24 | 25 | When :py:meth:`Microservice.configure` is called, the configurators 26 | search for values that can override the specified configurables defaults. 27 | 28 | The configurators resolve in the order they are declared. 29 | 30 | """ 31 | 32 | 33 | class Configurable(object): 34 | def __init__(self, name, *, template=None): 35 | """ 36 | Defines a configurable value for the application. 37 | 38 | Example (You should not use configurables in this way unless 39 | you are writing a custom ``Configurator``) 40 | 41 | :: 42 | 43 | c = Configurable("test", template=lambda x: x * 2) 44 | c.set_value("10") 45 | c.get_final_value() # "10" * 2 -> 1010 46 | 47 | c2 = Configurable("list_of_ints", template=lambda x: [int(y) for y in x.split(",")]) 48 | c.set_value("1,2,3,4,5") 49 | c.get_final_value() # [1,2,3,4,5] 50 | 51 | 52 | :param name: The name of the configurable parameter 53 | :param template: A callable template to apply over the extracted value 54 | """ 55 | self.name = name 56 | self.template = template or (lambda x: x) 57 | self.value = None 58 | 59 | def set_value(self, value): 60 | self.value = value 61 | 62 | def get_final_value(self): 63 | to_return = self.value 64 | return self.template(to_return) 65 | 66 | def __repr__(self): 67 | return "".format(self.name) 68 | 69 | def __str__(self): 70 | return repr(self) 71 | -------------------------------------------------------------------------------- /examples/example_webapp_vuejs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Vue.js example 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 |
19 | 20 |

Hello {{! userInfo.username }}

21 | 22 |

23 | Your email is {{! userInfo.email }} 24 |

25 | 26 |

27 | We last saw you {{! userInfo.last_seen }} 28 |

29 |
30 | 31 | 32 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /gemstone/discovery/cache.py: -------------------------------------------------------------------------------- 1 | import time 2 | import threading 3 | 4 | 5 | class DummyCache(object): 6 | """ 7 | Dummy remote service cache. Always returns ``None`` which triggers a 8 | service registry query. 9 | 10 | Example usage: 11 | 12 | :: 13 | 14 | class MyMicroService(MicroService): 15 | # ... 16 | service_registry_cache = DummyCache() 17 | # ... 18 | 19 | """ 20 | def __init__(self): 21 | pass 22 | 23 | def add_entry(self, name, remote_service): 24 | pass 25 | 26 | def get_entry(self, name): 27 | pass 28 | 29 | 30 | class CacheEntry(object): 31 | def __init__(self, name, remote_service): 32 | self.created = time.time() 33 | self.name = name 34 | self.remote_service = remote_service 35 | 36 | def is_still_valid(self, max_age): 37 | return (time.time() - self.created) < max_age 38 | 39 | 40 | class ServiceDiscoveryCache(object): 41 | def __init__(self, cache_lifetime_in_seconds): 42 | """ 43 | Service discovery cache that keeps entries in memory for 44 | a constant period of time. 45 | 46 | Example usage: 47 | 48 | :: 49 | 50 | class MyMicroService(MicroService): 51 | # ... 52 | service_registry_cache = ServiceDiscoveryCache(60) # keeps entries for one minute 53 | # ... 54 | 55 | :param cache_lifetime_in_seconds: int that specifies the cache entry lifetime 56 | """ 57 | 58 | self.ttl = cache_lifetime_in_seconds 59 | self.container = {} 60 | self.container_lock = threading.Lock() 61 | 62 | def add_entry(self, name, remote_service): 63 | with self.container_lock: 64 | self.container.setdefault(name, []) 65 | self.container[name].append(CacheEntry(name, remote_service)) 66 | 67 | def get_entry(self, name): 68 | self.expire_entries() 69 | 70 | with self.container_lock: 71 | for entry in self.container.get(name, []): 72 | if entry.is_still_valid(self.ttl): 73 | return entry 74 | 75 | def expire_entries(self): 76 | with self.container_lock: 77 | for _, entries in self.container.items(): 78 | i = 0 79 | while i < len(entries): 80 | current = entries[i] 81 | if current.is_still_valid(self.ttl): 82 | i += 1 83 | else: 84 | del entries[i] 85 | -------------------------------------------------------------------------------- /gemstone/event/transport/redis_transport.py: -------------------------------------------------------------------------------- 1 | import urllib.parse 2 | 3 | import simplejson as json 4 | 5 | try: 6 | import redis 7 | except ImportError: 8 | redis = None 9 | 10 | from gemstone.event.transport.base import BaseEventTransport 11 | 12 | 13 | class RedisEventTransport(BaseEventTransport): 14 | def __init__(self, redis_url): 15 | """ 16 | Event transport that uses a Redis server as message transport by using 17 | the PUBSUB mechanism. 18 | 19 | :param redis_url: A string that specifies the network location of the redis server: 20 | - ``redis://[:password@]hostaddr:port/dbnumber`` (plaintext) 21 | - ``rediss://[:password@]hostaddr:port/dbnumber`` (over TLS) 22 | - ``unix://[:password@]/path/to/socket?db=dbnumber`` (Unix socket) 23 | 24 | """ 25 | if not redis: 26 | raise RuntimeError("RedisEventTransport requires 'redis' to run") 27 | 28 | super(RedisEventTransport, self).__init__() 29 | conn_details = urllib.parse.urlparse(redis_url) 30 | if conn_details.scheme not in ("redis", "rediss", "unix"): 31 | raise ValueError( 32 | "Invalid redis url: scheme '{}' not allowed".format(conn_details.scheme)) 33 | 34 | self.connection_pool = redis.ConnectionPool.from_url(redis_url) 35 | self.handlers = {} 36 | 37 | def get_pubsub(self): 38 | conn = self.get_redis_connection() 39 | pubsub = conn.pubsub(ignore_subscribe_messages=True) 40 | return pubsub 41 | 42 | def get_redis_connection(self): 43 | return redis.StrictRedis(connection_pool=self.connection_pool) 44 | 45 | def start_accepting_events(self): 46 | pubsub = self.get_pubsub() 47 | pubsub.subscribe(*tuple(self.handlers.keys())) 48 | 49 | for message in pubsub.listen(): 50 | event_name = message["channel"].decode() 51 | event_data = message["data"].decode() 52 | 53 | self.on_event_received(event_name, json.loads(event_data)) 54 | 55 | def register_event_handler(self, handler_func, handled_event_name): 56 | self.handlers[handled_event_name] = handler_func 57 | 58 | def emit_event(self, event_name, event_body): 59 | conn = self.get_redis_connection() 60 | conn.publish(event_name, json.dumps(event_body)) 61 | 62 | def on_event_received(self, event_name, event_body): 63 | handler = self.handlers.get(event_name, None) 64 | if not handler: 65 | return 66 | self.run_on_main_thread(handler, (event_body,), {}) 67 | -------------------------------------------------------------------------------- /docs/topics/configuration.rst: -------------------------------------------------------------------------------- 1 | .. _configuration-tips: 2 | 3 | Configurable features 4 | ===================== 5 | 6 | In the context of this framework, configurables are 7 | entities that designate what properties of the microservice 8 | can be dynamically set and configurators are strategies 9 | that, on service startup, collects the required 10 | properties from the environment. 11 | 12 | Currently, the available confugurators are: 13 | 14 | - :py:class:`gemstone.config.configurator.CommandLineConfigurator` - collects values from 15 | the command line arguments 16 | 17 | In order to specify configurables for the microservice, you have to provide 18 | set the :py:data:`gemstone.MicroService.configurables` attribute to a list of 19 | :py:class:`Configurable` objects. 20 | 21 | Configurators are specified in the :py:data:`gemstone.MicroService.configurators` attribute. 22 | On service startup, each configurator tries to extract the required values from the environment in 23 | the order they are defined. 24 | 25 | 26 | In order to trigger the configurators, you need to explicitly call the 27 | :py:meth:`gemstone.MicroService.configure` method before calling 28 | :py:meth:`gemstone.MicroService.start` 29 | 30 | Defining configurators 31 | ---------------------- 32 | 33 | Configurators are defined in the :py:attr:`gemstone.MicroService.configurators` class attribute. 34 | 35 | :: 36 | 37 | class ExampleService(gemstone.MicroService): 38 | # ... 39 | configurators = [ 40 | gemstone.config.CommandLineConfigurator() 41 | ] 42 | # ... 43 | 44 | Defining configurables 45 | ---------------------- 46 | 47 | Configurables are defined in the :py:attr:`gemstone.MicroService.configurables` class attribute. 48 | 49 | :: 50 | 51 | class ExampleService(gemstone.MicroService): 52 | # ... 53 | configurables = [ 54 | gemstone.config.Configurable("port", template=lambda x: int(x)), 55 | gemstone.config.Configurable("discovery_strategies", 56 | template=lambda x: [RedisDiscoveryStrategy(a) for a in x.split(",")]), 57 | gemstone.config.Configurable("host"), 58 | gemstone.config.Configurable("accessible_at"), 59 | ] 60 | # ... 61 | 62 | In the example above, we defined 4 configurables: 63 | 64 | - ``gemstone.config.Configurable("port", template=lambda x: int(x))`` the ``--port`` command-line 65 | argument will be casted to ``int`` and assigned to ``gemstone.MicroService.port`` 66 | - ``gemstone.config.Configurable("host")`` the ``--host`` command-line 67 | argument assigned to ``gemstone.MicroService.host`` -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import re 3 | import sys 4 | from setuptools import setup, find_packages 5 | 6 | # Check the Python version. Currently only 3.4+ is supported 7 | 8 | if sys.version_info < (3, 4): 9 | sys.exit("Supported only for Python 3.4 or newer.") 10 | 11 | 12 | # Utility functions 13 | 14 | def read_dependencies(req_file): 15 | with open(req_file) as req: 16 | return [line.strip() for line in req] 17 | 18 | 19 | def get_file_content(filename): 20 | with open(filename) as f: 21 | return f.read() 22 | 23 | 24 | def get_meta_attr_from_string(meta_attr, content): 25 | result = re.search("{attrname}\s*=\s*['\"]([^'\"]+)['\"]".format(attrname=meta_attr), content) 26 | if not result: 27 | raise RuntimeError("Unable to extract {}".format(meta_attr)) 28 | return result.group(1) 29 | 30 | 31 | # Metadata 32 | 33 | CLASSIFIERS = """ 34 | Development Status :: 3 - Alpha 35 | License :: OSI Approved :: MIT License 36 | Operating System :: OS Independent 37 | Topic :: Utilities 38 | Programming Language :: Python :: 3.4 39 | Programming Language :: Python :: 3.5 40 | Programming Language :: Python :: 3.6 41 | """ 42 | URL = "https://github.com/vladcalin/gemstone" 43 | KEYWORDS = "microservice service gemstone jsonrpc rpc http asynchronous async tornado" 44 | DESCRIPTION = "Build microservices with Python" 45 | LICENSE = "MIT" 46 | 47 | module_content = get_file_content(os.path.join("gemstone", "__init__.py")) 48 | 49 | readme = get_file_content("README.rst") 50 | 51 | setup( 52 | # project metadata 53 | name="gemstone", 54 | version=get_meta_attr_from_string("__version__", module_content), 55 | license=LICENSE, 56 | 57 | author=get_meta_attr_from_string("__author__", module_content), 58 | author_email=get_meta_attr_from_string("__email__", module_content), 59 | 60 | maintainer=get_meta_attr_from_string("__author__", module_content), 61 | maintainer_email=get_meta_attr_from_string("__email__", module_content), 62 | 63 | long_description=readme, 64 | description=DESCRIPTION, 65 | keywords=KEYWORDS.split(), 66 | classifiers=[x.strip() for x in CLASSIFIERS.split("\n") if x != ""], 67 | url=URL, 68 | 69 | zip_safe=False, 70 | 71 | # packages 72 | packages=find_packages(), 73 | include_package_data=True, 74 | 75 | # tests 76 | test_suite="tests", 77 | test_requires="pytest", 78 | 79 | install_requires=[ 80 | "tornado", 81 | "simplejson", 82 | "click" 83 | ], 84 | 85 | extras_require={ 86 | "rabbitmq": ["pika"], 87 | "redis": ["redis"] 88 | } 89 | ) 90 | -------------------------------------------------------------------------------- /docs/reference/gemstone.core.rst: -------------------------------------------------------------------------------- 1 | The gemstone.core module 2 | ======================== 3 | 4 | 5 | .. automodule:: gemstone.core 6 | 7 | The gemstone.core.MicroService class 8 | ------------------------------------ 9 | 10 | .. autoclass:: gemstone.core.MicroService 11 | 12 | .. autoattribute:: gemstone.core.MicroService.name 13 | .. autoattribute:: gemstone.core.MicroService.host 14 | .. autoattribute:: gemstone.core.MicroService.port 15 | .. autoattribute:: gemstone.core.MicroService.accessible_at 16 | .. autoattribute:: gemstone.core.MicroService.endpoint 17 | .. autoattribute:: gemstone.core.MicroService.template_dir 18 | .. autoattribute:: gemstone.core.MicroService.static_dirs 19 | .. autoattribute:: gemstone.core.MicroService.extra_handlers 20 | .. autoattribute:: gemstone.core.MicroService.plugins 21 | .. autoattribute:: gemstone.core.MicroService.discovery_strategies 22 | .. autoattribute:: gemstone.core.MicroService.service_registry_ping_interval 23 | .. autoattribute:: gemstone.core.MicroService.periodic_tasks 24 | .. autoattribute:: gemstone.core.MicroService.event_transports 25 | .. autoattribute:: gemstone.core.MicroService.configurables 26 | .. autoattribute:: gemstone.core.MicroService.configurators 27 | .. autoattribute:: gemstone.core.MicroService.max_parallel_blocking_tasks 28 | 29 | Can be called 30 | """"""""""""" 31 | 32 | .. automethod:: gemstone.core.MicroService.get_service 33 | .. automethod:: gemstone.core.MicroService.emit_event 34 | .. automethod:: gemstone.core.MicroService.get_io_loop 35 | .. automethod:: gemstone.core.MicroService.get_executor 36 | .. automethod:: gemstone.core.MicroService.start 37 | .. automethod:: gemstone.core.MicroService.configure 38 | .. automethod:: gemstone.core.MicroService.register_plugin 39 | .. automethod:: gemstone.core.MicroService.get_plugin 40 | .. automethod:: gemstone.core.MicroService.start_thread 41 | .. automethod:: gemstone.core.MicroService.make_tornado_app 42 | 43 | Can be overridden 44 | """"""""""""""""" 45 | 46 | .. automethod:: gemstone.core.MicroService.get_logger 47 | .. automethod:: gemstone.core.MicroService.authenticate_request 48 | .. automethod:: gemstone.core.MicroService.on_service_start 49 | 50 | The gemstone.core.Container class 51 | --------------------------------- 52 | 53 | 54 | .. autoclass:: gemstone.core.Container 55 | :members: 56 | 57 | .. py:attribute:: microservice 58 | 59 | Decorators 60 | ---------- 61 | 62 | .. autofunction:: gemstone.core.exposed_method 63 | .. autofunction:: gemstone.core.event_handler 64 | 65 | -------------------------------------------------------------------------------- /gemstone/event/transport/base.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class BaseEventTransport(ABC): 5 | """ 6 | Base class for defining event transports. 7 | 8 | The basic workflow would be the following: 9 | 10 | - the handlers are registered with the :py:meth:`BaseEventTransport.register_event_handler` 11 | method 12 | - the :py:meth:`BaseEventTransport.start_accepting_events` is invoked 13 | - for each incoming event, call :py:meth:`BaseEventTransport.on_event_received` 14 | whose responsibility is to invoke the proper handler function (recommended 15 | to use the ``run_on_main_thread`` method) 16 | 17 | """ 18 | 19 | def __init__(self): 20 | self.microservice = None 21 | 22 | @abstractmethod 23 | def register_event_handler(self, handler_func, handled_event_name): 24 | """ 25 | Registers a function to handle all events of type ``handled_event_name`` 26 | 27 | :param handler_func: the handler function 28 | :param handled_event_name: the handled event type 29 | """ 30 | pass 31 | 32 | @abstractmethod 33 | def start_accepting_events(self): 34 | """ 35 | Starts accepting and handling events. 36 | """ 37 | pass 38 | 39 | @abstractmethod 40 | def on_event_received(self, event_name, event_body): 41 | """ 42 | Handles generic event. This function should treat every event that is received 43 | with the designated handler function. 44 | 45 | :param event_name: the name of the event to be handled 46 | :param event_body: the body of the event to be handled 47 | :return: 48 | """ 49 | pass 50 | 51 | @abstractmethod 52 | def emit_event(self, event_name, event_body): 53 | """ 54 | Emits an event of type ``event_name`` with the ``event_body`` content using the current 55 | event transport. 56 | 57 | :param event_name: 58 | :param event_body: 59 | :return: 60 | """ 61 | pass 62 | 63 | def set_microservice(self, microservice): 64 | """ 65 | Used by the microservice instance to send reference to itself. Do not override this. 66 | """ 67 | self.microservice = microservice 68 | 69 | def run_on_main_thread(self, func, args=None, kwargs=None): 70 | """ 71 | Runs the ``func`` callable on the main thread, by using the provided microservice 72 | instance's IOLoop. 73 | 74 | :param func: callable to run on the main thread 75 | :param args: tuple or list with the positional arguments. 76 | :param kwargs: dict with the keyword arguments. 77 | :return: 78 | """ 79 | if not args: 80 | args = () 81 | if not kwargs: 82 | kwargs = {} 83 | self.microservice.get_io_loop().add_callback(func, *args, **kwargs) 84 | -------------------------------------------------------------------------------- /tests/events/test_events.py: -------------------------------------------------------------------------------- 1 | import unittest.mock 2 | 3 | import pytest 4 | 5 | from gemstone.core import MicroService, event_handler 6 | from gemstone.event.transport import BaseEventTransport 7 | 8 | 9 | class MockedEventTransport(BaseEventTransport): 10 | register_event_handler = unittest.mock.MagicMock() 11 | on_event_received = unittest.mock.MagicMock() 12 | emit_event = unittest.mock.MagicMock() 13 | start_accepting_events = unittest.mock.MagicMock() 14 | 15 | 16 | class SemiMockedEventTransport(BaseEventTransport): 17 | def __init__(self): 18 | self.handlers = {} 19 | 20 | def start_accepting_events(self): 21 | pass 22 | 23 | def register_event_handler(self, handler_func, handled_event_name): 24 | self.handlers[handled_event_name] = handler_func 25 | 26 | def on_event_received(self, event_name, event_body): 27 | self.handlers[event_name](event_body) 28 | 29 | def emit_event(self, event_name, event_body): 30 | pass 31 | 32 | 33 | mocked_transport = MockedEventTransport() 34 | 35 | 36 | class TestServicePublisher(MicroService): 37 | name = "test" 38 | 39 | event_transports = [ 40 | mocked_transport 41 | ] 42 | 43 | handler = unittest.mock.MagicMock() 44 | # it is like a magic mock that is decorated 45 | handler = event_handler("test")(handler) 46 | 47 | 48 | def test_event_handler_discovery(): 49 | service = TestServicePublisher() 50 | service._gather_event_handlers() 51 | 52 | assert service.event_handlers == {"test": service.handler} 53 | 54 | 55 | def test_transport_init(): 56 | service = TestServicePublisher() 57 | service._gather_event_handlers() 58 | service._initialize_event_handlers() 59 | 60 | mocked_transport.register_event_handler.assert_has_calls([ 61 | unittest.mock.call(service.handler, "test") 62 | ]) 63 | 64 | 65 | def test_transport_start_in_threads(): 66 | service = TestServicePublisher() 67 | service.start_thread = unittest.mock.MagicMock() 68 | 69 | service._start_event_handlers() 70 | 71 | service.start_thread.assert_has_calls([ 72 | unittest.mock.call(target=mocked_transport.start_accepting_events, args=(), kwargs={}) 73 | ]) 74 | 75 | 76 | def test_emit_event(): 77 | service = TestServicePublisher() 78 | 79 | service.emit_event("testing", "test_value") 80 | service.emit_event("testing2", "test_value2") 81 | 82 | mocked_transport.emit_event.assert_has_calls([ 83 | unittest.mock.call("testing", "test_value"), 84 | unittest.mock.call("testing2", "test_value2") 85 | ]) 86 | 87 | 88 | def test_receive_event(): 89 | transport = SemiMockedEventTransport() 90 | TestServicePublisher.event_transports = [transport] 91 | service = TestServicePublisher() 92 | service._gather_event_handlers() 93 | service._initialize_event_handlers() 94 | service._start_event_handlers() 95 | 96 | transport.on_event_received("test", "hello") 97 | 98 | service.handler.assert_has_calls([ 99 | unittest.mock.call("hello") 100 | ]) 101 | -------------------------------------------------------------------------------- /tests/configuration/test_cmd_configuration.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pytest 3 | 4 | from gemstone.config import CommandLineConfigurator, Configurable 5 | 6 | 7 | def test_configurator_command_line_no_configurable(monkeypatch): 8 | configurables = [ 9 | Configurable("a"), 10 | Configurable("b"), 11 | Configurable("c") 12 | ] 13 | 14 | monkeypatch.setattr(sys, "argv", [sys.argv[0]]) 15 | 16 | configurator = CommandLineConfigurator() 17 | for c in configurables: 18 | configurator.register_configurable(c) 19 | 20 | configurator.load() 21 | assert configurator.get("a") is None 22 | assert configurator.get("b") is None 23 | assert configurator.get("c") is None 24 | assert configurator.get("d") is None 25 | 26 | 27 | def test_configurator_command_line_one_configurable(monkeypatch): 28 | configurables = [ 29 | Configurable("a"), 30 | Configurable("b"), 31 | Configurable("c") 32 | ] 33 | 34 | monkeypatch.setattr(sys, "argv", [sys.argv[0], "--a=1"]) 35 | 36 | configurator = CommandLineConfigurator() 37 | 38 | for c in configurables: 39 | configurator.register_configurable(c) 40 | configurator.load() 41 | 42 | assert configurator.get("a") == "1" 43 | assert configurator.get("b") is None 44 | assert configurator.get("c") is None 45 | assert configurator.get("d") is None 46 | 47 | 48 | def test_configurator_command_line_two_configurables(monkeypatch): 49 | configurables = [ 50 | Configurable("a"), 51 | Configurable("b"), 52 | Configurable("c") 53 | ] 54 | 55 | monkeypatch.setattr(sys, "argv", [sys.argv[0], "--a=1", "--b", "3"]) 56 | 57 | configurator = CommandLineConfigurator() 58 | 59 | for c in configurables: 60 | configurator.register_configurable(c) 61 | configurator.load() 62 | 63 | assert configurator.get("a") == "1" 64 | assert configurator.get("b") == "3" 65 | assert configurator.get("c") is None 66 | assert configurator.get("d") is None 67 | 68 | 69 | def test_configurator_command_line_three_configurables(monkeypatch): 70 | configurables = [ 71 | Configurable("a"), 72 | Configurable("b"), 73 | Configurable("c") 74 | ] 75 | 76 | monkeypatch.setattr(sys, "argv", [sys.argv[0], "--a=1", "--b", "3", "--c=2"]) 77 | 78 | configurator = CommandLineConfigurator() 79 | 80 | for c in configurables: 81 | configurator.register_configurable(c) 82 | configurator.load() 83 | 84 | assert configurator.get("a") == "1" 85 | assert configurator.get("b") == "3" 86 | assert configurator.get("c") == "2" 87 | assert configurator.get("d") is None 88 | 89 | 90 | def test_configurator_command_line_three_configurables_one_extra_param(monkeypatch): 91 | configurables = [ 92 | Configurable("a"), 93 | Configurable("b"), 94 | Configurable("c") 95 | ] 96 | 97 | monkeypatch.setattr(sys, "argv", [sys.argv[0], "--a=1", "--b", "3", "--c=2", "--d=4"]) 98 | 99 | configurator = CommandLineConfigurator() 100 | 101 | for c in configurables: 102 | configurator.register_configurable(c) 103 | 104 | with pytest.raises(SystemExit): 105 | # unrecognized parameter: --d 106 | configurator.load() 107 | -------------------------------------------------------------------------------- /gemstone/config/configurator.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import argparse 3 | 4 | 5 | class BaseConfigurator(abc.ABC): 6 | """ 7 | Base class for defining configurators. A configurator is a class that, starting 8 | from a set of name-configurable pairs, depending on the configurables' options 9 | and the environment, builds a configuration for the application. 10 | 11 | """ 12 | 13 | def __init__(self): 14 | self.configurables = [] 15 | 16 | @abc.abstractmethod 17 | def load(self): 18 | """ 19 | Loads the configuration for the application 20 | """ 21 | pass 22 | 23 | @abc.abstractmethod 24 | def get(self, name): 25 | """ 26 | Gets the extracted value for the specified name, if available. If no value could 27 | be loaded for the specified name, ``None`` must be returned. 28 | """ 29 | pass 30 | 31 | def register_configurable(self, configurable): 32 | """ 33 | Registers a configurable instance with this configurator 34 | 35 | :param configurable: a :py:class:`Configurable` instance 36 | """ 37 | self.configurables.append(configurable) 38 | 39 | def get_configurable_by_name(self, name): 40 | """ 41 | Returns the registered configurable with the specified name or ``None`` if no 42 | such configurator exists. 43 | """ 44 | l = [c for c in self.configurables if c.name == name] 45 | if l: 46 | return l[0] 47 | 48 | def __repr__(self): 49 | return "<{}>".format(self.__class__.__name__) 50 | 51 | def __str__(self): 52 | return repr(self) 53 | 54 | 55 | class CommandLineConfigurator(BaseConfigurator): 56 | """ 57 | Configurator that collects values from command line arguments. 58 | For each registered configurable, will attempt to get from command line 59 | the value designated by the argument ``--name`` where ``name`` is the name of the 60 | configurable. 61 | 62 | Example 63 | 64 | For the configurables 65 | 66 | - Configurable("a") 67 | - Configurable("b") 68 | - Configurable("c") 69 | 70 | the following command line interface will be exposed 71 | 72 | :: 73 | 74 | usage: service.py [-h] [--a A] [--b B] [--c C] 75 | 76 | optional arguments: 77 | -h, --help show this help message and exit 78 | --a A 79 | --b B 80 | --c C 81 | 82 | The ``service.py`` can be called like this 83 | 84 | :: 85 | 86 | python service.py --a=1 --b=2 --c=3 87 | 88 | 89 | """ 90 | 91 | def __init__(self): 92 | super(CommandLineConfigurator, self).__init__() 93 | self.args = None 94 | 95 | def load(self): 96 | parser = argparse.ArgumentParser() 97 | for configurable in self.configurables: 98 | parser.add_argument("--" + configurable.name) 99 | self.args = parser.parse_args() 100 | 101 | def get(self, name): 102 | configurable = self.get_configurable_by_name(name) 103 | if not configurable: 104 | return None 105 | 106 | value = getattr(self.args, name, None) 107 | if not value: 108 | return None 109 | 110 | configurable.set_value(value) 111 | return configurable.get_final_value() 112 | -------------------------------------------------------------------------------- /gemstone/event/transport/rabbitmq.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | try: 4 | import pika 5 | except ImportError: 6 | pika = None 7 | 8 | from gemstone.event.transport.base import BaseEventTransport 9 | 10 | 11 | class RabbitMqEventTransport(BaseEventTransport): 12 | EXCHANGE_PREFIX_BROADCAST = "gemstone.broadcast." 13 | 14 | def __init__(self, host="127.0.0.1", port=5672, username="", password="", **connection_options): 15 | """ 16 | Event transport via RabbitMQ server. 17 | 18 | :param host: ipv4 or hostname 19 | :param port: the port where the server listens 20 | :param username: username used for authentication 21 | :param password: password used for authentication 22 | :param connection_options: extra arguments that will be used in 23 | :py:class:`pika.BlockingConnection` initialization. 24 | """ 25 | if not pika: 26 | raise RuntimeError("RabbitMqEventTransport requires 'pika' to run") 27 | 28 | super(RabbitMqEventTransport, self).__init__() 29 | self._handlers = {} 30 | 31 | self.connection = pika.BlockingConnection( 32 | pika.ConnectionParameters( 33 | host=host, port=port, 34 | credentials=pika.PlainCredentials(username=username, password=password), 35 | **connection_options 36 | ) 37 | ) 38 | self.channel = self.connection.channel() 39 | 40 | def register_event_handler(self, handler_func, handled_event_name): 41 | self._handlers[handled_event_name] = handler_func 42 | 43 | def start_accepting_events(self): 44 | for event_name, event_handler in self._handlers.items(): 45 | # prepare broadcast queues 46 | current_exchange_name = self.EXCHANGE_PREFIX_BROADCAST + event_name 47 | self.channel.exchange_declare( 48 | exchange=current_exchange_name, 49 | type="fanout" 50 | ) 51 | result = self.channel.queue_declare(exclusive=True) 52 | queue_name = result.method.queue 53 | 54 | self.channel.queue_bind(exchange=current_exchange_name, queue=queue_name) 55 | self.channel.basic_consume(self._callback, queue=queue_name, no_ack=True) 56 | 57 | self.channel.start_consuming() 58 | 59 | def _callback(self, channel, method, properties, body): 60 | if not method.exchange.startswith(self.EXCHANGE_PREFIX_BROADCAST): 61 | return 62 | 63 | event_name = method.exchange[len(self.EXCHANGE_PREFIX_BROADCAST):] 64 | self.on_event_received(event_name, body) 65 | 66 | def on_event_received(self, event_name, event_body): 67 | handler = self._handlers.get(event_name) 68 | if not handler: 69 | return 70 | if isinstance(event_body, bytes): 71 | event_body = event_body.decode() 72 | 73 | event_body = json.loads(event_body) 74 | self.run_on_main_thread(handler, [event_body], {}) 75 | 76 | def emit_event(self, event_name, event_body): 77 | exchange_name = self.EXCHANGE_PREFIX_BROADCAST + event_name 78 | 79 | self.channel.basic_publish( 80 | exchange=exchange_name, 81 | routing_key='', 82 | body=json.dumps(event_body) 83 | ) 84 | 85 | def __del__(self): 86 | if hasattr(self, "channel"): 87 | self.channel.close() 88 | -------------------------------------------------------------------------------- /gemstone/util.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import importlib 3 | 4 | from gemstone.client import RemoteService, AsyncMethodCall 5 | 6 | 7 | def init_default_logger(): 8 | logging.basicConfig( 9 | level=logging.DEBUG, 10 | ) 11 | return logging.getLogger() 12 | 13 | 14 | def as_completed(*async_result_wrappers): 15 | """ 16 | Yields results as they become available from asynchronous method calls. 17 | 18 | Example usage 19 | 20 | :: 21 | 22 | async_calls = [service.call_method_async("do_stuff", (x,)) for x in range(25)] 23 | 24 | for async_call in gemstone.as_completed(*async_calls): 25 | print("just finished with result ", async_call.result()) 26 | 27 | :param async_result_wrappers: :py:class:`gemstone.client.structs.AsyncMethodCall` instances. 28 | :return: a generator that yields items as soon they results become available. 29 | 30 | .. versionadded:: 0.5.0 31 | """ 32 | for item in async_result_wrappers: 33 | if not isinstance(item, AsyncMethodCall): 34 | raise TypeError("Got non-AsyncMethodCall object: {}".format(item)) 35 | 36 | wrappers_copy = list(async_result_wrappers) 37 | 38 | while len(wrappers_copy): 39 | completed = list(filter(lambda x: x.finished(), wrappers_copy)) 40 | if not len(completed): 41 | continue 42 | 43 | for item in completed: 44 | wrappers_copy.remove(item) 45 | yield item 46 | 47 | 48 | def first_completed(*async_result_wrappers): 49 | """ 50 | Just like :py:func:`as_completed`, but returns only the first item and discards the 51 | rest. 52 | 53 | :param async_result_wrappers: 54 | :return: 55 | 56 | .. versionadded:: 0.5.0 57 | """ 58 | for item in async_result_wrappers: 59 | if not isinstance(item, AsyncMethodCall): 60 | raise TypeError("Got non-AsyncMethodCall object: {}".format(item)) 61 | wrappers_copy = list(async_result_wrappers) 62 | while True: 63 | completed = list(filter(lambda x: x.finished(), wrappers_copy)) 64 | if not len(completed): 65 | continue 66 | 67 | return completed[0].result() 68 | 69 | 70 | def get_remote_service_instance_for_url(url): 71 | return RemoteService(url) 72 | 73 | 74 | def dynamic_load(module_or_member): 75 | """ 76 | Dynamically loads a class or member of a class. 77 | 78 | If ``module_or_member`` is something like ``"a.b.c"``, will perform ``from a.b import c``. 79 | 80 | If ``module_or_member`` is something like ``"a"`` will perform ``import a`` 81 | 82 | :param module_or_member: the name of a module or member of a module to import. 83 | :return: the returned entity, be it a module or member of a module. 84 | """ 85 | parts = module_or_member.split(".") 86 | if len(parts) > 1: 87 | name_to_import = parts[-1] 88 | module_to_import = ".".join(parts[:-1]) 89 | else: 90 | name_to_import = None 91 | module_to_import = module_or_member 92 | 93 | module = importlib.import_module(module_to_import) 94 | if name_to_import: 95 | to_return = getattr(module, name_to_import) 96 | if not to_return: 97 | raise AttributeError("{} has no attribute {}".format(module, name_to_import)) 98 | return to_return 99 | else: 100 | return module 101 | -------------------------------------------------------------------------------- /gemstone/core/decorators.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import re 3 | 4 | import tornado.gen 5 | 6 | __all__ = [ 7 | 'event_handler', 8 | 'exposed_method' 9 | ] 10 | 11 | METHOD_NAME_REGEX = re.compile(r'^[a-zA-Z][a-zA-Z0-9_.]*$') 12 | 13 | 14 | def event_handler(event_name): 15 | """ 16 | Decorator for designating a handler for an event type. ``event_name`` must be a string 17 | representing the name of the event type. 18 | 19 | The decorated function must accept a parameter: the body of the received event, 20 | which will be a Python object that can be encoded as a JSON (dict, list, str, int, 21 | bool, float or None) 22 | 23 | :param event_name: The name of the event that will be handled. Only one handler per 24 | event name is supported by the same microservice. 25 | """ 26 | 27 | def wrapper(func): 28 | func._event_handler = True 29 | func._handled_event = event_name 30 | return func 31 | 32 | return wrapper 33 | 34 | 35 | def exposed_method(name=None, private=False, is_coroutine=True, requires_handler_reference=False): 36 | """ 37 | Marks a method as exposed via JSON RPC. 38 | 39 | :param name: the name of the exposed method. Must contains only letters, digits, dots and underscores. 40 | If not present or is set explicitly to ``None``, this parameter will default to the name 41 | of the exposed method. 42 | If two methods with the same name are exposed, a ``ValueError`` is raised. 43 | :type name: str 44 | :param private: Flag that specifies if the exposed method is private. 45 | :type private: bool 46 | :param is_coroutine: Flag that specifies if the method is a Tornado coroutine. If True, it will be wrapped 47 | with the :py:func:`tornado.gen.coroutine` decorator. 48 | :type is_coroutine: bool 49 | :param requires_handler_reference: If ``True``, the handler method will receive as the first 50 | parameter a ``handler`` argument with the Tornado 51 | request handler for the current request. This request handler 52 | can be further used to extract various information from the 53 | request, such as headers, cookies, etc. 54 | :type requires_handler_reference: bool 55 | 56 | .. versionadded:: 0.9.0 57 | 58 | """ 59 | 60 | def wrapper(func): 61 | 62 | # validation 63 | 64 | if name: 65 | method_name = name 66 | else: 67 | method_name = func.__name__ 68 | 69 | if not METHOD_NAME_REGEX.match(method_name): 70 | raise ValueError("Invalid method name: '{}'".format(method_name)) 71 | 72 | @functools.wraps(func) 73 | def real_wrapper(*args, **kwargs): 74 | return func(*args, **kwargs) 75 | 76 | # set appropriate flags 77 | if private: 78 | setattr(real_wrapper, "_exposed_private", True) 79 | else: 80 | setattr(real_wrapper, "_exposed_public", True) 81 | 82 | if is_coroutine: 83 | real_wrapper.__gemstone_is_coroutine = True 84 | real_wrapper = tornado.gen.coroutine(real_wrapper) 85 | setattr(real_wrapper, "_is_coroutine", True) 86 | 87 | if requires_handler_reference: 88 | setattr(real_wrapper, "_req_h_ref", True) 89 | 90 | setattr(real_wrapper, "_exposed_name", method_name) 91 | 92 | return real_wrapper 93 | 94 | return wrapper 95 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. pymicroservice documentation master file, created by 2 | sphinx-quickstart on Fri Nov 25 14:52:22 2016. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to gemstone's documentation! 7 | ==================================== 8 | 9 | 10 | The **gemstone** library aims to provide an easy way to develop simple and 11 | scalable microservices by using the asynchronous features of Python. 12 | 13 | This library offers support for writing a microservice that: 14 | 15 | - exposes a public Json RPC 2.0 HTTP API 16 | (see `The JSON RPC 2.0 specifications `_ ) 17 | - can communicate with other microservices through the JSON RPC protocol. 18 | - can communicate with other microservices through events (messages). 19 | 20 | This documentation is structured in multiple parts: 21 | 22 | - :ref:`topics-top` - A compilation in-depth explanations on various topics of interest. 23 | - :ref:`reference-top` - The reference to the classes, functions, constants that can be used. 24 | 25 | 26 | .. seealso:: 27 | - JSON RPC 2.0 specifications: http://www.jsonrpc.org/specification 28 | - Tornado: http://www.tornadoweb.org/en/stable/ 29 | 30 | 31 | Installation 32 | ------------ 33 | 34 | :: 35 | 36 | pip install gemstone 37 | # or 38 | pip install gemstone[redis] # to use the Redis features 39 | # or 40 | pip install gemstone[rabbitmq] # to use the RabbitMq features 41 | 42 | 43 | First look 44 | ---------- 45 | 46 | In a script ``hello_world.py`` write the following: 47 | 48 | :: 49 | 50 | import gemstone.core 51 | 52 | class HelloWorldService(gemstone.core.MicroService): 53 | name = "hello_world_service" 54 | host = "127.0.0.1" 55 | port = 8000 56 | 57 | @gemstone.core.exposed_method() 58 | def say_hello(self, name): 59 | return "hello {}".format(name) 60 | 61 | if __name__ == '__main__': 62 | service = HelloWorldService() 63 | service.start() 64 | 65 | We have now a microservice that exposes a public method ``say_hello`` and returns 66 | a ``"hello {name}"``. 67 | 68 | What we did is the following: 69 | 70 | - declared the class of our microservice by inheriting :py:class:`gemstone.MicroService` 71 | - assigned a name for our service (this is required) 72 | - assigned the ``host`` and the ``port`` where the microservice should listen 73 | - exposed a method by using the :py:func:`gemstone.exposed_method` decorator. 74 | - after that, when the script is directly executed, we start the service by calling 75 | the :py:meth:`gemstone.MicroService.start` method. 76 | 77 | To run it, run script 78 | 79 | :: 80 | 81 | python hello_world.py 82 | 83 | Now we have the service listening on ``http://localhost:8000/api`` (the default configuration 84 | for the URL endpoint). In order to test it, you have to do a HTTP ``POST`` request to 85 | that URL with the content: 86 | 87 | :: 88 | 89 | curl -i -X POST \ 90 | -H "Content-Type:application/json" \ 91 | -d '{"jsonrpc": "2.0","id": 1,"method": "say_hello","params": {"name": "world"}}' \ 92 | 'http://localhost:8000/api' 93 | 94 | The answer should be 95 | 96 | :: 97 | 98 | {"result": "hello world", "error": null, "jsonrpc": "2.0", "id": 1} 99 | 100 | 101 | 102 | Table of contents: 103 | 104 | .. toctree:: 105 | :maxdepth: 2 106 | 107 | topics/index 108 | reference/modules 109 | changes 110 | 111 | 112 | .. todoList:: 113 | 114 | Indices and tables 115 | ================== 116 | 117 | * :ref:`genindex` 118 | * :ref:`modindex` 119 | * :ref:`search` 120 | 121 | -------------------------------------------------------------------------------- /docs/topics/rpc.rst: -------------------------------------------------------------------------------- 1 | .. _rpc-communication: 2 | 3 | RPC communication via JSON RPC 2.0 4 | ================================== 5 | 6 | .. note:: 7 | 8 | Check out the `JSONRPC 2.0 protocol specifications `_ . 9 | 10 | The implementation 11 | ------------------ 12 | 13 | The RPC functionality is provided by the :py:class:`gemstone.TornadoJsonRpcHandler`. 14 | It is important to note that the blocking methods (that are not coroutines) 15 | are not executed in the main thread, but in a ``concurrent.features.ThreadPoolExecutor``. 16 | The methods that are coroutines are executed on the main thread. 17 | 18 | In order to create a basic microservice, you have to create a class that inherits the 19 | :py:class:`gemstone.MicroService` class as follows 20 | 21 | :: 22 | 23 | import gemstone 24 | 25 | class MyMicroService(gemstone.MicroService): 26 | 27 | name = "hello_world_service" 28 | ... 29 | 30 | Check out the :py:class:`gemstone.MicroService` documentation for the available attributes 31 | 32 | Public methods 33 | -------------- 34 | 35 | Any method decorated with :py:func:``gemstone.exposed_method`` is a public method that 36 | can be accessed by anybody. 37 | 38 | Example exposed method: 39 | 40 | :: 41 | 42 | class MyMicroService(gemstone.MicroService): 43 | # ... 44 | 45 | @gemstone.exposed_method() 46 | def say_hello(self, world): 47 | return "hello {}".format(world) 48 | 49 | # ... 50 | 51 | By default, a public method is blocking and will be executed in a threaded executor. 52 | 53 | You can make an exposed method a coroutine as shown in the next example. From a coroutine you can 54 | call other coroutines and can call blocking function by using the provided executor 55 | (:py:meth:`gemstone.MicroService.get_executor`) 56 | 57 | :: 58 | 59 | class MyMicroService(gemstone.MicroService): 60 | # ... 61 | 62 | @gemstone.exposed_method(is_coroutine=True) 63 | def say_hello_coroutine(self, world): 64 | yield self.get_executor.submit(time.sleep, 3) 65 | return "hello {}".format(world) 66 | 67 | # ... 68 | 69 | You can expose the public method under another name by giving the desired name as parameter 70 | to the :py:func:`gemstone.exposed_method` decorator. The new name can even use dots! 71 | 72 | :: 73 | 74 | class MyMicroService(gemstone.MicroService): 75 | # ... 76 | 77 | @gemstone.exposed_method("myservice.say_hello") 78 | def ugly_name(self, world): 79 | return "hello {}".format(world) 80 | 81 | # ... 82 | 83 | 84 | .. _private_methods: 85 | 86 | Private methods 87 | --------------- 88 | 89 | TODO 90 | 91 | Interacting with the microservice 92 | --------------------------------- 93 | 94 | Interaction with the microservice can be done by using the JSON RPC 2.0 protocol over HTTP. 95 | 96 | 97 | .. _interacting_with_another_microservice: 98 | 99 | Interacting with another microservice 100 | ------------------------------------- 101 | 102 | Interaction with another microservice can be done by using the :py:class:`gemstone.RemoteService` 103 | class. If at least one service registry was configured, you can use the 104 | :py:meth:`gemstone.MicroService.get_service` method. 105 | 106 | Or, as an alternative, if you know the exact network location of the remote service, you can 107 | instantiate a :py:class:`gemstone.RemoteService` yourself 108 | 109 | :: 110 | 111 | remote_service = gemstone.RemoteService("http://10.0.0.1:8000/api") 112 | r = remote_service.call_method("say_hello", ("world",)) 113 | print(r.result) 114 | # "hello world" 115 | 116 | FAQ 117 | --- 118 | 119 | TODO 120 | -------------------------------------------------------------------------------- /gemstone/client/structs.py: -------------------------------------------------------------------------------- 1 | import string 2 | import random 3 | 4 | 5 | class MethodCall(object): 6 | def __init__(self, method_name, params=None, id=None): 7 | self.method_name = method_name 8 | self.params = params or {} 9 | self.id = id or self._generate_id() 10 | 11 | def _generate_id(self): 12 | return "".join([random.choice(string.ascii_letters) for _ in range(10)]) 13 | 14 | def __repr__(self): 15 | return "MethodCall(id={}, method_name={}, params={})".format(self.id, self.method_name, 16 | self.params) 17 | 18 | def __hash__(self): 19 | return hash(self.id) 20 | 21 | def __eq__(self, other): 22 | if not isinstance(other, MethodCall): 23 | return False 24 | 25 | return hash(self) == hash(other) 26 | 27 | 28 | class Notification(object): 29 | def __init__(self, method_name, params=None, id=None): 30 | self.method_name = method_name 31 | self.params = params or {} 32 | self.id = None 33 | 34 | def __repr__(self): 35 | return "Notification(method_name={}, params={})".format(self.method_name, self.params) 36 | 37 | 38 | class Result(object): 39 | def __init__(self, result, error, id, method_call): 40 | self.result = result 41 | self.error = error 42 | self.id = id 43 | self.method_call = method_call 44 | 45 | def __repr__(self): 46 | return "Response(result={}, error={}, id={}, method_call={})".format( 47 | self.result, self.error, self.id, self.method_call 48 | ) 49 | 50 | 51 | class BatchResult(object): 52 | def __init__(self, *responses): 53 | self.responses = list(responses) 54 | 55 | def add_response(self, response): 56 | self.responses.append(response) 57 | 58 | def __iter__(self): 59 | return iter(self.responses) 60 | 61 | def __len__(self): 62 | return len(self.responses) 63 | 64 | def get_response_for_call(self, method_call): 65 | items = [x for x in self.responses if x.method_call == method_call] 66 | if not items: 67 | return None 68 | else: 69 | return items[0] 70 | 71 | 72 | class AsyncMethodCall(object): 73 | def __init__(self, req_obj, async_resp_object): 74 | self.request = req_obj 75 | self._async_resp = async_resp_object 76 | 77 | def finished(self): 78 | return self._async_resp.ready() 79 | 80 | def result(self, wait=False): 81 | """ 82 | Gets the result of the method call. If the call was successful, 83 | return the result, otherwise, reraise the exception. 84 | 85 | :param wait: Block until the result is available, or just get the result. 86 | :raises: RuntimeError when called and the result is not yet available. 87 | """ 88 | if wait: 89 | self._async_resp.wait() 90 | 91 | if not self.finished(): 92 | raise RuntimeError("Result is not ready yet") 93 | 94 | raw_response = self._async_resp.get() 95 | 96 | return Result(result=raw_response["result"], error=raw_response["error"], 97 | id=raw_response["id"], method_call=self.request) 98 | 99 | def successful(self): 100 | return self._async_resp.successful() 101 | 102 | def __repr__(self): 103 | return "AsyncMethodCall(ready={}, request={})".format( 104 | self.finished(), self.request 105 | ) 106 | 107 | def __hash__(self): 108 | return hash(self.request) 109 | 110 | def __eq__(self, other): 111 | if not isinstance(other, AsyncMethodCall): 112 | return False 113 | 114 | return self.request == other.request 115 | -------------------------------------------------------------------------------- /tests/test_client.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import urllib.request 3 | import json 4 | 5 | import pytest 6 | 7 | from gemstone.util import as_completed 8 | from gemstone.client.remote_service import RemoteService 9 | from gemstone.client.structs import Result, MethodCall, BatchResult, AsyncMethodCall 10 | 11 | DUMMY_SERVICE_URL = "http://example.com/api" 12 | 13 | 14 | def test_remote_service_initialize(): 15 | service = RemoteService(DUMMY_SERVICE_URL) 16 | 17 | assert service.url == DUMMY_SERVICE_URL 18 | 19 | 20 | def test_remote_service_make_request_obj(): 21 | service = RemoteService(DUMMY_SERVICE_URL) 22 | 23 | body = { 24 | "test": "ok" 25 | } 26 | 27 | req_obj = service.build_http_request_obj(body) 28 | 29 | assert isinstance(req_obj, urllib.request.Request) 30 | assert req_obj.get_full_url() == DUMMY_SERVICE_URL 31 | assert req_obj.method == "POST" 32 | 33 | assert req_obj.get_header("Content-Type".capitalize()) == 'application/json' 34 | assert req_obj.data == b'{"test": "ok"}' 35 | 36 | 37 | def dummy_urlopen(url, *args, **kwargs): 38 | class DummyResponse: 39 | def __init__(self, id=None, *a, **k): 40 | self.id = id 41 | 42 | def read(self): 43 | return json.dumps( 44 | {"jsonrpc": "2.0", "error": None, "result": None, "id": self.id}).encode() 45 | 46 | if not isinstance(url, urllib.request.Request): 47 | return DummyResponse() 48 | 49 | body = json.loads(url.data.decode()) 50 | return DummyResponse(body.get('id')) 51 | 52 | 53 | def dummy_urlopen_batch(url, *args, **kwargs): 54 | class DummyResponse: 55 | def __init__(self, *ids): 56 | self.ids = ids 57 | 58 | def read(self): 59 | return json.dumps([{"jsonrpc": "2.0", "error": None, "result": None, "id": i} for i in 60 | self.ids]).encode() 61 | 62 | if not isinstance(url, urllib.request.Request): 63 | return DummyResponse() 64 | 65 | body = json.loads(url.data.decode()) 66 | ids = [x["id"] for x in body] 67 | 68 | return DummyResponse(*ids) 69 | 70 | 71 | def test_simple_call(monkeypatch): 72 | service = RemoteService(DUMMY_SERVICE_URL) 73 | 74 | monkeypatch.setattr(urllib.request, 'urlopen', dummy_urlopen) 75 | 76 | result = service.call_method("test", []) 77 | assert isinstance(result, Result) 78 | assert result.id == result.method_call.id 79 | 80 | 81 | def test_simple_call_notify(monkeypatch): 82 | service = RemoteService(DUMMY_SERVICE_URL) 83 | 84 | monkeypatch.setattr(urllib.request, 'urlopen', dummy_urlopen) 85 | 86 | result = service.notify("test", []) 87 | assert result is None 88 | 89 | 90 | def test_simple_batch_call(monkeypatch): 91 | service = RemoteService(DUMMY_SERVICE_URL) 92 | monkeypatch.setattr(urllib.request, 'urlopen', dummy_urlopen_batch) 93 | 94 | calls = [ 95 | MethodCall("test", []), 96 | MethodCall("test2", []), 97 | MethodCall("test3", []) 98 | ] 99 | result = service.call_batch(*calls) 100 | 101 | assert isinstance(result, BatchResult) 102 | assert len(result) == 3 103 | assert result.get_response_for_call(calls[0]).id == calls[0].id 104 | assert result.get_response_for_call(calls[1]).id == calls[1].id 105 | assert result.get_response_for_call(calls[2]).id == calls[2].id 106 | assert result.get_response_for_call(MethodCall("invalid")) is None 107 | 108 | 109 | def test_async_call(monkeypatch): 110 | service = RemoteService(DUMMY_SERVICE_URL) 111 | monkeypatch.setattr(urllib.request, 'urlopen', dummy_urlopen) 112 | 113 | result = service.call_method_async("test", []) 114 | assert isinstance(result, AsyncMethodCall) 115 | 116 | result = result.result(wait=True) 117 | 118 | assert isinstance(result, Result) 119 | assert result.id == result.method_call.id 120 | 121 | 122 | def test_batch_call_errors(): 123 | service = RemoteService(DUMMY_SERVICE_URL) 124 | 125 | with pytest.raises(TypeError): 126 | service.call_batch(1, 2, 3) 127 | 128 | 129 | def test_as_completed(monkeypatch): 130 | service = RemoteService(DUMMY_SERVICE_URL) 131 | monkeypatch.setattr(urllib.request, 'urlopen', dummy_urlopen) 132 | 133 | items = [service.call_method_async("test", []) for _ in range(10)] 134 | 135 | data = as_completed(*items) 136 | 137 | assert inspect.isgenerator(data) 138 | 139 | results = list(data) 140 | assert len(results) == 10 141 | -------------------------------------------------------------------------------- /tests/test_client_functional.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import urllib.request 3 | import json 4 | import contextlib 5 | import threading 6 | import time 7 | 8 | import pytest 9 | 10 | from gemstone.util import first_completed, as_completed 11 | from gemstone.core import MicroService, exposed_method 12 | from gemstone.client.remote_service import RemoteService 13 | from gemstone.client.structs import Result, MethodCall, BatchResult, AsyncMethodCall, Notification 14 | 15 | 16 | class TestMicroservice(MicroService): 17 | name = "test_service" 18 | 19 | host = "127.0.0.1" 20 | port = 9019 21 | 22 | @exposed_method() 23 | def sum(self, a, b): 24 | return a + b 25 | 26 | @exposed_method() 27 | def divide(self, a, b): 28 | return a / b 29 | 30 | 31 | @pytest.fixture(scope="module") 32 | def microservice_url(): 33 | service = TestMicroservice() 34 | threading.Thread(target=service.start).start() 35 | time.sleep(1) # wait for the service to start 36 | yield service.accessible_at 37 | service.get_io_loop().stop() 38 | 39 | 40 | def test_client_simple_method_call(microservice_url): 41 | client = RemoteService(microservice_url) 42 | 43 | result = client.call_method("sum", params=[1, 2]) 44 | assert isinstance(result, Result) 45 | assert result.result == 3 46 | 47 | result = client.call_method("sum", params={"a": 1, "b": 2}) 48 | assert isinstance(result, Result) 49 | assert result.result == 3 50 | 51 | 52 | def test_client_simple_method_call_with_errors(microservice_url): 53 | client = RemoteService(microservice_url) 54 | 55 | # too few positional args 56 | result = client.call_method("sum", params=[1]) 57 | assert isinstance(result, Result) 58 | assert result.result is None 59 | assert result.error["code"] == -32602 60 | 61 | # too many positional args 62 | result = client.call_method("sum", params=[1, 2, 3]) 63 | assert isinstance(result, Result) 64 | assert result.result is None 65 | assert result.error["code"] == -32602 66 | 67 | # too few kw args 68 | result = client.call_method("sum", params={"a": 1}) 69 | assert isinstance(result, Result) 70 | assert result.result is None 71 | assert result.error["code"] == -32602 72 | 73 | # too many kw args 74 | result = client.call_method("sum", params={"a": 1, "b": 2, "c": 3}) 75 | assert isinstance(result, Result) 76 | assert result.result is None 77 | assert result.error["code"] == -32602 78 | 79 | # method not found 80 | result = client.call_method("invalid", params={"a": 1, "b": 2, "c": 3}) 81 | assert isinstance(result, Result) 82 | assert result.result is None 83 | assert result.error["code"] == -32601 84 | 85 | # internal error 86 | result = client.call_method("sum", params=[None, 3]) 87 | assert isinstance(result, Result) 88 | assert result.result is None 89 | assert result.error["code"] == -32603 90 | 91 | 92 | def test_client_simple_method_call_with_objects(microservice_url): 93 | client = RemoteService(microservice_url) 94 | 95 | req = MethodCall("sum", [1, 2]) 96 | result = client.call_method(req) 97 | assert isinstance(result, Result) 98 | assert result.result == 3 99 | 100 | req = MethodCall("sum", {"a": 1, "b": 2}) 101 | result = client.call_method(req) 102 | assert isinstance(result, Result) 103 | assert result.result == 3 104 | 105 | 106 | def test_client_batch_call(microservice_url): 107 | client = RemoteService(microservice_url) 108 | 109 | requests = [ 110 | MethodCall("sum", [1, 2]), 111 | MethodCall("divide", [10, 5]), 112 | MethodCall("sum", [10, -10]), 113 | MethodCall("sum", ["hello", " world"]), 114 | MethodCall("sum", [1, 2, 3]), # error 115 | Notification("sum", [1, 2]) 116 | ] 117 | resp = client.call_batch(*requests) 118 | assert len(resp) == 5 119 | assert resp.get_response_for_call(requests[0]).result == 3 120 | 121 | assert resp.get_response_for_call(requests[1]).result == 2. 122 | 123 | assert resp.get_response_for_call(requests[2]).result == 0 124 | 125 | assert resp.get_response_for_call(requests[3]).result == "hello world" 126 | 127 | assert resp.get_response_for_call(requests[5]) is None # it was a notification 128 | 129 | 130 | def test_client_async_call(microservice_url): 131 | client = RemoteService(microservice_url) 132 | 133 | async_call = client.call_method_async("sum", [1, 2]) 134 | assert isinstance(async_call, AsyncMethodCall) 135 | async_call.result(wait=True) 136 | assert async_call.finished() 137 | assert async_call.result().result == 3 138 | 139 | 140 | def test_client_async_as_completed(microservice_url): 141 | client = RemoteService(microservice_url) 142 | 143 | for ready_result in as_completed( 144 | *[client.call_method_async("sum", [i, i + 1]) for i in range(10)]): 145 | print(ready_result) 146 | assert ready_result.finished() 147 | 148 | 149 | def test_client_async_first_completed(microservice_url): 150 | client = RemoteService(microservice_url) 151 | 152 | res = first_completed(*[client.call_method_async("sum", [i, i + 1]) for i in range(10)]) 153 | 154 | assert isinstance(res, Result) 155 | assert res.error is None 156 | assert isinstance(res.result, int) 157 | assert 1 <= res.result <= 19 158 | -------------------------------------------------------------------------------- /tests/test_structs.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from gemstone.core.structs import JsonRpcParseError, JsonRpcInvalidRequestError, JsonRpcRequest, \ 4 | JsonRpcResponse, JsonRpcRequestBatch, JsonRpcResponseBatch 5 | 6 | 7 | def test_jsonrpc_request_from_dict_ok(): 8 | # complete 9 | jsonrpc_dict = { 10 | "jsonrpc": "2.0", 11 | "method": "test", 12 | "params": {"test": "ok"}, 13 | "id": 1 14 | } 15 | obj = JsonRpcRequest.from_dict(jsonrpc_dict) 16 | 17 | assert obj.method == "test" 18 | assert obj.id == 1 19 | assert obj.params == {"test": "ok"} 20 | 21 | # positional params 22 | jsonrpc_dict = { 23 | "jsonrpc": "2.0", 24 | "method": "test", 25 | "params": [1, 2, 3, 4, 5], 26 | "id": 1 27 | } 28 | obj = JsonRpcRequest.from_dict(jsonrpc_dict) 29 | 30 | assert obj.method == "test" 31 | assert obj.id == 1 32 | assert obj.params == [1, 2, 3, 4, 5] 33 | 34 | # without params 35 | jsonrpc_dict = { 36 | "jsonrpc": "2.0", 37 | "method": "test", 38 | "id": 1 39 | } 40 | obj = JsonRpcRequest.from_dict(jsonrpc_dict) 41 | 42 | assert obj.method == "test" 43 | assert obj.id == 1 44 | assert obj.params == {} 45 | 46 | # without id 47 | jsonrpc_dict = { 48 | "jsonrpc": "2.0", 49 | "method": "test", 50 | "params": {"test": "ok"}, 51 | } 52 | obj = JsonRpcRequest.from_dict(jsonrpc_dict) 53 | 54 | assert obj.method == "test" 55 | assert obj.id is None 56 | assert obj.params == {"test": "ok"} 57 | 58 | 59 | def test_jsonrpc_request_from_string_ok(): 60 | json1 = '{"jsonrpc": "2.0","method": "test","params": {"test": "ok"},"id": 1}' 61 | json2 = '{"jsonrpc": "2.0","method": "test","params": [1,2,3,4,5],"id": 1}' 62 | json3 = '{"jsonrpc": "2.0","method": "test","id": 1}' 63 | json4 = '{"jsonrpc": "2.0","method": "test","params": {"test": "ok"}}' 64 | 65 | json1_obj = JsonRpcRequest.from_string(json1) 66 | assert json1_obj.method == "test" 67 | assert json1_obj.id == 1 68 | assert json1_obj.params == {"test": "ok"} 69 | 70 | json2_obj = JsonRpcRequest.from_string(json2) 71 | assert json2_obj.method == "test" 72 | assert json2_obj.id == 1 73 | assert json2_obj.params == [1, 2, 3, 4, 5] 74 | 75 | json3_obj = JsonRpcRequest.from_string(json3) 76 | assert json3_obj.method == "test" 77 | assert json3_obj.id == 1 78 | assert json3_obj.params == {} 79 | 80 | json4_obj = JsonRpcRequest.from_string(json4) 81 | assert json4_obj.method == "test" 82 | assert json4_obj.id is None 83 | assert json4_obj.params == {"test": "ok"} 84 | 85 | 86 | def test_jsonrpc_request_from_dict_fail(): 87 | # no protocol version 88 | jsonrpc_dict = { 89 | "method": "test", 90 | "params": {"test": "ok"}, 91 | "id": 1 92 | } 93 | 94 | with pytest.raises(JsonRpcInvalidRequestError): 95 | JsonRpcRequest.from_dict(jsonrpc_dict) 96 | 97 | # no method 98 | jsonrpc_dict = { 99 | "jsonrpc": "2.0", 100 | "params": {"test": "ok"}, 101 | "id": 1 102 | } 103 | 104 | with pytest.raises(JsonRpcInvalidRequestError): 105 | JsonRpcRequest.from_dict(jsonrpc_dict) 106 | 107 | # wrong params type (string) 108 | jsonrpc_dict = { 109 | "jsonrpc": "2.0", 110 | "method": "test", 111 | "params": "wrong", 112 | "id": 1 113 | } 114 | 115 | with pytest.raises(JsonRpcInvalidRequestError): 116 | JsonRpcRequest.from_dict(jsonrpc_dict) 117 | 118 | # wrong params type (bool) 119 | jsonrpc_dict = { 120 | "jsonrpc": "2.0", 121 | "method": "test", 122 | "params": True, 123 | "id": 1 124 | } 125 | 126 | with pytest.raises(JsonRpcInvalidRequestError): 127 | JsonRpcRequest.from_dict(jsonrpc_dict) 128 | 129 | # wrong id type (bool) 130 | jsonrpc_dict = { 131 | "jsonrpc": "2.0", 132 | "method": "test", 133 | "params": "wrong", 134 | "id": False 135 | } 136 | 137 | with pytest.raises(JsonRpcInvalidRequestError): 138 | JsonRpcRequest.from_dict(jsonrpc_dict) 139 | 140 | 141 | def test_jsonrpc_request_from_string_fail(): 142 | # invalid json structure 143 | json1 = '{"jsonrpc": "2.0","method": "test"]' 144 | 145 | with pytest.raises(JsonRpcParseError): 146 | JsonRpcRequest.from_string(json1) 147 | 148 | # keys without " 149 | json2 = '{jsonrpc: "2.0",method: "test",params: [1,2,3,4,5],id: 1}' 150 | 151 | with pytest.raises(JsonRpcParseError): 152 | JsonRpcRequest.from_string(json2) 153 | 154 | 155 | def test_jsonrpc_request_batch_valid(): 156 | json_items = [ 157 | { 158 | "jsonrpc": "2.0", 159 | "method": "test", 160 | "params": {"a": 1, "b": 2}, 161 | "id": 1 162 | }, 163 | { 164 | "jsonrpc": "2.0", 165 | "method": "test", 166 | "params": {"a": 3, "b": 4}, 167 | "id": 2 168 | }, 169 | { 170 | "jsonrpc": "2.0", 171 | "method": "test", 172 | "params": {"a": 5, "b": 6}, 173 | "id": 3 174 | } 175 | ] 176 | 177 | parsed = JsonRpcRequestBatch.from_json_list(json_items) 178 | assert len(parsed.items) == 3 179 | assert [x.id for x in parsed.iter_items()] == [1, 2, 3] 180 | assert set([x.method for x in parsed.iter_items()]) == {"test"} 181 | assert [x.params for x in parsed.iter_items()] == [{"a": 1, "b": 2}, {"a": 3, "b": 4}, {"a": 5, "b": 6}] 182 | -------------------------------------------------------------------------------- /gemstone/client/remote_service.py: -------------------------------------------------------------------------------- 1 | import urllib.request 2 | import os 3 | 4 | from multiprocessing.pool import ThreadPool 5 | import simplejson as json 6 | 7 | from gemstone.client.structs import MethodCall, Notification, Result, BatchResult, AsyncMethodCall 8 | from gemstone.errors import CalledServiceError 9 | 10 | 11 | class RemoteService(object): 12 | RESPONSE_CODES = { 13 | -32001: "access_denied", 14 | -32603: "internal_error", 15 | -32601: "method_not_found", 16 | -32602: "invalid_params" 17 | } 18 | 19 | def __init__(self, service_endpoint, *, authentication_method=None): 20 | self.url = service_endpoint 21 | self.authentication_method = authentication_method 22 | self._thread_pool = None 23 | 24 | def _get_thread_pool(self): 25 | # lazily initialized 26 | if not self._thread_pool: 27 | self._thread_pool = ThreadPool(os.cpu_count()) 28 | return self._thread_pool 29 | 30 | def handle_single_request(self, request_object): 31 | """ 32 | Handles a single request object and returns the raw response 33 | 34 | :param request_object: 35 | """ 36 | if not isinstance(request_object, (MethodCall, Notification)): 37 | raise TypeError("Invalid type for request_object") 38 | 39 | method_name = request_object.method_name 40 | params = request_object.params 41 | req_id = request_object.id 42 | 43 | request_body = self.build_request_body(method_name, params, id=req_id) 44 | http_request = self.build_http_request_obj(request_body) 45 | 46 | try: 47 | response = urllib.request.urlopen(http_request) 48 | except urllib.request.HTTPError as e: 49 | raise CalledServiceError(e) 50 | 51 | if not req_id: 52 | return 53 | 54 | response_body = json.loads(response.read().decode()) 55 | return response_body 56 | 57 | def build_request_body(self, method_name, params, id=None): 58 | request_body = { 59 | "jsonrpc": "2.0", 60 | "method": method_name, 61 | "params": params 62 | 63 | } 64 | if id: 65 | request_body['id'] = id 66 | return request_body 67 | 68 | def build_http_request_obj(self, request_body): 69 | request = urllib.request.Request(self.url) 70 | request.add_header("Content-Type", "application/json") 71 | request.add_header("User-Agent", "gemstone-client") 72 | request.data = json.dumps(request_body).encode() 73 | request.method = "POST" 74 | return request 75 | 76 | def call_method(self, method_name_or_object, params=None): 77 | """ 78 | Calls the ``method_name`` method from the given service and returns a 79 | :py:class:`gemstone.client.structs.Result` instance. 80 | 81 | :param method_name_or_object: The name of te called method or a ``MethodCall`` instance 82 | :param params: A list of dict representing the parameters for the request 83 | :return: a :py:class:`gemstone.client.structs.Result` instance. 84 | """ 85 | if isinstance(method_name_or_object, MethodCall): 86 | req_obj = method_name_or_object 87 | else: 88 | req_obj = MethodCall(method_name_or_object, params) 89 | raw_response = self.handle_single_request(req_obj) 90 | response_obj = Result(result=raw_response["result"], error=raw_response['error'], 91 | id=raw_response["id"], method_call=req_obj) 92 | return response_obj 93 | 94 | def call_method_async(self, method_name_or_object, params=None): 95 | """ 96 | Calls the ``method_name`` method from the given service asynchronously 97 | and returns a :py:class:`gemstone.client.structs.AsyncMethodCall` instance. 98 | 99 | :param method_name_or_object: The name of te called method or a ``MethodCall`` instance 100 | :param params: A list of dict representing the parameters for the request 101 | :return: a :py:class:`gemstone.client.structs.AsyncMethodCall` instance. 102 | """ 103 | thread_pool = self._get_thread_pool() 104 | 105 | if isinstance(method_name_or_object, MethodCall): 106 | req_obj = method_name_or_object 107 | else: 108 | req_obj = MethodCall(method_name_or_object, params) 109 | 110 | async_result_mp = thread_pool.apply_async(self.handle_single_request, args=(req_obj,)) 111 | return AsyncMethodCall(req_obj=req_obj, async_resp_object=async_result_mp) 112 | 113 | def notify(self, method_name_or_object, params=None): 114 | """ 115 | Sends a notification to the service by calling the ``method_name`` 116 | method with the ``params`` parameters. Does not wait for a response, even 117 | if the response triggers an error. 118 | 119 | :param method_name_or_object: the name of the method to be called or a ``Notification`` 120 | instance 121 | :param params: a list of dict representing the parameters for the call 122 | :return: None 123 | """ 124 | if isinstance(method_name_or_object, Notification): 125 | req_obj = method_name_or_object 126 | else: 127 | req_obj = Notification(method_name_or_object, params) 128 | self.handle_single_request(req_obj) 129 | 130 | def call_batch(self, *requests): 131 | body = [] 132 | ids = {} 133 | for item in requests: 134 | if not isinstance(item, (MethodCall, Notification)): 135 | raise TypeError("Invalid type for batch item: {}".format(item)) 136 | 137 | body.append(self.build_request_body( 138 | method_name=item.method_name, 139 | params=item.params or {}, 140 | id=item.id 141 | )) 142 | if isinstance(item, MethodCall): 143 | ids[body[-1]["id"]] = item 144 | 145 | results = self.handle_batch_request(body) 146 | 147 | batch_result = BatchResult() 148 | for result in results: 149 | result_obj = Result(result["result"], result["error"], result["id"], 150 | method_call=ids[result["id"]]) 151 | batch_result.add_response(result_obj) 152 | return batch_result 153 | 154 | def handle_batch_request(self, body): 155 | request = self.build_http_request_obj(body) 156 | 157 | try: 158 | response = urllib.request.urlopen(request) 159 | except urllib.request.HTTPError as e: 160 | raise CalledServiceError(e) 161 | 162 | resp_body = json.loads(response.read().decode()) 163 | return resp_body 164 | -------------------------------------------------------------------------------- /docs/changes.rst: -------------------------------------------------------------------------------- 1 | Changes 2 | ======= 3 | 4 | 0.12.0 (22.04.2017) 5 | ~~~~~~~~~~~~~~~~~~~ 6 | 7 | - restructured modules 8 | - bug fixes 9 | - improved documentation 10 | - improved tests 11 | 12 | 0.11.0 (08.04.2017) 13 | ~~~~~~~~~~~~~~~~~~~ 14 | 15 | - added ``Container.get_io_loop`` method 16 | - added ``Container.get_executor`` method 17 | - added ``RedisEventTransport`` 18 | - ``emit_event`` now emits just events. Removed the ``broadcast`` parameter. 19 | A task handling functionality will be added in a further version 20 | - improved docs (still a work in progress) 21 | - added some more tests (still a work in progress) 22 | 23 | 24 | 0.10.1 (27.03.2017) 25 | ~~~~~~~~~~~~~~~~~~~ 26 | 27 | - removed some forgotten debug messages 28 | 29 | 30 | 0.10.0 (23.03.2017) 31 | ~~~~~~~~~~~~~~~~~~~ 32 | 33 | - added ``broadcast`` parameter to ``MicroService.emit_event`` 34 | - added the ``broadcast`` parameter to ``BaseEventTransport.emit_event`` 35 | - added the ``broadcast`` parameter to ``RabbitMqEventTransport.emit_event`` 36 | - improved tests and documentation 37 | - removed ``mappings`` and ``type`` parameters from ``Configurable`` 38 | - added ``gemstone.Module`` for better modularization of the microservice 39 | - added ``gemstone.MicroService.authenticate_request`` method for a more flexible 40 | authentication mechanism 41 | - deprecated ``gemstone.MicroService.api_token_is_valid`` method 42 | 43 | 0.9.0 (06.03.2017 44 | ~~~~~~~~~~~~~~~~~ 45 | 46 | - added the ``gemstone.exposed_method`` decorator for general usage that allows 47 | - to customize the name of the method 48 | - to specify if the method is a coroutine 49 | - to specify that the method requires a handler reference 50 | - to specify that the method is public or private 51 | - deprecated 52 | - ``gemstone.public_method`` decorator 53 | - ``gemstone.private_api_method`` decorator 54 | - ``gemstone.async_method`` decorator 55 | - ``gemstone.requires_handler_reference`` decorator 56 | - removed ``gemstone.MicroService.get_cli`` method in favor of the ``CommandLineConfigurator`` 57 | - improved documentation a little bit 58 | 59 | 0.8.0 (05.03.2017) 60 | ~~~~~~~~~~~~~~~~~~ 61 | 62 | - added the ``gemstone.requires_handler_reference`` decorator to enable 63 | the methods to get a reference to the Tornado request handler when called. 64 | - added the ``gemstone.async_method`` decorator to make a method a coroutine 65 | and be able to execute things asynchronously on the main thread. 66 | For example, a method decorated with ``async_method`` will be able to 67 | ``yield self._executor.submit(make_some_network_call)`` without blocking the main 68 | thread. 69 | - added two new examples: 70 | - ``example_coroutine_method`` - shows a basic usage if the ``async_method`` decorator 71 | - ``example_handler_ref`` - shows a basic usage if the ``requires_handler_reference`` decorator 72 | 73 | 74 | 0.7.0 (27.02.2017) 75 | ~~~~~~~~~~~~~~~~~~ 76 | 77 | - added ``gemstone.GemstoneCustomHandler`` class 78 | - modified the way one can add custom Tornado handler to the microservice. 79 | Now these handlers must inherit ``gemstone.GemstoneCustomHandler`` 80 | - restructured docs, now it is based more on docstrings 81 | - improved tests and code quality 82 | 83 | 0.6.0 (14.02.2017) 84 | ~~~~~~~~~~~~~~~~~~ 85 | 86 | - added configurable framework: 87 | - ``gemstone.config.configurable.Configurable`` class 88 | - ``gemstone.config.configurator.*`` classes 89 | - ``gemstone.MicroService.configurables`` and ``gemstone.MicroService.configurators`` attributes 90 | - switched testing to pytest 91 | - improved documentation (restructured and minor additions). Still a work in progress 92 | 93 | 94 | 95 | 0.5.0 (09.02.2017) 96 | ~~~~~~~~~~~~~~~~~~ 97 | 98 | - added support for publisher-subscriber communication method: 99 | - base class for event transports: ``gemstone.event.transport.BaseEventTransport`` 100 | - first concrete implementation: ``gemstone.event.transport.RabbitMqEventTransport`` 101 | - ``gemstone.MicroService.emit_event`` for publishing an event 102 | - ``gemstone.event_handler`` decorator for designating event handlers 103 | - restructured documentation (added tutorial, examples and howto sections). 104 | - added asynchronous method calls in ``gemstone.RemoteService``. 105 | - added ``gemstone.as_completed``, ``gemstone.first_completed``, ``gemstone.make_callbacks`` 106 | utility functions for dealing with asynchronous method calls. 107 | 108 | 109 | 0.4.0 (25.01.2017) 110 | ~~~~~~~~~~~~~~~~~~ 111 | 112 | - modified ``accessible_at`` attribute of the ``gemstone.MicroService`` class 113 | - added the ``endpoint`` attribute to the ``gemstone.MicroService`` class 114 | - improved how the microservice communicates with the service registry 115 | 116 | 0.3.1 (25.01.2017) 117 | ~~~~~~~~~~~~~~~~~~ 118 | 119 | - fixed event loop freezing on Windows 120 | - fixed a case when a ``TypeError`` was silenced when handling the bad parameters error 121 | in JSON RPC 2.0 handler (#21) 122 | - major refactoring (handling of JSON RPC objects as Python objects instead of dicts and lists) 123 | to improve readability and maintainability 124 | - improved documentation 125 | 126 | 0.3.0 (23.01.2017) 127 | ~~~~~~~~~~~~~~~~~~ 128 | - added validation strategies (method for extraction of api token from the request) 129 | - base subclass for implementing validation strategies 130 | - built in validation strategies: ``HeaderValidationStrategy``, ``BasicCookieStrategy`` 131 | - improved documentation 132 | 133 | 134 | 0.2.0 (17.01.2017) 135 | ~~~~~~~~~~~~~~~~~~ 136 | 137 | - added ``gemstone.RemoteService.get_service_by_name`` method 138 | - added ``call`` command to cli 139 | - added ``call_raw`` command to cli 140 | - improved documentation a little 141 | 142 | 0.1.3 (16.01.2017) 143 | ~~~~~~~~~~~~~~~~~~ 144 | 145 | - fixed manifest to include required missing files 146 | 147 | 0.1.2 (16.01.2017) 148 | ~~~~~~~~~~~~~~~~~~ 149 | 150 | - added py36 to travis-ci 151 | - refactored setup.py and reworked description files and documentation for better rendering 152 | 153 | 0.1.1 (13.01.2017) 154 | ~~~~~~~~~~~~~~~~~~ 155 | 156 | - changed the name of the library from ``pymicroservice`` to ``gemstone`` 157 | - added the ``gemstone.MicroService.accessible_at`` attribute 158 | 159 | 0.1.0 (09.01.2017) 160 | ~~~~~~~~~~~~~~~~~~ 161 | 162 | - added the ``pymicroservice.PyMicroService.get_cli`` method 163 | - improved documentation a little bit 164 | 165 | 0.0.4 166 | ~~~~~ 167 | 168 | - fixed bug when sending a notification that would result in an error 169 | was causing the microservice to respond abnormally (see #10) 170 | - fixed a bug that was causing the service to never respond with the 171 | invalid parameters status when calling a method with invalid parameters 172 | 173 | 0.0.3 174 | ~~~~~ 175 | 176 | - added ``pymicroservice.RemoteService`` class 177 | - added the ``pymicroservice.PyMicroService.get_service(name)`` 178 | - improved documentation 179 | -------------------------------------------------------------------------------- /gemstone/core/structs.py: -------------------------------------------------------------------------------- 1 | import simplejson as json 2 | 3 | 4 | class JsonRpcError(Exception): 5 | pass 6 | 7 | 8 | class JsonRpcParseError(JsonRpcError): 9 | pass 10 | 11 | 12 | class JsonRpcInvalidRequestError(JsonRpcError): 13 | pass 14 | 15 | 16 | class JsonRpcRequest(object): 17 | def __init__(self, method=None, params=None, id=None, extra=None): 18 | self.method = method 19 | self.params = params or {} 20 | self.id = id 21 | self.extra = extra or {} 22 | self.invalid = False 23 | 24 | def to_dict(self): 25 | to_ret = { 26 | "jsonrpc": "2.0", 27 | "method": self.method, 28 | "params": self.params, 29 | "id": self.id 30 | } 31 | for k, v in self.extra.items(): 32 | to_ret[k] = v 33 | return to_ret 34 | 35 | def to_string(self): 36 | return json.dumps(self.to_dict()) 37 | 38 | @classmethod 39 | def from_string(cls, string): 40 | try: 41 | return cls.from_dict(json.loads(string)) 42 | except json.JSONDecodeError: 43 | raise JsonRpcParseError() 44 | 45 | @classmethod 46 | def from_dict(cls, d): 47 | """ 48 | Validates a dict instance and transforms it in a 49 | :py:class:`gemstone.core.structs.JsonRpcRequest` 50 | instance 51 | 52 | :param d: The dict instance 53 | :return: A :py:class:`gemstone.core.structs.JsonRpcRequest` 54 | if everything goes well, or None if the validation fails 55 | """ 56 | for key in ("method", "jsonrpc"): 57 | if key not in d: 58 | raise JsonRpcInvalidRequestError() 59 | 60 | # check jsonrpc version 61 | jsonrpc = d.get("jsonrpc", None) 62 | if jsonrpc != "2.0": 63 | raise JsonRpcInvalidRequestError() 64 | 65 | # check method 66 | method = d.get("method", None) 67 | if not method: 68 | raise JsonRpcInvalidRequestError() 69 | if not isinstance(method, str): 70 | raise JsonRpcInvalidRequestError() 71 | 72 | # params 73 | params = d.get("params", {}) 74 | if not isinstance(params, (list, dict)): 75 | raise JsonRpcInvalidRequestError() 76 | 77 | req_id = d.get("id", None) 78 | if not isinstance(req_id, (int, str)) and req_id is not None: 79 | raise JsonRpcInvalidRequestError() 80 | 81 | extras = {k: d[k] for k in d if k not in ("jsonrpc", "id", "method", "params")} 82 | 83 | instance = cls( 84 | id=req_id, 85 | method=method, 86 | params=params, 87 | extra=extras 88 | ) 89 | return instance 90 | 91 | def is_notification(self): 92 | return self.id is None 93 | 94 | def __repr__(self): 95 | return "".format(self.id, self.method, 96 | self.params) 97 | 98 | 99 | class JsonRpcResponse(object): 100 | def __init__(self, result=None, id=None, error=None, send_id_field=False): 101 | self.response = result 102 | self.id = id 103 | self.error = error 104 | self.send_id_field = send_id_field 105 | 106 | def to_dict(self): 107 | to_return = { 108 | "jsonrpc": "2.0", 109 | "result": self.response, 110 | "error": self.error, 111 | } 112 | if self.id or self.send_id_field: 113 | to_return["id"] = self.id 114 | return to_return 115 | 116 | def to_string(self): 117 | return json.dumps(self.to_dict()) 118 | 119 | @classmethod 120 | def from_dict(cls, d): 121 | return cls( 122 | response=d.get("response", None), 123 | error=d.get("error", None), 124 | id=d.get("id", None) 125 | ) 126 | 127 | def __repr__(self): 128 | return "".format(self.id, self.response, 129 | self.error) 130 | 131 | 132 | class JsonRpcRequestBatch(object): 133 | def __init__(self, batch_of_jsonrpc_req): 134 | """ 135 | Makes a json rpc request batch object from a list of 136 | :py:class:`gemstone.core.structs.JSonRpcRequest` objects 137 | 138 | :param batch_of_jsonrpc_req: list of :py:class:`gemstone.core.structs.JSonRpcRequest` 139 | """ 140 | self.items = list(batch_of_jsonrpc_req) 141 | 142 | def add_item(self, item): 143 | """Adds an item to the batch""" 144 | self.items.append(item) 145 | 146 | def to_string(self): 147 | return json.dumps([x.to_string() for x in self.items]) 148 | 149 | @classmethod 150 | def from_json_list(cls, l): 151 | items = [] 152 | for item_raw in l: 153 | items.append(JsonRpcRequest.from_dict(item_raw)) 154 | return cls(items) 155 | 156 | def iter_items(self): 157 | for item in self.items: 158 | yield item 159 | 160 | 161 | class JsonRpcResponseBatch(object): 162 | def __init__(self, batch): 163 | self.items = batch 164 | 165 | def add_item(self, item): 166 | """Adds an item to the batch.""" 167 | 168 | if not isinstance(item, JsonRpcResponse): 169 | raise TypeError( 170 | "Expected JsonRpcResponse but got {} instead".format(type(item).__name__)) 171 | 172 | self.items.append(item) 173 | 174 | def iter_items(self): 175 | for item in self.items: 176 | yield item 177 | 178 | def to_string(self): 179 | return json.dumps([i.to_dict() for i in self.iter_items()]) 180 | 181 | 182 | class GenericResponse: 183 | PARSE_ERROR = JsonRpcResponse(error={"code": -32700, "message": "Parse error"}, 184 | send_id_field=True) 185 | INVALID_REQUEST = JsonRpcResponse(error={"code": -32600, "message": "Invalid Request"}, 186 | send_id_field=True) 187 | METHOD_NOT_FOUND = JsonRpcResponse(error={"code": -32601, "message": "Method not found"}) 188 | INVALID_PARAMS = JsonRpcResponse(error={"code": -32602, "message": "Invalid params"}) 189 | INTERNAL_ERROR = JsonRpcResponse(error={"code": -32603, "message": "Internal error"}) 190 | ACCESS_DENIED = JsonRpcResponse(error={"code": -32001, "message": "Access denied"}) 191 | 192 | NOTIFICATION_RESPONSE = JsonRpcResponse() 193 | 194 | 195 | def parse_json_structure(string_item): 196 | """ 197 | Given a raw representation of a json structure, returns the parsed corresponding data 198 | structure (``JsonRpcRequest`` or ``JsonRpcRequestBatch``) 199 | 200 | :param string_item: 201 | :return: 202 | """ 203 | if not isinstance(string_item, str): 204 | raise TypeError("Expected str but got {} instead".format(type(string_item).__name__)) 205 | 206 | try: 207 | item = json.loads(string_item) 208 | except json.JSONDecodeError: 209 | raise JsonRpcParseError() 210 | 211 | if isinstance(item, dict): 212 | return JsonRpcRequest.from_dict(item) 213 | elif isinstance(item, list): 214 | if len(item) == 0: 215 | raise JsonRpcInvalidRequestError() 216 | 217 | request_batch = JsonRpcRequestBatch([]) 218 | for d in item: 219 | try: 220 | # handles the case of valid batch but with invalid 221 | # requests. 222 | if not isinstance(d, dict): 223 | raise JsonRpcInvalidRequestError() 224 | # is dict, all fine 225 | parsed_entry = JsonRpcRequest.from_dict(d) 226 | except JsonRpcInvalidRequestError: 227 | parsed_entry = GenericResponse.INVALID_REQUEST 228 | request_batch.add_item(parsed_entry) 229 | return request_batch 230 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help 18 | help: 19 | @echo "Please use \`make ' where is one of" 20 | @echo " html to make standalone HTML files" 21 | @echo " dirhtml to make HTML files named index.html in directories" 22 | @echo " singlehtml to make a single large HTML file" 23 | @echo " pickle to make pickle files" 24 | @echo " json to make JSON files" 25 | @echo " htmlhelp to make HTML files and a HTML help project" 26 | @echo " qthelp to make HTML files and a qthelp project" 27 | @echo " applehelp to make an Apple Help Book" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " epub3 to make an epub3" 31 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 32 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 33 | @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" 34 | @echo " text to make text files" 35 | @echo " man to make manual pages" 36 | @echo " texinfo to make Texinfo files" 37 | @echo " info to make Texinfo files and run them through makeinfo" 38 | @echo " gettext to make PO message catalogs" 39 | @echo " changes to make an overview of all changed/added/deprecated items" 40 | @echo " xml to make Docutils-native XML files" 41 | @echo " pseudoxml to make pseudoxml-XML files for display purposes" 42 | @echo " linkcheck to check all external links for integrity" 43 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 44 | @echo " coverage to run coverage check of the documentation (if enabled)" 45 | @echo " dummy to check syntax errors of document sources" 46 | 47 | .PHONY: clean 48 | clean: 49 | rm -rf $(BUILDDIR)/* 50 | 51 | .PHONY: html 52 | html: 53 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 54 | @echo 55 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 56 | 57 | .PHONY: dirhtml 58 | dirhtml: 59 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 60 | @echo 61 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 62 | 63 | .PHONY: singlehtml 64 | singlehtml: 65 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 66 | @echo 67 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 68 | 69 | .PHONY: pickle 70 | pickle: 71 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 72 | @echo 73 | @echo "Build finished; now you can process the pickle files." 74 | 75 | .PHONY: json 76 | json: 77 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 78 | @echo 79 | @echo "Build finished; now you can process the JSON files." 80 | 81 | .PHONY: htmlhelp 82 | htmlhelp: 83 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 84 | @echo 85 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 86 | ".hhp project file in $(BUILDDIR)/htmlhelp." 87 | 88 | .PHONY: qthelp 89 | qthelp: 90 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 91 | @echo 92 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 93 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 94 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pymicroservice.qhcp" 95 | @echo "To view the help file:" 96 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pymicroservice.qhc" 97 | 98 | .PHONY: applehelp 99 | applehelp: 100 | $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp 101 | @echo 102 | @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." 103 | @echo "N.B. You won't be able to view it unless you put it in" \ 104 | "~/Library/Documentation/Help or install it in your application" \ 105 | "bundle." 106 | 107 | .PHONY: devhelp 108 | devhelp: 109 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 110 | @echo 111 | @echo "Build finished." 112 | @echo "To view the help file:" 113 | @echo "# mkdir -p $$HOME/.local/share/devhelp/pymicroservice" 114 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pymicroservice" 115 | @echo "# devhelp" 116 | 117 | .PHONY: epub 118 | epub: 119 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 120 | @echo 121 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 122 | 123 | .PHONY: epub3 124 | epub3: 125 | $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 126 | @echo 127 | @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." 128 | 129 | .PHONY: latex 130 | latex: 131 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 132 | @echo 133 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 134 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 135 | "(use \`make latexpdf' here to do that automatically)." 136 | 137 | .PHONY: latexpdf 138 | latexpdf: 139 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 140 | @echo "Running LaTeX files through pdflatex..." 141 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 142 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 143 | 144 | .PHONY: latexpdfja 145 | latexpdfja: 146 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 147 | @echo "Running LaTeX files through platex and dvipdfmx..." 148 | $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja 149 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 150 | 151 | .PHONY: text 152 | text: 153 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 154 | @echo 155 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 156 | 157 | .PHONY: man 158 | man: 159 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 160 | @echo 161 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 162 | 163 | .PHONY: texinfo 164 | texinfo: 165 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 166 | @echo 167 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 168 | @echo "Run \`make' in that directory to run these through makeinfo" \ 169 | "(use \`make info' here to do that automatically)." 170 | 171 | .PHONY: info 172 | info: 173 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 174 | @echo "Running Texinfo files through makeinfo..." 175 | make -C $(BUILDDIR)/texinfo info 176 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 177 | 178 | .PHONY: gettext 179 | gettext: 180 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 181 | @echo 182 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 183 | 184 | .PHONY: changes 185 | changes: 186 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 187 | @echo 188 | @echo "The overview file is in $(BUILDDIR)/changes." 189 | 190 | .PHONY: linkcheck 191 | linkcheck: 192 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 193 | @echo 194 | @echo "Link check complete; look for any errors in the above output " \ 195 | "or in $(BUILDDIR)/linkcheck/output.txt." 196 | 197 | .PHONY: doctest 198 | doctest: 199 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 200 | @echo "Testing of doctests in the sources finished, look at the " \ 201 | "results in $(BUILDDIR)/doctest/output.txt." 202 | 203 | .PHONY: coverage 204 | coverage: 205 | $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage 206 | @echo "Testing of coverage in the sources finished, look at the " \ 207 | "results in $(BUILDDIR)/coverage/python.txt." 208 | 209 | .PHONY: xml 210 | xml: 211 | $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml 212 | @echo 213 | @echo "Build finished. The XML files are in $(BUILDDIR)/xml." 214 | 215 | .PHONY: pseudoxml 216 | pseudoxml: 217 | $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml 218 | @echo 219 | @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." 220 | 221 | .PHONY: dummy 222 | dummy: 223 | $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy 224 | @echo 225 | @echo "Build finished. Dummy builder generates no files." 226 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=D:\VirtualEnvs\gemstone\Scripts\sphinx-build.exe 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | set I18NSPHINXOPTS=%SPHINXOPTS% . 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. dirhtml to make HTML files named index.html in directories 23 | echo. singlehtml to make a single large HTML file 24 | echo. pickle to make pickle files 25 | echo. json to make JSON files 26 | echo. htmlhelp to make HTML files and a HTML help project 27 | echo. qthelp to make HTML files and a qthelp project 28 | echo. devhelp to make HTML files and a Devhelp project 29 | echo. epub to make an epub 30 | echo. epub3 to make an epub3 31 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 32 | echo. text to make text files 33 | echo. man to make manual pages 34 | echo. texinfo to make Texinfo files 35 | echo. gettext to make PO message catalogs 36 | echo. changes to make an overview over all changed/added/deprecated items 37 | echo. xml to make Docutils-native XML files 38 | echo. pseudoxml to make pseudoxml-XML files for display purposes 39 | echo. linkcheck to check all external links for integrity 40 | echo. doctest to run all doctests embedded in the documentation if enabled 41 | echo. coverage to run coverage check of the documentation if enabled 42 | echo. dummy to check syntax errors of document sources 43 | goto end 44 | ) 45 | 46 | if "%1" == "clean" ( 47 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 48 | del /q /s %BUILDDIR%\* 49 | goto end 50 | ) 51 | 52 | 53 | REM Check if sphinx-build is available and fallback to Python version if any 54 | %SPHINXBUILD% 1>NUL 2>NUL 55 | if errorlevel 9009 goto sphinx_python 56 | goto sphinx_ok 57 | 58 | :sphinx_python 59 | 60 | set SPHINXBUILD=python -m sphinx.__init__ 61 | %SPHINXBUILD% 2> nul 62 | if errorlevel 9009 ( 63 | echo. 64 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 65 | echo.installed, then set the SPHINXBUILD environment variable to point 66 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 67 | echo.may add the Sphinx directory to PATH. 68 | echo. 69 | echo.If you don't have Sphinx installed, grab it from 70 | echo.http://sphinx-doc.org/ 71 | exit /b 1 72 | ) 73 | 74 | :sphinx_ok 75 | 76 | 77 | if "%1" == "html" ( 78 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 79 | if errorlevel 1 exit /b 1 80 | echo. 81 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 82 | goto end 83 | ) 84 | 85 | if "%1" == "dirhtml" ( 86 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 87 | if errorlevel 1 exit /b 1 88 | echo. 89 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 90 | goto end 91 | ) 92 | 93 | if "%1" == "singlehtml" ( 94 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 95 | if errorlevel 1 exit /b 1 96 | echo. 97 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 98 | goto end 99 | ) 100 | 101 | if "%1" == "pickle" ( 102 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 103 | if errorlevel 1 exit /b 1 104 | echo. 105 | echo.Build finished; now you can process the pickle files. 106 | goto end 107 | ) 108 | 109 | if "%1" == "json" ( 110 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 111 | if errorlevel 1 exit /b 1 112 | echo. 113 | echo.Build finished; now you can process the JSON files. 114 | goto end 115 | ) 116 | 117 | if "%1" == "htmlhelp" ( 118 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 119 | if errorlevel 1 exit /b 1 120 | echo. 121 | echo.Build finished; now you can run HTML Help Workshop with the ^ 122 | .hhp project file in %BUILDDIR%/htmlhelp. 123 | goto end 124 | ) 125 | 126 | if "%1" == "qthelp" ( 127 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 128 | if errorlevel 1 exit /b 1 129 | echo. 130 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 131 | .qhcp project file in %BUILDDIR%/qthelp, like this: 132 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pymicroservice.qhcp 133 | echo.To view the help file: 134 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pymicroservice.ghc 135 | goto end 136 | ) 137 | 138 | if "%1" == "devhelp" ( 139 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 140 | if errorlevel 1 exit /b 1 141 | echo. 142 | echo.Build finished. 143 | goto end 144 | ) 145 | 146 | if "%1" == "epub" ( 147 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 148 | if errorlevel 1 exit /b 1 149 | echo. 150 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 151 | goto end 152 | ) 153 | 154 | if "%1" == "epub3" ( 155 | %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3 156 | if errorlevel 1 exit /b 1 157 | echo. 158 | echo.Build finished. The epub3 file is in %BUILDDIR%/epub3. 159 | goto end 160 | ) 161 | 162 | if "%1" == "latex" ( 163 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 164 | if errorlevel 1 exit /b 1 165 | echo. 166 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 167 | goto end 168 | ) 169 | 170 | if "%1" == "latexpdf" ( 171 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 172 | cd %BUILDDIR%/latex 173 | make all-pdf 174 | cd %~dp0 175 | echo. 176 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 177 | goto end 178 | ) 179 | 180 | if "%1" == "latexpdfja" ( 181 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 182 | cd %BUILDDIR%/latex 183 | make all-pdf-ja 184 | cd %~dp0 185 | echo. 186 | echo.Build finished; the PDF files are in %BUILDDIR%/latex. 187 | goto end 188 | ) 189 | 190 | if "%1" == "text" ( 191 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 192 | if errorlevel 1 exit /b 1 193 | echo. 194 | echo.Build finished. The text files are in %BUILDDIR%/text. 195 | goto end 196 | ) 197 | 198 | if "%1" == "man" ( 199 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 200 | if errorlevel 1 exit /b 1 201 | echo. 202 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 203 | goto end 204 | ) 205 | 206 | if "%1" == "texinfo" ( 207 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 208 | if errorlevel 1 exit /b 1 209 | echo. 210 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 211 | goto end 212 | ) 213 | 214 | if "%1" == "gettext" ( 215 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 216 | if errorlevel 1 exit /b 1 217 | echo. 218 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 219 | goto end 220 | ) 221 | 222 | if "%1" == "changes" ( 223 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 224 | if errorlevel 1 exit /b 1 225 | echo. 226 | echo.The overview file is in %BUILDDIR%/changes. 227 | goto end 228 | ) 229 | 230 | if "%1" == "linkcheck" ( 231 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 232 | if errorlevel 1 exit /b 1 233 | echo. 234 | echo.Link check complete; look for any errors in the above output ^ 235 | or in %BUILDDIR%/linkcheck/output.txt. 236 | goto end 237 | ) 238 | 239 | if "%1" == "doctest" ( 240 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 241 | if errorlevel 1 exit /b 1 242 | echo. 243 | echo.Testing of doctests in the sources finished, look at the ^ 244 | results in %BUILDDIR%/doctest/output.txt. 245 | goto end 246 | ) 247 | 248 | if "%1" == "coverage" ( 249 | %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage 250 | if errorlevel 1 exit /b 1 251 | echo. 252 | echo.Testing of coverage in the sources finished, look at the ^ 253 | results in %BUILDDIR%/coverage/python.txt. 254 | goto end 255 | ) 256 | 257 | if "%1" == "xml" ( 258 | %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml 259 | if errorlevel 1 exit /b 1 260 | echo. 261 | echo.Build finished. The XML files are in %BUILDDIR%/xml. 262 | goto end 263 | ) 264 | 265 | if "%1" == "pseudoxml" ( 266 | %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml 267 | if errorlevel 1 exit /b 1 268 | echo. 269 | echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. 270 | goto end 271 | ) 272 | 273 | if "%1" == "dummy" ( 274 | %SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy 275 | if errorlevel 1 exit /b 1 276 | echo. 277 | echo.Build finished. Dummy builder generates no files. 278 | goto end 279 | ) 280 | 281 | :end 282 | -------------------------------------------------------------------------------- /tests/functional/test_jsonrpc_specs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Contains tests that assert the compliance with the 3 | JSON RPC 2.0 specifications (http://www.jsonrpc.org/specification) 4 | """ 5 | 6 | import logging 7 | 8 | import simplejson as json 9 | import pytest 10 | 11 | from tests.services.service_jsonrpc_specs import ServiceJsonRpcSpecs 12 | 13 | 14 | @pytest.fixture 15 | def app(): 16 | service = ServiceJsonRpcSpecs() 17 | service._initial_setup() 18 | return service.make_tornado_app() 19 | 20 | 21 | @pytest.mark.gen_test 22 | def test_incomplete_json(http_client, base_url): 23 | body = json.dumps({"jsonrpc": "2.0", "method": "subtract"})[:15] 24 | result = yield http_client.fetch(base_url + "/api", method="POST", body=body, 25 | headers={"content-type": "application/json"}) 26 | 27 | assert result.code == 200 28 | response_body = json.loads(result.body) 29 | assert response_body["result"] is None 30 | assert response_body["jsonrpc"] == "2.0" 31 | assert response_body["error"] == {"code": -32700, "message": "Parse error"} 32 | 33 | 34 | # examples from http://www.jsonrpc.org/specification 35 | 36 | @pytest.mark.gen_test 37 | def test_rpc_call_with_positional_parameters(http_client, base_url): 38 | base_url += "/api" 39 | body = json.dumps({"jsonrpc": "2.0", "method": "subtract", "params": [42, 23], "id": 1}) 40 | result = yield http_client.fetch(base_url, method="POST", body=body, 41 | headers={"content-type": "application/json"}) 42 | 43 | assert result.code == 200 44 | response_body = json.loads(result.body) 45 | assert response_body["jsonrpc"] == "2.0" 46 | assert response_body["result"] == 19 47 | assert response_body["id"] == 1 48 | 49 | body = json.dumps({"jsonrpc": "2.0", "method": "subtract", "params": [23, 42], "id": 2}) 50 | result = yield http_client.fetch(base_url, method="POST", body=body, 51 | headers={"content-type": "application/json"}) 52 | 53 | assert result.code == 200 54 | response_body = json.loads(result.body) 55 | assert response_body["jsonrpc"] == "2.0" 56 | assert response_body["result"] == -19 57 | assert response_body["id"] == 2 58 | 59 | 60 | @pytest.mark.gen_test 61 | def test_rpc_call_with_named_parameters(http_client, base_url): 62 | base_url += "/api" 63 | body = json.dumps( 64 | {"jsonrpc": "2.0", "method": "subtract", "params": {"a": 42, "b": 23}, "id": 3}) 65 | result = yield http_client.fetch(base_url, method="POST", body=body, 66 | headers={"content-type": "application/json"}) 67 | assert result.code == 200 68 | response_body = json.loads(result.body) 69 | assert response_body["jsonrpc"] == "2.0" 70 | assert response_body["result"] == 19 71 | assert response_body["id"] == 3 72 | 73 | body = json.dumps( 74 | {"jsonrpc": "2.0", "method": "subtract", "params": {"b": 42, "a": 23}, "id": 4}) 75 | result = yield http_client.fetch(base_url, method="POST", body=body, 76 | headers={"content-type": "application/json"}) 77 | assert result.code == 200 78 | response_body = json.loads(result.body) 79 | assert response_body["jsonrpc"] == "2.0" 80 | assert response_body["result"] == -19 81 | assert response_body["id"] == 4 82 | 83 | 84 | @pytest.mark.gen_test 85 | def test_a_notification(http_client, base_url): 86 | base_url += "/api" 87 | body = json.dumps({"jsonrpc": "2.0", "method": "update", "params": {"a": 23}}) 88 | result = yield http_client.fetch(base_url, method="POST", body=body, 89 | headers={"content-type": "application/json"}) 90 | assert result.code == 200 91 | response_body = json.loads(result.body) 92 | assert response_body["jsonrpc"] == "2.0" 93 | assert response_body["result"] is None 94 | assert response_body["error"] is None 95 | 96 | 97 | @pytest.mark.gen_test 98 | def test_a_notification_inexistent_method(http_client, base_url): 99 | base_url += "/api" 100 | body = json.dumps({"jsonrpc": "2.0", "method": "does_not_exist", "params": {"a": 23}}) 101 | result = yield http_client.fetch(base_url, method="POST", body=body, 102 | headers={"content-type": "application/json"}) 103 | assert result.code == 200 104 | response_body = json.loads(result.body) 105 | assert response_body["jsonrpc"] == "2.0" 106 | assert response_body["result"] is None 107 | assert response_body["error"] is None 108 | 109 | 110 | @pytest.mark.gen_test 111 | def test_rpc_call_of_non_existent_method(http_client, base_url): 112 | base_url += "/api" 113 | body = json.dumps({"jsonrpc": "2.0", "method": "does_not_exist", "params": {"a": 23}, "id": 1}) 114 | result = yield http_client.fetch(base_url, method="POST", body=body, 115 | headers={"content-type": "application/json"}) 116 | assert result.code == 200 117 | response_body = json.loads(result.body) 118 | assert response_body["jsonrpc"] == "2.0" 119 | assert response_body["result"] is None 120 | assert response_body["error"] == {"code": -32601, "message": "Method not found"} 121 | assert response_body["id"] == 1 122 | 123 | 124 | @pytest.mark.gen_test 125 | def test_rpc_call_with_invalid_json(http_client, base_url): 126 | base_url += "/api" 127 | body = '{"jsonrpc": "2.0", "method": "foobar, "params": "bar", "baz]' 128 | result = yield http_client.fetch(base_url, method="POST", body=body, 129 | headers={"content-type": "application/json"}) 130 | 131 | assert result.code == 200 132 | response_body = json.loads(result.body) 133 | assert response_body["jsonrpc"] == "2.0" 134 | assert response_body["result"] is None 135 | assert response_body["error"] == {"code": -32700, "message": "Parse error"} 136 | 137 | 138 | @pytest.mark.gen_test 139 | def test_rpc_call_with_invalid_request_object(http_client, base_url): 140 | base_url += "/api" 141 | body = json.dumps({"jsonrpc": "2.0", "method": "subtract", "params": "foobar"}) 142 | result = yield http_client.fetch(base_url, method="POST", body=body, 143 | headers={"content-type": "application/json"}) 144 | assert result.code == 200 145 | response_body = json.loads(result.body) 146 | assert response_body["jsonrpc"] == "2.0" 147 | assert response_body["result"] is None 148 | assert response_body["error"] == {"code": -32600, "message": "Invalid Request"} 149 | assert response_body["id"] is None 150 | 151 | 152 | @pytest.mark.gen_test 153 | def test_batch_call_with_invalid_json(http_client, base_url): 154 | base_url += "/api" 155 | body = '[{"jsonrpc": "2.0", "method": "sum", "params": [1,2,4], "id": "1"},{"jsonrpc": "2.0", "method"]' 156 | result = yield http_client.fetch(base_url, method="POST", body=body, 157 | headers={"content-type": "application/json"}) 158 | assert result.code == 200 159 | response_body = json.loads(result.body) 160 | print(response_body) 161 | assert isinstance(response_body, dict) 162 | assert response_body["jsonrpc"] == "2.0" 163 | assert response_body["result"] is None 164 | assert response_body["error"] == {"code": -32700, "message": "Parse error"} 165 | 166 | 167 | @pytest.mark.gen_test 168 | def test_batch_call_empty_array(http_client, base_url): 169 | base_url += "/api" 170 | body = "[]" 171 | result = yield http_client.fetch(base_url, method="POST", body=body, 172 | headers={"content-type": "application/json"}) 173 | assert result.code == 200 174 | response_body = json.loads(result.body) 175 | assert response_body["jsonrpc"] == "2.0" 176 | assert response_body["result"] is None 177 | assert response_body["error"] == {"code": -32600, "message": "Invalid Request"} 178 | assert response_body["id"] is None 179 | 180 | 181 | @pytest.mark.gen_test 182 | def test_batch_call_invalid_batch_but_not_empty(http_client, base_url): 183 | base_url += "/api" 184 | body = "[1]" 185 | result = yield http_client.fetch(base_url, method="POST", body=body, 186 | headers={"content-type": "application/json"}) 187 | assert result.code == 200 188 | response_body = json.loads(result.body) 189 | assert isinstance(response_body, list) 190 | assert len(response_body) == 1 191 | assert response_body[0]["jsonrpc"] == "2.0" 192 | assert response_body[0]["id"] is None 193 | assert response_body[0]["result"] is None 194 | assert response_body[0]["error"] == {"code": -32600, "message": "Invalid Request"} 195 | 196 | 197 | @pytest.mark.gen_test 198 | def test_batch_call_invalid_batch(http_client, base_url): 199 | base_url += "/api" 200 | body = "[1,2,3]" 201 | result = yield http_client.fetch(base_url, method="POST", body=body, 202 | headers={"content-type": "application/json"}) 203 | assert result.code == 200 204 | response_body = json.loads(result.body) 205 | assert isinstance(response_body, list) 206 | assert len(response_body) == 3 207 | 208 | for i in range(3): 209 | assert response_body[i]["jsonrpc"] == "2.0" 210 | assert response_body[i]["id"] is None 211 | assert response_body[i]["result"] is None 212 | assert response_body[i]["error"] == {"code": -32600, "message": "Invalid Request"} 213 | 214 | 215 | @pytest.mark.gen_test 216 | def test_batch_big_batch(http_client, base_url): 217 | base_url += "/api" 218 | body = [ 219 | {"jsonrpc": "2.0", "method": "sum", "params": [1, 2, 4], "id": "1"}, # valid 220 | {"jsonrpc": "2.0", "method": "notify_hello", "params": [7]}, # notification 221 | {"jsonrpc": "2.0", "method": "subtract", "params": [42, 23], "id": "2"}, # valid 222 | {"foo": "boo"}, # invalid request 223 | {"jsonrpc": "2.0", "method": "foo.get", "params": {"name": "myself"}, "id": "5"}, 224 | # method not found 225 | {"jsonrpc": "2.0", "method": "get_data", "id": "9"} # valid no params 226 | ] 227 | expected_results = { 228 | "1": {"jsonrpc": "2.0", "result": 7, "id": "1", "error": None}, 229 | "2": {"jsonrpc": "2.0", "result": 19, "id": "2", "error": None}, 230 | "5": {"jsonrpc": "2.0", "error": {"code": -32601, "message": "Method not found"}, "id": "5", 231 | "result": None}, 232 | "9": {"jsonrpc": "2.0", "result": ["hello", 5], "id": "9", "error": None} 233 | } 234 | 235 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 236 | headers={"content-type": "application/json"}) 237 | assert result.code == 200 238 | response_body = json.loads(result.body) 239 | assert isinstance(response_body, list) 240 | 241 | print(response_body) 242 | for request in body: 243 | req_id = request.get("id") 244 | if not req_id: 245 | continue 246 | 247 | expected = expected_results.get(req_id) 248 | actual_response = [x for x in response_body if x["id"] == req_id][0] 249 | 250 | assert expected == actual_response 251 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # gemstone documentation build configuration file, created by 5 | # sphinx-quickstart on Fri Nov 25 14:52:22 2016. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | import os 21 | import sys 22 | import gemstone 23 | 24 | sys.path.insert(0, os.path.abspath('.')) 25 | 26 | autoclass_content = 'both' 27 | 28 | # -- General configuration ------------------------------------------------ 29 | 30 | # If your documentation needs a minimal Sphinx version, state it here. 31 | # 32 | # needs_sphinx = '1.0' 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 36 | # ones. 37 | extensions = [ 38 | 'sphinx.ext.autodoc', 39 | 'sphinx.ext.todo', 40 | 'sphinx.ext.viewcode' 41 | ] 42 | 43 | # Add any paths that contain templates here, relative to this directory. 44 | templates_path = ['_templates'] 45 | 46 | # The suffix(es) of source filenames. 47 | # You can specify multiple suffix as a list of string: 48 | # 49 | # source_suffix = ['.rst', '.md'] 50 | source_suffix = '.rst' 51 | 52 | # The encoding of source files. 53 | # 54 | # source_encoding = 'utf-8-sig' 55 | 56 | # The master toctree document. 57 | master_doc = 'index' 58 | 59 | # General information about the project. 60 | project = 'gemstone' 61 | copyright = '2016, Vlad Calin' 62 | author = 'Vlad Calin' 63 | 64 | # The version info for the project you're documenting, acts as replacement for 65 | # |version| and |release|, also used in various other places throughout the 66 | # built documents. 67 | # 68 | # The short X.Y version. 69 | version = gemstone.__version__ 70 | # The full version, including alpha/beta/rc tags. 71 | release = gemstone.__version__ 72 | 73 | # The language for content autogenerated by Sphinx. Refer to documentation 74 | # for a list of supported languages. 75 | # 76 | # This is also used if you do content translation via gettext catalogs. 77 | # Usually you set "language" from the command line for these cases. 78 | language = None 79 | 80 | # There are two options for replacing |today|: either, you set today to some 81 | # non-false value, then it is used: 82 | # 83 | # today = '' 84 | # 85 | # Else, today_fmt is used as the format for a strftime call. 86 | # 87 | # today_fmt = '%B %d, %Y' 88 | 89 | # List of patterns, relative to source directory, that match files and 90 | # directories to ignore when looking for source files. 91 | # This patterns also effect to html_static_path and html_extra_path 92 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 93 | 94 | # The reST default role (used for this markup: `text`) to use for all 95 | # documents. 96 | # 97 | # default_role = None 98 | 99 | # If true, '()' will be appended to :func: etc. cross-reference text. 100 | # 101 | # add_function_parentheses = True 102 | 103 | # If true, the current module name will be prepended to all description 104 | # unit titles (such as .. function::). 105 | # 106 | # add_module_names = True 107 | 108 | # If true, sectionauthor and moduleauthor directives will be shown in the 109 | # output. They are ignored by default. 110 | # 111 | # show_authors = False 112 | 113 | # The name of the Pygments (syntax highlighting) style to use. 114 | pygments_style = 'sphinx' 115 | 116 | # A list of ignored prefixes for module index sorting. 117 | # modindex_common_prefix = [] 118 | 119 | # If true, keep warnings as "system message" paragraphs in the built documents. 120 | # keep_warnings = False 121 | 122 | # If true, `todo` and `todoList` produce output, else they produce nothing. 123 | todo_include_todos = True 124 | 125 | # -- Options for HTML output ---------------------------------------------- 126 | 127 | # The theme to use for HTML and HTML Help pages. See the documentation for 128 | # a list of builtin themes. 129 | # 130 | html_theme = 'nature' 131 | 132 | # Theme options are theme-specific and customize the look and feel of a theme 133 | # further. For a list of options available for each theme, see the 134 | # documentation. 135 | # 136 | # html_theme_options = {} 137 | 138 | # Add any paths that contain custom themes here, relative to this directory. 139 | # html_theme_path = [] 140 | 141 | # The name for this set of Sphinx documents. 142 | # " v documentation" by default. 143 | # 144 | # html_title = 'gemstone v0.1' 145 | 146 | # A shorter title for the navigation bar. Default is the same as html_title. 147 | # 148 | # html_short_title = None 149 | 150 | # The name of an image file (relative to this directory) to place at the top 151 | # of the sidebar. 152 | # 153 | # html_logo = None 154 | 155 | # The name of an image file (relative to this directory) to use as a favicon of 156 | # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 157 | # pixels large. 158 | # 159 | # html_favicon = None 160 | 161 | # Add any paths that contain custom static files (such as style sheets) here, 162 | # relative to this directory. They are copied after the builtin static files, 163 | # so a file named "default.css" will overwrite the builtin "default.css". 164 | html_static_path = ['_static'] 165 | 166 | # Add any extra paths that contain custom files (such as robots.txt or 167 | # .htaccess) here, relative to this directory. These files are copied 168 | # directly to the root of the documentation. 169 | # 170 | # html_extra_path = [] 171 | 172 | # If not None, a 'Last updated on:' timestamp is inserted at every page 173 | # bottom, using the given strftime format. 174 | # The empty string is equivalent to '%b %d, %Y'. 175 | # 176 | # html_last_updated_fmt = None 177 | 178 | # If true, SmartyPants will be used to convert quotes and dashes to 179 | # typographically correct entities. 180 | # 181 | # html_use_smartypants = True 182 | 183 | # Custom sidebar templates, maps document names to template names. 184 | # 185 | # html_sidebars = {} 186 | 187 | # Additional templates that should be rendered to pages, maps page names to 188 | # template names. 189 | # 190 | # html_additional_pages = {} 191 | 192 | # If false, no module index is generated. 193 | # 194 | # html_domain_indices = True 195 | 196 | # If false, no index is generated. 197 | # 198 | # html_use_index = True 199 | 200 | # If true, the index is split into individual pages for each letter. 201 | # 202 | # html_split_index = False 203 | 204 | # If true, links to the reST sources are added to the pages. 205 | # 206 | # html_show_sourcelink = True 207 | 208 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 209 | # 210 | # html_show_sphinx = True 211 | 212 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 213 | # 214 | # html_show_copyright = True 215 | 216 | # If true, an OpenSearch description file will be output, and all pages will 217 | # contain a tag referring to it. The value of this option must be the 218 | # base URL from which the finished HTML is served. 219 | # 220 | # html_use_opensearch = '' 221 | 222 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 223 | # html_file_suffix = None 224 | 225 | # Language to be used for generating the HTML full-text search index. 226 | # Sphinx supports the following languages: 227 | # 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' 228 | # 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh' 229 | # 230 | # html_search_language = 'en' 231 | 232 | # A dictionary with options for the search language support, empty by default. 233 | # 'ja' uses this config value. 234 | # 'zh' user can custom change `jieba` dictionary path. 235 | # 236 | # html_search_options = {'type': 'default'} 237 | 238 | # The name of a javascript file (relative to the configuration directory) that 239 | # implements a search results scorer. If empty, the default will be used. 240 | # 241 | # html_search_scorer = 'scorer.js' 242 | 243 | # Output file base name for HTML help builder. 244 | htmlhelp_basename = 'pymicroservicedoc' 245 | 246 | # -- Options for LaTeX output --------------------------------------------- 247 | 248 | latex_elements = { 249 | # The paper size ('letterpaper' or 'a4paper'). 250 | # 251 | # 'papersize': 'letterpaper', 252 | 253 | # The font size ('10pt', '11pt' or '12pt'). 254 | # 255 | # 'pointsize': '10pt', 256 | 257 | # Additional stuff for the LaTeX preamble. 258 | # 259 | # 'preamble': '', 260 | 261 | # Latex figure (float) alignment 262 | # 263 | # 'figure_align': 'htbp', 264 | } 265 | 266 | # Grouping the document tree into LaTeX files. List of tuples 267 | # (source start file, target name, title, 268 | # author, documentclass [howto, manual, or own class]). 269 | latex_documents = [ 270 | (master_doc, 'gemstone.tex', 'gemstone Documentation', 271 | 'Vlad Calin', 'manual'), 272 | ] 273 | 274 | # The name of an image file (relative to this directory) to place at the top of 275 | # the title page. 276 | # 277 | # latex_logo = None 278 | 279 | # For "manual" documents, if this is true, then toplevel headings are parts, 280 | # not chapters. 281 | # 282 | # latex_use_parts = False 283 | 284 | # If true, show page references after internal links. 285 | # 286 | # latex_show_pagerefs = False 287 | 288 | # If true, show URL addresses after external links. 289 | # 290 | # latex_show_urls = False 291 | 292 | # Documents to append as an appendix to all manuals. 293 | # 294 | # latex_appendices = [] 295 | 296 | # It false, will not define \strong, \code, itleref, \crossref ... but only 297 | # \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added 298 | # packages. 299 | # 300 | # latex_keep_old_macro_names = True 301 | 302 | # If false, no module index is generated. 303 | # 304 | # latex_domain_indices = True 305 | 306 | 307 | # -- Options for manual page output --------------------------------------- 308 | 309 | # One entry per manual page. List of tuples 310 | # (source start file, name, description, authors, manual section). 311 | man_pages = [ 312 | (master_doc, 'gemstone', 'gemstone Documentation', 313 | [author], 1) 314 | ] 315 | 316 | # If true, show URL addresses after external links. 317 | # 318 | # man_show_urls = False 319 | 320 | 321 | # -- Options for Texinfo output ------------------------------------------- 322 | 323 | # Grouping the document tree into Texinfo files. List of tuples 324 | # (source start file, target name, title, author, 325 | # dir menu entry, description, category) 326 | texinfo_documents = [ 327 | (master_doc, 'gemstone', 'gemstone Documentation', 328 | author, 'gemstone', 'One line description of project.', 329 | 'Miscellaneous'), 330 | ] 331 | 332 | # Documents to append as an appendix to all manuals. 333 | # 334 | # texinfo_appendices = [] 335 | 336 | # If false, no module index is generated. 337 | # 338 | # texinfo_domain_indices = True 339 | 340 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 341 | # 342 | # texinfo_show_urls = 'footnote' 343 | 344 | # If true, do not generate a @detailmenu in the "Top" node's menu. 345 | # 346 | # texinfo_no_detailmenu = False 347 | 348 | viewcode_import = True -------------------------------------------------------------------------------- /gemstone/core/handlers.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | import copy 3 | import time 4 | 5 | import simplejson as json 6 | from tornado.web import RequestHandler 7 | from tornado.gen import coroutine 8 | 9 | from gemstone.core.structs import JsonRpcResponse, JsonRpcRequest, JsonRpcResponseBatch, \ 10 | GenericResponse, JsonRpcInvalidRequestError 11 | 12 | __all__ = [ 13 | 'TornadoJsonRpcHandler', 14 | 'GemstoneCustomHandler' 15 | ] 16 | 17 | 18 | # noinspection PyAbstractClass 19 | class GemstoneCustomHandler(RequestHandler): 20 | """ 21 | Base class for custom Tornado handlers that 22 | can be added to the microservice. 23 | 24 | Offers a reference to the microservice through the ``self.microservice`` attribute. 25 | 26 | """ 27 | 28 | def __init__(self, *args, **kwargs): 29 | #: reference to the microservice that uses the request handler 30 | self.microservice = None 31 | super(GemstoneCustomHandler, self).__init__(*args, **kwargs) 32 | 33 | # noinspection PyMethodOverriding 34 | def initialize(self, microservice): 35 | self.microservice = microservice 36 | 37 | 38 | # noinspection PyAbstractClass 39 | class TornadoJsonRpcHandler(RequestHandler): 40 | def __init__(self, *args, **kwargs): 41 | self.response_is_sent = False 42 | self.methods = None 43 | self.executor = None 44 | self.validation_strategies = None 45 | self.api_token_handlers = None 46 | self.logger = None 47 | self.microservice = None 48 | super(TornadoJsonRpcHandler, self).__init__(*args, **kwargs) 49 | 50 | # noinspection PyMethodOverriding 51 | def initialize(self, microservice): 52 | self.logger = microservice.logger 53 | self.methods = microservice.methods 54 | self.executor = microservice.get_executor() 55 | self.response_is_sent = False 56 | self.microservice = microservice 57 | 58 | def get_current_user(self): 59 | return self.microservice.authenticate_request(self) 60 | 61 | @coroutine 62 | def post(self): 63 | if self.request.headers.get("Content-type").split(";")[0] != "application/json": 64 | self.write_single_response(GenericResponse.INVALID_REQUEST) 65 | return 66 | 67 | req_body_raw = self.request.body.decode() 68 | try: 69 | req_object = json.loads(req_body_raw) 70 | except json.JSONDecodeError: 71 | self.write_single_response(GenericResponse.PARSE_ERROR) 72 | return 73 | 74 | # handle the actual call 75 | if isinstance(req_object, dict): 76 | # single call 77 | try: 78 | req_object = JsonRpcRequest.from_dict(req_object) 79 | except JsonRpcInvalidRequestError: 80 | self.write_single_response(GenericResponse.INVALID_REQUEST) 81 | return 82 | 83 | if req_object.is_notification(): 84 | self.write_single_response(GenericResponse.NOTIFICATION_RESPONSE) 85 | 86 | result = yield self.handle_single_request(req_object) 87 | self.write_single_response(result) 88 | elif isinstance(req_object, list): 89 | if len(req_object) == 0: 90 | self.write_single_response(GenericResponse.INVALID_REQUEST) 91 | return 92 | 93 | # batch call 94 | invalid_requests = [] 95 | requests_futures = [] 96 | notification_futures = [] 97 | 98 | for item in req_object: 99 | try: 100 | if not isinstance(item, dict): 101 | raise JsonRpcInvalidRequestError() 102 | current_rpc_call = JsonRpcRequest.from_dict(item) 103 | 104 | # handle notifications 105 | if current_rpc_call.is_notification(): 106 | # we trigger their execution, but we don't yield for their results 107 | notification_futures.append(self.handle_single_request(current_rpc_call)) 108 | else: 109 | requests_futures.append(self.handle_single_request(current_rpc_call)) 110 | except JsonRpcInvalidRequestError: 111 | invalid_requests.append(GenericResponse.INVALID_REQUEST) 112 | 113 | finished_rpc_calls = yield requests_futures 114 | self.write_batch_response(JsonRpcResponseBatch(invalid_requests + finished_rpc_calls)) 115 | else: 116 | self.write_single_response(GenericResponse.INVALID_REQUEST) 117 | 118 | @coroutine 119 | def handle_single_request(self, request_object): 120 | """ 121 | Handles a single request object and returns the correct result as follows: 122 | 123 | - A valid response object if it is a regular request (with ID) 124 | - ``None`` if it was a notification (if None is returned, a response object with 125 | "received" body was already sent to the client. 126 | 127 | :param request_object: A :py:class:`gemstone.core.structs.JsonRpcRequest` object 128 | representing a Request object 129 | :return: A :py:class:`gemstone.core.structs.JsonRpcResponse` object representing a 130 | Response object or None if no response is expected (it was a notification) 131 | 132 | """ 133 | # don't handle responses? 134 | if isinstance(request_object, JsonRpcResponse): 135 | return request_object 136 | 137 | error = None 138 | result = None 139 | id_ = request_object.id 140 | 141 | # validate method name 142 | if request_object.method not in self.methods: 143 | resp = GenericResponse.METHOD_NOT_FOUND 144 | resp.id = id_ 145 | return resp 146 | 147 | # check for private access 148 | method = self.methods[request_object.method] 149 | 150 | if isinstance(request_object.params, (list, tuple)): 151 | self.call_method_from_all_plugins("on_method_call", request_object) 152 | else: 153 | self.call_method_from_all_plugins("on_method_call", request_object) 154 | 155 | if self._method_is_private(method): 156 | if not self.get_current_user(): 157 | resp = GenericResponse.ACCESS_DENIED 158 | resp.id = id_ 159 | return resp 160 | 161 | method = self.prepare_method_call(method, request_object.params) 162 | 163 | # before request hook 164 | _method_duration = time.time() 165 | 166 | try: 167 | result = yield self.call_method(method) 168 | except Exception as e: 169 | # catch all exceptions generated by method 170 | # and handle in a special manner only the TypeError 171 | if isinstance(e, TypeError): 172 | # TODO: find a proper way to check that the function got the wrong 173 | # parameters (with **kwargs) 174 | if "got an unexpected keyword argument" in e.args[0]: 175 | resp = GenericResponse.INVALID_PARAMS 176 | resp.id = id_ 177 | return resp 178 | # TODO: find a proper way to check that the function got the wrong 179 | # parameters (with *args) 180 | elif "takes" in e.args[0] and "positional argument" in e.args[0] and "were given" in \ 181 | e.args[0]: 182 | resp = GenericResponse.INVALID_PARAMS 183 | resp.id = id_ 184 | return resp 185 | elif "missing" in e.args[0] and "required positional argument" in e.args[0]: 186 | resp = GenericResponse.INVALID_PARAMS 187 | resp.id = id_ 188 | return resp 189 | # generic handling for any exception (even TypeError) that 190 | # is not generated because of bad parameters 191 | 192 | self.call_method_from_all_plugins("on_internal_error", e) 193 | 194 | err = GenericResponse.INTERNAL_ERROR 195 | err.id = id_ 196 | err.error["data"] = { 197 | "class": type(e).__name__, 198 | "info": str(e) 199 | } 200 | return err 201 | 202 | to_return_resp = JsonRpcResponse(result=result, error=error, id=id_) 203 | 204 | return to_return_resp 205 | 206 | def write_single_response(self, response_obj): 207 | """ 208 | Writes a json rpc response ``{"result": result, "error": error, "id": id}``. 209 | If the ``id`` is ``None``, the response will not contain an ``id`` field. 210 | The response is sent to the client as an ``application/json`` response. Only one call per 211 | response is allowed 212 | 213 | :param response_obj: A Json rpc response object 214 | :return: 215 | """ 216 | if not isinstance(response_obj, JsonRpcResponse): 217 | raise ValueError( 218 | "Expected JsonRpcResponse, but got {} instead".format(type(response_obj).__name__)) 219 | 220 | if not self.response_is_sent: 221 | self.set_status(200) 222 | self.set_header("Content-Type", "application/json") 223 | self.finish(response_obj.to_string()) 224 | self.response_is_sent = True 225 | 226 | def write_batch_response(self, batch_response): 227 | self.set_header("Content-Type", "application/json") 228 | self.write(batch_response.to_string()) 229 | 230 | def write_error(self, status_code, **kwargs): 231 | if status_code == 405: 232 | self.set_status(405) 233 | self.write_single_response( 234 | JsonRpcResponse(error={"code": 405, "message": "Method not allowed"})) 235 | return 236 | 237 | exc_info = kwargs["exc_info"] 238 | err = GenericResponse.INTERNAL_ERROR 239 | err.error["data"] = { 240 | "class": str(exc_info[0].__name__), 241 | "info": str(exc_info[1]) 242 | } 243 | self.set_status(200) 244 | self.write_single_response(err) 245 | 246 | def prepare_method_call(self, method, args): 247 | """ 248 | Wraps a method so that method() will call ``method(*args)`` or ``method(**args)``, 249 | depending of args type 250 | 251 | :param method: a callable object (method) 252 | :param args: dict or list with the parameters for the function 253 | :return: a 'patched' callable 254 | """ 255 | if self._method_requires_handler_ref(method): 256 | if isinstance(args, list): 257 | args = [self] + args 258 | elif isinstance(args, dict): 259 | args["handler"] = self 260 | 261 | if isinstance(args, list): 262 | to_call = partial(method, *args) 263 | elif isinstance(args, dict): 264 | to_call = partial(method, **args) 265 | else: 266 | raise TypeError( 267 | "args must be list or dict but got {} instead".format(type(args).__name__)) 268 | return to_call 269 | 270 | @coroutine 271 | def call_method(self, method): 272 | """ 273 | Calls a blocking method in an executor, in order to preserve the non-blocking behaviour 274 | 275 | If ``method`` is a coroutine, yields from it and returns, no need to execute in 276 | in an executor. 277 | 278 | :param method: The method or coroutine to be called (with no arguments). 279 | :return: the result of the method call 280 | """ 281 | if self._method_is_async_generator(method): 282 | result = yield method() 283 | else: 284 | result = yield self.executor.submit(method) 285 | return result 286 | 287 | @coroutine 288 | def handle_batch_request(self, batch_req_obj): 289 | responses = yield [self.handle_single_request(single_req) for single_req in 290 | batch_req_obj.iter_items()] 291 | return responses 292 | 293 | def _method_requires_handler_ref(self, method): 294 | return getattr(method, "_req_h_ref", False) 295 | 296 | def _method_is_async_generator(self, method): 297 | """ 298 | Given a simple callable or a callable wrapped in funtools.partial, determines 299 | if it was wrapped with the :py:func:`gemstone.async_method` decorator. 300 | 301 | :param method: 302 | :return: 303 | """ 304 | if hasattr(method, "func"): 305 | func = method.func 306 | else: 307 | func = method 308 | 309 | return getattr(func, "_is_coroutine", False) 310 | 311 | @staticmethod 312 | def _method_is_private(method): 313 | return getattr(method, "_exposed_private", False) 314 | 315 | def call_method_from_all_plugins(self, method, *args, **kwargs): 316 | for plugin in self.microservice.plugins: 317 | method_callable = getattr(plugin, method) 318 | if not method: 319 | continue 320 | method_callable(*args, **kwargs) 321 | -------------------------------------------------------------------------------- /tests/functional/test_microservice.py: -------------------------------------------------------------------------------- 1 | import simplejson as json 2 | 3 | import pytest 4 | 5 | from tests.services.service_microservice import TestService 6 | 7 | 8 | @pytest.fixture 9 | def app(): 10 | service = TestService() 11 | service.skip_configuration = True 12 | service._initial_setup() 13 | return service.make_tornado_app() 14 | 15 | 16 | @pytest.mark.gen_test 17 | def test_simple_call_no_parameters(http_client, base_url): 18 | base_url += "/api" 19 | body = { 20 | "id": 1, 21 | "method": "say_hello", 22 | "jsonrpc": "2.0" 23 | } 24 | 25 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 26 | headers={"content-type": "application/json"}) 27 | assert result.code == 200 28 | resp_body = json.loads(result.body) 29 | 30 | assert resp_body["id"] == 1 31 | assert resp_body["error"] is None 32 | assert resp_body["result"] == "hello" 33 | assert resp_body["jsonrpc"] == "2.0" 34 | 35 | 36 | @pytest.mark.gen_test 37 | def test_simple_call_positional_arguments(http_client, base_url): 38 | base_url += "/api" 39 | body = { 40 | "id": 1, 41 | "method": "subtract", 42 | "params": [10, 20], 43 | "jsonrpc": "2.0" 44 | } 45 | 46 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 47 | headers={"content-type": "application/json"}) 48 | assert result.code == 200 49 | resp_body = json.loads(result.body) 50 | 51 | assert resp_body["id"] == 1 52 | assert resp_body["error"] is None 53 | assert resp_body["result"] == -10 54 | assert resp_body["jsonrpc"] == "2.0" 55 | 56 | 57 | @pytest.mark.gen_test 58 | def test_simple_call_keyword_arguments(http_client, base_url): 59 | base_url += "/api" 60 | body = { 61 | "id": 1, 62 | "method": "subtract", 63 | "params": {"b": 20, "a": 10}, 64 | "jsonrpc": "2.0" 65 | } 66 | 67 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 68 | headers={"content-type": "application/json"}) 69 | assert result.code == 200 70 | resp_body = json.loads(result.body) 71 | 72 | assert resp_body["id"] == 1 73 | assert resp_body["error"] is None 74 | assert resp_body["result"] == -10 75 | assert resp_body["jsonrpc"] == "2.0" 76 | 77 | 78 | @pytest.mark.gen_test 79 | def test_simple_call_variable_length_positional_arguments(http_client, base_url): 80 | base_url += "/api" 81 | body = { 82 | "id": 1, 83 | "method": "sum", 84 | "params": list(range(0, 100)), 85 | "jsonrpc": "2.0" 86 | } 87 | 88 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 89 | headers={"content-type": "application/json"}) 90 | assert result.code == 200 91 | resp_body = json.loads(result.body) 92 | 93 | assert resp_body["id"] == 1 94 | assert resp_body["error"] is None 95 | assert resp_body["result"] == 4950 96 | assert resp_body["jsonrpc"] == "2.0" 97 | 98 | 99 | @pytest.mark.gen_test 100 | def test_simple_call_raise_exception(http_client, base_url): 101 | base_url += "/api" 102 | body = { 103 | "id": 1, 104 | "method": "divide", 105 | "params": {"a": 10, "b": 0}, 106 | "jsonrpc": "2.0" 107 | } 108 | 109 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 110 | headers={"content-type": "application/json"}) 111 | assert result.code == 200 112 | resp_body = json.loads(result.body) 113 | 114 | assert resp_body["id"] == 1 115 | assert resp_body["result"] is None 116 | assert resp_body["error"] is not None 117 | assert resp_body["error"]["code"] == -32603 118 | assert resp_body["error"]["message"] == "Internal error" 119 | assert resp_body["error"]["data"]["class"] == "ZeroDivisionError" 120 | assert resp_body["error"]["data"]["info"] == "division by zero" 121 | assert resp_body["jsonrpc"] == "2.0" 122 | 123 | 124 | @pytest.mark.gen_test 125 | def test_simple_call_wrong_arguments_too_few_positional_arguments(http_client, base_url): 126 | base_url += "/api" 127 | body = { 128 | "id": 1, 129 | "method": "subtract", 130 | "params": [10], 131 | "jsonrpc": "2.0" 132 | } 133 | 134 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 135 | headers={"content-type": "application/json"}) 136 | assert result.code == 200 137 | resp_body = json.loads(result.body) 138 | print(resp_body) 139 | assert resp_body["id"] == 1 140 | assert resp_body["result"] is None 141 | assert resp_body["error"] is not None 142 | assert resp_body["error"]["code"] == -32602 143 | assert resp_body["error"]["message"] == "Invalid params" 144 | assert resp_body["jsonrpc"] == "2.0" 145 | 146 | 147 | @pytest.mark.gen_test 148 | def test_simple_call_wrong_arguments_too_many_positional_arguments(http_client, base_url): 149 | base_url += "/api" 150 | body = { 151 | "id": 1, 152 | "method": "subtract", 153 | "params": [10, 20, 30], 154 | "jsonrpc": "2.0" 155 | } 156 | 157 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 158 | headers={"content-type": "application/json"}) 159 | assert result.code == 200 160 | resp_body = json.loads(result.body) 161 | print(resp_body) 162 | assert resp_body["id"] == 1 163 | assert resp_body["result"] is None 164 | assert resp_body["error"] is not None 165 | assert resp_body["error"]["code"] == -32602 166 | assert resp_body["error"]["message"] == "Invalid params" 167 | assert resp_body["jsonrpc"] == "2.0" 168 | 169 | 170 | @pytest.mark.gen_test 171 | def test_simple_call_wrong_arguments_too_few_keyword_arguments(http_client, base_url): 172 | base_url += "/api" 173 | body = { 174 | "id": 1, 175 | "method": "subtract", 176 | "params": {"a": 10}, 177 | "jsonrpc": "2.0" 178 | } 179 | 180 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 181 | headers={"content-type": "application/json"}) 182 | assert result.code == 200 183 | resp_body = json.loads(result.body) 184 | print(resp_body) 185 | assert resp_body["id"] == 1 186 | assert resp_body["result"] is None 187 | assert resp_body["error"] is not None 188 | assert resp_body["error"]["code"] == -32602 189 | assert resp_body["error"]["message"] == "Invalid params" 190 | assert resp_body["jsonrpc"] == "2.0" 191 | 192 | 193 | @pytest.mark.gen_test 194 | def test_simple_call_wrong_arguments_too_many_keyword_arguments(http_client, base_url): 195 | base_url += "/api" 196 | body = { 197 | "id": 1, 198 | "method": "subtract", 199 | "params": {"a": 10, "b": 20, "c": 30}, 200 | "jsonrpc": "2.0" 201 | } 202 | 203 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 204 | headers={"content-type": "application/json"}) 205 | assert result.code == 200 206 | resp_body = json.loads(result.body) 207 | print(resp_body) 208 | assert resp_body["id"] == 1 209 | assert resp_body["result"] is None 210 | assert resp_body["error"] is not None 211 | assert resp_body["error"]["code"] == -32602 212 | assert resp_body["error"]["message"] == "Invalid params" 213 | assert resp_body["jsonrpc"] == "2.0" 214 | 215 | 216 | @pytest.mark.gen_test 217 | def test_simple_call_wrong_arguments_no_arguments(http_client, base_url): 218 | base_url += "/api" 219 | body = { 220 | "id": 1, 221 | "method": "subtract", 222 | "jsonrpc": "2.0" 223 | } 224 | 225 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 226 | headers={"content-type": "application/json"}) 227 | assert result.code == 200 228 | resp_body = json.loads(result.body) 229 | print(resp_body) 230 | assert resp_body["id"] == 1 231 | assert resp_body["result"] is None 232 | assert resp_body["error"] is not None 233 | assert resp_body["error"]["code"] == -32602 234 | assert resp_body["error"]["message"] == "Invalid params" 235 | assert resp_body["jsonrpc"] == "2.0" 236 | 237 | 238 | @pytest.mark.gen_test 239 | def test_notification_valid(http_client, base_url): 240 | base_url += "/api" 241 | body = { 242 | "method": "subtract", 243 | "params": [20, 10], 244 | "jsonrpc": "2.0" 245 | } 246 | 247 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 248 | headers={"content-type": "application/json"}) 249 | assert result.code == 200 250 | resp_body = json.loads(result.body) 251 | print(resp_body) 252 | assert "id" not in resp_body 253 | assert resp_body["result"] is None 254 | assert resp_body["error"] is None 255 | assert resp_body["jsonrpc"] == "2.0" 256 | 257 | 258 | @pytest.mark.gen_test 259 | def test_notification_method_not_found(http_client, base_url): 260 | base_url += "/api" 261 | body = { 262 | "method": "foobar", 263 | "params": [20, 10], 264 | "jsonrpc": "2.0" 265 | } 266 | 267 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 268 | headers={"content-type": "application/json"}) 269 | assert result.code == 200 270 | resp_body = json.loads(result.body) 271 | print(resp_body) 272 | assert "id" not in resp_body 273 | assert resp_body["result"] is None 274 | assert resp_body["error"] is None 275 | assert resp_body["jsonrpc"] == "2.0" 276 | 277 | 278 | @pytest.mark.gen_test 279 | def test_notification_invalid_params(http_client, base_url): 280 | base_url += "/api" 281 | body = { 282 | "method": "divide", 283 | "params": [20, 10, 30], 284 | "jsonrpc": "2.0" 285 | } 286 | 287 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 288 | headers={"content-type": "application/json"}) 289 | assert result.code == 200 290 | resp_body = json.loads(result.body) 291 | print(resp_body) 292 | assert "id" not in resp_body 293 | assert resp_body["result"] is None 294 | assert resp_body["error"] is None 295 | assert resp_body["jsonrpc"] == "2.0" 296 | 297 | 298 | @pytest.mark.gen_test 299 | def test_notification_internal_error(http_client, base_url): 300 | base_url += "/api" 301 | body = { 302 | "method": "divide", 303 | "params": [20, 0], 304 | "jsonrpc": "2.0" 305 | } 306 | 307 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 308 | headers={"content-type": "application/json"}) 309 | assert result.code == 200 310 | resp_body = json.loads(result.body) 311 | print(resp_body) 312 | assert "id" not in resp_body 313 | assert resp_body["result"] is None 314 | assert resp_body["error"] is None 315 | assert resp_body["jsonrpc"] == "2.0" 316 | 317 | 318 | @pytest.mark.gen_test 319 | def test_private_call_wrong_token(http_client, base_url): 320 | base_url += "/api" 321 | body = { 322 | "id": 3, 323 | "method": "private_sum", 324 | "params": [13, -13], 325 | "jsonrpc": "2.0" 326 | } 327 | 328 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 329 | headers={"content-type": "application/json", 330 | "X-Testing-Token": "bad_token"}) 331 | assert result.code == 200 332 | resp_body = json.loads(result.body) 333 | print(resp_body) 334 | assert resp_body["id"] == 3 335 | assert resp_body["result"] is None 336 | assert resp_body["error"] is not None 337 | assert resp_body["error"]["code"] == -32001 338 | assert resp_body["error"]["message"] == "Access denied" 339 | assert resp_body["jsonrpc"] == "2.0" 340 | 341 | 342 | @pytest.mark.gen_test 343 | def test_private_call_wrong_token_and_wrong_parameters(http_client, base_url): 344 | # without a valid token, somebody should not be able to get information 345 | # about certain methods, such as the parameters 346 | base_url += "/api" 347 | body = { 348 | "id": 3, 349 | "method": "private_sum", 350 | "params": [13], 351 | "jsonrpc": "2.0" 352 | } 353 | 354 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 355 | headers={"content-type": "application/json", 356 | "X-Testing-Token": "bad_token"}) 357 | assert result.code == 200 358 | resp_body = json.loads(result.body) 359 | print(resp_body) 360 | assert resp_body["id"] == 3 361 | assert resp_body["result"] is None 362 | assert resp_body["error"] is not None 363 | assert resp_body["error"]["code"] == -32001 364 | assert resp_body["error"]["message"] == "Access denied" 365 | assert resp_body["jsonrpc"] == "2.0" 366 | 367 | 368 | @pytest.mark.gen_test 369 | def test_private_call_ok_token(http_client, base_url): 370 | # without a valid token, somebody should not be able to get information 371 | # about certain methods, such as the parameters 372 | base_url += "/api" 373 | body = { 374 | "id": 3, 375 | "method": "private_sum", 376 | "params": [13, -13], 377 | "jsonrpc": "2.0" 378 | } 379 | 380 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 381 | headers={"content-type": "application/json", 382 | "X-Testing-Token": "testing_token"}) 383 | assert result.code == 200 384 | resp_body = json.loads(result.body) 385 | print(resp_body) 386 | assert resp_body["id"] == 3 387 | assert resp_body["result"] == 0 388 | assert resp_body["error"] is None 389 | assert resp_body["jsonrpc"] == "2.0" 390 | 391 | 392 | @pytest.mark.gen_test 393 | def test_private_call_ok_token_wrong_parameters(http_client, base_url): 394 | # without a valid token, somebody should not be able to get information 395 | # about certain methods, such as the parameters 396 | base_url += "/api" 397 | body = { 398 | "id": 3, 399 | "method": "private_sum", 400 | "params": [13], 401 | "jsonrpc": "2.0" 402 | } 403 | 404 | result = yield http_client.fetch(base_url, method="POST", body=json.dumps(body), 405 | headers={"content-type": "application/json", 406 | "X-Testing-Token": "testing_token"}) 407 | assert result.code == 200 408 | resp_body = json.loads(result.body) 409 | print(resp_body) 410 | assert resp_body["id"] == 3 411 | assert resp_body["result"] is None 412 | assert resp_body["error"] is not None 413 | assert resp_body["error"]["code"] == -32602 414 | assert resp_body["error"]["message"] == "Invalid params" 415 | assert resp_body["jsonrpc"] == "2.0" 416 | --------------------------------------------------------------------------------