├── docs ├── api.rst ├── dev │ ├── todo.rst │ ├── authors.rst │ └── internals.rst ├── util.rst ├── user │ ├── advanced.rst │ ├── install.rst │ ├── intro.rst │ ├── quickstart.rst │ └── manual.rst ├── service.rst ├── index.rst ├── Makefile ├── make.bat └── conf.py ├── requirements.txt ├── .travis.yml ├── TODO ├── tox.ini ├── .gitignore ├── Makefile ├── tests ├── test_config.py ├── conf │ ├── threading.conf.py │ └── gevent.conf.py └── test_util.py ├── setup.py ├── LICENSE ├── UPGRADING ├── ginkgo ├── __init__.py ├── async │ ├── __init__.py │ ├── eventlet.py │ ├── threading.py │ └── gevent.py ├── logger.py ├── core.py ├── config.py ├── runner.py └── util.py └── README.md /docs/api.rst: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/dev/todo.rst: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/dev/authors.rst: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/dev/internals.rst: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | nose 2 | -------------------------------------------------------------------------------- /docs/util.rst: -------------------------------------------------------------------------------- 1 | Util module 2 | =========== 3 | 4 | .. automodule:: gevent_tools.util 5 | :members: -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - 2.6 5 | - 2.7 6 | 7 | before_install: 8 | - sudo apt-get install python-dev libevent-dev 9 | 10 | script: python setup.py test 11 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | - make sure exit codes are correct 2 | 3 | To think about / design: 4 | - Generalized timeouts. Part of AsyncManager interface? 5 | - AsyncManager "backends". gevent, eventlet, threads, subprocesses 6 | - Multiprocess support. Pistil? 7 | - Shared port bindings. Based on Pistil, expose with a Setting subclass Binding? 8 | 9 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (http://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py26, py27 8 | 9 | [testenv] 10 | commands = nosetests 11 | deps = 12 | gevent 13 | nose 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .virtualenv 2 | NOTES 3 | *.log 4 | *.pid 5 | .coverage 6 | docs/_build 7 | .tox 8 | 9 | .DS_Store 10 | *.py[co] 11 | 12 | # Packages 13 | *.egg 14 | *.egg-info 15 | dist 16 | build 17 | eggs 18 | parts 19 | bin 20 | var 21 | sdist 22 | develop-eggs 23 | .installed.cfg 24 | 25 | # Installer logs 26 | pip-log.txt 27 | 28 | # some emacs files 29 | *~ 30 | \#*\# 31 | .\#*\# 32 | .\#* 33 | -------------------------------------------------------------------------------- /docs/user/advanced.rst: -------------------------------------------------------------------------------- 1 | Advanced Usage and Patterns 2 | =========================== 3 | 4 | Service State Machine 5 | --------------------- 6 | 7 | TODO 8 | 9 | Service Factory in Config 10 | ------------------------- 11 | 12 | TODO 13 | 14 | Using Configuration Groups 15 | -------------------------- 16 | 17 | TODO 18 | 19 | Using ZeroMQ 20 | ------------ 21 | 22 | TODO 23 | 24 | Async Backends 25 | -------------- 26 | 27 | TODO 28 | 29 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: test 2 | 3 | test: 4 | nosetests 5 | 6 | dev: 7 | pip install -r requirements.txt 8 | python setup.py develop 9 | 10 | install: 11 | python setup.py install 12 | 13 | coverage: 14 | nosetests --with-coverage --cover-package=ginkgo 15 | 16 | build_pages: 17 | export branch=$$(git status | grep 'On branch' | cut -f 4 -d ' '); \ 18 | git checkout gh-pages; \ 19 | git commit --allow-empty -m 'trigger pages rebuild'; \ 20 | git push origin gh-pages; \ 21 | git checkout $$branch 22 | -------------------------------------------------------------------------------- /docs/user/install.rst: -------------------------------------------------------------------------------- 1 | .. _install: 2 | 3 | Installation 4 | ============ 5 | 6 | Ginkgo is currently only available via GitHub, as it won't be released on PyPI 7 | until it reaches a stable 1.0 release. 8 | 9 | Get the Code 10 | ------------ 11 | You can either clone the public repository: 12 | 13 | :: 14 | 15 | $ git clone git://github.com/progrium/ginkgo.git 16 | 17 | Download the tarball: 18 | 19 | :: 20 | 21 | $ curl -OL https://github.com/progrium/ginkgo/tarball/master 22 | 23 | Or, download the zipball: 24 | 25 | :: 26 | 27 | $ curl -OL https://github.com/progrium/ginkgo/zipball/master 28 | 29 | Once you have a copy of the source, you can embed it in your Python package, or install it into your site-packages easily: 30 | 31 | :: 32 | 33 | $ python setup.py install 34 | -------------------------------------------------------------------------------- /docs/service.rst: -------------------------------------------------------------------------------- 1 | Service component 2 | ================= 3 | 4 | .. autoclass:: ginkgo.core.Service 5 | 6 | .. attribute:: started 7 | 8 | This property returns whether this service has been started 9 | 10 | .. autoattribute:: ginkgo.core.Service.ready 11 | .. automethod:: ginkgo.core.Service._ready 12 | .. automethod:: ginkgo.core.Service.add_service 13 | .. automethod:: ginkgo.core.Service.remove_service 14 | .. automethod:: ginkgo.core.Service._start 15 | .. automethod:: ginkgo.core.Service._stop 16 | .. automethod:: ginkgo.core.Service.start 17 | .. automethod:: ginkgo.core.Service.stop 18 | .. automethod:: ginkgo.core.Service.serve_forever 19 | .. automethod:: ginkgo.core.Service.spawn 20 | .. automethod:: ginkgo.core.Service.spawn_later 21 | .. automethod:: ginkgo.core.Service.catch -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | from ginkgo import config 2 | 3 | def test_config(): 4 | c = config.Config() 5 | c.load({ 6 | "foo": "bar", 7 | "bar.foo": "qux", 8 | "bar.boo": "bar", 9 | "bar.baz.foo": "bar", 10 | "bar.baz.bar": "bar"}) 11 | 12 | assert c.get("foo") == "bar" 13 | 14 | class MyClass(object): 15 | foo = c.setting("foo", help="This is foo.", monitored=True) 16 | 17 | def override(self): 18 | self.foo = "qux" 19 | 20 | o = MyClass() 21 | assert o.foo == "bar" 22 | assert o.foo.changed == False 23 | assert o.foo.changed == False 24 | c.set("foo", "BAZ") 25 | assert o.foo.changed == True 26 | assert o.foo.changed == False 27 | assert MyClass.foo == "BAZ" 28 | 29 | o.override() 30 | assert MyClass.foo == "BAZ" 31 | assert o.foo == "qux" 32 | 33 | g = c.group() 34 | assert g.foo == "BAZ" 35 | assert g.bar.__class__ == config.Group 36 | assert g.bar.boo == "bar" 37 | assert g.bar.tree == None 38 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | from setuptools import setup, find_packages 4 | 5 | from ginkgo import __version__ 6 | 7 | setup( 8 | name='Ginkgo', 9 | version=__version__+"dev", 10 | author='Jeff Lindsay', 11 | author_email='jeff.lindsay@twilio.com', 12 | description='Lightweight service framework', 13 | packages=find_packages(), 14 | setup_requires=['nose'], 15 | tests_require=['nose'], 16 | test_suite='nose.collector', 17 | data_files=[], 18 | classifiers = [ 19 | "Development Status :: 4 - Beta", 20 | "Intended Audience :: Developers", 21 | "License :: OSI Approved :: MIT License", 22 | "Operating System :: OS Independent", 23 | "Programming Language :: Python", 24 | "Programming Language :: Python :: 2.6", 25 | "Programming Language :: Python :: 2.7", 26 | "Topic :: Software Development :: Libraries :: Python Modules", 27 | ], 28 | entry_points={ 29 | 'console_scripts': [ 30 | 'ginkgo = ginkgo.runner:run_ginkgo', 31 | 'ginkgoctl = ginkgo.runner:run_ginkgoctl']}, 32 | ) 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2012 by Jeff Lindsay and contributors 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /tests/conf/threading.conf.py: -------------------------------------------------------------------------------- 1 | ## 2 | ## This is not actually a good example! It's being used for testing 3 | ## purposes since there are no tests yet... 4 | ## 5 | 6 | import logging 7 | from ginkgo import Service as _Service 8 | from ginkgo import Setting 9 | 10 | 11 | delay = 1 12 | 13 | class flask: 14 | debug = False 15 | testing = True 16 | secret_key = "woifh28fhw93fh" 17 | 18 | class subgroup: 19 | foo = "bar" 20 | 21 | class MyService(_Service): 22 | foo = Setting("foo", default=("foo", 12), help="This is foo") 23 | bar = Setting("bar", help="This is bar", monitored=True) 24 | delay = Setting("delay", default=1, help="Delay between hello printing") 25 | 26 | def __init__(self): 27 | import logging 28 | self.log = logging.getLogger(__name__) 29 | 30 | def do_start(self): 31 | self.log.info("Hello here") 32 | self.spawn(self.loop) 33 | 34 | def do_reload(self): 35 | self.log.info("reloaded!") 36 | self.log.info("changed: {}".format(self.bar.changed)) 37 | 38 | def loop(self): 39 | while True: 40 | self.log.info("hello") 41 | self.async.sleep(self.delay) 42 | 43 | service = MyService 44 | -------------------------------------------------------------------------------- /UPGRADING: -------------------------------------------------------------------------------- 1 | 2 | * Change imports of `ginkgo.config.Setting` to `ginkgo.Setting` 3 | * Change imports of `ginkgo.core.Service` to `ginkgo.Service` 4 | * Use `.ready` instead of `.started` on Service objects 5 | * The `doc` named argument of Setting is now `help` 6 | * If you access settings outside of classes like this: 7 | 8 | Setting("setting").value 9 | 10 | Then you should just import ginkgo and do this: 11 | 12 | ginkgo.settings.get("setting") 13 | 14 | * If you use any gevent-based server (based on baseserver, 15 | StreamServer, etc), you need to explicitly wrap it when adding 16 | to a service using ginkgo.async.gevent.ServerWrapper. For 17 | example, this code: 18 | 19 | from gevent.server import StreamServer 20 | class MyService(Service): 21 | def __init__(self): 22 | self.server = StreamServer(...) 23 | self.add_service(self.server) 24 | 25 | Would need to become: 26 | 27 | from gevent.server import StreamServer 28 | from ginkgo.async.gevent import ServerWrapper 29 | class MyService(Service): 30 | def __init__(self): 31 | self.server = StreamServer(...) 32 | self.add_service(ServerWrapper(self.server)) 33 | -------------------------------------------------------------------------------- /tests/conf/gevent.conf.py: -------------------------------------------------------------------------------- 1 | ## 2 | ## This is not actually a good example! It's being used for testing 3 | ## purposes since there are no tests yet... 4 | ## 5 | 6 | import logging 7 | from ginkgo import Service as _Service 8 | from ginkgo import Setting 9 | 10 | 11 | delay = 1 12 | 13 | class flask: 14 | debug = False 15 | testing = True 16 | secret_key = "woifh28fhw93fh" 17 | 18 | class subgroup: 19 | foo = "bar" 20 | 21 | class MyService(_Service): 22 | foo = Setting("foo", default=("foo", 12), help="This is foo") 23 | bar = Setting("bar", help="This is bar", monitored=True) 24 | delay = Setting("delay", default=1, help="Delay between hello printing") 25 | 26 | def __init__(self): 27 | import logging 28 | self.log = logging.getLogger(__name__) 29 | 30 | def do_start(self): 31 | self.log.info("Hello here") 32 | self.spawn(self.loop) 33 | 34 | def do_reload(self): 35 | self.log.info("reloaded!") 36 | self.log.info("changed: {}".format(self.bar.changed)) 37 | 38 | def loop(self): 39 | while True: 40 | self.log.info("hello") 41 | self.async.sleep(self.delay) 42 | 43 | async = 'ginkgo.async.gevent' 44 | service = MyService 45 | -------------------------------------------------------------------------------- /ginkgo/__init__.py: -------------------------------------------------------------------------------- 1 | """Ginkgo module 2 | 3 | This toplevel module exposes most of the API you would use making Ginkgo 4 | applications. Rarely would you need to import from other modules, unless you're 5 | doing tests or something more advanced. This is a simple example of the common 6 | case building Ginkgo services:: 7 | 8 | from ginkgo import Service 9 | from ginkgo import Setting 10 | 11 | class HelloWorld(Service): 12 | message = Setting("message", default="Hello world") 13 | 14 | def do_start(self): 15 | self.spawn(self.message_forever) 16 | 17 | def message_forever(self): 18 | while True: 19 | print self.message 20 | self.async.sleep(1) 21 | 22 | """ 23 | import sys 24 | 25 | from .config import Config 26 | 27 | process = None 28 | settings = Config() 29 | Setting = lambda *args, **kwargs: settings.setting(*args, **kwargs) 30 | 31 | # Set the singleton location for Config global context 32 | Config.singleton_attr = (sys.modules[__name__], 'settings') 33 | 34 | from .core import Service 35 | 36 | __all__ = ["Service", "Setting", "process", "settings"] 37 | __author__ = "Jeff Lindsay " 38 | __license__ = "MIT" 39 | __version__ = ".".join(map(str, (0, 6, 0))) 40 | -------------------------------------------------------------------------------- /ginkgo/async/__init__.py: -------------------------------------------------------------------------------- 1 | """Async driver modules 2 | 3 | This module provides the base class for `AsyncManager` classes for different 4 | async drivers. This provides a unified interface to async primitives, 5 | regardless of whether you're using gevent, eventlet, threading, or 6 | multiprocessing. The `AsyncManager` also manages a pool of async workers, whatever 7 | they are. Since each `Service` has an `AsyncManager`, all `Service` objects 8 | also have their own pool of async workers. 9 | 10 | By default, all services use `ginkgo.async.gevent` to find their 11 | `AsyncManager`. But you can change this by setting `async` in your service 12 | class definition:: 13 | 14 | class NonGeventService(Service): 15 | async = "path.to.different.module" 16 | 17 | """ 18 | import signal 19 | from ..core import BasicService 20 | 21 | class AbstractAsyncManager(BasicService): 22 | def spawn(self, func, *args, **kwargs): 23 | raise NotImplementedError() 24 | 25 | def spawn_later(self, seconds, func, *args, **kwargs): 26 | raise NotImplementedError() 27 | 28 | def sleep(self, seconds): 29 | raise NotImplementedError() 30 | 31 | def queue(self, *args, **kwargs): 32 | raise NotImplementedError() 33 | 34 | def event(self, *args, **kwargs): 35 | raise NotImplementedError() 36 | 37 | def lock(self, *args, **kwargs): 38 | raise NotImplementedError() 39 | 40 | def signal(self, *args, **kwargs): 41 | return signal.signal(*args, **kwargs) 42 | 43 | def init(self): 44 | pass 45 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Ginkgo Service Framework 2 | ======================== 3 | 4 | | Release: v\ |version| (:ref:`Installation `) 5 | | License: MIT 6 | 7 | 8 | Ginkgo is a lightweight framework for writing network service daemons in 9 | Python. It currently focuses on gevent as its core networking and concurrency 10 | layer. 11 | 12 | The core idea behind Ginkgo is the "service model", where your primary building 13 | block or component of applications are composable services. A service is a 14 | mostly self-contained module of your application that can start/stop/reload, 15 | contain other services, manage async operations, and expose configuration. 16 | 17 | :: 18 | 19 | class ExampleService(Service): 20 | setting = Setting("example.setting", default="Foobar") 21 | 22 | def __init__(self): 23 | logging.info("Service is initializing.") 24 | 25 | self.child_service = AnotherService() 26 | self.add_service(self.child_service) 27 | 28 | def do_start(self): 29 | logging.info("Service is starting.") 30 | 31 | self.spawn(self.something_async) 32 | 33 | def do_stop(self): 34 | logging.info("Service is stopping.") 35 | 36 | def do_reload(self): 37 | logging.info("Service is reloading.") 38 | 39 | # ... 40 | 41 | Around this little bit of structure and convention, Ginkgo provides just a few 42 | baseline features to make building both complex and simple network daemons much 43 | easier: 44 | 45 | - Service class primitive for composing daemon apps from simple components 46 | - Dynamic configuration loaded from regular Python source files 47 | - Runner and service manager tool for easy, consistent usage and deployment 48 | - Integrated support for standard Python logging 49 | 50 | User Guide 51 | ========== 52 | 53 | .. toctree:: 54 | :maxdepth: 2 55 | 56 | user/intro 57 | user/install 58 | user/quickstart 59 | user/manual 60 | user/advanced 61 | 62 | API Reference 63 | ============= 64 | 65 | .. toctree:: 66 | :maxdepth: 2 67 | 68 | api 69 | 70 | Developer Guide 71 | =============== 72 | 73 | .. toctree:: 74 | :maxdepth: 1 75 | 76 | dev/internals 77 | dev/todo 78 | dev/authors 79 | 80 | -------------------------------------------------------------------------------- /ginkgo/async/eventlet.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import eventlet 4 | import eventlet.greenpool 5 | import eventlet.greenthread 6 | import eventlet.event 7 | import eventlet.queue 8 | import eventlet.timeout 9 | import eventlet.semaphore 10 | 11 | from ..core import BasicService, Service 12 | from ..util import defaultproperty 13 | 14 | class AsyncManager(BasicService): 15 | """Async primitives from eventlet""" 16 | stop_timeout = defaultproperty(int, 1) 17 | 18 | def __init__(self): 19 | self._greenlets = eventlet.greenpool.GreenPool() 20 | 21 | def do_stop(self): 22 | if eventlet.greenthread.getcurrent() in self._greenlets.coroutines_running: 23 | return eventlet.spawn(self.do_stop).join() 24 | if self._greenlets.running(): 25 | with eventlet.timeout.Timeout(self.stop_timeout, False): 26 | self._greenlets.waitall() # put in timeout for stop_timeout 27 | for g in list(self._greenlets.coroutines_running): 28 | with eventlet.timeout.Timeout(1, False): 29 | g.kill() # timeout of 1 sec? 30 | 31 | def spawn(self, func, *args, **kwargs): 32 | """Spawn a greenlet under this service""" 33 | return self._greenlets.spawn(func, *args, **kwargs) 34 | 35 | def spawn_later(self, seconds, func, *args, **kwargs): 36 | """Spawn a greenlet in the future under this service""" 37 | def spawner(): 38 | self.spawn(func, *args, **kwargs) 39 | return eventlet.spawn_after(seconds, spawner) 40 | 41 | def sleep(self, seconds): 42 | return eventlet.sleep(seconds) 43 | 44 | def queue(self, *args, **kwargs): 45 | return eventlet.queue.Queue(*args, **kwargs) 46 | 47 | def event(self, *args, **kwargs): 48 | return Event(*args, **kwargs) 49 | 50 | def lock(self, *args, **kwargs): 51 | return eventlet.semaphore.Semaphore(*args, **kwargs) 52 | 53 | class Event(eventlet.event.Event): 54 | def clear(self): 55 | if not self.ready(): 56 | return 57 | self.reset() 58 | 59 | def set(self): 60 | self.send() 61 | 62 | def wait(self, timeout=None): 63 | if timeout: 64 | with eventlet.timeout.Timeout(timeout, False): 65 | super(Event, self).wait() 66 | else: 67 | super(Event, self).wait() 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ginkgo v0.5.0dev 2 | 3 | Lightweight service framework on top of gevent, implementing the "service model" -- services all the way down. 4 | 5 | Please note that this version is major rewrite since 0.3.1, hence 6 | skipping 0.4.0. Take a look at UPGRADING and some of the documentation 7 | might be out of date during transition. 8 | 9 | ## Features 10 | * Service model -- Break your app down into services and sub-services. 11 | Modules, if you will, that can start, stop, and reload. Every service 12 | manages its own pool of greenlets. 13 | * Configuration -- Built-in, reloadable configuration based on Python 14 | files. Access configuration settings relative to services. 15 | * Runner -- Command-line tool to manage your service that can daemonize, 16 | chroot, drop privs, and set up or override configuration. 17 | 18 | ## Demo 19 | A talk was given at PyCon 2012 called "Throwing Together Distributed 20 | Services with gevent" that used Ginkgo to build a number of simple 21 | services combined to make a more complex distributed service. 22 | 23 | * [PyCon 2012 Video](http://pyvideo.org/video/642/throwing-together-distributed-services-with-geven) 24 | * [PyCon 2012 Slides](http://dl.dropbox.com/u/2096290/GinkgoPyCon.pdf) 25 | * [Source code used in talk](https://github.com/progrium/ginkgotutorial) 26 | 27 | ## Mailing List 28 | 29 | Pretty active discussion on this early microframework. Join it or just 30 | read what's being planned: 31 | 32 | * [Ginkgo-dev Google Group](http://groups.google.com/group/ginkgo-dev) 33 | 34 | ## Contributing 35 | 36 | Feel free to poke around the issues in the main repository and see if you can tackle any. From there you should: 37 | 38 | * Fork if you haven't 39 | * Create a branch for the feature / issue 40 | * Write code+tests 41 | * Pass tests (using nose) 42 | * Squash branch commits using [merge and reset](http://j.mp/vHLUoa) 43 | * Send pull request 44 | 45 | We highly recommend using branches for all features / issues and then squashing it into a single commit in your master before issuing a pull request. It's actually quite easy using [merge and reset](http://j.mp/vHLUoa). This helps keep features and issues consolidated, but also makes pull requests easier to read, which increases the speed and likelihood of being accepted. 46 | 47 | We're aiming for at least 90% test coverage. If you have the `coverage` Python package installed, you can run `python setup.py coverage` to get a coverage report of modules within gservice. 48 | 49 | ## Contributors 50 | 51 | * [Jeff Lindsay](jeff.lindsay@twilio.com) 52 | * [Sean McQuillan](sean@twilio.com) 53 | * [Alan Shreve](ashreve@twilio.com) 54 | * [Chad Selph](chad@twilio.com) 55 | * [Ryan Larrabure](ryan@twilio.com) 56 | * [Marc Abramowitz](marc@marc-abramowitz.com) 57 | * [David Wilemski](david@davidwilemski.com) 58 | 59 | ## License 60 | 61 | MIT 62 | 63 | -------------------------------------------------------------------------------- /docs/user/intro.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | ============ 3 | 4 | Origin 5 | ------ 6 | Ginkgo evolved from a project called "gevent_tools" that started as a 7 | collection of common features needed when building gevent applications. The 8 | author had previously made a habit of building lots of interesting little 9 | servers as a hobby, and then at work found himself writing and dealing with 10 | lots more given the company's service oriented architecture. Accustomed to using 11 | the application framework in Twisted, when he finally saw the light and 12 | discovered gevent, there was no such framework for that paradigm. 13 | 14 | Dealing with so many projects, it was not practical to reinvent the same basic 15 | features and architecture over and over again. The same way web frameworks made 16 | it easy to "throw together" a web application, there needed to be a way to 17 | quickly "throw together" network daemons. Not just simple one-off servers, but 18 | large-scale, complex applications -- often part of a larger distributed system. 19 | 20 | Through the experience of building large systems, a pattern emerged that was 21 | like a looser, more object-oriented version of the actor model based around the 22 | idea of services. This became the main feature of gevent_tools and it was later 23 | renamed gservice. However, with the hope of supporting other async mechanisms 24 | other than gevent's green threads (such as actual threads or processes, or 25 | other similar network libraries), the project was renamed Ginkgo. 26 | 27 | Vision 28 | ------ 29 | The Ginkgo microframework is a minimalist foundation for building very large 30 | systems, beyond individual daemons. There were originally plans for 31 | gevent_tools to include higher-level modules to aid in developing distributed 32 | applications, such as service discovery and messaging primitives. 33 | 34 | While Ginkgo will remain focused on "baseline" features common to pretty much 35 | all network daemons, a supplementary project to act as a "standard library" for 36 | Ginkgo applications is planned. Together with Ginkgo, the vision would be to 37 | quickly "throw together" distributed systems from simple primitives. 38 | 39 | Inspiration 40 | ----------- 41 | Most of Ginkgo was envisioned by taking good ideas from other projects, 42 | simplifying to their essential properties, and integrating them together. A lot 43 | of thanks goes out to these projects. 44 | 45 | Twisted is the first great Python evented daemon framework. The two big ideas 46 | borrowed from Twisted are their application framework and twistd. They directly 47 | inspired the service model and the Ginkgo runner. 48 | 49 | Trac is known for the problem it solves, and not so much for its great 50 | architecture. However, its component model and configuration API were a big 51 | influence on Ginkgo. Trac components are how we think of Ginkgo services, and 52 | the way Ginkgo defines configuration settings is directly inspired by the Trac 53 | configuration API. 54 | 55 | These projects also had some influence on Ginkgo's design and philosophy: 56 | Gunicorn, Mongrel, Apache, Django, Flask, python-daemon, Diesel, Tornado, 57 | Erlang/OTP, Typeface, Akka, Configgy, Ostrich, and others. 58 | -------------------------------------------------------------------------------- /ginkgo/async/threading.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import threading 4 | import Queue 5 | import time 6 | 7 | from ..util import defaultproperty 8 | from ..async import AbstractAsyncManager 9 | 10 | def _spin_wait(fn, timeout): 11 | """ Spin-wait blocking only 1 second at a time so we don't miss signals """ 12 | elapsed = 0 13 | while True: 14 | if fn(timeout=1): 15 | return True 16 | elapsed += 1 17 | if timeout is not None and elapsed >= timeout: 18 | return False 19 | 20 | 21 | class Event(threading._Event): 22 | def wait(self, timeout=None): 23 | return _spin_wait(super(Event, self).wait, timeout) 24 | 25 | 26 | class Thread(threading.Thread): 27 | def join(self, timeout=None): 28 | return _spin_wait(super(Thread, self).join, timeout) 29 | 30 | 31 | class Timer(threading._Timer): 32 | def join(self, timeout=None): 33 | return _spin_wait(super(Timer, self).join, timeout) 34 | 35 | 36 | class AsyncManager(AbstractAsyncManager): 37 | """Async manager for threads""" 38 | stop_timeout = defaultproperty(int, 1) 39 | 40 | def __init__(self): 41 | # _lock protects the _threads structure 42 | print ("The ginkgo.async.threading manager should not be used in " 43 | "production environments due to the known limitations of the GIL") 44 | self._lock = threading.Lock() 45 | self._threads = [] 46 | 47 | def do_stop(self): 48 | """ 49 | Beware! This function has different behavior than the gevent 50 | async manager in the following respects: 51 | - The stop timeout is used for joining each thread instead of 52 | for all of the threads collectively. 53 | - If the threads do not successfully join within their timeout, 54 | they will not be killed since there is no safe way to do this. 55 | """ 56 | with self._lock: 57 | reentrant_stop = threading.current_thread() in self._threads 58 | 59 | if reentrant_stop: 60 | t = Thread(target=self.do_stop) 61 | t.daemon=True 62 | t.start() 63 | return t.join() 64 | 65 | with self._lock: 66 | for t in self._threads: 67 | t.join(self.stop_timeout) 68 | 69 | def spawn(self, func, *args, **kwargs): 70 | """Spawn a greenlet under this service""" 71 | t = Thread(target=func, args=args, kwargs=kwargs) 72 | with self._lock: 73 | self._threads.append(t) 74 | t.daemon=True 75 | t.start() 76 | return t 77 | 78 | def spawn_later(self, seconds, func, *args, **kwargs): 79 | """Spawn a greenlet in the future under this service""" 80 | t = Timer(group=self, target=func, args=args, kwargs=kwargs) 81 | with self._lock: 82 | self._threads.append(t) 83 | t.daemon=True 84 | t.start() 85 | return t 86 | 87 | def sleep(self, seconds): 88 | return time.sleep(seconds) 89 | 90 | def queue(self, *args, **kwargs): 91 | return Queue.Queue(*args, **kwargs) 92 | 93 | def event(self, *args, **kwargs): 94 | return Event(*args, **kwargs) 95 | 96 | def lock(self, *args, **kwargs): 97 | return threading.Lock(*args, **kwargs) 98 | -------------------------------------------------------------------------------- /ginkgo/logger.py: -------------------------------------------------------------------------------- 1 | """Ginkgo logger 2 | 3 | This module provides the class for a logger object used by the runner module's 4 | `Process` object to manage, configure, and provide services around Python's 5 | standard logging module. Most notably it allows you to easily configure the 6 | Python logger using Ginkgo configuration. 7 | 8 | """ 9 | import logging 10 | import logging.config 11 | import os 12 | import os.path 13 | import sys 14 | 15 | import ginkgo 16 | 17 | DEFAULT_FORMAT = "%(asctime)s %(levelname) 7s %(module)s: %(message)s" 18 | 19 | class Logger(object): 20 | logfile = ginkgo.Setting("logfile", default=None, help=""" 21 | Path to primary log file. Ignored if logconfig is set. 22 | """) 23 | loglevel = ginkgo.Setting("loglevel", default='debug', help=""" 24 | Log level to use. Valid options: debug, info, warning, critical 25 | Ignored if logconfig is set. 26 | """) 27 | config = ginkgo.Setting("logconfig", default=None, help=""" 28 | Configuration of standard Python logger. Can be dict for basicConfig, 29 | dict with version key for dictConfig, or ini filepath for fileConfig. 30 | """) 31 | 32 | def __init__(self, process): 33 | self.process = process 34 | 35 | if self.logfile is None: 36 | process.config.set("logfile", os.path.expanduser( 37 | "~/.{}.log".format(process.service_name))) 38 | 39 | self.load_config() 40 | 41 | def load_config(self): 42 | if self.config is None: 43 | self._load_default_config() 44 | else: 45 | if isinstance(self.config, str) and os.path.exists(self.config): 46 | logging.config.fileConfig(self.config) 47 | elif 'version' in self.config: 48 | logging.config.dictConfig(self.config) 49 | else: 50 | self._reset_basic_config(self.config) 51 | 52 | def _load_default_config(self): 53 | default_config = dict( 54 | format=DEFAULT_FORMAT, 55 | level=getattr(logging, self.loglevel.upper())) 56 | if hasattr(self.process, 'pidfile'): 57 | default_config['filename'] = self.logfile 58 | self._reset_basic_config(default_config) 59 | 60 | def _reset_basic_config(self, config): 61 | for h in logging.root.handlers[:]: 62 | logging.root.removeHandler(h) 63 | logging.basicConfig(**config) 64 | 65 | def capture_stdio(self): 66 | # TODO: something smarter than this? 67 | try: 68 | os.dup2(logging._handlerList[0]().stream.fileno(), sys.stdout.fileno()) 69 | os.dup2(logging._handlerList[0]().stream.fileno(), sys.stderr.fileno()) 70 | except: 71 | pass 72 | 73 | @property 74 | def file_descriptors(self): 75 | return [handler.stream.fileno() for handler in [wr() for wr in 76 | logging._handlerList] if isinstance(handler, logging.FileHandler)] 77 | 78 | def shutdown(self): 79 | logging.shutdown() 80 | 81 | def print_log(self): 82 | with open(self.logfile, "r") as f: 83 | print f.read() 84 | 85 | def tail_log(self): 86 | with open(self.logfile, "r") as f: 87 | lines = f.readlines() 88 | for line in lines[-20:]: 89 | print line.strip() 90 | while True: 91 | line = f.readline() 92 | if line: 93 | print line.strip() 94 | -------------------------------------------------------------------------------- /tests/test_util.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from ginkgo import util 4 | 5 | class GlobalContextTest(unittest.TestCase): 6 | singleton_a = None 7 | singleton_b = None 8 | 9 | def setUp(self): 10 | GlobalContextTest.singleton_a = object() 11 | GlobalContextTest.singleton_b = object() 12 | 13 | 14 | def test_push_pop_of_singleton(self): 15 | class TestContext(util.GlobalContext): 16 | singleton_attr = (GlobalContextTest, 'singleton_a') 17 | def _singleton_id(): 18 | return id(GlobalContextTest.singleton_a) 19 | original_id = _singleton_id() 20 | new_object = object() 21 | TestContext._push_context(new_object) 22 | assert not _singleton_id() == original_id 23 | assert _singleton_id() == id(new_object) 24 | TestContext._pop_context() 25 | assert not _singleton_id() == id(new_object) 26 | assert _singleton_id() == original_id 27 | 28 | def test_nested_push_pop_of_two_singletons(self): 29 | class TestContext(util.GlobalContext): 30 | singleton_attr = (GlobalContextTest, 'singleton_a') 31 | def _singleton_id(): 32 | return id(GlobalContextTest.singleton_a) 33 | original_id = _singleton_id() 34 | first_object = object() 35 | second_object = object() 36 | assert _singleton_id() == original_id 37 | TestContext._push_context(first_object) 38 | assert _singleton_id() == id(first_object) 39 | TestContext._push_context(second_object) 40 | assert _singleton_id() == id(second_object) 41 | TestContext._pop_context() 42 | assert _singleton_id() == id(first_object) 43 | TestContext._pop_context() 44 | assert _singleton_id() == original_id 45 | 46 | def test_multiple_global_contexts(self): 47 | class FirstContext(util.GlobalContext): 48 | singleton_attr = (GlobalContextTest, 'singleton_a') 49 | class SecondContext(util.GlobalContext): 50 | singleton_attr = (GlobalContextTest, 'singleton_b') 51 | def _first_singleton_id(): 52 | return id(GlobalContextTest.singleton_a) 53 | def _second_singleton_id(): 54 | return id(GlobalContextTest.singleton_b) 55 | first_original_id = _first_singleton_id() 56 | second_original_id = _second_singleton_id() 57 | assert not first_original_id == second_original_id 58 | first_object = object() 59 | second_object = object() 60 | FirstContext._push_context(first_object) 61 | assert _first_singleton_id() == id(first_object) 62 | assert not _second_singleton_id() == id(first_object) 63 | SecondContext._push_context(second_object) 64 | assert _second_singleton_id() == id(second_object) 65 | assert not _first_singleton_id() == id(second_object) 66 | assert _first_singleton_id() == id(first_object) 67 | FirstContext._pop_context() 68 | assert _first_singleton_id() == first_original_id 69 | SecondContext._pop_context() 70 | assert _second_singleton_id() == second_original_id 71 | 72 | def test_context_manager(self): 73 | class TestContext(util.GlobalContext): pass 74 | TestContext.singleton_attr = (GlobalContextTest, 'singleton_a') 75 | GlobalContextTest.singleton_a = TestContext() 76 | original_id = id(GlobalContextTest.singleton_a) 77 | new_context = TestContext() 78 | with new_context: 79 | assert not original_id == id(GlobalContextTest.singleton_a) 80 | assert id(new_context) == id(GlobalContextTest.singleton_a) 81 | assert original_id == id(GlobalContextTest.singleton_a) 82 | 83 | def test_nested_context_managers(self): 84 | class TestContext(util.GlobalContext): pass 85 | TestContext.singleton_attr = (GlobalContextTest, 'singleton_a') 86 | GlobalContextTest.singleton_a = TestContext() 87 | original_id = id(GlobalContextTest.singleton_a) 88 | first_context = TestContext() 89 | second_context = TestContext() 90 | with first_context: 91 | assert id(first_context) == id(GlobalContextTest.singleton_a) 92 | with second_context: 93 | assert id(second_context) == id(GlobalContextTest.singleton_a) 94 | assert id(first_context) == id(GlobalContextTest.singleton_a) 95 | assert original_id == id(GlobalContextTest.singleton_a) 96 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | dirhtml: 45 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 48 | 49 | singlehtml: 50 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 51 | @echo 52 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 53 | 54 | pickle: 55 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 56 | @echo 57 | @echo "Build finished; now you can process the pickle files." 58 | 59 | json: 60 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 61 | @echo 62 | @echo "Build finished; now you can process the JSON files." 63 | 64 | htmlhelp: 65 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 66 | @echo 67 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 68 | ".hhp project file in $(BUILDDIR)/htmlhelp." 69 | 70 | qthelp: 71 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 72 | @echo 73 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 74 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 75 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/gevent-tools.qhcp" 76 | @echo "To view the help file:" 77 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/gevent-tools.qhc" 78 | 79 | devhelp: 80 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 81 | @echo 82 | @echo "Build finished." 83 | @echo "To view the help file:" 84 | @echo "# mkdir -p $$HOME/.local/share/devhelp/gevent-tools" 85 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/gevent-tools" 86 | @echo "# devhelp" 87 | 88 | epub: 89 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 90 | @echo 91 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 92 | 93 | latex: 94 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 95 | @echo 96 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 97 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 98 | "(use \`make latexpdf' here to do that automatically)." 99 | 100 | latexpdf: 101 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 102 | @echo "Running LaTeX files through pdflatex..." 103 | make -C $(BUILDDIR)/latex all-pdf 104 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 105 | 106 | text: 107 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 108 | @echo 109 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 110 | 111 | man: 112 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 113 | @echo 114 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 115 | 116 | changes: 117 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 118 | @echo 119 | @echo "The overview file is in $(BUILDDIR)/changes." 120 | 121 | linkcheck: 122 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 123 | @echo 124 | @echo "Link check complete; look for any errors in the above output " \ 125 | "or in $(BUILDDIR)/linkcheck/output.txt." 126 | 127 | doctest: 128 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 129 | @echo "Testing of doctests in the sources finished, look at the " \ 130 | "results in $(BUILDDIR)/doctest/output.txt." 131 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=_build 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . 10 | if NOT "%PAPER%" == "" ( 11 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 12 | ) 13 | 14 | if "%1" == "" goto help 15 | 16 | if "%1" == "help" ( 17 | :help 18 | echo.Please use `make ^` where ^ is one of 19 | echo. html to make standalone HTML files 20 | echo. dirhtml to make HTML files named index.html in directories 21 | echo. singlehtml to make a single large HTML file 22 | echo. pickle to make pickle files 23 | echo. json to make JSON files 24 | echo. htmlhelp to make HTML files and a HTML help project 25 | echo. qthelp to make HTML files and a qthelp project 26 | echo. devhelp to make HTML files and a Devhelp project 27 | echo. epub to make an epub 28 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 29 | echo. text to make text files 30 | echo. man to make manual pages 31 | echo. changes to make an overview over all changed/added/deprecated items 32 | echo. linkcheck to check all external links for integrity 33 | echo. doctest to run all doctests embedded in the documentation if enabled 34 | goto end 35 | ) 36 | 37 | if "%1" == "clean" ( 38 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 39 | del /q /s %BUILDDIR%\* 40 | goto end 41 | ) 42 | 43 | if "%1" == "html" ( 44 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 45 | if errorlevel 1 exit /b 1 46 | echo. 47 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 48 | goto end 49 | ) 50 | 51 | if "%1" == "dirhtml" ( 52 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 53 | if errorlevel 1 exit /b 1 54 | echo. 55 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 56 | goto end 57 | ) 58 | 59 | if "%1" == "singlehtml" ( 60 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 61 | if errorlevel 1 exit /b 1 62 | echo. 63 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 64 | goto end 65 | ) 66 | 67 | if "%1" == "pickle" ( 68 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 69 | if errorlevel 1 exit /b 1 70 | echo. 71 | echo.Build finished; now you can process the pickle files. 72 | goto end 73 | ) 74 | 75 | if "%1" == "json" ( 76 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 77 | if errorlevel 1 exit /b 1 78 | echo. 79 | echo.Build finished; now you can process the JSON files. 80 | goto end 81 | ) 82 | 83 | if "%1" == "htmlhelp" ( 84 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 85 | if errorlevel 1 exit /b 1 86 | echo. 87 | echo.Build finished; now you can run HTML Help Workshop with the ^ 88 | .hhp project file in %BUILDDIR%/htmlhelp. 89 | goto end 90 | ) 91 | 92 | if "%1" == "qthelp" ( 93 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 94 | if errorlevel 1 exit /b 1 95 | echo. 96 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 97 | .qhcp project file in %BUILDDIR%/qthelp, like this: 98 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\gevent-tools.qhcp 99 | echo.To view the help file: 100 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\gevent-tools.ghc 101 | goto end 102 | ) 103 | 104 | if "%1" == "devhelp" ( 105 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 106 | if errorlevel 1 exit /b 1 107 | echo. 108 | echo.Build finished. 109 | goto end 110 | ) 111 | 112 | if "%1" == "epub" ( 113 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 114 | if errorlevel 1 exit /b 1 115 | echo. 116 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 117 | goto end 118 | ) 119 | 120 | if "%1" == "latex" ( 121 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 122 | if errorlevel 1 exit /b 1 123 | echo. 124 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 125 | goto end 126 | ) 127 | 128 | if "%1" == "text" ( 129 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 130 | if errorlevel 1 exit /b 1 131 | echo. 132 | echo.Build finished. The text files are in %BUILDDIR%/text. 133 | goto end 134 | ) 135 | 136 | if "%1" == "man" ( 137 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 138 | if errorlevel 1 exit /b 1 139 | echo. 140 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 141 | goto end 142 | ) 143 | 144 | if "%1" == "changes" ( 145 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 146 | if errorlevel 1 exit /b 1 147 | echo. 148 | echo.The overview file is in %BUILDDIR%/changes. 149 | goto end 150 | ) 151 | 152 | if "%1" == "linkcheck" ( 153 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 154 | if errorlevel 1 exit /b 1 155 | echo. 156 | echo.Link check complete; look for any errors in the above output ^ 157 | or in %BUILDDIR%/linkcheck/output.txt. 158 | goto end 159 | ) 160 | 161 | if "%1" == "doctest" ( 162 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 163 | if errorlevel 1 exit /b 1 164 | echo. 165 | echo.Testing of doctests in the sources finished, look at the ^ 166 | results in %BUILDDIR%/doctest/output.txt. 167 | goto end 168 | ) 169 | 170 | :end 171 | -------------------------------------------------------------------------------- /ginkgo/async/gevent.py: -------------------------------------------------------------------------------- 1 | """Gevent async module 2 | 3 | This module provides the `AsyncManager` for gevent, as well as other utilities 4 | useful for building gevent based Ginkgo apps. Obviously, this is the only async 5 | module at the moment. 6 | 7 | Of note, this module provides wrapped versions of the gevent bundled servers 8 | that should be used instead of the gevent classes. These wrapped classes adapt 9 | gevent servers to be Ginkgo services. They currently include: 10 | 11 | * StreamServer 12 | * WSGIServer (based on the pywsgi.WSGIServer) 13 | * BackdoorServer 14 | 15 | """ 16 | from __future__ import absolute_import 17 | 18 | import gevent 19 | import gevent.event 20 | import gevent.queue 21 | import gevent.timeout 22 | import gevent.pool 23 | import gevent.baseserver 24 | import gevent.socket 25 | 26 | import gevent.backdoor 27 | import gevent.server 28 | import gevent.pywsgi 29 | 30 | from ..core import BasicService, Service 31 | from ..util import defaultproperty, ObjectWrapper 32 | from ..async import AbstractAsyncManager 33 | 34 | class AsyncManager(AbstractAsyncManager): 35 | """Async primitives from gevent""" 36 | stop_timeout = defaultproperty(int, 1) 37 | 38 | def __init__(self): 39 | self._greenlets = gevent.pool.Group() 40 | 41 | def do_stop(self): 42 | if gevent.getcurrent() in self._greenlets: 43 | return gevent.spawn(self.do_stop).join() 44 | if self._greenlets: 45 | self._greenlets.join(timeout=self.stop_timeout) 46 | self._greenlets.kill(block=True, timeout=1) 47 | 48 | def spawn(self, func, *args, **kwargs): 49 | """Spawn a greenlet under this service""" 50 | return self._greenlets.spawn(func, *args, **kwargs) 51 | 52 | def spawn_later(self, seconds, func, *args, **kwargs): 53 | """Spawn a greenlet in the future under this service""" 54 | group = self._greenlets 55 | g = group.greenlet_class(func, *args, **kwargs) 56 | g.start_later(seconds) 57 | group.add(g) 58 | return g 59 | 60 | def sleep(self, seconds): 61 | return gevent.sleep(seconds) 62 | 63 | def queue(self, *args, **kwargs): 64 | return gevent.queue.Queue(*args, **kwargs) 65 | 66 | def event(self, *args, **kwargs): 67 | return gevent.event.Event(*args, **kwargs) 68 | 69 | def lock(self, *args, **kwargs): 70 | return gevent.coros.Semaphore(*args, **kwargs) 71 | 72 | def signal(self, *args, **kwargs): 73 | gevent.signal(*args, **kwargs) 74 | 75 | def init(self): 76 | gevent.reinit() 77 | 78 | 79 | class ServerWrapper(Service): 80 | """Wrapper for gevent servers that are based on gevent.baseserver.BaseServer 81 | 82 | DEPRECATED: Please use the pre-wrapped gevent servers in the same module. 83 | 84 | Although BaseServer objects mostly look like they have the Service interface, 85 | there are certain extra methods (like reload, service_name, etc) that are assumed 86 | to be available. This class allows us to wrap gevent servers so they actually 87 | behave as a Service. There is no automatic wrapping like 0.3.0 and earlier, 88 | so you have to explicitly wrap gevent servers: 89 | 90 | from ginkgo.core import Service 91 | from ginkgo.async.gevent import ServerWrapper 92 | from gevent.pywsgi import WSGIServer 93 | 94 | class MyService(Service): 95 | def __init__(self): 96 | self.server = WSGIServer(('0.0.0.0', 80), self.handle) 97 | self.add_service(ServerWrapper(self.server)) 98 | 99 | """ 100 | def __init__(self, server, *args, **kwargs): 101 | super(ServerWrapper, self).__init__() 102 | if isinstance(server, gevent.baseserver.BaseServer): 103 | self.wrapped = server 104 | else: 105 | raise RuntimeError( 106 | "Object being wrapped is not a BaseServer instance") 107 | 108 | def do_start(self): 109 | self.spawn(self.wrapped.start) 110 | 111 | def do_stop(self): 112 | self.wrapped.stop() 113 | 114 | class StreamClient(Service): 115 | """StreamServer-like TCP client service""" 116 | 117 | def __init__(self, address, handler=None): 118 | self.address = address 119 | self.handler = handler 120 | 121 | def do_start(self): 122 | self.spawn(self.connect) 123 | 124 | def connect(self): 125 | self.handle( 126 | gevent.socket.create_connection(self.address)) 127 | 128 | def handle(self, socket): 129 | if self.handler: 130 | self.handler(socket) 131 | 132 | class _ServerWrapper(Service, ObjectWrapper): 133 | server = state = __subject__ = None 134 | _children = [] 135 | 136 | def __init__(self, *args, **kwargs): 137 | self.server = self.server(*args, **kwargs) 138 | ObjectWrapper.__init__(self, self.server) 139 | 140 | def do_start(self): 141 | self.spawn(self.server.start) 142 | 143 | def do_stop(self): 144 | self.server.stop() 145 | 146 | class StreamServer(_ServerWrapper): 147 | server = gevent.server.StreamServer 148 | 149 | class WSGIServer(_ServerWrapper): 150 | server = gevent.pywsgi.WSGIServer 151 | 152 | class BackdoorServer(_ServerWrapper): 153 | server = gevent.backdoor.BackdoorServer 154 | -------------------------------------------------------------------------------- /docs/user/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ========== 3 | 4 | Before you get started, be sure you have Ginkgo installed. 5 | 6 | Hello World Service 7 | ------------------- 8 | The simplest service you could write looks something like this:: 9 | 10 | from ginkgo import Service 11 | 12 | class HelloWorld(Service): 13 | def do_start(self): 14 | self.spawn(self.hello_forever) 15 | 16 | def hello_forever(self): 17 | while True: 18 | print "Hello World" 19 | self.async.sleep(1) 20 | 21 | If you save this as *hello.py* you can run it with the Ginkgo runner:: 22 | 23 | $ ginkgo hello.HelloWorld 24 | 25 | This should run your service, giving you a stream of "Hello World" lines. 26 | 27 | To stop your service, hit Ctrl+C. 28 | 29 | Writing a Server 30 | ---------------- 31 | A service is not a server until you make it one. Using gevent, this is 32 | easy using the StreamServer service to do the work of running a TCP 33 | server:: 34 | 35 | from ginkgo import Service 36 | from ginkgo.async.gevent import StreamServer 37 | 38 | class HelloWorldServer(Service): 39 | def __init__(self): 40 | self.add_service(StreamServer(('0.0.0.0', 7000), self.handle)) 41 | 42 | def handle(self, socket, address): 43 | while True: 44 | socket.send("Hello World\n") 45 | self.async.sleep(1) 46 | 47 | Save this as *quickstart.py* and run with:: 48 | 49 | $ ginkgo quickstart.HelloWorldServer 50 | 51 | It will start listening on port 7000. We can connect with netcat:: 52 | 53 | $ nc localhost 7000 54 | 55 | Again we see a stream of "Hello World" lines, but this time being sent over 56 | TCP. You can open more netcat connections to see it running concurrently 57 | just fine. 58 | 59 | Notice our HelloWorldServer implementation is *composed* of a generic 60 | StreamServer and doesn't need to implement anything else other than a 61 | handler for that StreamServer. 62 | 63 | Writing a Client 64 | ---------------- 65 | A client that maintains a persistent connection (or maybe pool of 66 | connections) to a server also makes sense to be modeled as a Service. 67 | Let's add a client to our HelloWorldServer in our quickstart module. Now 68 | it looks like this:: 69 | 70 | from ginkgo import Service 71 | from ginkgo.async.gevent import StreamServer 72 | from ginkgo.async.gevent import StreamClient 73 | 74 | class HelloWorldServer(Service): 75 | def __init__(self): 76 | self.add_service(StreamServer(('0.0.0.0', 7000), self.handle)) 77 | 78 | def handle(self, socket, address): 79 | while True: 80 | socket.send("Hello World\n") 81 | self.async.sleep(1) 82 | 83 | class HelloWorldClient(Service): 84 | def __init__(self): 85 | self.add_service(StreamClient(('0.0.0.0', 7000), self.handle)) 86 | 87 | def handle(self, socket): 88 | fileobj = socket.makefile() 89 | while True: 90 | print fileobj.readline().strip() 91 | 92 | Save and run the server first with:: 93 | 94 | $ ginkgo quickstart.HelloWorldServer 95 | 96 | Let that run, switch to a new terminal and run the client with:: 97 | 98 | $ ginkgo quickstart.HelloWorldClient 99 | 100 | As you'd expect, the client connects to the server and prints all the 101 | "Hello World" lines it receives. 102 | 103 | Service Composition 104 | ------------------- 105 | We've already been doing service composition by using generic TCP server 106 | and client services to build our HelloWorld services. These primitives 107 | are services themselves, just like the ones you've been making. So you 108 | can compose and aggregate your own services the same way. 109 | 110 | Let's combine our client and server by add a HelloWorld service in 111 | our quickstart module. It now looks like this:: 112 | 113 | from ginkgo import Service 114 | from ginkgo.async.gevent import StreamServer 115 | from ginkgo.async.gevent import StreamClient 116 | 117 | class HelloWorldServer(Service): 118 | def __init__(self): 119 | self.add_service(StreamServer(('0.0.0.0', 7000), self.handle)) 120 | 121 | def handle(self, socket, address): 122 | while True: 123 | socket.send("Hello World\n") 124 | self.async.sleep(1) 125 | 126 | class HelloWorldClient(Service): 127 | def __init__(self): 128 | self.add_service(StreamClient(('0.0.0.0', 7000), self.handle)) 129 | 130 | def handle(self, socket): 131 | fileobj = socket.makefile() 132 | while True: 133 | print fileobj.readline().strip() 134 | 135 | class HelloWorld(Service): 136 | def __init__(self): 137 | self.add_service(HelloWorldServer()) 138 | self.add_service(HelloWorldClient()) 139 | 140 | Save and we can run our new aggregate service:: 141 | 142 | $ ginkgo quickstart.HelloWorld 143 | 144 | Now the client and server are both running, giving us effectively what 145 | we came in with. 146 | 147 | Using a Web Framework 148 | --------------------- 149 | Adding a web server our HelloWorld service is quite trivial. Here we use 150 | gevent's WSGI server implementation:: 151 | 152 | from ginkgo import Service 153 | from ginkgo.async.gevent import StreamServer 154 | from ginkgo.async.gevent import StreamClient 155 | from ginkgo.async.gevent import WSGIServer 156 | 157 | class HelloWorldServer(Service): 158 | def __init__(self): 159 | self.add_service(StreamServer(('0.0.0.0', 7000), self.handle)) 160 | 161 | def handle(self, socket, address): 162 | while True: 163 | socket.send("Hello World\n") 164 | self.async.sleep(1) 165 | 166 | class HelloWorldClient(Service): 167 | def __init__(self): 168 | self.add_service(StreamClient(('0.0.0.0', 7000), self.handle)) 169 | 170 | def handle(self, socket): 171 | fileobj = socket.makefile() 172 | while True: 173 | print fileobj.readline().strip() 174 | 175 | class HelloWorldWebServer(Service): 176 | def __init__(self): 177 | self.add_service(WSGIServer(('0.0.0.0', 8000), self.handle)) 178 | 179 | def handle(self, environ, start_response): 180 | start_response('200 OK', [('Content-Type', 'text/html')]) 181 | return ["Hello World"] 182 | 183 | class HelloWorld(Service): 184 | def __init__(self): 185 | self.add_service(HelloWorldServer()) 186 | self.add_service(HelloWorldClient()) 187 | self.add_service(HelloWorldWebServer()) 188 | 189 | Running `quickstart.HelloWorld` with Ginkgo will run a server, a client, 190 | and a web server. The client will be printing our stream of "Hello 191 | World" lines. Our server is also available to be connected to via 192 | netcat. And we can also connect to our web server with curl:: 193 | 194 | $ curl http://localhost:8000 195 | 196 | And we see a strong declaration of "Hello World". 197 | 198 | In that example our web server implements a small WSGI application, but 199 | you can also use any WSGI compatible web framework. Here is an example 200 | of the Flask Hello World runnable with Ginkgo using `AppServer`:: 201 | 202 | from flask import Flask 203 | from ginkgo.async.gevent import WSGIServer 204 | 205 | app = Flask(__name__) 206 | 207 | @app.route("/") 208 | def hello(): 209 | return "Hello World!" 210 | 211 | def AppServer(): 212 | return WSGIServer(('0.0.0.0', 8000), app) 213 | 214 | Notice AppServer a callable that returns a service, in this case a 215 | pre-configured WSGIServer. 216 | 217 | Using Configuration 218 | ------------------- 219 | TODO 220 | 221 | 222 | 223 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # gevent-tools documentation build configuration file, created by 4 | # sphinx-quickstart on Sun May 8 03:34:49 2011. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys, os 15 | sys.path.insert(0, os.path.abspath('..')) 16 | from ginkgo import __version__ 17 | 18 | # If extensions (or modules to document with autodoc) are in another directory, 19 | # add these directories to sys.path here. If the directory is relative to the 20 | # documentation root, use os.path.abspath to make it absolute, like shown here. 21 | #sys.path.insert(0, os.path.abspath('.')) 22 | 23 | # -- General configuration ----------------------------------------------------- 24 | 25 | # If your documentation needs a minimal Sphinx version, state it here. 26 | #needs_sphinx = '1.0' 27 | 28 | # Add any Sphinx extension module names here, as strings. They can be extensions 29 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 30 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] 31 | 32 | # Add any paths that contain templates here, relative to this directory. 33 | templates_path = ['_templates'] 34 | 35 | # The suffix of source filenames. 36 | source_suffix = '.rst' 37 | 38 | # The encoding of source files. 39 | #source_encoding = 'utf-8-sig' 40 | 41 | # The master toctree document. 42 | master_doc = 'index' 43 | 44 | # General information about the project. 45 | project = u'Ginkgo' 46 | copyright = u'2012, Jeff Lindsay' 47 | 48 | # The version info for the project you're documenting, acts as replacement for 49 | # |version| and |release|, also used in various other places throughout the 50 | # built documents. 51 | # 52 | # The short X.Y version. 53 | version = __version__ 54 | # The full version, including alpha/beta/rc tags. 55 | release = __version__ 56 | 57 | # The language for content autogenerated by Sphinx. Refer to documentation 58 | # for a list of supported languages. 59 | #language = None 60 | 61 | # There are two options for replacing |today|: either, you set today to some 62 | # non-false value, then it is used: 63 | #today = '' 64 | # Else, today_fmt is used as the format for a strftime call. 65 | #today_fmt = '%B %d, %Y' 66 | 67 | # List of patterns, relative to source directory, that match files and 68 | # directories to ignore when looking for source files. 69 | exclude_patterns = ['_build'] 70 | 71 | # The reST default role (used for this markup: `text`) to use for all documents. 72 | #default_role = None 73 | 74 | # If true, '()' will be appended to :func: etc. cross-reference text. 75 | #add_function_parentheses = True 76 | 77 | # If true, the current module name will be prepended to all description 78 | # unit titles (such as .. function::). 79 | #add_module_names = True 80 | 81 | # If true, sectionauthor and moduleauthor directives will be shown in the 82 | # output. They are ignored by default. 83 | #show_authors = False 84 | 85 | # The name of the Pygments (syntax highlighting) style to use. 86 | pygments_style = 'sphinx' 87 | 88 | # A list of ignored prefixes for module index sorting. 89 | #modindex_common_prefix = [] 90 | 91 | 92 | # -- Options for HTML output --------------------------------------------------- 93 | 94 | # The theme to use for HTML and HTML Help pages. See the documentation for 95 | # a list of builtin themes. 96 | html_theme = 'default' 97 | 98 | # Theme options are theme-specific and customize the look and feel of a theme 99 | # further. For a list of options available for each theme, see the 100 | # documentation. 101 | #html_theme_options = {} 102 | 103 | # Add any paths that contain custom themes here, relative to this directory. 104 | #html_theme_path = [] 105 | 106 | # The name for this set of Sphinx documents. If None, it defaults to 107 | # " v documentation". 108 | #html_title = None 109 | 110 | # A shorter title for the navigation bar. Default is the same as html_title. 111 | #html_short_title = None 112 | 113 | # The name of an image file (relative to this directory) to place at the top 114 | # of the sidebar. 115 | #html_logo = None 116 | 117 | # The name of an image file (within the static path) to use as favicon of the 118 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 119 | # pixels large. 120 | #html_favicon = None 121 | 122 | # Add any paths that contain custom static files (such as style sheets) here, 123 | # relative to this directory. They are copied after the builtin static files, 124 | # so a file named "default.css" will overwrite the builtin "default.css". 125 | html_static_path = ['_static'] 126 | 127 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 128 | # using the given strftime format. 129 | #html_last_updated_fmt = '%b %d, %Y' 130 | 131 | # If true, SmartyPants will be used to convert quotes and dashes to 132 | # typographically correct entities. 133 | #html_use_smartypants = True 134 | 135 | # Custom sidebar templates, maps document names to template names. 136 | #html_sidebars = {} 137 | 138 | # Additional templates that should be rendered to pages, maps page names to 139 | # template names. 140 | #html_additional_pages = {} 141 | 142 | # If false, no module index is generated. 143 | #html_domain_indices = True 144 | 145 | # If false, no index is generated. 146 | #html_use_index = True 147 | 148 | # If true, the index is split into individual pages for each letter. 149 | #html_split_index = False 150 | 151 | # If true, links to the reST sources are added to the pages. 152 | #html_show_sourcelink = True 153 | 154 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 155 | #html_show_sphinx = True 156 | 157 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 158 | #html_show_copyright = True 159 | 160 | # If true, an OpenSearch description file will be output, and all pages will 161 | # contain a tag referring to it. The value of this option must be the 162 | # base URL from which the finished HTML is served. 163 | #html_use_opensearch = '' 164 | 165 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 166 | #html_file_suffix = None 167 | 168 | # Output file base name for HTML help builder. 169 | htmlhelp_basename = 'gevent-toolsdoc' 170 | 171 | 172 | # -- Options for LaTeX output -------------------------------------------------- 173 | 174 | # The paper size ('letter' or 'a4'). 175 | #latex_paper_size = 'letter' 176 | 177 | # The font size ('10pt', '11pt' or '12pt'). 178 | #latex_font_size = '10pt' 179 | 180 | # Grouping the document tree into LaTeX files. List of tuples 181 | # (source start file, target name, title, author, documentclass [howto/manual]). 182 | latex_documents = [ 183 | ('index', 'gevent-tools.tex', u'gevent-tools Documentation', 184 | u'Jeff Lindsay', 'manual'), 185 | ] 186 | 187 | # The name of an image file (relative to this directory) to place at the top of 188 | # the title page. 189 | #latex_logo = None 190 | 191 | # For "manual" documents, if this is true, then toplevel headings are parts, 192 | # not chapters. 193 | #latex_use_parts = False 194 | 195 | # If true, show page references after internal links. 196 | #latex_show_pagerefs = False 197 | 198 | # If true, show URL addresses after external links. 199 | #latex_show_urls = False 200 | 201 | # Additional stuff for the LaTeX preamble. 202 | #latex_preamble = '' 203 | 204 | # Documents to append as an appendix to all manuals. 205 | #latex_appendices = [] 206 | 207 | # If false, no module index is generated. 208 | #latex_domain_indices = True 209 | 210 | 211 | # -- Options for manual page output -------------------------------------------- 212 | 213 | # One entry per manual page. List of tuples 214 | # (source start file, name, description, authors, manual section). 215 | man_pages = [ 216 | ('index', 'gevent-tools', u'gevent-tools Documentation', 217 | [u'Jeff Lindsay'], 1) 218 | ] 219 | -------------------------------------------------------------------------------- /ginkgo/core.py: -------------------------------------------------------------------------------- 1 | """Ginkgo service core 2 | 3 | This module implements the core service model and several convenience 4 | decorators to use with your services. The primary export of this module is 5 | `Service`, but much of the implementation is in `BasicService`. `BasicService` 6 | uses a simple state machine defined by `ServiceStateMachine` and implements the 7 | core service interface. 8 | 9 | `BasicService` assumes no async model, whereas `Service` creates an 10 | `AsyncManager` from a driver in the `async` module. It's assumed the common 11 | case is to create async applications, but there are cases when you need a 12 | `Service` with no async. For example, `AsyncManager` classes inherit from 13 | `BasicService`, otherwise there would be a circular dependency. 14 | 15 | """ 16 | import functools 17 | import runpy 18 | 19 | from .util import AbstractStateMachine 20 | from .util import defaultproperty 21 | from . import Setting 22 | 23 | def require_ready(func): 24 | """ Decorator that blocks the call until the service is ready """ 25 | @functools.wraps(func) 26 | def wrapped(self, *args, **kwargs): 27 | try: 28 | self.state.wait("ready", self.ready_timeout) 29 | except Exception, e: 30 | pass 31 | if not self.ready: 32 | raise RuntimeWarning("Service must be ready to call this method.") 33 | return func(self, *args, **kwargs) 34 | return wrapped 35 | 36 | def autospawn(func): 37 | """ Decorator that will spawn the call in a local greenlet """ 38 | @functools.wraps(func) 39 | def wrapped(self, *args, **kwargs): 40 | self.spawn(func, self, *args, **kwargs) 41 | return wrapped 42 | 43 | class ServiceStateMachine(AbstractStateMachine): 44 | """ +------+ 45 | | init | 46 | +--+---+ 47 | | 48 | v 49 | +-------------------+ 50 | +--->| start() | 51 | | |-------------------| +-------------------+ 52 | | | starting +---+--->| stop() | 53 | | +-------------------+ | |-------------------| 54 | | | | | stopping | 55 | | v | +-------------------+ 56 | | +-----------+ | | 57 | | | ready() | | | 58 | | |-----------| | v 59 | | | ready +-------+ +-------------+ 60 | | +-----------+ | stopped() | 61 | | |-------------| 62 | +------------------------------------+ stopped | 63 | +-------------+ 64 | 65 | http://www.asciiflow.com/#7278337222084818599/1920677602 66 | """ 67 | initial_state = "init" 68 | allow_wait = ["ready", "stopped"] 69 | event_start = \ 70 | ["init", "stopped"], "starting", "pre_start" 71 | event_ready = \ 72 | ["starting"], "ready", "post_start" 73 | event_stop = \ 74 | ["ready", "starting"], "stopping", "pre_stop" 75 | event_stopped = \ 76 | ["stopping"], "stopped", "post_stop" 77 | 78 | class BasicService(object): 79 | _statemachine_class = ServiceStateMachine 80 | _children = defaultproperty(list) 81 | 82 | start_timeout = defaultproperty(int, 2) 83 | start_before = defaultproperty(bool, False) 84 | 85 | def pre_init(self): 86 | pass 87 | 88 | def __new__(cls, *args, **kwargs): 89 | s = super(BasicService, cls).__new__(cls, *args, **kwargs) 90 | s.pre_init() 91 | s.state = cls._statemachine_class(s) 92 | return s 93 | 94 | @property 95 | def service_name(self): 96 | return self.__class__.__name__ 97 | 98 | @property 99 | def ready(self): 100 | return self.state.current == 'ready' 101 | 102 | def add_service(self, service): 103 | """Add a child service to this service 104 | 105 | The service added will be started when this service starts, before 106 | its :meth:`_start` method is called. It will also be stopped when this 107 | service stops, before its :meth:`_stop` method is called. 108 | 109 | """ 110 | self._children.append(service) 111 | 112 | def remove_service(self, service): 113 | """Remove a child service from this service""" 114 | self._children.remove(service) 115 | 116 | def start(self, block_until_ready=True): 117 | """Starts children and then this service. By default it blocks until ready.""" 118 | self.state("start") 119 | if self.start_before: 120 | self.do_start() 121 | for child in self._children: 122 | if child.state.current not in ["ready", "starting"]: 123 | child.start(block_until_ready) 124 | if not self.start_before: 125 | ready = not self.do_start() 126 | if not ready and block_until_ready is True: 127 | self.state.wait("ready", self.start_timeout) 128 | elif ready: 129 | self.state("ready") 130 | else: 131 | self.state("ready") 132 | 133 | def pre_start(self): 134 | pass 135 | 136 | def do_start(self): 137 | """Empty implementation of service start. Implement me! 138 | 139 | Return `service.NOT_READY` to block until :meth:`set_ready` is 140 | called (or `ready_timeout` is reached). 141 | 142 | """ 143 | return 144 | 145 | def post_start(self): 146 | pass 147 | 148 | def stop(self): 149 | """Stop child services in reverse order and then this service""" 150 | if self.state.current in ["init", "stopped"]: 151 | return 152 | ready_before_stop = self.ready 153 | self.state("stop") 154 | for child in reversed(self._children): 155 | child.stop() 156 | if ready_before_stop: 157 | self.do_stop() 158 | self.state("stopped") 159 | 160 | def pre_stop(self): 161 | pass 162 | 163 | def post_stop(self): 164 | pass 165 | 166 | def do_stop(self): 167 | """Empty implementation of service stop. Implement me!""" 168 | return 169 | 170 | def reload(self): 171 | def _reload_children(): 172 | for child in self._children: 173 | child.reload() 174 | 175 | if self.start_before: 176 | self.do_reload() 177 | _reload_children() 178 | else: 179 | _reload_children() 180 | self.do_reload() 181 | 182 | def do_reload(self): 183 | """Empty implementation of service reload. Implement me!""" 184 | pass 185 | 186 | def serve_forever(self): 187 | """Start the service if it hasn't been already started and wait until it's stopped.""" 188 | try: 189 | self.start() 190 | except RuntimeWarning, e: 191 | # If it can't start because it's 192 | # already started, just move on 193 | pass 194 | 195 | self.state.wait("stopped") 196 | 197 | def __enter__(self): 198 | self.start() 199 | return self 200 | 201 | def __exit__(self, type, value, traceback): 202 | self.stop() 203 | 204 | 205 | class Service(BasicService): 206 | async_available = ["ginkgo.async." + m for m in ("gevent", "threading", 207 | "eventlet")] 208 | async = Setting("async", default="ginkgo.async.threading", help="""\ 209 | The async reactor to use. Available choices: 210 | ginkgo.async.gevent 211 | ginkgo.async.threading 212 | ginkgo.async.eventlet 213 | """) 214 | 215 | def pre_init(self): 216 | try: 217 | mod = runpy.run_module(self.async) 218 | self.async = mod['AsyncManager']() 219 | self.add_service(self.async) 220 | except (NotImplementedError, ImportError) as e: 221 | if self.async not in self.async_available: 222 | helptext = ("Please select a valid async module: \n\t" 223 | + "\n\t".join(self.async_available)) 224 | 225 | elif self.async.endswith("gevent"): 226 | helptext = ("Please make sure gevent is installed or use " 227 | "a different async manager.") 228 | else: 229 | helptext = "" 230 | 231 | raise RuntimeError( 232 | "Unable to load async manager from {}.\n{}".format(self.async, 233 | helptext)) 234 | 235 | def spawn(self, *args, **kwargs): 236 | return self.async.spawn(*args, **kwargs) 237 | 238 | def spawn_later(self, *args, **kwargs): 239 | return self.async.spawn_later(*args, **kwargs) 240 | 241 | 242 | -------------------------------------------------------------------------------- /ginkgo/config.py: -------------------------------------------------------------------------------- 1 | """Ginkgo config 2 | 3 | This module provides the class for a `Config` object, which represents an 4 | application configuration, often loaded by a configuration file. This is used 5 | by the runner module's `Process` object, but can be used completely 6 | independently. 7 | 8 | Configuration is described and accessed by Setting descriptors in your 9 | application. Configuration values can then be set by Python configuration 10 | files. However, using configuration files is completely optional. You can 11 | expose configuration to the end-user via command-line arguments, then load them 12 | into the `Config` object via `load()`. 13 | 14 | By default, Ginkgo creates a `Config` object singleton to use in your 15 | applications that you can import with `from ginkgo import settings`. You should 16 | only have to create a `Config` object in testing scenarios. Ginkgo also 17 | provides a shortcut for creating Setting descriptors associated with this 18 | singleton that you can import with `from ginkgo import Setting`. Often, this is 19 | the only API you need to use Ginkgo config. 20 | 21 | """ 22 | import collections 23 | import os.path 24 | import re 25 | import runpy 26 | 27 | import util 28 | 29 | class Config(util.GlobalContext): 30 | """Represents a collection of settings 31 | 32 | Provides access to a collection of settings that can be loaded from a 33 | Python module or file, or a dictionary. It allows classic-style classes to 34 | be used to indicate namespaces or groups, which can be nested. 35 | 36 | As a `GlobalContext`, you can specify the location of a singleton by setting 37 | `Config.singleton_attr` to a tuple of (object, attribute_name). Then any 38 | `Config` instance will be a context manager that will temporarily set that 39 | singleton to that instance. 40 | """ 41 | _settings = {} 42 | _descriptors = [] 43 | _forced_settings = set() 44 | _last_file = None 45 | 46 | def _normalize_path(self, path): 47 | return path.lower().lstrip(".") 48 | 49 | def get(self, path, default=None): 50 | """gets the current value of a setting""" 51 | return self._settings.get(self._normalize_path(path), default) 52 | 53 | def set(self, path, value, force=False): 54 | """sets the value of a setting""" 55 | path = self._normalize_path(path) 56 | if force or path not in self._forced_settings: 57 | self._settings[self._normalize_path(path)] = value 58 | if force: 59 | self._forced_settings.add(path) 60 | 61 | 62 | def group(self, path=''): 63 | """returns a Group object for the given path if exists""" 64 | if path not in self._settings: 65 | return Group(self, path) 66 | 67 | def setting(self, *args, **kwargs): 68 | """returns a _Setting descriptor attached to this configuration""" 69 | descriptor = _Setting(self, *args, **kwargs) 70 | self._descriptors.append(descriptor) 71 | return descriptor 72 | 73 | def load_module(self, module_path): 74 | """loads a module as configuration given a module path""" 75 | try: 76 | return self.load(runpy.run_module(module_path)) 77 | except Exception, e: 78 | raise RuntimeError("Config error: {}".format(e)) 79 | 80 | def load_file(self, file_path): 81 | """loads a module as configuration given a file path""" 82 | file_path = os.path.abspath(os.path.expanduser(file_path)) 83 | try: 84 | config_dict = runpy.run_path(file_path) 85 | except Exception, e: 86 | raise RuntimeError("Config error: {}".format(e)) 87 | self._last_file = file_path 88 | return self.load(config_dict) 89 | 90 | def reload_file(self): 91 | """reloads the last loaded configuration from load_file""" 92 | if self._last_file: 93 | return self.load_file(self._last_file) 94 | 95 | def load(self, config_dict): 96 | """loads a dictionary into settings""" 97 | def _load(d, prefix=''): 98 | """ 99 | Recursively loads configuration from a dictionary, putting 100 | configuration under classic-style classes in a namespace. 101 | """ 102 | items = (i for i in d.iteritems() if not i[0].startswith("_")) 103 | for key, value in items: 104 | path = ".".join((prefix, key)) 105 | if type(value).__name__ == 'classobj': 106 | _load(value.__dict__, path) 107 | else: 108 | self.set(path, value) 109 | _load(config_dict) 110 | return self._settings 111 | 112 | def print_help(self, only_default=False): 113 | print "config settings:" 114 | for d in sorted(self._descriptors, key=lambda d: d.path): 115 | if d.help: 116 | value = d.default if only_default else self.get(d.path, 117 | d.default) 118 | print " %- 14s %s [%s]" % ( 119 | d.path, d.help.replace('\n', '\n'+' '*18), value) 120 | 121 | 122 | class Group(collections.Mapping): 123 | """Provides read-only access to a group of config data 124 | 125 | These objects represent a 'view' into a particular scope of the entire 126 | config, whether the global group or a group created using classic-style 127 | classes in the config file. They're not created directly. Use the `group()` 128 | method on `Config`. 129 | 130 | c = Config() 131 | g1 = c.group() # global scope group 132 | g1.foo # gives you the setting named "foo" 133 | g2 = c.group("bar.baz") # bar.baz scoped group 134 | g2.qux # give you the setting named "bar.baz.qux" 135 | 136 | They are not intended to be the primary interface to settings. However, in 137 | some cases it is more convenient. You should usually use the `setting` 138 | function to embed config settings for particular values on relevant classes. 139 | """ 140 | def __init__(self, config, name=''): 141 | self._config = config 142 | self._name = name 143 | 144 | def __getattr__(self, name): 145 | path = self._config._normalize_path(".".join((self._name, name))) 146 | try: 147 | return self._config._settings[path] 148 | except KeyError: 149 | group_path = path + "." 150 | keys = self._config._settings.keys() 151 | if any(1 for k in keys if k.startswith(group_path)): 152 | return Group(self._config, path) 153 | return None 154 | 155 | def __repr__(self): 156 | return 'Group[{}:{}]'.format(self._name, self._dict()) 157 | 158 | def _dict(self): 159 | d = dict() 160 | group_path = self._name + "." 161 | for key in self._config._settings.keys(): 162 | if not self._name or key.startswith(group_path): 163 | if self._name: 164 | key = key.split(group_path, 1)[-1] 165 | name = key.split('.', 1)[0] 166 | if name not in d: 167 | d[name] = getattr(self, name) 168 | return d 169 | 170 | # Mapping protocol 171 | 172 | def __contains__(self, item): 173 | return item in self._dict() 174 | 175 | def __iter__(self): 176 | return self._dict().__iter__() 177 | 178 | def __len__(self): 179 | return self._dict().__len__() 180 | 181 | def __getitem__(self, key): 182 | return getattr(self, key) 183 | 184 | 185 | 186 | class _Setting(object): 187 | """Setting descriptor for embedding in component classes. 188 | 189 | Do not use this object directly, instead use `Config.setting()`. 190 | 191 | This is a descriptor for your component classes to define what settings 192 | your application uses and provides a way to access that setting. By 193 | accessing with a descriptor, if the configuration changes you 194 | will always have the current value. Example: 195 | 196 | class MyService(Service): 197 | foo = config.setting('foo', default='bar', 198 | help="This lets us set foo for MyService") 199 | 200 | def do_start(self): 201 | print self.foo 202 | """ 203 | _init = object() 204 | 205 | def __init__(self, config, path, default=None, monitored=False, help=''): 206 | self._last_value = self._init 207 | self.config = config 208 | self.path = path 209 | self.default = default 210 | self.monitored = monitored 211 | self.help = self.__doc__ = re.sub(r'\n\s+', '\n', help.strip()) 212 | 213 | def __get__(self, instance, type): 214 | if self.monitored: 215 | return SettingProxy(self.value, self) 216 | else: 217 | return self.value 218 | 219 | @property 220 | def value(self): 221 | return self.config.get(self.path, self.default) 222 | 223 | @property 224 | def changed(self): 225 | """ True if the value has changed since the last time accessing 226 | this property. False on first access. 227 | """ 228 | old, self._last_value = self._last_value, self.value 229 | return self.value != old and old is not self._init 230 | 231 | 232 | class SettingProxy(util.ObjectWrapper): 233 | """Wraps an object returned by a `Setting` descriptor 234 | 235 | Primarily it gives any object that comes from `Setting` a `changed` 236 | property that will determine if the value has been changed, such as when 237 | configuration is reloaded. 238 | """ 239 | descriptor = None 240 | 241 | def __init__(self, obj, descriptor): 242 | super(SettingProxy, self).__init__(obj) 243 | self.descriptor = descriptor 244 | 245 | @property 246 | def changed(self): 247 | return self.descriptor.changed 248 | 249 | @property 250 | def value(self): 251 | return self.__subject__ 252 | 253 | 254 | -------------------------------------------------------------------------------- /ginkgo/runner.py: -------------------------------------------------------------------------------- 1 | """Ginkgo runner 2 | 3 | The runner module is responsible for creating a "container" to run services in, 4 | and tools to manage that container. The container is itself a service based on 5 | a class called `Process`, which is intended to model the running process that 6 | contains the service. The process service takes an application service to run, 7 | associates a configuration with this "container", and then initializes the 8 | process to daemonize. This `Process` object is then assigned as a toplevel 9 | singleton, which you can use as a reference to the top of the service tree. 10 | 11 | The `ControlInterface` class models the commands you can use to start or 12 | control a daemonized service. This is exposed via two command line utilities 13 | `ginkgo` and `ginkgoctl`, both of which have their entry points defined in this 14 | module. 15 | 16 | The runner module and Ginkgo command line utilities are completely optional. 17 | You can always just write your own Python script or console command that takes 18 | your application service and calls `serve_forever()` on it. 19 | 20 | """ 21 | import argparse 22 | import logging 23 | import pwd 24 | import grp 25 | import os 26 | import os.path 27 | import runpy 28 | import signal 29 | import sys 30 | 31 | import ginkgo.core 32 | import ginkgo.logger 33 | import ginkgo.util 34 | 35 | STOP_SIGNAL = signal.SIGTERM 36 | RELOAD_SIGNAL = signal.SIGHUP 37 | 38 | sys.path.insert(0, os.getcwd()) 39 | 40 | logger = logging.getLogger(__name__) 41 | 42 | def run_ginkgo(): 43 | parser = argparse.ArgumentParser(prog="ginkgo", add_help=False) 44 | parser.add_argument("-v", "--version", 45 | action="version", version="%(prog)s {}".format(ginkgo.__version__)) 46 | parser.add_argument("-h", "--help", action="store_true", help=""" 47 | show program's help text and exit 48 | """.strip()) 49 | parser.add_argument("-d", "--daemonize", action="store_true", help=""" 50 | daemonize the service process 51 | """.strip()) 52 | parser.add_argument("target", nargs='?', help=""" 53 | service class path to run (modulename.ServiceClass) or 54 | configuration file path to use (/path/to/config.py) 55 | """.strip()) 56 | args = parser.parse_args() 57 | if args.help: 58 | parser.print_help() 59 | if args.target: 60 | print # blank line 61 | try: 62 | app = setup_process(args.target) 63 | app.config.print_help() 64 | except RuntimeError, e: 65 | parser.error(e) 66 | else: 67 | if args.target: 68 | try: 69 | ControlInterface().start(args.target, args.daemonize) 70 | except RuntimeError, e: 71 | parser.error(e) 72 | else: 73 | parser.print_usage() 74 | 75 | def run_ginkgoctl(): 76 | parser = argparse.ArgumentParser(prog="ginkgoctl") 77 | parser.add_argument("-v", "--version", 78 | action="version", version="%(prog)s {}".format(ginkgo.__version__)) 79 | parser.add_argument("-p", "--pid", help=""" 80 | pid or pidfile to use instead of target 81 | """.strip()) 82 | parser.add_argument("target", nargs='?', help=""" 83 | service class path to use (modulename.ServiceClass) or 84 | configuration file path to use (/path/to/config.py) 85 | """.strip()) 86 | parser.add_argument("action", 87 | choices="start stop restart reload status log logtail".split()) 88 | args = parser.parse_args() 89 | if args.pid and args.target: 90 | parser.error("You cannot specify both a target and a pid") 91 | try: 92 | if args.action in "start restart log logtail".split(): 93 | if not args.target: 94 | parser.error("You need to specify a target for {}".format(args.action)) 95 | getattr(ControlInterface(), args.action)(args.target) 96 | else: 97 | getattr(ControlInterface(), args.action)(resolve_pid(args.pid, args.target)) 98 | except RuntimeError, e: 99 | parser.error(e) 100 | 101 | def resolve_pid(pid=None, target=None): 102 | if pid and not os.path.exists(pid): 103 | return int(pid) 104 | if target is not None: 105 | setup_process(target, daemonize=True) 106 | pid = ginkgo.settings.get("pidfile") 107 | if pid is not None: 108 | if os.path.exists(pid): 109 | with open(pid, "r") as f: 110 | pid = f.read().strip() 111 | return int(pid) 112 | else: 113 | return 114 | raise RuntimeError("Unable to resolve pid from {}".format(pid or target)) 115 | 116 | def load_class(class_path): 117 | if '.' not in class_path: 118 | raise RuntimeError("Invalid class path") 119 | module_name, class_name = class_path.rsplit('.', 1) 120 | try: 121 | try: 122 | module = runpy.run_module(module_name) 123 | except ImportError: 124 | module = runpy.run_module(module_name + ".__init__") 125 | except ImportError, e: 126 | import traceback, pkgutil 127 | tb_tups = traceback.extract_tb(sys.exc_info()[2]) 128 | if pkgutil.__file__.startswith(tb_tups[-1][0]): 129 | # If the bottommost frame in our stack was in pkgutil, 130 | # then we can safely say that this ImportError occurred 131 | # because the top level class path was not found. 132 | raise RuntimeError("Unable to load class path: {}:\n{}".format( 133 | class_path, e)) 134 | else: 135 | # If the ImportError occurred further down, 136 | # raise original exception. 137 | raise 138 | try: 139 | return module[class_name] 140 | except KeyError, e: 141 | raise RuntimeError("Unable to find class in module: {}".format( 142 | class_path)) 143 | 144 | def resolve_target(target): 145 | if target.endswith('.py'): 146 | if os.path.exists(target): 147 | config = ginkgo.settings.load_file(target) 148 | try: 149 | return config['service'] 150 | except KeyError: 151 | raise RuntimeError( 152 | "Configuration does not specify a service factory") 153 | else: 154 | raise RuntimeError( 155 | 'Configuration file %s does not exist' % target) 156 | else: 157 | return target 158 | 159 | def setup_process(target, daemonize=True): 160 | service_factory = resolve_target(target) 161 | if isinstance(service_factory, str): 162 | service_factory = load_class(service_factory) 163 | 164 | if callable(service_factory): 165 | if daemonize: 166 | return DaemonProcess(service_factory) 167 | else: 168 | return Process(service_factory) 169 | else: 170 | raise RuntimeError("Does not appear to be a valid service factory") 171 | 172 | class ControlInterface(object): 173 | def start(self, target, daemonize=True): 174 | print "Starting process with {}...".format(target) 175 | app = setup_process(target, daemonize) 176 | try: 177 | app.serve_forever() 178 | except KeyboardInterrupt: 179 | pass 180 | finally: 181 | app.stop() 182 | 183 | def restart(self, target): 184 | self.stop(resolve_pid(target=target)) 185 | self.start(target) 186 | 187 | def stop(self, pid): 188 | if self._validate(pid): 189 | print "Stopping process {}...".format(pid) 190 | os.kill(pid, STOP_SIGNAL) 191 | 192 | def reload(self, pid): 193 | if self._validate(pid): 194 | print "Reloading process {}...".format(pid) 195 | os.kill(pid, RELOAD_SIGNAL) 196 | 197 | def status(self, pid): 198 | if self._validate(pid): 199 | print "Process is running as {}.".format(pid) 200 | 201 | def _validate(self, pid): 202 | try: 203 | os.kill(pid, 0) 204 | return pid 205 | except (OSError, TypeError): 206 | print "Process is NOT running." 207 | 208 | def log(self, target): 209 | app = setup_process(target) 210 | app.logger.print_log() 211 | 212 | def logtail(self, target): 213 | try: 214 | app = setup_process(target) 215 | app.logger.tail_log() 216 | except KeyboardInterrupt: 217 | pass 218 | 219 | class Process(ginkgo.core.Service, ginkgo.util.GlobalContext): 220 | singleton_attr = (ginkgo, 'process') 221 | start_before = True 222 | 223 | rundir = ginkgo.Setting("rundir", default=None, help=""" 224 | Change to a directory before running 225 | """) 226 | user = ginkgo.Setting("user", default=None, help=""" 227 | Change to a different user before running 228 | """) 229 | group = ginkgo.Setting("group", default=None, help=""" 230 | Change to a different group before running 231 | """) 232 | umask = ginkgo.Setting("umask", default=None, help=""" 233 | Change file mode creation mask before running 234 | """) 235 | 236 | def __init__(self, app_factory, config=None): 237 | self.app_factory = app_factory 238 | self.app = None 239 | 240 | self.config = config or ginkgo.settings 241 | self.logger = ginkgo.logger.Logger(self) 242 | 243 | self.pid = os.getpid() 244 | self.uid = os.geteuid() 245 | self.gid = os.getegid() 246 | self.environ = os.environ 247 | 248 | ginkgo.process = ginkgo.process or self 249 | 250 | @property 251 | def service_name(self): 252 | if self.app is None: 253 | # if the factory callable is called "service" 254 | # we need something better to name it, so we try 255 | # using first word of docstring if available 256 | if self.app_factory.__name__ == 'service': 257 | name = self.app_factory.__doc__ or self.app_factory.__name__ 258 | return name.split(' ', 1)[0] 259 | else: 260 | return self.app_factory.__name__ 261 | else: 262 | return self.app.service_name 263 | 264 | def do_start(self): 265 | if self.umask is not None: 266 | os.umask(self.umask) 267 | 268 | if self.rundir is not None: 269 | os.chdir(self.rundir) 270 | 271 | self.app = self.app_factory() 272 | self.add_service(self.app) 273 | 274 | self.async.init() 275 | self.async.signal(RELOAD_SIGNAL, self.reload) 276 | self.async.signal(STOP_SIGNAL, self.stop) 277 | 278 | def post_start(self): 279 | if self.group is not None: 280 | grp_record = grp.getgrnam(self.group) 281 | self.gid = grp_record.gr_gid 282 | os.setgid(self.gid) 283 | 284 | if self.user is not None: 285 | pw_record = pwd.getpwnam(self.user) 286 | self.uid = pw_record.pw_uid 287 | self.gid = pw_record.pw_gid 288 | os.setgid(self.gid) 289 | os.setuid(self.uid) 290 | 291 | def do_stop(self): 292 | logger.info("Stopping.") 293 | self.logger.shutdown() 294 | 295 | def do_reload(self): 296 | try: 297 | self.config.reload_file() 298 | self.logger.load_config() 299 | except RuntimeError, e: 300 | logger.warn(e) 301 | 302 | def trigger_hook(self, name, *args, **kwargs): 303 | """ Experimental """ 304 | hook = self.config.get(name) 305 | if hook is not None and callable(hook): 306 | try: 307 | hook(*args, **kwargs) 308 | except Exception, e: 309 | raise RuntimeError("Hook Error: {}".format(e)) 310 | 311 | def __enter__(self): 312 | self.__class__._push_context(self) 313 | Config._push_context(self.config) 314 | return self 315 | 316 | def __exit__(self, type, value, traceback): 317 | Config._pop_context() 318 | self.__class__._pop_context() 319 | 320 | 321 | class DaemonProcess(Process): 322 | pidfile = ginkgo.Setting("pidfile", default=None, help=""" 323 | Path to pidfile to use when daemonizing 324 | """) 325 | 326 | def __init__(self, app_factory, config=None): 327 | super(DaemonProcess, self).__init__(app_factory, config) 328 | 329 | if self.pidfile is None: 330 | self.config.set("pidfile", os.path.expanduser( 331 | "~/.{}.pid".format(self.service_name))) 332 | self.pidfile = ginkgo.util.Pidfile(str(self.pidfile)) 333 | 334 | 335 | def do_start(self): 336 | ginkgo.util.prevent_core_dump() 337 | ginkgo.util.daemonize( 338 | preserve_fds=self.logger.file_descriptors) 339 | self.logger.capture_stdio() 340 | self.pid = os.getpid() 341 | self.pidfile.create(self.pid) 342 | super(DaemonProcess, self).do_start() 343 | 344 | def do_stop(self): 345 | super(DaemonProcess, self).do_stop() 346 | self.pidfile.unlink() 347 | 348 | -------------------------------------------------------------------------------- /docs/user/manual.rst: -------------------------------------------------------------------------------- 1 | 2 | 3 | Ginkgo Manual 4 | ============= 5 | 6 | Service Model 7 | ------------- 8 | A service is an application component that starts and stops. It manages 9 | its own concurrency primitives and sub-service components. Creating a 10 | service just involves inheriting from the `Service` class and 11 | implementing any of the hooks needed of the service protocol: 12 | 13 | .. function:: Service.do_start() 14 | 15 | The do_start hook is where you implement what happens when the 16 | service is starting. Often this is where you bind to ports, open 17 | connectins, or spawn async loops. It should not be where actual 18 | service work is done, only the start up tasks. 19 | 20 | Before do_start is run, all child services are started, so you can 21 | assume they have been started. If you wish to make do_start run 22 | before child services, you can set the `start_before` class variable 23 | to True. 24 | 25 | .. function:: Service.do_stop() 26 | 27 | The do_stop hook is where you implement what happens when the 28 | service is stopping. This is often manipulating state in order to 29 | shutdown and clean up. It does not need to kill async operations or 30 | stop child services since that is taken care of by Ginkgo. 31 | 32 | .. function:: Service.do_reload() 33 | 34 | The do_reload hook is called when a parent service receives a reload 35 | call. In most cases, this is ultimately attached to SIGHUP signals. 36 | It is not meant to be a restart, which is a full stop and stop. 37 | Instead, you can use reload to reload state while running, such as 38 | configuration. 39 | 40 | Otherwise, a class that inherits from `Service` is like any other class 41 | and should be thought of as the primary interface to the component it 42 | represents. If the code for a component is too much to live in one 43 | service class, it's good practice to split it into sub-component services. 44 | In these cases, the parent service often doesn't do any work itself, but 45 | is just a container and API facade for the child component services. 46 | 47 | Here is a typical service implementation:: 48 | 49 | from ginkgo import Service 50 | 51 | class MyService(Service): 52 | def __init__(self): 53 | self.subcomponent = SubcomponentService() 54 | self.add_service(self.subcomponent) 55 | 56 | def do_start(self): 57 | 58 | 59 | TODO 60 | 61 | 62 | Using Configuration 63 | ------------------- 64 | Add the ``-h`` argument flag to our runner call:: 65 | 66 | $ ginkgo service.HelloWorld -h 67 | 68 | You'll see that the ``ginkgo`` runner command itself is very simple, but what's 69 | interesting is the last section:: 70 | 71 | config settings: 72 | daemon True or False whether to daemonize [False] 73 | group Change to a different group before running [None] 74 | logconfig Configuration of standard Python logger. Can be dict for basicConfig, 75 | dict with version key for dictConfig, or ini filepath for fileConfig. [None] 76 | logfile Path to primary log file. Ignored if logconfig is set. [/tmp/HelloWorld.log] 77 | loglevel Log level to use. Valid options: debug, info, warning, critical 78 | Ignored if logconfig is set. [debug] 79 | pidfile Path to pidfile to use when daemonizing [None] 80 | rundir Change to a directory before running [None] 81 | umask Change file mode creation mask before running [None] 82 | user Change to a different user before running [None] 83 | 84 | These are builtin settings and their default values. If you want to set any of 85 | these, you have to create a configuration file. But you can also create your 86 | own settings, so let's first change our Hello World service to be configurable:: 87 | 88 | from ginkgo import Service, Setting 89 | 90 | class HelloWorld(Service): 91 | message = Setting("message", default="Hello World", 92 | help="Message to print out while running") 93 | 94 | def do_start(self): 95 | self.spawn(self.message_forever) 96 | 97 | def message_forever(self): 98 | while True: 99 | print self.message 100 | self.async.sleep(1) 101 | 102 | Running ``ginkgo service.HelloWorld -h`` again should now include your new 103 | setting. Let's create a configuration file now called *service.conf.py*:: 104 | 105 | import os 106 | daemon = bool(os.environ.get("DAEMONIZE", False)) 107 | message = "Services all the way down." 108 | service = "service.HelloWorld" 109 | 110 | A configuration file is simply a valid Python source file. In it, you define 111 | variables of any type using the setting name to set them. 112 | 113 | There's a special setting calling ``service`` that must be set, which is the 114 | class path target telling it what service to run. To run with this 115 | configuration, you just point ``ginkgo`` to the configuration file:: 116 | 117 | $ ginkgo service.conf.py 118 | 119 | And it should start and you should see "Services all the way down" repeating. 120 | 121 | You don't have direct access to set config settings from the ``ginkgo`` tool, 122 | but you can set values in your config to pull from the environment. For 123 | example, our configuration above lets us force our service to daemonize by 124 | setting the ``DAEMONIZE`` environment variable:: 125 | 126 | $ DAEMONIZE=yes ginkgo service.conf.py 127 | 128 | To stop the daemonized process, you can manually kill it or use the service 129 | management tool ``ginkgoctl``:: 130 | 131 | $ ginkgoctl service.conf.py stop 132 | 133 | Service Manager 134 | --------------- 135 | Running and stopping your service is easy with ``ginkgo``, but once you 136 | daemonize, it gets harder to interface with it. The ``ginkgoctl`` utility is 137 | for managing a daemonized service process. 138 | 139 | :: 140 | 141 | $ ginkgoctl -h 142 | usage: ginkgoctl [-h] [-v] [-p PID] 143 | [target] {start,stop,restart,reload,status,log,logtail} 144 | 145 | positional arguments: 146 | target service class path to use (modulename.ServiceClass) or 147 | configuration file path to use (/path/to/config.py) 148 | {start,stop,restart,reload,status,log,logtail} 149 | 150 | optional arguments: 151 | -h, --help show this help message and exit 152 | -v, --version show program's version number and exit 153 | -p PID, --pid PID pid or pidfile to use instead of target 154 | 155 | Like ``ginkgo`` it takes a target class path or configuration file. For 156 | ``stop``, ``reload``, and ``status`` it can also just take a pid or pidfile 157 | with the ``pid`` argument. 158 | 159 | Using ``ginkgoctl`` will always force your service to daemonize 160 | when you use the ``start`` action. 161 | 162 | Service Model and Reloading 163 | --------------------------- 164 | Our service model lets you implement three main hooks on services: 165 | ``do_start()``, ``do_stop()``, and ``do_reload()``. We've used ``do_start()``, 166 | which is run when a service is starting up. Not surprisingly, ``do_stop()`` is 167 | run when a service is shutting down. When is ``do_reload()`` run? Well, 168 | whenever ``reload()`` is called. :) 169 | 170 | Services are designed to contain other services like object composition. Though 171 | after adding services to a service, when you call any of the service interface 172 | methods, they will propogate down to child services. This is done in the actual 173 | ``start()``, ``stop()``, and ``reload()`` methods. The ``do_`` methods are for 174 | you to implement specifically what happens for *that* service to 175 | start/stop/reload. 176 | 177 | So when is ``reload()`` called? Okay, I'll skip ahead and just say it gets 178 | called when the process receives a SIGHUP signal. As you may have guessed, for 179 | convenience, this is exposed in ``ginkgoctl`` with the ``reload`` action. 180 | 181 | The semantics of ``reload`` are up to you and your application or service. 182 | Though one thing happens automatically when a process gets a reload signal: 183 | configuration is reloaded. 184 | 185 | One use of ``do_reload()`` is to take new configuration and perform any 186 | operations to apply that configuration to your running service. However, as 187 | long as you access a configuration setting by reference via the ``Setting`` 188 | descriptor, you may not need to do anything -- the value will just update in 189 | real-time. 190 | 191 | Let's see this in action. We'll change our Hello World service to have a 192 | ``rate_per_minute`` setting that will be used for our delay between messages:: 193 | 194 | from ginkgo import Service, Setting 195 | 196 | class HelloWorld(Service): 197 | message = Setting("message", default="Hello World", 198 | help="Message to print out while running") 199 | 200 | rate = Setting("rate_per_minute", default=60, 201 | help="Rate at which to emit message") 202 | 203 | def do_start(self): 204 | self.spawn(self.message_forever) 205 | 206 | def message_forever(self): 207 | while True: 208 | print self.message 209 | self.async.sleep(60.0 / self.rate) 210 | 211 | The default is 60 messages a minute, which results in the same behavior as 212 | before. So let's change our configuration to use a different rate:: 213 | 214 | import os 215 | daemon = bool(os.environ.get("DAEMONIZE", False)) 216 | message = "Services all the way down." 217 | rate_per_minute = 180 218 | service = "service.HelloWorld" 219 | 220 | Use ``ginkgo`` to start the service:: 221 | 222 | $ ginkgo service.conf.py 223 | 224 | As you can see, it's emitting messages a bit faster now. About 3 per second. 225 | Now while that's running, open the configuration file and change 226 | rate_per_minute to some other value. Then, in another terminal, change to that 227 | directory and reload:: 228 | 229 | $ ginkgoctl service.conf.py reload 230 | 231 | Look back at your running service to see that it's now using the new emit rate. 232 | 233 | Using Logging 234 | ------------- 235 | Logging with Ginkgo is based on standard Python logging. We make sure it works 236 | with daemonization and provide Ginkgo-friendly ways to configure it with good 237 | defaults. We even support reloading logging configuration. 238 | 239 | Out of the box, you can just start logging. We encourage you to use the common 240 | convention of module level loggers, but obviously there is a lot of freedom in 241 | how you use Python logging. Let's add some logging to our Hello World, 242 | including changing our print call to a logger call as it's better practice:: 243 | 244 | import logging 245 | from ginkgo import Service, Setting 246 | 247 | logger = logging.getLogger(__name__) 248 | 249 | class HelloWorld(Service): 250 | message = Setting("message", default="Hello World", 251 | help="Message to print out while running") 252 | 253 | rate = Setting("rate_per_minute", default=60, 254 | help="Rate at which to emit message") 255 | 256 | def do_start(self): 257 | logger.info("Starting up!") 258 | self.spawn(self.message_forever) 259 | 260 | def do_stop(self): 261 | logger.info("Goodbye.") 262 | 263 | def message_forever(self): 264 | while True: 265 | logger.info(self.message) 266 | self.async.sleep(60.0 / self.rate) 267 | 268 | Let's run it with our existing configuration for a bit and then stop:: 269 | 270 | $ ginkgo service.conf.py 271 | Starting process with service.conf.py... 272 | 2012-04-28 17:21:32,608 INFO service: Starting up! 273 | 2012-04-28 17:21:32,608 INFO service: Services all the way down. 274 | 2012-04-28 17:21:33,609 INFO service: Services all the way down. 275 | 2012-04-28 17:21:34,610 INFO service: Services all the way down. 276 | 2012-04-28 17:21:35,714 INFO service: Goodbye. 277 | 2012-04-28 17:21:35,714 INFO runner: Stopping. 278 | 279 | Running ``-h`` will show you that the default logfile is going to be 280 | */tmp/HelloWorld.log*, which logging will create and append to if you 281 | daemonize. 282 | 283 | To configure logging, Ginkgo exposes two settings for simple case 284 | configuration: ``logfile`` and ``loglevel``. If that's not enough, you can use 285 | ``logconfig``, which will override any value for ``logfile`` and ``loglevel``. 286 | 287 | Using ``logconfig`` you can configure logging as expressed by 288 | ``logging.basicConfig``. By default, if you set ``logconfig`` to a dictionary, 289 | it will apply those keyword arguments to ``logging.basicConfig``. You can 290 | learn more about ``logging.basicConfig`` 291 | `here `_. 292 | 293 | For advanced configuration, we also let you use ``logging.config`` from the 294 | ``logconfig`` setting. If ``logconfig`` is a dictionary with a ``version`` key, 295 | we will load it into ``logging.config.dictConfig``. If ``logconfig`` is a path 296 | to a file, we load it into ``logging.config.fileConfig``. Both of these are 297 | ways to define a configuration structure that lets you create just about any 298 | logging configuration. Read more about ``logging.config`` 299 | `here `_. 300 | 301 | -------------------------------------------------------------------------------- /ginkgo/util.py: -------------------------------------------------------------------------------- 1 | """Ginkgo utility functions and classes 2 | 3 | This module contains functions and classes that are shared across modules or 4 | are more general utilities that aren't specific to Ginkgo. This way we keep 5 | Ginkgo modules very dense in readable domain specific code. 6 | """ 7 | import resource 8 | import os 9 | import errno 10 | import tempfile 11 | 12 | 13 | class defaultproperty(object): 14 | """ 15 | Allow for default-valued properties to be added to classes. 16 | 17 | Example usage: 18 | 19 | class Foo(object): 20 | bar = defaultproperty(list) 21 | """ 22 | def __init__(self, default_factory, *args, **kwargs): 23 | self.default_factory = default_factory 24 | self.args = args 25 | self.kwargs = kwargs 26 | 27 | def __get__(self, instance, owner): 28 | if instance is None: 29 | return None 30 | for kls in owner.__mro__: 31 | for key, value in kls.__dict__.iteritems(): 32 | if value == self: 33 | newval = self.default_factory(*self.args, **self.kwargs) 34 | instance.__dict__[key] = newval 35 | return newval 36 | 37 | 38 | def daemonize(preserve_fds=None): 39 | """\ 40 | Standard daemonization of a process. 41 | http://www.svbug.com/documentation/comp.unix.programmer-FAQ/faq_2.html#SEC16 42 | """ 43 | def _maxfd(limit=1024): 44 | maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] 45 | if maxfd == resource.RLIM_INFINITY: 46 | return limit 47 | else: 48 | return maxfd 49 | 50 | def _devnull(default="/dev/null"): 51 | if hasattr(os, "devnull"): 52 | return os.devnull 53 | else: 54 | return default 55 | 56 | def _close_fds(preserve=None): 57 | preserve = preserve or [] 58 | for fd in xrange(0, _maxfd()): 59 | if fd not in preserve: 60 | try: 61 | os.close(fd) 62 | except OSError: # fd wasn't open to begin with (ignored) 63 | pass 64 | 65 | if os.fork(): 66 | os._exit(0) 67 | os.setsid() 68 | 69 | if os.fork(): 70 | os._exit(0) 71 | 72 | os.umask(0) 73 | _close_fds(preserve_fds) 74 | 75 | os.open(_devnull(), os.O_RDWR) 76 | os.dup2(0, 1) 77 | os.dup2(0, 2) 78 | 79 | def prevent_core_dump(): 80 | """ Prevent this process from generating a core dump. 81 | 82 | Sets the soft and hard limits for core dump size to zero. On 83 | Unix, this prevents the process from creating core dump 84 | altogether. 85 | 86 | """ 87 | core_resource = resource.RLIMIT_CORE 88 | 89 | try: 90 | # Ensure the resource limit exists on this platform, by requesting 91 | # its current value 92 | core_limit_prev = resource.getrlimit(core_resource) 93 | except ValueError, e: 94 | raise RuntimeWarning( 95 | "System does not support RLIMIT_CORE resource limit ({})".format(e)) 96 | 97 | # Set hard and soft limits to zero, i.e. no core dump at all 98 | resource.setrlimit(core_resource, (0, 0)) 99 | 100 | class PassthroughEvent(object): 101 | def wait(self, timeout=None): return 102 | def set(self): return 103 | def clear(self): return 104 | 105 | class AbstractStateMachine(object): 106 | event_class = PassthroughEvent 107 | 108 | def __init__(self, subject): 109 | self._state = self.initial_state 110 | self._subject = subject 111 | self._waitables = {} 112 | if hasattr(self._subject, 'async'): 113 | self.event_class = self._subject.async.event 114 | for state in self.allow_wait: 115 | self._waitables[state] = self.event_class() 116 | 117 | @property 118 | def current(self): 119 | return self._state 120 | 121 | def wait(self, state, timeout=None): 122 | if state in self._waitables: 123 | self._waitables[state].wait(timeout) 124 | else: 125 | raise RuntimeWarning("Unable to wait for state '{}'".format(state)) 126 | 127 | def __call__(self, event): 128 | from_states, to_state, callback = self._lookup_event(event) 129 | if self._state in from_states: 130 | self._callback(callback) 131 | self._transition(to_state) 132 | else: 133 | raise RuntimeWarning(""" 134 | Unable to enter '{}' from state '{}' 135 | """.format(to_state, self.current).strip()) 136 | 137 | def _lookup_event(self, event): 138 | event_definition = "event_{}".format(event) 139 | if hasattr(self, event_definition): 140 | return getattr(self, event_definition) 141 | else: 142 | raise AttributeError("No event '{}' on {}".format(event, self)) 143 | 144 | def _callback(self, name): 145 | if name is not None and hasattr(self._subject, name): 146 | getattr(self._subject, name)() 147 | 148 | def _transition(self, new_state): 149 | for state in self._waitables: 150 | self._waitables[state].clear() 151 | self._state = new_state 152 | if new_state in self._waitables: 153 | self._waitables[new_state].set() 154 | 155 | 156 | class GlobalContext(object): 157 | """Context manager mixin for stackable singletons 158 | 159 | Use this mixin when a class has a global singleton set somewhere that can 160 | be temporarily set while in the context of an instance of that class:: 161 | 162 | class Foo(GlobalContext): 163 | instance = None # where we'll keep the singleton 164 | singleton_attr = (Foo, 'instance') # tell mixin where it is 165 | 166 | Foo.instance = Foo() # set an initial Foo singleton 167 | temporary_foo = Foo() # create another Foo 168 | # now use it as a context 169 | with temporary_foo: 170 | # the singleton will be set to this instance 171 | assert Foo.instance is temporary_foo 172 | # then set back when you exit the context 173 | assert Foo.instance is not temporary_foo 174 | 175 | You can also nest global contexts if necessary. The main API is just 176 | setting where the singleton is with `singleton_attr`, which is a tuple of 177 | (object, attribute name). If `singleton_attr` is not set, there is no 178 | effect when you use the context manager. You can define `singleton_attr` 179 | outside the class definition to decouple your class definition from your 180 | use of a singleton. For example:: 181 | 182 | class Foo(GlobalContext): 183 | pass 184 | 185 | singleton = Foo() # module level singleton 186 | Foo.singleton_attr = (sys.modules[__name__], 'singleton') 187 | 188 | """ 189 | singleton_attr = None 190 | _singleton_stacks = {} 191 | 192 | @classmethod 193 | def _get_singleton(cls): 194 | if cls.singleton_attr: 195 | return getattr(*cls.singleton_attr) 196 | 197 | @classmethod 198 | def _set_singleton(cls, value): 199 | if cls.singleton_attr: 200 | setattr(*list(cls.singleton_attr)+[value]) 201 | 202 | @classmethod 203 | def _push_context(cls, obj): 204 | if cls.singleton_attr: 205 | klass = cls.__name__ 206 | if klass not in cls._singleton_stacks: 207 | cls._singleton_stacks[klass] = [] 208 | cls._singleton_stacks[klass].append(cls._get_singleton()) 209 | cls._set_singleton(obj) 210 | 211 | @classmethod 212 | def _pop_context(cls): 213 | if cls.singleton_attr: 214 | klass = cls.__name__ 215 | cls._set_singleton( 216 | cls._singleton_stacks.get(klass, []).pop()) 217 | 218 | def __enter__(self): 219 | self.__class__._push_context(self) 220 | return self 221 | 222 | def __exit__(self, type, value, traceback): 223 | self.__class__._pop_context() 224 | 225 | 226 | class Pidfile(object): 227 | """\ 228 | Manage a PID file. If a specific name is provided 229 | it and '"%s.oldpid" % name' will be used. Otherwise 230 | we create a temp file using os.mkstemp. 231 | """ 232 | 233 | def __init__(self, fname): 234 | self.fname = fname 235 | self.pid = None 236 | 237 | def create(self, pid): 238 | oldpid = self.validate() 239 | if oldpid: 240 | if oldpid == os.getpid(): 241 | return 242 | raise RuntimeError("Already running on PID %s " \ 243 | "(or pid file '%s' is stale)" % (os.getpid(), self.fname)) 244 | 245 | self.pid = pid 246 | 247 | # Write pidfile 248 | fdir = os.path.dirname(self.fname) 249 | if fdir and not os.path.isdir(fdir): 250 | raise RuntimeError("%s doesn't exist. Can't create pidfile." % fdir) 251 | fd, fname = tempfile.mkstemp(dir=fdir) 252 | os.write(fd, "%s\n" % self.pid) 253 | if self.fname: 254 | os.rename(fname, self.fname) 255 | else: 256 | self.fname = fname 257 | os.close(fd) 258 | 259 | # set permissions to -rw-r--r-- 260 | os.chmod(self.fname, 420) 261 | 262 | def rename(self, path): 263 | self.unlink() 264 | self.fname = path 265 | self.create(self.pid) 266 | 267 | def unlink(self): 268 | """ delete pidfile""" 269 | try: 270 | with open(self.fname, "r") as f: 271 | pid1 = int(f.read() or 0) 272 | 273 | if pid1 == self.pid: 274 | os.unlink(self.fname) 275 | except: 276 | pass 277 | 278 | def validate(self): 279 | """ Validate pidfile and make it stale if needed""" 280 | if not self.fname: 281 | return 282 | try: 283 | with open(self.fname, "r") as f: 284 | wpid = int(f.read() or 0) 285 | 286 | if wpid <= 0: 287 | return 288 | 289 | try: 290 | os.kill(wpid, 0) 291 | return wpid 292 | except OSError, e: 293 | if e[0] == errno.ESRCH: 294 | return 295 | raise 296 | except IOError, e: 297 | if e[0] == errno.ENOENT: 298 | return 299 | raise 300 | 301 | ## The following is extracted from ProxyTypes 0.9 package, licensed under ZPL: 302 | ## http://pypi.python.org/pypi/ProxyTypes 303 | 304 | class AbstractProxy(object): 305 | """Delegates all operations (except ``.__subject__``) to another object""" 306 | __slots__ = () 307 | 308 | def __call__(self,*args,**kw): 309 | return self.__subject__(*args,**kw) 310 | 311 | def __getattribute__(self, attr, oga=object.__getattribute__): 312 | subject = oga(self,'__subject__') 313 | if attr=='__subject__': 314 | return subject 315 | return getattr(subject,attr) 316 | 317 | def __setattr__(self,attr,val, osa=object.__setattr__): 318 | if attr=='__subject__': 319 | osa(self,attr,val) 320 | else: 321 | setattr(self.__subject__,attr,val) 322 | 323 | def __delattr__(self,attr, oda=object.__delattr__): 324 | if attr=='__subject__': 325 | oda(self,attr) 326 | else: 327 | delattr(self.__subject__,attr) 328 | 329 | def __nonzero__(self): 330 | return bool(self.__subject__) 331 | 332 | def __getitem__(self,arg): 333 | return self.__subject__[arg] 334 | 335 | def __setitem__(self,arg,val): 336 | self.__subject__[arg] = val 337 | 338 | def __delitem__(self,arg): 339 | del self.__subject__[arg] 340 | 341 | def __getslice__(self,i,j): 342 | return self.__subject__[i:j] 343 | 344 | 345 | def __setslice__(self,i,j,val): 346 | self.__subject__[i:j] = val 347 | 348 | def __delslice__(self,i,j): 349 | del self.__subject__[i:j] 350 | 351 | def __contains__(self,ob): 352 | return ob in self.__subject__ 353 | 354 | for name in 'repr str hash len abs complex int long float iter oct hex'.split(): 355 | exec "def __%s__(self): return %s(self.__subject__)" % (name,name) 356 | 357 | for name in 'cmp', 'coerce', 'divmod': 358 | exec "def __%s__(self,ob): return %s(self.__subject__,ob)" % (name,name) 359 | 360 | for name,op in [ 361 | ('lt','<'), ('gt','>'), ('le','<='), ('ge','>='), 362 | ('eq','=='), ('ne','!=') 363 | ]: 364 | exec "def __%s__(self,ob): return self.__subject__ %s ob" % (name,op) 365 | 366 | for name,op in [('neg','-'), ('pos','+'), ('invert','~')]: 367 | exec "def __%s__(self): return %s self.__subject__" % (name,op) 368 | 369 | for name, op in [ 370 | ('or','|'), ('and','&'), ('xor','^'), ('lshift','<<'), ('rshift','>>'), 371 | ('add','+'), ('sub','-'), ('mul','*'), ('div','/'), ('mod','%'), 372 | ('truediv','/'), ('floordiv','//') 373 | ]: 374 | exec ( 375 | "def __%(name)s__(self,ob):\n" 376 | " return self.__subject__ %(op)s ob\n" 377 | "\n" 378 | "def __r%(name)s__(self,ob):\n" 379 | " return ob %(op)s self.__subject__\n" 380 | "\n" 381 | "def __i%(name)s__(self,ob):\n" 382 | " self.__subject__ %(op)s=ob\n" 383 | " return self\n" 384 | ) % locals() 385 | 386 | del name, op 387 | 388 | # Oddball signatures 389 | 390 | def __rdivmod__(self,ob): 391 | return divmod(ob, self.__subject__) 392 | 393 | def __pow__(self,*args): 394 | return pow(self.__subject__,*args) 395 | 396 | def __ipow__(self,ob): 397 | self.__subject__ **= ob 398 | return self 399 | 400 | def __rpow__(self,ob): 401 | return pow(ob, self.__subject__) 402 | 403 | class ObjectProxy(AbstractProxy): 404 | """Proxy for a specific object""" 405 | 406 | __slots__ = "__subject__" 407 | 408 | def __init__(self,subject): 409 | self.__subject__ = subject 410 | 411 | class AbstractWrapper(AbstractProxy): 412 | """Mixin to allow extra behaviors and attributes on proxy instance""" 413 | __slots__ = () 414 | 415 | def __getattribute__(self, attr, oga=object.__getattribute__): 416 | if attr.startswith('__'): 417 | subject = oga(self,'__subject__') 418 | if attr=='__subject__': 419 | return subject 420 | return getattr(subject,attr) 421 | return oga(self,attr) 422 | 423 | def __getattr__(self,attr, oga=object.__getattribute__): 424 | return getattr(oga(self,'__subject__'), attr) 425 | 426 | def __setattr__(self,attr,val, osa=object.__setattr__): 427 | if ( 428 | attr=='__subject__' 429 | or hasattr(type(self),attr) and not attr.startswith('__') 430 | ): 431 | osa(self,attr,val) 432 | else: 433 | setattr(self.__subject__,attr,val) 434 | 435 | def __delattr__(self,attr, oda=object.__delattr__): 436 | if ( 437 | attr=='__subject__' 438 | or hasattr(type(self),attr) and not attr.startswith('__') 439 | ): 440 | oda(self,attr) 441 | else: 442 | delattr(self.__subject__,attr) 443 | 444 | class ObjectWrapper(ObjectProxy, AbstractWrapper): __slots__ = () 445 | 446 | --------------------------------------------------------------------------------