├── THANKS
├── examples
    ├── static
    │   ├── hello.html
    │   ├── images
    │   │   ├── logo.png
    │   │   ├── gunicorn.png
    │   │   └── large_gunicorn.png
    │   └── index.html
    ├── .hello.py.swo
    ├── hello_pool.py
    ├── hello.py
    ├── multiworker.py
    ├── tcp_hello.py
    ├── multiworker2.py
    ├── _sendfile.py
    ├── amqp.py
    └── serve_file.py
├── pistil
    ├── tcp
    │   ├── __init__.py
    │   ├── arbiter.py
    │   ├── sync_worker.py
    │   ├── gevent_worker.py
    │   └── sock.py
    ├── __init__.py
    ├── errors.py
    ├── workertmp.py
    ├── pidfile.py
    ├── worker.py
    ├── pool.py
    ├── util.py
    └── arbiter.py
├── MANIFEST.in
├── .gitignore
├── NOTICE
├── LICENSE
├── setup.py
└── README.rst


/THANKS:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/examples/static/hello.html:
--------------------------------------------------------------------------------
1 | hello world
2 | 


--------------------------------------------------------------------------------
/examples/.hello.py.swo:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/meebo/pistil/HEAD/examples/.hello.py.swo


--------------------------------------------------------------------------------
/examples/static/images/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/meebo/pistil/HEAD/examples/static/images/logo.png


--------------------------------------------------------------------------------
/examples/static/images/gunicorn.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/meebo/pistil/HEAD/examples/static/images/gunicorn.png


--------------------------------------------------------------------------------
/pistil/tcp/__init__.py:
--------------------------------------------------------------------------------
1 | from pistil.tcp.arbiter import TcpArbiter
2 | from pistil.tcp.sync_worker import TcpSyncWorker
3 | 


--------------------------------------------------------------------------------
/examples/static/images/large_gunicorn.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/meebo/pistil/HEAD/examples/static/images/large_gunicorn.png


--------------------------------------------------------------------------------
/examples/static/index.html:
--------------------------------------------------------------------------------
1 | Welcome !
2 | 
3 | Go to <a href="hello.html">Hello world</a>.
4 | 
5 | Or list <a href="images/">images/</a>
6 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include .gitignore
2 | include LICENSE
3 | include NOTICE
4 | include README.rst
5 | include THANKS
6 | 
7 | recursive-include examples *
8 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | *.gem
 2 | *.swp
 3 | *.pyc
 4 | *#*
 5 | build
 6 | dist
 7 | setuptools-*
 8 | .svn/*
 9 | .DS_Store
10 | *.so
11 | distribute-0.6.8-py2.6.egg
12 | distribute-0.6.8.tar.gz
13 | pistil.egg-info
14 | nohup.out
15 | .coverage
16 | doc/.sass-cache
17 | 


--------------------------------------------------------------------------------
/pistil/__init__.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of pistil released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | 
 7 | version_info = (0, 2,0)
 8 | __version__ = ".".join(map(str, version_info))
 9 | 
10 | SERVER_SOFTWARE = "pistil/%s" % __version__
11 | 


--------------------------------------------------------------------------------
/pistil/errors.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of pistil released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | 
 7 | class HaltServer(Exception):
 8 |     def __init__(self, reason, exit_status=1):
 9 |         self.reason = reason
10 |         self.exit_status = exit_status
11 |     
12 |     def __str__(self):
13 |         return "<HaltServer %r %d>" % (self.reason, self.exit_status)
14 | 


--------------------------------------------------------------------------------
/NOTICE:
--------------------------------------------------------------------------------
 1 | Pistil
 2 | 
 3 | 2011 (c) Benoît Chesneau <benoitc@e-engura.org>
 4 | 2011 (c) Meebo
 5 | 
 6 | Pistil is released under the MIT license. See the LICENSE
 7 | file for the complete license.
 8 | 
 9 | 
10 | Following files are under MIT License and copyrights:
11 | 2009-2011 (c) Benoît Chesneau <benoitc@e-engura.org>
12 | 2009-2011 (c) Paul J. Davis <paul.joseph.davis@gmail.com>
13 | 
14 | 
15 | pistil/arbiter.py,
16 | pistil/util.py
17 | pistil/sock.py
18 | pistil/pidfile.py
19 | pistil/workertmp.py
20 | pistil/worker.py
21 | 


--------------------------------------------------------------------------------
/examples/hello_pool.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of pistil released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | from pistil.pool import PoolArbiter
 7 | from pistil.worker import Worker
 8 | 
 9 | class MyWorker(Worker):
10 | 
11 |     def handle(self):
12 |         print "hello from worker n°%s" % self.pid
13 | 
14 | if __name__ == "__main__":
15 |     conf = {"num_workers": 3 }
16 |     spec = (MyWorker, 30, "worker", {}, "test",)
17 |     a = PoolArbiter(conf, spec)
18 |     a.run()
19 | 


--------------------------------------------------------------------------------
/examples/hello.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of pistil released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | import time
 7 | 
 8 | from pistil.arbiter import Arbiter
 9 | from pistil.worker import Worker
10 | 
11 | 
12 | class MyWorker(Worker):
13 | 
14 |     def handle(self):
15 |         print "hello from worker n°%s" % self.pid
16 | 
17 | 
18 | if __name__ == "__main__":
19 |     conf = {}
20 |     specs = [(MyWorker, 30, "worker", {}, "test")]
21 |     a = Arbiter(conf, specs)
22 |     a.run()
23 | 


--------------------------------------------------------------------------------
/examples/multiworker.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of pistil released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | from pistil.arbiter import Arbiter
 7 | from pistil.worker import Worker
 8 | 
 9 | 
10 | class MyWorker(Worker):
11 | 
12 |     def handle(self): 
13 |         print "hello worker 1 from %s" % self.name
14 | 
15 | class MyWorker2(Worker):
16 | 
17 |     def handle(self):
18 |         print "hello worker 2 from %s" % self.name
19 | 
20 | 
21 | if __name__ == '__main__':
22 |     conf = {}
23 | 
24 |     specs = [
25 |         (MyWorker, 30, "worker", {}, "w1"),
26 |         (MyWorker2, 30, "worker", {}, "w2"),
27 |         (MyWorker2, 30, "kill", {}, "w3")
28 |     ]
29 |     # launchh the arbiter
30 |     arbiter = Arbiter(conf, specs)
31 |     arbiter.run()
32 | 


--------------------------------------------------------------------------------
/examples/tcp_hello.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of pistil released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | from pistil.tcp.sync_worker import TcpSyncWorker
 7 | from pistil.tcp.arbiter import TcpArbiter
 8 | 
 9 | from http_parser.http import HttpStream
10 | from http_parser.reader import SocketReader
11 | 
12 | class MyTcpWorker(TcpSyncWorker):
13 | 
14 |     def handle(self, sock, addr):
15 |         p = HttpStream(SocketReader(sock))
16 | 
17 |         path = p.path()
18 |         data = "hello world"
19 |         sock.send("".join(["HTTP/1.1 200 OK\r\n", 
20 |                         "Content-Type: text/html\r\n",
21 |                         "Content-Length:" + str(len(data)) + "\r\n",
22 |                          "Connection: close\r\n\r\n",
23 |                          data]))
24 | 
25 | if __name__ == '__main__':
26 |     conf = {"num_workers": 3}
27 |     spec = (MyTcpWorker, 30, "worker", {}, "worker",)
28 |     
29 |     arbiter = TcpArbiter(conf, spec)
30 | 
31 |     arbiter.run()
32 | 
33 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | 2011 (c) Benoît Chesneau <benoitc@e-engura.org>
 2 | 2011 (c) Meebo
 3 | 
 4 | Permission is hereby granted, free of charge, to any person
 5 | obtaining a copy of this software and associated documentation
 6 | files (the "Software"), to deal in the Software without
 7 | restriction, including without limitation the rights to use,
 8 | copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the
10 | Software is furnished to do so, subject to the following
11 | conditions:
12 | 
13 | The above copyright notice and this permission notice shall be
14 | included in all copies or substantial portions of the Software.
15 | 
16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
18 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
20 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
21 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
23 | OTHER DEALINGS IN THE SOFTWARE.
24 | 


--------------------------------------------------------------------------------
/pistil/tcp/arbiter.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of pistil released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | import logging
 7 | import os
 8 | 
 9 | from pistil import util
10 | from pistil.pool import PoolArbiter
11 | from pistil.tcp.sock import create_socket
12 | 
13 | log = logging.getLogger(__name__)
14 | 
15 | 
16 | class TcpArbiter(PoolArbiter):
17 | 
18 |     _LISTENER = None
19 | 
20 |     def on_init(self, args):
21 |         self.address = util.parse_address(args.get('address',
22 |             ('127.0.0.1', 8000)))
23 |         if not self._LISTENER:
24 |             self._LISTENER = create_socket(args)
25 | 
26 |         # we want to pass the socket to the worker.
27 |         self.conf.update({"sock": self._LISTENER})
28 |         
29 | 
30 |     def when_ready(self):
31 |         log.info("Listening at: %s (%s)", self._LISTENER,
32 |             self.pid)
33 |    
34 |     def on_reexec(self):
35 |         # save the socket file descriptor number in environ to reuse the
36 |         # socket after forking a new master.
37 |         os.environ['PISTIL_FD'] = str(self._LISTENER.fileno())
38 | 
39 |     def on_stop(self, graceful=True):
40 |         self._LISTENER = None
41 | 
42 | 
43 | 
44 | 


--------------------------------------------------------------------------------
/pistil/workertmp.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of pistil released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | import os
 7 | import tempfile
 8 | 
 9 | from pistil import util
10 | 
11 | class WorkerTmp(object):
12 | 
13 |     def __init__(self, cfg):
14 |         old_umask = os.umask(cfg.get("umask", 0))
15 |         fd, name = tempfile.mkstemp(prefix="wgunicorn-")
16 |         
17 |         # allows the process to write to the file
18 |         util.chown(name, cfg.get("uid", os.geteuid()), cfg.get("gid",
19 |             os.getegid()))
20 |         os.umask(old_umask)
21 | 
22 |         # unlink the file so we don't leak tempory files
23 |         try:
24 |             os.unlink(name)
25 |             self._tmp = os.fdopen(fd, 'w+b', 1)
26 |         except:
27 |             os.close(fd)
28 |             raise
29 | 
30 |         self.spinner = 0
31 | 
32 |     def notify(self): 
33 |         try:
34 |             self.spinner = (self.spinner+1) % 2
35 |             os.fchmod(self._tmp.fileno(), self.spinner)
36 |         except AttributeError:
37 |             # python < 2.6
38 |             self._tmp.truncate(0)
39 |             os.write(self._tmp.fileno(), "X")
40 | 
41 |     def fileno(self):
42 |         return self._tmp.fileno()
43 |        
44 |     def close(self):
45 |         return self._tmp.close()
46 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of gunicorn released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | 
 7 | import os
 8 | from setuptools import setup, find_packages
 9 | import sys
10 | 
11 | from pistil import __version__
12 | 
13 | setup(
14 |     name = 'pistil',
15 |     version = __version__,
16 | 
17 |     description = 'Multiprocessing toolkit',
18 |     long_description = file(
19 |         os.path.join(
20 |             os.path.dirname(__file__),
21 |             'README.rst'
22 |         )
23 |     ).read(),
24 |     author = 'Benoit Chesneau',
25 |     author_email = 'benoitc@e-engura.com',
26 |     license = 'MIT',
27 |     url = 'http://github.com/meebo/pistil',
28 | 
29 |     classifiers = [
30 |         'Development Status :: 4 - Beta',
31 |         'Environment :: Other Environment',
32 |         'Intended Audience :: Developers',
33 |         'License :: OSI Approved :: MIT License',
34 |         'Operating System :: MacOS :: MacOS X',
35 |         'Operating System :: POSIX',
36 |         'Programming Language :: Python',
37 |         'Topic :: Internet',
38 |         'Topic :: Utilities',
39 |         'Topic :: Software Development :: Libraries :: Python Modules',
40 |     ],
41 |     zip_safe = False,
42 |     packages = find_packages(exclude=['examples', 'tests']),
43 |     include_package_data = True
44 | )
45 | 


--------------------------------------------------------------------------------
/examples/multiworker2.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of pistil released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | import time
 7 | import urllib2
 8 | 
 9 | from pistil.arbiter import Arbiter
10 | from pistil.worker import Worker
11 | from pistil.tcp.sync_worker import TcpSyncWorker
12 | from pistil.tcp.arbiter import TcpArbiter
13 | 
14 | from http_parser.http import HttpStream
15 | from http_parser.reader import SocketReader
16 | 
17 | class MyTcpWorker(TcpSyncWorker):
18 | 
19 |     def handle(self, sock, addr):
20 |         p = HttpStream(SocketReader(sock))
21 | 
22 |         path = p.path()
23 |         data = "welcome wold"
24 |         sock.send("".join(["HTTP/1.1 200 OK\r\n", 
25 |                         "Content-Type: text/html\r\n",
26 |                         "Content-Length:" + str(len(data)) + "\r\n",
27 |                          "Connection: close\r\n\r\n",
28 |                          data]))
29 | 
30 | 
31 | class UrlWorker(Worker):
32 | 
33 |     def run(self):
34 |         print "ici"
35 |         while self.alive: 
36 |             time.sleep(0.1)
37 |             f = urllib2.urlopen("http://localhost:5000")
38 |             print f.read()
39 |             self.notify() 
40 | 
41 | class MyPoolArbiter(TcpArbiter):
42 | 
43 |     def on_init(self, conf):
44 |         TcpArbiter.on_init(self, conf)
45 |         # we return a spec
46 |         return (MyTcpWorker, 30, "worker", {}, "http_welcome",)
47 | 
48 | 
49 | if __name__ == '__main__':
50 |     conf = {"num_workers": 3, "address": ("127.0.0.1", 5000)}
51 | 
52 |     specs = [
53 |         (MyPoolArbiter, 30, "supervisor", {}, "tcp_pool"),
54 |         (UrlWorker, 30, "worker", {}, "grabber")
55 |     ]
56 | 
57 |     arbiter = Arbiter(conf, specs)
58 |     arbiter.run()
59 | 
60 | 
61 | 
62 | 
63 | 


--------------------------------------------------------------------------------
/pistil/tcp/sync_worker.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of pistil released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | import errno
 7 | import logging
 8 | import os
 9 | import select
10 | import socket
11 | 
12 | 
13 | from pistil import util
14 | from pistil.worker import Worker
15 | 
16 | 
17 | log = logging.getLogger(__name__)
18 | 
19 | 
20 | class TcpSyncWorker(Worker):
21 | 
22 |     def on_init_process(self):
23 |         self.socket = self.conf.get('sock')
24 |         self.address = self.socket.getsockname()
25 |         util.close_on_exec(self.socket)
26 |         
27 |     def run(self):
28 |         self.socket.setblocking(0)
29 | 
30 |         while self.alive:
31 |             self.notify()
32 | 
33 |             # Accept a connection. If we get an error telling us
34 |             # that no connection is waiting we fall down to the
35 |             # select which is where we'll wait for a bit for new
36 |             # workers to come give us some love.
37 |             try:
38 |                 client, addr = self.socket.accept()
39 |                 client.setblocking(1)
40 |                 util.close_on_exec(client)
41 |                 self.handle(client, addr)
42 | 
43 |                 # Keep processing clients until no one is waiting. This
44 |                 # prevents the need to select() for every client that we
45 |                 # process.
46 |                 continue
47 | 
48 |             except socket.error, e:
49 |                 if e[0] not in (errno.EAGAIN, errno.ECONNABORTED):
50 |                     raise
51 | 
52 |             # If our parent changed then we shut down.
53 |             if self.ppid != os.getppid():
54 |                 log.info("Parent changed, shutting down: %s", self)
55 |                 return
56 |             
57 |             try:
58 |                 self.notify()
59 |                 ret = select.select([self.socket], [], self._PIPE,
60 |                         self.timeout / 2.0)
61 |                 if ret[0]:
62 |                     continue
63 |             except select.error, e:
64 |                 if e[0] == errno.EINTR:
65 |                     continue
66 |                 if e[0] == errno.EBADF:
67 |                     if self.nr < 0:
68 |                         continue
69 |                     else:
70 |                         return
71 |                 raise
72 | 
73 |     def handle(self, client, addr):
74 |         raise NotImplementedError
75 | 


--------------------------------------------------------------------------------
/examples/_sendfile.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of gunicorn released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | import errno
 7 | import os
 8 | import sys
 9 | 
10 | try:
11 |     import ctypes
12 |     import ctypes.util
13 | except MemoryError:
14 |     # selinux execmem denial
15 |     # https://bugzilla.redhat.com/show_bug.cgi?id=488396
16 |     raise ImportError
17 | 
18 | SUPPORTED_PLATFORMS = (
19 |         'darwin',
20 |         'freebsd',
21 |         'dragonfly',
22 |         'linux2')
23 | 
24 | if sys.version_info < (2, 6) or \
25 |         sys.platform not in SUPPORTED_PLATFORMS:
26 |     raise ImportError("sendfile isn't supported on this platform")
27 | 
28 | _libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True)
29 | _sendfile = _libc.sendfile
30 | 
31 | def sendfile(fdout, fdin, offset, nbytes):
32 |     if sys.platform == 'darwin':
33 |         _sendfile.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_uint64,
34 |                               ctypes.POINTER(ctypes.c_uint64), ctypes.c_voidp,
35 |                               ctypes.c_int]
36 |         _nbytes = ctypes.c_uint64(nbytes)
37 |         result = _sendfile(fdin, fdout, offset, _nbytes, None, 0)
38 |         
39 |         if result == -1:
40 |             e = ctypes.get_errno()
41 |             if e == errno.EAGAIN and _nbytes.value:
42 |                 return nbytes.value
43 |             raise OSError(e, os.strerror(e))
44 |         return _nbytes.value
45 |     elif sys.platform in ('freebsd', 'dragonfly',):
46 |         _sendfile.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_uint64,
47 |                               ctypes.c_uint64, ctypes.c_voidp,
48 |                               ctypes.POINTER(ctypes.c_uint64), ctypes.c_int]
49 |         _sbytes = ctypes.c_uint64()
50 |         result = _sendfile(fdin, fdout, offset, nbytes, None, _sbytes, 0)
51 |         if result == -1:
52 |             e = ctypes.get_errno()
53 |             if e == errno.EAGAIN and _sbytes.value:
54 |                 return _sbytes.value
55 |             raise OSError(e, os.strerror(e))
56 |         return _sbytes.value
57 | 
58 |     else:
59 |         _sendfile.argtypes = [ctypes.c_int, ctypes.c_int,
60 |                 ctypes.POINTER(ctypes.c_uint64), ctypes.c_size_t]
61 | 
62 |         _offset = ctypes.c_uint64(offset)
63 |         sent = _sendfile(fdout, fdin, _offset, nbytes) 
64 |         if sent == -1:
65 |             e = ctypes.get_errno()
66 |             raise OSError(e, os.strerror(e))
67 |         return sent
68 | 
69 | 


--------------------------------------------------------------------------------
/pistil/tcp/gevent_worker.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of pistil released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | from __future__ import with_statement
 7 | 
 8 | import os
 9 | import sys
10 | import logging
11 | log = logging.getLogger(__name__)
12 | 
13 | try:
14 |     import gevent
15 | except ImportError:
16 |     raise RuntimeError("You need gevent installed to use this worker.")
17 | 
18 | 
19 | from gevent.pool import Pool
20 | from gevent.server import StreamServer
21 | 
22 | from pistil import util
23 | from pistil.tcp.sync_worker import TcpSyncWorker 
24 | 
25 | # workaround on osx, disable kqueue
26 | if sys.platform == "darwin":
27 |     os.environ['EVENT_NOKQUEUE'] = "1"
28 | 
29 | 
30 | class PStreamServer(StreamServer):
31 |     def __init__(self, listener, handle, spawn='default', worker=None):
32 |         StreamServer.__init__(self, listener, spawn=spawn)
33 |         self.handle_func = handle
34 |         self.worker = worker
35 | 
36 |     def stop(self, timeout=None):
37 |         super(PStreamServer, self).stop(timeout=timeout)
38 | 
39 |     def handle(self, sock, addr):
40 |         self.handle_func(sock, addr)
41 | 
42 | 
43 | class TcpGeventWorker(TcpSyncWorker):
44 | 
45 |     def on_init(self, conf):
46 |         self.worker_connections = conf.get("worker_connections", 
47 |                 10000)
48 |         self.pool = Pool(self.worker_connections)
49 | 
50 |     def run(self):
51 |         self.socket.setblocking(1)
52 |         
53 |         # start gevent stream server
54 |         server = PStreamServer(self.socket, self.handle, spawn=self.pool,
55 |                 worker=self)
56 |         server.start()
57 | 
58 |         try:
59 |             while self.alive:
60 |                 self.notify()
61 |                 if self.ppid != os.getppid():
62 |                     log.info("Parent changed, shutting down: %s", self)
63 |                     break
64 |         
65 |                 gevent.sleep(1.0)
66 |                 
67 |         except KeyboardInterrupt:
68 |             pass
69 | 
70 |         try:
71 |             # Try to stop connections until timeout
72 |             self.notify()
73 |             server.stop(timeout=self.timeout)
74 |         except:
75 |             pass
76 | 
77 | 
78 |     if hasattr(gevent.core, 'dns_shutdown'):
79 | 
80 |         def init_process(self):
81 |             #gevent 0.13 and older doesn't reinitialize dns for us after forking
82 |             #here's the workaround
83 |             gevent.core.dns_shutdown(fail_requests=1)
84 |             gevent.core.dns_init()
85 |             super(TcpGeventWorker, self).init_process()
86 | 
87 | 


--------------------------------------------------------------------------------
/pistil/pidfile.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -
 2 | #
 3 | # This file is part of pistil released under the MIT license. 
 4 | # See the NOTICE for more information.
 5 | 
 6 | from __future__ import with_statement
 7 | 
 8 | import errno
 9 | import os
10 | import tempfile
11 | 
12 | 
13 | class Pidfile(object):
14 |     """\
15 |     Manage a PID file. If a specific name is provided
16 |     it and '"%s.oldpid" % name' will be used. Otherwise
17 |     we create a temp file using os.mkstemp.
18 |     """
19 | 
20 |     def __init__(self, fname):
21 |         self.fname = fname
22 |         self.pid = None
23 |         
24 |     def create(self, pid):
25 |         oldpid = self.validate()
26 |         if oldpid:
27 |             if oldpid == os.getpid():
28 |                 return
29 |             raise RuntimeError("Already running on PID %s " \
30 |                 "(or pid file '%s' is stale)" % (os.getpid(), self.fname))
31 | 
32 |         self.pid = pid
33 |         
34 |         # Write pidfile
35 |         fdir = os.path.dirname(self.fname)
36 |         if fdir and not os.path.isdir(fdir):
37 |             raise RuntimeError("%s doesn't exist. Can't create pidfile." % fdir)
38 |         fd, fname = tempfile.mkstemp(dir=fdir)
39 |         os.write(fd, "%s\n" % self.pid)
40 |         if self.fname:
41 |             os.rename(fname, self.fname)
42 |         else:
43 |             self.fname = fname
44 |         os.close(fd)
45 | 
46 |         # set permissions to -rw-r--r-- 
47 |         os.chmod(self.fname, 420)
48 |         
49 |     def rename(self, path):
50 |         self.unlink()
51 |         self.fname = path
52 |         self.create(self.pid)
53 |         
54 |     def unlink(self):
55 |         """ delete pidfile"""
56 |         try:
57 |             with open(self.fname, "r") as f:
58 |                 pid1 =  int(f.read() or 0)
59 | 
60 |             if pid1 == self.pid:
61 |                 os.unlink(self.fname)
62 |         except:
63 |             pass
64 |        
65 |     def validate(self):
66 |         """ Validate pidfile and make it stale if needed"""
67 |         if not self.fname:
68 |             return
69 |         try:
70 |             with open(self.fname, "r") as f:
71 |                 wpid = int(f.read() or 0)
72 | 
73 |                 if wpid <= 0:
74 |                     return
75 | 
76 |                 try:
77 |                     os.kill(wpid, 0)
78 |                     return wpid
79 |                 except OSError, e:
80 |                     if e[0] == errno.ESRCH:
81 |                         return
82 |                     raise
83 |         except IOError, e:
84 |             if e[0] == errno.ENOENT:
85 |                 return
86 |             raise
87 | 
88 | 


--------------------------------------------------------------------------------
/examples/amqp.py:
--------------------------------------------------------------------------------
 1 | #-*- coding: utf-8 -*-
 2 | """
 3 | A simple worker with a AMQP consumer.
 4 | 
 5 | This example shows how to implement a simple AMQP consumer
 6 | based on `Kombu <http://github.com/ask/kombu>`_ and shows you
 7 | what different kind of workers you can put to a arbiter
 8 | to manage the worker lifetime, event handling and shutdown/reload szenarios.
 9 | """
10 | import sys
11 | import time
12 | import socket
13 | import logging
14 | from pistil.arbiter import Arbiter
15 | from pistil.worker import Worker
16 | from kombu.connection import BrokerConnection
17 | from kombu.messaging import Exchange, Queue, Consumer, Producer
18 | 
19 | 
20 | CONNECTION = ('localhost', 'guest', 'default', '/')
21 | 
22 | log = logging.getLogger(__name__)
23 | 
24 | 
25 | class AMQPWorker(Worker):
26 | 
27 |     queues = [
28 |         {'routing_key': 'test',
29 |          'name': 'test',
30 |          'handler': 'handle_test'
31 |         }
32 |     ]
33 | 
34 |     _connection = None
35 | 
36 |     def handle_test(self, body, message):
37 |         log.debug("Handle message: %s" % body)
38 |         message.ack()
39 | 
40 |     def handle(self):
41 |         log.debug("Start consuming")
42 |         exchange = Exchange('amqp.topic', type='direct', durable=True)
43 |         self._connection = BrokerConnection(*CONNECTION)
44 |         channel = self._connection.channel()
45 | 
46 |         for entry in self.queues:
47 |             log.debug("prepare to consume %s" % entry['routing_key'])
48 |             queue = Queue(entry['name'], exchange=exchange,
49 |                           routing_key=entry['routing_key'])
50 |             consumer = Consumer(channel, queue)
51 |             consumer.register_callback(getattr(self, entry['handler']))
52 |             consumer.consume()
53 | 
54 |         log.debug("start consuming...")
55 |         while True:
56 |             try:
57 |                 self._connection.drain_events()
58 |             except socket.timeout:
59 |                 log.debug("nothing to consume...")
60 |                 break
61 |         self._connection.close()
62 | 
63 |     def run(self):
64 |         while self.alive:
65 |             try:
66 |                 self.handle()
67 |             except Exception:
68 |                 self.alive = False
69 |                 raise
70 | 
71 |     def handle_quit(self, sig, frame):
72 |         if self._connection is not None:
73 |             self._connection.close()
74 |         self.alive = False
75 | 
76 |     def handle_exit(self, sig, frame):
77 |         if self._connection is not None:
78 |             self._connection.close()
79 |         self.alive = False
80 |         sys.exit(0)
81 | 
82 | 
83 | if __name__ == "__main__":
84 |     conf = {}
85 |     specs = [(AMQPWorker, None, "worker", {}, "test")]
86 |     a = Arbiter(conf, specs)
87 |     a.run()
88 | 


--------------------------------------------------------------------------------
/pistil/worker.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -
  2 | #
  3 | # This file is part of pistil released under the MIT license. 
  4 | # See the NOTICE for more information.
  5 | 
  6 | from __future__ import with_statement
  7 | 
  8 | import logging
  9 | import os
 10 | import signal
 11 | import sys
 12 | import time
 13 | import traceback
 14 | 
 15 | 
 16 | from pistil import util
 17 | from pistil.workertmp import WorkerTmp
 18 | 
 19 | log = logging.getLogger(__name__)
 20 | 
 21 | class Worker(object):
 22 |     
 23 |     _SIGNALS = map(
 24 |         lambda x: getattr(signal, "SIG%s" % x),
 25 |         "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split()
 26 |     )
 27 |     
 28 |     _PIPE = []
 29 | 
 30 | 
 31 |     def __init__(self, conf, name=None, child_type="worker", 
 32 |             age=0, ppid=0, timeout=30):
 33 | 
 34 |         if name is None:
 35 |             name =  self.__class__.__name__
 36 |         self.name = name
 37 | 
 38 |         self.child_type = child_type
 39 |         self.age = age
 40 |         self.ppid = ppid
 41 |         self.timeout = timeout
 42 |         self.conf = conf
 43 | 
 44 | 
 45 |         # initialize
 46 |         self.booted = False
 47 |         self.alive = True
 48 |         self.debug =self.conf.get("debug", False)
 49 |         self.tmp = WorkerTmp(self.conf)
 50 | 
 51 |         self.on_init(self.conf)
 52 | 
 53 |     def on_init(self, conf):
 54 |         pass
 55 | 
 56 | 
 57 |     def pid(self):
 58 |         return os.getpid()
 59 |     pid = util.cached_property(pid)
 60 | 
 61 |     def notify(self):
 62 |         """\
 63 |         Your worker subclass must arrange to have this method called
 64 |         once every ``self.timeout`` seconds. If you fail in accomplishing
 65 |         this task, the master process will murder your workers.
 66 |         """
 67 |         self.tmp.notify()
 68 | 
 69 |     
 70 |     def handle(self):
 71 |         raise NotImplementedError
 72 | 
 73 |     def run(self):
 74 |         """\
 75 |         This is the mainloop of a worker process. You should override
 76 |         this method in a subclass to provide the intended behaviour
 77 |         for your particular evil schemes.
 78 |         """
 79 |         while True:
 80 |             self.notify()
 81 |             self.handle()
 82 |             time.sleep(0.1)
 83 | 
 84 |     def on_init_process(self):
 85 |         """ method executed when we init a process """
 86 |         pass
 87 | 
 88 |     def init_process(self):
 89 |         """\
 90 |         If you override this method in a subclass, the last statement
 91 |         in the function should be to call this method with
 92 |         super(MyWorkerClass, self).init_process() so that the ``run()``
 93 |         loop is initiated.
 94 |         """
 95 |         util.set_owner_process(self.conf.get("uid", os.geteuid()),
 96 |                 self.conf.get("gid", os.getegid()))
 97 | 
 98 |         # Reseed the random number generator
 99 |         util.seed()
100 | 
101 |         # For waking ourselves up
102 |         self._PIPE = os.pipe()
103 |         map(util.set_non_blocking, self._PIPE)
104 |         map(util.close_on_exec, self._PIPE)
105 |         
106 |         # Prevent fd inherientence
107 |         util.close_on_exec(self.tmp.fileno())
108 |         self.init_signals()
109 |         
110 |         self.on_init_process()
111 | 
112 |         # Enter main run loop
113 |         self.booted = True
114 |         self.run()
115 | 
116 |     def init_signals(self):
117 |         map(lambda s: signal.signal(s, signal.SIG_DFL), self._SIGNALS)
118 |         signal.signal(signal.SIGQUIT, self.handle_quit)
119 |         signal.signal(signal.SIGTERM, self.handle_exit)
120 |         signal.signal(signal.SIGINT, self.handle_exit)
121 |         signal.signal(signal.SIGWINCH, self.handle_winch)
122 |             
123 |     def handle_quit(self, sig, frame):
124 |         self.alive = False
125 | 
126 |     def handle_exit(self, sig, frame):
127 |         self.alive = False
128 |         sys.exit(0)
129 |         
130 |     def handle_winch(self, sig, fname):
131 |         # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD.
132 |         return
133 | 


--------------------------------------------------------------------------------
/pistil/tcp/sock.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -
  2 | #
  3 | # This file is part of gunicorn released under the MIT license. 
  4 | # See the NOTICE for more information.
  5 | 
  6 | import errno
  7 | import logging
  8 | import os
  9 | import socket
 10 | import sys
 11 | import time
 12 | 
 13 | from pistil import util
 14 | 
 15 | log = logging.getLogger(__name__)
 16 | 
 17 | class BaseSocket(object):
 18 |     
 19 |     def __init__(self, conf, fd=None):
 20 |         self.conf = conf
 21 |         self.address = util.parse_address(conf.get('address',
 22 |             ('127.0.0.1', 8000)))
 23 |         if fd is None:
 24 |             sock = socket.socket(self.FAMILY, socket.SOCK_STREAM)
 25 |         else:
 26 |             sock = socket.fromfd(fd, self.FAMILY, socket.SOCK_STREAM)
 27 |         self.sock = self.set_options(sock, bound=(fd is not None))
 28 |     
 29 |     def __str__(self, name):
 30 |         return "<socket %d>" % self.sock.fileno()
 31 |     
 32 |     def __getattr__(self, name):
 33 |         return getattr(self.sock, name)
 34 |     
 35 |     def set_options(self, sock, bound=False):
 36 |         sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 37 |         if not bound:
 38 |             self.bind(sock)
 39 |         sock.setblocking(0)
 40 |         sock.listen(self.conf.get('backlog', 2048))
 41 |         return sock
 42 |         
 43 |     def bind(self, sock):
 44 |         sock.bind(self.address)
 45 |         
 46 |     def close(self):
 47 |         try:
 48 |             self.sock.close()
 49 |         except socket.error, e:
 50 |             log.info("Error while closing socket %s", str(e))
 51 |         time.sleep(0.3)
 52 |         del self.sock
 53 | 
 54 | class TCPSocket(BaseSocket):
 55 |     
 56 |     FAMILY = socket.AF_INET
 57 |     
 58 |     def __str__(self):
 59 |         return "http://%s:%d" % self.sock.getsockname()
 60 |     
 61 |     def set_options(self, sock, bound=False):
 62 |         sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
 63 |         return super(TCPSocket, self).set_options(sock, bound=bound)
 64 | 
 65 | class TCP6Socket(TCPSocket):
 66 | 
 67 |     FAMILY = socket.AF_INET6
 68 | 
 69 |     def __str__(self):
 70 |         (host, port, fl, sc) = self.sock.getsockname()
 71 |         return "http://[%s]:%d" % (host, port)
 72 | 
 73 | class UnixSocket(BaseSocket):
 74 |     
 75 |     FAMILY = socket.AF_UNIX
 76 |     
 77 |     def __init__(self, conf, fd=None):
 78 |         if fd is None:
 79 |             try:
 80 |                 os.remove(conf.address)
 81 |             except OSError:
 82 |                 pass
 83 |         super(UnixSocket, self).__init__(conf, fd=fd)
 84 |     
 85 |     def __str__(self):
 86 |         return "unix:%s" % self.address
 87 |         
 88 |     def bind(self, sock):
 89 |         old_umask = os.umask(self.conf.get("umask", 0))
 90 |         sock.bind(self.address)
 91 |         util.chown(self.address, self.conf.get("uid", os.geteuid()),
 92 |                 self.conf.get("gid", os.getegid()))
 93 |         os.umask(old_umask)
 94 |         
 95 |     def close(self):
 96 |         super(UnixSocket, self).close()
 97 |         os.unlink(self.address)
 98 | 
 99 | def create_socket(conf):
100 |     """
101 |     Create a new socket for the given address. If the
102 |     address is a tuple, a TCP socket is created. If it
103 |     is a string, a Unix socket is created. Otherwise
104 |     a TypeError is raised.
105 |     """
106 |     # get it only once
107 |     addr = conf.get("address", ('127.0.0.1', 8000))
108 |     
109 |     if isinstance(addr, tuple):
110 |         if util.is_ipv6(addr[0]):
111 |             sock_type = TCP6Socket
112 |         else:
113 |             sock_type = TCPSocket
114 |     elif isinstance(addr, basestring):
115 |         sock_type = UnixSocket
116 |     else:
117 |         raise TypeError("Unable to create socket from: %r" % addr)
118 | 
119 |     if 'PISTIL_FD' in os.environ:
120 |         fd = int(os.environ.pop('PISTIL_FD'))
121 |         try:
122 |             return sock_type(conf, fd=fd)
123 |         except socket.error, e:
124 |             if e[0] == errno.ENOTCONN:
125 |                 log.error("PISTIL_FD should refer to an open socket.")
126 |             else:
127 |                 raise
128 | 
129 |     # If we fail to create a socket from GUNICORN_FD
130 |     # we fall through and try and open the socket
131 |     # normally.
132 |     
133 |     for i in range(5):
134 |         try:
135 |             return sock_type(conf)
136 |         except socket.error, e:
137 |             if e[0] == errno.EADDRINUSE:
138 |                 log.error("Connection in use: %s", str(addr))
139 |             if e[0] == errno.EADDRNOTAVAIL:
140 |                 log.error("Invalid address: %s", str(addr))
141 |                 sys.exit(1)
142 |             if i < 5:
143 |                 log.error("Retrying in 1 second.")
144 |                 time.sleep(1)
145 |           
146 |     log.error("Can't connect to %s", str(addr))
147 |     sys.exit(1)
148 | 
149 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
  1 | pistil
  2 | ------
  3 | 
  4 | 
  5 | Simple multiprocessing toolkit. This is based on the `Gunicorn <http://gunicorn.org>`_ multiprocessing engine. 
  6 | 
  7 | This library allows you to supervise multiple type of workers and chain
  8 | supervisors. Gracefull, reload, signaling between workers is handled.  
  9 | 
 10 | 
 11 | Simple Arbiter launching one worker::
 12 | 
 13 |     from pistil.arbiter import Arbiter
 14 |     from pistil.worker import Worker
 15 | 
 16 |     class MyWorker(Worker):
 17 | 
 18 |         def handle(self):
 19 |             print "hello from worker n°%s" % self.pid
 20 | 
 21 |     if __name__ == "__main__":
 22 |         conf = {}
 23 |         specs = [(MyWorker, 30, "worker", {}, "test")]
 24 |         a = Arbiter(conf, specs)
 25 |         a.run()
 26 | 
 27 | The same with different with the Pool arbiter. This time we send the
 28 | same worker on 3 os processes::
 29 |     
 30 |     from pistil.pool import PoolArbiter
 31 |     from pistil.worker import Worker
 32 | 
 33 |     class MyWorker(Worker):
 34 | 
 35 |         def handle(self):
 36 |             print "hello from worker n°%s" % self.pid
 37 | 
 38 |     if __name__ == "__main__":
 39 |         conf = {"num_workers": 3 }
 40 |         spec = (MyWorker, 30, "worker", {}, "test",)
 41 |         a = PoolArbiter(conf, spec)
 42 |         a.run()
 43 | 
 44 | A common use for that pattern is a tcp server tjhat share the same
 45 | socket between them. For that purpose pistil provides the TcpArbiter and
 46 | TcpSyncWorker and the GeventTcpWorker to use with gevent.
 47 | 
 48 | Pistil allows you to mix diffrent kind of workers in an arbiter::
 49 | 
 50 |     from pistil.arbiter import Arbiter
 51 |     from pistil.worker import Worker
 52 | 
 53 |     class MyWorker(Worker):
 54 | 
 55 |         def handle(self): 
 56 |             print "hello worker 1 from %s" % self.name
 57 | 
 58 |     class MyWorker2(Worker):
 59 | 
 60 |         def handle(self):
 61 |             print "hello worker 2 from %s" % self.name
 62 | 
 63 | 
 64 |     if __name__ == '__main__':
 65 |         conf = {}
 66 | 
 67 |         specs = [
 68 |             (MyWorker, 30, "worker", {}, "w1"),
 69 |             (MyWorker2, 30, "worker", {}, "w2"),
 70 |             (MyWorker2, 30, "kill", {}, "w3")
 71 |         ]
 72 |         # launchh the arbiter
 73 |         arbiter = Arbiter(conf, specs)
 74 |         arbiter.run()
 75 | 
 76 | You can also chain arbiters:: 
 77 | 
 78 |     import time
 79 |     import urllib2
 80 | 
 81 |     from pistil.arbiter import Arbiter
 82 |     from pistil.worker import Worker
 83 |     from pistil.tcp.sync_worker import TcpSyncWorker
 84 |     from pistil.tcp.arbiter import TcpArbiter
 85 | 
 86 |     from http_parser.http import HttpStream
 87 |     from http_parser.reader import SocketReader
 88 | 
 89 |     class MyTcpWorker(TcpSyncWorker):
 90 | 
 91 |         def handle(self, sock, addr):
 92 |             p = HttpStream(SocketReader(sock))
 93 |             path = p.path()
 94 |             data = "welcome wold"
 95 |             sock.send("".join(["HTTP/1.1 200 OK\r\n", 
 96 |                             "Content-Type: text/html\r\n",
 97 |                             "Content-Length:" + str(len(data)) + "\r\n",
 98 |                              "Connection: close\r\n\r\n",
 99 |                              data]))
100 | 
101 | 
102 |     class UrlWorker(Worker):
103 | 
104 |         def handle(self):
105 |             f = urllib2.urlopen("http://localhost:5000")
106 |             print f.read()
107 | 
108 |     class MyPoolArbiter(TcpArbiter):
109 | 
110 |         def on_init(self, conf):
111 |             TcpArbiter.on_init(self, conf)
112 |             # we return a spec
113 |             return (MyTcpWorker, 30, "worker", {}, "http_welcome",)
114 | 
115 | 
116 |     if __name__ == '__main__':
117 |         conf = {"num_workers": 3, "address": ("127.0.0.1", 5000)}
118 | 
119 |         specs = [
120 |             (MyPoolArbiter, 30, "supervisor", {}, "tcp_pool"),
121 |             (UrlWorker, 30, "worker", {}, "grabber")
122 |         ]
123 | 
124 |         arbiter = Arbiter(conf, specs)
125 |         arbiter.run()
126 | 
127 | 
128 | This examplelaunch a web server with 3 workers on port 5000 and another
129 | worker fetching the welcome page hosted by this server::
130 | 
131 | 
132 |     $ python examples/multiworker2.py 
133 | 
134 |     2011-08-08 00:05:42 [13195] [DEBUG] Arbiter master booted on 13195
135 |     2011-08-08 00:05:42 [13196] [INFO] Booting grabber (worker) with pid: 13196
136 |     ici
137 |     2011-08-08 00:05:42 [13197] [INFO] Booting pool (supervisor) with pid: 13197
138 |     2011-08-08 00:05:42 [13197] [DEBUG] Arbiter pool booted on 13197
139 |     2011-08-08 00:05:42 [13197] [INFO] Listening at: http://127.0.0.1:5000 (13197)
140 |     2011-08-08 00:05:42 [13198] [INFO] Booting worker (worker) with pid: 13198
141 |     2011-08-08 00:05:42 [13199] [INFO] Booting worker (worker) with pid: 13199
142 |     welcome world
143 |     welcome world
144 | 
145 | 
146 | More documentation is comming. See also examples in the examples/
147 | folder.
148 | 


--------------------------------------------------------------------------------
/examples/serve_file.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -
  2 | #
  3 | # This file is part of gunicorn released under the MIT license. 
  4 | # See the NOTICE for more information.
  5 | 
  6 | import mimetypes
  7 | import os
  8 | 
  9 | 
 10 | from gevent import monkey
 11 | monkey.noisy = False
 12 | monkey.patch_all()
 13 | 
 14 | 
 15 | from http_parser.http import HttpStream
 16 | from http_parser.reader import SocketReader
 17 | 
 18 | from pistil import util
 19 | from pistil.tcp.arbiter import TcpArbiter
 20 | from pistil.tcp.gevent_worker import TcpGeventWorker
 21 | 
 22 | CURDIR = os.path.dirname(__file__)
 23 | 
 24 | try:
 25 |     # Python 3.3 has os.sendfile().
 26 |     from os import sendfile
 27 | except ImportError:
 28 |     try:
 29 |         from _sendfile import sendfile
 30 |     except ImportError:
 31 |         sendfile = None
 32 | 
 33 | def write_error(sock, status_int, reason, mesg):
 34 |     html = textwrap.dedent("""\
 35 |     <html>
 36 |       <head>
 37 |         <title>%(reason)s</title>
 38 |       </head>
 39 |       <body>
 40 |         <h1>%(reason)s</h1>
 41 |         %(mesg)s
 42 |       </body>
 43 |     </html>
 44 |     """) % {"reason": reason, "mesg": mesg}
 45 | 
 46 |     http = textwrap.dedent("""\
 47 |     HTTP/1.1 %s %s\r
 48 |     Connection: close\r
 49 |     Content-Type: text/html\r
 50 |     Content-Length: %d\r
 51 |     \r
 52 |     %s
 53 |     """) % (str(status_int), reason, len(html), html)
 54 |     write_nonblock(sock, http)
 55 | 
 56 | 
 57 | 
 58 | class HttpWorker(TcpGeventWorker):
 59 | 
 60 |     def handle(self, sock, addr):
 61 |         p = HttpStream(SocketReader(sock))
 62 | 
 63 |         path = p.path()
 64 | 
 65 |         if not path or path == "/":
 66 |             path = "index.html"
 67 |         
 68 |         if path.startswith("/"):
 69 |             path = path[1:]
 70 |         
 71 |         real_path = os.path.join(CURDIR, "static", path)
 72 | 
 73 |         if os.path.isdir(real_path):
 74 |             lines = ["<ul>"]
 75 |             for d in os.listdir(real_path):
 76 |                 fpath = os.path.join(real_path, d)
 77 |                 lines.append("<li><a href=" + d + ">" + d + "</a>")
 78 | 
 79 |             data = "".join(lines)
 80 |             resp = "".join(["HTTP/1.1 200 OK\r\n", 
 81 |                             "Content-Type: text/html\r\n",
 82 |                             "Content-Length:" + str(len(data)) + "\r\n",
 83 |                             "Connection: close\r\n\r\n",
 84 |                             data])
 85 |             sock.sendall(resp)
 86 | 
 87 |         elif not os.path.exists(real_path):
 88 |             util.write_error(sock, 404, "Not found", real_path + " not found")
 89 |         else:
 90 |             ctype = mimetypes.guess_type(real_path)[0]
 91 | 
 92 |             if ctype.startswith('text') or 'html' in ctype:
 93 | 
 94 |                 try:
 95 |                     f = open(real_path, 'rb')
 96 |                     data = f.read()
 97 |                     resp = "".join(["HTTP/1.1 200 OK\r\n", 
 98 |                                 "Content-Type: " + ctype + "\r\n",
 99 |                                 "Content-Length:" + str(len(data)) + "\r\n",
100 |                                 "Connection: close\r\n\r\n",
101 |                                 data])
102 |                     sock.sendall(resp)
103 |                 finally:
104 |                     f.close()
105 |             else:
106 | 
107 |                 try:
108 |                     f = open(real_path, 'r')
109 |                     clen = int(os.fstat(f.fileno())[6])
110 |                     
111 |                     # send headers
112 |                     sock.send("".join(["HTTP/1.1 200 OK\r\n", 
113 |                                 "Content-Type: " + ctype + "\r\n",
114 |                                 "Content-Length:" + str(clen) + "\r\n",
115 |                                  "Connection: close\r\n\r\n"]))
116 | 
117 |                     if not sendfile:
118 |                         while True:
119 |                             data = f.read(4096)
120 |                             if not data:
121 |                                 break
122 |                             sock.send(data)
123 |                     else:
124 |                         fileno = f.fileno()
125 |                         sockno = sock.fileno()
126 |                         sent = 0
127 |                         offset = 0
128 |                         nbytes = clen
129 |                         sent += sendfile(sockno, fileno, offset+sent, nbytes-sent)
130 |                         while sent != nbytes:
131 |                             sent += sendfile(sock.fileno(), fileno, offset+sent, nbytes-sent)
132 | 
133 | 
134 |                 finally:
135 |                     f.close()
136 | 
137 | 
138 | def main():
139 |     conf = {"address": ("127.0.0.1", 5000), "debug": True,
140 |             "num_workers": 3}
141 |     spec = (HttpWorker, 30, "send_file", {}, "worker",)
142 |     
143 |     arbiter = TcpArbiter(conf, spec)
144 |     arbiter.run()
145 | 
146 | 
147 | if __name__ == '__main__':
148 |     main()
149 | 


--------------------------------------------------------------------------------
/pistil/pool.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -
  2 | #
  3 | # This file is part of pistil released under the MIT license. 
  4 | # See the NOTICE for more information.
  5 | 
  6 | import errno
  7 | import os
  8 | import signal
  9 | 
 10 | from pistil.errors import HaltServer
 11 | from pistil.arbiter import Arbiter, Child
 12 | from pistil.workertmp import WorkerTmp
 13 | from pistil import util
 14 | 
 15 | DEFAULT_CONF = dict(
 16 |     uid = os.geteuid(),
 17 |     gid = os.getegid(),
 18 |     umask = 0,
 19 |     debug = False,
 20 |     num_workers = 1,
 21 | )
 22 | 
 23 | 
 24 | class PoolArbiter(Arbiter):
 25 | 
 26 | 
 27 |     _SIGNALS = map(
 28 |         lambda x: getattr(signal, "SIG%s" % x),
 29 |         "HUP QUIT INT TERM TTIN TTOU USR1 USR2 WINCH".split()
 30 |     )
 31 | 
 32 |     def __init__(self, args, spec=(), name=None,
 33 |             child_type="supervisor", age=0, ppid=0,
 34 |             timeout=30):
 35 | 
 36 |         if not isinstance(spec, tuple):
 37 |             raise TypeError("spec should be a tuple")
 38 | 
 39 |         # set conf
 40 |         conf = DEFAULT_CONF.copy()
 41 |         conf.update(args)
 42 |         self.conf = conf
 43 | 
 44 |         # set number of workers
 45 |         self.num_workers = conf.get('num_workers', 1)
 46 |        
 47 |         ret =  self.on_init(conf)
 48 |         if not ret: 
 49 |             self._SPEC = Child(*spec)
 50 |         else:
 51 |             self._SPEC = Child(*ret)
 52 |         
 53 |         if name is None:
 54 |             name =  self.__class__.__name__
 55 | 
 56 |         self.name = name
 57 |         self.child_type = child_type
 58 |         self.age = age
 59 |         self.ppid = ppid
 60 |         self.timeout = timeout
 61 | 
 62 | 
 63 |         self.pid = None
 64 |         self.child_age = 0
 65 |         self.booted = False
 66 |         self.stopping = False
 67 |         self.debug =self.conf.get("debug", False)
 68 |         self.tmp = WorkerTmp(self.conf) 
 69 | 
 70 |     def update_proc_title(self):
 71 |         util._setproctitle("arbiter [%s running %s workers]" % (self.name,  
 72 |             self.num_workers))
 73 | 
 74 |     def on_init(self, conf):
 75 |         return None
 76 | 
 77 |     def on_init_process(self):
 78 |         self.update_proc_title()
 79 |         
 80 |     def handle_ttin(self):
 81 |         """\
 82 |         SIGTTIN handling.
 83 |         Increases the number of workers by one.
 84 |         """
 85 |         self.num_workers += 1
 86 |         self.update_proc_title()
 87 |         self.manage_workers()
 88 |     
 89 |     def handle_ttou(self):
 90 |         """\
 91 |         SIGTTOU handling.
 92 |         Decreases the number of workers by one.
 93 |         """
 94 |         if self.num_workers <= 1:
 95 |             return
 96 |         self.num_workers -= 1
 97 |         self.update_proc_title()
 98 |         self.manage_workers()
 99 | 
100 |     def reload(self):
101 |         """ 
102 |         used on HUP
103 |         """
104 | 
105 |         # exec on reload hook
106 |         self.on_reload()
107 | 
108 |         # spawn new workers with new app & conf
109 |         for i in range(self.conf.get("num_workers", 1)):
110 |             self.spawn_child(self._SPEC)
111 |             
112 |         # set new proc_name
113 |         util._setproctitle("master [%s]" % self.name)
114 |         
115 |         # manage workers
116 |         self.manage_workers() 
117 | 
118 |     def reap_workers(self):
119 |         """\
120 |         Reap workers to avoid zombie processes
121 |         """
122 |         try:
123 |             while True:
124 |                 wpid, status = os.waitpid(-1, os.WNOHANG)
125 |                 if not wpid:
126 |                     break
127 |                 
128 |                 # A worker said it cannot boot. We'll shutdown
129 |                 # to avoid infinite start/stop cycles.
130 |                 exitcode = status >> 8
131 |                 if exitcode == self._WORKER_BOOT_ERROR:
132 |                     reason = "Worker failed to boot."
133 |                     raise HaltServer(reason, self._WORKER_BOOT_ERROR)
134 |                 child_info = self._WORKERS.pop(wpid, None)
135 | 
136 |                 if not child_info:
137 |                     continue
138 | 
139 |                 child, state = child_info
140 |                 child.tmp.close()
141 |         except OSError, e:
142 |             if e.errno == errno.ECHILD:
143 |                 pass
144 | 
145 |     def manage_workers(self):
146 |         """\
147 |         Maintain the number of workers by spawning or killing
148 |         as required.
149 |         """
150 |         if len(self._WORKERS.keys()) < self.num_workers:
151 |             self.spawn_workers()
152 | 
153 |         workers = self._WORKERS.items()
154 |         workers.sort(key=lambda w: w[1][0].age)
155 |         while len(workers) > self.num_workers:
156 |             (pid, _) = workers.pop(0)
157 |             self.kill_worker(pid, signal.SIGQUIT)
158 | 
159 |     def spawn_workers(self):
160 |         """\
161 |         Spawn new workers as needed.
162 |         
163 |         This is where a worker process leaves the main loop
164 |         of the master process.
165 |         """
166 |         for i in range(self.num_workers - len(self._WORKERS.keys())):
167 |             self.spawn_child(self._SPEC)
168 |             
169 |     
170 |     def kill_worker(self, pid, sig):
171 |         """\
172 |                Kill a worker
173 | 
174 |         :attr pid: int, worker pid
175 |         :attr sig: `signal.SIG*` value
176 |          """
177 |         if not isinstance(pid, int):
178 |             return
179 | 
180 |         try:
181 |             os.kill(pid, sig)
182 |         except OSError, e:
183 |             if e.errno == errno.ESRCH:
184 |                 try:
185 |                     (child, info) = self._WORKERS.pop(pid)
186 |                     child.tmp.close()
187 |                     return
188 |                 except (KeyError, OSError):
189 |                     return
190 |             raise  
191 | 


--------------------------------------------------------------------------------
/pistil/util.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -
  2 | #
  3 | # This file is part of gunicorn released under the MIT license. 
  4 | # See the NOTICE for more information.
  5 | 
  6 | 
  7 | try:
  8 |     import ctypes
  9 | except MemoryError:
 10 |     # selinux execmem denial
 11 |     # https://bugzilla.redhat.com/show_bug.cgi?id=488396
 12 |     ctypes = None
 13 | except ImportError:
 14 |     # Python on Solaris compiled with Sun Studio doesn't have ctypes
 15 |     ctypes = None
 16 | 
 17 | import fcntl
 18 | import os
 19 | import pkg_resources
 20 | import random
 21 | import resource
 22 | import socket
 23 | import sys
 24 | import textwrap
 25 | import time
 26 | 
 27 | 
 28 | MAXFD = 1024
 29 | if (hasattr(os, "devnull")):
 30 |    REDIRECT_TO = os.devnull
 31 | else:
 32 |    REDIRECT_TO = "/dev/null"
 33 | 
 34 | timeout_default = object()
 35 | 
 36 | CHUNK_SIZE = (16 * 1024)
 37 | 
 38 | MAX_BODY = 1024 * 132
 39 | 
 40 | weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
 41 | monthname = [None,
 42 |              'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
 43 |              'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
 44 | 
 45 | # Server and Date aren't technically hop-by-hop
 46 | # headers, but they are in the purview of the
 47 | # origin server which the WSGI spec says we should
 48 | # act like. So we drop them and add our own.
 49 | #
 50 | # In the future, concatenation server header values
 51 | # might be better, but nothing else does it and
 52 | # dropping them is easier.
 53 | hop_headers = set("""
 54 |     connection keep-alive proxy-authenticate proxy-authorization
 55 |     te trailers transfer-encoding upgrade
 56 |     server date
 57 |     """.split())
 58 |             
 59 | try:
 60 |     from setproctitle import setproctitle
 61 |     def _setproctitle(title):
 62 |         setproctitle(title) 
 63 | except ImportError:
 64 |     def _setproctitle(title):
 65 |         return
 66 | 
 67 | def load_worker_class(uri):
 68 |     if uri.startswith("egg:"):
 69 |         # uses entry points
 70 |         entry_str = uri.split("egg:")[1]
 71 |         try:
 72 |             dist, name = entry_str.rsplit("#",1)
 73 |         except ValueError:
 74 |             dist = entry_str
 75 |             name = "sync"
 76 | 
 77 |         return pkg_resources.load_entry_point(dist, "gunicorn.workers", name)
 78 |     else:
 79 |         components = uri.split('.')
 80 |         if len(components) == 1:
 81 |             try:
 82 |                 if uri.startswith("#"):
 83 |                     uri = uri[1:]
 84 |                 return pkg_resources.load_entry_point("gunicorn", 
 85 |                             "gunicorn.workers", uri)
 86 |             except ImportError: 
 87 |                 raise RuntimeError("arbiter uri invalid or not found")
 88 |         klass = components.pop(-1)
 89 |         mod = __import__('.'.join(components))
 90 |         for comp in components[1:]:
 91 |             mod = getattr(mod, comp)
 92 |         return getattr(mod, klass)
 93 | 
 94 | def set_owner_process(uid,gid):
 95 |     """ set user and group of workers processes """
 96 |     if gid:
 97 |         try:
 98 |             os.setgid(gid)
 99 |         except OverflowError:
100 |             if not ctypes:
101 |                 raise
102 |             # versions of python < 2.6.2 don't manage unsigned int for
103 |             # groups like on osx or fedora
104 |             os.setgid(-ctypes.c_int(-gid).value)
105 |             
106 |     if uid:
107 |         os.setuid(uid)
108 |         
109 | def chown(path, uid, gid):
110 |     try:
111 |         os.chown(path, uid, gid)
112 |     except OverflowError:
113 |         if not ctypes:
114 |             raise
115 |         os.chown(path, uid, -ctypes.c_int(-gid).value)
116 | 
117 | 
118 | def is_ipv6(addr):
119 |     try:
120 |         socket.inet_pton(socket.AF_INET6, addr)
121 |     except socket.error: # not a valid address
122 |         return False
123 |     return True
124 |         
125 | def parse_address(netloc, default_port=8000):
126 |     if isinstance(netloc, tuple):
127 |         return netloc
128 | 
129 |     if netloc.startswith("unix:"):
130 |         return netloc.split("unix:")[1]
131 | 
132 |     # get host
133 |     if '[' in netloc and ']' in netloc:
134 |         host = netloc.split(']')[0][1:].lower()
135 |     elif ':' in netloc:
136 |         host = netloc.split(':')[0].lower()
137 |     elif netloc == "":
138 |         host = "0.0.0.0"
139 |     else:
140 |         host = netloc.lower()
141 |     
142 |     #get port
143 |     netloc = netloc.split(']')[-1]
144 |     if ":" in netloc:
145 |         port = netloc.split(':', 1)[1]
146 |         if not port.isdigit():
147 |             raise RuntimeError("%r is not a valid port number." % port)
148 |         port = int(port)
149 |     else:
150 |         port = default_port 
151 |     return (host, port)
152 |     
153 | def get_maxfd():
154 |     maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
155 |     if (maxfd == resource.RLIM_INFINITY):
156 |         maxfd = MAXFD
157 |     return maxfd
158 | 
159 | def close_on_exec(fd):
160 |     flags = fcntl.fcntl(fd, fcntl.F_GETFD)
161 |     flags |= fcntl.FD_CLOEXEC
162 |     fcntl.fcntl(fd, fcntl.F_SETFD, flags)
163 |     
164 | def set_non_blocking(fd):
165 |     flags = fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK
166 |     fcntl.fcntl(fd, fcntl.F_SETFL, flags)
167 | 
168 | def close(sock):
169 |     try:
170 |         sock.close()
171 |     except socket.error:
172 |         pass
173 | 
174 | def write_chunk(sock, data):
175 |     chunk = "".join(("%X\r\n" % len(data), data, "\r\n"))
176 |     sock.sendall(chunk)
177 |     
178 | def write(sock, data, chunked=False):
179 |     if chunked:
180 |         return write_chunk(sock, data)
181 |     sock.sendall(data)
182 | 
183 | def write_nonblock(sock, data, chunked=False):
184 |     timeout = sock.gettimeout()
185 |     if timeout != 0.0:
186 |         try:
187 |             sock.setblocking(0)
188 |             return write(sock, data, chunked)
189 |         finally:
190 |             sock.setblocking(1)
191 |     else:
192 |         return write(sock, data, chunked)
193 |     
194 | def writelines(sock, lines, chunked=False):
195 |     for line in list(lines):
196 |         write(sock, line, chunked)
197 | 
198 | def write_error(sock, status_int, reason, mesg):
199 |     html = textwrap.dedent("""\
200 |     <html>
201 |       <head>
202 |         <title>%(reason)s</title>
203 |       </head>
204 |       <body>
205 |         <h1>%(reason)s</h1>
206 |         %(mesg)s
207 |       </body>
208 |     </html>
209 |     """) % {"reason": reason, "mesg": mesg}
210 | 
211 |     http = textwrap.dedent("""\
212 |     HTTP/1.1 %s %s\r
213 |     Connection: close\r
214 |     Content-Type: text/html\r
215 |     Content-Length: %d\r
216 |     \r
217 |     %s
218 |     """) % (str(status_int), reason, len(html), html)
219 |     write_nonblock(sock, http)
220 | 
221 | def normalize_name(name):
222 |     return  "-".join([w.lower().capitalize() for w in name.split("-")])
223 |     
224 | def import_app(module):
225 |     parts = module.split(":", 1)
226 |     if len(parts) == 1:
227 |         module, obj = module, "application"
228 |     else:
229 |         module, obj = parts[0], parts[1]
230 | 
231 |     try:
232 |         __import__(module)
233 |     except ImportError:
234 |         if module.endswith(".py") and os.path.exists(module):
235 |             raise ImportError("Failed to find application, did "
236 |                 "you mean '%s:%s'?" % (module.rsplit(".",1)[0], obj))
237 |         else:
238 |             raise
239 | 
240 |     mod = sys.modules[module]
241 |     app = eval(obj, mod.__dict__)
242 |     if app is None:
243 |         raise ImportError("Failed to find application object: %r" % obj)
244 |     if not callable(app):
245 |         raise TypeError("Application object must be callable.")
246 |     return app
247 | 
248 | def http_date(timestamp=None):
249 |     """Return the current date and time formatted for a message header."""
250 |     if timestamp is None:
251 |         timestamp = time.time()
252 |     year, month, day, hh, mm, ss, wd, y, z = time.gmtime(timestamp)
253 |     s = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % (
254 |             weekdayname[wd],
255 |             day, monthname[month], year,
256 |             hh, mm, ss)
257 |     return s
258 |     
259 | def to_bytestring(s):
260 |     """ convert to bytestring an unicode """
261 |     if not isinstance(s, basestring):
262 |         return s
263 |     if isinstance(s, unicode):
264 |         return s.encode('utf-8')
265 |     else:
266 |         return s
267 | 
268 | def is_hoppish(header):
269 |     return header.lower().strip() in hop_headers
270 | 
271 | def daemonize():
272 |     """\
273 |     Standard daemonization of a process.
274 |     http://www.svbug.com/documentation/comp.unix.programmer-FAQ/faq_2.html#SEC16
275 |     """
276 |     if not 'GUNICORN_FD' in os.environ:
277 |         if os.fork():
278 |             os._exit(0)
279 |         os.setsid()
280 | 
281 |         if os.fork():
282 |             os._exit(0)
283 |         
284 |         os.umask(0)
285 |         maxfd = get_maxfd()
286 | 
287 |         # Iterate through and close all file descriptors.
288 |         for fd in range(0, maxfd):
289 |             try:
290 |                 os.close(fd)
291 |             except OSError:	# ERROR, fd wasn't open to begin with (ignored)
292 |                 pass
293 |         
294 |         os.open(REDIRECT_TO, os.O_RDWR)
295 |         os.dup2(0, 1)
296 |         os.dup2(0, 2)
297 | 
298 | def seed():
299 |     try:
300 |         random.seed(os.urandom(64))
301 |     except NotImplementedError:
302 |         random.seed(random.random())
303 | 
304 | 
305 | class _Missing(object):
306 | 
307 |     def __repr__(self):
308 |         return 'no value'
309 | 
310 |     def __reduce__(self):
311 |         return '_missing'
312 | 
313 | _missing = _Missing()
314 | 
315 | class cached_property(object):
316 |     
317 |     def __init__(self, func, name=None, doc=None):
318 |         self.__name__ = name or func.__name__
319 |         self.__module__ = func.__module__
320 |         self.__doc__ = doc or func.__doc__
321 |         self.func = func
322 | 
323 |     def __get__(self, obj, type=None):
324 |         if obj is None:
325 |             return self
326 |         value = obj.__dict__.get(self.__name__, _missing)
327 |         if value is _missing:
328 |             value = self.func(obj)
329 |             obj.__dict__[self.__name__] = value
330 |         return value
331 | 


--------------------------------------------------------------------------------
/pistil/arbiter.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -
  2 | #
  3 | # This file is part of pistil released under the MIT license. 
  4 | # See the NOTICE for more information.
  5 | 
  6 | from __future__ import with_statement
  7 | 
  8 | import errno
  9 | import logging
 10 | import os
 11 | import select
 12 | import signal
 13 | import sys
 14 | import time
 15 | import traceback
 16 | 
 17 | from pistil.errors import HaltServer
 18 | from pistil.workertmp import WorkerTmp
 19 | from pistil import util
 20 | from pistil import __version__, SERVER_SOFTWARE
 21 | 
 22 | LOG_LEVELS = {
 23 |     "critical": logging.CRITICAL,
 24 |     "error": logging.ERROR,
 25 |     "warning": logging.WARNING,
 26 |     "info": logging.INFO,
 27 |     "debug": logging.DEBUG
 28 | }
 29 | 
 30 | DEFAULT_CONF = dict(
 31 |     uid = os.geteuid(),
 32 |     gid = os.getegid(),
 33 |     umask = 0,
 34 |     debug = False,
 35 | )
 36 | 
 37 | 
 38 | RESTART_WORKERS = ("worker", "supervisor")
 39 | 
 40 | log = logging.getLogger(__name__)
 41 | 
 42 | 
 43 | logging.basicConfig(format="%(asctime)s [%(process)d] [%(levelname)s] %(message)s",
 44 |         datefmt="%Y-%m-%d %H:%M:%S", level=logging.DEBUG)
 45 | 
 46 | 
 47 | 
 48 | 
 49 | class Child(object):
 50 | 
 51 |     def __init__(self, child_class, timeout, child_type,
 52 |             args, name):
 53 |         self.child_class= child_class
 54 |         self.timeout = timeout
 55 |         self.child_type = child_type
 56 |         self.args = args
 57 |         self.name = name
 58 | 
 59 | 
 60 | # chaine init worker:
 61 | # (WorkerClass, max_requests, timeout, type, args, name)
 62 | # types: supervisor, kill, brutal_kill, worker
 63 | # timeout: integer in seconds or None
 64 | 
 65 | class Arbiter(object):
 66 |     """
 67 |     Arbiter maintain the workers processes alive. It launches or
 68 |     kills them if needed. It also manages application reloading
 69 |     via SIGHUP/USR2.
 70 |     """
 71 | 
 72 |     _SPECS_BYNAME = {}
 73 |     _CHILDREN_SPECS = []
 74 | 
 75 |     # A flag indicating if a worker failed to
 76 |     # to boot. If a worker process exist with
 77 |     # this error code, the arbiter will terminate.
 78 |     _WORKER_BOOT_ERROR = 3
 79 | 
 80 |     _WORKERS = {}    
 81 |     _PIPE = []
 82 | 
 83 |     # I love dynamic languages
 84 |     _SIG_QUEUE = []
 85 |     _SIGNALS = map(
 86 |         lambda x: getattr(signal, "SIG%s" % x),
 87 |         "HUP QUIT INT TERM USR1 WINCH".split()
 88 |     )
 89 |     _SIG_NAMES = dict(
 90 |         (getattr(signal, name), name[3:].lower()) for name in dir(signal)
 91 |         if name[:3] == "SIG" and name[3] != "_"
 92 |     )
 93 |     
 94 |     def __init__(self, args, specs=[], name=None,
 95 |             child_type="supervisor", age=0, ppid=0,
 96 |             timeout=30):
 97 | 
 98 |         # set conf
 99 |         conf = DEFAULT_CONF.copy()
100 |         conf.update(args)
101 |         self.conf = conf
102 | 
103 | 
104 |         specs.extend(self.on_init(conf))
105 | 
106 |         for spec in specs:
107 |             c = Child(*spec)
108 |             self._CHILDREN_SPECS.append(c)
109 |             self._SPECS_BYNAME[c.name] = c
110 | 
111 | 
112 |         if name is None:
113 |             name =  self.__class__.__name__
114 |         self.name = name
115 |         self.child_type = child_type
116 |         self.age = age
117 |         self.ppid = ppid
118 |         self.timeout = timeout
119 | 
120 | 
121 |         self.pid = None
122 |         self.num_children = len(self._CHILDREN_SPECS)
123 |         self.child_age = 0
124 |         self.booted = False
125 |         self.stopping = False
126 |         self.debug =self.conf.get("debug", False)
127 |         self.tmp = WorkerTmp(self.conf)
128 |         
129 |     def on_init(self, args):
130 |         return []
131 | 
132 | 
133 |     def on_init_process(self):
134 |         """ method executed when we init a process """
135 |         pass
136 | 
137 | 
138 |     def init_process(self):
139 |         """\
140 |         If you override this method in a subclass, the last statement
141 |         in the function should be to call this method with
142 |         super(MyWorkerClass, self).init_process() so that the ``run()``
143 |         loop is initiated.
144 |         """
145 | 
146 |         # set current pid
147 |         self.pid = os.getpid()
148 |         
149 |         util.set_owner_process(self.conf.get("uid", os.geteuid()),
150 |                 self.conf.get("gid", os.getegid()))
151 | 
152 |         # Reseed the random number generator
153 |         util.seed()
154 | 
155 |         # prevent fd inheritance
156 |         util.close_on_exec(self.tmp.fileno())
157 | 
158 |          # init signals
159 |         self.init_signals()
160 | 
161 |         util._setproctitle("arbiter [%s]" % self.name)
162 |         self.on_init_process()
163 | 
164 |         log.debug("Arbiter %s booted on %s", self.name, self.pid)
165 |         self.when_ready()
166 |         # Enter main run loop
167 |         self.booted = True
168 |         self.run()
169 |     
170 | 
171 |     def when_ready(self):
172 |         pass
173 | 
174 |     def init_signals(self):
175 |         """\
176 |         Initialize master signal handling. Most of the signals
177 |         are queued. Child signals only wake up the master.
178 |         """
179 |         if self._PIPE:
180 |             map(os.close, self._PIPE)
181 |         self._PIPE = pair = os.pipe()
182 |         map(util.set_non_blocking, pair)
183 |         map(util.close_on_exec, pair)
184 |         map(lambda s: signal.signal(s, self.signal), self._SIGNALS)
185 |         signal.signal(signal.SIGCHLD, self.handle_chld)
186 | 
187 |     def signal(self, sig, frame):
188 |         if len(self._SIG_QUEUE) < 5:
189 |             self._SIG_QUEUE.append(sig)
190 |             self.wakeup()
191 |         else:
192 |             log.warn("Dropping signal: %s", sig)
193 | 
194 |     def run(self):
195 |         "Main master loop." 
196 |         if not self.booted:
197 |             return self.init_process()
198 | 
199 |         self.spawn_workers()
200 |         while True:
201 |             try:
202 |                 # notfy the master
203 |                 self.tmp.notify()
204 |                 self.reap_workers()
205 |                 sig = self._SIG_QUEUE.pop(0) if len(self._SIG_QUEUE) else None
206 |                 if sig is None:
207 |                     self.sleep()
208 |                     self.murder_workers()
209 |                     self.manage_workers()
210 |                     continue
211 |                 
212 |                 if sig not in self._SIG_NAMES:
213 |                     log.info("Ignoring unknown signal: %s", sig)
214 |                     continue
215 |                 
216 |                 signame = self._SIG_NAMES.get(sig)
217 |                 handler = getattr(self, "handle_%s" % signame, None)
218 |                 if not handler:
219 |                     log.error("Unhandled signal: %s", signame)
220 |                     continue
221 |                 log.info("Handling signal: %s", signame)
222 |                 handler()
223 |                 self.tmp.notify()
224 |                 self.wakeup()
225 |             except StopIteration:
226 |                 self.halt()
227 |             except KeyboardInterrupt:
228 |                 self.halt()
229 |             except HaltServer, inst:
230 |                 self.halt(reason=inst.reason, exit_status=inst.exit_status)
231 |             except SystemExit:
232 |                 raise
233 |             except Exception:
234 |                 log.info("Unhandled exception in main loop:\n%s",  
235 |                             traceback.format_exc())
236 |                 self.stop(False)
237 |                 sys.exit(-1)
238 | 
239 |     def handle_chld(self, sig, frame):
240 |         "SIGCHLD handling"
241 |         self.wakeup()
242 |         self.reap_workers()
243 |         
244 |     def handle_hup(self):
245 |         """\
246 |         HUP handling.
247 |         - Reload configuration
248 |         - Start the new worker processes with a new configuration
249 |         - Gracefully shutdown the old worker processes
250 |         """
251 |         log.info("Hang up: %s", self.name)
252 |         self.reload()
253 |         
254 |     def handle_quit(self):
255 |         "SIGQUIT handling"
256 |         raise StopIteration
257 |     
258 |     def handle_int(self):
259 |         "SIGINT handling"
260 |         raise StopIteration
261 |     
262 |     def handle_term(self):
263 |         "SIGTERM handling"
264 |         self.stop(False)
265 |         raise StopIteration
266 | 
267 |     def handle_usr1(self):
268 |         """\
269 |         SIGUSR1 handling.
270 |         Kill all workers by sending them a SIGUSR1
271 |         """
272 |         self.kill_workers(signal.SIGUSR1)
273 |     
274 |     def handle_winch(self):
275 |         "SIGWINCH handling"
276 |         if os.getppid() == 1 or os.getpgrp() != os.getpid():
277 |             log.info("graceful stop of workers")
278 |             self.num_workers = 0
279 |             self.kill_workers(signal.SIGQUIT)
280 |         else:
281 |             log.info("SIGWINCH ignored. Not daemonized")
282 |     
283 |     def wakeup(self):
284 |         """\
285 |         Wake up the arbiter by writing to the _PIPE
286 |         """
287 |         try:
288 |             os.write(self._PIPE[1], '.')
289 |         except IOError, e:
290 |             if e.errno not in [errno.EAGAIN, errno.EINTR]:
291 |                 raise
292 |         
293 |                     
294 |     def halt(self, reason=None, exit_status=0):
295 |         """ halt arbiter """
296 |         log.info("Shutting down: %s", self.name)
297 |         if reason is not None:
298 |             log.info("Reason: %s", reason)
299 |         self.stop()
300 |         log.info("See you next")
301 |         sys.exit(exit_status)
302 |         
303 |     def sleep(self):
304 |         """\
305 |         Sleep until _PIPE is readable or we timeout.
306 |         A readable _PIPE means a signal occurred.
307 |         """
308 |         try:
309 |             ready = select.select([self._PIPE[0]], [], [], 1.0)
310 |             if not ready[0]:
311 |                 return
312 |             while os.read(self._PIPE[0], 1):
313 |                 pass
314 |         except select.error, e:
315 |             if e[0] not in [errno.EAGAIN, errno.EINTR]:
316 |                 raise
317 |         except OSError, e:
318 |             if e.errno not in [errno.EAGAIN, errno.EINTR]:
319 |                 raise
320 |         except KeyboardInterrupt:
321 |             sys.exit()
322 |             
323 |     
324 |     def on_stop(self, graceful=True):
325 |         """ method used to pass code when the server start """
326 | 
327 |     def stop(self, graceful=True):
328 |         """\
329 |         Stop workers
330 |         
331 |         :attr graceful: boolean, If True (the default) workers will be
332 |         killed gracefully  (ie. trying to wait for the current connection)
333 |         """
334 |         
335 |         ## pass any actions before we effectively stop
336 |         self.on_stop(graceful=graceful)
337 |         self.stopping = True
338 |         sig = signal.SIGQUIT
339 |         if not graceful:
340 |             sig = signal.SIGTERM
341 |         limit = time.time() + self.timeout
342 |         while True:
343 |             if time.time() >= limit or not self._WORKERS:
344 |                 break
345 |             self.kill_workers(sig)
346 |             time.sleep(0.1)
347 |             self.reap_workers()
348 |         self.kill_workers(signal.SIGKILL)   
349 |         self.stopping = False
350 | 
351 |     def on_reload(self):
352 |         """ method executed on reload """
353 | 
354 | 
355 |     def reload(self):
356 |         """ 
357 |         used on HUP
358 |         """
359 |     
360 |         # exec on reload hook
361 |         self.on_reload()
362 | 
363 |         OLD__WORKERS = self._WORKERS.copy()
364 | 
365 |         # don't kill
366 |         to_reload = []
367 | 
368 |         # spawn new workers with new app & conf
369 |         for child in self._CHILDREN_SPECS:
370 |             if child.child_type != "supervisor":
371 |                 to_reload.append(child)
372 | 
373 |         # set new proc_name
374 |         util._setproctitle("arbiter [%s]" % self.name)
375 |         
376 |         # kill old workers
377 |         for wpid, (child, state) in OLD__WORKERS.items():
378 |             if state and child.timeout is not None:
379 |                 if child.child_type == "supervisor":
380 |                     # we only reload suprvisors.
381 |                     sig = signal.SIGHUP
382 |                 elif child.child_type == "brutal_kill":
383 |                     sig =  signal.SIGTERM
384 |                 else:
385 |                     sig =  signal.SIGQUIT
386 |                 self.kill_worker(wpid, sig)
387 | 
388 |         
389 |     def murder_workers(self):
390 |         """\
391 |         Kill unused/idle workers
392 |         """
393 |         for (pid, child_info) in self._WORKERS.items():
394 |             (child, state) = child_info
395 |             if state and child.timeout is not None:
396 |                 try:
397 |                     diff = time.time() - os.fstat(child.tmp.fileno()).st_ctime
398 |                     if diff <= child.timeout:
399 |                         continue
400 |                 except ValueError:
401 |                     continue
402 |             elif state and child.timeout is None:
403 |                 continue
404 | 
405 |             log.critical("WORKER TIMEOUT (pid:%s)", pid)
406 |             self.kill_worker(pid, signal.SIGKILL)
407 |         
408 |     def reap_workers(self):
409 |         """\
410 |         Reap workers to avoid zombie processes
411 |         """
412 |         try:
413 |             while True:
414 |                 wpid, status = os.waitpid(-1, os.WNOHANG)
415 |                 if not wpid:
416 |                     break
417 |                 
418 |                 # A worker said it cannot boot. We'll shutdown
419 |                 # to avoid infinite start/stop cycles.
420 |                 exitcode = status >> 8
421 |                 if exitcode == self._WORKER_BOOT_ERROR:
422 |                     reason = "Worker failed to boot."
423 |                     raise HaltServer(reason, self._WORKER_BOOT_ERROR)
424 |                 child_info = self._WORKERS.pop(wpid, None)
425 | 
426 |                 if not child_info:
427 |                     continue
428 | 
429 |                 child, state = child_info
430 |                 child.tmp.close()
431 |                 if child.child_type in RESTART_WORKERS and not self.stopping:
432 |                     self._WORKERS["<killed %s>"  % id(child)] = (child, 0)
433 |         except OSError, e:
434 |             if e.errno == errno.ECHILD:
435 |                 pass
436 |     
437 |     def manage_workers(self):
438 |         """\
439 |         Maintain the number of workers by spawning or killing
440 |         as required.
441 |         """
442 | 
443 |         for pid, (child, state) in self._WORKERS.items():
444 |             if not state:
445 |                 del self._WORKERS[pid]
446 |                 self.spawn_child(self._SPECS_BYNAME[child.name])
447 | 
448 |     def pre_fork(self, worker):
449 |         """ methode executed on prefork """
450 | 
451 |     def post_fork(self, worker):
452 |         """ method executed after we forked a worker """
453 |             
454 |     def spawn_child(self, child_spec):
455 |         self.child_age += 1
456 |         name = child_spec.name
457 |         child_type = child_spec.child_type
458 | 
459 |         child_args = self.conf
460 |         child_args.update(child_spec.args)
461 | 
462 |         try:
463 |             # initialize child class
464 |             child = child_spec.child_class(
465 |                         child_args,
466 |                         name = name,
467 |                         child_type = child_type, 
468 |                         age = self.child_age,
469 |                         ppid = self.pid,
470 |                         timeout = child_spec.timeout)
471 |         except:
472 |             log.info("Unhandled exception while creating '%s':\n%s",  
473 |                             name, traceback.format_exc())
474 |             return
475 | 
476 | 
477 |         self.pre_fork(child)
478 |         pid = os.fork()
479 |         if pid != 0:
480 |             self._WORKERS[pid] = (child, 1)
481 |             return
482 | 
483 |         # Process Child
484 |         worker_pid = os.getpid()
485 |         try:
486 |             util._setproctitle("worker %s [%s]" % (name,  worker_pid))
487 |             log.info("Booting %s (%s) with pid: %s", name,
488 |                     child_type, worker_pid)
489 |             self.post_fork(child)
490 |             child.init_process()
491 |             sys.exit(0)
492 |         except SystemExit:
493 |             raise
494 |         except:
495 |             log.exception("Exception in worker process:")
496 |             if not child.booted:
497 |                 sys.exit(self._WORKER_BOOT_ERROR)
498 |             sys.exit(-1)
499 |         finally:
500 |             log.info("Worker exiting (pid: %s)", worker_pid)
501 |             try:
502 |                 child.tmp.close()
503 |             except:
504 |                 pass
505 | 
506 |     def spawn_workers(self):
507 |         """\
508 |         Spawn new workers as needed.
509 |         
510 |         This is where a worker process leaves the main loop
511 |         of the master process.
512 |         """
513 |         
514 |         for child in self._CHILDREN_SPECS: 
515 |             self.spawn_child(child)
516 | 
517 |     def kill_workers(self, sig):
518 |         """\
519 |         Lill all workers with the signal `sig`
520 |         :attr sig: `signal.SIG*` value
521 |         """
522 |         for pid in self._WORKERS.keys():
523 |             self.kill_worker(pid, sig)
524 |                    
525 | 
526 |     def kill_worker(self, pid, sig):
527 |         """\
528 |         Kill a worker
529 |         
530 |         :attr pid: int, worker pid
531 |         :attr sig: `signal.SIG*` value
532 |          """
533 |         if not isinstance(pid, int):
534 |             return
535 | 
536 |         try:
537 |             os.kill(pid, sig)
538 |         except OSError, e:
539 |             if e.errno == errno.ESRCH:
540 |                 try:
541 |                     (child, info) = self._WORKERS.pop(pid)
542 |                     child.tmp.close()
543 |            
544 |                     if not self.stopping:
545 |                         self._WORKERS["<killed %s>"  % id(child)] = (child, 0)
546 |                     return
547 |                 except (KeyError, OSError):
548 |                     return
549 |             raise            
550 | 


--------------------------------------------------------------------------------