├── .gitignore ├── AUTHORS.txt ├── LICENSE.txt ├── MANIFEST.in ├── NEWS.txt ├── README.rst ├── djangotestapp ├── __init__.py ├── manage.py ├── settings.py └── urls.py ├── rc-scripts ├── conf.d │ └── spawning ├── init.d │ ├── spawning │ ├── spawning.debian │ └── spawning.fedora └── sysconfig │ └── spawning.fedora ├── setup.cfg ├── setup.py └── spawning ├── __init__.py ├── django_factory.py ├── memory_watcher.py ├── paste_factory.py ├── reloader_dev.py ├── reloader_svn.py ├── spawning_child.py ├── spawning_controller.py ├── util ├── __init__.py ├── log_parser.py ├── status.py └── system.py └── wsgi_factory.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.bak 3 | *.swp 4 | dist 5 | *.egg-info 6 | build 7 | -------------------------------------------------------------------------------- /AUTHORS.txt: -------------------------------------------------------------------------------- 1 | Authors 2 | 3 | - Donovan Preston (creator) 4 | - Ben Bangert (contributor) 5 | - Ludvig Ericson (contributor) 6 | - Elliot Murphy (contributor) 7 | - Steve 'Ashcrow' Milner (contributor) 8 | - Ryan Williams (contributor) 9 | - R. Tyler Croy (current maintainer) 10 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2008, Donovan Preston 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt *.rst *.py 2 | recursive-include spawning *.py 3 | recursive-include rc-scripts * 4 | -------------------------------------------------------------------------------- /NEWS.txt: -------------------------------------------------------------------------------- 1 | 0.9 2 | ==== 3 | - Remove the unnecessary dependency on eventlet.jsonhttp. This makes spawning work with eventlet 0.9, which removed jsonhttp and put it in another library. (fzzzy) 4 | - Preserve the order of sys.path. (verterok) 5 | - Work around a python deadlock bug with forked child processes (statik) 6 | 7 | 0.9.1: Fix news file to move entries from 0.8.13 to 0.9 since I decided to call this release 0.9. 8 | 9 | 0.9.2: Fix a small typo in the preserve order of sys.path patch from 0.9 that would cause spawning to crash. (schmir) 10 | 11 | 0.9.3: 12 | - Add the ability to disallow HTTP Keepalive via --no-keepalive 13 | - Enable "progressive" spawning and reaping of children processes; in 14 | effect the parent will receive a signal from children about their 15 | impending death and the parent will spin up a new child while the old one 16 | expires 17 | - SIGHUP to the parent will cause it to cycle children without itself dying 18 | 19 | 0.9.4: 20 | - Script for spawning renamed from "spawn" to "spawning" to avoid confusion 21 | - Use O_APPEND when opening files when daemonizing 22 | - Debian/Ubuntu init.d scripts using start-stop-daemon 23 | - spawning.util module added to house methods that don't belong elsewhere 24 | - Update Spawning to use newer Eventlet API calls 25 | - Properly handle errors when calling `setproctitle` 26 | - Allow optional `eventlet.backdoor` bound to localhost 27 | - Operate more cleanly with Eventlet's websockets support with changes to 28 | eventlet.tpool integration 29 | - Avoid passing certain objects through `eventlet.tpool.Proxy` when using 30 | threads for a performance gain 31 | - Prevent leaking pipes when handling children processes 32 | 33 | 34 | 0.9.5: 35 | - Fixed a number of issues with reloading of Spawning children 36 | - Added /_sysinfo to provide information about the machine 37 | - Added health page that listens on a different port from the controller, 38 | providing HTML and JSON formatted information about children 39 | - Introduced a basic logfile analyzer for processsing Spawning logs for 40 | further information not provided by the health page 41 | 42 | 43 | 44 | 0.8 45 | ==== 46 | 47 | Fixed a problem where eventlet monkeypatching was inappropriately installed in child processes which use threads. This would result in certain operations (primarily DNS lookup operations) resulting in a greenlet "cannot switch to another thread" exception. 48 | 49 | Changed the paste factory to use 10 worker threads by default, to match the paste default. Previously if the paste ini file did not mention how many worker threads to use, spawning would default to 0 and switch into cooperative, non-blocking mode. 50 | 51 | Added a deadman timeout to child processes which have been told to exit and are waiting for outstanding requests to finish. If the timeout expires before all requests have completed, we assume the process is hung and kill -9 it. The default timeout is 120 seconds. 52 | 53 | If an i/o child process dies with an exit code other than 0, the controller decides something must have gone horribly wrong and restarts all of the children. 54 | 55 | 0.8.1: Fix a bug where the reloader didn't work with paster serve. 56 | 57 | 0.8.2: In the svn reloader, watch both spawning's directory and the directory of the wsgi application we are serving. Also, fix the django_factory which was broken by 0.8. 58 | 59 | 0.8.3: For all svn repositories the svn reloader is watching, also watch any svn:externals repositories contained therein. 60 | 61 | 0.8.4: Fixed a bug where the controller process dying unexpectedly (such as from a kill -9) would cause the children to have an exception but then keep running forever, preventing any other processes from using the ports again in the future. 62 | 63 | 0.8.5: Fixed a bug in the svn reloader where files that svn reported as 'not under version control' would cause the reloader to crash and exit immediately. I now use svn's exit code instead of sniffing svn's output, which should also help avoid problems for anyone who is using a localized copy of svn, or if svn ever changes the content of these messages. 64 | 65 | 0.8.6: Fix a file descriptor leak that occurred when the controller reloaded. If your code changed enough times over the lifetime of the server (thousands of times) it would eventually run out of file descriptors and refuse to start up. Now the number of file descriptors stays constant no matter how many times the server restarts. 66 | 67 | 0.8.7: 68 | - In the svn reloader process, check to see if the controller is still alive, and if not, just exit. 69 | - Don't hold on to the web port at all in the reloader_svn process. 70 | - Add an exponential backoff to the controller's "panic" restart. Before spawning would restart as fast as possible; now it backs off the time between restarts. 71 | - If we can't import the wsgi app, panic. 72 | - If we can't fork a child process (out of memory), panic. 73 | 74 | 0.8.8: 75 | - Added --access-log-file command line option to allow writing access logs to someplace other than stdout. Useful for redirecting to /dev/null for speed 76 | - Correctly extract the child's exit code and clean up the logging of child exit events. 77 | - Add coverage gathering using figleaf if the --coverage command line option is given. When gathering coverage, the figleaf report can be downloaded from the /_coverage url. 78 | - Add a --max-memory option to limit the total amount of memory spawning will use. If this is exceeded a SIGHUP will be sent to the controller causing the children to restart. 79 | - Add a --max-age option to limit the total amount of time a spawning child is allowed to run. After the time limit is exceeded a SIGHUP will be sent to the controller to cause the children to restart. 80 | - Instead of just passing the PYTHONPATH environment variable through to the children, construct the PYTHONPATH from the contents of sys.path. 81 | - Instead of just trying to run 'spawn' with /usr/bin/env when restarting, just run sys.executable -m spawning.spawning_controller, making it more likely that the controller will run correctly when restarting. 82 | - Add a --verbose option and change the default startup procedure to not log the detailed dictionary of configuration information. 83 | 84 | 0.8.9: Minor release which provides compatibility with running servers which are using 0.8.7. With 0.8.8, any running servers which are upgraded from 0.8.7->0.8.8 will crash with a KeyError and need to be restarted manually. 85 | 86 | 0.8.10: When spawning starts up, add the current working directory to sys.path if it is not already there. Also, when calculating the PYTHONPATH to give to the child from sys.path, remove any path which does not exist, preventing setuptools "DistributionNotFound" errors. 87 | 88 | 0.8.11: 89 | - Added Python 2.4 compatibility. (kiorky) 90 | - Added license headers to all source files. (statik) 91 | - Print exceptions to stderr instead of stdout. (lericson) 92 | - Don't assume every OSError the controller process gets is EINTR. (lericson) 93 | - Added simple daemonizing support. (lericson) 94 | - Added an OpenRC init script. (lericson) 95 | - Added an explicit manifest. (lericson) 96 | 97 | 0.8.12: 98 | - Remove the processpool implementation added in 0.7 because an equivalent setup can be achieved using controller processes and 1 thread; now we can just talk about the 'number of processes' and 'number of threads' instead of having two levels of different kind of processes when using the processpool. (fzzzy) 99 | - Minor release to fix the explicit manifest and OpenRC init script added in the last release. (lericson) 100 | - When running under 2.6 we no longer produce a deprecation warning about the removal of the sets module. (fzzzy) 101 | 102 | 103 | 104 | 105 | 106 | 0.7 107 | ==== 108 | 109 | Added django_factory. 110 | 111 | Add an optional worker processpool which can be used as an equivalent to the worker threadpool. 112 | 113 | Added command-line script to launch a wsgi application, 'spawn'. 114 | 115 | spawn mymodule.my_wsgi_app 116 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Spawning is a fast, easy to use, and flexible HTTP server for hosting python web applications which conform to the WSGI interface. 2 | 3 | Spawning uses eventlet to do non-blocking I/O for http requests and responses. This means the server will scale to a large number of idle keep-alive connections easily. Spawning can be configured to use multiple OS processes and either POSIX threads or eventlet's green threads, which are implemented using greenlet. 4 | 5 | Spawning is open source software, licensed under the MIT license. If you wish to contribute to development, please check out the source from http://github.com/rtyler/Spawning/ and either submit patches or fork spawning and submit a pull request. 6 | 7 | Single or Multiple Process 8 | ========================== 9 | 10 | If your wsgi applications store state in memory, Spawning can be configured to run only one Python process. In this configuration your application state will be available to all requests but your application will not be able to take full advantage of multiple processors. Using multiple processes will take advantage of all processors and thus should be used for applications which do not share state. 11 | 12 | Single or Multiple Worker Thread 13 | ================================================================ 14 | 15 | If your wsgi applications perform a certain subset of blocking calls which have been monkeypatched by eventlet to cooperate instead (such as operations in the socket module), you can configure each process to run only a single main thread and cooperate using eventlet's green threads instead. This can be useful if your application needs to scale to a large number of simultaneous open connections, such as a COMET server or an application which uses AJAX polling. However, most existing wsgi applications will probably perform blocking operations (for example, calling database adapter libraries which perform blocking socket operations). Therefore, for most wsgi applications a combination of multiple processes and multiple threads will be ideal. 16 | 17 | Graceful Code Reloading 18 | ======================= 19 | Spawning can watch all Python files that are imported into sys.modules for changes and performs a graceful reload on change. To enable this behavior, specify --reload=dev on the command line. Old processes are told to stop accepting requests and finish any outstanding requests they are servicing, and shutdown. Meanwhile, new processes are started and begin accepting requests and servicing them with the new code. At no point will users of your site see "connection refused" errors because the server is continuously listening during reload. 20 | 21 | Running spawning 22 | ================ 23 | 24 | Spawning can be used to launch a wsgi application from the command line using the "spawn" script, or using Python Paste. To use with paste, specify use = egg:Spawning in the [server:main] section of a paste ini file. 25 | 26 | Spawning can also be used to run a Django application by using --factory=spawning.django_factory.config_factory. 27 | 28 | Examples of running spawning 29 | ============================ 30 | 31 | Run the wsgi application callable called "my_wsgi_application" inside the my_wsgi_module.py file:: 32 | 33 | % spawning my_wsgi_module.my_wsgi_application 34 | 35 | Run whatever is configured inside of the paste-style configuration file development.ini. Equivalent to using paster serve with an ini file configured to use Spawning as the server:: 36 | 37 | % spawning --factory=spawning.paste_factory.config_factory development.ini 38 | 39 | Run the Django app mysite:: 40 | 41 | % spawning --factory=spawning.django_factory.config_factory mysite.settings 42 | 43 | Run the wsgi application wrapped with some middleware. Pass as many middleware strings as desired after the wsgi application name:: 44 | 45 | % spawning my_wsgi_module.my_wsgi_application other_wsgi_module.some_wsgi_middleware 46 | 47 | Run the wsgi application on port 80, with 4 processes each using a threadpool of size 8:: 48 | 49 | % sudo spawning --port=80 --processes=4 --threads=8 my_wsgi_module.my_wsgi_application 50 | 51 | Use a threadpool of size 0, which indicates that eventlet monkeypatching should be performed and wsgi applications should all be called in the same thread. Useful for writing a comet-style application where a lot of requests are simply waiting on a server-side event or internal network io to complete:: 52 | 53 | % spawning --processes=4 --threads=0 my_wsgi_module.my_comet_application 54 | 55 | Additional Useful Arguments 56 | =========================== 57 | 58 | -l ACCESS_LOG_FILE, --access-log-file=ACCESS_LOG_FILE 59 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 60 | 61 | The file to log access log lines to. If not given, log 62 | to stdout. Pass /dev/null to discard logs. 63 | 64 | -c, --coverage 65 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 66 | 67 | If given, gather coverage data from the running 68 | program and make the coverage report available from 69 | the /_coverage url. See the figleaf docs for more 70 | info: http://darcs.idyll.org/~t/projects/figleaf/doc/ 71 | 72 | -m MAX_MEMORY, --max-memory=MAX_MEMORY 73 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 74 | 75 | If given, the maximum amount of memory this instance 76 | of Spawning is allowed to use. If all of the processes 77 | started by this Spawning controller use more than this 78 | amount of memory, send a SIGHUP to the controller to 79 | get the children to restart. 80 | 81 | -a MAX_AGE, --max-age=MAX_AGE 82 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 83 | 84 | If given, the maximum amount of time (in seconds) an 85 | instance of spawning_child is allowed to run. Once 86 | this time limit has expired a SIGHUP will be sent to 87 | spawning_controller, causing it to restart all of the 88 | child processes. 89 | 90 | --status-port=PORT, --status-host=HOST 91 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 92 | 93 | If given, starts up a small web service to give 94 | health status reports on the Spawning server. The 95 | service listens on two urls, 96 | 97 | * http://status_host:status_port/status 98 | * http://status_host:status_port/status.json 99 | 100 | The first is an HTML page that displays the status 101 | of the server in a human-pleasing manner. The .json 102 | url is a JSON formatting of the same data. 103 | 104 | The status web service is only started if the 105 | --status-port option is supplied and different than 106 | the service port. --status-host is useful if 107 | monitoring happens on a different ip address than 108 | web application requests. 109 | -------------------------------------------------------------------------------- /djangotestapp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rtyler/Spawning/f56c0f52330c0fb5c6a7322bc5b3a83f73863440/djangotestapp/__init__.py -------------------------------------------------------------------------------- /djangotestapp/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from django.core.management import execute_manager 3 | try: 4 | import settings # Assumed to be in the same directory. 5 | except ImportError: 6 | import sys 7 | sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__) 8 | sys.exit(1) 9 | 10 | if __name__ == "__main__": 11 | execute_manager(settings) 12 | -------------------------------------------------------------------------------- /djangotestapp/settings.py: -------------------------------------------------------------------------------- 1 | # Django settings for djangotestapp project. 2 | 3 | DEBUG = True 4 | TEMPLATE_DEBUG = DEBUG 5 | 6 | ADMINS = ( 7 | # ('Your Name', 'your_email@domain.com'), 8 | ) 9 | 10 | MANAGERS = ADMINS 11 | 12 | DATABASE_ENGINE = '' # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 13 | DATABASE_NAME = '' # Or path to database file if using sqlite3. 14 | DATABASE_USER = '' # Not used with sqlite3. 15 | DATABASE_PASSWORD = '' # Not used with sqlite3. 16 | DATABASE_HOST = '' # Set to empty string for localhost. Not used with sqlite3. 17 | DATABASE_PORT = '' # Set to empty string for default. Not used with sqlite3. 18 | 19 | # Local time zone for this installation. Choices can be found here: 20 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 21 | # although not all choices may be available on all operating systems. 22 | # If running in a Windows environment this must be set to the same as your 23 | # system time zone. 24 | TIME_ZONE = 'America/Chicago' 25 | 26 | # Language code for this installation. All choices can be found here: 27 | # http://www.i18nguy.com/unicode/language-identifiers.html 28 | LANGUAGE_CODE = 'en-us' 29 | 30 | SITE_ID = 1 31 | 32 | # If you set this to False, Django will make some optimizations so as not 33 | # to load the internationalization machinery. 34 | USE_I18N = True 35 | 36 | # Absolute path to the directory that holds media. 37 | # Example: "/home/media/media.lawrence.com/" 38 | MEDIA_ROOT = '' 39 | 40 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 41 | # trailing slash if there is a path component (optional in other cases). 42 | # Examples: "http://media.lawrence.com", "http://example.com/media/" 43 | MEDIA_URL = '' 44 | 45 | # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a 46 | # trailing slash. 47 | # Examples: "http://foo.com/media/", "/media/". 48 | ADMIN_MEDIA_PREFIX = '/media/' 49 | 50 | # Make this unique, and don't share it with anybody. 51 | SECRET_KEY = 'r04y1gw5-^%1)@(gbh$oa#wajdpa2yzij7eqj$gzs1hjb^-jbu' 52 | 53 | # List of callables that know how to import templates from various sources. 54 | TEMPLATE_LOADERS = ( 55 | 'django.template.loaders.filesystem.load_template_source', 56 | 'django.template.loaders.app_directories.load_template_source', 57 | # 'django.template.loaders.eggs.load_template_source', 58 | ) 59 | 60 | MIDDLEWARE_CLASSES = ( 61 | 'django.middleware.common.CommonMiddleware', 62 | 'django.contrib.sessions.middleware.SessionMiddleware', 63 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 64 | ) 65 | 66 | ROOT_URLCONF = 'djangotestapp.urls' 67 | 68 | TEMPLATE_DIRS = ( 69 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 70 | # Always use forward slashes, even on Windows. 71 | # Don't forget to use absolute paths, not relative paths. 72 | ) 73 | 74 | INSTALLED_APPS = ( 75 | 'django.contrib.auth', 76 | 'django.contrib.contenttypes', 77 | 'django.contrib.sessions', 78 | 'django.contrib.sites', 79 | ) 80 | -------------------------------------------------------------------------------- /djangotestapp/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import * 2 | 3 | # Uncomment the next two lines to enable the admin: 4 | # from django.contrib import admin 5 | # admin.autodiscover() 6 | 7 | urlpatterns = patterns('', 8 | # Example: 9 | # (r'^djangotestapp/', include('djangotestapp.foo.urls')), 10 | 11 | # Uncomment the admin/doc line below and add 'django.contrib.admindocs' 12 | # to INSTALLED_APPS to enable admin documentation: 13 | # (r'^admin/doc/', include('django.contrib.admindocs.urls')), 14 | 15 | # Uncomment the next line to enable the admin: 16 | # (r'^admin/', include(admin.site.urls)), 17 | ) 18 | -------------------------------------------------------------------------------- /rc-scripts/conf.d/spawning: -------------------------------------------------------------------------------- 1 | # Python Spawning configuration 2 | 3 | # DO NOT MODIFY THIS FILE DIRECTLY! CREATE A COPY AND MODIFY THAT INSTEAD! 4 | 5 | # Spawning factory name. 6 | # If you're using Django, you should use: 7 | #FACTORY=spawning.django_factory.config_factory. 8 | # If you're using Paste, you should use: 9 | #FACTORY=spawning.paste_factory.config_factory 10 | # Or, for regular WSGI, the default is: 11 | #FACTORY=spawning.wsgi_factory.config_factory 12 | 13 | # The WSGI application you want to spawn. 14 | # This means different things with different factories, for example with 15 | # Django, the WSGI_APP should be the Django settings module to load. 16 | WSGI_APP= 17 | 18 | # Serve on : 19 | HOST=0.0.0.0 20 | PORT=8080 21 | 22 | # Change to user (and group if given). `[user[:group]]` 23 | #CHUID= 24 | 25 | # A colon-separated list of the Python path. 26 | #PYTHON_LIBS=/usr/local/lib/my-python-packages 27 | 28 | # stderr is redirected to ERROR_LOG. The status output from the workers, among 29 | # other things, will end up there. 30 | ERROR_LOG=/var/log/spawning/error.log 31 | 32 | # stdout is redirected to INFO_LOG. 33 | INFO_LOG=/var/log/spawning/info.log 34 | 35 | # The access log is written separately from stdout and stderr. 36 | ACCESS_LOG=/var/log/spawning/access.log 37 | 38 | # Concurrency model. 39 | 40 | # Choose either forks (NUM_WORKERS) or threads (NUM_THREADS). 41 | # NUM_WORKERS defines the number of forks to make a process-based pool out of. 42 | #NUM_WORKERS=10 43 | #NUM_THREADS=0 44 | 45 | # NUM_THREADS defines the number of threads to make a thread-based pool out of. 46 | #NUM_THREADS=10 47 | #NUM_WORKERS=0 48 | 49 | # NUM_PROCS defines the number of forks to make, each fork has its own 50 | # NUM_WORKERS and NUM_THREADS. 51 | #NUM_PROCS=1 52 | 53 | # Maximum age in seconds of a worker before it's restarted. 54 | #MAX_AGE= 55 | 56 | # Maximum memory usage of _all_ workers before it's restarted. 57 | # Counts resident set size (RSS) in kilobytes, see ps(1). 58 | #MAX_MEMORY= 59 | 60 | # Watch file. When the file changes, Spawning reloads. 61 | #WATCH= 62 | 63 | # Extra options to pass to Spawning. 64 | #EXTRA_OPTS= 65 | 66 | # Path to Spawning binary. 67 | #SPAWN_BIN= 68 | -------------------------------------------------------------------------------- /rc-scripts/init.d/spawning: -------------------------------------------------------------------------------- 1 | #!/sbin/runscript 2 | # Copyright 2009 Ludvig Ericson 3 | # Distributed under the terms of the 3-clause BSD license 4 | 5 | [[ -z "${SPAWN_BIN}" ]] && SPAWN_BIN="$(which spawning)" 6 | PIDFILE="/var/run/${SVCNAME}.pid" 7 | 8 | opts="${opts} reload" 9 | 10 | depend() { 11 | need net 12 | } 13 | 14 | check_params() { 15 | if [[ "${SVCNAME}" == "spawning" && -z "${I_KNOW}" ]]; then 16 | ewarn "It is highly recommended to use a symbolic link for this" 17 | ewarn "script and start via that instead. This allows you to run" 18 | ewarn "multiple spawn services simultaneously. To do this, simply:" 19 | ewarn 20 | ewarn " ln -s /etc/init.d/spawning /etc/init.d/spawning.mysvc" 21 | ewarn " cp /etc/conf.d/spawning /etc/conf.d/spawning.mysvc" 22 | ewarn 23 | ewarn "If you don't want to be bothered by this message, set I_KNOW=yes" 24 | ewarn "in your configuration file." 25 | ewarn 26 | fi 27 | 28 | if [[ -z "${SPAWN_BIN}" ]]; then 29 | eerror "Couldn't find spawning binary and no explicit" 30 | eerror "path set in configuration file." 31 | return 1 32 | fi 33 | 34 | for CONF_VAR in WSGI_APP HOST PORT ERROR_LOG; do 35 | if [[ -z "$(eval echo \$${CONF_VAR})" ]]; then 36 | eerror "Required configuration variable ${CONF_VAR} not" 37 | eerror "set in configuration file." 38 | return 1 39 | fi 40 | done 41 | 42 | if [[ ! -z "${WORKERS}" && -z "${NUM_WORKERS}" ]]; then 43 | eerror "WORKERS has changed name to NUM_WORKERS, please update" 44 | eerror "your configuration file(s) accordingly." 45 | return 1 46 | fi 47 | } 48 | 49 | start() { 50 | ebegin "Starting ${SVCNAME} on ${HOST}:${PORT}" 51 | local OPTS 52 | 53 | check_params || return 1 54 | 55 | OPTS="${OPTS} --host ${HOST} --port ${PORT}" 56 | [[ ! -z "${NUM_PROCS}" ]] && OPTS="${OPTS} --processes ${NUM_PROCS}" 57 | if [[ ! -z "${NUM_THREADS}" ]]; then 58 | OPTS="${OPTS} --threads ${NUM_THREADS}" 59 | elif [[ ! -z "${NUM_WORKERS}" ]]; then 60 | OPTS="${OPTS} --workers ${NUM_WORKERS}" 61 | fi 62 | OPTS="${OPTS} --access-log ${ACCESS_LOG:-/dev/null}" 63 | 64 | [[ ! -z "${CHUID}" ]] && OPTS="${OPTS} --chuid ${CHUID}" 65 | [[ ! -z "${FACTORY}" ]] && OPTS="${OPTS} --factory ${FACTORY}" 66 | [[ ! -z "${WATCH}" ]] && OPTS="${OPTS} --watch ${WATCH}" 67 | [[ ! -z "${MAX_MEMORY}" ]] && OPTS="${OPTS} --max-memory ${MAX_MEMORY}" 68 | [[ ! -z "${MAX_AGE}" ]] && OPTS="${OPTS} --max-age ${MAX_AGE}" 69 | OPTS="${OPTS} ${EXTRA_OPTS}" 70 | 71 | start-stop-daemon --start --pidfile "${PIDFILE}" \ 72 | --exec "${SPAWN_BIN}" --env PYTHONPATH="${PYTHON_LIBS}" -- \ 73 | --daemonize --pidfile "${PIDFILE}" \ 74 | --stderr "${ERROR_LOG}" --stdout "${INFO_LOG:-/dev/null}" \ 75 | ${OPTS} ${WSGI_APP} 76 | eend $? 77 | } 78 | 79 | stop() { 80 | check_params || return 1 81 | ebegin "Stopping ${SVCNAME}" 82 | start-stop-daemon --stop --pidfile "${PIDFILE}" --signal INT 83 | eend $? 84 | } 85 | 86 | reload() { 87 | check_params || return 1 88 | ebegin "Telling ${SVCNAME} to reload itself" 89 | kill -HUP "$(cat ${PIDFILE})" 90 | eend $? 91 | } 92 | -------------------------------------------------------------------------------- /rc-scripts/init.d/spawning.debian: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ### BEGIN INIT INFO 4 | # Provides: spawning.debian 5 | # Required-Start: $all 6 | # Required-Stop: $all 7 | # Default-Start: 2 3 4 5 8 | # Default-Stop: 0 1 6 9 | # Short-Description: starts a Spawning daemon 10 | # Description: starts a Spawning daemon 11 | ### END INIT INFO 12 | 13 | # Usage instructions: 14 | # - Copy this file to your /etc/init.d directory: 15 | # $ sudo cp spawning.debian /etc/init.d/spawning.mysite 16 | # - Change the options below in the "Configuration" section. 17 | # Spawning arguments currently supported are: 18 | # - host (defaults to 127.0.0.1) 19 | # - port (defaults to 8080) 20 | # - access-log (defaults to /dev/null) 21 | # - stderr (defaults to /dev/null) 22 | # - chuid (optional) 23 | # - factory (optional) 24 | # - processes (optional) 25 | # If you have additional arguments to provide (such as 26 | # max-age, watch, workers etc., add them as follows: 27 | # EXTRA_ARGS="max-age=1000 --watch=file" etc. 28 | # - Initialise the script to survive reboots: 29 | # $ sudo update-rc.d spawning.mysite defaults 30 | # - Start your server: 31 | # $ sudo /etc/init.d/spawning.mysite start 32 | 33 | NAME="spawning.debian" # change debian to your own site 34 | DESC="Starts a Spawning daemon to run [domain here]" 35 | # This is where your app or settings.py lives 36 | SITE_DIR=/var/www/mysite/ 37 | 38 | # CONFIGURATION - edit this stuff 39 | # Configure your own spawning here 40 | SPAWNING_BIN=/usr/local/bin/spawning #Path to Spawning executable 41 | PROCESSES=2 # Number of processes to spawn 42 | # Django by default: delete this var to run a WSGI app 43 | FACTORY="spawning.django_factory.config_factory" 44 | PORT="8080" 45 | HOST="127.0.0.1" # listen on loopback by default 46 | CHUID="" 47 | ACCESS_LOG=/dev/null 48 | ERROR_LOG=/dev/null 49 | APP="settings" # Django by default, but put your own app here 50 | # END CONFIGURATION 51 | 52 | # Don't edit this 53 | PIDFILE=/var/run/$NAME.pid 54 | 55 | # make sure the access_log file exists 56 | if [ ! -e $ACCESS_LOG ]; then 57 | touch $ACCESS_LOG 58 | fi 59 | # make sure the error log file exists 60 | if [ ! -e $ERROR_LOG ]; then 61 | touch $ERROR_LOG 62 | fi 63 | 64 | start() { 65 | echo -n "Starting $NAME on $HOST:$PORT...: " 66 | DAEMON_ARGS="--host=$HOST --port=$PORT" 67 | DAEMON_ARGS="$DAEMON_ARGS --stderr=$ERROR_LOG --access-log-file=$ACCESS_LOG" 68 | [ -n "$PROCESSES" ] && DAEMON_ARGS="$DAEMON_ARGS --processes=$PROCESSES" 69 | [ -n "$FACTORY" ] && DAEMON_ARGS="$DAEMON_ARGS --factory=$FACTORY" 70 | [ -n "$CHUID" ] && DAEMON_ARGS="$DAEMON_ARGS --chuid=$CHUID" 71 | [ -n "$EXTRA_ARGS" ] && DAEMON_ARGS="$DAEMON_ARGS $EXTRA_ARGS" 72 | DAEMON_ARGS="$DAEMON_ARGS $APP" 73 | /sbin/start-stop-daemon --start --background --make-pidfile --pidfile=$PIDFILE --chdir $SITE_DIR --exec $SPAWNING_BIN -- $DAEMON_ARGS || return 2 74 | echo $NAME 75 | return 0 76 | } 77 | 78 | stop () { 79 | echo -n "Stopping $NAME: " 80 | /sbin/start-stop-daemon --stop --pidfile $PIDFILE 81 | rm -f $PIDFILE 82 | echo $NAME 83 | return 84 | } 85 | 86 | status() { 87 | if [ -f "$PIDFILE" ]; then 88 | echo -n "$NAME already running with PIDs: " && cat $PIDFILE && echo 89 | else 90 | echo "$NAME not running" 91 | fi 92 | return 93 | } 94 | 95 | case "$1" in 96 | start) 97 | start 98 | ;; 99 | stop) 100 | stop 101 | ;; 102 | status) 103 | status 104 | ;; 105 | restart) 106 | stop 107 | sleep 1 108 | start 109 | ;; 110 | *) 111 | echo "Usage: $NAME (start|stop|status|restart)" 112 | exit 1 113 | ;; 114 | esac 115 | exit $? 116 | -------------------------------------------------------------------------------- /rc-scripts/init.d/spawning.fedora: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # spawning.$app_name Starts Spawning 4 | # 5 | # 6 | # chkconfig: 345 88 12 7 | # description: Spawning is a lightweight http daemon. 8 | ### BEGIN INIT INFO 9 | # Provides: $spawning 10 | ### END INIT INFO 11 | 12 | # Source function library. 13 | . /etc/rc.d/init.d/functions 14 | 15 | # config 16 | if [ -f /etc/sysconfig/spawning.$app_name ]; then 17 | . /etc/sysconfig/spawning.$app_name 18 | fi 19 | 20 | name="spawning.$app_name" 21 | prog="spawning" 22 | exec=$exec #Path to Spawning executable 23 | 24 | description="Starts a Spawning daemon to run the internets" 25 | # This is where your app or settings.py lives 26 | 27 | # Don't edit this 28 | PIDFILE=/var/run/$name.pid 29 | lockfile=/var/lock/subsys/${name} 30 | 31 | if [ $python_path ]; then 32 | export PYTHONPATH=${python_path} 33 | fi 34 | 35 | # make sure the access_log file exists 36 | if [ ! -e $access_log ]; then 37 | touch $access_log 38 | fi 39 | # make sure the error log file exists 40 | if [ ! -e $error_log ]; then 41 | touch $error_log 42 | fi 43 | 44 | start() { 45 | 46 | 47 | echo -n "Starting $name on $host:$port...: " 48 | echo -e "\n" 49 | 50 | conf="--host=$host --port=$port" 51 | conf="$conf --stderr=$error_log --access-log-file=$access_log" 52 | [ -n "$pocesses" ] && conf="$conf --processes=$processes" 53 | [ -n "$threads" ] && conf="$conf --threads=$threads" 54 | [ -n "$factory" ] && conf="$conf --factory=$factory" 55 | [ -n "$dev" ] && conf="$conf --reload=dev" 56 | [ -n "$CHUID" ] && conf="$conf --chuid=$CHUID" 57 | 58 | conf="$conf $app" 59 | 60 | # if not running, start it up here, usually something like "daemon $exec" 61 | daemon $exec --daemonize $conf 62 | retval=$? 63 | 64 | echo -e "\n" 65 | 66 | [ $retval -eq 0 ] && touch $lockfile 67 | return $retval 68 | } 69 | 70 | stop () { 71 | echo -n $"Stopping $prog: " 72 | # stop it here, often "killproc $prog" 73 | killproc $prog 74 | retval=$? 75 | echo 76 | return $retval 77 | } 78 | 79 | status() { 80 | if [ -f "$PIDFILE" ]; then 81 | echo -n "$name already running with PIDs: " && cat $PIDFILE && echo 82 | else 83 | echo "$name not running" 84 | fi 85 | return 86 | } 87 | 88 | case "$1" in 89 | start) 90 | start 91 | ;; 92 | stop) 93 | stop 94 | ;; 95 | status) 96 | status 97 | ;; 98 | restart) 99 | stop 100 | sleep 1 101 | start 102 | ;; 103 | *) 104 | echo "Usage: $name (start|stop|status|restart)" 105 | exit 1 106 | ;; 107 | esac 108 | exit $? -------------------------------------------------------------------------------- /rc-scripts/sysconfig/spawning.fedora: -------------------------------------------------------------------------------- 1 | # CONFIGURATION - edit this stuff with your keyboard. 2 | 3 | app_name=app_name # necessary for namespacing 4 | # this will effect the site directory, log paths, user name 5 | # and expected location of this file from the init.d script 6 | # though there is nothing stopping you from changing each of those settings by hand 7 | 8 | processes=2 # Number of processes to spawn 9 | threads=8 # Number of threads to spawn 10 | 11 | port="8080" 12 | host="localhost" # listen on loopback by default 13 | 14 | # Django by default: delete this var to run a WSGI app 15 | factory="spawning.django_factory.config_factory" 16 | site_dir=/var/www/$app_name 17 | exec=${site_dir}/bin/spawning 18 | 19 | app="settings" # Django by default, but put your own app here 20 | 21 | CHUID="$app_name" # if running multiple insances of spawning 22 | # setup a user for each application 23 | # in addition .bashrc files that set paths are likely necessary as well 24 | 25 | access_log=/var/log/spawning.$app_name/access_log # to have logging "off" ... set these vars to /dev/null 26 | error_log=/var/log/spawning.$app_name/error_log 27 | 28 | #python_path=$site_dir:/usr/lib/python26.zip:/usr/lib/python2.6:/usr/lib/python2.6/lib-old:/usr/lib/python2.6/lib-dynload:/usr/lib/python2.6/site-packages 29 | # a default fedora install will not have a pythonpath available to init scripts ... 30 | # these are the paths for fedora 13 ... 31 | # your milage may vary... though this variable is very likely necessary to get your script to work -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rtyler/Spawning/f56c0f52330c0fb5c6a7322bc5b3a83f73863440/setup.cfg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2008, Donovan Preston 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | # THE SOFTWARE. 21 | 22 | import sys 23 | 24 | from spawning import __version__ 25 | 26 | from os import path 27 | 28 | from setuptools import find_packages, setup 29 | 30 | install_requires = ['eventlet >= 0.9.12',] 31 | 32 | try: 33 | import json 34 | except ImportError: 35 | install_requires.append('simplejson') 36 | 37 | setup( 38 | name='Spawning', 39 | description='Spawning is a wsgi server which supports multiple processes, multiple threads, green threads, non-blocking HTTP io, and automatic graceful upgrading of code.', 40 | long_description=file( 41 | path.join( 42 | path.dirname(__file__), 43 | 'README.rst' 44 | ) 45 | ).read(), 46 | author='Donovan Preston', 47 | author_email='dsposx@mac.com', 48 | maintainer='R. Tyler Croy', 49 | maintainer_email='tyler@monkeypox.org', 50 | include_package_data = True, 51 | packages = ['spawning'], 52 | package_dir = {'': '.'}, 53 | version=__version__, 54 | install_requires=install_requires, 55 | entry_points={ 56 | 'console_scripts': [ 57 | 'spawning=spawning.spawning_controller:main', 58 | ], 59 | 'paste.server_factory': [ 60 | 'main=spawning.paste_factory:server_factory' 61 | ] 62 | }, 63 | classifiers=[ 64 | "License :: OSI Approved :: MIT License", 65 | "Programming Language :: Python", 66 | "Operating System :: MacOS :: MacOS X", 67 | "Operating System :: POSIX", 68 | "Topic :: Internet", 69 | "Topic :: Software Development :: Libraries :: Python Modules", 70 | "Intended Audience :: Developers", 71 | "Development Status :: 4 - Beta" 72 | ] 73 | ) 74 | 75 | -------------------------------------------------------------------------------- /spawning/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2008, Donovan Preston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to 5 | # deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | # FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | """ 24 | """ 25 | 26 | __version__ = '0.9.7' 27 | 28 | setproctitle = lambda v: NotImplemented 29 | 30 | try: 31 | from setproctitle import setproctitle 32 | except ImportError: 33 | try: 34 | from procname import setprocname as setproctitle 35 | except ImportError: 36 | pass 37 | 38 | -------------------------------------------------------------------------------- /spawning/django_factory.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2008, Donovan Preston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to 5 | # deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | # FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | 24 | import inspect 25 | import os 26 | import django.core.handlers.wsgi 27 | 28 | import spawning.util 29 | 30 | def config_factory(args): 31 | args['django_settings_module'] = args.get('args', [None])[0] 32 | args['app_factory'] = 'spawning.django_factory.app_factory' 33 | 34 | ## TODO More directories 35 | ## INSTALLED_APPS (list of quals) 36 | ## ROOT_URL_CONF (qual) 37 | ## MIDDLEWARE_CLASSES (list of quals) 38 | ## TEMPLATE_CONTEXT_PROCESSORS (list of quals) 39 | settings_module = spawning.util.named(args['django_settings_module']) 40 | 41 | dirs = [os.path.split( 42 | inspect.getfile( 43 | inspect.getmodule( 44 | settings_module)))[0]] 45 | args['source_directories'] = dirs 46 | 47 | return args 48 | 49 | 50 | def app_factory(config): 51 | os.environ['DJANGO_SETTINGS_MODULE'] = config['django_settings_module'] 52 | 53 | app = django.core.handlers.wsgi.WSGIHandler() 54 | 55 | return app 56 | 57 | -------------------------------------------------------------------------------- /spawning/memory_watcher.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2008, Donovan Preston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to 5 | # deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | # FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | import commands 24 | import os 25 | import optparse 26 | import signal 27 | import sys 28 | import time 29 | 30 | 31 | MEMORY_WATCH_INTERVAL = 60 32 | 33 | 34 | def watch_memory(controller_pid, max_memory, max_age): 35 | if max_age: 36 | end_time = time.time() + max_age 37 | else: 38 | end_time = None 39 | 40 | process_group = os.getpgrp() 41 | while True: 42 | if max_age: 43 | now = time.time() 44 | if now + MEMORY_WATCH_INTERVAL > end_time: 45 | time.sleep(end_time - now) 46 | print "(%s) *** watcher restarting processes! Time limit exceeded." % ( 47 | os.getpid(), ) 48 | os.kill(controller_pid, signal.SIGHUP) 49 | end_time = time.time() + max_age 50 | continue 51 | 52 | time.sleep(MEMORY_WATCH_INTERVAL) 53 | if max_memory: 54 | out = commands.getoutput('ps -o rss -g %s' % (process_group, )) 55 | used_mem = sum(int(x) for x in out.split('\n')[1:]) 56 | if used_mem > max_memory: 57 | print "(%s) *** memory watcher restarting processes! Memory usage of %s exceeded %s." % ( 58 | os.getpid(), used_mem, max_memory) 59 | os.kill(controller_pid, signal.SIGHUP) 60 | 61 | 62 | if __name__ == '__main__': 63 | parser = optparse.OptionParser( 64 | description="Watch all the processes in the process group" 65 | " and if the total memory used goes over a configurable amount, send a SIGHUP" 66 | " to a given pid.") 67 | parser.add_option('-a', '--max-age', dest='max_age', type='int', 68 | help='If given, the maximum amount of time (in seconds) to run before sending a ' 69 | 'SIGHUP to the given pid.') 70 | 71 | options, positional_args = parser.parse_args() 72 | 73 | if len(positional_args) < 2: 74 | parser.error("Usage: %s controller_pid max_memory_in_megabytes") 75 | 76 | controller_pid = int(positional_args[0]) 77 | max_memory = int(positional_args[1]) 78 | if max_memory: 79 | info = 'memory to %s' % (max_memory, ) 80 | else: 81 | info = '' 82 | 83 | if options.max_age: 84 | if info: 85 | info += ' and' 86 | info = " time to %s" % (options.max_age, ) 87 | 88 | print "(%s) watcher starting up, limiting%s." % ( 89 | os.getpid(), info) 90 | 91 | try: 92 | watch_memory(controller_pid, max_memory, options.max_age) 93 | except KeyboardInterrupt: 94 | pass 95 | -------------------------------------------------------------------------------- /spawning/paste_factory.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2008, Donovan Preston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to 5 | # deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | # FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | import os 24 | import sys 25 | 26 | from paste.deploy import loadwsgi 27 | 28 | from spawning import spawning_controller 29 | 30 | 31 | def config_factory(args): 32 | if 'config_url' in args: 33 | config_url = args['config_url'] 34 | relative_to = args['relative_to'] 35 | global_conf = args['global_conf'] 36 | else: 37 | config_file = os.path.abspath(args['args'][0]) 38 | config_url = 'config:%s' % (os.path.basename(config_file), ) 39 | relative_to = os.path.dirname(config_file) 40 | global_conf = {} 41 | for arg in args['args'][1:]: 42 | key, value = arg.split('=') 43 | global_conf[key] = value 44 | 45 | ctx = loadwsgi.loadcontext( 46 | loadwsgi.SERVER, 47 | config_url, 48 | relative_to=relative_to, 49 | global_conf=global_conf) 50 | 51 | watch = args.get('watch', None) 52 | if watch is None: 53 | watch = [] 54 | if ctx.global_conf['__file__'] not in watch: 55 | watch.append(ctx.global_conf['__file__']) 56 | args['watch'] = watch 57 | 58 | args['app_factory'] = 'spawning.paste_factory.app_factory' 59 | args['config_url'] = config_url 60 | args['relative_to'] = relative_to 61 | args['source_directories'] = [relative_to] 62 | args['global_conf'] = ctx.global_conf 63 | 64 | debug = ctx.global_conf.get('debug', None) 65 | if debug is not None: 66 | args['dev'] = (debug == 'true') 67 | host = ctx.local_conf.get('host', None) 68 | if host is not None: 69 | args['host'] = host 70 | port = ctx.local_conf.get('port', None) 71 | if port is not None: 72 | args['port'] = int(port) 73 | num_processes = ctx.local_conf.get('num_processes', None) 74 | if num_processes is not None: 75 | args['num_processes'] = int(num_processes) 76 | threadpool_workers = ctx.local_conf.get('threadpool_workers', None) 77 | if threadpool_workers is not None: 78 | args['threadpool_workers'] = int(threadpool_workers) 79 | 80 | return args 81 | 82 | 83 | def app_factory(config): 84 | return loadwsgi.loadapp( 85 | config['config_url'], 86 | relative_to=config['relative_to'], 87 | global_conf=config['global_conf']) 88 | 89 | 90 | def server_factory(global_conf, host, port, *args, **kw): 91 | config_url = 'config:' + os.path.split(global_conf['__file__'])[1] 92 | relative_to = global_conf['here'] 93 | 94 | def run(app): 95 | args = spawning_controller.DEFAULTS.copy() 96 | args.update( 97 | {'config_url': config_url, 'relative_to': relative_to, 'global_conf': global_conf}) 98 | 99 | spawning_controller.run_controller( 100 | 'spawning.paste_factory.config_factory', args) 101 | 102 | return run 103 | -------------------------------------------------------------------------------- /spawning/reloader_dev.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2008, Donovan Preston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to 5 | # deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | # FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | """Watch files and send a SIGHUP signal to another process 24 | if any of the files change. 25 | """ 26 | 27 | try: 28 | set 29 | except NameError: 30 | import sets 31 | set = sets.Set 32 | 33 | import optparse, os, signal, sys, tempfile, time 34 | from os.path import join 35 | from distutils import sysconfig 36 | 37 | import eventlet 38 | 39 | try: 40 | from procname import setprocname 41 | except ImportError, e: 42 | setprocname = lambda n: None 43 | 44 | def watch_forever(pid, interval, files=None): 45 | """ 46 | """ 47 | limiter = eventlet.GreenPool() 48 | module_mtimes = {} 49 | last_changed_time = None 50 | while True: 51 | uniques = set() 52 | 53 | uniques.add(join(sysconfig.get_python_lib(), 'easy-install.pth')) 54 | uniques.update(list(get_sys_modules_files())) 55 | 56 | if files: 57 | uniques.update(files) 58 | ##print uniques 59 | changed = False 60 | for filename in uniques: 61 | try: 62 | stat = os.stat(filename) 63 | if stat: 64 | mtime = stat.st_mtime 65 | else: 66 | mtime = 0 67 | except (OSError, IOError): 68 | continue 69 | if filename.endswith('.pyc') and os.path.exists(filename[:-1]): 70 | mtime = max(os.stat(filename[:-1]).st_mtime, mtime) 71 | if not module_mtimes.has_key(filename): 72 | module_mtimes[filename] = mtime 73 | elif module_mtimes[filename] < mtime: 74 | changed = True 75 | last_changed_time = mtime 76 | module_mtimes[filename] = mtime 77 | print "(%s) * File %r changed" % (os.getpid(), filename) 78 | 79 | if not changed and last_changed_time is not None: 80 | last_changed_time = None 81 | if pid: 82 | print "(%s) ** Sending SIGHUP to %s at %s" % ( 83 | os.getpid(), pid, time.asctime()) 84 | os.kill(pid, signal.SIGHUP) 85 | return ## this process is going to die now, no need to keep watching 86 | else: 87 | print "EXIT??!!!" 88 | os._exit(5) 89 | 90 | eventlet.sleep(interval) 91 | 92 | 93 | def get_sys_modules_files(): 94 | for module in sys.modules.values(): 95 | fn = getattr(module, '__file__', None) 96 | if fn is not None: 97 | yield os.path.abspath(fn) 98 | 99 | 100 | def main(): 101 | parser = optparse.OptionParser() 102 | parser.add_option("-p", "--pid", 103 | type="int", dest="pid", 104 | help="A pid to SIGHUP when a monitored file changes. " 105 | "If not given, just print a message to stdout and kill this process instead.") 106 | parser.add_option("-i", "--interval", 107 | type="int", dest="interval", 108 | help="The time to wait between scans, in seconds.", default=1) 109 | options, args = parser.parse_args() 110 | 111 | try: 112 | watch_forever(options.pid, options.interval) 113 | except KeyboardInterrupt: 114 | pass 115 | 116 | 117 | if __name__ == '__main__': 118 | main() 119 | 120 | -------------------------------------------------------------------------------- /spawning/reloader_svn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2008, Donovan Preston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to 5 | # deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | # FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | """Watch the svn revision returned from svn info and send a SIGHUP 24 | to a process when the revision changes. 25 | """ 26 | 27 | 28 | import commands, optparse, os, signal, sys, tempfile, time 29 | 30 | try: 31 | from procname import setprocname 32 | except ImportError, e: 33 | setprocname = lambda n: None 34 | 35 | 36 | def get_revision(directory): 37 | cmd = 'svn info' 38 | if directory is not None: 39 | cmd = '%s %s' % (cmd, directory) 40 | 41 | try: 42 | out = commands.getoutput(cmd).split('\n') 43 | except IOError: 44 | return 45 | 46 | for line in out: 47 | if line.startswith('Revision: '): 48 | return int(line[len('Revision: '):]) 49 | 50 | 51 | def watch_forever(directories, pid, interval): 52 | setprocname("spawn: svn reloader") 53 | if directories is None: 54 | directories = ['.'] 55 | ## Look for externals 56 | all_svn_repos = set(directories) 57 | 58 | def visit(parent, subdirname, children): 59 | if '.svn' in children: 60 | children.remove('.svn') 61 | status, out = commands.getstatusoutput('svn propget svn:externals %s' % (subdirname, )) 62 | if status: 63 | return 64 | 65 | for line in out.split('\n'): 66 | line = line.strip() 67 | if not line: 68 | continue 69 | name, _external_url = line.split() 70 | fulldir = os.path.join(parent, subdirname, name) 71 | ## Don't keep going into the external in the walk() 72 | try: 73 | children.remove(name) 74 | except ValueError: 75 | print "*** An entry in svn externals doesn't exist, ignoring:", name 76 | else: 77 | directories.append(fulldir) 78 | all_svn_repos.add(fulldir) 79 | 80 | while directories: 81 | dirname = directories.pop(0) 82 | os.path.walk(dirname, visit, dirname) 83 | 84 | revisions = {} 85 | for dirname in all_svn_repos: 86 | revisions[dirname] = get_revision(dirname) 87 | 88 | print "(%s) svn watcher watching directories: %s" % ( 89 | os.getpid(), list(all_svn_repos)) 90 | 91 | while True: 92 | if pid: 93 | ## Check to see if our controller is still alive; if not, just exit. 94 | try: 95 | os.getpgid(pid) 96 | except OSError: 97 | print "(%s) reloader_svn is orphaned; controller %s no longer running. Exiting." % ( 98 | os.getpid(), pid) 99 | os._exit(0) 100 | 101 | for dirname in all_svn_repos: 102 | new_revision = get_revision(dirname) 103 | 104 | if new_revision is not None and new_revision != revisions[dirname]: 105 | revisions[dirname] = new_revision 106 | if pid: 107 | print "(%s) * SVN revision changed on %s to %s; Sending SIGHUP to %s at %s" % ( 108 | os.getpid(), dirname, new_revision, pid, time.asctime()) 109 | os.kill(pid, signal.SIGHUP) 110 | os._exit(0) 111 | else: 112 | print "(%s) Revision changed, dying at %s" % ( 113 | os.getpid(), time.asctime()) 114 | os._exit(5) 115 | 116 | time.sleep(interval) 117 | 118 | 119 | def main(): 120 | parser = optparse.OptionParser() 121 | parser.add_option("-d", "--dir", dest='dirs', action="append", 122 | help="The directories to do svn info in. If not given, use cwd.") 123 | parser.add_option("-p", "--pid", 124 | type="int", dest="pid", 125 | help="A pid to SIGHUP when the svn revision changes. " 126 | "If not given, just print a message to stdout and kill this process instead.") 127 | parser.add_option("-i", "--interval", 128 | type="int", dest="interval", 129 | help="The time to wait between scans, in seconds.", default=10) 130 | options, args = parser.parse_args() 131 | 132 | print "(%s) svn watcher running, controller pid %s" % (os.getpid(), options.pid) 133 | if options.pid is None: 134 | options.pid = os.getpid() 135 | try: 136 | watch_forever(options.dirs, int(options.pid), options.interval) 137 | except KeyboardInterrupt: 138 | pass 139 | 140 | 141 | if __name__ == '__main__': 142 | main() 143 | 144 | -------------------------------------------------------------------------------- /spawning/spawning_child.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2008, Donovan Preston 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to 6 | # deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | # FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | # THE SOFTWARE. 23 | 24 | """spawning_child.py 25 | """ 26 | 27 | import eventlet 28 | import eventlet.event 29 | import eventlet.greenio 30 | import eventlet.greenthread 31 | import eventlet.hubs 32 | import eventlet.wsgi 33 | 34 | import errno 35 | import optparse 36 | import os 37 | import signal 38 | import socket 39 | import sys 40 | import time 41 | 42 | import spawning.util 43 | from spawning import setproctitle, reloader_dev 44 | 45 | try: 46 | import simplejson as json 47 | except ImportError: 48 | import json 49 | 50 | 51 | class URLInterceptor(object): 52 | """ 53 | Intercepts one or more paths. 54 | """ 55 | 56 | paths = [] 57 | 58 | def __init__(self, app, paths=[]): 59 | """ 60 | Creates an instance. 61 | 62 | :Parameters: 63 | - `app`: Application to fall through to 64 | """ 65 | self.app = app 66 | 67 | def _intercept(self, env, start_response): 68 | """ 69 | Executes business logic. 70 | 71 | :Parameters: 72 | - `env`: environment information 73 | - `start_response`: wsgi response function 74 | """ 75 | raise NotImplementedError('_intercept must be overridden') 76 | 77 | def __call__(self, env, start_response): 78 | """ 79 | Dispatches input to the proper method. 80 | 81 | :Parameters: 82 | - `env`: environment information 83 | - `start_response`: wsgi response function 84 | """ 85 | if env['PATH_INFO'] in self.paths: 86 | return self._intercept(env, start_response) 87 | return self.app(env, start_response) 88 | 89 | 90 | class FigleafCoverage(URLInterceptor): 91 | 92 | paths = ['/_coverage'] 93 | 94 | def __init__(self, app): 95 | URLInterceptor.__init__(self, app) 96 | import figleaf 97 | figleaf.start() 98 | 99 | def _intercept(self, env, start_response): 100 | import figleaf 101 | try: 102 | import cPickle as pickle 103 | except ImportError: 104 | import pickle 105 | 106 | coverage = figleaf.get_info() 107 | s = pickle.dumps(coverage) 108 | start_response("200 OK", [('Content-type', 'application/x-pickle')]) 109 | return [s] 110 | 111 | 112 | class SystemInfo(URLInterceptor): 113 | """ 114 | Intercepts /_sysinfo path and returns json data. 115 | """ 116 | 117 | paths = ['/_sysinfo'] 118 | 119 | def _intercept(self, env, start_response): 120 | """ 121 | Executes business logic. 122 | 123 | :Parameters: 124 | - `env`: environment information 125 | - `start_response`: wsgi response function 126 | """ 127 | import spawning.util.system 128 | start_response("200 OK", [('Content-type', 'application/json')]) 129 | return [json.dumps(spawning.util.system.System())] 130 | 131 | 132 | class ExitChild(Exception): 133 | pass 134 | 135 | class ChildStatus(object): 136 | def __init__(self, controller_port): 137 | self.controller_url = "http://127.0.0.1:%s/" % controller_port 138 | self.server = None 139 | 140 | def send_status_to_controller(self): 141 | try: 142 | child_status = {'pid':os.getpid()} 143 | if self.server: 144 | child_status['concurrent_requests'] = \ 145 | self.server.outstanding_requests 146 | else: 147 | child_status['error'] = 'Starting...' 148 | body = json.dumps(child_status) 149 | import urllib2 150 | urllib2.urlopen(self.controller_url, body) 151 | except (KeyboardInterrupt, SystemExit, 152 | eventlet.greenthread.greenlet.GreenletExit): 153 | raise 154 | except Exception, e: 155 | # we really don't want exceptions here to stop read_pipe_and_die 156 | pass 157 | 158 | _g_status = None 159 | def init_statusobj(status_port): 160 | global _g_status 161 | if status_port: 162 | _g_status = ChildStatus(status_port) 163 | def get_statusobj(): 164 | return _g_status 165 | 166 | 167 | def read_pipe_and_die(the_pipe, server_coro): 168 | dying = False 169 | try: 170 | while True: 171 | eventlet.hubs.trampoline(the_pipe, read=True) 172 | c = os.read(the_pipe, 1) 173 | # this is how the controller tells the child to send a status update 174 | if c == 's' and get_statusobj(): 175 | get_statusobj().send_status_to_controller() 176 | elif not dying: 177 | dying = True # only send ExitChild once 178 | eventlet.greenthread.kill(server_coro, ExitChild) 179 | # continue to listen for status pings while dying 180 | except socket.error: 181 | pass 182 | # if here, perhaps the controller's process went down; we should die too if 183 | # we aren't already 184 | if not dying: 185 | eventlet.greenthread.kill(server_coro, KeyboardInterrupt) 186 | 187 | 188 | def deadman_timeout(signum, frame): 189 | print "(%s) !!! Deadman timer expired, killing self with extreme prejudice" % ( 190 | os.getpid(), ) 191 | os.kill(os.getpid(), signal.SIGKILL) 192 | 193 | def tpool_wsgi(app): 194 | from eventlet import tpool 195 | def tpooled_application(e, s): 196 | result = tpool.execute(app, e, s) 197 | # return builtins directly 198 | if isinstance(result, (basestring, list, tuple)): 199 | return result 200 | else: 201 | # iterators might execute code when iterating over them, 202 | # so we wrap them in a Proxy object so every call to 203 | # next() goes through tpool 204 | return tpool.Proxy(result) 205 | return tpooled_application 206 | 207 | 208 | def warn_controller_of_imminent_death(controller_pid): 209 | # The controller responds to a SIGUSR1 by kicking off a new child process. 210 | try: 211 | os.kill(controller_pid, signal.SIGUSR1) 212 | except OSError, e: 213 | if not e.errno == errno.ESRCH: 214 | raise 215 | 216 | 217 | def serve_from_child(sock, config, controller_pid): 218 | threads = config.get('threadpool_workers', 0) 219 | wsgi_application = spawning.util.named(config['app_factory'])(config) 220 | 221 | if config.get('coverage'): 222 | wsgi_application = FigleafCoverage(wsgi_application) 223 | if config.get('sysinfo'): 224 | wsgi_application = SystemInfo(wsgi_application) 225 | 226 | if threads >= 1: 227 | # proxy calls of the application through tpool 228 | wsgi_application = tpool_wsgi(wsgi_application) 229 | elif threads != 1: 230 | print "(%s) not using threads, installing eventlet cooperation monkeypatching" % ( 231 | os.getpid(), ) 232 | eventlet.patcher.monkey_patch(all=False, socket=True) 233 | 234 | host, port = sock.getsockname() 235 | 236 | access_log_file = config.get('access_log_file') 237 | if access_log_file is not None: 238 | access_log_file = open(access_log_file, 'a') 239 | 240 | max_age = 0 241 | if config.get('max_age'): 242 | max_age = int(config.get('max_age')) 243 | 244 | server_event = eventlet.event.Event() 245 | # the status object wants to have a reference to the server object 246 | if config.get('status_port'): 247 | def send_server_to_status(server_event): 248 | server = server_event.wait() 249 | get_statusobj().server = server 250 | eventlet.spawn(send_server_to_status, server_event) 251 | 252 | http_version = config.get('no_keepalive') and 'HTTP/1.0' or 'HTTP/1.1' 253 | try: 254 | wsgi_args = (sock, wsgi_application) 255 | wsgi_kwargs = {'log' : access_log_file, 'server_event' : server_event, 'max_http_version' : http_version} 256 | if config.get('no_keepalive'): 257 | wsgi_kwargs.update({'keepalive' : False}) 258 | if max_age: 259 | wsgi_kwargs.update({'timeout_value' : True}) 260 | eventlet.with_timeout(max_age, eventlet.wsgi.server, *wsgi_args, 261 | **wsgi_kwargs) 262 | warn_controller_of_imminent_death(controller_pid) 263 | else: 264 | eventlet.wsgi.server(*wsgi_args, **wsgi_kwargs) 265 | except KeyboardInterrupt: 266 | # controller probably doesn't know that we got killed by a SIGINT 267 | warn_controller_of_imminent_death(controller_pid) 268 | except ExitChild: 269 | pass # parent killed us, it already knows we're dying 270 | 271 | ## Set a deadman timer to violently kill the process if it doesn't die after 272 | ## some long timeout. 273 | signal.signal(signal.SIGALRM, deadman_timeout) 274 | signal.alarm(config['deadman_timeout']) 275 | 276 | ## Once we get here, we just need to handle outstanding sockets, not 277 | ## accept any new sockets, so we should close the server socket. 278 | sock.close() 279 | 280 | server = server_event.wait() 281 | 282 | last_outstanding = None 283 | while server.outstanding_requests: 284 | if last_outstanding != server.outstanding_requests: 285 | print "(%s) %s requests remaining, waiting... (timeout after %s)" % ( 286 | os.getpid(), server.outstanding_requests, config['deadman_timeout']) 287 | last_outstanding = server.outstanding_requests 288 | eventlet.sleep(0.1) 289 | 290 | print "(%s) *** Child exiting: all requests completed at %s" % ( 291 | os.getpid(), time.asctime()) 292 | 293 | 294 | def child_sighup(*args, **kwargs): 295 | exit(0) 296 | 297 | 298 | def main(): 299 | parser = optparse.OptionParser() 300 | parser.add_option("-r", "--reload", 301 | action='store_true', dest='reload', 302 | help='If --reload is passed, reload the server any time ' 303 | 'a loaded module changes.') 304 | 305 | options, args = parser.parse_args() 306 | 307 | if len(args) != 5: 308 | print "Usage: %s controller_pid httpd_fd death_fd factory_qual factory_args" % ( 309 | sys.argv[0], ) 310 | sys.exit(1) 311 | 312 | controller_pid, httpd_fd, death_fd, factory_qual, factory_args = args 313 | controller_pid = int(controller_pid) 314 | config = spawning.util.named(factory_qual)(json.loads(factory_args)) 315 | 316 | setproctitle("spawn: child (%s)" % ", ".join(config.get("args"))) 317 | 318 | ## Set up status reporter, if requested 319 | init_statusobj(config.get('status_port')) 320 | 321 | ## Set up the reloader 322 | if config.get('reload'): 323 | watch = config.get('watch', None) 324 | if watch: 325 | watching = ' and %s' % watch 326 | else: 327 | watching = '' 328 | print "(%s) reloader watching sys.modules%s" % (os.getpid(), watching) 329 | eventlet.spawn( 330 | reloader_dev.watch_forever, controller_pid, 1, watch) 331 | 332 | ## The parent will catch sigint and tell us to shut down 333 | signal.signal(signal.SIGINT, signal.SIG_IGN) 334 | ## Expect a SIGHUP when we want the child to die 335 | signal.signal(signal.SIGHUP, child_sighup) 336 | eventlet.spawn(read_pipe_and_die, int(death_fd), eventlet.getcurrent()) 337 | 338 | ## Make the socket object from the fd given to us by the controller 339 | sock = eventlet.greenio.GreenSocket( 340 | socket.fromfd(int(httpd_fd), socket.AF_INET, socket.SOCK_STREAM)) 341 | 342 | serve_from_child( 343 | sock, config, controller_pid) 344 | 345 | if __name__ == '__main__': 346 | main() 347 | -------------------------------------------------------------------------------- /spawning/spawning_controller.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # Copyright (c) 2008, Donovan Preston 3 | # 4 | # Permission is hereby granted, free of charge, to any person obtaining a copy 5 | # of this software and associated documentation files (the "Software"), to 6 | # deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in 13 | # all copies or substantial portions of the Software. 14 | # 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | # FROM, 21 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | # THE SOFTWARE. 23 | from __future__ import with_statement 24 | 25 | import eventlet 26 | eventlet.monkey_patch() 27 | 28 | import commands 29 | import datetime 30 | import errno 31 | import logging 32 | import optparse 33 | import pprint 34 | import signal 35 | import socket 36 | import sys 37 | import time 38 | import traceback 39 | 40 | try: 41 | import simplejson as json 42 | except ImportError: 43 | import json 44 | 45 | 46 | import eventlet.backdoor 47 | from eventlet.green import os 48 | 49 | import spawning 50 | import spawning.util 51 | 52 | KEEP_GOING = True 53 | RESTART_CONTROLLER = False 54 | PANIC = False 55 | 56 | 57 | DEFAULTS = { 58 | 'num_processes': 4, 59 | 'threadpool_workers': 4, 60 | 'watch': [], 61 | 'dev': True, 62 | 'host': '', 63 | 'port': 8080, 64 | 'deadman_timeout': 10, 65 | 'max_memory': None, 66 | } 67 | 68 | def print_exc(msg="Exception occured!"): 69 | print >>sys.stderr, "(%d) %s" % (os.getpid(), msg) 70 | traceback.print_exc() 71 | 72 | def environ(): 73 | env = os.environ.copy() 74 | # to avoid duplicates in the new sys.path 75 | revised_paths = set() 76 | new_path = list() 77 | for path in sys.path: 78 | if os.path.exists(path) and path not in revised_paths: 79 | revised_paths.add(path) 80 | new_path.append(path) 81 | current_directory = os.path.realpath('.') 82 | if current_directory not in revised_paths: 83 | new_path.append(current_directory) 84 | 85 | env['PYTHONPATH'] = ':'.join(new_path) 86 | return env 87 | 88 | class Child(object): 89 | def __init__(self, pid, kill_pipe): 90 | self.pid = pid 91 | self.kill_pipe = kill_pipe 92 | self.active = True 93 | self.forked_at = datetime.datetime.now() 94 | 95 | class Controller(object): 96 | sock = None 97 | factory = None 98 | args = None 99 | config = None 100 | children = None 101 | keep_going = True 102 | panic = False 103 | log = None 104 | controller_pid = None 105 | num_processes = 0 106 | 107 | def __init__(self, sock, factory, args, **kwargs): 108 | self.sock = sock 109 | self.factory = factory 110 | self.config = spawning.util.named(factory)(args) 111 | self.args = args 112 | self.children = {} 113 | self.log = logging.getLogger('Spawning') 114 | if not kwargs.get('log_handler'): 115 | self.log.addHandler(logging.StreamHandler()) 116 | self.log.setLevel(logging.DEBUG) 117 | self.controller_pid = os.getpid() 118 | self.num_processes = int(self.config.get('num_processes', 0)) 119 | self.started_at = datetime.datetime.now() 120 | 121 | def spawn_children(self, number=1): 122 | parent_pid = os.getpid() 123 | 124 | for i in range(number): 125 | child_side, parent_side = os.pipe() 126 | try: 127 | child_pid = os.fork() 128 | except: 129 | print_exc('Could not fork child! Panic!') 130 | ### TODO: restart 131 | 132 | if not child_pid: # child process 133 | os.close(parent_side) 134 | command = [sys.executable, '-c', 135 | 'import sys; from spawning import spawning_child; spawning_child.main()', 136 | str(parent_pid), 137 | str(self.sock.fileno()), 138 | str(child_side), 139 | self.factory, 140 | json.dumps(self.args)] 141 | if self.args['reload'] == 'dev': 142 | command.append('--reload') 143 | env = environ() 144 | tpool_size = int(self.config.get('threadpool_workers', 0)) 145 | assert tpool_size >= 0, (tpool_size, 'Cannot have a negative --threads argument') 146 | if not tpool_size in (0, 1): 147 | env['EVENTLET_THREADPOOL_SIZE'] = str(tpool_size) 148 | os.execve(sys.executable, command, env) 149 | 150 | # controller process 151 | os.close(child_side) 152 | self.children[child_pid] = Child(child_pid, parent_side) 153 | 154 | def children_count(self): 155 | return len(self.children) 156 | 157 | def runloop(self): 158 | while self.keep_going: 159 | eventlet.sleep(0.1) 160 | ## Only start the number of children we need 161 | number = self.num_processes - self.children_count() 162 | if number > 0: 163 | self.log.debug('Should start %d new children', number) 164 | self.spawn_children(number=number) 165 | continue 166 | 167 | if not self.children: 168 | ## If we don't yet have children, let's loop 169 | continue 170 | 171 | pid, result = None, None 172 | try: 173 | pid, result = os.wait() 174 | except OSError, e: 175 | if e.errno != errno.EINTR: 176 | raise 177 | 178 | if pid and self.children.get(pid): 179 | try: 180 | child = self.children.pop(pid) 181 | os.close(child.kill_pipe) 182 | except (IOError, OSError): 183 | pass 184 | 185 | if result: 186 | signum = os.WTERMSIG(result) 187 | exitcode = os.WEXITSTATUS(result) 188 | self.log.info('(%s) Child died from signal %s with code %s', 189 | pid, signum, exitcode) 190 | 191 | def handle_sighup(self, *args, **kwargs): 192 | ''' Pass `no_restart` to prevent restarting the run loop ''' 193 | self.kill_children() 194 | self.spawn_children(number=self.num_processes) 195 | # TODO: nothing seems to use no_restart, can it be removed? 196 | if not kwargs.get('no_restart', True): 197 | self.runloop() 198 | 199 | def kill_children(self): 200 | for pid, child in self.children.items(): 201 | try: 202 | os.write(child.kill_pipe, 'k') 203 | child.active = False 204 | # all maintenance of children's membership happens in runloop() 205 | # as children die and os.wait() gets results 206 | except OSError, e: 207 | if e.errno != errno.EPIPE: 208 | raise 209 | 210 | def handle_deadlychild(self, *args, **kwargs): 211 | """ 212 | SIGUSR1 handler, will spin up an extra child to handle the load 213 | left over after a previously running child stops taking connections 214 | and "dies" gracefully 215 | """ 216 | if self.keep_going: 217 | self.spawn_children(number=1) 218 | 219 | def run(self): 220 | self.log.info('(%s) *** Controller starting at %s' % (self.controller_pid, 221 | time.asctime())) 222 | 223 | if self.config.get('pidfile'): 224 | with open(self.config.get('pidfile'), 'w') as fd: 225 | fd.write('%s\n' % self.controller_pid) 226 | 227 | spawning.setproctitle("spawn: controller " + self.args.get('argv_str', '')) 228 | 229 | if self.sock is None: 230 | self.sock = bind_socket(self.config) 231 | 232 | signal.signal(signal.SIGHUP, self.handle_sighup) 233 | signal.signal(signal.SIGUSR1, self.handle_deadlychild) 234 | 235 | if self.config.get('status_port'): 236 | from spawning.util import status 237 | eventlet.spawn(status.Server, self, 238 | self.config['status_host'], self.config['status_port']) 239 | 240 | try: 241 | self.runloop() 242 | except KeyboardInterrupt: 243 | self.keep_going = False 244 | self.kill_children() 245 | self.log.info('(%s) *** Controller exiting' % (self.controller_pid)) 246 | 247 | def bind_socket(config): 248 | sleeptime = 0.5 249 | host = config.get('host', '') 250 | port = config.get('port', 8080) 251 | for x in range(8): 252 | try: 253 | sock = eventlet.listen((host, port)) 254 | break 255 | except socket.error, e: 256 | if e[0] != errno.EADDRINUSE: 257 | raise 258 | print "(%s) socket %s:%s already in use, retrying after %s seconds..." % ( 259 | os.getpid(), host, port, sleeptime) 260 | eventlet.sleep(sleeptime) 261 | sleeptime *= 2 262 | else: 263 | print "(%s) could not bind socket %s:%s, dying." % ( 264 | os.getpid(), host, port) 265 | sys.exit(1) 266 | return sock 267 | 268 | def set_process_owner(spec): 269 | import pwd, grp 270 | if ":" in spec: 271 | user, group = spec.split(":", 1) 272 | else: 273 | user, group = spec, None 274 | if group: 275 | os.setgid(grp.getgrnam(group).gr_gid) 276 | if user: 277 | os.setuid(pwd.getpwnam(user).pw_uid) 278 | return user, group 279 | 280 | def start_controller(sock, factory, factory_args): 281 | c = Controller(sock, factory, factory_args) 282 | installGlobal(c) 283 | c.run() 284 | 285 | def main(): 286 | current_directory = os.path.realpath('.') 287 | if current_directory not in sys.path: 288 | sys.path.append(current_directory) 289 | 290 | parser = optparse.OptionParser(description="Spawning is an easy-to-use and flexible wsgi server. It supports graceful restarting so that your site finishes serving any old requests while starting new processes to handle new requests with the new code. For the simplest usage, simply pass the dotted path to your wsgi application: 'spawn my_module.my_wsgi_app'", version=spawning.__version__) 291 | parser.add_option('-v', '--verbose', dest='verbose', action='store_true', help='Display verbose configuration ' 292 | 'information when starting up or restarting.') 293 | parser.add_option("-f", "--factory", dest='factory', default='spawning.wsgi_factory.config_factory', 294 | help="""Dotted path (eg mypackage.mymodule.myfunc) to a callable which takes a dictionary containing the command line arguments and figures out what needs to be done to start the wsgi application. Current valid values are: spawning.wsgi_factory.config_factory, spawning.paste_factory.config_factory, and spawning.django_factory.config_factory. The factory used determines what the required positional command line arguments will be. See the spawning.wsgi_factory module for documentation on how to write a new factory. 295 | """) 296 | parser.add_option("-i", "--host", 297 | dest='host', default=DEFAULTS['host'], 298 | help='The local ip address to bind.') 299 | parser.add_option("-p", "--port", 300 | dest='port', type='int', default=DEFAULTS['port'], 301 | help='The local port address to bind.') 302 | parser.add_option("-s", "--processes", 303 | dest='processes', type='int', default=DEFAULTS['num_processes'], 304 | help='The number of unix processes to start to use for handling web i/o.') 305 | parser.add_option("-t", "--threads", 306 | dest='threads', type='int', default=DEFAULTS['threadpool_workers'], 307 | help="The number of posix threads to use for handling web requests. " 308 | "If threads is 0, do not use threads but instead use eventlet's cooperative " 309 | "greenlet-based microthreads, monkeypatching the socket and pipe operations which normally block " 310 | "to cooperate instead. Note that most blocking database api modules will not " 311 | "automatically cooperate.") 312 | parser.add_option('-d', '--daemonize', dest='daemonize', action='store_true', 313 | help="Daemonize after starting children.") 314 | parser.add_option('-u', '--chuid', dest='chuid', metavar="ID", 315 | help="Change user ID in daemon mode (and group ID if given, " 316 | "separate with colon.)") 317 | parser.add_option('--pidfile', dest='pidfile', metavar="FILE", 318 | help="Write own process ID to FILE in daemon mode.") 319 | parser.add_option('--stdout', dest='stdout', metavar="FILE", 320 | help="Redirect stdout to FILE in daemon mode.") 321 | parser.add_option('--stderr', dest='stderr', metavar="FILE", 322 | help="Redirect stderr to FILE in daemon mode.") 323 | parser.add_option('-w', '--watch', dest='watch', action='append', 324 | help="Watch the given file's modification time. If the file changes, the web server will " 325 | 'restart gracefully, allowing old requests to complete in the old processes ' 326 | 'while starting new processes with the latest code or configuration.') 327 | ## TODO Hook up the svn reloader again 328 | parser.add_option("-r", "--reload", 329 | type='str', dest='reload', 330 | help='If --reload=dev is passed, reload any time ' 331 | 'a loaded module or configuration file changes.') 332 | parser.add_option("--deadman", "--deadman_timeout", 333 | type='int', dest='deadman_timeout', default=DEFAULTS['deadman_timeout'], 334 | help='When killing an old i/o process because the code has changed, don\'t wait ' 335 | 'any longer than the deadman timeout value for the process to gracefully exit. ' 336 | 'If all requests have not completed by the deadman timeout, the process will be mercilessly killed.') 337 | parser.add_option('-l', '--access-log-file', dest='access_log_file', default=None, 338 | help='The file to log access log lines to. If not given, log to stdout. Pass /dev/null to discard logs.') 339 | parser.add_option('-c', '--coverage', dest='coverage', action='store_true', 340 | help='If given, gather coverage data from the running program and make the ' 341 | 'coverage report available from the /_coverage url. See the figleaf docs ' 342 | 'for more info: http://darcs.idyll.org/~t/projects/figleaf/doc/') 343 | parser.add_option('--sysinfo', dest='sysinfo', action='store_true', 344 | help='If given, gather system information data and make the ' 345 | 'report available from the /_sysinfo url.') 346 | parser.add_option('-m', '--max-memory', dest='max_memory', type='int', default=0, 347 | help='If given, the maximum amount of memory this instance of Spawning ' 348 | 'is allowed to use. If all of the processes started by this Spawning controller ' 349 | 'use more than this amount of memory, send a SIGHUP to the controller ' 350 | 'to get the children to restart.') 351 | parser.add_option('--backdoor', dest='backdoor', action='store_true', 352 | help='Start a backdoor bound to localhost:3000') 353 | parser.add_option('-a', '--max-age', dest='max_age', type='int', 354 | help='If given, the maximum amount of time (in seconds) an instance of spawning_child ' 355 | 'is allowed to run. Once this time limit has expired the child will' 356 | 'gracefully kill itself while the server starts a replacement.') 357 | parser.add_option('--no-keepalive', dest='no_keepalive', action='store_true', 358 | help='Disable HTTP/1.1 KeepAlive') 359 | parser.add_option('-z', '--z-restart-args', dest='restart_args', 360 | help='For internal use only') 361 | parser.add_option('--status-port', dest='status_port', type='int', default=0, 362 | help='If given, hosts a server status page at that port. Two pages are served: a human-readable HTML version at http://host:status_port/status, and a machine-readable version at http://host:status_port/status.json') 363 | parser.add_option('--status-host', dest='status_host', type='string', default='', 364 | help='If given, binds the server status page to the specified local ip address. Defaults to the same value as --host. If --status-port is not supplied, the status page will not be activated.') 365 | 366 | options, positional_args = parser.parse_args() 367 | 368 | if len(positional_args) < 1 and not options.restart_args: 369 | parser.error("At least one argument is required. " 370 | "For the default factory, it is the dotted path to the wsgi application " 371 | "(eg my_package.my_module.my_wsgi_application). For the paste factory, it " 372 | "is the ini file to load. Pass --help for detailed information about available options.") 373 | 374 | if options.backdoor: 375 | try: 376 | eventlet.spawn(eventlet.backdoor.backdoor_server, eventlet.listen(('localhost', 3000))) 377 | except Exception, ex: 378 | sys.stderr.write('**> Error opening backdoor: %s\n' % ex) 379 | 380 | sock = None 381 | 382 | if options.restart_args: 383 | restart_args = json.loads(options.restart_args) 384 | factory = restart_args['factory'] 385 | factory_args = restart_args['factory_args'] 386 | 387 | start_delay = restart_args.get('start_delay') 388 | if start_delay is not None: 389 | factory_args['start_delay'] = start_delay 390 | print "(%s) delaying startup by %s" % (os.getpid(), start_delay) 391 | time.sleep(start_delay) 392 | 393 | fd = restart_args.get('fd') 394 | if fd is not None: 395 | sock = socket.fromfd(restart_args['fd'], socket.AF_INET, socket.SOCK_STREAM) 396 | ## socket.fromfd doesn't result in a socket object that has the same fd. 397 | ## The old fd is still open however, so we close it so we don't leak. 398 | os.close(restart_args['fd']) 399 | return start_controller(sock, factory, factory_args) 400 | 401 | ## We're starting up for the first time. 402 | if options.daemonize: 403 | # Do the daemon dance. Note that this isn't what is considered good 404 | # daemonization, because frankly it's convenient to keep the file 405 | # descriptiors open (especially when there are prints scattered all 406 | # over the codebase.) 407 | # What we do instead is fork off, create a new session, fork again. 408 | # This leaves the process group in a state without a session 409 | # leader. 410 | pid = os.fork() 411 | if not pid: 412 | os.setsid() 413 | pid = os.fork() 414 | if pid: 415 | os._exit(0) 416 | else: 417 | os._exit(0) 418 | print "(%s) now daemonized" % (os.getpid(),) 419 | # Close _all_ open (and othewise!) files. 420 | import resource 421 | maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] 422 | if maxfd == resource.RLIM_INFINITY: 423 | maxfd = 4096 424 | for fdnum in xrange(maxfd): 425 | try: 426 | os.close(fdnum) 427 | except OSError, e: 428 | if e.errno != errno.EBADF: 429 | raise 430 | # Remap std{in,out,err} 431 | devnull = os.open(os.path.devnull, os.O_RDWR) 432 | oflags = os.O_WRONLY | os.O_CREAT | os.O_APPEND 433 | if devnull != 0: # stdin 434 | os.dup2(devnull, 0) 435 | if options.stdout: 436 | stdout_fd = os.open(options.stdout, oflags) 437 | if stdout_fd != 1: 438 | os.dup2(stdout_fd, 1) 439 | os.close(stdout_fd) 440 | else: 441 | os.dup2(devnull, 1) 442 | if options.stderr: 443 | stderr_fd = os.open(options.stderr, oflags) 444 | if stderr_fd != 2: 445 | os.dup2(stderr_fd, 2) 446 | os.close(stderr_fd) 447 | else: 448 | os.dup2(devnull, 2) 449 | # Change user & group ID. 450 | if options.chuid: 451 | user, group = set_process_owner(options.chuid) 452 | print "(%s) set user=%s group=%s" % (os.getpid(), user, group) 453 | else: 454 | # Become a process group leader only if not daemonizing. 455 | os.setpgrp() 456 | 457 | ## Fork off the thing that watches memory for this process group. 458 | controller_pid = os.getpid() 459 | if options.max_memory and not os.fork(): 460 | env = environ() 461 | from spawning import memory_watcher 462 | basedir, cmdname = os.path.split(memory_watcher.__file__) 463 | if cmdname.endswith('.pyc'): 464 | cmdname = cmdname[:-1] 465 | 466 | os.chdir(basedir) 467 | command = [ 468 | sys.executable, 469 | cmdname, 470 | '--max-age', str(options.max_age), 471 | str(controller_pid), 472 | str(options.max_memory)] 473 | os.execve(sys.executable, command, env) 474 | 475 | factory = options.factory 476 | 477 | # If you tell me to watch something, I'm going to reload then 478 | if options.watch: 479 | options.reload = True 480 | 481 | if options.status_port == options.port: 482 | options.status_port = None 483 | sys.stderr.write('**> Status port cannot be the same as the service port, disabling status.\n') 484 | 485 | 486 | factory_args = { 487 | 'verbose': options.verbose, 488 | 'host': options.host, 489 | 'port': options.port, 490 | 'num_processes': options.processes, 491 | 'threadpool_workers': options.threads, 492 | 'watch': options.watch, 493 | 'reload': options.reload, 494 | 'deadman_timeout': options.deadman_timeout, 495 | 'access_log_file': options.access_log_file, 496 | 'pidfile': options.pidfile, 497 | 'coverage': options.coverage, 498 | 'sysinfo': options.sysinfo, 499 | 'no_keepalive' : options.no_keepalive, 500 | 'max_age' : options.max_age, 501 | 'argv_str': " ".join(sys.argv[1:]), 502 | 'args': positional_args, 503 | 'status_port': options.status_port, 504 | 'status_host': options.status_host or options.host 505 | } 506 | start_controller(sock, factory, factory_args) 507 | 508 | _global_attr_name_ = '_spawning_controller_' 509 | def installGlobal(controller): 510 | setattr(sys, _global_attr_name_, controller) 511 | 512 | def globalController(): 513 | return getattr(sys, _global_attr_name_, None) 514 | 515 | 516 | if __name__ == '__main__': 517 | main() 518 | 519 | 520 | 521 | -------------------------------------------------------------------------------- /spawning/util/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010, R. Tyler Ballance 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to 5 | # deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | # FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | def named(name): 24 | """Return an object given its name. 25 | 26 | The name uses a module-like syntax, eg:: 27 | 28 | os.path.join 29 | 30 | or:: 31 | 32 | mulib.mu.Resource 33 | """ 34 | toimport = name 35 | obj = None 36 | import_err_strings = [] 37 | while toimport: 38 | try: 39 | obj = __import__(toimport) 40 | break 41 | except ImportError, err: 42 | # print 'Import error on %s: %s' % (toimport, err) # debugging spam 43 | import_err_strings.append(err.__str__()) 44 | toimport = '.'.join(toimport.split('.')[:-1]) 45 | if obj is None: 46 | raise ImportError('%s could not be imported. Import errors: %r' % (name, import_err_strings)) 47 | for seg in name.split('.')[1:]: 48 | try: 49 | obj = getattr(obj, seg) 50 | except AttributeError: 51 | dirobj = dir(obj) 52 | dirobj.sort() 53 | raise AttributeError('attribute %r missing from %r (%r) %r. Import errors: %r' % ( 54 | seg, obj, dirobj, name, import_err_strings)) 55 | return obj 56 | -------------------------------------------------------------------------------- /spawning/util/log_parser.py: -------------------------------------------------------------------------------- 1 | import time 2 | from datetime import datetime, timedelta 3 | import sys 4 | import optparse 5 | import re 6 | 7 | __all__ = ['parse_line', 'parse_lines', 'parse_casual_time', 8 | 'group_parsed_lines', 'select_timerange'] 9 | 10 | month_names = {'Jan': 1, 'Feb': 2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6, 'Jul':7, 11 | 'Aug':8, 'Sep': 9, 'Oct':10, 'Nov': 11, 'Dec': 12} 12 | 13 | 14 | def parse_line(line): 15 | """ Parses a Spawning log line into a dictionary of fields. 16 | 17 | Returns the following fields: 18 | * client_ip : The remote IP address. 19 | * date : datetime object representing when the request completed 20 | * method : HTTP method 21 | * path : url path 22 | * version : HTTP version 23 | * status_code : HTTP status code 24 | * size : length of the body 25 | * duration : time in seconds to complete the request 26 | """ 27 | # note that a split-based version of the function is faster than 28 | # a regexp-based version 29 | segs = line.split() 30 | if len(segs) != 11: 31 | return None 32 | retval = {} 33 | try: 34 | retval['client_ip'] = segs[0] 35 | if segs[1] != '-' or segs[2] != '-': 36 | return None 37 | if segs[3][0] != '[' or segs[4][-1] != ']': 38 | return None 39 | # time parsing by explicitly poking at string slices is much faster 40 | # than strptime, but it won't work in non-English locales because of 41 | # the month names 42 | d = segs[3] 43 | t = segs[4] 44 | retval['date'] = datetime( 45 | int(d[8:12]), # year 46 | month_names[d[4:7]], # month 47 | int(d[1:3]), # day 48 | int(t[0:2]), # hour 49 | int(t[3:5]), # minute 50 | int(t[6:8])) # second 51 | if segs[5][0] != '"' or segs[7][-1] != '"': 52 | return None 53 | retval['method'] = segs[5][1:] 54 | retval['path'] = segs[6] 55 | retval['version'] = segs[7][:-1] 56 | retval['status_code'] = int(segs[8]) 57 | retval['size'] = int(segs[9]) 58 | retval['duration'] = float(segs[10]) 59 | except (IndexError, ValueError): 60 | return None 61 | return retval 62 | 63 | 64 | def parse_lines(fd): 65 | """Generator function that accepts an iterable file-like object and 66 | yields all the parseable lines found in it. 67 | """ 68 | for line in fd: 69 | parsed = parse_line(line) 70 | if parsed is not None: 71 | yield parsed 72 | 73 | 74 | time_intervals = {"sec":1, "min":60, "hr":3600, "day": 86400, 75 | "second":1, "minute":60, "hour":3600, 76 | "s":1, "m":60, "h":3600, "d":86400} 77 | for k,v in time_intervals.items(): # pluralize 78 | time_intervals[k + "s"] = v 79 | 80 | 81 | def parse_casual_time(timestr, relative_to): 82 | """Lenient relative time parser. Returns a datetime object if it can. 83 | 84 | Accepts such human-friendly times as "-1 hour", "-30s", "15min", "2d", "now". 85 | Any such relative time is interpreted as a delta applied to the relative_to 86 | argument, which should be a datetime. 87 | """ 88 | timestr = timestr.lower() 89 | try: 90 | return datetime(*(time.strptime(timestr)[0:6])) 91 | except ValueError: 92 | pass 93 | if timestr == "now": 94 | return datetime.now() 95 | # match stuff like "-1 hour", "-30s" 96 | m = re.match(r'([-0-9.]+)\s*(\w+)?', timestr) 97 | if m: 98 | intervalsz = 1 99 | if len(m.groups()) > 1 and m.group(2) in time_intervals: 100 | intervalsz = time_intervals[m.group(2)] 101 | relseconds = float(m.group(1)) * intervalsz 102 | return relative_to + timedelta(seconds=relseconds) 103 | 104 | def group_parsed_lines(lines, field): 105 | """Aggregates the parsed log lines by a field. Counts 106 | the log lines in each group and their average duration. The return 107 | value is a dict, where the keys are the unique field values, and the values 108 | are dicts of count, avg_duration, and the key. 109 | """ 110 | grouped = {} 111 | for parsed in lines: 112 | key = parsed[field] 113 | summary = grouped.setdefault(key, {'count':0, 'total_duration':0.0}) 114 | summary[field] = key 115 | summary['count'] += 1 116 | summary['total_duration'] += parsed['duration'] 117 | # average dat up 118 | for summary in grouped.values(): 119 | summary['avg_duration'] = summary['total_duration']/summary['count'] 120 | del summary['total_duration'] 121 | return grouped 122 | 123 | def select_timerange(lines, earliest=None, latest=None): 124 | """ Generator that accepts an iterable of parsed log lines and yields 125 | the log lines that are between the earliest and latest dates. If 126 | either earliest or latest is None, it is ignored.""" 127 | for parsed in lines: 128 | if earliest and parsed['date'] < earliest: 129 | continue 130 | if latest and parsed['date'] > latest: 131 | continue 132 | yield parsed 133 | 134 | 135 | if __name__ == "__main__": 136 | parser = optparse.OptionParser() 137 | parser.add_option('--earliest', dest='earliest', default=None, 138 | help='Earliest date to count, either as a full date or a relative time \ 139 | such as "-1 hour". Relative to --latest, so you generally want to\ 140 | specify a negative relative.') 141 | parser.add_option('--latest', dest='latest', default=None, 142 | help='Latest date to count, either as a full date or a relative time\ 143 | such as "-30s". Relative to now.') 144 | parser.add_option('--group-by', dest='group_by', default='path', 145 | help='Compute counts and aggregates for log lines grouped by this\ 146 | attribute. Good values include "status_code", "method", and\ 147 | "path" (the default).') 148 | opts, args = parser.parse_args() 149 | 150 | if opts.latest: 151 | opts.latest = parse_casual_time(opts.latest, datetime.now()) 152 | if opts.earliest: 153 | opts.earliest = parse_casual_time(opts.earliest, 154 | opts.latest or datetime.now()) 155 | if opts.earliest or opts.latest: 156 | print "Including dates between", \ 157 | opts.earliest or "the beginning of time", "and", opts.latest or "now" 158 | 159 | parsed_lines = parse_lines(sys.stdin) 160 | grouped = group_parsed_lines( 161 | select_timerange(parsed_lines, opts.earliest, opts.latest), 162 | opts.group_by) 163 | 164 | flat = grouped.values() 165 | flat.sort(key=lambda x: x['count']) 166 | flat.reverse() 167 | print "Count\tAvg Dur\t%s" % opts.group_by 168 | for summary in flat: 169 | print "%d\t%.4f\t%s" % (summary['count'], 170 | summary['avg_duration'], summary[opts.group_by]) 171 | 172 | -------------------------------------------------------------------------------- /spawning/util/status.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | try: 3 | import json 4 | except ImportError: 5 | import simplejson as json 6 | 7 | import eventlet 8 | from eventlet import event 9 | from eventlet import wsgi 10 | from eventlet.green import os 11 | 12 | class Server(object): 13 | def __init__(self, controller, host, port): 14 | self.controller = controller 15 | self.host = host 16 | self.port = port 17 | self.status_waiter = None 18 | self.child_events = {} 19 | socket = eventlet.listen((host, port)) 20 | wsgi.server(socket, self.application) 21 | 22 | def get_status_data(self): 23 | # using a waiter because we only want one child collection ping 24 | # happening at a time; if there are multiple concurrent status requests, 25 | # they all simply share the same set of data results 26 | if self.status_waiter is None: 27 | self.status_waiter = eventlet.spawn(self._collect_status_data) 28 | return self.status_waiter.wait() 29 | 30 | def _collect_status_data(self): 31 | try: 32 | now = datetime.datetime.now() 33 | children = self.controller.children.values() 34 | status_data = { 35 | 'active_children_count':len([c 36 | for c in children 37 | if c.active]), 38 | 'killed_children_count':len([c 39 | for c in children 40 | if not c.active]), 41 | 'configured_children_count':self.controller.num_processes, 42 | 'now':now.ctime(), 43 | 'pid':os.getpid(), 44 | 'uptime':format_timedelta(now - self.controller.started_at), 45 | 'started_at':self.controller.started_at.ctime(), 46 | 'config':self.controller.config} 47 | # fire up a few greenthreads to wait on children's responses 48 | p = eventlet.GreenPile() 49 | for child in self.controller.children.values(): 50 | p.spawn(self.collect_child_status, child) 51 | status_data['children'] = dict([pid_cd for pid_cd in p]) 52 | 53 | # total concurrent connections 54 | status_data['concurrent_requests'] = sum([ 55 | child.get('concurrent_requests', 0) 56 | for child in status_data['children'].values()]) 57 | finally: 58 | # wipe out the waiter so that subsequent requests create new ones 59 | self.status_waiter = None 60 | return status_data 61 | 62 | def collect_child_status(self, child): 63 | self.child_events[child.pid] = event.Event() 64 | try: 65 | try: 66 | # tell the child to POST its status to us, we handle it in the 67 | # wsgi application below 68 | eventlet.hubs.trampoline(child.kill_pipe, write=True) 69 | os.write(child.kill_pipe, 's') 70 | t = eventlet.Timeout(1) 71 | results = self.child_events[child.pid].wait() 72 | t.cancel() 73 | except (OSError, IOError), e: 74 | results = {'error': "%s %s" % (type(e), e)} 75 | except eventlet.Timeout: 76 | results = {'error':'Timed out'} 77 | finally: 78 | self.child_events.pop(child.pid, None) 79 | 80 | results.update({ 81 | 'pid':child.pid, 82 | 'active':child.active, 83 | 'uptime':format_timedelta(datetime.datetime.now() - child.forked_at), 84 | 'forked_at':child.forked_at.ctime()}) 85 | return child.pid, results 86 | 87 | def application(self, environ, start_response): 88 | if environ['REQUEST_METHOD'] == 'GET': 89 | status_data = self.get_status_data() 90 | if environ['PATH_INFO'] == '/status': 91 | start_response('200 OK', [('content-type', 'text/html')]) 92 | return [fill_template(status_data)] 93 | elif environ['PATH_INFO'] == '/status.json': 94 | start_response('200 OK', [('content-type', 'application/json')]) 95 | return [json.dumps(status_data, indent=2)] 96 | 97 | elif environ['REQUEST_METHOD'] == 'POST': 98 | # it's a client posting its stats to us 99 | body = environ['wsgi.input'].read() 100 | child_status = json.loads(body) 101 | pid = child_status['pid'] 102 | if pid in self.child_events: 103 | self.child_events[pid].send(child_status) 104 | start_response('200 OK', [('content-type', 'application/json')]) 105 | else: 106 | start_response('500 Internal Server Error', 107 | [('content-type', 'text/plain')]) 108 | print "Don't know about child pid %s" % pid 109 | return [""] 110 | 111 | # fallthrough case 112 | start_response('404 Not Found', [('content-type', 'text/plain')]) 113 | return [""] 114 | 115 | def format_timedelta(t): 116 | """Based on how HAProxy's status page shows dates. 117 | 10d 14h 118 | 3h 20m 119 | 1h 0m 120 | 12m 121 | 15s 122 | """ 123 | seconds = t.seconds 124 | if t.days > 0: 125 | return "%sd %sh" % (t.days, int(seconds/3600)) 126 | else: 127 | if seconds > 3600: 128 | hours = int(seconds/3600) 129 | seconds -= hours*3600 130 | return "%sh %sm" % (hours, int(seconds/60)) 131 | else: 132 | if seconds > 60: 133 | return "%sm" % int(seconds/60) 134 | else: 135 | return "%ss" % seconds 136 | 137 | class Tag(object): 138 | """Yeah, there's a templating DSL in this status module. Deal with it.""" 139 | def __init__(self, name, *children, **attrs): 140 | self.name = name 141 | self.attrs = attrs 142 | self.children = list(children) 143 | 144 | def __str__(self): 145 | al = [] 146 | for name, val in self.attrs.iteritems(): 147 | if name == 'cls': 148 | name = "class" 149 | if isinstance(val, (list, tuple)): 150 | val = " ".join(val) 151 | else: 152 | val = str(val) 153 | al.append('%s="%s"' % (name, val)) 154 | if al: 155 | attrstr = " " + " ".join(al) + " " 156 | else: 157 | attrstr = "" 158 | cl = [] 159 | for child in self.children: 160 | cl.append(str(child)) 161 | if cl: 162 | childstr = "\n" + "\n".join(cl) + "\n" 163 | else: 164 | childstr = "" 165 | return "<%s%s>%s" % (self.name, attrstr, childstr, self.name) 166 | 167 | def make_tag(name): 168 | return lambda *c, **a: Tag(name, *c, **a) 169 | p = make_tag('p') 170 | div = make_tag('div') 171 | table = make_tag('table') 172 | tr = make_tag('tr') 173 | th = make_tag('th') 174 | td = make_tag('td') 175 | h2 = make_tag('h2') 176 | span = make_tag('span') 177 | 178 | def fill_template(status_data): 179 | # controller status 180 | cont_div = table(id='controller') 181 | cont_div.children.append(tr(th("PID:", title="Controller Process ID"), 182 | td(status_data['pid']))) 183 | cont_div.children.append(tr(th("Uptime:", title="Time since launch"), 184 | td(status_data['uptime']))) 185 | cont_div.children.append(tr(th("Host:", title="Host and port server is listening on, all means all interfaces."), 186 | td("%s:%s" % (status_data['config']['host'] or "all", 187 | status_data['config']['port'])))) 188 | cont_div.children.append(tr(th("Threads:", title="Threads per child"), 189 | td(status_data['config']['threadpool_workers']))) 190 | cont_div = div(cont_div) 191 | 192 | # children headers and summaries 193 | child_div = div(h2("Child Processes")) 194 | count_td = td(status_data['active_children_count'], "/", 195 | status_data['configured_children_count']) 196 | if status_data['active_children_count'] < \ 197 | status_data['configured_children_count']: 198 | count_td.attrs['cls'] = "error" 199 | count_td.children.append( 200 | span("(", status_data['killed_children_count'], ")")) 201 | children_table = table( 202 | tr( 203 | th('PID', title="Process ID"), 204 | th('Active', title="Accepting New Requests"), 205 | th('Uptime', title="Uptime"), 206 | th('Concurrent', title="Concurrent Requests")), 207 | tr( 208 | td("Total"), 209 | count_td, 210 | td(), # no way to "total" uptime 211 | td(status_data['concurrent_requests'])), 212 | id="children") 213 | child_div.children.append(children_table) 214 | 215 | # children themselves 216 | odd = True 217 | for pid in sorted(status_data['children'].keys()): 218 | child = status_data['children'][pid] 219 | row = tr(td(pid), cls=['child']) 220 | if odd: 221 | row.attrs['cls'].append('odd') 222 | odd = not odd 223 | 224 | # active handling 225 | row.children.append(td({True:'Y', False:'N'}[child['active']])) 226 | if not child['active']: 227 | row.attrs['cls'].append('dying') 228 | 229 | # errors 230 | if child.get('error'): 231 | row.attrs['cls'].append('error') 232 | row.children.append(td(child['error'], colspan=2)) 233 | else: 234 | # no errors 235 | row.children.append(td(child['uptime'])) 236 | row.children.append(td(child['concurrent_requests'])) 237 | 238 | children_table.children.append(row) 239 | 240 | # config dump 241 | config_div = div( 242 | h2("Configuration"), 243 | table(*[tr(th(key), td(status_data['config'][key])) 244 | for key in sorted(status_data['config'].keys())]), 245 | id='config') 246 | 247 | to_format = {'cont_div': cont_div, 'child_div':child_div, 248 | 'config_div':config_div} 249 | to_format.update(status_data) 250 | return HTML_SHELL % to_format 251 | 252 | HTML_SHELL = """ 253 | 254 | 255 | Spawning Status 256 | 331 | 332 |

Spawning Status

333 |
334 |

%(now)s

335 |
336 | Refresh (
337 | every 338 | s 339 |
) 340 |
341 | JSON 342 |
343 | %(cont_div)s 344 | %(child_div)s 345 | %(config_div)s 346 | 347 | 365 | 366 | """ 367 | -------------------------------------------------------------------------------- /spawning/util/system.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010, Steve 'Ashcrow' MIlner 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to 5 | # deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | # FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | """ 23 | Platform related items. 24 | """ 25 | 26 | import os 27 | import platform 28 | import sys 29 | import tempfile 30 | 31 | 32 | class System(dict): 33 | """ 34 | Class to make finding out system information all in one place. 35 | 36 | **Note**: You can not add attributes to an instance of this class. 37 | """ 38 | 39 | def __init__(self): 40 | dict.__init__(self, { 41 | 'architecture': platform.architecture(), 42 | 'max_int': sys.maxint, 43 | 'max_size': sys.maxsize, 44 | 'max_unicode': sys.maxunicode, 45 | 'name': platform.node(), 46 | 'path_seperator': os.path.sep, 47 | 'processor': platform.processor(), 48 | 'python_version': platform.python_version(), 49 | 'python_branch': platform.python_branch(), 50 | 'python_build': platform.python_build(), 51 | 'python_compiler': platform.python_compiler(), 52 | 'python_implementation': platform.python_implementation(), 53 | 'python_revision': platform.python_revision(), 54 | 'python_version_tuple': platform.python_version_tuple(), 55 | 'python_path': sys.path, 56 | 'login': os.getlogin(), 57 | 'system': platform.system(), 58 | 'temp_directory': tempfile.gettempdir(), 59 | 'uname': platform.uname(), 60 | }) 61 | 62 | def __getattr__(self, name): 63 | """ 64 | Looks in the dictionary for items **only**. 65 | 66 | :Parameters: 67 | - 'name': name of the attribute to get. 68 | """ 69 | data = dict(self).get(name) 70 | if data == None: 71 | raise AttributeError("'%s' has no attribute '%s'" % ( 72 | self.__class__.__name__, name)) 73 | return data 74 | 75 | def __setattr__(self, key, value): 76 | """ 77 | Setting attributes is **not** allowed. 78 | 79 | :Parameters: 80 | - `key`: attribute name to set. 81 | - `value`: value to set attribute to. 82 | """ 83 | raise AttributeError("can't set attribute") 84 | 85 | def __repr__(self): 86 | """ 87 | Nice object representation. 88 | """ 89 | return unicode( 90 | "" % ( 91 | self.system, self.name, self.architecture, self.processor)) 92 | 93 | # Method aliases 94 | __str__ = __repr__ 95 | __unicode__ = __repr__ 96 | __setitem__ = __setattr__ 97 | -------------------------------------------------------------------------------- /spawning/wsgi_factory.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2008, Donovan Preston 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to 5 | # deal 6 | # in the Software without restriction, including without limitation the rights 7 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | # copies of the Software, and to permit persons to whom the Software is 9 | # furnished to do so, subject to the following conditions: 10 | 11 | # The above copyright notice and this permission notice shall be included in 12 | # all copies or substantial portions of the Software. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | # FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | # THE SOFTWARE. 22 | 23 | """The config_factory takes a dictionary containing the command line arguments 24 | and should return the same dictionary after modifying any of the settings it wishes. 25 | At the very least the config_factory must set the 'app_factory' key in the returned 26 | argument dictionary, which should be the dotted path to the function which will be 27 | called to actually return the wsgi application which will be served. Also, the 28 | config_factory can look at the 'args' key for any additional positional command-line 29 | arguments that were passed to spawn, and modify the configuration dictionary 30 | based on it's contents. 31 | 32 | Return value of config_factory should be a dict containing: 33 | app_factory: The dotted path to the wsgi application factory. 34 | Will be called with the result of factory_qual as the argument. 35 | host: The local ip to bind to. 36 | port: The local port to bind to. 37 | num_processes: The number of processes to spawn. 38 | num_threads: The number of threads to use in the threadpool in each process. 39 | If 0, install the eventlet monkeypatching and do not use the threadpool. 40 | Code which blocks instead of cooperating will block the process, possibly 41 | causing stalls. (TODO sigalrm?) 42 | dev: If True, watch all files in sys.modules, easy-install.pth, and any additional 43 | file paths in the 'watch' list for changes and restart child 44 | processes on change. If False, only reload if the svn revision of the 45 | current directory changes. 46 | watch: List of additional files to watch for changes and reload when changed. 47 | """ 48 | import inspect 49 | import os 50 | import time 51 | 52 | import spawning.util 53 | 54 | def config_factory(args): 55 | args['app_factory'] = 'spawning.wsgi_factory.app_factory' 56 | args['app'] = args['args'][0] 57 | args['middleware'] = args['args'][1:] 58 | 59 | args['source_directories'] = [os.path.split( 60 | inspect.getfile( 61 | inspect.getmodule( 62 | spawning.util.named(args['app']))))[0]] 63 | return args 64 | 65 | 66 | def app_factory(config): 67 | app = spawning.util.named(config['app']) 68 | for mid in config['middleware']: 69 | app = spawning.util.named(mid)(app) 70 | return app 71 | 72 | 73 | def hello_world(env, start_response): 74 | start_response('200 OK', [('Content-type', 'text/plain')]) 75 | return ['Hello, World!\r\n'] 76 | 77 | 78 | def really_long(env, start_response): 79 | start_response('200 OK', [('Content-type', 'text/plain')]) 80 | time.sleep(180) 81 | return ['Goodbye, World!\r\n'] 82 | 83 | --------------------------------------------------------------------------------