├── MANIFEST.in ├── .gitignore ├── setup.py ├── README.md ├── lib └── gevent_selfpipe │ └── __init__.py └── distribute_setup.py /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include distribute_setup.py 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.pyc 3 | *.pyo 4 | .DS_Store 5 | build 6 | dist 7 | MANIFEST 8 | test/example/*.sqlite3 9 | doc/.build 10 | distribute-*.egg 11 | distribute-*.tar.gz 12 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from distribute_setup import use_setuptools 2 | use_setuptools() 3 | 4 | from setuptools import setup, find_packages 5 | 6 | 7 | setup( 8 | name='gevent-selfpipe', 9 | version='0.0.3', 10 | description='Hack to enable gevent-based scheduling of blocking calls.', 11 | author='Zachary Voase', 12 | author_email='z@zacharyvoase.com', 13 | url='http://github.com/zacharyvoase/gevent-selfpipe', 14 | packages=find_packages(where='lib'), 15 | package_dir={'': 'lib'}, 16 | install_requires=[ 17 | 'gevent>=0.13.6', 18 | ], 19 | ) 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gevent-selfpipe 2 | 3 | A simple hack to allow blocking calls to be made in Python background threads 4 | but with gevent-based scheduling (so that other greenlets can run while the 5 | call completes). It does this using the [self-pipe trick][], normally used for 6 | avoiding race conditions in signal handling but surprisingly nifty in this 7 | circumstance. 8 | 9 | [self-pipe trick]: http://cr.yp.to/docs/selfpipe.html 10 | 11 | 12 | ## Example 13 | 14 | ```pycon 15 | >>> import time 16 | >>> import gevent 17 | >>> from gevent_selfpipe import selfpipe 18 | >>> def print_something(): 19 | ... print "If this message appears it means the main thread yielded." 20 | >>> gl = gevent.spawn(print_something) 21 | >>> time.sleep(1) # Waits 1 second, nothing printed. 22 | >>> selfpipe(time.sleep, 1) # Waits 1 second. 23 | If this message appears it means the main thread yielded. 24 | ``` 25 | 26 | 27 | ## Installation 28 | 29 | pip install gevent-selfpipe 30 | 31 | 32 | ## (Un)license 33 | 34 | This is free and unencumbered software released into the public domain. 35 | 36 | Anyone is free to copy, modify, publish, use, compile, sell, or distribute this 37 | software, either in source code form or as a compiled binary, for any purpose, 38 | commercial or non-commercial, and by any means. 39 | 40 | In jurisdictions that recognize copyright laws, the author or authors of this 41 | software dedicate any and all copyright interest in the software to the public 42 | domain. We make this dedication for the benefit of the public at large and to 43 | the detriment of our heirs and successors. We intend this dedication to be an 44 | overt act of relinquishment in perpetuity of all present and future rights to 45 | this software under copyright law. 46 | 47 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 48 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 49 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 50 | AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 51 | ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 52 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 53 | 54 | For more information, please refer to 55 | -------------------------------------------------------------------------------- /lib/gevent_selfpipe/__init__.py: -------------------------------------------------------------------------------- 1 | """Self-pipe trick utilities for greenlet.""" 2 | 3 | from functools import wraps 4 | import os 5 | import sys 6 | import threading 7 | 8 | import gevent.socket 9 | 10 | 11 | def selfpipe(func, *args, **kwargs): 12 | 13 | """ 14 | Run a thread using the self-pipe trick for gevent synchronization. 15 | 16 | Runs the provided function, with arguments, in a separate thread, using 17 | the self-pipe trick to synchronize the completion of that thread with the 18 | gevent loop in the main thread. Any value returned or exception raised by 19 | the provided function will propagate. 20 | 21 | You can call this function directly to co-operatively yield while making a 22 | normally-blocking call, or use ``gevent.spawn()`` with it to run a 23 | traditional blocking function as a background greenlet. 24 | 25 | Note that the called function should *not* use gevent in any way (gevent 26 | raises an exception if used from more than one thread), and if you want 27 | good performance characteristics, it should probably be a GIL-releasing 28 | C or Cython function. 29 | """ 30 | 31 | read_fd, write_fd = os.pipe() 32 | 33 | result_container = [] 34 | 35 | def runner(): 36 | try: 37 | result = func(*args, **kwargs) 38 | result_container.append((True, result)) 39 | except Exception, exc: 40 | result_container.append((False,) + sys.exc_info()) 41 | finally: 42 | os.write(write_fd, '\x00') 43 | 44 | thread = threading.Thread(target=runner) 45 | thread.start() 46 | 47 | try: 48 | gevent.socket.wait_read(read_fd) 49 | os.read(read_fd, 1) 50 | finally: 51 | thread.join() 52 | os.close(read_fd) 53 | os.close(write_fd) 54 | 55 | success, value = result_container[0][:2] 56 | if success: 57 | return value 58 | raise value 59 | 60 | 61 | def selfpiped(function): 62 | 63 | """ 64 | A decorator to declare a function as always using the self-pipe trick. 65 | 66 | This means the decorated function will always be run in a background thread 67 | with self-pipe gevent synchronization. 68 | 69 | An example using ZeroMQ: 70 | 71 | >>> import zmq 72 | >>> @selfpiped 73 | ... def device(d_type, isock, osock): 74 | ... try: 75 | ... zmq.device(d_type, isock, osock) 76 | ... finally: 77 | ... isock.close(); osock.close() 78 | >>> dev_gl = gevent.spawn(device, zmq.STREAMER, isock, osock) 79 | """ 80 | 81 | @wraps(function) 82 | def wrapper(*args, **kwargs): 83 | return selfpipe(function, *args, **kwargs) 84 | return wrapper 85 | -------------------------------------------------------------------------------- /distribute_setup.py: -------------------------------------------------------------------------------- 1 | #!python 2 | """Bootstrap distribute installation 3 | 4 | If you want to use setuptools in your package's setup.py, just include this 5 | file in the same directory with it, and add this to the top of your setup.py:: 6 | 7 | from distribute_setup import use_setuptools 8 | use_setuptools() 9 | 10 | If you want to require a specific version of setuptools, set a download 11 | mirror, or use an alternate download directory, you can do so by supplying 12 | the appropriate options to ``use_setuptools()``. 13 | 14 | This file can also be run as a script to install or upgrade setuptools. 15 | """ 16 | import os 17 | import sys 18 | import time 19 | import fnmatch 20 | import tempfile 21 | import tarfile 22 | from distutils import log 23 | 24 | try: 25 | from site import USER_SITE 26 | except ImportError: 27 | USER_SITE = None 28 | 29 | try: 30 | import subprocess 31 | 32 | def _python_cmd(*args): 33 | args = (sys.executable,) + args 34 | return subprocess.call(args) == 0 35 | 36 | except ImportError: 37 | # will be used for python 2.3 38 | def _python_cmd(*args): 39 | args = (sys.executable,) + args 40 | # quoting arguments if windows 41 | if sys.platform == 'win32': 42 | def quote(arg): 43 | if ' ' in arg: 44 | return '"%s"' % arg 45 | return arg 46 | args = [quote(arg) for arg in args] 47 | return os.spawnl(os.P_WAIT, sys.executable, *args) == 0 48 | 49 | DEFAULT_VERSION = "0.6.24" 50 | DEFAULT_URL = "http://pypi.python.org/packages/source/d/distribute/" 51 | SETUPTOOLS_FAKED_VERSION = "0.6c11" 52 | 53 | SETUPTOOLS_PKG_INFO = """\ 54 | Metadata-Version: 1.0 55 | Name: setuptools 56 | Version: %s 57 | Summary: xxxx 58 | Home-page: xxx 59 | Author: xxx 60 | Author-email: xxx 61 | License: xxx 62 | Description: xxx 63 | """ % SETUPTOOLS_FAKED_VERSION 64 | 65 | 66 | def _install(tarball): 67 | # extracting the tarball 68 | tmpdir = tempfile.mkdtemp() 69 | log.warn('Extracting in %s', tmpdir) 70 | old_wd = os.getcwd() 71 | try: 72 | os.chdir(tmpdir) 73 | tar = tarfile.open(tarball) 74 | _extractall(tar) 75 | tar.close() 76 | 77 | # going in the directory 78 | subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) 79 | os.chdir(subdir) 80 | log.warn('Now working in %s', subdir) 81 | 82 | # installing 83 | log.warn('Installing Distribute') 84 | if not _python_cmd('setup.py', 'install'): 85 | log.warn('Something went wrong during the installation.') 86 | log.warn('See the error message above.') 87 | finally: 88 | os.chdir(old_wd) 89 | 90 | 91 | def _build_egg(egg, tarball, to_dir): 92 | # extracting the tarball 93 | tmpdir = tempfile.mkdtemp() 94 | log.warn('Extracting in %s', tmpdir) 95 | old_wd = os.getcwd() 96 | try: 97 | os.chdir(tmpdir) 98 | tar = tarfile.open(tarball) 99 | _extractall(tar) 100 | tar.close() 101 | 102 | # going in the directory 103 | subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) 104 | os.chdir(subdir) 105 | log.warn('Now working in %s', subdir) 106 | 107 | # building an egg 108 | log.warn('Building a Distribute egg in %s', to_dir) 109 | _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) 110 | 111 | finally: 112 | os.chdir(old_wd) 113 | # returning the result 114 | log.warn(egg) 115 | if not os.path.exists(egg): 116 | raise IOError('Could not build the egg.') 117 | 118 | 119 | def _do_download(version, download_base, to_dir, download_delay): 120 | egg = os.path.join(to_dir, 'distribute-%s-py%d.%d.egg' 121 | % (version, sys.version_info[0], sys.version_info[1])) 122 | if not os.path.exists(egg): 123 | tarball = download_setuptools(version, download_base, 124 | to_dir, download_delay) 125 | _build_egg(egg, tarball, to_dir) 126 | sys.path.insert(0, egg) 127 | import setuptools 128 | setuptools.bootstrap_install_from = egg 129 | 130 | 131 | def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, 132 | to_dir=os.curdir, download_delay=15, no_fake=True): 133 | # making sure we use the absolute path 134 | to_dir = os.path.abspath(to_dir) 135 | was_imported = 'pkg_resources' in sys.modules or \ 136 | 'setuptools' in sys.modules 137 | try: 138 | try: 139 | import pkg_resources 140 | if not hasattr(pkg_resources, '_distribute'): 141 | if not no_fake: 142 | _fake_setuptools() 143 | raise ImportError 144 | except ImportError: 145 | return _do_download(version, download_base, to_dir, download_delay) 146 | try: 147 | pkg_resources.require("distribute>="+version) 148 | return 149 | except pkg_resources.VersionConflict: 150 | e = sys.exc_info()[1] 151 | if was_imported: 152 | sys.stderr.write( 153 | "The required version of distribute (>=%s) is not available,\n" 154 | "and can't be installed while this script is running. Please\n" 155 | "install a more recent version first, using\n" 156 | "'easy_install -U distribute'." 157 | "\n\n(Currently using %r)\n" % (version, e.args[0])) 158 | sys.exit(2) 159 | else: 160 | del pkg_resources, sys.modules['pkg_resources'] # reload ok 161 | return _do_download(version, download_base, to_dir, 162 | download_delay) 163 | except pkg_resources.DistributionNotFound: 164 | return _do_download(version, download_base, to_dir, 165 | download_delay) 166 | finally: 167 | if not no_fake: 168 | _create_fake_setuptools_pkg_info(to_dir) 169 | 170 | def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, 171 | to_dir=os.curdir, delay=15): 172 | """Download distribute from a specified location and return its filename 173 | 174 | `version` should be a valid distribute version number that is available 175 | as an egg for download under the `download_base` URL (which should end 176 | with a '/'). `to_dir` is the directory where the egg will be downloaded. 177 | `delay` is the number of seconds to pause before an actual download 178 | attempt. 179 | """ 180 | # making sure we use the absolute path 181 | to_dir = os.path.abspath(to_dir) 182 | try: 183 | from urllib.request import urlopen 184 | except ImportError: 185 | from urllib2 import urlopen 186 | tgz_name = "distribute-%s.tar.gz" % version 187 | url = download_base + tgz_name 188 | saveto = os.path.join(to_dir, tgz_name) 189 | src = dst = None 190 | if not os.path.exists(saveto): # Avoid repeated downloads 191 | try: 192 | log.warn("Downloading %s", url) 193 | src = urlopen(url) 194 | # Read/write all in one block, so we don't create a corrupt file 195 | # if the download is interrupted. 196 | data = src.read() 197 | dst = open(saveto, "wb") 198 | dst.write(data) 199 | finally: 200 | if src: 201 | src.close() 202 | if dst: 203 | dst.close() 204 | return os.path.realpath(saveto) 205 | 206 | def _no_sandbox(function): 207 | def __no_sandbox(*args, **kw): 208 | try: 209 | from setuptools.sandbox import DirectorySandbox 210 | if not hasattr(DirectorySandbox, '_old'): 211 | def violation(*args): 212 | pass 213 | DirectorySandbox._old = DirectorySandbox._violation 214 | DirectorySandbox._violation = violation 215 | patched = True 216 | else: 217 | patched = False 218 | except ImportError: 219 | patched = False 220 | 221 | try: 222 | return function(*args, **kw) 223 | finally: 224 | if patched: 225 | DirectorySandbox._violation = DirectorySandbox._old 226 | del DirectorySandbox._old 227 | 228 | return __no_sandbox 229 | 230 | def _patch_file(path, content): 231 | """Will backup the file then patch it""" 232 | existing_content = open(path).read() 233 | if existing_content == content: 234 | # already patched 235 | log.warn('Already patched.') 236 | return False 237 | log.warn('Patching...') 238 | _rename_path(path) 239 | f = open(path, 'w') 240 | try: 241 | f.write(content) 242 | finally: 243 | f.close() 244 | return True 245 | 246 | _patch_file = _no_sandbox(_patch_file) 247 | 248 | def _same_content(path, content): 249 | return open(path).read() == content 250 | 251 | def _rename_path(path): 252 | new_name = path + '.OLD.%s' % time.time() 253 | log.warn('Renaming %s into %s', path, new_name) 254 | os.rename(path, new_name) 255 | return new_name 256 | 257 | def _remove_flat_installation(placeholder): 258 | if not os.path.isdir(placeholder): 259 | log.warn('Unkown installation at %s', placeholder) 260 | return False 261 | found = False 262 | for file in os.listdir(placeholder): 263 | if fnmatch.fnmatch(file, 'setuptools*.egg-info'): 264 | found = True 265 | break 266 | if not found: 267 | log.warn('Could not locate setuptools*.egg-info') 268 | return 269 | 270 | log.warn('Removing elements out of the way...') 271 | pkg_info = os.path.join(placeholder, file) 272 | if os.path.isdir(pkg_info): 273 | patched = _patch_egg_dir(pkg_info) 274 | else: 275 | patched = _patch_file(pkg_info, SETUPTOOLS_PKG_INFO) 276 | 277 | if not patched: 278 | log.warn('%s already patched.', pkg_info) 279 | return False 280 | # now let's move the files out of the way 281 | for element in ('setuptools', 'pkg_resources.py', 'site.py'): 282 | element = os.path.join(placeholder, element) 283 | if os.path.exists(element): 284 | _rename_path(element) 285 | else: 286 | log.warn('Could not find the %s element of the ' 287 | 'Setuptools distribution', element) 288 | return True 289 | 290 | _remove_flat_installation = _no_sandbox(_remove_flat_installation) 291 | 292 | def _after_install(dist): 293 | log.warn('After install bootstrap.') 294 | placeholder = dist.get_command_obj('install').install_purelib 295 | _create_fake_setuptools_pkg_info(placeholder) 296 | 297 | def _create_fake_setuptools_pkg_info(placeholder): 298 | if not placeholder or not os.path.exists(placeholder): 299 | log.warn('Could not find the install location') 300 | return 301 | pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1]) 302 | setuptools_file = 'setuptools-%s-py%s.egg-info' % \ 303 | (SETUPTOOLS_FAKED_VERSION, pyver) 304 | pkg_info = os.path.join(placeholder, setuptools_file) 305 | if os.path.exists(pkg_info): 306 | log.warn('%s already exists', pkg_info) 307 | return 308 | 309 | log.warn('Creating %s', pkg_info) 310 | f = open(pkg_info, 'w') 311 | try: 312 | f.write(SETUPTOOLS_PKG_INFO) 313 | finally: 314 | f.close() 315 | 316 | pth_file = os.path.join(placeholder, 'setuptools.pth') 317 | log.warn('Creating %s', pth_file) 318 | f = open(pth_file, 'w') 319 | try: 320 | f.write(os.path.join(os.curdir, setuptools_file)) 321 | finally: 322 | f.close() 323 | 324 | _create_fake_setuptools_pkg_info = _no_sandbox(_create_fake_setuptools_pkg_info) 325 | 326 | def _patch_egg_dir(path): 327 | # let's check if it's already patched 328 | pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') 329 | if os.path.exists(pkg_info): 330 | if _same_content(pkg_info, SETUPTOOLS_PKG_INFO): 331 | log.warn('%s already patched.', pkg_info) 332 | return False 333 | _rename_path(path) 334 | os.mkdir(path) 335 | os.mkdir(os.path.join(path, 'EGG-INFO')) 336 | pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') 337 | f = open(pkg_info, 'w') 338 | try: 339 | f.write(SETUPTOOLS_PKG_INFO) 340 | finally: 341 | f.close() 342 | return True 343 | 344 | _patch_egg_dir = _no_sandbox(_patch_egg_dir) 345 | 346 | def _before_install(): 347 | log.warn('Before install bootstrap.') 348 | _fake_setuptools() 349 | 350 | 351 | def _under_prefix(location): 352 | if 'install' not in sys.argv: 353 | return True 354 | args = sys.argv[sys.argv.index('install')+1:] 355 | for index, arg in enumerate(args): 356 | for option in ('--root', '--prefix'): 357 | if arg.startswith('%s=' % option): 358 | top_dir = arg.split('root=')[-1] 359 | return location.startswith(top_dir) 360 | elif arg == option: 361 | if len(args) > index: 362 | top_dir = args[index+1] 363 | return location.startswith(top_dir) 364 | if arg == '--user' and USER_SITE is not None: 365 | return location.startswith(USER_SITE) 366 | return True 367 | 368 | 369 | def _fake_setuptools(): 370 | log.warn('Scanning installed packages') 371 | try: 372 | import pkg_resources 373 | except ImportError: 374 | # we're cool 375 | log.warn('Setuptools or Distribute does not seem to be installed.') 376 | return 377 | ws = pkg_resources.working_set 378 | try: 379 | setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools', 380 | replacement=False)) 381 | except TypeError: 382 | # old distribute API 383 | setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools')) 384 | 385 | if setuptools_dist is None: 386 | log.warn('No setuptools distribution found') 387 | return 388 | # detecting if it was already faked 389 | setuptools_location = setuptools_dist.location 390 | log.warn('Setuptools installation detected at %s', setuptools_location) 391 | 392 | # if --root or --preix was provided, and if 393 | # setuptools is not located in them, we don't patch it 394 | if not _under_prefix(setuptools_location): 395 | log.warn('Not patching, --root or --prefix is installing Distribute' 396 | ' in another location') 397 | return 398 | 399 | # let's see if its an egg 400 | if not setuptools_location.endswith('.egg'): 401 | log.warn('Non-egg installation') 402 | res = _remove_flat_installation(setuptools_location) 403 | if not res: 404 | return 405 | else: 406 | log.warn('Egg installation') 407 | pkg_info = os.path.join(setuptools_location, 'EGG-INFO', 'PKG-INFO') 408 | if (os.path.exists(pkg_info) and 409 | _same_content(pkg_info, SETUPTOOLS_PKG_INFO)): 410 | log.warn('Already patched.') 411 | return 412 | log.warn('Patching...') 413 | # let's create a fake egg replacing setuptools one 414 | res = _patch_egg_dir(setuptools_location) 415 | if not res: 416 | return 417 | log.warn('Patched done.') 418 | _relaunch() 419 | 420 | 421 | def _relaunch(): 422 | log.warn('Relaunching...') 423 | # we have to relaunch the process 424 | # pip marker to avoid a relaunch bug 425 | if sys.argv[:3] == ['-c', 'install', '--single-version-externally-managed']: 426 | sys.argv[0] = 'setup.py' 427 | args = [sys.executable] + sys.argv 428 | sys.exit(subprocess.call(args)) 429 | 430 | 431 | def _extractall(self, path=".", members=None): 432 | """Extract all members from the archive to the current working 433 | directory and set owner, modification time and permissions on 434 | directories afterwards. `path' specifies a different directory 435 | to extract to. `members' is optional and must be a subset of the 436 | list returned by getmembers(). 437 | """ 438 | import copy 439 | import operator 440 | from tarfile import ExtractError 441 | directories = [] 442 | 443 | if members is None: 444 | members = self 445 | 446 | for tarinfo in members: 447 | if tarinfo.isdir(): 448 | # Extract directories with a safe mode. 449 | directories.append(tarinfo) 450 | tarinfo = copy.copy(tarinfo) 451 | tarinfo.mode = 448 # decimal for oct 0700 452 | self.extract(tarinfo, path) 453 | 454 | # Reverse sort directories. 455 | if sys.version_info < (2, 4): 456 | def sorter(dir1, dir2): 457 | return cmp(dir1.name, dir2.name) 458 | directories.sort(sorter) 459 | directories.reverse() 460 | else: 461 | directories.sort(key=operator.attrgetter('name'), reverse=True) 462 | 463 | # Set correct owner, mtime and filemode on directories. 464 | for tarinfo in directories: 465 | dirpath = os.path.join(path, tarinfo.name) 466 | try: 467 | self.chown(tarinfo, dirpath) 468 | self.utime(tarinfo, dirpath) 469 | self.chmod(tarinfo, dirpath) 470 | except ExtractError: 471 | e = sys.exc_info()[1] 472 | if self.errorlevel > 1: 473 | raise 474 | else: 475 | self._dbg(1, "tarfile: %s" % e) 476 | 477 | 478 | def main(argv, version=DEFAULT_VERSION): 479 | """Install or upgrade setuptools and EasyInstall""" 480 | tarball = download_setuptools() 481 | _install(tarball) 482 | 483 | 484 | if __name__ == '__main__': 485 | main(sys.argv[1:]) 486 | --------------------------------------------------------------------------------