├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.rst ├── media ├── logo-small.png └── logo.png ├── proclib ├── __init__.py ├── api.py ├── helpers.py ├── pipe.py ├── process.py └── response.py ├── setup.py └── tests ├── __init__.py ├── test_api.py ├── test_helpers.py ├── test_pipe.py └── test_process.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: python 3 | python: 4 | - "2.7" 5 | - "3.2" 6 | - "3.3" 7 | - "3.4" 8 | - "pypy" 9 | - "pypy3" 10 | install: pip install . 11 | script: py.test -v tests 12 | notifications: 13 | email: 14 | recepients: 15 | - packwolf58@gmail.com 16 | on_success: never 17 | on_failure: always 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Eeo Jun 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst 2 | include LICENSE 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://raw.githubusercontent.com/datalib/proclib/master/media/logo-small.png 2 | 3 | Proclib is a high level wrapper/abstraction around the standard 4 | library subprocess module, written in Python, with proper piping 5 | support which aims to simplify the usage of Unix utilities right 6 | from Python and help the developer focus on the commands and not 7 | the code which calls the commands. 8 | 9 | .. image:: https://travis-ci.org/datalib/proclib.svg?branch=master 10 | :target: https://travis-ci.org/datalib/proclib 11 | 12 | Overview 13 | -------- 14 | 15 | `proclib.api.spawn(cmd)` 16 | Given a string or list making up commands *cmd*, return 17 | a Response object which is the result of piping the commands, 18 | i.e. they are run in *parallel*. The ``data`` parameter can be 19 | used to configure the data passed in to the initial process. 20 | Usage example:: 21 | 22 | >>> from proclib.api import spawn 23 | >>> r = spawn('yes | head') 24 | >>> r.stdout.read() 25 | 'y\ny\ny\ny\ny\ny\ny\ny\ny\ny\n' 26 | >>> r.close() 27 | >>> r.history[0].explain_signal() 28 | {'action': 'kill', 29 | 'description': 'write on a pipe with no readers', 30 | 'id': 13, 31 | 'signal': 'SIGPIPE'} 32 | 33 | Streaming support is built-in- that is that the stdout of 34 | any process can be streamed lazily instead of read and stored 35 | in memory all in one go. Also, any kind of iterable can be 36 | piped to the process:: 37 | 38 | def gen(): 39 | yield 'hi\n' 40 | yield 'ho\n' 41 | 42 | r = spawn('cat', data=gen()) 43 | assert r.out.split() == ['hi', 'ho'] 44 | -------------------------------------------------------------------------------- /media/logo-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalib/proclib/12268ee3d2b0b590cb7cdaff5828a5099612bf66/media/logo-small.png -------------------------------------------------------------------------------- /media/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalib/proclib/12268ee3d2b0b590cb7cdaff5828a5099612bf66/media/logo.png -------------------------------------------------------------------------------- /proclib/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | proclib 3 | ~~~~~~~ 4 | 5 | Simple high level convenience wrapper over the 6 | stdlib subprocess module. 7 | 8 | :copyright: (c) 2015 Eeo Jun 9 | :license: MIT, see LICENSE for details. 10 | """ 11 | 12 | 13 | __version__ = '0.2.0' 14 | -------------------------------------------------------------------------------- /proclib/api.py: -------------------------------------------------------------------------------- 1 | """ 2 | proclib.api 3 | ~~~~~~~~~~~ 4 | Module that exposes the public, easy-to-use 5 | functional wrappers. 6 | """ 7 | 8 | from .helpers import str_parse, list_parse 9 | from .pipe import Pipe 10 | 11 | __all__ = ('spawn',) 12 | 13 | 14 | def parse(cmds): 15 | """ 16 | Given a string or list of lists/strings *cmds*, 17 | determine and use the correct parser to use and 18 | return the results as a list. 19 | :param cmds: List/String of commands. 20 | """ 21 | parser = str_parse if isinstance(cmds, str) else list_parse 22 | return list(parser(cmds)) 23 | 24 | 25 | def spawn(cmd, data=(), env=None, cwd=None): 26 | """ 27 | Given a string or list making up commands *cmd*, 28 | return a response object which is the result of 29 | piping the commands in parallel. Optionally, pass 30 | in some *data* in the form of an iterable, file 31 | object, or string to the first process. 32 | 33 | :param cmd: String/List of commands. 34 | :param data: Data to be passed to the first command. 35 | :param env: Override environment variables. 36 | :param cwd: Override working directory. 37 | """ 38 | if isinstance(data, str): 39 | data = [data] 40 | 41 | pipe = Pipe( 42 | commands=parse(cmd), 43 | data=data, 44 | env=env, 45 | cwd=cwd, 46 | ) 47 | return pipe.run() 48 | -------------------------------------------------------------------------------- /proclib/helpers.py: -------------------------------------------------------------------------------- 1 | """ 2 | proclib.helpers 3 | ~~~~~~~~~~~~~~~ 4 | 5 | Helper utility functions. 6 | """ 7 | 8 | import shlex 9 | import signal 10 | 11 | 12 | TO_RESTORE = tuple( 13 | getattr(signal, sig) for sig in ('SIGPIPE', 'SIGXFZ', 'SIGXFSZ') 14 | if hasattr(signal, sig) 15 | ) 16 | 17 | 18 | def restore_signals(signals=TO_RESTORE): 19 | """ 20 | Restores signals before the process is 21 | executed so that they can be terminated 22 | with SIGPIPE. 23 | 24 | :param signals: Optimization detail 25 | that defaults to integers corresponding 26 | to SIGPIPE, SIGXFZ, and SIGXFSZ (if 27 | they are available). 28 | """ 29 | for sig in signals: 30 | signal.signal(sig, signal.SIG_DFL) 31 | 32 | 33 | def str_parse(cmds, pipe_operator='|'): 34 | """ 35 | Given a string of commands *cmds* yield the 36 | command in chunks, separated by the *pipe_operator* 37 | defaulting to '|'. 38 | 39 | :param cmds: String of commands. 40 | :param pipe_operator: The pipe operator. 41 | """ 42 | buff = [] 43 | for item in shlex.split(cmds): 44 | if item == pipe_operator: 45 | yield buff 46 | buff = [] 47 | continue 48 | buff.append(item) 49 | if buff: 50 | yield buff 51 | 52 | 53 | def list_parse(cmds): 54 | """ 55 | Given a list of commands, if they are a 56 | string then parse them, else yield them 57 | as if they were already correctly formatted. 58 | 59 | :param cmds: List of commands. 60 | """ 61 | for item in cmds: 62 | if isinstance(item, str): 63 | for item in str_parse(item): 64 | yield item 65 | continue 66 | yield list(item) 67 | 68 | 69 | class cached_property(object): 70 | """ 71 | Property that is computed only once during 72 | the lifetime of an object, i.e. the second 73 | time the attribute is looked up there is 74 | zero cost overhead. 75 | 76 | :param func: Function to wrap over. 77 | """ 78 | 79 | def __init__(self, func): 80 | self.getter = func 81 | self.__name__ = func.__name__ 82 | self.__doc__ = func.__doc__ 83 | 84 | def __get__(self, instance, objtype=None): 85 | if instance is None: 86 | return self 87 | res = instance.__dict__[self.__name__] = self.getter(instance) 88 | return res 89 | -------------------------------------------------------------------------------- /proclib/pipe.py: -------------------------------------------------------------------------------- 1 | """ 2 | proclib.pipe 3 | ~~~~~~~~~~~~ 4 | 5 | Implements the Pipe object. 6 | """ 7 | 8 | from subprocess import PIPE 9 | from .process import Process 10 | 11 | 12 | class Pipe(object): 13 | """ 14 | A Pipe object represents and starts the parallel 15 | execution and piping of multiple Processes. 16 | 17 | :param commands: A list of commands. 18 | :param data: Data to be piped in to the first process. 19 | :param opts: Extra options to be passed to every 20 | spawned process. 21 | """ 22 | 23 | process_class = Process 24 | 25 | def __init__(self, commands, data=(), **opts): 26 | self.commands = commands 27 | self.data = data 28 | self.opts = opts 29 | 30 | def spawn_procs(self): 31 | """ 32 | Return a list of processes that have had their 33 | file handles configured the correct order. 34 | """ 35 | stdout = PIPE 36 | for item in self.commands: 37 | proc = self.process_class( 38 | command=item, 39 | stdin=stdout, 40 | **self.opts 41 | ) 42 | stdout = proc.process.stdout 43 | yield proc 44 | 45 | @staticmethod 46 | def make_response(procs): 47 | """ 48 | Given an iterable of processes *procs*, run all of 49 | them and pop the last one, returning it as the 50 | result of running all of them in a pipe. 51 | 52 | :param procs: An iterable of processes. 53 | """ 54 | history = [p.run() for p in procs] 55 | r = history.pop() 56 | r.history = history 57 | 58 | for res in r.history: 59 | res.stdout.close() 60 | 61 | return r 62 | 63 | def run(self): 64 | """ 65 | Runs the processes. Internally this calls the 66 | ``spawn_procs`` method but converts them into 67 | responses via their ``run`` method and the 68 | ``make_response`` method. 69 | """ 70 | procs = list(self.spawn_procs()) 71 | procs[0].pipe(self.data) 72 | return self.make_response(procs) 73 | -------------------------------------------------------------------------------- /proclib/process.py: -------------------------------------------------------------------------------- 1 | """ 2 | proclib.process 3 | ~~~~~~~~~~~~~~~ 4 | 5 | Implements the Process class. 6 | """ 7 | 8 | from subprocess import Popen, PIPE 9 | from .response import Response 10 | from .helpers import restore_signals 11 | 12 | 13 | class Process(object): 14 | """ 15 | A Process object is a wrapper around a regular 16 | `subprocess.Popen` instance that knows how to 17 | call it with the correct arguments. 18 | 19 | :param command: The command. 20 | :param opts: Keyword argument of additional 21 | options to be pased to the `subprocess.Popen` 22 | constructor. 23 | """ 24 | 25 | response_class = Response 26 | defaults = dict(close_fds=True, 27 | universal_newlines=True, 28 | preexec_fn=restore_signals, 29 | stdout=PIPE, 30 | stderr=PIPE, 31 | stdin=PIPE) 32 | 33 | def __init__(self, command, **opts): 34 | conf = self.defaults.copy() 35 | conf.update(opts) 36 | 37 | self.command = command 38 | self.process = Popen(args=command, **conf) 39 | 40 | def pipe(self, lines): 41 | """ 42 | Given a Process with a valid stdin (not a 43 | file-handle that is not PIPE), write *lines* 44 | of data to the process where *lines* can 45 | be any iterable. 46 | 47 | :param lines: Iterable of data to be piped 48 | in to the process. 49 | """ 50 | with self.process.stdin as stdin: 51 | for line in lines: 52 | stdin.write(line) 53 | 54 | def run(self): 55 | """ 56 | Wraps the Popen instance in a ``Response`` 57 | object. 58 | """ 59 | return self.response_class( 60 | self.command, 61 | self.process, 62 | ) 63 | -------------------------------------------------------------------------------- /proclib/response.py: -------------------------------------------------------------------------------- 1 | """ 2 | proclib.response 3 | ~~~~~~~~~~~~~~~~ 4 | Implements the Response class. 5 | """ 6 | 7 | from .helpers import cached_property 8 | from signalsdb.api import explain 9 | import warnings 10 | 11 | 12 | class Response(object): 13 | """ 14 | A Response object represents the result of running 15 | a process. The parameters supplied are stored and 16 | available as attributes as well. 17 | 18 | :param command: A command in the form of a list. 19 | :param process: A `subprocess.Popen` object. 20 | """ 21 | 22 | def __init__(self, command, process): 23 | self.history = [] 24 | self.command = command 25 | self.process = process 26 | 27 | self.pid = process.pid 28 | self.stdout = process.stdout 29 | self.stderr = process.stderr 30 | 31 | def __iter__(self): 32 | for line in self.stdout: 33 | yield line 34 | 35 | @cached_property 36 | def out(self): 37 | """ 38 | Reads the entire stdout of the Popen instance 39 | and then returns it. Not recommended for long 40 | running processes. 41 | """ 42 | with self.stdout: 43 | return self.stdout.read() 44 | 45 | @cached_property 46 | def err(self): 47 | """ 48 | Similar to ``out``, reads the entirety of 49 | ``stderr``. 50 | """ 51 | with self.stderr: 52 | return self.stderr.read() 53 | 54 | @property 55 | def status_code(self): 56 | """ 57 | Returns the exit code of the process, None 58 | if the process hasn't completed. 59 | """ 60 | return self.process.poll() 61 | 62 | @property 63 | def finished(self): 64 | """ 65 | Returns a boolean stating if the process has 66 | completed or not. Internally this calls 67 | `status_code` to determine the exit code 68 | of the process. 69 | """ 70 | return self.status_code is not None 71 | 72 | def close(self): 73 | """ 74 | If possible, close the stdout and stderr file 75 | handles. 76 | """ 77 | if self.stdout: self.stdout.close() 78 | if self.stderr: self.stderr.close() 79 | 80 | @property 81 | def ok(self): 82 | """ 83 | Returns a boolean depending on whether the 84 | `returncode` attribute equals zero. 85 | """ 86 | return self.status_code == 0 87 | 88 | def wait(self): 89 | """ 90 | Block until the process to complete. 91 | """ 92 | self.process.wait() 93 | 94 | def terminate(self): 95 | """ 96 | Terminate the process if it is not finished. 97 | """ 98 | if not self.finished: 99 | self.process.terminate() 100 | 101 | def __repr__(self): 102 | return '' % self.command[0] 103 | 104 | def __enter__(self): 105 | """ 106 | Similar to a file object, will return the 107 | Response object and then will safely close 108 | all open file handles when it exits. 109 | """ 110 | return self 111 | 112 | def __exit__(self, *_): 113 | self.close() 114 | 115 | def explain_signal(self): 116 | """ 117 | Explains (provides the name, the description, 118 | and the default action of) the signal that 119 | killed the process. 120 | """ 121 | status = self.status_code 122 | if status and status < 0: 123 | return explain(abs(status)) 124 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | import proclib 3 | 4 | 5 | setup( 6 | name='proclib', 7 | version=proclib.__version__, 8 | 9 | description='pythonic processes', 10 | long_description=open('README.rst').read(), 11 | license='MIT', 12 | 13 | author='Eeo Jun', 14 | author_email='packwolf58@gmail.com', 15 | url='https://github.com/datalib/proclib', 16 | 17 | packages=['proclib'], 18 | install_requires=[ 19 | 'signalsdb==0.1.2', 20 | ], 21 | extras_require={ 22 | 'test': ['pytest'], 23 | }, 24 | 25 | include_package_data=True, 26 | zip_safe=False, 27 | platforms='any', 28 | 29 | keywords='processes unix process datalib', 30 | classifiers=[ 31 | 'Development Status :: 4 - Beta', 32 | 'Intended Audience :: Developers', 33 | 'License :: OSI Approved :: MIT License', 34 | 35 | 'Programming Language :: Python :: 2', 36 | 'Programming Language :: Python :: 3', 37 | ], 38 | ) 39 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalib/proclib/12268ee3d2b0b590cb7cdaff5828a5099612bf66/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_api.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | try: 3 | from cStringIO import StringIO 4 | except: 5 | from io import StringIO 6 | from proclib.api import spawn 7 | 8 | 9 | @pytest.fixture(params=[ 10 | ['cat', ['grep', 'at']], 11 | ['cat | grep at'], 12 | 'cat | grep at', 13 | ]) 14 | def command(request): 15 | return request.param 16 | 17 | 18 | @pytest.fixture(params=[ 19 | lambda: iter(['at\n'] * 2), 20 | lambda: 'at\nat\n', 21 | lambda: ['at\nat\n'], 22 | lambda: StringIO('at\nat\n'), 23 | ]) 24 | def data(request): 25 | return request.param() 26 | 27 | 28 | def test_spawn_no_data(): 29 | r = spawn('echo m') 30 | r.wait() 31 | 32 | assert r.out.strip() == 'm' 33 | assert r.ok 34 | 35 | 36 | def test_spawn_with_data(command, data): 37 | r = spawn(command, data=data) 38 | r.wait() 39 | 40 | assert r.out == 'at\nat\n' 41 | assert r.ok 42 | -------------------------------------------------------------------------------- /tests/test_helpers.py: -------------------------------------------------------------------------------- 1 | from pytest import fixture 2 | from proclib.helpers import str_parse, list_parse, cached_property 3 | 4 | 5 | @fixture 6 | def expected(): 7 | return [['cat'], ['grep', 'at']] 8 | 9 | 10 | @fixture(params=[ 11 | ['cat | grep at'], 12 | ['cat', 'grep at'], 13 | ['cat', ['grep', 'at']], 14 | ]) 15 | def cmd(request): 16 | return request.param 17 | 18 | 19 | def test_str_parse(expected): 20 | assert list(str_parse('cat | grep at')) == expected 21 | 22 | 23 | def test_list_parse(cmd, expected): 24 | assert list(list_parse(cmd)) == expected 25 | 26 | 27 | def test_parsers_equiv(): 28 | assert list(str_parse('')) == list(list_parse([])) 29 | assert list(str_parse('ng | cat')) == list(list_parse(['ng | cat'])) 30 | 31 | 32 | def test_cached_property(): 33 | class Obj(object): 34 | ctx = [] 35 | 36 | @cached_property 37 | def name(self): 38 | self.ctx.append(1) 39 | return self 40 | 41 | obj = Obj() 42 | assert obj.name is obj 43 | assert obj.name is obj 44 | assert obj.ctx == [1] 45 | -------------------------------------------------------------------------------- /tests/test_pipe.py: -------------------------------------------------------------------------------- 1 | from itertools import chain 2 | from pytest import fixture 3 | from proclib.pipe import Pipe 4 | from proclib.process import Process 5 | 6 | 7 | @fixture 8 | def pipe(): 9 | return Pipe( 10 | [['cat'], ['grep', 'at']], 11 | data=['c\nat\n'], 12 | ) 13 | 14 | 15 | def test_spawn_procs(pipe): 16 | procs = pipe.spawn_procs() 17 | assert [p.command for p in procs] == pipe.commands 18 | for p in procs: 19 | p.popen.kill() 20 | 21 | 22 | def test_run_pipes_data(pipe): 23 | r = pipe.run() 24 | r.wait() 25 | assert r.ok 26 | assert r.finished 27 | assert r.out == 'at\n' 28 | 29 | 30 | @fixture(scope='module') 31 | def res(request): 32 | r = Pipe([['yes'], ['head', '-n', '2']]).run() 33 | r.wait() 34 | request.addfinalizer(r.close) 35 | return r 36 | 37 | 38 | def test_sigpipe_was_used(res): 39 | yes = res.history[0] 40 | yes.wait() 41 | assert yes.explain_signal()['signal'] == 'SIGPIPE' 42 | assert not yes.ok 43 | 44 | 45 | def test_correct_data(res): 46 | assert len(res.out.split()) == 2 47 | assert res.ok 48 | 49 | 50 | def test_make_response_pops_proc(): 51 | procs = chain( 52 | [Process(['echo', 'm']) for _ in range(2)], 53 | [Process(['echo', 'a'])], 54 | ) 55 | r = Pipe.make_response(procs) 56 | 57 | assert len(r.history) == 2 58 | assert r.out == 'a\n' 59 | assert r.command == ['echo', 'a'] 60 | -------------------------------------------------------------------------------- /tests/test_process.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from contextlib import closing 3 | from pytest import fixture 4 | from proclib.process import Process 5 | 6 | 7 | @fixture 8 | def proc(): 9 | return Process('cat') 10 | 11 | 12 | def test_pipe_passes_data(proc): 13 | proc.pipe(['at']) 14 | r = proc.run() 15 | r.wait() 16 | 17 | with closing(r): 18 | assert r.out == 'at' 19 | assert r.err == '' 20 | assert r.ok 21 | assert r.pid 22 | 23 | 24 | def test_terminate_cat(proc): 25 | r = proc.run() 26 | r.terminate() 27 | r.wait() 28 | 29 | sig = r.explain_signal() 30 | assert r.finished 31 | assert sig['signal'] == 'SIGTERM' 32 | assert sig['id'] == 15 33 | assert not r.ok 34 | 35 | 36 | def test_context_manager(proc): 37 | proc.pipe(['data']) 38 | with proc.run() as r: 39 | assert r.stdout.read() == 'data' 40 | assert r.stderr.read() == '' 41 | 42 | assert r.stdout.closed 43 | assert r.stderr.closed 44 | r.wait() 45 | 46 | 47 | def test_iter(proc): 48 | proc.pipe(['data\ndata\ndata\n']) 49 | with proc.run() as r: 50 | assert list(r) == (['data\n'] * 3) 51 | assert not r.stdout.closed 52 | --------------------------------------------------------------------------------