├── .coveragerc ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ └── package-tests.yml ├── .gitignore ├── .hgignore ├── .hgtags ├── .readthedocs.yaml ├── .travis.yml ├── CHANGES ├── LICENSE ├── MANIFEST.in ├── README.rst ├── appveyor.yml ├── docs ├── Makefile ├── conf.py ├── index.rst ├── internals.rst ├── overview.rst ├── reference.rst ├── requirements.txt ├── spelling_wordlist.txt └── tutorial.rst ├── echoer.py ├── lister.py ├── package.json ├── pyproject.toml ├── retrier.py ├── sarge ├── __init__.py ├── shlext.py └── utils.py ├── setup.cfg ├── stack_tracer.py ├── test_expect.py ├── test_expect2.py ├── test_feeder.py ├── test_progress.py ├── test_sarge.py ├── tox.ini └── waiter.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | omit = 4 | .tox/* 5 | setup.py 6 | stack_tracer.py 7 | 8 | [report] 9 | exclude_lines = 10 | pragma: no cover 11 | raise NotImplementedError 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve this library. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Environment** 27 | - OS, including version 28 | - Version of this library 29 | 30 | **Additional information** 31 | Add any other information about the problem here. 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/package-tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | paths-ignore: 7 | - 'LICENSE.*' 8 | - 'README.*' 9 | - '.github/ISSUE-TEMPLATE/**' 10 | - 'docs/**' 11 | - '.hgignore' 12 | - '.gitignore' 13 | 14 | pull_request: 15 | branches: [ master ] 16 | paths-ignore: 17 | - 'LICENSE.*' 18 | - 'README.*' 19 | - '.github/ISSUE-TEMPLATE/**' 20 | - 'docs/**' 21 | - '.hgignore' 22 | - '.gitignore' 23 | 24 | jobs: 25 | build: 26 | runs-on: ${{ matrix.os }} 27 | strategy: 28 | fail-fast: false 29 | matrix: 30 | os: [ubuntu-latest, macos-latest, windows-latest] 31 | python-version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13', 'pypy-2.7', 'pypy-3.9'] 32 | 33 | steps: 34 | - uses: actions/checkout@v4 35 | - name: Set up Python ${{ matrix.python-version }} 36 | uses: actions/setup-python@v5 37 | with: 38 | python-version: ${{ matrix.python-version }} 39 | - name: Install Windows-only dependencies 40 | run: | 41 | python retrier.py --delay 10 --retries 5 choco install gnuwin32-coreutils.install 42 | echo "C:\Program Files (x86)\GnuWin32\bin" >> $env:GITHUB_PATH 43 | if: ${{ matrix.os == 'windows-latest' }} 44 | - name: Test with unittest 45 | run: | 46 | python test_sarge.py 47 | - name: Test with coverage 48 | run: | 49 | pip install coverage 50 | coverage run --branch test_sarge.py 51 | coverage xml 52 | - name: Upload coverage to Codecov 53 | uses: codecov/codecov-action@v4 54 | with: 55 | flags: unittests 56 | files: coverage.xml 57 | fail_ci_if_error: false 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.log 3 | .idea 4 | .tox 5 | build 6 | dist 7 | htmlcov 8 | docs/_build 9 | MANIFEST 10 | testfile.txt 11 | random.bin 12 | .dict-validwords 13 | .coverage 14 | watcher.conf 15 | test1.py 16 | emitter.py 17 | hello.py 18 | 19 | -------------------------------------------------------------------------------- /.hgignore: -------------------------------------------------------------------------------- 1 | \.(pyc|log|yapf|json)$ 2 | (\.(idea|tox|egg-info)|build|dist|htmlcov|logs)/ 3 | docs/_build/ 4 | ^MANIFEST$ 5 | testfile.txt 6 | random.bin 7 | \.(dict-validwords|coverage)$ 8 | watcher.conf 9 | (test\d+|emitter|hello)\.py 10 | -------------------------------------------------------------------------------- /.hgtags: -------------------------------------------------------------------------------- 1 | c5e1974ad4b3b8f90c543f62ab55030166b528e0 0.1 2 | a2af27af604953355240fc66c90d88866b8834cb 0.1.1 3 | fe5a2bc0f8eb991787dbb571884c94c7bb009c27 0.1.2 4 | 2bc636121db12819d52d58fa69d206e42732828e 0.1.3 5 | efe59a1e35345391388b58e7765e5fec4e80e48d 0.1.4 6 | 5380b6360f7d0fbf3e3a7b59aa1b3a2552b56e29 0.1.5 7 | 1e8ec0027165721e1ff56ea8301a9755eea8722e 0.1.6 8 | e29c3b5dff70c44f5411b73667de0a80586b69be 0.1.7 9 | a9c5a766cf693fff9a99c0bb0f3ed15f119ca3d0 0.1.7.post0 10 | 7ffd992bb7dc5f050db00194765ed84c532cdfd1 0.1.7.post1 11 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the version of Python and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/conf.py 17 | 18 | # We recommend specifying your dependencies to enable reproducible builds: 19 | # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 20 | python: 21 | install: 22 | - requirements: docs/requirements.txt 23 | 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | python: 4 | # - "2.6" removed because not available on Travis 5 | - "2.7" 6 | # - "3.2" removed because Coveralls/coverage 4.0 fails on 3.2 7 | # - "3.3" removed because not available on Travis 8 | - "3.4" 9 | - "3.5" 10 | - "3.6" 11 | - "3.7" 12 | - "3.8" 13 | - "pypy" 14 | #jobs: 15 | #exclude: 16 | #- os: osx 17 | #python: "2.7" 18 | #- os: osx 19 | #python: "3.4" 20 | install: 21 | - pip install coveralls 22 | script: 23 | - "PYTHONHASHSEED=0 python setup.py test" 24 | - "PYTHONHASHSEED=0 coverage run setup.py test" 25 | after_success: coveralls 26 | -------------------------------------------------------------------------------- /CHANGES: -------------------------------------------------------------------------------- 1 | :orphan: 2 | 3 | Please see the online change log at 4 | 5 | https://pythonhosted.org/sarge/overview.html#change-log 6 | 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2022 by Vinay Sajip. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | * The name(s) of the copyright holder(s) may not be used to endorse or 13 | promote products derived from this software without specific prior 14 | written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) "AS IS" AND ANY EXPRESS OR 17 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 18 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 19 | EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, 20 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 21 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 22 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 23 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 24 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | 27 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include README.rst 3 | include setup.py 4 | include sarge/*.py 5 | include test_sarge.py 6 | include stack_tracer.py 7 | include lister.py 8 | include echoer.py 9 | include waiter.py 10 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | |badge1| |badge2| 2 | 3 | .. |badge1| image:: https://img.shields.io/github/workflow/status/vsajip/sarge/Tests 4 | :alt: GitHub test status 5 | 6 | .. |badge2| image:: https://img.shields.io/codecov/c/github/vsajip/sarge 7 | :target: https://app.codecov.io/gh/vsajip/sarge 8 | :alt: GitHub coverage status 9 | 10 | Overview 11 | ======== 12 | The sarge package provides a wrapper for subprocess which provides command 13 | pipeline functionality. 14 | 15 | This package leverages subprocess to provide easy-to-use cross-platform command 16 | pipelines with a POSIX flavour: you can have chains of commands using ``;``, ``&``, 17 | pipes using ``|`` and ``|&``, and redirection. 18 | 19 | Requirements & Installation 20 | --------------------------- 21 | 22 | The sarge package requires Python 2.7 or Python 3.6 or greater, and can be installed 23 | with the standard Python installation procedure:: 24 | 25 | 26 | pip install sarge 27 | 28 | There is a set of unit tests which you can invoke with:: 29 | 30 | python setup.py test 31 | 32 | before running the installation. 33 | 34 | Availability & Documentation 35 | ---------------------------- 36 | 37 | The latest version of sarge can be found on `GitHub 38 | `_. 39 | 40 | The latest documentation (kept updated between releases) is on `Read The Docs 41 | `_. 42 | 43 | Please report any problems or suggestions for improvement either via the `mailing list 44 | `_ or the `issue tracker 45 | `_. 46 | 47 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | version: 1.0.{build} 2 | image: 3 | - Visual Studio 2017 4 | - ubuntu1804 5 | - macos 6 | - macos-mojave 7 | environment: 8 | matrix: 9 | - TOXENV: py27 10 | - TOXENV: py35 11 | - TOXENV: py36 12 | - TOXENV: py37 13 | - TOXENV: py38 14 | install: 15 | - cmd: pip install tox 16 | build: off 17 | test_script: 18 | - cmd: >- 19 | set USE_MSYS=1 20 | 21 | tox 22 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | 15 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest remote apidocs 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " singlehtml to make a single large HTML file" 22 | @echo " pickle to make pickle files" 23 | @echo " json to make JSON files" 24 | @echo " htmlhelp to make HTML files and a HTML help project" 25 | @echo " qthelp to make HTML files and a qthelp project" 26 | @echo " devhelp to make HTML files and a Devhelp project" 27 | @echo " epub to make an epub" 28 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 29 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 30 | @echo " text to make text files" 31 | @echo " man to make manual pages" 32 | @echo " changes to make an overview of all changed/added/deprecated items" 33 | @echo " linkcheck to check all external links for integrity" 34 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 35 | 36 | clean: 37 | -rm -rf $(BUILDDIR)/* 38 | 39 | html: 40 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 41 | @echo 42 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 43 | 44 | pdoc: 45 | mkdir -p $(BUILDDIR)/html/apidocs 46 | pdoc -o $(BUILDDIR)/html/apidocs --no-show-source --docformat google --logo https://www.red-dove.com/assets/img/rdclogo.gif ../sarge 47 | 48 | apidocs: 49 | docfrag --libs .. sarge -f hovertip > hover.json 50 | 51 | remote: 52 | rsync -avz $(BUILDDIR)/html/* vopal:~/apps/rdc_docs/sarge 53 | 54 | spelling: 55 | $(SPHINXBUILD) -b spelling $(ALLSPHINXOPTS) $(BUILDDIR) 56 | 57 | dirhtml: 58 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 59 | @echo 60 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 61 | 62 | singlehtml: 63 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 64 | @echo 65 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 66 | 67 | pickle: 68 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 69 | @echo 70 | @echo "Build finished; now you can process the pickle files." 71 | 72 | json: 73 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 74 | @echo 75 | @echo "Build finished; now you can process the JSON files." 76 | 77 | htmlhelp: 78 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 79 | @echo 80 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 81 | ".hhp project file in $(BUILDDIR)/htmlhelp." 82 | 83 | qthelp: 84 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 85 | @echo 86 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 87 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 88 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Sarge.qhcp" 89 | @echo "To view the help file:" 90 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Sarge.qhc" 91 | 92 | devhelp: 93 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 94 | @echo 95 | @echo "Build finished." 96 | @echo "To view the help file:" 97 | @echo "# mkdir -p $$HOME/.local/share/devhelp/Sarge" 98 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Sarge" 99 | @echo "# devhelp" 100 | 101 | epub: 102 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 103 | @echo 104 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 105 | 106 | latex: 107 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 108 | @echo 109 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 110 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 111 | "(use \`make latexpdf' here to do that automatically)." 112 | 113 | latexpdf: 114 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 115 | @echo "Running LaTeX files through pdflatex..." 116 | make -C $(BUILDDIR)/latex all-pdf 117 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 118 | 119 | text: 120 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 121 | @echo 122 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 123 | 124 | man: 125 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 126 | @echo 127 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 128 | 129 | changes: 130 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 131 | @echo 132 | @echo "The overview file is in $(BUILDDIR)/changes." 133 | 134 | linkcheck: 135 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 136 | @echo 137 | @echo "Link check complete; look for any errors in the above output " \ 138 | "or in $(BUILDDIR)/linkcheck/output.txt." 139 | 140 | doctest: 141 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 142 | @echo "Testing of doctests in the sources finished, look at the " \ 143 | "results in $(BUILDDIR)/doctest/output.txt." 144 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Sarge documentation build configuration file, created by 4 | # sphinx-quickstart on Thu Jan 19 18:01:03 2012. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import datetime, os, sys 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | sys.path.insert(0, os.path.abspath('..')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # If your documentation needs a minimal Sphinx version, state it here. 24 | #needs_sphinx = '1.0' 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be extensions 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 28 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.imgmath', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinxcontrib.spelling' 29 | ] 30 | 31 | # Add any paths that contain templates here, relative to this directory. 32 | templates_path = ['_templates'] 33 | 34 | # The suffix of source filenames. 35 | source_suffix = '.rst' 36 | 37 | # The encoding of source files. 38 | #source_encoding = 'utf-8-sig' 39 | 40 | # The master toctree document. 41 | master_doc = 'index' 42 | 43 | # General information about the project. 44 | project = u'Sarge' 45 | copyright = u'2012-%s, Vinay Sajip' % datetime.date.today().year 46 | 47 | # The version info for the project you're documenting, acts as replacement for 48 | # |version| and |release|, also used in various other places throughout the 49 | # built documents. 50 | # 51 | # The full version, including alpha/beta/rc tags. 52 | from sarge import __version__ as release, __date__ as today 53 | # The short X.Y version. 54 | version = '.'.join(release.split('.')[:2]) 55 | if '.dev' in release: 56 | today = datetime.date.today() 57 | else: 58 | today = datetime.datetime.strptime(today, '%Y-%m-%d').date() 59 | today = today.strftime('%b %d, %Y') 60 | 61 | # The language for content autogenerated by Sphinx. Refer to documentation 62 | # for a list of supported languages. 63 | #language = None 64 | 65 | # There are two options for replacing |today|: either, you set today to some 66 | # non-false value, then it is used: 67 | #today = '' 68 | # Else, today_fmt is used as the format for a strftime call. 69 | #today_fmt = '%B %d, %Y' 70 | 71 | # List of patterns, relative to source directory, that match files and 72 | # directories to ignore when looking for source files. 73 | exclude_patterns = ['_build'] 74 | 75 | # The reST default role (used for this markup: `text`) to use for all documents. 76 | #default_role = None 77 | 78 | # If true, '()' will be appended to :func: etc. cross-reference text. 79 | #add_function_parentheses = True 80 | 81 | # If true, the current module name will be prepended to all description 82 | # unit titles (such as .. function::). 83 | add_module_names = False 84 | 85 | # If true, sectionauthor and moduleauthor directives will be shown in the 86 | # output. They are ignored by default. 87 | #show_authors = False 88 | 89 | # The name of the Pygments (syntax highlighting) style to use. 90 | pygments_style = 'sphinx' 91 | 92 | # A list of ignored prefixes for module index sorting. 93 | #modindex_common_prefix = [] 94 | 95 | spelling_lang='en_GB' 96 | spelling_word_list_filename='spelling_wordlist.txt' 97 | 98 | # -- Options for HTML output --------------------------------------------------- 99 | 100 | # The theme to use for HTML and HTML Help pages. See the documentation for 101 | # a list of builtin themes. 102 | html_theme = os.environ.get('DOCS_THEME', 'default') 103 | 104 | THEME_OPTIONS = { 105 | 'sizzle': { 106 | } 107 | } 108 | 109 | if html_theme == 'sizzle' and os.path.isfile('hover.json'): 110 | import json 111 | 112 | with open('hover.json', encoding='utf-8') as f: 113 | THEME_OPTIONS['sizzle']['custom_data'] = {'hovers': json.load(f) } 114 | 115 | # Theme options are theme-specific and customize the look and feel of a theme 116 | # further. For a list of options available for each theme, see the 117 | # documentation. 118 | if html_theme in THEME_OPTIONS: 119 | html_theme_options = THEME_OPTIONS[html_theme] 120 | 121 | # Add any paths that contain custom themes here, relative to this directory. 122 | html_theme_path = ['themes'] 123 | 124 | # The name for this set of Sphinx documents. If None, it defaults to 125 | # " v documentation". 126 | #html_title = None 127 | 128 | # A shorter title for the navigation bar. Default is the same as html_title. 129 | #html_short_title = None 130 | 131 | # The name of an image file (relative to this directory) to place at the top 132 | # of the sidebar. 133 | #html_logo = None 134 | 135 | # The name of an image file (within the static path) to use as favicon of the 136 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 137 | # pixels large. 138 | #html_favicon = None 139 | 140 | # Add any paths that contain custom static files (such as style sheets) here, 141 | # relative to this directory. They are copied after the builtin static files, 142 | # so a file named "default.css" will overwrite the builtin "default.css". 143 | #html_static_path = ['_static'] 144 | 145 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 146 | # using the given strftime format. 147 | #html_last_updated_fmt = '%b %d, %Y' 148 | 149 | # If true, SmartyPants will be used to convert quotes and dashes to 150 | # typographically correct entities. 151 | #html_use_smartypants = True 152 | 153 | # Custom sidebar templates, maps document names to template names. 154 | #html_sidebars = {} 155 | 156 | # Additional templates that should be rendered to pages, maps page names to 157 | # template names. 158 | #html_additional_pages = {} 159 | 160 | # If false, no module index is generated. 161 | #html_domain_indices = True 162 | 163 | # If false, no index is generated. 164 | #html_use_index = True 165 | 166 | # If true, the index is split into individual pages for each letter. 167 | #html_split_index = False 168 | 169 | # If true, links to the reST sources are added to the pages. 170 | #html_show_sourcelink = True 171 | 172 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 173 | #html_show_sphinx = True 174 | 175 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 176 | #html_show_copyright = True 177 | 178 | # If true, an OpenSearch description file will be output, and all pages will 179 | # contain a tag referring to it. The value of this option must be the 180 | # base URL from which the finished HTML is served. 181 | #html_use_opensearch = '' 182 | 183 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 184 | #html_file_suffix = None 185 | 186 | # Output file base name for HTML help builder. 187 | htmlhelp_basename = 'Sargedoc' 188 | 189 | 190 | # -- Options for LaTeX output -------------------------------------------------- 191 | 192 | # The paper size ('letter' or 'a4'). 193 | #latex_paper_size = 'letter' 194 | 195 | # The font size ('10pt', '11pt' or '12pt'). 196 | #latex_font_size = '10pt' 197 | 198 | # Grouping the document tree into LaTeX files. List of tuples 199 | # (source start file, target name, title, author, documentclass [howto/manual]). 200 | latex_documents = [ 201 | ('index', 'Sarge.tex', u'Sarge Documentation', 202 | u'Vinay Sajip', 'manual'), 203 | ] 204 | 205 | # The name of an image file (relative to this directory) to place at the top of 206 | # the title page. 207 | #latex_logo = None 208 | 209 | # For "manual" documents, if this is true, then toplevel headings are parts, 210 | # not chapters. 211 | #latex_use_parts = False 212 | 213 | # If true, show page references after internal links. 214 | #latex_show_pagerefs = False 215 | 216 | # If true, show URL addresses after external links. 217 | #latex_show_urls = False 218 | 219 | # Additional stuff for the LaTeX preamble. 220 | #latex_preamble = '' 221 | 222 | # Documents to append as an appendix to all manuals. 223 | #latex_appendices = [] 224 | 225 | # If false, no module index is generated. 226 | #latex_domain_indices = True 227 | 228 | 229 | # -- Options for manual page output -------------------------------------------- 230 | 231 | # One entry per manual page. List of tuples 232 | # (source start file, name, description, authors, manual section). 233 | man_pages = [ 234 | ('index', 'sarge', u'Sarge Documentation', 235 | [u'Vinay Sajip'], 1) 236 | ] 237 | 238 | 239 | # -- Options for Epub output --------------------------------------------------- 240 | 241 | # Bibliographic Dublin Core info. 242 | epub_title = u'Sarge' 243 | epub_author = u'Vinay Sajip' 244 | epub_publisher = u'Vinay Sajip' 245 | epub_copyright = u'2012-2013, Vinay Sajip' 246 | 247 | # The language of the text. It defaults to the language option 248 | # or en if the language is not set. 249 | #epub_language = '' 250 | 251 | # The scheme of the identifier. Typical schemes are ISBN or URL. 252 | #epub_scheme = '' 253 | 254 | # The unique identifier of the text. This can be a ISBN number 255 | # or the project homepage. 256 | #epub_identifier = '' 257 | 258 | # A unique identification for the text. 259 | #epub_uid = '' 260 | 261 | # HTML files that should be inserted before the pages created by sphinx. 262 | # The format is a list of tuples containing the path and title. 263 | #epub_pre_files = [] 264 | 265 | # HTML files shat should be inserted after the pages created by sphinx. 266 | # The format is a list of tuples containing the path and title. 267 | #epub_post_files = [] 268 | 269 | # A list of files that should not be packed into the epub file. 270 | #epub_exclude_files = [] 271 | 272 | # The depth of the table of contents in toc.ncx. 273 | #epub_tocdepth = 3 274 | 275 | # Allow duplicate toc entries. 276 | #epub_tocdup = True 277 | 278 | 279 | # Example configuration for intersphinx: refer to the Python standard library. 280 | intersphinx_mapping = {'python': ('http://docs.python.org/', None)} 281 | 282 | def skip_module_docstring(app, what, name, obj, options, lines): 283 | if (what, name) == ('module', 'sarge'): 284 | del lines[:] 285 | 286 | def setup(app): 287 | app.connect('autodoc-process-docstring', skip_module_docstring) 288 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Sarge documentation master file, created by 2 | sphinx-quickstart on Thu Jan 19 18:01:03 2012. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | .. _index: 7 | 8 | Welcome to sarge's documentation! 9 | ================================= 10 | 11 | .. rst-class:: release-info 12 | 13 | .. list-table:: 14 | :widths: auto 15 | :stub-columns: 1 16 | 17 | * - Release: 18 | - |release| 19 | * - Date: 20 | - |today| 21 | 22 | Welcome to the documentation for ``sarge``, a wrapper for :mod:`subprocess` 23 | which aims to make life easier for anyone who needs to interact with external 24 | applications from their Python code. 25 | 26 | **Please note:** this documentation is *work in progress*. 27 | 28 | .. toctree:: 29 | :maxdepth: 3 30 | 31 | overview 32 | tutorial 33 | internals 34 | reference 35 | 36 | Please report any problems or suggestions for improvement either via the 37 | `mailing list `_ or the `issue 38 | tracker `_. 39 | 40 | -------------------------------------------------------------------------------- /docs/internals.rst: -------------------------------------------------------------------------------- 1 | .. _internals: 2 | 3 | .. currentmodule:: sarge 4 | 5 | Under the hood 6 | ============== 7 | 8 | This is the section where some description of how ``sarge`` works internally 9 | will be provided, as and when time permits. 10 | 11 | How capturing works 12 | ------------------- 13 | 14 | This section describes how :class:`Capture` is implemented. 15 | 16 | Basic approach 17 | ^^^^^^^^^^^^^^ 18 | 19 | A :class:`~sarge.Capture` consists of a queue, some output streams from 20 | sub-processes, and some threads to read from those streams into the queue. One 21 | thread is created for each stream, and the thread exits when its stream has 22 | been completely read. When you read from a :class:`~sarge.Capture` instance 23 | using methods like :meth:`~sarge.Capture.read`, :meth:`~sarge.Capture.readline` 24 | and :meth:`~sarge.Capture.readlines`, you are effectively reading from the 25 | queue. 26 | 27 | Blocking and timeouts 28 | ^^^^^^^^^^^^^^^^^^^^^ 29 | 30 | Each of the :meth:`~Capture.read`, :meth:`~Capture.readline` and 31 | :meth:`~Capture.readlines` methods has optional ``block`` and ``timeout`` 32 | keyword arguments. These default to ``True`` and ``None`` respectively, 33 | which means block indefinitely until there's some data -- the standard 34 | behaviour for file-like objects. However, these can be overridden internally 35 | in a couple of ways: 36 | 37 | * The :class:`Capture` constructor takes an optional ``timeout`` keyword 38 | argument. This defaults to ``None``, but if specified, that's the timeout used 39 | by the ``readXXX`` methods unless you specify values in the method calls. 40 | If ``None`` is specified in the constructor, the module attribute 41 | :attr:`default_capture_timeout` is used, which is currently set to 0.02 42 | seconds. If you need to change this default, you can do so before any 43 | :class:`Capture` instances are created (or just provide an alternative default 44 | in every :class:`Capture` creation). 45 | * If all streams feeding into the capture have been completely read, 46 | then ``block`` is always set to ``False``. 47 | 48 | 49 | Implications when handling large amounts of data 50 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 51 | There shouldn't be any special implications of handling large amounts of 52 | data, other than buffering, buffer sizes and memory usage (which you would 53 | have to think about anyway). Here's an example of piping a 20MB file into a 54 | capture across several process boundaries:: 55 | 56 | $ ls -l random.bin 57 | -rw-rw-r-- 1 vinay vinay 20971520 2012-01-17 17:57 random.bin 58 | $ python 59 | [snip] 60 | >>> from sarge import run, Capture 61 | >>> p = run('cat random.bin|cat|cat|cat|cat|cat', stdout=Capture(), async_=True) 62 | >>> for i in range(8): 63 | ... data = p.stdout.read(2621440) 64 | ... print('Read chunk %d: %d bytes' % (i, len(data))) 65 | ... 66 | Read chunk 0: 2621440 bytes 67 | Read chunk 1: 2621440 bytes 68 | Read chunk 2: 2621440 bytes 69 | Read chunk 3: 2621440 bytes 70 | Read chunk 4: 2621440 bytes 71 | Read chunk 5: 2621440 bytes 72 | Read chunk 6: 2621440 bytes 73 | Read chunk 7: 2621440 bytes 74 | >>> p.stdout.read() 75 | '' 76 | 77 | 78 | Swapping output streams 79 | ----------------------- 80 | 81 | A new constant, ``STDERR``, is defined by ``sarge``. If you specify 82 | ``stdout=STDERR``, this means that you want the child process ``stdout`` to 83 | be the same as its ``stderr``. This is analogous to the core functionality in 84 | :class:`subprocess.Popen` where you can specify ``stderr=STDOUT`` to have the 85 | child process ``stderr`` be the same as its ``stdout``. The use of this 86 | constant also allows you to swap the child's ``stdout`` and ``stderr``, 87 | which can be useful in some cases. 88 | 89 | This functionality works through a class :class:`sarge.Popen` which subclasses 90 | :class:`subprocess.Popen` and overrides the internal ``_get_handles`` method to 91 | work the necessary magic -- which is to duplicate, close and swap handles as 92 | needed. 93 | 94 | How shell quoting works 95 | ----------------------- 96 | 97 | The :func:`shell_quote` function works as follows. Firstly, 98 | an empty string is converted to ``''``. Next, a check is made to see if the 99 | string has already been quoted (i.e. it begins and ends with the ``'`` 100 | character), and if so, it is returned enclosed in ``"`` and with any contained 101 | `"` characters escaped with a backslash. Otherwise, it's bracketed with the 102 | ``'`` character and every internal instance of ``'`` is replaced with 103 | ``'"'"'``. 104 | 105 | How shell command formatting works 106 | ---------------------------------- 107 | 108 | This is inspired by Nick Coghlan's `shell_command `_ project. An internal :class:`ShellFormatter` 110 | class is derived from :class:`string.Formatter` and overrides the 111 | :meth:`string.Formatter.convert_field` method to provide quoting for placeholder 112 | values. This formatter is simpler than Nick's in that it forces you to 113 | explicitly provide the indices of positional arguments: You have to use e.g. 114 | ``'cp {0} {1}`` instead of ``cp {} {}``. This avoids the need to keep an 115 | internal counter in the formatter, which would make its implementation be not 116 | thread-safe without additional work. 117 | 118 | How command parsing works 119 | ------------------------- 120 | 121 | Internally ``sarge`` uses a simple recursive descent parser to parse commands. 122 | A simple BNF grammar for the parser would be:: 123 | 124 | ::= ((";" | "&") )* 125 | ::= (("&&" | "||") )* 126 | ::= ( (("|" | "|&") )*) | "(" ")" 127 | ::= + 128 | ::= WORD (()? (">" | ">>") ( | ("&" )))* 129 | 130 | where WORD and NUM are terminal tokens with the meanings you would expect. 131 | 132 | The parser constructs a parse tree, which is used internally by the 133 | :class:`Pipeline` class to manage the running of the pipeline. 134 | 135 | The standard library's :mod:`shlex` module contains a class which is used for 136 | lexical scanning. Since the :class:`shlex.shlex` class is not able to provide 137 | the needed functionality, ``sarge`` includes a module, ``shlext``, 138 | which defines a subclass, ``shell_shlex``, which provides the necessary 139 | functionality. This is not part of the public API of ``sarge``, though it has 140 | been `submitted as an 141 | enhancement `_ on the Python 142 | issue tracker. 143 | 144 | Thread debugging 145 | ---------------- 146 | 147 | Sometimes, you can get deadlocks even though you think you've taken 148 | sufficient measures to avoid them. To help identify where deadlocks are 149 | occurring, the ``sarge`` source distribution includes a module, 150 | ``stack_tracer``, which is based on MIT-licensed code by László Nagy in an 151 | `ActiveState recipe `_. To see how 152 | it's invoked, you can look at the ``sarge`` test harness ``test_sarge.py`` -- 153 | this is set to invoke the tracer if the ``TRACE_THREADS`` variable is set (which 154 | it is, by default). If the unit tests hang on your system, then the 155 | ``threads-X.Y.log`` file will show where the deadlock is (just look and see what 156 | all the threads are waiting for). 157 | 158 | Future changes 159 | -------------- 160 | 161 | At the moment, if a :class:`Capture` is used, it will read from its sub-process 162 | output streams into a queue, which can then be read by your code. If you don't 163 | read from the :class:`Capture` in a timely fashion, a lot of data could 164 | potentially be buffered in memory -- the same thing that happens when you use 165 | :meth:`subprocess.Popen.communicate`. There might be added some means of 166 | "turning the tap off", i.e. pausing the reader threads so that the capturing 167 | threads stop reading from the sub-process streams. This will, of course, cause 168 | those sub-processes to block on their I/O, so at some point the tap would need 169 | to be turned back on. However, such a facility would afford better 170 | sub-process control in some scenarios. 171 | 172 | Next steps 173 | ---------- 174 | 175 | You might find it helpful to look at the :ref:`reference`. 176 | -------------------------------------------------------------------------------- /docs/overview.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | ======== 3 | 4 | Start here for all things ``sarge``. 5 | 6 | What is Sarge for? 7 | ------------------ 8 | 9 | If you want to interact with external programs from your Python applications, 10 | Sarge is a library which is intended to make your life easier than using the 11 | :mod:`subprocess` module in Python's standard library. 12 | 13 | Sarge is, of course, short for sergeant -- and like any good non-commissioned 14 | officer, ``sarge`` works to issue commands on your behalf and to inform you 15 | about the results of running those commands. 16 | 17 | The acronym lovers among you might be amused to learn that sarge can also 18 | stand for "Subprocess Allegedly Rewards Good Encapsulation" :-) 19 | 20 | Here's a taster (example suggested by Kenneth Reitz's Envoy documentation):: 21 | 22 | >>> from sarge import capture_stdout 23 | >>> p = capture_stdout('fortune|cowthink') 24 | >>> p.returncode 25 | 0 26 | >>> p.commands 27 | [Command('fortune'), Command('cowthink')] 28 | >>> p.returncodes 29 | [0, 0] 30 | >>> print(p.stdout.text) 31 | ____________________________________ 32 | ( The last thing one knows in ) 33 | ( constructing a work is what to put ) 34 | ( first. ) 35 | ( ) 36 | ( -- Blaise Pascal ) 37 | ------------------------------------ 38 | o ^__^ 39 | o (oo)\_______ 40 | (__)\ )\/\ 41 | ||----w | 42 | || || 43 | 44 | The :func:`capture_stdout` function is a convenient form of an underlying 45 | function, :func:`run`. You can also use conditionals:: 46 | 47 | >>> from sarge import run 48 | >>> p = run('false && echo foo') 49 | >>> p.commands 50 | [Command('false')] 51 | >>> p.returncodes 52 | [1] 53 | >>> p.returncode 54 | 1 55 | >>> p = run('false || echo foo') 56 | foo 57 | >>> p.commands 58 | [Command('false'), Command('echo foo')] 59 | >>> p.returncodes 60 | [1, 0] 61 | >>> p.returncode 62 | 0 63 | 64 | The conditional logic is being done by sarge and not the shell -- which means 65 | you can use the identical code on Windows. Here's an example of some more 66 | involved use of pipes, which also works identically on POSIX and Windows:: 67 | 68 | >>> cmd = 'echo foo | tee stdout.log 3>&1 1>&2 2>&3 | tee stderr.log > %s' % os.devnull 69 | >>> p = run(cmd) 70 | >>> p.commands 71 | [Command('echo foo'), Command('tee stdout.log'), Command('tee stderr.log')] 72 | >>> p.returncodes 73 | [0, 0, 0] 74 | >>> 75 | vinay@eta-oneiric64:~/projects/sarge$ cat stdout.log 76 | foo 77 | vinay@eta-oneiric64:~/projects/sarge$ cat stderr.log 78 | foo 79 | 80 | In the above example, the first tee invocation swaps its ``stderr`` and 81 | ``stdout`` -- see `this post `_ for a longer explanation 82 | of this somewhat esoteric usage. 83 | 84 | Why not just use ``subprocess``? 85 | -------------------------------- 86 | 87 | The :mod:`subprocess` module in the standard library contains some very 88 | powerful functionality. It encapsulates the nitty-gritty details of subprocess 89 | creation and communication on POSIX and Windows platforms, and presents the 90 | application programmer with a uniform interface to the OS-level facilities. 91 | However, :mod:`subprocess` does not do much more than this, 92 | and is difficult to use in some scenarios. For example: 93 | 94 | * You want to use command pipelines, but using ``subprocess`` out of the box 95 | often leads to deadlocks because pipe buffers get filled up. 96 | * You want to use bash-style pipe syntax on Windows, 97 | but Windows shells don't support some of the syntax you want to use, 98 | like ``&&``, ``||``, ``|&`` and so on. 99 | * You want to process output from commands in a flexible way, 100 | and :meth:`~subprocess.Popen.communicate` is not flexible enough for your 101 | needs -- for example, you need to process output a line at a time. 102 | * You want to avoid `shell injection 103 | `_ problems by 104 | having the ability to quote your command arguments safely. 105 | * :mod:`subprocess` allows you to let ``stderr`` be the same as ``stdout``, 106 | but not the other way around -- and you need to do that. 107 | 108 | Main features 109 | ------------- 110 | 111 | Sarge offers the following features: 112 | 113 | * A simple run command which allows a rich subset of Bash-style shell 114 | command syntax, but parsed and run by sarge so that you can run on Windows 115 | without ``cygwin``. 116 | * The ability to format shell commands with placeholders, 117 | such that variables are quoted to prevent shell injection attacks:: 118 | 119 | >>> from sarge import shell_format 120 | >>> shell_format('ls {0}', '*.py') 121 | "ls '*.py'" 122 | >>> shell_format('cat {0}', 'a file name with spaces') 123 | "cat 'a file name with spaces'" 124 | 125 | * The ability to capture output streams without requiring you to program your 126 | own threads. You just use a :class:`Capture` object and then you can read 127 | from it as and when you want:: 128 | 129 | >>> from sarge import Capture, run 130 | >>> with Capture() as out: 131 | ... run('echo foobarbaz', stdout=out) 132 | ... 133 | 134 | >>> out.read(3) 135 | 'foo' 136 | >>> out.read(3) 137 | 'bar' 138 | >>> out.read(3) 139 | 'baz' 140 | >>> out.read(3) 141 | '\n' 142 | >>> out.read(3) 143 | '' 144 | 145 | A :class:`Capture` object can capture the output from multiple commands:: 146 | 147 | >>> from sarge import run, Capture 148 | >>> p = run('echo foo; echo bar; echo baz', stdout=Capture()) 149 | >>> p.stdout.readline() 150 | 'foo\n' 151 | >>> p.stdout.readline() 152 | 'bar\n' 153 | >>> p.stdout.readline() 154 | 'baz\n' 155 | >>> p.stdout.readline() 156 | '' 157 | 158 | Delays in commands are honoured in asynchronous calls:: 159 | 160 | >>> from sarge import run, Capture 161 | >>> cmd = 'echo foo & (sleep 2; echo bar) & (sleep 1; echo baz)' 162 | >>> p = run(cmd, stdout=Capture(), async_=True) # returns immediately 163 | >>> p.close() # wait for completion 164 | >>> p.stdout.readline() 165 | 'foo\n' 166 | >>> p.stdout.readline() 167 | 'baz\n' 168 | >>> p.stdout.readline() 169 | 'bar\n' 170 | >>> 171 | 172 | Here, the ``sleep`` commands ensure that the asynchronous ``echo`` calls 173 | occur in the order ``foo`` (no delay), ``baz`` (after a delay of one second) 174 | and ``bar`` (after a delay of two seconds); the capturing works as expected. 175 | 176 | 177 | Python version and platform compatibility 178 | ----------------------------------------- 179 | 180 | Sarge is intended to be used on and is tested on Python versions 2.7, 3.6 - 3.10, pypy 181 | and pypy3 on Linux, Windows, and macOS. 182 | 183 | 184 | Project status 185 | -------------- 186 | 187 | The project has reached production/stable status in its development: there is a test 188 | suite and it has been exercised on Windows, Ubuntu and Mac OS X. However, 189 | because of the timing sensitivity of the functionality, testing needs to be 190 | performed on as wide a range of hardware and platforms as possible. 191 | 192 | The source repository for the project is on GitHub: 193 | 194 | https://github.com/vsajip/sarge/ 195 | 196 | You can leave feedback by raising a new issue on the `issue 197 | tracker `_. 198 | 199 | .. note:: For testing under Windows, you need to install the `GnuWin32 200 | coreutils `_ 201 | package, and copy the relevant executables (currently ``libiconv2.dll``, 202 | ``libintl3.dll``, ``cat.exe``, ``echo.exe``, ``tee.exe``, ``false.exe``, 203 | ``true.exe``, ``sleep.exe`` and ``touch.exe``) to the directory from which 204 | you run the test harness (``test_sarge.py``). 205 | 206 | 207 | API stability 208 | ------------- 209 | 210 | Although every attempt will be made to keep API changes to the absolute minimum, 211 | it should be borne in mind that the software is in its very early stages. For 212 | example, the asynchronous feature (where commands are run in separate threads 213 | when you specify ``&`` in a command pipeline) can be considered experimental, 214 | and there may be changes in this area. However, you aren't forced to use this 215 | feature, and ``sarge`` should be useful without it. 216 | 217 | .. _changelog: 218 | 219 | Change log 220 | ---------- 221 | 222 | 0.1.8 (future) 223 | ~~~~~~~~~~~~~~ 224 | 225 | Released: Not yet. 226 | 227 | 228 | - Fixed #55: Polled subcommands in order to return up-to-date return codes in 229 | ``Pipeline.returncodes``. 230 | 231 | - Fixed #56: Ensure process_ready event is set at the appropriate time. 232 | 233 | - Fixed #57: Stored exception in command node for use when in asynchronous mode. 234 | 235 | 0.1.7 236 | ~~~~~ 237 | 238 | Released: 2021-12-10 239 | 240 | - Fixed #50: Initialized `commands` attribute in a constructor. 241 | 242 | - Fixed #52: Updated documentation to show improved command line parsing under Windows. 243 | 244 | - Fixed #53: Added waiter.py to the manifest so that the test suite can be run. 245 | 246 | 0.1.6 247 | ~~~~~ 248 | 249 | Released: 2020-08-24 250 | 251 | - Fixed #44: Added an optional timeout to :meth:`Command.wait` and 252 | :meth:`Pipeline.wait`, which only takes effect on Python >= 3.3. 253 | 254 | - Fixed #47: Added the ``replace_env`` keyword argument which allows a complete 255 | replacement for ``os.environ`` to be passed. 256 | 257 | - Fixed #51: Improved error handling around a ``Popen`` call. 258 | 259 | 0.1.5 260 | ~~~~~ 261 | 262 | Released: 2018-06-18 263 | 264 | - Fixed #37: Instead of an OSError with a "no such file or directory" message, 265 | a ValueError is raised with a more informative "Command not found" message. 266 | 267 | - Fixed #38: Replaced ``async`` keyword argument with ``async_``, as ``async`` 268 | has become a keyword in Python 3.7. 269 | 270 | - Fixed #39: Updated tutorial example on progress monitoring. 271 | 272 | 0.1.4 273 | ~~~~~ 274 | 275 | Released: 2015-01-24 276 | 277 | - Fixed issue #21: Don't parse if shell=True. 278 | 279 | - Fixed issue #20: Run pipeline in separate thread if async. 280 | 281 | - Fixed issue #23: Return the correct return code when shell=True. 282 | 283 | - Improved logging. 284 | 285 | - Minor documentation updates. 286 | 287 | - Minor additions to tests. 288 | 289 | 0.1.3 290 | ~~~~~ 291 | 292 | Released: 2014-01-17 293 | 294 | - Fixed issue #15: Handled subprocess internal changes in Python 2.7.6. 295 | 296 | - Improved logging support. 297 | 298 | - Minor documentation updates. 299 | 300 | 0.1.2 301 | ~~~~~ 302 | 303 | Released: 2013-12-17 304 | 305 | - Fixed issue #13: Removed module globals to improve thread safety. 306 | 307 | - Fixed issue #12: Fixed a hang which occurred when a redirection failed. 308 | 309 | - Fixed issue #11: Added ``+`` to the characters allowed in parameters. 310 | 311 | - Fixed issue #10: Removed a spurious debugger breakpoint. 312 | 313 | - Fixed issue #9: Relative pathnames in redirections are now relative to the 314 | current working directory for the redirected process. 315 | 316 | - Added the ability to pass objects with ``fileno()`` methods as values 317 | to the ``input`` argument of ``run()``, and a ``Feeder`` class which 318 | facilitates passing data to child processes dynamically over time (rather 319 | than just an initial string, byte-string or file). 320 | 321 | - Added functionality under Windows to use PATH, PATHEXT and the 322 | registry to find appropriate commands. This can e.g. convert a 323 | command ``'foo bar'``, if ``'foo.py'`` is a Python script in the 324 | ``c:\Tools`` directory which is on the path, to the equivalent 325 | ``'c:\Python26\Python.exe c:\Tools\foo.py bar'``. This is done internally 326 | when a command is parsed, before it is passed to ``subprocess``. 327 | 328 | - Fixed issue #7: Corrected handling of whitespace and redirections. 329 | 330 | - Fixed issue #8: Added a missing import. 331 | 332 | - Added Travis integration. 333 | 334 | - Added encoding parameter to the ``Capture`` initializer. 335 | 336 | - Fixed issue #6: addressed bugs in Capture logic so that iterating over 337 | captures is closer to ``subprocess`` behaviour. 338 | 339 | - Tests added to cover added functionality and reported issues. 340 | 341 | - Numerous documentation updates. 342 | 343 | 344 | 0.1.1 345 | ~~~~~ 346 | 347 | Released: 2013-06-04 348 | 349 | - ``expect`` method added to ``Capture`` class, to allow searching 350 | for specific patterns in subprocess output streams. 351 | 352 | - added ``terminate``, ``kill`` and ``poll`` methods to ``Command`` 353 | class to operate on the wrapped subprocess. 354 | 355 | - ``Command.run`` now propagates exceptions which occur while spawning 356 | subprocesses. 357 | 358 | - Fixed issue #4: ``shell_shlex`` does not split on ``@``. 359 | 360 | - Fixed issue #3: ``run`` et al now accept commands as lists, just as 361 | ``subprocess.Popen`` does. 362 | 363 | - Fixed issue #2: ``shell_quote`` implementation improved. 364 | 365 | - Improved ``shell_shlex`` resilience by handling Unicode on 2.x (where 366 | ``shlex`` breaks if passed Unicode). 367 | 368 | - Added ``get_stdout``, ``get_stderr`` and ``get_both`` for when subprocess 369 | output is not expected to be voluminous. 370 | 371 | - Added an internal lock to serialise access to shared data. 372 | 373 | - Tests added to cover added functionality and reported issues. 374 | 375 | - Numerous documentation updates. 376 | 377 | 378 | 0.1 379 | ~~~ 380 | 381 | Released: 2012-02-10 382 | 383 | - Initial release. 384 | 385 | 386 | Next steps 387 | ---------- 388 | 389 | You might find it helpful to look at the :ref:`tutorial`, or the 390 | :ref:`reference`. 391 | -------------------------------------------------------------------------------- /docs/reference.rst: -------------------------------------------------------------------------------- 1 | .. _reference: 2 | 3 | .. currentmodule:: sarge 4 | 5 | API Reference 6 | ============= 7 | 8 | This is the place where the functions and classes in ``sarge's`` public API 9 | are described. 10 | 11 | Attributes 12 | ---------- 13 | 14 | .. attribute:: default_capture_timeout 15 | 16 | This is the default timeout which will be used by :class:`Capture` 17 | instances when you don't specify one in the :class:`Capture` constructor. 18 | This is currently set to **0.02 seconds**. 19 | 20 | Functions 21 | --------- 22 | 23 | .. function:: run(command, input=None, async_=False, **kwargs) 24 | 25 | This function is a convenience wrapper which constructs a 26 | :class:`Pipeline` instance from the passed parameters, and then invokes 27 | :meth:`~Pipeline.run` and :meth:`~Pipeline.close` on that instance. 28 | 29 | :param command: The command(s) to run. 30 | :type command: str 31 | :param input: Input data to be passed to the command(s). If text is passed, 32 | it's converted to ``bytes`` using the default encoding. The 33 | bytes are converted to a file-like object (a 34 | :class:`BytesIO` instance). If a value such as a file-like 35 | object, integer file descriptor or special value like 36 | ``subprocess.PIPE`` is passed, it is passed through 37 | unchanged to :class:`subprocess.Popen`. 38 | :type input: Text, bytes or a file-like object containing bytes (not text). 39 | :param kwargs: Any keyword parameters which you might want to pass to the 40 | wrapped :class:`Pipeline` instance. Apart from the ``input`` 41 | and ``async_`` keyword arguments described above, other 42 | keyword arguments are passed to the wrapped 43 | :class:`Pipeline` instance, and thence to 44 | :class:`subprocess.Popen` via a :class:`Command` instance. 45 | Note that the ``env`` kwarg is treated differently to how it 46 | is in :class:`~subprocess.Popen`: it is treated as a set of 47 | *additional* environment variables to be added to the values 48 | in ``os.environ``, unless ``replace_env`` is also specified 49 | as true, in which case the value of ``env`` becomes the 50 | entire child process environment. 51 | 52 | 53 | Under Windows, you might find it useful to pass the keyword argument 54 | ``posix=True``, which will cause `command` to be parsed using POSIX 55 | conventions. This makes it easier to pass parameters with spaces in 56 | them. 57 | 58 | :return: The created :class:`Pipeline` instance. 59 | 60 | .. versionchanged:: 0.1.5 61 | The ``async`` keyword parameter was changed to ``async_``, as ``async`` 62 | is a keyword in Python 3.7 and later. 63 | 64 | .. function:: capture_stdout(command, input=None, async_=False, **kwargs) 65 | 66 | This function is a convenience wrapper which does the same as :func:`run` 67 | while capturing the ``stdout`` of the subprocess(es). This captured output 68 | is available through the ``stdout`` attribute of the return value from 69 | this function. 70 | 71 | :param command: As for :func:`run`. 72 | :param input: As for :func:`run`. 73 | :param kwargs: As for :func:`run`. 74 | :return: As for :func:`run`. 75 | 76 | .. versionchanged:: 0.1.5 77 | The ``async`` keyword parameter was changed to ``async_``, as ``async`` 78 | is a keyword in Python 3.7 and later. 79 | 80 | .. function:: get_stdout(command, input=None, async_=False, **kwargs) 81 | 82 | This function is a convenience wrapper which does the same as 83 | :func:`capture_stdout` but also returns the text captured. Use this when 84 | you know the output is not voluminous, so it doesn't matter that it's 85 | buffered in memory. 86 | 87 | :param command: As for :func:`run`. 88 | :param input: As for :func:`run`. 89 | :param kwargs: As for :func:`run`. 90 | :return: The captured text. 91 | 92 | .. versionadded:: 0.1.1 93 | 94 | .. versionchanged:: 0.1.5 95 | The ``async`` keyword parameter was changed to ``async_``, as ``async`` 96 | is a keyword in Python 3.7 and later. 97 | 98 | .. function:: capture_stderr(command, input=None, async_=False, **kwargs) 99 | 100 | This function is a convenience wrapper which does the same as :func:`run` 101 | while capturing the ``stderr`` of the subprocess(es). This captured output 102 | is available through the ``stderr`` attribute of the return value from 103 | this function. 104 | 105 | :param command: As for :func:`run`. 106 | :param input: As for :func:`run`. 107 | :param kwargs: As for :func:`run`. 108 | :return: As for :func:`run`. 109 | 110 | .. versionchanged:: 0.1.5 111 | The ``async`` keyword parameter was changed to ``async_``, as ``async`` 112 | is a keyword in Python 3.7 and later. 113 | 114 | .. function:: get_stderr(command, input=None, async_=False, **kwargs) 115 | 116 | This function is a convenience wrapper which does the same as 117 | :func:`capture_stderr` but also returns the text captured. Use this when 118 | you know the output is not voluminous, so it doesn't matter that it's 119 | buffered in memory. 120 | 121 | :param command: As for :func:`run`. 122 | :param input: As for :func:`run`. 123 | :param kwargs: As for :func:`run`. 124 | :return: The captured text. 125 | 126 | .. versionadded:: 0.1.1 127 | 128 | .. versionchanged:: 0.1.5 129 | The ``async`` keyword parameter was changed to ``async_``, as ``async`` 130 | is a keyword in Python 3.7 and later. 131 | 132 | .. function:: capture_both(command, input=None, async_=False, **kwargs) 133 | 134 | This function is a convenience wrapper which does the same as :func:`run` 135 | while capturing the ``stdout`` and the ``stderr`` of the subprocess(es). 136 | This captured output is available through the ``stdout`` and 137 | ``stderr`` attributes of the return value from this function. 138 | 139 | :param command: As for :func:`run`. 140 | :param input: As for :func:`run`. 141 | :param kwargs: As for :func:`run`. 142 | :return: As for :func:`run`. 143 | 144 | .. versionchanged:: 0.1.5 145 | The ``async`` keyword parameter was changed to ``async_``, as ``async`` 146 | is a keyword in Python 3.7 and later. 147 | 148 | .. function:: get_both(command, input=None, async_=False, **kwargs) 149 | 150 | This function is a convenience wrapper which does the same as 151 | :func:`capture_both` but also returns the text captured. Use this when 152 | you know the output is not voluminous, so it doesn't matter that it's 153 | buffered in memory. 154 | 155 | :param command: As for :func:`run`. 156 | :param input: As for :func:`run`. 157 | :param kwargs: As for :func:`run`. 158 | :return: The captured text as a 2-element tuple, with the ``stdout`` text 159 | in the first element and the ``stderr`` text in the second. 160 | 161 | .. versionadded:: 0.1.1 162 | 163 | 164 | .. versionchanged:: 0.1.5 165 | The ``async`` keyword parameter was changed to ``async_``, as ``async`` 166 | is a keyword in Python 3.7 and later. 167 | 168 | .. function:: shell_quote(s) 169 | 170 | Quote text so that it is safe for POSIX command shells. 171 | 172 | For example, "*.py" would be converted to "'*.py'". If the text is 173 | considered safe it is returned unquoted. 174 | 175 | :param s: The value to quote 176 | :type s: str, or unicode on 2.x 177 | :return: A safe version of the input, from the point of view of POSIX 178 | command shells 179 | 180 | .. function:: shell_format(fmt, *args, **kwargs) 181 | 182 | Format a shell command with format placeholders and variables to fill 183 | those placeholders. 184 | 185 | Note: you must specify positional parameters explicitly, i.e. as {0}, {1} 186 | instead of {}, {}. Requiring the formatter to maintain its own counter can 187 | lead to thread safety issues unless a thread local is used to maintain 188 | the counter. It's not that hard to specify the values explicitly 189 | yourself :-) 190 | 191 | :param fmt: The shell command as a format string. Note that you will need 192 | to double up braces you want in the result, i.e. { -> {{ and 193 | } -> }}, due to the way :meth:`str.format` works. 194 | :type fmt: str, or unicode on 2.x 195 | :param args: Positional arguments for use with ``fmt``. 196 | :param kwargs: Keyword arguments for use with ``fmt``. 197 | :return: The formatted shell command, which should be safe for use in 198 | shells from the point of view of shell injection. 199 | :rtype: The type of ``fmt``. 200 | 201 | Classes 202 | ------- 203 | 204 | .. class:: Command(args, **kwargs) 205 | 206 | This represents a single command to be spawned as a subprocess. 207 | 208 | :param args: The command to run. 209 | :type args: str if ``shell=True``, or an array of str 210 | :param kwargs: Any keyword parameters you might pass to 211 | :class:`~subprocess.Popen`, other than ``stdin`` (for which, 212 | you need to see the ``input`` argument of 213 | :meth:`~Command.run`). 214 | 215 | 216 | .. cssclass:: class-members-heading 217 | 218 | Attributes 219 | 220 | .. attribute:: process 221 | 222 | The `subprocess.Popen` instance for the subprocess, once it has been created. It 223 | is initially `None`. 224 | 225 | .. attribute:: returncode 226 | 227 | The subprocess returncode, when that is available. It is initially `None`. 228 | 229 | .. attribute:: exception 230 | 231 | Any exception which occurred when trying to create the subprocess. Note that once 232 | a subprocess has been created, any exceptions in the subprocess can only be 233 | communicated via the :attr:`returncode` - this value is *only* for exceptions 234 | during subprocess creation. 235 | 236 | .. versionadded:: 0.1.8 237 | 238 | .. cssclass:: class-members-heading 239 | 240 | Methods 241 | 242 | .. method:: run(input=None, async_=False) 243 | 244 | Run the command. 245 | 246 | :param input: Input data to be passed to the command. If text is 247 | passed, it's converted to ``bytes`` using the default 248 | encoding. The bytes are converted to a file-like object (a 249 | :class:`BytesIO` instance). The contents of the 250 | file-like object are written to the ``stdin`` 251 | stream of the sub-process. 252 | :type input: Text, bytes or a file-like object containing bytes. 253 | :param async_: If ``True``, the command is run asynchronously -- that is 254 | to say, :meth:`wait` is not called on the underlying 255 | :class:`~subprocess.Popen` instance. 256 | :type async_: bool 257 | 258 | .. versionchanged:: 0.1.5 259 | The ``async`` keyword parameter was changed to ``async_``, as ``async`` 260 | is a keyword in Python 3.7 and later. 261 | 262 | .. method:: wait(timeout=None) 263 | 264 | Wait for the command's underlying sub-process to complete, with a specified 265 | timeout. If the timeout is reached, a ``subprocess.TimeoutExpired`` exception 266 | is raised. The timeout is ignored in versions of Python < 3.3. 267 | 268 | .. versionchanged:: 0.1.6 269 | The ``timeout`` parameter was added. 270 | 271 | .. method:: terminate() 272 | 273 | Terminate the command's underlying sub-process by calling 274 | :meth:`subprocess.Popen.terminate` on it. 275 | 276 | .. versionadded:: 0.1.1 277 | 278 | .. method:: kill() 279 | 280 | Kill the command's underlying sub-process by calling 281 | :meth:`subprocess.Popen.kill` on it. 282 | 283 | .. versionadded:: 0.1.1 284 | 285 | .. method:: poll() 286 | 287 | Poll the command's underlying sub-process by calling 288 | :meth:`subprocess.Popen.poll` on it. Returns the result of that call. 289 | 290 | .. versionadded:: 0.1.1 291 | 292 | 293 | .. class:: Pipeline(source, posix=True, **kwargs) 294 | 295 | This represents a set of commands which need to be run as a unit. 296 | 297 | :param source: The source text with the command(s) to run. 298 | :type source: str 299 | :param posix: Whether the source will be parsed using POSIX conventions. 300 | :type posix: bool 301 | :param kwargs: Any keyword parameters you would pass to 302 | :class:`subprocess.Popen`, other than ``stdin`` (for which, 303 | you need to use the ``input`` parameter of the 304 | :meth:`~Pipeline.run` method instead). You can pass 305 | :class:`Capture` instances for ``stdout`` and ``stderr`` 306 | keyword arguments, which will cause those streams to be 307 | captured to those instances. 308 | 309 | .. cssclass:: class-members-heading 310 | 311 | Attributes 312 | 313 | .. attribute:: returncodes 314 | 315 | A list of the return codes of all sub-processes which were actually run. This 316 | will internally poll the commands in the pipeline to find the latest known return 317 | codes. 318 | 319 | .. attribute:: returncode 320 | 321 | The return code of the last sub-process which was actually run. 322 | 323 | .. attribute:: commands 324 | 325 | The :class:`Command` instances which were actually created. 326 | 327 | .. attribute:: exceptions 328 | 329 | A list of any exceptions creating subprocesses. This should be of use in 330 | diagnosing problems with commands (e.g. typos, or executables correctly spelled 331 | but not found on the system path). 332 | 333 | .. versionadded:: 0.1.8 334 | 335 | .. cssclass:: class-members-heading 336 | 337 | Methods 338 | 339 | .. method:: run(input=None, async_=False) 340 | 341 | Run the pipeline. 342 | 343 | :param input: The same as for the :meth:`Command.run` method. 344 | :param async_: The same as for the :meth:`Command.run` method. Note that 345 | parts of the pipeline may specify synchronous or 346 | asynchronous running -- this flag refers to the pipeline 347 | as a whole. 348 | 349 | .. versionchanged:: 0.1.5 350 | The ``async`` keyword parameter was changed to ``async_``, as ``async`` 351 | is a keyword in Python 3.7 and later. 352 | 353 | .. method:: wait(timeout=None) 354 | 355 | Wait for all command sub-processes to finish, with an optional timeout. If the 356 | timeout is reached, a ``subprocess.TimeoutExpired`` exception is raised. The 357 | timeout is ignored in versions of Python < 3.3. If applied, it is applied to each 358 | of the pipeline's commands in turn, which means that the effective timeout might 359 | be cumulative. 360 | 361 | .. versionchanged:: 0.1.6 362 | The ``timeout`` parameter was added. 363 | 364 | .. method:: poll_last() 365 | 366 | Check if the last command in the pipeline has terminated, and return its exit 367 | code, if available, or else `None`. Note that :meth:~Pipeline.poll_all` should be 368 | called when you want to ensure that all commands in a pipeline have completed. 369 | 370 | .. method:: poll_all() 371 | 372 | Check if all commands to run have terminated. Return a list of exit codes, where 373 | available, or `None` values where return codes are not yet available. 374 | 375 | .. method:: close() 376 | 377 | Wait for all command sub-processes to finish, and close all opened 378 | streams. 379 | 380 | .. class:: Capture(timeout=None, buffer_size=0) 381 | 382 | A class which allows an output stream from a sub-process to be captured. 383 | 384 | :param timeout: The default timeout, in seconds. Note that you can 385 | override this in particular calls to read input. If 386 | ``None`` is specified, the value of the module attribute 387 | ``default_capture_timeout`` is used instead. 388 | :type timeout: float 389 | :param buffer_size: The buffer size to use when reading from the underlying 390 | streams. If not specified or specified as zero, a 4K 391 | buffer is used. For interactive applications, use a value 392 | of 1. 393 | :type buffer_size: int 394 | 395 | .. cssclass:: class-members-heading 396 | 397 | Methods 398 | 399 | .. method:: read(size=-1, block=True, timeout=None) 400 | 401 | Like the ``read`` method of any file-like object. 402 | 403 | :param size: The number of bytes to read. If not specified, the intent is 404 | to read the stream until it is exhausted. 405 | :type size: int 406 | :param block: Whether to block waiting for input to be available, 407 | :type block: bool 408 | :param timeout: How long to wait for input. If ``None``, 409 | use the default timeout that this instance was 410 | initialised with. If the result is ``None``, wait 411 | indefinitely. 412 | :type timeout: float 413 | 414 | .. method:: readline(size=-1, block=True, timeout=None) 415 | 416 | Like the ``readline`` method of any file-like object. 417 | 418 | :param size: As for the :meth:`~Capture.read` method. 419 | :param block: As for the :meth:`~Capture.read` method. 420 | :param timeout: As for the :meth:`~Capture.read` method. 421 | 422 | .. method:: readlines(sizehint=-1, block=True, timeout=None) 423 | 424 | Like the ``readlines`` method of any file-like object. 425 | 426 | :param sizehint: As for the :meth:`~Capture.read` method's ``size``. 427 | :param block: As for the :meth:`~Capture.read` method. 428 | :param timeout: As for the :meth:`~Capture.read` method. 429 | 430 | .. method:: expect(string_or_pattern, timeout=None) 431 | 432 | This looks for a pattern in the captured output stream. If found, it 433 | returns immediately; otherwise, it will block until the timeout expires, 434 | waiting for a match as bytes from the captured stream continue to be read. 435 | 436 | :param string_or_pattern: A string or pattern representing a regular 437 | expression to match. Note that this needs to 438 | be a bytestring pattern if you pass a pattern 439 | in; if you pass in text, it is converted to 440 | bytes using the ``utf-8`` codec and then to 441 | a pattern used for matching (using ``search``). 442 | If you pass in a pattern, you may want to 443 | ensure that its flags include ``re/MULTILINE`` 444 | so that you can make use of ``^`` and ``$`` in 445 | matching line boundaries. Note that on Windows, 446 | you may need to use ``\r?$`` to match ends of 447 | lines, as ``$`` matches Unix newlines (LF) and 448 | not Windows newlines (CRLF). 449 | 450 | :param timeout: If not specified, the module's ``default_expect_timeout`` 451 | is used. 452 | :returns: A regular expression match instance, if a match was found 453 | within the specified timeout, or ``None`` if no match was 454 | found. 455 | 456 | .. method:: close(stop_threads=False): 457 | 458 | Close the capture object. By default, this waits for the threads which 459 | read the captured streams to terminate (which may not happen unless the 460 | child process is killed, and the streams read to exhaustion). To ensure 461 | that the threads are stopped immediately, specify ``True`` for the 462 | ``stop_threads`` parameter, which will asks the threads to terminate 463 | immediately. This may lead to losing data from the captured streams 464 | which has not yet been read. 465 | 466 | 467 | .. class:: Popen 468 | 469 | This is a subclass of :class:`subprocess.Popen` which is provided mainly 470 | to allow a process' ``stdout`` to be mapped to its ``stderr``. The 471 | standard library version allows you to specify ``stderr=STDOUT`` to 472 | indicate that the standard error stream of the sub-process be the same as 473 | its standard output stream. However. there's no facility in the standard 474 | library to do ``stdout=STDERR`` -- but it *is* provided in this subclass. 475 | 476 | In fact, the two streams can be swapped by doing ``stdout=STDERR, 477 | stderr=STDOUT`` in a call. The ``STDERR`` value is defined in ``sarge`` 478 | as an integer constant which is understood by ``sarge`` (much as 479 | ``STDOUT`` is an integer constant which is understood by ``subprocess``). 480 | 481 | 482 | Shell syntax understood by ``sarge`` 483 | ------------------------------------ 484 | 485 | Shell commands are parsed by ``sarge`` using a simple parser. 486 | 487 | Command syntax 488 | ^^^^^^^^^^^^^^ 489 | 490 | The ``sarge`` parser looks for commands which are separated by ``;`` and ``&``:: 491 | 492 | echo foo; echo bar & echo baz 493 | 494 | which means to run `echo foo`, wait for its completion, 495 | and then run ``echo bar`` and then ``echo baz`` without waiting for ``echo 496 | bar`` to complete. 497 | 498 | The commands which are separated by ``&`` and ``;`` are *conditional* commands, 499 | of the form:: 500 | 501 | a && b 502 | 503 | or:: 504 | 505 | c || d 506 | 507 | Here, command ``b`` is executed only if ``a`` returns success (i.e. a 508 | return code of 0), whereas ``d`` is only executed if ``c`` returns failure, 509 | i.e. a return code other than 0. Of course, in practice all of ``a``, ``b``, 510 | ``c`` and ``d`` could have arguments, not shown above for simplicity's sake. 511 | 512 | Each operand on either side of ``&&`` or ``||`` could also consist of a 513 | pipeline -- a set of commands connected such that the output streams of one 514 | feed into the input stream of another. For example:: 515 | 516 | echo foo | cat 517 | 518 | or:: 519 | 520 | command-a |& command-b 521 | 522 | where the use of ``|`` indicates that the standard output of ``echo foo`` is 523 | piped to the input of ``cat``, whereas the standard error of ``command-a`` is 524 | piped to the input of ``command-b``. 525 | 526 | Redirections 527 | ^^^^^^^^^^^^ 528 | 529 | The ``sarge`` parser also understands redirections such as are shown in the 530 | following examples:: 531 | 532 | command arg-1 arg-2 > stdout.txt 533 | command arg-1 arg-2 2> stderr.txt 534 | command arg-1 arg-2 2>&1 535 | command arg-1 arg-2 >&2 536 | 537 | In general, file descriptors other than 1 and 2 are not allowed, 538 | as the functionality needed to provided them (``dup2``) is not properly 539 | supported on Windows. However, an esoteric special case *is* recognised:: 540 | 541 | echo foo | tee stdout.log 3>&1 1>&2 2>&3 | tee stderr.log > /dev/null 542 | 543 | This redirection construct will put ``foo`` in both ``stdout.log`` *and* 544 | ``stderr.log``. The effect of this construct is to swap the standard output 545 | and standard error streams, using file descriptor 3 as a temporary as in the 546 | code analogue for swapping variables ``a`` and ``b`` using temporary variable 547 | ``c``:: 548 | 549 | c = a 550 | a = b 551 | b = c 552 | 553 | This is recognised by ``sarge`` and used to swap the two streams, 554 | though it doesn't literally use file descriptor ``3``, 555 | instead using a cross-platform mechanism to fulfill the requirement. 556 | 557 | You can see `this post `_ for a longer explanation of 558 | this somewhat esoteric usage of redirection. 559 | 560 | Next steps 561 | ---------- 562 | 563 | You might find it helpful to look at the 564 | `mailing list `_. 565 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinxcontrib-spelling==7.6.2 2 | sphinx<7 3 | sphinx-rtd-theme>=1.2.2 4 | -------------------------------------------------------------------------------- /docs/spelling_wordlist.txt: -------------------------------------------------------------------------------- 1 | Coghlan 2 | Sarge 3 | Reitz 4 | nitty 5 | pypy 6 | macOS 7 | kwarg 8 | es 9 | py 10 | returncode 11 | bytestring 12 | fulfill 13 | virtualenv 14 | ftype 15 | assoc 16 | tty 17 | etc 18 | -------------------------------------------------------------------------------- /docs/tutorial.rst: -------------------------------------------------------------------------------- 1 | .. _tutorial: 2 | 3 | Tutorial 4 | ======== 5 | 6 | This is the place to start your practical exploration of ``sarge``. 7 | 8 | Installation and testing 9 | ------------------------ 10 | 11 | sarge is a pure-Python library. You should be able to install it using:: 12 | 13 | pip install sarge 14 | 15 | for installing ``sarge`` into a virtualenv or other directory where you have 16 | write permissions. On POSIX platforms, you may need to invoke using ``sudo`` 17 | if you need to install ``sarge`` in a protected location such as your system 18 | Python's ``site-packages`` directory. 19 | 20 | A full test suite is included with ``sarge``. To run it, you'll need to unpack 21 | a source tarball and run ``python setup.py test`` in the top-level directory 22 | of the unpack location. You can of course also run ``pip install `` 23 | to install from the source tarball (perhaps invoking with ``sudo`` if you need 24 | to install to a protected location). 25 | 26 | Common usage patterns 27 | --------------------- 28 | 29 | In the simplest cases, sarge doesn't provide any major advantage over 30 | ``subprocess``:: 31 | 32 | >>> from sarge import run 33 | >>> run('echo "Hello, world!"') 34 | Hello, world! 35 | 36 | 37 | The ``echo`` command got run, as expected, and printed its output on the 38 | console. In addition, a ``Pipeline`` object got returned. Don't worry too much 39 | about what this is for now -- it's more useful when more complex combinations 40 | of commands are run. 41 | 42 | By comparison, the analogous case with ``subprocess`` would be:: 43 | 44 | >>> from subprocess import call 45 | >>> call('echo "Hello, world!"'.split()) 46 | "Hello, world!" 47 | 0 48 | 49 | We had to call :meth:`split` on the command (or we could have passed 50 | ``shell=True``), and as well as running the command, the :meth:`call` method 51 | returned the exit code of the subprocess. To get the same effect with ``sarge`` 52 | you have to do:: 53 | 54 | >>> from sarge import run 55 | >>> run('echo "Hello, world!"').returncode 56 | Hello, world! 57 | 0 58 | 59 | If that's as simple as you want to get, then of course you don't need 60 | ``sarge``. Let's look at more demanding uses next. 61 | 62 | Finding commands under Windows 63 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 64 | 65 | In versions 0.1.1 and earlier, ``sarge``, like ``subprocess``, did not do 66 | anything special to find the actual executable to run -- it was expected to be 67 | found in the current directory or the path. Specifically, ``PATHEXT`` was not 68 | supported: where you might type ``yada`` in a command shell and have it run 69 | ``python yada.py`` because ``.py`` is in the ``PATHEXT`` environment variable 70 | and Python is registered to handle files with that extension, neither 71 | ``subprocess`` (with ``shell=False``) nor ``sarge`` did this. You needed to 72 | specify the executable name explicitly in the command passed to ``sarge``. 73 | 74 | In 0.1.2 and later versions, ``sarge`` has improved command-line handling. The 75 | "which" functionality has been backported from Python 3.3, which takes care of 76 | using ``PATHEXT`` to resolve a command ``yada`` as ``c:\Tools\yada.py`` where 77 | ``c:\Tools`` is on the PATH and ``yada.py`` is in there. In addition, ``sarge`` 78 | queries the registry to see which programs are associated with the extension, 79 | and updates the command line accordingly. Thus, a command line ``foo bar`` 80 | passed to ``sarge`` may actually result in ``c:\Windows\py.exe c:\Tools\foo.py 81 | bar`` being passed to ``subprocess`` (assuming the Python Launcher for Windows, 82 | ``py.exe``, is associated with ``.py`` files). 83 | 84 | This new functionality is not limited to Python scripts - it should 85 | work for any extensions which are in ``PATHEXT`` and have an ftype/assoc 86 | binding them to an executable through ``shell``, ``open`` and ``command`` 87 | subkeys in the registry, and where the command line is of the form 88 | ``"" "%1" %*`` (this is the standard form used by several 89 | languages). 90 | 91 | Parsing commands under Windows 92 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 93 | 94 | Sometimes, you may find it useful to pass ``posix=True`` for parsing command lines 95 | under Windows (this is the default on POSIX platforms, so it doesn't need to be passed 96 | explicitly there). Note the differences:: 97 | 98 | >>> from sarge import parse_command_line 99 | >>> parse_command_line('git rev-list origin/master --since="1 hours ago"') 100 | CommandNode(command=['git', 'rev-list', 'origin/master', '--since="1', 'hours', 'ago"'] redirects={}) 101 | >>> parse_command_line('git rev-list origin/master --since="1 hours ago"', posix=True) 102 | CommandNode(command=['git', 'rev-list', 'origin/master', '--since=1 hours ago'] redirects={}) 103 | >>> 104 | 105 | As you can see, passing ``posix=True`` has allowed the ``--since`` parameter to be 106 | correctly handled. 107 | 108 | Chaining commands 109 | ^^^^^^^^^^^^^^^^^ 110 | 111 | It's easy to chain commands together with ``sarge``. For example:: 112 | 113 | >>> run('echo "Hello,"; echo "world!"') 114 | Hello, 115 | world! 116 | 117 | 118 | whereas this would have been more involved if you were just using 119 | ``subprocess``:: 120 | 121 | >>> call('echo "Hello,"'.split()); call('echo "world!"'.split()) 122 | "Hello," 123 | 0 124 | "world!" 125 | 0 126 | 127 | You get two return codes, one for each command. The same information is 128 | available from ``sarge``, in one place -- the :class:`~sarge.Pipeline` instance that's 129 | returned from a :func:`~sarge.run` call:: 130 | 131 | >>> run('echo "Hello,"; echo "world!"').returncodes 132 | Hello, 133 | world! 134 | [0, 0] 135 | 136 | The :attr:`returncodes` property of a :class:`~sarge.Pipeline` instance returns a 137 | list of the return codes of all the commands that were run, 138 | whereas the :attr:`returncode` property just returns the last element of 139 | this list. The :class:`~sarge.Pipeline` class defines a number of useful properties 140 | - see the reference for full details. 141 | 142 | Handling user input safely 143 | ^^^^^^^^^^^^^^^^^^^^^^^^^^ 144 | 145 | By default, ``sarge`` does not run commands via the shell. This means that 146 | wildcard characters in user input do not have potentially dangerous 147 | consequences:: 148 | 149 | >>> run('ls *.py') 150 | ls: cannot access *.py: No such file or directory 151 | 152 | 153 | This behaviour helps to avoid `shell injection 154 | `_ attacks. 155 | 156 | There might be circumstances where you need to use ``shell=True``, 157 | in which case you should consider formatting your commands with placeholders 158 | and quoting any variable parts that you get from external sources (such as 159 | user input). Which brings us on to ... 160 | 161 | Formatting commands with placeholders for safe usage 162 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 163 | 164 | If you need to merge commands with external inputs (e.g. user inputs) and you 165 | want to prevent shell injection attacks, you can use the :func:`~sarge.shell_format` 166 | function. This takes a format string, positional and keyword arguments and 167 | uses the new formatting (:meth:`str.format`) to produce the result:: 168 | 169 | >>> from sarge import shell_format 170 | >>> shell_format('ls {0}', '*.py') 171 | "ls '*.py'" 172 | 173 | Note how the potentially unsafe input has been quoted. With a safe input, 174 | no quoting is done:: 175 | 176 | >>> shell_format('ls {0}', 'test.py') 177 | 'ls test.py' 178 | 179 | If you really want to prevent quoting, even for potentially unsafe inputs, 180 | just use the ``s`` conversion:: 181 | 182 | >>> shell_format('ls {0!s}', '*.py') 183 | 'ls *.py' 184 | 185 | There is also a :func:`~sarge.shell_quote` function which quotes potentially unsafe 186 | input:: 187 | 188 | >>> from sarge import shell_quote 189 | >>> shell_quote('abc') 190 | 'abc' 191 | >>> shell_quote('ab?') 192 | "'ab?'" 193 | >>> shell_quote('"ab?"') 194 | '\'"ab?"\'' 195 | >>> shell_quote("'ab?'") 196 | '"\'ab?\'"' 197 | 198 | This function is used internally by :func:`~sarge.shell_format`, so you shouldn't need 199 | to call it directly except in unusual cases. 200 | 201 | Passing input data to commands 202 | ------------------------------ 203 | 204 | You can pass input to a command pipeline using the ``input`` keyword parameter 205 | to :func:`~sarge.run`:: 206 | 207 | >>> from sarge import run 208 | >>> p = run('cat|cat', input='foo') 209 | foo>>> 210 | 211 | Here's how the value passed as ``input`` is processed: 212 | 213 | * Text is encoded to bytes using UTF-8, which is then wrapped in a ``BytesIO`` 214 | object. 215 | * Bytes are wrapped in a ``BytesIO`` object. 216 | * Starting with 0.1.2, if you pass an object with a ``fileno`` attribute, 217 | that will be called as a method and the resulting value will be passed to 218 | the ``subprocess`` layer. This would normally be a readable file descriptor. 219 | * Other values (such as integers representing OS-level file descriptors, or 220 | special values like ``subprocess.PIPE``) are passed to the ``subprocess`` 221 | layer as-is. 222 | 223 | If the result of the above process is a ``BytesIO`` instance (or if you passed 224 | in a ``BytesIO`` instance), then ``sarge`` will spin up an internal thread to 225 | write the data to the child process when it is spawned. The reason for a 226 | separate thread is that if the child process consumes data slowly, or the size 227 | of data is large, then the calling thread would block for potentially long 228 | periods of time. 229 | 230 | Passing input data to commands dynamically 231 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 232 | 233 | Sometimes, you may want to pass quite a lot of data to a child process which 234 | is not conveniently available as a string, byte-string or a file, but which 235 | is generated in the parent process (the one using ``sarge``) by some other 236 | means. Starting with 0.1.2, ``sarge`` facilitates this by supporting objects 237 | with ``fileno()`` attributes as described above, and includes a ``Feeder`` 238 | class which has a suitable ``fileno()`` implementation. 239 | 240 | Creating and using a feeder is simple:: 241 | 242 | import sys 243 | from sarge import Feeder, run 244 | 245 | feeder = Feeder() 246 | run([sys.executable, 'echoer.py'], input=feeder, async_=True) 247 | 248 | After this, you can feed data to the child process' ``stdin`` by calling the 249 | ``feed()`` method of the ``Feeder`` instance:: 250 | 251 | feeder.feed('Hello') 252 | feeder.feed(b'Goodbye') 253 | 254 | If you pass in text, it will be encoded to bytes using UTF-8. 255 | 256 | Once you've finished with the feeder, you can close it:: 257 | 258 | feeder.close() 259 | 260 | Depending on how quickly the child process consumes data, the thread calling 261 | ``feed()`` might block on I/O. If this is a problem, you can spawn a separate 262 | thread which does the feeding. 263 | 264 | Here's a complete working example:: 265 | 266 | import os 267 | import subprocess 268 | import sys 269 | import time 270 | 271 | import sarge 272 | 273 | try: 274 | text_type = unicode 275 | except NameError: 276 | text_type = str 277 | 278 | def main(args=None): 279 | feeder = sarge.Feeder() 280 | p = sarge.run([sys.executable, 'echoer.py'], input=feeder, async_=True) 281 | try: 282 | lines = ('hello', 'goodbye') 283 | gen = iter(lines) 284 | while p.commands[0].returncode is None: 285 | try: 286 | data = next(gen) 287 | except StopIteration: 288 | break 289 | feeder.feed(data + '\n') 290 | p.commands[0].poll() 291 | time.sleep(0.05) # wait for child to return echo 292 | finally: 293 | p.commands[0].terminate() 294 | feeder.close() 295 | 296 | if __name__ == '__main__': 297 | try: 298 | rc = main() 299 | except Exception as e: 300 | print(e) 301 | rc = 9 302 | sys.exit(rc) 303 | 304 | In the above example, the ``echoer.py`` script (included in the ``sarge`` 305 | source distribution, as it's part of the test suite) just reads lines from its 306 | ``stdin``, duplicates and prints to its ``stdout``. Since we passed in the 307 | strings ``hello`` and ``goodbye``, the output from the script should be:: 308 | 309 | hello hello 310 | goodbye goodbye 311 | 312 | 313 | Chaining commands conditionally 314 | ------------------------------- 315 | 316 | You can use ``&&`` and ``||`` to chain commands conditionally using 317 | short-circuit Boolean semantics. For example:: 318 | 319 | >>> from sarge import run 320 | >>> run('false && echo foo') 321 | 322 | 323 | Here, ``echo foo`` wasn't called, because the ``false`` command evaluates to 324 | ``False`` in the shell sense (by returning an exit code other than zero). 325 | Conversely:: 326 | 327 | >>> run('false || echo foo') 328 | foo 329 | 330 | 331 | Here, ``foo`` is output because we used the ``||`` condition; because the left- 332 | hand operand evaluates to ``False``, the right-hand operand is evaluated (i.e. 333 | run, in this context). Similarly, using the ``true`` command:: 334 | 335 | >>> run('true && echo foo') 336 | foo 337 | 338 | >>> run('true || echo foo') 339 | 340 | 341 | 342 | Creating command pipelines 343 | -------------------------- 344 | 345 | It's just as easy to construct command pipelines:: 346 | 347 | >>> run('echo foo | cat') 348 | foo 349 | 350 | >>> run('echo foo; echo bar | cat') 351 | foo 352 | bar 353 | 354 | 355 | Using redirection 356 | ----------------- 357 | 358 | You can also use redirection to files as you might expect. For example:: 359 | 360 | >>> run('echo foo | cat > /tmp/junk') 361 | 362 | ^D (to exit Python) 363 | $ cat /tmp/junk 364 | foo 365 | 366 | You can use ``>``, ``>>``, ``2>``, ``2>>`` which all work as on POSIX systems. 367 | However, you can't use ``<`` or ``<<``. 368 | 369 | To send things to the bit-bucket in a cross-platform way, 370 | you can do something like:: 371 | 372 | >>> run('echo foo | cat > %s' % os.devnull) 373 | 374 | 375 | Capturing ``stdout`` and ``stderr`` from commands 376 | ------------------------------------------------- 377 | 378 | To capture output for commands, just pass a :class:`~sarge.Capture` instance for the 379 | relevant stream:: 380 | 381 | >>> from sarge import run, Capture 382 | >>> p = run('echo foo; echo bar | cat', stdout=Capture()) 383 | >>> p.stdout.text 384 | u'foo\nbar\n' 385 | 386 | 387 | The :class:`~sarge.Capture` instance acts like a stream you can read from: it has 388 | :meth:`~sarge.Capture.read`, :meth:`~sarge.Capture.readline` and 389 | :meth:`~sarge.Capture.readlines` methods which you can call just like on any 390 | file-like object, except that they offer additional options through ``block`` 391 | and ``timeout`` keyword parameters. 392 | 393 | As in the above example, you can use the ``bytes`` or ``text`` property of a 394 | :class:`~sarge.Capture` instance to read all the bytes or text captured. The latter 395 | just decodes the former using UTF-8 (the default encoding isn't used, 396 | because on Python 2.x, the default encoding isn't UTF-8 -- it's ASCII). 397 | 398 | There are some convenience functions -- :func:`~sarge.capture_stdout`, 399 | :func:`~sarge.capture_stderr` and :func:`~sarge.capture_both` -- which work just like 400 | :func:`~sarge.run` but capture the relevant streams to :class:`~sarge.Capture` instances, 401 | which can be accessed using the appropriate attribute on the 402 | :class:`~sarge.Pipeline` instance returned from the functions. 403 | 404 | There are more convenience functions, :func:`~sarge.get_stdout`, :func:`~sarge.get_stderr` 405 | and :func:`~sarge.get_both`, which work just like :func:`~sarge.capture_stdout`, 406 | :func:`~sarge.capture_stderr` and :func:`~sarge.capture_both` respectively, but return the 407 | captured text. For example:: 408 | 409 | >>> from sarge import get_stdout 410 | >>> get_stdout('echo foo; echo bar') 411 | u'foo\nbar\n' 412 | 413 | .. versionadded:: 0.1.1 414 | The :func:`~sarge.get_stdout`, :func:`~sarge.get_stderr` and :func:`~sarge.get_both` functions 415 | were added. 416 | 417 | 418 | A :class:`~sarge.Capture` instance can capture output from one or 419 | more sub-process streams, and will create a thread for each such stream so 420 | that it can read all sub-process output without causing the sub-processes to 421 | block on their output I/O. However, if you use a :class:`~sarge.Capture`, 422 | you should be prepared either to consume what it's read from the 423 | sub-processes, or else be prepared for it all to be buffered in memory (which 424 | may be problematic if the sub-processes generate a *lot* of output). 425 | 426 | Iterating over captures 427 | ----------------------- 428 | 429 | You can iterate over :class:`~sarge.Capture` instances. By default you will get 430 | successive lines from the captured data, as bytes; if you want text, 431 | you can wrap with :class:`io.TextIOWrapper`. Here's an example using Python 432 | 3.2:: 433 | 434 | >>> from sarge import capture_stdout 435 | >>> p = capture_stdout('echo foo; echo bar') 436 | >>> for line in p.stdout: print(repr(line)) 437 | ... 438 | b'foo\n' 439 | b'bar\n' 440 | >>> p = capture_stdout('echo bar; echo baz') 441 | >>> from io import TextIOWrapper 442 | >>> for line in TextIOWrapper(p.stdout): print(repr(line)) 443 | ... 444 | 'bar\n' 445 | 'baz\n' 446 | 447 | This works the same way in Python 2.x. Using Python 2.7:: 448 | 449 | >>> from sarge import capture_stdout 450 | >>> p = capture_stdout('echo foo; echo bar') 451 | >>> for line in p.stdout: print(repr(line)) 452 | ... 453 | 'foo\n' 454 | 'bar\n' 455 | >>> p = capture_stdout('echo bar; echo baz') 456 | >>> from io import TextIOWrapper 457 | >>> for line in TextIOWrapper(p.stdout): print(repr(line)) 458 | ... 459 | u'bar\n' 460 | u'baz\n' 461 | 462 | 463 | Interacting with child processes 464 | -------------------------------- 465 | 466 | Sometimes you need to interact with a child process in an interactive manner. 467 | To illustrate how to do this, consider the following simple program, 468 | named ``receiver``, which will be used as the child process:: 469 | 470 | #!/usr/bin/env python 471 | import sys 472 | 473 | def main(args=None): 474 | while True: 475 | user_input = sys.stdin.readline().strip() 476 | if not user_input: 477 | break 478 | s = 'Hi, %s!\n' % user_input 479 | sys.stdout.write(s) 480 | sys.stdout.flush() # need this when run as a subprocess 481 | 482 | if __name__ == '__main__': 483 | sys.exit(main()) 484 | 485 | This just reads lines from the input and echoes them back as a greeting. If 486 | we run it interactively:: 487 | 488 | $ ./receiver 489 | Fred 490 | Hi, Fred! 491 | Jim 492 | Hi, Jim! 493 | Sheila 494 | Hi, Sheila! 495 | 496 | The program exits on seeing an empty line. 497 | 498 | We can now show how to interact with this program from a parent process:: 499 | 500 | >>> from sarge import Command, Capture 501 | >>> from subprocess import PIPE 502 | >>> p = Command('./receiver', stdout=Capture(buffer_size=1)) 503 | >>> p.run(input=PIPE, async_=True) 504 | Command('./receiver') 505 | >>> p.stdin.write('Fred\n') 506 | >>> p.stdout.readline() 507 | 'Hi, Fred!\n' 508 | >>> p.stdin.write('Jim\n') 509 | >>> p.stdout.readline() 510 | 'Hi, Jim!\n' 511 | >>> p.stdin.write('Sheila\n') 512 | >>> p.stdout.readline() 513 | 'Hi, Sheila!\n' 514 | >>> p.stdin.write('\n') 515 | >>> p.stdout.readline() 516 | '' 517 | >>> p.returncode 518 | >>> p.wait() 519 | 0 520 | 521 | Note that the above code is for Python 2.x. If you're using Python 3.x, you need 522 | to do some things slightly differently: 523 | 524 | * Pass byte-strings to the streams, because interprocess communication occurs 525 | in bytes rather than text. In other words, use for example 526 | ``p.stdin.write(b'Fred\n')`` to send bytes to the child (otherwise you will 527 | get a ``TypeError``). Note that you'll also get byte-strings back. 528 | * Add explicit ``p.stdin.flush()`` calls following ``p.stdin.write()`` calls, to 529 | ensure that the child process sees your output. You should do this even if 530 | you are running Python unbuffered (``-u``) in both parent and child processes 531 | (see https://github.com/vsajip/sarge/issues/43 and 532 | https://bugs.python.org/issue21332 for more information). 533 | 534 | The ``p.returncode`` didn't print anything, indicating that the return code 535 | was ``None``. This means that although the child process has exited, 536 | it's still a zombie because we haven't "reaped" it by making a call to 537 | :meth:`~sarge.Command.wait`. Once that's done, the zombie disappears and we get the 538 | return code. 539 | 540 | Buffering issues 541 | ^^^^^^^^^^^^^^^^ 542 | 543 | From the point of view of buffering, note that two elements are needed for 544 | the above example to work: 545 | 546 | * We specify ``buffer_size=1`` in the Capture constructor. Without this, 547 | data would only be read into the Capture's queue after an I/O completes -- 548 | which would depend on how many bytes the Capture reads at a time. You can 549 | also pass a ``buffer_size=-1`` to indicate that you want to use line- 550 | buffering, i.e. read a line at a time from the child process. (This may only 551 | work as expected if the child process flushes its output buffers after every 552 | line.) 553 | * We make a ``flush`` call in the ``receiver`` script, to ensure that the pipe 554 | is flushed to the capture queue. You could avoid the ``flush`` call in the 555 | above example if you used ``python -u receiver`` as the command (which runs 556 | the script unbuffered). 557 | 558 | This example illustrates that in order for this sort of interaction to work, 559 | you need cooperation from the child process. If the child process has large 560 | output buffers and doesn't flush them, you could be kept waiting for input 561 | until the buffers fill up or a flush occurs. 562 | 563 | If a third party package you're trying to interact with gives you buffering 564 | problems, you may or may not have luck (on POSIX, at least) using the 565 | ``unbuffer`` utility from the ``expect-dev`` package (do a Web search to find 566 | it). This invokes a program directing its output to a pseudo-tty device which 567 | gives line buffering behaviour. This doesn't always work, though :-( 568 | 569 | Looking for specific patterns in child process output 570 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 571 | 572 | You can look for specific patterns in the output of a child process, by using 573 | the :meth:`~sarge.Capture.expect` method of the :class:`~sarge.Capture` class. This takes a 574 | string, bytestring or regular expression pattern object and a timeout, and 575 | either returns a regular expression match object (if a match was found in the 576 | specified timeout) or ``None`` (if no match was found in the specified 577 | timeout). If you pass in a bytestring, it will be converted to a regular 578 | expression pattern. If you pass in text, it will be encoded to bytes using the 579 | ``utf-8`` codec and then to a regular expression pattern. This pattern will be 580 | used to look for a match (using ``search``). If you pass in a regular 581 | expression pattern, make sure it is meant for bytes rather than text (to avoid 582 | ``TypeError`` on Python 3.x). You may also find it useful to specify 583 | ``re.MULTILINE`` in the pattern flags, so that you can match using ``^`` and 584 | ``$`` at line boundaries. Note that on Windows, you may need to use ``\r?$`` 585 | to match ends of lines, as ``$`` matches Unix newlines (LF) and not Windows 586 | newlines (CRLF). 587 | 588 | .. versionadded:: 0.1.1 589 | The ``expect`` method was added. 590 | 591 | To illustrate usage of :meth:`~sarge.Capture.expect`, consider the program 592 | ``lister.py`` (which is provided as part of the source distribution, as it's 593 | used in the tests). This prints ``line 1``, ``line 2`` etc. indefinitely with 594 | a configurable delay, flushing its output stream after each line. We can 595 | capture the output from a run of ``lister.py``, ensuring that we use 596 | line-buffering in the parent process:: 597 | 598 | >>> from sarge import Capture, run 599 | >>> c = Capture(buffer_size=-1) # line-buffering 600 | >>> p = run('python lister.py -d 0.01', async_=True, stdout=c) 601 | >>> m = c.expect('^line 1$') 602 | >>> m.span() 603 | (0, 6) 604 | >>> m = c.expect('^line 5$') 605 | >>> m.span() 606 | (28, 34) 607 | >>> m = c.expect('^line 1.*$') 608 | >>> m.span() 609 | (63, 70) 610 | >>> c.close(True) # close immediately, discard any unread input 611 | >>> p.commands[0].kill() # kill the subprocess 612 | >>> c.bytes[63:70] 613 | 'line 10' 614 | >>> m = c.expect(r'^line 1\d\d$') 615 | >>> m.span() 616 | (783, 791) 617 | >>> c.bytes[783:791] 618 | 'line 100' 619 | 620 | 621 | Displaying progress as a child process runs 622 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 623 | 624 | You can display progress as a child process runs, assuming that its output 625 | allows you to track that progress. Consider the following script, 626 | ``test_progress.py`` (which is included in the source distribution):: 627 | 628 | import optparse # because of 2.6 support 629 | import sys 630 | import threading 631 | import time 632 | import logging 633 | 634 | from sarge import capture_stdout, run, Capture 635 | 636 | logger = logging.getLogger(__name__) 637 | 638 | def progress(capture, options): 639 | lines_seen = 0 640 | messages = { 641 | b'line 25\n': 'Getting going ...\n', 642 | b'line 50\n': 'Well on the way ...\n', 643 | b'line 75\n': 'Almost there ...\n', 644 | } 645 | while True: 646 | s = capture.readline(timeout=1.0) 647 | if not s: 648 | logger.debug('No more data, breaking out') 649 | break 650 | if options.dots: 651 | sys.stderr.write('.') 652 | sys.stderr.flush() # needed for Python 3.x 653 | else: 654 | msg = messages.get(s) 655 | if msg: 656 | sys.stderr.write(msg) 657 | lines_seen += 1 658 | if options.dots: 659 | sys.stderr.write('\n') 660 | sys.stderr.write('Done - %d lines seen.\n' % lines_seen) 661 | 662 | def main(): 663 | parser = optparse.OptionParser() 664 | parser.add_option('-n', '--no-dots', dest='dots', default=True, 665 | action='store_false', help='Show dots for progress') 666 | options, args = parser.parse_args() 667 | 668 | #~ p = capture_stdout('ncat -k -l -p 42421', async_=True) 669 | p = capture_stdout('python lister.py -d 0.1 -c 100', async_=True) 670 | 671 | time.sleep(0.01) 672 | t = threading.Thread(target=progress, args=(p.stdout, options)) 673 | t.start() 674 | 675 | while(p.returncodes[0] is None): 676 | # We could do other useful work here. If we have no useful 677 | # work to do here, we can call readline() and process it 678 | # directly in this loop, instead of creating a thread to do it in. 679 | p.commands[0].poll() 680 | time.sleep(0.05) 681 | t.join() 682 | 683 | if __name__ == '__main__': 684 | logging.basicConfig(level=logging.DEBUG, filename='test_progress.log', 685 | filemode='w', format='%(asctime)s %(threadName)-10s %(name)-15s %(lineno)4d %(message)s') 686 | sys.exit(main()) 687 | 688 | When this is run without the ``--no-dots`` argument, you should see the 689 | following:: 690 | 691 | $ python progress.py 692 | ....................................................... (100 dots printed) 693 | Done - 100 lines seen. 694 | 695 | If run *with* the ``--no-dots`` argument, you should see:: 696 | 697 | $ python progress.py --no-dots 698 | Getting going ... 699 | Well on the way ... 700 | Almost there ... 701 | Done - 100 lines seen. 702 | 703 | with short pauses between the output lines. 704 | 705 | 706 | Direct terminal usage 707 | ^^^^^^^^^^^^^^^^^^^^^ 708 | 709 | Some programs don't work through their ``stdin``/``stdout``/``stderr`` 710 | streams, instead opting to work directly with their controlling terminal. In 711 | such cases, you can't work with these programs using ``sarge``; you need to use 712 | a pseudo-terminal approach, such as is provided by (for example) 713 | `pexpect `_. ``Sarge`` works within the limits 714 | of the :mod:`subprocess` module, which means sticking to ``stdin``, ``stdout`` 715 | and ``stderr`` as ordinary streams or pipes (but not pseudo-terminals). 716 | 717 | Examples of programs which work directly through their controlling terminal 718 | are ``ftp`` and ``ssh`` - the password prompts for these programs are 719 | generally always printed to the controlling terminal rather than ``stdout`` or 720 | ``stderr``. 721 | 722 | .. _environments: 723 | 724 | Environments 725 | ------------ 726 | 727 | In the :class:`subprocess.Popen` constructor, the ``env`` keyword argument, if 728 | supplied, is expected to be the *complete* environment passed to the child 729 | process. This can lead to problems on Windows, where if you don't pass the 730 | ``SYSTEMROOT`` environment variable, things can break. With ``sarge``, it's 731 | assumed by default that anything you pass in ``env`` is *added* to the 732 | contents of ``os.environ``. This is almost always what you want -- after all, 733 | in a POSIX shell, the environment is generally inherited with certain 734 | additions for a specific command invocation. However, if you want to pass a 735 | complete environment rather than an augmented ``os.environ``, you can do this 736 | by passing ``replace_env=True`` in the keyword arguments. In that case, the 737 | value of the ``env`` keyword argument is passed as-is to the child process. 738 | 739 | .. versionadded:: 0.1.6 740 | The ``replace_env`` keyword parameter was added. 741 | 742 | .. note:: On Python 2.x on Windows, environment keys and values must be of 743 | type ``str`` - Unicode values will cause a ``TypeError``. Be careful of 744 | this if you use ``from __future__ import unicode_literals``. For example, 745 | the test harness for sarge uses Unicode literals on 2.x, 746 | necessitating the use of different logic for 2.x and 3.x:: 747 | 748 | if PY3: 749 | env = {'FOO': 'BAR'} 750 | else: 751 | # Python 2.x wants native strings, at least on Windows 752 | env = { b'FOO': b'BAR' } 753 | 754 | 755 | Working directory and other options 756 | ----------------------------------- 757 | 758 | You can set the working directory for a :class:`~sarge.Command` or :class:`~sarge.Pipeline` 759 | using the ``cwd`` keyword argument to the constructor, which is passed through 760 | to the subprocess when it's created. Likewise, you can use the other keyword 761 | arguments which are accepted by the :class:`subprocess.Popen` constructor. 762 | 763 | Avoid using the ``stdin`` keyword argument -- instead, use the ``input`` keyword 764 | argument to the :meth:`~sarge.Command.run` and :meth:`~sarge.Pipeline.run` methods, or the 765 | :func:`~sarge.run`, :func:`~sarge.capture_stdout`, :func:`~sarge.capture_stderr`, and 766 | :func:`~sarge.capture_both` functions. The ``input`` keyword makes it easier for you 767 | to pass literal text or byte data. 768 | 769 | Unicode and bytes 770 | ----------------- 771 | 772 | All data between your process and sub-processes is communicated as bytes. Any 773 | text passed as input to :func:`~sarge.run` or a :meth:`~sarge.Pipeline.run` method will be 774 | converted to bytes using UTF-8 (the default encoding isn't used, because on 775 | Python 2.x, the default encoding isn't UTF-8 -- it's ASCII). 776 | 777 | As ``sarge`` requires Python 2.6 or later, you can use ``from __future__ 778 | import unicode_literals`` and byte literals like ``b'foo'`` so that your code 779 | looks and behaves the same under Python 2.x and Python 3.x. (See the note on 780 | using native string keys and values in :ref:`environments`.) 781 | 782 | As mentioned above, :class:`~sarge.Capture` instances return bytes, but you can wrap 783 | with :class:`io.TextIOWrapper` if you want text. 784 | 785 | 786 | Use as context managers 787 | ----------------------- 788 | 789 | The :class:`~sarge.Capture` and :class:`~sarge.Pipeline` classes can be used as context 790 | managers:: 791 | 792 | >>> with Capture() as out: 793 | ... with Pipeline('cat; echo bar | cat', stdout=out) as p: 794 | ... p.run(input='foo\n') 795 | ... 796 | 797 | >>> out.read().split() 798 | ['foo', 'bar'] 799 | 800 | 801 | Synchronous and asynchronous execution of commands 802 | -------------------------------------------------- 803 | 804 | By default. commands passed to :func:`~sarge.run` run synchronously, 805 | i.e. all commands run to completion before the call returns. However, you can 806 | pass ``async_=True`` to run, in which case the call returns a :class:`~sarge.Pipeline` 807 | instance before all the commands in it have run. You will need to call 808 | :meth:`~sarge.Pipeline.wait` or :meth:`~sarge.Pipeline.close` on this instance when you 809 | are ready to synchronise with it; this is needed so that the sub processes 810 | can be properly disposed of (otherwise, you will leave zombie processes 811 | hanging around, which show up, for example, as ```` on Linux systems 812 | when you run ``ps -ef``). Here's an example:: 813 | 814 | >>> p = run('echo foo|cat|cat|cat|cat', async_=True) 815 | >>> foo 816 | 817 | Here, ``foo`` is printed to the terminal by the last ``cat`` command, but all 818 | the sub-processes are zombies. (The ``run`` function returned immediately, 819 | so the interpreter got to issue the ``>>>` prompt *before* the ``foo`` output 820 | was printed.) 821 | 822 | In another terminal, you can see the zombies:: 823 | 824 | $ ps -ef | grep defunct | grep -v grep 825 | vinay 4219 4217 0 19:27 pts/0 00:00:00 [echo] 826 | vinay 4220 4217 0 19:27 pts/0 00:00:00 [cat] 827 | vinay 4221 4217 0 19:27 pts/0 00:00:00 [cat] 828 | vinay 4222 4217 0 19:27 pts/0 00:00:00 [cat] 829 | vinay 4223 4217 0 19:27 pts/0 00:00:00 [cat] 830 | 831 | Now back in the interactive Python session, we call :meth:`~sarge.Pipeline.close` on 832 | the pipeline:: 833 | 834 | >>> p.close() 835 | 836 | and now, in the other terminal, look for defunct processes again:: 837 | 838 | $ ps -ef | grep defunct | grep -v grep 839 | $ 840 | 841 | No zombies found :-) 842 | 843 | Handling errors in asynchronous mode 844 | ------------------------------------ 845 | 846 | If an exception occurs calling :meth:`~sarge.Command.run` when trying to start a child 847 | process in synchronous mode, it's raised immediately. However, when running in 848 | asynchronous mode (`async_=True`), there will be a command pipeline which is run in a 849 | separate thread. In this case, the exception is caught in that thread but not 850 | propagated to the calling thread; instead, it is stored in the `exception` attribute of 851 | the :class:`~sarge.Command` instance. The `process` attribute of that instance will be `None`, 852 | as the child process couldn't be started. 853 | 854 | You can check the `exception` attributes of commands in a :class:`~sarge.Pipeline` instance to 855 | see if any have occurred. 856 | 857 | .. versionadded:: 0.18 858 | The `exception` attribute was added to :class:`~sarge.Command`. 859 | 860 | About threading and forking on POSIX 861 | ------------------------------------ 862 | 863 | If you run commands asynchronously by using ``&`` in a command pipeline, then a 864 | thread is spawned to run each such command asynchronously. Remember that thread 865 | scheduling behaviour can be unexpected -- things may not always run in the order 866 | you expect. For example, the command line:: 867 | 868 | echo foo & echo bar & echo baz 869 | 870 | should run all of the ``echo`` commands concurrently as far as possible, 871 | but you can't be sure of the exact sequence in which these commands complete -- 872 | it may vary from machine to machine and even from one run to the next. This has 873 | nothing to do with ``sarge`` -- there are no guarantees with just plain Bash, 874 | either. 875 | 876 | On POSIX, :mod:`subprocess` uses :func:`os.fork` to create the child process, 877 | and you may see dire warnings on the Internet about mixing threads, processes 878 | and ``fork()``. It *is* a heady mix, to be sure: you need to understand what's 879 | going on in order to avoid nasty surprises. If you run into any such, it may be 880 | hard to get help because others can't reproduce the problems. However, that's 881 | no reason to shy away from providing the functionality altogether. Such issues 882 | do not occur on Windows, for example: because Windows doesn't have a 883 | ``fork()`` system call, child processes are created in a different way which 884 | doesn't give rise to the issues which sometimes crop up in a POSIX environment. 885 | 886 | For an exposition of the sort of things which might bite you if you are using locks, 887 | threading and ``fork()`` on POSIX, see `this post 888 | `_. 889 | 890 | Other resources on this topic: 891 | 892 | * http://bugs.python.org/issue6721 893 | 894 | Please report any problems you find in this area (or any other) either via the 895 | `mailing list `_ or the `issue 896 | tracker `_. 897 | 898 | Next steps 899 | ---------- 900 | 901 | You might find it helpful to look at information about how ``sarge`` works 902 | internally -- :ref:`internals` -- or peruse the :ref:`reference`. 903 | -------------------------------------------------------------------------------- /echoer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2013 Vinay M. Sajip. See LICENSE for licensing information. 4 | # 5 | # Part of the test harness for sarge: Subprocess Allegedly Rewards Good Encapsulation :-) 6 | # 7 | import sys 8 | 9 | 10 | def main(args=None): 11 | while True: 12 | data = sys.stdin.readline() 13 | if not data: 14 | break 15 | data = data.strip() 16 | data = '%s %s\n' % (data, data) 17 | sys.stdout.write(data) 18 | sys.stdout.flush() 19 | 20 | 21 | if __name__ == '__main__': 22 | try: 23 | rc = main() 24 | except Exception as e: 25 | print(e) 26 | rc = 9 27 | sys.exit(rc) 28 | -------------------------------------------------------------------------------- /lister.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2013 Vinay M. Sajip. See LICENSE for licensing information. 4 | # 5 | # Part of the test harness for sarge: Subprocess Allegedly Rewards Good Encapsulation :-) 6 | # 7 | import logging 8 | import optparse 9 | import os 10 | import re 11 | import sys 12 | import time 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | 17 | def _file_lines(fn): 18 | with open(fn) as f: 19 | for line in f: 20 | yield line 21 | 22 | 23 | def _auto_lines(): 24 | i = 1 25 | while True: 26 | line = 'line %d\n' % i 27 | i += 1 28 | yield line 29 | 30 | 31 | def main(args=None): 32 | parser = optparse.OptionParser(usage='usage: %prog [options] [filename]', 33 | description='Print lines optionally from ' 34 | 'a file, with a delay between ' 35 | 'lines. If no filename is ' 36 | 'specified, lines of the form ' 37 | '"line N" are generated ' 38 | 'internally and printed.') 39 | parser.add_option('-d', 40 | '--delay', 41 | default=None, 42 | type=float, 43 | help='Delay between lines (seconds)') 44 | parser.add_option('-c', 45 | '--count', 46 | default=0, 47 | type=int, 48 | help='Maximum number of lines to output') 49 | parser.add_option('-i', 50 | '--interest', 51 | default=None, 52 | help='Indicate patterns of interest for logging') 53 | if args is None: 54 | args = sys.argv[1:] 55 | options, args = parser.parse_args(args) 56 | if not args: 57 | liner = _auto_lines() 58 | else: 59 | fn = args[0] 60 | if not os.path.isfile(fn): 61 | sys.stderr.write('not a file: %r\n' % fn) 62 | return 2 63 | liner = _file_lines(fn) 64 | bytes_written = 0 65 | pattern = options.interest 66 | if pattern: 67 | pattern = re.compile(pattern) 68 | nlines = 0 69 | for line in liner: 70 | sys.stdout.write(line) 71 | sys.stdout.flush() 72 | nlines += 1 73 | bytes_written += len(line) 74 | if pattern and pattern.search(line): 75 | s = ': %r' % line 76 | level = logging.INFO 77 | else: 78 | s = '' 79 | level = logging.DEBUG 80 | logger.log(level, 'Wrote out %d bytes%s', bytes_written, s) 81 | if options.count and nlines >= options.count: 82 | break 83 | if options.delay: 84 | time.sleep(options.delay) 85 | 86 | 87 | if __name__ == '__main__': 88 | if not os.path.exists('logs'): 89 | os.makedirs('logs') 90 | logging.basicConfig(level=logging.INFO, 91 | filename=os.path.join('logs', 'lister.log'), 92 | filemode='w', 93 | format='%(asctime)s %(levelname)s %(message)s') 94 | try: 95 | rc = main() 96 | except Exception as e: 97 | print(e) 98 | rc = 9 99 | sys.exit(rc) 100 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "index-metadata": { 3 | "extensions": { 4 | "python.details": { 5 | "classifiers": [ 6 | "Development Status :: 5 - Production/Stable", 7 | "Intended Audience :: Developers", 8 | "License :: OSI Approved :: BSD License", 9 | "Programming Language :: Python", 10 | "Programming Language :: Python :: 2", 11 | "Programming Language :: Python :: 3", 12 | "Programming Language :: Python :: 2.7", 13 | "Programming Language :: Python :: 3.6", 14 | "Programming Language :: Python :: 3.7", 15 | "Programming Language :: Python :: 3.8", 16 | "Programming Language :: Python :: 3.9", 17 | "Programming Language :: Python :: 3.10", 18 | "Operating System :: OS Independent", 19 | "Topic :: Software Development :: Libraries :: Python Modules" 20 | ], 21 | "license": "Copyright (C) 2008-2022 by Vinay Sajip. All Rights Reserved. See LICENSE for license." 22 | }, 23 | "python.project": { 24 | "contacts": [ 25 | { 26 | "email": "vinay_sajip@red-dove.com", 27 | "name": "Vinay Sajip", 28 | "role": "author" 29 | }, 30 | { 31 | "email": "vinay_sajip@red-dove.com", 32 | "name": "Vinay Sajip", 33 | "role": "maintainer" 34 | } 35 | ], 36 | "project_urls": { 37 | "Home": "https://docs.red-dove.com/sarge/" 38 | } 39 | } 40 | }, 41 | "metadata_version": "2.0", 42 | "name": "sarge", 43 | "python.exports": { 44 | "modules": [ 45 | "sarge" 46 | ] 47 | }, 48 | "source_url": "https://github.com/vsajip/sarge/releases/download/0.1.8.dev0/sarge-0.1.8.dev0.tar.gz", 49 | "summary": "A wrapper for subprocess which provides command pipeline functionality.", 50 | "version": "0.1.8.dev0" 51 | }, 52 | "metadata": { 53 | "description": "The sarge package provides a wrapper for subprocess which provides command pipeline functionality. This package leverages subprocess to provide easy-to-use cross-platform command pipelines with a Posix flavour: you can have chains of commands using ;, &, pipes using ``|`` and ``|&``, and redirection.", 54 | "name": "sarge", 55 | "platform": "No particular restrictions", 56 | "version": "0.1.8.dev0" 57 | }, 58 | "source": { 59 | "packages": [ 60 | "sarge" 61 | ], 62 | "license-files": ["LICENSE"] 63 | }, 64 | "version": 1 65 | } 66 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools >= 44", 4 | "wheel >= 0.29.0", 5 | ] 6 | build-backend = 'setuptools.build_meta' 7 | -------------------------------------------------------------------------------- /retrier.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (C) 2022 Red Dove Consultants Limited 5 | # 6 | import argparse 7 | import logging 8 | import os 9 | import subprocess 10 | import sys 11 | import time 12 | 13 | DEBUGGING = 'PY_DEBUG' in os.environ 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def main(): 19 | fn = os.path.basename(__file__) 20 | fn = os.path.splitext(fn)[0] 21 | lfn = os.path.expanduser('~/logs/%s.log' % fn) 22 | if os.path.isdir(os.path.dirname(lfn)): 23 | logging.basicConfig(level=logging.DEBUG, filename=lfn, filemode='w', 24 | format='%(message)s') 25 | adhf = argparse.ArgumentDefaultsHelpFormatter 26 | ap = argparse.ArgumentParser(formatter_class=adhf, prog=fn) 27 | aa = ap.add_argument 28 | aa('-r', '--retries', default=3, type=int, help='Number of times to retry') 29 | aa('-d', '--delay', default=5, type=int, help='Number of seconds to delay between retries') 30 | options, args = ap.parse_known_args() 31 | rc = 1 32 | while options.retries > 0: 33 | p = subprocess.Popen(args) 34 | p.wait() 35 | if p.returncode == 0: 36 | rc = 0 37 | break 38 | options.retries -= 1 39 | if options.delay: 40 | time.sleep(options.delay) 41 | return rc 42 | 43 | if __name__ == '__main__': 44 | try: 45 | rc = main() 46 | except KeyboardInterrupt: 47 | rc = 2 48 | except Exception as e: 49 | if DEBUGGING: 50 | s = ' %s:' % type(e).__name__ 51 | else: 52 | s = '' 53 | sys.stderr.write('Failed:%s %s\n' % (s, e)) 54 | if DEBUGGING: import traceback; traceback.print_exc() 55 | rc = 1 56 | sys.exit(rc) 57 | -------------------------------------------------------------------------------- /sarge/shlext.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2012-2019 Vinay M. Sajip. See LICENSE for licensing information. 4 | # 5 | # Enhancements in shlex to tokenize closer to the way real shells do 6 | # 7 | from collections import deque 8 | import shlex 9 | import sys 10 | 11 | # We need to behave differently on 2,x and 3,x, because on 2.x 12 | # shlex barfs on Unicode, and must be given str. 13 | 14 | if sys.version_info[0] < 3: 15 | PY3 = False 16 | text_type = unicode 17 | else: 18 | PY3 = True 19 | text_type = str 20 | 21 | 22 | class shell_shlex(shlex.shlex): 23 | 24 | def __init__(self, instream=None, **kwargs): 25 | if 'control' not in kwargs: 26 | control = '' 27 | else: 28 | control = kwargs.pop('control') 29 | if control is True: 30 | control = '();<>|&' 31 | # shlex on 2.x doesn't like being passed Unicode :-( 32 | if not PY3 and isinstance(instream, text_type): 33 | instream = instream.encode('utf-8') 34 | shlex.shlex.__init__(self, instream, **kwargs) 35 | self.control = control 36 | self.wordchars += ',+-./*?=$%:@\\' # these chars allowed in params 37 | if self.control: 38 | self.pbchars = deque() 39 | 40 | def read_token(self): 41 | quoted = False 42 | escapedstate = ' ' 43 | self.preceding = '' 44 | while True: 45 | if self.control and self.pbchars: 46 | nextchar = self.pbchars.pop() 47 | else: 48 | nextchar = self.instream.read(1) 49 | if nextchar == '\n': 50 | self.lineno += 1 51 | if self.debug >= 3: # pragma: no cover 52 | print("shlex: in state %r saw %r" % (self.state, nextchar)) 53 | if self.state is None: 54 | self.token = '' # past end of file 55 | break 56 | elif self.state == ' ': 57 | if not nextchar: 58 | self.token_type = self.state 59 | self.state = None # end of file 60 | break 61 | elif nextchar in self.whitespace: 62 | self.preceding = nextchar 63 | if self.debug >= 2: # pragma: no cover 64 | print("shlex: whitespace in whitespace state") 65 | if self.token or (self.posix and quoted): # pragma: no cover 66 | break # emit current token 67 | else: # pragma: no cover 68 | continue 69 | elif nextchar in self.commenters: 70 | self.instream.readline() 71 | self.lineno += 1 72 | self.preceding = '\n' 73 | elif self.posix and nextchar in self.escape: 74 | escapedstate = 'a' 75 | self.token_type = self.state 76 | self.state = nextchar 77 | elif nextchar in self.wordchars: 78 | self.token = nextchar 79 | self.token_type = self.state 80 | self.state = 'a' 81 | elif nextchar in self.control: 82 | self.token = nextchar 83 | self.token_type = self.state 84 | self.state = 'c' 85 | elif nextchar in self.quotes: 86 | if not self.posix: 87 | self.token = nextchar 88 | self.token_type = self.state 89 | self.state = nextchar 90 | elif self.whitespace_split: # pragma: no cover 91 | self.token = nextchar 92 | self.token_type = self.state 93 | self.state = 'a' 94 | else: 95 | self.token = nextchar 96 | if self.token or (self.posix and quoted): 97 | break # emit current token 98 | else: # pragma: no cover 99 | continue 100 | elif self.state in self.quotes: 101 | quoted = True 102 | if not nextchar: # end of file 103 | if self.debug >= 2: # pragma: no cover 104 | print("shlex: I see EOF in quotes state") 105 | # XXX what error should be raised here? 106 | raise ValueError("No closing quotation") 107 | if nextchar == self.state: 108 | self.token_type = self.state 109 | if not self.posix: 110 | self.token += nextchar 111 | self.state = ' ' 112 | break 113 | else: 114 | self.state = 'a' 115 | elif (self.posix and nextchar in self.escape 116 | and self.state in self.escapedquotes): # pragma: no cover 117 | escapedstate = self.state 118 | self.token_type = self.state 119 | self.state = nextchar 120 | else: 121 | self.token += nextchar 122 | elif self.state in self.escape: 123 | if not nextchar: # pragma: no cover 124 | if self.debug >= 2: 125 | print("shlex: I see EOF in escape state") 126 | # XXX what error should be raised here? 127 | raise ValueError("No escaped character") 128 | # In posix shells, only the quote itself or the escape 129 | # character may be escaped within quotes. 130 | if (escapedstate in self.quotes and nextchar != self.state 131 | and nextchar != escapedstate): # pragma: no cover 132 | self.token += self.state 133 | self.token += nextchar 134 | self.token_type = self.state 135 | self.state = escapedstate 136 | elif self.state in ('a', 'c'): 137 | if not nextchar: 138 | self.token_type = self.state 139 | self.state = None # end of file 140 | break 141 | elif nextchar in self.whitespace: 142 | if self.debug >= 2: # pragma: no cover 143 | print("shlex: I see whitespace in word state") 144 | self.token_type = self.state 145 | self.state = ' ' 146 | if self.token or (self.posix and quoted): 147 | # push back so that preceding is set 148 | # correctly for the next token 149 | if self.control: 150 | self.pbchars.append(nextchar) 151 | break # emit current token 152 | else: # pragma: no cover 153 | continue 154 | elif nextchar in self.commenters: 155 | self.instream.readline() 156 | self.lineno += 1 157 | if self.posix: 158 | self.token_type = self.state 159 | self.state = ' ' 160 | if self.token or (self.posix and quoted): 161 | break # emit current token 162 | else: # pragma: no cover 163 | continue 164 | elif self.posix and nextchar in self.quotes: 165 | self.token_type = self.state 166 | self.state = nextchar 167 | elif self.posix and nextchar in self.escape: # pragma: no cover 168 | escapedstate = 'a' 169 | self.token_type = self.state 170 | self.state = nextchar 171 | elif self.state == 'c': 172 | if nextchar in self.control: 173 | self.token += nextchar 174 | else: 175 | if nextchar not in self.whitespace: 176 | self.pbchars.append(nextchar) 177 | else: 178 | self.preceding = nextchar 179 | self.token_type = self.state 180 | self.state = ' ' 181 | break 182 | elif (nextchar in self.wordchars or nextchar in self.quotes 183 | or self.whitespace_split): 184 | self.token += nextchar 185 | else: 186 | if self.control: 187 | self.pbchars.append(nextchar) 188 | else: 189 | self.pushback.appendleft(nextchar) 190 | if self.debug >= 2: # pragma: no cover 191 | print("shlex: I see punctuation in word state") 192 | self.token_type = self.state 193 | self.state = ' ' 194 | if self.token: 195 | break # emit current token 196 | else: # pragma: no cover 197 | continue 198 | result = self.token 199 | self.token = '' 200 | if self.posix and not quoted and result == '': 201 | result = None 202 | if self.debug > 1: # pragma: no cover 203 | if result: 204 | print("shlex: raw token=" + repr(result)) 205 | else: 206 | print("shlex: raw token=EOF") 207 | return result 208 | -------------------------------------------------------------------------------- /sarge/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2012-2019 Vinay M. Sajip. See LICENSE for licensing information. 4 | # 5 | # sarge: Subprocess Allegedly Rewards Good Encapsulation :-) 6 | # 7 | import os 8 | import re 9 | import sys 10 | import threading 11 | 12 | try: 13 | from shutil import which 14 | except ImportError: 15 | # Copied from Python 3.3. 16 | def which(cmd, mode=os.F_OK | os.X_OK, path=None): 17 | """Given a command, mode, and a PATH string, return the path which 18 | conforms to the given mode on the PATH, or None if there is no such 19 | file. 20 | 21 | `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result 22 | of os.environ.get("PATH"), or can be overridden with a custom search 23 | path. 24 | 25 | """ 26 | 27 | # Check that a given file can be accessed with the correct mode. 28 | # Additionally check that `file` is not a directory, as on Windows 29 | # directories pass the os.access check. 30 | def _access_check(fn, mode): 31 | return (os.path.exists(fn) and os.access(fn, mode) and not os.path.isdir(fn)) 32 | 33 | # If we're given a path with a directory part, look it up directly rather 34 | # than referring to PATH directories. This includes checking relative to the 35 | # current directory, e.g. ./script 36 | if os.path.dirname(cmd): 37 | if _access_check(cmd, mode): 38 | return cmd 39 | return None 40 | 41 | if path is None: 42 | path = os.environ.get("PATH", os.defpath) 43 | if not path: # pragma: no cover 44 | return None 45 | path = path.split(os.pathsep) 46 | 47 | if sys.platform == "win32": 48 | # The current directory takes precedence on Windows. 49 | if os.curdir not in path: 50 | path.insert(0, os.curdir) 51 | 52 | # PATHEXT is necessary to check on Windows. 53 | pathext = os.environ.get("PATHEXT", "").split(os.pathsep) 54 | # See if the given file matches any of the expected path extensions. 55 | # This will allow us to short circuit when given "python.exe". 56 | # If it does match, only test that one, otherwise we have to try 57 | # others. 58 | if any(cmd.lower().endswith(ext.lower()) for ext in pathext): # pragma: no cover 59 | files = [cmd] 60 | else: 61 | files = [cmd + ext for ext in pathext] 62 | else: # pragma: no cover 63 | # On other platforms you don't have things like PATHEXT to tell you 64 | # what file suffixes are executable, so just pass on cmd as-is. 65 | files = [cmd] 66 | 67 | seen = set() 68 | for dir in path: 69 | normdir = os.path.normcase(dir) 70 | if normdir not in seen: 71 | seen.add(normdir) 72 | for thefile in files: 73 | name = os.path.join(dir, thefile) 74 | if _access_check(name, mode): 75 | return name 76 | return None 77 | 78 | 79 | if sys.platform == 'win32': 80 | try: 81 | import winreg 82 | except ImportError: 83 | import _winreg as winreg 84 | 85 | COMMAND_RE = re.compile(r'^"([^"]*)" "%1" %\*$') 86 | 87 | def find_command(cmd): 88 | """ 89 | Convert a command into a script name, if possible, and find 90 | any associated executable. 91 | :param cmd: The command (e.g. 'hello') 92 | :returns: A 2-tuple. The first element is an executable associated 93 | with the extension of the command script. The second 94 | element is the script name, including the extension and 95 | pathname if found on the path. Example for 'hello' might be 96 | (r'c:/Python/python.exe', r'c:/MyTools/hello.py'). 97 | """ 98 | result = None 99 | cmd = which(cmd) 100 | if cmd: 101 | if cmd.startswith('.\\'): # pragma: no cover 102 | cmd = cmd[2:] 103 | _, extn = os.path.splitext(cmd) 104 | HKCR = winreg.HKEY_CLASSES_ROOT 105 | try: 106 | ftype = winreg.QueryValue(HKCR, extn) 107 | path = os.path.join(ftype, 'shell', 'open', 'command') 108 | s = winreg.QueryValue(HKCR, path) 109 | exe = None 110 | m = COMMAND_RE.match(s) 111 | if m: # pragma: no cover 112 | exe = m.groups()[0] 113 | result = exe, cmd 114 | except OSError: # pragma: no cover 115 | pass 116 | return result 117 | 118 | 119 | if sys.version_info[:2] < (3, 4): 120 | def is_main_thread(): 121 | return isinstance(threading.current_thread(), threading._MainThread) 122 | else: 123 | def is_main_thread(): 124 | return threading.current_thread() is threading.main_thread() 125 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = sarge 3 | version = attr: sarge.__version__ 4 | description = A wrapper for subprocess which provides command pipeline functionality. 5 | long_description = The sarge package provides a wrapper for subprocess which provides 6 | command pipeline functionality. 7 | 8 | This package leverages subprocess to provide easy-to-use cross-platform command 9 | pipelines with a POSIX flavour: you can have chains of commands using ``;``, ``&``, 10 | pipes using ``|`` and ``|&``, and redirection. 11 | url = https://github.com/vsajip/sarge 12 | author = Vinay Sajip 13 | author_email = vinay_sajip@yahoo.co.uk 14 | license = BSD 15 | license_file = LICENSE 16 | classifiers = 17 | Development Status :: 5 - Production/Stable 18 | Environment :: Console 19 | Environment :: MacOS X 20 | Environment :: Win32 (MS Windows) 21 | Intended Audience :: Developers 22 | Intended Audience :: System Administrators 23 | License :: OSI Approved :: BSD License 24 | Operating System :: MacOS :: MacOS X 25 | Operating System :: Microsoft :: Windows 26 | Operating System :: POSIX 27 | Operating System :: Unix 28 | Programming Language :: Python 29 | Programming Language :: Python :: 2 30 | Programming Language :: Python :: 2.7 31 | Programming Language :: Python :: 3 32 | Programming Language :: Python :: 3.6 33 | Programming Language :: Python :: 3.7 34 | Programming Language :: Python :: 3.8 35 | Programming Language :: Python :: 3.9 36 | Programming Language :: Python :: 3.10 37 | Programming Language :: Python :: 3.11 38 | Programming Language :: Python :: 3.12 39 | Programming Language :: Python :: 3.13 40 | Programming Language :: Python :: Implementation 41 | Topic :: Software Development :: Libraries 42 | Topic :: Software Development :: Libraries :: Python Modules 43 | Topic :: System :: Shells 44 | project_urls = 45 | Documentation = https://sarge.readthedocs.io/ 46 | Source = https://github.com/vsajip/sarge 47 | Tracker = https://github.com/vsajip/sarge/issues 48 | keywords = subprocess, wrapper, external, command 49 | platforms = any 50 | 51 | [options] 52 | packages = sarge 53 | 54 | [bdist_wheel] 55 | universal = 1 56 | -------------------------------------------------------------------------------- /stack_tracer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Adapted from http://code.activestate.com/recipes/577334/ 4 | 5 | by László Nagy, released under the MIT license. 6 | """ 7 | 8 | import os 9 | import sys 10 | import threading 11 | import time 12 | import traceback 13 | 14 | 15 | def _get_stack_traces(): 16 | code = [] 17 | threads = dict((t.ident, t.name) for t in threading.enumerate()) 18 | for threadId, stack in sys._current_frames().items(): 19 | if threadId == threading.current_thread().ident: 20 | continue 21 | threadName = threads.get(threadId, 'Unknown') 22 | code.append('\n# Thread: %s (%s)' % (threadId, threadName)) 23 | for filename, lineno, name, line in traceback.extract_stack(stack): 24 | code.append('File: %r, line %d, in %s' % (filename, lineno, name)) 25 | if line: 26 | code.append(' %s' % (line.strip())) 27 | 28 | return '\n'.join(code) 29 | 30 | 31 | class TraceDumper(threading.Thread): 32 | """Dump stack traces into a given file periodically.""" 33 | 34 | def __init__(self, path, interval): 35 | """ 36 | @param path: File path to output stack trace info. 37 | @param interval: in seconds - how often to update the trace file. 38 | """ 39 | assert (interval > 0.1) 40 | self.interval = interval 41 | self.path = os.path.abspath(path) 42 | self.stop_requested = threading.Event() 43 | threading.Thread.__init__(self) 44 | 45 | def run(self): 46 | while not self.stop_requested.is_set(): 47 | time.sleep(self.interval) 48 | self.write_stack_traces() 49 | 50 | def stop(self): 51 | self.stop_requested.set() 52 | self.join() 53 | 54 | def write_stack_traces(self): 55 | with open(self.path, 'w') as out: 56 | out.write(_get_stack_traces()) 57 | 58 | 59 | _tracer = None 60 | 61 | 62 | def start_trace(path, interval=5): 63 | """Start tracing into the given file.""" 64 | global _tracer 65 | if _tracer is None: 66 | _tracer = TraceDumper(path, interval) 67 | _tracer.daemon = True 68 | _tracer.start() 69 | else: 70 | raise Exception('Already tracing to %s' % _tracer.path) 71 | 72 | 73 | def stop_trace(): 74 | """Stop tracing.""" 75 | global _tracer 76 | if _tracer is not None: 77 | _tracer.stop() 78 | _tracer = None 79 | -------------------------------------------------------------------------------- /test_expect.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from sarge import run, Capture 3 | import time 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | logging.basicConfig( 8 | filename='test_expect.log', 9 | filemode='w', 10 | level=logging.INFO, 11 | format='%(asctime)s %(levelname)-8s %(name)s %(threadName)s %(lineno)4d %(message)s') 12 | cap = Capture(buffer_size=-1) # line buffered 13 | p = run('python lister.py -d 0.01 -i "" docs/_build/html/tutorial.html', 14 | async_=True, 15 | stdout=cap) 16 | stime = time.time() 17 | logger.info('Calling expect for head') 18 | cap.expect('', 60.0) 19 | logger.info('Returned from expect for head') 20 | elapsed = time.time() - stime 21 | if not cap.match: 22 | print(' not found within time limit.') 23 | else: 24 | print(' found at %s in %.1f seconds.' % (cap.match.span(), elapsed)) 25 | stime = time.time() 26 | logger.info('Calling expect for body') 27 | cap.expect('', 60.0) 28 | logger.info('Returned from expect for body') 29 | elapsed = time.time() - stime 30 | if not cap.match: 31 | print(' not found within time limit.') 32 | else: 33 | print(' found at %s in %.1f seconds.' % (cap.match.span(), elapsed)) 34 | logger.debug('Killing subprocess') 35 | p.commands[0].kill() 36 | logger.debug('Closing capture') 37 | cap.close() 38 | -------------------------------------------------------------------------------- /test_expect2.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from sarge import run, Capture 3 | import time 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | logging.basicConfig( 8 | filename='test_expect.log', 9 | filemode='w', 10 | level=logging.INFO, 11 | format='%(asctime)s %(levelname)-8s %(name)s %(threadName)s %(lineno)4d %(message)s') 12 | cap = Capture(buffer_size=-1) # line buffered 13 | p = run('python lister.py -d 0.01', async_=True, stdout=cap) 14 | 15 | WAIT_TIME = 1.0 16 | 17 | 18 | def do_expect(pattern, timeout=None): 19 | stime = time.time() 20 | cap.expect(pattern, timeout or WAIT_TIME) 21 | elapsed = time.time() - stime 22 | if not cap.match: 23 | print('%r not found within time limit.' % pattern) 24 | result = False 25 | else: 26 | print('%r found at %s in %.1f seconds.' % (pattern, cap.match.span(), elapsed)) 27 | result = True 28 | return result 29 | 30 | 31 | if do_expect('line 1$'): 32 | if do_expect('line 5$'): 33 | if do_expect('line 1.*$'): 34 | cap.close(True) 35 | print(cap.bytes[cap.match.start():cap.match.end()]) 36 | 37 | p.commands[0].kill() 38 | -------------------------------------------------------------------------------- /test_feeder.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2013-2021 Vinay M. Sajip. See LICENSE for licensing information. 4 | # 5 | # Part of the test harness for sarge: Subprocess Allegedly Rewards Good Encapsulation :-) 6 | # 7 | import sys 8 | import time 9 | 10 | import sarge 11 | 12 | try: 13 | text_type = unicode 14 | except NameError: 15 | text_type = str 16 | 17 | 18 | def main(args=None): 19 | feeder = sarge.Feeder() 20 | p = sarge.run([sys.executable, 'echoer.py'], input=feeder, async_=True) 21 | try: 22 | lines = ('hello', 'goodbye') 23 | gen = iter(lines) 24 | while p.commands[0].returncode is None: 25 | try: 26 | data = next(gen) 27 | except StopIteration: 28 | break 29 | feeder.feed(data + '\n') 30 | p.commands[0].poll() 31 | time.sleep(0.05) # wait for child to return echo 32 | finally: 33 | p.commands[0].terminate() 34 | feeder.close() 35 | 36 | 37 | if __name__ == '__main__': 38 | try: 39 | rc = main() 40 | except Exception as e: 41 | print(e) 42 | rc = 9 43 | sys.exit(rc) 44 | -------------------------------------------------------------------------------- /test_progress.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2013-2021 Vinay M. Sajip. See LICENSE for licensing information. 4 | # 5 | # Part of the test harness for sarge: Subprocess Allegedly Rewards Good Encapsulation :-) 6 | # 7 | import optparse # because of 2.6 support 8 | import sys 9 | import threading 10 | import time 11 | import logging 12 | 13 | from sarge import capture_stdout 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def progress(capture, options): 19 | lines_seen = 0 20 | messages = { 21 | b'line 25\n': 'Getting going ...\n', 22 | b'line 50\n': 'Well on the way ...\n', 23 | b'line 75\n': 'Almost there ...\n', 24 | } 25 | while True: 26 | s = capture.readline(timeout=1.0) 27 | if not s: 28 | logger.debug('No more data, breaking out') 29 | break 30 | if options.dots: 31 | sys.stderr.write('.') 32 | sys.stderr.flush() # needed for Python 3.x 33 | else: 34 | msg = messages.get(s) 35 | if msg: 36 | sys.stderr.write(msg) 37 | lines_seen += 1 38 | if options.dots: 39 | sys.stderr.write('\n') 40 | sys.stderr.write('Done - %d lines seen.\n' % lines_seen) 41 | 42 | 43 | def main(): 44 | parser = optparse.OptionParser() 45 | parser.add_option('-n', 46 | '--no-dots', 47 | dest='dots', 48 | default=True, 49 | action='store_false', 50 | help='Show dots for progress') 51 | options, args = parser.parse_args() 52 | 53 | # p = capture_stdout('ncat -k -l -p 42421', async_=True) 54 | p = capture_stdout('python lister.py -d 0.1 -c 100', async_=True) 55 | 56 | time.sleep(0.01) 57 | t = threading.Thread(target=progress, args=(p.stdout, options)) 58 | t.start() 59 | 60 | while (p.returncodes[0] is None): 61 | # We could do other useful work here. If we have no useful 62 | # work to do here, we can call readline() and process it 63 | # directly in this loop, instead of creating a thread to do it in. 64 | p.commands[0].poll() 65 | time.sleep(0.05) 66 | t.join() 67 | 68 | 69 | if __name__ == '__main__': 70 | logging.basicConfig(level=logging.DEBUG, 71 | filename='test_progress.log', 72 | filemode='w', 73 | format='%(asctime)s %(threadName)-10s %(name)-15s %(lineno)4d %(message)s') 74 | sys.exit(main()) 75 | -------------------------------------------------------------------------------- /test_sarge.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2012-2022 Vinay M. Sajip. See LICENSE for licensing information. 4 | # 5 | # Test harness for sarge: Subprocess Allegedly Rewards Good Encapsulation :-) 6 | # 7 | 8 | from __future__ import unicode_literals 9 | 10 | from io import TextIOWrapper 11 | import logging 12 | import os 13 | import re 14 | import shutil 15 | import subprocess 16 | import sys 17 | import tempfile 18 | import time 19 | import unittest 20 | 21 | from sarge import (shell_quote, Capture, Command, CommandLineParser, Pipeline, shell_format, run, 22 | parse_command_line, capture_stdout, get_stdout, capture_stderr, get_stderr, 23 | capture_both, get_both, Popen, Feeder) 24 | from sarge.shlext import shell_shlex 25 | from stack_tracer import start_trace, stop_trace 26 | 27 | if sys.platform == 'win32': # pragma: no cover 28 | from sarge.utils import find_command 29 | 30 | TRACE_THREADS = sys.platform not in ('cli', ) # debugging only 31 | 32 | PY3 = sys.version_info[0] >= 3 33 | 34 | logger = logging.getLogger(__name__) 35 | 36 | EMITTER = '''#!/usr/bin/env python 37 | import sys 38 | 39 | sys.stdout.write('foo\\n') 40 | sys.stderr.write('bar\\n') 41 | ''' 42 | 43 | SEP = '=' * 60 44 | 45 | 46 | def missing_files(): 47 | result = [] # on POSIX, nothing missing 48 | if os.name == 'nt': # pragma: no cover 49 | 50 | def found_file(fn): 51 | if os.path.exists(fn): 52 | return True 53 | for d in os.environ['PATH'].split(os.pathsep): 54 | p = os.path.join(d, fn) 55 | if os.path.exists(p): 56 | return True 57 | return False 58 | 59 | files = ('cat.exe', 'echo.exe', 'tee.exe', 'false.exe', 'true.exe', 'sleep.exe', 60 | 'touch.exe') 61 | 62 | # Looking for the DLLs used by the above - perhaps this check isn't 63 | # needed, as if the .exes were installed properly, we should be OK. The 64 | # DLL check is relevant for GnuWin32 but may not be for MSYS, MSYS2 etc. 65 | if not os.environ.get('USE_MSYS', ''): 66 | files = ('libiconv2.dll', 'libintl3.dll') + files 67 | 68 | path_dirs = os.environ['PATH'].split(os.pathsep) 69 | 70 | for fn in files: 71 | if os.path.exists(fn): 72 | found = True # absolute, or in current directory 73 | else: 74 | found = False 75 | for d in path_dirs: 76 | p = os.path.join(d, fn) 77 | if os.path.exists(p): 78 | found = True 79 | break 80 | if not found: 81 | result.append(fn) 82 | 83 | return result 84 | 85 | 86 | ERROR_MESSAGE = ''' 87 | Can't find one or more of the files needed for testing: 88 | 89 | %s 90 | 91 | You may need to install the GnuWin32 coreutils package, MSYS, or an equivalent. 92 | '''.strip() 93 | 94 | missing = missing_files() 95 | if missing: 96 | missing = ', '.join(missing) 97 | print(ERROR_MESSAGE % missing) 98 | sys.exit(1) 99 | del missing 100 | 101 | 102 | class SargeTest(unittest.TestCase): 103 | 104 | def setUp(self): 105 | logger.debug(SEP) 106 | logger.debug(self.id().rsplit('.', 1)[-1]) 107 | logger.debug(SEP) 108 | 109 | def test_quote(self): 110 | self.assertEqual(shell_quote(''), "''") 111 | self.assertEqual(shell_quote('a'), 'a') 112 | self.assertEqual(shell_quote('*'), "'*'") 113 | self.assertEqual(shell_quote('foo'), 'foo') 114 | self.assertEqual(shell_quote("'*.py'"), "''\\''*.py'\\'''") 115 | self.assertEqual(shell_quote("'a'; rm -f b; true 'c'"), 116 | "''\\''a'\\''; rm -f b; true '\\''c'\\'''") 117 | self.assertEqual(shell_quote("*.py"), "'*.py'") 118 | self.assertEqual(shell_quote("'*.py"), "''\\''*.py'") 119 | 120 | def test_quote_with_shell(self): 121 | from subprocess import PIPE, Popen 122 | 123 | if os.name != 'posix': # pragma: no cover 124 | raise unittest.SkipTest('This test works only on POSIX') 125 | 126 | workdir = tempfile.mkdtemp() 127 | try: 128 | s = "'\\\"; touch %s/foo #'" % workdir 129 | cmd = 'echo %s' % shell_quote(s) 130 | p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) 131 | p.communicate() 132 | self.assertEqual(p.returncode, 0) 133 | files = os.listdir(workdir) 134 | self.assertEqual(files, []) 135 | fn = "'ab?'" 136 | cmd = 'touch %s/%s' % (workdir, shell_quote(fn)) 137 | p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) 138 | p.communicate() 139 | self.assertEqual(p.returncode, 0) 140 | files = os.listdir(workdir) 141 | self.assertEqual(files, ["'ab?'"]) 142 | finally: 143 | shutil.rmtree(workdir) 144 | 145 | def test_formatter(self): 146 | self.assertEqual(shell_format('ls {0}', '*.py'), "ls '*.py'") 147 | self.assertEqual(shell_format('ls {0!s}', '*.py'), "ls *.py") 148 | 149 | def send_to_capture(self, c, s): 150 | rd, wr = os.pipe() 151 | c.add_stream(os.fdopen(rd, 'rb')) 152 | os.write(wr, s) 153 | os.close(wr) 154 | 155 | def test_capture(self): 156 | logger.debug('test_capture started') 157 | with Capture() as c: 158 | self.send_to_capture(c, b'foofoo') 159 | self.assertEqual(c.read(3), b'foo') 160 | self.assertEqual(c.read(3), b'foo') 161 | self.assertEqual(c.read(), b'') 162 | logger.debug('test_capture finished') 163 | 164 | def test_command_splitting(self): 165 | logger.debug('test_command started') 166 | cmd = 'echo foo' 167 | c = Command(cmd) 168 | self.assertEqual(c.args, cmd.split()) 169 | c = Command(cmd, shell=True) 170 | self.assertEqual(c.args, cmd) 171 | 172 | def test_command_no_stdin(self): 173 | self.assertRaises(ValueError, Command, 'cat', stdin='xyz') 174 | 175 | def test_literal_input(self): 176 | with Capture() as out: 177 | self.assertEqual(run('cat', stdout=out, input='foo').returncode, 0) 178 | self.assertEqual(out.read(), b'foo') 179 | 180 | def test_read_extra(self): 181 | with Capture() as out: 182 | self.assertEqual(run('cat', stdout=out, input='bar').returncode, 0) 183 | self.assertEqual(out.read(5), b'bar') 184 | 185 | def test_shell_redirection(self): 186 | with Capture() as err: 187 | self.assertEqual(run('cat >&2', stderr=err, shell=True, input='bar').returncode, 0) 188 | self.assertEqual(err.read(), b'bar') 189 | 190 | def test_capture_bytes(self): 191 | with Capture() as err: 192 | self.assertEqual(run('cat >&2', stderr=err, shell=True, input='bar').returncode, 0) 193 | self.assertEqual(err.bytes, b'bar') 194 | with Capture() as err: 195 | self.assertEqual(run('cat >&2', stderr=err, shell=True, input='bar').returncode, 0) 196 | self.assertEqual(err.text, 'bar') 197 | 198 | def ensure_testfile(self): 199 | if not os.path.exists('testfile.txt'): # pragma: no cover 200 | with open('testfile.txt', 'w') as f: 201 | for i in range(10000): 202 | f.write('Line %d\n' % (i + 1)) 203 | 204 | def test_run_sync(self): 205 | self.ensure_testfile() 206 | with open('testfile.txt') as f: 207 | content = f.readlines() 208 | with Capture() as out: 209 | self.assertEqual(run('cat testfile.txt testfile.txt', stdout=out).returncode, 0) 210 | lines = out.readlines() 211 | self.assertEqual(len(lines), len(content) * 2) 212 | # run with a list (see Issue #3) 213 | with Capture() as out: 214 | self.assertEqual( 215 | run(['cat', 'testfile.txt', 'testfile.txt'], stdout=out).returncode, 0) 216 | lines = out.readlines() 217 | self.assertEqual(len(lines), len(content) * 2) 218 | 219 | def test_run_async(self): 220 | self.ensure_testfile() 221 | with open('testfile.txt', 'rb') as f: 222 | content = f.read().splitlines(True) 223 | with Capture(timeout=1) as out: 224 | p = run('cat testfile.txt testfile.txt', stdout=out, async_=True) 225 | # Do some other work in parallel, including reading from the 226 | # concurrently running child process 227 | read_count = 0 228 | if out.readline(): 229 | read_count += 1 230 | if out.readline(): 231 | read_count += 1 232 | # kill some time ... 233 | for i in range(10): 234 | with open('testfile.txt') as f: 235 | f.read() 236 | p.wait() 237 | self.assertEqual(p.returncode, 0) 238 | lines = out.readlines() 239 | self.assertEqual(len(lines), len(content) * 2 - read_count) 240 | 241 | def test_env(self): 242 | e = os.environ 243 | if PY3: 244 | env = {'FOO': 'BAR'} 245 | else: 246 | # Python 2.x wants native strings, at least on Windows 247 | # (and literals are Unicode in this module) 248 | env = {b'FOO': b'BAR'} 249 | c = Command('echo foo', env=env) 250 | d = c.kwargs['env'] 251 | ek = set(e) 252 | dk = set(d) 253 | ek.add('FOO') 254 | self.assertEqual(dk, ek) 255 | self.assertEqual(d['FOO'], 'BAR') 256 | c = Command('echo foo', env=env, replace_env=True) 257 | ek = set(env) 258 | dk = set(c.kwargs['env']) 259 | self.assertEqual(dk, ek) 260 | self.assertEqual(dk, {'FOO'}) 261 | 262 | def test_env_usage(self): 263 | if os.name == 'nt': 264 | cmd = 'echo %FOO%' 265 | else: 266 | cmd = 'echo $FOO' 267 | if PY3: 268 | env = {'FOO': 'BAR'} 269 | else: 270 | # Python 2.x wants native strings, at least on Windows 271 | # (and literals are Unicode in this module) 272 | env = {b'FOO': b'BAR'} 273 | c = Command(cmd, env=env, stdout=Capture(), shell=True) 274 | c.run() 275 | self.assertEqual(c.stdout.text.strip(), 'BAR') 276 | 277 | def test_shlex(self): 278 | TESTS = (('', []), ('a', [('a', 'a')]), ('a && b\n', [('a', 'a'), ('&&', 'c'), 279 | ('b', 'a')]), 280 | ('a | b; c>/fred/jim-sheila.txt|&d;e&', [('a', 'a'), ('|', 'c'), ('b', 'a'), 281 | (';', 'c'), ('c', 'a'), ('>', 'c'), 282 | ('/fred/jim-sheila.txt', 'a'), 283 | ('|&', 'c'), ('d', 'a'), (';', 'c'), 284 | ('e', 'a'), ('&', 'c')])) 285 | for posix in False, True: 286 | for s, expected in TESTS: 287 | s = shell_shlex(s, posix=posix, control=True) 288 | actual = [] 289 | while True: 290 | t, tt = s.get_token(), s.token_type 291 | if not t: 292 | break 293 | actual.append((t, tt)) 294 | self.assertEqual(actual, expected) 295 | 296 | def test_shlex_without_control(self): 297 | TESTS = (('', []), ('a', [('a', 'a')]), ('a && b\n', [('a', 'a'), ('&', 'a'), ('&', 'a'), 298 | ('b', 'a')]), 299 | ('a | b; c>/fred/jim-sheila.txt|&d;e&', [('a', 'a'), ('|', 'a'), ('b', 'a'), 300 | (';', 'a'), ('c', 'a'), ('>', 'a'), 301 | ('/fred/jim-sheila.txt', 'a'), 302 | ('|', 'a'), ('&', 'a'), ('d', 'a'), 303 | (';', 'a'), ('e', 'a'), ('&', 'a')])) 304 | for posix in False, True: 305 | for s, expected in TESTS: 306 | s = shell_shlex(s, posix=posix) 307 | actual = [] 308 | while True: 309 | t, tt = s.get_token(), s.token_type 310 | if not t: 311 | break 312 | actual.append((t, tt)) 313 | self.assertEqual(actual, expected) 314 | 315 | def test_shlex_with_quoting(self): 316 | TESTS = ( 317 | ('"a b"', False, [('"a b"', '"')]), 318 | ('"a b"', True, [('a b', 'a')]), 319 | ('"a b" c# comment', False, [('"a b"', '"'), ('c', 'a')]), 320 | ('"a b" c# comment', True, [('a b', 'a'), ('c', 'a')]), 321 | ) 322 | for s, posix, expected in TESTS: 323 | s = shell_shlex(s, posix=posix) 324 | actual = [] 325 | while True: 326 | t, tt = s.get_token(), s.token_type 327 | if not t: 328 | break 329 | actual.append((t, tt)) 330 | self.assertEqual(actual, expected) 331 | s = shell_shlex('"abc') 332 | self.assertRaises(ValueError, s.get_token) 333 | 334 | def test_shlex_with_misc_chars(self): 335 | TESTS = ( 336 | ('rsync user.name@host.domain.tld:path dest', 337 | ('rsync', 'user.name@host.domain.tld:path', 'dest')), 338 | (r'c:\Python26\Python lister.py -d 0.01', (r'c:\Python26\Python', 'lister.py', '-d', 339 | '0.01')), 340 | ) 341 | for s, t in TESTS: 342 | sh = shell_shlex(s) 343 | self.assertEqual(tuple(sh), t) 344 | 345 | def test_shlex_issue_31(self): 346 | cmd = "python -c 'print('\''ok'\'')'" 347 | list(shell_shlex(cmd, control='();>|&', posix=True)) 348 | shell_format("python -c {0}", "print('ok')") 349 | list(shell_shlex(cmd, control='();>|&', posix=True)) 350 | 351 | def test_shlex_issue_34(self): 352 | cmd = "ls foo,bar" 353 | actual = list(shell_shlex(cmd)) 354 | self.assertEqual(actual, ['ls', 'foo,bar']) 355 | 356 | def test_parsing(self): 357 | parse_command_line('abc') 358 | parse_command_line('abc " " # comment') 359 | parse_command_line('abc \ "def"') 360 | parse_command_line('(abc)') 361 | self.assertRaises(ValueError, parse_command_line, '(abc') 362 | self.assertRaises(ValueError, parse_command_line, '&&') 363 | parse_command_line('(abc>def)') 364 | parse_command_line('(abc 2>&1; def >>&2)') 365 | parse_command_line('(a|b;c d && e || f >ghi jkl 2> mno)') 366 | parse_command_line('(abc; (def)); ghi & ((((jkl & mno)))); pqr') 367 | c = parse_command_line('git rev-list origin/master --since="1 hours ago"', posix=True) 368 | self.assertEqual(c.command, ['git', 'rev-list', 'origin/master', '--since=1 hours ago']) 369 | 370 | def test_parsing_special(self): 371 | for cmd in ('ls -l --color=auto', 'sleep 0.5', 'ls /tmp/abc.def', 'ls *.py?', 372 | r'c:\Python26\Python lister.py -d 0.01'): 373 | node = parse_command_line(cmd, posix=False) 374 | if sys.platform != 'win32': 375 | self.assertEqual(node.command, cmd.split()) 376 | else: 377 | split = cmd.split()[1:] 378 | self.assertEqual(node.command[1:], split) 379 | 380 | def test_parsing_controls(self): 381 | clp = CommandLineParser() 382 | gvc = clp.get_valid_controls 383 | self.assertEqual(gvc('>>>>'), ['>>', '>>']) 384 | self.assertEqual(gvc('>>'), ['>>']) 385 | self.assertEqual(gvc('>>>'), ['>>', '>']) 386 | self.assertEqual(gvc('>>>>>'), ['>>', '>>', '>']) 387 | self.assertEqual(gvc('))))'), [')', ')', ')', ')']) 388 | self.assertEqual(gvc('>>;>>'), ['>>', ';', '>>']) 389 | self.assertEqual(gvc(';'), [';']) 390 | self.assertEqual(gvc(';;'), [';', ';']) 391 | self.assertEqual(gvc(');'), [')', ';']) 392 | self.assertEqual(gvc('>&'), ['>', '&']) 393 | self.assertEqual(gvc('>>&'), ['>>', '&']) 394 | self.assertEqual(gvc('||&'), ['||', '&']) 395 | self.assertEqual(gvc('|&'), ['|&']) 396 | 397 | # def test_scratch(self): 398 | # import pdb; pdb.set_trace() 399 | # parse_command_line('(a|b;c d && e || f >ghi jkl 2> mno)') 400 | 401 | def test_parsing_errors(self): 402 | self.assertRaises(ValueError, parse_command_line, '(abc') 403 | self.assertRaises(ValueError, parse_command_line, '(abc |&| def') 404 | self.assertRaises(ValueError, parse_command_line, '&&') 405 | self.assertRaises(ValueError, parse_command_line, 'abc>') 406 | self.assertRaises(ValueError, parse_command_line, 'a 3> b') 407 | self.assertRaises(ValueError, parse_command_line, 'abc >&x') 408 | self.assertRaises(ValueError, parse_command_line, 'a > b | c') 409 | self.assertRaises(ValueError, parse_command_line, 'a 2> b |& c') 410 | self.assertRaises(ValueError, parse_command_line, 'a > b > c') 411 | self.assertRaises(ValueError, parse_command_line, 'a > b >> c') 412 | self.assertRaises(ValueError, parse_command_line, 'a 2> b 2> c') 413 | self.assertRaises(ValueError, parse_command_line, 'a 2>> b 2>> c') 414 | self.assertRaises(ValueError, parse_command_line, 'a 3> b') 415 | 416 | def test_pipeline_no_input_stdout(self): 417 | with Capture() as out: 418 | with Pipeline('echo foo 2> %s | cat | cat' % os.devnull, stdout=out) as pl: 419 | pl.run() 420 | self.assertEqual(out.read().strip(), b'foo') 421 | 422 | def test_pipeline_no_input_stderr(self): 423 | if os.name != 'posix': 424 | raise unittest.SkipTest('This test works only on POSIX') 425 | with Capture() as err: 426 | with Pipeline('echo foo 2> %s | cat | cat >&2' % os.devnull, stderr=err) as pl: 427 | pl.run() 428 | self.assertEqual(err.read().strip(), b'foo') 429 | 430 | def test_pipeline_no_input_pipe_stderr(self): 431 | if os.name != 'posix': 432 | raise unittest.SkipTest('This test works only on POSIX') 433 | with Capture() as err: 434 | with Pipeline('echo foo 2> %s | cat >&2 |& cat >&2' % os.devnull, stderr=err) as pl: 435 | pl.run() 436 | self.assertEqual(err.read().strip(), b'foo') 437 | 438 | def test_pipeline_with_input_stdout(self): 439 | logger.debug('starting') 440 | with Capture() as out: 441 | with Pipeline('cat 2>> %s | cat | cat' % os.devnull, stdout=out) as pl: 442 | pl.run(input='foo' * 1000) 443 | self.assertEqual(out.read().strip(), b'foo' * 1000) 444 | 445 | def test_pipeline_no_input_redirect_stderr(self): 446 | if os.name != 'posix': 447 | raise unittest.SkipTest('This test works only on POSIX') 448 | with Capture() as err: 449 | with Pipeline('echo foo 2> %s | cat 2>&1 | cat >&2' % os.devnull, stderr=err) as pl: 450 | pl.run() 451 | self.assertEqual(err.read().strip(), b'foo') 452 | 453 | def test_pipeline_swap_outputs(self): 454 | for fn in ('stdout.log', 'stderr.log'): 455 | if os.path.exists(fn): 456 | os.unlink(fn) 457 | with Pipeline('echo foo | tee stdout.log 3>&1 1>&2 2>&3 | ' 458 | 'tee stderr.log > %s' % os.devnull) as pl: 459 | pl.run() 460 | with open('stdout.log') as f: 461 | self.assertEqual(f.read().strip(), 'foo') 462 | with open('stderr.log') as f: 463 | self.assertEqual(f.read().strip(), 'foo') 464 | for fn in ('stdout.log', 'stderr.log'): 465 | os.unlink(fn) 466 | 467 | def test_pipeline_large_file(self): 468 | if os.path.exists('dest.bin'): # pragma: no cover 469 | os.unlink('dest.bin') 470 | if not os.path.exists('random.bin'): # pragma: no cover 471 | with open('random.bin', 'wb') as f: 472 | f.write(os.urandom(20 * 1048576)) 473 | with Pipeline('cat random.bin | cat | cat | cat | cat | ' 474 | 'cat > dest.bin ') as pl: 475 | pl.run() 476 | with open('random.bin', 'rb') as f: 477 | data1 = f.read() 478 | with open('dest.bin', 'rb') as f: 479 | data2 = f.read() 480 | os.unlink('dest.bin') 481 | self.assertEqual(data1, data2) 482 | 483 | def test_logical_and(self): 484 | with Capture() as out: 485 | with Pipeline('false && echo foo', stdout=out) as pl: 486 | pl.run() 487 | self.assertEqual(out.read().strip(), b'') 488 | with Capture() as out: 489 | with Pipeline('true && echo foo', stdout=out) as pl: 490 | pl.run() 491 | self.assertEqual(out.read().strip(), b'foo') 492 | with Capture() as out: 493 | with Pipeline('false | cat && echo foo', stdout=out) as pl: 494 | pl.run() 495 | self.assertEqual(out.read().strip(), b'foo') 496 | 497 | def test_logical_or(self): 498 | with Capture() as out: 499 | with Pipeline('false || echo foo', stdout=out) as pl: 500 | pl.run() 501 | self.assertEqual(out.read().strip(), b'foo') 502 | with Capture() as out: 503 | with Pipeline('true || echo foo', stdout=out) as pl: 504 | pl.run() 505 | self.assertEqual(out.read().strip(), b'') 506 | 507 | def test_list(self): 508 | with Capture() as out: 509 | with Pipeline('echo foo > %s; echo bar' % os.devnull, stdout=out) as pl: 510 | pl.run() 511 | self.assertEqual(out.read().strip(), b'bar') 512 | 513 | def test_list_merge(self): 514 | with Capture() as out: 515 | with Pipeline('echo foo; echo bar; echo baz', stdout=out) as pl: 516 | pl.run() 517 | self.assertEqual(out.read().split(), [b'foo', b'bar', b'baz']) 518 | 519 | def test_capture_when_other_piped(self): 520 | with Capture() as out: 521 | with Pipeline('echo foo; echo bar |& cat', stdout=out) as pl: 522 | pl.run() 523 | self.assertEqual(out.read().split(), [b'foo', b'bar']) 524 | 525 | def test_pipeline_func(self): 526 | self.assertEqual(run('false').returncode, 1) 527 | with Capture() as out: 528 | self.assertEqual(run('echo foo', stdout=out).returncode, 0) 529 | self.assertEqual(out.bytes.strip(), b'foo') 530 | 531 | def test_double_redirect(self): 532 | with Capture() as out: 533 | self.assertRaises(ValueError, run, 'echo foo > %s' % os.devnull, stdout=out) 534 | with Capture() as out: 535 | with Capture() as err: 536 | self.assertRaises(ValueError, 537 | run, 538 | 'echo foo 2> %s' % os.devnull, 539 | stdout=out, 540 | stderr=err) 541 | 542 | def test_pipeline_async(self): 543 | logger.debug('starting') 544 | with Capture() as out: 545 | p = run('echo foo & (sleep 2; echo bar) & (sleep 1; echo baz)', stdout=out) 546 | self.assertEqual(p.returncode, 0) 547 | items = out.bytes.split() 548 | for item in (b'foo', b'bar', b'baz'): 549 | self.assertTrue(item in items) 550 | self.assertTrue(items.index(b'bar') > items.index(b'baz')) 551 | 552 | def ensure_emitter(self): 553 | if not os.path.exists('emitter.py'): # pragma: no cover 554 | with open('emitter.py', 'w') as f: 555 | f.write(EMITTER) 556 | 557 | def test_capture_stdout(self): 558 | p = capture_stdout('echo foo') 559 | self.assertEqual(p.stdout.text.strip(), 'foo') 560 | 561 | def test_get_stdout(self): 562 | s = get_stdout('echo foo; echo bar') 563 | parts = s.split() 564 | possibilities = (['foo', 'bar'], ['bar', 'foo']) 565 | self.assertIn(parts, possibilities) 566 | 567 | def test_capture_stderr(self): 568 | self.ensure_emitter() 569 | p = capture_stderr('"%s" emitter.py > %s' % (sys.executable, os.devnull)) 570 | self.assertEqual(p.stderr.text.strip(), 'bar') 571 | 572 | def test_get_stderr(self): 573 | self.ensure_emitter() 574 | s = get_stderr('"%s" emitter.py > %s' % (sys.executable, os.devnull)) 575 | self.assertEqual(s.strip(), 'bar') 576 | 577 | def test_get_both(self): 578 | self.ensure_emitter() 579 | t = get_both('"%s" emitter.py' % sys.executable) 580 | self.assertEqual([s.strip() for s in t], ['foo', 'bar']) 581 | 582 | def test_capture_both(self): 583 | self.ensure_emitter() 584 | p = capture_both('"%s" emitter.py' % sys.executable) 585 | self.assertEqual(p.stdout.text.strip(), 'foo') 586 | self.assertEqual(p.stderr.text.strip(), 'bar') 587 | 588 | def test_byte_iterator(self): 589 | p = capture_stdout('echo foo; echo bar') 590 | lines = [] 591 | for line in p.stdout: 592 | lines.append(line.strip()) 593 | self.assertEqual(lines, [b'foo', b'bar']) 594 | 595 | def test_text_iterator(self): 596 | p = capture_stdout('echo foo; echo bar') 597 | lines = [] 598 | for line in TextIOWrapper(p.stdout): 599 | lines.append(line) 600 | self.assertEqual(lines, ['foo\n', 'bar\n']) 601 | 602 | def test_partial_line(self): 603 | p = capture_stdout('echo foobarbaz') 604 | lines = [p.stdout.readline(6), p.stdout.readline().strip()] 605 | self.assertEqual(lines, [b'foobar', b'baz']) 606 | 607 | def test_returncodes(self): 608 | p = capture_stdout('echo foo; echo bar; echo baz; false') 609 | self.assertEqual(p.returncodes, [0, 0, 0, 1]) 610 | self.assertEqual(p.returncode, 1) 611 | 612 | def test_processes(self): 613 | p = capture_stdout('echo foo; echo bar; echo baz; false') 614 | plist = p.processes 615 | for p in plist: 616 | self.assertTrue(isinstance(p, Popen)) 617 | 618 | def test_command_run(self): 619 | c = Command('echo foo'.split(), stdout=Capture()) 620 | c.run() 621 | self.assertEqual(c.returncode, 0) 622 | 623 | def test_command_nonexistent(self): 624 | c = Command('nonesuch foo'.split(), stdout=Capture()) 625 | if PY3: 626 | ARR = self.assertRaisesRegex 627 | else: 628 | ARR = self.assertRaisesRegexp 629 | ARR(ValueError, 'Command not found: nonesuch', c.run) 630 | 631 | def test_pipeline_nonexistent(self): 632 | p = Pipeline('nonesuch foo'.split(), stdout=Capture()) 633 | self.assertEqual(p.commands, []) 634 | self.assertEqual(p.returncodes, []) 635 | self.assertEqual(p.processes, []) 636 | if PY3: 637 | ARR = self.assertRaisesRegex 638 | else: 639 | ARR = self.assertRaisesRegexp 640 | ARR(ValueError, 'Command not found: nonesuch', p.run) 641 | 642 | def test_working_dir(self): 643 | d = tempfile.mkdtemp() 644 | try: 645 | run('touch newfile.txt', cwd=d) 646 | files = os.listdir(d) 647 | self.assertEqual(files, ['newfile.txt']) 648 | finally: 649 | shutil.rmtree(d) 650 | 651 | def test_expect(self): 652 | cap = Capture(buffer_size=-1) # line buffered 653 | p = run('%s lister.py -d 0.01' % sys.executable, async_=True, stdout=cap) 654 | timeout = 1.0 655 | m1 = cap.expect('^line 1\r?$', timeout) 656 | self.assertTrue(m1) 657 | m2 = cap.expect('^line 5\r?$', timeout) 658 | self.assertTrue(m2) 659 | m3 = cap.expect('^line 1.*\r?$', timeout) 660 | self.assertTrue(m3) 661 | cap.close(True) 662 | p.commands[0].kill() 663 | p.commands[0].wait() 664 | data = cap.bytes 665 | self.assertEqual(data[m1.start():m1.end()].rstrip(), b'line 1') 666 | self.assertEqual(data[m2.start():m2.end()].rstrip(), b'line 5') 667 | self.assertEqual(data[m3.start():m3.end()].rstrip(), b'line 10') 668 | 669 | def test_redirection_with_whitespace(self): 670 | node = parse_command_line('a 2 > b') 671 | self.assertEqual(node.command, ['a', '2']) 672 | self.assertEqual(node.redirects, {1: ('>', 'b')}) 673 | node = parse_command_line('a 2> b') 674 | self.assertEqual(node.command, ['a']) 675 | self.assertEqual(node.redirects, {2: ('>', 'b')}) 676 | node = parse_command_line('a 2 >> b') 677 | self.assertEqual(node.command, ['a', '2']) 678 | self.assertEqual(node.redirects, {1: ('>>', 'b')}) 679 | node = parse_command_line('a 2>> b') 680 | self.assertEqual(node.command, ['a']) 681 | self.assertEqual(node.redirects, {2: ('>>', 'b')}) 682 | 683 | def test_redirection_with_cwd(self): 684 | workdir = tempfile.mkdtemp() 685 | try: 686 | run('echo hello > world', cwd=workdir) 687 | p = os.path.join(workdir, 'world') 688 | self.assertTrue(os.path.exists(p)) 689 | with open(p) as f: 690 | self.assertEqual(f.read().strip(), 'hello') 691 | finally: 692 | shutil.rmtree(workdir) 693 | 694 | if sys.platform == 'win32': # pragma: no cover 695 | pyrunner_re = re.compile(r'.*py.*\.exe', re.I) 696 | pywrunner_re = re.compile(r'.*py.*w\.exe', re.I) 697 | 698 | def test_find_command(self): 699 | cmd = find_command('dummy.py') 700 | self.assertTrue(cmd is None or self.pyrunner_re.match(cmd)) 701 | cmd = find_command('dummy.pyw') 702 | self.assertTrue(cmd is None or self.pywrunner_re.match(cmd)) 703 | 704 | def test_run_found_command(self): 705 | with open('hello.py', 'w') as f: 706 | f.write('print("Hello, world!")') 707 | cmd = find_command('hello') 708 | if not cmd: 709 | raise unittest.SkipTest('.py not in PATHEXT or not registered') 710 | p = capture_stdout('hello') 711 | self.assertEqual(p.stdout.text.rstrip(), 'Hello, world!') 712 | 713 | def test_feeder(self): 714 | feeder = Feeder() 715 | p = capture_stdout([sys.executable, 'echoer.py'], input=feeder, async_=True) 716 | try: 717 | lines = ('hello', 'goodbye') 718 | gen = iter(lines) 719 | # p.commands may not be set yet (separate thread) 720 | while not p.commands or p.commands[0].returncode is None: 721 | logger.debug('commands: %s', p.commands) 722 | try: 723 | data = next(gen) 724 | except StopIteration: 725 | break 726 | feeder.feed(data + '\n') 727 | if p.commands: 728 | p.commands[0].poll() 729 | time.sleep(0.05) # wait for child to return echo 730 | finally: 731 | # p.commands may not be set yet (separate thread) 732 | if p.commands: 733 | p.commands[0].terminate() 734 | p.commands[0].wait() 735 | feeder.close() 736 | self.assertEqual(p.stdout.text.splitlines(), ['hello hello', 'goodbye goodbye']) 737 | 738 | def test_timeout(self): 739 | if sys.version_info[:2] < (3, 3): 740 | raise unittest.SkipTest('test is only valid for Python >= 3.3') 741 | cap = Capture(buffer_size=1) 742 | p = run('%s waiter.py 5.0' % sys.executable, async_=True, stdout=cap) 743 | with self.assertRaises(subprocess.TimeoutExpired): 744 | p.wait(2.5) 745 | self.assertEqual(p.returncodes, [None]) 746 | self.assertEqual(cap.read(block=False), b'Waiting ... ') 747 | p.wait(3.0) # ensure the child process finishes 748 | self.assertEqual(p.returncodes, [0]) 749 | expected = b'done.\n' if os.name != 'nt' else b'done.\r\n' 750 | self.assertEqual(cap.read(), expected) 751 | 752 | def test_exceptions(self): 753 | cmd = 'echo "Hello" && eco "Goodbye"' 754 | cap = Capture(buffer_size=1) 755 | p = run(cmd, async_=True, stdout=cap) 756 | time.sleep(0.1) # Can be slow on Windows! 757 | exceptions = p.exceptions 758 | self.assertEqual(2, len(exceptions)) 759 | self.assertIsNone(exceptions[0]) 760 | e = exceptions[1] 761 | self.assertIsNotNone(e) 762 | self.assertIsInstance(e, ValueError) 763 | self.assertEqual(e.args, ('Command not found: eco',)) 764 | returncodes = p.returncodes 765 | self.assertEqual(returncodes, [0, None]) 766 | 767 | if __name__ == '__main__': # pragma: no cover 768 | # switch the level to DEBUG for in-depth logging. 769 | if not os.path.exists('logs'): 770 | os.makedirs('logs') 771 | fn = 'test_sarge-%d.%d.log' % sys.version_info[:2] 772 | logging.basicConfig(level=logging.DEBUG, 773 | filename=os.path.join('logs', fn), 774 | filemode='w', 775 | format='%(threadName)s %(funcName)s %(lineno)d ' 776 | '%(message)s') 777 | logging.getLogger('sarge.parse').setLevel(logging.WARNING) 778 | fn = 'threads-%d.%d.log' % sys.version_info[:2] 779 | fn = os.path.join('logs', fn) 780 | if TRACE_THREADS: 781 | start_trace(fn) 782 | try: 783 | unittest.main() 784 | finally: 785 | if TRACE_THREADS: 786 | stop_trace() 787 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # tox (https://tox.readthedocs.io/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | envlist = py27, py36, py37, py38, py39, py310, py311, py312, pypy, pypy3 8 | isolated_build = True 9 | 10 | [testenv] 11 | passenv = USE_MSYS 12 | commands = 13 | {envpython} test_sarge.py 14 | # coverage run -a test_sarge.py 15 | #deps = 16 | # coveralls 17 | -------------------------------------------------------------------------------- /waiter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Copyright (C) 2020 Vinay M. Sajip. See LICENSE for licensing information. 4 | # 5 | # Part of the test harness for sarge: Subprocess Allegedly Rewards Good Encapsulation :-) 6 | # 7 | import sys 8 | import time 9 | 10 | 11 | def main(args=None): 12 | sys.stdout.write('Waiting ... ') 13 | sys.stdout.flush() 14 | if len(sys.argv) < 2: 15 | timeout = 5.0 16 | else: 17 | timeout = float(sys.argv[1]) 18 | time.sleep(timeout) 19 | sys.stdout.write('done.\n') 20 | 21 | 22 | if __name__ == '__main__': 23 | try: 24 | rc = main() 25 | except Exception as e: 26 | print(e) 27 | rc = 9 28 | sys.exit(rc) 29 | --------------------------------------------------------------------------------