├── .mailmap ├── command_runner ├── requirements.txt ├── elevate.py └── __init__.py ├── .gitattributes ├── .codespellrc ├── .github └── workflows │ ├── codespell.yml │ ├── linux.yaml │ ├── macos.yaml │ ├── windows.yaml │ ├── pylint-windows.yaml │ ├── pylint-linux.yaml │ └── codeql-analysis.yml ├── LICENSE ├── .gitignore ├── setup.py ├── CHANGELOG.md ├── README.md └── tests └── test_command_runner.py /.mailmap: -------------------------------------------------------------------------------- 1 | Orsiris de Jong 2 | -------------------------------------------------------------------------------- /command_runner/requirements.txt: -------------------------------------------------------------------------------- 1 | psutil>=5.6.0 -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.codespellrc: -------------------------------------------------------------------------------- 1 | [codespell] 2 | # Ref: https://github.com/codespell-project/codespell#using-a-config-file 3 | skip = .git*,.codespellrc 4 | check-hidden = true 5 | # ignore-regex = 6 | # ignore-words-list = 7 | -------------------------------------------------------------------------------- /.github/workflows/codespell.yml: -------------------------------------------------------------------------------- 1 | # Codespell configuration is within .codespellrc 2 | --- 3 | name: Codespell 4 | 5 | on: 6 | push: 7 | branches: [master] 8 | pull_request: 9 | branches: [master] 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | codespell: 16 | name: Check for spelling errors 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v4 22 | - name: Annotate locations with typos 23 | uses: codespell-project/codespell-problem-matcher@v1 24 | - name: Codespell 25 | uses: codespell-project/actions-codespell@v2 26 | -------------------------------------------------------------------------------- /.github/workflows/linux.yaml: -------------------------------------------------------------------------------- 1 | name: linux-tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [ubuntu-latest] 12 | # Python 3.3 and 3.4 have been removed since github won't provide these anymore 13 | # As of 2023/01/09, we have removed python 3.5 and 3.6 as they don't work anymore with linux on github 14 | # As of 2023/08/30, we have removed python 2.7 since github actions won't provide it anymore 15 | # As of 2025/01/20, we have removed python 3.7 since github actions won't povide it anymore 16 | python-version: [3.8, 3.9, "3.10", "3.11", "3.12", "3.13", 'pypy-3.6', 'pypy-3.7', 'pypy-3.8', 'pypy-3.10'] 17 | 18 | steps: 19 | - uses: actions/checkout@v3 20 | - name: Set up Python ${{ matrix.python-version }} 21 | uses: actions/setup-python@v3 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | python -m pip install --upgrade setuptools 28 | if [ -f command_runner/requirements.txt ]; then pip install -r command_runner/requirements.txt; fi 29 | - name: Generate Report 30 | env: 31 | RUNNING_ON_GITHUB_ACTIONS: true 32 | run: | 33 | pip install pytest coverage 34 | python -m coverage run -m pytest -vvs tests 35 | - name: Upload Coverage to Codecov 36 | uses: codecov/codecov-action@v3 37 | -------------------------------------------------------------------------------- /.github/workflows/macos.yaml: -------------------------------------------------------------------------------- 1 | name: macos-tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [macos-latest] 12 | # Python 3.3 and 3.4 have been removed since github won't provide these anymore 13 | # As of 2023/01/09, we have removed python 3.5 and 3.6 as they don't work anymore with linux on github 14 | # As of 2023/08/30, we have removed python 2.7 since github actions won't provide it anymore 15 | # As of 2025/01/20, we have removed python 3.7 since github actions won't povide it anymore 16 | # As of 2025/02/20, we have removed pypy-3.6 and pypy-3.7 since github actions won't povide it anymore 17 | python-version: [3.8, 3.9, "3.10", "3.11", "3.12", "3.13", 'pypy-3.8', 'pypy-3.10'] 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v3 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | python -m pip install --upgrade setuptools 29 | if [ -f command_runner/requirements.txt ]; then pip install -r command_runner/requirements.txt; fi 30 | - name: Generate Report 31 | env: 32 | RUNNING_ON_GITHUB_ACTIONS: true 33 | run: | 34 | pip install pytest coverage 35 | python -m coverage run -m pytest -vvs tests 36 | - name: Upload Coverage to Codecov 37 | uses: codecov/codecov-action@v3 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2015-2025, NetInvent, Orsiris de Jong, contact@netperfect.fr 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /.github/workflows/windows.yaml: -------------------------------------------------------------------------------- 1 | name: windows-tests 2 | 3 | # Remember kids, the default shell here is Powershell 4 | # Don't run with python 3.3 as using python -m to run flake8 or pytest will fail. 5 | # Hence, without python -m, pytest will not have it's PYTHONPATH set to current dir and imports will fail 6 | # Don't run with python 3.4 as github cannot install it (pip install --upgrade pip fails) 7 | 8 | on: [push, pull_request] 9 | 10 | jobs: 11 | build: 12 | 13 | runs-on: ${{ matrix.os }} 14 | strategy: 15 | matrix: 16 | os: [windows-latest] 17 | # As of 2023/08/30, we have removed python 2.7 since github actions won't provide it anymore 18 | # As of 2024/09/15, we have removed python 3.5 since we cannot gather the dependencies anymore 19 | python-version: [3.6, 3.7, 3.8, 3.9, "3.10", "3.11", "3.12", "3.13", 'pypy-3.6', 'pypy-3.7', 'pypy-3.8', 'pypy-3.10'] 20 | 21 | steps: 22 | - uses: actions/checkout@v3 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v3 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | - name: Install dependencies 28 | run: | 29 | python -m pip install --upgrade pip 30 | python -m pip install --upgrade setuptools 31 | if (Test-Path "command_runner/requirements.txt") { pip install -r command_runner/requirements.txt } 32 | - name: Generate Report 33 | env: 34 | RUNNING_ON_GITHUB_ACTIONS: true 35 | run: | 36 | pip install pytest coverage 37 | python -m coverage run -m pytest -vvs tests 38 | - name: Upload Coverage to Codecov 39 | uses: codecov/codecov-action@v3 40 | -------------------------------------------------------------------------------- /.github/workflows/pylint-windows.yaml: -------------------------------------------------------------------------------- 1 | name: pylint-windows-tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [windows-latest] 12 | # Don't use pypy on windows since it does not have pywin32 module 13 | # python-version: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10"] 14 | python-version: ["3.12"] 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Set up Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v2 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install --upgrade pip 25 | python -m pip install --upgrade setuptools 26 | if (Test-Path "command_runner/requirements.txt") { pip install -r command_runner/requirements.txt } 27 | - name: Lint with Pylint 28 | run: | 29 | python -m pip install pylint 30 | # Do not run pylint on python 3.3 because isort is not available for python 3.3, don't run on python 3.4 because pylint: disable=xxxx does not exist 31 | python -m pylint --disable=C,W,R --max-line-length=127 command_runner 32 | - name: Lint with flake8 33 | run: | 34 | python -m pip install flake8 35 | # stop the build if there are Python syntax errors or undefined names 36 | python -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics command_runner 37 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 38 | python -m flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics command_runner 39 | - name: Lint with Black 40 | run: | 41 | pip install black 42 | python -m black --check command_runner -------------------------------------------------------------------------------- /.github/workflows/pylint-linux.yaml: -------------------------------------------------------------------------------- 1 | name: pylint-linux-tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [ubuntu-latest] 12 | # python-version: [3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, "3.10", 'pypy-3.6', 'pypy-3.7'] 13 | python-version: ["3.8", "3.12"] 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | - name: Set up Python ${{ matrix.python-version }} 18 | uses: actions/setup-python@v2 19 | with: 20 | python-version: ${{ matrix.python-version }} 21 | - name: Install dependencies 22 | run: | 23 | python -m pip install --upgrade pip 24 | python -m pip install --upgrade setuptools 25 | if [ -f command_runner/requirements.txt ]; then pip install -r command_runner/requirements.txt; fi 26 | - name: Lint with Pylint 27 | run: | 28 | python -m pip install pylint 29 | # Do not run pylint on python 3.3 because isort is not available for python 3.3, don't run on python 3.4 because pylint: disable=xxxx does not exist 30 | # Disable E0401 import error since we lint on linux and pywin32 is obviously missing 31 | python -m pylint --disable=C,W,R,E0401 --max-line-length=127 command_runner 32 | - name: Lint with flake8 33 | run: | 34 | python -m pip install flake8 35 | # stop the build if there are Python syntax errors or undefined names 36 | python -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics command_runner 37 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 38 | python -m flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics command_runner 39 | - name: Lint with Black 40 | run: | 41 | pip install black 42 | python -m black --check command_runner -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # celery beat schedule file 95 | celerybeat-schedule 96 | 97 | # SageMath parsed files 98 | *.sage.py 99 | 100 | # Environments 101 | .env 102 | .venv 103 | env/ 104 | venv/ 105 | ENV/ 106 | env.bak/ 107 | venv.bak/ 108 | 109 | # Spyder project settings 110 | .spyderproject 111 | .spyproject 112 | 113 | # Rope project settings 114 | .ropeproject 115 | 116 | # mkdocs documentation 117 | /site 118 | 119 | # mypy 120 | .mypy_cache/ 121 | .dmypy.json 122 | dmypy.json 123 | 124 | # Pyre type checker 125 | .pyre/ 126 | 127 | # PyCharm 128 | .idea/ 129 | 130 | # VSCode 131 | .vscode/ 132 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ master ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ master ] 20 | schedule: 21 | - cron: '37 13 * * 5' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'python' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v2 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v2 73 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of command_runner package 5 | 6 | 7 | __intname__ = "command_runner.setup" 8 | __author__ = "Orsiris de Jong" 9 | __copyright__ = "Copyright (C) 2021-2025 Orsiris de Jong for NetInvent" 10 | __licence__ = "BSD 3 Clause" 11 | __build__ = "2022092801" 12 | 13 | 14 | PACKAGE_NAME = "command_runner" 15 | DESCRIPTION = "Platform agnostic command and shell execution tool, also allows UAC/sudo privilege elevation" 16 | 17 | import sys 18 | import os 19 | 20 | import pkg_resources 21 | import setuptools 22 | 23 | 24 | def _read_file(filename): 25 | here = os.path.abspath(os.path.dirname(__file__)) 26 | if sys.version_info[0] < 3: 27 | # With python 2.7, open has no encoding parameter, resulting in TypeError 28 | # Fix with io.open (slow but works) 29 | from io import open as io_open 30 | 31 | try: 32 | with io_open( 33 | os.path.join(here, filename), "r", encoding="utf-8" 34 | ) as file_handle: 35 | return file_handle.read() 36 | except IOError: 37 | # Ugly fix for missing requirements.txt file when installing via pip under Python 2 38 | return "psutil\n" 39 | else: 40 | with open(os.path.join(here, filename), "r", encoding="utf-8") as file_handle: 41 | return file_handle.read() 42 | 43 | 44 | def get_metadata(package_file): 45 | """ 46 | Read metadata from package file 47 | """ 48 | 49 | _metadata = {} 50 | 51 | for line in _read_file(package_file).splitlines(): 52 | if line.startswith("__version__") or line.startswith("__description__"): 53 | delim = "=" 54 | _metadata[line.split(delim)[0].strip().strip("__")] = ( 55 | line.split(delim)[1].strip().strip("'\"") 56 | ) 57 | return _metadata 58 | 59 | 60 | def parse_requirements(filename): 61 | """ 62 | There is a parse_requirements function in pip but it keeps changing import path 63 | Let's build a simple one 64 | """ 65 | try: 66 | requirements_txt = _read_file(filename) 67 | install_requires = [ 68 | str(requirement) 69 | for requirement in pkg_resources.parse_requirements(requirements_txt) 70 | ] 71 | return install_requires 72 | except OSError: 73 | print( 74 | 'WARNING: No requirements.txt file found as "{}". Please check path or create an empty one'.format( 75 | filename 76 | ) 77 | ) 78 | 79 | 80 | package_path = os.path.abspath(PACKAGE_NAME) 81 | package_file = os.path.join(package_path, "__init__.py") 82 | metadata = get_metadata(package_file) 83 | requirements = parse_requirements(os.path.join(package_path, "requirements.txt")) 84 | long_description = _read_file("README.md") 85 | 86 | setuptools.setup( 87 | name=PACKAGE_NAME, 88 | # We may use find_packages in order to not specify each package manually 89 | # packages = ['command_runner'], 90 | packages=setuptools.find_packages(), 91 | version=metadata["version"], 92 | install_requires=requirements, 93 | classifiers=[ 94 | # command_runner is mature 95 | "Development Status :: 5 - Production/Stable", 96 | "Intended Audience :: Developers", 97 | "Topic :: Software Development", 98 | "Topic :: System", 99 | "Topic :: System :: Operating System", 100 | "Topic :: System :: Shells", 101 | "Programming Language :: Python", 102 | "Programming Language :: Python :: 3", 103 | "Programming Language :: Python :: Implementation :: CPython", 104 | "Programming Language :: Python :: Implementation :: PyPy", 105 | "Operating System :: POSIX :: Linux", 106 | "Operating System :: POSIX :: BSD :: FreeBSD", 107 | "Operating System :: POSIX :: BSD :: NetBSD", 108 | "Operating System :: POSIX :: BSD :: OpenBSD", 109 | "Operating System :: Microsoft", 110 | "Operating System :: Microsoft :: Windows", 111 | "License :: OSI Approved :: BSD License", 112 | ], 113 | description=DESCRIPTION, 114 | license="BSD", 115 | author="NetInvent - Orsiris de Jong", 116 | author_email="contact@netinvent.fr", 117 | url="https://github.com/netinvent/command_runner", 118 | keywords=[ 119 | "shell", 120 | "execution", 121 | "subprocess", 122 | "check_output", 123 | "wrapper", 124 | "uac", 125 | "sudo", 126 | "elevate", 127 | "privilege", 128 | ], 129 | long_description=long_description, 130 | long_description_content_type="text/markdown", 131 | python_requires=">=2.7", 132 | ) 133 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # v1.5.2 - Leave the queues alone 2 | 3 | - New `no_close_queues` parameter to leave stdout/stderr queues open for later usage by parent functions 4 | - Updated ofunctions.threading implementation to v2.1.0 5 | - Added python 3.12 and pypy 3.10 to test matrix 6 | - poller/monitor tests now have less rounds in pypy python implementation (takes too long on Github actions) 7 | - Various minor linter fixes 8 | 9 | # v1.5.1 - Let's get your priorities right 10 | 11 | - IO priority was set with process priority values instead of IO priority values 12 | - Failing to set process/IO priority because of insufficient permissions now shows a proper warning message in logs 13 | - priorities are now case insensitive 14 | 15 | # v1.5.0 - command and conquer them all, nod if you're happy 16 | 17 | - New silent parameter disabling all logger calls except of logging.DEBUG levels 18 | - New on_exit parameter that takes a callback function as argument 19 | - valid_exit_codes now accept boolean True which means "all" exit codes 20 | - New priority parameter 21 | - New io_priority parameter 22 | - Fix output capture failure should be an error log instead of debug 23 | - Fix no longer show debug logging for stdout or stderr when empty 24 | 25 | # v1.4.1 - command and conquer them all, don't nod 26 | 27 | - Fix endoding always was set to os default unless explicitly disabled by setting `encoding=False` 28 | 29 | # v1.4.0 - command and conquer them all 30 | 31 | ## Features 32 | 33 | - command_runner now has a `command_runner_threaded()` function which allows to run in background, but still provide live stdout/stderr stream output via queues/callbacks 34 | - Refactor poller mode to allow multiple stdout / stderr stream redirectors 35 | - Passing a queue.Queue() instance to stdout/stderr arguments will fill queue with live stream output 36 | - Passing a function to stdout/stderr arguments will callback said function with live stream output 37 | - Passing a string to stdout/stderr arguments will redirect stream into filename described by string 38 | - Added `split_stream` argument which will make command_runner return (exit_code, stdout, stderr) instead of (exit_code, output) tuple 39 | - Added `check_interval` argument which decides how much time we sleep between two checks, defaults to 0.05 seconds. 40 | Lowering this improves responsiveness, but increases CPU usage. Default value should be more than reasaonable for most applications 41 | - Added `stop_on` argument which takes a function, which is called every `check_interval` and will interrupt execution if it returns True 42 | - Added `process_callback` argument which takes a function(process), which is called upon execution with a subprocess.Popen object as argument for optional external process control 43 | - Possibility to disable command_runner stream encoding with `encoding=False` so we get raw output (bytes) 44 | - Added more unit tests (stop_on, process_callback, stream callback / queues, to_null_redirections, split_streams) 45 | 46 | ## Fixes 47 | 48 | - Fix unix command provided as list didn't work with `shell=True` 49 | - Fixed more Python 2.7 UnicodedecodeErrors on corner case exceptions catches 50 | - Fixed python 2.7 TimeoutException output can fail with UnicodedecodeError 51 | - Fix Python 2.7 does not have subprocess.DEVNULL 52 | - Ensure output is always None if process didn't return any string on stdout/stderr on Python 2.7 53 | - Fix python 2.7 process.communicate() multiple calls endup without output (non blocking process.poll() needs communicate() when using shell=True) 54 | 55 | ## Misc 56 | 57 | - Removed queue usage in monitor mode (needs lesser threads) 58 | - Optimized performance 59 | - Added new exit code -250 when queue/callbacks are used with monitor method or unknown method has been called 60 | - Optimized tests 61 | 62 | # v1.3.1 - command & conquer the standard out/err reloaded 63 | 64 | ## Misc 65 | 66 | - Packaging fixes for Python 2.7 when using `pip install command_runner` 67 | 68 | # v1.3.0 - command & conquer the standard out/err 69 | 70 | ## Features 71 | 72 | - Adds the possibility to redirect stdout/stderr to null with `stdout=False` or `stderr=False` arguments 73 | 74 | ## Misc 75 | 76 | - Add python 3.10 to the test matrix 77 | 78 | # v1.2.1 - command (threads) & conquer 79 | 80 | ## Fixes 81 | 82 | - Timeout race condition with pypy 3.7 (!) where sometimes exit code wasn't -254 83 | - Try to use signal.SIGTERM (if exists) to kill a process instead of os API that uses PID in order to prevent possible collision when process is already dead and another process with the same PID exists 84 | 85 | ## Misc 86 | 87 | - Unit tests are more verbose 88 | - Black formatter is now enforced 89 | - Timeout tests are less strict to cope with some platform delays 90 | 91 | # v1.2.0 - command (runner) & conquer 92 | 93 | ## Features 94 | 95 | - Added a new capture method (monitor) 96 | - There are now two distinct methods to capture output 97 | - Spawning a thread to enforce timeouts, and using process.communicate() (monitor method) 98 | - Spawning a thread to readlines from stdout pipe to an output queue, and reading from that output queue while enforcing timeouts (polller method) 99 | - On the fly output (live_output=True) option is now explicit (uses poller method only) 100 | - Returns partial stdout output when timeouts are reached 101 | - Returns partial stdout output when CTRL+C signal is received (only with poller method) 102 | 103 | ## Fixes 104 | 105 | - CRITICAL: Fixed rare annoying but where output wasn't complete 106 | - Use process signals in favor of direct os.kill API to avoid potential race conditions when PID is reused too fast 107 | - Allow full process subtree killing on Windows & Linux, hence not blocking multiple commands like echo "test" && sleep 100 && echo "done" 108 | - Windows does not maintain an explicit process subtree, so we runtime walk processes to establish the child processes to kill. Obviously, orphaned processes cannot be killed that way.- 109 | 110 | ## Misc 111 | 112 | - Adds a default 16K stdout buffer 113 | - Default command execution timeout is 3600s (1 hour) 114 | - Highly improved tests 115 | - All tests are done for both capture methods 116 | - Timeout tests are more accurate 117 | - Added missing encoding tests 118 | - 2500 rounds of file reading and comparison are added to detect rare queue read misses 119 | 120 | # v0.7.0 (yanked - do not use; see v1.2.0 critical fixes) - The windows GUI 121 | 122 | ## Features 123 | 124 | - Added threaded pipe reader (poller) in order to enforce timeouts on Windows GUI apps 125 | 126 | # v0.6.4 - Keep it working more 127 | 128 | ## Fixes 129 | 130 | - Fixed possible encoding issue with Python < 3.4 and powershell containing non unicode output 131 | - More packaging fixes 132 | 133 | # v0.6.3 - keep_it_working 134 | 135 | ## Fixes 136 | 137 | - Packaging fixes for Python 2.7 138 | 139 | # v0.6.2 - make_it_work 140 | 141 | ## Fixes 142 | 143 | - Improve CI tests 144 | - Fixed possible use of `windows_no_window` with Python < 3.7 should not be allowed 145 | 146 | # v0.6.0 - Initial public release - make_it_simple -------------------------------------------------------------------------------- /command_runner/elevate.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of command_runner module 5 | 6 | """ 7 | elevate is a Windows/ unix compatible function elevator for Python 3+ 8 | 9 | usage: 10 | import sys 11 | from elevate import elevate 12 | 13 | def main(argv): 14 | print('Hello world, with arguments %s' % argv) 15 | 16 | # Hey, check my exit code ;) 17 | sys.exit(123) 18 | 19 | if __name__ == '__main__': 20 | elevate(main, sys.argv) 21 | 22 | Versioning semantics: 23 | Major version: backward compatibility breaking changes 24 | Minor version: New functionality 25 | Patch version: Backwards compatible bug fixes 26 | 27 | """ 28 | 29 | __intname__ = "command_runner.elevate" 30 | __author__ = "Orsiris de Jong" 31 | __copyright__ = "Copyright (C) 2017-2025 Orsiris de Jong for NetInvent" 32 | __licence__ = "BSD 3 Clause" 33 | __version__ = "0.3.3" 34 | __build__ = "2024091501" 35 | 36 | from typing import Tuple 37 | from logging import getLogger 38 | import os 39 | import sys 40 | from command_runner import command_runner 41 | 42 | OS_NAME = os.name 43 | if OS_NAME == "nt": 44 | try: 45 | import win32event # monitor process 46 | import win32process # monitor process 47 | from win32com.shell.shell import ShellExecuteEx 48 | from win32com.shell.shell import IsUserAnAdmin 49 | from win32com.shell import shellcon 50 | except ImportError: 51 | raise ImportError( 52 | "Cannot import ctypes for checking admin privileges on Windows platform." 53 | ) 54 | 55 | logger = getLogger(__name__) 56 | 57 | 58 | def is_admin(): 59 | # type: () -> bool 60 | """ 61 | Checks whether current program has administrative privileges in OS 62 | Works with Windows XP SP2+ and most Unixes 63 | 64 | :return: Boolean, True if admin privileges present 65 | """ 66 | 67 | # Works with XP SP2 + 68 | if OS_NAME == "nt": 69 | try: 70 | return IsUserAnAdmin() 71 | except Exception: 72 | raise EnvironmentError("Cannot check admin privileges") 73 | elif OS_NAME == "posix": 74 | # Check for root on Posix 75 | # os.getuid only exists on postix OSes 76 | # pylint: disable=E1101 (no-member) 77 | return os.getuid() == 0 78 | else: 79 | raise EnvironmentError( 80 | "OS does not seem to be supported for admin check. OS: {}".format(OS_NAME) 81 | ) 82 | 83 | 84 | def get_absolute_path(executable): 85 | # type: (str) -> str 86 | """ 87 | Search for full executable path in preferred shell paths 88 | This allows avoiding usage of shell=True with subprocess 89 | """ 90 | 91 | executable_path = None 92 | exit_code, output = command_runner(["type", "-p", "sudo"]) 93 | if exit_code == 0: 94 | # Remove ending '\n'' character 95 | output = output.strip() 96 | if os.path.isfile(output): 97 | return output 98 | 99 | if OS_NAME == "nt": 100 | split_char = ";" 101 | else: 102 | split_char = ":" 103 | for path in os.environ.get("PATH", "").split(split_char): 104 | if os.path.isfile(os.path.join(path, executable)): 105 | executable_path = os.path.join(path, executable) 106 | return executable_path 107 | 108 | 109 | def _windows_runner(runner, arguments): 110 | # type: (str, str) -> int 111 | # Old method using ctypes which does not wait for executable to exit nor does get exit code 112 | # See https://docs.microsoft.com/en-us/windows/desktop/api/shellapi/nf-shellapi-shellexecutew 113 | # int 0 means SH_HIDE window, 1 is SW_SHOWNORMAL 114 | # needs the following imports 115 | # import ctypes 116 | # ctypes.windll.shell32.ShellExecuteW(None, 'runas', runner, arguments, None, 0) 117 | 118 | # Method with exit code that waits for executable to exit, needs the following imports 119 | # import win32event # monitor process 120 | # import win32process # monitor process 121 | # from win32com.shell.shell import ShellExecuteEx 122 | # from win32com.shell import shellcon 123 | # pylint: disable=C0103 (invalid-name) 124 | # pylint: disable=E0606 (possibly-used-before-assignment) 125 | childProcess = ShellExecuteEx( 126 | nShow=0, 127 | fMask=shellcon.SEE_MASK_NOCLOSEPROCESS, 128 | lpVerb="runas", 129 | lpFile=runner, 130 | lpParameters=arguments, 131 | ) 132 | 133 | # pylint: disable=C0103 (invalid-name) 134 | procHandle = childProcess["hProcess"] 135 | # pylint: disable=I1101 (c-extension-no-member) 136 | # pylint: disable=E0606 (possibly-used-before-assignment) 137 | win32event.WaitForSingleObject(procHandle, win32event.INFINITE) 138 | # pylint: disable=I1101 (c-extension-no-member) 139 | # pylint: disable=E0606 (possibly-used-before-assignment) 140 | exit_code = win32process.GetExitCodeProcess(procHandle) 141 | return exit_code 142 | 143 | 144 | def _check_environment(): 145 | # type: () -> Tuple[str, str] 146 | # Regardless of the runner (CPython, Nuitka or frozen CPython), sys.argv[0] is the relative path to script, 147 | # sys.argv[1] are the arguments 148 | # The only exception being CPython on Windows where sys.argv[0] contains absolute path to script 149 | # Regardless of OS, sys.executable will contain full path to python binary for CPython and Nuitka, 150 | # and full path to frozen executable on frozen CPython 151 | 152 | # Recapitulative table create with 153 | # (CentOS 7x64 / Python 3.4 / Nuitka 0.6.1 / PyInstaller 3.4) and 154 | # (Windows 10 x64 / Python 3.7x32 / Nuitka 0.6.2.10 / PyInstaller 3.4) 155 | # -------------------------------------------------------------------------------------------------------------- 156 | # | OS | Variable | CPython | Nuitka | PyInstaller | 157 | # |------------------------------------------------------------------------------------------------------------| 158 | # | Lin | argv | ['./test.py', '-h'] | ['./test', '-h'] | ['./test.py', -h'] | 159 | # | Lin | sys.executable | /usr/bin/python3.4 | /usr/bin/python3.4 | /absolute/path/to/test | 160 | # | Win | argv | ['C:\\Python\\test.py', '-h'] | ['test', '-h'] | ['test', '-h'] | 161 | # | Win | sys.executable | C:\Python\python.exe | C:\Python\Python.exe | C:\absolute\path\to\test.exe | 162 | # -------------------------------------------------------------------------------------------------------------- 163 | 164 | # Nuitka > 0.8 just declares __compiled__ variables 165 | # Nuitka 0.6.2 and newer define builtin __nuitka_binary_dir 166 | # Nuitka does not set the frozen attribute on sys 167 | # Nuitka < 0.6.2 can be detected in sloppy ways, ie if not sys.argv[0].endswith('.py') or len(sys.path) < 3 168 | # Let's assume this will only be compiled with newer nuitka, and remove sloppy detections 169 | is_nuitka_compiled = False 170 | try: 171 | # Actual if statement not needed, but keeps code inspectors more happy 172 | if "__compiled__" in globals(): 173 | is_nuitka_compiled = True 174 | except NameError: 175 | pass 176 | 177 | if is_nuitka_compiled: 178 | # On nuitka, sys.executable is the python binary, even if it does not exist in standalone, 179 | # so we need to fill runner with sys.argv[0] absolute path 180 | runner = os.path.abspath(sys.argv[0]) 181 | arguments = sys.argv[1:] 182 | # current_dir = os.path.dirname(runner) 183 | logger.debug('Running elevator as Nuitka with runner "{}"'.format(runner)) 184 | # If a freezer is used (PyInstaller, cx_freeze, py2exe) 185 | elif getattr(sys, "frozen", False): 186 | runner = os.path.abspath(sys.executable) 187 | arguments = sys.argv[1:] 188 | # current_dir = os.path.dirname(runner) 189 | logger.debug('Running elevator as Frozen with runner "{}"'.format(runner)) 190 | # If standard interpreter CPython is used 191 | else: 192 | runner = os.path.abspath(sys.executable) 193 | arguments = [os.path.abspath(sys.argv[0])] + sys.argv[1:] 194 | # current_dir = os.path.abspath(sys.argv[0]) 195 | logger.debug('Running elevator as CPython with runner "{}"'.format(runner)) 196 | logger.debug('Arguments are "{}"'.format(arguments)) 197 | return runner, arguments 198 | 199 | 200 | def elevate(callable_function, *args, **kwargs): 201 | """ 202 | UAC elevation / sudo code working for CPython, Nuitka >= 0.6.2, PyInstaller, PyExe, CxFreeze 203 | """ 204 | if is_admin(): 205 | # Don't bother if we already got mighty admin privileges 206 | callable_function(*args, **kwargs) 207 | else: 208 | runner, arguments = _check_environment() 209 | # Windows runner 210 | if OS_NAME == "nt": 211 | # Re-run the script with admin rights 212 | # Join arguments and double quote each argument in order to prevent space separation 213 | arguments = " ".join('"' + arg + '"' for arg in arguments) 214 | try: 215 | exit_code = _windows_runner(runner, arguments) 216 | logger.debug('Child exited with code "{}"'.format(exit_code)) 217 | sys.exit(exit_code) 218 | 219 | except Exception as exc: 220 | logger.info(exc) 221 | logger.debug("Trace:", exc_info=True) 222 | sys.exit(255) 223 | # Linux runner and hopefully Unixes 224 | else: 225 | # Re-run the script but with sudo 226 | sudo_path = get_absolute_path("sudo") 227 | if sudo_path is None: 228 | logger.error( 229 | "Cannot find sudo executable. Trying to run without privileges elevation." 230 | ) 231 | callable_function(*args, **kwargs) 232 | else: 233 | command = ["sudo", runner] + arguments 234 | # Optionally might also pass a stdout PIPE to command_runner so we get live output 235 | exit_code, output = command_runner(command, shell=False, timeout=None) 236 | 237 | logger.info("Child output: {}".format(output)) 238 | sys.exit(exit_code) 239 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # command_runner 2 | # Platform agnostic command execution, timed background jobs with live stdout/stderr output capture, and UAC/sudo elevation 3 | 4 | [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) 5 | [![Percentage of issues still open](http://isitmaintained.com/badge/open/netinvent/command_runner.svg)](http://isitmaintained.com/project/netinvent/command_runner "Percentage of issues still open") 6 | [![Maintainability](https://api.codeclimate.com/v1/badges/defbe10a354d3705f287/maintainability)](https://codeclimate.com/github/netinvent/command_runner/maintainability) 7 | [![codecov](https://codecov.io/gh/netinvent/command_runner/branch/master/graph/badge.svg?token=rXqlphOzMh)](https://codecov.io/gh/netinvent/command_runner) 8 | [![linux-tests](https://github.com/netinvent/command_runner/actions/workflows/linux.yaml/badge.svg)](https://github.com/netinvent/command_runner/actions/workflows/linux.yaml) 9 | [![windows-tests](https://github.com/netinvent/command_runner/actions/workflows/windows.yaml/badge.svg)](https://github.com/netinvent/command_runner/actions/workflows/windows.yaml) 10 | [![GitHub Release](https://img.shields.io/github/release/netinvent/command_runner.svg?label=Latest)](https://github.com/netinvent/command_runner/releases/latest) 11 | 12 | 13 | command_runner's purpose is to run external commands from python, just like subprocess on which it relies, 14 | while solving various problems a developer may face among: 15 | - Handling of all possible subprocess.popen / subprocess.check_output scenarios / python versions in one handy function without encoding / timeout hassle 16 | - Allow stdout/stderr stream output to be redirected to callback functions / output queues / files so you get to handle output in your application while commands are running 17 | - Callback to optional stop check so we can stop execution from outside command_runner 18 | - Callback with optional process information so we get to control the process from outside command_runner 19 | - Callback once we're finished to easen thread usage 20 | - Optional process priority and io_priority settings 21 | - System agnostic functionality, the developer shouldn't carry the burden of Windows & Linux differences 22 | - Optional Windows UAC elevation module compatible with CPython, PyInstaller & Nuitka 23 | - Optional Linux sudo elevation compatible with CPython, PyInstaller & Nuitka 24 | - Optional heartbeat for command execution 25 | 26 | It is compatible with Python 2.7+, tested up to Python 3.13 (backports some newer functionality to Python 3.5) and is tested on Linux, Windows and MacOS. 27 | It is also compatible with PyPy Python implementation. 28 | ...and yes, keeping Python 2.7 compatibility has proven to be quite challenging. 29 | 30 | ## command_runner 31 | 32 | command_runner is a replacement package for subprocess.popen and subprocess.check_output 33 | The main promise command_runner can do is to make sure to never have a blocking command, and always get results. 34 | 35 | It works as wrapper for subprocess.popen and subprocess.communicate that solves: 36 | - Platform differences 37 | - Handle timeouts even for windows GUI applications that don't return anything to stdout 38 | - Python language version differences 39 | - Handle timeouts even on earlier Python implementations 40 | - Handle encoding even on earlier Python implementations 41 | - Keep the promise to always return an exit code (so we don't have to deal with exit codes and exception logic at the same time) 42 | - Keep the promise to always return the command output regardless of the execution state (even with timeouts, callback interrupts and keyboard interrupts) 43 | - Can show command output on the fly without waiting the end of execution (with `live_output=True` argument) 44 | - Can give command output on the fly to application by using queues or callback functions 45 | - Catch all possible exceptions and log them properly with encoding fixes 46 | - Be compatible, and always return the same result regardless of platform 47 | 48 | command_runner also promises to properly kill commands when timeouts are reached, including spawned subprocesses of such commands. 49 | This specific behavior is achieved via psutil module, which is an optional dependency. 50 | 51 | 52 | ### command_runner in a nutshell 53 | 54 | ### In a nutshell 55 | 56 | Install with `pip install command_runner` 57 | 58 | The following example will work regardless of the host OS and the Python version. 59 | 60 | ```python 61 | from command_runner import command_runner 62 | 63 | exit_code, output = command_runner('ping 127.0.0.1', timeout=10) 64 | ``` 65 | 66 | 67 | ## Advanced command_runner usage 68 | 69 | 70 | ### Special exit codes 71 | 72 | In order to keep the promise to always provide an exit_code, special exit codes have been added for the case where none is given. 73 | Those exit codes are: 74 | 75 | - -250 : command_runner called with incompatible arguments 76 | - -251 : stop_on function returned True 77 | - -252 : KeyboardInterrupt 78 | - -253 : FileNotFoundError, OSError, IOError 79 | - -254 : Timeout 80 | - -255 : Any other uncatched exceptions 81 | 82 | This allows you to use the standard exit code logic, without having to deal with various exceptions. 83 | 84 | ### Default encoding 85 | 86 | command_runner has an `encoding` argument which defaults to `utf-8` for Unixes and `cp437` for Windows platforms. 87 | Using `cp437` ensures that most `cmd.exe` output is encoded properly, including accents and special characters, on most locale systems. 88 | Still you can specify your own encoding for other usages, like Powershell where `unicode_escape` is preferred. 89 | 90 | ```python 91 | from command_runner import command_runner 92 | 93 | command = r'C:\Windows\sysnative\WindowsPowerShell\v1.0\powershell.exe --help' 94 | exit_code, output = command_runner(command, encoding='unicode_escape') 95 | ``` 96 | 97 | Earlier subprocess.popen implementations didn't have an encoding setting so command_runner will deal with encoding for those. 98 | You can also disable command_runner's internal encoding in order to get raw process output (bytes) by passing False boolean. 99 | 100 | Example: 101 | ```python 102 | from command_runner import command_runner 103 | 104 | exit_code, raw_output = command_runner('ping 127.0.0.1', encoding=False) 105 | ``` 106 | 107 | ### On the fly (interactive screen) output 108 | 109 | **Note: for live output capture and threading, see stream redirection. If you want to run your application while command_runner gives back command output, the best way to achieve this is using queues / callbacks.** 110 | 111 | command_runner can output a command output on the fly to stdout, eg show output on screen during execution. 112 | This is helpful when the command is long, and we need to know the output while execution is ongoing. 113 | It is also helpful in order to catch partial command output when timeout is reached or a CTRL+C signal is received. 114 | Example: 115 | 116 | ```python 117 | from command_runner import command_runner 118 | 119 | exit_code, output = command_runner('ping 127.0.0.1', shell=True, live_output=True) 120 | ``` 121 | 122 | Note: using live output relies on stdout pipe polling, which has lightly higher cpu usage. 123 | 124 | ### Timeouts 125 | 126 | **command_runner has a `timeout` argument which defaults to 3600 seconds.** 127 | This default setting ensures commands will not block the main script execution. 128 | Feel free to lower / higher that setting with `timeout` argument. 129 | Note that a command_runner will try to kill the whole process tree that the command may have generated. 130 | 131 | ```python 132 | from command_runner import command_runner 133 | 134 | exit_code, output = command_runner('ping 127.0.0.1', timeout=30) 135 | ``` 136 | 137 | #### Remarks on processes termination 138 | 139 | When we instruct command_runner to stop a process (because of one of the requirements met, example timeouts), using `shell=True` will spawn a shell which will spawn the desired child process. Under MS Windows, there is no direct process tree, so we cannot easily kill the whole process tree. 140 | We fixed this by walking processes during runtime. The drawback is that orphaned processes cannot be identified this way. 141 | 142 | ### Disabling logs / silencing 143 | 144 | `command_runner` has it's own logging system, which will log all sorts of error logs. 145 | If you need to disable it's logging, just run with argument silent. 146 | Be aware that logging.DEBUG log levels won't be silenced, by design. 147 | 148 | Example: 149 | ```python 150 | from command_runner import command_runner 151 | 152 | exit_code, output = command_runner('ping 127.0.0.1', silent=True) 153 | ``` 154 | 155 | If you also need to disable logging.DEBUG level, you can run the following code which will required logging.CRITICAL only messages which `command_runner` never does: 156 | 157 | ```python 158 | import logging 159 | import command_runner 160 | 161 | logging.getLogger('command_runner').setLevel(logging.CRITICAL) 162 | ``` 163 | 164 | ### Capture method 165 | 166 | `command_runner` allows two different process output capture methods: 167 | 168 | `method='monitor'` which is default: 169 | - A thread is spawned in order to check stop conditions and kill process if needed 170 | - A main loop waits for the process to finish, then uses proc.communicate() to get it's output 171 | - Pros: 172 | - Less CPU usage 173 | - Less threads 174 | - Cons: 175 | - It cannot read partial output on KeyboardInterrupt or stop_on (still works for partial timeout output) 176 | - It cannot use queues or callback functions redirectors 177 | - It is 0.1 seconds slower than poller method 178 | 179 | 180 | `method='poller'`: 181 | - A thread is spawned and reads stdout/stderr pipes into output queues 182 | - A poller loop reads from the output queues, checks stop conditions and kills process if needed 183 | - Pros: 184 | - Reads on the fly, allowing interactive commands (is also used with `live_output=True`) 185 | - Allows stdout/stderr output to be written live to callback functions, queues or files (useful when threaded) 186 | - It is 0.1 seconds faster than monitor method and is the preferred method for fast batch runnings 187 | - Cons: 188 | - Slightly higher CPU usage 189 | 190 | Example: 191 | ```python 192 | from command_runner import command_runner 193 | 194 | 195 | exit_code, output = command_runner('ping 127.0.0.1', method='poller') 196 | exit_code, output = command_runner('ping 127.0.0.1', method='monitor') 197 | ``` 198 | 199 | #### stdout / stderr stream redirection using poller capture method 200 | 201 | command_runner can redirect the command's stdout and/or stderr streams to different outputs: 202 | - subprocess pipes 203 | - /dev/null or NUL 204 | - files 205 | - queues 206 | - callback functions 207 | 208 | Unless an output redirector is given for `stderr` argument, stderr will be redirected to `stdout` stream. 209 | Note that both queues and callback function redirectors require `poller` method and will fail if method is not set. 210 | 211 | Output redirector descriptions: 212 | 213 | - subprocess pipes 214 | 215 | - By default, stdout writes into a subprocess.PIPE which is read by command_runner and returned as `output` variable. 216 | - You may also pass any other subprocess.PIPE int values to `stdout` or `stderr` arguments. 217 | 218 | - /dev/null or NUL 219 | 220 | - If `stdout=False` and/or `stderr=False` argument(s) are given, command output will not be saved. 221 | - stdout/stderr streams will be redirected to `/dev/null` or `NUL` depending on platform. 222 | - Output will always be `None`. See `split_streams` for more details using multiple outputs. 223 | 224 | - files 225 | 226 | - Giving `stdout` and/or `stderr` arguments a string, `command_runner` will consider the string to be a file path where stream output will be written live. 227 | - Examples: 228 | ```python 229 | from command_runner import command_runner 230 | exit_code, output = command_runner('dir', stdout=r"C:/tmp/command_result", stderr=r"C:/tmp/command_error", shell=True) 231 | ``` 232 | ```python 233 | from command_runner import command_runner 234 | exit_code, output = command_runner('dir', stdout='/tmp/stdout.log', stderr='/tmp/stderr.log', shell=True) 235 | ``` 236 | - Opening a file with the wrong encoding (especially opening a CP437 encoded file on Windows with UTF-8 coded might endup with UnicodedecodeError.) 237 | 238 | - queues 239 | 240 | - Queue(s) will be filled up by command_runner. 241 | - In order to keep your program "live", we'll use the threaded version of command_runner which is basically the same except it returns a future result instead of a tuple. 242 | - Note: With all the best will, there's no good way to achieve this under Python 2.7 without using more queues, so the threaded version is only compatible with Python 3.3+. 243 | - For Python 2.7, you must create your thread and queue reader yourself (see footnote for a Python 2.7 compatible example). 244 | - Threaded command_runner plus queue example: 245 | 246 | ```python 247 | import queue 248 | from command_runner import command_runner_threaded 249 | 250 | output_queue = queue.Queue() 251 | stream_output = "" 252 | thread_result = command_runner_threaded('ping 127.0.0.1', shell=True, method='poller', stdout=output_queue) 253 | 254 | read_queue = True 255 | while read_queue: 256 | try: 257 | line = output_queue.get(timeout=0.1) 258 | except queue.Empty: 259 | pass 260 | else: 261 | if line is None: 262 | read_queue = False 263 | else: 264 | stream_output += line 265 | # ADD YOUR LIVE CODE HERE 266 | 267 | # Now we may get exit_code and output since result has become available at this point 268 | exit_code, output = thread_result.result() 269 | ``` 270 | - You might also want to read both stdout and stderr queues. In that case, you can create a read loop just like in the following example. 271 | - Here we're reading both queues in one loop, so we need to observe a couple of conditions before stopping the loop, in order to catch all queue output: 272 | ```python 273 | import queue 274 | from time import sleep 275 | from command_runner import command_runner_threaded 276 | 277 | stdout_queue = queue.Queue() 278 | stderr_queue = queue.Queue() 279 | thread_result = command_runner_threaded('ping 127.0.0.1', method='poller', shell=True, stdout=stdout_queue, stderr=stderr_queue) 280 | 281 | read_stdout = read_stderr = True 282 | while read_stdout or read_stderr: 283 | 284 | try: 285 | stdout_line = stdout_queue.get(timeout=0.1) 286 | except queue.Empty: 287 | pass 288 | else: 289 | if stdout_line is None: 290 | read_stdout = False 291 | else: 292 | print('STDOUT:', stdout_line) 293 | 294 | try: 295 | stderr_line = stderr_queue.get(timeout=0.1) 296 | except queue.Empty: 297 | pass 298 | else: 299 | if stderr_line is None: 300 | read_stderr = False 301 | else: 302 | print('STDERR:', stderr_line) 303 | 304 | # ADD YOUR LIVE CODE HERE 305 | 306 | exit_code, output = thread_result.result() 307 | assert exit_code == 0, 'We did not succeed in running the thread' 308 | 309 | ``` 310 | 311 | - callback functions 312 | 313 | - The callback function will get one argument, being a str of current stream readings. 314 | - It will be executed on every line that comes from streams. Example: 315 | ```python 316 | from command_runner import command_runner 317 | 318 | def callback_function(string): 319 | # ADD YOUR CODE HERE 320 | print('CALLBACK GOT:', string) 321 | 322 | # Launch command_runner 323 | exit_code, output = command_runner('ping 127.0.0.1', stdout=callback_function, method='poller') 324 | ``` 325 | 326 | ### stdin stream redirection 327 | 328 | `command_runner` allows to redirect some stream directly into the subprocess it spawns. 329 | 330 | Example code 331 | ```python 332 | import sys 333 | from command_runner import command_runner 334 | 335 | 336 | exit_code, output = command_runner("gzip -d", stdin=sys.stdin.buffer) 337 | print("Uncompressed data", output) 338 | ``` 339 | The above program, when run with `echo "Hello, World!" | gzip | python myscript.py` will show the uncompressed string `Hello, World!` 340 | 341 | You can use whatever file descriptor you want, basic ones being sys.stdin for text input and sys.stdin.buffer for binary input. 342 | 343 | ### Checking intervals 344 | 345 | By default, command_runner checks timeouts and outputs every 0.05 seconds. 346 | You can increase/decrease this setting via `check_interval` setting which accepts floats. 347 | Example: `command_runner(cmd, check_interval=0.2)` 348 | Note that lowering `check_interval` will increase CPU usage. 349 | 350 | ### stop_on 351 | 352 | In some situations, you want a command to be aborted on some external triggers. 353 | That's where `stop_on` argument comes in handy, 354 | Just pass a function to `stop_on`, which will be executed every on every `check_interval`. As soon as function result becomes True, execution will halt with exit code -251. 355 | 356 | As a side note, when using `stop_on=my_func`, if `my_func` is cpu/io intensive, you should set `check_interval` to something reasonable, which generally counts in seconds. 357 | 358 | Example: 359 | ```python 360 | from command_runner import command_runner 361 | 362 | def some_function(): 363 | return True if we_must_stop_execution 364 | exit_code, output = command_runner('ping 127.0.0.1', stop_on=some_function, check_interval=2) 365 | ``` 366 | 367 | ### Getting current process information 368 | 369 | `command_runner` can provide an instance of subprocess.Popen of currently run command as external data, in order to retrieve process data like pids. 370 | In order to do so, just declare a function and give it as `process_callback` argument. 371 | 372 | Example: 373 | ```python 374 | from command_runner import command_runner 375 | 376 | def show_process_info(process): 377 | print('My process has pid: {}'.format(process.pid)) 378 | 379 | exit_code, output = command_runner('ping 127.0.0.1', process_callback=show_process_info) 380 | ``` 381 | 382 | ### Split stdout and stderr 383 | 384 | By default, `command_runner` returns a tuple like `(exit_code, output)` in which output contains both stdout and stderr stream outputs. 385 | You can alter that behavior by using argument `split_stream=True`. 386 | In that case, `command_runner` will return a tuple like `(exit_code, stdout, stderr)`. 387 | 388 | Example: 389 | ```python 390 | from command_runner import command_runner 391 | 392 | exit_code, stdout, stderr = command_runner('ping 127.0.0.1', split_streams=True) 393 | print('exit code:', exit_code) 394 | print('stdout', stdout) 395 | print('stderr', stderr) 396 | ``` 397 | 398 | ### On-exit Callback 399 | 400 | `command_runner` allows to execute a callback function once it has finished it's execution. 401 | This might help building threaded programs where a callback is needed to disable GUI elements for example, or make the program aware that execution has finished without the need for polling checks. 402 | 403 | Example: 404 | ```python 405 | from command_runner import command_runner 406 | 407 | def do_something(): 408 | print("We're done running") 409 | 410 | exit_code, output = command_runner('ping 127.0.0.1', on_exit=do_something) 411 | ``` 412 | 413 | ### Process and IO priority 414 | `command_runner` can set it's subprocess priority to 'low', 'normal' or 'high', which translate to 15, 0, -15 niceness on Linux and BELOW_NORMAL_PRIORITY_CLASS and HIGH_PRIORITY_CLASS in Windows. 415 | On Linux, you may also directly use priority with niceness int values. 416 | 417 | You may also set subprocess io priority to 'low', 'normal' or 'high'. 418 | 419 | Example: 420 | ```python 421 | from command_runner import command_runner 422 | 423 | exit_code, output = command_runner('some_intensive_process', priority='low', io_priority='high') 424 | ``` 425 | 426 | ### Heartbeat 427 | When running long commands, one might want to know that the program is still running. 428 | The following example will log a message every hour stating that we're still running our command 429 | 430 | ```python 431 | from command_runner import command_runner 432 | 433 | exit_code, output = command_runner('/some/long/command', timeout=None, heartbeat=3600) 434 | ``` 435 | 436 | ### Other arguments 437 | 438 | `command_runner` takes **any** argument that `subprocess.Popen()` would take. 439 | 440 | It also uses the following standard arguments: 441 | - command (str/list): The command, doesn't need to be a list, a simple string works 442 | - valid_exit_codes (list): List of exit codes which won't trigger error logs 443 | - timeout (int): seconds before a process tree is killed forcefully, defaults to 3600 444 | - shell (bool): Shall we use the cmd.exe or /usr/bin/env shell for command execution, defaults to False 445 | - encoding (str/bool): Which text encoding the command produces, defaults to cp437 under Windows and utf-8 under Linux 446 | - stdin (sys.stdin/int): Optional stdin file descriptor, sent to the process command_runner spawns 447 | - stdout (str/queue.Queue/function/False/None): Optional path to filename where to dump stdout, or queue where to write stdout, or callback function which is called when stdout has output 448 | - stderr (str/queue.Queue/function/False/None): Optional path to filename where to dump stderr, or queue where to write stderr, or callback function which is called when stderr has output 449 | - no_close_queues (bool): Normally, command_runner sends None to stdout / stderr queues when process is finished. This behavior can be disabled allowing to reuse those queues for other functions wrapping command_runner 450 | - windows_no_window (bool): Shall a command create a console window (MS Windows only), defaults to False 451 | - live_output (bool): Print output to stdout while executing command, defaults to False 452 | - method (str): Accepts 'poller' or 'monitor' stdout capture and timeout monitoring methods 453 | - check interval (float): Defaults to 0.05 seconds, which is the time between stream readings and timeout checks 454 | - stop_on (function): Optional function that when returns True stops command_runner execution 455 | - on_exit (function): Optional function that gets executed when command_runner has finished (callback function) 456 | - process_callback (function): Optional function that will take command_runner spawned process as argument, in order to deal with process info outside of command_runner 457 | - split_streams (bool): Split stdout and stderr into two separate results 458 | - silent (bool): Allows to disable command_runner's internal logs, except for logging.DEBUG levels which for obvious reasons should never be silenced 459 | - priority (str): Allows to set CPU bound process priority (takes 'low', 'normal' or 'high' parameter) 460 | - io_priority (str): Allows to set IO priority for process (takes 'low', 'normal' or 'high' parameter) 461 | - heartbeat (int): Optional seconds on which command runner should log a heartbeat message 462 | - close_fds (bool): Like Popen, defaults to True on Linux and False on Windows 463 | - universal_newlines (bool): Like Popen, defaults to False 464 | - creation_flags (int): Like Popen, defaults to 0 465 | - bufsize (int): Like Popen, defaults to 16384. Line buffering (bufsize=1) is deprecated since Python 3.7 466 | 467 | **Note that ALL other subprocess.Popen arguments are supported, since they are directly passed to subprocess.** 468 | 469 | 470 | ### command_runner Python 2.7 compatible queue reader 471 | 472 | The following example is a Python 2.7 compatible threaded implementation that reads stdout / stderr queue in a thread. 473 | This only exists for compatibility reasons. 474 | 475 | ```python 476 | import queue 477 | import threading 478 | from command_runner import command_runner 479 | 480 | def read_queue(output_queue): 481 | """ 482 | Read the queue as thread 483 | Our problem here is that the thread can live forever if we don't check a global value, which is...well ugly 484 | """ 485 | stream_output = "" 486 | read_queue = True 487 | while read_queue: 488 | try: 489 | line = output_queue.get(timeout=1) 490 | except queue.Empty: 491 | pass 492 | else: 493 | # The queue reading can be stopped once 'None' is received. 494 | if line is None: 495 | read_queue = False 496 | else: 497 | stream_output += line 498 | # ADD YOUR LIVE CODE HERE 499 | 500 | 501 | # Create a new queue that command_runner will fill up 502 | output_queue = queue.Queue() 503 | 504 | # Create a thread of read_queue() in order to read the queue while command_runner executes the command 505 | read_thread = threading.Thread( 506 | target=read_queue, args=(output_queue) 507 | ) 508 | read_thread.daemon = True # thread dies with the program 509 | read_thread.start() 510 | 511 | # Launch command_runner, which will be blocking. Your live code goes directly into the threaded function 512 | exit_code, output = command_runner('ping 127.0.0.1', stdout=output_queue, method='poller') 513 | ``` 514 | 515 | # UAC Elevation / sudo elevation 516 | 517 | command_runner package allowing privilege elevation. 518 | Becoming an admin is fairly easy with command_runner.elevate 519 | You only have to import the elevate module, and then launch your main function with the elevate function. 520 | 521 | ### elevation In a nutshell 522 | 523 | ```python 524 | from command_runner.elevate import elevate 525 | 526 | def main(): 527 | """My main function that should be elevated""" 528 | print("Who's the administrator, now ?") 529 | 530 | if __name__ == '__main__': 531 | elevate(main) 532 | ``` 533 | 534 | elevate function handles arguments (positional and keyword arguments). 535 | `elevate(main, arg, arg2, kw=somearg)` will call `main(arg, arg2, kw=somearg)` 536 | 537 | ### Advanced elevate usage 538 | 539 | #### is_admin() function 540 | 541 | The elevate module has a nifty is_admin() function that returns a boolean according to your current root/administrator privileges. 542 | Usage: 543 | 544 | ```python 545 | from command_runner.elevate import is_admin 546 | 547 | print('Am I an admin ? %s' % is_admin()) 548 | ``` 549 | 550 | #### sudo elevation 551 | 552 | Initially designed for Windows UAC, command_runner.elevate can also elevate privileges on Linux, using the sudo command. 553 | This is mainly designed for PyInstaller / Nuitka executables, as it's really not safe to allow automatic privilege elevation of a Python interpreter. 554 | 555 | Example for a binary in `/usr/local/bin/my_compiled_python_binary` 556 | 557 | You'll have to allow this file to be run with sudo without a password prompt. 558 | This can be achieved in `/etc/sudoers` file. 559 | 560 | Example for Redhat / Rocky Linux, where adding the following line will allow the elevation process to succeed without password: 561 | ``` 562 | someuser ALL= NOPASSWD:/usr/local/bin/my_compiled_python_binary 563 | ``` 564 | -------------------------------------------------------------------------------- /tests/test_command_runner.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of command_runner module 5 | 6 | """ 7 | command_runner is a quick tool to launch commands from Python, get exit code 8 | and output, and handle most errors that may happen 9 | 10 | Versioning semantics: 11 | Major version: backward compatibility breaking changes 12 | Minor version: New functionality 13 | Patch version: Backwards compatible bug fixes 14 | 15 | """ 16 | 17 | __intname__ = "command_runner_tests" 18 | __author__ = "Orsiris de Jong" 19 | __copyright__ = "Copyright (C) 2015-2025 Orsiris de Jong" 20 | __licence__ = "BSD 3 Clause" 21 | __build__ = "2025090901" 22 | 23 | 24 | import sys 25 | import os 26 | import platform 27 | import re 28 | import threading 29 | import logging 30 | import collections 31 | 32 | try: 33 | from command_runner import * 34 | except ImportError: # would be ModuleNotFoundError in Python 3+ 35 | # In case we run tests without actually having installed command_runner 36 | sys.path.insert(0, os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))) 37 | from command_runner import * 38 | 39 | # Python 2.7 compat where datetime.now() does not have .timestamp() method 40 | if sys.version_info[0] < 3 or sys.version_info[1] < 4: 41 | # python version < 3.3 42 | import time 43 | 44 | def timestamp(date): 45 | return time.mktime(date.timetuple()) 46 | 47 | else: 48 | 49 | def timestamp(date): 50 | return date.timestamp() 51 | 52 | 53 | # We need a logging unit here 54 | logger = logging.getLogger() 55 | logger.setLevel(logging.WARNING) 56 | handler = logging.StreamHandler(sys.stdout) 57 | handler.setLevel(logging.WARNING) 58 | formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") 59 | handler.setFormatter(formatter) 60 | logger.addHandler(handler) 61 | 62 | streams = ["stdout", "stderr"] 63 | methods = ["monitor", "poller"] 64 | 65 | TEST_FILENAME = "README.md" 66 | if os.name == "nt": 67 | ENCODING = "cp437" 68 | PING_CMD = "ping 127.0.0.1 -n 4" 69 | PING_CMD_10S = "ping 127.0.0.1 -n 10" 70 | PING_CMD_REDIR = PING_CMD + " 1>&2" 71 | # Make sure we run the failure command first so end result is okay 72 | PING_CMD_AND_FAILURE = "ping 0.0.0.0 -n 2 1>&2 & ping 127.0.0.1 -n 2" 73 | PING_FAILURE = "ping 0.0.0.0 -n 2 1>&2" 74 | 75 | PRINT_FILE_CMD = "type {}".format(TEST_FILENAME) 76 | # On widows, we cannot print binary files to console, type would transliterate it into text 77 | # This is a dummy test on windows 78 | PRINT_BINARY_FILE_CMD = "type C:\\Windows\\System32\\cmd.exe" 79 | else: 80 | ENCODING = "utf-8" 81 | PING_CMD = ["ping", "-c", "4", "127.0.0.1"] 82 | PING_CMD_10S = ["ping", "-c", "10", "127.0.0.1"] 83 | PING_CMD_REDIR = "ping -c 4 127.0.0.1 1>&2" 84 | PING_CMD_AND_FAILURE = "ping -c 2 0.0.0.0 1>&2; ping -c 2 127.0.0.1" 85 | PRINT_FILE_CMD = "cat {}".format(TEST_FILENAME) 86 | PRINT_BINARY_FILE_CMD = "cat /bin/sh" 87 | PING_FAILURE = "ping -c 2 0.0.0.0 1>&2" 88 | 89 | 90 | ELAPSED_TIME = timestamp(datetime.now()) 91 | PROCESS_ID = None 92 | STREAM_OUTPUT = "" 93 | PROC = None 94 | ON_EXIT_CALLED = False 95 | 96 | 97 | def reset_elapsed_time(): 98 | global ELAPSED_TIME 99 | ELAPSED_TIME = timestamp(datetime.now()) 100 | 101 | 102 | def get_elapsed_time(): 103 | return timestamp(datetime.now()) - ELAPSED_TIME 104 | 105 | 106 | def running_on_github_actions(): 107 | """ 108 | This is set in github actions workflow with 109 | env: 110 | RUNNING_ON_GITHUB_ACTIONS: true 111 | """ 112 | return os.environ.get("RUNNING_ON_GITHUB_ACTIONS") == "true" # bash 'true' 113 | 114 | 115 | def is_pypy(): 116 | """ 117 | Checks interpreter 118 | """ 119 | return True if platform.python_implementation().lower() == "pypy" else False 120 | 121 | 122 | def is_macos(): 123 | """ 124 | Checks if under Mac OS 125 | """ 126 | return platform.system().lower() == "darwin" 127 | 128 | 129 | def test_standard_ping_with_encoding(): 130 | """ 131 | Test command_runner with a standard ping and encoding parameter 132 | """ 133 | for method in methods: 134 | print("method={}".format(method)) 135 | exit_code, output = command_runner(PING_CMD, encoding=ENCODING, method=method) 136 | print(output) 137 | assert ( 138 | exit_code == 0 139 | ), "Exit code should be 0 for ping command with method {}".format(method) 140 | 141 | 142 | def test_standard_ping_with_default_encoding(): 143 | """ 144 | Without encoding, iter(stream.readline, '') will hang since the expected sentinel char would be b'': 145 | This could only happen on python <3.6 since command_runner decides to use an encoding anyway 146 | """ 147 | for method in methods: 148 | exit_code, output = command_runner(PING_CMD, encoding=None, method=method) 149 | print(output) 150 | assert ( 151 | exit_code == 0 152 | ), "Exit code should be 0 for ping command with method {}".format(method) 153 | 154 | 155 | def test_standard_ping_with_encoding_disabled(): 156 | """ 157 | Without encoding disabled, we should have binary output 158 | """ 159 | for method in methods: 160 | exit_code, output = command_runner(PING_CMD, encoding=False, method=method) 161 | print(output) 162 | assert ( 163 | exit_code == 0 164 | ), "Exit code should be 0 for ping command with method {}".format(method) 165 | assert isinstance(output, bytes), "Output should be binary." 166 | 167 | 168 | def test_direct_binary_output_to_stdout(): 169 | """ 170 | Without encoding disabled, we should have binary output 171 | """ 172 | for method in methods: 173 | exit_code, output = command_runner(PRINT_BINARY_FILE_CMD, encoding=False, shell=True, method=method) 174 | print(output) 175 | assert ( 176 | exit_code == 0 177 | ), "Exit code should be 0 for ping command with method {}".format(method) 178 | assert isinstance(output, bytes), "Output should be binary." 179 | 180 | def test_timeout(): 181 | """ 182 | Test command_runner with a timeout 183 | """ 184 | for method in methods: 185 | begin_time = datetime.now() 186 | exit_code, output = command_runner(PING_CMD, timeout=1, method=method) 187 | print(output) 188 | end_time = datetime.now() 189 | assert ( 190 | end_time - begin_time 191 | ).total_seconds() < 2, "It took more than 2 seconds for a timeout=1 command to finish with method {}".format( 192 | method 193 | ) 194 | assert ( 195 | exit_code == -254 196 | ), "Exit code should be -254 on timeout with method {}".format(method) 197 | assert "Timeout" in output, "Output should have timeout with method {}".format( 198 | method 199 | ) 200 | 201 | 202 | def test_timeout_with_subtree_killing(): 203 | """ 204 | Launch a subtree of long commands and see if timeout actually kills them in time 205 | """ 206 | if os.name != "nt": 207 | cmd = 'echo "test" && sleep 5 && echo "done"' 208 | else: 209 | cmd = "echo test && {} && echo done".format(PING_CMD) 210 | 211 | for method in methods: 212 | begin_time = datetime.now() 213 | exit_code, output = command_runner(cmd, shell=True, timeout=1, method=method) 214 | print(output) 215 | end_time = datetime.now() 216 | elapsed_time = (end_time - begin_time).total_seconds() 217 | assert ( 218 | elapsed_time < 4 219 | ), "It took more than 2 seconds for a timeout=1 command to finish with method {}".format( 220 | method 221 | ) 222 | assert ( 223 | exit_code == -254 224 | ), "Exit code should be -254 on timeout with method {}".format(method) 225 | assert "Timeout" in output, "Output should have timeout with method {}".format( 226 | method 227 | ) 228 | 229 | 230 | def test_no_timeout(): 231 | """ 232 | Test with setting timeout=None 233 | """ 234 | for method in methods: 235 | exit_code, output = command_runner(PING_CMD, timeout=None, method=method) 236 | print(output) 237 | assert ( 238 | exit_code == 0 239 | ), "Without timeout, command should have run with method {}".format(method) 240 | 241 | 242 | def test_live_output(): 243 | """ 244 | Test command_runner with live output to stdout 245 | """ 246 | for method in methods: 247 | exit_code, _ = command_runner( 248 | PING_CMD, stdout=PIPE, encoding=ENCODING, method=method 249 | ) 250 | assert ( 251 | exit_code == 0 252 | ), "Exit code should be 0 for ping command with method {}".format(method) 253 | 254 | 255 | def test_not_found(): 256 | """ 257 | Test command_runner with an unexisting command 258 | """ 259 | for method in methods: 260 | print("The following command should fail with method {}".format(method)) 261 | exit_code, output = command_runner("unknown_command_nowhere_to_be_found_1234") 262 | assert ( 263 | exit_code == -253 264 | ), "Unknown command should trigger a -253 exit code with method {}".format( 265 | method 266 | ) 267 | assert "failed" in output, "Error code -253 should be Command x failed, reason" 268 | 269 | 270 | def test_file_output(): 271 | """ 272 | Test command_runner with file output instead of stdout 273 | """ 274 | for method in methods: 275 | stdout_filename = "temp.test" 276 | stderr_filename = "temp.test.err" 277 | print("The following command should timeout") 278 | exit_code, output = command_runner( 279 | PING_CMD, 280 | timeout=1, 281 | stdout=stdout_filename, 282 | stderr=stderr_filename, 283 | method=method, 284 | ) 285 | assert os.path.isfile( 286 | stdout_filename 287 | ), "Log file does not exist with method {}".format(method) 288 | 289 | # We don't have encoding argument in Python 2, yet we need it for PyPy 290 | if sys.version_info[0] < 3: 291 | with open(stdout_filename, "r") as file_handle: 292 | output = file_handle.read() 293 | else: 294 | with open(stdout_filename, "r", encoding=ENCODING) as file_handle: 295 | output = file_handle.read() 296 | 297 | assert os.path.isfile( 298 | stderr_filename 299 | ), "stderr log file does not exist with method {}".format(method) 300 | assert ( 301 | exit_code == -254 302 | ), "Exit code should be -254 for timeouts with method {}".format(method) 303 | assert "Timeout" in output, "Output should have timeout with method {}".format( 304 | method 305 | ) 306 | 307 | # arbitrary time to make sure file handle was closed 308 | sleep(3) 309 | os.remove(stdout_filename) 310 | os.remove(stderr_filename) 311 | 312 | 313 | def test_valid_exit_codes(): 314 | """ 315 | Test command_runner with a failed ping but that should not trigger an error 316 | 317 | # WIP We could improve tests here by capturing logs 318 | """ 319 | valid_exit_codes = [0, 1, 2] 320 | if is_macos(): 321 | valid_exit_codes.append(68) # ping non-existent exits with such on Mac 322 | for method in methods: 323 | 324 | exit_code, _ = command_runner( 325 | "ping nonexistent_host", 326 | shell=True, 327 | valid_exit_codes=valid_exit_codes, 328 | method=method, 329 | ) 330 | assert ( 331 | exit_code in valid_exit_codes 332 | ), "Exit code not in valid list with method {}".format(method) 333 | 334 | exit_code, _ = command_runner( 335 | "ping nonexistent_host", shell=True, valid_exit_codes=True, method=method 336 | ) 337 | assert exit_code != 0, "Exit code should not be equal to 0" 338 | 339 | exit_code, _ = command_runner( 340 | "ping nonexistent_host", shell=True, valid_exit_codes=False, method=method 341 | ) 342 | assert exit_code != 0, "Exit code should not be equal to 0" 343 | 344 | exit_code, _ = command_runner( 345 | "ping nonexistent_host", shell=True, valid_exit_codes=None, method=method 346 | ) 347 | assert exit_code != 0, "Exit code should not be equal to 0" 348 | 349 | 350 | def test_unix_only_split_command(): 351 | """ 352 | This test is specifically written when command_runner receives a str command instead of a list on unix 353 | """ 354 | if os.name == "posix": 355 | for method in methods: 356 | exit_code, _ = command_runner(" ".join(PING_CMD), method=method) 357 | assert ( 358 | exit_code == 0 359 | ), "Non split command should not trigger an error with method {}".format( 360 | method 361 | ) 362 | 363 | 364 | def test_create_no_window(): 365 | """ 366 | Only used on windows, when we don't want to create a cmd visible windows 367 | """ 368 | for method in methods: 369 | exit_code, _ = command_runner(PING_CMD, windows_no_window=True, method=method) 370 | assert exit_code == 0, "Should have worked too with method {}".format(method) 371 | 372 | 373 | def test_read_file(): 374 | """ 375 | Read a couple of times the same file to be sure we don't get garbage from _read_pipe() 376 | This is a random failure detection test 377 | """ 378 | 379 | # We don't have encoding argument in Python 2, yet we need it for PyPy 380 | if sys.version_info[0] < 3: 381 | with open(TEST_FILENAME, "r") as file: 382 | file_content = file.read() 383 | else: 384 | with open(TEST_FILENAME, "r", encoding=ENCODING) as file: 385 | file_content = file.read() 386 | for method in methods: 387 | # pypy is quite slow with poller method on github actions. 388 | # Lets lower rounds 389 | max_rounds = 100 if is_pypy() else 1000 390 | print("\nSetting up test_read_file for {} rounds".format(max_rounds)) 391 | for round in range(0, max_rounds): 392 | print("Comparison round {} with method {}".format(round, method)) 393 | exit_code, output = command_runner( 394 | PRINT_FILE_CMD, shell=True, method=method 395 | ) 396 | if os.name == "nt": 397 | output = output.replace("\r\n", "\n") 398 | 399 | assert ( 400 | exit_code == 0 401 | ), "Did not succeed to read {}, method={}, exit_code: {}, output: {}".format( 402 | TEST_FILENAME, method, exit_code, output 403 | ) 404 | assert ( 405 | file_content == output 406 | ), "Round {} File content and output are not identical, method={}".format( 407 | round, method 408 | ) 409 | 410 | 411 | def test_stop_on_argument(): 412 | expected_output_regex = "Command .* was stopped because stop_on function returned True. Original output was:" 413 | 414 | def stop_on(): 415 | """ 416 | Simple function that returns True two seconds after reset_elapsed_time() has been called 417 | """ 418 | if get_elapsed_time() > 2: 419 | return True 420 | 421 | for method in methods: 422 | reset_elapsed_time() 423 | print("method={}".format(method)) 424 | exit_code, output = command_runner(PING_CMD, stop_on=stop_on, method=method) 425 | 426 | # On github actions only with Python 2.7.18, we sometimes get -251 failed because of OS: [Error 5] Access is denied 427 | # when os.kill(pid) is called in kill_childs_mod 428 | # On my windows platform using the same Python version, it works... 429 | # well nothing I can debug on github actions 430 | if running_on_github_actions() and os.name == "nt" and sys.version_info[0] < 3: 431 | assert exit_code in [ 432 | -253, 433 | -251, 434 | ], "Not as expected, we should get a permission error on github actions windows platform" 435 | else: 436 | assert ( 437 | exit_code == -251 438 | ), "Monitor mode should have been stopped by stop_on with exit_code -251. method={}, exit_code: {}, output: {}".format( 439 | method, exit_code, output 440 | ) 441 | assert ( 442 | re.match(expected_output_regex, output, re.MULTILINE) is not None 443 | ), "stop_on output is bogus. method={}, exit_code: {}, output: {}".format( 444 | method, exit_code, output 445 | ) 446 | 447 | 448 | def test_process_callback(): 449 | def callback(process_id): 450 | global PROCESS_ID 451 | PROCESS_ID = process_id 452 | 453 | for method in methods: 454 | exit_code, output = command_runner( 455 | PING_CMD, method=method, process_callback=callback 456 | ) 457 | assert ( 458 | exit_code == 0 459 | ), "Wrong exit code. method={}, exit_code: {}, output: {}".format( 460 | method, exit_code, output 461 | ) 462 | assert isinstance( 463 | PROCESS_ID, subprocess.Popen 464 | ), 'callback did not work properly. PROCESS_ID="{}"'.format(PROCESS_ID) 465 | 466 | 467 | def test_stream_callback(): 468 | global STREAM_OUTPUT 469 | 470 | def stream_callback(string): 471 | global STREAM_OUTPUT 472 | STREAM_OUTPUT += string 473 | print("CALLBACK: ", string) 474 | 475 | for stream in streams: 476 | stream_args = {stream: stream_callback} 477 | for method in methods: 478 | STREAM_OUTPUT = "" 479 | try: 480 | print("Method={}, stream={}, output=callback".format(method, stream)) 481 | exit_code, output = command_runner( 482 | PING_CMD_REDIR, shell=True, method=method, **stream_args 483 | ) 484 | except ValueError: 485 | if method == "poller": 486 | assert False, "ValueError should not be produced in poller mode." 487 | if method == "poller": 488 | assert ( 489 | exit_code == 0 490 | ), "Wrong exit code. method={}, exit_code: {}, output: {}".format( 491 | method, exit_code, output 492 | ) 493 | 494 | # Since we redirect STDOUT to STDERR 495 | assert ( 496 | STREAM_OUTPUT == output 497 | ), "Callback stream should contain same result as output" 498 | else: 499 | assert ( 500 | exit_code == -250 501 | ), "stream_callback exit_code is bogus. method={}, exit_code: {}, output: {}".format( 502 | method, exit_code, output 503 | ) 504 | 505 | 506 | def test_queue_output(): 507 | """ 508 | Thread command runner and get it's output queue 509 | """ 510 | 511 | if sys.version_info[0] < 3: 512 | print("Queue test uses concurrent futures. Won't run on python 2.7, sorry.") 513 | return 514 | 515 | # pypy is quite slow with poller method on github actions. 516 | # Lets lower rounds 517 | max_rounds = 100 if is_pypy() else 1000 518 | print("\nSetting up test_read_file for {} rounds".format(max_rounds)) 519 | for i in range(0, max_rounds): 520 | for stream in streams: 521 | for method in methods: 522 | if method == "monitor" and i > 1: 523 | # Dont bother to repeat the test for monitor mode more than once 524 | continue 525 | output_queue = queue.Queue() 526 | stream_output = "" 527 | stream_args = {stream: output_queue} 528 | print( 529 | "Round={}, Method={}, stream={}, output=queue".format( 530 | i, method, stream 531 | ) 532 | ) 533 | thread_result = command_runner_threaded( 534 | PRINT_FILE_CMD, shell=True, method=method, **stream_args 535 | ) 536 | 537 | read_queue = True 538 | while read_queue: 539 | try: 540 | line = output_queue.get(timeout=0.1) 541 | except queue.Empty: 542 | pass 543 | else: 544 | if line is None: 545 | break 546 | else: 547 | stream_output += line 548 | 549 | exit_code, output = thread_result.result() 550 | 551 | if method == "poller": 552 | assert ( 553 | exit_code == 0 554 | ), "Wrong exit code. method={}, exit_code: {}, output: {}".format( 555 | method, exit_code, output 556 | ) 557 | # Since we redirect STDOUT to STDERR 558 | if stream == "stdout": 559 | assert ( 560 | stream_output == output 561 | ), "stdout queue output should contain same result as output" 562 | if stream == "stderr": 563 | assert ( 564 | len(stream_output) == 0 565 | ), "stderr queue output should be empty" 566 | else: 567 | assert ( 568 | exit_code == -250 569 | ), "stream_queue exit_code is bogus. method={}, exit_code: {}, output: {}".format( 570 | method, exit_code, output 571 | ) 572 | 573 | 574 | def test_queue_non_threaded_command_runner(): 575 | """ 576 | Test case for Python 2.7 without proper threading return values 577 | """ 578 | 579 | def read_queue(output_queue, stream_output): 580 | """ 581 | Read the queue as thread 582 | Our problem here is that the thread can live forever if we don't check a global value, which is...well ugly 583 | """ 584 | read_queue = True 585 | while read_queue: 586 | try: 587 | line = output_queue.get(timeout=1) 588 | except queue.Empty: 589 | pass 590 | else: 591 | # The queue reading can be stopped once 'None' is received. 592 | if line is None: 593 | read_queue = False 594 | else: 595 | stream_output["value"] += line 596 | # ADD YOUR LIVE CODE HERE 597 | return stream_output 598 | 599 | for i in range(0, 20): 600 | for cmd in [PING_CMD, PRINT_FILE_CMD]: 601 | if cmd == PRINT_FILE_CMD: 602 | shell_args = {"shell": True} 603 | else: 604 | shell_args = {"shell": False} 605 | # Create a new queue that command_runner will fill up 606 | output_queue = queue.Queue() 607 | stream_output = {"value": ""} 608 | # Create a thread of read_queue() in order to read the queue while command_runner executes the command 609 | read_thread = threading.Thread( 610 | target=read_queue, args=(output_queue, stream_output) 611 | ) 612 | read_thread.daemon = True # thread dies with the program 613 | read_thread.start() 614 | 615 | # Launch command_runner 616 | print("Round={}, cmd={}".format(i, cmd)) 617 | exit_code, output = command_runner( 618 | cmd, stdout=output_queue, method="poller", **shell_args 619 | ) 620 | assert ( 621 | exit_code == 0 622 | ), "PING_CMD Exit code is not okay. exit_code={}, output={}".format( 623 | exit_code, output 624 | ) 625 | 626 | # Wait until we are sure that we emptied the queue 627 | while not output_queue.empty(): 628 | sleep(0.1) 629 | 630 | assert stream_output["value"] == output, "Output should be identical" 631 | 632 | 633 | def test_double_queue_threaded_stop(): 634 | """ 635 | Use both stdout and stderr queues and make them stop 636 | """ 637 | 638 | if sys.version_info[0] < 3: 639 | print("Queue test uses concurrent futures. Won't run on python 2.7, sorry.") 640 | return 641 | 642 | stdout_queue = queue.Queue() 643 | stderr_queue = queue.Queue() 644 | thread_result = command_runner_threaded( 645 | PING_CMD_AND_FAILURE, 646 | method="poller", 647 | shell=True, 648 | stdout=stdout_queue, 649 | stderr=stderr_queue, 650 | ) 651 | 652 | print("Begin to read queues") 653 | read_stdout = read_stderr = True 654 | while read_stdout or read_stderr: 655 | try: 656 | stdout_line = stdout_queue.get(timeout=0.1) 657 | except queue.Empty: 658 | pass 659 | else: 660 | if stdout_line is None: 661 | read_stdout = False 662 | print("stdout is finished") 663 | else: 664 | print("STDOUT:", stdout_line) 665 | 666 | try: 667 | stderr_line = stderr_queue.get(timeout=0.1) 668 | except queue.Empty: 669 | pass 670 | else: 671 | if stderr_line is None: 672 | read_stderr = False 673 | print("stderr is finished") 674 | else: 675 | print("STDERR:", stderr_line) 676 | 677 | while True: 678 | done = thread_result.done() 679 | print("Thread is done:", done) 680 | if done: 681 | break 682 | sleep(1) 683 | 684 | exit_code, _ = thread_result.result() 685 | assert exit_code == 0, "We did not succeed in running the thread" 686 | 687 | 688 | def test_deferred_command(): 689 | """ 690 | Using deferred_command in order to run a command after a given timespan 691 | """ 692 | test_filename = "deferred_test_file" 693 | if os.path.isfile(test_filename): 694 | os.remove(test_filename) 695 | deferred_command("echo test > {}".format(test_filename), defer_time=5) 696 | assert os.path.isfile(test_filename) is False, "File should not exist yet" 697 | sleep(6) 698 | assert os.path.isfile(test_filename) is True, "File should exist now" 699 | os.remove(test_filename) 700 | 701 | 702 | def test_powershell_output(): 703 | # Don't bother to test powershell on other platforms than windows 704 | if os.name != "nt": 705 | return 706 | """ 707 | Parts from windows_tools.powershell are used here 708 | """ 709 | 710 | powershell_interpreter = None 711 | # Try to guess powershell path if no valid path given 712 | interpreter_executable = "powershell.exe" 713 | for syspath in ["sysnative", "system32"]: 714 | try: 715 | # Let's try native powershell (64 bit) first or else 716 | # Import-Module may fail when running 32 bit powershell on 64 bit arch 717 | best_guess = os.path.join( 718 | os.environ.get("SYSTEMROOT", "C:"), 719 | syspath, 720 | "WindowsPowerShell", 721 | "v1.0", 722 | interpreter_executable, 723 | ) 724 | if os.path.isfile(best_guess): 725 | powershell_interpreter = best_guess 726 | break 727 | except KeyError: 728 | pass 729 | if powershell_interpreter is None: 730 | try: 731 | ps_paths = os.path.dirname(os.environ["PSModulePath"]).split(";") 732 | for ps_path in ps_paths: 733 | if ps_path.endswith("Modules"): 734 | ps_path = ps_path.strip("Modules") 735 | possible_ps_path = os.path.join(ps_path, interpreter_executable) 736 | if os.path.isfile(possible_ps_path): 737 | powershell_interpreter = possible_ps_path 738 | break 739 | except KeyError: 740 | pass 741 | 742 | if powershell_interpreter is None: 743 | raise OSError("Could not find any valid powershell interpreter") 744 | 745 | # Do not add -NoProfile so we don't end up in a path we're not supposed to 746 | command = powershell_interpreter + " -NonInteractive -NoLogo %s" % PING_CMD 747 | exit_code, output = command_runner(command, encoding="unicode_escape") 748 | print("powershell: ", exit_code, output) 749 | assert exit_code == 0, "Powershell execution failed." 750 | 751 | 752 | def test_null_redir(): 753 | for method in methods: 754 | print("method={}".format(method)) 755 | exit_code, output = command_runner(PING_CMD, stdout=False) 756 | print(exit_code) 757 | print("OUTPUT:", output) 758 | assert output is None, "We should not have any output here" 759 | 760 | exit_code, output = command_runner( 761 | PING_CMD_AND_FAILURE, shell=True, stderr=False 762 | ) 763 | print(exit_code) 764 | print("OUTPUT:", output) 765 | assert "0.0.0.0" not in output, "We should not get error output from here" 766 | 767 | for method in methods: 768 | print("method={}".format(method)) 769 | exit_code, stdout, stderr = command_runner( 770 | PING_CMD, split_streams=True, stdout=False, stderr=False 771 | ) 772 | print(exit_code) 773 | print("STDOUT:", stdout) 774 | print("STDERR:", stderr) 775 | assert stdout is None, "We should not have any output from stdout" 776 | assert stderr is None, "We should not have any output from stderr" 777 | 778 | exit_code, stdout, stderr = command_runner( 779 | PING_CMD_AND_FAILURE, 780 | shell=True, 781 | split_streams=True, 782 | stdout=False, 783 | stderr=False, 784 | ) 785 | print(exit_code) 786 | print("STDOUT:", stdout) 787 | print("STDERR:", stderr) 788 | assert stdout is None, "We should not have any output from stdout" 789 | assert stderr is None, "We should not have any output from stderr" 790 | 791 | 792 | def test_split_streams(): 793 | """ 794 | Test replacing output with stdout and stderr output 795 | """ 796 | for cmd in [PING_CMD, PING_CMD_AND_FAILURE]: 797 | for method in methods: 798 | print("cmd={}, method={}".format(cmd, method)) 799 | 800 | try: 801 | exit_code, _ = command_runner( 802 | cmd, method=method, shell=True, split_streams=True 803 | ) 804 | except ValueError: 805 | # Should generate a valueError 806 | pass 807 | except Exception as exc: 808 | assert ( 809 | False 810 | ), "We should have too many values to unpack here: {}".format(exc) 811 | 812 | exit_code, stdout, stderr = command_runner( 813 | cmd, method=method, shell=True, split_streams=True 814 | ) 815 | print("exit_code:", exit_code) 816 | print("STDOUT:", stdout) 817 | print("STDERR:", stderr) 818 | if cmd == PING_CMD: 819 | assert ( 820 | exit_code == 0 821 | ), "Exit code should be 0 for ping command with method {}".format( 822 | method 823 | ) 824 | assert "127.0.0.1" in stdout 825 | assert stderr is None 826 | if cmd == PING_CMD_AND_FAILURE: 827 | assert ( 828 | exit_code == 0 829 | ), "Exit code should be 0 for ping command with method {}".format( 830 | method 831 | ) 832 | assert "127.0.0.1" in stdout 833 | assert "0.0.0.0" in stderr 834 | 835 | 836 | def test_on_exit(): 837 | def on_exit(): 838 | global ON_EXIT_CALLED 839 | ON_EXIT_CALLED = True 840 | 841 | exit_code, _ = command_runner(PING_CMD, on_exit=on_exit) 842 | assert exit_code == 0, "Exit code is not null" 843 | assert ON_EXIT_CALLED is True, "On exit was never called" 844 | 845 | 846 | def test_no_close_queues(): 847 | """ 848 | Test no_close_queues 849 | """ 850 | 851 | if sys.version_info[0] < 3: 852 | print("Queue test uses concurrent futures. Won't run on python 2.7, sorry.") 853 | return 854 | 855 | stdout_queue = queue.Queue() 856 | stderr_queue = queue.Queue() 857 | thread_result = command_runner_threaded( 858 | PING_CMD_AND_FAILURE, 859 | method="poller", 860 | shell=True, 861 | stdout=stdout_queue, 862 | stderr=stderr_queue, 863 | no_close_queues=True, 864 | ) 865 | 866 | print("Begin to read queues") 867 | read_stdout = read_stderr = True 868 | wait_period = 50 # let's have 100 rounds of 2x timeout 0.1s = 10 seconds, which should be enough for exec to terminate 869 | while read_stdout or read_stderr: 870 | try: 871 | stdout_line = stdout_queue.get(timeout=0.1) 872 | except queue.Empty: 873 | pass 874 | else: 875 | if stdout_line is None: 876 | assert False, "STDOUT queue has been closed with no_close_queues" 877 | else: 878 | print("STDOUT:", stdout_line) 879 | 880 | try: 881 | stderr_line = stderr_queue.get(timeout=0.1) 882 | except queue.Empty: 883 | pass 884 | else: 885 | if stderr_line is None: 886 | assert False, "STDOUT queue has been closed with no_close_queues" 887 | else: 888 | print("STDERR:", stderr_line) 889 | wait_period -= 1 890 | if wait_period < 1: 891 | break 892 | 893 | while True: 894 | done = thread_result.done() 895 | print("Thread is done:", done) 896 | if done: 897 | break 898 | sleep(1) 899 | 900 | exit_code, _ = thread_result.result() 901 | assert exit_code == 0, "We did not succeed in running the thread" 902 | 903 | 904 | def test_low_priority(): 905 | def check_low_priority(process): 906 | niceness = psutil.Process(process.pid).nice() 907 | io_niceness = psutil.Process(process.pid).ionice() 908 | if os.name == "nt": 909 | assert niceness == 16384, "Process low prio niceness not properly set: {}".format( 910 | niceness 911 | ) 912 | assert io_niceness == 1, "Process low prio io niceness not set properly: {}".format( 913 | io_niceness 914 | ) 915 | else: 916 | assert niceness == 15, "Process low prio niceness not properly set: {}".format( 917 | niceness 918 | ) 919 | assert io_niceness == 3, "Process low prio io niceness not set properly: {}".format( 920 | io_niceness 921 | ) 922 | print("Nice !") 923 | 924 | def command_runner_thread(): 925 | return command_runner_threaded( 926 | PING_CMD, 927 | priority="low", 928 | io_priority="low", 929 | process_callback=check_low_priority, 930 | ) 931 | 932 | thread = threading.Thread(target=command_runner_thread, args=()) 933 | thread.daemon = True # thread dies with the program 934 | thread.start() 935 | 936 | 937 | def test_high_priority(): 938 | def check_high_priority(process): 939 | niceness = psutil.Process(process.pid).nice() 940 | io_niceness = psutil.Process(process.pid).ionice() 941 | if os.name == "nt": 942 | assert niceness == 128, "Process high prio niceness not properly set: {}".format( 943 | niceness 944 | ) 945 | # So se actually don't test this here, since high prio cannot be set on Windows unless 946 | # we have NtSetInformationProcess privilege 947 | # assert io_niceness == 3, "Process high prio io niceness not set properly: {}".format( 948 | # io_niceness 949 | # ) 950 | else: 951 | assert niceness == -15, "Process high prio niceness not properly set: {}".format( 952 | niceness 953 | ) 954 | assert io_niceness == 1, "Process high prio io niceness not set properly: {}".format( 955 | io_niceness 956 | ) 957 | print("Nice !") 958 | 959 | def command_runner_thread(): 960 | return command_runner_threaded( 961 | PING_CMD, 962 | priority="high", 963 | # io_priority="high", 964 | process_callback=check_high_priority, 965 | ) 966 | 967 | thread = threading.Thread(target=command_runner_thread, args=()) 968 | thread.daemon = True # thread dies with the program 969 | thread.start() 970 | 971 | 972 | def test_heartbeat(): 973 | # Log capture class, blatantly copied from https://stackoverflow.com/a/37967421/2635443 974 | class TailLogHandler(logging.Handler): 975 | 976 | def __init__(self, log_queue): 977 | logging.Handler.__init__(self) 978 | self.log_queue = log_queue 979 | 980 | def emit(self, record): 981 | self.log_queue.append(self.format(record)) 982 | 983 | 984 | class TailLogger(object): 985 | 986 | def __init__(self, maxlen): 987 | self._log_queue = collections.deque(maxlen=maxlen) 988 | self._log_handler = TailLogHandler(self._log_queue) 989 | 990 | def contents(self): 991 | return "\n".join(self._log_queue) 992 | 993 | @property 994 | def log_handler(self): 995 | return self._log_handler 996 | 997 | tail = TailLogger(10) 998 | 999 | formatter = logging.Formatter( 1000 | "%(asctime)s - %(name)s - %(levelname)s - %(message)s" 1001 | ) 1002 | 1003 | log_handler = tail.log_handler 1004 | log_handler.setFormatter(formatter) 1005 | logger.addHandler(log_handler) # Add the handler to the logger 1006 | logger.setLevel(logging.INFO) 1007 | 1008 | exit_code, output = command_runner( 1009 | PING_CMD_10S, heartbeat=2, shell=False 1010 | ) 1011 | log_contents = tail.contents() 1012 | print("LOGS:\n", log_contents) 1013 | print("END LOGS") 1014 | print("COMMAND_OUTPUT:\n", output) 1015 | assert exit_code == 0, "Exit code should be 0 for ping command with heartbeat" 1016 | # We should have a modulo 2 heeatbeat 1017 | assert ( 1018 | "Still running command after 4 seconds" in log_contents 1019 | ), "Output should have heartbeat" 1020 | 1021 | 1022 | if __name__ == "__main__": 1023 | print("Example code for %s, %s" % (__intname__, __build__)) 1024 | 1025 | test_standard_ping_with_encoding() 1026 | test_standard_ping_with_default_encoding() 1027 | test_standard_ping_with_encoding_disabled() 1028 | test_timeout() 1029 | test_timeout_with_subtree_killing() 1030 | test_no_timeout() 1031 | test_live_output() 1032 | test_not_found() 1033 | test_file_output() 1034 | test_valid_exit_codes() 1035 | test_unix_only_split_command() 1036 | test_create_no_window() 1037 | test_read_file() 1038 | test_stop_on_argument() 1039 | test_process_callback() 1040 | test_stream_callback() 1041 | test_queue_output() 1042 | test_queue_non_threaded_command_runner() 1043 | test_double_queue_threaded_stop() 1044 | test_deferred_command() 1045 | test_powershell_output() 1046 | test_null_redir() 1047 | test_split_streams() 1048 | test_on_exit() 1049 | test_no_close_queues() 1050 | test_low_priority() 1051 | test_high_priority() 1052 | test_heartbeat() 1053 | 1054 | -------------------------------------------------------------------------------- /command_runner/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of command_runner module 5 | 6 | """ 7 | command_runner is a quick tool to launch commands from Python, get exit code 8 | and output, and handle most errors that may happen 9 | 10 | Versioning semantics: 11 | Major version: backward compatibility breaking changes 12 | Minor version: New functionality 13 | Patch version: Backwards compatible bug fixes 14 | 15 | """ 16 | # python 2.7 compat fixes so all strings are considered unicode 17 | from __future__ import unicode_literals 18 | 19 | 20 | __intname__ = "command_runner" 21 | __author__ = "Orsiris de Jong" 22 | __copyright__ = "Copyright (C) 2015-2025 Orsiris de Jong for NetInvent" 23 | __licence__ = "BSD 3 Clause" 24 | __version__ = "1.7.5" 25 | __build__ = "2025090901" 26 | __compat__ = "python2.7+" 27 | 28 | import os 29 | import shlex 30 | import subprocess 31 | import sys 32 | from datetime import datetime 33 | from logging import getLogger 34 | from time import sleep 35 | 36 | 37 | # Avoid checking os type numerous times 38 | os_name = os.name 39 | 40 | 41 | # Don't bother with an ImportError since we need command_runner to work without dependencies 42 | try: 43 | import psutil 44 | 45 | # Also make sure we directly import priority classes so we can reuse them 46 | if os_name == "nt": 47 | from psutil import ( 48 | # ABOVE_NORMAL_PRIORITY_CLASS, 49 | BELOW_NORMAL_PRIORITY_CLASS, 50 | HIGH_PRIORITY_CLASS, 51 | IDLE_PRIORITY_CLASS, 52 | NORMAL_PRIORITY_CLASS, 53 | REALTIME_PRIORITY_CLASS, 54 | ) 55 | from psutil import ( 56 | IOPRIO_HIGH, 57 | IOPRIO_NORMAL, 58 | IOPRIO_LOW, 59 | # IOPRIO_VERYLOW, 60 | ) 61 | else: 62 | from psutil import ( 63 | IOPRIO_CLASS_BE, 64 | IOPRIO_CLASS_IDLE, 65 | # IOPRIO_CLASS_NONE, 66 | IOPRIO_CLASS_RT, 67 | ) 68 | except (ImportError, AttributeError): 69 | if os_name == "nt": 70 | BELOW_NORMAL_PRIORITY_CLASS = 16384 71 | HIGH_PRIORITY_CLASS = 128 72 | NORMAL_PRIORITY_CLASS = 32 73 | REALTIME_PRIORITY_CLASS = 256 74 | IDLE_PRIORITY_CLASS = 64 75 | IOPRIO_HIGH = 3 76 | IOPRIO_NORMAL = 2 77 | IOPRIO_LOW = 1 78 | else: 79 | IOPRIO_CLASS_IDLE = 3 80 | IOPRIO_CLASS_BE = 2 81 | IOPRIO_CLASS_RT = 1 82 | 83 | 84 | # Python 2.7 does not have priorities defined in subprocess module, but psutil has 85 | # Since Windows and Linux use different possible values, let's simplify things by 86 | # allowing 5 process priorities: verylow (idle), low, normal, high, rt 87 | # and 3 process io priorities: low, normal, high 88 | # For IO, rt == high 89 | if os_name == "nt": 90 | PRIORITIES = { 91 | "process": { 92 | "verylow": IDLE_PRIORITY_CLASS, 93 | "low": BELOW_NORMAL_PRIORITY_CLASS, 94 | "normal": NORMAL_PRIORITY_CLASS, 95 | "high": HIGH_PRIORITY_CLASS, 96 | "rt": REALTIME_PRIORITY_CLASS, 97 | }, 98 | "io": { 99 | "low": IOPRIO_LOW, 100 | "normal": IOPRIO_NORMAL, 101 | "high": IOPRIO_HIGH, 102 | }, 103 | } 104 | else: 105 | PRIORITIES = { 106 | "process": { 107 | "verylow": 20, 108 | "low": 15, 109 | "normal": 0, 110 | "high": -15, 111 | "rt": -20, 112 | }, 113 | "io": { 114 | "low": IOPRIO_CLASS_IDLE, 115 | "normal": IOPRIO_CLASS_BE, 116 | "high": IOPRIO_CLASS_RT, 117 | }, 118 | } 119 | 120 | 121 | try: 122 | import signal 123 | except ImportError: 124 | pass 125 | 126 | # Python 2.7 compat fixes (queue was Queue) 127 | try: 128 | import queue 129 | except ImportError: 130 | import Queue as queue 131 | import threading 132 | 133 | # Python 2.7 compat fixes (missing typing) 134 | try: 135 | from typing import Union, Optional, List, Tuple, Any, Callable 136 | except ImportError: 137 | pass 138 | 139 | # Python 2.7 compat fixes (no concurrent futures) 140 | try: 141 | from concurrent.futures import Future 142 | from functools import wraps 143 | except ImportError: 144 | # Python 2.7 just won't have concurrent.futures, so we just declare threaded and wraps in order to 145 | # avoid NameError 146 | def threaded(fn): 147 | """ 148 | Simple placeholder for python 2.7 149 | """ 150 | return fn 151 | 152 | def wraps(fn): 153 | """ 154 | Simple placeholder for python 2.7 155 | """ 156 | return fn 157 | 158 | 159 | # Python 2.7 compat fixes (no FileNotFoundError class) 160 | try: 161 | # pylint: disable=E0601 (used-before-assignment) 162 | FileNotFoundError 163 | except NameError: 164 | # pylint: disable=W0622 (redefined-builtin) 165 | FileNotFoundError = IOError 166 | 167 | # python <= 3.3 compat fixes (missing TimeoutExpired class) 168 | try: 169 | TimeoutExpired = subprocess.TimeoutExpired 170 | except AttributeError: 171 | 172 | class TimeoutExpired(BaseException): 173 | """ 174 | Basic redeclaration when subprocess.TimeoutExpired does not exist, python <= 3.3 175 | """ 176 | 177 | def __init__(self, cmd, timeout, output=None, stderr=None, *args, **kwargs): 178 | self.cmd = cmd 179 | self.timeout = timeout 180 | self.output = output 181 | self.stderr = stderr 182 | try: 183 | super().__init__(*args, **kwargs) 184 | except TypeError: 185 | # python 2.7 needs super(Baseclass, self) 186 | super(BaseException, self).__init__(*args, **kwargs) 187 | 188 | def __str__(self): 189 | return "Command '%s' timed out after %s seconds" % (self.cmd, self.timeout) 190 | 191 | @property 192 | def stdout(self): 193 | return self.output 194 | 195 | @stdout.setter 196 | def stdout(self, value): 197 | # There's no obvious reason to set this, but allow it anyway so 198 | # .stdout is a transparent alias for .output 199 | self.output = value 200 | 201 | 202 | class InterruptGetOutput(BaseException): 203 | """ 204 | Make sure we get the current output when process is stopped mid-execution 205 | """ 206 | 207 | def __init__(self, output, *args, **kwargs): 208 | self._output = output 209 | try: 210 | super().__init__(*args, **kwargs) 211 | except TypeError: 212 | # python 2.7 needs super(Baseclass, self) 213 | super(BaseException, self).__init__(*args, **kwargs) 214 | 215 | @property 216 | def output(self): 217 | return self._output 218 | 219 | 220 | class KbdInterruptGetOutput(InterruptGetOutput): 221 | """ 222 | Make sure we get the current output when KeyboardInterrupt is made 223 | """ 224 | 225 | def __init__(self, output, *args, **kwargs): 226 | self._output = output 227 | try: 228 | super().__init__(*args, **kwargs) 229 | except TypeError: 230 | # python 2.7 needs super(Baseclass, self) 231 | super(InterruptGetOutput, self).__init__(*args, **kwargs) 232 | 233 | @property 234 | def output(self): 235 | return self._output 236 | 237 | 238 | class StopOnInterrupt(InterruptGetOutput): 239 | """ 240 | Make sure we get the current output when optional stop_on function execution returns True 241 | """ 242 | 243 | def __init__(self, output, *args, **kwargs): 244 | self._output = output 245 | try: 246 | super().__init__(*args, **kwargs) 247 | except TypeError: 248 | # python 2.7 needs super(Baseclass, self) 249 | super(InterruptGetOutput, self).__init__(*args, **kwargs) 250 | 251 | @property 252 | def output(self): 253 | return self._output 254 | 255 | 256 | ### BEGIN DIRECT IMPORT FROM ofunctions.threading 257 | def call_with_future(fn, future, args, kwargs): 258 | """ 259 | Threading a function with return info using Future 260 | from https://stackoverflow.com/a/19846691/2635443 261 | 262 | Example: 263 | 264 | @threaded 265 | def somefunc(arg): 266 | return 'arg was %s' % arg 267 | 268 | 269 | thread = somefunc('foo') 270 | while thread.done() is False: 271 | time.sleep(1) 272 | 273 | print(thread.result()) 274 | """ 275 | try: 276 | result = fn(*args, **kwargs) 277 | future.set_result(result) 278 | except Exception as exc: 279 | future.set_exception(exc) 280 | 281 | 282 | # pylint: disable=E0102 (function-redefined) 283 | def threaded(fn): 284 | """ 285 | @threaded wrapper in order to thread any function 286 | 287 | @wraps decorator sole purpose is for function.__name__ to be the real function 288 | instead of 'wrapper' 289 | 290 | """ 291 | 292 | @wraps(fn) 293 | def wrapper(*args, **kwargs): 294 | if kwargs.pop("__no_threads", False): 295 | return fn(*args, **kwargs) 296 | future = Future() 297 | thread = threading.Thread( 298 | target=call_with_future, args=(fn, future, args, kwargs) 299 | ) 300 | thread.daemon = True 301 | thread.start() 302 | return future 303 | 304 | return wrapper 305 | 306 | 307 | ### END DIRECT IMPORT FROM ofunctions.threading 308 | 309 | 310 | logger = getLogger(__intname__) 311 | PIPE = subprocess.PIPE 312 | 313 | 314 | def _validate_process_priority( 315 | priority, # type: Union[int, str] 316 | ): 317 | # type: (...) -> int 318 | """ 319 | Check if priority int is valid 320 | """ 321 | 322 | def _raise_prio_error(priority, reason): 323 | raise ValueError( 324 | "Priority not valid ({}): {}. Please use one of {}".format( 325 | reason, priority, ", ".join(list(PRIORITIES["process"].keys())) 326 | ) 327 | ) 328 | 329 | if isinstance(priority, int): 330 | if os_name == "nt": 331 | _raise_prio_error(priority, "windows does not accept ints as priority") 332 | if -20 <= priority <= 20: 333 | _raise_prio_error(priority, "priority out of range") 334 | elif isinstance(priority, str): 335 | try: 336 | priority = PRIORITIES["process"][priority.lower()] 337 | except KeyError: 338 | _raise_prio_error(priority, "priority does not exist") 339 | return priority 340 | 341 | 342 | def _set_priority( 343 | pid, # type: int 344 | priority, # type: Union[int, str] 345 | priority_type, # type: str 346 | ): 347 | """ 348 | Set process and / or io prioritie 349 | """ 350 | priority = priority.lower() 351 | 352 | if priority_type == "process": 353 | _priority = _validate_process_priority(priority) 354 | psutil.Process(pid).nice(_priority) 355 | elif priority_type == "io": 356 | valid_io_priorities = list(PRIORITIES["io"].keys()) 357 | if priority not in valid_io_priorities: 358 | raise ValueError( 359 | "Bogus {} priority given: {}. Please use one of {}".format( 360 | priority_type, priority, ", ".join(valid_io_priorities) 361 | ) 362 | ) 363 | psutil.Process(pid).ionice(PRIORITIES[priority_type][priority]) 364 | else: 365 | raise ValueError("Bogus priority type given.") 366 | 367 | 368 | def set_priority( 369 | pid, # type: int 370 | priority, # type: Union[int, str] 371 | ): 372 | """ 373 | Shorthand for _set_priority 374 | """ 375 | _set_priority(pid, priority, "process") 376 | 377 | 378 | def set_io_priority( 379 | pid, # type: int 380 | priority, # type: str 381 | ): 382 | """ 383 | Shorthand for _set_priority 384 | """ 385 | _set_priority(pid, priority, "io") 386 | 387 | 388 | def to_encoding( 389 | process_output, # type: Union[str, bytes] 390 | encoding, # type: Optional[str] 391 | errors, # type: str 392 | ): 393 | # type: (...) -> str 394 | """ 395 | Convert bytes output to string and handles conversion errors 396 | Variation of ofunctions.string_handling.safe_string_convert 397 | """ 398 | 399 | if not encoding: 400 | return process_output 401 | 402 | # Compatibility for earlier Python versions where Popen has no 'encoding' nor 'errors' arguments 403 | if isinstance(process_output, bytes): 404 | try: 405 | process_output = process_output.decode(encoding, errors=errors) 406 | except TypeError: 407 | try: 408 | # handle TypeError: don't know how to handle UnicodeDecodeError in error callback 409 | process_output = process_output.decode(encoding, errors="ignore") 410 | except (ValueError, TypeError): 411 | # What happens when str cannot be concatenated 412 | logger.error("Output cannot be captured {}".format(process_output)) 413 | elif process_output is None: 414 | # We deal with strings. Alter output string to avoid NoneType errors 415 | process_output = "" 416 | return process_output 417 | 418 | 419 | def kill_childs_mod( 420 | pid=None, # type: int 421 | itself=False, # type: bool 422 | soft_kill=False, # type: bool 423 | ): 424 | # type: (...) -> bool 425 | """ 426 | Inline version of ofunctions.kill_childs that has no hard dependency on psutil 427 | 428 | Kills all children of pid (current pid can be obtained with os.getpid()) 429 | If no pid given current pid is taken 430 | Good idea when using multiprocessing, is to call with atexit.register(ofunctions.kill_childs, os.getpid(),) 431 | 432 | Beware: MS Windows does not maintain a process tree, so child dependencies are computed on the fly 433 | Knowing this, orphaned processes (where parent process died) cannot be found and killed this way 434 | 435 | Prefer using process.send_signal() in favor of process.kill() to avoid race conditions when PID was reused too fast 436 | 437 | :param pid: Which pid tree we'll kill 438 | :param itself: Should parent be killed too ? 439 | """ 440 | sig = None 441 | 442 | ### BEGIN COMMAND_RUNNER MOD 443 | if "psutil" not in sys.modules: 444 | logger.error( 445 | "No psutil module present. Can only kill direct pids, not child subtree." 446 | ) 447 | if "signal" not in sys.modules: 448 | logger.error( 449 | "No signal module present. Using direct psutil kill API which might have race conditions when PID is reused too fast." 450 | ) 451 | else: 452 | """ 453 | Warning: There are only a couple of signals supported on Windows platform 454 | 455 | Extract from signal.valid_signals(): 456 | 457 | Windows / Python 3.9-64 458 | {, , , , , , } 459 | 460 | Linux / Python 3.8-64 461 | {, , , , , , , , , , , , , , , 16, , , , , , , , , , , , , , , , , 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, } 462 | 463 | A ValueError will be raised in any other case. Note that not all systems define the same set of signal names; 464 | an AttributeError will be raised if a signal name is not defined as SIG* module level constant. 465 | """ 466 | try: 467 | if not soft_kill and hasattr(signal, "SIGKILL"): 468 | # Don't bother to make pylint go crazy on Windows 469 | # pylint: disable=E1101 470 | sig = signal.SIGKILL 471 | else: 472 | sig = signal.SIGTERM 473 | except NameError: 474 | sig = None 475 | ### END COMMAND_RUNNER MOD 476 | 477 | def _process_killer( 478 | process, # type: Union[subprocess.Popen, psutil.Process] 479 | sig, # type: signal.valid_signals 480 | soft_kill, # type: bool 481 | ): 482 | # (...) -> None 483 | """ 484 | Simple abstract process killer that works with signals in order to avoid reused PID race conditions 485 | and can prefers using terminate than kill 486 | """ 487 | if sig: 488 | try: 489 | process.send_signal(sig) 490 | # psutil.NoSuchProcess might not be available, let's be broad 491 | # pylint: disable=W0703 492 | except Exception: 493 | pass 494 | else: 495 | if soft_kill: 496 | process.terminate() 497 | else: 498 | process.kill() 499 | 500 | try: 501 | current_process = psutil.Process(pid) 502 | # psutil.NoSuchProcess might not be available, let's be broad 503 | # pylint: disable=W0703 504 | except Exception: 505 | if itself: 506 | ### BEGIN COMMAND_RUNNER MOD 507 | try: 508 | os.kill( 509 | pid, 15 510 | ) # 15 being signal.SIGTERM or SIGKILL depending on the platform 511 | except OSError as exc: 512 | if os_name == "nt": 513 | # We'll do an ugly hack since os.kill() has some pretty big caveats on Windows 514 | # especially for Python 2.7 where we can get Access Denied 515 | os.system("taskkill /F /pid {}".format(pid)) 516 | else: 517 | logger.error( 518 | "Could not properly kill process with pid {}: {}".format( 519 | pid, 520 | to_encoding(exc.__str__(), "utf-8", "backslashreplace"), 521 | ) 522 | ) 523 | raise 524 | ### END COMMAND_RUNNER MOD 525 | return False 526 | else: 527 | for child in current_process.children(recursive=True): 528 | _process_killer(child, sig, soft_kill) 529 | 530 | if itself: 531 | _process_killer(current_process, sig, soft_kill) 532 | return True 533 | 534 | 535 | def command_runner( 536 | command, # type: Union[str, List[str]] 537 | valid_exit_codes=False, # type: Union[List[int], bool] 538 | timeout=3600, # type: Optional[int] 539 | shell=False, # type: bool 540 | encoding=None, # type: Optional[Union[str, bool]] 541 | stdin=None, # type: Optional[Union[int, str, Callable, queue.Queue]] 542 | stdout=None, # type: Optional[Union[int, str, Callable, queue.Queue]] 543 | stderr=None, # type: Optional[Union[int, str, Callable, queue.Queue]] 544 | no_close_queues=False, # type: Optional[bool] 545 | windows_no_window=False, # type: bool 546 | live_output=False, # type: bool 547 | method="monitor", # type: str 548 | check_interval=0.05, # type: float 549 | stop_on=None, # type: Callable 550 | on_exit=None, # type: Callable 551 | process_callback=None, # type: Callable 552 | split_streams=False, # type: bool 553 | silent=False, # type: bool 554 | priority=None, # type: Union[int, str] 555 | io_priority=None, # type: str 556 | heartbeat=0, # type: int 557 | **kwargs # type: Any 558 | ): 559 | # type: (...) -> Union[Tuple[int, Optional[Union[bytes, str]]], Tuple[int, Optional[Union[bytes, str]], Optional[Union[bytes, str]]]] 560 | """ 561 | Unix & Windows compatible subprocess wrapper that handles output encoding and timeouts 562 | Newer Python check_output already handles encoding and timeouts, but this one is retro-compatible 563 | It is still recommended to set cp437 for windows and utf-8 for unix 564 | 565 | Also allows a list of various valid exit codes (ie no error when exit code = arbitrary int) 566 | 567 | command should be a list of strings, eg ['ping', '-c 2', '127.0.0.1'] 568 | command can also be a single string, ex 'ping -c 2 127.0.0.1' if shell=True or if os is Windows 569 | 570 | Accepts all of subprocess.popen arguments 571 | 572 | Whenever we can, we need to avoid shell=True in order to preserve better security 573 | Avoiding shell=True involves passing absolute paths to executables since we don't have shell PATH environment 574 | 575 | When no stdout option is given, we'll get output into the returned (exit_code, output) tuple 576 | When stdout = filename or stderr = filename, we'll write output to the given file 577 | 578 | live_output will poll the process for output and show it on screen (output may be non reliable, don't use it if 579 | your program depends on the commands' stdout output) 580 | 581 | windows_no_window will disable visible window (MS Windows platform only) 582 | 583 | stop_on is an optional function that will stop execution if function returns True 584 | 585 | priority and io_priority can be set to 'low', 'normal' or 'high' 586 | priority may also be an int from -20 to 20 on Unix 587 | 588 | heartbeat will log a line every heartbeat seconds informing that we're still alive 589 | 590 | Returns a tuple (exit_code, output) 591 | """ 592 | 593 | # Choose default encoding when none set 594 | # cp437 encoding assures we catch most special characters from cmd.exe 595 | # Unless encoding=False in which case nothing gets encoded except Exceptions and logger strings for Python 2 596 | error_encoding = "cp437" if os_name == "nt" else "utf-8" 597 | if encoding is None: 598 | encoding = error_encoding 599 | 600 | # Fix when unix command was given as single string 601 | # This is more secure than setting shell=True 602 | if os_name == "posix": 603 | if not shell and isinstance(command, str): 604 | command = shlex.split(command) 605 | elif shell and isinstance(command, list): 606 | command = " ".join(command) 607 | 608 | # Set default values for kwargs 609 | errors = kwargs.pop( 610 | "errors", "backslashreplace" 611 | ) # Don't let encoding issues make you mad 612 | universal_newlines = kwargs.pop("universal_newlines", False) 613 | creationflags = kwargs.pop("creationflags", 0) 614 | # subprocess.CREATE_NO_WINDOW was added in Python 3.7 for Windows OS only 615 | if ( 616 | windows_no_window 617 | and sys.version_info[0] >= 3 618 | and sys.version_info[1] >= 7 619 | and os_name == "nt" 620 | ): 621 | # Disable the following pylint error since the code also runs on nt platform, but 622 | # triggers an error on Unix 623 | # pylint: disable=E1101 624 | creationflags = creationflags | subprocess.CREATE_NO_WINDOW 625 | close_fds = kwargs.pop("close_fds", "posix" in sys.builtin_module_names) 626 | 627 | # Default buffer size. line buffer (1) is deprecated in Python 3.7+ 628 | bufsize = kwargs.pop("bufsize", 16384) 629 | 630 | # Decide whether we write to output variable only (stdout=None), to output variable and stdout (stdout=PIPE) 631 | # or to output variable and to file (stdout='path/to/file') 632 | if stdout is None: 633 | _stdout = PIPE 634 | stdout_destination = "pipe" 635 | elif callable(stdout): 636 | _stdout = PIPE 637 | stdout_destination = "callback" 638 | elif isinstance(stdout, queue.Queue): 639 | _stdout = PIPE 640 | stdout_destination = "queue" 641 | elif isinstance(stdout, str): 642 | # We will send anything to file 643 | _stdout = open(stdout, "wb") 644 | stdout_destination = "file" 645 | elif stdout is False: 646 | # Python 2.7 does not have subprocess.DEVNULL, hence we need to use a file descriptor 647 | try: 648 | _stdout = subprocess.DEVNULL 649 | except AttributeError: 650 | _stdout = PIPE 651 | stdout_destination = None 652 | else: 653 | # We will send anything to given stdout pipe 654 | _stdout = stdout 655 | stdout_destination = "pipe" 656 | 657 | # The only situation where we don't add stderr to stdout is if a specific target file was given 658 | if callable(stderr): 659 | _stderr = PIPE 660 | stderr_destination = "callback" 661 | elif isinstance(stderr, queue.Queue): 662 | _stderr = PIPE 663 | stderr_destination = "queue" 664 | elif isinstance(stderr, str): 665 | _stderr = open(stderr, "wb") 666 | stderr_destination = "file" 667 | elif stderr is False: 668 | try: 669 | _stderr = subprocess.DEVNULL 670 | except AttributeError: 671 | _stderr = PIPE 672 | stderr_destination = None 673 | elif stderr is not None: 674 | _stderr = stderr 675 | stderr_destination = "pipe" 676 | # Automagically add a pipe so we are sure not to redirect to stdout 677 | elif split_streams: 678 | _stderr = PIPE 679 | stderr_destination = "pipe" 680 | else: 681 | _stderr = subprocess.STDOUT 682 | stderr_destination = "stdout" 683 | 684 | def _read_pipe( 685 | stream, # type: io.StringIO 686 | output_queue, # type: queue.Queue 687 | ): 688 | # type: (...) -> None 689 | """ 690 | will read from subprocess.PIPE 691 | Must be threaded since readline() might be blocking on Windows GUI apps 692 | 693 | Partly based on https://stackoverflow.com/a/4896288/2635443 694 | """ 695 | 696 | # WARNING: Depending on the stream type (binary or text), the sentinel character 697 | # needs to be of the same type, or the iterator won't have an end 698 | 699 | # We also need to check that stream has readline, in case we're writing to files instead of PIPE 700 | 701 | # Another magnificent python 2.7 fix 702 | # So we need to convert sentinel_char which would be unicode because of unicode_litterals 703 | # to str which is the output format from stream.readline() 704 | 705 | if hasattr(stream, "readline"): 706 | sentinel_char = str("") if hasattr(stream, "encoding") else b"" 707 | for line in iter(stream.readline, sentinel_char): 708 | output_queue.put(line) 709 | output_queue.put(None) 710 | stream.close() 711 | 712 | def _get_error_output(output_stdout, output_stderr): 713 | """ 714 | Try to concatenate output for exceptions if possible 715 | """ 716 | try: 717 | return output_stdout + output_stderr 718 | except TypeError: 719 | if output_stdout: 720 | return output_stdout 721 | if output_stderr: 722 | return output_stderr 723 | return None 724 | 725 | def _heartbeat_thread( 726 | process, # type: Union[subprocess.Popen[str], subprocess.Popen] 727 | heartbeat, # type: int 728 | ): 729 | begin_time = datetime.now() 730 | while True: 731 | elapsed_time = int((datetime.now() - begin_time).total_seconds()) 732 | if elapsed_time > heartbeat and elapsed_time % heartbeat == 0: 733 | logger.info("Still running command after %s seconds" % elapsed_time) 734 | if process.poll() is not None: 735 | break 736 | sleep(1) 737 | 738 | def heartbeat_thread( 739 | process, # type: Union[subprocess.Popen[str], subprocess.Popen] 740 | heartbeat, # type: int 741 | ): 742 | """ 743 | Just a shorthand to run the heartbeat thread 744 | """ 745 | if heartbeat: 746 | heartbeat_thread = threading.Thread( 747 | target=_heartbeat_thread, 748 | args=( 749 | process, 750 | heartbeat, 751 | ), 752 | ) 753 | heartbeat_thread.daemon = True 754 | heartbeat_thread.start() 755 | 756 | def _poll_process( 757 | process, # type: Union[subprocess.Popen[str], subprocess.Popen] 758 | timeout, # type: int 759 | encoding, # type: str 760 | errors, # type: str 761 | ): 762 | # type: (...) -> Union[Tuple[int, Optional[str]], Tuple[int, Optional[str], Optional[str]]] 763 | """ 764 | Process stdout/stderr output polling is only used in live output mode 765 | since it takes more resources than using communicate() 766 | 767 | Reads from process output pipe until: 768 | - Timeout is reached, in which case we'll terminate the process 769 | - Process ends by itself 770 | 771 | Returns an encoded string of the pipe output 772 | """ 773 | 774 | def __check_timeout( 775 | begin_time, # type: datetime.timestamp 776 | timeout, # type: int 777 | ): 778 | # type: (...) -> None 779 | """ 780 | Simple subfunction to check whether timeout is reached 781 | Since we check this a lot, we put it into a function 782 | """ 783 | if timeout and (datetime.now() - begin_time).total_seconds() > timeout: 784 | kill_childs_mod(process.pid, itself=True, soft_kill=False) 785 | raise TimeoutExpired( 786 | process, timeout, _get_error_output(output_stdout, output_stderr) 787 | ) 788 | if stop_on and stop_on(): 789 | kill_childs_mod(process.pid, itself=True, soft_kill=False) 790 | raise StopOnInterrupt(_get_error_output(output_stdout, output_stderr)) 791 | 792 | begin_time = datetime.now() 793 | 794 | heartbeat_thread(process, heartbeat) 795 | 796 | if encoding is False: 797 | output_stdout = output_stderr = b"" 798 | else: 799 | output_stdout = output_stderr = "" 800 | 801 | try: 802 | if stdout_destination is not None: 803 | stdout_read_queue = True 804 | stdout_queue = queue.Queue() 805 | stdout_read_thread = threading.Thread( 806 | target=_read_pipe, args=(process.stdout, stdout_queue) 807 | ) 808 | stdout_read_thread.daemon = True # thread dies with the program 809 | stdout_read_thread.start() 810 | else: 811 | stdout_read_queue = False 812 | 813 | # Don't bother to read stderr if we redirect to stdout 814 | if stderr_destination not in ["stdout", None]: 815 | stderr_read_queue = True 816 | stderr_queue = queue.Queue() 817 | stderr_read_thread = threading.Thread( 818 | target=_read_pipe, args=(process.stderr, stderr_queue) 819 | ) 820 | stderr_read_thread.daemon = True # thread dies with the program 821 | stderr_read_thread.start() 822 | else: 823 | stderr_read_queue = False 824 | 825 | while stdout_read_queue or stderr_read_queue: 826 | if stdout_read_queue: 827 | try: 828 | line = stdout_queue.get(timeout=check_interval) 829 | except queue.Empty: 830 | pass 831 | else: 832 | if line is None: 833 | stdout_read_queue = False 834 | else: 835 | line = to_encoding(line, encoding, errors) 836 | if stdout_destination == "callback": 837 | stdout(line) 838 | if stdout_destination == "queue": 839 | stdout.put(line) 840 | if live_output: 841 | if encoding == False: 842 | # We need to allow binary output too, hence using sys.stdout.buffer instead of sys.stdout 843 | sys.stdout.buffer.write(line) 844 | else: 845 | sys.stdout.write(line) 846 | output_stdout += line 847 | 848 | if stderr_read_queue: 849 | try: 850 | line = stderr_queue.get(timeout=check_interval) 851 | except queue.Empty: 852 | pass 853 | else: 854 | if line is None: 855 | stderr_read_queue = False 856 | else: 857 | line = to_encoding(line, encoding, errors) 858 | if stderr_destination == "callback": 859 | stderr(line) 860 | if stderr_destination == "queue": 861 | stderr.put(line) 862 | if live_output: 863 | sys.stderr.write(line) 864 | if split_streams: 865 | output_stderr += line 866 | else: 867 | output_stdout += line 868 | 869 | __check_timeout(begin_time, timeout) 870 | 871 | # Make sure we wait for the process to terminate, even after 872 | # output_queue has finished sending data, so we catch the exit code 873 | while process.poll() is None: 874 | __check_timeout(begin_time, timeout) 875 | # Additional timeout check to make sure we don't return an exit code from processes 876 | # that were killed because of timeout 877 | __check_timeout(begin_time, timeout) 878 | exit_code = process.poll() 879 | if split_streams: 880 | return exit_code, output_stdout, output_stderr 881 | return exit_code, output_stdout 882 | 883 | except KeyboardInterrupt: 884 | raise KbdInterruptGetOutput(_get_error_output(output_stdout, output_stderr)) 885 | 886 | def _timeout_check_thread( 887 | process, # type: Union[subprocess.Popen[str], subprocess.Popen] 888 | timeout, # type: int 889 | must_stop, # type dict 890 | ): 891 | # type: (...) -> None 892 | 893 | """ 894 | Since elder python versions don't have timeout, we need to manually check the timeout for a process 895 | when working in process monitor mode 896 | """ 897 | 898 | begin_time = datetime.now() 899 | while True: 900 | if timeout and (datetime.now() - begin_time).total_seconds() > timeout: 901 | kill_childs_mod(process.pid, itself=True, soft_kill=False) 902 | must_stop["value"] = "T" # T stands for TIMEOUT REACHED 903 | break 904 | if stop_on and stop_on(): 905 | kill_childs_mod(process.pid, itself=True, soft_kill=False) 906 | must_stop["value"] = "S" # S stands for STOP_ON RETURNED TRUE 907 | break 908 | if process.poll() is not None: 909 | break 910 | # We definitely need some sleep time here or else we will overload CPU 911 | sleep(check_interval) 912 | 913 | def _monitor_process( 914 | process, # type: Union[subprocess.Popen[str], subprocess.Popen] 915 | timeout, # type: int 916 | encoding, # type: str 917 | errors, # type: str 918 | ): 919 | # type: (...) -> Union[Tuple[int, Optional[str]], Tuple[int, Optional[str], Optional[str]]] 920 | """ 921 | Create a thread in order to enforce timeout or a stop_on condition 922 | Get stdout output and return it 923 | """ 924 | 925 | # Shared mutable objects have proven to have race conditions with PyPy 3.7 (mutable object 926 | # is changed in thread, but outer monitor function has still old mutable object state) 927 | # Strangely, this happened only sometimes on github actions/ubuntu 20.04.3 & pypy 3.7 928 | # Just make sure the thread is done before using mutable object 929 | must_stop = {"value": False} 930 | 931 | thread = threading.Thread( 932 | target=_timeout_check_thread, 933 | args=(process, timeout, must_stop), 934 | ) 935 | thread.daemon = True # was setDaemon(True) which has been deprecated 936 | thread.start() 937 | 938 | heartbeat_thread(process, heartbeat) 939 | 940 | if encoding is False: 941 | output_stdout = output_stderr = b"" 942 | output_stdout_end = output_stderr_end = b"" 943 | else: 944 | output_stdout = output_stderr = "" 945 | output_stdout_end = output_stderr_end = "" 946 | 947 | try: 948 | # Don't use process.wait() since it may deadlock on old Python versions 949 | # Also it won't allow communicate() to get incomplete output on timeouts 950 | while process.poll() is None: 951 | if must_stop["value"]: 952 | break 953 | # We still need to use process.communicate() in this loop so we don't get stuck 954 | # with poll() is not None even after process is finished, when using shell=True 955 | # Behavior validated on python 3.7 956 | try: 957 | output_stdout, output_stderr = process.communicate() 958 | # ValueError is raised on closed IO file 959 | except (TimeoutExpired, ValueError): 960 | pass 961 | exit_code = process.poll() 962 | 963 | try: 964 | output_stdout_end, output_stderr_end = process.communicate() 965 | except (TimeoutExpired, ValueError): 966 | pass 967 | 968 | # Fix python 2.7 first process.communicate() call will have output whereas other python versions 969 | # will give output in second process.communicate() call 970 | if output_stdout_end and len(output_stdout_end) > 0: 971 | output_stdout = output_stdout_end 972 | if output_stderr_end and len(output_stderr_end) > 0: 973 | output_stderr = output_stderr_end 974 | 975 | if split_streams: 976 | if stdout_destination is not None: 977 | output_stdout = to_encoding(output_stdout, encoding, errors) 978 | if stderr_destination is not None: 979 | output_stderr = to_encoding(output_stderr, encoding, errors) 980 | else: 981 | if stdout_destination is not None: 982 | output_stdout = to_encoding(output_stdout, encoding, errors) 983 | 984 | # On PyPy 3.7 only, we can have a race condition where we try to read the queue before 985 | # the thread could write to it, failing to register a timeout. 986 | # This workaround prevents reading the mutable object while the thread is still alive 987 | while thread.is_alive(): 988 | sleep(check_interval) 989 | 990 | if must_stop["value"] == "T": 991 | raise TimeoutExpired( 992 | process, timeout, _get_error_output(output_stdout, output_stderr) 993 | ) 994 | if must_stop["value"] == "S": 995 | raise StopOnInterrupt(_get_error_output(output_stdout, output_stderr)) 996 | if split_streams: 997 | return exit_code, output_stdout, output_stderr 998 | return exit_code, output_stdout 999 | except KeyboardInterrupt: 1000 | raise KbdInterruptGetOutput(_get_error_output(output_stdout, output_stderr)) 1001 | 1002 | # After all the stuff above, here's finally the function main entry point 1003 | output_stdout = output_stderr = None 1004 | 1005 | try: 1006 | # Don't allow monitor method when stdout or stderr is callback/queue redirection (makes no sense) 1007 | if method == "monitor" and ( 1008 | stdout_destination 1009 | in [ 1010 | "callback", 1011 | "queue", 1012 | ] 1013 | or stderr_destination in ["callback", "queue"] 1014 | ): 1015 | raise ValueError( 1016 | 'Cannot use callback or queue destination in monitor mode. Please use method="poller" argument.' 1017 | ) 1018 | 1019 | # Finally, we won't use encoding & errors arguments for Popen 1020 | # since it would defeat the idea of binary pipe reading in live mode 1021 | 1022 | # Python >= 3.3 has SubProcessError(TimeoutExpired) class 1023 | # Python >= 3.6 has encoding & error arguments 1024 | # Python >= 3.7 has creationflags arguments for process priority under windows 1025 | # universal_newlines=True makes netstat command fail under windows 1026 | # timeout does not work under Python 2.7 with subprocess32 < 3.5 1027 | # decoder may be cp437 or unicode_escape for dos commands or utf-8 for powershell 1028 | 1029 | if priority: 1030 | process_prio = _validate_process_priority(priority) 1031 | # Don't bother to make pylint go crazy on Windows missing os.nice() 1032 | # pylint: disable=E1101 1033 | if os_name == "nt" and sys.version_info >= (3, 7): 1034 | creationflags |= process_prio 1035 | 1036 | # Actually we don't want preexec_fn since it's not thread safe, neither supported 1037 | # in subinterpreters. We'll use set_priority() once the process is spawned 1038 | # else: 1039 | # kwargs["preexec_fn"] = lambda: os.nice(process_prio) 1040 | 1041 | # Disabling pylint error for the same reason as above 1042 | # pylint: disable=E1123 1043 | if sys.version_info >= (3, 6): 1044 | process = subprocess.Popen( 1045 | command, 1046 | stdin=stdin, 1047 | stdout=_stdout, 1048 | stderr=_stderr, 1049 | shell=shell, 1050 | universal_newlines=universal_newlines, 1051 | encoding=encoding if encoding is not False else None, 1052 | errors=errors if encoding is not False else None, 1053 | creationflags=creationflags, 1054 | bufsize=bufsize, # 1 = line buffered 1055 | close_fds=close_fds, 1056 | **kwargs 1057 | ) 1058 | else: 1059 | process = subprocess.Popen( 1060 | command, 1061 | stdin=stdin, 1062 | stdout=_stdout, 1063 | stderr=_stderr, 1064 | shell=shell, 1065 | universal_newlines=universal_newlines, 1066 | creationflags=creationflags, 1067 | bufsize=bufsize, 1068 | close_fds=close_fds, 1069 | **kwargs 1070 | ) 1071 | 1072 | # Set process priority if not set earlier by creationflags 1073 | if priority and ( 1074 | os_name != "nt" or (sys.version_info < (3, 7) and os_name == "nt") 1075 | ): 1076 | try: 1077 | try: 1078 | set_priority(process.pid, priority) 1079 | except psutil.AccessDenied as exc: 1080 | logger.warning( 1081 | "Cannot set process priority {}: {}. Access denied.".format( 1082 | priority, exc 1083 | ) 1084 | ) 1085 | logger.debug("Trace:", exc_info=True) 1086 | except Exception as exc: 1087 | logger.warning( 1088 | "Cannot set process priority {}: {}".format(priority, exc) 1089 | ) 1090 | logger.debug("Trace:", exc_info=True) 1091 | except NameError: 1092 | logger.warning( 1093 | "Cannot set process priority. No psutil module installed." 1094 | ) 1095 | logger.debug("Trace:", exc_info=True) 1096 | # Set io priority if given 1097 | if io_priority: 1098 | try: 1099 | try: 1100 | set_io_priority(process.pid, io_priority) 1101 | except psutil.AccessDenied as exc: 1102 | logger.warning( 1103 | "Cannot set io priority {}: {} Access denied.".format( 1104 | io_priority, exc 1105 | ) 1106 | ) 1107 | logger.debug("Trace:", exc_info=True) 1108 | except Exception as exc: 1109 | logger.warning( 1110 | "Cannot set io priority {}: {}".format(io_priority, exc) 1111 | ) 1112 | logger.debug("Trace:", exc_info=True) 1113 | raise 1114 | except NameError: 1115 | logger.warning("Cannot set io priority. No psutil module installed.") 1116 | 1117 | try: 1118 | # let's return process information if callback was given 1119 | if callable(process_callback): 1120 | process_callback(process) 1121 | if method == "poller" or live_output and _stdout is not False: 1122 | if split_streams: 1123 | exit_code, output_stdout, output_stderr = _poll_process( 1124 | process, timeout, encoding, errors 1125 | ) 1126 | else: 1127 | exit_code, output_stdout = _poll_process( 1128 | process, timeout, encoding, errors 1129 | ) 1130 | elif method == "monitor": 1131 | if split_streams: 1132 | exit_code, output_stdout, output_stderr = _monitor_process( 1133 | process, timeout, encoding, errors 1134 | ) 1135 | else: 1136 | exit_code, output_stdout = _monitor_process( 1137 | process, timeout, encoding, errors 1138 | ) 1139 | else: 1140 | raise ValueError("Unknown method {} provided.".format(method)) 1141 | except KbdInterruptGetOutput as exc: 1142 | exit_code = -252 1143 | output_stdout = "KeyboardInterrupted. Partial output\n{}".format(exc.output) 1144 | try: 1145 | kill_childs_mod(process.pid, itself=True, soft_kill=False) 1146 | except AttributeError: 1147 | pass 1148 | if stdout_destination == "file" and output_stdout: 1149 | _stdout.write(output_stdout.encode(encoding, errors=errors)) 1150 | if stderr_destination == "file" and output_stderr: 1151 | _stderr.write(output_stderr.encode(encoding, errors=errors)) 1152 | elif stdout_destination == "file" and output_stderr: 1153 | _stdout.write(output_stderr.encode(encoding, errors=errors)) 1154 | 1155 | logger.debug( 1156 | 'Command "{}" returned with exit code "{}"'.format(command, exit_code) 1157 | ) 1158 | except subprocess.CalledProcessError as exc: 1159 | exit_code = exc.returncode 1160 | try: 1161 | output_stdout = exc.output 1162 | except AttributeError: 1163 | output_stdout = "command_runner: Could not obtain output from command." 1164 | 1165 | logger_fn = logger.error 1166 | valid_exit_codes_msg = "" 1167 | if valid_exit_codes: 1168 | if valid_exit_codes is True or exit_code in valid_exit_codes: 1169 | logger_fn = logger.info 1170 | valid_exit_codes_msg = " allowed" 1171 | 1172 | if not silent: 1173 | logger_fn( 1174 | 'Command "{}" failed with{} exit code "{}"'.format( 1175 | command, 1176 | valid_exit_codes_msg, 1177 | exc.returncode, 1178 | ) 1179 | ) 1180 | except FileNotFoundError as exc: 1181 | message = 'Command "{}" failed, file not found: {}'.format( 1182 | command, to_encoding(exc.__str__(), error_encoding, errors) 1183 | ) 1184 | if not silent: 1185 | logger.error(message) 1186 | if stdout_destination == "file": 1187 | _stdout.write(message.encode(error_encoding, errors=errors)) 1188 | exit_code, output_stdout = (-253, message) 1189 | # On python 2.7, OSError is also raised when file is not found (no FileNotFoundError) 1190 | # pylint: disable=W0705 (duplicate-except) 1191 | except (OSError, IOError) as exc: 1192 | message = 'Command "{}" failed because of OS: {}'.format( 1193 | command, to_encoding(exc.__str__(), error_encoding, errors) 1194 | ) 1195 | if not silent: 1196 | logger.error(message) 1197 | if stdout_destination == "file": 1198 | _stdout.write(message.encode(error_encoding, errors=errors)) 1199 | exit_code, output_stdout = (-253, message) 1200 | except TimeoutExpired as exc: 1201 | message = 'Timeout {} seconds expired for command "{}" execution. Original output was: {}'.format( 1202 | timeout, command, to_encoding(exc.output, error_encoding, errors)[-1000:] 1203 | ) 1204 | if not silent: 1205 | logger.error(message) 1206 | if stdout_destination == "file": 1207 | _stdout.write(message.encode(error_encoding, errors=errors)) 1208 | exit_code, output_stdout = (-254, message) 1209 | except StopOnInterrupt as exc: 1210 | message = "Command {} was stopped because stop_on function returned True. Original output was: {}".format( 1211 | command, to_encoding(exc.output, error_encoding, errors)[-1000:] 1212 | ) 1213 | if not silent: 1214 | logger.info(message) 1215 | if stdout_destination == "file": 1216 | _stdout.write(message.encode(error_encoding, errors=errors)) 1217 | exit_code, output_stdout = (-251, message) 1218 | except ValueError as exc: 1219 | message = to_encoding(exc.__str__(), error_encoding, errors) 1220 | if not silent: 1221 | logger.error(message, exc_info=True) 1222 | if stdout_destination == "file": 1223 | _stdout.write(message) 1224 | exit_code, output_stdout = (-250, message) 1225 | # We need to be able to catch a broad exception 1226 | # pylint: disable=W0703 1227 | except Exception as exc: 1228 | if not silent: 1229 | logger.error( 1230 | 'Command "{}" failed for unknown reasons: {}'.format( 1231 | command, to_encoding(exc.__str__(), error_encoding, errors) 1232 | ), 1233 | exc_info=True, 1234 | ) 1235 | exit_code, output_stdout = ( 1236 | -255, 1237 | to_encoding(exc.__str__(), error_encoding, errors), 1238 | ) 1239 | finally: 1240 | if stdout_destination == "file": 1241 | _stdout.close() 1242 | if stderr_destination == "file": 1243 | _stderr.close() 1244 | 1245 | # Make sure we send a simple queue end before leaving to make sure any queue read process will stop regardless 1246 | # of command_runner state (useful when launching with queue and method poller which isn't supposed to write queues) 1247 | if not no_close_queues: 1248 | if stdout_destination == "queue": 1249 | stdout.put(None) 1250 | if stderr_destination == "queue": 1251 | stderr.put(None) 1252 | 1253 | # With polling, we return None if nothing has been send to the queues 1254 | # With monitor, process.communicate() will result in '' even if nothing has been sent 1255 | # Let's fix this here 1256 | # Python 2.7 will return False to u'' == '' (UnicodeWarning: Unicode equal comparison failed) 1257 | # so we have to make the following statement 1258 | if stdout_destination is None or ( 1259 | output_stdout is not None and len(output_stdout) == 0 1260 | ): 1261 | output_stdout = None 1262 | if stderr_destination is None or ( 1263 | output_stderr is not None and len(output_stderr) == 0 1264 | ): 1265 | output_stderr = None 1266 | 1267 | stdout_output = to_encoding(output_stdout, error_encoding, errors) 1268 | if stdout_output: 1269 | logger.debug("STDOUT: " + stdout_output) 1270 | if stderr_destination not in ["stdout", None]: 1271 | stderr_output = to_encoding(output_stderr, error_encoding, errors) 1272 | if stderr_output and not silent: 1273 | if exit_code == 0 or ( 1274 | valid_exit_codes 1275 | and valid_exit_codes is True 1276 | or exit_code in valid_exit_codes 1277 | ): 1278 | logger.debug("STDERR: " + stderr_output) 1279 | else: 1280 | logger.error("STDERR: " + stderr_output) 1281 | 1282 | if on_exit: 1283 | logger.debug("Running on_exit callable.") 1284 | on_exit() 1285 | 1286 | if split_streams: 1287 | return exit_code, output_stdout, output_stderr 1288 | return exit_code, _get_error_output(output_stdout, output_stderr) 1289 | 1290 | 1291 | if sys.version_info[0] >= 3: 1292 | 1293 | @threaded 1294 | def command_runner_threaded(*args, **kwargs): 1295 | """ 1296 | Threaded version of command_runner_threaded which returns concurrent.Future result 1297 | Not available for Python 2.7 1298 | """ 1299 | return command_runner(*args, **kwargs) 1300 | 1301 | 1302 | def deferred_command(command, defer_time=300): 1303 | # type: (str, int) -> None 1304 | """ 1305 | This is basically an ugly hack to launch commands which are detached from parent process 1306 | Especially useful to launch an auto update/deletion of a running executable after a given amount of 1307 | seconds after it finished 1308 | """ 1309 | # Use ping as a standard timer in shell since it's present on virtually *any* system 1310 | if os_name == "nt": 1311 | deferrer = "ping 127.0.0.1 -n {} > NUL & ".format(defer_time) 1312 | else: 1313 | deferrer = "sleep {} && ".format(defer_time) 1314 | 1315 | # We'll create a independent shell process that will not be attached to any stdio interface 1316 | # Our command shall be a single string since shell=True 1317 | subprocess.Popen( 1318 | deferrer + command, 1319 | shell=True, 1320 | stdin=None, 1321 | stdout=None, 1322 | stderr=None, 1323 | close_fds=True, 1324 | ) 1325 | --------------------------------------------------------------------------------