├── mirakuru ├── py.typed ├── compat.py ├── __init__.py ├── pid.py ├── tcp.py ├── unixsocket.py ├── exceptions.py ├── base_env.py ├── http.py ├── output.py └── base.py ├── newsfragments ├── .gitignore ├── 960.misc.rst ├── 955.misc.rst ├── +143e55e6.misc.rst └── +2959172e.misc.rst ├── oldest-requirements.rq ├── .rstcheck.cfg ├── tests ├── executors │ ├── __init__.py │ ├── test_unixsocket_executor.py │ ├── test_output_executor.py │ ├── test_output_executor_regression_issue_98.py │ ├── test_pid_executor.py │ ├── test_tcp_executor.py │ ├── test_popen_kwargs.py │ ├── test_executor_kill.py │ ├── test_http_executor.py │ └── test_executor.py ├── sample_daemon.py ├── test_base.py ├── __init__.py ├── signals.py ├── retry.py ├── unixsocketserver_for_tests.py └── server_for_tests.py ├── .coveragerc ├── logo.png ├── MANIFEST.in ├── .gitignore ├── .github ├── workflows │ ├── pr-check.yml │ ├── build.yml │ ├── pre-commit.yml │ ├── pypi.yml │ ├── automerge.yml │ └── tests.yml └── dependabot.yml ├── AUTHORS.rst ├── Pipfile ├── mypy.ini ├── .pre-commit-config.yaml ├── CONTRIBUTING.rst ├── pyproject.toml ├── LICENSE ├── README.rst ├── CHANGES.rst └── logo.svg /mirakuru/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /newsfragments/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /oldest-requirements.rq: -------------------------------------------------------------------------------- 1 | psutil==4.0.0 2 | -------------------------------------------------------------------------------- /.rstcheck.cfg: -------------------------------------------------------------------------------- 1 | [rstcheck] 2 | report_level = warning 3 | -------------------------------------------------------------------------------- /tests/executors/__init__.py: -------------------------------------------------------------------------------- 1 | """Executors tests.""" 2 | -------------------------------------------------------------------------------- /newsfragments/960.misc.rst: -------------------------------------------------------------------------------- 1 | Do not install mypy on pypy. 2 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | include = 3 | mirakuru/* 4 | tests/* 5 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dbfixtures/mirakuru/HEAD/logo.png -------------------------------------------------------------------------------- /newsfragments/955.misc.rst: -------------------------------------------------------------------------------- 1 | Update pytest configuration to toml-native 2 | -------------------------------------------------------------------------------- /newsfragments/+143e55e6.misc.rst: -------------------------------------------------------------------------------- 1 | Lint required-version and classifiers within pyproject.toml 2 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE *.rst *.py mirakuru/py.typed 2 | recursive-include mirakuru/ *.py 3 | prune tests 4 | -------------------------------------------------------------------------------- /newsfragments/+2959172e.misc.rst: -------------------------------------------------------------------------------- 1 | Small refactoring to test_tcp_executor in an attempt to avoid random Address already in use. 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .cache/* 2 | *.egg-info 3 | dist 4 | build 5 | *.pyc 6 | .coverage 7 | venv/* 8 | 9 | # Pipenv 10 | Pipfile.lock 11 | 12 | .idea/ 13 | atlassian-ide-plugin.xml 14 | /.pytest_cache/ 15 | 16 | /.mypy_cache/ 17 | -------------------------------------------------------------------------------- /.github/workflows/pr-check.yml: -------------------------------------------------------------------------------- 1 | name: Run test commands 2 | 3 | on: 4 | pull_request: 5 | branches: [ main ] 6 | 7 | jobs: 8 | pr-check: 9 | uses: fizyk/actions-reuse/.github/workflows/shared-pr-check.yml@v4.1.1 10 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Test build package 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | uses: fizyk/actions-reuse/.github/workflows/shared-pypi.yml@v4.1.1 12 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | name: Run pre-commit 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | pre-commit: 11 | uses: fizyk/actions-reuse/.github/workflows/shared-pre-commit.yml@v4.1.1 12 | -------------------------------------------------------------------------------- /.github/workflows/pypi.yml: -------------------------------------------------------------------------------- 1 | name: Package and publish 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | 7 | 8 | 9 | jobs: 10 | build: 11 | uses: fizyk/actions-reuse/.github/workflows/shared-pypi.yml@v4.1.1 12 | with: 13 | publish: true 14 | secrets: 15 | pypi_token: ${{ secrets.pypi_password }} 16 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "04:00" 8 | open-pull-requests-limit: 1 9 | - package-ecosystem: github-actions 10 | directory: "/" 11 | schedule: 12 | interval: weekly 13 | time: "04:00" 14 | open-pull-requests-limit: 1 15 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | Authors 2 | ======= 3 | 4 | This file contains the list of people involved in the development of 5 | mirakuru along its history. 6 | 7 | * Mateusz Lenik 8 | * Tomasz Święcicki 9 | * Tomasz Krzyszczyk 10 | * Grzegorz Śliwiński 11 | * Paweł Wilczyński 12 | * Daniel O'Connell 13 | * Michał Pawłowski 14 | * Grégoire Détrez 15 | * Lars Gohr 16 | 17 | Great thanks to `Mateusz Lenik `_ for original package! 18 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [packages] 7 | psutil = "==7.2.0" 8 | 9 | [dev-packages] 10 | towncrier = "==25.8.0" 11 | pytest = "==9.0.2" 12 | pytest-cov = "==7.0.0" 13 | coverage = "==7.13.0" 14 | python-daemon = "==3.1.2" 15 | mypy = {version = "==1.19.1", markers="implementation_name == 'cpython'"} 16 | pygments = "==2.19.2" 17 | tbump = "==6.11.0" 18 | pytest-rerunfailures = "==16.1" 19 | pytest-xdist = "==3.8.0" 20 | -------------------------------------------------------------------------------- /.github/workflows/automerge.yml: -------------------------------------------------------------------------------- 1 | name: Merge me test dependencies! 2 | 3 | on: 4 | workflow_run: 5 | types: 6 | - completed 7 | workflows: 8 | # List all required workflow names here. 9 | - 'Run linters' 10 | - 'Run tests' 11 | - 'Run tests on macos' 12 | - 'Test build package' 13 | 14 | jobs: 15 | automerge: 16 | uses: fizyk/actions-reuse/.github/workflows/shared-automerge.yml@v4.1.1 17 | secrets: 18 | app_id: ${{ secrets.MERGE_APP_ID }} 19 | private_key: ${{ secrets.MERGE_APP_PRIVATE_KEY }} 20 | -------------------------------------------------------------------------------- /tests/sample_daemon.py: -------------------------------------------------------------------------------- 1 | """Daemon sample application for tests purposes. 2 | 3 | Stopping this process is possible only by the SIGKILL signal. 4 | 5 | Usage: 6 | 7 | python tests/sample_daemon.py 8 | 9 | """ 10 | 11 | import os 12 | import sys 13 | import time 14 | 15 | import daemon 16 | 17 | sys.path.append(os.getcwd()) 18 | 19 | from tests.signals import block_signals # noqa: E402 20 | 21 | with daemon.DaemonContext(initgroups=False): 22 | block_signals() 23 | while True: 24 | print("Sleeping mirakuru daemon...") 25 | time.sleep(1) 26 | -------------------------------------------------------------------------------- /tests/test_base.py: -------------------------------------------------------------------------------- 1 | """General mirakuru library tests.""" 2 | 3 | from mirakuru import * # noqa: F403 4 | 5 | 6 | def test_importing_mirakuru() -> None: 7 | """Test if all most commonly used classes are imported by default.""" 8 | assert "Executor" in globals() 9 | assert "SimpleExecutor" in globals() 10 | assert "OutputExecutor" in globals() 11 | assert "TCPExecutor" in globals() 12 | assert "HTTPExecutor" in globals() 13 | assert "PidExecutor" in globals() 14 | assert "ExecutorError" in globals() 15 | assert "TimeoutExpired" in globals() 16 | assert "AlreadyRunning" in globals() 17 | assert "ProcessExitedWithError" in globals() 18 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | allow_redefinition = False 3 | allow_untyped_globals = False 4 | check_untyped_defs = True 5 | disallow_incomplete_defs = True 6 | disallow_subclassing_any = True 7 | disallow_untyped_calls = True 8 | disallow_untyped_decorators = True 9 | disallow_untyped_defs = True 10 | follow_imports = silent 11 | ignore_missing_imports = False 12 | implicit_reexport = False 13 | no_implicit_optional = True 14 | pretty = True 15 | show_error_codes = True 16 | strict_equality = True 17 | warn_no_return = True 18 | warn_return_any = True 19 | warn_unreachable = True 20 | warn_unused_ignores = True 21 | 22 | [mypy-daemon.*] 23 | ignore_missing_imports = True 24 | 25 | [mypy-psutil.*] 26 | ignore_missing_imports = True 27 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Package of tests for mirakuru. 2 | 3 | Tests are written using py.test framework which dictates patterns that should 4 | be followed in test cases. 5 | """ 6 | 7 | import sys 8 | from os import path 9 | from subprocess import check_output 10 | 11 | TEST_PATH = path.abspath(path.dirname(__file__)) 12 | 13 | TEST_SERVER_PATH = path.join(TEST_PATH, "server_for_tests.py") 14 | TEST_SOCKET_SERVER_PATH = path.join(TEST_PATH, "unixsocketserver_for_tests.py") 15 | SAMPLE_DAEMON_PATH = path.join(TEST_PATH, "sample_daemon.py") 16 | 17 | HTTP_SERVER_CMD = f"{sys.executable} -m http.server" 18 | 19 | 20 | def ps_aux() -> str: 21 | """Return output of systems `ps aux -w` call.""" 22 | return check_output(("ps", "aux", "-w")).decode() 23 | -------------------------------------------------------------------------------- /tests/signals.py: -------------------------------------------------------------------------------- 1 | """Contains `block_signals` function for tests purposes.""" 2 | 3 | import signal 4 | from typing import Any 5 | 6 | 7 | def block_signals() -> None: 8 | """Catch all of the signals that it is possible. 9 | 10 | Reject their default behaviour. The process is actually mortal but the 11 | only way to kill is to send SIGKILL signal (kill -9). 12 | """ 13 | 14 | def sighandler(signum: int, _: Any) -> None: 15 | """Signal handling function.""" 16 | print(f"Tried to kill with signal {signum}.") 17 | 18 | for sgn in [x for x in dir(signal) if x.startswith("SIG")]: 19 | try: 20 | signum = getattr(signal, sgn) 21 | signal.signal(signum, sighandler) 22 | except (ValueError, RuntimeError, OSError): 23 | pass 24 | -------------------------------------------------------------------------------- /tests/retry.py: -------------------------------------------------------------------------------- 1 | """Small retry callable in case of specific error occurred.""" 2 | 3 | from datetime import datetime, timedelta, timezone 4 | from time import sleep 5 | from typing import Callable, Type, TypeVar 6 | 7 | from mirakuru import ExecutorError 8 | 9 | T = TypeVar("T") 10 | 11 | 12 | def retry( 13 | func: Callable[[], T], 14 | timeout: int = 60, 15 | possible_exception: Type[Exception] = ExecutorError, 16 | ) -> T: 17 | """Attempt to retry the function for timeout time.""" 18 | time: datetime = datetime.now(timezone.utc) 19 | timeout_diff: timedelta = timedelta(seconds=timeout) 20 | i = 0 21 | while True: 22 | i += 1 23 | try: 24 | res = func() 25 | return res 26 | except possible_exception as e: 27 | if time + timeout_diff < datetime.now(timezone.utc): 28 | raise TimeoutError("Failed after {i} attempts".format(i=i)) from e 29 | sleep(1) 30 | -------------------------------------------------------------------------------- /mirakuru/compat.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 by Clearcode 2 | # and associates (see AUTHORS). 3 | 4 | # This file is part of mirakuru. 5 | 6 | # mirakuru is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # mirakuru is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with mirakuru. If not, see . 18 | """Mirakuru compatibility module.""" 19 | 20 | import signal 21 | 22 | # Windows does not have SIGKILL, fall back to SIGTERM. 23 | SIGKILL = getattr(signal, "SIGKILL", signal.SIGTERM) 24 | 25 | 26 | __all__ = ("SIGKILL",) 27 | -------------------------------------------------------------------------------- /tests/executors/test_unixsocket_executor.py: -------------------------------------------------------------------------------- 1 | """TCPExecutor tests. 2 | 3 | Some of these tests run ``nc``: when running Debian, make sure the 4 | ``netcat-openbsd`` package is used, not ``netcat-traditional``. 5 | """ 6 | 7 | import sys 8 | 9 | import pytest 10 | 11 | from mirakuru import TimeoutExpired 12 | from mirakuru.unixsocket import UnixSocketExecutor 13 | from tests import TEST_SOCKET_SERVER_PATH 14 | 15 | 16 | def test_start_and_wait( 17 | tmp_path_factory: pytest.TempPathFactory, 18 | ) -> None: 19 | """Test if executor await for process to accept connections.""" 20 | socker_path = tmp_path_factory.getbasetemp() / "mirakuru.sock" 21 | socket_server_cmd = f"{sys.executable} {TEST_SOCKET_SERVER_PATH} {socker_path}" 22 | executor = UnixSocketExecutor(socket_server_cmd + " 2", socket_name=str(socker_path), timeout=5) 23 | with executor: 24 | assert executor.running() is True 25 | 26 | 27 | def test_start_and_timeout( 28 | tmp_path_factory: pytest.TempPathFactory, 29 | ) -> None: 30 | """Test if executor will properly time out.""" 31 | socker_path = tmp_path_factory.getbasetemp() / "mirakuru.sock" 32 | socket_server_cmd = f"{sys.executable} {TEST_SOCKET_SERVER_PATH} {socker_path}" 33 | executor = UnixSocketExecutor( 34 | socket_server_cmd + " 10", socket_name=str(socker_path), timeout=5 35 | ) 36 | 37 | with pytest.raises(TimeoutExpired): 38 | executor.start() 39 | 40 | assert executor.running() is False 41 | -------------------------------------------------------------------------------- /mirakuru/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 by Clearcode 2 | # and associates (see AUTHORS). 3 | 4 | # This file is part of mirakuru. 5 | 6 | # mirakuru is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # mirakuru is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with mirakuru. If not, see . 18 | 19 | """Mirakuru main module.""" 20 | 21 | import logging 22 | 23 | from mirakuru.base import Executor, SimpleExecutor 24 | from mirakuru.exceptions import ( 25 | AlreadyRunning, 26 | ExecutorError, 27 | ProcessExitedWithError, 28 | TimeoutExpired, 29 | ) 30 | from mirakuru.http import HTTPExecutor 31 | from mirakuru.output import OutputExecutor 32 | from mirakuru.pid import PidExecutor 33 | from mirakuru.tcp import TCPExecutor 34 | 35 | __version__ = "3.0.1" 36 | 37 | __all__ = ( 38 | "Executor", 39 | "SimpleExecutor", 40 | "OutputExecutor", 41 | "TCPExecutor", 42 | "HTTPExecutor", 43 | "PidExecutor", 44 | "ExecutorError", 45 | "TimeoutExpired", 46 | "AlreadyRunning", 47 | "ProcessExitedWithError", 48 | ) 49 | 50 | 51 | # Set default logging handler to avoid "No handler found" warnings. 52 | logging.getLogger(__name__).addHandler(logging.NullHandler()) 53 | -------------------------------------------------------------------------------- /tests/executors/test_output_executor.py: -------------------------------------------------------------------------------- 1 | # mypy: no-strict-optional 2 | """Output executor test.""" 3 | 4 | import subprocess 5 | 6 | import pytest 7 | 8 | from mirakuru import OutputExecutor 9 | from mirakuru.exceptions import TimeoutExpired 10 | 11 | 12 | def test_executor_waits_for_process_output() -> None: 13 | """Check if executor waits for specified output.""" 14 | command = 'bash -c "sleep 2 && echo foo && echo bar && sleep 100"' 15 | executor = OutputExecutor(command, "foo", timeout=10).start() 16 | 17 | assert executor.running() is True 18 | # foo has been used for start as a banner. 19 | assert executor.output().readline() == "bar\n" 20 | executor.stop() 21 | 22 | # check proper __str__ and __repr__ rendering: 23 | assert "OutputExecutor" in repr(executor) 24 | assert "foo" in str(executor) 25 | 26 | 27 | def test_executor_waits_for_process_err_output() -> None: 28 | """Check if executor waits for specified error output.""" 29 | command = 'bash -c "sleep 2 && >&2 echo foo && >&2 echo bar && sleep 100"' 30 | executor = OutputExecutor( 31 | command, "foo", timeout=10, stdin=None, stderr=subprocess.PIPE 32 | ).start() 33 | 34 | assert executor.running() is True 35 | # foo has been used for start as a banner. 36 | assert executor.err_output().readline() == "bar\n" 37 | executor.stop() 38 | 39 | # check proper __str__ and __repr__ rendering: 40 | assert "OutputExecutor" in repr(executor) 41 | assert "foo" in str(executor) 42 | 43 | 44 | def test_executor_dont_start() -> None: 45 | """Executor should not start.""" 46 | command = 'bash -c "sleep 2 && echo foo && echo bar && sleep 100"' 47 | executor = OutputExecutor(command, "foobar", timeout=3) 48 | with pytest.raises(TimeoutExpired): 49 | executor.start() 50 | 51 | assert executor.running() is False 52 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | ci: 3 | skip: [pipenv, mypy] 4 | 5 | # See https://pre-commit.com for more information 6 | # See https://pre-commit.com/hooks.html for more hooks 7 | minimum_pre_commit_version: 4.0.0 8 | default_stages: [pre-commit] 9 | repos: 10 | - repo: https://github.com/pre-commit/pre-commit-hooks 11 | rev: v6.0.0 12 | hooks: 13 | - id: check-added-large-files 14 | - id: check-case-conflict 15 | - id: check-merge-conflict 16 | - id: trailing-whitespace 17 | - id: check-toml 18 | - id: end-of-file-fixer 19 | - id: mixed-line-ending 20 | - id: check-yaml 21 | - id: pretty-format-json 22 | - id: detect-private-key 23 | - id: debug-statements 24 | 25 | - repo: https://github.com/astral-sh/ruff-pre-commit 26 | rev: v0.14.10 27 | hooks: 28 | - id: ruff-check 29 | args: [--fix, --exit-non-zero-on-fix, --respect-gitignore, --show-fixes] 30 | - id: ruff-format 31 | 32 | - repo: https://github.com/rstcheck/rstcheck 33 | rev: v6.2.5 34 | hooks: 35 | - id: rstcheck 36 | additional_dependencies: [sphinx, toml] 37 | 38 | - repo: https://github.com/fizyk/pyproject-validator 39 | rev: v0.1.0 40 | hooks: 41 | - id: check-python-version-consistency 42 | 43 | - repo: local 44 | hooks: 45 | - id: pipenv 46 | stages: [pre-commit, manual] 47 | language: system 48 | name: Install dependencies for the local linters 49 | entry: bash -c "pip install pipenv && pipenv install --dev" 50 | types_or: 51 | - python 52 | - toml # Pipfile 53 | pass_filenames: false 54 | - id: mypy 55 | stages: [pre-commit, manual] 56 | name: mypy 57 | entry: pipenv run mypy . 58 | language: system 59 | types_or: 60 | - python 61 | - toml # Pipfile 62 | pass_filenames: false 63 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Run tests 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | tests: 11 | uses: fizyk/actions-reuse/.github/workflows/shared-tests-pytests.yml@v4.1.1 12 | with: 13 | python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14", "pypy-3.11"]' 14 | pytest_opts: '-n auto --dist loadgroup' 15 | secrets: 16 | codecov_token: ${{ secrets.CODECOV_TOKEN }} 17 | macostests: 18 | uses: fizyk/actions-reuse/.github/workflows/shared-tests-pytests.yml@v4.1.1 19 | needs: [tests] 20 | with: 21 | python-versions: '["3.12", "3.13", "3.14", "pypy-3.11"]' 22 | pytest_opts: '-n auto --dist loadgroup --basetemp=$RUNNER_TEMP/pt' 23 | os: macos-14 # macos-latest is on macos-15 which misbehaves 24 | secrets: 25 | codecov_token: ${{ secrets.CODECOV_TOKEN }} 26 | oldest: 27 | needs: [tests] 28 | runs-on: 'ubuntu-latest' 29 | strategy: 30 | fail-fast: false 31 | matrix: 32 | python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "pypy-3.11"] 33 | steps: 34 | - uses: actions/checkout@v6 35 | - name: Set up Pipenv on python ${{ matrix.python-version }} 36 | uses: fizyk/actions-reuse/.github/actions/pipenv-setup@v4.1.1 37 | with: 38 | python-version: ${{ matrix.python-version }} 39 | cache: false 40 | allow-prereleases: true 41 | - name: Install oldest supported versions 42 | uses: fizyk/actions-reuse/.github/actions/pipenv-run@v4.1.1 43 | with: 44 | command: pip install -r oldest-requirements.rq 45 | - name: Run tests 46 | uses: fizyk/actions-reuse/.github/actions/pipenv-run@v4.1.1 47 | with: 48 | command: pytest -v --cov --cov-report=xml -n auto --dist loadgroup 49 | - name: Upload coverage to Codecov 50 | uses: codecov/codecov-action@v5.5.2 51 | with: 52 | token: ${{ secrets.CODECOV_TOKEN }} 53 | flags: unittests 54 | env_vars: OS,PYTHON 55 | fail_ci_if_error: false 56 | -------------------------------------------------------------------------------- /tests/executors/test_output_executor_regression_issue_98.py: -------------------------------------------------------------------------------- 1 | # mypy: no-strict-optional 2 | """Regression test for mirakuru issue #98. 3 | 4 | The problem: OutputExecutor hangs when there is too much output 5 | produced before the expected banner appears. 6 | 7 | This test intentionally generates a large amount of stdout noise before 8 | printing the banner. On a correct implementation, OutputExecutor should 9 | handle the stream and detect the banner within the timeout, starting the 10 | process successfully. Due to the bug, this currently times out/hangs, 11 | so we mark the test as xfail until the bug is fixed. 12 | 13 | See: https://github.com/dbfixtures/mirakuru/issues/98 14 | """ 15 | 16 | import subprocess 17 | 18 | from mirakuru import OutputExecutor, TimeoutExpired 19 | 20 | 21 | def test_output_executor_handles_large_output_before_banner() -> None: 22 | """OutputExecutor should not hang even with large pre-banner output. 23 | 24 | The command prints many large lines to stdout, then finally prints the 25 | banner and sleeps, keeping the process alive. If OutputExecutor properly 26 | drains stdout, it should detect the banner and start within the timeout. 27 | """ 28 | # Generate ~4-5 MB of output quickly, then print the banner and sleep. 29 | # Use only POSIX shell + bash built-ins to keep environment-simple. 30 | long_line = "x" * 80 31 | # The brace expansion {1..60000} relies on bash; we explicitly use bash -c. 32 | command = ( 33 | "bash -c '" 34 | f"for i in {{1..60000}}; do echo {long_line!s}; done; " 35 | "echo BANNER_READY; sleep 100'" 36 | ) 37 | 38 | # Use a reasonably generous timeout to allow draining and detection. 39 | executor = OutputExecutor( 40 | command, 41 | banner="BANNER_READY", 42 | timeout=15, 43 | stdin=None, 44 | stdout=subprocess.PIPE, 45 | stderr=subprocess.PIPE, 46 | ) 47 | 48 | try: 49 | with executor: 50 | assert executor.running() is True 51 | except TimeoutExpired: 52 | assert False, f"OutputExecutor should not hang. {executor.output()}" 53 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | Contribute to mirakuru 2 | ====================== 3 | 4 | Thank you for taking time to contribute to mirakuru! 5 | 6 | The following is a set of guidelines for contributing to mirakuru. These are just guidelines, not rules, use your best judgment and feel free to propose changes to this document in a pull request. 7 | 8 | Bug Reports 9 | ----------- 10 | 11 | #. Use a clear and descriptive title for the issue - it'll be much easier to identify the problem. 12 | #. Describe the steps to reproduce the problems in as many details as possible. 13 | #. If possible, provide a code snippet to reproduce the issue. 14 | 15 | Feature requests/proposals 16 | -------------------------- 17 | 18 | #. Use a clear and descriptive title for the proposal 19 | #. Provide as detailed description as possible 20 | * Use case is great to have 21 | #. There'll be a bit of discussion for the feature. Don't worry, if it is to be accepted, we'd like to support it, so we need to understand it thoroughly. 22 | 23 | 24 | Pull requests 25 | ------------- 26 | 27 | #. Start with a bug report or feature request 28 | #. Use a clear and descriptive title 29 | #. Provide a description - which issue does it refers to, and what part of the issue is being solved 30 | #. Be ready for code review :) 31 | 32 | Commits 33 | ------- 34 | 35 | #. Make sure commits are atomic, and each atomic change is being followed by test. 36 | #. If the commit solves part of the issue reported, include *refs #[Issue number]* in a commit message. 37 | #. If the commit solves whole issue reported, please refer to `Closing issues via commit messages `_ for ways to close issues when commits will be merged. 38 | 39 | Coding style 40 | ------------ 41 | 42 | #. Coding style is being handled by black and doublechecked by pycodestyle and pydocstyle 43 | 44 | Testing 45 | ------- 46 | 47 | # Tests are writen using pytest. 48 | # PR tests run on Github Actions. 49 | 50 | Release 51 | ------- 52 | 53 | Install pipenv and --dev dependencies first, Then run: 54 | 55 | .. code-block:: bash 56 | 57 | pipenv run tbump [NEW_VERSION] 58 | -------------------------------------------------------------------------------- /tests/executors/test_pid_executor.py: -------------------------------------------------------------------------------- 1 | """PidExecutor tests.""" 2 | 3 | import os 4 | from typing import Iterator 5 | 6 | import pytest 7 | 8 | from mirakuru import AlreadyRunning, PidExecutor, TimeoutExpired 9 | 10 | FILENAME = f"pid-test-tmp{os.getpid()}" 11 | SLEEP = f'bash -c "sleep 1 && touch {FILENAME} && sleep 1"' 12 | 13 | 14 | @pytest.fixture(autouse=True) 15 | def run_around_tests() -> Iterator[None]: 16 | """Make sure the **FILENAME** file is not present. 17 | 18 | This executor actually removes FILENAME as process used to test 19 | PidExecutor only creates it. 20 | """ 21 | try: 22 | os.remove(FILENAME) 23 | except OSError: 24 | pass 25 | 26 | yield 27 | 28 | try: 29 | os.remove(FILENAME) 30 | except OSError: 31 | pass 32 | 33 | 34 | def test_start_and_wait() -> None: 35 | """Test if the executor will await for the process to create a file.""" 36 | process = f'bash -c "sleep 2 && touch {FILENAME} && sleep 10"' 37 | with PidExecutor(process, FILENAME, timeout=5) as executor: 38 | assert executor.running() is True 39 | 40 | # check proper __str__ and __repr__ rendering: 41 | assert "PidExecutor" in repr(executor) 42 | assert process in str(executor) 43 | 44 | 45 | @pytest.mark.parametrize("pid_file", (None, "")) 46 | def test_empty_filename(pid_file: str | None) -> None: 47 | """Check whether an exception is raised if an empty FILENAME is given.""" 48 | with pytest.raises(ValueError): 49 | PidExecutor(SLEEP, pid_file) # type: ignore[arg-type] 50 | 51 | 52 | def test_if_file_created() -> None: 53 | """Check whether the process really created the given file.""" 54 | assert os.path.isfile(FILENAME) is False 55 | executor = PidExecutor(SLEEP, FILENAME) 56 | with executor: 57 | assert os.path.isfile(FILENAME) is True 58 | 59 | 60 | def test_timeout_error() -> None: 61 | """Check if timeout properly expires.""" 62 | executor = PidExecutor(SLEEP, FILENAME, timeout=1) 63 | 64 | with pytest.raises(TimeoutExpired): 65 | executor.start() 66 | 67 | assert executor.running() is False 68 | 69 | 70 | def test_fail_if_other_executor_running() -> None: 71 | """Test raising AlreadyRunning exception when port is blocked.""" 72 | process = f'bash -c "sleep 2 && touch {FILENAME} && sleep 10"' 73 | executor = PidExecutor(process, FILENAME) 74 | executor2 = PidExecutor(process, FILENAME) 75 | 76 | with executor: 77 | assert executor.running() is True 78 | 79 | with pytest.raises(AlreadyRunning): 80 | executor2.start() 81 | -------------------------------------------------------------------------------- /tests/executors/test_tcp_executor.py: -------------------------------------------------------------------------------- 1 | """TCPExecutor tests. 2 | 3 | Some of these tests run ``nc``: when running Debian, make sure the 4 | ``netcat-openbsd`` package is used, not ``netcat-traditional``. 5 | """ 6 | 7 | import logging 8 | import socket 9 | 10 | import pytest 11 | from _pytest.logging import LogCaptureFixture 12 | 13 | from mirakuru import AlreadyRunning, TCPExecutor, TimeoutExpired 14 | from tests import HTTP_SERVER_CMD 15 | 16 | 17 | # Allocate a random free port to avoid hardcoding and risking hitting an 18 | # occupied one; then let its number be reused elsewhere (by `nc`). 19 | def _find_free_port() -> int: 20 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 21 | s.bind(("localhost", 0)) 22 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 23 | return int(s.getsockname()[1]) 24 | 25 | 26 | def nc_command(port: int, sleep_seconds: int = 2) -> str: 27 | """Construct a command to start a netcat listener on the specified port.""" 28 | return f'bash -c "sleep {sleep_seconds} && nc -lk {port}"' 29 | 30 | 31 | def test_start_and_wait(caplog: LogCaptureFixture) -> None: 32 | """Test if the executor awaits for a process to accept connections.""" 33 | test_port = _find_free_port() 34 | caplog.set_level(logging.DEBUG, logger="mirakuru") 35 | executor = TCPExecutor(nc_command(test_port), "localhost", port=test_port, timeout=5) 36 | executor.start() 37 | assert executor.running() is True 38 | executor.stop() 39 | 40 | 41 | def test_repr_and_str() -> None: 42 | """Check the proper str and repr conversion.""" 43 | test_port = _find_free_port() 44 | nc = nc_command(test_port) 45 | executor = TCPExecutor(nc, "localhost", port=test_port, timeout=5) 46 | # check proper __str__ and __repr__ rendering: 47 | assert "TCPExecutor" in repr(executor) 48 | assert nc in str(executor) 49 | 50 | 51 | def test_it_raises_error_on_timeout() -> None: 52 | """Check if TimeoutExpired gets raised correctly.""" 53 | test_port = _find_free_port() 54 | executor = TCPExecutor(nc_command(test_port, 10), host="localhost", port=test_port, timeout=2) 55 | 56 | with pytest.raises(TimeoutExpired): 57 | executor.start() 58 | 59 | assert executor.running() is False 60 | 61 | 62 | def test_fail_if_other_executor_running() -> None: 63 | """Test raising AlreadyRunning exception.""" 64 | test_port = _find_free_port() 65 | http_server = f"{HTTP_SERVER_CMD} {test_port}" 66 | executor = TCPExecutor(http_server, host="localhost", port=test_port) 67 | executor2 = TCPExecutor(http_server, host="localhost", port=test_port) 68 | 69 | with executor: 70 | assert executor.running() is True 71 | 72 | with pytest.raises(AlreadyRunning): 73 | executor2.start() 74 | 75 | with pytest.raises(AlreadyRunning): 76 | with executor2: 77 | pass 78 | -------------------------------------------------------------------------------- /tests/unixsocketserver_for_tests.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2015, Doug Hellmann, All Rights Reserved 2 | # 3 | # Redistribution and use in source and binary forms, with or without 4 | # modification, are permitted provided that the following conditions are met: 5 | 6 | # * Redistributions of source code must retain the above copyright notice, 7 | # this list of conditions and the following disclaimer. 8 | # * Redistributions in binary form must reproduce the above copyright notice, 9 | # this list of conditions and the following disclaimer in the documentation 10 | # and/or other materials provided with the distribution. 11 | 12 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” 13 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 14 | # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 15 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 16 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 17 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 18 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 19 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 20 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 21 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 22 | # POSSIBILITY OF SUCH DAMAGE. 23 | """Sample unixsocket server with small modifications.""" 24 | 25 | import os 26 | import socket 27 | import sys 28 | from time import sleep 29 | 30 | SOCKET_ADDRESS = "./uds_socket" 31 | 32 | SLEEP = 0 33 | 34 | if len(sys.argv) >= 2: 35 | SOCKET_ADDRESS = sys.argv[1] 36 | 37 | if len(sys.argv) >= 3: 38 | SLEEP = int(sys.argv[2]) 39 | 40 | # Make sure the socket does not already exist 41 | try: 42 | os.unlink(SOCKET_ADDRESS) 43 | except OSError: 44 | if os.path.exists(SOCKET_ADDRESS): 45 | raise 46 | 47 | # Create a UDS socket 48 | SOCK = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 49 | 50 | # Bind the socket to the address 51 | print(f"starting up on {SOCKET_ADDRESS}") 52 | SOCK.bind(SOCKET_ADDRESS) 53 | sleep(SLEEP) 54 | 55 | # Listen for incoming connections 56 | SOCK.listen(1) 57 | 58 | while True: 59 | # Wait for a connection 60 | print("waiting for a connection") 61 | CONNECTION, CLIENT_ADDRESS = SOCK.accept() 62 | try: 63 | print("connection from", CLIENT_ADDRESS) 64 | 65 | # Receive the data in small chunks and retransmit it 66 | while True: 67 | RECEIVED_DATA = CONNECTION.recv(16) 68 | print(f"received {RECEIVED_DATA!r}") 69 | if RECEIVED_DATA: 70 | print("sending data back to the client") 71 | CONNECTION.sendall(RECEIVED_DATA) 72 | else: 73 | print("no data from", CLIENT_ADDRESS) 74 | break 75 | 76 | finally: 77 | # Clean up the connection 78 | CONNECTION.close() 79 | -------------------------------------------------------------------------------- /tests/executors/test_popen_kwargs.py: -------------------------------------------------------------------------------- 1 | """Test passing additional Popen kwargs to subprocess.Popen.""" 2 | 3 | import os 4 | from unittest import mock 5 | 6 | import pytest 7 | 8 | from mirakuru.base import SimpleExecutor 9 | 10 | 11 | @pytest.mark.parametrize("command", ("echo test", ["echo", "test"])) 12 | def test_additional_popen_kwargs_are_passed_through(command: str | list[str]) -> None: 13 | """Test that additional Popen kwargs are correctly passed through to subprocess.Popen. 14 | 15 | Given: 16 | Additional kwargs to be passed to popen. 17 | When: 18 | Executor starts 19 | Then: 20 | Additional kwargs are passed through. 21 | """ 22 | popen_kwargs = {"bufsize": 1, "close_fds": True} 23 | 24 | with mock.patch("subprocess.Popen") as popen_mock: 25 | # Make the mocked Popen act like a context manager-compatible Popen 26 | proc = mock.MagicMock() 27 | # Make poll() return None initially (running), but 0 after the first call 28 | # to simulate stopping 29 | proc.poll.side_effect = [None, 0] 30 | proc.wait.return_value = 0 31 | proc.__exit__.return_value = None 32 | proc.pid = 12345 33 | popen_mock.return_value = proc 34 | 35 | with SimpleExecutor(command, popen_kwargs=popen_kwargs): 36 | assert popen_mock.called 37 | args, kwargs = popen_mock.call_args 38 | 39 | assert kwargs.get("shell") is False 40 | 41 | # Additional kwargs passed through 42 | for k, v in popen_kwargs.items(): 43 | assert kwargs.get(k) == v 44 | 45 | # Executor still sets its standard kwargs 46 | assert "env" in kwargs and isinstance(kwargs["env"], dict) 47 | assert kwargs.get("cwd") is None 48 | assert kwargs.get("universal_newlines") is True 49 | assert callable(kwargs.get("preexec_fn")) 50 | 51 | 52 | def test_additional_popen_kwargs_do_not_override_standard_streams() -> None: 53 | """Test that additional Popen kwargs do not override standard streams. 54 | 55 | Given: 56 | Additional kwargs to be passed to popen. 57 | When: 58 | Executor starts 59 | Then: 60 | Standard streams are not overridden. 61 | """ 62 | with mock.patch("subprocess.Popen") as popen_mock: 63 | proc = mock.MagicMock() 64 | # Make poll() return None initially (running), but 0 after the first call 65 | # to simulate stopping 66 | proc.poll.side_effect = [None, 0] 67 | proc.wait.return_value = 0 68 | proc.__exit__.return_value = None 69 | proc.pid = 12345 70 | popen_mock.return_value = proc 71 | 72 | with SimpleExecutor("echo test", popen_kwargs={"stdout": os.devnull}): 73 | _, kwargs = popen_mock.call_args 74 | # stdout should not be os.devnull; it should be whatever SimpleExecutor sets 75 | # by default (PIPE). We cannot import subprocess.PIPE directly here without 76 | # shadowing the module under test; check that it's not the devnull we passed. 77 | assert kwargs.get("stdout") != os.devnull 78 | -------------------------------------------------------------------------------- /mirakuru/pid.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 by Clearcode 2 | # and associates (see AUTHORS). 3 | 4 | # This file is part of mirakuru. 5 | 6 | # mirakuru is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # mirakuru is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with mirakuru. If not, see . 18 | """Pid executor definition.""" 19 | 20 | import os.path 21 | from typing import Any 22 | 23 | from mirakuru.base import Executor 24 | 25 | 26 | class PidExecutor(Executor): 27 | """File existence checking process executor. 28 | 29 | Used to start processes that create pid files (or any other for that 30 | matter). Starts the given process and waits for the given file to be 31 | created. 32 | """ 33 | 34 | def __init__( 35 | self, 36 | command: str | list[str] | tuple[str, ...], 37 | filename: str, 38 | **kwargs: Any, 39 | ) -> None: 40 | """Initialize the PidExecutor executor. 41 | 42 | If the filename is empty, a ValueError is thrown. 43 | 44 | :param (str, list) command: command to be run by the subprocess 45 | :param str filename: the file which is to exist 46 | :param bool shell: same as the `subprocess.Popen` shell definition 47 | :param int timeout: number of seconds to wait for the process to start 48 | or stop. If None or False, wait indefinitely. 49 | :param float sleep: how often to check for start/stop condition 50 | :param int sig_stop: signal used to stop process run by the executor. 51 | default is `signal.SIGTERM` 52 | :param int sig_kill: signal used to kill process run by the executor. 53 | default is `signal.SIGKILL` (`signal.SIGTERM` on Windows) 54 | 55 | :raises: ValueError 56 | 57 | """ 58 | super().__init__(command, **kwargs) 59 | if not filename: 60 | raise ValueError("filename must be defined") 61 | self.filename = filename 62 | """the name of the file which the process is to create.""" 63 | 64 | def pre_start_check(self) -> bool: 65 | """Check if the specified file has been created. 66 | 67 | .. note:: 68 | 69 | The process will be considered started when it will have created 70 | the specified file as defined in the initializer. 71 | """ 72 | return os.path.isfile(self.filename) 73 | 74 | def after_start_check(self) -> bool: 75 | """Check if the process has created the specified file. 76 | 77 | .. note:: 78 | 79 | The process will be considered started when it will have created 80 | the specified file as defined in the initializer. 81 | """ 82 | return self.pre_start_check() # we can reuse logic from `pre_start()` 83 | -------------------------------------------------------------------------------- /mirakuru/tcp.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 by Clearcode 2 | # and associates (see AUTHORS). 3 | 4 | # This file is part of mirakuru. 5 | 6 | # mirakuru is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # mirakuru is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with mirakuru. If not, see . 18 | """TCP executor definition.""" 19 | 20 | import socket 21 | from typing import Any 22 | 23 | from mirakuru.base import Executor 24 | 25 | 26 | class TCPExecutor(Executor): 27 | """TCP-listening process executor. 28 | 29 | Used to start (and wait to actually be running) processes that can accept 30 | TCP connections. 31 | """ 32 | 33 | def __init__( 34 | self, 35 | command: str | list[str] | tuple[str, ...], 36 | host: str, 37 | port: int, 38 | **kwargs: Any, 39 | ) -> None: 40 | """Initialize TCPExecutor executor. 41 | 42 | :param (str, list) command: command to be run by the subprocess 43 | :param str host: host under which process is accessible 44 | :param int port: port under which process is accessible 45 | :param bool shell: same as the `subprocess.Popen` shell definition 46 | :param int timeout: number of seconds to wait for the process to start 47 | or stop. If None or False, wait indefinitely. 48 | :param float sleep: how often to check for start/stop condition 49 | :param int sig_stop: signal used to stop process run by the executor. 50 | default is `signal.SIGTERM` 51 | :param int sig_kill: signal used to kill process run by the executor. 52 | default is `signal.SIGKILL` (`signal.SIGTERM` on Windows) 53 | 54 | """ 55 | super().__init__(command, **kwargs) 56 | self.host = host 57 | """Host name, process is listening on.""" 58 | self.port = port 59 | """Port number, process is listening on.""" 60 | 61 | def pre_start_check(self) -> bool: 62 | """Check if process accepts connections. 63 | 64 | .. note:: 65 | 66 | Process will be considered started, when it'll be able to accept 67 | TCP connections as defined in initializer. 68 | """ 69 | try: 70 | sock = socket.socket() 71 | sock.connect((self.host, self.port)) 72 | return True 73 | except (socket.error, socket.timeout): 74 | return False 75 | finally: 76 | # close socket manually for sake of PyPy 77 | sock.close() 78 | 79 | def after_start_check(self) -> bool: 80 | """Check if process accepts connections. 81 | 82 | .. note:: 83 | 84 | Process will be considered started, when it'll be able to accept 85 | TCP connections as defined in initializer. 86 | """ 87 | return self.pre_start_check() # we can reuse logic from `pre_start()` 88 | -------------------------------------------------------------------------------- /mirakuru/unixsocket.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2019 by Clearcode 2 | # and associates (see AUTHORS). 3 | 4 | # This file is part of mirakuru. 5 | 6 | # mirakuru is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # mirakuru is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with mirakuru. If not, see . 18 | """TCP Socket executor definition.""" 19 | 20 | import logging 21 | import socket 22 | from typing import Any, List, Tuple, Union 23 | 24 | from mirakuru import Executor 25 | 26 | LOG = logging.getLogger(__name__) 27 | 28 | 29 | class UnixSocketExecutor(Executor): 30 | """Unixsocket listening process executor. 31 | 32 | Used to start (and wait to actually be running) processes that can accept 33 | stream Unix socket connections. 34 | """ 35 | 36 | def __init__( 37 | self, 38 | command: Union[str, List[str], Tuple[str, ...]], 39 | socket_name: str, 40 | **kwargs: Any, 41 | ) -> None: 42 | """Initialize UnixSocketExecutor executor. 43 | 44 | :param (str, list) command: command to be run by the subprocess 45 | :param str socket_name: unix socket path 46 | :param bool shell: same as the `subprocess.Popen` shell definition 47 | :param int timeout: number of seconds to wait for the process to start 48 | or stop. If None or False, wait indefinitely. 49 | :param float sleep: how often to check for start/stop condition 50 | :param int sig_stop: signal used to stop process run by the executor. 51 | default is `signal.SIGTERM` 52 | :param int sig_kill: signal used to kill process run by the executor. 53 | default is `signal.SIGKILL` (`signal.SIGTERM` on Windows) 54 | """ 55 | super().__init__(command, **kwargs) 56 | self.socket = socket_name 57 | 58 | def pre_start_check(self) -> bool: 59 | """Check if process accepts connections. 60 | 61 | .. note:: 62 | 63 | Process will be considered started, when it'll be able to accept 64 | Unix Socket connections as defined in initializer. 65 | """ 66 | exec_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 67 | try: 68 | exec_sock.connect(self.socket) 69 | return True 70 | except socket.error as msg: 71 | LOG.debug("Can not connect to socket: %s", msg) 72 | return False 73 | finally: 74 | # close socket manually for sake of PyPy 75 | exec_sock.close() 76 | 77 | def after_start_check(self) -> bool: 78 | """Check if process accepts connections. 79 | 80 | .. note:: 81 | 82 | Process will be considered started, when it'll be able to accept 83 | Unix Socket connections as defined in initializer. 84 | """ 85 | return self.pre_start_check() # we can reuse logic from `pre_start()` 86 | -------------------------------------------------------------------------------- /mirakuru/exceptions.py: -------------------------------------------------------------------------------- 1 | """Mirakuru exceptions.""" 2 | 3 | from typing import TYPE_CHECKING 4 | 5 | if TYPE_CHECKING: # pragma: no cover 6 | from mirakuru.base import SimpleExecutor # pylint:disable=cyclic-import 7 | 8 | 9 | class ExecutorError(Exception): 10 | """Base exception for executor failures.""" 11 | 12 | def __init__(self, executor: "SimpleExecutor") -> None: 13 | """Exception initialization. 14 | 15 | :param mirakuru.base.SimpleExecutor executor: for which exception 16 | occurred 17 | """ 18 | super().__init__(self) 19 | self.executor = executor 20 | 21 | 22 | class TimeoutExpired(ExecutorError): 23 | """Is raised when the timeout expires while starting an executor.""" 24 | 25 | def __init__(self, executor: "SimpleExecutor", timeout: int | float) -> None: 26 | """Exception initialization with an extra ``timeout`` argument. 27 | 28 | :param mirakuru.base.SimpleExecutor executor: for which exception 29 | occurred 30 | :param int timeout: timeout for which exception occurred 31 | """ 32 | super().__init__(executor) 33 | self.timeout = timeout 34 | 35 | def __str__(self) -> str: 36 | """Return Exception's string representation. 37 | 38 | :returns: string representation 39 | :rtype: str 40 | """ 41 | return f"Executor {self.executor} timed out after {self.timeout} seconds" 42 | 43 | 44 | class AlreadyRunning(ExecutorError): 45 | """Is raised when the executor seems to be already running. 46 | 47 | When some other process (not necessary executor) seems to be started with 48 | same configuration we can't bind to same port. 49 | """ 50 | 51 | def __str__(self) -> str: 52 | """Return Exception's string representation. 53 | 54 | :returns: string representation 55 | :rtype: str 56 | """ 57 | port = getattr(self.executor, "port") 58 | return ( 59 | f"Executor {self.executor} seems to be already running. " 60 | f"It looks like the previous executor process hasn't been " 61 | f"terminated or killed." 62 | + ( 63 | "" 64 | if port is None 65 | else f" Also there might be some completely " 66 | f"different service listening on {port} port." 67 | ) 68 | ) 69 | 70 | 71 | class ProcessExitedWithError(ExecutorError): 72 | """Raised when the process invoked by the executor returns a non-zero code. 73 | 74 | We allow the process to exit with zero because we support daemonizing 75 | subprocesses. We assume that when double-forking, the parent process will 76 | exit with 0 in case of successful daemonization. 77 | """ 78 | 79 | def __init__(self, executor: "SimpleExecutor", exit_code: int) -> None: 80 | """Exception initialization with an extra ``exit_code`` argument. 81 | 82 | :param mirakuru.base.SimpleExecutor executor: for which exception 83 | occurred 84 | :param int exit_code: code the subprocess exited with 85 | """ 86 | super().__init__(executor) 87 | self.exit_code = exit_code 88 | 89 | def __str__(self) -> str: 90 | """Return Exception's string representation. 91 | 92 | :returns: string representation 93 | :rtype: str 94 | """ 95 | return ( 96 | f"The process invoked by the {self.executor} executor has " 97 | f"exited with a non-zero code: {self.exit_code}." 98 | ) 99 | 100 | 101 | class ProcessFinishedWithError(ProcessExitedWithError): 102 | """Raised when the process invoked by the executor fails when stopping. 103 | 104 | When a process is stopped, it should shut down cleanly and return zero as 105 | exit code. When is returns a non-zero exit code, this exception is raised. 106 | """ 107 | -------------------------------------------------------------------------------- /mirakuru/base_env.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 by Clearcode 2 | # and associates (see AUTHORS). 3 | 4 | # This file is part of mirakuru. 5 | 6 | # mirakuru is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # mirakuru is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with mirakuru. If not, see . 18 | """Module contains functions used for finding process descendants.""" 19 | 20 | import errno 21 | import logging 22 | import re 23 | import subprocess 24 | 25 | try: 26 | import psutil 27 | except ImportError: 28 | psutil = None 29 | 30 | 31 | LOG = logging.getLogger(__name__) 32 | 33 | 34 | PS_XE_PID_MATCH = re.compile(r"^.*?(\d+).+$") 35 | """_sre.SRE_Pattern matching PIDs in result from `$ ps xe -o pid,cmd`.""" 36 | 37 | 38 | def processes_with_env_psutil(env_name: str, env_value: str) -> set[int]: 39 | """Find PIDs of processes having environment variable matching given one. 40 | 41 | Internally it uses `psutil` library. 42 | 43 | :param str env_name: name of environment variable to be found 44 | :param str env_value: environment variable value prefix 45 | :return: process identifiers (PIDs) of processes that have certain 46 | environment variable equal certain value 47 | :rtype: set 48 | """ 49 | pids = set() 50 | 51 | for proc in psutil.process_iter(): 52 | try: 53 | pinfo = proc.as_dict(attrs=["pid", "environ"]) 54 | except (psutil.NoSuchProcess, IOError): 55 | # can't do much if psutil is not able to get this process details 56 | pass 57 | else: 58 | penv = pinfo.get("environ") 59 | if penv and env_value in penv.get(env_name, ""): 60 | pids.add(pinfo["pid"]) 61 | 62 | return pids 63 | 64 | 65 | def processes_with_env_ps(env_name: str, env_value: str) -> set[int]: 66 | """Find PIDs of processes having environment variable matching given one. 67 | 68 | It uses `$ ps xe -o pid,cmd` command so it works only on systems 69 | having such command available (Linux, MacOS). If not available function 70 | will just LOG error. 71 | 72 | :param str env_name: name of environment variable to be found 73 | :param str env_value: environment variable value prefix 74 | :return: process identifiers (PIDs) of processes that have certain 75 | environment variable equal certain value 76 | :rtype: set 77 | """ 78 | pids: set[int] = set() 79 | ps_xe: list[bytes] = [] 80 | try: 81 | cmd = "ps", "xe", "-o", "pid,cmd" 82 | ps_xe = subprocess.check_output(cmd).splitlines() 83 | except OSError as err: 84 | if err.errno == errno.ENOENT: 85 | LOG.error( 86 | "`$ ps xe -o pid,cmd` command was called but it is not " 87 | "available on this operating system. Mirakuru will not " 88 | "be able to list the process tree and find if there are " 89 | "any leftovers of the Executor." 90 | ) 91 | return pids 92 | except subprocess.CalledProcessError: 93 | LOG.error("`$ ps xe -o pid,cmd` command exited with non-zero code.") 94 | 95 | env = f"{env_name}={env_value}" 96 | 97 | for line in ps_xe: 98 | sline = str(line) 99 | if env in sline: 100 | match = PS_XE_PID_MATCH.match(sline) 101 | # This always matches: all lines other than the header (not 102 | # containing our environment variable) have a PID required by the 103 | # reggex. Still check it for mypy. 104 | if match: 105 | pids.add(int(match.group(1))) 106 | return pids 107 | 108 | 109 | if psutil: 110 | processes_with_env = processes_with_env_psutil 111 | else: 112 | # In case psutil can't be imported (on pypy3) we try to use '$ ps xe' 113 | processes_with_env = processes_with_env_ps 114 | -------------------------------------------------------------------------------- /tests/executors/test_executor_kill.py: -------------------------------------------------------------------------------- 1 | # mypy: no-strict-optional 2 | """Tests that check various kill behaviours.""" 3 | 4 | import errno 5 | import os 6 | import signal 7 | import sys 8 | import time 9 | from typing import NoReturn, Set 10 | from unittest.mock import patch 11 | 12 | import pytest 13 | 14 | from mirakuru import HTTPExecutor, SimpleExecutor 15 | from mirakuru.compat import SIGKILL 16 | from mirakuru.exceptions import ProcessFinishedWithError 17 | from tests import SAMPLE_DAEMON_PATH, TEST_SERVER_PATH, ps_aux 18 | 19 | SLEEP_300 = "sleep 300" 20 | 21 | 22 | def test_custom_signal_kill() -> None: 23 | """Start process and shuts it down using signal SIGQUIT.""" 24 | executor = SimpleExecutor(SLEEP_300, kill_signal=signal.SIGQUIT) 25 | executor.start() 26 | assert executor.running() is True 27 | executor.kill() 28 | assert executor.running() is False 29 | 30 | 31 | def test_kill_custom_signal_kill() -> None: 32 | """Start process and shuts it down using signal SIGQUIT passed to kill.""" 33 | executor = SimpleExecutor(SLEEP_300) 34 | executor.start() 35 | assert executor.running() is True 36 | executor.kill(sig=signal.SIGQUIT) 37 | assert executor.running() is False 38 | 39 | 40 | def test_already_closed() -> None: 41 | """Check that the executor cleans after itself after it exited earlier.""" 42 | with pytest.raises(ProcessFinishedWithError) as excinfo: 43 | with SimpleExecutor("python") as executor: 44 | assert executor.running() 45 | os.killpg(executor.process.pid, SIGKILL) 46 | 47 | def process_stopped() -> bool: 48 | """Return True only only when self.process is not running.""" 49 | return executor.running() is False 50 | 51 | executor.wait_for(process_stopped) 52 | assert executor.process 53 | assert excinfo.value.exit_code == -9 54 | assert not executor.process 55 | 56 | 57 | @pytest.mark.xdist_group(name="sample-deamon") 58 | def test_daemons_killing() -> None: 59 | """Test if all subprocesses of SimpleExecutor can be killed. 60 | 61 | The most problematic subprocesses are daemons or other services that 62 | change the process group ID. This test verifies that daemon process 63 | is killed after executor's kill(). 64 | """ 65 | executor = SimpleExecutor(("python", SAMPLE_DAEMON_PATH), shell=True) 66 | executor.start() 67 | time.sleep(2) 68 | assert executor.running() is not True, ( 69 | "Executor should not have subprocess running as it started a daemon." 70 | ) 71 | 72 | assert SAMPLE_DAEMON_PATH in ps_aux() 73 | executor.kill() 74 | assert SAMPLE_DAEMON_PATH not in ps_aux() 75 | 76 | 77 | def test_stopping_brutally() -> None: 78 | """Test if SimpleExecutor is stopping insubordinate process. 79 | 80 | Check if the process that doesn't react to SIGTERM signal will be killed 81 | by executor with SIGKILL automatically. 82 | """ 83 | host_port = "127.0.0.1:8000" 84 | cmd = f"{sys.executable} {TEST_SERVER_PATH} {host_port} True" 85 | executor = HTTPExecutor(cmd, f"http://{host_port!s}/", timeout=20) 86 | executor.start() 87 | assert executor.running() is True 88 | 89 | stop_at = time.time() + 10 90 | executor.stop() 91 | assert executor.running() is False 92 | assert stop_at <= time.time(), "Subprocess killed earlier than in 10 secs" 93 | 94 | 95 | def test_stopping_children_of_stopped_process() -> None: 96 | """Check that children exiting between listing and killing are ignored. 97 | 98 | Given: 99 | Executor is running and it's process spawn children, 100 | and we requested it's stop, and it's stopped 101 | When: 102 | At the time of the check for subprocesses they're still active, 103 | but before we start killing them, they are already dead. 104 | Then: 105 | We ignore and skip OsError indicates there's no such process. 106 | """ 107 | 108 | # pylint: disable=protected-access, missing-docstring 109 | def raise_os_error(*_: int, **__: int) -> NoReturn: 110 | os_error = OSError() 111 | os_error.errno = errno.ESRCH 112 | raise os_error 113 | 114 | def processes_with_env_mock(*_: str, **__: str) -> Set[int]: 115 | return {1} 116 | 117 | with ( 118 | patch("mirakuru.base.processes_with_env", new=processes_with_env_mock), 119 | patch("os.kill", new=raise_os_error), 120 | ): 121 | executor = SimpleExecutor(SLEEP_300) 122 | executor._kill_all_kids(executor._stop_signal) 123 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "mirakuru" 3 | version = "3.0.1" 4 | description = "Process executor (not only) for tests." 5 | readme = "README.rst" 6 | keywords = ["process", "executor", "tests", "orchestration"] 7 | license = "LGPL-3.0-or-later" 8 | license-files = ["LICENSE"] 9 | authors = [ 10 | {name = "Grzegorz Śliwiński", email = "fizyk+pypi@fizyk.dev"} 11 | ] 12 | classifiers = [ 13 | "Development Status :: 5 - Production/Stable", 14 | "Intended Audience :: Developers", 15 | "Natural Language :: English", 16 | "Operating System :: OS Independent", 17 | "Programming Language :: Python", 18 | "Programming Language :: Python :: 3", 19 | "Programming Language :: Python :: 3.10", 20 | "Programming Language :: Python :: 3.11", 21 | "Programming Language :: Python :: 3.12", 22 | "Programming Language :: Python :: 3.13", 23 | "Programming Language :: Python :: 3.14", 24 | "Programming Language :: Python :: 3 :: Only", 25 | "Topic :: Software Development :: Libraries :: Python Modules", 26 | ] 27 | dependencies = [ 28 | # psutil is used to find processes leaked during termination. 29 | # It runs on many platforms but not Cygwin: 30 | # . 31 | "psutil>=4.0.0; sys_platform != 'cygwin'", 32 | ] 33 | requires-python = ">= 3.10" 34 | 35 | [project.urls] 36 | "Source" = "https://github.com/dbfixtures/mirakuru" 37 | "Bug Tracker" = "https://github.com/dbfixtures/mirakuru/issues" 38 | "Changelog" = "https://github.com/dbfixtures/mirakuru/blob/v3.0.1/CHANGES.rst" 39 | 40 | [build-system] 41 | requires = ["setuptools >= 77.0.3", "wheel"] 42 | build-backend = "setuptools.build_meta" 43 | 44 | [tool.setuptools] 45 | zip-safe = true 46 | 47 | [tool.setuptools.packages.find] 48 | include = ["mirakuru*"] 49 | exclude = ["tests*"] 50 | namespaces = false 51 | 52 | [tool.towncrier] 53 | directory = "newsfragments" 54 | single_file=true 55 | filename="CHANGES.rst" 56 | issue_format="`#{issue} `_" 57 | 58 | [[tool.towncrier.type]] 59 | directory = "break" 60 | name = "Breaking changes" 61 | showcontent = true 62 | 63 | [[tool.towncrier.type]] 64 | directory = "depr" 65 | name = "Deprecations" 66 | showcontent = true 67 | 68 | [[tool.towncrier.type]] 69 | directory = "feature" 70 | name = "Features" 71 | showcontent = true 72 | 73 | [[tool.towncrier.type]] 74 | directory = "bugfix" 75 | name = "Bugfixes" 76 | showcontent = true 77 | 78 | [[tool.towncrier.type]] 79 | directory = "misc" 80 | name = "Miscellaneus" 81 | showcontent = false 82 | 83 | [tool.pytest] 84 | filterwarnings = ["error"] 85 | strict_xfail = true 86 | 87 | [tool.ruff] 88 | line-length = 100 89 | target-version = 'py310' 90 | 91 | [tool.ruff.lint] 92 | select = [ 93 | "E", # pycodestyle 94 | "F", # pyflakes 95 | "I", # isort 96 | "D", # pydocstyle 97 | ] 98 | 99 | [tool.tbump] 100 | # Uncomment this if your project is hosted on GitHub: 101 | # github_url = "https://github.com///" 102 | 103 | [tool.tbump.version] 104 | current = "3.0.1" 105 | 106 | # Example of a semver regexp. 107 | # Make sure this matches current_version before 108 | # using tbump 109 | regex = ''' 110 | (?P\d+) 111 | \. 112 | (?P\d+) 113 | \. 114 | (?P\d+) 115 | (\- 116 | (?P.+) 117 | )? 118 | ''' 119 | 120 | [tool.tbump.git] 121 | message_template = "Release {new_version}" 122 | tag_template = "v{new_version}" 123 | 124 | [[tool.tbump.field]] 125 | # the name of the field 126 | name = "extra" 127 | # the default value to use, if there is no match 128 | default = "" 129 | 130 | 131 | # For each file to patch, add a [[file]] config 132 | # section containing the path of the file, relative to the 133 | # tbump.toml location. 134 | [[tool.tbump.file]] 135 | src = "mirakuru/__init__.py" 136 | 137 | [[tool.tbump.file]] 138 | src = "pyproject.toml" 139 | search = 'version = "{current_version}"' 140 | 141 | [[tool.tbump.file]] 142 | src = "pyproject.toml" 143 | search = '"Changelog" = "https://github.com/dbfixtures/mirakuru/blob/v{current_version}/CHANGES.rst"' 144 | 145 | # You can specify a list of commands to 146 | # run after the files have been patched 147 | # and before the git commit is made 148 | 149 | [[tool.tbump.before_commit]] 150 | name = "Build changelog" 151 | cmd = "pipenv run towncrier build --version {new_version} --yes" 152 | 153 | # Or run some commands after the git tag and the branch 154 | # have been pushed: 155 | # [[tool.tbump.after_push]] 156 | # name = "publish" 157 | # cmd = "./publish.sh" 158 | -------------------------------------------------------------------------------- /tests/server_for_tests.py: -------------------------------------------------------------------------------- 1 | """HTTP server that responses with delays used for tests. 2 | 3 | Example usage: 4 | 5 | python tests/slow_server.py [HOST:PORT] 6 | 7 | - run HTTP Server, HOST and PORT are optional 8 | 9 | python tests/slow_server.py [HOST:PORT] True 10 | 11 | - run IMMORTAL server (stopping process only by SIGKILL) 12 | 13 | """ 14 | 15 | import ast 16 | import os 17 | import sys 18 | import time 19 | from http.server import BaseHTTPRequestHandler, HTTPServer 20 | from urllib.parse import parse_qs 21 | 22 | sys.path.append(os.getcwd()) 23 | 24 | from tests.signals import block_signals # noqa: E402 25 | 26 | 27 | class SlowServerHandler(BaseHTTPRequestHandler): 28 | """Slow server handler.""" 29 | 30 | timeout = 2 31 | endtime = None 32 | 33 | def do_GET(self) -> None: # pylint:disable=invalid-name 34 | """Serve GET request.""" 35 | self.send_response(200) 36 | self.send_header("Content-type", "text/html") 37 | self.end_headers() 38 | self.wfile.write(b"Hi. I am very slow.") 39 | 40 | def do_HEAD(self) -> None: # pylint:disable=invalid-name 41 | """Serve HEAD request. 42 | 43 | but count to wait and return 500 response if wait time not exceeded 44 | due to the fact that HTTPServer will hang waiting for response 45 | to return otherwise if none response will be returned. 46 | """ 47 | self.timeout_status() 48 | self.end_headers() 49 | 50 | def timeout_status(self) -> None: 51 | """Set proper response status based on timeout.""" 52 | if self.count_timeout(): 53 | self.send_response(200) 54 | else: 55 | self.send_response(500) 56 | 57 | def count_timeout(self) -> bool: # pylint: disable=no-self-use 58 | """Count down the timeout time.""" 59 | if SlowServerHandler.endtime is None: 60 | SlowServerHandler.endtime = time.time() + SlowServerHandler.timeout 61 | return time.time() >= SlowServerHandler.endtime 62 | 63 | 64 | class SlowGetServerHandler(SlowServerHandler): 65 | """Responds only on GET after a while.""" 66 | 67 | def do_GET(self) -> None: # pylint:disable=invalid-name 68 | """Serve GET request.""" 69 | self.timeout_status() 70 | self.send_header("Content-type", "text/html") 71 | self.end_headers() 72 | self.wfile.write(b"Hi. I am very slow.") 73 | 74 | def do_HEAD(self) -> None: # pylint:disable=invalid-name 75 | """Serve HEAD request.""" 76 | self.send_response(500) 77 | self.end_headers() 78 | 79 | 80 | class SlowPostServerHandler(SlowServerHandler): 81 | """Responds only on POST after a while.""" 82 | 83 | def do_POST(self) -> None: # pylint:disable=invalid-name 84 | """Serve POST request.""" 85 | self.timeout_status() 86 | self.end_headers() 87 | self.wfile.write(b"Hi. I am very slow.") 88 | 89 | def do_HEAD(self) -> None: # pylint:disable=invalid-name 90 | """Serve HEAD request.""" 91 | self.send_response(500) 92 | self.end_headers() 93 | 94 | 95 | class SlowPostKeyServerHandler(SlowServerHandler): 96 | """Responds only on POST after a while.""" 97 | 98 | def do_POST(self) -> None: # pylint:disable=invalid-name 99 | """Serve POST request.""" 100 | content_len = int(self.headers["Content-Length"]) 101 | post_body = self.rfile.read(content_len) 102 | form = parse_qs(post_body) 103 | if form.get(b"key") == [b"hole"]: 104 | self.timeout_status() 105 | else: 106 | self.send_response(500) 107 | self.end_headers() 108 | self.wfile.write(b"Hi. I am very slow.") 109 | 110 | def do_HEAD(self) -> None: # pylint:disable=invalid-name 111 | """Serve HEAD request.""" 112 | self.send_response(500) 113 | self.end_headers() 114 | 115 | 116 | HANDLERS = { 117 | "HEAD": SlowServerHandler, 118 | "GET": SlowGetServerHandler, 119 | "POST": SlowPostServerHandler, 120 | "Key": SlowPostKeyServerHandler, 121 | } 122 | 123 | if __name__ == "__main__": 124 | HOST, PORT, IMMORTAL, METHOD = "127.0.0.1", "8000", "False", "HEAD" 125 | if len(sys.argv) >= 2: 126 | HOST, PORT = sys.argv[1].split(":") 127 | 128 | if len(sys.argv) >= 3: 129 | IMMORTAL = sys.argv[2] 130 | 131 | if len(sys.argv) == 4: 132 | METHOD = sys.argv[3] 133 | 134 | if ast.literal_eval(IMMORTAL): 135 | block_signals() 136 | 137 | server = HTTPServer((HOST, int(PORT)), HANDLERS[METHOD]) # pylint: disable=invalid-name 138 | print(f"Starting slow server on {HOST}:{PORT}...") 139 | server.serve_forever() 140 | -------------------------------------------------------------------------------- /mirakuru/http.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 by Clearcode 2 | # and associates (see AUTHORS). 3 | 4 | # This file is part of mirakuru. 5 | 6 | # mirakuru is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # mirakuru is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with mirakuru. If not, see . 18 | """HTTP enabled process executor.""" 19 | 20 | import re 21 | import socket 22 | from http.client import HTTPConnection, HTTPException 23 | from logging import getLogger 24 | from typing import Any 25 | from urllib.parse import urlencode, urlparse 26 | 27 | from mirakuru.tcp import TCPExecutor 28 | 29 | LOG = getLogger(__name__) 30 | 31 | 32 | class HTTPExecutor(TCPExecutor): 33 | """Http enabled process executor.""" 34 | 35 | DEFAULT_PORT = 80 36 | """Default TCP port for the HTTP protocol.""" 37 | 38 | def __init__( 39 | self, 40 | command: str | list[str] | tuple[str, ...], 41 | url: str, 42 | status: str | int = r"^2\d\d$", 43 | method: str = "HEAD", 44 | payload: dict[str, str] | None = None, 45 | headers: dict[str, str] | None = None, 46 | **kwargs: Any, 47 | ) -> None: 48 | """Initialize HTTPExecutor executor. 49 | 50 | :param (str, list) command: command to be run by the subprocess 51 | :param str url: URL that executor checks to verify 52 | if process has already started. 53 | :param bool shell: same as the `subprocess.Popen` shell definition 54 | :param str|int status: HTTP status code(s) that an endpoint must 55 | return for the executor being considered as running. This argument 56 | is interpreted as a single status code - e.g. '200' or '404' but 57 | also it can be a regular expression - e.g. '4..' or '(200|404)'. 58 | Default: any 2XX HTTP status code. 59 | :param str method: request method to check status on. 60 | Defaults to HEAD. 61 | :param dict payload: Payload to send along the request 62 | :param dict headers: 63 | :param int timeout: number of seconds to wait for the process to start 64 | or stop. If None or False, wait indefinitely. 65 | :param float sleep: how often to check for start/stop condition 66 | :param int sig_stop: signal used to stop process run by the executor. 67 | default is `signal.SIGTERM` 68 | :param int sig_kill: signal used to kill process run by the executor. 69 | default is `signal.SIGKILL` 70 | 71 | """ 72 | self.url = urlparse(url) 73 | """ 74 | An :func:`urlparse.urlparse` representation of an url. 75 | 76 | It'll be used to check process status on. 77 | """ 78 | 79 | if not self.url.hostname: 80 | raise ValueError("Url provided does not contain hostname") 81 | 82 | port = self.url.port 83 | if port is None: 84 | port = self.DEFAULT_PORT 85 | 86 | self.status = str(status) 87 | self.status_re = re.compile(str(status)) 88 | self.method = method 89 | self.payload = payload 90 | self.headers = headers 91 | 92 | super().__init__(command, host=self.url.hostname, port=port, **kwargs) 93 | 94 | def after_start_check(self) -> bool: 95 | """Check if defined URL returns expected status to a check request.""" 96 | conn = HTTPConnection(self.host, self.port) 97 | try: 98 | body = urlencode(self.payload) if self.payload else None 99 | headers = self.headers if self.headers else {} 100 | conn.request( 101 | self.method, 102 | self.url.path, 103 | body, 104 | headers, 105 | ) 106 | try: 107 | status = str(conn.getresponse().status) 108 | finally: 109 | conn.close() 110 | 111 | if status == self.status or self.status_re.match(status): 112 | return True 113 | return False 114 | 115 | except (HTTPException, socket.timeout, socket.error) as ex: 116 | LOG.debug("Encounter %s while trying to check if service has started.", ex) 117 | return False 118 | -------------------------------------------------------------------------------- /tests/executors/test_http_executor.py: -------------------------------------------------------------------------------- 1 | """HTTP Executor tests.""" 2 | 3 | import socket 4 | import sys 5 | from functools import partial 6 | from http.client import OK, HTTPConnection 7 | from typing import Any 8 | from unittest.mock import patch 9 | 10 | import pytest 11 | 12 | from mirakuru import AlreadyRunning, HTTPExecutor, TCPExecutor, TimeoutExpired 13 | from tests import HTTP_SERVER_CMD, TEST_SERVER_PATH 14 | 15 | HOST = "127.0.0.1" 16 | PORT = 7987 17 | 18 | HTTP_NORMAL_CMD = f"{HTTP_SERVER_CMD} {PORT}" 19 | HTTP_SLOW_CMD = f"{sys.executable} {TEST_SERVER_PATH} {HOST}:{PORT}" 20 | 21 | pytestmark = pytest.mark.xdist_group(name=f"http-port-{PORT}") 22 | 23 | 24 | slow_server_executor = partial( # pylint: disable=invalid-name 25 | HTTPExecutor, 26 | HTTP_SLOW_CMD, 27 | f"http://{HOST}:{PORT}/", 28 | ) 29 | 30 | 31 | def connect_to_server() -> None: 32 | """Connect to http server and assert 200 response.""" 33 | conn = HTTPConnection(HOST, PORT) 34 | conn.request("GET", "/") 35 | assert conn.getresponse().status == OK 36 | conn.close() 37 | 38 | 39 | def test_executor_starts_and_waits() -> None: 40 | """Test if process awaits for HEAD request to be completed.""" 41 | command = f'bash -c "sleep 3 && {HTTP_NORMAL_CMD}"' 42 | 43 | executor = HTTPExecutor(command, f"http://{HOST}:{PORT}/", timeout=20) 44 | executor.start() 45 | assert executor.running() is True 46 | 47 | connect_to_server() 48 | 49 | executor.stop() 50 | 51 | # check proper __str__ and __repr__ rendering: 52 | assert "HTTPExecutor" in repr(executor) 53 | assert command in str(executor) 54 | 55 | 56 | def test_shell_started_server_stops() -> None: 57 | """Test if executor terminates properly executor with shell=True.""" 58 | executor = HTTPExecutor(HTTP_NORMAL_CMD, f"http://{HOST}:{PORT}/", timeout=20, shell=True) 59 | 60 | with pytest.raises(socket.error): 61 | connect_to_server() 62 | 63 | with executor: 64 | assert executor.running() is True 65 | connect_to_server() 66 | 67 | assert executor.running() is False 68 | 69 | with pytest.raises(socket.error): 70 | connect_to_server() 71 | 72 | 73 | @pytest.mark.parametrize("method", ("HEAD", "GET", "POST")) 74 | def test_slow_method_server_starting(method: str) -> None: 75 | """Test whether or not executor awaits for slow starting servers. 76 | 77 | Simple example. You run Gunicorn and it is working but you have to 78 | wait for worker processes. 79 | """ 80 | http_method_slow_cmd = f"{sys.executable} {TEST_SERVER_PATH} {HOST}:{PORT} False {method}" 81 | with HTTPExecutor( 82 | http_method_slow_cmd, 83 | f"http://{HOST}:{PORT}/", 84 | method=method, 85 | timeout=30, 86 | ) as executor: 87 | assert executor.running() is True 88 | connect_to_server() 89 | 90 | 91 | def test_slow_post_payload_server_starting() -> None: 92 | """Test whether or not executor awaits for slow starting servers. 93 | 94 | Simple example. You run Gunicorn and it is working but you have to 95 | wait for worker processes. 96 | """ 97 | http_method_slow_cmd = f"{sys.executable} {TEST_SERVER_PATH} {HOST}:{PORT} False Key" 98 | with HTTPExecutor( 99 | http_method_slow_cmd, 100 | f"http://{HOST}:{PORT}/", 101 | method="POST", 102 | timeout=30, 103 | payload={"key": "hole"}, 104 | ) as executor: 105 | assert executor.running() is True 106 | connect_to_server() 107 | 108 | 109 | @pytest.mark.parametrize("method", ("HEAD", "GET", "POST")) 110 | def test_slow_method_server_timed_out(method: str) -> None: 111 | """Check if timeout properly expires.""" 112 | http_method_slow_cmd = f"{sys.executable} {TEST_SERVER_PATH} {HOST}:{PORT} False {method}" 113 | executor = HTTPExecutor( 114 | http_method_slow_cmd, f"http://{HOST}:{PORT}/", method=method, timeout=1 115 | ) 116 | 117 | with pytest.raises(TimeoutExpired) as exc: 118 | executor.start() 119 | 120 | assert executor.running() is False 121 | assert "timed out after" in str(exc.value) 122 | 123 | 124 | def test_fail_if_other_running() -> None: 125 | """Test raising AlreadyRunning exception when port is blocked.""" 126 | executor = HTTPExecutor( 127 | HTTP_NORMAL_CMD, 128 | f"http://{HOST}:{PORT}/", 129 | ) 130 | executor2 = HTTPExecutor( 131 | HTTP_NORMAL_CMD, 132 | f"http://{HOST}:{PORT}/", 133 | ) 134 | 135 | with executor: 136 | assert executor.running() is True 137 | 138 | with pytest.raises(AlreadyRunning): 139 | executor2.start() 140 | 141 | with pytest.raises(AlreadyRunning) as exc: 142 | with executor2: 143 | pass 144 | assert "seems to be already running" in str(exc.value) 145 | 146 | 147 | @patch.object(HTTPExecutor, "DEFAULT_PORT", PORT) 148 | def test_default_port() -> None: 149 | """Test default port for the base TCP check. 150 | 151 | Check if HTTP executor fills in the default port for the TCP check 152 | from the base class if no port is provided in the URL. 153 | """ 154 | executor = HTTPExecutor(HTTP_NORMAL_CMD, f"http://{HOST}/") 155 | 156 | assert executor.url.port is None 157 | assert executor.port == PORT 158 | 159 | assert TCPExecutor.pre_start_check(executor) is False 160 | executor.start() 161 | assert TCPExecutor.pre_start_check(executor) is True 162 | executor.stop() 163 | 164 | 165 | @pytest.mark.parametrize( 166 | "accepted_status, expected_timeout", 167 | ( 168 | # default behaviour - only 2XX HTTP status codes are accepted 169 | (None, True), 170 | # one explicit integer status code 171 | (200, True), 172 | # one explicit status code as a string 173 | ("404", False), 174 | # status codes as a regular expression 175 | (r"(2|4)\d\d", False), 176 | # status codes as a regular expression 177 | ("(200|404)", False), 178 | ), 179 | ) 180 | def test_http_status_codes(accepted_status: None | int | str, expected_timeout: bool) -> None: 181 | """Test how 'status' argument influences executor start. 182 | 183 | :param int|str accepted_status: Executor 'status' value 184 | :param bool expected_timeout: if Executor raises TimeoutExpired or not 185 | """ 186 | kwargs: dict[str, Any] = { 187 | "command": HTTP_NORMAL_CMD, 188 | "url": f"http://{HOST}:{PORT}/badpath", 189 | "timeout": 2, 190 | } 191 | if accepted_status: 192 | kwargs["status"] = accepted_status 193 | executor = HTTPExecutor(**kwargs) 194 | 195 | if not expected_timeout: 196 | executor.start() 197 | executor.stop() 198 | else: 199 | with pytest.raises(TimeoutExpired): 200 | executor.start() 201 | executor.stop() 202 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /mirakuru/output.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 by Clearcode 2 | # and associates (see AUTHORS). 3 | 4 | # This file is part of mirakuru. 5 | 6 | # mirakuru is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # mirakuru is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with mirakuru. If not, see . 18 | """Executor that awaits for appearance of a predefined banner in output.""" 19 | 20 | import platform 21 | import re 22 | import select 23 | from typing import IO, Any, TypeVar 24 | 25 | from mirakuru.base import SimpleExecutor 26 | 27 | IS_DARWIN = platform.system() == "Darwin" 28 | 29 | 30 | OutputExecutorType = TypeVar("OutputExecutorType", bound="OutputExecutor") 31 | 32 | 33 | class OutputExecutor(SimpleExecutor): 34 | """Executor that awaits for string output being present in output.""" 35 | 36 | def __init__( 37 | self, 38 | command: str | list[str] | tuple[str, ...], 39 | banner: str, 40 | **kwargs: Any, 41 | ) -> None: 42 | """Initialize OutputExecutor executor. 43 | 44 | :param (str, list) command: command to be run by the subprocess 45 | :param str banner: string that has to appear in process output - 46 | should compile to regular expression. 47 | :param bool shell: same as the `subprocess.Popen` shell definition 48 | :param int timeout: number of seconds to wait for the process to start 49 | or stop. If None or False, wait indefinitely. 50 | :param float sleep: how often to check for start/stop condition 51 | :param int sig_stop: signal used to stop process run by the executor. 52 | default is `signal.SIGTERM` 53 | :param int sig_kill: signal used to kill process run by the executor. 54 | default is `signal.SIGKILL` (`signal.SIGTERM` on Windows) 55 | 56 | """ 57 | super().__init__(command, **kwargs) 58 | self._banner = re.compile(banner) 59 | # Also keep a bytes-compiled regex to operate on raw peeked bytes. 60 | try: 61 | self._banner_bytes = re.compile(self._banner.pattern.encode("utf-8")) 62 | except Exception: 63 | # Fallback: a simple utf-8 encode of provided banner string 64 | self._banner_bytes = re.compile(str(banner).encode("utf-8")) 65 | if not any((self._stdout, self._stderr)): 66 | raise TypeError("At least one of stdout or stderr has to be initialized") 67 | 68 | def start(self: OutputExecutorType) -> OutputExecutorType: 69 | """Start the process. 70 | 71 | .. note:: 72 | 73 | Process will be considered started when a defined banner appears 74 | in the process output. 75 | """ 76 | super().start() 77 | 78 | if not IS_DARWIN: 79 | polls: list[tuple[select.poll, IO[Any]]] = [] 80 | for output_handle, output_method in ( 81 | (self._stdout, self.output), 82 | (self._stderr, self.err_output), 83 | ): 84 | if output_handle is not None: 85 | # get a polling object 86 | std_poll = select.poll() 87 | 88 | output_file = output_method() 89 | if output_file is None: 90 | raise ValueError("The process is started but the output file is None") 91 | # register a file descriptor 92 | # POLLIN because we will wait for data to read 93 | std_poll.register(output_file, select.POLLIN) 94 | polls.append((std_poll, output_file)) 95 | 96 | try: 97 | 98 | def await_for_output() -> bool: 99 | return self._wait_for_output(*polls) 100 | 101 | self.wait_for(await_for_output) 102 | 103 | for poll, output in polls: 104 | # unregister the file descriptor 105 | # and delete the polling object 106 | poll.unregister(output) 107 | finally: 108 | while len(polls) > 0: 109 | poll_and_output = polls.pop() 110 | del poll_and_output 111 | else: 112 | outputs = [] 113 | for output_handle, output_method in ( 114 | (self._stdout, self.output), 115 | (self._stderr, self.err_output), 116 | ): 117 | if output_handle is not None: 118 | outputs.append(output_method()) 119 | 120 | def await_for_output() -> bool: 121 | return self._wait_for_darwin_output(*outputs) 122 | 123 | self.wait_for(await_for_output) 124 | 125 | return self 126 | 127 | def _consume_until_banner_or_block(self, output: IO[Any]) -> tuple[bool, bool]: 128 | """Consume available data from a ready stream and check for banner. 129 | 130 | Returns a pair (found, should_break): 131 | - found: banner was detected and consumed up to end-of-line. 132 | - should_break: no more data immediately available for this descriptor, 133 | so the caller's inner draining loop should break. 134 | """ 135 | raw = getattr(output, "buffer", None) 136 | if raw is None: 137 | # Fallback to safe line reads on text wrappers 138 | line = output.readline() 139 | if not line: 140 | return False, True 141 | if self._banner.match(line): 142 | return True, True 143 | return False, False 144 | preview = raw.peek(65536) # 64KB 145 | if not preview: 146 | return False, True 147 | m = self._banner_bytes.search(preview) 148 | if m is None: 149 | to_read = min(len(preview), 8192) # 8KB 150 | _ = raw.read(to_read) 151 | return False, False 152 | nl_pos = preview.find(b"\n", m.end()) 153 | if nl_pos == -1: 154 | _ = raw.read(len(preview)) 155 | return False, False 156 | _ = raw.read(nl_pos + 1) 157 | return True, True 158 | 159 | def _wait_for_darwin_output(self, *fds: IO[Any] | None) -> bool: 160 | """Select an implementation to be used on macOS using select(). 161 | 162 | Drain all immediately available data in small chunks from ready 163 | descriptors and look for the banner using regex.search on a rolling 164 | buffer. This avoids blocking on TextIOWrapper.readline() with partial 165 | data and prevents pipe backpressure under heavy pre-banner output. 166 | """ 167 | # Filter out Nones defensively 168 | valid_fds = tuple(fd for fd in fds if fd is not None) 169 | if not valid_fds: 170 | return False 171 | 172 | found = False 173 | # Keep draining while there is data immediately available. 174 | while True: 175 | rlist, _, _ = select.select(valid_fds, [], [], 0) 176 | if not rlist: 177 | break 178 | for output in rlist: 179 | while True: 180 | rready, _, _ = select.select([output], [], [], 0) 181 | if not rready: 182 | break 183 | found, should_break = self._consume_until_banner_or_block(output) 184 | if found: 185 | return True 186 | if should_break: 187 | break 188 | # else continue draining 189 | return found 190 | 191 | def _wait_for_output(self, *polls: tuple["select.poll", IO[Any]]) -> bool: 192 | """Check if output matches banner. 193 | 194 | Drain as much data as available from ready descriptors in bursts using 195 | non-blocking chunked reads to avoid stalling on text line buffering. 196 | Returns True as soon as the banner is detected using regex.search(). 197 | 198 | .. warning:: 199 | Waiting for I/O completion. It does not work on Windows. Sorry. 200 | """ 201 | found = False 202 | any_ready = True 203 | # Keep draining while something is ready; exit when nothing is immediately ready. 204 | while any_ready: 205 | any_ready = False 206 | for p, output in polls: 207 | # Poll for readiness; when ready, drain in a controlled manner. 208 | while p.poll(0): 209 | any_ready = True 210 | found, should_break = self._consume_until_banner_or_block(output) 211 | if found: 212 | return True 213 | if should_break: 214 | break 215 | # else continue draining 216 | # loop continues if any_ready set 217 | return found 218 | -------------------------------------------------------------------------------- /tests/executors/test_executor.py: -------------------------------------------------------------------------------- 1 | # mypy: no-strict-optional 2 | """Test basic executor functionality.""" 3 | 4 | import gc 5 | import shlex 6 | import signal 7 | import uuid 8 | from subprocess import check_output 9 | from typing import List, Union 10 | from unittest import mock 11 | 12 | import pytest 13 | 14 | from mirakuru import Executor 15 | from mirakuru.base import SimpleExecutor 16 | from mirakuru.exceptions import ProcessExitedWithError, TimeoutExpired 17 | from tests import SAMPLE_DAEMON_PATH, ps_aux 18 | from tests.retry import retry 19 | 20 | SLEEP_300 = "sleep 300" 21 | 22 | 23 | @pytest.mark.parametrize("command", (SLEEP_300, SLEEP_300.split())) 24 | def test_running_process(command: Union[str, List[str]]) -> None: 25 | """Start process and shuts it down.""" 26 | executor = SimpleExecutor(command) 27 | executor.start() 28 | assert executor.running() is True 29 | executor.stop() 30 | assert executor.running() is False 31 | 32 | # check proper __str__ and __repr__ rendering: 33 | assert "SimpleExecutor" in repr(executor) 34 | assert SLEEP_300 in str(executor) 35 | 36 | 37 | @pytest.mark.parametrize("command", (SLEEP_300, SLEEP_300.split())) 38 | def test_command(command: Union[str, List[str]]) -> None: 39 | """Check that the command and command parts are equivalent.""" 40 | executor = SimpleExecutor(command) 41 | assert executor.command == SLEEP_300 42 | assert executor.command_parts == SLEEP_300.split() 43 | 44 | 45 | def test_custom_signal_stop() -> None: 46 | """Start process and shuts it down using signal SIGQUIT.""" 47 | executor = SimpleExecutor(SLEEP_300, stop_signal=signal.SIGQUIT) 48 | executor.start() 49 | assert executor.running() is True 50 | executor.stop() 51 | assert executor.running() is False 52 | 53 | 54 | def test_stop_custom_signal_stop() -> None: 55 | """Start process and shuts it down using signal SIGQUIT passed to stop.""" 56 | executor = SimpleExecutor(SLEEP_300) 57 | executor.start() 58 | assert executor.running() is True 59 | executor.stop(stop_signal=signal.SIGQUIT) 60 | assert executor.running() is False 61 | 62 | 63 | def test_stop_custom_exit_signal_stop() -> None: 64 | """Start process and expect it to finish with custom signal.""" 65 | executor = SimpleExecutor("false", shell=True) 66 | executor.start() 67 | # false exits instant, so there should not be a process to stop 68 | retry(lambda: executor.stop(stop_signal=signal.SIGQUIT, expected_returncode=-3)) 69 | assert executor.running() is False 70 | 71 | 72 | @pytest.mark.flaky(reruns=5, reruns_delay=1, only_rerun="ProcessFinishedWithError") 73 | def test_stop_custom_exit_signal_context() -> None: 74 | """Start a process and expect a custom exit signal in the context manager.""" 75 | with SimpleExecutor("false", expected_returncode=-3, shell=True) as executor: 76 | executor.stop(stop_signal=signal.SIGQUIT) 77 | assert executor.running() is False 78 | 79 | 80 | def test_running_context() -> None: 81 | """Start process and shuts it down.""" 82 | executor = SimpleExecutor(SLEEP_300) 83 | with executor: 84 | assert executor.running() is True 85 | 86 | assert executor.running() is False 87 | 88 | 89 | def test_executor_in_context_only() -> None: 90 | """Start a process and shuts it down only in context.""" 91 | with SimpleExecutor(SLEEP_300) as executor: 92 | assert executor.running() is True 93 | 94 | 95 | def test_context_stopped() -> None: 96 | """Start for context and shuts it for nested context.""" 97 | executor = SimpleExecutor(SLEEP_300) 98 | with executor: 99 | assert executor.running() is True 100 | with executor.stopped(): 101 | assert executor.running() is False 102 | assert executor.running() is True 103 | 104 | assert executor.running() is False 105 | 106 | 107 | ECHO_FOOBAR = 'echo "foobar"' 108 | 109 | 110 | @pytest.mark.parametrize("command", (ECHO_FOOBAR, shlex.split(ECHO_FOOBAR))) 111 | def test_process_output(command: Union[str, List[str]]) -> None: 112 | """Start a process, check output and shut it down.""" 113 | executor = SimpleExecutor(command) 114 | executor.start() 115 | 116 | assert executor.output().read() == "foobar\n" 117 | executor.stop() 118 | 119 | 120 | @pytest.mark.parametrize("command", (ECHO_FOOBAR, shlex.split(ECHO_FOOBAR))) 121 | def test_process_output_shell(command: Union[str, List[str]]) -> None: 122 | """Start process, check output and shut it down with shell set to True.""" 123 | executor = SimpleExecutor(command, shell=True) 124 | executor.start() 125 | 126 | assert executor.output().read().strip() == "foobar" 127 | executor.stop() 128 | 129 | 130 | def test_start_check_executor() -> None: 131 | """Validate Executor base class having NotImplemented methods.""" 132 | executor = Executor(SLEEP_300) 133 | with pytest.raises(NotImplementedError): 134 | executor.pre_start_check() 135 | with pytest.raises(NotImplementedError): 136 | executor.after_start_check() 137 | 138 | 139 | def test_stopping_not_yet_running_executor() -> None: 140 | """Test if SimpleExecutor can be stopped even it was never running. 141 | 142 | We must make sure that it's possible to call .stop() and SimpleExecutor 143 | will not raise any exception and .start() can be called afterwards. 144 | """ 145 | executor = SimpleExecutor(SLEEP_300) 146 | executor.stop() 147 | executor.start() 148 | assert executor.running() is True 149 | executor.stop() 150 | 151 | 152 | def test_forgotten_stop() -> None: 153 | """Test if SimpleExecutor subprocess is killed after an instance is deleted. 154 | 155 | Existence can end because of context scope end or by calling 'del'. 156 | If someone forgot to stop() or kill() subprocess it should be killed 157 | by default on instance cleanup. 158 | """ 159 | mark = uuid.uuid1().hex 160 | # We cannot simply do `sleep 300 #` in a shell because in that 161 | # case bash (default shell on some systems) does `execve` without cloning 162 | # itself - that means there will be no process with commandline like: 163 | # '/bin/sh -c sleep 300 && true #' - instead that process would 164 | # get substituted with 'sleep 300' and the marked commandline would be 165 | # overwritten. 166 | # Injecting some flow control (`&&`) forces bash to fork properly. 167 | marked_command = f"sleep 300 && true #{mark}" 168 | executor = SimpleExecutor(marked_command, shell=True) 169 | executor.start() 170 | assert executor.running() is True 171 | ps_output = ps_aux() 172 | assert mark in ps_output, ( 173 | f"The test command {marked_command} should be running in \n\n {ps_output}." 174 | ) 175 | del executor 176 | gc.collect() # to force 'del' immediate effect 177 | assert mark not in ps_aux(), "The test process should not be running at this point." 178 | 179 | 180 | def test_executor_raises_if_process_exits_with_error() -> None: 181 | """Test process exit detection. 182 | 183 | If the process exits with an error while checks are being polled, executor 184 | should raise an exception. 185 | """ 186 | error_code = 12 187 | failing_executor = Executor(["bash", "-c", f"exit {error_code!s}"], timeout=5) 188 | failing_executor.pre_start_check = mock.Mock(return_value=False) # type: ignore 189 | # After-start check will keep returning False to let the process terminate. 190 | failing_executor.after_start_check = mock.Mock(return_value=False) # type: ignore 191 | 192 | with pytest.raises(ProcessExitedWithError) as exc: 193 | failing_executor.start() 194 | 195 | assert exc.value.exit_code == 12 196 | error_msg = f"exited with a non-zero code: {error_code!s}" 197 | assert error_msg in str(exc.value) 198 | 199 | # Pre-start check should have been called - after-start check might or 200 | # might not have been called - depending on the timing. 201 | assert failing_executor.pre_start_check.called is True 202 | 203 | 204 | def test_executor_ignores_processes_exiting_with_0() -> None: 205 | """Test process exit detection. 206 | 207 | Subprocess exiting with zero should be tolerated in order to support 208 | double-forking applications. 209 | """ 210 | # We execute a process that will return zero. In order to give the process 211 | # enough time to return we keep the polling loop spinning for a second. 212 | executor = Executor(["bash", "-c", "exit 0"], timeout=1.0) 213 | executor.pre_start_check = mock.Mock(return_value=False) # type: ignore 214 | executor.after_start_check = mock.Mock(return_value=False) # type: ignore 215 | 216 | with pytest.raises(TimeoutExpired): 217 | # We keep the post-checks spinning forever so it eventually times out. 218 | executor.start() 219 | 220 | # Both checks should have been called. 221 | assert executor.pre_start_check.called is True 222 | assert executor.after_start_check.called is True 223 | 224 | 225 | def test_executor_methods_returning_self() -> None: 226 | """Test if SimpleExecutor lets to chain start, stop and kill methods.""" 227 | executor = SimpleExecutor(SLEEP_300).start().stop().kill().stop() 228 | assert not executor.running() 229 | 230 | # Check if context manager returns executor to use it in 'as' phrase: 231 | with SimpleExecutor(SLEEP_300) as executor: 232 | assert executor.running() 233 | 234 | with SimpleExecutor(SLEEP_300).start().stopped() as executor: 235 | assert not executor.running() 236 | 237 | assert SimpleExecutor(SLEEP_300).start().stop().output 238 | 239 | 240 | @pytest.mark.xdist_group(name="sample-deamon") 241 | def test_mirakuru_cleanup() -> None: 242 | """Test if cleanup_subprocesses is fired correctly on python exit.""" 243 | cmd = f""" 244 | python -c 'from mirakuru import SimpleExecutor; 245 | from time import sleep; 246 | import gc; 247 | gc.disable(); 248 | ex = SimpleExecutor( 249 | ("python", "{SAMPLE_DAEMON_PATH}")).start(); 250 | sleep(1); 251 | ' 252 | """ 253 | check_output(shlex.split(cmd.replace("\n", ""))) 254 | assert SAMPLE_DAEMON_PATH not in ps_aux() 255 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: https://raw.githubusercontent.com/dbfixtures/mirakuru/master/logo.png 2 | :height: 100px 3 | 4 | mirakuru 5 | ======== 6 | 7 | Mirakuru is a process orchestration tool designed for functional and integration tests. 8 | 9 | When your application or tests rely on external processes (like databases, APIs, or other services), 10 | ensuring these processes are started and ready *before* your main code executes can be challenging. 11 | **Mirakuru** solves this by orchestrating the startup of these processes and waiting until they 12 | are fully operational (e.g., accepting connections, producing specific output) before allowing 13 | your program or tests to continue. 14 | 15 | 16 | .. image:: https://img.shields.io/pypi/v/mirakuru.svg 17 | :target: https://pypi.python.org/pypi/mirakuru/ 18 | :alt: Latest PyPI version 19 | 20 | .. image:: https://img.shields.io/pypi/wheel/mirakuru.svg 21 | :target: https://pypi.python.org/pypi/mirakuru/ 22 | :alt: Wheel Status 23 | 24 | .. image:: https://img.shields.io/pypi/pyversions/mirakuru.svg 25 | :target: https://pypi.python.org/pypi/mirakuru/ 26 | :alt: Supported Python Versions 27 | 28 | .. image:: https://img.shields.io/pypi/l/mirakuru.svg 29 | :target: https://pypi.python.org/pypi/mirakuru/ 30 | :alt: License 31 | 32 | Installation 33 | ------------ 34 | 35 | Install mirakuru using pip: 36 | 37 | .. code-block:: bash 38 | 39 | pip install mirakuru 40 | 41 | Quick Start 42 | ----------- 43 | 44 | Here's a simple example showing how mirakuru ensures a Redis server is ready before your code runs: 45 | 46 | .. code-block:: python 47 | 48 | from mirakuru import TCPExecutor 49 | 50 | # Start Redis server and wait until it accepts connections on port 6379 51 | redis_executor = TCPExecutor('redis-server', host='localhost', port=6379) 52 | redis_executor.start() 53 | 54 | # Redis is now running and ready to accept connections 55 | # ... your code that uses Redis here ... 56 | 57 | # Clean up - stop the Redis server 58 | redis_executor.stop() 59 | 60 | The key benefit: ``start()`` blocks until Redis is actually ready, so you never try to connect too early. 61 | 62 | 63 | 64 | Usage 65 | ----- 66 | 67 | In projects that rely on multiple processes, there might be a need to guard code 68 | with tests that verify interprocess communication. You need to set up all the 69 | required databases, auxiliary and application services to verify their cooperation. 70 | Synchronizing (or orchestrating) test procedures with tested processes can be challenging. 71 | 72 | If so, then **mirakuru** is what you need. 73 | 74 | ``Mirakuru`` starts your process and waits for a clear indication that it's running. 75 | The library provides seven executors to fit different cases: 76 | 77 | .. list-table:: 78 | :header-rows: 1 79 | :widths: 25 75 80 | 81 | * - Executor 82 | - Use When 83 | * - **SimpleExecutor** 84 | - You just need to start/stop a process without waiting for readiness. Base class for all other executors. 85 | * - **Executor** 86 | - Base class for executors that verify process startup. 87 | * - **OutputExecutor** 88 | - Your process prints a specific message when ready (e.g., "Server started on port 8080") 89 | * - **TCPExecutor** 90 | - Your process opens a TCP port when ready (e.g., Redis, PostgreSQL, Memcached) 91 | * - **UnixSocketExecutor** 92 | - Your process opens a Unix socket when ready (e.g., Docker daemon, some databases) 93 | * - **HTTPExecutor** 94 | - Your process serves HTTP requests when ready (e.g., web servers, REST APIs) 95 | * - **PidExecutor** 96 | - Your process creates a .pid file when ready (e.g., traditional Unix daemons) 97 | 98 | SimpleExecutor 99 | ++++++++++++++ 100 | 101 | The simplest executor implementation. 102 | It simply starts the process passed to constructor, and reports it as running. 103 | 104 | .. code-block:: python 105 | 106 | from mirakuru import SimpleExecutor 107 | 108 | process = SimpleExecutor('my_special_process') 109 | process.start() 110 | 111 | # Here you can do your stuff, e.g. communicate with the started process 112 | 113 | process.stop() 114 | 115 | OutputExecutor 116 | ++++++++++++++ 117 | 118 | OutputExecutor starts a process and monitors its output for a specific text marker 119 | (banner). The process is not reported as started until this marker appears in the output. 120 | 121 | .. code-block:: python 122 | 123 | from mirakuru import OutputExecutor 124 | 125 | process = OutputExecutor('my_special_process', banner='processed!') 126 | process.start() 127 | 128 | # Here you can do your stuff, e.g. communicate with the started process 129 | 130 | process.stop() 131 | 132 | What happens during start here, is that the executor constantly checks output 133 | produced by started process, and looks for the banner part occurring within the 134 | output. 135 | Once the output is identified, as in example `processed!` is found in output. 136 | It is considered as started, and executor releases your script from wait to work. 137 | 138 | 139 | TCPExecutor 140 | +++++++++++ 141 | 142 | TCPExecutor should be used to start processes that communicate over TCP connections. 143 | This executor tries to connect to the process on the specified host and port to check 144 | if it started accepting connections. Once it successfully connects, the process is 145 | reported as started and control returns to your code. 146 | 147 | .. code-block:: python 148 | 149 | from mirakuru import TCPExecutor 150 | 151 | process = TCPExecutor('my_special_process', host='localhost', port=1234) 152 | process.start() 153 | 154 | # Here you can do your stuff, e.g. communicate with the started process 155 | 156 | process.stop() 157 | 158 | HTTPExecutor 159 | ++++++++++++ 160 | 161 | HTTPExecutor is designed for starting web applications and HTTP services. 162 | In addition to the command, you need to pass a URL that will be used to check 163 | if the service is ready. By default, it makes a HEAD request to this URL. 164 | Once the request succeeds, the executor reports the process as started and 165 | control returns to your code. 166 | 167 | .. code-block:: python 168 | 169 | from mirakuru import HTTPExecutor 170 | 171 | process = HTTPExecutor('my_special_process', url='http://localhost:6543/status') 172 | process.start() 173 | 174 | # Here you can do your stuff, e.g. communicate with the started process 175 | 176 | process.stop() 177 | 178 | This executor, however, apart from HEAD request, also inherits TCPExecutor, 179 | so it'll try to connect to process over TCP first, to determine, 180 | if it can try to make a HEAD request already. 181 | 182 | By default HTTPExecutor waits until its subprocess responds with 2XX HTTP status code. 183 | If you consider other codes as valid you need to specify them in 'status' argument. 184 | 185 | .. code-block:: python 186 | 187 | from mirakuru import HTTPExecutor 188 | 189 | process = HTTPExecutor('my_special_process', url='http://localhost:6543/status', status='(200|404)') 190 | process.start() 191 | 192 | The "status" argument can be a single code integer like 200, 404, 500 or a regular expression string - 193 | '^(2|4)00$', '2\d\d', '\d{3}', etc. 194 | 195 | There's also a possibility to change the request method used to perform request to the server. 196 | By default it's HEAD, but GET, POST or other are also possible. 197 | 198 | .. code-block:: python 199 | 200 | from mirakuru import HTTPExecutor 201 | 202 | process = HTTPExecutor('my_special_process', url='http://localhost:6543/status', status='(200|404)', method='GET') 203 | process.start() 204 | 205 | 206 | PidExecutor 207 | +++++++++++ 208 | 209 | Is an executor that starts the given 210 | process, and then waits for a given file to be found before it gives back control. 211 | An example use for this class is writing integration tests for processes that 212 | notify their running by creating a .pid file. 213 | 214 | .. code-block:: python 215 | 216 | from mirakuru import PidExecutor 217 | 218 | process = PidExecutor('my_special_process', filename='/var/msp/my_special_process.pid') 219 | process.start() 220 | 221 | # Here you can do your stuff, e.g. communicate with the started process 222 | 223 | process.stop() 224 | 225 | 226 | .. code-block:: python 227 | 228 | from mirakuru import HTTPExecutor 229 | from http.client import HTTPConnection, OK 230 | 231 | 232 | def test_it_works(): 233 | # The ``./http_server`` here launches some HTTP server on the 6543 port, 234 | # but naturally it is not immediate and takes a non-deterministic time: 235 | executor = HTTPExecutor("./http_server", url="http://127.0.0.1:6543/") 236 | 237 | # Start the server and wait for it to run (blocking): 238 | executor.start() 239 | # Here the server should be running! 240 | conn = HTTPConnection("127.0.0.1", 6543) 241 | conn.request("GET", "/") 242 | assert conn.getresponse().status is OK 243 | executor.stop() 244 | 245 | 246 | A command by which executor spawns a process can be defined by either string or list. 247 | 248 | .. code-block:: python 249 | 250 | # command as string 251 | TCPExecutor('python -m smtpd -n -c DebuggingServer localhost:1025', host='localhost', port=1025) 252 | # command as list 253 | TCPExecutor( 254 | ['python', '-m', 'smtpd', '-n', '-c', 'DebuggingServer', 'localhost:1025'], 255 | host='localhost', port=1025 256 | ) 257 | 258 | Use as a Context manager 259 | ------------------------ 260 | 261 | Starting 262 | ++++++++ 263 | 264 | Mirakuru executors can also work as a context managers. 265 | 266 | .. code-block:: python 267 | 268 | from mirakuru import HTTPExecutor 269 | 270 | with HTTPExecutor('my_special_process', url='http://localhost:6543/status') as process: 271 | 272 | # Here you can do your stuff, e.g. communicate with the started process 273 | assert process.running() is True 274 | 275 | assert process.running() is False 276 | 277 | Defined process starts upon entering context, and exit upon exiting it. 278 | 279 | Stopping 280 | ++++++++ 281 | 282 | Mirakuru also allows to stop process for given context. 283 | To do this, simply use built-in stopped context manager. 284 | 285 | .. code-block:: python 286 | 287 | from mirakuru import HTTPExecutor 288 | 289 | process = HTTPExecutor('my_special_process', url='http://localhost:6543/status').start() 290 | 291 | # Here you can do your stuff, e.g. communicate with the started process 292 | 293 | with process.stopped(): 294 | 295 | # Here you will not be able to communicate with the process as it is killed here 296 | assert process.running() is False 297 | 298 | assert process.running() is True 299 | 300 | Defined process stops upon entering context, and starts upon exiting it. 301 | 302 | 303 | Methods chaining 304 | ++++++++++++++++ 305 | 306 | Mirakuru encourages methods chaining so you can inline some operations, e.g.: 307 | 308 | .. code-block:: python 309 | 310 | from mirakuru import SimpleExecutor 311 | 312 | command_stdout = SimpleExecutor('my_special_process').start().stop().output 313 | 314 | Contributing and reporting bugs 315 | ------------------------------- 316 | 317 | Source code is available at: `dbfixtures/mirakuru `_. 318 | Issue tracker is located at `GitHub Issues `_. 319 | Projects `PyPI page `_. 320 | 321 | Windows support 322 | --------------- 323 | 324 | Frankly, there's none, Python's support differs a bit in required places 325 | and the team has no experience in developing for Windows. 326 | However we'd welcome contributions that will allow the windows support. 327 | 328 | See: 329 | 330 | * `#392 `_ 331 | * `#336 `_ 332 | 333 | Also, with the introduction of `WSL `_ 334 | the need for raw Windows support might not be that urgent... If you've got any thoughts or are willing to contribute, 335 | please start with the issues listed above. 336 | -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | CHANGELOG 2 | ========= 3 | 4 | .. towncrier release notes start 5 | 6 | mirakuru 3.0.1 (2025-11-01) 7 | =========================== 8 | 9 | Bugfixes 10 | -------- 11 | 12 | - Correctly point the minimum required python version. 13 | 14 | Not updating it causes installs without pinned mirakuru versions to fail. 15 | 16 | 17 | mirakuru 3.0.0 (2025-10-29) 18 | =========================== 19 | 20 | Breaking changes 21 | ---------------- 22 | 23 | - Drop support for Python 3.9 24 | 25 | 26 | Features 27 | -------- 28 | 29 | - Executors now accept additional kwargs that can be passed to popen. (`#905 `_) 30 | - Add support for Python 3.14 31 | 32 | 33 | Bugfixes 34 | -------- 35 | 36 | - Prevent ``OutputExecutor`` from hanging when a child process produces 37 | a large amount of output before the banner. 38 | This was resolved by implementing non-blocking reads and fragmented output consumption 39 | to avoid pipe backpressure and ensure timely banner detection. (`#98 `_) 40 | 41 | 42 | Miscellaneus 43 | ------------ 44 | 45 | - Downgrade MacOS runner to 14, as the 15 running as macos-latest at the moment is misbehaving. 46 | 47 | Tests on PyPy are running fine, while those on regular python vversion constantly fail. 48 | - Improve README 49 | - Replace black with ruff-format 50 | - Speed up tests with pytest-xdist 51 | - Update pipelines for fizyk/actions-reuse 4 52 | 53 | 54 | 2.6.1 (2025-07-02) 55 | ================== 56 | 57 | Miscellaneus 58 | ------------ 59 | 60 | - Adjust links after repository transfer 61 | - Adjust workflows for actions-reuse 3.0.0 62 | - Fix rerunfailures configuration 63 | - Run tests against oldest supported dependencies. 64 | - Standardized license field in `pyproject.toml` to use SPDX identifier and avoid deprecation warnings from setuptools. 65 | - Tests: pick free port instead of hardcoding to 3000 66 | 67 | 68 | 2.6.0 (2025-02-07) 69 | ================== 70 | 71 | Features 72 | -------- 73 | 74 | - SimpleExecutor now have an envvars property, that returns defined envvars with added os.envvars and mirakuru_uuid envvar. 75 | 76 | This will allow to re-use same envvars for starting process and any additional process runs in inheriting executors. (`#842 `_) 77 | 78 | 79 | Miscellaneus 80 | ------------ 81 | 82 | - `#810 `_, `#841 `_ 83 | - Add pytest-rerunfailures for flaky tests. 84 | - Defer to pytest's tmp_path_factory instead of manual tmpdir setting/detection 85 | 86 | 87 | 2.5.3 (2024-10-11) 88 | ================== 89 | 90 | Breaking changes 91 | ---------------- 92 | 93 | - Dropped support for Python 3.8 (As it reached EOL) 94 | 95 | 96 | Features 97 | -------- 98 | 99 | - Added support for Python 3.13 100 | 101 | 102 | Miscellaneus 103 | ------------ 104 | 105 | - `#724 `_, `#726 `_, `#742 `_ 106 | - * Extended line-lenght to 100 characters 107 | * updated test_forgotten_stop as on CI on 108 | Python 3.13 it lost one character out of the marker 109 | 110 | 111 | 2.5.2 (2023-10-17) 112 | ================== 113 | 114 | Breaking changes 115 | ---------------- 116 | 117 | - Drop support for Python 3.7 (`#667 `_) 118 | 119 | 120 | Features 121 | -------- 122 | 123 | - Support Python 3.12 (`#685 `_) 124 | 125 | 126 | Miscellaneus 127 | ------------ 128 | 129 | - `#639 `_, `#653 `_, `#655 `_, `#664 `_, `#686 `_ 130 | 131 | 132 | 2.5.1 (2023-02-27) 133 | ================== 134 | 135 | Bugfixes 136 | -------- 137 | 138 | - Include py.typed to the package published on pypi (`#633 `_) 139 | 140 | 141 | 2.5.0 (2023-02-17) 142 | ================== 143 | 144 | Features 145 | -------- 146 | 147 | - Support Python 3.11 (`#617 `_) 148 | 149 | 150 | Miscellaneus 151 | ------------ 152 | 153 | - Reformatted code with black 23 (`#613 `_) 154 | - Introduce towncrier as changelog management too. (`#615 `_) 155 | - Moved Development dependency management to pipfile/pipenv (`#616 `_) 156 | - Move package definition into the pyproject.toml file (`#618 `_) 157 | - Use shared automerge flow and github app. (`#619 `_) 158 | - Use tbump to manage versioning (`#628 `_) 159 | 160 | 161 | 2.4.2 162 | ===== 163 | 164 | Misc 165 | ---- 166 | 167 | + Added Python 3.10 to classifiers 168 | 169 | 2.4.1 170 | ===== 171 | 172 | Misc 173 | ---- 174 | 175 | - Use strictier mypy checks 176 | 177 | 2.4.0 178 | ===== 179 | 180 | Features 181 | -------- 182 | 183 | - Replace `exp_sig` executor parameter with `expected_returncode`. 184 | Parameter description already assumed that, however handing it assumed full 185 | POSIX compatibility on the process side. Now the POSIX is only assumed if no 186 | `expected_returncode` is passed to the executor, and returncode is simply that, 187 | a returncode, nothing more 188 | 189 | 2.3.1 190 | ===== 191 | 192 | Misc 193 | ---- 194 | 195 | - Moved CI to Github Actions 196 | - Blackified codebase 197 | - Compacted Documentation into readme (was pretty small anyway) 198 | 199 | 2.3.0 200 | ===== 201 | 202 | - [enhancement] Ability to set up expected exit code for executor. In Java exit codes 1- 127 have 203 | special meaning, and the regular exit codes are offset by those of special meaning. 204 | 205 | 2.2.0 206 | ===== 207 | 208 | - [enhancement] If process is being closed and the shutdown won't be clean (won't return exit code 0) 209 | mirakuru will now rise ProcessFinishedWithError exception with exit_code 210 | 211 | 2.1.2 212 | ===== 213 | 214 | - [bugfix][macos] Fixed typing issue on macOS 215 | 216 | 2.1.1 217 | ===== 218 | 219 | - [bug] Always close connection for HTTPExecutor after_start_check 220 | - [enhancement] Log debug message if execption occured during 221 | HTTPExecutor start check 222 | - [ehnancement] adjust typing handling in HTTPExecutor 223 | 224 | 2.1.0 225 | ===== 226 | 227 | - [feature] Drop support for python 3.5. Rely on typing syntax and fstrings that 228 | is available since python 3.6 only 229 | - [ehnancement] For output executor on MacOs fallback to `select.select` for OutputExecutor. 230 | Increases compatibility with MacOS where presence of `select.poll` depends 231 | on the compiler used. 232 | - [enhancement] Apply shelx.quote on command parts if command is given as a list 233 | Should result in similar results when running such command with or without shell. 234 | 235 | 2.0.1 236 | ===== 237 | 238 | - [repackage] - mark python 3.5 as required. Should disallow installing on python 2 239 | 240 | 2.0.0 241 | ===== 242 | 243 | - [feature] Add UnixSocketExecutor for executors that communicate with Unix Sockets 244 | - [feature] Mirakuru is now fully type hinted 245 | - [feature] Drop support for python 2 246 | - [feature] Allow for configuring process outputs to pipe to 247 | - [feature] OutputExecutor can now check for banner in stderr 248 | - [feature] HTTPEecutor now can check status on different method. 249 | Along with properly configured payload and headers. 250 | - [feature] Ability to set custom env vars for orchestrated process 251 | - [feature] Ability to set custom cwd path for orchestrated process 252 | - [enhancement] psutil is no longer required on cygwin 253 | 254 | 1.1.0 255 | ===== 256 | 257 | - [enhancement] Executor's timeout to be set for both executor's start and stop 258 | - [enhancement] It's no longer possible to hang indefinitely on the start 259 | or stop. Timeout is set to 3600 seconds by default, with values possible 260 | between `0` and `sys.maxsize` with the latter still bit longer 261 | than `2924712086` centuries. 262 | 263 | 1.0.0 264 | ===== 265 | 266 | - [enhancement] Do not fail if processes child throw EPERM error 267 | during clean up phase 268 | - [enhancement] Run subprocesses in shell by default on Windows 269 | - [ehnancement] Do not pass preexec_fn on windows 270 | 271 | 0.9.0 272 | ===== 273 | 274 | - [enhancement] Fallback to kill through SIGTERM on Windows, 275 | since SIGKILL is not available 276 | - [enhancement] detect cases where during stop process already exited, 277 | and simply clean up afterwards 278 | 279 | 0.8.3 280 | ===== 281 | 282 | - [enhancement] when killing the process ignore OsError with errno `no such process` as the process have already died. 283 | - [enhancement] small context manager code cleanup 284 | 285 | 286 | 0.8.2 287 | ===== 288 | 289 | - [bugfix] atexit cleanup_subprocesses() function now reimports needed functions 290 | 291 | 292 | 0.8.1 293 | ===== 294 | 295 | - [bugfix] Handle IOErrors from psutil (#112) 296 | - [bugfix] Pass global vars to atexit cleanup_subprocesses function (#111) 297 | 298 | 299 | 0.8.0 300 | ===== 301 | 302 | - [feature] Kill all running mirakuru subprocesses on python exit. 303 | - [enhancement] Prefer psutil library (>=4.0.0) over calling 'ps xe' command to find leaked subprocesses. 304 | 305 | 306 | 0.7.0 307 | ===== 308 | 309 | - [feature] HTTPExecutor enriched with the 'status' argument. 310 | It allows to define which HTTP status code(s) signify that a HTTP server is running. 311 | - [feature] Changed executor methods to return itself to allow method chaining. 312 | - [feature] Context Manager to return Executor instance, allows creating Executor instance on the fly. 313 | - [style] Migrated `%` string formating to `format()`. 314 | - [style] Explicitly numbered replacement fields in string. 315 | - [docs] Added documentation for timeouts. 316 | 317 | 0.6.1 318 | ===== 319 | 320 | - [refactoring] Moved source to src directory. 321 | - [fix, feature] Python 3.5 fixes. 322 | - [fix] Docstring changes for updated pep257. 323 | 324 | 0.6.0 325 | ===== 326 | 327 | - [fix] Modify MANIFEST to prune tests folder. 328 | - [feature] HTTPExecutor will now set the default 80 if not present in a URL. 329 | - [feature] Detect subprocesses exiting erroneously while polling the checks and error early. 330 | - [fix] Make test_forgotten_stop pass by preventing the shell from optimizing forking out. 331 | 332 | 0.5.0 333 | ===== 334 | 335 | - [style] Corrected code to conform with W503, D210 and E402 linters errors as reported by pylama `6.3.1`. 336 | - [feature] Introduced a hack that kills all subprocesses of executor process. 337 | It requires 'ps xe -ww' command being available in OS otherwise logs error. 338 | - [refactoring] Classes name convention change. 339 | Executor class got renamed into SimpleExecutor and StartCheckExecutor class got renamed into Executor. 340 | 341 | 0.4.0 342 | ===== 343 | 344 | - [feature] Ability to set up custom signal for stopping and killing processes managed by executors. 345 | - [feature] Replaced explicit parameters with keywords for kwargs handled by basic Executor init method. 346 | - [feature] Executor now accepts both list and string as a command. 347 | - [fix] Even it's not recommended to import all but `from mirakuru import *` didn't worked. Now it's fixed. 348 | - [tests] increased tests coverage. 349 | Even test cover 100% of code it doesn't mean they cover 100% of use cases! 350 | - [code quality] Increased Pylint code evaluation. 351 | 352 | 0.3.0 353 | ===== 354 | 355 | - [feature] Introduced PidExecutor that waits for specified file to be created. 356 | - [feature] Provided PyPy compatibility. 357 | - [fix] Closing all resources explicitly. 358 | 359 | 0.2.0 360 | ===== 361 | 362 | - [fix] Kill all children processes of Executor started with shell=True. 363 | - [feature] Executors are now context managers - to start executors for given context. 364 | - [feature] Executor.stopped - context manager for stopping executors for given context. 365 | - [feature] HTTPExecutor and TCPExecutor before .start() check whether port 366 | is already used by other processes and raise AlreadyRunning if detects it. 367 | - [refactoring] Moved python version conditional imports into compat.py module. 368 | 369 | 370 | 0.1.4 371 | ===== 372 | 373 | - [fix] Fixed an issue where setting shell to True would execute only part of the command. 374 | 375 | 0.1.3 376 | ===== 377 | 378 | - [fix] Fixed an issue where OutputExecutor would hang, if started process stopped producing output. 379 | 380 | 0.1.2 381 | ===== 382 | 383 | - [fix] Removed leftover sleep from TCPExecutor._wait_for_connection. 384 | 385 | 0.1.1 386 | ===== 387 | 388 | - [fix] Fixed `MANIFEST.in`. 389 | - Updated packaging options. 390 | 391 | 0.1.0 392 | ===== 393 | 394 | - Exposed process attribute on Executor. 395 | - Exposed port and host on TCPExecutor. 396 | - Exposed URL on HTTPExecutor. 397 | - Simplified package structure. 398 | - Simplified executors operating API. 399 | - Updated documentation. 400 | - Added docblocks for every function. 401 | - Applied license headers. 402 | - Stripped orchestrators. 403 | - Forked off from `summon_process`. 404 | -------------------------------------------------------------------------------- /logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 17 | 18 | 20 | image/svg+xml 21 | 23 | 24 | 25 | 26 | 27 | 30 | 34 | 38 | 45 | 49 | 53 | 57 | 61 | 65 | 73 | 77 | 81 | 85 | MIRAKURU 100 | 101 | -------------------------------------------------------------------------------- /mirakuru/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 by Clearcode 2 | # and associates (see AUTHORS). 3 | 4 | # This file is part of mirakuru. 5 | 6 | # mirakuru is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU Lesser General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | 11 | # mirakuru is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU Lesser General Public License for more details. 15 | 16 | # You should have received a copy of the GNU Lesser General Public License 17 | # along with mirakuru. If not, see . 18 | """Executor with the core functionality.""" 19 | 20 | import atexit 21 | import errno 22 | import logging 23 | import os 24 | import platform 25 | import shlex 26 | import signal 27 | import subprocess 28 | import time 29 | import uuid 30 | from contextlib import contextmanager 31 | from types import TracebackType 32 | from typing import ( 33 | IO, 34 | Any, 35 | Callable, 36 | Iterator, 37 | Type, 38 | TypeVar, 39 | ) 40 | 41 | from mirakuru.base_env import processes_with_env 42 | from mirakuru.compat import SIGKILL 43 | from mirakuru.exceptions import ( 44 | AlreadyRunning, 45 | ProcessExitedWithError, 46 | ProcessFinishedWithError, 47 | TimeoutExpired, 48 | ) 49 | 50 | LOG = logging.getLogger(__name__) 51 | 52 | ENV_UUID = "mirakuru_uuid" 53 | """ 54 | Name of the environment variable used by mirakuru to mark its subprocesses. 55 | """ 56 | 57 | IGNORED_ERROR_CODES = [errno.ESRCH] 58 | if platform.system() == "Darwin": 59 | IGNORED_ERROR_CODES = [errno.ESRCH, errno.EPERM] 60 | 61 | # Type variables used for self in functions returning self, so it's correctly 62 | # typed in derived classes. 63 | SimpleExecutorType = TypeVar("SimpleExecutorType", bound="SimpleExecutor") 64 | ExecutorType = TypeVar("ExecutorType", bound="Executor") 65 | 66 | 67 | @atexit.register 68 | def cleanup_subprocesses() -> None: 69 | """On python exit: find possibly running subprocesses and kill them.""" 70 | # atexit functions tends to loose global imports sometimes so reimport 71 | # everything what is needed again here: 72 | import errno 73 | import os 74 | 75 | from mirakuru.base_env import processes_with_env 76 | from mirakuru.compat import SIGKILL 77 | 78 | pids = processes_with_env(ENV_UUID, str(os.getpid())) 79 | for pid in pids: 80 | try: 81 | os.kill(pid, SIGKILL) 82 | except OSError as err: 83 | if err.errno != errno.ESRCH: 84 | print("Can not kill the", pid, "leaked process", err) 85 | 86 | 87 | class SimpleExecutor: # pylint:disable=too-many-instance-attributes 88 | """Simple subprocess executor with start/stop/kill functionality.""" 89 | 90 | def __init__( # pylint:disable=too-many-arguments 91 | self, 92 | command: str | list[str] | tuple[str, ...], 93 | cwd: str | None = None, 94 | shell: bool = False, 95 | timeout: int | float = 3600, 96 | sleep: float = 0.1, 97 | stop_signal: int = signal.SIGTERM, 98 | kill_signal: int = SIGKILL, 99 | expected_returncode: int | None = None, 100 | envvars: dict[str, str] | None = None, 101 | stdin: None | int | IO[Any] = subprocess.PIPE, 102 | stdout: None | int | IO[Any] = subprocess.PIPE, 103 | stderr: None | int | IO[Any] = None, 104 | popen_kwargs: dict[str, Any] | None = None, 105 | ) -> None: 106 | """Initialize executor. 107 | 108 | :param command: command to be run by the subprocess 109 | :param cwd: current working directory to be set for executor 110 | :param shell: same as the `subprocess.Popen` shell definition. 111 | On Windows always set to True. 112 | :param timeout: number of seconds to wait for the process to start 113 | or stop. 114 | :param sleep: how often to check for start/stop condition 115 | :param stop_signal: signal used to stop a process run by the executor. 116 | default is `signal.SIGTERM` 117 | :param kill_signal: signal used to kill a process run by the executor. 118 | default is `signal.SIGKILL` (`signal.SIGTERM` on Windows) 119 | :param expected_returncode: expected exit code. 120 | default is None, which means, Executor will determine a POSIX 121 | compatible return code based on signal sent. 122 | :param envvars: Additional environment variables 123 | :param stdin: file descriptor for stdin 124 | :param stdout: file descriptor for stdout 125 | :param stderr: file descriptor for stderr 126 | :param popen_kwargs: additional keyword arguments to be passed to 127 | `subprocess.Popen` when starting the process. 128 | 129 | .. note:: 130 | 131 | **timeout** set for an executor is valid for all the levels of waits 132 | on the way up. That means that if some more advanced executor 133 | establishes the timeout to 10 seconds, and it will take 5 seconds 134 | for the first check, the second check will only have 5 seconds left. 135 | 136 | Your executor will raise an exception if something goes wrong 137 | during this time. The default value of timeout is ``None``, so it 138 | is a good practice to set this. 139 | 140 | """ 141 | if isinstance(command, (list, tuple)): 142 | self.command = " ".join((shlex.quote(c) for c in command)) 143 | """Command that the executor runs.""" 144 | self.command_parts = command 145 | else: 146 | self.command = command 147 | self.command_parts = shlex.split(command) 148 | 149 | self._cwd = cwd 150 | self._shell = True 151 | if platform.system() != "Windows": 152 | self._shell = shell 153 | 154 | self._timeout = timeout 155 | self._sleep = sleep 156 | self._stop_signal = stop_signal 157 | self._kill_signal = kill_signal 158 | self._expected_returncode = expected_returncode 159 | self._envvars = envvars or {} 160 | 161 | self._stdin = stdin 162 | self._stdout = stdout 163 | self._stderr = stderr 164 | 165 | self._endtime: float | None = None 166 | self._additional_popen_kwargs = popen_kwargs or {} 167 | self.process: subprocess.Popen | None = None 168 | """A :class:`subprocess.Popen` instance once process is started.""" 169 | 170 | self._uuid = f"{os.getpid()}:{uuid.uuid4()}" 171 | 172 | def __enter__(self: SimpleExecutorType) -> SimpleExecutorType: 173 | """Enter context manager starting the subprocess. 174 | 175 | :returns: itself 176 | :rtype: SimpleExecutor 177 | """ 178 | return self.start() 179 | 180 | def __exit__( 181 | self, 182 | exc_type: Type[BaseException] | None, 183 | exc_value: BaseException | None, 184 | traceback: TracebackType | None, 185 | ) -> None: 186 | """Exit context manager stopping the subprocess.""" 187 | self.stop() 188 | 189 | def running(self) -> bool: 190 | """Check if executor is running. 191 | 192 | :returns: True if process is running, False otherwise 193 | :rtype: bool 194 | """ 195 | if self.process is None: 196 | LOG.debug("There is no process running!") 197 | return False 198 | return self.process.poll() is None 199 | 200 | @property 201 | def envvars(self) -> dict[str, str]: 202 | """Combines required environment variables with os.environ and mirakuru_uuid.""" 203 | envs = os.environ.copy() 204 | envs.update(self._envvars) 205 | # Trick with marking subprocesses with an environment variable. 206 | # 207 | # There is no easy way to recognize all subprocesses that were 208 | # spawned during lifetime of a certain subprocess so mirakuru does 209 | # this hack in order to mark who was the original parent. Even if 210 | # some subprocess got daemonized or changed original process group 211 | # mirakuru will be able to find it by this environment variable. 212 | # 213 | # There may be a situation when some subprocess will abandon 214 | # original envs from parents and then it won't be later found. 215 | envs[ENV_UUID] = self._uuid 216 | return envs 217 | 218 | @property 219 | def _popen_kwargs(self) -> dict[str, Any]: 220 | """Get kwargs for the process instance. 221 | 222 | .. note:: 223 | We want to open ``stdin``, ``stdout`` and ``stderr`` as text 224 | streams in universal newlines mode, so we have to set 225 | ``universal_newlines`` to ``True``. 226 | 227 | :return: 228 | """ 229 | kwargs: dict[str, Any] = {} 230 | kwargs.update(self._additional_popen_kwargs) 231 | 232 | if self._stdin: 233 | kwargs["stdin"] = self._stdin 234 | if self._stdout: 235 | kwargs["stdout"] = self._stdout 236 | if self._stderr: 237 | kwargs["stderr"] = self._stderr 238 | kwargs["universal_newlines"] = True 239 | 240 | kwargs["shell"] = self._shell 241 | kwargs["env"] = self.envvars 242 | kwargs["cwd"] = self._cwd 243 | if platform.system() != "Windows": 244 | kwargs["preexec_fn"] = os.setsid 245 | 246 | return kwargs 247 | 248 | def start(self: SimpleExecutorType) -> SimpleExecutorType: 249 | """Start defined process. 250 | 251 | After process gets started, timeout countdown begins as well. 252 | 253 | :returns: itself 254 | :rtype: SimpleExecutor 255 | """ 256 | if self.process is None: 257 | command: str | list[str] | tuple[str, ...] = self.command 258 | if not self._shell: 259 | command = self.command_parts 260 | LOG.debug("Starting process: %s", command) 261 | self.process = subprocess.Popen(command, **self._popen_kwargs) 262 | 263 | self._set_timeout() 264 | return self 265 | 266 | def _set_timeout(self) -> None: 267 | """Set timeout for possible wait.""" 268 | self._endtime = time.time() + self._timeout 269 | 270 | def _clear_process(self) -> None: 271 | """Close stdin/stdout of subprocess. 272 | 273 | It is required because of ResourceWarning in Python 3. 274 | """ 275 | if self.process: 276 | self.process.__exit__(None, None, None) 277 | self.process = None 278 | 279 | self._endtime = None 280 | 281 | def _kill_all_kids(self, sig: int) -> set[int]: 282 | """Kill all subprocesses (and its subprocesses) that executor started. 283 | 284 | This function tries to kill all leftovers in process tree that current 285 | executor may have left. It uses environment variable to recognise if 286 | process have origin in this Executor so it does not give 100 % and 287 | some daemons fired by subprocess may still be running. 288 | 289 | :param int sig: signal used to stop process run by executor. 290 | :return: process ids (pids) of killed processes 291 | :rtype: set 292 | """ 293 | pids = processes_with_env(ENV_UUID, self._uuid) 294 | for pid in pids: 295 | LOG.debug("Killing process %d ...", pid) 296 | try: 297 | os.kill(pid, sig) 298 | except OSError as err: 299 | if err.errno in IGNORED_ERROR_CODES: 300 | # the process has died before we tried to kill it. 301 | pass 302 | else: 303 | raise 304 | LOG.debug("Killed process %d.", pid) 305 | return pids 306 | 307 | def stop( 308 | self: SimpleExecutorType, 309 | stop_signal: int | None = None, 310 | expected_returncode: int | None = None, 311 | ) -> SimpleExecutorType: 312 | """Stop process running. 313 | 314 | Wait 10 seconds for the process to end, then just kill it. 315 | 316 | :param int stop_signal: signal used to stop process run by executor. 317 | None for default. 318 | :param int expected_returncode: expected exit code. 319 | None for default - POSIX compatible behaviour. 320 | :returns: self 321 | :rtype: SimpleExecutor 322 | 323 | .. note:: 324 | 325 | When gathering coverage for the subprocess in tests, 326 | you have to allow subprocesses to end gracefully. 327 | """ 328 | if self.process is None: 329 | return self 330 | 331 | if stop_signal is None: 332 | stop_signal = self._stop_signal 333 | 334 | try: 335 | os.killpg(self.process.pid, stop_signal) 336 | except OSError as err: 337 | if err.errno in IGNORED_ERROR_CODES: 338 | pass 339 | else: 340 | raise 341 | 342 | def process_stopped() -> bool: 343 | """Return True only only when self.process is not running.""" 344 | return self.running() is False 345 | 346 | self._set_timeout() 347 | try: 348 | self.wait_for(process_stopped) 349 | except TimeoutExpired: 350 | # at this moment, process got killed, 351 | pass 352 | 353 | if self.process is None: 354 | # the process has already been force killed and cleaned up by the 355 | # `wait_for` above. 356 | return self # type: ignore[unreachable] 357 | self._kill_all_kids(stop_signal) 358 | exit_code = self.process.wait() 359 | self._clear_process() 360 | 361 | if expected_returncode is None: 362 | expected_returncode = self._expected_returncode 363 | if expected_returncode is None: 364 | # Assume a POSIX approach where sending a SIGNAL means 365 | # that the process should exist with -SIGNAL exit code. 366 | # https://docs.python.org/3/library/subprocess.html#subprocess.Popen.returncode 367 | expected_returncode = -stop_signal 368 | 369 | if exit_code and exit_code != expected_returncode: 370 | raise ProcessFinishedWithError(self, exit_code) 371 | 372 | return self 373 | 374 | @contextmanager 375 | def stopped(self: SimpleExecutorType) -> Iterator[SimpleExecutorType]: 376 | """Stop process for given context and starts it afterwards. 377 | 378 | Allows for easier writing resistance integration tests whenever one of 379 | the service fails. 380 | :yields: itself 381 | :rtype: SimpleExecutor 382 | """ 383 | if self.running(): 384 | self.stop() 385 | yield self 386 | self.start() 387 | 388 | def kill( 389 | self: SimpleExecutorType, wait: bool = True, sig: int | None = None 390 | ) -> SimpleExecutorType: 391 | """Kill the process if running. 392 | 393 | :param bool wait: set to `True` to wait for the process to end, 394 | or False, to simply proceed after sending signal. 395 | :param int sig: signal used to kill process run by the executor. 396 | None by default. 397 | :returns: itself 398 | :rtype: SimpleExecutor 399 | """ 400 | if sig is None: 401 | sig = self._kill_signal 402 | if self.process and self.running(): 403 | os.killpg(self.process.pid, sig) 404 | if wait: 405 | self.process.wait() 406 | 407 | self._kill_all_kids(sig) 408 | self._clear_process() 409 | return self 410 | 411 | def output(self) -> IO[Any] | None: 412 | """Return subprocess output.""" 413 | if self.process is not None: 414 | return self.process.stdout 415 | return None # pragma: no cover 416 | 417 | def err_output(self) -> IO[Any] | None: 418 | """Return subprocess stderr.""" 419 | if self.process is not None: 420 | return self.process.stderr 421 | return None # pragma: no cover 422 | 423 | def wait_for(self: SimpleExecutorType, wait_for: Callable[[], bool]) -> SimpleExecutorType: 424 | """Wait for callback to return True. 425 | 426 | Simply returns if wait_for condition has been met, 427 | raises TimeoutExpired otherwise and kills the process. 428 | 429 | :param callback wait_for: callback to call 430 | :raises: mirakuru.exceptions.TimeoutExpired 431 | :returns: itself 432 | :rtype: SimpleExecutor 433 | """ 434 | while self.check_timeout(): 435 | if wait_for(): 436 | return self 437 | time.sleep(self._sleep) 438 | 439 | self.kill() 440 | raise TimeoutExpired(self, timeout=self._timeout) 441 | 442 | def check_timeout(self) -> bool: 443 | """Check if timeout has expired. 444 | 445 | Returns True if there is no timeout set or the timeout has not expired. 446 | Kills the process and raises TimeoutExpired exception otherwise. 447 | 448 | This method should be used in while loops waiting for some data. 449 | 450 | :return: True if timeout expired, False if not 451 | :rtype: bool 452 | """ 453 | return self._endtime is None or time.time() <= self._endtime 454 | 455 | def __del__(self) -> None: 456 | """Cleanup subprocesses created during Executor lifetime.""" 457 | try: 458 | if self.process: 459 | self.kill() 460 | except Exception: # pragma: no cover 461 | print("*" * 80) 462 | print("Exception while deleting Executor. It is strongly suggested that you use") 463 | print("it as a context manager instead.") 464 | print("*" * 80) 465 | raise 466 | 467 | def __repr__(self) -> str: 468 | """Return unambiguous executor representation.""" 469 | command = self.command 470 | if len(command) > 10: 471 | command = command[:10] + "..." 472 | module = self.__class__.__module__ 473 | executor = self.__class__.__name__ 474 | return f'<{module}.{executor}: "{command}" {hex(id(self))}>' 475 | 476 | def __str__(self) -> str: 477 | """Return readable executor representation.""" 478 | module = self.__class__.__module__ 479 | executor = self.__class__.__name__ 480 | return f'<{module}.{executor}: "{self.command}" {hex(id(self))}>' 481 | 482 | 483 | class Executor(SimpleExecutor): 484 | """Base class for executors with a pre- and after-start checks.""" 485 | 486 | def pre_start_check(self) -> bool: 487 | """Check process before the start of executor. 488 | 489 | Should be overridden in order to return True when some other 490 | executor (or process) has already started with the same configuration. 491 | :rtype: bool 492 | """ 493 | raise NotImplementedError 494 | 495 | def start(self: ExecutorType) -> ExecutorType: 496 | """Start executor with additional checks. 497 | 498 | Checks if previous executor isn't running then start process 499 | (executor) and wait until it's started. 500 | :returns: itself 501 | :rtype: Executor 502 | """ 503 | if self.pre_start_check(): 504 | # Some other executor (or process) is running with same config: 505 | raise AlreadyRunning(self) 506 | 507 | super().start() 508 | 509 | self.wait_for(self.check_subprocess) 510 | return self 511 | 512 | def check_subprocess(self) -> bool: 513 | """Make sure the process didn't exit with an error and run the checks. 514 | 515 | :rtype: bool 516 | :return: the actual check status or False before starting the process 517 | :raise ProcessExitedWithError: when the main process exits with 518 | an error 519 | """ 520 | if self.process is None: # pragma: no cover 521 | # No process was started. 522 | return False 523 | exit_code = self.process.poll() 524 | if exit_code is not None and exit_code != 0: 525 | # The main process exited with an error. Clean up the children 526 | # if any. 527 | self._kill_all_kids(self._kill_signal) 528 | self._clear_process() 529 | raise ProcessExitedWithError(self, exit_code) 530 | 531 | return self.after_start_check() 532 | 533 | def after_start_check(self) -> bool: 534 | """Check process after the start of executor. 535 | 536 | Should be overridden in order to return boolean value if executor 537 | can be treated as started. 538 | :rtype: bool 539 | """ 540 | raise NotImplementedError 541 | --------------------------------------------------------------------------------