├── .bumpversion.cfg
├── .circleci
    ├── config.yml
    └── merge_pr.sh
├── .github
    ├── ISSUE_TEMPLATE.md
    └── PULL_REQUEST_TEMPLATE.md
├── .gitignore
├── .project-template
    ├── fill_template_vars.sh
    ├── refill_template_vars.sh
    └── template_vars.txt
├── .pydocstyle.ini
├── LICENSE
├── Makefile
├── README.md
├── asyncio_run_in_process
    ├── __init__.py
    ├── _child.py
    ├── _child_trio.py
    ├── _utils.py
    ├── abc.py
    ├── constants.py
    ├── exceptions.py
    ├── main.py
    ├── process.py
    ├── state.py
    ├── tools
    │   ├── __init__.py
    │   └── sleep.py
    ├── typing.py
    └── typing_compat.py
├── docs
    ├── Makefile
    ├── _static
    │   └── .suppress-sphinx-build-warning
    ├── asyncio_run_in_process.rst
    ├── conf.py
    ├── index.rst
    └── releases.rst
├── mypy.ini
├── pytest.ini
├── requirements-docs.txt
├── setup.py
├── tests
    └── core
    │   ├── conftest.py
    │   ├── test_import.py
    │   ├── test_open_in_process.py
    │   ├── test_process_object.py
    │   ├── test_remote_exception.py
    │   ├── test_run_in_process.py
    │   └── test_state.py
└── tox.ini


/.bumpversion.cfg:
--------------------------------------------------------------------------------
 1 | [bumpversion]
 2 | current_version = 0.1.0-alpha.10
 3 | commit = True
 4 | tag = True
 5 | parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(-(?P<stage>[^.]*)\.(?P<devnum>\d+))?
 6 | serialize = 
 7 | 	{major}.{minor}.{patch}-{stage}.{devnum}
 8 | 	{major}.{minor}.{patch}
 9 | 
10 | [bumpversion:part:stage]
11 | optional_value = stable
12 | first_value = stable
13 | values = 
14 | 	alpha
15 | 	beta
16 | 	stable
17 | 
18 | [bumpversion:part:devnum]
19 | 
20 | [bumpversion:file:setup.py]
21 | search = version='{current_version}',
22 | replace = version='{new_version}',
23 | 


--------------------------------------------------------------------------------
/.circleci/config.yml:
--------------------------------------------------------------------------------
 1 | version: 2.0
 2 | 
 3 | # heavily inspired by https://raw.githubusercontent.com/pinax/pinax-wiki/6bd2a99ab6f702e300d708532a6d1d9aa638b9f8/.circleci/config.yml
 4 | 
 5 | common: &common
 6 |   working_directory: ~/repo
 7 |   steps:
 8 |     - checkout
 9 |     - run:
10 |         name: merge pull request base
11 |         command: ./.circleci/merge_pr.sh
12 |     - run:
13 |         name: merge pull request base (2nd try)
14 |         command: ./.circleci/merge_pr.sh
15 |         when: on_fail
16 |     - run:
17 |         name: merge pull request base (3nd try)
18 |         command: ./.circleci/merge_pr.sh
19 |         when: on_fail
20 |     - restore_cache:
21 |         keys:
22 |           - cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "tox.ini" }}
23 |     - run:
24 |         name: install dependencies
25 |         command: pip install --user tox
26 |     - run:
27 |         name: run tox
28 |         command: ~/.local/bin/tox -r
29 |     - save_cache:
30 |         paths:
31 |           - .hypothesis
32 |           - .tox
33 |           - ~/.cache/pip
34 |           - ~/.local
35 |           - ./eggs
36 |         key: cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "tox.ini" }}
37 | 
38 | jobs:
39 |   doctest:
40 |     <<: *common
41 |     docker:
42 |       - image: circleci/python:3.6
43 |         environment:
44 |           TOXENV: doctest
45 |   lint:
46 |     <<: *common
47 |     docker:
48 |       - image: circleci/python:3.6
49 |         environment:
50 |           TOXENV: lint
51 |   py36-core:
52 |     <<: *common
53 |     docker:
54 |       - image: circleci/python:3.6
55 |         environment:
56 |           TOXENV: py36-core
57 |   py37-core:
58 |     <<: *common
59 |     docker:
60 |       - image: circleci/python:3.7
61 |         environment:
62 |           TOXENV: py37-core
63 |   py38-core:
64 |     <<: *common
65 |     docker:
66 |       - image: circleci/python:3.8
67 |         environment:
68 |           TOXENV: py38-core
69 | workflows:
70 |   version: 2
71 |   test:
72 |     jobs:
73 |       - doctest
74 |       - lint
75 |       - py36-core
76 |       - py37-core
77 |       - py38-core
78 | 


--------------------------------------------------------------------------------
/.circleci/merge_pr.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | if [[ -n "${CIRCLE_PR_NUMBER}" ]]; then
 4 |   PR_INFO_URL=https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pulls/$CIRCLE_PR_NUMBER
 5 |   PR_BASE_BRANCH=$(curl -L "$PR_INFO_URL" | python -c 'import json, sys; obj = json.load(sys.stdin); sys.stdout.write(obj["base"]["ref"])')
 6 |   git fetch origin +"$PR_BASE_BRANCH":circleci/pr-base
 7 |   # We need these config values or git complains when creating the
 8 |   # merge commit
 9 |   git config --global user.name "Circle CI"
10 |   git config --global user.email "circleci@example.com"
11 |   git merge --no-edit circleci/pr-base
12 | fi
13 | 


--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE.md:
--------------------------------------------------------------------------------
 1 |  _If this is a bug report, please fill in the following sections.
 2 | If this is a feature request, delete and describe what you would like with examples._
 3 | 
 4 | ## What was wrong?
 5 | 
 6 | ### Code that produced the error
 7 | 
 8 | ```py
 9 | CODE_TO_REPRODUCE 
10 | ```
11 | 
12 | ### Full error output 
13 | 
14 | ```sh
15 | ERROR_HERE 
16 | ```
17 | 
18 | ### Expected Result
19 | 
20 | _This section may be deleted if the expectation is "don't crash"._
21 | 
22 | ```sh
23 | EXPECTED_RESULT
24 | ``` 
25 | 
26 | ### Environment
27 | 
28 | ```sh
29 | # run this:
30 | $ python -m eth_utils
31 | 
32 | # then copy the output here:
33 | OUTPUT_HERE
34 | ``` 
35 | 
36 | ## How can it be fixed?
37 | 
38 | Fill this section in if you know how this could or should be fixed.
39 | 


--------------------------------------------------------------------------------
/.github/PULL_REQUEST_TEMPLATE.md:
--------------------------------------------------------------------------------
 1 | ## What was wrong?
 2 | 
 3 | Issue #
 4 | 
 5 | ## How was it fixed?
 6 | 
 7 | Summary of approach.
 8 | 
 9 | #### Cute Animal Picture
10 | 
11 | ![put a cute animal picture link inside the parentheses]()
12 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | *.py[cod]
  2 | 
  3 | # C extensions
  4 | *.so
  5 | 
  6 | # Packages
  7 | *.egg
  8 | *.egg-info
  9 | dist
 10 | build
 11 | eggs
 12 | .eggs
 13 | parts
 14 | bin
 15 | var
 16 | sdist
 17 | develop-eggs
 18 | .installed.cfg
 19 | lib
 20 | lib64
 21 | venv*
 22 | 
 23 | # Installer logs
 24 | pip-log.txt
 25 | 
 26 | # Unit test / coverage reports
 27 | .coverage
 28 | .tox
 29 | nosetests.xml
 30 | 
 31 | # Translations
 32 | *.mo
 33 | 
 34 | # Mr Developer
 35 | .mr.developer.cfg
 36 | .project
 37 | .pydevproject
 38 | 
 39 | # Complexity
 40 | output/*.html
 41 | output/*/index.html
 42 | 
 43 | # Sphinx
 44 | docs/_build
 45 | docs/modules.rst
 46 | docs/*.internal.rst
 47 | docs/*.utils.rst
 48 | docs/*._utils.*
 49 | 
 50 | # Blockchain
 51 | chains
 52 | 
 53 | # Hypothese Property base testing
 54 | .hypothesis
 55 | 
 56 | # tox/pytest cache
 57 | .cache
 58 | 
 59 | # Test output logs
 60 | logs
 61 | ### JetBrains template
 62 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
 63 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 64 | 
 65 | # User-specific stuff:
 66 | .idea/workspace.xml
 67 | .idea/tasks.xml
 68 | .idea/dictionaries
 69 | .idea/vcs.xml
 70 | .idea/jsLibraryMappings.xml
 71 | 
 72 | # Sensitive or high-churn files:
 73 | .idea/dataSources.ids
 74 | .idea/dataSources.xml
 75 | .idea/dataSources.local.xml
 76 | .idea/sqlDataSources.xml
 77 | .idea/dynamic.xml
 78 | .idea/uiDesigner.xml
 79 | 
 80 | # Gradle:
 81 | .idea/gradle.xml
 82 | .idea/libraries
 83 | 
 84 | # Mongo Explorer plugin:
 85 | .idea/mongoSettings.xml
 86 | 
 87 | # VIM temp files
 88 | *.sw[op]
 89 | 
 90 | # mypy
 91 | .mypy_cache
 92 | 
 93 | ## File-based project format:
 94 | *.iws
 95 | 
 96 | ## Plugin-specific files:
 97 | 
 98 | # IntelliJ
 99 | /out/
100 | 
101 | # mpeltonen/sbt-idea plugin
102 | .idea_modules/
103 | 
104 | # JIRA plugin
105 | atlassian-ide-plugin.xml
106 | 
107 | # Crashlytics plugin (for Android Studio and IntelliJ)
108 | com_crashlytics_export_strings.xml
109 | crashlytics.properties
110 | crashlytics-build.properties
111 | fabric.properties
112 | 
113 | 


--------------------------------------------------------------------------------
/.project-template/fill_template_vars.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | 
 3 | set -o errexit
 4 | set -o nounset
 5 | set -o pipefail
 6 | 
 7 | PROJECT_ROOT=$(dirname $(dirname $(python -c 'import os, sys; sys.stdout.write(os.path.realpath(sys.argv[1]))' "$0")))
 8 | 
 9 | echo "What is your python module name?"
10 | read MODULE_NAME
11 | 
12 | echo "What is your pypi package name? (default: $MODULE_NAME)"
13 | read PYPI_INPUT
14 | PYPI_NAME=${PYPI_INPUT:-$MODULE_NAME}
15 | 
16 | echo "What is your github project name? (default: $PYPI_NAME)"
17 | read REPO_INPUT
18 | REPO_NAME=${REPO_INPUT:-$PYPI_NAME}
19 | 
20 | echo "What is your readthedocs.org project name? (default: $PYPI_NAME)"
21 | read RTD_INPUT
22 | RTD_NAME=${RTD_INPUT:-$PYPI_NAME}
23 | 
24 | echo "What is your project name (ex: at the top of the README)? (default: $REPO_NAME)"
25 | read PROJECT_INPUT
26 | PROJECT_NAME=${PROJECT_INPUT:-$REPO_NAME}
27 | 
28 | echo "What is a one-liner describing the project?"
29 | read SHORT_DESCRIPTION
30 | 
31 | _replace() {
32 |   local find_cmd=(find "$PROJECT_ROOT" ! -perm -u=x ! -path '*/.git/*' -type f)
33 | 
34 |   if [[ $(uname) == Darwin ]]; then
35 |     "${find_cmd[@]}" -exec sed -i '' "$1" {} +
36 |   else
37 |     "${find_cmd[@]}" -exec sed -i "$1" {} +
38 |   fi
39 | }
40 | _replace "s/<MODULE_NAME>/$MODULE_NAME/g"
41 | _replace "s/<PYPI_NAME>/$PYPI_NAME/g"
42 | _replace "s/<REPO_NAME>/$REPO_NAME/g"
43 | _replace "s/<RTD_NAME>/$RTD_NAME/g"
44 | _replace "s/<PROJECT_NAME>/$PROJECT_NAME/g"
45 | _replace "s/<SHORT_DESCRIPTION>/$SHORT_DESCRIPTION/g"
46 | 
47 | mkdir -p "$PROJECT_ROOT/$MODULE_NAME"
48 | touch "$PROJECT_ROOT/$MODULE_NAME/__init__.py"
49 | 


--------------------------------------------------------------------------------
/.project-template/refill_template_vars.sh:
--------------------------------------------------------------------------------
1 | TEMPLATE_DIR=$(dirname $(readlink -f "$0"))
2 | <"$TEMPLATE_DIR/template_vars.txt" "$TEMPLATE_DIR/fill_template_vars.sh"
3 | 


--------------------------------------------------------------------------------
/.project-template/template_vars.txt:
--------------------------------------------------------------------------------
1 | asyncio_run_in_process
2 | asyncio-run-in-process
3 | asyncio-run-in-process
4 | asyncio-run-in-process
5 | asyncio-run-in-process
6 | Simple asyncio friendly replacement for multiprocessing
7 | 


--------------------------------------------------------------------------------
/.pydocstyle.ini:
--------------------------------------------------------------------------------
 1 | [pydocstyle]
 2 | ; All error codes found here:
 3 | ; http://www.pydocstyle.org/en/3.0.0/error_codes.html
 4 | ;
 5 | ; Ignored:
 6 | ; D1 - Missing docstring error codes
 7 | ;
 8 | ; Selected:
 9 | ; D2 - Whitespace error codes
10 | ; D3 - Quote error codes
11 | ; D4 - Content related error codes
12 | select=D2,D3,D4
13 | 
14 | ; Extra ignores:
15 | ; D200 - One-line docstring should fit on one line with quotes
16 | ; D203 - 1 blank line required before class docstring
17 | ; D204 - 1 blank line required after class docstring
18 | ; D205 - 1 blank line required between summary line and description
19 | ; D212 - Multi-line docstring summary should start at the first line
20 | ; D302 - Use u""" for Unicode docstrings
21 | ; D400 - First line should end with a period
22 | ; D401 - First line should be in imperative mood
23 | ; D412 - No blank lines allowed between a section header and its content
24 | add-ignore=D200,D203,D204,D205,D212,D302,D400,D401,D412
25 | 
26 | ; Explanation:
27 | ; D400 - Enabling this error code seems to make it a requirement that the first
28 | ; sentence in a docstring is not split across two lines.  It also makes it a
29 | ; requirement that no docstring can have a multi-sentence description without a
30 | ; summary line.  Neither one of those requirements seem appropriate.
31 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2019 The Ethereum Foundation
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | CURRENT_SIGN_SETTING := $(shell git config commit.gpgSign)
 2 | 
 3 | .PHONY: clean-pyc clean-build docs
 4 | 
 5 | help:
 6 | 	@echo "clean-build - remove build artifacts"
 7 | 	@echo "clean-pyc - remove Python file artifacts"
 8 | 	@echo "lint - check style with flake8"
 9 | 	@echo "test - run tests quickly with the default Python"
10 | 	@echo "testall - run tests on every Python version with tox"
11 | 	@echo "release - package and upload a release"
12 | 	@echo "dist - package"
13 | 
14 | clean: clean-build clean-pyc
15 | 
16 | clean-build:
17 | 	rm -fr build/
18 | 	rm -fr dist/
19 | 	rm -fr *.egg-info
20 | 
21 | clean-pyc:
22 | 	find . -name '*.pyc' -exec rm -f {} +
23 | 	find . -name '*.pyo' -exec rm -f {} +
24 | 	find . -name '*~' -exec rm -f {} +
25 | 
26 | lint:
27 | 	tox -elint
28 | 
29 | lint-roll:
30 | 	isort --recursive asyncio_run_in_process tests
31 | 	$(MAKE) lint
32 | 
33 | test:
34 | 	pytest tests
35 | 
36 | test-all:
37 | 	tox
38 | 
39 | build-docs:
40 | 	sphinx-apidoc -o docs/ . setup.py "*conftest*"
41 | 	$(MAKE) -C docs clean
42 | 	$(MAKE) -C docs html
43 | 	$(MAKE) -C docs doctest
44 | 
45 | docs: build-docs
46 | 	open docs/_build/html/index.html
47 | 
48 | linux-docs: build-docs
49 | 	xdg-open docs/_build/html/index.html
50 | 
51 | release: clean
52 | 	git config commit.gpgSign true
53 | 	bumpversion $(bump)
54 | 	git push upstream && git push upstream --tags
55 | 	python setup.py sdist bdist_wheel
56 | 	twine upload dist/*
57 | 	git config commit.gpgSign "$(CURRENT_SIGN_SETTING)"
58 | 
59 | dist: clean
60 | 	python setup.py sdist bdist_wheel
61 | 	ls -l dist
62 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # asyncio-run-in-process
 2 | 
 3 | [![Join the chat at https://gitter.im/ethereum/asyncio-run-in-process](https://badges.gitter.im/ethereum/asyncio-run-in-process.svg)](https://gitter.im/ethereum/asyncio-run-in-process?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 4 | [![Build Status](https://circleci.com/gh/ethereum/asyncio-run-in-process.svg?style=shield)](https://circleci.com/gh/ethereum/asyncio-run-in-process)
 5 | [![PyPI version](https://badge.fury.io/py/asyncio-run-in-process.svg)](https://badge.fury.io/py/asyncio-run-in-process)
 6 | [![Python versions](https://img.shields.io/pypi/pyversions/asyncio-run-in-process.svg)](https://pypi.python.org/pypi/asyncio-run-in-process)
 7 | [![Docs build](https://readthedocs.org/projects/asyncio-run-in-process/badge/?version=latest)](http://asyncio-run-in-process.readthedocs.io/en/latest/?badge=latest)
 8 |    
 9 | 
10 | Simple asyncio friendly replacement for multiprocessing
11 | 
12 | Read more in the [documentation on ReadTheDocs](https://asyncio-run-in-process.readthedocs.io/). [View the change log](https://asyncio-run-in-process.readthedocs.io/en/latest/releases.html).
13 | 
14 | ## Quickstart
15 | 
16 | ```sh
17 | pip install asyncio-run-in-process
18 | ```
19 | 
20 | ## Developer Setup
21 | 
22 | If you would like to hack on asyncio-run-in-process, please check out the [Snake Charmers
23 | Tactical Manual](https://github.com/ethereum/snake-charmers-tactical-manual)
24 | for information on how we do:
25 | 
26 | - Testing
27 | - Pull Requests
28 | - Code Style
29 | - Documentation
30 | 
31 | ### Development Environment Setup
32 | 
33 | You can set up your dev environment with:
34 | 
35 | ```sh
36 | git clone git@github.com:ethereum/asyncio-run-in-process.git
37 | cd asyncio-run-in-process
38 | virtualenv -p python3 venv
39 | . venv/bin/activate
40 | pip install -e .[dev]
41 | ```
42 | 
43 | ### Testing Setup
44 | 
45 | During development, you might like to have tests run on every file save.
46 | 
47 | Show flake8 errors on file change:
48 | 
49 | ```sh
50 | # Test flake8
51 | when-changed -v -s -r -1 asyncio_run_in_process/ tests/ -c "clear; flake8 asyncio_run_in_process tests && echo 'flake8 success' || echo 'error'"
52 | ```
53 | 
54 | Run multi-process tests in one command, but without color:
55 | 
56 | ```sh
57 | # in the project root:
58 | pytest --numprocesses=4 --looponfail --maxfail=1
59 | # the same thing, succinctly:
60 | pytest -n 4 -f --maxfail=1
61 | ```
62 | 
63 | Run in one thread, with color and desktop notifications:
64 | 
65 | ```sh
66 | cd venv
67 | ptw --onfail "notify-send -t 5000 'Test failure ⚠⚠⚠⚠⚠' 'python 3 test on asyncio-run-in-process failed'" ../tests ../asyncio_run_in_process
68 | ```
69 | 
70 | ### Release setup
71 | 
72 | For Debian-like systems:
73 | ```
74 | apt install pandoc
75 | ```
76 | 
77 | To release a new version:
78 | 
79 | ```sh
80 | make release bump=$$VERSION_PART_TO_BUMP$$
81 | ```
82 | 
83 | #### How to bumpversion
84 | 
85 | The version format for this repo is `{major}.{minor}.{patch}` for stable, and
86 | `{major}.{minor}.{patch}-{stage}.{devnum}` for unstable (`stage` can be alpha or beta).
87 | 
88 | To issue the next version in line, specify which part to bump,
89 | like `make release bump=minor` or `make release bump=devnum`. This is typically done from the
90 | master branch, except when releasing a beta (in which case the beta is released from master,
91 | and the previous stable branch is released from said branch). To include changes made with each
92 | release, update "docs/releases.rst" with the changes, and apply commit directly to master 
93 | before release.
94 | 
95 | If you are in a beta version, `make release bump=stage` will switch to a stable.
96 | 
97 | To issue an unstable version when the current version is stable, specify the
98 | new version explicitly, like `make release bump="--new-version 4.0.0-alpha.1 devnum"`
99 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/__init__.py:
--------------------------------------------------------------------------------
 1 | from .exceptions import (  # noqa: F401
 2 |     BaseRunInProcessException,
 3 |     InvalidState,
 4 |     ProcessKilled,
 5 | )
 6 | from .main import (  # noqa: F401
 7 |     open_in_process,
 8 |     open_in_process_with_trio,
 9 |     run_in_process,
10 |     run_in_process_with_trio,
11 | )
12 | from .state import (  # noqa: F401
13 |     State,
14 | )
15 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/_child.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import asyncio
  3 | import logging
  4 | import os
  5 | import signal
  6 | import sys
  7 | from typing import (
  8 |     Any,
  9 |     BinaryIO,
 10 |     Coroutine,
 11 |     Sequence,
 12 |     cast,
 13 | )
 14 | 
 15 | from ._utils import (
 16 |     RemoteException,
 17 |     cleanup_tasks,
 18 |     pickle_value,
 19 |     receive_pickled_value,
 20 | )
 21 | from .abc import (
 22 |     TAsyncFn,
 23 |     TEngineRunner,
 24 | )
 25 | from .exceptions import (
 26 |     ChildCancelled,
 27 | )
 28 | from .state import (
 29 |     State,
 30 |     update_state,
 31 |     update_state_finished,
 32 |     update_state_initialized,
 33 | )
 34 | from .typing import (
 35 |     TReturn,
 36 | )
 37 | 
 38 | logger = logging.getLogger("asyncio_run_in_process")
 39 | 
 40 | 
 41 | SHUTDOWN_SIGNALS = {signal.SIGTERM}
 42 | 
 43 | 
 44 | async def _handle_coro(coro: Coroutine[Any, Any, TReturn], got_SIGINT: asyncio.Event) -> TReturn:
 45 |     """
 46 |     Understanding this function requires some detailed knowledge of how
 47 |     coroutines work.
 48 | 
 49 |     The goal here is to run a coroutine function and wait for the result.
 50 |     However, if a SIGINT signal is received then we want to inject a
 51 |     `KeyboardInterrupt` exception into the running coroutine.
 52 | 
 53 |     Some additional nuance:
 54 | 
 55 |     - The `SIGINT` signal can happen multiple times and each time we want to
 56 |       throw a `KeyboardInterrupt` into the running coroutine which may choose to
 57 |       ignore the exception and continue executing.
 58 |     - When the `KeyboardInterrupt` hits the coroutine it can return a value
 59 |       which is sent using a `StopIteration` exception.  We treat this as the
 60 |       return value of the coroutine.
 61 |     """
 62 |     # The `coro` is first wrapped in `asyncio.shield` to protect us in the case
 63 |     # that a SIGINT happens.  In this case, the coro has a chance to return a
 64 |     # value during exception handling, in which case we are left with
 65 |     # `coro_task` which has not been awaited and thus will cause asyncio to
 66 |     # issue a warning.  However, since the coroutine has already exited, if we
 67 |     # await the `coro_task` then we will encounter a `RuntimeError`.  By
 68 |     # wrapping the coroutine in `asyncio.shield` we side-step this by
 69 |     # preventing the cancellation to actually penetrate the coroutine, allowing
 70 |     # us to await the `coro_task` without causing the `RuntimeError`.
 71 |     coro_task = asyncio.ensure_future(asyncio.shield(coro))
 72 |     async with cleanup_tasks(coro_task):
 73 |         while True:
 74 |             # Run the coroutine until it either returns, or a SIGINT is received.
 75 |             # This is done in a loop because the function *could* choose to ignore
 76 |             # the `KeyboardInterrupt` and continue processing, in which case we
 77 |             # reset the signal and resume waiting.
 78 |             done, pending = await asyncio.wait(
 79 |                 (coro_task, got_SIGINT.wait()),
 80 |                 return_when=asyncio.FIRST_COMPLETED,
 81 |             )
 82 | 
 83 |             if coro_task.done():
 84 |                 async with cleanup_tasks(*done, *pending):
 85 |                     return await coro_task
 86 |             elif got_SIGINT.is_set():
 87 |                 got_SIGINT.clear()
 88 | 
 89 |                 # In the event that a SIGINT was recieve we need to inject a
 90 |                 # KeyboardInterrupt exception into the running coroutine.
 91 |                 try:
 92 |                     coro.throw(KeyboardInterrupt)
 93 |                 except StopIteration as err:
 94 |                     # StopIteration is how coroutines signal their return values.
 95 |                     # If the exception was raised, we treat the argument as the
 96 |                     # return value of the function.
 97 |                     async with cleanup_tasks(*done, *pending):
 98 |                         return cast(TReturn, err.value)
 99 |                 except BaseException:
100 |                     raise
101 |             else:
102 |                 raise Exception("Code path should not be reachable")
103 | 
104 | 
105 | async def _do_async_fn(
106 |     async_fn: TAsyncFn,
107 |     args: Sequence[Any],
108 |     to_parent: BinaryIO,
109 |     loop: asyncio.AbstractEventLoop,
110 | ) -> TReturn:
111 |     # state: STARTED
112 |     update_state(to_parent, State.STARTED)
113 | 
114 |     # A Future that will be set if any of the SHUTDOWN_SIGNALS signals are
115 |     # received causing _do_async_fn to raise a SystemExit
116 |     system_exit_signum: 'asyncio.Future[int]' = asyncio.Future()
117 | 
118 |     # setup signal handlers.
119 |     for signum in SHUTDOWN_SIGNALS:
120 |         loop.add_signal_handler(
121 |             signum.value,
122 |             system_exit_signum.set_result,
123 |             signum,
124 |         )
125 | 
126 |     # Install a signal handler to set an asyncio.Event upon receiving a SIGINT
127 |     got_SIGINT = asyncio.Event()
128 |     loop.add_signal_handler(
129 |         signal.SIGINT,
130 |         got_SIGINT.set,
131 |     )
132 | 
133 |     # state: EXECUTING
134 |     update_state(to_parent, State.EXECUTING)
135 | 
136 |     # First we need to generate a coroutine.  We need this so we can throw
137 |     # exceptions into the running coroutine to allow it to handle keyboard
138 |     # interrupts.
139 |     async_fn_coro: Coroutine[Any, Any, TReturn] = async_fn(*args)
140 | 
141 |     # The coroutine is then given to `_handle_coro` which waits for either the
142 |     # coroutine to finish, returning the result, or for a SIGINT signal at
143 |     # which point injects a `KeyboardInterrupt` into the running coroutine.
144 |     async_fn_task: 'asyncio.Future[TReturn]' = asyncio.ensure_future(
145 |         _handle_coro(async_fn_coro, got_SIGINT),
146 |     )
147 | 
148 |     # Now we wait for either a result from the coroutine or a SIGTERM which
149 |     # triggers immediate cancellation of the running coroutine.
150 |     done, pending = await asyncio.wait(
151 |         (async_fn_task, system_exit_signum),
152 |         return_when=asyncio.FIRST_COMPLETED,
153 |     )
154 | 
155 |     # We prioritize the `SystemExit` case.
156 |     async with cleanup_tasks(*done, *pending):
157 |         if system_exit_signum.done():
158 |             raise SystemExit(await system_exit_signum)
159 |         elif async_fn_task.done():
160 |             return await async_fn_task
161 |         else:
162 |             raise Exception("unreachable")
163 | 
164 | 
165 | def _run_on_asyncio(async_fn: TAsyncFn, args: Sequence[Any], to_parent: BinaryIO) -> None:
166 |     loop = asyncio.get_event_loop()
167 |     try:
168 |         result: Any = loop.run_until_complete(_do_async_fn(async_fn, args, to_parent, loop))
169 |     except BaseException:
170 |         exc_type, exc_value, exc_tb = sys.exc_info()
171 |         # `mypy` thinks that `exc_value` and `exc_tb` are `Optional[..]` types
172 |         if exc_type is asyncio.CancelledError:
173 |             exc_value = ChildCancelled(*exc_value.args)  # type: ignore
174 |         remote_exc = RemoteException(exc_value, exc_tb)  # type: ignore
175 |         finished_payload = pickle_value(remote_exc)
176 |         raise
177 |     else:
178 |         finished_payload = pickle_value(result)
179 |     finally:
180 |         update_state_finished(to_parent, finished_payload)
181 | 
182 | 
183 | def run_process(runner: TEngineRunner, fd_read: int, fd_write: int) -> None:
184 |     """
185 |     Run the child process.
186 | 
187 |     This communicates the status of the child process back to the parent
188 |     process over the given file descriptor, runs the coroutine, handles error
189 |     cases, and transmits the result back to the parent process.
190 |     """
191 |     # state: INITIALIZING (default initial state)
192 |     with os.fdopen(fd_write, "wb") as to_parent:
193 |         # state: INITIALIZED
194 |         update_state_initialized(to_parent)
195 | 
196 |         with os.fdopen(fd_read, "rb", closefd=True) as from_parent:
197 |             # state: WAIT_EXEC_DATA
198 |             update_state(to_parent, State.WAIT_EXEC_DATA)
199 |             async_fn, args = receive_pickled_value(from_parent)
200 | 
201 |         # state: BOOTING
202 |         update_state(to_parent, State.BOOTING)
203 | 
204 |         try:
205 |             runner(async_fn, args, to_parent)
206 |         except KeyboardInterrupt:
207 |             code = 2
208 |         except SystemExit as err:
209 |             code = err.args[0]
210 |         except BaseException:
211 |             logger.exception("%s raised an unexpected exception", async_fn)
212 |             code = 1
213 |         else:
214 |             code = 0
215 |         finally:
216 |             sys.exit(code)
217 | 
218 | 
219 | #
220 | # CLI invocation for subprocesses
221 | #
222 | parser = argparse.ArgumentParser(description="asyncio-run-in-process")
223 | parser.add_argument(
224 |     "--fd-read",
225 |     type=int,
226 |     required=True,
227 |     help=(
228 |         "The file descriptor that the child process can use to read data that "
229 |         "has been written by the parent process"
230 |     ),
231 | )
232 | parser.add_argument(
233 |     "--fd-write",
234 |     type=int,
235 |     required=True,
236 |     help=(
237 |         "The file descriptor that the child process can use for writing data "
238 |         "meant to be read by the parent process"
239 |     ),
240 | )
241 | 
242 | 
243 | if __name__ == "__main__":
244 |     args = parser.parse_args()
245 |     run_process(
246 |         runner=_run_on_asyncio,
247 |         fd_read=args.fd_read,
248 |         fd_write=args.fd_write,
249 |     )
250 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/_child_trio.py:
--------------------------------------------------------------------------------
 1 | import signal
 2 | import sys
 3 | from typing import (
 4 |     Any,
 5 |     AsyncIterator,
 6 |     Awaitable,
 7 |     BinaryIO,
 8 |     Callable,
 9 |     Sequence,
10 | )
11 | 
12 | import trio
13 | import trio_typing
14 | 
15 | from ._utils import (
16 |     RemoteException,
17 |     pickle_value,
18 | )
19 | from .abc import (
20 |     TAsyncFn,
21 | )
22 | from .state import (
23 |     State,
24 |     update_state,
25 |     update_state_finished,
26 | )
27 | from .typing import (
28 |     TReturn,
29 | )
30 | 
31 | SHUTDOWN_SIGNALS = {signal.SIGTERM}
32 | 
33 | 
34 | async def _do_monitor_signals(signal_aiter: AsyncIterator[int]) -> None:
35 |     async for signum in signal_aiter:
36 |         raise SystemExit(signum)
37 | 
38 | 
39 | @trio_typing.takes_callable_and_args
40 | async def _do_async_fn(
41 |     async_fn: Callable[..., Awaitable[TReturn]],
42 |     args: Sequence[Any],
43 |     to_parent: BinaryIO,
44 | ) -> TReturn:
45 |     with trio.open_signal_receiver(*SHUTDOWN_SIGNALS) as signal_aiter:
46 |         # state: STARTED
47 |         update_state(to_parent, State.STARTED)
48 | 
49 |         async with trio.open_nursery() as nursery:
50 |             nursery.start_soon(_do_monitor_signals, signal_aiter)
51 | 
52 |             # state: EXECUTING
53 |             update_state(to_parent, State.EXECUTING)
54 | 
55 |             result = await async_fn(*args)
56 | 
57 |             nursery.cancel_scope.cancel()
58 |         return result
59 | 
60 | 
61 | def _run_on_trio(async_fn: TAsyncFn, args: Sequence[Any], to_parent: BinaryIO) -> None:
62 |     try:
63 |         result = trio.run(_do_async_fn, async_fn, args, to_parent)
64 |     except BaseException:
65 |         _, exc_value, exc_tb = sys.exc_info()
66 |         # `mypy` thinks that `exc_value` and `exc_tb` are `Optional[..]` types
67 |         remote_exc = RemoteException(exc_value, exc_tb)  # type: ignore
68 |         finished_payload = pickle_value(remote_exc)
69 |         raise
70 |     else:
71 |         finished_payload = pickle_value(result)
72 |     finally:
73 |         update_state_finished(to_parent, finished_payload)
74 | 
75 | 
76 | if __name__ == "__main__":
77 |     from asyncio_run_in_process._child import parser, run_process
78 |     args = parser.parse_args()
79 |     run_process(
80 |         runner=_run_on_trio,
81 |         fd_read=args.fd_read,
82 |         fd_write=args.fd_write,
83 |     )
84 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/_utils.py:
--------------------------------------------------------------------------------
  1 | import asyncio
  2 | import io
  3 | import os
  4 | import sys
  5 | import traceback
  6 | from types import (
  7 |     TracebackType,
  8 | )
  9 | from typing import (
 10 |     Any,
 11 |     AsyncContextManager,
 12 |     AsyncIterator,
 13 |     BinaryIO,
 14 |     Tuple,
 15 | )
 16 | 
 17 | from async_generator import (
 18 |     asynccontextmanager,
 19 | )
 20 | import cloudpickle
 21 | 
 22 | from asyncio_run_in_process.exceptions import (
 23 |     UnpickleableValue,
 24 | )
 25 | 
 26 | 
 27 | def get_subprocess_command(child_r: int, child_w: int, use_trio: bool) -> Tuple[str, ...]:
 28 |     if use_trio:
 29 |         from . import _child_trio as child_runner
 30 |     else:
 31 |         from . import _child as child_runner  # type: ignore
 32 | 
 33 |     return (
 34 |         sys.executable,
 35 |         "-m",
 36 |         child_runner.__name__,
 37 |         "--fd-read",
 38 |         str(child_r),
 39 |         "--fd-write",
 40 |         str(child_w),
 41 |     )
 42 | 
 43 | 
 44 | def pickle_value(value: Any) -> bytes:
 45 |     serialized_value = cloudpickle.dumps(value)
 46 |     # mypy doesn't recognize that this line produces a bytes type.
 47 |     return len(serialized_value).to_bytes(4, 'big') + serialized_value  # type: ignore
 48 | 
 49 | 
 50 | def read_exactly(stream: BinaryIO, num_bytes: int) -> bytes:
 51 |     buffer = io.BytesIO()
 52 |     bytes_remaining = num_bytes
 53 |     while bytes_remaining > 0:
 54 |         data = stream.read(bytes_remaining)
 55 |         if data == b"":
 56 |             raise ConnectionError("Got end of stream")
 57 |         buffer.write(data)
 58 |         bytes_remaining -= len(data)
 59 | 
 60 |     return buffer.getvalue()
 61 | 
 62 | 
 63 | def receive_pickled_value(stream: BinaryIO) -> Any:
 64 |     len_bytes = read_exactly(stream, 4)
 65 |     serialized_len = int.from_bytes(len_bytes, "big")
 66 |     serialized_result = read_exactly(stream, serialized_len)
 67 |     try:
 68 |         return cloudpickle.loads(serialized_result)
 69 |     except BaseException as e:
 70 |         raise UnpickleableValue(*e.args) from e
 71 | 
 72 | 
 73 | class RemoteTraceback(Exception):
 74 | 
 75 |     def __init__(self, tb: str) -> None:
 76 |         self.tb = tb
 77 | 
 78 |     def __str__(self) -> str:
 79 |         return self.tb
 80 | 
 81 | 
 82 | class RemoteException(Exception):
 83 |     def __init__(self, exc: BaseException, tb: TracebackType) -> None:
 84 |         self.tb = (
 85 |             f'\n""" (exception from process: {os.getpid()})\n'
 86 |             f"{''.join(traceback.format_exception(type(exc), exc, tb))}"
 87 |             '"""'
 88 |         )
 89 |         self.exc = exc
 90 | 
 91 |     def __reduce__(self) -> Any:
 92 |         """
 93 |         Trick the `pickle` module into recreating this as the original
 94 |         exception when the value gets unpickled.
 95 |         """
 96 |         return rebuild_exc, (self.exc, self.tb)
 97 | 
 98 | 
 99 | def rebuild_exc(exc, tb):  # type: ignore
100 |     exc.__cause__ = RemoteTraceback(tb)
101 |     return exc
102 | 
103 | 
104 | def cleanup_tasks(*tasks: 'asyncio.Future[Any]') -> AsyncContextManager[None]:
105 |     """
106 |     Context manager that ensures that all tasks are properly cancelled and awaited.
107 | 
108 |     The order in which tasks are cleaned is such that the first task will be
109 |     the last to be cancelled/awaited.
110 | 
111 |     This function **must** be called with at least one task.
112 |     """
113 |     return _cleanup_tasks(*tasks)
114 | 
115 | 
116 | @asynccontextmanager
117 | async def _cleanup_tasks(task: 'asyncio.Future[Any]',
118 |                          *tasks: 'asyncio.Future[Any]',
119 |                          ) -> AsyncIterator[None]:
120 |     try:
121 |         if tasks:
122 |             async with cleanup_tasks(*tasks):
123 |                 yield
124 |         else:
125 |             yield
126 |     finally:
127 |         if not task.done():
128 |             task.cancel()
129 | 
130 |         try:
131 |             await task
132 |         except asyncio.CancelledError:
133 |             pass
134 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/abc.py:
--------------------------------------------------------------------------------
  1 | from abc import (
  2 |     ABC,
  3 |     abstractmethod,
  4 | )
  5 | import signal
  6 | from typing import (
  7 |     Any,
  8 |     BinaryIO,
  9 |     Callable,
 10 |     Coroutine,
 11 |     Generic,
 12 |     Optional,
 13 |     Sequence,
 14 |     TypeVar,
 15 | )
 16 | 
 17 | from .state import (
 18 |     State,
 19 | )
 20 | from .typing import (
 21 |     TReturn,
 22 | )
 23 | 
 24 | TAsyncFn = TypeVar("TAsyncFn", bound=Callable[..., Coroutine[Any, Any, TReturn]])
 25 | TEngineRunner = TypeVar("TEngineRunner", bound=Callable[[TAsyncFn, Sequence[Any], BinaryIO], None])
 26 | 
 27 | 
 28 | class ProcessAPI(ABC, Generic[TReturn]):
 29 |     sub_proc_payload: bytes
 30 | 
 31 |     #
 32 |     # State
 33 |     #
 34 |     @property
 35 |     @abstractmethod
 36 |     def state(self) -> State:
 37 |         ...
 38 | 
 39 |     @abstractmethod
 40 |     async def update_state(self, value: State) -> None:
 41 |         ...
 42 | 
 43 |     @abstractmethod
 44 |     async def wait_for_state(self, state: State) -> None:
 45 |         ...
 46 | 
 47 |     #
 48 |     # PID
 49 |     #
 50 |     @property
 51 |     @abstractmethod
 52 |     def pid(self) -> int:
 53 |         ...
 54 | 
 55 |     @pid.setter
 56 |     def pid(self, value: int) -> None:
 57 |         raise NotImplementedError
 58 | 
 59 |     @abstractmethod
 60 |     async def wait_pid(self) -> int:
 61 |         ...
 62 | 
 63 |     #
 64 |     # Return Value
 65 |     #
 66 |     @property
 67 |     @abstractmethod
 68 |     def return_value(self) -> TReturn:
 69 |         ...
 70 | 
 71 |     @return_value.setter
 72 |     def return_value(self, value: TReturn) -> None:
 73 |         raise NotImplementedError
 74 | 
 75 |     @abstractmethod
 76 |     async def wait_return_value(self) -> TReturn:
 77 |         ...
 78 | 
 79 |     #
 80 |     # Return Code
 81 |     #
 82 |     @property
 83 |     @abstractmethod
 84 |     def returncode(self) -> int:
 85 |         ...
 86 | 
 87 |     @returncode.setter
 88 |     def returncode(self, value: int) -> None:
 89 |         raise NotImplementedError
 90 | 
 91 |     @abstractmethod
 92 |     async def wait_returncode(self) -> int:
 93 |         ...
 94 | 
 95 |     #
 96 |     # Error
 97 |     #
 98 |     @property
 99 |     @abstractmethod
100 |     def error(self) -> Optional[BaseException]:
101 |         ...
102 | 
103 |     @error.setter
104 |     def error(self, value: BaseException) -> None:
105 |         raise NotImplementedError
106 | 
107 |     @abstractmethod
108 |     async def wait_error(self) -> Optional[BaseException]:
109 |         ...
110 | 
111 |     #
112 |     # Result
113 |     #
114 |     @abstractmethod
115 |     def get_result_or_raise(self) -> TReturn:
116 |         ...
117 | 
118 |     @abstractmethod
119 |     async def wait_result_or_raise(self) -> TReturn:
120 |         ...
121 | 
122 |     #
123 |     # Lifecycle management APIs
124 |     #
125 |     @abstractmethod
126 |     async def wait(self) -> None:
127 |         ...
128 | 
129 |     @abstractmethod
130 |     async def kill(self) -> None:
131 |         ...
132 | 
133 |     @abstractmethod
134 |     def terminate(self) -> None:
135 |         ...
136 | 
137 |     @abstractmethod
138 |     def send_signal(self, sig: signal.Signals) -> None:
139 |         ...
140 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/constants.py:
--------------------------------------------------------------------------------
 1 | # Default number of seconds given to a child to reach the EXECUTING state before we yield in
 2 | # open_in_process(). Can be overwritten by the ASYNCIO_RUN_IN_PROCESS_STARTUP_TIMEOUT
 3 | # environment variable.
 4 | STARTUP_TIMEOUT_SECONDS = 5
 5 | 
 6 | # The number of seconds that are given to a child process to exit after the
 7 | # parent process gets a KeyboardInterrupt/SIGINT-signal and sends a `SIGINT` to
 8 | # the child process.
 9 | SIGINT_TIMEOUT_SECONDS = 2
10 | 
11 | # The number of seconds that are givent to a child process to exit after the
12 | # parent process gets an `asyncio.CancelledError` which results in sending a
13 | # `SIGTERM` to the child process.
14 | SIGTERM_TIMEOUT_SECONDS = 2
15 | 
16 | # Default maximum number of process that can be running at the same time. Can be overwritten via
17 | # the ASYNCIO_RUN_IN_PROCESS_MAX_PROCS environment variable.
18 | MAX_PROCESSES = 16
19 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/exceptions.py:
--------------------------------------------------------------------------------
 1 | class BaseRunInProcessException(Exception):
 2 |     pass
 3 | 
 4 | 
 5 | class ProcessKilled(BaseRunInProcessException):
 6 |     pass
 7 | 
 8 | 
 9 | class InvalidState(BaseRunInProcessException):
10 |     pass
11 | 
12 | 
13 | class ChildCancelled(BaseRunInProcessException):
14 |     pass
15 | 
16 | 
17 | class UnpickleableValue(BaseRunInProcessException):
18 |     pass
19 | 
20 | 
21 | class InvalidDataFromChild(BaseRunInProcessException):
22 |     """
23 |     The child's return value cannot be unpickled.
24 | 
25 |     This seems to happen only when the child raises a custom exception whose constructor has more
26 |     than one required argument: https://github.com/ethereum/asyncio-run-in-process/issues/28
27 |     """
28 |     pass
29 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/main.py:
--------------------------------------------------------------------------------
  1 | import asyncio
  2 | import concurrent
  3 | import logging
  4 | import os
  5 | import signal
  6 | import sys
  7 | from typing import (
  8 |     TYPE_CHECKING,
  9 |     Any,
 10 |     AsyncContextManager,
 11 |     AsyncIterator,
 12 |     Callable,
 13 |     Optional,
 14 | )
 15 | 
 16 | from async_generator import (
 17 |     asynccontextmanager,
 18 | )
 19 | 
 20 | from . import (
 21 |     constants,
 22 | )
 23 | from ._utils import (
 24 |     cleanup_tasks,
 25 |     get_subprocess_command,
 26 |     read_exactly,
 27 |     receive_pickled_value,
 28 | )
 29 | from .abc import (
 30 |     ProcessAPI,
 31 | )
 32 | from .exceptions import (
 33 |     InvalidDataFromChild,
 34 |     InvalidState,
 35 |     UnpickleableValue,
 36 | )
 37 | from .process import (
 38 |     Process,
 39 | )
 40 | from .state import (
 41 |     State,
 42 | )
 43 | from .typing import (
 44 |     TReturn,
 45 | )
 46 | 
 47 | if TYPE_CHECKING:
 48 |     from typing import Tuple  # noqa: F401
 49 |     from .typing import SubprocessKwargs  # noqa: F401
 50 | 
 51 | 
 52 | logger = logging.getLogger("asyncio_run_in_process")
 53 | _executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
 54 | 
 55 | 
 56 | def _get_executor() -> concurrent.futures.ThreadPoolExecutor:
 57 |     global _executor
 58 |     if _executor is None:
 59 |         max_procs = int(os.getenv('ASYNCIO_RUN_IN_PROCESS_MAX_PROCS', constants.MAX_PROCESSES))
 60 |         _executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_procs)
 61 |     return _executor
 62 | 
 63 | 
 64 | async def _monitor_sub_proc(
 65 |     proc: ProcessAPI[TReturn], sub_proc: asyncio.subprocess.Process, parent_w: int
 66 | ) -> None:
 67 |     logger.debug("starting subprocess to run %s", proc)
 68 | 
 69 |     await proc.wait_pid()
 70 |     if proc.pid != sub_proc.pid:
 71 |         raise Exception("Process id mismatch.  This should not be possible")
 72 |     logger.debug("subprocess for %s started.  pid=%d", proc, sub_proc.pid)
 73 | 
 74 |     # we write the execution data immediately without waiting for the
 75 |     # `WAIT_EXEC_DATA` state to ensure that the child process doesn't have
 76 |     # to wait for that data due to the round trip times between processes.
 77 |     logger.debug("writing execution data for %s over fd=%d", proc, parent_w)
 78 |     # pass the child process the serialized `async_fn` and `args`
 79 | 
 80 |     with os.fdopen(parent_w, "wb") as to_child:
 81 |         to_child.write(proc.sub_proc_payload)
 82 |         to_child.flush()
 83 | 
 84 |     await proc.wait_for_state(State.WAIT_EXEC_DATA)
 85 |     logger.debug("child process %s (pid=%d) waiting for exec data", proc, sub_proc.pid)
 86 | 
 87 |     await proc.wait_for_state(State.STARTED)
 88 |     logger.debug("waiting for process %s (pid=%d) to finish", proc, sub_proc.pid)
 89 | 
 90 |     await sub_proc.wait()
 91 | 
 92 |     proc.returncode = sub_proc.returncode
 93 |     logger.debug("process %s (pid=%d) finished: returncode=%d", proc, sub_proc.pid, proc.returncode)
 94 | 
 95 | 
 96 | async def _relay_signals(
 97 |     proc: ProcessAPI[Any],
 98 |     queue: 'asyncio.Queue[signal.Signals]',
 99 | ) -> None:
100 |     if proc.state.is_before(State.EXECUTING):
101 |         # If the process has not reached the state where the child process
102 |         # can properly handle the signal, give it a moment to reach the
103 |         # `EXECUTING` stage.
104 |         logger.debug("_relay_signals(): Waiting for %s to reach EXECUTING state", proc)
105 |         await proc.wait_for_state(State.EXECUTING)
106 |     elif proc.state is State.FINISHED:
107 |         logger.debug("_relay_signals(): %s is already finished, exiting %s", proc)
108 |         return
109 | 
110 |     logger.debug("_relay_signals(): Waiting for signals to relay to %s", proc)
111 |     while True:
112 |         signum = await queue.get()
113 |         logger.debug("relaying signal %s to child process %s", signum, proc)
114 |         proc.send_signal(signum)
115 | 
116 | 
117 | async def _monitor_state(
118 |     proc: ProcessAPI[TReturn],
119 |     parent_read_fd: int,
120 |     child_write_fd: int,
121 |     loop: asyncio.AbstractEventLoop,
122 | ) -> None:
123 |     with os.fdopen(parent_read_fd, "rb", closefd=True) as from_child:
124 |         for expected_state in State:
125 |             if proc.state is not expected_state:
126 |                 raise InvalidState(
127 |                     f"Process in state {proc.state} but expected state {expected_state}"
128 |                 )
129 | 
130 |             next_expected_state = State(proc.state + 1)
131 |             logger.debug(
132 |                 "Waiting for next expected state (%s) from child (%s)", next_expected_state, proc)
133 |             try:
134 |                 child_state_as_byte = await loop.run_in_executor(
135 |                     _get_executor(), read_exactly, from_child, 1)
136 |             except asyncio.CancelledError:
137 |                 # When the sub process is sent a SIGKILL, the write end of the pipe used in
138 |                 # read_exactly is never closed and the thread above attempting to read from it
139 |                 # will prevent us from leaving this fdopen() context, so we need to close the
140 |                 # write end ourselves to ensure the read_exactly() returns.
141 |                 logger.debug(
142 |                     "_monitor_state() cancelled while waiting data from child, closing "
143 |                     "child_write_fd to ensure we exit")
144 |                 os.close(child_write_fd)
145 |                 raise
146 | 
147 |             try:
148 |                 child_state = State(ord(child_state_as_byte))
149 |             except TypeError:
150 |                 raise InvalidState(f"Child sent state: {child_state_as_byte.hex()}")
151 | 
152 |             if not proc.state.is_next(child_state):
153 |                 raise InvalidState(
154 |                     f"Invalid state transition: {proc.state} -> {child_state}"
155 |                 )
156 | 
157 |             logger.debug("Got next state (%s) from child (%s)", child_state, proc)
158 | 
159 |             if child_state is State.FINISHED:
160 |                 # For the FINISHED state we delay updating the state until we also
161 |                 # have a return value.
162 |                 break
163 |             elif child_state is State.INITIALIZED:
164 |                 # For the INITIALIZED state we expect an additional payload of the
165 |                 # process id.  The process ID is gotten via this mechanism to
166 |                 # prevent the need for ugly sleep based code in
167 |                 # `_monitor_sub_proc`.
168 |                 pid_bytes = await loop.run_in_executor(_get_executor(), read_exactly, from_child, 4)
169 |                 proc.pid = int.from_bytes(pid_bytes, 'big')
170 | 
171 |             await proc.update_state(child_state)
172 |             logger.debug(
173 |                 "Updated process %s state %s -> %s",
174 |                 proc,
175 |                 expected_state.name,
176 |                 child_state.name,
177 |             )
178 | 
179 |         # This is mostly a sanity check but it ensures that we don't try to get a result from a
180 |         # process which hasn't finished.
181 |         if child_state is not State.FINISHED:
182 |             raise InvalidState(f"Invalid final state: {proc.state}")
183 | 
184 |         logger.debug("Waiting for result from %s", proc)
185 |         try:
186 |             result = await loop.run_in_executor(_get_executor(), receive_pickled_value, from_child)
187 |         except UnpickleableValue as e:
188 |             result = InvalidDataFromChild(
189 |                 "Unable to unpickle data from child. This may be a custom exception class; see "
190 |                 "https://github.com/ethereum/asyncio-run-in-process/issues/28 for more details. "
191 |                 "Original error: %s" % e.args)
192 |             result.__cause__ = e
193 |         except asyncio.CancelledError:
194 |             # See comment above as to why we need to do this.
195 |             logger.debug(
196 |                 "_monitor_state() cancelled while waiting data from child, closing "
197 |                 "child_write_fd to ensure we exit")
198 |             os.close(child_write_fd)
199 |             raise
200 | 
201 |     logger.debug("Waiting for returncode from %s", proc)
202 |     await proc.wait_returncode()
203 | 
204 |     if isinstance(result, InvalidDataFromChild):
205 |         # When we're unable to unpickle the result from the child, we need to force an error
206 |         # so that the InvalidDataFromChild is raised in .wait_result_or_raise().
207 |         proc.error = result
208 |         proc.returncode = -99
209 |     elif proc.returncode == 0:
210 |         proc.return_value = result
211 |     else:
212 |         proc.error = result
213 | 
214 |     await proc.update_state(child_state)
215 |     logger.debug(
216 |         "Updated process %s state %s -> %s",
217 |         proc,
218 |         expected_state.name,
219 |         child_state.name,
220 |     )
221 | 
222 | 
223 | # SIGINT isn't included here because it's handled by catching the
224 | # `KeyboardInterrupt` exception.
225 | RELAY_SIGNALS = (signal.SIGTERM, signal.SIGHUP)
226 | 
227 | 
228 | def open_in_process(
229 |     async_fn: Callable[..., TReturn],
230 |     *args: Any,
231 |     loop: asyncio.AbstractEventLoop = None,
232 |     subprocess_kwargs: 'SubprocessKwargs' = None,
233 | ) -> AsyncContextManager[ProcessAPI[TReturn]]:
234 |     return _open_in_process(
235 |         async_fn, *args, loop=loop, subprocess_kwargs=subprocess_kwargs, use_trio=False)
236 | 
237 | 
238 | def open_in_process_with_trio(
239 |     async_fn: Callable[..., TReturn],
240 |     *args: Any,
241 |     subprocess_kwargs: 'SubprocessKwargs' = None,
242 | ) -> AsyncContextManager[ProcessAPI[TReturn]]:
243 |     return _open_in_process(
244 |         async_fn, *args, loop=None, subprocess_kwargs=subprocess_kwargs, use_trio=True)
245 | 
246 | 
247 | def _update_subprocess_kwargs(subprocess_kwargs: Optional['SubprocessKwargs'],
248 |                               child_r: int,
249 |                               child_w: int) -> 'SubprocessKwargs':
250 |     if subprocess_kwargs is None:
251 |         subprocess_kwargs = {}
252 | 
253 |     base_pass_fds = subprocess_kwargs.get('pass_fds', ())
254 |     pass_fds: Tuple[int, ...]
255 | 
256 |     if base_pass_fds is None:
257 |         pass_fds = (child_r, child_w)
258 |     else:
259 |         pass_fds = tuple(set(base_pass_fds).union((child_r, child_w)))
260 | 
261 |     updated_kwargs = subprocess_kwargs.copy()
262 |     updated_kwargs['pass_fds'] = pass_fds
263 | 
264 |     return updated_kwargs
265 | 
266 | 
267 | @asynccontextmanager
268 | async def _open_in_process(
269 |     async_fn: Callable[..., TReturn],
270 |     *args: Any,
271 |     loop: asyncio.AbstractEventLoop = None,
272 |     subprocess_kwargs: 'SubprocessKwargs' = None,
273 |     use_trio: bool = False,
274 | ) -> AsyncIterator[ProcessAPI[TReturn]]:
275 |     if use_trio and loop is not None:
276 |         raise ValueError("If using trio, cannot specify a loop")
277 | 
278 |     proc: Process[TReturn] = Process(async_fn, args)
279 | 
280 |     parent_r, child_w = os.pipe()
281 |     child_r, parent_w = os.pipe()
282 | 
283 |     command = get_subprocess_command(child_r, child_w, use_trio)
284 | 
285 |     sub_proc = await asyncio.create_subprocess_exec(
286 |         *command,
287 |         **_update_subprocess_kwargs(subprocess_kwargs, child_r, child_w),
288 |     )
289 |     if loop is None:
290 |         loop = asyncio.get_event_loop()
291 | 
292 |     signal_queue: asyncio.Queue[signal.Signals] = asyncio.Queue()
293 | 
294 |     for signum in RELAY_SIGNALS:
295 |         loop.add_signal_handler(
296 |             signum,
297 |             signal_queue.put_nowait,
298 |             signum,
299 |         )
300 | 
301 |     # Monitoring
302 |     monitor_sub_proc_task = asyncio.ensure_future(_monitor_sub_proc(proc, sub_proc, parent_w))
303 |     relay_signals_task = asyncio.ensure_future(_relay_signals(proc, signal_queue))
304 |     monitor_state_task = asyncio.ensure_future(_monitor_state(proc, parent_r, child_w, loop))
305 | 
306 |     startup_timeout = int(
307 |         os.getenv('ASYNCIO_RUN_IN_PROCESS_STARTUP_TIMEOUT', constants.STARTUP_TIMEOUT_SECONDS))
308 |     async with cleanup_tasks(monitor_sub_proc_task, relay_signals_task, monitor_state_task):
309 |         try:
310 |             await asyncio.wait_for(proc.wait_pid(), timeout=startup_timeout)
311 |         except asyncio.TimeoutError:
312 |             sub_proc.kill()
313 |             raise asyncio.TimeoutError(
314 |                 f"{proc} took more than {startup_timeout} seconds to start up")
315 | 
316 |         logger.debug(
317 |             "Got pid %d for %s, waiting for it to reach EXECUTING state before yielding",
318 |             proc.pid, proc)
319 |         # Wait until the child process has reached the EXECUTING
320 |         # state before yielding the context.  This ensures that any
321 |         # calls to things like `terminate` or `kill` will be handled
322 |         # properly in the child process.
323 |         #
324 |         # The timeout ensures that if something is fundamentally wrong
325 |         # with the subprocess we don't hang indefinitely.
326 |         try:
327 |             logger.debug("Waiting for proc pid=%d to reach EXECUTING state", proc.pid)
328 |             await asyncio.wait_for(proc.wait_for_state(State.EXECUTING), timeout=startup_timeout)
329 |         except asyncio.TimeoutError:
330 |             sub_proc.kill()
331 |             raise asyncio.TimeoutError(
332 |                 f"{proc} took more than {startup_timeout} seconds to start up")
333 | 
334 |         try:
335 |             try:
336 |                 yield proc
337 |             except KeyboardInterrupt as err:
338 |                 # If a keyboard interrupt is encountered relay it to the
339 |                 # child process and then give it a moment to cleanup before
340 |                 # re-raising
341 |                 logger.debug("Relaying SIGINT to pid=%d", sub_proc.pid)
342 |                 try:
343 |                     proc.send_signal(signal.SIGINT)
344 |                     try:
345 |                         await asyncio.wait_for(
346 |                             proc.wait(), timeout=constants.SIGINT_TIMEOUT_SECONDS)
347 |                     except asyncio.TimeoutError:
348 |                         logger.debug(
349 |                             "Timed out waiting for pid=%d to exit after relaying SIGINT",
350 |                             sub_proc.pid,
351 |                         )
352 |                 except BaseException:
353 |                     logger.exception(
354 |                         "Unexpected error when terminating child; pid=%d", sub_proc.pid)
355 |                 finally:
356 |                     raise err
357 |             except asyncio.CancelledError as err:
358 |                 # Send the child a SIGINT and wait SIGINT_TIMEOUT_SECONDS for it to terminate. If
359 |                 # that times out, send a SIGTERM and wait SIGTERM_TIMEOUT_SECONDS before
360 |                 # re-raising.
361 |                 logger.debug(
362 |                     "Got CancelledError while running subprocess pid=%d.  Sending SIGINT.",
363 |                     sub_proc.pid,
364 |                 )
365 |                 try:
366 |                     proc.send_signal(signal.SIGINT)
367 |                     try:
368 |                         await asyncio.wait_for(
369 |                             proc.wait(), timeout=constants.SIGINT_TIMEOUT_SECONDS)
370 |                     except asyncio.TimeoutError:
371 |                         logger.debug(
372 |                             "Timed out waiting for pid=%d to exit after SIGINT, sending SIGTERM",
373 |                             sub_proc.pid,
374 |                         )
375 |                         proc.terminate()
376 |                         try:
377 |                             await asyncio.wait_for(
378 |                                 proc.wait(), timeout=constants.SIGTERM_TIMEOUT_SECONDS)
379 |                         except asyncio.TimeoutError:
380 |                             logger.debug(
381 |                                 "Timed out waiting for pid=%d to exit after SIGTERM", sub_proc.pid)
382 |                 except BaseException:
383 |                     logger.exception(
384 |                         "Unexpected error when terminating child; pid=%d", sub_proc.pid)
385 |                 finally:
386 |                     raise err
387 |             else:
388 |                 # In the case that the yielded context block exits without an
389 |                 # error we wait for the process to finish naturally.  This can
390 |                 # hang indefinitely.
391 |                 logger.debug(
392 |                     "Waiting for %s (pid=%d) to finish naturally, this can hang forever",
393 |                     proc,
394 |                     proc.pid,
395 |                 )
396 |                 await proc.wait()
397 |         finally:
398 |             if sub_proc.returncode is None:
399 |                 # If the process has not returned at this stage we need to hard
400 |                 # kill it to prevent it from hanging.
401 |                 logger.warning(
402 |                     "Child process pid=%d failed to exit cleanly.  Sending SIGKILL",
403 |                     sub_proc.pid,
404 |                     # The `any` call is to include a stacktrace if this
405 |                     # happened due to an exception but to omit it if this is
406 |                     # somehow happening outside of an exception context.
407 |                     exc_info=any(sys.exc_info()),
408 |                 )
409 |                 sub_proc.kill()
410 | 
411 | 
412 | async def run_in_process(async_fn: Callable[..., TReturn],
413 |                          *args: Any,
414 |                          loop: asyncio.AbstractEventLoop = None,
415 |                          subprocess_kwargs: 'SubprocessKwargs' = None) -> TReturn:
416 |     proc_ctx = open_in_process(
417 |         async_fn,
418 |         *args,
419 |         loop=loop,
420 |         subprocess_kwargs=subprocess_kwargs,
421 |     )
422 |     async with proc_ctx as proc:
423 |         await proc.wait()
424 |     return proc.get_result_or_raise()
425 | 
426 | 
427 | async def run_in_process_with_trio(async_fn: Callable[..., TReturn],
428 |                                    *args: Any,
429 |                                    subprocess_kwargs: 'SubprocessKwargs' = None) -> TReturn:
430 |     proc_ctx = open_in_process_with_trio(
431 |         async_fn, *args, subprocess_kwargs=subprocess_kwargs)
432 |     async with proc_ctx as proc:
433 |         await proc.wait()
434 |     return proc.get_result_or_raise()
435 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/process.py:
--------------------------------------------------------------------------------
  1 | import asyncio
  2 | import os
  3 | import signal
  4 | from typing import (
  5 |     Any,
  6 |     Callable,
  7 |     Optional,
  8 |     Sequence,
  9 | )
 10 | 
 11 | from ._utils import (
 12 |     pickle_value,
 13 | )
 14 | from .abc import (
 15 |     ProcessAPI,
 16 | )
 17 | from .exceptions import (
 18 |     ProcessKilled,
 19 | )
 20 | from .state import (
 21 |     State,
 22 | )
 23 | from .typing import (
 24 |     TReturn,
 25 | )
 26 | 
 27 | 
 28 | class Process(ProcessAPI[TReturn]):
 29 |     _pid: Optional[int] = None
 30 |     _returncode: Optional[int] = None
 31 |     _return_value: TReturn
 32 |     _error: Optional[BaseException] = None
 33 |     _state: State = State.INITIALIZING
 34 | 
 35 |     sub_proc_payload: bytes
 36 | 
 37 |     def __init__(
 38 |         self, async_fn: Callable[..., TReturn], args: Sequence[Any]
 39 |     ) -> None:
 40 |         self._async_fn = async_fn
 41 |         self._args = args
 42 |         self.sub_proc_payload = pickle_value((self._async_fn, self._args))
 43 | 
 44 |         self._has_pid = asyncio.Event()
 45 |         self._has_returncode = asyncio.Event()
 46 |         self._has_return_value = asyncio.Event()
 47 |         self._has_error = asyncio.Event()
 48 |         self._state_changed = asyncio.Condition()
 49 | 
 50 |     def __str__(self) -> str:
 51 |         return f"Process[{self._async_fn}]"
 52 | 
 53 |     #
 54 |     # State
 55 |     #
 56 |     @property
 57 |     def state(self) -> State:
 58 |         """
 59 |         Return the current state of the process.
 60 |         """
 61 |         return self._state
 62 | 
 63 |     async def update_state(self, state: State) -> None:
 64 |         """
 65 |         Update the state of the process.
 66 | 
 67 |         .. warning::
 68 | 
 69 |             This is an internal API and should not be used in user code.
 70 |         """
 71 |         async with self._state_changed:
 72 |             self._state = state
 73 |             self._state_changed.notify_all()
 74 | 
 75 |     async def wait_for_state(self, state: State) -> None:
 76 |         """
 77 |         Block until the process has reached or surpassed the given state.
 78 |         """
 79 |         if self.state.is_on_or_after(state):
 80 |             return
 81 | 
 82 |         # We use a loop since there should be a finite number of possible state
 83 |         # transitions and thus we should arrived at the desired state within a
 84 |         # number of iterations equal to that of the number of possible states.
 85 |         for _ in range(len(State)):
 86 |             async with self._state_changed:
 87 |                 await self._state_changed.wait()
 88 |                 if self.state.is_on_or_after(state):
 89 |                     break
 90 |         else:
 91 |             raise BaseException(
 92 |                 f"This code path should not be reachable since there are a "
 93 |                 f"finite number of state transitions.  Current state is "
 94 |                 f"{self.state}"
 95 |             )
 96 | 
 97 |     #
 98 |     # PID
 99 |     #
100 |     @property
101 |     def pid(self) -> int:
102 |         """
103 |         Return the process id of the process
104 | 
105 |         Raises an `AttributeError` if the process id is not yet available.
106 |         """
107 |         if self._pid is None:
108 |             raise AttributeError("No PID set for process")
109 |         return self._pid
110 | 
111 |     @pid.setter
112 |     def pid(self, value: int) -> None:
113 |         self._pid = value
114 |         self._has_pid.set()
115 | 
116 |     async def wait_pid(self) -> int:
117 |         """
118 |         Block until the process id is available.
119 |         """
120 |         await self._has_pid.wait()
121 |         return self.pid
122 | 
123 |     #
124 |     # Return Value
125 |     #
126 |     @property
127 |     def return_value(self) -> TReturn:
128 |         """
129 |         Return the return value of the proc
130 | 
131 |         Raises an `AttributeError` if the process has not exited.
132 |         """
133 |         if not hasattr(self, "_return_value"):
134 |             raise AttributeError("No return_value set")
135 |         return self._return_value
136 | 
137 |     @return_value.setter
138 |     def return_value(self, value: TReturn) -> None:
139 |         self._return_value = value
140 |         self._has_return_value.set()
141 | 
142 |     async def wait_return_value(self) -> TReturn:
143 |         """
144 |         Block until the return code of the process has been set.
145 | 
146 |         This will block indefinitely if the process exits with an error.
147 |         """
148 |         await self._has_return_value.wait()
149 |         return self.return_value
150 | 
151 |     #
152 |     # Return Code
153 |     #
154 |     @property
155 |     def returncode(self) -> int:
156 |         """
157 |         Return the integer return code of the process.
158 | 
159 |         Raises an `AttributeError` if the process has not exited.
160 |         """
161 |         if self._returncode is None:
162 |             raise AttributeError("No returncode set")
163 |         return self._returncode
164 | 
165 |     @returncode.setter
166 |     def returncode(self, value: int) -> None:
167 |         self._returncode = value
168 |         self._has_returncode.set()
169 | 
170 |     async def wait_returncode(self) -> int:
171 |         """
172 |         Block until the return code of the process has been set.
173 |         """
174 |         await self._has_returncode.wait()
175 |         return self.returncode
176 | 
177 |     #
178 |     # Error
179 |     #
180 |     @property
181 |     def error(self) -> Optional[BaseException]:
182 |         """
183 |         Return the error raised by the process.
184 | 
185 |         Raises an `AttributeError` if the process has not raised an exception.
186 |         """
187 |         if self._error is None and not hasattr(self, "_return_value"):
188 |             raise AttributeError("No error set")
189 |         return self._error
190 | 
191 |     @error.setter
192 |     def error(self, value: BaseException) -> None:
193 |         self._error = value
194 |         self._has_error.set()
195 | 
196 |     async def wait_error(self) -> BaseException:
197 |         """
198 |         Block until the process has an error.
199 | 
200 |         This will block indefinitely if the process does not throw an exception.
201 |         """
202 |         await self._has_error.wait()
203 |         # mypy is unable to tell that `self.error` **must** be non-null in this
204 |         # case.
205 |         return self.error  # type: ignore
206 | 
207 |     #
208 |     # Result
209 |     #
210 |     def get_result_or_raise(self) -> TReturn:
211 |         """
212 |         Return the computed result from the process, raising if it was an exception.
213 | 
214 |         If the process has not finished then raises an `AttributeError`
215 |         """
216 |         if self._error is None and not hasattr(self, "_return_value"):
217 |             raise AttributeError("Process not done")
218 |         elif self._error is not None:
219 |             raise self._error
220 |         elif hasattr(self, "_return_value"):
221 |             return self._return_value
222 |         else:
223 |             raise BaseException("Code path should be unreachable")
224 | 
225 |     async def wait_result_or_raise(self) -> TReturn:
226 |         """
227 |         Block until the process has exited, either returning the return value
228 |         if execution was successful, or raising an exception if it failed
229 |         """
230 |         await self.wait_returncode()
231 | 
232 |         if self.returncode == 0:
233 |             return await self.wait_return_value()
234 |         else:
235 |             raise await self.wait_error()
236 | 
237 |     #
238 |     # Lifecycle management APIs
239 |     #
240 |     async def wait(self) -> None:
241 |         """
242 |         Block until the process has exited.
243 |         """
244 |         await self.wait_returncode()
245 | 
246 |         if self.returncode == 0:
247 |             await self.wait_return_value()
248 |         else:
249 |             await self.wait_error()
250 | 
251 |     async def kill(self) -> None:
252 |         """
253 |         Issue a `SIGKILL` signal to the process.
254 | 
255 |         This immediately transitions the process state to `FINISHED` and sets
256 |         the error to `ProcessKilled`
257 |         """
258 |         self.send_signal(signal.SIGKILL)
259 |         await self.update_state(State.FINISHED)
260 |         self.error = ProcessKilled("Process terminated with SIGKILL")
261 | 
262 |     def terminate(self) -> None:
263 |         """
264 |         Issues a `SIGTERM` to the process.
265 |         """
266 |         self.send_signal(signal.SIGTERM)
267 | 
268 |     def send_signal(self, signum: signal.Signals) -> None:
269 |         """
270 |         Issues the provided signal to the process.
271 |         """
272 |         os.kill(self.pid, signum.value)
273 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/state.py:
--------------------------------------------------------------------------------
 1 | import enum
 2 | import os
 3 | from typing import (
 4 |     BinaryIO,
 5 | )
 6 | 
 7 | 
 8 | class State(enum.IntEnum):
 9 |     """
10 |     Child process lifecycle
11 |     """
12 | 
13 |     INITIALIZING = 0
14 |     INITIALIZED = 1
15 |     WAIT_EXEC_DATA = 2
16 |     BOOTING = 3
17 |     STARTED = 4
18 |     EXECUTING = 5
19 |     FINISHED = 6
20 | 
21 |     def is_next(self, other: "State") -> bool:
22 |         return other == self + 1
23 | 
24 |     def is_on_or_after(self, other: "State") -> bool:
25 |         return self >= other
26 | 
27 |     def is_before(self, other: "State") -> bool:
28 |         return self < other
29 | 
30 | 
31 | def update_state(to_parent: BinaryIO, state: State) -> None:
32 |     to_parent.write(state.value.to_bytes(1, 'big'))
33 |     to_parent.flush()
34 | 
35 | 
36 | def update_state_initialized(to_parent: BinaryIO) -> None:
37 |     payload = State.INITIALIZED.value.to_bytes(1, 'big') + os.getpid().to_bytes(4, 'big')
38 |     to_parent.write(payload)
39 |     to_parent.flush()
40 | 
41 | 
42 | def update_state_finished(to_parent: BinaryIO, finished_payload: bytes) -> None:
43 |     payload = State.FINISHED.value.to_bytes(1, 'big') + finished_payload
44 |     to_parent.write(payload)
45 |     to_parent.flush()
46 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/tools/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ethereum/asyncio-run-in-process/0ebfcea6ce9f0ae50abe7bf98d4c2d3783b4e86b/asyncio_run_in_process/tools/__init__.py


--------------------------------------------------------------------------------
/asyncio_run_in_process/tools/sleep.py:
--------------------------------------------------------------------------------
 1 | from sniffio import (
 2 |     current_async_library,
 3 | )
 4 | 
 5 | 
 6 | async def sleep(duration: float) -> None:
 7 |     if current_async_library() == 'trio':
 8 |         import trio
 9 |         await trio.sleep(duration)
10 |     elif current_async_library() == 'asyncio':
11 |         import asyncio
12 |         await asyncio.sleep(duration)
13 |     else:
14 |         raise Exception("Invariant")
15 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/typing.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | from typing import (
 3 |     IO,
 4 |     TYPE_CHECKING,
 5 |     Any,
 6 |     Callable,
 7 |     Dict,
 8 |     TypeVar,
 9 | )
10 | 
11 | if TYPE_CHECKING:
12 |     #
13 |     # This block ensures that typing_extensions isn't needed during runtime and
14 |     # is only necessary for linting.
15 |     #
16 |     import subprocess
17 |     from typing import Mapping, Optional, Sequence, Union
18 | 
19 |     from .typing_compat import Literal, TypedDict
20 | 
21 |     try:
22 |         # mypy knows this isn't present in <=3.6
23 |         from subprocess import STARTUPINFO  # type: ignore
24 |     except ImportError:
25 |         # STARTUPINFO added in 3.7+
26 |         STARTUPINFO = Any
27 | 
28 |     PIPE = Literal[subprocess.PIPE]
29 |     DEVNULL = Literal[subprocess.DEVNULL]
30 |     STDOUT = Literal[subprocess.STDOUT]
31 | 
32 |     SubprocessKwargs = TypedDict(
33 |         'SubprocessKwargs',
34 |         # no: bufsize, universal_newlines, shell, text, encoding and errors
35 |         {
36 |             'stdin': Optional[Union[IO, PIPE, DEVNULL]],
37 |             'stdout': Optional[Union[IO, PIPE, DEVNULL]],
38 |             'stderr': Optional[Union[IO, PIPE, DEVNULL, STDOUT]],
39 | 
40 |             'limit': Optional[int],
41 | 
42 |             'preexec_fn': Callable[..., Any],
43 |             'close_fds': Optional[bool],
44 |             'pass_fds': Optional[Sequence[int]],
45 |             'cdw': Optional[os.PathLike],
46 |             'restore_signals': Optional[bool],
47 |             'start_new_session': Optional[bool],
48 |             'env': Optional[Mapping[str, str]],
49 |             'startupinfo': Optional[STARTUPINFO],
50 |             'creationflags': Optional[Union[
51 |                 Literal[
52 |                     subprocess.CREATE_NEW_CONSOLE,  # type: ignore
53 |                     subprocess.CREATE_NEW_PROCESS_GROUP,  # type: ignore
54 |                     subprocess.ABOVE_NORMAL_PRIORITY_CLASS,  # type: ignore
55 |                     subprocess.BELOW_NORMAL_PRIORITY_CLASS,  # type: ignore
56 |                     subprocess.HIGH_PRIORITY_CLASS,  # type: ignore
57 |                     subprocess.IDLE_PRIORITY_CLASS,  # type: ignore
58 |                     subprocess.NORMAL_PRIORITY_CLASS,  # type: ignore
59 |                     subprocess.REALTIME_PRIORITY_CLASS,  # type: ignore
60 |                     subprocess.CREATE_NO_WINDOW,  # type: ignore
61 |                     subprocess.DETACHED_PROCESS,  # type: ignore
62 |                     subprocess.CREATE_DEFAULT_ERROR_MODE,  # type: ignore
63 |                     subprocess.CREATE_BREAKAWAY_FROM_JOB,  # type: ignore
64 |                 ],
65 |                 Sequence[Literal[
66 |                     subprocess.CREATE_NEW_CONSOLE,
67 |                     subprocess.CREATE_NEW_PROCESS_GROUP,
68 |                     subprocess.ABOVE_NORMAL_PRIORITY_CLASS,
69 |                     subprocess.BELOW_NORMAL_PRIORITY_CLASS,
70 |                     subprocess.HIGH_PRIORITY_CLASS,
71 |                     subprocess.IDLE_PRIORITY_CLASS,
72 |                     subprocess.NORMAL_PRIORITY_CLASS,
73 |                     subprocess.REALTIME_PRIORITY_CLASS,
74 |                     subprocess.CREATE_NO_WINDOW,
75 |                     subprocess.DETACHED_PROCESS,
76 |                     subprocess.CREATE_DEFAULT_ERROR_MODE,
77 |                     subprocess.CREATE_BREAKAWAY_FROM_JOB,
78 |                 ]],
79 |             ]],
80 |         },
81 |         total=False,
82 |     )
83 | else:
84 |     # Ensure this is importable outside of the `TYPE_CHECKING` context so that
85 |     # 3rd party libraries can use this as a type without having to think too
86 |     # much about what they are doing.
87 |     SubprocessKwargs = Dict[str, Any]
88 | 
89 | 
90 | TReturn = TypeVar("TReturn")
91 | 


--------------------------------------------------------------------------------
/asyncio_run_in_process/typing_compat.py:
--------------------------------------------------------------------------------
1 | try:
2 |     # mypy knows that `TypedDict` and `Literal` don't exist in <3.8
3 |     from typing import TypedDict, Literal  # type: ignore
4 | except ImportError:
5 |     # TypedDict is only available in python 3.8+
6 |     # Literal is only available in python 3.8+
7 |     from typing_extensions import TypedDict, Literal  # noqa: F401
8 | 


--------------------------------------------------------------------------------
/docs/Makefile:
--------------------------------------------------------------------------------
  1 | # Makefile for Sphinx documentation
  2 | #
  3 | 
  4 | # You can set these variables from the command line.
  5 | SPHINXOPTS    =
  6 | SPHINXBUILD   = sphinx-build
  7 | PAPER         =
  8 | BUILDDIR      = _build
  9 | 
 10 | # User-friendly check for sphinx-build
 11 | ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
 12 | $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
 13 | endif
 14 | 
 15 | # Internal variables.
 16 | PAPEROPT_a4     = -D latex_paper_size=a4
 17 | PAPEROPT_letter = -D latex_paper_size=letter
 18 | ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 19 | # the i18n builder cannot share the environment and doctrees with the others
 20 | I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
 21 | 
 22 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
 23 | 
 24 | help:
 25 | 	@echo "Please use \`make <target>' where <target> is one of"
 26 | 	@echo "  html       to make standalone HTML files"
 27 | 	@echo "  dirhtml    to make HTML files named index.html in directories"
 28 | 	@echo "  singlehtml to make a single large HTML file"
 29 | 	@echo "  pickle     to make pickle files"
 30 | 	@echo "  json       to make JSON files"
 31 | 	@echo "  htmlhelp   to make HTML files and a HTML help project"
 32 | 	@echo "  qthelp     to make HTML files and a qthelp project"
 33 | 	@echo "  devhelp    to make HTML files and a Devhelp project"
 34 | 	@echo "  epub       to make an epub"
 35 | 	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
 36 | 	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
 37 | 	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
 38 | 	@echo "  text       to make text files"
 39 | 	@echo "  man        to make manual pages"
 40 | 	@echo "  texinfo    to make Texinfo files"
 41 | 	@echo "  info       to make Texinfo files and run them through makeinfo"
 42 | 	@echo "  gettext    to make PO message catalogs"
 43 | 	@echo "  changes    to make an overview of all changed/added/deprecated items"
 44 | 	@echo "  xml        to make Docutils-native XML files"
 45 | 	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
 46 | 	@echo "  linkcheck  to check all external links for integrity"
 47 | 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
 48 | 
 49 | clean:
 50 | 	rm -rf $(BUILDDIR)/*
 51 | 
 52 | html:
 53 | 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 54 | 	@echo
 55 | 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 56 | 
 57 | dirhtml:
 58 | 	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
 59 | 	@echo
 60 | 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
 61 | 
 62 | singlehtml:
 63 | 	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
 64 | 	@echo
 65 | 	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
 66 | 
 67 | pickle:
 68 | 	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
 69 | 	@echo
 70 | 	@echo "Build finished; now you can process the pickle files."
 71 | 
 72 | json:
 73 | 	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
 74 | 	@echo
 75 | 	@echo "Build finished; now you can process the JSON files."
 76 | 
 77 | htmlhelp:
 78 | 	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
 79 | 	@echo
 80 | 	@echo "Build finished; now you can run HTML Help Workshop with the" \
 81 | 	      ".hhp project file in $(BUILDDIR)/htmlhelp."
 82 | 
 83 | qthelp:
 84 | 	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
 85 | 	@echo
 86 | 	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
 87 | 	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
 88 | 	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/web3.qhcp"
 89 | 	@echo "To view the help file:"
 90 | 	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/web3.qhc"
 91 | 
 92 | devhelp:
 93 | 	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
 94 | 	@echo
 95 | 	@echo "Build finished."
 96 | 	@echo "To view the help file:"
 97 | 	@echo "# mkdir -p $$HOME/.local/share/devhelp/web3"
 98 | 	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/web3"
 99 | 	@echo "# devhelp"
100 | 
101 | epub:
102 | 	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
103 | 	@echo
104 | 	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
105 | 
106 | latex:
107 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
108 | 	@echo
109 | 	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
110 | 	@echo "Run \`make' in that directory to run these through (pdf)latex" \
111 | 	      "(use \`make latexpdf' here to do that automatically)."
112 | 
113 | latexpdf:
114 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
115 | 	@echo "Running LaTeX files through pdflatex..."
116 | 	$(MAKE) -C $(BUILDDIR)/latex all-pdf
117 | 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
118 | 
119 | latexpdfja:
120 | 	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
121 | 	@echo "Running LaTeX files through platex and dvipdfmx..."
122 | 	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
123 | 	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
124 | 
125 | text:
126 | 	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
127 | 	@echo
128 | 	@echo "Build finished. The text files are in $(BUILDDIR)/text."
129 | 
130 | man:
131 | 	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
132 | 	@echo
133 | 	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
134 | 
135 | texinfo:
136 | 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
137 | 	@echo
138 | 	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
139 | 	@echo "Run \`make' in that directory to run these through makeinfo" \
140 | 	      "(use \`make info' here to do that automatically)."
141 | 
142 | info:
143 | 	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
144 | 	@echo "Running Texinfo files through makeinfo..."
145 | 	make -C $(BUILDDIR)/texinfo info
146 | 	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
147 | 
148 | gettext:
149 | 	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
150 | 	@echo
151 | 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
152 | 
153 | changes:
154 | 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
155 | 	@echo
156 | 	@echo "The overview file is in $(BUILDDIR)/changes."
157 | 
158 | linkcheck:
159 | 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
160 | 	@echo
161 | 	@echo "Link check complete; look for any errors in the above output " \
162 | 	      "or in $(BUILDDIR)/linkcheck/output.txt."
163 | 
164 | doctest:
165 | 	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
166 | 	@echo "Testing of doctests in the sources finished, look at the " \
167 | 	      "results in $(BUILDDIR)/doctest/output.txt."
168 | 
169 | xml:
170 | 	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
171 | 	@echo
172 | 	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
173 | 
174 | pseudoxml:
175 | 	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
176 | 	@echo
177 | 	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
178 | 


--------------------------------------------------------------------------------
/docs/_static/.suppress-sphinx-build-warning:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ethereum/asyncio-run-in-process/0ebfcea6ce9f0ae50abe7bf98d4c2d3783b4e86b/docs/_static/.suppress-sphinx-build-warning


--------------------------------------------------------------------------------
/docs/asyncio_run_in_process.rst:
--------------------------------------------------------------------------------
 1 | asyncio\_run\_in\_process package
 2 | =================================
 3 | 
 4 | Submodules
 5 | ----------
 6 | 
 7 | asyncio\_run\_in\_process.abc module
 8 | ------------------------------------
 9 | 
10 | .. automodule:: asyncio_run_in_process.abc
11 |     :members:
12 |     :undoc-members:
13 |     :show-inheritance:
14 | 
15 | asyncio\_run\_in\_process.exceptions module
16 | -------------------------------------------
17 | 
18 | .. automodule:: asyncio_run_in_process.exceptions
19 |     :members:
20 |     :undoc-members:
21 |     :show-inheritance:
22 | 
23 | asyncio\_run\_in\_process.process module
24 | ----------------------------------------
25 | 
26 | .. automodule:: asyncio_run_in_process.process
27 |     :members:
28 |     :undoc-members:
29 |     :show-inheritance:
30 | 
31 | asyncio\_run\_in\_process.run\_in\_process module
32 | -------------------------------------------------
33 | 
34 | .. automodule:: asyncio_run_in_process.run_in_process
35 |     :members:
36 |     :undoc-members:
37 |     :show-inheritance:
38 | 
39 | asyncio\_run\_in\_process.state module
40 | --------------------------------------
41 | 
42 | .. automodule:: asyncio_run_in_process.state
43 |     :members:
44 |     :undoc-members:
45 |     :show-inheritance:
46 | 
47 | asyncio\_run\_in\_process.typing module
48 | ---------------------------------------
49 | 
50 | .. automodule:: asyncio_run_in_process.typing
51 |     :members:
52 |     :undoc-members:
53 |     :show-inheritance:
54 | 
55 | 
56 | Module contents
57 | ---------------
58 | 
59 | .. automodule:: asyncio_run_in_process
60 |     :members:
61 |     :undoc-members:
62 |     :show-inheritance:
63 | 


--------------------------------------------------------------------------------
/docs/conf.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #
  3 | # asyncio-run-in-process documentation build configuration file, created by
  4 | # sphinx-quickstart on Thu Oct 16 20:43:24 2014.
  5 | #
  6 | # This file is execfile()d with the current directory set to its
  7 | # containing dir.
  8 | #
  9 | # Note that not all possible configuration values are present in this
 10 | # autogenerated file.
 11 | #
 12 | # All configuration values have a default; values that are commented out
 13 | # serve to show the default.
 14 | 
 15 | # If extensions (or modules to document with autodoc) are in another directory,
 16 | # add these directories to sys.path here. If the directory is relative to the
 17 | # documentation root, use os.path.abspath to make it absolute, like shown here.
 18 | #sys.path.insert(0, os.path.abspath('.'))
 19 | 
 20 | import os
 21 | 
 22 | DIR = os.path.dirname('__file__')
 23 | with open (os.path.join(DIR, '../setup.py'), 'r') as f:
 24 |     for line in f:
 25 |         if 'version=' in line:
 26 |             setup_version = line.split('\'')[1]
 27 |             break
 28 | 
 29 | # -- General configuration ------------------------------------------------
 30 | 
 31 | # If your documentation needs a minimal Sphinx version, state it here.
 32 | #needs_sphinx = '1.0'
 33 | 
 34 | # Add any Sphinx extension module names here, as strings. They can be
 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 36 | # ones.
 37 | extensions = [
 38 |     'sphinx.ext.autodoc',
 39 |     'sphinx.ext.doctest',
 40 |     'sphinx.ext.intersphinx',
 41 | ]
 42 | 
 43 | # Add any paths that contain templates here, relative to this directory.
 44 | templates_path = ['_templates']
 45 | 
 46 | # The suffix of source filenames.
 47 | source_suffix = '.rst'
 48 | 
 49 | # The encoding of source files.
 50 | #source_encoding = 'utf-8-sig'
 51 | 
 52 | # The master toctree document.
 53 | master_doc = 'index'
 54 | 
 55 | # General information about the project.
 56 | project = 'asyncio-run-in-process'
 57 | copyright = '2019, The Ethereum Foundation'
 58 | 
 59 | __version__ = setup_version
 60 | # The version info for the project you're documenting, acts as replacement for
 61 | # |version| and |release|, also used in various other places throughout the
 62 | # built documents.
 63 | #
 64 | # The short X.Y version.
 65 | version = '.'.join(__version__.split('.')[:2])
 66 | # The full version, including alpha/beta/rc tags.
 67 | release = __version__
 68 | 
 69 | # The language for content autogenerated by Sphinx. Refer to documentation
 70 | # for a list of supported languages.
 71 | #language = None
 72 | 
 73 | # There are two options for replacing |today|: either, you set today to some
 74 | # non-false value, then it is used:
 75 | #today = ''
 76 | # Else, today_fmt is used as the format for a strftime call.
 77 | #today_fmt = '%B %d, %Y'
 78 | 
 79 | # List of patterns, relative to source directory, that match files and
 80 | # directories to ignore when looking for source files.
 81 | exclude_patterns = [
 82 |     '_build',
 83 |     'modules.rst',
 84 | ]
 85 | 
 86 | # The reST default role (used for this markup: `text`) to use for all
 87 | # documents.
 88 | #default_role = None
 89 | 
 90 | # If true, '()' will be appended to :func: etc. cross-reference text.
 91 | #add_function_parentheses = True
 92 | 
 93 | # If true, the current module name will be prepended to all description
 94 | # unit titles (such as .. function::).
 95 | #add_module_names = True
 96 | 
 97 | # If true, sectionauthor and moduleauthor directives will be shown in the
 98 | # output. They are ignored by default.
 99 | #show_authors = False
100 | 
101 | # The name of the Pygments (syntax highlighting) style to use.
102 | pygments_style = 'sphinx'
103 | 
104 | # A list of ignored prefixes for module index sorting.
105 | #modindex_common_prefix = []
106 | 
107 | # If true, keep warnings as "system message" paragraphs in the built documents.
108 | #keep_warnings = False
109 | 
110 | 
111 | # -- Options for HTML output ----------------------------------------------
112 | 
113 | # The theme to use for HTML and HTML Help pages.  See the documentation for
114 | # a list of builtin themes.
115 | html_theme = 'sphinx_rtd_theme'
116 | 
117 | # Theme options are theme-specific and customize the look and feel of a theme
118 | # further.  For a list of options available for each theme, see the
119 | # documentation.
120 | #html_theme_options = {}
121 | 
122 | # Add any paths that contain custom themes here, relative to this directory.
123 | 
124 | # The name for this set of Sphinx documents.  If None, it defaults to
125 | # "<project> v<release> documentation".
126 | #html_title = None
127 | 
128 | # A shorter title for the navigation bar.  Default is the same as html_title.
129 | #html_short_title = None
130 | 
131 | # The name of an image file (relative to this directory) to place at the top
132 | # of the sidebar.
133 | #html_logo = None
134 | 
135 | # The name of an image file (within the static path) to use as favicon of the
136 | # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
137 | # pixels large.
138 | #html_favicon = None
139 | 
140 | # Add any paths that contain custom static files (such as style sheets) here,
141 | # relative to this directory. They are copied after the builtin static files,
142 | # so a file named "default.css" will overwrite the builtin "default.css".
143 | html_static_path = ['_static']
144 | 
145 | # Add any extra paths that contain custom files (such as robots.txt or
146 | # .htaccess) here, relative to this directory. These files are copied
147 | # directly to the root of the documentation.
148 | #html_extra_path = []
149 | 
150 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
151 | # using the given strftime format.
152 | #html_last_updated_fmt = '%b %d, %Y'
153 | 
154 | # If true, SmartyPants will be used to convert quotes and dashes to
155 | # typographically correct entities.
156 | #html_use_smartypants = True
157 | 
158 | # Custom sidebar templates, maps document names to template names.
159 | #html_sidebars = {}
160 | 
161 | # Additional templates that should be rendered to pages, maps page names to
162 | # template names.
163 | #html_additional_pages = {}
164 | 
165 | # If false, no module index is generated.
166 | #html_domain_indices = True
167 | 
168 | # If false, no index is generated.
169 | #html_use_index = True
170 | 
171 | # If true, the index is split into individual pages for each letter.
172 | #html_split_index = False
173 | 
174 | # If true, links to the reST sources are added to the pages.
175 | #html_show_sourcelink = True
176 | 
177 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
178 | #html_show_sphinx = True
179 | 
180 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
181 | #html_show_copyright = True
182 | 
183 | # If true, an OpenSearch description file will be output, and all pages will
184 | # contain a <link> tag referring to it.  The value of this option must be the
185 | # base URL from which the finished HTML is served.
186 | #html_use_opensearch = ''
187 | 
188 | # This is the file name suffix for HTML files (e.g. ".xhtml").
189 | #html_file_suffix = None
190 | 
191 | # Output file base name for HTML help builder.
192 | htmlhelp_basename = 'asyncio_run_in_processdoc'
193 | 
194 | 
195 | # -- Options for LaTeX output ---------------------------------------------
196 | 
197 | latex_elements = {
198 | # The paper size ('letterpaper' or 'a4paper').
199 | #'papersize': 'letterpaper',
200 | 
201 | # The font size ('10pt', '11pt' or '12pt').
202 | #'pointsize': '10pt',
203 | 
204 | # Additional stuff for the LaTeX preamble.
205 | #'preamble': '',
206 | }
207 | 
208 | # Grouping the document tree into LaTeX files. List of tuples
209 | # (source start file, target name, title,
210 | #  author, documentclass [howto, manual, or own class]).
211 | latex_documents = [
212 |   ('index', 'asyncio_run_in_process.tex', 'asyncio-run-in-process Documentation',
213 |    'The Ethereum Foundation', 'manual'),
214 | ]
215 | 
216 | # The name of an image file (relative to this directory) to place at the top of
217 | # the title page.
218 | #latex_logo = None
219 | 
220 | # For "manual" documents, if this is true, then toplevel headings are parts,
221 | # not chapters.
222 | #latex_use_parts = False
223 | 
224 | # If true, show page references after internal links.
225 | #latex_show_pagerefs = False
226 | 
227 | # If true, show URL addresses after external links.
228 | #latex_show_urls = False
229 | 
230 | # Documents to append as an appendix to all manuals.
231 | #latex_appendices = []
232 | 
233 | # If false, no module index is generated.
234 | #latex_domain_indices = True
235 | 
236 | 
237 | # -- Options for manual page output ---------------------------------------
238 | 
239 | # One entry per manual page. List of tuples
240 | # (source start file, name, description, authors, manual section).
241 | man_pages = [
242 |     ('index', 'asyncio_run_in_process', 'asyncio-run-in-process Documentation',
243 |      ['The Ethereum Foundation'], 1)
244 | ]
245 | 
246 | # If true, show URL addresses after external links.
247 | #man_show_urls = False
248 | 
249 | 
250 | # -- Options for Texinfo output -------------------------------------------
251 | 
252 | # Grouping the document tree into Texinfo files. List of tuples
253 | # (source start file, target name, title, author,
254 | #  dir menu entry, description, category)
255 | texinfo_documents = [
256 |   ('index', 'asyncio-run-in-process', 'asyncio-run-in-process Documentation',
257 |    'The Ethereum Foundation', 'asyncio-run-in-process', 'Simple asyncio friendly replacement for multiprocessing',
258 |    'Miscellaneous'),
259 | ]
260 | 
261 | # Documents to append as an appendix to all manuals.
262 | #texinfo_appendices = []
263 | 
264 | # If false, no module index is generated.
265 | #texinfo_domain_indices = True
266 | 
267 | # How to display URL addresses: 'footnote', 'no', or 'inline'.
268 | #texinfo_show_urls = 'footnote'
269 | 
270 | # If true, do not generate a @detailmenu in the "Top" node's menu.
271 | #texinfo_no_detailmenu = False
272 | 
273 | # -- Intersphinx configuration ------------------------------------------------
274 | 
275 | intersphinx_mapping = {
276 |     'python': ('https://docs.python.org/3.6', None),
277 | }
278 | 
279 | # -- Doctest configuration ----------------------------------------
280 | 
281 | import doctest
282 | 
283 | doctest_default_flags = (0
284 |     | doctest.DONT_ACCEPT_TRUE_FOR_1
285 |     | doctest.ELLIPSIS
286 |     | doctest.IGNORE_EXCEPTION_DETAIL
287 |     | doctest.NORMALIZE_WHITESPACE
288 | )
289 | 


--------------------------------------------------------------------------------
/docs/index.rst:
--------------------------------------------------------------------------------
 1 | asyncio-run-in-process
 2 | ==============================
 3 | 
 4 | Simple asyncio friendly replacement for multiprocessing to run a coroutine in an isolated process
 5 | 
 6 | Contents
 7 | --------
 8 | 
 9 | .. toctree::
10 |     :maxdepth: 3
11 | 
12 |     asyncio_run_in_process
13 |     releases
14 | 
15 | 
16 | Quickstart
17 | ----------
18 | 
19 | We use ``run_in_process`` for something we want run in a process in a blocking manner.
20 | 
21 | .. code-block:: python
22 | 
23 |     from asyncio_run_in_process import run_in_process
24 | 
25 |     async def fib(n):
26 |         if n <= 1:
27 |             return n
28 |         else:
29 |             return await fib(n - 1) + await fib(n - 2)
30 | 
31 |     # runs in a separate process
32 |     result = await run_in_process(fib, 10)
33 |     print(f"The 10th fibbonacci number is {result}")
34 | 
35 | 
36 | We use ``open_in_process`` for something we want to run in the background.
37 | 
38 | .. code-block:: python
39 | 
40 |     from asyncio_run_in_process import open_in_process:
41 | 
42 |     async def fib(n):
43 |         if n <= 1:
44 |             return n
45 |         else:
46 |             return await fib(n - 1) + await fib(n - 2)
47 | 
48 |     # runs in a separate process
49 |     async with open_in_process(fib, 10) as proc:
50 |         # do some other things here while it runs in the background.
51 |         ...
52 |         # the context will block here until the process has finished.
53 |     # once the context exits the result is available on the process.
54 |     print(f"The 10th fibbonacci number is {proc.result}")
55 | 
56 | 
57 | Both functions above will run the coroutine in an asyncio event loop, but should you want to
58 | run them with ``trio``, the ``run_in_process_with_trio`` and ``open_in_process_with_trio``
59 | functions can be used.
60 | 
61 | 
62 | Maximum number of running processes
63 | -----------------------------------
64 | 
65 | By default we can only have up to ``MAX_PROCESSES`` running at any given moment, but that can
66 | be changed via the ``ASYNCIO_RUN_IN_PROCESS_MAX_PROCS`` environment variable.
67 | 
68 | 
69 | Gotchas
70 | -------
71 | 
72 | If a function passed to ``open_in_process`` uses asyncio's ``loop.run_in_executor()``
73 | to run synchronous code, you must ensure the task/process terminates in case the
74 | ``loop.run_in_executor()`` call is cancelled, or else ``open_in_process`` will not
75 | ever return. This is necessary because asyncio does not cancel the thread/process it
76 | starts (in ``loop.run_in_executor()``), and that prevents ``open_in_process`` from
77 | terminating. One way to ensure that is to have the code running in the executor react
78 | to an event that gets set when ``loop.run_in_executor()`` returns.
79 | 
80 | 
81 | Indices and tables
82 | ------------------
83 | 
84 | * :ref:`genindex`
85 | * :ref:`modindex`
86 | 


--------------------------------------------------------------------------------
/docs/releases.rst:
--------------------------------------------------------------------------------
 1 | Release Notes
 2 | =============
 3 | 
 4 | v0.1.0-alpha.7
 5 | --------------
 6 | 
 7 | - Added new APIs to run the coroutine with trio in the subprocess
 8 | - Several bug fixes
 9 | 
10 | v0.1.0-alpha.1
11 | --------------
12 | 
13 | - Launched repository, claimed names for pip, RTD, github, etc
14 | 


--------------------------------------------------------------------------------
/mypy.ini:
--------------------------------------------------------------------------------
 1 | [mypy]
 2 | 
 3 | check_untyped_defs = True
 4 | disallow_incomplete_defs = True
 5 | disallow_untyped_defs = True
 6 | disallow_any_generics = True
 7 | disallow_untyped_calls = True
 8 | disallow_untyped_decorators = True
 9 | disallow_subclassing_any = True
10 | ignore_missing_imports = True
11 | strict_optional = True
12 | strict_equality = True
13 | warn_redundant_casts = True
14 | warn_return_any = True
15 | warn_unused_configs = True
16 | warn_unused_ignores = True
17 | 


--------------------------------------------------------------------------------
/pytest.ini:
--------------------------------------------------------------------------------
1 | [pytest]
2 | addopts= -v --showlocals --durations 10
3 | python_paths= .
4 | xfail_strict=true
5 | 
6 | [pytest-watch]
7 | runner= pytest --failed-first --maxfail=1 --no-success-flaky-report
8 | 


--------------------------------------------------------------------------------
/requirements-docs.txt:
--------------------------------------------------------------------------------
1 | asyncio-run-in-process[doc]
2 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | # -*- coding: utf-8 -*-
 3 | from setuptools import (
 4 |     setup,
 5 |     find_packages,
 6 | )
 7 | 
 8 | extras_require = {
 9 |     'test': [
10 |         "async-exit-stack==1.0.1",
11 |         "pytest==5.2.2",
12 |         "pytest-asyncio==0.10.0",
13 |         "pytest-xdist",
14 |         "tox>=2.9.1,<3",
15 |     ],
16 |     'lint': [
17 |         "flake8==3.7.9",
18 |         "isort>=4.2.15,<5",
19 |         "mypy==0.770",
20 |         "pydocstyle>=3.0.0,<4",
21 |         "typing-extensions>=3.7.4.1,<4;python_version<'3.8'",
22 |     ],
23 |     'doc': [
24 |         "Sphinx>=1.6.5,<2",
25 |         "sphinx_rtd_theme>=0.1.9",
26 |     ],
27 |     'dev': [
28 |         "bumpversion>=0.5.3,<1",
29 |         "pytest-watch>=4.1.0,<5",
30 |         "wheel",
31 |         "twine",
32 |         "ipython",
33 |     ],
34 | }
35 | 
36 | extras_require['dev'] = (
37 |     extras_require['dev'] +  # noqa: W504
38 |     extras_require['test'] +  # noqa: W504
39 |     extras_require['lint'] +  # noqa: W504
40 |     extras_require['doc']
41 | )
42 | 
43 | 
44 | with open('./README.md') as readme:
45 |     long_description = readme.read()
46 | 
47 | 
48 | setup(
49 |     name='asyncio-run-in-process',
50 |     # *IMPORTANT*: Don't manually change the version here. Use `make bump`, as described in readme
51 |     version='0.1.0-alpha.10',
52 |     description="""asyncio-run-in-process: Asyncio friendly replacement for multiprocessing""",
53 |     long_description=long_description,
54 |     long_description_content_type='text/markdown',
55 |     author='The Ethereum Foundation',
56 |     author_email='snakecharmers@ethereum.org',
57 |     url='https://github.com/ethereum/asyncio-run-in-process',
58 |     include_package_data=True,
59 |     install_requires=[
60 |         "async-generator>=1.10,<2",
61 |         "cloudpickle>=1.2.1,<2",
62 |         "trio>=0.16,<0.17",
63 |         "trio-typing>=0.5.0,<0.6",
64 |     ],
65 |     python_requires='>=3.6, <4',
66 |     extras_require=extras_require,
67 |     py_modules=['asyncio_run_in_process'],
68 |     license="MIT",
69 |     zip_safe=False,
70 |     keywords='ethereum',
71 |     packages=find_packages(exclude=["tests", "tests.*"]),
72 |     classifiers=[
73 |         'Development Status :: 3 - Alpha',
74 |         'Intended Audience :: Developers',
75 |         'License :: OSI Approved :: MIT License',
76 |         'Natural Language :: English',
77 |         'Programming Language :: Python :: 3',
78 |         'Programming Language :: Python :: 3.6',
79 |         'Programming Language :: Python :: 3.7',
80 |         'Programming Language :: Python :: Implementation :: PyPy',
81 |     ],
82 | )
83 | 


--------------------------------------------------------------------------------
/tests/core/conftest.py:
--------------------------------------------------------------------------------
 1 | from pathlib import (
 2 |     Path,
 3 | )
 4 | import tempfile
 5 | 
 6 | import pytest
 7 | 
 8 | from asyncio_run_in_process import (
 9 |     run_in_process,
10 |     run_in_process_with_trio,
11 | )
12 | 
13 | 
14 | @pytest.fixture
15 | def touch_path():
16 |     with tempfile.TemporaryDirectory() as base_dir:
17 |         yield Path(base_dir) / "touch.txt"
18 | 
19 | 
20 | @pytest.fixture(params=('use_trio', 'use_asyncio'))
21 | def runner(request):
22 |     if request.param == 'use_trio':
23 |         return run_in_process_with_trio
24 |     elif request.param == 'use_asyncio':
25 |         return run_in_process
26 |     else:
27 |         raise Exception("Invariant")
28 | 


--------------------------------------------------------------------------------
/tests/core/test_import.py:
--------------------------------------------------------------------------------
1 | def test_import():
2 |     import asyncio_run_in_process  # noqa: F401
3 | 


--------------------------------------------------------------------------------
/tests/core/test_open_in_process.py:
--------------------------------------------------------------------------------
  1 | import asyncio
  2 | import concurrent
  3 | import pickle
  4 | import signal
  5 | 
  6 | import pytest
  7 | import trio
  8 | 
  9 | from async_exit_stack import (
 10 |     AsyncExitStack,
 11 | )
 12 | from asyncio_run_in_process import (
 13 |     ProcessKilled,
 14 |     constants,
 15 |     main,
 16 |     open_in_process,
 17 |     open_in_process_with_trio,
 18 | )
 19 | from asyncio_run_in_process.exceptions import (
 20 |     ChildCancelled,
 21 |     InvalidDataFromChild,
 22 | )
 23 | from asyncio_run_in_process.process import (
 24 |     Process,
 25 | )
 26 | from asyncio_run_in_process.state import (
 27 |     State,
 28 | )
 29 | from asyncio_run_in_process.tools.sleep import (
 30 |     sleep,
 31 | )
 32 | 
 33 | 
 34 | @pytest.fixture(params=('use_trio', 'use_asyncio'))
 35 | def open_in_proc(request):
 36 |     if request.param == 'use_trio':
 37 |         return open_in_process_with_trio
 38 |     elif request.param == 'use_asyncio':
 39 |         return open_in_process
 40 |     else:
 41 |         raise Exception("Invariant")
 42 | 
 43 | 
 44 | @pytest.mark.asyncio
 45 | async def test_SIGINT_on_method_using_run_in_executor():
 46 |     # This test exists only to show that one needs to be carefull when using run_in_executor() as
 47 |     # asyncio does not cancel the thread/process it starts, so we need to make sure they return or
 48 |     # else open_in_process() hangs forever. In the code below, this is achieved by setting the
 49 |     # stop_loop event before the method passed to open_in_process() returns. If we don't set that
 50 |     # event, the test hangs forever.
 51 |     async def loop_forever_in_executor():
 52 |         import threading
 53 |         stop_loop = threading.Event()
 54 | 
 55 |         def thread_loop():
 56 |             import time
 57 |             while not stop_loop.is_set():
 58 |                 time.sleep(0.01)
 59 | 
 60 |         loop = asyncio.get_event_loop()
 61 |         try:
 62 |             await loop.run_in_executor(None, thread_loop)
 63 |         finally:
 64 |             stop_loop.set()
 65 | 
 66 |     async with open_in_process(loop_forever_in_executor) as proc:
 67 |         proc.send_signal(signal.SIGINT)
 68 |     assert proc.returncode == 2
 69 | 
 70 | 
 71 | @pytest.mark.asyncio
 72 | async def test_open_in_proc_SIGTERM_while_running(open_in_proc):
 73 |     async def do_sleep_forever():
 74 |         while True:
 75 |             await sleep(0)
 76 | 
 77 |     async with open_in_proc(do_sleep_forever) as proc:
 78 |         proc.terminate()
 79 |     assert proc.returncode == 15
 80 | 
 81 | 
 82 | @pytest.mark.asyncio
 83 | async def test_open_in_proc_SIGKILL_while_running(open_in_proc):
 84 |     async def do_sleep_forever():
 85 |         while True:
 86 |             await sleep(0)
 87 | 
 88 |     async with open_in_proc(do_sleep_forever) as proc:
 89 |         await proc.kill()
 90 |     assert proc.returncode == -9
 91 |     assert isinstance(proc.error, ProcessKilled)
 92 | 
 93 | 
 94 | @pytest.mark.asyncio
 95 | async def test_open_proc_SIGINT_while_running(open_in_proc):
 96 |     async def do_sleep_forever():
 97 |         while True:
 98 |             await sleep(0)
 99 | 
100 |     async with open_in_proc(do_sleep_forever) as proc:
101 |         proc.send_signal(signal.SIGINT)
102 | 
103 |     assert proc.returncode == 2
104 | 
105 | 
106 | @pytest.mark.asyncio
107 | async def test_open_proc_SIGINT_can_be_handled(open_in_proc):
108 |     async def do_sleep_forever():
109 |         try:
110 |             while True:
111 |                 await sleep(0)
112 |         except KeyboardInterrupt:
113 |             return 9999
114 | 
115 |     async with open_in_proc(do_sleep_forever) as proc:
116 |         proc.send_signal(signal.SIGINT)
117 | 
118 |     assert proc.returncode == 0
119 |     assert proc.get_result_or_raise() == 9999
120 | 
121 | 
122 | @pytest.mark.asyncio
123 | async def test_open_proc_SIGINT_can_be_ignored(open_in_proc):
124 |     async def do_sleep_forever():
125 |         try:
126 |             while True:
127 |                 await sleep(0)
128 |         except KeyboardInterrupt:
129 |             # silence the first SIGINT
130 |             pass
131 | 
132 |         try:
133 |             while True:
134 |                 await sleep(0)
135 |         except KeyboardInterrupt:
136 |             return 9999
137 | 
138 |     async with open_in_proc(do_sleep_forever) as proc:
139 |         proc.send_signal(signal.SIGINT)
140 |         await asyncio.sleep(0.01)
141 |         proc.send_signal(signal.SIGINT)
142 | 
143 |     assert proc.returncode == 0
144 |     assert proc.get_result_or_raise() == 9999
145 | 
146 | 
147 | @pytest.mark.asyncio
148 | async def test_open_proc_invalid_function_call(open_in_proc):
149 |     async def takes_no_args():
150 |         pass
151 | 
152 |     async with open_in_proc(takes_no_args, 1, 2, 3) as proc:
153 |         pass
154 |     assert proc.returncode == 1
155 |     assert isinstance(proc.error, TypeError)
156 | 
157 | 
158 | @pytest.mark.asyncio
159 | async def test_open_proc_unpickleable_params(touch_path, open_in_proc):
160 |     async def takes_open_file(f):
161 |         pass
162 | 
163 |     with pytest.raises(pickle.PickleError):
164 |         with open(touch_path, "w") as touch_file:
165 |             async with open_in_proc(takes_open_file, touch_file):
166 |                 # this code block shouldn't get executed
167 |                 assert False  # noqa: B011
168 | 
169 | 
170 | @pytest.mark.asyncio
171 | async def test_open_proc_KeyboardInterrupt_while_running():
172 |     async def do_sleep_forever():
173 |         while True:
174 |             await asyncio.sleep(0)
175 | 
176 |     with pytest.raises(KeyboardInterrupt):
177 |         async with open_in_process(do_sleep_forever) as proc:
178 |             raise KeyboardInterrupt
179 |     assert proc.returncode == 2
180 | 
181 | 
182 | # XXX: For some reason this test hangs forever if we use the open_in_proc fixture, so
183 | # we have to have duplicate versions of it for trio/asyncio.
184 | @pytest.mark.asyncio
185 | async def test_open_proc_with_trio_KeyboardInterrupt_while_running():
186 |     sleep = trio.sleep
187 | 
188 |     async def do_sleep_forever():
189 |         while True:
190 |             await sleep(0)
191 | 
192 |     with pytest.raises(KeyboardInterrupt):
193 |         async with open_in_process_with_trio(do_sleep_forever) as proc:
194 |             raise KeyboardInterrupt
195 |     assert proc.returncode == 2
196 | 
197 | 
198 | @pytest.mark.asyncio
199 | async def test_open_proc_does_not_hang_on_exception(open_in_proc):
200 |     class CustomException(BaseException):
201 |         pass
202 | 
203 |     async def raise_():
204 |         await sleep(0.01)
205 |         raise CustomException("Just a boring exception")
206 | 
207 |     async def _do_inner():
208 |         with pytest.raises(CustomException):
209 |             async with open_in_proc(raise_) as proc:
210 |                 await proc.wait_result_or_raise()
211 | 
212 |     await asyncio.wait_for(_do_inner(), timeout=1)
213 | 
214 | 
215 | @pytest.mark.asyncio
216 | async def test_open_proc_unpickleable_exc(open_in_proc):
217 |     # Custom exception classes requiring multiple arguments cannot be pickled:
218 |     # https://bugs.python.org/issue32696
219 |     class CustomException(BaseException):
220 |         def __init__(self, msg, arg2):
221 |             super().__init__(msg)
222 |             self.arg2 = arg2
223 | 
224 |     async def raise_():
225 |         await sleep(0.01)
226 |         raise CustomException('msg', 'arg2')
227 | 
228 |     async def _do_inner():
229 |         with pytest.raises(InvalidDataFromChild):
230 |             async with open_in_proc(raise_) as proc:
231 |                 await proc.wait_result_or_raise()
232 | 
233 |     await asyncio.wait_for(_do_inner(), timeout=1)
234 | 
235 | 
236 | @pytest.mark.asyncio
237 | async def test_cancelled_error_in_child():
238 |     # An asyncio.CancelledError from the child process will be converted into a ChildCancelled.
239 |     async def raise_err():
240 |         await asyncio.sleep(0.01)
241 |         raise asyncio.CancelledError()
242 | 
243 |     async def _do_inner():
244 |         async with open_in_process(raise_err) as proc:
245 |             await proc.wait_result_or_raise()
246 | 
247 |     with pytest.raises(ChildCancelled):
248 |         await asyncio.wait_for(_do_inner(), timeout=1)
249 | 
250 | 
251 | @pytest.mark.asyncio
252 | async def test_task_cancellation(monkeypatch):
253 |     # If the task executing open_in_process() is cancelled, we will ask the child proc to
254 |     # terminate and propagate the CancelledError.
255 | 
256 |     async def store_received_signals():
257 |         # Return only when we receive a SIGTERM, also checking that we received a SIGINT before
258 |         # the SIGTERM.
259 |         received_signals = []
260 |         loop = asyncio.get_event_loop()
261 |         for sig in [signal.SIGINT, signal.SIGTERM]:
262 |             loop.add_signal_handler(sig, received_signals.append, sig)
263 |         while True:
264 |             if signal.SIGTERM in received_signals:
265 |                 assert [signal.SIGINT, signal.SIGTERM] == received_signals
266 |                 return
267 |             await asyncio.sleep(0)
268 | 
269 |     child_started = asyncio.Event()
270 | 
271 |     async def runner():
272 |         async with open_in_process(store_received_signals) as proc:
273 |             child_started.set()
274 |             await proc.wait_result_or_raise()
275 | 
276 |     monkeypatch.setattr(constants, 'SIGINT_TIMEOUT_SECONDS', 0.2)
277 |     monkeypatch.setattr(constants, 'SIGTERM_TIMEOUT_SECONDS', 0.2)
278 |     task = asyncio.ensure_future(runner())
279 |     await asyncio.wait_for(child_started.wait(), timeout=1)
280 |     assert not task.done()
281 |     task.cancel()
282 |     # For some reason, using pytest.raises() here doesn't seem to prevent the
283 |     # asyncio.CancelledError from closing the event loop, causing subsequent tests to fail.
284 |     raised_cancelled_error = False
285 |     try:
286 |         await asyncio.wait_for(task, timeout=1)
287 |     except asyncio.CancelledError:
288 |         raised_cancelled_error = True
289 |     assert raised_cancelled_error
290 | 
291 | 
292 | @pytest.mark.asyncio
293 | async def test_timeout_waiting_for_executing_state(open_in_proc, monkeypatch):
294 |     async def wait_for_state(self, state):
295 |         if state is State.EXECUTING:
296 |             await asyncio.sleep(constants.STARTUP_TIMEOUT_SECONDS + 0.1)
297 | 
298 |     monkeypatch.setattr(Process, 'wait_for_state', wait_for_state)
299 |     monkeypatch.setattr(constants, 'STARTUP_TIMEOUT_SECONDS', 1)
300 | 
301 |     async def do_sleep_forever():
302 |         while True:
303 |             await sleep(0.1)
304 | 
305 |     with pytest.raises(asyncio.TimeoutError):
306 |         async with open_in_proc(do_sleep_forever):
307 |             pass
308 | 
309 | 
310 | @pytest.mark.asyncio
311 | async def test_timeout_waiting_for_pid(open_in_proc, monkeypatch):
312 |     async def wait_pid(self):
313 |         await asyncio.sleep(constants.STARTUP_TIMEOUT_SECONDS + 0.1)
314 | 
315 |     monkeypatch.setattr(Process, 'wait_pid', wait_pid)
316 |     monkeypatch.setattr(constants, 'STARTUP_TIMEOUT_SECONDS', 1)
317 | 
318 |     async def do_sleep_forever():
319 |         while True:
320 |             await sleep(0.1)
321 | 
322 |     with pytest.raises(asyncio.TimeoutError):
323 |         async with open_in_proc(do_sleep_forever):
324 |             pass
325 | 
326 | 
327 | @pytest.mark.asyncio
328 | async def test_max_processes(monkeypatch, open_in_proc):
329 |     async def do_sleep_forever():
330 |         while True:
331 |             await sleep(0.2)
332 | 
333 |     max_procs = 4
334 |     executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_procs)
335 |     # We need to monkeypatch _get_executor() instead of setting the
336 |     # ASYNCIO_RUN_IN_PROCESS_MAX_PROCS environment variable because if another test runs before us
337 |     # it will cause _get_executor() to store a global executor created using the default number of
338 |     # max procs and then when it got called again here it'd reuse that executor.
339 |     monkeypatch.setattr(main, '_get_executor', lambda: executor)
340 |     monkeypatch.setenv('ASYNCIO_RUN_IN_PROCESS_STARTUP_TIMEOUT', str(1))
341 | 
342 |     above_limit_proc_created = False
343 |     procs = []
344 |     async with AsyncExitStack() as stack:
345 |         for _ in range(max_procs):
346 |             proc = await stack.enter_async_context(open_in_proc(do_sleep_forever))
347 |             procs.append(proc)
348 | 
349 |         for proc in procs:
350 |             assert proc.state is State.EXECUTING
351 | 
352 |         try:
353 |             async with open_in_proc(do_sleep_forever) as proc:
354 |                 # This should not execute as the above should raise a TimeoutError, but in case it
355 |                 # doesn't happen we need to ensure the proc is terminated so we can leave the
356 |                 # context and fail the test below.
357 |                 proc.send_signal(signal.SIGINT)
358 |         except asyncio.TimeoutError:
359 |             pass
360 |         else:
361 |             above_limit_proc_created = True
362 |         finally:
363 |             for proc in procs:
364 |                 proc.send_signal(signal.SIGINT)
365 | 
366 |     # We want to fail the test only after we leave the AsyncExitStack(), or else it will pass the
367 |     # exception along when returning control to open_in_proc(), which will interpret it as a
368 |     # failure of the process to exit and send a SIGKILL (together with a warning).
369 |     if above_limit_proc_created:
370 |         raise AssertionError("This process must not have been created successfully")
371 | 


--------------------------------------------------------------------------------
/tests/core/test_process_object.py:
--------------------------------------------------------------------------------
  1 | import asyncio
  2 | 
  3 | import pytest
  4 | 
  5 | from asyncio_run_in_process import (
  6 |     State,
  7 |     open_in_process,
  8 | )
  9 | 
 10 | 
 11 | @pytest.mark.asyncio
 12 | async def test_Process_object_state_api():
 13 |     async def return7():
 14 |         return 7
 15 | 
 16 |     async with open_in_process(return7) as proc:
 17 |         assert proc.state.is_on_or_after(State.STARTED)
 18 | 
 19 |         await asyncio.wait_for(proc.wait_for_state(State.FINISHED), timeout=2)
 20 |         assert proc.state is State.FINISHED
 21 |         assert proc.return_value == 7
 22 | 
 23 | 
 24 | @pytest.mark.asyncio
 25 | async def test_Process_object_wait_for_return_value():
 26 |     async def return7():
 27 |         return 7
 28 | 
 29 |     async with open_in_process(return7) as proc:
 30 |         await asyncio.wait_for(proc.wait_return_value(), timeout=2)
 31 |         assert proc.return_value == 7
 32 | 
 33 | 
 34 | @pytest.mark.asyncio
 35 | async def test_Process_object_wait_for_pid():
 36 |     async def return7():
 37 |         return 7
 38 | 
 39 |     async with open_in_process(return7) as proc:
 40 |         await asyncio.wait_for(proc.wait_pid(), timeout=2)
 41 |         assert isinstance(proc.pid, int)
 42 | 
 43 | 
 44 | @pytest.mark.asyncio
 45 | async def test_Process_object_wait_for_returncode():
 46 |     async def system_exit_123():
 47 |         raise SystemExit(123)
 48 | 
 49 |     async with open_in_process(system_exit_123) as proc:
 50 |         await asyncio.wait_for(proc.wait_returncode(), timeout=2)
 51 |         assert proc.returncode == 123
 52 | 
 53 | 
 54 | @pytest.mark.asyncio
 55 | async def test_Process_object_wait_for_error():
 56 |     async def raise_error():
 57 |         raise ValueError("child-error")
 58 | 
 59 |     async with open_in_process(raise_error) as proc:
 60 |         await asyncio.wait_for(proc.wait_error(), timeout=2)
 61 |         assert isinstance(proc.error, ValueError)
 62 | 
 63 | 
 64 | @pytest.mark.asyncio
 65 | async def test_Process_object_wait_for_result_when_error():
 66 |     async def raise_error():
 67 |         raise ValueError("child-error")
 68 | 
 69 |     async with open_in_process(raise_error) as proc:
 70 |         with pytest.raises(ValueError, match="child-error"):
 71 |             await asyncio.wait_for(proc.wait_result_or_raise(), timeout=2)
 72 | 
 73 | 
 74 | @pytest.mark.asyncio
 75 | async def test_Process_object_wait_for_result_when_return_value():
 76 |     async def return7():
 77 |         return 7
 78 | 
 79 |     async with open_in_process(return7) as proc:
 80 |         result = await asyncio.wait_for(proc.wait_result_or_raise(), timeout=2)
 81 |         assert result == 7
 82 |         assert proc.error is None
 83 | 
 84 | 
 85 | @pytest.mark.asyncio
 86 | async def test_Process_object_wait_when_return_value():
 87 |     async def return7():
 88 |         return 7
 89 | 
 90 |     async with open_in_process(return7) as proc:
 91 |         await asyncio.wait_for(proc.wait(), timeout=2)
 92 |         assert proc.return_value == 7
 93 |         assert proc.error is None
 94 | 
 95 | 
 96 | @pytest.mark.asyncio
 97 | async def test_Process_object_wait_when_error():
 98 |     async def raise_error():
 99 |         raise ValueError("child-error")
100 | 
101 |     async with open_in_process(raise_error) as proc:
102 |         await asyncio.wait_for(proc.wait(), timeout=2)
103 |         assert isinstance(proc.error, ValueError)
104 | 


--------------------------------------------------------------------------------
/tests/core/test_remote_exception.py:
--------------------------------------------------------------------------------
 1 | import logging
 2 | import os
 3 | import sys
 4 | 
 5 | import cloudpickle
 6 | 
 7 | from asyncio_run_in_process._utils import (
 8 |     RemoteException,
 9 | )
10 | 
11 | 
12 | def test_RemoteException(caplog):
13 |     logger = logging.getLogger('asyncio_run_in_process.testing')
14 | 
15 |     def recorder_func():
16 |         try:
17 |             outer_func()
18 |         except Exception:
19 |             _, exc_value, exc_tb = sys.exc_info()
20 |             remote_err = RemoteException(exc_value, exc_tb)
21 | 
22 |         local_err = cloudpickle.loads(cloudpickle.dumps(remote_err))
23 | 
24 |         try:
25 |             raise local_err
26 |         except BaseException:
27 |             logger.debug("Got Error:", exc_info=True)
28 | 
29 |         return local_err
30 | 
31 |     def outer_func():
32 |         inner_func()
33 | 
34 |     def inner_func():
35 |         raise ValueError("Some Error")
36 | 
37 |     with caplog.at_level(logging.DEBUG):
38 |         local_err = recorder_func()
39 |     assert isinstance(local_err, ValueError)
40 |     assert "Some Error" in caplog.text
41 |     assert "outer_func" in caplog.text
42 |     assert "inner_func" in caplog.text
43 |     assert str(os.getpid()) in caplog.text
44 | 


--------------------------------------------------------------------------------
/tests/core/test_run_in_process.py:
--------------------------------------------------------------------------------
 1 | import asyncio
 2 | import os
 3 | from pathlib import (
 4 |     Path,
 5 | )
 6 | 
 7 | import pytest
 8 | 
 9 | 
10 | @pytest.mark.asyncio
11 | async def test_run_in_process_touch_file(touch_path, runner):
12 |     async def touch_file(path: Path):
13 |         path.touch()
14 | 
15 |     assert not touch_path.exists()
16 |     await asyncio.wait_for(runner(touch_file, touch_path), timeout=2)
17 |     assert touch_path.exists()
18 | 
19 | 
20 | @pytest.mark.asyncio
21 | async def test_run_in_process_with_result(runner):
22 |     async def return7():
23 |         return 7
24 | 
25 |     result = await asyncio.wait_for(runner(return7), timeout=2)
26 |     assert result == 7
27 | 
28 | 
29 | @pytest.mark.asyncio
30 | async def test_run_in_process_with_error(runner):
31 |     async def raise_err():
32 |         raise ValueError("Some err")
33 | 
34 |     with pytest.raises(ValueError, match="Some err"):
35 |         await asyncio.wait_for(runner(raise_err), timeout=2)
36 | 
37 | 
38 | @pytest.mark.asyncio
39 | async def test_run_in_process_pass_environment_variables(runner):
40 |     # sanity
41 |     assert 'ASYNC_RUN_IN_PROCESS_ENV_TEST' not in os.environ
42 | 
43 |     async def return_env():
44 |         return os.environ['ASYNC_RUN_IN_PROCESS_ENV_TEST']
45 | 
46 |     value = await runner(
47 |         return_env,
48 |         subprocess_kwargs={'env': {'ASYNC_RUN_IN_PROCESS_ENV_TEST': 'test-value'}},
49 |     )
50 |     assert value == 'test-value'
51 | 


--------------------------------------------------------------------------------
/tests/core/test_state.py:
--------------------------------------------------------------------------------
 1 | import pytest
 2 | 
 3 | from asyncio_run_in_process.state import (
 4 |     State,
 5 | )
 6 | 
 7 | 
 8 | @pytest.mark.parametrize('state', State)
 9 | def test_state_properties(state):
10 |     all_states = tuple(State)
11 |     before_states = all_states[:state]
12 |     after_states = all_states[state.value + 1:]
13 | 
14 |     assert state.is_on_or_after(state)
15 |     assert not state.is_before(state)
16 |     assert not state.is_next(state)
17 | 
18 |     assert all(other.is_before(state) for other in before_states)
19 |     assert all(state.is_on_or_after(other) for other in before_states)
20 |     if before_states:
21 |         assert before_states[-1].is_next(state)
22 | 
23 |     assert all(other.is_on_or_after(state) for other in after_states)
24 |     assert all(state.is_before(other) for other in after_states)
25 | 
26 |     if after_states:
27 |         assert state.is_next(after_states[0])
28 | 


--------------------------------------------------------------------------------
/tox.ini:
--------------------------------------------------------------------------------
 1 | [tox]
 2 | envlist=
 3 |     py{36,37,38}-core
 4 |     lint
 5 |     doctest
 6 | 
 7 | [isort]
 8 | combine_as_imports=True
 9 | force_sort_within_sections=True
10 | include_trailing_comma=True
11 | known_third_party=hypothesis,pytest
12 | known_first_party=asyncio_run_in_process
13 | line_length=21
14 | multi_line_output=3
15 | use_parentheses=True
16 | 
17 | [flake8]
18 | max-line-length= 100
19 | exclude= venv*,.tox,docs,build
20 | ignore=
21 | 
22 | [testenv]
23 | usedevelop=True
24 | commands=
25 |     core: pytest {posargs:tests/core}
26 |     doctest: make -C {toxinidir}/docs doctest
27 | basepython =
28 |     doctest: python
29 |     py36: python3.6
30 |     py37: python3.7
31 |     py38: python3.8
32 | extras=
33 |     test
34 |     doctest: doc
35 | whitelist_externals=make
36 | 
37 | [testenv:lint]
38 | basepython=python
39 | extras=lint
40 | commands=
41 |     mypy -p asyncio_run_in_process --config-file {toxinidir}/mypy.ini
42 |     flake8 {toxinidir}/asyncio_run_in_process {toxinidir}/tests
43 |     isort --recursive --check-only --diff {toxinidir}/asyncio_run_in_process {toxinidir}/tests
44 |     pydocstyle {toxinidir}/asyncio_run_in_process {toxinidir}/tests
45 | 


--------------------------------------------------------------------------------