├── tests ├── __init__.py ├── devx │ └── __init__.py ├── ipc │ ├── __init__.py │ ├── test_server.py │ ├── test_multi_tpt.py │ └── test_each_tpt.py ├── test_runtime.py ├── test_multi_program.py ├── test_clustering.py ├── test_local.py ├── test_root_runtime.py ├── test_2way.py ├── test_shm.py ├── test_docs_examples.py ├── test_rpc.py └── test_child_manages_service_nursery.py ├── examples ├── __init__.py ├── debugging │ ├── debug_mode_hang.py │ ├── root_actor_error.py │ ├── root_actor_breakpoint.py │ ├── root_actor_breakpoint_forever.py │ ├── subactor_breakpoint.py │ ├── per_actor_debug.py │ ├── root_timeout_while_child_crashed.py │ ├── subactor_error.py │ ├── open_ctx_modnofound.py │ ├── subactor_bp_in_ctx.py │ ├── fast_error_in_root_after_spawn.py │ ├── root_self_cancelled_w_error.py │ ├── multi_daemon_subactors.py │ ├── multi_subactors.py │ ├── pm_in_subactor.py │ ├── restore_builtin_breakpoint.py │ ├── multi_subactor_root_errors.py │ ├── root_cancelled_but_child_is_in_tty_lock.py │ ├── shield_hang_in_sub.py │ ├── shielded_pause.py │ ├── multi_nested_subactors_error_up_through_nurseries.py │ └── asyncio_bp.py ├── service_discovery.py ├── actor_spawning_and_causality.py ├── __main__.py ├── remote_error_propagation.py ├── parallelism │ ├── single_func.py │ ├── we_are_processes.py │ ├── _concurrent_futures_primes.py │ └── concurrent_actors_primes.py ├── actor_spawning_and_causality_with_daemon.py ├── quick_cluster.py ├── asynchronous_generators.py ├── integration │ └── open_context_and_sleep.py ├── a_trynamic_first_scene.py ├── multiple_streams_one_portal.py ├── rpc_bidir_streaming.py ├── trio │ └── lockacquire_not_unmasked.py ├── infected_asyncio_echo_server.py └── full_fledged_streaming_service.py ├── nooz ├── .gitignore ├── 336.trivial.rst ├── 324.bugfix.rst ├── 335.trivial.rst ├── 356.trivial.rst ├── HOWTO.rst ├── 344.bugfix.rst ├── 349.trivial.rst ├── 358.feature.rst ├── 346.bugfix.rst ├── 322.trivial.rst ├── 343.trivial.rst ├── _template.rst ├── 333.feature.rst └── 337.feature.rst ├── mypy.ini ├── docs ├── mk_gh_readme.sh ├── Makefile ├── github_readme │ ├── conf.py │ └── _sphinx_readme.rst ├── dev_tips.rst └── conf.py ├── MANIFEST.in ├── pytest.ini ├── default.nix ├── notes_to_self └── howtorelease.md ├── tractor ├── _testing │ ├── samples.py │ ├── addr.py │ ├── fault_simulation.py │ └── __init__.py ├── ipc │ ├── __init__.py │ ├── _mp_bs.py │ ├── _types.py │ ├── _fd_share.py │ └── _linux.py ├── experimental │ └── __init__.py ├── trionics │ └── __init__.py ├── _child.py ├── msg │ ├── __init__.py │ ├── _exts.py │ └── ptr.py ├── __init__.py ├── _clustering.py ├── devx │ ├── __init__.py │ ├── debug │ │ └── __init__.py │ └── cli.py ├── _multiaddr.py ├── _mp_fixup_main.py └── _entry.py ├── .gitignore ├── ruff.toml ├── pyproject.toml └── .github └── workflows └── ci.yml /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /examples/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/devx/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /nooz/.gitignore: -------------------------------------------------------------------------------- 1 | !.gitignore 2 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | plugins = trio_typing.plugin 3 | -------------------------------------------------------------------------------- /tests/ipc/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | `tractor.ipc` subsystem(s)/unit testing suites. 3 | 4 | ''' 5 | -------------------------------------------------------------------------------- /docs/mk_gh_readme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sphinx-build -b rst ./github_readme ./ 3 | 4 | mv _sphinx_readme.rst _README.rst 5 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # https://packaging.python.org/en/latest/guides/using-manifest-in/#using-manifest-in 2 | include docs/README.rst 3 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | # vim: ft=ini 2 | # pytest.ini for tractor 3 | 4 | [pytest] 5 | # don't show frickin captured logs AGAIN in the report.. 6 | addopts = --show-capture='no' 7 | log_cli = false 8 | ; minversion = 6.0 9 | -------------------------------------------------------------------------------- /examples/debugging/debug_mode_hang.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Reproduce a bug where enabling debug mode for a sub-actor actually causes 3 | a hang on teardown... 4 | 5 | ''' 6 | import asyncio 7 | 8 | import trio 9 | import tractor 10 | -------------------------------------------------------------------------------- /nooz/336.trivial.rst: -------------------------------------------------------------------------------- 1 | Add ``key: Callable[..., Hashable]`` support to ``.trionics.maybe_open_context()`` 2 | 3 | Gives users finer grained control over cache hit behaviour using 4 | a callable which receives the input ``kwargs: dict``. 5 | -------------------------------------------------------------------------------- /nooz/324.bugfix.rst: -------------------------------------------------------------------------------- 1 | Only set `._debug.Lock.local_pdb_complete` if has been created. 2 | 3 | This can be triggered by a very rare race condition (and thus we have no 4 | working test yet) but it is known to exist in (a) consumer project(s). 5 | -------------------------------------------------------------------------------- /examples/debugging/root_actor_error.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | async def main(): 6 | async with tractor.open_root_actor( 7 | debug_mode=True, 8 | ): 9 | assert 0 10 | 11 | 12 | if __name__ == '__main__': 13 | trio.run(main) 14 | -------------------------------------------------------------------------------- /nooz/335.trivial.rst: -------------------------------------------------------------------------------- 1 | Establish an explicit "backend spawning" method table; use it from CI 2 | 3 | More clearly lays out the current set of (3) backends: ``['trio', 4 | 'mp_spawn', 'mp_forkserver']`` and adjusts the ``._spawn.py`` internals 5 | as well as the test suite to accommodate. 6 | -------------------------------------------------------------------------------- /examples/debugging/root_actor_breakpoint.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | async def main(): 6 | 7 | async with tractor.open_root_actor( 8 | debug_mode=True, 9 | ): 10 | 11 | await trio.sleep(0.1) 12 | 13 | await tractor.pause() 14 | 15 | await trio.sleep(0.1) 16 | 17 | 18 | if __name__ == '__main__': 19 | trio.run(main) 20 | -------------------------------------------------------------------------------- /default.nix: -------------------------------------------------------------------------------- 1 | { pkgs ? import {} }: 2 | let 3 | nativeBuildInputs = with pkgs; [ 4 | stdenv.cc.cc.lib 5 | uv 6 | ]; 7 | 8 | in 9 | pkgs.mkShell { 10 | inherit nativeBuildInputs; 11 | 12 | LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath nativeBuildInputs; 13 | TMPDIR = "/tmp"; 14 | 15 | shellHook = '' 16 | set -e 17 | uv venv .venv --python=3.12 18 | ''; 19 | } 20 | -------------------------------------------------------------------------------- /nooz/356.trivial.rst: -------------------------------------------------------------------------------- 1 | Drop `trio.Process.aclose()` usage, copy into our spawning code. 2 | 3 | The details are laid out in https://github.com/goodboy/tractor/issues/330. 4 | `trio` changed is process running quite some time ago, this just copies 5 | out the small bit we needed (from the old `.aclose()`) for hard kills 6 | where a soft runtime cancel request fails and our "zombie killer" 7 | implementation kicks in. 8 | -------------------------------------------------------------------------------- /examples/debugging/root_actor_breakpoint_forever.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | async def main( 6 | registry_addrs: tuple[str, int]|None = None 7 | ): 8 | 9 | async with tractor.open_root_actor( 10 | debug_mode=True, 11 | # loglevel='runtime', 12 | ): 13 | while True: 14 | await tractor.pause() 15 | 16 | 17 | if __name__ == '__main__': 18 | trio.run(main) 19 | -------------------------------------------------------------------------------- /notes_to_self/howtorelease.md: -------------------------------------------------------------------------------- 1 | First generate a built disti: 2 | 3 | ``` 4 | python -m pip install --upgrade build 5 | python -m build --sdist --outdir dist/alpha5/ 6 | ``` 7 | 8 | Then try a test ``pypi`` upload: 9 | 10 | ``` 11 | python -m twine upload --repository testpypi dist/alpha5/* 12 | ``` 13 | 14 | The push to `pypi` for realz. 15 | 16 | ``` 17 | python -m twine upload --repository testpypi dist/alpha5/* 18 | ``` 19 | -------------------------------------------------------------------------------- /nooz/HOWTO.rst: -------------------------------------------------------------------------------- 1 | See both the `towncrier docs`_ and the `pluggy release readme`_ for hot 2 | tips. We basically have the most minimal setup and release process right 3 | now and use the default `fragment set`_. 4 | 5 | 6 | .. _towncrier docs: https://github.com/twisted/towncrier#quick-start 7 | .. _pluggy release readme: https://github.com/pytest-dev/pluggy/blob/main/changelog/README.rst 8 | .. _fragment set: https://github.com/twisted/towncrier#news-fragments 9 | -------------------------------------------------------------------------------- /examples/service_discovery.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | tractor.log.get_console_log("INFO") 5 | 6 | 7 | async def main(service_name): 8 | 9 | async with tractor.open_nursery() as an: 10 | await an.start_actor(service_name) 11 | 12 | async with tractor.get_registry() as portal: 13 | print(f"Arbiter is listening on {portal.channel}") 14 | 15 | async with tractor.wait_for_actor(service_name) as sockaddr: 16 | print(f"my_service is found at {sockaddr}") 17 | 18 | await an.cancel() 19 | 20 | 21 | if __name__ == '__main__': 22 | trio.run(main, 'some_actor_name') 23 | -------------------------------------------------------------------------------- /nooz/344.bugfix.rst: -------------------------------------------------------------------------------- 1 | Always ``list``-cast the ``mngrs`` input to 2 | ``.trionics.gather_contexts()`` and ensure its size otherwise raise 3 | a ``ValueError``. 4 | 5 | Turns out that trying to pass an inline-style generator comprehension 6 | doesn't seem to work inside the ``async with`` expression? Further, in 7 | such a case we can get a hang waiting on the all-entered event 8 | completion when the internal mngrs iteration is a noop. Instead we 9 | always greedily check a size and error on empty input; the lazy 10 | iteration of a generator input is not beneficial anyway since we're 11 | entering all manager instances in concurrent tasks. 12 | -------------------------------------------------------------------------------- /nooz/349.trivial.rst: -------------------------------------------------------------------------------- 1 | Always redraw the `pdbpp` prompt on `SIGINT` during REPL use. 2 | 3 | There was recent changes todo with Python 3.10 that required us to pin 4 | to a specific commit in `pdbpp` which have recently been fixed minus 5 | this last issue with `SIGINT` shielding: not clobbering or not 6 | showing the `(Pdb++)` prompt on ctlr-c by the user. This repairs all 7 | that by firstly removing the standard KBI intercepting of the std lib's 8 | `pdb.Pdb._cmdloop()` as well as ensuring that only the actor with REPL 9 | control ever reports `SIGINT` handler log msgs and prompt redraws. With 10 | this we move back to using pypi `pdbpp` release. 11 | -------------------------------------------------------------------------------- /examples/debugging/subactor_breakpoint.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | async def breakpoint_forever(): 6 | ''' 7 | Indefinitely re-enter debugger in child actor. 8 | 9 | ''' 10 | while True: 11 | await trio.sleep(0.1) 12 | await tractor.pause() 13 | 14 | 15 | async def main(): 16 | 17 | async with tractor.open_nursery( 18 | debug_mode=True, 19 | loglevel='cancel', 20 | ) as n: 21 | 22 | portal = await n.run_in_actor( 23 | breakpoint_forever, 24 | ) 25 | await portal.result() 26 | 27 | 28 | if __name__ == '__main__': 29 | trio.run(main) 30 | -------------------------------------------------------------------------------- /nooz/358.feature.rst: -------------------------------------------------------------------------------- 1 | Switch to using the fork & fix of `pdb++`, `pdbp`: 2 | https://github.com/mdmintz/pdbp 3 | 4 | Allows us to sidestep a variety of issues that aren't being maintained 5 | in the upstream project thanks to the hard work of @mdmintz! 6 | 7 | We also include some default settings adjustments as per recent 8 | development on the fork: 9 | 10 | - sticky mode is still turned on by default but now activates when 11 | a using the `ll` repl command. 12 | - turn off line truncation by default to avoid inter-line gaps when 13 | resizing the terimnal during use. 14 | - when using the backtrace cmd either by `w` or `bt`, the config 15 | automatically switches to non-sticky mode. 16 | -------------------------------------------------------------------------------- /examples/actor_spawning_and_causality.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | async def cellar_door(): 6 | assert not tractor.is_root_process() 7 | return "Dang that's beautiful" 8 | 9 | 10 | async def main(): 11 | """The main ``tractor`` routine. 12 | """ 13 | async with tractor.open_nursery() as n: 14 | 15 | portal = await n.run_in_actor( 16 | cellar_door, 17 | name='some_linguist', 18 | ) 19 | 20 | # The ``async with`` will unblock here since the 'some_linguist' 21 | # actor has completed its main task ``cellar_door``. 22 | 23 | print(await portal.result()) 24 | 25 | 26 | if __name__ == '__main__': 27 | trio.run(main) 28 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /examples/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Needed on Windows. 3 | 4 | This module is needed as the program entry point for invocation 5 | with ``python -m ``. See the solution from @chrizzFTD 6 | here: 7 | 8 | https://github.com/goodboy/tractor/pull/61#issuecomment-470053512 9 | 10 | """ 11 | if __name__ == '__main__': 12 | import multiprocessing 13 | multiprocessing.freeze_support() 14 | # ``tests/test_docs_examples.py::test_example`` will copy each 15 | # script from this examples directory into a module in a new 16 | # temporary dir and name it test_example.py. We import that script 17 | # module here and invoke it's ``main()``. 18 | from . import test_example 19 | test_example.trio.run(test_example.main) 20 | -------------------------------------------------------------------------------- /examples/debugging/per_actor_debug.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | async def die(): 5 | raise RuntimeError 6 | 7 | 8 | async def main(): 9 | async with tractor.open_nursery() as tn: 10 | 11 | debug_actor = await tn.start_actor( 12 | 'debugged_boi', 13 | enable_modules=[__name__], 14 | debug_mode=True, 15 | ) 16 | crash_boi = await tn.start_actor( 17 | 'crash_boi', 18 | enable_modules=[__name__], 19 | # debug_mode=True, 20 | ) 21 | 22 | async with trio.open_nursery() as n: 23 | n.start_soon(debug_actor.run, die) 24 | n.start_soon(crash_boi.run, die) 25 | 26 | 27 | if __name__ == '__main__': 28 | trio.run(main) 29 | -------------------------------------------------------------------------------- /nooz/346.bugfix.rst: -------------------------------------------------------------------------------- 1 | Fixes to ensure IPC (channel) breakage doesn't result in hung actor 2 | trees; the zombie reaping and general supervision machinery will always 3 | clean up and terminate. 4 | 5 | This includes not only the (mostly minor) fixes to solve these cases but 6 | also a new extensive test suite in `test_advanced_faults.py` with an 7 | accompanying highly configurable example module-script in 8 | `examples/advanced_faults/ipc_failure_during_stream.py`. Tests ensure we 9 | never get hang or zombies despite operating in debug mode and attempt to 10 | simulate all possible IPC transport failure cases for a local-host actor 11 | tree. 12 | 13 | Further we simplify `Context.open_stream.__aexit__()` to just call 14 | `MsgStream.aclose()` directly more or less avoiding a pure duplicate 15 | code path. 16 | -------------------------------------------------------------------------------- /examples/debugging/root_timeout_while_child_crashed.py: -------------------------------------------------------------------------------- 1 | 2 | import trio 3 | import tractor 4 | 5 | 6 | async def key_error(): 7 | "Raise a ``NameError``" 8 | return {}['doggy'] 9 | 10 | 11 | async def main(): 12 | """Root dies 13 | 14 | """ 15 | async with tractor.open_nursery( 16 | debug_mode=True, 17 | loglevel='debug' 18 | ) as n: 19 | 20 | # spawn both actors 21 | portal = await n.run_in_actor(key_error) 22 | 23 | # XXX: originally a bug caused by this is where root would enter 24 | # the debugger and clobber the tty used by the repl even though 25 | # child should have it locked. 26 | with trio.fail_after(1): 27 | await trio.Event().wait() 28 | 29 | 30 | if __name__ == '__main__': 31 | trio.run(main) 32 | -------------------------------------------------------------------------------- /tractor/_testing/samples.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | 4 | 5 | def generate_sample_messages( 6 | amount: int, 7 | rand_min: int = 0, 8 | rand_max: int = 0, 9 | silent: bool = False 10 | ) -> tuple[list[bytes], int]: 11 | 12 | msgs = [] 13 | size = 0 14 | 15 | if not silent: 16 | print(f'\ngenerating {amount} messages...') 17 | 18 | for i in range(amount): 19 | msg = f'[{i:08}]'.encode('utf-8') 20 | 21 | if rand_max > 0: 22 | msg += os.urandom( 23 | random.randint(rand_min, rand_max)) 24 | 25 | size += len(msg) 26 | 27 | msgs.append(msg) 28 | 29 | if not silent and i and i % 10_000 == 0: 30 | print(f'{i} generated') 31 | 32 | if not silent: 33 | print(f'done, {size:,} bytes in total') 34 | 35 | return msgs, size 36 | -------------------------------------------------------------------------------- /examples/remote_error_propagation.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | async def assert_err(): 6 | assert 0 7 | 8 | 9 | async def main(): 10 | async with tractor.open_nursery() as n: 11 | real_actors = [] 12 | for i in range(3): 13 | real_actors.append(await n.start_actor( 14 | f'actor_{i}', 15 | enable_modules=[__name__], 16 | )) 17 | 18 | # start one actor that will fail immediately 19 | await n.run_in_actor(assert_err) 20 | 21 | # should error here with a ``RemoteActorError`` containing 22 | # an ``AssertionError`` and all the other actors have been cancelled 23 | 24 | 25 | if __name__ == '__main__': 26 | try: 27 | # also raises 28 | trio.run(main) 29 | except tractor.RemoteActorError: 30 | print("Look Maa that actor failed hard, hehhh!") 31 | -------------------------------------------------------------------------------- /examples/debugging/subactor_error.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | async def name_error(): 6 | getattr(doggypants) # noqa (on purpose) 7 | 8 | 9 | async def main(): 10 | async with tractor.open_nursery( 11 | debug_mode=True, 12 | # loglevel='transport', 13 | ) as an: 14 | 15 | # TODO: ideally the REPL arrives at this frame in the parent, 16 | # ABOVE the @api_frame of `Portal.run_in_actor()` (which 17 | # should eventually not even be a portal method ... XD) 18 | # await tractor.pause() 19 | p: tractor.Portal = await an.run_in_actor(name_error) 20 | 21 | # with this style, should raise on this line 22 | await p.result() 23 | 24 | # with this alt style should raise at `open_nusery()` 25 | # return await p.result() 26 | 27 | 28 | if __name__ == '__main__': 29 | trio.run(main) 30 | -------------------------------------------------------------------------------- /examples/debugging/open_ctx_modnofound.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | @tractor.context 6 | async def just_sleep( 7 | 8 | ctx: tractor.Context, 9 | **kwargs, 10 | 11 | ) -> None: 12 | ''' 13 | Start and sleep. 14 | 15 | ''' 16 | await ctx.started() 17 | await trio.sleep_forever() 18 | 19 | 20 | async def main() -> None: 21 | 22 | async with tractor.open_nursery( 23 | debug_mode=True, 24 | ) as n: 25 | portal = await n.start_actor( 26 | 'ctx_child', 27 | 28 | # XXX: we don't enable the current module in order 29 | # to trigger `ModuleNotFound`. 30 | enable_modules=[], 31 | ) 32 | 33 | async with portal.open_context( 34 | just_sleep, # taken from pytest parameterization 35 | ) as (ctx, sent): 36 | raise KeyboardInterrupt 37 | 38 | 39 | if __name__ == '__main__': 40 | trio.run(main) 41 | -------------------------------------------------------------------------------- /nooz/322.trivial.rst: -------------------------------------------------------------------------------- 1 | Strictly support Python 3.10+, start runtime machinery reorg 2 | 3 | Since we want to push forward using the new `match:` syntax for our 4 | internal RPC-msg loops, we officially drop 3.9 support for the next 5 | release which should coincide well with the first release of 3.11. 6 | 7 | This patch set also officially removes the ``tractor.run()`` API (which 8 | has been deprecated for some time) as well as starts an initial re-org 9 | of the internal runtime core by: 10 | - renaming ``tractor._actor`` -> ``._runtime`` 11 | - moving the ``._runtime.ActorActor._process_messages()`` and 12 | ``._async_main()`` to be module level singleton-task-functions since 13 | they are only started once for each connection and actor spawn 14 | respectively; this internal API thus looks more similar to (at the 15 | time of writing) the ``trio``-internals in ``trio._core._run``. 16 | - officially remove ``tractor.run()``, now deprecated for some time. 17 | -------------------------------------------------------------------------------- /examples/parallelism/single_func.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run with a process monitor from a terminal using:: 3 | 4 | $TERM -e watch -n 0.1 "pstree -a $$" \ 5 | & python examples/parallelism/single_func.py \ 6 | && kill $! 7 | 8 | """ 9 | import os 10 | 11 | import tractor 12 | import trio 13 | 14 | 15 | async def burn_cpu(): 16 | 17 | pid = os.getpid() 18 | 19 | # burn a core @ ~ 50kHz 20 | for _ in range(50000): 21 | await trio.sleep(1/50000/50) 22 | 23 | return os.getpid() 24 | 25 | 26 | async def main(): 27 | 28 | async with tractor.open_nursery() as n: 29 | 30 | portal = await n.run_in_actor(burn_cpu) 31 | 32 | # burn rubber in the parent too 33 | await burn_cpu() 34 | 35 | # wait on result from target function 36 | pid = await portal.result() 37 | 38 | # end of nursery block 39 | print(f"Collected subproc {pid}") 40 | 41 | 42 | if __name__ == '__main__': 43 | trio.run(main) 44 | -------------------------------------------------------------------------------- /tractor/ipc/__init__.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | ''' 18 | A modular IPC layer supporting the power of cross-process SC! 19 | 20 | ''' 21 | from ._chan import ( 22 | _connect_chan as _connect_chan, 23 | Channel as Channel 24 | ) 25 | -------------------------------------------------------------------------------- /nooz/343.trivial.rst: -------------------------------------------------------------------------------- 1 | Rework our ``.trionics.BroadcastReceiver`` internals to avoid method 2 | recursion and approach a design and interface closer to ``trio``'s 3 | ``MemoryReceiveChannel``. 4 | 5 | The details of the internal changes include: 6 | 7 | - implementing a ``BroadcastReceiver.receive_nowait()`` and using it 8 | within the async ``.receive()`` thus avoiding recursion from 9 | ``.receive()``. 10 | - failing over to an internal ``._receive_from_underlying()`` when the 11 | ``_nowait()`` call raises ``trio.WouldBlock`` 12 | - adding ``BroadcastState.statistics()`` for debugging and testing both 13 | internals and by users. 14 | - add an internal ``BroadcastReceiver._raise_on_lag: bool`` which can be 15 | set to avoid ``Lagged`` raising for possible use cases where a user 16 | wants to choose between a [cheap or nasty 17 | pattern](https://zguide.zeromq.org/docs/chapter7/#The-Cheap-or-Nasty-Pattern) 18 | the the particular stream (we use this in ``piker``'s dark clearing 19 | engine to avoid fast feeds breaking during HFT periods). 20 | -------------------------------------------------------------------------------- /tractor/experimental/__init__.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | ''' 18 | Experimental APIs and subsystems not yet validated to be included as 19 | built-ins. 20 | 21 | This is a staging area for ``tractor.builtin``. 22 | 23 | ''' 24 | from ._pubsub import pub as msgpub 25 | 26 | 27 | __all__ = [ 28 | 'msgpub', 29 | ] 30 | -------------------------------------------------------------------------------- /nooz/_template.rst: -------------------------------------------------------------------------------- 1 | {% for section in sections %} 2 | {% set underline = "-" %} 3 | {% if section %} 4 | {{section}} 5 | {{ underline * section|length }}{% set underline = "~" %} 6 | 7 | {% endif %} 8 | {% if sections[section] %} 9 | {% for category, val in definitions.items() if category in sections[section] %} 10 | 11 | {{ definitions[category]['name'] }} 12 | {{ underline * definitions[category]['name']|length }} 13 | 14 | {% if definitions[category]['showcontent'] %} 15 | {% for text, values in sections[section][category]|dictsort(by='value') %} 16 | {% set issue_joiner = joiner(', ') %} 17 | - {% for value in values|sort %}{{ issue_joiner() }}`{{ value }} `_{% endfor %}: {{ text }} 18 | 19 | {% endfor %} 20 | {% else %} 21 | - {{ sections[section][category]['']|sort|join(', ') }} 22 | 23 | 24 | {% endif %} 25 | {% if sections[section][category]|length == 0 %} 26 | 27 | No significant changes. 28 | 29 | {% else %} 30 | {% endif %} 31 | {% endfor %} 32 | {% else %} 33 | 34 | No significant changes. 35 | 36 | {% endif %} 37 | {% endfor %} 38 | -------------------------------------------------------------------------------- /examples/actor_spawning_and_causality_with_daemon.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | async def movie_theatre_question(): 6 | """A question asked in a dark theatre, in a tangent 7 | (errr, I mean different) process. 8 | """ 9 | return 'have you ever seen a portal?' 10 | 11 | 12 | async def main(): 13 | """The main ``tractor`` routine. 14 | """ 15 | async with tractor.open_nursery() as n: 16 | 17 | portal = await n.start_actor( 18 | 'frank', 19 | # enable the actor to run funcs from this current module 20 | enable_modules=[__name__], 21 | ) 22 | 23 | print(await portal.run(movie_theatre_question)) 24 | # call the subactor a 2nd time 25 | print(await portal.run(movie_theatre_question)) 26 | 27 | # the async with will block here indefinitely waiting 28 | # for our actor "frank" to complete, but since it's an 29 | # "outlive_main" actor it will never end until cancelled 30 | await portal.cancel_actor() 31 | 32 | 33 | if __name__ == '__main__': 34 | trio.run(main) 35 | -------------------------------------------------------------------------------- /examples/parallelism/we_are_processes.py: -------------------------------------------------------------------------------- 1 | """ 2 | Run with a process monitor from a terminal using:: 3 | 4 | $TERM -e watch -n 0.1 "pstree -a $$" \ 5 | & python examples/parallelism/we_are_processes.py \ 6 | && kill $! 7 | 8 | """ 9 | from multiprocessing import cpu_count 10 | import os 11 | 12 | import tractor 13 | import trio 14 | 15 | 16 | async def target(): 17 | print( 18 | f"Yo, i'm '{tractor.current_actor().name}' " 19 | f"running in pid {os.getpid()}" 20 | ) 21 | 22 | await trio.sleep_forever() 23 | 24 | 25 | async def main(): 26 | 27 | async with tractor.open_nursery() as n: 28 | 29 | for i in range(cpu_count()): 30 | await n.run_in_actor(target, name=f'worker_{i}') 31 | 32 | print('This process tree will self-destruct in 1 sec...') 33 | await trio.sleep(1) 34 | 35 | # you could have done this yourself 36 | raise Exception('Self Destructed') 37 | 38 | 39 | if __name__ == '__main__': 40 | try: 41 | trio.run(main) 42 | except Exception: 43 | print('Zombies Contained') 44 | -------------------------------------------------------------------------------- /examples/parallelism/_concurrent_futures_primes.py: -------------------------------------------------------------------------------- 1 | import time 2 | import concurrent.futures 3 | import math 4 | 5 | PRIMES = [ 6 | 112272535095293, 7 | 112582705942171, 8 | 112272535095293, 9 | 115280095190773, 10 | 115797848077099, 11 | 1099726899285419] 12 | 13 | 14 | def is_prime(n): 15 | if n < 2: 16 | return False 17 | if n == 2: 18 | return True 19 | if n % 2 == 0: 20 | return False 21 | 22 | sqrt_n = int(math.floor(math.sqrt(n))) 23 | for i in range(3, sqrt_n + 1, 2): 24 | if n % i == 0: 25 | return False 26 | return True 27 | 28 | 29 | def main(): 30 | with concurrent.futures.ProcessPoolExecutor() as executor: 31 | start = time.time() 32 | 33 | for number, prime in zip(PRIMES, executor.map(is_prime, PRIMES)): 34 | print('%d is prime: %s' % (number, prime)) 35 | 36 | print(f'processing took {time.time() - start} seconds') 37 | 38 | 39 | if __name__ == '__main__': 40 | 41 | start = time.time() 42 | main() 43 | print(f'script took {time.time() - start} seconds') 44 | -------------------------------------------------------------------------------- /examples/quick_cluster.py: -------------------------------------------------------------------------------- 1 | 2 | import trio 3 | import tractor 4 | 5 | 6 | async def sleepy_jane() -> None: 7 | uid: tuple = tractor.current_actor().uid 8 | print(f'Yo i am actor {uid}') 9 | await trio.sleep_forever() 10 | 11 | 12 | async def main(): 13 | ''' 14 | Spawn a flat actor cluster, with one process per detected core. 15 | 16 | ''' 17 | portal_map: dict[str, tractor.Portal] 18 | 19 | # look at this hip new syntax! 20 | async with ( 21 | 22 | tractor.open_actor_cluster( 23 | modules=[__name__] 24 | ) as portal_map, 25 | 26 | tractor.trionics.collapse_eg(), 27 | trio.open_nursery() as tn, 28 | ): 29 | 30 | for (name, portal) in portal_map.items(): 31 | tn.start_soon( 32 | portal.run, 33 | sleepy_jane, 34 | ) 35 | 36 | await trio.sleep(0.5) 37 | 38 | # kill the cluster with a cancel 39 | raise KeyboardInterrupt 40 | 41 | 42 | if __name__ == '__main__': 43 | try: 44 | trio.run(main) 45 | except KeyboardInterrupt: 46 | print('trio cancelled by KBI') 47 | -------------------------------------------------------------------------------- /examples/asynchronous_generators.py: -------------------------------------------------------------------------------- 1 | from typing import AsyncIterator 2 | from itertools import repeat 3 | 4 | import trio 5 | import tractor 6 | 7 | 8 | async def stream_forever() -> AsyncIterator[int]: 9 | 10 | for i in repeat("I can see these little future bubble things"): 11 | # each yielded value is sent over the ``Channel`` to the parent actor 12 | yield i 13 | await trio.sleep(0.01) 14 | 15 | 16 | async def main(): 17 | 18 | async with tractor.open_nursery() as n: 19 | 20 | portal = await n.start_actor( 21 | 'donny', 22 | enable_modules=[__name__], 23 | ) 24 | 25 | # this async for loop streams values from the above 26 | # async generator running in a separate process 27 | async with portal.open_stream_from(stream_forever) as stream: 28 | count = 0 29 | async for letter in stream: 30 | print(letter) 31 | count += 1 32 | 33 | if count > 50: 34 | break 35 | 36 | print('stream terminated') 37 | 38 | await portal.cancel_actor() 39 | 40 | 41 | if __name__ == '__main__': 42 | trio.run(main) 43 | -------------------------------------------------------------------------------- /examples/integration/open_context_and_sleep.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import click 3 | import tractor 4 | import pydantic 5 | # from multiprocessing import shared_memory 6 | 7 | 8 | @tractor.context 9 | async def just_sleep( 10 | 11 | ctx: tractor.Context, 12 | **kwargs, 13 | 14 | ) -> None: 15 | ''' 16 | Test a small ping-pong 2-way streaming server. 17 | 18 | ''' 19 | await ctx.started() 20 | await trio.sleep_forever() 21 | 22 | 23 | async def main() -> None: 24 | 25 | proc = await trio.open_process( ( 26 | 'python', 27 | '-c', 28 | 'import trio; trio.run(trio.sleep_forever)', 29 | )) 30 | await proc.wait() 31 | # await trio.sleep_forever() 32 | # async with tractor.open_nursery() as n: 33 | 34 | # portal = await n.start_actor( 35 | # 'rpc_server', 36 | # enable_modules=[__name__], 37 | # ) 38 | 39 | # async with portal.open_context( 40 | # just_sleep, # taken from pytest parameterization 41 | # ) as (ctx, sent): 42 | # await trio.sleep_forever() 43 | 44 | 45 | 46 | if __name__ == '__main__': 47 | import time 48 | # time.sleep(999) 49 | trio.run(main) 50 | -------------------------------------------------------------------------------- /examples/debugging/subactor_bp_in_ctx.py: -------------------------------------------------------------------------------- 1 | import tractor 2 | import trio 3 | 4 | 5 | async def gen(): 6 | yield 'yo' 7 | await tractor.pause() 8 | yield 'yo' 9 | await tractor.pause() 10 | 11 | 12 | @tractor.context 13 | async def just_bp( 14 | ctx: tractor.Context, 15 | ) -> None: 16 | 17 | await ctx.started() 18 | await tractor.pause() 19 | 20 | # TODO: bps and errors in this call.. 21 | async for val in gen(): 22 | print(val) 23 | 24 | # await trio.sleep(0.5) 25 | 26 | # prematurely destroy the connection 27 | await ctx.chan.aclose() 28 | 29 | # THIS CAUSES AN UNRECOVERABLE HANG 30 | # without latest ``pdbpp``: 31 | assert 0 32 | 33 | 34 | 35 | async def main(): 36 | 37 | async with tractor.open_nursery( 38 | debug_mode=True, 39 | enable_transports=['uds'], 40 | loglevel='devx', 41 | ) as n: 42 | p = await n.start_actor( 43 | 'bp_boi', 44 | enable_modules=[__name__], 45 | ) 46 | async with p.open_context( 47 | just_bp, 48 | ) as (ctx, first): 49 | await trio.sleep_forever() 50 | 51 | 52 | if __name__ == '__main__': 53 | trio.run(main) 54 | -------------------------------------------------------------------------------- /examples/a_trynamic_first_scene.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | _this_module = __name__ 5 | the_line = 'Hi my name is {}' 6 | 7 | 8 | tractor.log.get_console_log("INFO") 9 | 10 | 11 | async def hi(): 12 | return the_line.format(tractor.current_actor().name) 13 | 14 | 15 | async def say_hello(other_actor): 16 | async with tractor.wait_for_actor(other_actor) as portal: 17 | return await portal.run(hi) 18 | 19 | 20 | async def main(): 21 | """Main tractor entry point, the "master" process (for now 22 | acts as the "director"). 23 | """ 24 | async with tractor.open_nursery() as n: 25 | print("Alright... Action!") 26 | 27 | donny = await n.run_in_actor( 28 | say_hello, 29 | name='donny', 30 | # arguments are always named 31 | other_actor='gretchen', 32 | ) 33 | gretchen = await n.run_in_actor( 34 | say_hello, 35 | name='gretchen', 36 | other_actor='donny', 37 | ) 38 | print(await gretchen.result()) 39 | print(await donny.result()) 40 | print("CUTTTT CUUTT CUT!!! Donny!! You're supposed to say...") 41 | 42 | 43 | if __name__ == '__main__': 44 | trio.run(main) 45 | -------------------------------------------------------------------------------- /examples/debugging/fast_error_in_root_after_spawn.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Fast fail test with a `Context`. 3 | 4 | Ensure the partially initialized sub-actor process 5 | doesn't cause a hang on error/cancel of the parent 6 | nursery. 7 | 8 | ''' 9 | import trio 10 | import tractor 11 | 12 | 13 | @tractor.context 14 | async def sleep( 15 | ctx: tractor.Context, 16 | ): 17 | await trio.sleep(0.5) 18 | await ctx.started() 19 | await trio.sleep_forever() 20 | 21 | 22 | async def open_ctx( 23 | n: tractor._supervise.ActorNursery 24 | ): 25 | 26 | # spawn both actors 27 | portal = await n.start_actor( 28 | name='sleeper', 29 | enable_modules=[__name__], 30 | ) 31 | 32 | async with portal.open_context( 33 | sleep, 34 | ) as (ctx, first): 35 | assert first is None 36 | 37 | 38 | async def main(): 39 | 40 | async with tractor.open_nursery( 41 | debug_mode=True, 42 | loglevel='runtime', 43 | ) as an: 44 | 45 | async with trio.open_nursery() as n: 46 | n.start_soon(open_ctx, an) 47 | 48 | await trio.sleep(0.2) 49 | await trio.sleep(0.1) 50 | assert 0 51 | 52 | 53 | if __name__ == '__main__': 54 | trio.run(main) 55 | -------------------------------------------------------------------------------- /examples/debugging/root_self_cancelled_w_error.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | async def main(): 6 | async with tractor.open_root_actor( 7 | debug_mode=True, 8 | loglevel='cancel', 9 | ) as _root: 10 | 11 | # manually trigger self-cancellation and wait 12 | # for it to fully trigger. 13 | _root.cancel_soon() 14 | await _root._cancel_complete.wait() 15 | print('root cancelled') 16 | 17 | # now ensure we can still use the REPL 18 | try: 19 | await tractor.pause() 20 | except trio.Cancelled as _taskc: 21 | assert (root_cs := _root._root_tn.cancel_scope).cancel_called 22 | # NOTE^^ above logic but inside `open_root_actor()` and 23 | # passed to the `shield=` expression is effectively what 24 | # we're testing here! 25 | await tractor.pause(shield=root_cs.cancel_called) 26 | 27 | # XXX, if shield logic *is wrong* inside `open_root_actor()`'s 28 | # crash-handler block this should never be interacted, 29 | # instead `trio.Cancelled` would be bubbled up: the original 30 | # BUG. 31 | assert 0 32 | 33 | 34 | if __name__ == '__main__': 35 | trio.run(main) 36 | -------------------------------------------------------------------------------- /examples/multiple_streams_one_portal.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | log = tractor.log.get_logger('multiportal') 6 | 7 | 8 | async def stream_data(seed=10): 9 | log.info("Starting stream task") 10 | 11 | for i in range(seed): 12 | yield i 13 | await trio.sleep(0) # trigger scheduler 14 | 15 | 16 | async def stream_from_portal(p, consumed): 17 | 18 | async with p.open_stream_from(stream_data) as stream: 19 | async for item in stream: 20 | if item in consumed: 21 | consumed.remove(item) 22 | else: 23 | consumed.append(item) 24 | 25 | 26 | async def main(): 27 | 28 | async with tractor.open_nursery(loglevel='info') as an: 29 | 30 | p = await an.start_actor('stream_boi', enable_modules=[__name__]) 31 | 32 | consumed = [] 33 | 34 | async with trio.open_nursery() as n: 35 | for i in range(2): 36 | n.start_soon(stream_from_portal, p, consumed) 37 | 38 | # both streaming consumer tasks have completed and so we should 39 | # have nothing in our list thanks to single threadedness 40 | assert not consumed 41 | 42 | await an.cancel() 43 | 44 | 45 | if __name__ == '__main__': 46 | trio.run(main) 47 | -------------------------------------------------------------------------------- /nooz/333.feature.rst: -------------------------------------------------------------------------------- 1 | Add support for ``trio >= 0.22`` and support for the new Python 3.11 2 | ``[Base]ExceptionGroup`` from `pep 654`_ via the backported 3 | `exceptiongroup`_ package and some final fixes to the debug mode 4 | subsystem. 5 | 6 | This port ended up driving some (hopefully) final fixes to our debugger 7 | subsystem including the solution to all lingering stdstreams locking 8 | race-conditions and deadlock scenarios. This includes extending the 9 | debugger tests suite as well as cancellation and ``asyncio`` mode cases. 10 | Some of the notable details: 11 | 12 | - always reverting to the ``trio`` SIGINT handler when leaving debug 13 | mode. 14 | - bypassing child attempts to acquire the debug lock when detected 15 | to be amdist actor-runtime-cancellation. 16 | - allowing the root actor to cancel local but IPC-stale subactor 17 | requests-tasks for the debug lock when in a "no IPC peers" state. 18 | 19 | Further we refined our ``ActorNursery`` semantics to be more similar to 20 | ``trio`` in the sense that parent task errors are always packed into the 21 | actor-nursery emitted exception group and adjusted all tests and 22 | examples accordingly. 23 | 24 | .. _pep 654: https://peps.python.org/pep-0654/#handling-exception-groups 25 | .. _exceptiongroup: https://github.com/python-trio/exceptiongroup 26 | -------------------------------------------------------------------------------- /tractor/trionics/__init__.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | ''' 18 | Sugary patterns for trio + tractor designs. 19 | 20 | ''' 21 | from ._mngrs import ( 22 | gather_contexts as gather_contexts, 23 | maybe_open_context as maybe_open_context, 24 | maybe_open_nursery as maybe_open_nursery, 25 | ) 26 | from ._broadcast import ( 27 | AsyncReceiver as AsyncReceiver, 28 | broadcast_receiver as broadcast_receiver, 29 | BroadcastReceiver as BroadcastReceiver, 30 | Lagged as Lagged, 31 | ) 32 | from ._beg import ( 33 | collapse_eg as collapse_eg, 34 | get_collapsed_eg as get_collapsed_eg, 35 | is_multi_cancelled as is_multi_cancelled, 36 | ) 37 | from ._taskc import ( 38 | maybe_raise_from_masking_exc as maybe_raise_from_masking_exc, 39 | ) 40 | -------------------------------------------------------------------------------- /examples/debugging/multi_daemon_subactors.py: -------------------------------------------------------------------------------- 1 | import tractor 2 | import trio 3 | 4 | 5 | async def breakpoint_forever(): 6 | "Indefinitely re-enter debugger in child actor." 7 | try: 8 | while True: 9 | yield 'yo' 10 | await tractor.pause() 11 | except BaseException: 12 | tractor.log.get_console_log().exception( 13 | 'Cancelled while trying to enter pause point!' 14 | ) 15 | raise 16 | 17 | 18 | async def name_error(): 19 | "Raise a ``NameError``" 20 | getattr(doggypants) # noqa 21 | 22 | 23 | async def main(): 24 | ''' 25 | Test breakpoint in a streaming actor. 26 | 27 | ''' 28 | async with tractor.open_nursery( 29 | debug_mode=True, 30 | loglevel='cancel', 31 | # loglevel='devx', 32 | ) as n: 33 | 34 | p0 = await n.start_actor('bp_forever', enable_modules=[__name__]) 35 | p1 = await n.start_actor('name_error', enable_modules=[__name__]) 36 | 37 | # retreive results 38 | async with p0.open_stream_from(breakpoint_forever) as stream: 39 | 40 | # triggers the first name error 41 | try: 42 | await p1.run(name_error) 43 | except tractor.RemoteActorError as rae: 44 | assert rae.boxed_type is NameError 45 | 46 | async for i in stream: 47 | 48 | # a second time try the failing subactor and this tie 49 | # let error propagate up to the parent/nursery. 50 | await p1.run(name_error) 51 | 52 | 53 | if __name__ == '__main__': 54 | trio.run(main) 55 | -------------------------------------------------------------------------------- /examples/debugging/multi_subactors.py: -------------------------------------------------------------------------------- 1 | import tractor 2 | import trio 3 | 4 | 5 | async def breakpoint_forever(): 6 | "Indefinitely re-enter debugger in child actor." 7 | while True: 8 | await trio.sleep(0.1) 9 | await tractor.pause() 10 | 11 | 12 | async def name_error(): 13 | "Raise a ``NameError``" 14 | getattr(doggypants) # noqa 15 | 16 | 17 | async def spawn_error(): 18 | """"A nested nursery that triggers another ``NameError``. 19 | """ 20 | async with tractor.open_nursery() as n: 21 | portal = await n.run_in_actor( 22 | name_error, 23 | name='name_error_1', 24 | ) 25 | return await portal.result() 26 | 27 | 28 | async def main(): 29 | """The main ``tractor`` routine. 30 | 31 | The process tree should look as approximately as follows: 32 | 33 | -python examples/debugging/multi_subactors.py 34 | |-python -m tractor._child --uid ('name_error', 'a7caf490 ...) 35 | |-python -m tractor._child --uid ('bp_forever', '1f787a7e ...) 36 | `-python -m tractor._child --uid ('spawn_error', '52ee14a5 ...) 37 | `-python -m tractor._child --uid ('name_error', '3391222c ...) 38 | """ 39 | async with tractor.open_nursery( 40 | debug_mode=True, 41 | # loglevel='runtime', 42 | ) as n: 43 | 44 | # Spawn both actors, don't bother with collecting results 45 | # (would result in a different debugger outcome due to parent's 46 | # cancellation). 47 | await n.run_in_actor(breakpoint_forever) 48 | await n.run_in_actor(name_error) 49 | await n.run_in_actor(spawn_error) 50 | 51 | 52 | if __name__ == '__main__': 53 | trio.run(main) 54 | -------------------------------------------------------------------------------- /examples/debugging/pm_in_subactor.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | @tractor.context 6 | async def name_error( 7 | ctx: tractor.Context, 8 | ): 9 | ''' 10 | Raise a `NameError`, catch it and enter `.post_mortem()`, then 11 | expect the `._rpc._invoke()` crash handler to also engage. 12 | 13 | ''' 14 | try: 15 | getattr(doggypants) # noqa (on purpose) 16 | except NameError: 17 | await tractor.post_mortem() 18 | raise 19 | 20 | 21 | async def main(): 22 | ''' 23 | Test 3 `PdbREPL` entries: 24 | - one in the child due to manual `.post_mortem()`, 25 | - another in the child due to runtime RPC crash handling. 26 | - final one here in parent from the RAE. 27 | 28 | ''' 29 | # XXX NOTE: ideally the REPL arrives at this frame in the parent 30 | # ONE UP FROM the inner ctx block below! 31 | async with tractor.open_nursery( 32 | debug_mode=True, 33 | # loglevel='cancel', 34 | ) as an: 35 | p: tractor.Portal = await an.start_actor( 36 | 'child', 37 | enable_modules=[__name__], 38 | ) 39 | 40 | # XXX should raise `RemoteActorError[NameError]` 41 | # AND be the active frame when REPL enters! 42 | try: 43 | async with p.open_context(name_error) as (ctx, first): 44 | assert first 45 | except tractor.RemoteActorError as rae: 46 | assert rae.boxed_type is NameError 47 | 48 | # manually handle in root's parent task 49 | await tractor.post_mortem() 50 | raise 51 | else: 52 | raise RuntimeError('IPC ctx should have remote errored!?') 53 | 54 | 55 | if __name__ == '__main__': 56 | trio.run(main) 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /examples/rpc_bidir_streaming.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | @tractor.context 6 | async def simple_rpc( 7 | 8 | ctx: tractor.Context, 9 | data: int, 10 | 11 | ) -> None: 12 | '''Test a small ping-pong 2-way streaming server. 13 | 14 | ''' 15 | # signal to parent that we're up much like 16 | # ``trio.TaskStatus.started()`` 17 | await ctx.started(data + 1) 18 | 19 | async with ctx.open_stream() as stream: 20 | 21 | count = 0 22 | async for msg in stream: 23 | 24 | assert msg == 'ping' 25 | await stream.send('pong') 26 | count += 1 27 | 28 | else: 29 | assert count == 10 30 | 31 | 32 | async def main() -> None: 33 | 34 | async with tractor.open_nursery() as n: 35 | 36 | portal = await n.start_actor( 37 | 'rpc_server', 38 | enable_modules=[__name__], 39 | ) 40 | 41 | # XXX: syntax requires py3.9 42 | async with ( 43 | 44 | portal.open_context( 45 | simple_rpc, # taken from pytest parameterization 46 | data=10, 47 | 48 | ) as (ctx, sent), 49 | 50 | ctx.open_stream() as stream, 51 | ): 52 | 53 | assert sent == 11 54 | 55 | count = 0 56 | # receive msgs using async for style 57 | await stream.send('ping') 58 | 59 | async for msg in stream: 60 | assert msg == 'pong' 61 | await stream.send('ping') 62 | count += 1 63 | 64 | if count >= 9: 65 | break 66 | 67 | # explicitly teardown the daemon-actor 68 | await portal.cancel_actor() 69 | 70 | 71 | if __name__ == '__main__': 72 | trio.run(main) 73 | -------------------------------------------------------------------------------- /docs/github_readme/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | 3 | # this config is for the rst generation extension and thus 4 | # requires only basic settings: 5 | # https://github.com/sphinx-contrib/restbuilder 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | # Warn about all references to unknown targets 18 | nitpicky = True 19 | 20 | # The master toctree document. 21 | master_doc = '_sphinx_readme' 22 | 23 | # -- Project information ----------------------------------------------------- 24 | 25 | project = 'tractor' 26 | copyright = '2018, Tyler Goodlet' 27 | author = 'Tyler Goodlet' 28 | 29 | # The full version, including alpha/beta/rc tags 30 | release = '0.0.0a0.dev0' 31 | 32 | # -- General configuration --------------------------------------------------- 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 36 | # ones. 37 | extensions = [ 38 | 'sphinx.ext.autodoc', 39 | 'sphinx.ext.intersphinx', 40 | 'sphinx.ext.todo', 41 | 'sphinxcontrib.restbuilder', 42 | 43 | ] 44 | 45 | # Add any paths that contain templates here, relative to this directory. 46 | templates_path = ['_templates'] 47 | 48 | # List of patterns, relative to source directory, that match files and 49 | # directories to ignore when looking for source files. 50 | # This pattern also affects html_static_path and html_extra_path. 51 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 52 | -------------------------------------------------------------------------------- /examples/debugging/restore_builtin_breakpoint.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | import trio 5 | import tractor 6 | 7 | # ensure mod-path is correct! 8 | from tractor.devx.debug import ( 9 | _sync_pause_from_builtin as _sync_pause_from_builtin, 10 | ) 11 | 12 | 13 | async def main() -> None: 14 | 15 | # intially unset, no entry. 16 | orig_pybp_var: int = os.environ.get('PYTHONBREAKPOINT') 17 | assert orig_pybp_var in {None, "0"} 18 | 19 | async with tractor.open_nursery( 20 | debug_mode=True, 21 | loglevel='devx', 22 | maybe_enable_greenback=True, 23 | # ^XXX REQUIRED to enable `breakpoint()` support (from sync 24 | # fns) and thus required here to avoid an assertion err 25 | # on the next line 26 | ): 27 | assert ( 28 | (pybp_var := os.environ['PYTHONBREAKPOINT']) 29 | == 30 | 'tractor.devx.debug._sync_pause_from_builtin' 31 | ) 32 | 33 | # TODO: an assert that verifies the hook has indeed been, hooked 34 | # XD 35 | assert ( 36 | (pybp_hook := sys.breakpointhook) 37 | is not tractor.devx.debug._set_trace 38 | ) 39 | 40 | print( 41 | f'$PYTHONOBREAKPOINT: {pybp_var!r}\n' 42 | f'`sys.breakpointhook`: {pybp_hook!r}\n' 43 | ) 44 | breakpoint() # first bp, tractor hook set. 45 | 46 | # XXX AFTER EXIT (of actor-runtime) verify the hook is unset.. 47 | # 48 | # YES, this is weird but it's how stdlib docs say to do it.. 49 | # https://docs.python.org/3/library/sys.html#sys.breakpointhook 50 | assert os.environ.get('PYTHONBREAKPOINT') is orig_pybp_var 51 | assert sys.breakpointhook 52 | 53 | # now ensure a regular builtin pause still works 54 | breakpoint() # last bp, stdlib hook restored 55 | 56 | 57 | if __name__ == '__main__': 58 | trio.run(main) 59 | -------------------------------------------------------------------------------- /tests/test_runtime.py: -------------------------------------------------------------------------------- 1 | """ 2 | Verifying internal runtime state and undocumented extras. 3 | 4 | """ 5 | import os 6 | 7 | import pytest 8 | import trio 9 | import tractor 10 | 11 | from tractor._testing import tractor_test 12 | 13 | 14 | _file_path: str = '' 15 | 16 | 17 | def unlink_file(): 18 | print('Removing tmp file!') 19 | os.remove(_file_path) 20 | 21 | 22 | async def crash_and_clean_tmpdir( 23 | tmp_file_path: str, 24 | error: bool = True, 25 | ): 26 | global _file_path 27 | _file_path = tmp_file_path 28 | 29 | actor = tractor.current_actor() 30 | actor.lifetime_stack.callback(unlink_file) 31 | 32 | assert os.path.isfile(tmp_file_path) 33 | await trio.sleep(0.1) 34 | if error: 35 | assert 0 36 | else: 37 | actor.cancel_soon() 38 | 39 | 40 | @pytest.mark.parametrize( 41 | 'error_in_child', 42 | [True, False], 43 | ) 44 | @tractor_test 45 | async def test_lifetime_stack_wipes_tmpfile( 46 | tmp_path, 47 | error_in_child: bool, 48 | ): 49 | child_tmp_file = tmp_path / "child.txt" 50 | child_tmp_file.touch() 51 | assert child_tmp_file.exists() 52 | path = str(child_tmp_file) 53 | 54 | try: 55 | with trio.move_on_after(0.5): 56 | async with tractor.open_nursery() as n: 57 | await ( # inlined portal 58 | await n.run_in_actor( 59 | crash_and_clean_tmpdir, 60 | tmp_file_path=path, 61 | error=error_in_child, 62 | ) 63 | ).result() 64 | 65 | except ( 66 | tractor.RemoteActorError, 67 | # tractor.BaseExceptionGroup, 68 | BaseExceptionGroup, 69 | ): 70 | pass 71 | 72 | # tmp file should have been wiped by 73 | # teardown stack. 74 | assert not child_tmp_file.exists() 75 | -------------------------------------------------------------------------------- /examples/debugging/multi_subactor_root_errors.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Test that a nested nursery will avoid clobbering 3 | the debugger latched by a broken child. 4 | 5 | ''' 6 | import trio 7 | import tractor 8 | 9 | 10 | async def name_error(): 11 | "Raise a ``NameError``" 12 | getattr(doggypants) # noqa 13 | 14 | 15 | async def spawn_error(): 16 | """"A nested nursery that triggers another ``NameError``. 17 | """ 18 | async with tractor.open_nursery() as n: 19 | portal = await n.run_in_actor( 20 | name_error, 21 | name='name_error_1', 22 | ) 23 | return await portal.result() 24 | 25 | 26 | async def main(): 27 | """The main ``tractor`` routine. 28 | 29 | The process tree should look as approximately as follows: 30 | 31 | python examples/debugging/multi_subactors.py 32 | ├─ python -m tractor._child --uid ('name_error', 'a7caf490 ...) 33 | `-python -m tractor._child --uid ('spawn_error', '52ee14a5 ...) 34 | `-python -m tractor._child --uid ('name_error', '3391222c ...) 35 | 36 | Order of failure: 37 | - nested name_error sub-sub-actor 38 | - root actor should then fail on assert 39 | - program termination 40 | """ 41 | async with tractor.open_nursery( 42 | debug_mode=True, 43 | loglevel='devx', 44 | ) as n: 45 | 46 | # spawn both actors 47 | portal = await n.run_in_actor( 48 | name_error, 49 | name='name_error', 50 | ) 51 | portal1 = await n.run_in_actor( 52 | spawn_error, 53 | name='spawn_error', 54 | ) 55 | 56 | # trigger a root actor error 57 | assert 0 58 | 59 | # attempt to collect results (which raises error in parent) 60 | # still has some issues where the parent seems to get stuck 61 | await portal.result() 62 | await portal1.result() 63 | 64 | 65 | if __name__ == '__main__': 66 | trio.run(main) 67 | -------------------------------------------------------------------------------- /tests/ipc/test_server.py: -------------------------------------------------------------------------------- 1 | ''' 2 | High-level `.ipc._server` unit tests. 3 | 4 | ''' 5 | from __future__ import annotations 6 | 7 | import pytest 8 | import trio 9 | from tractor import ( 10 | devx, 11 | ipc, 12 | log, 13 | ) 14 | from tractor._testing.addr import ( 15 | get_rando_addr, 16 | ) 17 | # TODO, use/check-roundtripping with some of these wrapper types? 18 | # 19 | # from .._addr import Address 20 | # from ._chan import Channel 21 | # from ._transport import MsgTransport 22 | # from ._uds import UDSAddress 23 | # from ._tcp import TCPAddress 24 | 25 | 26 | @pytest.mark.parametrize( 27 | '_tpt_proto', 28 | ['uds', 'tcp'] 29 | ) 30 | def test_basic_ipc_server( 31 | _tpt_proto: str, 32 | debug_mode: bool, 33 | loglevel: str, 34 | ): 35 | 36 | # so we see the socket-listener reporting on console 37 | log.get_console_log("INFO") 38 | 39 | rando_addr: tuple = get_rando_addr( 40 | tpt_proto=_tpt_proto, 41 | ) 42 | async def main(): 43 | async with ipc._server.open_ipc_server() as server: 44 | 45 | assert ( 46 | server._parent_tn 47 | and 48 | server._parent_tn is server._stream_handler_tn 49 | ) 50 | assert server._no_more_peers.is_set() 51 | 52 | eps: list[ipc._server.Endpoint] = await server.listen_on( 53 | accept_addrs=[rando_addr], 54 | stream_handler_nursery=None, 55 | ) 56 | assert ( 57 | len(eps) == 1 58 | and 59 | (ep := eps[0])._listener 60 | and 61 | not ep.peer_tpts 62 | ) 63 | 64 | server._parent_tn.cancel_scope.cancel() 65 | 66 | # !TODO! actually make a bg-task connection from a client 67 | # using `ipc._chan._connect_chan()` 68 | 69 | with devx.maybe_open_crash_handler( 70 | pdb=debug_mode, 71 | ): 72 | trio.run(main) 73 | -------------------------------------------------------------------------------- /docs/dev_tips.rst: -------------------------------------------------------------------------------- 1 | Hot tips for ``tractor`` hackers 2 | ================================ 3 | 4 | This is a WIP guide for newcomers to the project mostly to do with 5 | dev, testing, CI and release gotchas, reminders and best practises. 6 | 7 | ``tractor`` is a fairly novel project compared to most since it is 8 | effectively a new way of doing distributed computing in Python and is 9 | much closer to working with an "application level runtime" (like erlang 10 | OTP or scala's akka project) then it is a traditional Python library. 11 | As such, having an arsenal of tools and recipes for figuring out the 12 | right way to debug problems when they do arise is somewhat of 13 | a necessity. 14 | 15 | 16 | Making a Release 17 | ---------------- 18 | We currently do nothing special here except the traditional 19 | PyPa release recipe as in `documented by twine`_. I personally 20 | create sub-dirs within the generated `dist/` with an explicit 21 | release name such as `alpha3/` when there's been a sequence of 22 | releases I've made, but it really is up to you how you like to 23 | organize generated sdists locally. 24 | 25 | The resulting build cmds are approximately: 26 | 27 | .. code:: bash 28 | 29 | python setup.py sdist -d ./dist/XXX.X/ 30 | 31 | twine upload -r testpypi dist/XXX.X/* 32 | 33 | twine upload dist/XXX.X/* 34 | 35 | 36 | 37 | .. _documented by twine: https://twine.readthedocs.io/en/latest/#using-twine 38 | 39 | 40 | Debugging and monitoring actor trees 41 | ------------------------------------ 42 | TODO: but there are tips in the readme for some terminal commands 43 | which can be used to see the process trees easily on Linux. 44 | 45 | 46 | Using the log system to trace `trio` task flow 47 | ---------------------------------------------- 48 | TODO: the logging system is meant to be oriented around 49 | stack "layers" of the runtime such that you can track 50 | "logical abstraction layers" in the code such as errors, cancellation, 51 | IPC and streaming, and the low level transport and wire protocols. 52 | -------------------------------------------------------------------------------- /tests/test_multi_program.py: -------------------------------------------------------------------------------- 1 | """ 2 | Multiple python programs invoking the runtime. 3 | """ 4 | import platform 5 | import time 6 | 7 | import pytest 8 | import trio 9 | import tractor 10 | from tractor._testing import ( 11 | tractor_test, 12 | ) 13 | from .conftest import ( 14 | sig_prog, 15 | _INT_SIGNAL, 16 | _INT_RETURN_CODE, 17 | ) 18 | 19 | 20 | def test_abort_on_sigint(daemon): 21 | assert daemon.returncode is None 22 | time.sleep(0.1) 23 | sig_prog(daemon, _INT_SIGNAL) 24 | assert daemon.returncode == _INT_RETURN_CODE 25 | 26 | # XXX: oddly, couldn't get capfd.readouterr() to work here? 27 | if platform.system() != 'Windows': 28 | # don't check stderr on windows as its empty when sending CTRL_C_EVENT 29 | assert "KeyboardInterrupt" in str(daemon.stderr.read()) 30 | 31 | 32 | @tractor_test 33 | async def test_cancel_remote_arbiter(daemon, reg_addr): 34 | assert not tractor.current_actor().is_arbiter 35 | async with tractor.get_registry(reg_addr) as portal: 36 | await portal.cancel_actor() 37 | 38 | time.sleep(0.1) 39 | # the arbiter channel server is cancelled but not its main task 40 | assert daemon.returncode is None 41 | 42 | # no arbiter socket should exist 43 | with pytest.raises(OSError): 44 | async with tractor.get_registry(reg_addr) as portal: 45 | pass 46 | 47 | 48 | def test_register_duplicate_name(daemon, reg_addr): 49 | 50 | async def main(): 51 | 52 | async with tractor.open_nursery( 53 | registry_addrs=[reg_addr], 54 | ) as n: 55 | 56 | assert not tractor.current_actor().is_arbiter 57 | 58 | p1 = await n.start_actor('doggy') 59 | p2 = await n.start_actor('doggy') 60 | 61 | async with tractor.wait_for_actor('doggy') as portal: 62 | assert portal.channel.uid in (p2.channel.uid, p1.channel.uid) 63 | 64 | await n.cancel() 65 | 66 | # run it manually since we want to start **after** 67 | # the other "daemon" program 68 | trio.run(main) 69 | -------------------------------------------------------------------------------- /examples/debugging/root_cancelled_but_child_is_in_tty_lock.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | async def name_error(): 6 | "Raise a ``NameError``" 7 | getattr(doggypants) # noqa 8 | 9 | 10 | async def spawn_until(depth=0): 11 | """"A nested nursery that triggers another ``NameError``. 12 | """ 13 | async with tractor.open_nursery() as n: 14 | if depth < 1: 15 | # await n.run_in_actor('breakpoint_forever', breakpoint_forever) 16 | await n.run_in_actor(name_error) 17 | else: 18 | depth -= 1 19 | await n.run_in_actor( 20 | spawn_until, 21 | depth=depth, 22 | name=f'spawn_until_{depth}', 23 | ) 24 | 25 | 26 | async def main(): 27 | ''' 28 | The process tree should look as approximately as follows when the 29 | debugger first engages: 30 | 31 | python examples/debugging/multi_nested_subactors_bp_forever.py 32 | ├─ python -m tractor._child --uid ('spawner1', '7eab8462 ...) 33 | │ └─ python -m tractor._child --uid ('spawn_until_0', '3720602b ...) 34 | │ └─ python -m tractor._child --uid ('name_error', '505bf71d ...) 35 | │ 36 | └─ python -m tractor._child --uid ('spawner0', '1d42012b ...) 37 | └─ python -m tractor._child --uid ('name_error', '6c2733b8 ...) 38 | 39 | ''' 40 | async with tractor.open_nursery( 41 | debug_mode=True, 42 | loglevel='devx', 43 | enable_transports=['uds'], 44 | ) as n: 45 | 46 | # spawn both actors 47 | portal = await n.run_in_actor( 48 | spawn_until, 49 | depth=0, 50 | name='spawner0', 51 | ) 52 | portal1 = await n.run_in_actor( 53 | spawn_until, 54 | depth=1, 55 | name='spawner1', 56 | ) 57 | 58 | # nursery cancellation should be triggered due to propagated 59 | # error from child. 60 | await portal.result() 61 | await portal1.result() 62 | 63 | 64 | if __name__ == '__main__': 65 | trio.run(main) 66 | -------------------------------------------------------------------------------- /tractor/_child.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | """ 18 | This is the "bootloader" for actors started using the native trio backend. 19 | 20 | """ 21 | import argparse 22 | 23 | from ast import literal_eval 24 | 25 | from ._runtime import Actor 26 | from ._entry import _trio_main 27 | 28 | 29 | def parse_uid(arg): 30 | name, uuid = literal_eval(arg) # ensure 2 elements 31 | return str(name), str(uuid) # ensures str encoding 32 | 33 | def parse_ipaddr(arg): 34 | try: 35 | return literal_eval(arg) 36 | 37 | except (ValueError, SyntaxError): 38 | # UDS: try to interpret as a straight up str 39 | return arg 40 | 41 | 42 | if __name__ == "__main__": 43 | __tracebackhide__: bool = True 44 | 45 | parser = argparse.ArgumentParser() 46 | parser.add_argument("--uid", type=parse_uid) 47 | parser.add_argument("--loglevel", type=str) 48 | parser.add_argument("--parent_addr", type=parse_ipaddr) 49 | parser.add_argument("--asyncio", action='store_true') 50 | args = parser.parse_args() 51 | 52 | subactor = Actor( 53 | name=args.uid[0], 54 | uuid=args.uid[1], 55 | loglevel=args.loglevel, 56 | spawn_method="trio" 57 | ) 58 | 59 | _trio_main( 60 | subactor, 61 | parent_addr=args.parent_addr, 62 | infect_asyncio=args.asyncio, 63 | ) 64 | -------------------------------------------------------------------------------- /tractor/msg/__init__.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | ''' 18 | Built-in messaging patterns, types, APIs and helpers. 19 | 20 | ''' 21 | from typing import ( 22 | TypeAlias, 23 | ) 24 | from .ptr import ( 25 | NamespacePath as NamespacePath, 26 | ) 27 | from .pretty_struct import ( 28 | Struct as Struct, 29 | ) 30 | from ._codec import ( 31 | _def_msgspec_codec as _def_msgspec_codec, 32 | _ctxvar_MsgCodec as _ctxvar_MsgCodec, 33 | 34 | apply_codec as apply_codec, 35 | mk_codec as mk_codec, 36 | mk_dec as mk_dec, 37 | MsgCodec as MsgCodec, 38 | MsgDec as MsgDec, 39 | current_codec as current_codec, 40 | ) 41 | # currently can't bc circular with `._context` 42 | # from ._ops import ( 43 | # PldRx as PldRx, 44 | # _drain_to_final_msg as _drain_to_final_msg, 45 | # ) 46 | 47 | from .types import ( 48 | PayloadMsg as PayloadMsg, 49 | 50 | Aid as Aid, 51 | SpawnSpec as SpawnSpec, 52 | 53 | Start as Start, 54 | StartAck as StartAck, 55 | 56 | Started as Started, 57 | Yield as Yield, 58 | Stop as Stop, 59 | Return as Return, 60 | CancelAck as CancelAck, 61 | 62 | Error as Error, 63 | 64 | # type-var for `.pld` field 65 | PayloadT as PayloadT, 66 | 67 | # full msg class set from above as list 68 | __msg_types__ as __msg_types__, 69 | 70 | # type-alias for union of all msgs 71 | MsgType as MsgType, 72 | ) 73 | 74 | __msg_spec__: TypeAlias = MsgType 75 | -------------------------------------------------------------------------------- /examples/trio/lockacquire_not_unmasked.py: -------------------------------------------------------------------------------- 1 | from contextlib import ( 2 | asynccontextmanager as acm, 3 | ) 4 | from functools import partial 5 | 6 | import tractor 7 | import trio 8 | 9 | 10 | log = tractor.log.get_logger( 11 | name=__name__ 12 | ) 13 | 14 | _lock: trio.Lock|None = None 15 | 16 | 17 | @acm 18 | async def acquire_singleton_lock( 19 | ) -> None: 20 | global _lock 21 | if _lock is None: 22 | log.info('Allocating LOCK') 23 | _lock = trio.Lock() 24 | 25 | log.info('TRYING TO LOCK ACQUIRE') 26 | async with _lock: 27 | log.info('ACQUIRED') 28 | yield _lock 29 | 30 | log.info('RELEASED') 31 | 32 | 33 | 34 | async def hold_lock_forever( 35 | task_status=trio.TASK_STATUS_IGNORED 36 | ): 37 | async with ( 38 | tractor.trionics.maybe_raise_from_masking_exc(), 39 | acquire_singleton_lock() as lock, 40 | ): 41 | task_status.started(lock) 42 | await trio.sleep_forever() 43 | 44 | 45 | async def main( 46 | ignore_special_cases: bool, 47 | loglevel: str = 'info', 48 | debug_mode: bool = True, 49 | ): 50 | async with ( 51 | trio.open_nursery() as tn, 52 | 53 | # tractor.trionics.maybe_raise_from_masking_exc() 54 | # ^^^ XXX NOTE, interestingly putting the unmasker 55 | # here does not exhibit the same behaviour ?? 56 | ): 57 | if not ignore_special_cases: 58 | from tractor.trionics import _taskc 59 | _taskc._mask_cases.clear() 60 | 61 | _lock = await tn.start( 62 | hold_lock_forever, 63 | ) 64 | with trio.move_on_after(0.2): 65 | await tn.start( 66 | hold_lock_forever, 67 | ) 68 | 69 | tn.cancel_scope.cancel() 70 | 71 | 72 | # XXX, manual test as script 73 | if __name__ == '__main__': 74 | tractor.log.get_console_log(level='info') 75 | for case in [True, False]: 76 | log.info( 77 | f'\n' 78 | f'------ RUNNING SCRIPT TRIAL ------\n' 79 | f'ignore_special_cases: {case!r}\n' 80 | ) 81 | trio.run(partial( 82 | main, 83 | ignore_special_cases=case, 84 | loglevel='info', 85 | )) 86 | -------------------------------------------------------------------------------- /tests/test_clustering.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | import pytest 4 | import trio 5 | import tractor 6 | from tractor import open_actor_cluster 7 | from tractor.trionics import gather_contexts 8 | from tractor._testing import tractor_test 9 | 10 | MESSAGE = 'tractoring at full speed' 11 | 12 | 13 | def test_empty_mngrs_input_raises() -> None: 14 | 15 | async def main(): 16 | with trio.fail_after(3): 17 | async with ( 18 | open_actor_cluster( 19 | modules=[__name__], 20 | 21 | # NOTE: ensure we can passthrough runtime opts 22 | loglevel='cancel', 23 | debug_mode=False, 24 | 25 | ) as portals, 26 | 27 | gather_contexts(mngrs=()), 28 | ): 29 | # should fail before this? 30 | assert portals 31 | 32 | # test should fail if we mk it here! 33 | assert 0, 'Should have raised val-err !?' 34 | 35 | with pytest.raises(ValueError): 36 | trio.run(main) 37 | 38 | 39 | @tractor.context 40 | async def worker( 41 | ctx: tractor.Context, 42 | 43 | ) -> None: 44 | 45 | await ctx.started() 46 | 47 | async with ctx.open_stream( 48 | allow_overruns=True, 49 | ) as stream: 50 | 51 | # TODO: this with the below assert causes a hang bug? 52 | # with trio.move_on_after(1): 53 | 54 | async for msg in stream: 55 | # do something with msg 56 | print(msg) 57 | assert msg == MESSAGE 58 | 59 | # TODO: does this ever cause a hang 60 | # assert 0 61 | 62 | 63 | @tractor_test 64 | async def test_streaming_to_actor_cluster() -> None: 65 | 66 | async with ( 67 | open_actor_cluster(modules=[__name__]) as portals, 68 | 69 | gather_contexts( 70 | mngrs=[p.open_context(worker) for p in portals.values()], 71 | ) as contexts, 72 | 73 | gather_contexts( 74 | mngrs=[ctx[0].open_stream() for ctx in contexts], 75 | ) as streams, 76 | 77 | ): 78 | with trio.move_on_after(1): 79 | for stream in itertools.cycle(streams): 80 | await stream.send(MESSAGE) 81 | -------------------------------------------------------------------------------- /tests/test_local.py: -------------------------------------------------------------------------------- 1 | """ 2 | Arbiter and "local" actor api 3 | """ 4 | import time 5 | 6 | import pytest 7 | import trio 8 | import tractor 9 | 10 | from tractor._testing import tractor_test 11 | 12 | 13 | @pytest.mark.trio 14 | async def test_no_runtime(): 15 | """An arbitter must be established before any nurseries 16 | can be created. 17 | 18 | (In other words ``tractor.open_root_actor()`` must be engaged at 19 | some point?) 20 | """ 21 | with pytest.raises(RuntimeError) : 22 | async with tractor.find_actor('doggy'): 23 | pass 24 | 25 | 26 | @tractor_test 27 | async def test_self_is_registered(reg_addr): 28 | "Verify waiting on the arbiter to register itself using the standard api." 29 | actor = tractor.current_actor() 30 | assert actor.is_arbiter 31 | with trio.fail_after(0.2): 32 | async with tractor.wait_for_actor('root') as portal: 33 | assert portal.channel.uid[0] == 'root' 34 | 35 | 36 | @tractor_test 37 | async def test_self_is_registered_localportal(reg_addr): 38 | "Verify waiting on the arbiter to register itself using a local portal." 39 | actor = tractor.current_actor() 40 | assert actor.is_arbiter 41 | async with tractor.get_registry(reg_addr) as portal: 42 | assert isinstance(portal, tractor._portal.LocalPortal) 43 | 44 | with trio.fail_after(0.2): 45 | sockaddr = await portal.run_from_ns( 46 | 'self', 'wait_for_actor', name='root') 47 | assert sockaddr[0] == reg_addr 48 | 49 | 50 | def test_local_actor_async_func(reg_addr): 51 | """Verify a simple async function in-process. 52 | """ 53 | nums = [] 54 | 55 | async def print_loop(): 56 | 57 | async with tractor.open_root_actor( 58 | registry_addrs=[reg_addr], 59 | ): 60 | # arbiter is started in-proc if dne 61 | assert tractor.current_actor().is_arbiter 62 | 63 | for i in range(10): 64 | nums.append(i) 65 | await trio.sleep(0.1) 66 | 67 | start = time.time() 68 | trio.run(print_loop) 69 | 70 | # ensure the sleeps were actually awaited 71 | assert time.time() - start >= 1 72 | assert nums == list(range(10)) 73 | -------------------------------------------------------------------------------- /ruff.toml: -------------------------------------------------------------------------------- 1 | # from default `ruff.toml` @ 2 | # https://docs.astral.sh/ruff/configuration/ 3 | 4 | # Exclude a variety of commonly ignored directories. 5 | exclude = [ 6 | ".bzr", 7 | ".direnv", 8 | ".eggs", 9 | ".git", 10 | ".git-rewrite", 11 | ".hg", 12 | ".ipynb_checkpoints", 13 | ".mypy_cache", 14 | ".nox", 15 | ".pants.d", 16 | ".pyenv", 17 | ".pytest_cache", 18 | ".pytype", 19 | ".ruff_cache", 20 | ".svn", 21 | ".tox", 22 | ".venv", 23 | ".vscode", 24 | "__pypackages__", 25 | "_build", 26 | "buck-out", 27 | "build", 28 | "dist", 29 | "node_modules", 30 | "site-packages", 31 | "venv", 32 | ] 33 | 34 | # Same as Black. 35 | line-length = 88 36 | indent-width = 4 37 | 38 | # Assume Python 3.9 39 | target-version = "py311" 40 | 41 | [lint] 42 | # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. 43 | # Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or 44 | # McCabe complexity (`C901`) by default. 45 | select = ["E4", "E7", "E9", "F"] 46 | ignore = [ 47 | 'E402', # https://docs.astral.sh/ruff/rules/module-import-not-at-top-of-file/ 48 | ] 49 | 50 | # Allow fix for all enabled rules (when `--fix`) is provided. 51 | fixable = ["ALL"] 52 | unfixable = [] 53 | 54 | # Allow unused variables when underscore-prefixed. 55 | # dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" 56 | 57 | [format] 58 | # Use single quotes in `ruff format`. 59 | quote-style = "single" 60 | 61 | # Like Black, indent with spaces, rather than tabs. 62 | indent-style = "space" 63 | 64 | # Like Black, respect magic trailing commas. 65 | skip-magic-trailing-comma = false 66 | 67 | # Like Black, automatically detect the appropriate line ending. 68 | line-ending = "auto" 69 | 70 | # Enable auto-formatting of code examples in docstrings. Markdown, 71 | # reStructuredText code/literal blocks and doctests are all supported. 72 | # 73 | # This is currently disabled by default, but it is planned for this 74 | # to be opt-out in the future. 75 | docstring-code-format = false 76 | 77 | # Set the line length limit used when formatting code snippets in 78 | # docstrings. 79 | # 80 | # This only has an effect when the `docstring-code-format` setting is 81 | # enabled. 82 | docstring-code-line-length = "dynamic" 83 | -------------------------------------------------------------------------------- /nooz/337.feature.rst: -------------------------------------------------------------------------------- 1 | Add support for debug-lock blocking using a ``._debug.Lock._blocked: 2 | set[tuple]`` and add ids when no-more IPC connections with the 3 | root actor are detected. 4 | 5 | This is an enhancement which (mostly) solves a lingering debugger 6 | locking race case we needed to handle: 7 | 8 | - child crashes acquires TTY lock in root and attaches to ``pdb`` 9 | - child IPC goes down such that all channels to the root are broken 10 | / non-functional. 11 | - root is stuck thinking the child is still in debug even though it 12 | can't be contacted and the child actor machinery hasn't been 13 | cancelled by its parent. 14 | - root get's stuck in deadlock with child since it won't send a cancel 15 | request until the child is finished debugging (to avoid clobbering 16 | a child that is actually using the debugger), but the child can't 17 | unlock the debugger bc IPC is down and it can't contact the root. 18 | 19 | To avoid this scenario add debug lock blocking list via 20 | `._debug.Lock._blocked: set[tuple]` which holds actor uids for any actor 21 | that is detected by the root as having no transport channel connections 22 | (of which at least one should exist if this sub-actor at some point 23 | acquired the debug lock). The root consequently checks this list for any 24 | actor that tries to (re)acquire the lock and blocks with 25 | a ``ContextCancelled``. Further, when a debug condition is tested in 26 | ``._runtime._invoke``, the context's ``._enter_debugger_on_cancel`` is 27 | set to `False` if the actor was put on the block list then all 28 | post-mortem / crash handling will be bypassed for that task. 29 | 30 | In theory this approach to block list management may cause problems 31 | where some nested child actor acquires and releases the lock multiple 32 | times and it gets stuck on the block list after the first use? If this 33 | turns out to be an issue we can try changing the strat so blocks are 34 | only added when the root has zero IPC peers left? 35 | 36 | Further, this adds a root-locking-task side cancel scope, 37 | ``Lock._root_local_task_cs_in_debug``, which can be ``.cancel()``-ed by the root 38 | runtime when a stale lock is detected during the IPC channel testing. 39 | However, right now we're NOT using this since it seems to cause test 40 | failures likely due to causing pre-mature cancellation and maybe needs 41 | a bit more experimenting? 42 | -------------------------------------------------------------------------------- /tractor/__init__.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | """ 18 | tractor: structured concurrent ``trio``-"actors". 19 | 20 | """ 21 | 22 | from ._clustering import ( 23 | open_actor_cluster as open_actor_cluster, 24 | ) 25 | from ._context import ( 26 | Context as Context, # the type 27 | context as context, # a func-decorator 28 | ) 29 | from ._streaming import ( 30 | MsgStream as MsgStream, 31 | stream as stream, 32 | ) 33 | from ._discovery import ( 34 | get_registry as get_registry, 35 | find_actor as find_actor, 36 | wait_for_actor as wait_for_actor, 37 | query_actor as query_actor, 38 | ) 39 | from ._supervise import ( 40 | open_nursery as open_nursery, 41 | ActorNursery as ActorNursery, 42 | ) 43 | from ._state import ( 44 | current_actor as current_actor, 45 | is_root_process as is_root_process, 46 | current_ipc_ctx as current_ipc_ctx, 47 | debug_mode as debug_mode 48 | ) 49 | from ._exceptions import ( 50 | ContextCancelled as ContextCancelled, 51 | ModuleNotExposed as ModuleNotExposed, 52 | MsgTypeError as MsgTypeError, 53 | RemoteActorError as RemoteActorError, 54 | TransportClosed as TransportClosed, 55 | ) 56 | from .devx import ( 57 | breakpoint as breakpoint, 58 | pause as pause, 59 | pause_from_sync as pause_from_sync, 60 | post_mortem as post_mortem, 61 | ) 62 | from . import msg as msg 63 | from ._root import ( 64 | run_daemon as run_daemon, 65 | open_root_actor as open_root_actor, 66 | ) 67 | from .ipc import Channel as Channel 68 | from ._portal import Portal as Portal 69 | from ._runtime import Actor as Actor 70 | # from . import hilevel as hilevel 71 | -------------------------------------------------------------------------------- /tractor/_testing/addr.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | ''' 18 | Random IPC addr generation for isolating 19 | the discovery space between test sessions. 20 | 21 | Might be eventually useful to expose as a util set from 22 | our `tractor.discovery` subsys? 23 | 24 | ''' 25 | import random 26 | from typing import ( 27 | Type, 28 | ) 29 | from tractor import ( 30 | _addr, 31 | ) 32 | 33 | 34 | def get_rando_addr( 35 | tpt_proto: str, 36 | *, 37 | 38 | # choose random port at import time 39 | _rando_port: str = random.randint(1000, 9999) 40 | 41 | ) -> tuple[str, str|int]: 42 | ''' 43 | Used to globally override the runtime to the 44 | per-test-session-dynamic addr so that all tests never conflict 45 | with any other actor tree using the default. 46 | 47 | ''' 48 | addr_type: Type[_addr.Addres] = _addr._address_types[tpt_proto] 49 | def_reg_addr: tuple[str, int] = _addr._default_lo_addrs[tpt_proto] 50 | 51 | # this is the "unwrapped" form expected to be passed to 52 | # `.open_root_actor()` by test body. 53 | testrun_reg_addr: tuple[str, int|str] 54 | match tpt_proto: 55 | case 'tcp': 56 | testrun_reg_addr = ( 57 | addr_type.def_bindspace, 58 | _rando_port, 59 | ) 60 | 61 | # NOTE, file-name uniqueness (no-collisions) will be based on 62 | # the runtime-directory and root (pytest-proc's) pid. 63 | case 'uds': 64 | testrun_reg_addr = addr_type.get_random().unwrap() 65 | 66 | # XXX, as sanity it should never the same as the default for the 67 | # host-singleton registry actor. 68 | assert def_reg_addr != testrun_reg_addr 69 | 70 | return testrun_reg_addr 71 | -------------------------------------------------------------------------------- /examples/debugging/shield_hang_in_sub.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Verify we can dump a `stackscope` tree on a hang. 3 | 4 | ''' 5 | import os 6 | import signal 7 | 8 | import trio 9 | import tractor 10 | 11 | @tractor.context 12 | async def start_n_shield_hang( 13 | ctx: tractor.Context, 14 | ): 15 | # actor: tractor.Actor = tractor.current_actor() 16 | 17 | # sync to parent-side task 18 | await ctx.started(os.getpid()) 19 | 20 | print('Entering shield sleep..') 21 | with trio.CancelScope(shield=True): 22 | await trio.sleep_forever() # in subactor 23 | 24 | # XXX NOTE ^^^ since this shields, we expect 25 | # the zombie reaper (aka T800) to engage on 26 | # SIGINT from the user and eventually hard-kill 27 | # this subprocess! 28 | 29 | 30 | async def main( 31 | from_test: bool = False, 32 | ) -> None: 33 | 34 | async with ( 35 | tractor.open_nursery( 36 | debug_mode=True, 37 | enable_stack_on_sig=True, 38 | # maybe_enable_greenback=False, 39 | loglevel='devx', 40 | enable_transports=['uds'], 41 | ) as an, 42 | ): 43 | ptl: tractor.Portal = await an.start_actor( 44 | 'hanger', 45 | enable_modules=[__name__], 46 | debug_mode=True, 47 | ) 48 | async with ptl.open_context( 49 | start_n_shield_hang, 50 | ) as (ctx, cpid): 51 | 52 | _, proc, _ = an._children[ptl.chan.uid] 53 | assert cpid == proc.pid 54 | 55 | print( 56 | 'Yo my child hanging..?\n' 57 | # "i'm a user who wants to see a `stackscope` tree!\n" 58 | ) 59 | 60 | # XXX simulate the wrapping test's "user actions" 61 | # (i.e. if a human didn't run this manually but wants to 62 | # know what they should do to reproduce test behaviour) 63 | if from_test: 64 | print( 65 | f'Sending SIGUSR1 to {cpid!r}!\n' 66 | ) 67 | os.kill( 68 | cpid, 69 | signal.SIGUSR1, 70 | ) 71 | 72 | # simulate user cancelling program 73 | await trio.sleep(0.5) 74 | os.kill( 75 | os.getpid(), 76 | signal.SIGINT, 77 | ) 78 | else: 79 | # actually let user send the ctl-c 80 | await trio.sleep_forever() # in root 81 | 82 | 83 | if __name__ == '__main__': 84 | trio.run(main) 85 | -------------------------------------------------------------------------------- /tests/ipc/test_multi_tpt.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Verify the `enable_transports` param drives various 3 | per-root/sub-actor IPC endpoint/server settings. 4 | 5 | ''' 6 | from __future__ import annotations 7 | 8 | import pytest 9 | import trio 10 | import tractor 11 | from tractor import ( 12 | Actor, 13 | Portal, 14 | ipc, 15 | msg, 16 | _state, 17 | _addr, 18 | ) 19 | 20 | @tractor.context 21 | async def chk_tpts( 22 | ctx: tractor.Context, 23 | tpt_proto_key: str, 24 | ): 25 | rtvars = _state._runtime_vars 26 | assert ( 27 | tpt_proto_key 28 | in 29 | rtvars['_enable_tpts'] 30 | ) 31 | actor: Actor = tractor.current_actor() 32 | spec: msg.types.SpawnSpec = actor._spawn_spec 33 | assert spec._runtime_vars == rtvars 34 | 35 | # ensure individual IPC ep-addr types 36 | serv: ipc._server.Server = actor.ipc_server 37 | addr: ipc._types.Address 38 | for addr in serv.addrs: 39 | assert addr.proto_key == tpt_proto_key 40 | 41 | # Actor delegate-props enforcement 42 | assert ( 43 | actor.accept_addrs 44 | == 45 | serv.accept_addrs 46 | ) 47 | 48 | await ctx.started(serv.accept_addrs) 49 | 50 | 51 | # TODO, parametrize over mis-matched-proto-typed `registry_addrs` 52 | # since i seems to work in `piker` but not exactly sure if both tcp 53 | # & uds are being deployed then? 54 | # 55 | @pytest.mark.parametrize( 56 | 'tpt_proto_key', 57 | ['tcp', 'uds'], 58 | ids=lambda item: f'ipc_tpt={item!r}' 59 | ) 60 | def test_root_passes_tpt_to_sub( 61 | tpt_proto_key: str, 62 | reg_addr: tuple, 63 | debug_mode: bool, 64 | ): 65 | async def main(): 66 | async with tractor.open_nursery( 67 | enable_transports=[tpt_proto_key], 68 | registry_addrs=[reg_addr], 69 | debug_mode=debug_mode, 70 | ) as an: 71 | 72 | assert ( 73 | tpt_proto_key 74 | in 75 | _state._runtime_vars['_enable_tpts'] 76 | ) 77 | 78 | ptl: Portal = await an.start_actor( 79 | name='sub', 80 | enable_modules=[__name__], 81 | ) 82 | async with ptl.open_context( 83 | chk_tpts, 84 | tpt_proto_key=tpt_proto_key, 85 | ) as (ctx, accept_addrs): 86 | 87 | uw_addr: tuple 88 | for uw_addr in accept_addrs: 89 | addr = _addr.wrap_address(uw_addr) 90 | assert addr.is_valid 91 | 92 | # shudown sub-actor(s) 93 | await an.cancel() 94 | 95 | trio.run(main) 96 | -------------------------------------------------------------------------------- /tractor/_clustering.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | ''' 18 | Actor cluster helpers. 19 | 20 | ''' 21 | from __future__ import annotations 22 | from contextlib import ( 23 | asynccontextmanager as acm, 24 | ) 25 | from multiprocessing import cpu_count 26 | from typing import ( 27 | AsyncGenerator, 28 | ) 29 | 30 | import trio 31 | import tractor 32 | 33 | 34 | @acm 35 | async def open_actor_cluster( 36 | modules: list[str], 37 | count: int = cpu_count(), 38 | names: list[str] | None = None, 39 | hard_kill: bool = False, 40 | 41 | # passed through verbatim to ``open_root_actor()`` 42 | **runtime_kwargs, 43 | 44 | ) -> AsyncGenerator[ 45 | dict[str, tractor.Portal], 46 | None, 47 | ]: 48 | 49 | portals: dict[str, tractor.Portal] = {} 50 | 51 | if not names: 52 | names = [f'worker_{i}' for i in range(count)] 53 | 54 | if not len(names) == count: 55 | raise ValueError( 56 | 'Number of names is {len(names)} but count it {count}') 57 | 58 | async with ( 59 | # tractor.trionics.collapse_eg(), 60 | tractor.open_nursery( 61 | **runtime_kwargs, 62 | ) as an 63 | ): 64 | async with ( 65 | # tractor.trionics.collapse_eg(), 66 | trio.open_nursery() as tn, 67 | tractor.trionics.maybe_raise_from_masking_exc() 68 | ): 69 | uid = tractor.current_actor().uid 70 | 71 | async def _start(name: str) -> None: 72 | name = f'{uid[0]}.{name}' 73 | portals[name] = await an.start_actor( 74 | enable_modules=modules, 75 | name=name, 76 | ) 77 | 78 | for name in names: 79 | tn.start_soon(_start, name) 80 | 81 | assert len(portals) == count 82 | yield portals 83 | await an.cancel(hard_kill=hard_kill) 84 | -------------------------------------------------------------------------------- /examples/infected_asyncio_echo_server.py: -------------------------------------------------------------------------------- 1 | ''' 2 | An SC compliant infected ``asyncio`` echo server. 3 | 4 | ''' 5 | import asyncio 6 | from statistics import mean 7 | import time 8 | 9 | import trio 10 | import tractor 11 | 12 | 13 | async def aio_echo_server( 14 | to_trio: trio.MemorySendChannel, 15 | from_trio: asyncio.Queue, 16 | 17 | ) -> None: 18 | 19 | # a first message must be sent **from** this ``asyncio`` 20 | # task or the ``trio`` side will never unblock from 21 | # ``tractor.to_asyncio.open_channel_from():`` 22 | to_trio.send_nowait('start') 23 | 24 | # XXX: this uses an ``from_trio: asyncio.Queue`` currently but we 25 | # should probably offer something better. 26 | while True: 27 | # echo the msg back 28 | to_trio.send_nowait(await from_trio.get()) 29 | await asyncio.sleep(0) 30 | 31 | 32 | @tractor.context 33 | async def trio_to_aio_echo_server( 34 | ctx: tractor.Context, 35 | ): 36 | # this will block until the ``asyncio`` task sends a "first" 37 | # message. 38 | async with tractor.to_asyncio.open_channel_from( 39 | aio_echo_server, 40 | ) as (first, chan): 41 | 42 | assert first == 'start' 43 | await ctx.started(first) 44 | 45 | async with ctx.open_stream() as stream: 46 | 47 | async for msg in stream: 48 | await chan.send(msg) 49 | 50 | out = await chan.receive() 51 | # echo back to parent actor-task 52 | await stream.send(out) 53 | 54 | 55 | async def main(): 56 | 57 | async with tractor.open_nursery() as n: 58 | p = await n.start_actor( 59 | 'aio_server', 60 | enable_modules=[__name__], 61 | infect_asyncio=True, 62 | ) 63 | async with p.open_context( 64 | trio_to_aio_echo_server, 65 | ) as (ctx, first): 66 | 67 | assert first == 'start' 68 | 69 | count = 0 70 | async with ctx.open_stream() as stream: 71 | 72 | delays = [] 73 | send = time.time() 74 | 75 | await stream.send(count) 76 | async for msg in stream: 77 | recv = time.time() 78 | delays.append(recv - send) 79 | assert msg == count 80 | count += 1 81 | send = time.time() 82 | await stream.send(count) 83 | 84 | if count >= 1e3: 85 | break 86 | 87 | print(f'mean round trip rate (Hz): {1/mean(delays)}') 88 | await p.cancel_actor() 89 | 90 | 91 | if __name__ == '__main__': 92 | trio.run(main) 93 | -------------------------------------------------------------------------------- /tractor/ipc/_mp_bs.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | ''' 17 | Utils to tame mp non-SC madeness 18 | 19 | ''' 20 | import platform 21 | 22 | 23 | def disable_mantracker(): 24 | ''' 25 | Disable all `multiprocessing` "resource tracking" machinery since 26 | it's an absolute multi-threaded mess of non-SC madness. 27 | 28 | ''' 29 | from multiprocessing.shared_memory import SharedMemory 30 | 31 | 32 | # 3.13+ only.. can pass `track=False` to disable 33 | # all the resource tracker bs. 34 | # https://docs.python.org/3/library/multiprocessing.shared_memory.html 35 | if (_py_313 := ( 36 | platform.python_version_tuple()[:-1] 37 | >= 38 | ('3', '13') 39 | ) 40 | ): 41 | from functools import partial 42 | return partial( 43 | SharedMemory, 44 | track=False, 45 | ) 46 | 47 | # !TODO, once we drop 3.12- we can obvi remove all this! 48 | else: 49 | from multiprocessing import ( 50 | resource_tracker as mantracker, 51 | ) 52 | 53 | # Tell the "resource tracker" thing to fuck off. 54 | class ManTracker(mantracker.ResourceTracker): 55 | def register(self, name, rtype): 56 | pass 57 | 58 | def unregister(self, name, rtype): 59 | pass 60 | 61 | def ensure_running(self): 62 | pass 63 | 64 | # "know your land and know your prey" 65 | # https://www.dailymotion.com/video/x6ozzco 66 | mantracker._resource_tracker = ManTracker() 67 | mantracker.register = mantracker._resource_tracker.register 68 | mantracker.ensure_running = mantracker._resource_tracker.ensure_running 69 | mantracker.unregister = mantracker._resource_tracker.unregister 70 | mantracker.getfd = mantracker._resource_tracker.getfd 71 | 72 | # use std type verbatim 73 | shmT = SharedMemory 74 | 75 | return shmT 76 | -------------------------------------------------------------------------------- /tractor/msg/_exts.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | ''' 18 | Type-extension-utils for codec-ing (python) objects not 19 | covered by the `msgspec.msgpack` protocol. 20 | 21 | See the various API docs from `msgspec`. 22 | 23 | extending from native types, 24 | - https://jcristharif.com/msgspec/extending.html#mapping-to-from-native-types 25 | 26 | converters, 27 | - https://jcristharif.com/msgspec/converters.html 28 | - https://jcristharif.com/msgspec/api.html#msgspec.convert 29 | 30 | `Raw` fields, 31 | - https://jcristharif.com/msgspec/api.html#raw 32 | - support for `.convert()` and `Raw`, 33 | |_ https://jcristharif.com/msgspec/changelog.html 34 | 35 | ''' 36 | from types import ( 37 | ModuleType, 38 | ) 39 | import typing 40 | from typing import ( 41 | Type, 42 | Union, 43 | ) 44 | 45 | def dec_type_union( 46 | type_names: list[str], 47 | mods: list[ModuleType] = [] 48 | ) -> Type|Union[Type]: 49 | ''' 50 | Look up types by name, compile into a list and then create and 51 | return a `typing.Union` from the full set. 52 | 53 | ''' 54 | # import importlib 55 | types: list[Type] = [] 56 | for type_name in type_names: 57 | for mod in [ 58 | typing, 59 | # importlib.import_module(__name__), 60 | ] + mods: 61 | if type_ref := getattr( 62 | mod, 63 | type_name, 64 | False, 65 | ): 66 | types.append(type_ref) 67 | 68 | # special case handling only.. 69 | # ipc_pld_spec: Union[Type] = eval( 70 | # pld_spec_str, 71 | # {}, # globals 72 | # {'typing': typing}, # locals 73 | # ) 74 | 75 | return Union[*types] 76 | 77 | 78 | def enc_type_union( 79 | union_or_type: Union[Type]|Type, 80 | ) -> list[str]: 81 | ''' 82 | Encode a type-union or single type to a list of type-name-strings 83 | ready for IPC interchange. 84 | 85 | ''' 86 | type_strs: list[str] = [] 87 | for typ in getattr( 88 | union_or_type, 89 | '__args__', 90 | {union_or_type,}, 91 | ): 92 | type_strs.append(typ.__qualname__) 93 | 94 | return type_strs 95 | -------------------------------------------------------------------------------- /examples/debugging/shielded_pause.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | async def cancellable_pause_loop( 6 | task_status: trio.TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED 7 | ): 8 | with trio.CancelScope() as cs: 9 | task_status.started(cs) 10 | for _ in range(3): 11 | try: 12 | # ON first entry, there is no level triggered 13 | # cancellation yet, so this cp does a parent task 14 | # ctx-switch so that this scope raises for the NEXT 15 | # checkpoint we hit. 16 | await trio.lowlevel.checkpoint() 17 | await tractor.pause() 18 | 19 | cs.cancel() 20 | 21 | # parent should have called `cs.cancel()` by now 22 | await trio.lowlevel.checkpoint() 23 | 24 | except trio.Cancelled: 25 | print('INSIDE SHIELDED PAUSE') 26 | await tractor.pause(shield=True) 27 | else: 28 | # should raise it again, bubbling up to parent 29 | print('BUBBLING trio.Cancelled to parent task-nursery') 30 | await trio.lowlevel.checkpoint() 31 | 32 | 33 | async def pm_on_cancelled(): 34 | async with trio.open_nursery() as tn: 35 | tn.cancel_scope.cancel() 36 | try: 37 | await trio.sleep_forever() 38 | except trio.Cancelled: 39 | # should also raise `Cancelled` since 40 | # we didn't pass `shield=True`. 41 | try: 42 | await tractor.post_mortem(hide_tb=False) 43 | except trio.Cancelled as taskc: 44 | 45 | # should enter just fine, in fact it should 46 | # be debugging the internals of the previous 47 | # sin-shield call above Bo 48 | await tractor.post_mortem( 49 | hide_tb=False, 50 | shield=True, 51 | ) 52 | raise taskc 53 | 54 | else: 55 | raise RuntimeError('Dint cancel as expected!?') 56 | 57 | 58 | async def cancelled_before_pause( 59 | ): 60 | ''' 61 | Verify that using a shielded pause works despite surrounding 62 | cancellation called state in the calling task. 63 | 64 | ''' 65 | async with trio.open_nursery() as tn: 66 | cs: trio.CancelScope = await tn.start(cancellable_pause_loop) 67 | await trio.sleep(0.1) 68 | 69 | assert cs.cancelled_caught 70 | 71 | await pm_on_cancelled() 72 | 73 | 74 | async def main(): 75 | async with tractor.open_nursery( 76 | debug_mode=True, 77 | ) as n: 78 | portal: tractor.Portal = await n.run_in_actor( 79 | cancelled_before_pause, 80 | ) 81 | await portal.result() 82 | 83 | # ensure the same works in the root actor! 84 | await pm_on_cancelled() 85 | 86 | 87 | if __name__ == '__main__': 88 | trio.run(main) 89 | -------------------------------------------------------------------------------- /tractor/devx/__init__.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | """ 18 | Runtime "developer experience" utils and addons to aid our 19 | (advanced) users and core devs in building distributed applications 20 | and working with/on the actor runtime. 21 | 22 | """ 23 | from .debug import ( 24 | maybe_wait_for_debugger as maybe_wait_for_debugger, 25 | acquire_debug_lock as acquire_debug_lock, 26 | breakpoint as breakpoint, 27 | pause as pause, 28 | pause_from_sync as pause_from_sync, 29 | sigint_shield as sigint_shield, 30 | open_crash_handler as open_crash_handler, 31 | maybe_open_crash_handler as maybe_open_crash_handler, 32 | maybe_init_greenback as maybe_init_greenback, 33 | post_mortem as post_mortem, 34 | mk_pdb as mk_pdb, 35 | ) 36 | from ._stackscope import ( 37 | enable_stack_on_sig as enable_stack_on_sig, 38 | ) 39 | from .pformat import ( 40 | add_div as add_div, 41 | pformat_caller_frame as pformat_caller_frame, 42 | pformat_boxed_tb as pformat_boxed_tb, 43 | ) 44 | 45 | 46 | # TODO, move this to a new `.devx._pdbp` mod? 47 | def _enable_readline_feats() -> str: 48 | ''' 49 | Handle `readline` when compiled with `libedit` to avoid breaking 50 | tab completion in `pdbp` (and its dep `tabcompleter`) 51 | particularly since `uv` cpython distis are compiled this way.. 52 | 53 | See docs for deats, 54 | https://docs.python.org/3/library/readline.html#module-readline 55 | 56 | Originally discovered soln via SO answer, 57 | https://stackoverflow.com/q/49287102 58 | 59 | ''' 60 | import readline 61 | if ( 62 | # 3.13+ attr 63 | # https://docs.python.org/3/library/readline.html#readline.backend 64 | (getattr(readline, 'backend', False) == 'libedit') 65 | or 66 | 'libedit' in readline.__doc__ 67 | ): 68 | readline.parse_and_bind("python:bind -v") 69 | readline.parse_and_bind("python:bind ^I rl_complete") 70 | return 'libedit' 71 | else: 72 | readline.parse_and_bind("tab: complete") 73 | readline.parse_and_bind("set editing-mode vi") 74 | readline.parse_and_bind("set keymap vi") 75 | return 'readline' 76 | 77 | 78 | _enable_readline_feats() 79 | -------------------------------------------------------------------------------- /tests/test_root_runtime.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Runtime boot/init sanity. 3 | 4 | ''' 5 | 6 | import pytest 7 | import trio 8 | 9 | import tractor 10 | from tractor._exceptions import RuntimeFailure 11 | 12 | 13 | @tractor.context 14 | async def open_new_root_in_sub( 15 | ctx: tractor.Context, 16 | ) -> None: 17 | 18 | async with tractor.open_root_actor(): 19 | pass 20 | 21 | 22 | @pytest.mark.parametrize( 23 | 'open_root_in', 24 | ['root', 'sub'], 25 | ids='open_2nd_root_in={}'.format, 26 | ) 27 | def test_only_one_root_actor( 28 | open_root_in: str, 29 | reg_addr: tuple, 30 | debug_mode: bool 31 | ): 32 | ''' 33 | Verify we specially fail whenever more then one root actor 34 | is attempted to be opened within an already opened tree. 35 | 36 | ''' 37 | async def main(): 38 | async with tractor.open_nursery() as an: 39 | 40 | if open_root_in == 'root': 41 | async with tractor.open_root_actor( 42 | registry_addrs=[reg_addr], 43 | ): 44 | pass 45 | 46 | ptl: tractor.Portal = await an.start_actor( 47 | name='bad_rooty_boi', 48 | enable_modules=[__name__], 49 | ) 50 | 51 | async with ptl.open_context( 52 | open_new_root_in_sub, 53 | ) as (ctx, first): 54 | pass 55 | 56 | if open_root_in == 'root': 57 | with pytest.raises( 58 | RuntimeFailure 59 | ) as excinfo: 60 | trio.run(main) 61 | 62 | else: 63 | with pytest.raises( 64 | tractor.RemoteActorError, 65 | ) as excinfo: 66 | trio.run(main) 67 | 68 | assert excinfo.value.boxed_type is RuntimeFailure 69 | 70 | 71 | def test_implicit_root_via_first_nursery( 72 | reg_addr: tuple, 73 | debug_mode: bool 74 | ): 75 | ''' 76 | The first `ActorNursery` open should implicitly call 77 | `_root.open_root_actor()`. 78 | 79 | ''' 80 | async def main(): 81 | async with tractor.open_nursery() as an: 82 | assert an._implicit_runtime_started 83 | assert tractor.current_actor().aid.name == 'root' 84 | 85 | trio.run(main) 86 | 87 | 88 | def test_runtime_vars_unset( 89 | reg_addr: tuple, 90 | debug_mode: bool 91 | ): 92 | ''' 93 | Ensure any `._state._runtime_vars` are restored to default values 94 | after the root actor-runtime exits! 95 | 96 | ''' 97 | assert not tractor._state._runtime_vars['_debug_mode'] 98 | async def main(): 99 | assert not tractor._state._runtime_vars['_debug_mode'] 100 | async with tractor.open_nursery( 101 | debug_mode=True, 102 | ): 103 | assert tractor._state._runtime_vars['_debug_mode'] 104 | 105 | # after runtime closure, should be reverted! 106 | assert not tractor._state._runtime_vars['_debug_mode'] 107 | 108 | trio.run(main) 109 | -------------------------------------------------------------------------------- /tests/ipc/test_each_tpt.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Unit-ish tests for specific IPC transport protocol backends. 3 | 4 | ''' 5 | from __future__ import annotations 6 | from pathlib import Path 7 | 8 | import pytest 9 | import trio 10 | import tractor 11 | from tractor import ( 12 | Actor, 13 | _state, 14 | _addr, 15 | ) 16 | 17 | 18 | @pytest.fixture 19 | def bindspace_dir_str() -> str: 20 | 21 | rt_dir: Path = tractor._state.get_rt_dir() 22 | bs_dir: Path = rt_dir / 'doggy' 23 | bs_dir_str: str = str(bs_dir) 24 | assert not bs_dir.is_dir() 25 | 26 | yield bs_dir_str 27 | 28 | # delete it on suite teardown. 29 | # ?TODO? should we support this internally 30 | # or is leaking it ok? 31 | if bs_dir.is_dir(): 32 | bs_dir.rmdir() 33 | 34 | 35 | def test_uds_bindspace_created_implicitly( 36 | debug_mode: bool, 37 | bindspace_dir_str: str, 38 | ): 39 | registry_addr: tuple = ( 40 | f'{bindspace_dir_str}', 41 | 'registry@doggy.sock', 42 | ) 43 | bs_dir_str: str = registry_addr[0] 44 | 45 | # XXX, ensure bindspace-dir DNE beforehand! 46 | assert not Path(bs_dir_str).is_dir() 47 | 48 | async def main(): 49 | async with tractor.open_nursery( 50 | enable_transports=['uds'], 51 | registry_addrs=[registry_addr], 52 | debug_mode=debug_mode, 53 | ) as _an: 54 | 55 | # XXX MUST be created implicitly by 56 | # `.ipc._uds.start_listener()`! 57 | assert Path(bs_dir_str).is_dir() 58 | 59 | root: Actor = tractor.current_actor() 60 | assert root.is_registrar 61 | 62 | assert registry_addr in root.reg_addrs 63 | assert ( 64 | registry_addr 65 | in 66 | _state._runtime_vars['_registry_addrs'] 67 | ) 68 | assert ( 69 | _addr.wrap_address(registry_addr) 70 | in 71 | root.registry_addrs 72 | ) 73 | 74 | trio.run(main) 75 | 76 | 77 | def test_uds_double_listen_raises_connerr( 78 | debug_mode: bool, 79 | bindspace_dir_str: str, 80 | ): 81 | registry_addr: tuple = ( 82 | f'{bindspace_dir_str}', 83 | 'registry@doggy.sock', 84 | ) 85 | 86 | async def main(): 87 | async with tractor.open_nursery( 88 | enable_transports=['uds'], 89 | registry_addrs=[registry_addr], 90 | debug_mode=debug_mode, 91 | ) as _an: 92 | 93 | # runtime up 94 | root: Actor = tractor.current_actor() 95 | 96 | from tractor.ipc._uds import ( 97 | start_listener, 98 | UDSAddress, 99 | ) 100 | ya_bound_addr: UDSAddress = root.registry_addrs[0] 101 | try: 102 | await start_listener( 103 | addr=ya_bound_addr, 104 | ) 105 | except ConnectionError as connerr: 106 | assert type(src_exc := connerr.__context__) is OSError 107 | assert 'Address already in use' in src_exc.args 108 | # complete, exit test. 109 | 110 | else: 111 | pytest.fail('It dint raise a connerr !?') 112 | 113 | 114 | trio.run(main) 115 | -------------------------------------------------------------------------------- /tractor/_testing/fault_simulation.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | ''' 18 | `pytest` utils helpers and plugins for testing `tractor`'s runtime 19 | and applications. 20 | 21 | ''' 22 | 23 | from tractor import ( 24 | MsgStream, 25 | ) 26 | 27 | async def break_ipc( 28 | stream: MsgStream, 29 | method: str|None = None, 30 | pre_close: bool = False, 31 | 32 | def_method: str = 'socket_close', 33 | 34 | ) -> None: 35 | ''' 36 | XXX: close the channel right after an error is raised 37 | purposely breaking the IPC transport to make sure the parent 38 | doesn't get stuck in debug or hang on the connection join. 39 | this more or less simulates an infinite msg-receive hang on 40 | the other end. 41 | 42 | ''' 43 | # close channel via IPC prot msging before 44 | # any transport breakage 45 | if pre_close: 46 | await stream.aclose() 47 | 48 | method: str = method or def_method 49 | print( 50 | '#################################\n' 51 | 'Simulating CHILD-side IPC BREAK!\n' 52 | f'method: {method}\n' 53 | f'pre `.aclose()`: {pre_close}\n' 54 | '#################################\n' 55 | ) 56 | 57 | match method: 58 | case 'socket_close': 59 | await stream._ctx.chan.transport.stream.aclose() 60 | 61 | case 'socket_eof': 62 | # NOTE: `trio` does the following underneath this 63 | # call in `src/trio/_highlevel_socket.py`: 64 | # `Stream.socket.shutdown(tsocket.SHUT_WR)` 65 | await stream._ctx.chan.transport.stream.send_eof() 66 | 67 | # TODO: remove since now this will be invalid with our 68 | # new typed msg spec? 69 | # case 'msg': 70 | # await stream._ctx.chan.send(None) 71 | 72 | # TODO: the actual real-world simulated cases like 73 | # transport layer hangs and/or lower layer 2-gens type 74 | # scenarios.. 75 | # 76 | # -[ ] already have some issues for this general testing 77 | # area: 78 | # - https://github.com/goodboy/tractor/issues/97 79 | # - https://github.com/goodboy/tractor/issues/124 80 | # - PR from @guille: 81 | # https://github.com/goodboy/tractor/pull/149 82 | # case 'hang': 83 | # TODO: framework research: 84 | # 85 | # - https://github.com/GuoTengda1993/pynetem 86 | # - https://github.com/shopify/toxiproxy 87 | # - https://manpages.ubuntu.com/manpages/trusty/man1/wirefilter.1.html 88 | 89 | case _: 90 | raise RuntimeError( 91 | f'IPC break method unsupported: {method}' 92 | ) 93 | -------------------------------------------------------------------------------- /examples/debugging/multi_nested_subactors_error_up_through_nurseries.py: -------------------------------------------------------------------------------- 1 | import trio 2 | import tractor 3 | 4 | 5 | async def name_error(): 6 | "Raise a ``NameError``" 7 | getattr(doggypants) # noqa 8 | 9 | 10 | async def breakpoint_forever(): 11 | "Indefinitely re-enter debugger in child actor." 12 | while True: 13 | await tractor.pause() 14 | 15 | # NOTE: if the test never sent 'q'/'quit' commands 16 | # on the pdb repl, without this checkpoint line the 17 | # repl would spin in this actor forever. 18 | # await trio.sleep(0) 19 | 20 | 21 | async def spawn_until(depth=0): 22 | """"A nested nursery that triggers another ``NameError``. 23 | """ 24 | async with tractor.open_nursery() as n: 25 | if depth < 1: 26 | 27 | await n.run_in_actor(breakpoint_forever) 28 | 29 | p = await n.run_in_actor( 30 | name_error, 31 | name='name_error' 32 | ) 33 | await trio.sleep(0.5) 34 | # rx and propagate error from child 35 | await p.result() 36 | 37 | else: 38 | # recusrive call to spawn another process branching layer of 39 | # the tree 40 | depth -= 1 41 | await n.run_in_actor( 42 | spawn_until, 43 | depth=depth, 44 | name=f'spawn_until_{depth}', 45 | ) 46 | 47 | 48 | # TODO: notes on the new boxed-relayed errors through proxy actors 49 | async def main(): 50 | """The main ``tractor`` routine. 51 | 52 | The process tree should look as approximately as follows when the debugger 53 | first engages: 54 | 55 | python examples/debugging/multi_nested_subactors_bp_forever.py 56 | ├─ python -m tractor._child --uid ('spawner1', '7eab8462 ...) 57 | │ └─ python -m tractor._child --uid ('spawn_until_3', 'afcba7a8 ...) 58 | │ └─ python -m tractor._child --uid ('spawn_until_2', 'd2433d13 ...) 59 | │ └─ python -m tractor._child --uid ('spawn_until_1', '1df589de ...) 60 | │ └─ python -m tractor._child --uid ('spawn_until_0', '3720602b ...) 61 | │ 62 | └─ python -m tractor._child --uid ('spawner0', '1d42012b ...) 63 | └─ python -m tractor._child --uid ('spawn_until_2', '2877e155 ...) 64 | └─ python -m tractor._child --uid ('spawn_until_1', '0502d786 ...) 65 | └─ python -m tractor._child --uid ('spawn_until_0', 'de918e6d ...) 66 | 67 | """ 68 | async with tractor.open_nursery( 69 | debug_mode=True, 70 | # loglevel='cancel', 71 | ) as n: 72 | 73 | # spawn both actors 74 | portal = await n.run_in_actor( 75 | spawn_until, 76 | depth=3, 77 | name='spawner0', 78 | ) 79 | portal1 = await n.run_in_actor( 80 | spawn_until, 81 | depth=4, 82 | name='spawner1', 83 | ) 84 | 85 | # TODO: test this case as well where the parent don't see 86 | # the sub-actor errors by default and instead expect a user 87 | # ctrl-c to kill the root. 88 | with trio.move_on_after(3): 89 | await trio.sleep_forever() 90 | 91 | # gah still an issue here. 92 | await portal.result() 93 | 94 | # should never get here 95 | await portal1.result() 96 | 97 | 98 | if __name__ == '__main__': 99 | trio.run(main) 100 | -------------------------------------------------------------------------------- /tests/test_2way.py: -------------------------------------------------------------------------------- 1 | """ 2 | Bidirectional streaming. 3 | 4 | """ 5 | import pytest 6 | import trio 7 | import tractor 8 | 9 | 10 | @tractor.context 11 | async def simple_rpc( 12 | 13 | ctx: tractor.Context, 14 | data: int, 15 | 16 | ) -> None: 17 | ''' 18 | Test a small ping-pong server. 19 | 20 | ''' 21 | # signal to parent that we're up 22 | await ctx.started(data + 1) 23 | 24 | print('opening stream in callee') 25 | async with ctx.open_stream() as stream: 26 | 27 | count = 0 28 | while True: 29 | try: 30 | await stream.receive() == 'ping' 31 | except trio.EndOfChannel: 32 | assert count == 10 33 | break 34 | else: 35 | print('pong') 36 | await stream.send('pong') 37 | count += 1 38 | 39 | 40 | @tractor.context 41 | async def simple_rpc_with_forloop( 42 | 43 | ctx: tractor.Context, 44 | data: int, 45 | 46 | ) -> None: 47 | """Same as previous test but using ``async for`` syntax/api. 48 | 49 | """ 50 | 51 | # signal to parent that we're up 52 | await ctx.started(data + 1) 53 | 54 | print('opening stream in callee') 55 | async with ctx.open_stream() as stream: 56 | 57 | count = 0 58 | async for msg in stream: 59 | 60 | assert msg == 'ping' 61 | print('pong') 62 | await stream.send('pong') 63 | count += 1 64 | 65 | else: 66 | assert count == 10 67 | 68 | 69 | @pytest.mark.parametrize( 70 | 'use_async_for', 71 | [True, False], 72 | ) 73 | @pytest.mark.parametrize( 74 | 'server_func', 75 | [simple_rpc, simple_rpc_with_forloop], 76 | ) 77 | def test_simple_rpc(server_func, use_async_for): 78 | ''' 79 | The simplest request response pattern. 80 | 81 | ''' 82 | async def main(): 83 | async with tractor.open_nursery() as n: 84 | 85 | portal = await n.start_actor( 86 | 'rpc_server', 87 | enable_modules=[__name__], 88 | ) 89 | 90 | async with portal.open_context( 91 | server_func, # taken from pytest parameterization 92 | data=10, 93 | ) as (ctx, sent): 94 | 95 | assert sent == 11 96 | 97 | async with ctx.open_stream() as stream: 98 | 99 | if use_async_for: 100 | 101 | count = 0 102 | # receive msgs using async for style 103 | print('ping') 104 | await stream.send('ping') 105 | 106 | async for msg in stream: 107 | assert msg == 'pong' 108 | print('ping') 109 | await stream.send('ping') 110 | count += 1 111 | 112 | if count >= 9: 113 | break 114 | 115 | else: 116 | # classic send/receive style 117 | for _ in range(10): 118 | 119 | print('ping') 120 | await stream.send('ping') 121 | assert await stream.receive() == 'pong' 122 | 123 | # stream should terminate here 124 | 125 | # final context result(s) should be consumed here in __aexit__() 126 | 127 | await portal.cancel_actor() 128 | 129 | trio.run(main) 130 | -------------------------------------------------------------------------------- /tractor/ipc/_types.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | ''' 18 | IPC subsys type-lookup helpers? 19 | 20 | ''' 21 | from typing import ( 22 | Type, 23 | # TYPE_CHECKING, 24 | ) 25 | 26 | import trio 27 | import socket 28 | 29 | from tractor.ipc._transport import ( 30 | MsgTransportKey, 31 | MsgTransport 32 | ) 33 | from tractor.ipc._tcp import ( 34 | TCPAddress, 35 | MsgpackTCPStream, 36 | ) 37 | from tractor.ipc._uds import ( 38 | UDSAddress, 39 | MsgpackUDSStream, 40 | ) 41 | 42 | # if TYPE_CHECKING: 43 | # from tractor._addr import Address 44 | 45 | 46 | Address = TCPAddress|UDSAddress 47 | 48 | # manually updated list of all supported msg transport types 49 | _msg_transports = [ 50 | MsgpackTCPStream, 51 | MsgpackUDSStream 52 | ] 53 | 54 | 55 | # convert a MsgTransportKey to the corresponding transport type 56 | _key_to_transport: dict[ 57 | MsgTransportKey, 58 | Type[MsgTransport], 59 | ] = { 60 | ('msgpack', 'tcp'): MsgpackTCPStream, 61 | ('msgpack', 'uds'): MsgpackUDSStream, 62 | } 63 | 64 | # convert an Address wrapper to its corresponding transport type 65 | _addr_to_transport: dict[ 66 | Type[TCPAddress|UDSAddress], 67 | Type[MsgTransport] 68 | ] = { 69 | TCPAddress: MsgpackTCPStream, 70 | UDSAddress: MsgpackUDSStream, 71 | } 72 | 73 | 74 | def transport_from_addr( 75 | addr: Address, 76 | codec_key: str = 'msgpack', 77 | ) -> Type[MsgTransport]: 78 | ''' 79 | Given a destination address and a desired codec, find the 80 | corresponding `MsgTransport` type. 81 | 82 | ''' 83 | try: 84 | return _addr_to_transport[type(addr)] 85 | 86 | except KeyError: 87 | raise NotImplementedError( 88 | f'No known transport for address {repr(addr)}' 89 | ) 90 | 91 | 92 | def transport_from_stream( 93 | stream: trio.abc.Stream, 94 | codec_key: str = 'msgpack' 95 | ) -> Type[MsgTransport]: 96 | ''' 97 | Given an arbitrary `trio.abc.Stream` and a desired codec, 98 | find the corresponding `MsgTransport` type. 99 | 100 | ''' 101 | transport = None 102 | if isinstance(stream, trio.SocketStream): 103 | sock: socket.socket = stream.socket 104 | match sock.family: 105 | case socket.AF_INET | socket.AF_INET6: 106 | transport = 'tcp' 107 | 108 | case socket.AF_UNIX: 109 | transport = 'uds' 110 | 111 | case _: 112 | raise NotImplementedError( 113 | f'Unsupported socket family: {sock.family}' 114 | ) 115 | 116 | if not transport: 117 | raise NotImplementedError( 118 | f'Could not figure out transport type for stream type {type(stream)}' 119 | ) 120 | 121 | key = (codec_key, transport) 122 | 123 | return _key_to_transport[key] 124 | -------------------------------------------------------------------------------- /examples/parallelism/concurrent_actors_primes.py: -------------------------------------------------------------------------------- 1 | """ 2 | Demonstration of the prime number detector example from the 3 | ``concurrent.futures`` docs: 4 | 5 | https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor-example 6 | 7 | This uses no extra threads, fancy semaphores or futures; all we need 8 | is ``tractor``'s channels. 9 | 10 | """ 11 | from contextlib import ( 12 | asynccontextmanager as acm, 13 | aclosing, 14 | ) 15 | from typing import Callable 16 | import itertools 17 | import math 18 | import time 19 | 20 | import tractor 21 | import trio 22 | 23 | 24 | PRIMES = [ 25 | 112272535095293, 26 | 112582705942171, 27 | 112272535095293, 28 | 115280095190773, 29 | 115797848077099, 30 | 1099726899285419, 31 | ] 32 | 33 | 34 | async def is_prime(n): 35 | if n < 2: 36 | return False 37 | if n == 2: 38 | return True 39 | if n % 2 == 0: 40 | return False 41 | 42 | sqrt_n = int(math.floor(math.sqrt(n))) 43 | for i in range(3, sqrt_n + 1, 2): 44 | if n % i == 0: 45 | return False 46 | return True 47 | 48 | 49 | @acm 50 | async def worker_pool(workers=4): 51 | """Though it's a trivial special case for ``tractor``, the well 52 | known "worker pool" seems to be the defacto "but, I want this 53 | process pattern!" for most parallelism pilgrims. 54 | 55 | Yes, the workers stay alive (and ready for work) until you close 56 | the context. 57 | """ 58 | async with tractor.open_nursery() as tn: 59 | 60 | portals = [] 61 | snd_chan, recv_chan = trio.open_memory_channel(len(PRIMES)) 62 | 63 | for i in range(workers): 64 | 65 | # this starts a new sub-actor (process + trio runtime) and 66 | # stores it's "portal" for later use to "submit jobs" (ugh). 67 | portals.append( 68 | await tn.start_actor( 69 | f'worker_{i}', 70 | enable_modules=[__name__], 71 | ) 72 | ) 73 | 74 | async def _map( 75 | worker_func: Callable[[int], bool], 76 | sequence: list[int] 77 | ) -> list[bool]: 78 | 79 | # define an async (local) task to collect results from workers 80 | async def send_result(func, value, portal): 81 | await snd_chan.send((value, await portal.run(func, n=value))) 82 | 83 | async with trio.open_nursery() as n: 84 | 85 | for value, portal in zip(sequence, itertools.cycle(portals)): 86 | n.start_soon( 87 | send_result, 88 | worker_func, 89 | value, 90 | portal 91 | ) 92 | 93 | # deliver results as they arrive 94 | for _ in range(len(sequence)): 95 | yield await recv_chan.receive() 96 | 97 | # deliver the parallel "worker mapper" to user code 98 | yield _map 99 | 100 | # tear down all "workers" on pool close 101 | await tn.cancel() 102 | 103 | 104 | async def main(): 105 | 106 | async with worker_pool() as actor_map: 107 | 108 | start = time.time() 109 | 110 | async with aclosing(actor_map(is_prime, PRIMES)) as results: 111 | async for number, prime in results: 112 | 113 | print(f'{number} is prime: {prime}') 114 | 115 | print(f'processing took {time.time() - start} seconds') 116 | 117 | 118 | if __name__ == '__main__': 119 | start = time.time() 120 | trio.run(main) 121 | print(f'script took {time.time() - start} seconds') 122 | -------------------------------------------------------------------------------- /tractor/_testing/__init__.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | ''' 18 | Various helpers/utils for auditing your `tractor` app and/or the 19 | core runtime. 20 | 21 | ''' 22 | from contextlib import ( 23 | asynccontextmanager as acm, 24 | ) 25 | import os 26 | import pathlib 27 | 28 | import tractor 29 | from tractor.devx.debug import ( 30 | BoxedMaybeException, 31 | ) 32 | from .pytest import ( 33 | tractor_test as tractor_test 34 | ) 35 | from .fault_simulation import ( 36 | break_ipc as break_ipc, 37 | ) 38 | 39 | 40 | # TODO, use dulwhich for this instead? 41 | # -> we're going to likely need it (or something similar) 42 | # for supporting hot-coad reload feats eventually anyway! 43 | def repodir() -> pathlib.Path: 44 | ''' 45 | Return the abspath to the repo directory. 46 | 47 | ''' 48 | # 2 parents up to step up through tests/ 49 | return pathlib.Path( 50 | __file__ 51 | 52 | # 3 .parents bc: 53 | # <._testing-pkg>.. 54 | # /$HOME/..//tractor/_testing/__init__.py 55 | ).parent.parent.parent.absolute() 56 | 57 | 58 | def examples_dir() -> pathlib.Path: 59 | ''' 60 | Return the abspath to the examples directory as `pathlib.Path`. 61 | 62 | ''' 63 | return repodir() / 'examples' 64 | 65 | 66 | def mk_cmd( 67 | ex_name: str, 68 | exs_subpath: str = 'debugging', 69 | ) -> str: 70 | ''' 71 | Generate a shell command suitable to pass to `pexpect.spawn()` 72 | which runs the script as a python program's entrypoint. 73 | 74 | In particular ensure we disable the new tb coloring via unsetting 75 | `$PYTHON_COLORS` so that `pexpect` can pattern match without 76 | color-escape-codes. 77 | 78 | ''' 79 | script_path: pathlib.Path = ( 80 | examples_dir() 81 | / exs_subpath 82 | / f'{ex_name}.py' 83 | ) 84 | py_cmd: str = ' '.join([ 85 | 'python', 86 | str(script_path) 87 | ]) 88 | # XXX, required for py 3.13+ 89 | # https://docs.python.org/3/using/cmdline.html#using-on-controlling-color 90 | # https://docs.python.org/3/using/cmdline.html#envvar-PYTHON_COLORS 91 | os.environ['PYTHON_COLORS'] = '0' 92 | return py_cmd 93 | 94 | 95 | @acm 96 | async def expect_ctxc( 97 | yay: bool, 98 | reraise: bool = False, 99 | ) -> None: 100 | ''' 101 | Small acm to catch `ContextCancelled` errors when expected 102 | below it in a `async with ()` block. 103 | 104 | ''' 105 | if yay: 106 | try: 107 | yield (maybe_exc := BoxedMaybeException()) 108 | raise RuntimeError('Never raised ctxc?') 109 | except tractor.ContextCancelled as ctxc: 110 | maybe_exc.value = ctxc 111 | if reraise: 112 | raise 113 | else: 114 | return 115 | else: 116 | yield (maybe_exc := BoxedMaybeException()) 117 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | # Warn about all references to unknown targets 18 | nitpicky = True 19 | 20 | # The master toctree document. 21 | master_doc = 'index' 22 | 23 | # -- Project information ----------------------------------------------------- 24 | 25 | project = 'tractor' 26 | copyright = '2018, Tyler Goodlet' 27 | author = 'Tyler Goodlet' 28 | 29 | # The full version, including alpha/beta/rc tags 30 | release = '0.0.0a0.dev0' 31 | 32 | # -- General configuration --------------------------------------------------- 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 36 | # ones. 37 | extensions = [ 38 | 'sphinx.ext.autodoc', 39 | 'sphinx.ext.intersphinx', 40 | 'sphinx.ext.todo', 41 | ] 42 | 43 | # Add any paths that contain templates here, relative to this directory. 44 | templates_path = ['_templates'] 45 | 46 | # List of patterns, relative to source directory, that match files and 47 | # directories to ignore when looking for source files. 48 | # This pattern also affects html_static_path and html_extra_path. 49 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 50 | 51 | 52 | # -- Options for HTML output ------------------------------------------------- 53 | 54 | # The theme to use for HTML and HTML Help pages. See the documentation for 55 | # a list of builtin themes. 56 | # 57 | html_theme = 'sphinx_book_theme' 58 | 59 | pygments_style = 'algol_nu' 60 | 61 | # Theme options are theme-specific and customize the look and feel of a theme 62 | # further. For a list of options available for each theme, see the 63 | # documentation. 64 | html_theme_options = { 65 | # 'logo': 'tractor_logo_side.svg', 66 | # 'description': 'Structured concurrent "actors"', 67 | "repository_url": "https://github.com/goodboy/tractor", 68 | "use_repository_button": True, 69 | "home_page_in_toc": False, 70 | "show_toc_level": 1, 71 | "path_to_docs": "docs", 72 | 73 | } 74 | html_sidebars = { 75 | "**": [ 76 | "sbt-sidebar-nav.html", 77 | # "sidebar-search-bs.html", 78 | # 'localtoc.html', 79 | ], 80 | # 'logo.html', 81 | # 'github.html', 82 | # 'relations.html', 83 | # 'searchbox.html' 84 | # ] 85 | } 86 | 87 | # doesn't seem to work? 88 | # extra_navbar = "

nextttt-gennnnn

" 89 | 90 | html_title = '' 91 | html_logo = '_static/tractor_logo_side.svg' 92 | html_favicon = '_static/tractor_logo_side.svg' 93 | # show_navbar_depth = 1 94 | 95 | # Add any paths that contain custom static files (such as style sheets) here, 96 | # relative to this directory. They are copied after the builtin static files, 97 | # so a file named "default.css" will overwrite the builtin "default.css". 98 | html_static_path = ['_static'] 99 | 100 | # Example configuration for intersphinx: refer to the Python standard library. 101 | intersphinx_mapping = { 102 | "python": ("https://docs.python.org/3", None), 103 | "pytest": ("https://docs.pytest.org/en/latest", None), 104 | "setuptools": ("https://setuptools.readthedocs.io/en/latest", None), 105 | } 106 | -------------------------------------------------------------------------------- /tractor/devx/debug/__init__.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or 5 | # modify it under the terms of the GNU Affero General Public License 6 | # as published by the Free Software Foundation, either version 3 of 7 | # the License, or (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, but 10 | # WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 12 | # Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public 15 | # License along with this program. If not, see 16 | # . 17 | 18 | ''' 19 | Multi-actor debugging for da peeps! 20 | 21 | ''' 22 | from __future__ import annotations 23 | from tractor.log import get_logger 24 | from ._repl import ( 25 | PdbREPL as PdbREPL, 26 | mk_pdb as mk_pdb, 27 | TractorConfig as TractorConfig, 28 | ) 29 | from ._tty_lock import ( 30 | DebugStatus as DebugStatus, 31 | DebugStateError as DebugStateError, 32 | ) 33 | from ._trace import ( 34 | Lock as Lock, 35 | _pause_msg as _pause_msg, 36 | _repl_fail_msg as _repl_fail_msg, 37 | _set_trace as _set_trace, 38 | _sync_pause_from_builtin as _sync_pause_from_builtin, 39 | breakpoint as breakpoint, 40 | maybe_init_greenback as maybe_init_greenback, 41 | maybe_import_greenback as maybe_import_greenback, 42 | pause as pause, 43 | pause_from_sync as pause_from_sync, 44 | ) 45 | from ._post_mortem import ( 46 | BoxedMaybeException as BoxedMaybeException, 47 | maybe_open_crash_handler as maybe_open_crash_handler, 48 | open_crash_handler as open_crash_handler, 49 | post_mortem as post_mortem, 50 | _crash_msg as _crash_msg, 51 | _maybe_enter_pm as _maybe_enter_pm, 52 | ) 53 | from ._sync import ( 54 | maybe_wait_for_debugger as maybe_wait_for_debugger, 55 | acquire_debug_lock as acquire_debug_lock, 56 | ) 57 | from ._sigint import ( 58 | sigint_shield as sigint_shield, 59 | _ctlc_ignore_header as _ctlc_ignore_header 60 | ) 61 | 62 | log = get_logger(__name__) 63 | 64 | # ---------------- 65 | # XXX PKG TODO XXX 66 | # ---------------- 67 | # refine the internal impl and APIs! 68 | # 69 | # -[ ] rework `._pause()` and it's branch-cases for root vs. 70 | # subactor: 71 | # -[ ] `._pause_from_root()` + `_pause_from_subactor()`? 72 | # -[ ] do the de-factor based on bg-thread usage in 73 | # `.pause_from_sync()` & `_pause_from_bg_root_thread()`. 74 | # -[ ] drop `debug_func == None` case which is confusing af.. 75 | # -[ ] factor out `_enter_repl_sync()` into a util func for calling 76 | # the `_set_trace()` / `_post_mortem()` APIs? 77 | # 78 | # -[ ] figure out if we need `acquire_debug_lock()` and/or re-implement 79 | # it as part of the `.pause_from_sync()` rework per above? 80 | # 81 | # -[ ] pair the `._pause_from_subactor()` impl with a "debug nursery" 82 | # that's dynamically allocated inside the `._rpc` task thus 83 | # avoiding the `._service_n.start()` usage for the IPC request? 84 | # -[ ] see the TODO inside `._rpc._errors_relayed_via_ipc()` 85 | # 86 | # -[ ] impl a `open_debug_request()` which encaps all 87 | # `request_root_stdio_lock()` task scheduling deats 88 | # + `DebugStatus` state mgmt; which should prolly be re-branded as 89 | # a `DebugRequest` type anyway AND with suppoort for bg-thread 90 | # (from root actor) usage? 91 | # 92 | # -[ ] handle the `xonsh` case for bg-root-threads in the SIGINT 93 | # handler! 94 | # -[ ] do we need to do the same for subactors? 95 | # -[ ] make the failing tests finally pass XD 96 | # 97 | # -[ ] simplify `maybe_wait_for_debugger()` to be a root-task only 98 | # API? 99 | # -[ ] currently it's implemented as that so might as well make it 100 | # formal? 101 | -------------------------------------------------------------------------------- /examples/debugging/asyncio_bp.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Examples of using the builtin `breakpoint()` from an `asyncio.Task` 3 | running in a subactor spawned with `infect_asyncio=True`. 4 | 5 | ''' 6 | import asyncio 7 | 8 | import trio 9 | import tractor 10 | from tractor import ( 11 | to_asyncio, 12 | Portal, 13 | ) 14 | 15 | 16 | async def aio_sleep_forever(): 17 | await asyncio.sleep(float('inf')) 18 | 19 | 20 | async def bp_then_error( 21 | to_trio: trio.MemorySendChannel, 22 | from_trio: asyncio.Queue, 23 | 24 | raise_after_bp: bool = True, 25 | 26 | ) -> None: 27 | 28 | # sync with `trio`-side (caller) task 29 | to_trio.send_nowait('start') 30 | 31 | # NOTE: what happens here inside the hook needs some refinement.. 32 | # => seems like it's still `.debug._set_trace()` but 33 | # we set `Lock.local_task_in_debug = 'sync'`, we probably want 34 | # some further, at least, meta-data about the task/actor in debug 35 | # in terms of making it clear it's `asyncio` mucking about. 36 | breakpoint() # asyncio-side 37 | 38 | # short checkpoint / delay 39 | await asyncio.sleep(0.5) # asyncio-side 40 | 41 | if raise_after_bp: 42 | raise ValueError('asyncio side error!') 43 | 44 | # TODO: test case with this so that it gets cancelled? 45 | else: 46 | # XXX NOTE: this is required in order to get the SIGINT-ignored 47 | # hang case documented in the module script section! 48 | await aio_sleep_forever() 49 | 50 | 51 | @tractor.context 52 | async def trio_ctx( 53 | ctx: tractor.Context, 54 | bp_before_started: bool = False, 55 | ): 56 | 57 | # this will block until the ``asyncio`` task sends a "first" 58 | # message, see first line in above func. 59 | async with ( 60 | to_asyncio.open_channel_from( 61 | bp_then_error, 62 | # raise_after_bp=not bp_before_started, 63 | ) as (first, chan), 64 | 65 | trio.open_nursery() as tn, 66 | ): 67 | assert first == 'start' 68 | 69 | if bp_before_started: 70 | await tractor.pause() # trio-side 71 | 72 | await ctx.started(first) # trio-side 73 | 74 | tn.start_soon( 75 | to_asyncio.run_task, 76 | aio_sleep_forever, 77 | ) 78 | await trio.sleep_forever() 79 | 80 | 81 | async def main( 82 | bps_all_over: bool = True, 83 | 84 | # TODO, WHICH OF THESE HAZ BUGZ? 85 | cancel_from_root: bool = False, 86 | err_from_root: bool = False, 87 | 88 | ) -> None: 89 | 90 | async with tractor.open_nursery( 91 | debug_mode=True, 92 | maybe_enable_greenback=True, 93 | # loglevel='devx', 94 | ) as an: 95 | ptl: Portal = await an.start_actor( 96 | 'aio_daemon', 97 | enable_modules=[__name__], 98 | infect_asyncio=True, 99 | debug_mode=True, 100 | # loglevel='cancel', 101 | ) 102 | 103 | async with ptl.open_context( 104 | trio_ctx, 105 | bp_before_started=bps_all_over, 106 | ) as (ctx, first): 107 | 108 | assert first == 'start' 109 | 110 | # pause in parent to ensure no cross-actor 111 | # locking problems exist! 112 | await tractor.pause() # trio-root 113 | 114 | if cancel_from_root: 115 | await ctx.cancel() 116 | 117 | if err_from_root: 118 | assert 0 119 | else: 120 | await trio.sleep_forever() 121 | 122 | 123 | # TODO: case where we cancel from trio-side while asyncio task 124 | # has debugger lock? 125 | # await ptl.cancel_actor() 126 | 127 | 128 | if __name__ == '__main__': 129 | 130 | # works fine B) 131 | trio.run(main) 132 | 133 | # will hang and ignores SIGINT !! 134 | # NOTE: you'll need to send a SIGQUIT (via ctl-\) to kill it 135 | # manually.. 136 | # trio.run(main, True) 137 | -------------------------------------------------------------------------------- /tractor/devx/cli.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | """ 18 | CLI framework extensions for hacking on the actor runtime. 19 | 20 | Currently popular frameworks supported are: 21 | 22 | - `typer` via the `@callback` API 23 | 24 | """ 25 | from __future__ import annotations 26 | from typing import ( 27 | Any, 28 | Callable, 29 | ) 30 | from typing_extensions import Annotated 31 | 32 | import typer 33 | 34 | 35 | _runtime_vars: dict[str, Any] = {} 36 | 37 | 38 | def load_runtime_vars( 39 | ctx: typer.Context, 40 | callback: Callable, 41 | pdb: bool = False, # --pdb 42 | ll: Annotated[ 43 | str, 44 | typer.Option( 45 | '--loglevel', 46 | '-l', 47 | help='BigD logging level', 48 | ), 49 | ] = 'cancel', # -l info 50 | ): 51 | ''' 52 | Maybe engage crash handling with `pdbp` when code inside 53 | a `typer` CLI endpoint cmd raises. 54 | 55 | To use this callback simply take your `app = typer.Typer()` instance 56 | and decorate this function with it like so: 57 | 58 | .. code:: python 59 | 60 | from tractor.devx import cli 61 | 62 | app = typer.Typer() 63 | 64 | # manual decoration to hook into `click`'s context system! 65 | cli.load_runtime_vars = app.callback( 66 | invoke_without_command=True, 67 | ) 68 | 69 | And then you can use the now augmented `click` CLI context as so, 70 | 71 | .. code:: python 72 | 73 | @app.command( 74 | context_settings={ 75 | "allow_extra_args": True, 76 | "ignore_unknown_options": True, 77 | } 78 | ) 79 | def my_cli_cmd( 80 | ctx: typer.Context, 81 | ): 82 | rtvars: dict = ctx.runtime_vars 83 | pdb: bool = rtvars['pdb'] 84 | 85 | with tractor.devx.cli.maybe_open_crash_handler(pdb=pdb): 86 | trio.run( 87 | partial( 88 | my_tractor_main_task_func, 89 | debug_mode=pdb, 90 | loglevel=rtvars['ll'], 91 | ) 92 | ) 93 | 94 | which will enable log level and debug mode globally for the entire 95 | `tractor` + `trio` runtime thereafter! 96 | 97 | Bo 98 | 99 | ''' 100 | global _runtime_vars 101 | _runtime_vars |= { 102 | 'pdb': pdb, 103 | 'll': ll, 104 | } 105 | 106 | ctx.runtime_vars: dict[str, Any] = _runtime_vars 107 | print( 108 | f'`typer` sub-cmd: {ctx.invoked_subcommand}\n' 109 | f'`tractor` runtime vars: {_runtime_vars}' 110 | ) 111 | 112 | # XXX NOTE XXX: hackzone.. if no sub-cmd is specified (the 113 | # default if the user just invokes `bigd`) then we simply 114 | # invoke the sole `_bigd()` cmd passing in the "parent" 115 | # typer.Context directly to that call since we're treating it 116 | # as a "non sub-command" or wtv.. 117 | # TODO: ideally typer would have some kinda built-in way to get 118 | # this behaviour without having to construct and manually 119 | # invoke our own cmd.. 120 | if ( 121 | ctx.invoked_subcommand is None 122 | or ctx.invoked_subcommand == callback.__name__ 123 | ): 124 | cmd: typer.core.TyperCommand = typer.core.TyperCommand( 125 | name='bigd', 126 | callback=callback, 127 | ) 128 | ctx.params = {'ctx': ctx} 129 | cmd.invoke(ctx) 130 | -------------------------------------------------------------------------------- /examples/full_fledged_streaming_service.py: -------------------------------------------------------------------------------- 1 | import time 2 | import trio 3 | import tractor 4 | from tractor import ( 5 | ActorNursery, 6 | MsgStream, 7 | Portal, 8 | ) 9 | 10 | 11 | # this is the first 2 actors, streamer_1 and streamer_2 12 | async def stream_data(seed): 13 | for i in range(seed): 14 | yield i 15 | await trio.sleep(0.0001) # trigger scheduler 16 | 17 | 18 | # this is the third actor; the aggregator 19 | async def aggregate(seed): 20 | ''' 21 | Ensure that the two streams we receive match but only stream 22 | a single set of values to the parent. 23 | 24 | ''' 25 | an: ActorNursery 26 | async with tractor.open_nursery() as an: 27 | portals: list[Portal] = [] 28 | for i in range(1, 3): 29 | 30 | # fork/spawn call 31 | portal = await an.start_actor( 32 | name=f'streamer_{i}', 33 | enable_modules=[__name__], 34 | ) 35 | 36 | portals.append(portal) 37 | 38 | send_chan, recv_chan = trio.open_memory_channel(500) 39 | 40 | async def push_to_chan(portal, send_chan): 41 | 42 | # TODO: https://github.com/goodboy/tractor/issues/207 43 | async with send_chan: 44 | async with portal.open_stream_from(stream_data, seed=seed) as stream: 45 | async for value in stream: 46 | # leverage trio's built-in backpressure 47 | await send_chan.send(value) 48 | 49 | print(f"FINISHED ITERATING {portal.channel.uid}") 50 | 51 | # spawn 2 trio tasks to collect streams and push to a local queue 52 | async with trio.open_nursery() as n: 53 | 54 | for portal in portals: 55 | n.start_soon( 56 | push_to_chan, 57 | portal, 58 | send_chan.clone(), 59 | ) 60 | 61 | # close this local task's reference to send side 62 | await send_chan.aclose() 63 | 64 | unique_vals = set() 65 | async with recv_chan: 66 | async for value in recv_chan: 67 | if value not in unique_vals: 68 | unique_vals.add(value) 69 | # yield upwards to the spawning parent actor 70 | yield value 71 | 72 | assert value in unique_vals 73 | 74 | print("FINISHED ITERATING in aggregator") 75 | 76 | await an.cancel() 77 | print("WAITING on `ActorNursery` to finish") 78 | print("AGGREGATOR COMPLETE!") 79 | 80 | 81 | async def main() -> list[int]: 82 | ''' 83 | This is the "root" actor's main task's entrypoint. 84 | 85 | By default (and if not otherwise specified) that root process 86 | also acts as a "registry actor" / "registrar" on the localhost 87 | for the purposes of multi-actor "service discovery". 88 | 89 | ''' 90 | # yes, a nursery which spawns `trio`-"actors" B) 91 | an: ActorNursery 92 | async with tractor.open_nursery( 93 | loglevel='cancel', 94 | # debug_mode=True, 95 | ) as an: 96 | 97 | seed = int(1e3) 98 | pre_start = time.time() 99 | 100 | portal: Portal = await an.start_actor( 101 | name='aggregator', 102 | enable_modules=[__name__], 103 | ) 104 | 105 | stream: MsgStream 106 | async with portal.open_stream_from( 107 | aggregate, 108 | seed=seed, 109 | ) as stream: 110 | 111 | start = time.time() 112 | # the portal call returns exactly what you'd expect 113 | # as if the remote "aggregate" function was called locally 114 | result_stream: list[int] = [] 115 | async for value in stream: 116 | result_stream.append(value) 117 | 118 | cancelled: bool = await portal.cancel_actor() 119 | assert cancelled 120 | 121 | print(f"STREAM TIME = {time.time() - start}") 122 | print(f"STREAM + SPAWN TIME = {time.time() - pre_start}") 123 | assert result_stream == list(range(seed)) 124 | return result_stream 125 | 126 | 127 | if __name__ == '__main__': 128 | final_stream = trio.run(main) 129 | -------------------------------------------------------------------------------- /tractor/ipc/_fd_share.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | ''' 17 | File-descriptor-sharing on `linux` by "wilhelm_of_bohemia". 18 | 19 | ''' 20 | from __future__ import annotations 21 | import os 22 | import array 23 | import socket 24 | import tempfile 25 | from pathlib import Path 26 | from contextlib import ExitStack 27 | 28 | import trio 29 | import tractor 30 | from tractor.ipc import RBToken 31 | 32 | 33 | actor_name = 'ringd' 34 | 35 | 36 | _rings: dict[str, dict] = {} 37 | 38 | 39 | async def _attach_to_ring( 40 | ring_name: str 41 | ) -> tuple[int, int, int]: 42 | actor = tractor.current_actor() 43 | 44 | fd_amount = 3 45 | sock_path = ( 46 | Path(tempfile.gettempdir()) 47 | / 48 | f'{os.getpid()}-pass-ring-fds-{ring_name}-to-{actor.name}.sock' 49 | ) 50 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 51 | sock.bind(sock_path) 52 | sock.listen(1) 53 | 54 | async with ( 55 | tractor.find_actor(actor_name) as ringd, 56 | ringd.open_context( 57 | _pass_fds, 58 | name=ring_name, 59 | sock_path=sock_path 60 | ) as (ctx, _sent) 61 | ): 62 | # prepare array to receive FD 63 | fds = array.array("i", [0] * fd_amount) 64 | 65 | conn, _ = sock.accept() 66 | 67 | # receive FD 68 | msg, ancdata, flags, addr = conn.recvmsg( 69 | 1024, 70 | socket.CMSG_LEN(fds.itemsize * fd_amount) 71 | ) 72 | 73 | for ( 74 | cmsg_level, 75 | cmsg_type, 76 | cmsg_data, 77 | ) in ancdata: 78 | if ( 79 | cmsg_level == socket.SOL_SOCKET 80 | and 81 | cmsg_type == socket.SCM_RIGHTS 82 | ): 83 | fds.frombytes(cmsg_data[:fds.itemsize * fd_amount]) 84 | break 85 | else: 86 | raise RuntimeError("Receiver: No FDs received") 87 | 88 | conn.close() 89 | sock.close() 90 | sock_path.unlink() 91 | 92 | return RBToken.from_msg( 93 | await ctx.wait_for_result() 94 | ) 95 | 96 | 97 | @tractor.context 98 | async def _pass_fds( 99 | ctx: tractor.Context, 100 | name: str, 101 | sock_path: str 102 | ) -> RBToken: 103 | global _rings 104 | token = _rings[name] 105 | client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 106 | client.connect(sock_path) 107 | await ctx.started() 108 | fds = array.array('i', token.fds) 109 | client.sendmsg([b'FDs'], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)]) 110 | client.close() 111 | return token 112 | 113 | 114 | @tractor.context 115 | async def _open_ringbuf( 116 | ctx: tractor.Context, 117 | name: str, 118 | buf_size: int 119 | ) -> RBToken: 120 | global _rings 121 | is_owner = False 122 | if name not in _rings: 123 | stack = ExitStack() 124 | token = stack.enter_context( 125 | tractor.open_ringbuf( 126 | name, 127 | buf_size=buf_size 128 | ) 129 | ) 130 | _rings[name] = { 131 | 'token': token, 132 | 'stack': stack, 133 | } 134 | is_owner = True 135 | 136 | ring = _rings[name] 137 | await ctx.started() 138 | 139 | try: 140 | await trio.sleep_forever() 141 | 142 | except tractor.ContextCancelled: 143 | ... 144 | 145 | finally: 146 | if is_owner: 147 | ring['stack'].close() 148 | 149 | 150 | async def open_ringbuf( 151 | name: str, 152 | buf_size: int 153 | ) -> RBToken: 154 | async with ( 155 | tractor.find_actor(actor_name) as ringd, 156 | ringd.open_context( 157 | _open_ringbuf, 158 | name=name, 159 | buf_size=buf_size 160 | ) as (rd_ctx, _) 161 | ): 162 | yield await _attach_to_ring(name) 163 | await rd_ctx.cancel() 164 | -------------------------------------------------------------------------------- /tractor/ipc/_linux.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | ''' 17 | Linux specifics, for now we are only exposing EventFD 18 | 19 | ''' 20 | import os 21 | import errno 22 | 23 | import cffi 24 | import trio 25 | 26 | ffi = cffi.FFI() 27 | 28 | # Declare the C functions and types we plan to use. 29 | # - eventfd: for creating the event file descriptor 30 | # - write: for writing to the file descriptor 31 | # - read: for reading from the file descriptor 32 | # - close: for closing the file descriptor 33 | ffi.cdef( 34 | ''' 35 | int eventfd(unsigned int initval, int flags); 36 | 37 | ssize_t write(int fd, const void *buf, size_t count); 38 | ssize_t read(int fd, void *buf, size_t count); 39 | 40 | int close(int fd); 41 | ''' 42 | ) 43 | 44 | 45 | # Open the default dynamic library (essentially 'libc' in most cases) 46 | C = ffi.dlopen(None) 47 | 48 | 49 | # Constants from , if needed. 50 | EFD_SEMAPHORE = 1 51 | EFD_CLOEXEC = 0o2000000 52 | EFD_NONBLOCK = 0o4000 53 | 54 | 55 | def open_eventfd(initval: int = 0, flags: int = 0) -> int: 56 | ''' 57 | Open an eventfd with the given initial value and flags. 58 | Returns the file descriptor on success, otherwise raises OSError. 59 | 60 | ''' 61 | fd = C.eventfd(initval, flags) 62 | if fd < 0: 63 | raise OSError(errno.errorcode[ffi.errno], 'eventfd failed') 64 | return fd 65 | 66 | 67 | def write_eventfd(fd: int, value: int) -> int: 68 | ''' 69 | Write a 64-bit integer (uint64_t) to the eventfd's counter. 70 | 71 | ''' 72 | # Create a uint64_t* in C, store `value` 73 | data_ptr = ffi.new('uint64_t *', value) 74 | 75 | # Call write(fd, data_ptr, 8) 76 | # We expect to write exactly 8 bytes (sizeof(uint64_t)) 77 | ret = C.write(fd, data_ptr, 8) 78 | if ret < 0: 79 | raise OSError(errno.errorcode[ffi.errno], 'write to eventfd failed') 80 | return ret 81 | 82 | 83 | def read_eventfd(fd: int) -> int: 84 | ''' 85 | Read a 64-bit integer (uint64_t) from the eventfd, returning the value. 86 | Reading resets the counter to 0 (unless using EFD_SEMAPHORE). 87 | 88 | ''' 89 | # Allocate an 8-byte buffer in C for reading 90 | buf = ffi.new('char[]', 8) 91 | 92 | ret = C.read(fd, buf, 8) 93 | if ret < 0: 94 | raise OSError(errno.errorcode[ffi.errno], 'read from eventfd failed') 95 | # Convert the 8 bytes we read into a Python integer 96 | data_bytes = ffi.unpack(buf, 8) # returns a Python bytes object of length 8 97 | value = int.from_bytes(data_bytes, byteorder='little', signed=False) 98 | return value 99 | 100 | 101 | def close_eventfd(fd: int) -> int: 102 | ''' 103 | Close the eventfd. 104 | 105 | ''' 106 | ret = C.close(fd) 107 | if ret < 0: 108 | raise OSError(errno.errorcode[ffi.errno], 'close failed') 109 | 110 | 111 | class EventFD: 112 | ''' 113 | Use a previously opened eventfd(2), meant to be used in 114 | sub-actors after root actor opens the eventfds then passes 115 | them through pass_fds 116 | 117 | ''' 118 | 119 | def __init__( 120 | self, 121 | fd: int, 122 | omode: str 123 | ): 124 | self._fd: int = fd 125 | self._omode: str = omode 126 | self._fobj = None 127 | 128 | @property 129 | def fd(self) -> int | None: 130 | return self._fd 131 | 132 | def write(self, value: int) -> int: 133 | return write_eventfd(self._fd, value) 134 | 135 | async def read(self) -> int: 136 | return await trio.to_thread.run_sync( 137 | read_eventfd, self._fd, 138 | abandon_on_cancel=True 139 | ) 140 | 141 | def open(self): 142 | self._fobj = os.fdopen(self._fd, self._omode) 143 | 144 | def close(self): 145 | if self._fobj: 146 | self._fobj.close() 147 | 148 | def __enter__(self): 149 | self.open() 150 | return self 151 | 152 | def __exit__(self, exc_type, exc_value, traceback): 153 | self.close() 154 | -------------------------------------------------------------------------------- /tests/test_shm.py: -------------------------------------------------------------------------------- 1 | """ 2 | Shared mem primitives and APIs. 3 | 4 | """ 5 | import uuid 6 | 7 | # import numpy 8 | import pytest 9 | import trio 10 | import tractor 11 | from tractor.ipc._shm import ( 12 | open_shm_list, 13 | attach_shm_list, 14 | ) 15 | 16 | 17 | @tractor.context 18 | async def child_attach_shml_alot( 19 | ctx: tractor.Context, 20 | shm_key: str, 21 | ) -> None: 22 | 23 | await ctx.started(shm_key) 24 | 25 | # now try to attach a boatload of times in a loop.. 26 | for _ in range(1000): 27 | shml = attach_shm_list( 28 | key=shm_key, 29 | readonly=False, 30 | ) 31 | assert shml.shm.name == shm_key 32 | await trio.sleep(0.001) 33 | 34 | 35 | def test_child_attaches_alot(): 36 | async def main(): 37 | async with tractor.open_nursery() as an: 38 | 39 | # allocate writeable list in parent 40 | key = f'shml_{uuid.uuid4()}' 41 | shml = open_shm_list( 42 | key=key, 43 | ) 44 | 45 | portal = await an.start_actor( 46 | 'shm_attacher', 47 | enable_modules=[__name__], 48 | ) 49 | 50 | async with ( 51 | portal.open_context( 52 | child_attach_shml_alot, 53 | shm_key=shml.key, 54 | ) as (ctx, start_val), 55 | ): 56 | assert start_val == key 57 | await ctx.result() 58 | 59 | await portal.cancel_actor() 60 | 61 | trio.run(main) 62 | 63 | 64 | @tractor.context 65 | async def child_read_shm_list( 66 | ctx: tractor.Context, 67 | shm_key: str, 68 | use_str: bool, 69 | frame_size: int, 70 | ) -> None: 71 | 72 | # attach in child 73 | shml = attach_shm_list( 74 | key=shm_key, 75 | # dtype=str if use_str else float, 76 | ) 77 | await ctx.started(shml.key) 78 | 79 | async with ctx.open_stream() as stream: 80 | async for i in stream: 81 | print(f'(child): reading shm list index: {i}') 82 | 83 | if use_str: 84 | expect = str(float(i)) 85 | else: 86 | expect = float(i) 87 | 88 | if frame_size == 1: 89 | val = shml[i] 90 | assert expect == val 91 | print(f'(child): reading value: {val}') 92 | else: 93 | frame = shml[i - frame_size:i] 94 | print(f'(child): reading frame: {frame}') 95 | 96 | 97 | @pytest.mark.parametrize( 98 | 'use_str', 99 | [False, True], 100 | ids=lambda i: f'use_str_values={i}', 101 | ) 102 | @pytest.mark.parametrize( 103 | 'frame_size', 104 | [1, 2**6, 2**10], 105 | ids=lambda i: f'frame_size={i}', 106 | ) 107 | def test_parent_writer_child_reader( 108 | use_str: bool, 109 | frame_size: int, 110 | ): 111 | 112 | async def main(): 113 | async with tractor.open_nursery( 114 | # debug_mode=True, 115 | ) as an: 116 | 117 | portal = await an.start_actor( 118 | 'shm_reader', 119 | enable_modules=[__name__], 120 | debug_mode=True, 121 | ) 122 | 123 | # allocate writeable list in parent 124 | key = 'shm_list' 125 | seq_size = int(2 * 2 ** 10) 126 | shml = open_shm_list( 127 | key=key, 128 | size=seq_size, 129 | dtype=str if use_str else float, 130 | readonly=False, 131 | ) 132 | 133 | async with ( 134 | portal.open_context( 135 | child_read_shm_list, 136 | shm_key=key, 137 | use_str=use_str, 138 | frame_size=frame_size, 139 | ) as (ctx, sent), 140 | 141 | ctx.open_stream() as stream, 142 | ): 143 | 144 | assert sent == key 145 | 146 | for i in range(seq_size): 147 | 148 | val = float(i) 149 | if use_str: 150 | val = str(val) 151 | 152 | # print(f'(parent): writing {val}') 153 | shml[i] = val 154 | 155 | # only on frame fills do we 156 | # signal to the child that a frame's 157 | # worth is ready. 158 | if (i % frame_size) == 0: 159 | print(f'(parent): signalling frame full on {val}') 160 | await stream.send(i) 161 | else: 162 | print(f'(parent): signalling final frame on {val}') 163 | await stream.send(i) 164 | 165 | await portal.cancel_actor() 166 | 167 | trio.run(main) 168 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | # ------ build-system ------ 6 | 7 | [project] 8 | name = "tractor" 9 | version = "0.1.0a6dev0" 10 | description = 'structured concurrent `trio`-"actors"' 11 | authors = [{ name = "Tyler Goodlet", email = "goodboy_foss@protonmail.com" }] 12 | requires-python = ">= 3.11" 13 | readme = "docs/README.rst" 14 | license = "AGPL-3.0-or-later" 15 | keywords = [ 16 | "trio", 17 | "async", 18 | "concurrency", 19 | "structured concurrency", 20 | "actor model", 21 | "distributed", 22 | "multiprocessing", 23 | ] 24 | classifiers = [ 25 | "Development Status :: 3 - Alpha", 26 | "Operating System :: POSIX :: Linux", 27 | "Framework :: Trio", 28 | "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", 29 | "Programming Language :: Python :: Implementation :: CPython", 30 | "Programming Language :: Python :: 3 :: Only", 31 | "Programming Language :: Python :: 3.11", 32 | "Topic :: System :: Distributed Computing", 33 | ] 34 | dependencies = [ 35 | # trio runtime and friends 36 | # (poetry) proper range specs, 37 | # https://packaging.python.org/en/latest/discussions/install-requires-vs-requirements/#id5 38 | # TODO, for 3.13 we must go go `0.27` which means we have to 39 | # disable strict egs or port to handling them internally! 40 | "trio>0.27", 41 | "tricycle>=0.4.1,<0.5", 42 | "wrapt>=1.16.0,<2", 43 | "colorlog>=6.8.2,<7", 44 | # built-in multi-actor `pdb` REPL 45 | "pdbp>=1.6,<2", # windows only (from `pdbp`) 46 | # typed IPC msging 47 | "msgspec>=0.19.0", 48 | "cffi>=1.17.1", 49 | "bidict>=0.23.1", 50 | ] 51 | 52 | # ------ project ------ 53 | 54 | [dependency-groups] 55 | dev = [ 56 | # test suite 57 | # TODO: maybe some of these layout choices? 58 | # https://docs.pytest.org/en/8.0.x/explanation/goodpractices.html#choosing-a-test-layout-import-rules 59 | "pytest>=8.3.5", 60 | "pexpect>=4.9.0,<5", 61 | # `tractor.devx` tooling 62 | "greenback>=1.2.1,<2", 63 | "stackscope>=0.2.2,<0.3", 64 | # ^ requires this? 65 | "typing-extensions>=4.14.1", 66 | 67 | "pyperclip>=1.9.0", 68 | "prompt-toolkit>=3.0.50", 69 | "xonsh>=0.19.2", 70 | "psutil>=7.0.0", 71 | ] 72 | # TODO, add these with sane versions; were originally in 73 | # `requirements-docs.txt`.. 74 | # docs = [ 75 | # "sphinx>=" 76 | # "sphinx_book_theme>=" 77 | # ] 78 | 79 | # ------ dependency-groups ------ 80 | 81 | # ------ dependency-groups ------ 82 | 83 | [tool.uv.sources] 84 | # XXX NOTE, only for @goodboy's hacking on `pprint(sort_dicts=False)` 85 | # for the `pp` alias.. 86 | # pdbp = { path = "../pdbp", editable = true } 87 | 88 | # ------ tool.uv.sources ------ 89 | # TODO, distributed (multi-host) extensions 90 | # linux kernel networking 91 | # 'pyroute2 92 | 93 | # ------ tool.uv.sources ------ 94 | 95 | [tool.uv] 96 | # XXX NOTE, prefer the sys python bc apparently the distis from 97 | # `astral` are built in a way that breaks `pdbp`+`tabcompleter`'s 98 | # likely due to linking against `libedit` over `readline`.. 99 | # |_https://docs.astral.sh/uv/concepts/python-versions/#managed-python-distributions 100 | # |_https://gregoryszorc.com/docs/python-build-standalone/main/quirks.html#use-of-libedit-on-linux 101 | # 102 | # https://docs.astral.sh/uv/reference/settings/#python-preference 103 | python-preference = 'system' 104 | 105 | # ------ tool.uv ------ 106 | 107 | [tool.hatch.build.targets.sdist] 108 | include = ["tractor"] 109 | 110 | [tool.hatch.build.targets.wheel] 111 | include = ["tractor"] 112 | 113 | # ------ tool.hatch ------ 114 | 115 | [tool.towncrier] 116 | package = "tractor" 117 | filename = "NEWS.rst" 118 | directory = "nooz/" 119 | version = "0.1.0a6" 120 | title_format = "tractor {version} ({project_date})" 121 | template = "nooz/_template.rst" 122 | all_bullets = true 123 | 124 | [[tool.towncrier.type]] 125 | directory = "feature" 126 | name = "Features" 127 | showcontent = true 128 | 129 | [[tool.towncrier.type]] 130 | directory = "bugfix" 131 | name = "Bug Fixes" 132 | showcontent = true 133 | 134 | [[tool.towncrier.type]] 135 | directory = "doc" 136 | name = "Improved Documentation" 137 | showcontent = true 138 | 139 | [[tool.towncrier.type]] 140 | directory = "trivial" 141 | name = "Trivial/Internal Changes" 142 | showcontent = true 143 | 144 | # ------ tool.towncrier ------ 145 | 146 | [tool.pytest.ini_options] 147 | minversion = '6.0' 148 | testpaths = [ 149 | 'tests' 150 | ] 151 | addopts = [ 152 | # TODO: figure out why this isn't working.. 153 | '--rootdir=./tests', 154 | 155 | '--import-mode=importlib', 156 | # don't show frickin captured logs AGAIN in the report.. 157 | '--show-capture=no', 158 | ] 159 | log_cli = false 160 | # TODO: maybe some of these layout choices? 161 | # https://docs.pytest.org/en/8.0.x/explanation/goodpractices.html#choosing-a-test-layout-import-rules 162 | # pythonpath = "src" 163 | 164 | # ------ tool.pytest ------ 165 | -------------------------------------------------------------------------------- /docs/github_readme/_sphinx_readme.rst: -------------------------------------------------------------------------------- 1 | tractor 2 | ======= 3 | The Python async-native multi-core system *you always wanted*. 4 | 5 | 6 | |gh_actions| 7 | |docs| 8 | 9 | .. _actor model: https://en.wikipedia.org/wiki/Actor_model 10 | .. _trio: https://github.com/python-trio/trio 11 | .. _multi-processing: https://en.wikipedia.org/wiki/Multiprocessing 12 | .. _trionic: https://trio.readthedocs.io/en/latest/design.html#high-level-design-principles 13 | .. _async sandwich: https://trio.readthedocs.io/en/latest/tutorial.html#async-sandwich 14 | .. _structured concurrent: https://trio.discourse.group/t/concise-definition-of-structured-concurrency/228 15 | 16 | 17 | ``tractor`` is a `structured concurrent`_ "`actor model`_" built on trio_ and multi-processing_. 18 | 19 | It is an attempt to pair trionic_ `structured concurrency`_ with 20 | distributed Python. You can think of it as a ``trio`` 21 | *-across-processes* or simply as an opinionated replacement for the 22 | stdlib's ``multiprocessing`` but built on async programming primitives 23 | from the ground up. 24 | 25 | Don't be scared off by this description. ``tractor`` **is just ``trio``** 26 | but with nurseries for process management and cancel-able IPC. 27 | If you understand how to work with ``trio``, ``tractor`` will give you 28 | the parallelism you've been missing. 29 | 30 | ``tractor``'s nurseries let you spawn ``trio`` *"actors"*: new Python 31 | processes which each run a ``trio`` scheduled task tree (also known as 32 | an `async sandwich`_ - a call to ``trio.run()``). That is, each 33 | "*Actor*" is a new process plus a ``trio`` runtime. 34 | 35 | "Actors" communicate by exchanging asynchronous messages_ and avoid 36 | sharing state. The intention of this model is to allow for highly 37 | distributed software that, through the adherence to *structured 38 | concurrency*, results in systems which fail in predictable and 39 | recoverable ways. 40 | 41 | The first step to grok ``tractor`` is to get the basics of ``trio`` down. 42 | A great place to start is the `trio docs`_ and this `blog post`_. 43 | 44 | .. _messages: https://en.wikipedia.org/wiki/Message_passing 45 | .. _trio docs: https://trio.readthedocs.io/en/latest/ 46 | .. _blog post: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/ 47 | .. _structured concurrency: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/ 48 | .. _3 axioms: https://en.wikipedia.org/wiki/Actor_model#Fundamental_concepts 49 | .. _unrequirements: https://en.wikipedia.org/wiki/Actor_model#Direct_communication_and_asynchrony 50 | .. _async generators: https://www.python.org/dev/peps/pep-0525/ 51 | 52 | 53 | Install 54 | ------- 55 | No PyPi release yet! 56 | 57 | :: 58 | 59 | pip install git+git://github.com/goodboy/tractor.git 60 | 61 | 62 | Alluring Features 63 | ----------------- 64 | - **It's just** ``trio``, but with SC applied to processes (aka "actors") 65 | - Infinitely nesteable process trees 66 | - Built-in API for inter-process streaming 67 | - A (first ever?) "native" multi-core debugger for Python using `pdb++`_ 68 | - (Soon to land) ``asyncio`` support allowing for "infected" actors where 69 | `trio` drives the `asyncio` scheduler via the astounding "`guest mode`_" 70 | 71 | 72 | Example: self-destruct a process tree 73 | ------------------------------------- 74 | .. literalinclude:: ../../examples/parallelism/we_are_processes.py 75 | :language: python 76 | 77 | 78 | The example you're probably after... 79 | ------------------------------------ 80 | It seems the initial query from most new users is "how do I make a worker 81 | pool thing?". 82 | 83 | ``tractor`` is built to handle any SC process tree you can 84 | imagine; the "worker pool" pattern is a trivial special case: 85 | 86 | .. literalinclude:: ../../examples/parallelism/concurrent_actors_primes.py 87 | :language: python 88 | 89 | 90 | Feel like saying hi? 91 | -------------------- 92 | This project is very much coupled to the ongoing development of 93 | ``trio`` (i.e. ``tractor`` gets most of its ideas from that brilliant 94 | community). If you want to help, have suggestions or just want to 95 | say hi, please feel free to reach us in our `matrix channel`_. If 96 | matrix seems too hip, we're also mostly all in the the `trio gitter 97 | channel`_! 98 | 99 | .. _trio gitter channel: https://gitter.im/python-trio/general 100 | .. _matrix channel: https://matrix.to/#/!tractor:matrix.org 101 | .. _pdb++: https://github.com/pdbpp/pdbpp 102 | .. _guest mode: https://trio.readthedocs.io/en/stable/reference-lowlevel.html?highlight=guest%20mode#using-guest-mode-to-run-trio-on-top-of-other-event-loops 103 | 104 | 105 | .. |gh_actions| image:: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fgoodboy%2Ftractor%2Fbadge&style=popout-square 106 | :target: https://actions-badge.atrox.dev/goodboy/tractor/goto 107 | .. |docs| image:: https://readthedocs.org/projects/tractor/badge/?version=latest 108 | :target: https://tractor.readthedocs.io/en/latest/?badge=latest 109 | :alt: Documentation Status 110 | -------------------------------------------------------------------------------- /tractor/_multiaddr.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | ''' 18 | Multiaddress parser and utils according the spec(s) defined by 19 | `libp2p` and used in dependent project such as `ipfs`: 20 | 21 | - https://docs.libp2p.io/concepts/fundamentals/addressing/ 22 | - https://github.com/libp2p/specs/blob/master/addressing/README.md 23 | 24 | ''' 25 | from typing import Iterator 26 | 27 | from bidict import bidict 28 | 29 | # TODO: see if we can leverage libp2p ecosys projects instead of 30 | # rolling our own (parser) impls of the above addressing specs: 31 | # - https://github.com/libp2p/py-libp2p 32 | # - https://docs.libp2p.io/concepts/nat/circuit-relay/#relay-addresses 33 | # prots: bidict[int, str] = bidict({ 34 | prots: bidict[int, str] = { 35 | 'ipv4': 3, 36 | 'ipv6': 3, 37 | 'wg': 3, 38 | 39 | 'tcp': 4, 40 | 'udp': 4, 41 | 42 | # TODO: support the next-gen shite Bo 43 | # 'quic': 4, 44 | # 'ssh': 7, # via rsyscall bootstrapping 45 | } 46 | 47 | prot_params: dict[str, tuple[str]] = { 48 | 'ipv4': ('addr',), 49 | 'ipv6': ('addr',), 50 | 'wg': ('addr', 'port', 'pubkey'), 51 | 52 | 'tcp': ('port',), 53 | 'udp': ('port',), 54 | 55 | # 'quic': ('port',), 56 | # 'ssh': ('port',), 57 | } 58 | 59 | 60 | def iter_prot_layers( 61 | multiaddr: str, 62 | ) -> Iterator[ 63 | tuple[ 64 | int, 65 | list[str] 66 | ] 67 | ]: 68 | ''' 69 | Unpack a libp2p style "multiaddress" into multiple "segments" 70 | for each "layer" of the protocoll stack (in OSI terms). 71 | 72 | ''' 73 | tokens: list[str] = multiaddr.split('/') 74 | root, tokens = tokens[0], tokens[1:] 75 | assert not root # there is a root '/' on LHS 76 | itokens = iter(tokens) 77 | 78 | prot: str | None = None 79 | params: list[str] = [] 80 | for token in itokens: 81 | # every prot path should start with a known 82 | # key-str. 83 | if token in prots: 84 | if prot is None: 85 | prot: str = token 86 | else: 87 | yield prot, params 88 | prot = token 89 | 90 | params = [] 91 | 92 | elif token not in prots: 93 | params.append(token) 94 | 95 | else: 96 | yield prot, params 97 | 98 | 99 | def parse_maddr( 100 | multiaddr: str, 101 | ) -> dict[str, str | int | dict]: 102 | ''' 103 | Parse a libp2p style "multiaddress" into its distinct protocol 104 | segments where each segment is of the form: 105 | 106 | `..////../` 107 | 108 | and is loaded into a (order preserving) `layers: dict[str, 109 | dict[str, Any]` which holds each protocol-layer-segment of the 110 | original `str` path as a separate entry according to its approx 111 | OSI "layer number". 112 | 113 | Any `paramN` in the path must be distinctly defined by a str-token in the 114 | (module global) `prot_params` table. 115 | 116 | For eg. for wireguard which requires an address, port number and publickey 117 | the protocol params are specified as the entry: 118 | 119 | 'wg': ('addr', 'port', 'pubkey'), 120 | 121 | and are thus parsed from a maddr in that order: 122 | `'/wg/1.1.1.1/51820/'` 123 | 124 | ''' 125 | layers: dict[str, str | int | dict] = {} 126 | for ( 127 | prot_key, 128 | params, 129 | ) in iter_prot_layers(multiaddr): 130 | 131 | layer: int = prots[prot_key] # OSI layer used for sorting 132 | ep: dict[str, int | str] = {'layer': layer} 133 | layers[prot_key] = ep 134 | 135 | # TODO; validation and resolving of names: 136 | # - each param via a validator provided as part of the 137 | # prot_params def? (also see `"port"` case below..) 138 | # - do a resolv step that will check addrs against 139 | # any loaded network.resolv: dict[str, str] 140 | rparams: list = list(reversed(params)) 141 | for key in prot_params[prot_key]: 142 | val: str | int = rparams.pop() 143 | 144 | # TODO: UGHH, dunno what we should do for validation 145 | # here, put it in the params spec somehow? 146 | if key == 'port': 147 | val = int(val) 148 | 149 | ep[key] = val 150 | 151 | return layers 152 | -------------------------------------------------------------------------------- /tractor/msg/ptr.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | ''' 18 | IPC-compat cross-mem-boundary object pointer. 19 | 20 | ''' 21 | 22 | # TODO: integration with our ``enable_modules: list[str]`` caps sys. 23 | 24 | # ``pkgutil.resolve_name()`` internally uses 25 | # ``importlib.import_module()`` which can be filtered by inserting 26 | # a ``MetaPathFinder`` into ``sys.meta_path`` (which we could do before 27 | # entering the ``_runtime.process_messages()`` loop). 28 | # - https://github.com/python/cpython/blob/main/Lib/pkgutil.py#L645 29 | # - https://stackoverflow.com/questions/1350466/preventing-python-code-from-importing-certain-modules 30 | # - https://stackoverflow.com/a/63320902 31 | # - https://docs.python.org/3/library/sys.html#sys.meta_path 32 | 33 | # the new "Implicit Namespace Packages" might be relevant? 34 | # - https://www.python.org/dev/peps/pep-0420/ 35 | 36 | # add implicit serialized message type support so that paths can be 37 | # handed directly to IPC primitives such as streams and `Portal.run()` 38 | # calls: 39 | # - via ``msgspec``: 40 | # - https://jcristharif.com/msgspec/api.html#struct 41 | # - https://jcristharif.com/msgspec/extending.html 42 | # via ``msgpack-python``: 43 | # - https://github.com/msgpack/msgpack-python#packingunpacking-of-custom-data-type 44 | 45 | from __future__ import annotations 46 | from inspect import ( 47 | isfunction, 48 | ismethod, 49 | ) 50 | from pkgutil import resolve_name 51 | 52 | 53 | class NamespacePath(str): 54 | ''' 55 | A serializeable `str`-subtype implementing a "namespace 56 | pointer" to any Python object reference (like a function) 57 | using the same format as the built-in `pkgutil.resolve_name()` 58 | system. 59 | 60 | A value describes a target's module-path and namespace-key 61 | separated by a ':' and thus can be easily used as 62 | a IPC-message-native reference-type allowing memory isolated 63 | actors to point-and-load objects via a minimal `str` value. 64 | 65 | ''' 66 | _ref: object | type | None = None 67 | 68 | # TODO: support providing the ns instance in 69 | # order to support 'self.` style to make 70 | # `Portal.run_from_ns()` work! 71 | # _ns: ModuleType|type|None = None 72 | 73 | def load_ref(self) -> object | type: 74 | if self._ref is None: 75 | self._ref = resolve_name(self) 76 | return self._ref 77 | 78 | @staticmethod 79 | def _mk_fqnp( 80 | ref: type|object, 81 | ) -> tuple[str, str]: 82 | ''' 83 | Generate a minial `str` pair which describes a python 84 | object's namespace path and object/type name. 85 | 86 | In more precise terms something like: 87 | - 'py.namespace.path:object_name', 88 | - eg.'tractor.msg:NamespacePath' will be the ``str`` form 89 | of THIS type XD 90 | 91 | ''' 92 | if isfunction(ref): 93 | name: str = getattr(ref, '__name__') 94 | mod_name: str = ref.__module__ 95 | 96 | elif ismethod(ref): 97 | # build out the path manually i guess..? 98 | # TODO: better way? 99 | name: str = '.'.join([ 100 | type(ref.__self__).__name__, 101 | ref.__func__.__name__, 102 | ]) 103 | mod_name: str = ref.__self__.__module__ 104 | 105 | else: # object or other? 106 | # isinstance(ref, object) 107 | # and not isfunction(ref) 108 | name: str = type(ref).__name__ 109 | mod_name: str = ref.__module__ 110 | 111 | # TODO: return static value direactly? 112 | # 113 | # fully qualified namespace path, tuple. 114 | fqnp: tuple[str, str] = ( 115 | mod_name, 116 | name, 117 | ) 118 | return fqnp 119 | 120 | @classmethod 121 | def from_ref( 122 | cls, 123 | ref: type|object, 124 | 125 | ) -> NamespacePath: 126 | 127 | fqnp: tuple[str, str] = cls._mk_fqnp(ref) 128 | return cls(':'.join(fqnp)) 129 | 130 | def to_tuple( 131 | self, 132 | 133 | # TODO: could this work re `self:` case from above? 134 | # load_ref: bool = True, 135 | 136 | ) -> tuple[str, str]: 137 | return self._mk_fqnp( 138 | self.load_ref() 139 | ) 140 | -------------------------------------------------------------------------------- /tests/test_docs_examples.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Let's make sure them docs work yah? 3 | 4 | ''' 5 | from contextlib import contextmanager 6 | import itertools 7 | import os 8 | import sys 9 | import subprocess 10 | import platform 11 | import shutil 12 | 13 | import pytest 14 | from tractor._testing import ( 15 | examples_dir, 16 | ) 17 | 18 | 19 | @pytest.fixture 20 | def run_example_in_subproc( 21 | loglevel: str, 22 | testdir: pytest.Pytester, 23 | reg_addr: tuple[str, int], 24 | ): 25 | 26 | @contextmanager 27 | def run(script_code): 28 | kwargs = dict() 29 | 30 | if platform.system() == 'Windows': 31 | # on windows we need to create a special __main__.py which will 32 | # be executed with ``python -m `` on windows.. 33 | shutil.copyfile( 34 | examples_dir() / '__main__.py', 35 | str(testdir / '__main__.py'), 36 | ) 37 | 38 | # drop the ``if __name__ == '__main__'`` guard onwards from 39 | # the *NIX version of each script 40 | windows_script_lines = itertools.takewhile( 41 | lambda line: "if __name__ ==" not in line, 42 | script_code.splitlines() 43 | ) 44 | script_code = '\n'.join(windows_script_lines) 45 | script_file = testdir.makefile('.py', script_code) 46 | 47 | # without this, tests hang on windows forever 48 | kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP 49 | 50 | # run the testdir "libary module" as a script 51 | cmdargs = [ 52 | sys.executable, 53 | '-m', 54 | # use the "module name" of this "package" 55 | 'test_example' 56 | ] 57 | else: 58 | script_file = testdir.makefile('.py', script_code) 59 | cmdargs = [ 60 | sys.executable, 61 | str(script_file), 62 | ] 63 | 64 | # XXX: BE FOREVER WARNED: if you enable lots of tractor logging 65 | # in the subprocess it may cause infinite blocking on the pipes 66 | # due to backpressure!!! 67 | proc = testdir.popen( 68 | cmdargs, 69 | stdin=subprocess.PIPE, 70 | stdout=subprocess.PIPE, 71 | stderr=subprocess.PIPE, 72 | **kwargs, 73 | ) 74 | assert not proc.returncode 75 | yield proc 76 | proc.wait() 77 | assert proc.returncode == 0 78 | 79 | yield run 80 | 81 | 82 | @pytest.mark.parametrize( 83 | 'example_script', 84 | 85 | # walk yields: (dirpath, dirnames, filenames) 86 | [ 87 | (p[0], f) 88 | for p in os.walk(examples_dir()) 89 | for f in p[2] 90 | 91 | if ( 92 | '__' not in f 93 | and f[0] != '_' 94 | and 'debugging' not in p[0] 95 | and 'integration' not in p[0] 96 | and 'advanced_faults' not in p[0] 97 | and 'multihost' not in p[0] 98 | and 'trio' not in p[0] 99 | ) 100 | ], 101 | ids=lambda t: t[1], 102 | ) 103 | def test_example( 104 | run_example_in_subproc, 105 | example_script, 106 | ): 107 | ''' 108 | Load and run scripts from this repo's ``examples/`` dir as a user 109 | would copy and pasing them into their editor. 110 | 111 | On windows a little more "finessing" is done to make 112 | ``multiprocessing`` play nice: we copy the ``__main__.py`` into the 113 | test directory and invoke the script as a module with ``python -m 114 | test_example``. 115 | 116 | ''' 117 | ex_file: str = os.path.join(*example_script) 118 | 119 | if 'rpc_bidir_streaming' in ex_file and sys.version_info < (3, 9): 120 | pytest.skip("2-way streaming example requires py3.9 async with syntax") 121 | 122 | with open(ex_file, 'r') as ex: 123 | code = ex.read() 124 | 125 | with run_example_in_subproc(code) as proc: 126 | err = None 127 | try: 128 | if not proc.poll(): 129 | _, err = proc.communicate(timeout=15) 130 | 131 | except subprocess.TimeoutExpired as e: 132 | proc.kill() 133 | err = e.stderr 134 | 135 | # if we get some gnarly output let's aggregate and raise 136 | if err: 137 | errmsg = err.decode() 138 | errlines = errmsg.splitlines() 139 | last_error = errlines[-1] 140 | if ( 141 | 'Error' in last_error 142 | 143 | # XXX: currently we print this to console, but maybe 144 | # shouldn't eventually once we figure out what's 145 | # a better way to be explicit about aio side 146 | # cancels? 147 | and 148 | 'asyncio.exceptions.CancelledError' not in last_error 149 | ): 150 | raise Exception(errmsg) 151 | 152 | assert proc.returncode == 0 153 | -------------------------------------------------------------------------------- /tractor/_mp_fixup_main.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | """ 18 | Helpers pulled mostly verbatim from ``multiprocessing.spawn`` 19 | to aid with "fixing up" the ``__main__`` module in subprocesses. 20 | 21 | These helpers are needed for any spawing backend that doesn't already 22 | handle this. For example when using ``trio_run_in_process`` it is needed 23 | but obviously not when we're already using ``multiprocessing``. 24 | 25 | """ 26 | import os 27 | import sys 28 | import platform 29 | import types 30 | import runpy 31 | 32 | 33 | ORIGINAL_DIR = os.path.abspath(os.getcwd()) 34 | 35 | 36 | def _mp_figure_out_main() -> dict[str, str]: 37 | """Taken from ``multiprocessing.spawn.get_preparation_data()``. 38 | 39 | Retrieve parent actor `__main__` module data. 40 | """ 41 | d = {} 42 | # Figure out whether to initialise main in the subprocess as a module 43 | # or through direct execution (or to leave it alone entirely) 44 | main_module = sys.modules['__main__'] 45 | main_mod_name = getattr(main_module.__spec__, "name", None) 46 | if main_mod_name is not None: 47 | d['init_main_from_name'] = main_mod_name 48 | # elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE): 49 | elif platform.system() != 'Windows': 50 | main_path = getattr(main_module, '__file__', None) 51 | if main_path is not None: 52 | if ( 53 | not os.path.isabs(main_path) and ( 54 | ORIGINAL_DIR is not None) 55 | ): 56 | # process.ORIGINAL_DIR is not None): 57 | # main_path = os.path.join(process.ORIGINAL_DIR, main_path) 58 | main_path = os.path.join(ORIGINAL_DIR, main_path) 59 | d['init_main_from_path'] = os.path.normpath(main_path) 60 | 61 | return d 62 | 63 | 64 | # Multiprocessing module helpers to fix up the main module in 65 | # spawned subprocesses 66 | def _fixup_main_from_name(mod_name: str) -> None: 67 | # __main__.py files for packages, directories, zip archives, etc, run 68 | # their "main only" code unconditionally, so we don't even try to 69 | # populate anything in __main__, nor do we make any changes to 70 | # __main__ attributes 71 | current_main = sys.modules['__main__'] 72 | if mod_name == "__main__" or mod_name.endswith(".__main__"): 73 | return 74 | 75 | # If this process was forked, __main__ may already be populated 76 | if getattr(current_main.__spec__, "name", None) == mod_name: 77 | return 78 | 79 | # Otherwise, __main__ may contain some non-main code where we need to 80 | # support unpickling it properly. We rerun it as __mp_main__ and make 81 | # the normal __main__ an alias to that 82 | # old_main_modules.append(current_main) 83 | main_module = types.ModuleType("__mp_main__") 84 | main_content = runpy.run_module(mod_name, 85 | run_name="__mp_main__", 86 | alter_sys=True) # type: ignore 87 | main_module.__dict__.update(main_content) 88 | sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module 89 | 90 | 91 | def _fixup_main_from_path(main_path: str) -> None: 92 | # If this process was forked, __main__ may already be populated 93 | current_main = sys.modules['__main__'] 94 | 95 | # Unfortunately, the main ipython launch script historically had no 96 | # "if __name__ == '__main__'" guard, so we work around that 97 | # by treating it like a __main__.py file 98 | # See https://github.com/ipython/ipython/issues/4698 99 | main_name = os.path.splitext(os.path.basename(main_path))[0] 100 | if main_name == 'ipython': 101 | return 102 | 103 | # Otherwise, if __file__ already has the setting we expect, 104 | # there's nothing more to do 105 | if getattr(current_main, '__file__', None) == main_path: 106 | return 107 | 108 | # If the parent process has sent a path through rather than a module 109 | # name we assume it is an executable script that may contain 110 | # non-main code that needs to be executed 111 | # old_main_modules.append(current_main) 112 | main_module = types.ModuleType("__mp_main__") 113 | main_content = runpy.run_path(main_path, 114 | run_name="__mp_main__") # type: ignore 115 | main_module.__dict__.update(main_content) 116 | sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module 117 | -------------------------------------------------------------------------------- /tests/test_rpc.py: -------------------------------------------------------------------------------- 1 | ''' 2 | RPC (or maybe better labelled as "RTS: remote task scheduling"?) 3 | related API and error checks. 4 | 5 | ''' 6 | import itertools 7 | 8 | import pytest 9 | import tractor 10 | import trio 11 | 12 | 13 | async def sleep_back_actor( 14 | actor_name, 15 | func_name, 16 | func_defined, 17 | exposed_mods, 18 | *, 19 | reg_addr: tuple, 20 | ): 21 | if actor_name: 22 | async with tractor.find_actor( 23 | actor_name, 24 | # NOTE: must be set manually since 25 | # the subactor doesn't have the reg_addr 26 | # fixture code run in it! 27 | # TODO: maybe we should just set this once in the 28 | # _state mod and derive to all children? 29 | registry_addrs=[reg_addr], 30 | ) as portal: 31 | try: 32 | await portal.run(__name__, func_name) 33 | except tractor.RemoteActorError as err: 34 | if not func_defined: 35 | expect = AttributeError 36 | if not exposed_mods: 37 | expect = tractor.ModuleNotExposed 38 | 39 | assert err.boxed_type is expect 40 | raise 41 | else: 42 | await trio.sleep(float('inf')) 43 | 44 | 45 | async def short_sleep(): 46 | await trio.sleep(0) 47 | 48 | 49 | @pytest.mark.parametrize( 50 | 'to_call', [ 51 | ([], 'short_sleep', tractor.RemoteActorError), 52 | ([__name__], 'short_sleep', tractor.RemoteActorError), 53 | ([__name__], 'fake_func', tractor.RemoteActorError), 54 | (['tmp_mod'], 'import doggy', ModuleNotFoundError), 55 | (['tmp_mod'], '4doggy', SyntaxError), 56 | ], 57 | ids=[ 58 | 'no_mods', 59 | 'this_mod', 60 | 'this_mod_bad_func', 61 | 'fail_to_import', 62 | 'fail_on_syntax', 63 | ], 64 | ) 65 | def test_rpc_errors( 66 | reg_addr, 67 | to_call, 68 | testdir, 69 | ): 70 | ''' 71 | Test errors when making various RPC requests to an actor 72 | that either doesn't have the requested module exposed or doesn't define 73 | the named function. 74 | 75 | ''' 76 | exposed_mods, funcname, inside_err = to_call 77 | subactor_exposed_mods = [] 78 | func_defined = globals().get(funcname, False) 79 | subactor_requests_to = 'root' 80 | remote_err = tractor.RemoteActorError 81 | 82 | # remote module that fails at import time 83 | if exposed_mods == ['tmp_mod']: 84 | # create an importable module with a bad import 85 | testdir.syspathinsert() 86 | # module should raise a ModuleNotFoundError at import 87 | testdir.makefile('.py', tmp_mod=funcname) 88 | 89 | # no need to expose module to the subactor 90 | subactor_exposed_mods = exposed_mods 91 | exposed_mods = [] 92 | func_defined = False 93 | # subactor should not try to invoke anything 94 | subactor_requests_to = None 95 | # the module will be attempted to be imported locally but will 96 | # fail in the initial local instance of the actor 97 | remote_err = inside_err 98 | 99 | async def main(): 100 | 101 | # spawn a subactor which calls us back 102 | async with tractor.open_nursery( 103 | registry_addrs=[reg_addr], 104 | enable_modules=exposed_mods.copy(), 105 | 106 | # NOTE: will halt test in REPL if uncommented, so only 107 | # do that if actually debugging subactor but keep it 108 | # disabled for the test. 109 | # debug_mode=True, 110 | ) as n: 111 | 112 | actor = tractor.current_actor() 113 | assert actor.is_arbiter 114 | await n.run_in_actor( 115 | sleep_back_actor, 116 | actor_name=subactor_requests_to, 117 | 118 | name='subactor', 119 | 120 | # function from the local exposed module space 121 | # the subactor will invoke when it RPCs back to this actor 122 | func_name=funcname, 123 | exposed_mods=exposed_mods, 124 | func_defined=True if func_defined else False, 125 | enable_modules=subactor_exposed_mods, 126 | reg_addr=reg_addr, 127 | ) 128 | 129 | def run(): 130 | trio.run(main) 131 | 132 | # handle both parameterized cases 133 | if exposed_mods and func_defined: 134 | run() 135 | else: 136 | # underlying errors aren't propagated upwards (yet) 137 | with pytest.raises( 138 | expected_exception=(remote_err, ExceptionGroup), 139 | ) as err: 140 | run() 141 | 142 | # get raw instance from pytest wrapper 143 | value = err.value 144 | 145 | # might get multiple `trio.Cancelled`s as well inside an inception 146 | if isinstance(value, ExceptionGroup): 147 | value = next(itertools.dropwhile( 148 | lambda exc: not isinstance(exc, tractor.RemoteActorError), 149 | value.exceptions 150 | )) 151 | 152 | if getattr(value, 'type', None): 153 | assert value.boxed_type is inside_err 154 | -------------------------------------------------------------------------------- /tests/test_child_manages_service_nursery.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Test a service style daemon that maintains a nursery for spawning 3 | "remote async tasks" including both spawning other long living 4 | sub-sub-actor daemons. 5 | 6 | ''' 7 | from typing import Optional 8 | import asyncio 9 | from contextlib import ( 10 | asynccontextmanager as acm, 11 | aclosing, 12 | ) 13 | 14 | import pytest 15 | import trio 16 | import tractor 17 | from tractor import RemoteActorError 18 | 19 | 20 | async def aio_streamer( 21 | from_trio: asyncio.Queue, 22 | to_trio: trio.abc.SendChannel, 23 | ) -> trio.abc.ReceiveChannel: 24 | 25 | # required first msg to sync caller 26 | to_trio.send_nowait(None) 27 | 28 | from itertools import cycle 29 | for i in cycle(range(10)): 30 | to_trio.send_nowait(i) 31 | await asyncio.sleep(0.01) 32 | 33 | 34 | async def trio_streamer(): 35 | from itertools import cycle 36 | for i in cycle(range(10)): 37 | yield i 38 | await trio.sleep(0.01) 39 | 40 | 41 | async def trio_sleep_and_err(delay: float = 0.5): 42 | await trio.sleep(delay) 43 | # name error 44 | doggy() # noqa 45 | 46 | 47 | _cached_stream: Optional[ 48 | trio.abc.ReceiveChannel 49 | ] = None 50 | 51 | 52 | @acm 53 | async def wrapper_mngr( 54 | ): 55 | from tractor.trionics import broadcast_receiver 56 | global _cached_stream 57 | in_aio = tractor.current_actor().is_infected_aio() 58 | 59 | if in_aio: 60 | if _cached_stream: 61 | 62 | from_aio = _cached_stream 63 | 64 | # if we already have a cached feed deliver a rx side clone 65 | # to consumer 66 | async with broadcast_receiver(from_aio, 6) as from_aio: 67 | yield from_aio 68 | return 69 | else: 70 | async with tractor.to_asyncio.open_channel_from( 71 | aio_streamer, 72 | ) as (first, from_aio): 73 | assert not first 74 | 75 | # cache it so next task uses broadcast receiver 76 | _cached_stream = from_aio 77 | 78 | yield from_aio 79 | else: 80 | async with aclosing(trio_streamer()) as stream: 81 | # cache it so next task uses broadcast receiver 82 | _cached_stream = stream 83 | yield stream 84 | 85 | 86 | _nursery: trio.Nursery = None 87 | 88 | 89 | @tractor.context 90 | async def trio_main( 91 | ctx: tractor.Context, 92 | ): 93 | # sync 94 | await ctx.started() 95 | 96 | # stash a "service nursery" as "actor local" (aka a Python global) 97 | global _nursery 98 | tn = _nursery 99 | assert tn 100 | 101 | async def consume_stream(): 102 | async with wrapper_mngr() as stream: 103 | async for msg in stream: 104 | print(msg) 105 | 106 | # run 2 tasks to ensure broadcaster chan use 107 | tn.start_soon(consume_stream) 108 | tn.start_soon(consume_stream) 109 | 110 | tn.start_soon(trio_sleep_and_err) 111 | 112 | await trio.sleep_forever() 113 | 114 | 115 | @tractor.context 116 | async def open_actor_local_nursery( 117 | ctx: tractor.Context, 118 | ): 119 | global _nursery 120 | async with ( 121 | tractor.trionics.collapse_eg(), 122 | trio.open_nursery() as tn 123 | ): 124 | _nursery = tn 125 | await ctx.started() 126 | await trio.sleep(10) 127 | # await trio.sleep(1) 128 | 129 | # XXX: this causes the hang since 130 | # the caller does not unblock from its own 131 | # ``trio.sleep_forever()``. 132 | 133 | # TODO: we need to test a simple ctx task starting remote tasks 134 | # that error and then blocking on a ``Nursery.start()`` which 135 | # never yields back.. aka a scenario where the 136 | # ``tractor.context`` task IS NOT in the service n's cancel 137 | # scope. 138 | tn.cancel_scope.cancel() 139 | 140 | 141 | @pytest.mark.parametrize( 142 | 'asyncio_mode', 143 | [True, False], 144 | ids='asyncio_mode={}'.format, 145 | ) 146 | def test_actor_managed_trio_nursery_task_error_cancels_aio( 147 | asyncio_mode: bool, 148 | reg_addr: tuple, 149 | ): 150 | ''' 151 | Verify that a ``trio`` nursery created managed in a child actor 152 | correctly relays errors to the parent actor when one of its spawned 153 | tasks errors even when running in infected asyncio mode and using 154 | broadcast receivers for multi-task-per-actor subscription. 155 | 156 | ''' 157 | async def main(): 158 | 159 | # cancel the nursery shortly after boot 160 | async with tractor.open_nursery() as n: 161 | p = await n.start_actor( 162 | 'nursery_mngr', 163 | infect_asyncio=asyncio_mode, # TODO, is this enabling debug mode? 164 | enable_modules=[__name__], 165 | ) 166 | async with ( 167 | p.open_context(open_actor_local_nursery) as (ctx, first), 168 | p.open_context(trio_main) as (ctx, first), 169 | ): 170 | await trio.sleep_forever() 171 | 172 | with pytest.raises(RemoteActorError) as excinfo: 173 | trio.run(main) 174 | 175 | # verify boxed error 176 | err = excinfo.value 177 | assert err.boxed_type is NameError 178 | -------------------------------------------------------------------------------- /tractor/_entry.py: -------------------------------------------------------------------------------- 1 | # tractor: structured concurrent "actors". 2 | # Copyright 2018-eternity Tyler Goodlet. 3 | 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as published by 6 | # the Free Software Foundation, either version 3 of the License, or 7 | # (at your option) any later version. 8 | 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | """ 18 | Sub-process entry points. 19 | 20 | """ 21 | from __future__ import annotations 22 | from functools import partial 23 | import multiprocessing as mp 24 | # import os 25 | from typing import ( 26 | Any, 27 | TYPE_CHECKING, 28 | ) 29 | 30 | import trio # type: ignore 31 | 32 | from .log import ( 33 | get_console_log, 34 | get_logger, 35 | ) 36 | from . import _state 37 | from .devx import ( 38 | _frame_stack, 39 | pformat, 40 | ) 41 | # from .msg import pretty_struct 42 | from .to_asyncio import run_as_asyncio_guest 43 | from ._addr import UnwrappedAddress 44 | from ._runtime import ( 45 | async_main, 46 | Actor, 47 | ) 48 | 49 | if TYPE_CHECKING: 50 | from ._spawn import SpawnMethodKey 51 | 52 | 53 | log = get_logger(__name__) 54 | 55 | 56 | def _mp_main( 57 | 58 | actor: Actor, 59 | accept_addrs: list[UnwrappedAddress], 60 | forkserver_info: tuple[Any, Any, Any, Any, Any], 61 | start_method: SpawnMethodKey, 62 | parent_addr: UnwrappedAddress | None = None, 63 | infect_asyncio: bool = False, 64 | 65 | ) -> None: 66 | ''' 67 | The routine called *after fork* which invokes a fresh `trio.run()` 68 | 69 | ''' 70 | actor._forkserver_info = forkserver_info 71 | from ._spawn import try_set_start_method 72 | spawn_ctx: mp.context.BaseContext = try_set_start_method(start_method) 73 | assert spawn_ctx 74 | 75 | if actor.loglevel is not None: 76 | log.info( 77 | f'Setting loglevel for {actor.uid} to {actor.loglevel}' 78 | ) 79 | get_console_log(actor.loglevel) 80 | 81 | # TODO: use scops headers like for `trio` below! 82 | # (well after we libify it maybe..) 83 | log.info( 84 | f'Started new {spawn_ctx.current_process()} for {actor.uid}' 85 | # f"parent_addr is {parent_addr}" 86 | ) 87 | _state._current_actor: Actor = actor 88 | trio_main = partial( 89 | async_main, 90 | actor=actor, 91 | accept_addrs=accept_addrs, 92 | parent_addr=parent_addr 93 | ) 94 | try: 95 | if infect_asyncio: 96 | actor._infected_aio = True 97 | run_as_asyncio_guest(trio_main) 98 | else: 99 | trio.run(trio_main) 100 | except KeyboardInterrupt: 101 | pass # handle it the same way trio does? 102 | 103 | finally: 104 | log.info( 105 | f'`mp`-subactor {actor.uid} exited' 106 | ) 107 | 108 | 109 | def _trio_main( 110 | actor: Actor, 111 | *, 112 | parent_addr: UnwrappedAddress|None = None, 113 | infect_asyncio: bool = False, 114 | 115 | ) -> None: 116 | ''' 117 | Entry point for a `trio_run_in_process` subactor. 118 | 119 | ''' 120 | _frame_stack.hide_runtime_frames() 121 | 122 | _state._current_actor = actor 123 | trio_main = partial( 124 | async_main, 125 | actor, 126 | parent_addr=parent_addr 127 | ) 128 | 129 | if actor.loglevel is not None: 130 | get_console_log(actor.loglevel) 131 | log.info( 132 | f'Starting `trio` subactor from parent @ ' 133 | f'{parent_addr}\n' 134 | + 135 | pformat.nest_from_op( 136 | input_op='>(', # see syntax ideas above 137 | text=f'{actor}', 138 | ) 139 | ) 140 | logmeth = log.info 141 | exit_status: str = ( 142 | 'Subactor exited\n' 143 | + 144 | pformat.nest_from_op( 145 | input_op=')>', # like a "closed-to-play"-icon from super perspective 146 | text=f'{actor}', 147 | nest_indent=1, 148 | ) 149 | ) 150 | try: 151 | if infect_asyncio: 152 | actor._infected_aio = True 153 | run_as_asyncio_guest(trio_main) 154 | else: 155 | trio.run(trio_main) 156 | 157 | except KeyboardInterrupt: 158 | logmeth = log.cancel 159 | exit_status: str = ( 160 | 'Actor received KBI (aka an OS-cancel)\n' 161 | + 162 | pformat.nest_from_op( 163 | input_op='c)>', # closed due to cancel (see above) 164 | text=f'{actor}', 165 | ) 166 | ) 167 | except BaseException as err: 168 | logmeth = log.error 169 | exit_status: str = ( 170 | 'Main actor task exited due to crash?\n' 171 | + 172 | pformat.nest_from_op( 173 | input_op='x)>', # closed by error 174 | text=f'{actor}', 175 | ) 176 | ) 177 | # NOTE since we raise a tb will already be shown on the 178 | # console, thus we do NOT use `.exception()` above. 179 | raise err 180 | 181 | finally: 182 | logmeth(exit_status) 183 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | # any time someone pushes a new branch to origin 5 | push: 6 | 7 | # Allows you to run this workflow manually from the Actions tab 8 | workflow_dispatch: 9 | 10 | jobs: 11 | # ------ sdist ------ 12 | # test that we can generate a software distribution and install it 13 | # thus avoid missing file issues after packaging. 14 | # 15 | # -[x] produce sdist with uv 16 | # ------ - ------ 17 | sdist-linux: 18 | name: 'sdist' 19 | runs-on: ubuntu-latest 20 | 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v4 24 | 25 | - name: Install latest uv 26 | uses: astral-sh/setup-uv@v6 27 | 28 | - name: Build sdist as tar.gz 29 | run: uv build --sdist --python=3.13 30 | 31 | - name: Install sdist from .tar.gz 32 | run: python -m pip install dist/*.tar.gz 33 | 34 | # ------ type-check ------ 35 | # mypy: 36 | # name: 'MyPy' 37 | # runs-on: ubuntu-latest 38 | 39 | # steps: 40 | # - name: Checkout 41 | # uses: actions/checkout@v4 42 | 43 | # - name: Install latest uv 44 | # uses: astral-sh/setup-uv@v6 45 | 46 | # # faster due to server caching? 47 | # # https://docs.astral.sh/uv/guides/integration/github/#setting-up-python 48 | # - name: "Set up Python" 49 | # uses: actions/setup-python@v6 50 | # with: 51 | # python-version-file: "pyproject.toml" 52 | 53 | # # w uv 54 | # # - name: Set up Python 55 | # # run: uv python install 56 | 57 | # - name: Setup uv venv 58 | # run: uv venv .venv --python=3.13 59 | 60 | # - name: Install 61 | # run: uv sync --dev 62 | 63 | # # TODO, ty cmd over repo 64 | # # - name: type check with ty 65 | # # run: ty ./tractor/ 66 | 67 | # # - uses: actions/cache@v3 68 | # # name: Cache uv virtenv as default .venv 69 | # # with: 70 | # # path: ./.venv 71 | # # key: venv-${{ hashFiles('uv.lock') }} 72 | 73 | # - name: Run MyPy check 74 | # run: mypy tractor/ --ignore-missing-imports --show-traceback 75 | 76 | 77 | testing-linux: 78 | name: '${{ matrix.os }} Python ${{ matrix.python }} - ${{ matrix.spawn_backend }}' 79 | timeout-minutes: 10 80 | runs-on: ${{ matrix.os }} 81 | 82 | strategy: 83 | fail-fast: false 84 | matrix: 85 | os: [ubuntu-latest] 86 | python-version: ['3.13'] 87 | spawn_backend: [ 88 | 'trio', 89 | # 'mp_spawn', 90 | # 'mp_forkserver', 91 | ] 92 | 93 | steps: 94 | 95 | - uses: actions/checkout@v4 96 | 97 | - name: 'Install uv + py-${{ matrix.python-version }}' 98 | uses: astral-sh/setup-uv@v6 99 | with: 100 | python-version: ${{ matrix.python-version }} 101 | 102 | # GH way.. faster? 103 | # - name: setup-python@v6 104 | # uses: actions/setup-python@v6 105 | # with: 106 | # python-version: '${{ matrix.python-version }}' 107 | 108 | # consider caching for speedups? 109 | # https://docs.astral.sh/uv/guides/integration/github/#caching 110 | 111 | - name: Install the project w uv 112 | run: uv sync --all-extras --dev 113 | 114 | # - name: Install dependencies 115 | # run: pip install -U . -r requirements-test.txt -r requirements-docs.txt --upgrade-strategy eager 116 | 117 | - name: List deps tree 118 | run: uv tree 119 | 120 | - name: Run tests 121 | run: uv run pytest tests/ --spawn-backend=${{ matrix.spawn_backend }} -rsx 122 | 123 | # XXX legacy NOTE XXX 124 | # 125 | # We skip 3.10 on windows for now due to not having any collabs to 126 | # debug the CI failures. Anyone wanting to hack and solve them is very 127 | # welcome, but our primary user base is not using that OS. 128 | 129 | # TODO: use job filtering to accomplish instead of repeated 130 | # boilerplate as is above XD: 131 | # - https://docs.github.com/en/actions/learn-github-actions/managing-complex-workflows 132 | # - https://docs.github.com/en/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix 133 | # - https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idif 134 | # testing-windows: 135 | # name: '${{ matrix.os }} Python ${{ matrix.python }} - ${{ matrix.spawn_backend }}' 136 | # timeout-minutes: 12 137 | # runs-on: ${{ matrix.os }} 138 | 139 | # strategy: 140 | # fail-fast: false 141 | # matrix: 142 | # os: [windows-latest] 143 | # python: ['3.10'] 144 | # spawn_backend: ['trio', 'mp'] 145 | 146 | # steps: 147 | 148 | # - name: Checkout 149 | # uses: actions/checkout@v2 150 | 151 | # - name: Setup python 152 | # uses: actions/setup-python@v2 153 | # with: 154 | # python-version: '${{ matrix.python }}' 155 | 156 | # - name: Install dependencies 157 | # run: pip install -U . -r requirements-test.txt -r requirements-docs.txt --upgrade-strategy eager 158 | 159 | # # TODO: pretty sure this solves debugger deps-issues on windows, but it needs to 160 | # # be verified by someone with a native setup. 161 | # # - name: Force pyreadline3 162 | # # run: pip uninstall pyreadline; pip install -U pyreadline3 163 | 164 | # - name: List dependencies 165 | # run: pip list 166 | 167 | # - name: Run tests 168 | # run: pytest tests/ --spawn-backend=${{ matrix.spawn_backend }} -rsx 169 | --------------------------------------------------------------------------------