├── lox ├── py.typed ├── exceptions.py ├── debug.py ├── worker │ ├── __init__.py │ ├── process.py │ └── thread.py ├── queue │ ├── __init__.py │ ├── funnel.py │ └── announcement.py ├── lock │ ├── __init__.py │ ├── qlock.py │ ├── index_semaphore.py │ ├── rw_lock.py │ └── light_switch.py ├── __init__.py └── helper.py ├── docs ├── _static │ └── .gitkeep ├── authors.rst ├── history.rst ├── contributing.rst ├── modules.rst ├── Makefile ├── make.bat ├── installation.rst ├── FAQ.rst ├── index.rst ├── conf.py └── examples.rst ├── requirements.txt ├── assets ├── lox.ai ├── lox.eps ├── lox.pdf ├── lox.png ├── logo.ico ├── logo.png ├── lox_200w.png ├── lox_padded.jpg └── lox_padded.png ├── pytest.ini ├── AUTHORS.rst ├── requirements_dev.txt ├── .editorconfig ├── .github ├── ISSUE_TEMPLATE.md ├── FUNDING.yml └── workflows │ ├── test.yml │ └── deploy.yaml ├── MANIFEST.in ├── tox.ini ├── .coveragerc ├── .readthedocs.yaml ├── tests ├── test_helper.py ├── conftest.py ├── queue │ ├── test_Funnel.py │ └── test_Announcement.py ├── lock │ ├── test_QLock.py │ ├── test_IndexSemaphore.py │ ├── test_LightSwitch.py │ └── test_RWLock.py └── worker │ ├── test_process.py │ └── test_thread.py ├── setup.cfg ├── setup.py ├── .gitignore ├── Makefile ├── .pre-commit-config.yaml ├── CONTRIBUTING.rst ├── HISTORY.rst ├── README.rst └── LICENSE /lox/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/_static/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pathos 2 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /lox/exceptions.py: -------------------------------------------------------------------------------- 1 | class Timeout(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /assets/lox.ai: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BrianPugh/lox/HEAD/assets/lox.ai -------------------------------------------------------------------------------- /assets/lox.eps: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BrianPugh/lox/HEAD/assets/lox.eps -------------------------------------------------------------------------------- /assets/lox.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BrianPugh/lox/HEAD/assets/lox.pdf -------------------------------------------------------------------------------- /assets/lox.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BrianPugh/lox/HEAD/assets/lox.png -------------------------------------------------------------------------------- /assets/logo.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BrianPugh/lox/HEAD/assets/logo.ico -------------------------------------------------------------------------------- /assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BrianPugh/lox/HEAD/assets/logo.png -------------------------------------------------------------------------------- /assets/lox_200w.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BrianPugh/lox/HEAD/assets/lox_200w.png -------------------------------------------------------------------------------- /assets/lox_padded.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BrianPugh/lox/HEAD/assets/lox_padded.jpg -------------------------------------------------------------------------------- /assets/lox_padded.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/BrianPugh/lox/HEAD/assets/lox_padded.png -------------------------------------------------------------------------------- /docs/modules.rst: -------------------------------------------------------------------------------- 1 | lox 2 | === 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | lox 8 | -------------------------------------------------------------------------------- /lox/debug.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | _truelike = {"true", "t", "yes", "y", "1"} 4 | LOX_DEBUG = os.getenv("LOX_DEBUG", "False").lower() in _truelike 5 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | addopts = --strict-markers 3 | markers = 4 | visual 5 | requires_pathos: tests that require pathos (multiprocessing) dependency 6 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * Brian Pugh 9 | 10 | Contributors 11 | ------------ 12 | 13 | None yet. Why not be the first? 14 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | autoflake 2 | autopep8 3 | bumpversion 4 | coverage 5 | flake8 6 | pytest 7 | pytest-benchmark 8 | pytest-mock 9 | pytest-runner 10 | Sphinx 11 | sphinx_rtd_theme 12 | tox 13 | tqdm 14 | twine 15 | watchdog 16 | wheel 17 | -------------------------------------------------------------------------------- /lox/worker/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Add concurrency to methods and functions in a single line of code.""" 4 | 5 | __all__ = [ 6 | "process", 7 | "thread", 8 | ] 9 | 10 | import lox.worker.process 11 | import lox.worker.thread 12 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.bat] 14 | indent_style = tab 15 | end_of_line = crlf 16 | 17 | [LICENSE] 18 | insert_final_newline = false 19 | 20 | [Makefile] 21 | indent_style = tab 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | * lox version: 2 | * Python version: 3 | * Operating System: 4 | 5 | ### Description 6 | 7 | Describe what you were trying to get done. 8 | Tell us what happened, what went wrong, and what you expected to happen. 9 | 10 | ### What I Did 11 | 12 | ``` 13 | Paste the command(s) you ran and the output. 14 | If there was a crash, please include the traceback here. 15 | ``` 16 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft lox 2 | include AUTHORS.rst 3 | include CONTRIBUTING.rst 4 | include HISTORY.rst 5 | include LICENSE 6 | include README.rst 7 | 8 | include requirements.txt 9 | include requirements_dev.txt 10 | 11 | recursive-include tests * 12 | recursive-exclude * __pycache__ 13 | recursive-exclude * *.py[co] 14 | 15 | recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif 16 | -------------------------------------------------------------------------------- /lox/queue/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Objects to help shuttle data beteween tasks. 5 | """ 6 | __all__ = [ 7 | "Announcement", 8 | "SubscribeFinalizedError", 9 | "Funnel", 10 | "FunnelPutError", 11 | "FunnelPutTopError", 12 | ] 13 | from .announcement import Announcement, SubscribeFinalizedError 14 | from .funnel import Funnel, FunnelPutError, FunnelPutTopError 15 | -------------------------------------------------------------------------------- /lox/lock/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Concurrency control objects to help parallelized tasks communicate and share resources. 5 | """ 6 | 7 | __all__ = [ 8 | "IndexSemaphore", 9 | "LightSwitch", 10 | "QLock", 11 | "RWLock", 12 | ] 13 | 14 | from .index_semaphore import IndexSemaphore 15 | from .light_switch import LightSwitch 16 | from .qlock import QLock 17 | from .rw_lock import RWLock 18 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py36, flake8 3 | 4 | [travis] 5 | python = 6 | 3.6: py36 7 | 8 | [testenv:flake8] 9 | basepython = python 10 | deps = flake8 11 | commands = flake8 lox 12 | 13 | [testenv] 14 | setenv = 15 | PYTHONPATH = {toxinidir} 16 | deps = 17 | -r{toxinidir}/requirements_dev.txt 18 | -r{toxinidir}/requirements.txt 19 | commands = 20 | pip install -U pip 21 | py.test --basetemp={envtmpdir} 22 | 23 | [pytest] 24 | addopts = --benchmark-columns="median,ops,rounds" 25 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | 2 | [run] 3 | branch = True 4 | omit = */tests/* 5 | 6 | [report] 7 | exclude_lines = 8 | # Have to re-enable the standard pragma 9 | pragma: no cover 10 | 11 | # Don't complain about missing debug-only code: 12 | def __repr__ 13 | if self\.debug: 14 | if debug: 15 | if DEBUG: 16 | 17 | # Don't complain if tests don't hit defensive assertion code: 18 | raise AssertionError 19 | raise NotImplementedError 20 | 21 | # Don't complain if non-runnable code isn't run: 22 | if 0: 23 | if False: 24 | if __name__ == .__main__.: 25 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yaml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Set the OS, Python version and other tools you might need 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/conf.py 17 | fail_on_warning: false 18 | 19 | # Optionally declare the Python requirements required to build your docs 20 | python: 21 | install: 22 | - requirements: requirements_dev.txt 23 | - method: pip 24 | path: . 25 | -------------------------------------------------------------------------------- /tests/test_helper.py: -------------------------------------------------------------------------------- 1 | import lox 2 | from lox.helper import term_colors 3 | 4 | 5 | def test_ctx(capsys): 6 | with capsys.disabled(): 7 | print("") 8 | with term_colors("red"): 9 | print("test red") 10 | with term_colors("green"): 11 | print("test green") 12 | with term_colors("blue"): 13 | print("test blue") 14 | with term_colors("orange"): 15 | print("test orange") 16 | with term_colors("underline"): 17 | print("test underline") 18 | with term_colors("bold"): 19 | print("test bold") 20 | with term_colors("header"): 21 | print("test header") 22 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python3 -msphinx 7 | SPHINXPROJ = lox 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | def pytest_addoption(parser): 5 | try: 6 | parser.addoption( 7 | "--visual", 8 | action="store_true", 9 | default=False, 10 | help="run interactive visual tests", 11 | ) 12 | except ValueError: 13 | pass 14 | 15 | 16 | def pytest_collection_modifyitems(config, items): 17 | if config.getoption("--visual"): 18 | # --visual given in cli: do not skip visual tests 19 | return 20 | skip_visual = pytest.mark.skip(reason="need --visual option to run") 21 | for item in items: 22 | if "visual" in item.keywords: 23 | item.add_marker(skip_visual) 24 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 1.0.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | search = version = "{current_version}" 8 | replace = version = "{new_version}" 9 | 10 | [bumpversion:file:lox/__init__.py] 11 | search = __version__ = "{current_version}" 12 | replace = __version__ = "{new_version}" 13 | 14 | [bdist_wheel] 15 | universal = 1 16 | 17 | [flake8] 18 | exclude = docs 19 | max-line-length = 88 20 | extend-ignore = E203,E402,F401,D100,D101,D102,D103,D104,D105,D106,D107,D200,D202,D205,D400 21 | docstring-convention = numpy 22 | per-file-ignores = **/tests/:D100,D101,D102,D103,D104,D105,D106,D107 23 | 24 | [aliases] 25 | test = pytest 26 | 27 | [tool:pytest] 28 | collect_ignore = ['setup.py'] 29 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [BrianPugh] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 13 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 14 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=python -msphinx 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=lox 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The Sphinx module was not found. Make sure you have Sphinx installed, 20 | echo.then set the SPHINXBUILD environment variable to point to the full 21 | echo.path of the 'sphinx-build' executable. Alternatively you may add the 22 | echo.Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /lox/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | Easy multithreading for every project. 5 | """ 6 | 7 | __author__ = """Brian Pugh""" 8 | __email__ = "bnp117@gmail.com" 9 | __version__ = "1.0.0" 10 | 11 | __all__ = [ 12 | "Announcement", 13 | "Funnel", 14 | "FunnelPutError", 15 | "FunnelPutTopError", 16 | "IndexSemaphore", 17 | "LOX_DEBUG", 18 | "LightSwitch", 19 | "QLock", 20 | "RWLock", 21 | "SubscribeFinalizedError", 22 | "Timeout", 23 | "process", 24 | "thread", 25 | ] 26 | 27 | import lox.worker 28 | from lox.worker.process import process 29 | from lox.worker.thread import thread 30 | 31 | from .debug import LOX_DEBUG 32 | from .exceptions import Timeout 33 | from .lock import IndexSemaphore, LightSwitch, QLock, RWLock 34 | from .queue import ( 35 | Announcement, 36 | Funnel, 37 | FunnelPutError, 38 | FunnelPutTopError, 39 | SubscribeFinalizedError, 40 | ) 41 | -------------------------------------------------------------------------------- /tests/queue/test_Funnel.py: -------------------------------------------------------------------------------- 1 | import queue 2 | from threading import Lock, Thread 3 | from time import sleep 4 | 5 | import lox 6 | from lox import Announcement 7 | 8 | 9 | def test_1(): 10 | """Test most basic usecase with 1 subscribers.""" 11 | funnel = lox.Funnel() 12 | sub_1 = funnel.subscribe() 13 | 14 | sub_1.put("foo", "job_id") 15 | res = funnel.get() 16 | assert len(res) == 2 17 | assert res[0] == "job_id" 18 | assert res[1] == "foo" 19 | assert funnel.d == {} 20 | 21 | 22 | def test_2(): 23 | """Test most basic usecase with 2 subscribers.""" 24 | 25 | funnel = lox.Funnel() 26 | sub_1 = funnel.subscribe() 27 | sub_2 = funnel.subscribe() 28 | 29 | sub_1.put("foo", "job_id") 30 | try: 31 | res = funnel.get(timeout=0.01) 32 | assert 0 33 | except queue.Empty: 34 | assert 1 35 | 36 | sub_2.put("bar", "job_id") 37 | 38 | res = funnel.get() 39 | assert len(res) == 3 40 | assert res[0] == "job_id" 41 | assert res[1] == "foo" 42 | assert res[2] == "bar" 43 | assert funnel.d == {} 44 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] 17 | 18 | steps: 19 | - uses: actions/checkout@v4 20 | 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v4 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip 29 | pip install -e . 30 | pip install -r requirements_dev.txt 31 | pip install pre-commit 32 | 33 | - name: Run pre-commit 34 | run: | 35 | pre-commit run --all-files 36 | 37 | - name: Test without pathos 38 | run: | 39 | python -m pytest -m "not requires_pathos" 40 | 41 | - name: Install pathos 42 | run: | 43 | pip install -e ".[multiprocessing]" 44 | 45 | - name: Test with pathos 46 | run: | 47 | python -m pytest -m "requires_pathos" 48 | -------------------------------------------------------------------------------- /tests/lock/test_QLock.py: -------------------------------------------------------------------------------- 1 | from threading import Lock, Thread 2 | from time import sleep, time 3 | 4 | from lox import QLock 5 | 6 | SLEEP_TIME = 0.1 7 | 8 | 9 | def test_1(): 10 | res = "" 11 | sol = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 12 | qlock = QLock() 13 | 14 | def worker(x): 15 | nonlocal res 16 | with qlock: 17 | res += x 18 | 19 | threads = [] 20 | for name in sol: 21 | t = Thread(target=worker, args=(name,)) 22 | t.start() 23 | threads.append(t) 24 | sleep(SLEEP_TIME) # probably enough time for .acquire() to run 25 | 26 | # Wait for all threads to complete 27 | for t in threads: 28 | t.join() 29 | 30 | for r, s in zip(res, sol): 31 | assert r == s 32 | 33 | 34 | def test_timeout(): 35 | qlock = QLock() 36 | assert qlock.acquire() 37 | assert qlock.acquire(timeout=SLEEP_TIME) is False 38 | 39 | 40 | def test_perf_qlock(benchmark): 41 | lock = QLock() 42 | 43 | @benchmark 44 | def acquire_release(): 45 | lock.acquire() 46 | lock.release() 47 | 48 | 49 | def test_perf_lock(benchmark): 50 | lock = Lock() 51 | 52 | @benchmark 53 | def acquire_release(): 54 | lock.acquire() 55 | lock.release() 56 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yaml: -------------------------------------------------------------------------------- 1 | name: Build package and push to PyPi 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | tags: 7 | - "v*.*.*" 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Check out repository 14 | uses: actions/checkout@v4 15 | with: 16 | fetch-depth: 0 # Includes getting tags 17 | 18 | - name: Set up python 3.11 19 | uses: actions/setup-python@v5 20 | with: 21 | python-version: "3.11" 22 | 23 | - name: Build Package 24 | run: | 25 | python -m pip install --upgrade pip 26 | make dev 27 | make package 28 | 29 | - name: Run tests 30 | run: | 31 | python -m pytest 32 | 33 | - name: Publish to PyPI 34 | if: github.event_name != 'workflow_dispatch' 35 | env: 36 | TWINE_USERNAME: __token__ 37 | TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} 38 | run: | 39 | python -m twine upload dist/* 40 | 41 | - uses: actions/upload-artifact@v4 42 | if: always() 43 | with: 44 | name: dist 45 | path: dist/ 46 | -------------------------------------------------------------------------------- /lox/helper.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: helper 3 | :synopsis: Private general lox helper functions/classes 4 | """ 5 | 6 | __all__ = [ 7 | "term_colors", 8 | ] 9 | 10 | 11 | class term_colors: 12 | """Escape sequences to colorize text in the terminal""" 13 | 14 | HEADER = "\033[95m" 15 | BLUE = "\033[94m" 16 | GREEN = "\033[92m" 17 | ORANGE = "\033[93m" 18 | RED = "\033[91m" 19 | ENDC = "\033[0m" 20 | BOLD = "\033[1m" 21 | UNDERLINE = "\033[4m" 22 | 23 | def __init__(self, color: str): 24 | self.color = color.lower() 25 | 26 | def __enter__( 27 | self, 28 | ): 29 | if self.color == "red": 30 | print(self.RED, end="") 31 | elif self.color == "green": 32 | print(self.GREEN, end="") 33 | elif self.color == "blue": 34 | print(self.BLUE, end="") 35 | elif self.color == "orange" or self.color == "yellow": 36 | print(self.ORANGE, end="") 37 | elif self.color == "bold": 38 | print(self.BOLD, end="") 39 | elif self.color == "underline": 40 | print(self.UNDERLINE, end="") 41 | elif self.color == "header": 42 | print(self.HEADER, end="") 43 | else: 44 | raise ValueError("Invalid color string") 45 | 46 | def __exit__(self, type, value, traceback): 47 | print(self.ENDC, end="", flush=True) 48 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Installation 5 | ============ 6 | 7 | 8 | Stable release 9 | -------------- 10 | 11 | To install lox, run this command in your terminal: 12 | 13 | .. code-block:: console 14 | 15 | $ pip install lox 16 | 17 | For multiprocessing support, install with the optional extra: 18 | 19 | .. code-block:: console 20 | 21 | $ pip install 'lox[multiprocessing]' 22 | 23 | This is the preferred method to install lox, as it will always install the most recent stable release. 24 | 25 | If you don't have `pip`_ installed, this `Python installation guide`_ can guide 26 | you through the process. 27 | 28 | .. _pip: https://pip.pypa.io 29 | .. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ 30 | 31 | 32 | From sources 33 | ------------ 34 | 35 | The sources for lox can be downloaded from the `Github repo`_. 36 | 37 | You can either clone the public repository: 38 | 39 | .. code-block:: console 40 | 41 | $ git clone git://github.com/BrianPugh/lox 42 | 43 | Or download the `tarball`_: 44 | 45 | .. code-block:: console 46 | 47 | $ curl -OL https://github.com/BrianPugh/lox/tarball/main 48 | 49 | Once you have a copy of the source, you can install it with: 50 | 51 | .. code-block:: console 52 | 53 | $ python setup.py install 54 | 55 | 56 | .. _Github repo: https://github.com/BrianPugh/lox 57 | .. _tarball: https://github.com/BrianPugh/lox/tarball/main 58 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from setuptools import find_packages, setup 5 | from setuptools.command.install import install 6 | 7 | version = "1.0.0" 8 | 9 | 10 | with open("README.rst") as readme_file: 11 | readme = readme_file.read() 12 | readme = readme.replace( 13 | "assets/lox_200w.png", 14 | "https://raw.githubusercontent.com/BrianPugh/lox/main/assets/lox_200w.png", 15 | ) 16 | 17 | with open("HISTORY.rst") as history_file: 18 | history = history_file.read() 19 | 20 | setup( 21 | author="Brian Pugh", 22 | author_email="bnp117@gmail.com", 23 | classifiers=[ 24 | "Development Status :: 2 - Pre-Alpha", 25 | "Intended Audience :: Developers", 26 | "License :: OSI Approved :: MIT License", 27 | "Natural Language :: English", 28 | "Programming Language :: Python :: 3", 29 | "Programming Language :: Python :: 3.8", 30 | "Programming Language :: Python :: 3.9", 31 | "Programming Language :: Python :: 3.10", 32 | "Programming Language :: Python :: 3.11", 33 | "Programming Language :: Python :: 3.12", 34 | ], 35 | description="Threading and Multiprocessing for every project.", 36 | install_requires=[], 37 | extras_require={ 38 | "multiprocessing": ["pathos"], 39 | }, 40 | license="MIT license", 41 | long_description=readme + "\n\n" + history, 42 | long_description_content_type="text/x-rst", 43 | include_package_data=True, 44 | keywords="lox", 45 | name="lox", 46 | packages=find_packages(include=["lox"]), 47 | test_suite="tests", 48 | url="https://github.com/BrianPugh/lox", 49 | version=version, 50 | zip_safe=False, 51 | ) 52 | -------------------------------------------------------------------------------- /docs/FAQ.rst: -------------------------------------------------------------------------------- 1 | === 2 | FAQ 3 | === 4 | 5 | **Q: Whats the difference between multithreading and multiprocessing?** 6 | 7 | **A:** Multithreading and Multiprocessing are two different methods to provide concurrency (parallelism) to your code. 8 | 9 | Threading has low overhead for sharing resources between threads. Threads share the same heap, meaning global variables are easily accessible from each thread. However, at any given moment, only a single line of python is being executed, meaning if your code is CPU-bound, using threading will have the same performance (actually worse due to overhead) as not using threading. 10 | 11 | Multiprocessing is basically several copies of your python code running at once, communicating over pipes. Each worker has it's own python interpretter, it's own stack, it's own heap, it's own everything. Any data transferred between your main program and the workers must first be serialized (using **dill**, a library very similar to **pickle**) passed over a pipe, then deserialized. 12 | 13 | In short, if your project is I/O bound (web requests, reading/writing files, waiting for responses from compiled code/binaries, etc), threading is probably the better choice. However, if your code is computation bound, and if the libraries you are using aren't using compiled backends that are already maxing out your CPU, multiprocessing might be the better option. 14 | 15 | **Q: Why not just use the built-in** ``await`` **?** 16 | 17 | **A:** Trying to shove ``await`` into a project typically requires great care both in the code written and the packages used. Ontop of this, using await may require a substantial refactor of the layout of the code. The goal of **lox** is to require the smallest, least risky changes in your codebase. 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | 58 | # Flask stuff: 59 | instance/ 60 | .webassets-cache 61 | 62 | # Scrapy stuff: 63 | .scrapy 64 | 65 | # Sphinx documentation 66 | docs/_build/ 67 | docs/lox.* 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # dotenv 85 | .env 86 | 87 | # virtualenv 88 | .venv 89 | venv/ 90 | ENV/ 91 | 92 | # Spyder project settings 93 | .spyderproject 94 | .spyproject 95 | 96 | # Rope project settings 97 | .ropeproject 98 | 99 | # mkdocs documentation 100 | /site 101 | 102 | # mypy 103 | .mypy_cache/ 104 | 105 | # MacOS 106 | *.DS_Store 107 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | lox: Concurrency Made Easy 2 | ========================== 3 | 4 | Many programs are `embaressingly parallel `_ and can gain large performance boost by simply parallelizing portions of the code. However, multithreading a program is still typically seen as a difficult task and placed at the bottom of the TODO list. **lox** aims to make it as simple and intuitive as possible to parallelize functions and methods in python. This includes both invoking functions, as well as providing easy-to-use guards for 5 | shared resources. 6 | 7 | **lox** provides a simple, shallow learning-curve toolset to implement multithreading or multiprocessing that will work in most projects. **lox** is not meant to be the bleeding edge of performance; for absolute maximum performance, you code will have to be more fine tuned and may benefit from python3's builtin **asyncio**, **greenlet**, or other async libraries. **lox**'s primary goal is to provide that maximum concurrency performance in the least amount of time and the 8 | smallest refactor. 9 | 10 | A very simple example is as follows. 11 | 12 | .. doctest:: 13 | 14 | >>> import lox 15 | >>> 16 | >>> @lox.thread(4) # Will operate with a maximum of 4 threads 17 | ... def foo(x, y): 18 | ... return x * y 19 | ... 20 | >>> foo(3, 4) 21 | 12 22 | >>> for i in range(5): 23 | ... foo.scatter(i, i + 1) 24 | ... 25 | -ignore- 26 | >>> # foo is currently being executed in 4 threads 27 | >>> results = foo.gather() # block until results are ready 28 | >>> print(results) # Results are in the same order as scatter() calls 29 | [0, 2, 6, 12, 20] 30 | 31 | Features 32 | ======== 33 | 34 | * **Multithreading**: Powerful, intuitive multithreading in just 2 additional lines of code. 35 | 36 | * **Multiprocessing**: Truly parallel function execution with the same interface as **multithreading**. 37 | 38 | * **Synchronization**: Advanced thread synchronization, communication, and resource management tools. 39 | 40 | 41 | Contents 42 | ======== 43 | 44 | .. toctree:: 45 | :maxdepth: 2 46 | 47 | installation 48 | modules 49 | examples 50 | 51 | .. toctree:: 52 | :maxdepth: 1 53 | 54 | FAQ 55 | contributing 56 | authors 57 | history 58 | 59 | 60 | Indices and tables 61 | ================== 62 | * :ref:`genindex` 63 | * :ref:`modindex` 64 | * :ref:`search` 65 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean clean-test clean-pyc clean-build docs help 2 | .DEFAULT_GOAL := help 3 | 4 | define BROWSER_PYSCRIPT 5 | import os, webbrowser, sys 6 | 7 | try: 8 | from urllib import pathname2url 9 | except: 10 | from urllib.request import pathname2url 11 | 12 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 13 | endef 14 | export BROWSER_PYSCRIPT 15 | 16 | define PRINT_HELP_PYSCRIPT 17 | import re, sys 18 | 19 | for line in sys.stdin: 20 | match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) 21 | if match: 22 | target, help = match.groups() 23 | print("%-20s %s" % (target, help)) 24 | endef 25 | export PRINT_HELP_PYSCRIPT 26 | 27 | BROWSER := python -c "$$BROWSER_PYSCRIPT" 28 | 29 | help: 30 | @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) 31 | 32 | clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts 33 | 34 | clean-build: ## remove build artifacts 35 | rm -fr build/ 36 | rm -fr dist/ 37 | rm -fr .eggs/ 38 | find . -name '*.egg-info' -exec rm -fr {} + 39 | find . -name '*.egg' -exec rm -f {} + 40 | 41 | clean-pyc: ## remove Python file artifacts 42 | find . -name '*.pyc' -exec rm -f {} + 43 | find . -name '*.pyo' -exec rm -f {} + 44 | find . -name '*~' -exec rm -f {} + 45 | find . -name '__pycache__' -exec rm -fr {} + 46 | 47 | clean-test: ## remove test and coverage artifacts 48 | rm -fr .tox/ 49 | rm -f .coverage 50 | rm -fr htmlcov/ 51 | rm -fr .pytest_cache 52 | 53 | lint: ## check style with flake8 54 | flake8 lox tests 55 | 56 | test: ## run tests quickly with the default Python 57 | python3 -m pytest 58 | 59 | test-thread: 60 | python3 -m pytest tests/worker/test_thread.py --pdb -s 61 | 62 | test-process: 63 | python3 -m pytest tests/worker/test_thread.py --pdb -s 64 | 65 | test-all: ## run tests on every Python version with tox 66 | tox 67 | 68 | coverage: ## check code coverage quickly with the default Python 69 | coverage run --source lox -m pytest 70 | coverage report -m 71 | coverage html 72 | $(BROWSER) htmlcov/index.html 73 | 74 | docs: ## generate Sphinx HTML documentation, including API docs 75 | rm -f docs/lox.rst 76 | rm -f docs/modules.rst 77 | sphinx-apidoc -o docs/ lox 78 | $(MAKE) -C docs clean 79 | $(MAKE) -C docs html 80 | $(BROWSER) docs/_build/html/index.html 81 | 82 | servedocs: docs ## compile the docs watching for changes 83 | watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . 84 | 85 | release: dist ## package and upload a release 86 | twine upload dist/* 87 | 88 | dist: clean ## builds source and wheel package 89 | python setup.py sdist 90 | python setup.py bdist_wheel 91 | ls -l dist 92 | 93 | install: clean ## install the package to the active Python's site-packages 94 | python setup.py install 95 | 96 | dev: 97 | python3 -m pip install -r requirements_dev.txt 98 | python3 -m pip install -r requirements.txt 99 | python3 -m pip install -e . 100 | 101 | package: 102 | python3 setup.py sdist 103 | python3 setup.py bdist_wheel 104 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/PyCQA/isort 3 | rev: 5.13.2 4 | hooks: 5 | - id: isort 6 | name: isort 7 | args: ["--profile=black"] 8 | - id: isort 9 | name: isort (cython) 10 | types: [cython] 11 | args: ["--profile=black"] 12 | 13 | - repo: https://github.com/psf/black 14 | rev: 22.3.0 15 | hooks: 16 | - id: black 17 | args: 18 | - "--target-version=py36" 19 | - "--target-version=py37" 20 | - "--target-version=py38" 21 | - "--target-version=py39" 22 | types: [python] 23 | 24 | - repo: https://github.com/kynan/nbstripout 25 | rev: 0.5.0 26 | hooks: 27 | - id: nbstripout 28 | 29 | - repo: https://github.com/nbQA-dev/nbQA 30 | rev: 1.3.1 31 | hooks: 32 | - id: nbqa-isort 33 | args: ["--profile=black"] 34 | - id: nbqa-black 35 | args: 36 | - "--target-version=py36" 37 | - "--target-version=py37" 38 | - "--target-version=py38" 39 | - "--target-version=py39" 40 | additional_dependencies: [black==21.7b0] 41 | - id: nbqa-flake8 42 | 43 | - repo: https://github.com/pre-commit/pygrep-hooks 44 | rev: v1.9.0 45 | hooks: 46 | - id: python-check-blanket-noqa 47 | - id: python-check-blanket-type-ignore 48 | - id: python-check-mock-methods 49 | - id: python-no-log-warn 50 | - id: rst-backticks 51 | - id: rst-directive-colons 52 | types: [text] 53 | - id: rst-inline-touching-normal 54 | types: [text] 55 | 56 | - repo: https://github.com/pre-commit/pre-commit-hooks 57 | rev: v4.2.0 58 | hooks: 59 | - id: check-added-large-files 60 | - id: check-ast 61 | - id: check-builtin-literals 62 | - id: check-case-conflict 63 | - id: check-docstring-first 64 | - id: check-shebang-scripts-are-executable 65 | - id: check-merge-conflict 66 | - id: check-json 67 | - id: check-toml 68 | - id: check-xml 69 | - id: check-yaml 70 | - id: debug-statements 71 | - id: destroyed-symlinks 72 | - id: detect-private-key 73 | - id: end-of-file-fixer 74 | exclude: ^LICENSE|\.(html|csv|txt|svg|py)$ 75 | - id: pretty-format-json 76 | args: ["--autofix", "--no-ensure-ascii", "--no-sort-keys"] 77 | - id: requirements-txt-fixer 78 | - id: trailing-whitespace 79 | args: [--markdown-linebreak-ext=md] 80 | exclude: \.(html|svg)$ 81 | 82 | - repo: https://github.com/PyCQA/flake8 83 | rev: 7.0.0 84 | hooks: 85 | - id: flake8 86 | additional_dependencies: 87 | - flake8-2020 88 | - flake8-bugbear 89 | - flake8-comprehensions 90 | - flake8-docstrings 91 | - flake8-implicit-str-concat 92 | -------------------------------------------------------------------------------- /lox/lock/qlock.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: QLock 3 | :synopsis: FIFO Lock 4 | 5 | .. moduleauthor:: Brian Pugh 6 | """ 7 | 8 | from collections import deque 9 | from threading import Lock 10 | 11 | __all__ = [ 12 | "QLock", 13 | ] 14 | 15 | 16 | class QLock: 17 | """Lock that guarentees FIFO operation. ~6x slower than ``Lock()``. 18 | 19 | Modified from https://stackoverflow.com/a/19695878 20 | """ 21 | 22 | def __init__(self): 23 | """Create a ``QLock`` object.""" 24 | 25 | self.queue = deque() 26 | self.lock = Lock() 27 | self.count = 0 28 | 29 | def __enter__(self): 30 | """Acquire ``QLock`` at context enter.""" 31 | 32 | self.acquire() 33 | 34 | def __exit__(self, exc_type, exc_val, exc_tb): 35 | """Release ``QLock`` at context exit.""" 36 | 37 | self.release() 38 | 39 | def __len__( 40 | self, 41 | ): 42 | """Return number of tasks waiting to acquire. 43 | 44 | Returns 45 | ------- 46 | int 47 | Number of tasks waiting to acquire. 48 | """ 49 | 50 | return self.count 51 | 52 | @property 53 | def locked(self): 54 | """Whether or not the ``QLock`` is acquired""" 55 | 56 | return self.count > 0 57 | 58 | def acquire(self, timeout=-1): 59 | """Block until resource is available. 60 | 61 | Threads that call ``acquire`` obtain resource FIFO. 62 | 63 | Parameters 64 | ---------- 65 | timeout : float 66 | Maximum number of seconds to wait before aborting. 67 | 68 | Return 69 | ------ 70 | bool 71 | True on successful acquire, False on timeout. 72 | """ 73 | 74 | with self.lock: 75 | if self.count > 0: 76 | # Append a new acquired lock the queue 77 | lock = Lock() 78 | lock.acquire() 79 | self.queue.append(lock) 80 | self.lock.release() 81 | 82 | # Block until the lock has been released from another thread 83 | acquired = lock.acquire(timeout=timeout) 84 | self.lock.acquire() 85 | if acquired: 86 | # lock acquired 87 | pass 88 | else: 89 | # timed out 90 | try: 91 | self.queue.remove(self.lock) 92 | except ValueError: 93 | # lock must have been released between the timeout 94 | # and self.lock.acquire() 95 | pass 96 | return False 97 | self.count += 1 98 | return True 99 | 100 | def release(self): 101 | """Release exclusive access to resource. 102 | 103 | Raises 104 | ------ 105 | ValueError 106 | Lock released more than it has been acquired. 107 | """ 108 | 109 | with self.lock: 110 | if self.count == 0: 111 | raise ValueError("lock not acquired") 112 | self.count -= 1 113 | if self.queue: 114 | self.queue.popleft().release() 115 | -------------------------------------------------------------------------------- /lox/lock/index_semaphore.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: index_semaphore 3 | :synopsis: BoundedSemaphore where acquires return an index from [0, val). 4 | 5 | .. moduleauthor:: Brian Pugh 6 | """ 7 | 8 | import logging as log 9 | from contextlib import contextmanager 10 | from queue import Empty, Full, Queue 11 | 12 | __all__ = [ 13 | "IndexSemaphore", 14 | ] 15 | 16 | 17 | class IndexSemaphore: 18 | """``BoundedSemaphore``-like object where acquires return an index from [0, val). 19 | 20 | Example acquiring a gpu resource from a thread: 21 | >>> sem = IndexSemaphore(4) 22 | >>> with sem() as index: 23 | >>> print("Obtained resource %d" % (index,)) 24 | >>> 25 | Obtained resource 0 26 | """ 27 | 28 | def __init__(self, val): 29 | """Create an ``IndexSemaphore`` object. 30 | 31 | Parameters 32 | ---------- 33 | val : int 34 | Number of resources available. 35 | """ 36 | 37 | if val <= 0: 38 | raise ValueError("val must be >0") 39 | 40 | self.queue = Queue(maxsize=val) 41 | for i in range(val): 42 | self.queue.put(i) 43 | 44 | @contextmanager 45 | def __call__(self, timeout=None): 46 | """Enter context manager when a timeout wants to be specified. 47 | Only to be call as part of a "with" statement. 48 | 49 | Parameters 50 | ---------- 51 | timeout : float 52 | Maximum number of seconds to wait before aborting. 53 | Set timeout=None for no timeout. 54 | 55 | Returns 56 | ------- 57 | int 58 | Resource index. None on timeout/failure. 59 | """ 60 | 61 | index = self.acquire(timeout=timeout) 62 | try: 63 | yield index 64 | except Exception as e: 65 | log.error(e) 66 | finally: 67 | if index is not None: 68 | self.release(index) 69 | 70 | def __len__(self): 71 | """Return current blocked queue size. 72 | 73 | Returns 74 | ------- 75 | int 76 | Current blocked queue size. 77 | """ 78 | 79 | return self.queue.qsize() 80 | 81 | def acquire(self, timeout=None): 82 | """Blocking acquire resource. 83 | 84 | Parameters 85 | ---------- 86 | timeout : float 87 | Maximum number of seconds to wait before returning. 88 | 89 | Returns 90 | ------- 91 | int 92 | Resource index on successful acquire. None on timeout. 93 | """ 94 | 95 | try: 96 | return self.queue.get(timeout=timeout) 97 | except Empty: 98 | return None 99 | 100 | def release(self, index): 101 | """Release resource at index. 102 | 103 | Parameters 104 | ---------- 105 | index : int 106 | Index of resource to release. 107 | 108 | Raises 109 | ------ 110 | Exception 111 | Resource has been released more times than acquired. 112 | """ 113 | 114 | try: 115 | self.queue.put_nowait(index) 116 | except Full: 117 | raise Exception("IndexSemaphore released more times than acquired") 118 | -------------------------------------------------------------------------------- /tests/queue/test_Announcement.py: -------------------------------------------------------------------------------- 1 | import queue 2 | from threading import Lock, Thread 3 | from time import sleep 4 | 5 | import pytest 6 | 7 | import lox 8 | from lox import Announcement 9 | 10 | 11 | def test_1(): 12 | """Test most basic usecase.""" 13 | 14 | ann = lox.Announcement() 15 | x_in = [1, 2, 3, 4, 5] 16 | foo_soln, bar_soln = [], [] 17 | 18 | foo_q = ann.subscribe() 19 | bar_q = ann.subscribe() 20 | 21 | assert isinstance(foo_q, lox.Announcement) 22 | assert isinstance(bar_q, lox.Announcement) 23 | assert foo_q != bar_q 24 | 25 | def foo(): 26 | x = foo_q.get() 27 | foo_soln.append(x**2) 28 | 29 | def bar(): 30 | x = bar_q.get() 31 | bar_soln.append(x**3) 32 | 33 | threads = [] 34 | for _ in x_in: 35 | threads.append(Thread(target=foo)) 36 | threads.append(Thread(target=bar)) 37 | 38 | for x in x_in: 39 | ann.put(x) 40 | 41 | for t in threads: 42 | t.start() 43 | 44 | for t in threads: 45 | t.join() 46 | 47 | assert len(foo_soln) == len(x_in) 48 | assert len(bar_soln) == len(x_in) 49 | 50 | for x, r in zip(x_in, foo_soln): 51 | assert r == x**2 52 | 53 | for x, r in zip(x_in, bar_soln): 54 | assert r == x**3 55 | 56 | 57 | def test_2(): 58 | """Test the backlog.""" 59 | 60 | ann = lox.Announcement(backlog=0) 61 | x_in = [1, 2, 3, 4, 5] 62 | foo_soln, bar_soln = [], [] 63 | 64 | foo_q = ann.subscribe() 65 | 66 | def foo(): 67 | x = foo_q.get() 68 | foo_soln.append(x**2) 69 | 70 | def bar(): 71 | x = bar_q.get() 72 | bar_soln.append(x**3) 73 | 74 | threads = [] 75 | for _ in x_in: 76 | threads.append(Thread(target=foo)) 77 | threads.append(Thread(target=bar)) 78 | 79 | for x in x_in: 80 | ann.put(x) 81 | 82 | bar_q = ann.subscribe() 83 | ann.finalize() 84 | 85 | for t in threads: 86 | t.start() 87 | 88 | for t in threads: 89 | t.join() 90 | 91 | assert len(foo_soln) == len(x_in) 92 | assert len(bar_soln) == len(x_in) 93 | 94 | for x, r in zip(x_in, foo_soln): 95 | assert r == x**2 96 | 97 | for x, r in zip(x_in, bar_soln): 98 | assert r == x**3 99 | 100 | 101 | def test_3(): 102 | """Test subscribing after finalize.""" 103 | 104 | ann = lox.Announcement() 105 | ann.finalize() 106 | with pytest.raises(lox.SubscribeFinalizedError): 107 | ann.subscribe() 108 | 109 | 110 | def test_4(): 111 | """Testing many-to-many capability.""" 112 | 113 | ann_foo = Announcement() 114 | ann_bar = ann_foo.subscribe() 115 | ann_baz = ann_bar.subscribe() 116 | 117 | ann_foo.put(1) 118 | 119 | assert ann_bar.get() == 1 120 | assert ann_baz.get() == 1 121 | 122 | ann_bar.put(2) 123 | assert ann_foo.get() == 2 124 | assert ann_baz.get() == 2 125 | 126 | ann_bar.put(3) 127 | assert ann_foo.get() == 3 128 | assert ann_baz.get() == 3 129 | 130 | 131 | def test_5(): 132 | """Testing many-to-many backlog and finalization.""" 133 | 134 | ann_foo = lox.Announcement(backlog=-1) 135 | ann_bar = ann_foo.subscribe() 136 | 137 | ann_foo.put(1) 138 | ann_foo.put(3) 139 | ann_foo.put(7) 140 | 141 | assert ann_bar.get() == 1 142 | assert ann_bar.get() == 3 143 | assert ann_bar.get() == 7 144 | 145 | ann_baz = ann_bar.subscribe() 146 | 147 | assert ann_baz.get() == 1 148 | assert ann_baz.get() == 3 149 | assert ann_baz.get() == 7 150 | 151 | ann_boo = ann_bar.subscribe() 152 | ann_boo.finalize() 153 | 154 | assert ann_boo.get() == 1 155 | assert ann_boo.get() == 3 156 | assert ann_boo.get() == 7 157 | 158 | with pytest.raises(queue.Empty): 159 | ann_boo.get(timeout=0.01) 160 | 161 | with pytest.raises(lox.SubscribeFinalizedError): 162 | ann_foo.subscribe() 163 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Contributing 5 | ============ 6 | 7 | Contributions are welcome, and they are greatly appreciated! Every little bit 8 | helps, and credit will always be given. 9 | 10 | You can contribute in many ways: 11 | 12 | Types of Contributions 13 | ---------------------- 14 | 15 | Report Bugs 16 | ~~~~~~~~~~~ 17 | 18 | Report bugs at https://github.com/BrianPugh/lox/issues. 19 | 20 | If you are reporting a bug, please include: 21 | 22 | * Your operating system name and version. 23 | * Any details about your local setup that might be helpful in troubleshooting. 24 | * Detailed steps to reproduce the bug. 25 | 26 | Fix Bugs 27 | ~~~~~~~~ 28 | 29 | Look through the GitHub issues for bugs. Anything tagged with "bug" and "help 30 | wanted" is open to whoever wants to implement it. 31 | 32 | Implement Features 33 | ~~~~~~~~~~~~~~~~~~ 34 | 35 | Look through the GitHub issues for features. Anything tagged with "enhancement" 36 | and "help wanted" is open to whoever wants to implement it. 37 | 38 | Write Documentation 39 | ~~~~~~~~~~~~~~~~~~~ 40 | 41 | lox could always use more documentation, whether as part of the 42 | official lox docs, in docstrings, or even on the web in blog posts, 43 | articles, and such. 44 | 45 | Submit Feedback 46 | ~~~~~~~~~~~~~~~ 47 | 48 | The best way to send feedback is to file an issue at https://github.com/BrianPugh/lox/issues. 49 | 50 | If you are proposing a feature: 51 | 52 | * Explain in detail how it would work. 53 | * Keep the scope as narrow as possible, to make it easier to implement. 54 | * Remember that this is a volunteer-driven project, and that contributions 55 | are welcome :) 56 | 57 | Get Started! 58 | ------------ 59 | 60 | Ready to contribute? Here's how to set up ``lox`` for local development. 61 | 62 | 1. Fork the ``lox`` repo on GitHub. 63 | 2. Clone your fork locally:: 64 | 65 | $ git clone git@github.com:your_name_here/lox.git 66 | 67 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 68 | 69 | $ mkvirtualenv lox 70 | $ cd lox/ 71 | $ python setup.py develop 72 | 73 | 4. Create a branch for local development:: 74 | 75 | $ git checkout -b name-of-your-bugfix-or-feature 76 | 77 | Now you can make your changes locally. 78 | 79 | 5. When you're done making changes, check that your changes pass flake8 and the 80 | tests, including testing other Python versions with tox:: 81 | 82 | $ flake8 lox tests 83 | $ python setup.py test or py.test 84 | $ tox 85 | 86 | To get flake8 and tox, just pip install them into your virtualenv. 87 | 88 | 6. Commit your changes and push your branch to GitHub:: 89 | 90 | $ git add . 91 | $ git commit -m "Your detailed description of your changes." 92 | $ git push origin name-of-your-bugfix-or-feature 93 | 94 | 7. Submit a pull request through the GitHub website. 95 | 96 | Pull Request Guidelines 97 | ----------------------- 98 | 99 | Before you submit a pull request, check that it meets these guidelines: 100 | 101 | 1. The pull request should include tests. 102 | 2. If the pull request adds functionality, the docs should be updated. Put 103 | your new functionality into a function with a docstring, and add the 104 | feature to the list in README.rst. 105 | 3. The pull request should work for Python >=3.6, and for PyPy. Check 106 | https://travis-ci.org/BrianPugh/lox/pull_requests 107 | and make sure that the tests pass for all supported Python versions. 108 | 109 | Tips 110 | ---- 111 | 112 | To run a subset of tests:: 113 | 114 | $ py.test tests.test_lox 115 | 116 | 117 | Deploying 118 | --------- 119 | 120 | A reminder for the maintainers on how to deploy. 121 | Make sure all your changes are committed (including an entry in HISTORY.rst). 122 | Then run:: 123 | 124 | $ bumpversion patch # possible: major / minor / patch 125 | $ git push 126 | $ git push --tags 127 | 128 | Travis will then deploy to PyPI if tests pass. 129 | -------------------------------------------------------------------------------- /tests/lock/test_IndexSemaphore.py: -------------------------------------------------------------------------------- 1 | from collections import deque 2 | from threading import Lock, Thread 3 | from time import sleep, time 4 | 5 | import pytest 6 | 7 | from lox import IndexSemaphore 8 | 9 | SLEEP_TIME = 0.01 10 | n_resource = 5 11 | n_threads = 20 12 | 13 | 14 | def test_multithread_args(): 15 | resp = deque() 16 | sem = IndexSemaphore(n_resource) 17 | locks = [Lock() for _ in range(n_resource)] 18 | 19 | def func(): 20 | nonlocal locks 21 | index = sem.acquire(timeout=100) 22 | if locks[index].acquire(timeout=0): 23 | # Acquired Test Lock with no waiting, indicating this index is unique 24 | sleep(SLEEP_TIME) 25 | locks[index].release() 26 | resp.append(True) 27 | else: 28 | # timeout (bad) 29 | resp.append(False) 30 | sem.release(index) 31 | 32 | threads = [Thread(target=func) for _ in range(n_threads)] 33 | 34 | for t in threads: 35 | t.start() 36 | for t in threads: 37 | t.join() 38 | for r in resp: 39 | assert r 40 | assert len(resp) == n_threads 41 | 42 | 43 | def test_multithread_no_args(): 44 | resp = deque() 45 | sem = IndexSemaphore(n_resource) 46 | locks = [Lock() for _ in range(n_resource)] 47 | 48 | def func(): 49 | nonlocal locks 50 | index = sem.acquire() 51 | if locks[index].acquire(timeout=0): 52 | # Acquired Test Lock with no waiting, indicating this index is unique 53 | sleep(SLEEP_TIME) 54 | locks[index].release() 55 | resp.append(True) 56 | else: 57 | # timeout (bad) 58 | resp.append(False) 59 | sem.release(index) 60 | 61 | threads = [Thread(target=func) for _ in range(n_threads)] 62 | 63 | for t in threads: 64 | t.start() 65 | for t in threads: 66 | t.join() 67 | for r in resp: 68 | assert r 69 | assert len(resp) == n_threads 70 | 71 | 72 | def test_multithread_context_args(): 73 | resp = deque() 74 | sem = IndexSemaphore(n_resource) 75 | locks = [Lock() for _ in range(n_resource)] 76 | 77 | def func(): 78 | nonlocal locks 79 | with sem(timeout=None) as index: 80 | if locks[index].acquire(timeout=0): 81 | # Acquired Test Lock with no waiting, indicating this index is unique 82 | sleep(SLEEP_TIME) 83 | locks[index].release() 84 | resp.append(True) 85 | else: 86 | # timeout (bad) 87 | resp.append(False) 88 | 89 | threads = [Thread(target=func) for _ in range(n_threads)] 90 | 91 | for t in threads: 92 | t.start() 93 | for t in threads: 94 | t.join() 95 | for r in resp: 96 | assert r 97 | assert len(resp) == n_threads 98 | 99 | 100 | def test_multithread_context_no_args(): 101 | resp = deque() 102 | sem = IndexSemaphore(n_resource) 103 | locks = [Lock() for _ in range(n_resource)] 104 | 105 | def func(): 106 | nonlocal locks 107 | with sem() as index: 108 | if locks[index].acquire(timeout=0): 109 | # Acquired Test Lock with no waiting, indicating this index is unique 110 | sleep(SLEEP_TIME) 111 | locks[index].release() 112 | resp.append(True) 113 | else: 114 | # timeout (bad) 115 | resp.append(False) 116 | 117 | threads = [Thread(target=func) for _ in range(n_threads)] 118 | 119 | for t in threads: 120 | t.start() 121 | for t in threads: 122 | t.join() 123 | for r in resp: 124 | assert r 125 | assert len(resp) == n_threads 126 | 127 | 128 | def test_invalid_constructor(): 129 | with pytest.raises(ValueError): 130 | IndexSemaphore(0) 131 | 132 | 133 | def test_timeout(): 134 | sem = IndexSemaphore(1) 135 | with sem(timeout=None) as index1: 136 | assert index1 == 0 137 | with sem(timeout=0.1) as index2: 138 | assert index2 is None 139 | -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | History 3 | ======= 4 | 5 | 0.11.0 (2022-04-07) 6 | ------------------- 7 | * Set number of workers to 0 (in thread execution) if the environment variable ``LOX_DEBUG`` is set to a true-like value (``true``, ``1``, etc.). Makes it easier to set breakpoints in multithreaded code without having to manually edit the decorator. 8 | 9 | 0.10.0 (2021-12-18) 10 | ------------------- 11 | * Remove dependency pinning. 12 | * Allow ``@lox.thread(0)``. This will execute ``scatter`` calls in parent thread. 13 | Useful for debugging breakpoints in parallelized code. 14 | 15 | 0.9.0 (2020-11-25) 16 | ------------------ 17 | * ``tqdm`` support on ``lox.process.gather``. See v0.8.0 release notes for usage. 18 | 19 | 0.8.0 (2020-11-25) 20 | ------------------ 21 | * ``tqdm`` support on ``lox.thread.gather`` 22 | * Can be a bool:: 23 | 24 | >>> my_func.gather(tqdm=True) 25 | 26 | * Can be a ``tqdm`` object:: 27 | 28 | >>> from tqdm import tqdm 29 | >>> pbar = tqdm(total=100) 30 | >>> for _ in range(100): 31 | >>> my_func.scatter() 32 | >>> my_func.gather(tqdm=pbar) 33 | 34 | 0.7.0 (2020-07-20) 35 | ------------------ 36 | * Complete rework of workers 37 | + Fix memory leaks 38 | * Drop support for python3.5 39 | * Drop support for chaining in favor of simpler codebase 40 | 41 | 0.6.3 (2019-07-30) 42 | ------------------ 43 | * Alternative fix for 0.6.2. 44 | 45 | 0.6.2 (2019-07-21) 46 | ------------------ 47 | * Update dependencies 48 | * Fix garbage-collecting exclusiviity 49 | 50 | 0.6.1 (2019-07-21) 51 | ------------------ 52 | * Fix memory leak in ``lox.process``. 53 | 54 | 0.6.0 (2019-07-21) 55 | ------------------ 56 | 57 | * ``lox.Announcement`` ``subscribe()`` calls now return another ``Announcement`` 58 | object that behaves like a queue instead of an actual queue. Allows for 59 | many-queue-to-many-queue communications. 60 | 61 | * New Object: ``lox.Funnel``. allows for waiting on many queues for a complete 62 | set of inputs indicated by a job ID. 63 | 64 | 0.5.0 (2019-07-01) 65 | ------------------ 66 | 67 | * New Object: ``lox.Announcement``. Allows a one-to-many thread queue with 68 | backlog support so that late subscribers can still get all (or most recent) 69 | announcements before they subscribed. 70 | 71 | * New Feature: ``lox.thread`` ``scatter`` calls can now be chained together. 72 | ``scatter`` now returns an ``int`` subclass that contains metadata to allow 73 | chaining. Each scatter call can have a maximum of 1 previous ``scatter`` result. 74 | 75 | * Documentation updates, theming, and logos 76 | 77 | 0.4.3 (2019-06-24) 78 | ------------------ 79 | * Garbage collect cached decorated object methods 80 | 81 | 0.4.2 (2019-06-23) 82 | ------------------ 83 | * Fixed multiple instances and successive scatter and gather calls to wrapped methods 84 | 85 | 0.4.1 (2019-06-23) 86 | ------------------ 87 | * Fixed broken workers and unit tests for workers 88 | 89 | 0.4.0 (2019-06-22) 90 | ------------------ 91 | * Semi-breaking change: **lox.thread** and **lox.process** now automatically pass 92 | the object instance when decorating a method. 93 | 94 | 0.3.4 (2019-06-20) 95 | ------------------ 96 | * Print traceback in red when a thread crashes 97 | 98 | 0.3.3 (2019-06-19) 99 | ------------------ 100 | * Fix bug where thread in scatter of lox.thread double releases on empty queue 101 | 102 | 0.3.2 (2019-06-17) 103 | ------------------ 104 | 105 | * Fix manifest for installation from wheel 106 | 107 | 0.3.1 (2019-06-17) 108 | ------------------ 109 | 110 | * Fix package on pypi 111 | 112 | 0.3.0 (2019-06-01) 113 | ------------------ 114 | 115 | * Multiprocessing decorator. **lox.pool** renamed to **lox.thread** 116 | 117 | * Substantial pytest bug fixes 118 | 119 | * Documentation examples 120 | 121 | * timeout for RWLock 122 | 123 | 0.2.1 (2019-05-25) 124 | ------------------ 125 | 126 | * Fix IndexSemaphore context manager 127 | 128 | 0.2.0 (2019-05-24) 129 | ------------------ 130 | 131 | * Added QLock 132 | 133 | * Documentation syntax fixes 134 | 135 | 0.1.1 (2019-05-24) 136 | ------------------ 137 | 138 | * CICD test 139 | 140 | 0.1.0 (2019-05-24) 141 | ------------------ 142 | 143 | * First release on PyPI. 144 | -------------------------------------------------------------------------------- /lox/lock/rw_lock.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: rw_Lock 3 | :synopsis: Synchronization primitive to solves the readers–writers problem. 4 | 5 | .. moduleauthor:: Brian Pugh 6 | """ 7 | 8 | from contextlib import contextmanager 9 | from threading import Lock 10 | 11 | from . import LightSwitch 12 | 13 | __all__ = [ 14 | "RWLock", 15 | ] 16 | 17 | 18 | class RWLock: 19 | """Lock for a Multi-Reader-Single-Writer scenario. 20 | 21 | Unlimited numbers of reader can obtain the lock, but as soon as a writer 22 | attempts to acquire the lock, all reads are blocked until the current 23 | readers are finished, the writer acquires the lock, and finally 24 | releases it. 25 | 26 | Similar to a ``lox.LightSwitch``, but blocks incoming "readers" while a "write" 27 | is trying to be performed. 28 | 29 | Attributes 30 | ---------- 31 | read_counter : int 32 | Number of readers that have acquired the lock. 33 | """ 34 | 35 | def __init__(self): 36 | """Create RWLock object""" 37 | 38 | self._no_writers = Lock() 39 | self._no_readers = Lock() 40 | self.read_counter = LightSwitch(self._no_writers) 41 | self._write_counter = LightSwitch(self._no_readers) 42 | self._readers_queue = Lock() 43 | 44 | @contextmanager 45 | def __call__(self, rw_flag: str, timeout=-1): 46 | """Use in contextmanager to specify acquire/release type""" 47 | 48 | self.acquire(rw_flag, timeout=timeout) 49 | try: 50 | yield self 51 | finally: 52 | self.release(rw_flag) 53 | 54 | def __len__(self): 55 | """Get the read_counter value 56 | 57 | Returns 58 | ------- 59 | int 60 | Number of current readers 61 | """ 62 | 63 | return len(self.read_counter) 64 | 65 | def _check_rw_flag(self, rw_flag): 66 | """Check if passed in flag is a valid value""" 67 | 68 | rw_flag = rw_flag.lower() 69 | if rw_flag == "r": 70 | pass 71 | elif rw_flag == "w": 72 | pass 73 | else: 74 | raise ValueError("rw_flag must be 'r' or 'w'") 75 | return rw_flag 76 | 77 | def acquire(self, rw_flag: str, timeout=-1): 78 | """Acquire the lock as a "reader" or a "writer". 79 | 80 | Parameters 81 | ---------- 82 | rw_flag : str 83 | Either 'r' for 'read' or 'w' for 'write' acquire. 84 | 85 | timeout : float 86 | Time in seconds before timeout occurs for acquiring lock. 87 | 88 | Returns 89 | ------- 90 | bool 91 | ``True`` if lock was acquired, ``False`` otherwise. 92 | """ 93 | 94 | obtained = False 95 | rw_flag = self._check_rw_flag(rw_flag) 96 | if rw_flag == "r": 97 | obtained = self._readers_queue.acquire(timeout=timeout) 98 | if not obtained: 99 | return False 100 | 101 | obtained = self._no_readers.acquire(timeout=timeout) 102 | if not obtained: 103 | self._readers_queue.release() 104 | return False 105 | 106 | obtained = self.read_counter.acquire(timeout=timeout) 107 | 108 | self._no_readers.release() 109 | self._readers_queue.release() 110 | elif rw_flag == "w": 111 | obtained = self._write_counter.acquire(timeout=timeout) 112 | if not obtained: 113 | return False 114 | 115 | obtained = self._no_writers.acquire(timeout=timeout) 116 | if not obtained: 117 | self._write_counter.release() 118 | return obtained 119 | 120 | def release(self, rw_flag: str): 121 | """Release acquired lock. 122 | 123 | Parameters 124 | ---------- 125 | rw_flag : str 126 | Either 'r' for 'read' or 'w' for 'write' acquire. 127 | """ 128 | 129 | rw_flag = self._check_rw_flag(rw_flag) 130 | if rw_flag == "r": 131 | self.read_counter.release() 132 | elif rw_flag == "w": 133 | self._no_writers.release() 134 | self._write_counter.release() 135 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. image:: assets/lox_200w.png 2 | 3 | 4 | .. image:: https://img.shields.io/pypi/v/lox.svg 5 | :target: https://pypi.python.org/pypi/lox 6 | 7 | .. image:: https://readthedocs.org/projects/lox/badge/?version=latest 8 | :target: https://lox.readthedocs.io/en/latest/?badge=latest 9 | :alt: Documentation Status 10 | 11 | 12 | Threading and multiprocessing made easy. 13 | 14 | 15 | * Free software: Apache-2.0 license 16 | * Documentation: https://lox.readthedocs.io. 17 | * Python >=3.6 18 | 19 | 20 | **Lox** provides decorators and synchronization primitives to quickly add 21 | concurrency to your projects. 22 | 23 | Installation 24 | ------------ 25 | 26 | pip3 install --user lox 27 | 28 | For multiprocessing support, install with the optional extra: 29 | 30 | pip3 install --user 'lox[multiprocessing]' 31 | 32 | Features 33 | -------- 34 | 35 | * **Multithreading**: Powerful, intuitive multithreading in just 2 additional lines of code. 36 | 37 | * **Multiprocessing**: Truly parallel function execution with the same interface as **multithreading**. 38 | 39 | * **Synchronization**: Advanced thread synchronization, communication, and resource management tools. 40 | 41 | Todos 42 | ----- 43 | 44 | * All objects except ``lox.process`` are for threads. These will eventually be multiprocess friendly. 45 | 46 | Usage 47 | ----- 48 | 49 | Easy Multithreading 50 | ^^^^^^^^^^^^^^^^^^^ 51 | 52 | >>> import lox 53 | >>> 54 | >>> @lox.thread(4) # Will operate with a maximum of 4 threads 55 | ... def foo(x,y): 56 | ... return x*y 57 | >>> foo(3,4) # normal function calls still work 58 | 12 59 | >>> for i in range(5): 60 | ... foo.scatter(i, i+1) 61 | -ignore- 62 | >>> # foo is currently being executed in 4 threads 63 | >>> results = foo.gather() # block until results are ready 64 | >>> print(results) # Results are in the same order as scatter() calls 65 | [0, 2, 6, 12, 20] 66 | 67 | Or, for example, if you aren't allowed to directly decorate the function you 68 | would like multithreaded/multiprocessed, you can just directly invoke the 69 | decorator: 70 | 71 | .. code-block:: pycon 72 | 73 | >>> # Lets say we don't have direct access to this function 74 | ... def foo(x, y): 75 | ... return x * y 76 | ... 77 | >>> 78 | >>> def my_func(): 79 | ... foo_threaded = lox.thread(foo) 80 | ... for i in range(5): 81 | ... foo_threaded.scatter(i, i + 1) 82 | ... results = foo_threaded.gather() 83 | ... # foo is currently being executed in default 50 thread executor pool 84 | ... return results 85 | ... 86 | 87 | 88 | This also makes it easier to dynamically control the number of 89 | thread/processes in the executor pool. The syntax is a little weird, but 90 | this is just explicitly invoking a decorator that has optional arguments: 91 | 92 | .. code-block:: pycon 93 | 94 | >>> # Set the number of executer threads to 10 95 | >>> foo_threaded = lox.thread(10)(foo) 96 | 97 | 98 | Easy Multiprocessing 99 | ^^^^^^^^^^^^^^^^^^^^ 100 | 101 | .. code-block:: pycon 102 | 103 | >>> import lox 104 | >>> 105 | >>> @lox.process(4) # Will operate with a pool of 4 processes 106 | ... def foo(x, y): 107 | ... return x * y 108 | ... 109 | >>> foo(3, 4) # normal function calls still work 110 | 12 111 | >>> for i in range(5): 112 | ... foo.scatter(i, i + 1) 113 | ... 114 | -ignore- 115 | >>> # foo is currently being executed in 4 processes 116 | >>> results = foo.gather() # block until results are ready 117 | >>> print(results) # Results are in the same order as scatter() calls 118 | [0, 2, 6, 12, 20] 119 | 120 | 121 | Progress Bar Support (tqdm) 122 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ 123 | 124 | .. code-block:: pycon 125 | 126 | >>> import lox 127 | >>> from random import random 128 | >>> from time import sleep 129 | >>> 130 | >>> @lox.thread(2) 131 | ... def foo(multiplier): 132 | ... sleep(multiplier * random()) 133 | ... 134 | >>> for i in range(10): 135 | >>> foo.scatter(i) 136 | >>> results = foo.gather(tqdm=True) 137 | 90%|████████████████████████████████▌ | 9/10 [00:03<00:00, 1.32it/s] 138 | 100%|███████████████████████████████████████| 10/10 [00:06<00:00, 1.46s/it] 139 | -------------------------------------------------------------------------------- /lox/lock/light_switch.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: LightSwitch 3 | :synopsis: Lock acquire on first entry and release on last exit. 4 | 5 | .. moduleauthor:: Brian Pugh 6 | 7 | See https://w3.cs.jmu.edu/kirkpams/OpenCSF/Books/cs361/html/DesignAdv.html 8 | 9 | """ 10 | 11 | import threading 12 | from time import time 13 | 14 | try: 15 | import pathos.multiprocessing as mp 16 | 17 | _PATHOS_AVAILABLE = True 18 | except ImportError: 19 | mp = None 20 | _PATHOS_AVAILABLE = False 21 | 22 | __all__ = [ 23 | "LightSwitch", 24 | ] 25 | 26 | 27 | class LightSwitch: 28 | """Acquires a provided lock while ``LightSwitch`` is in use. 29 | 30 | The lightswitch pattern creates a first-in-last-out synchronization 31 | mechanism. The name of the pattern is inspired by people entering a 32 | room in the physical world. The first person to enter the room turns 33 | on the lights; then, when everyone is leaving, the last person to exit 34 | turns the lights off. 35 | """ 36 | 37 | def __init__(self, lock, multiprocessing=False): 38 | """Create a ``LightSwitch`` object. 39 | 40 | Parameters 41 | ---------- 42 | lock : ``threading.Lock`` or ``pathos.multiprocessing.Lock`` 43 | Lock to acquire when internal counter is incremented from zero. 44 | Lock to release when internal counter is decremented to zero. 45 | """ 46 | 47 | if multiprocessing: 48 | if not _PATHOS_AVAILABLE: 49 | raise RuntimeError( 50 | "pathos is not installed. " 51 | "Install the full package with: pip install lox[multiprocessing]" 52 | ) 53 | self._library = mp 54 | else: 55 | self._library = threading 56 | 57 | self.lock = lock 58 | self.counter = 0 59 | self._counter_lock = self._library.Lock() 60 | 61 | @property 62 | def lock(self): 63 | """threading.Lock: The lock provided to the constructor that may be 64 | acquired/released by ``LightSwitch``. 65 | """ 66 | 67 | return self._lock 68 | 69 | @lock.setter 70 | def lock(self, lock): 71 | self._lock = lock 72 | 73 | @property 74 | def counter(self): 75 | """int: Times the ``LightSwitch`` has been acquired without release.""" 76 | 77 | return self._counter 78 | 79 | @counter.setter 80 | def counter(self, counter): 81 | self._counter = counter 82 | 83 | def __enter__(self): 84 | """Acquire ``LightSwitch`` at context enter.""" 85 | 86 | self.acquire() 87 | 88 | def __exit__(self, exc_type, exc_val, exc_tb): 89 | """Release ``LightSwitch`` at context exit.""" 90 | 91 | self.release() 92 | 93 | def __len__(self): 94 | """Get the counter value. 95 | 96 | Returns 97 | ------- 98 | int 99 | counter value (number of times lightswitch has been acquired). 100 | """ 101 | 102 | return self.counter 103 | 104 | def acquire(self, timeout=-1): 105 | """Acquire the ``LightSwitch`` and increment the internal counter. 106 | 107 | When the internal counter is incremented from zero, it will acquire 108 | the provided lock. 109 | 110 | Parameters 111 | ---------- 112 | timeout : float 113 | Maximum number of seconds to wait before aborting. 114 | 115 | Returns 116 | ------- 117 | bool 118 | ``True`` on success, ``False`` on failure (like timeout). 119 | """ 120 | 121 | # Acquire the counter_lock while keeping track of approximately time 122 | t_start = time() 123 | if not self._counter_lock.acquire(timeout=timeout): 124 | return False 125 | t_remaining = timeout - (time() - t_start) 126 | 127 | if timeout < 0: 128 | t_remaining = -1 129 | elif t_remaining < 0: 130 | t_remaining = 0 131 | 132 | # Acquire lock if first to acquire 133 | if self.counter == 0: 134 | if not self._lock.acquire(timeout=t_remaining): 135 | self._counter_lock.release() 136 | return False 137 | self.counter += 1 138 | 139 | self._counter_lock.release() 140 | return True 141 | 142 | def release(self): 143 | """Release the ``LightSwitch`` by decrementing the internal counter. 144 | 145 | When the internal counter is decremented to zero, it will release 146 | the provided lock. 147 | """ 148 | 149 | with self._counter_lock: 150 | self.counter -= 1 151 | if self.counter == 0: 152 | self._lock.release() 153 | -------------------------------------------------------------------------------- /tests/lock/test_LightSwitch.py: -------------------------------------------------------------------------------- 1 | from copy import copy 2 | from threading import Lock, Thread 3 | from time import sleep, time 4 | 5 | from lox import LightSwitch 6 | 7 | SLEEP_TIME = 0.01 8 | N_WORKERS = 5 9 | 10 | common_lock = None 11 | counting_lock = None 12 | resp_lock = None 13 | resource = None 14 | 15 | 16 | def common_setup(): 17 | global common_lock, counting_lock, resp_lock 18 | 19 | common_lock = Lock() 20 | counting_lock = LightSwitch(common_lock) 21 | resp_lock = Lock() 22 | 23 | 24 | def test_LightSwitch_1(): 25 | global common_lock, counting_lock, resp_lock, resource 26 | common_setup() 27 | 28 | with counting_lock: 29 | acquired_lock = common_lock.acquire(timeout=0) 30 | assert acquired_lock is False 31 | acquired_lock = common_lock.acquire(timeout=0) 32 | assert acquired_lock is True 33 | common_lock.release() 34 | 35 | 36 | def test_LightSwitch_len(): 37 | global common_lock, counting_lock, resp_lock, resource 38 | common_setup() 39 | assert 0 == len(counting_lock) 40 | counting_lock.acquire() 41 | assert 1 == len(counting_lock) 42 | counting_lock.acquire() 43 | assert 2 == len(counting_lock) 44 | counting_lock.release() 45 | assert 1 == len(counting_lock) 46 | counting_lock.release() 47 | assert 0 == len(counting_lock) 48 | 49 | 50 | def test_LightSwitch_timeout(): 51 | lock = Lock() 52 | ls = LightSwitch(lock) 53 | lock.acquire() 54 | assert ls.acquire(timeout=SLEEP_TIME) is False 55 | assert ls.acquire(timeout=0) is False 56 | 57 | 58 | def test_bathroom_example(): 59 | sol = [ 60 | "p_0_enter", 61 | "p_1_enter", 62 | "p_0_exit", 63 | "p_2_enter", 64 | "p_1_exit", 65 | "p_3_enter", 66 | "p_2_exit", 67 | "p_4_enter", 68 | "p_3_exit", 69 | "p_4_exit", 70 | "j_enter", 71 | "j_exit", 72 | ] 73 | 74 | res = bathroom_example() 75 | 76 | for r, s in zip(res, sol): 77 | assert r == s 78 | 79 | 80 | def bathroom_example(): 81 | """Bathroom analogy for light switch. 82 | 83 | Scenario: 84 | A janitor needs to clean a restroom, but is not allowed to enter until 85 | all people are out of the restroom. How do we implement this? 86 | """ 87 | restroom_occupied = Lock() 88 | restroom = LightSwitch(restroom_occupied) 89 | res = [] 90 | n_people = 5 91 | sleep_time = 0.2 92 | 93 | def janitor(): 94 | with restroom_occupied: # block until the restroom is no longer occupied 95 | res.append("j_enter") 96 | print("(%0.3f s) Janitor entered the restroom" % (time() - t_start,)) 97 | sleep(sleep_time) # clean the restroom 98 | res.append("j_exit") 99 | print("(%0.3f s) Janitor exited the restroom" % (time() - t_start,)) 100 | 101 | def people(id): 102 | if id == 0: # Get the starting time of execution for display purposes 103 | global t_start 104 | t_start = time() 105 | with restroom: # block if a janitor is in the restroom 106 | res.append("p_%d_enter" % (id,)) 107 | print( 108 | "(%0.3f s) Person %d entered the restroom" 109 | % ( 110 | time() - t_start, 111 | id, 112 | ) 113 | ) 114 | sleep(sleep_time) # use the restroom 115 | res.append("p_%d_exit" % (id,)) 116 | print( 117 | "(%0.3f s) Person %d exited the restroom" 118 | % ( 119 | time() - t_start, 120 | id, 121 | ) 122 | ) 123 | 124 | people_threads = [Thread(target=people, args=(i,)) for i in range(n_people)] 125 | janitor_thread = Thread(target=janitor) 126 | 127 | for i, person in enumerate(people_threads): 128 | person.start() # Person i will now attempt to enter the restroom 129 | sleep(sleep_time * 0.6) # wait for 60% the time a person spends in the restroom 130 | if i == 0: # While the first person is in the restroom... 131 | janitor_thread.start() # the janitor would like to enter. HOWEVER... 132 | print("(%0.3f s) Janitor Dispatched" % (time() - t_start)) 133 | # Wait for all threads to finish 134 | for t in people_threads: 135 | t.join() 136 | janitor_thread.join() 137 | 138 | # The results will look like: 139 | """ 140 | Running Restroom Demo 141 | (0.000 s) Person 0 entered the restroom 142 | (0.061 s) Person 1 entered the restroom 143 | (0.100 s) Person 0 exited the restroom 144 | (0.122 s) Person 2 entered the restroom 145 | (0.162 s) Person 1 exited the restroom 146 | (0.182 s) Person 3 entered the restroom 147 | (0.222 s) Person 2 exited the restroom 148 | (0.243 s) Person 4 entered the restroom 149 | (0.282 s) Person 3 exited the restroom 150 | (0.343 s) Person 4 exited the restroom 151 | (0.343 s) Janitor entered the restroom 152 | (0.443 s) Janitor exited the restroom 153 | """ 154 | # Note that multiple people can be in the restroom. 155 | # If people kept using the restroom, the Janitor would never be able 156 | # to enter (technically known as thread starvation). 157 | # If this is undesired for your application, look at RWLock 158 | 159 | return res 160 | 161 | 162 | if __name__ == "__main__": 163 | print("Running Restroom Demo") 164 | bathroom_example() 165 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # lox documentation build configuration file, created by 5 | # sphinx-quickstart on Fri Jun 9 13:47:02 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | import doctest 17 | 18 | # If extensions (or modules to document with autodoc) are in another 19 | # directory, add these directories to sys.path here. If the directory is 20 | # relative to the documentation root, use os.path.abspath to make it 21 | # absolute, like shown here. 22 | # 23 | import os 24 | import sys 25 | 26 | sys.path.insert(0, os.path.abspath("..")) 27 | 28 | import lox 29 | 30 | # -- General configuration --------------------------------------------- 31 | 32 | # If your documentation needs a minimal Sphinx version, state it here. 33 | # 34 | # needs_sphinx = '1.0' 35 | 36 | # Add any Sphinx extension module names here, as strings. They can be 37 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 38 | extensions = [ 39 | "sphinx.ext.autodoc", 40 | "sphinx.ext.viewcode", 41 | "sphinx.ext.napoleon", 42 | "sphinx.ext.doctest", 43 | ] 44 | 45 | napoleon_google_docstring = True 46 | napoleon_numpy_docstring = True 47 | napoleon_include_init_with_doc = False 48 | napoleon_include_private_with_doc = False 49 | napoleon_include_special_with_doc = True 50 | napoleon_use_admonition_for_examples = False 51 | napoleon_use_admonition_for_notes = False 52 | napoleon_use_admonition_for_references = False 53 | napoleon_use_ivar = False 54 | napoleon_use_param = True 55 | napoleon_use_rtype = True 56 | 57 | doctest_global_setup = """ 58 | import lox 59 | """ 60 | 61 | doctest.ELLIPSIS_MARKER = "-ignore-" # monkeypatch to play nicely with sphinx 62 | 63 | # doctest_default_flags = doctest.ELLIPSIS 64 | 65 | # Add any paths that contain templates here, relative to this directory. 66 | templates_path = ["_templates"] 67 | 68 | # The suffix(es) of source filenames. 69 | # You can specify multiple suffix as a list of string: 70 | # 71 | # source_suffix = ['.rst', '.md'] 72 | source_suffix = ".rst" 73 | 74 | # The master toctree document. 75 | master_doc = "index" 76 | 77 | # General information about the project. 78 | project = "lox" 79 | copyright = "2019, Brian Pugh" 80 | author = "Brian Pugh" 81 | 82 | # The version info for the project you're documenting, acts as replacement 83 | # for |version| and |release|, also used in various other places throughout 84 | # the built documents. 85 | # 86 | # The short X.Y version. 87 | version = lox.__version__ 88 | # The full version, including alpha/beta/rc tags. 89 | release = lox.__version__ 90 | 91 | # The language for content autogenerated by Sphinx. Refer to documentation 92 | # for a list of supported languages. 93 | # 94 | # This is also used if you do content translation via gettext catalogs. 95 | # Usually you set "language" from the command line for these cases. 96 | language = None 97 | 98 | # List of patterns, relative to source directory, that match files and 99 | # directories to ignore when looking for source files. 100 | # This patterns also effect to html_static_path and html_extra_path 101 | exclude_patterns = ["_build", "_templates", "Thumbs.db", ".DS_Store", ".gitkeep"] 102 | 103 | # The name of the Pygments (syntax highlighting) style to use. 104 | pygments_style = "sphinx" 105 | 106 | # If true, `todo` and `todoList` produce output, else they produce nothing. 107 | todo_include_todos = False 108 | 109 | 110 | # -- Options for HTML output ------------------------------------------- 111 | 112 | # The theme to use for HTML and HTML Help pages. See the documentation for 113 | # a list of builtin themes. 114 | # 115 | html_theme = "sphinx_rtd_theme" 116 | 117 | # Theme options are theme-specific and customize the look and feel of a 118 | # theme further. For a list of options available for each theme, see the 119 | # documentation. 120 | # 121 | html_theme_options = { 122 | "canonical_url": "", 123 | "logo_only": True, 124 | "prev_next_buttons_location": "bottom", 125 | "style_external_links": False, 126 | "style_nav_header_background": "white", 127 | # Toc options 128 | "collapse_navigation": True, 129 | "sticky_navigation": True, 130 | "navigation_depth": 4, 131 | "includehidden": True, 132 | "titles_only": False, 133 | } 134 | 135 | html_logo = "../assets/lox_200w.png" 136 | html_favicon = "../assets/logo.ico" 137 | 138 | # Add any paths that contain custom static files (such as style sheets) here, 139 | # relative to this directory. They are copied after the builtin static files, 140 | # so a file named "default.css" will overwrite the builtin "default.css". 141 | html_static_path = ["_static"] 142 | 143 | 144 | # -- Options for HTMLHelp output --------------------------------------- 145 | 146 | # Output file base name for HTML help builder. 147 | htmlhelp_basename = "loxdoc" 148 | 149 | 150 | # -- Options for LaTeX output ------------------------------------------ 151 | 152 | latex_elements = { 153 | # The paper size ('letterpaper' or 'a4paper'). 154 | # 155 | # 'papersize': 'letterpaper', 156 | # The font size ('10pt', '11pt' or '12pt'). 157 | # 158 | # 'pointsize': '10pt', 159 | # Additional stuff for the LaTeX preamble. 160 | # 161 | # 'preamble': '', 162 | # Latex figure (float) alignment 163 | # 164 | # 'figure_align': 'htbp', 165 | } 166 | 167 | # Grouping the document tree into LaTeX files. List of tuples 168 | # (source start file, target name, title, author, documentclass 169 | # [howto, manual, or own class]). 170 | latex_documents = [ 171 | (master_doc, "lox.tex", "lox Documentation", "Brian Pugh", "manual"), 172 | ] 173 | 174 | 175 | # -- Options for manual page output ------------------------------------ 176 | 177 | # One entry per manual page. List of tuples 178 | # (source start file, name, description, authors, manual section). 179 | man_pages = [(master_doc, "lox", "lox Documentation", [author], 1)] 180 | 181 | 182 | # -- Options for Texinfo output ---------------------------------------- 183 | 184 | # Grouping the document tree into Texinfo files. List of tuples 185 | # (source start file, target name, title, author, 186 | # dir menu entry, description, category) 187 | texinfo_documents = [ 188 | ( 189 | master_doc, 190 | "lox", 191 | "lox Documentation", 192 | author, 193 | "lox", 194 | "Threading made easy.", 195 | "Miscellaneous", 196 | ), 197 | ] 198 | -------------------------------------------------------------------------------- /lox/queue/funnel.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: funnel 3 | :synopsis: Wait on many queues. 4 | 5 | Wait on a queue until a set of inputs are ready. 6 | """ 7 | 8 | import logging as log 9 | import queue 10 | import threading 11 | from collections import deque 12 | 13 | __all__ = [ 14 | "Funnel", 15 | "FunnelPutError", 16 | "FunnelPutTopError", 17 | ] 18 | 19 | 20 | class FunnelPutError(Exception): 21 | """Cannot ``put`` to a top/main Funnel object; 22 | can only ``put`` to subscribers. 23 | """ 24 | 25 | 26 | class FunnelPutTopError(Exception): 27 | """Can only put onto subscribers, not the top/main ``Funnel``.""" 28 | 29 | 30 | class FunnelElement: 31 | def __init__(self, item=None, complete=False): 32 | """ 33 | Parameters 34 | ---------- 35 | item 36 | User data 37 | complete : bool 38 | True if item contains valid data. 39 | """ 40 | 41 | self.item = item 42 | self.complete = complete 43 | 44 | def set(self, item): 45 | self.item = item 46 | self.complete = True 47 | 48 | 49 | class Funnel: 50 | """Wait on many queues. 51 | 52 | .. doctest:: 53 | 54 | >>> funnel = lox.Funnel() 55 | >>> sub_1 = funnel.subscribe() 56 | >>> sub_2 = funnel.subscribe() 57 | 58 | >>> sub_1.put("foo", "job_id") 59 | >>> try: 60 | ... res = funnel.get(timeout=0.01) 61 | ... except queue.Empty: 62 | ... print("Timed Out") 63 | ... 64 | Timed Out 65 | >>> sub_2.put("bar", "job_id") 66 | >>> res = funnel.get() 67 | >>> print(len(res)) 68 | 3 69 | >>> print(res) 70 | ['job_id','foo','bar'] 71 | 72 | 73 | Attributes 74 | ---------- 75 | index : int 76 | Index into list of solutions (if a subscriber). ``-1`` otherwise. 77 | Note: if ``get(return_jid=True)`` then this is offset by one. 78 | """ 79 | 80 | def __init__(self): 81 | """Create a Funnel Object""" 82 | 83 | self.lock = threading.RLock() 84 | self.index = -1 # Index into list of solutions 85 | self.subscribers = deque() 86 | self.d = {} 87 | self.q = queue.Queue() # Object that complete sets are placed on. 88 | 89 | @classmethod 90 | def _clone(cls, funnel): 91 | """Create a new ``Funnel`` object that shares subscribers and resources 92 | with an existing ``Funnel``. 93 | 94 | Only difference is it's index is 1 higher than the funnel's last subscriber. 95 | 96 | Parameters 97 | ---------- 98 | funnel : lox.Funnel 99 | ``Funnel`` object to clone from 100 | 101 | Returns 102 | ------- 103 | Funnel 104 | New Funnel object. 105 | """ 106 | 107 | new_funnel = cls() 108 | new_funnel.lock = funnel.lock 109 | 110 | with new_funnel.lock: 111 | new_funnel.d = funnel.d 112 | new_funnel.q = funnel.q 113 | new_funnel.subscribers = funnel.subscribers 114 | new_funnel.index = len(funnel) 115 | log.debug("new_funnel.index == %d" % new_funnel.index) 116 | new_funnel.subscribers.append(new_funnel) 117 | 118 | return new_funnel 119 | 120 | def __len__( 121 | self, 122 | ): 123 | """Return number of input queues. 124 | 125 | Returns 126 | ------- 127 | int 128 | Number of input queues. 129 | """ 130 | 131 | return len(self.subscribers) 132 | 133 | def subscribe( 134 | self, 135 | ): 136 | """Create a new Funnel for data to be ``put`` on. 137 | 138 | Returns 139 | ------- 140 | Funnel 141 | A funnel object that is a required input on ``get`` calls. 142 | """ 143 | 144 | new_funnel = Funnel._clone(self) 145 | return new_funnel 146 | 147 | def put( 148 | self, 149 | item, 150 | jid, 151 | blocking=True, 152 | timeout=-1, 153 | ): 154 | """ 155 | Parameters 156 | ---------- 157 | item 158 | data to put onto all subscribers' queues 159 | 160 | jid : hashable 161 | unique identifier for job. 162 | 163 | block : bool 164 | Block until data is put on queues or timeout. 165 | 166 | timeout : float 167 | Wait up to ``timeout`` seconds before raising ``queue.Full``. 168 | Defaults to no timeout. 169 | 170 | Returns 171 | ------- 172 | bool 173 | True if item was successfully added; False otherwise. 174 | 175 | Raises 176 | ------ 177 | FunnelPutTopError 178 | Can only put onto subscribers, not the top/main ``Funnel``. 179 | """ 180 | 181 | if self.index < 0: 182 | raise FunnelPutTopError 183 | 184 | if not self.lock.acquire(blocking=blocking, timeout=timeout): 185 | return False 186 | 187 | # put Item into dict's key's deque 188 | val = self.d.get(jid) 189 | if val is None: 190 | self.d[jid] = deque() 191 | 192 | for _ in range(len(self) - len(self.d[jid])): 193 | # Add a empty, incomplete funnel elements until it matches 194 | # the number of input queues. 195 | self.d[jid].append(FunnelElement()) 196 | 197 | self.d[jid][self.index].set(item) 198 | 199 | # Add item to queue if all subscribers are accounted for. 200 | if all(elem.complete for elem in self.d[jid]): 201 | log.debug( 202 | 'JID "%s" complete; putting onto queue %s' % (str(jid), str(self.q)) 203 | ) 204 | self.q.put(tuple([jid] + [elem.item for elem in self.d[jid]])) 205 | del self.d[jid] 206 | 207 | self.lock.release() 208 | return True 209 | 210 | def get(self, block=True, timeout=None, return_jid=True): 211 | """Get from the receive queue. Will return the contents of each 212 | input queue in the order subscribed as a tuple 213 | 214 | Parameters 215 | ---------- 216 | block : bool 217 | Block until data is obtained from receive queue or timeout. 218 | 219 | timeout : float 220 | Wait up to ``timeout`` seconds before raising ``queue.Full``. 221 | Defaults to no timeout. 222 | 223 | return_jid : bool 224 | Have the Job ID as the first element of the returned tuple. 225 | Defaults to True 226 | 227 | Returns 228 | ------- 229 | tuple 230 | items from input queues. 231 | 232 | Raises 233 | ------ 234 | queue.Empty 235 | When there are no elements in queue and timeout has been reached. 236 | """ 237 | items = self.q.get(block=block, timeout=timeout) 238 | 239 | if return_jid: 240 | return items 241 | else: 242 | return items[1:] 243 | -------------------------------------------------------------------------------- /tests/lock/test_RWLock.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from collections import deque 3 | from copy import copy 4 | from time import sleep, time 5 | 6 | from lox import RWLock 7 | 8 | SLEEP_TIME = 0.01 9 | N_WORKERS = 5 10 | 11 | rw_lock = None 12 | resource = None 13 | resp = None 14 | 15 | 16 | def common_setup(): 17 | global rw_lock, resp 18 | rw_lock = RWLock() 19 | resp = deque() 20 | 21 | 22 | def common_create_workers(func, n, *args): 23 | threads = [] 24 | for _ in range(n): 25 | t = threading.Thread(target=func, args=args) 26 | threads.append(t) 27 | 28 | t_start = time() 29 | for t in threads: 30 | t.start() 31 | 32 | return threads, t_start 33 | 34 | 35 | def read_worker(): 36 | global rw_lock, resp 37 | with rw_lock("r"): 38 | local_copy = copy(resource) 39 | sleep(SLEEP_TIME) 40 | 41 | resp.append(local_copy) 42 | return 43 | 44 | 45 | def write_worker(val): 46 | global rw_lock, resource 47 | with rw_lock("w"): 48 | resource = val 49 | return 50 | 51 | 52 | def test_RWLock_r(): 53 | global rw_lock, resource, resp 54 | common_setup() 55 | resource = 0 56 | 57 | threads, t_start = common_create_workers(read_worker, N_WORKERS) 58 | for t in threads: 59 | t.join() 60 | t_end = time() 61 | t_diff = t_end - t_start 62 | 63 | assert N_WORKERS > 2 64 | # for this to be true, readers have to access at same time (good) 65 | assert t_diff < (N_WORKERS - 1) * SLEEP_TIME 66 | for r in resp: 67 | assert r == resource 68 | 69 | 70 | def test_RWLock_w(): 71 | global rw_lock, resource 72 | common_setup() 73 | resource = 0 74 | new_val = 5 75 | 76 | threads_w1, t_start_w1 = common_create_workers(write_worker, 1, new_val) 77 | 78 | for t in threads_w1: 79 | t.join() 80 | 81 | assert resource == new_val 82 | 83 | 84 | def test_RWLock_rw(): 85 | global rw_lock, resource, resp 86 | common_setup() 87 | 88 | resource = 0 89 | soln = [0] * N_WORKERS + [ 90 | 5, 91 | ] * N_WORKERS 92 | 93 | threads_r1, _t_start_r1 = common_create_workers(read_worker, N_WORKERS) 94 | threads_w1, _t_start_w1 = common_create_workers(write_worker, N_WORKERS, 5) 95 | threads_r2, _t_start_r2 = common_create_workers(read_worker, N_WORKERS) 96 | 97 | for t in threads_r1: 98 | t.join() 99 | for t in threads_w1: 100 | t.join() 101 | for t in threads_r2: 102 | t.join() 103 | 104 | for r, s in zip(resp, soln): 105 | assert r == s 106 | 107 | 108 | def test_RWLock_timeout(): 109 | lock = RWLock() 110 | 111 | assert lock.acquire("r", timeout=0.01) is True 112 | assert lock.acquire("w", timeout=0.01) is False 113 | assert lock.acquire("r", timeout=0.01) is True 114 | lock.release("r") 115 | lock.release("r") 116 | 117 | assert lock.acquire("w", timeout=0.01) is True 118 | assert lock.acquire("w", timeout=0.01) is False 119 | assert lock.acquire("r", timeout=0.01) is False 120 | lock.release("w") 121 | 122 | 123 | def test_bathroom_example(): 124 | # Note: after the janitor exits, the remaining people are nondeterministic 125 | sol = [ 126 | "p_0_enter", 127 | "p_0_exit", 128 | "j_enter", 129 | "j_exit", 130 | ] 131 | 132 | res = bathroom_example()[:4] 133 | 134 | for r, s in zip(res, sol): 135 | assert r == s 136 | 137 | 138 | def bathroom_example(): 139 | """ 140 | Scenario: 141 | A janitor needs to clean a restroom, but is not allowed to enter until 142 | all people are out of the restroom. How do we implement this? 143 | """ 144 | restroom = RWLock() 145 | res = [] 146 | n_people = 5 147 | sleep_time = 0.1 148 | 149 | def janitor(): 150 | with restroom("w"): # block until the restroom is no longer occupied 151 | res.append("j_enter") 152 | print("(%0.3f s) Janitor entered the restroom" % (time() - t_start,)) 153 | sleep(sleep_time) # clean the restroom 154 | res.append("j_exit") 155 | print("(%0.3f s) Janitor exited the restroom" % (time() - t_start,)) 156 | 157 | def people(id): 158 | if id == 0: # Get the starting time of execution for display purposes 159 | global t_start 160 | t_start = time() 161 | with restroom("r"): # block if a janitor is in the restroom 162 | res.append("p_%d_enter" % (id,)) 163 | print( 164 | "(%0.3f s) Person %d entered the restroom" 165 | % ( 166 | time() - t_start, 167 | id, 168 | ) 169 | ) 170 | sleep(sleep_time) # use the restroom 171 | res.append("p_%d_exit" % (id,)) 172 | print( 173 | "(%0.3f s) Person %d exited the restroom" 174 | % ( 175 | time() - t_start, 176 | id, 177 | ) 178 | ) 179 | 180 | people_threads = [ 181 | threading.Thread(target=people, args=(i,)) for i in range(n_people) 182 | ] 183 | janitor_thread = threading.Thread(target=janitor) 184 | 185 | for i, person in enumerate(people_threads): 186 | person.start() # Person i will now attempt to enter the restroom 187 | sleep(sleep_time * 0.6) # wait for 60% the time a person spends in the restroom 188 | if i == 0: # While the first person is in the restroom... 189 | janitor_thread.start() # the janitor would like to enter. HOWEVER... 190 | # A new person (until all n_people are done) enters every 0.5 seconds. 191 | # Wait for all threads to finish 192 | for t in people_threads: 193 | t.join() 194 | janitor_thread.join() 195 | 196 | # The results will look like: 197 | """ 198 | Running Restroom Demo 199 | (0.000 s) Person 0 entered the restroom 200 | (0.100 s) Person 0 exited the restroom 201 | (0.101 s) Janitor entered the restroom 202 | (0.201 s) Janitor exited the restroom 203 | (0.201 s) Person 1 entered the restroom 204 | (0.202 s) Person 2 entered the restroom 205 | (0.202 s) Person 3 entered the restroom 206 | (0.243 s) Person 4 entered the restroom 207 | (0.302 s) Person 1 exited the restroom 208 | (0.302 s) Person 2 exited the restroom 209 | (0.303 s) Person 3 exited the restroom 210 | (0.343 s) Person 4 exited the restroom 211 | """ 212 | # While Person 0 is in the restroom, Janitor is waiting to enter (at ~0.500 s). 213 | # While the Janitor is waiting, he doesn't let anyone else into the room. 214 | # After Person 0, leaves the room, the Janitor enters. 215 | # After cleaning, the Janitor leaves at the 2.000 second mark. 216 | # Ever since the janitor was waiting (at 0.500 s), Person 1, Person 2, 217 | # Person 3, and Person 4 have been lining up to enter. 218 | # Since the Janitor left the restroom, all waiting people go in at the same time. 219 | return res 220 | 221 | 222 | if __name__ == "__main__": 223 | print("Running Restroom Demo") 224 | bathroom_example() 225 | -------------------------------------------------------------------------------- /tests/worker/test_process.py: -------------------------------------------------------------------------------- 1 | from random import random 2 | from time import sleep, time 3 | 4 | import pytest 5 | 6 | import lox 7 | 8 | SLEEP_TIME = 0.01 9 | N_WORKERS = 2 10 | 11 | 12 | @pytest.mark.requires_pathos 13 | def test_basic_args(): 14 | in_x = [ 15 | 1, 16 | 2, 17 | 3, 18 | 4, 19 | 5, 20 | 6, 21 | 7, 22 | 8, 23 | 9, 24 | 10, 25 | 11, 26 | 12, 27 | ] 28 | in_y = [ 29 | 13, 30 | 14, 31 | 15, 32 | 16, 33 | 17, 34 | 18, 35 | 19, 36 | 20, 37 | 21, 38 | 22, 39 | 23, 40 | 24, 41 | ] 42 | 43 | @lox.process(N_WORKERS) # specifying maximum number of processes 44 | def worker_task(x, y): 45 | sleep(SLEEP_TIME) 46 | return x * y 47 | 48 | # Vanilla function execution still works 49 | assert 10 == worker_task(2, 5) 50 | 51 | assert len(worker_task) == 0 52 | print("test_process: %s" % (worker_task)) 53 | 54 | for _ in range(2): 55 | for x, y in zip(in_x, in_y): 56 | worker_task.scatter(x, y) 57 | 58 | res = worker_task.gather() 59 | assert len(res) == len(in_x) 60 | 61 | for r, x, y in zip(res, in_x, in_y): 62 | assert (x * y) == r 63 | 64 | 65 | @pytest.mark.requires_pathos 66 | def test_basic_noargs(): 67 | in_x = [ 68 | 1, 69 | 2, 70 | 3, 71 | 4, 72 | 5, 73 | 6, 74 | 7, 75 | 8, 76 | 9, 77 | 10, 78 | 11, 79 | 12, 80 | ] 81 | in_y = [ 82 | 13, 83 | 14, 84 | 15, 85 | 16, 86 | 17, 87 | 18, 88 | 19, 89 | 20, 90 | 21, 91 | 22, 92 | 23, 93 | 24, 94 | ] 95 | 96 | @lox.process # maximum number of processes = # cores 97 | def worker_task(x, y): 98 | sleep(SLEEP_TIME) 99 | return x * y 100 | 101 | # Vanilla function execution still works 102 | assert 10 == worker_task(2, 5) 103 | 104 | assert len(worker_task) == 0 105 | print("test_process: %s" % (worker_task)) 106 | 107 | for _ in range(2): 108 | for x, y in zip(in_x, in_y): 109 | worker_task.scatter(x, y) 110 | 111 | res = worker_task.gather() 112 | assert len(res) == len(in_x) 113 | 114 | for r, x, y in zip(res, in_x, in_y): 115 | assert (x * y) == r 116 | 117 | 118 | class Class1: 119 | def __init__(self, z): 120 | self.z = z 121 | 122 | @lox.process(2) 123 | def test_method1(self, x, y): 124 | sleep(SLEEP_TIME) 125 | return x * y + self.z 126 | 127 | @lox.process 128 | def test_method2(self, x, y): 129 | sleep(SLEEP_TIME) 130 | return x * y + self.z 131 | 132 | 133 | @pytest.mark.requires_pathos 134 | def test_method_1(): 135 | in_x = [ 136 | 1, 137 | 2, 138 | 3, 139 | 4, 140 | 5, 141 | 6, 142 | 7, 143 | 8, 144 | 9, 145 | 10, 146 | 11, 147 | 12, 148 | ] 149 | in_y = [ 150 | 13, 151 | 14, 152 | 15, 153 | 16, 154 | 17, 155 | 18, 156 | 19, 157 | 20, 158 | 21, 159 | 22, 160 | 23, 161 | 24, 162 | ] 163 | z = 5 164 | 165 | test_obj = Class1(z) 166 | 167 | assert (2 * 5 + z) == test_obj.test_method1(2, 5) 168 | assert (2 * 5 + z) == test_obj.test_method2(2, 5) 169 | 170 | for i in range(2): 171 | for x, y in zip(in_x, in_y): 172 | test_obj.test_method1.scatter(x, y) 173 | res = test_obj.test_method1.gather() 174 | assert len(res) == len(in_x), f"iteration {i}" 175 | 176 | for r, x, y in zip(res, in_x, in_y): 177 | assert (x * y + z) == r 178 | 179 | for x, y in zip(in_x, in_y): 180 | test_obj.test_method2.scatter(x, y) 181 | res = test_obj.test_method2.gather() 182 | assert len(res) == len(in_x), f"iteration {i}" 183 | 184 | for r, x, y in zip(res, in_x, in_y): 185 | assert (x * y + z) == r 186 | 187 | 188 | @pytest.fixture 189 | def mock_tqdm(mocker): 190 | return mocker.patch("lox.worker.process.TQDM") 191 | 192 | 193 | @pytest.mark.requires_pathos 194 | def test_tqdm_bool(mock_tqdm): 195 | 196 | in_x = [ 197 | 1, 198 | 2, 199 | 3, 200 | 4, 201 | 5, 202 | 6, 203 | 7, 204 | 8, 205 | 9, 206 | 10, 207 | 11, 208 | 12, 209 | ] 210 | in_y = [ 211 | 13, 212 | 14, 213 | 15, 214 | 16, 215 | 17, 216 | 18, 217 | 19, 218 | 20, 219 | 21, 220 | 22, 221 | 23, 222 | 24, 223 | ] 224 | 225 | @lox.process(N_WORKERS) # specifying maximum number of processes 226 | def worker_task(x, y): 227 | sleep(random()) 228 | return x * y 229 | 230 | for x, y in zip(in_x, in_y): 231 | worker_task.scatter(x, y) 232 | res = worker_task.gather(tqdm=True) 233 | 234 | mock_tqdm.assert_called_once_with(total=len(in_x)) 235 | 236 | assert len(res) == len(in_x) 237 | 238 | for r, x, y in zip(res, in_x, in_y): 239 | assert (x * y) == r 240 | 241 | 242 | @pytest.mark.requires_pathos 243 | def test_tqdm_tqdm(mocker): 244 | in_x = [ 245 | 1, 246 | 2, 247 | 3, 248 | 4, 249 | 5, 250 | 6, 251 | 7, 252 | 8, 253 | 9, 254 | 10, 255 | 11, 256 | 12, 257 | ] 258 | in_y = [ 259 | 13, 260 | 14, 261 | 15, 262 | 16, 263 | 17, 264 | 18, 265 | 19, 266 | 20, 267 | 21, 268 | 22, 269 | 23, 270 | 24, 271 | ] 272 | 273 | @lox.process(N_WORKERS) # specifying maximum number of processes 274 | def worker_task(x, y): 275 | sleep(random()) 276 | return x * y 277 | 278 | pbar = mocker.MagicMock() 279 | for x, y in zip(in_x, in_y): 280 | worker_task.scatter(x, y) 281 | res = worker_task.gather(tqdm=pbar) 282 | 283 | n_update = 0 284 | for args, _kwargs in pbar.update.call_args_list: 285 | n_update += args[0] 286 | assert n_update == len(in_x) 287 | 288 | assert len(res) == len(in_x) 289 | 290 | for r, x, y in zip(res, in_x, in_y): 291 | assert (x * y) == r 292 | 293 | 294 | @pytest.mark.visual 295 | @pytest.mark.requires_pathos 296 | def test_tqdm_bool_visual(): 297 | """Primarily for visually asserting our tqdm mocks are correct.""" 298 | 299 | in_x = [ 300 | 1, 301 | 2, 302 | 3, 303 | 4, 304 | 5, 305 | 6, 306 | 7, 307 | 8, 308 | 9, 309 | 10, 310 | 11, 311 | 12, 312 | ] 313 | in_y = [ 314 | 13, 315 | 14, 316 | 15, 317 | 16, 318 | 17, 319 | 18, 320 | 19, 321 | 20, 322 | 21, 323 | 22, 324 | 23, 325 | 24, 326 | ] 327 | 328 | @lox.process(N_WORKERS) # specifying maximum number of processes 329 | def worker_task(x, y): 330 | sleep(random()) 331 | return x * y 332 | 333 | for x, y in zip(in_x, in_y): 334 | worker_task.scatter(x, y) 335 | res = worker_task.gather(tqdm=True) 336 | 337 | assert len(res) == len(in_x) 338 | 339 | for r, x, y in zip(res, in_x, in_y): 340 | assert (x * y) == r 341 | -------------------------------------------------------------------------------- /lox/worker/process.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: process 3 | :synopsis: Easily execute a function or method in multiple processes. 4 | 5 | Still allows the decorated function/method as normal. 6 | 7 | Example 8 | ------- 9 | .. doctest:: 10 | :skipif: True 11 | 12 | >>> import lox 13 | >>> 14 | >>> @lox.process(4) # Will operate with a maximum of 4 processes 15 | ... def foo(x, y): 16 | ... return x * y 17 | ... 18 | >>> foo(3, 4) 19 | 12 20 | >>> for i in range(5): 21 | ... foo.scatter(i, i + 1) 22 | ... 23 | >>> # foo is currently being executed in 4 processes 24 | >>> results = foo.gather() 25 | >>> print(results) 26 | [0, 2, 6, 12, 20] 27 | """ 28 | 29 | import os 30 | import threading 31 | from collections import deque 32 | from typing import Callable 33 | 34 | try: 35 | import pathos.multiprocessing as mp 36 | 37 | _PATHOS_AVAILABLE = True 38 | except ImportError: 39 | mp = None 40 | _PATHOS_AVAILABLE = False 41 | 42 | from ..debug import LOX_DEBUG 43 | 44 | try: 45 | from tqdm import tqdm as TQDM # to avoid argument namespace collisions. 46 | except ModuleNotFoundError: 47 | TQDM = None 48 | 49 | 50 | __all__ = [ 51 | "process", 52 | ] 53 | 54 | 55 | class ScatterGatherCallable: 56 | def __init__(self, fn, instance, executor, pending, n_workers): 57 | self._fn = fn 58 | self._instance = instance 59 | self._executor = executor 60 | self._pending = pending 61 | self._n_workers = n_workers 62 | self._results_thread_lock = threading.Lock() 63 | self._tqdm = None 64 | self._tqdm_pre_update = 0 # Stores updates before we have a tqdm object. 65 | 66 | def __call__(self, *args, **kwargs): 67 | if self._instance is not None: 68 | args = (self._instance,) + args 69 | return self._fn(*args, **kwargs) 70 | 71 | def scatter(self, *args, **kwargs): 72 | if not _PATHOS_AVAILABLE: 73 | raise ValueError( 74 | "pathos is not installed. " 75 | "Install the full package with: pip install lox[multiprocessing]" 76 | ) 77 | 78 | if self._instance is not None: 79 | args = (self._instance,) + args 80 | 81 | if self._executor[0] is None: 82 | # We create the pool here for greater serialization compatability 83 | self._executor[0] = mp.Pool(self._n_workers) 84 | 85 | def callback(res): 86 | # All of these are executed in a single "results" thread 87 | with self._results_thread_lock: 88 | if self._tqdm is not None: 89 | self._tqdm.update(1) 90 | else: 91 | self._tqdm_pre_update += 1 92 | 93 | fut = self._executor[0].apply_async( 94 | self._fn, args=args, kwds=kwargs, callback=callback 95 | ) 96 | self._pending.append(fut) 97 | 98 | return fut 99 | 100 | def gather(self, *, tqdm=None): 101 | if not _PATHOS_AVAILABLE: 102 | raise ValueError( 103 | "pathos is not installed. " 104 | "Install the full package with: pip install lox[multiprocessing]" 105 | ) 106 | 107 | if tqdm is not None: 108 | if TQDM is None: 109 | raise ModuleNotFoundError("No module named 'tqdm'") 110 | 111 | if isinstance(tqdm, bool) and tqdm: 112 | tqdm = TQDM(total=len(self._pending)) 113 | 114 | with self._results_thread_lock: 115 | self._tqdm = tqdm 116 | # Update the progressbar with all of the results from before 117 | # the TQDM object was declared. 118 | self._tqdm.update(self._tqdm_pre_update) 119 | 120 | self._executor[0].close() 121 | self._executor[0].join() 122 | fetched = [x.get() for x in self._pending] 123 | self._pending.clear() 124 | self._executor[0] = None 125 | return fetched 126 | 127 | 128 | class ScatterGatherDescriptor: 129 | def __init__(self, fn, n_workers): 130 | self._executor = [None] # Make a list to share a "pointer" 131 | self._n_workers = n_workers 132 | self._fn = fn 133 | self._pending = deque() 134 | self._base_callable = ScatterGatherCallable( 135 | self._fn, None, self._executor, self._pending, self._n_workers 136 | ) 137 | 138 | def __call__(self, *args, **kwargs): 139 | """ 140 | Vanilla passthrough function execution. Default user function behavior. 141 | 142 | Returns 143 | ------- 144 | Decorated function return type. 145 | Return of decorated function. 146 | """ 147 | 148 | return self._fn(*args, **kwargs) 149 | 150 | def __len__(self): 151 | """Return length of unprocessed job queue. 152 | 153 | Returns 154 | ------- 155 | Approximate length of unprocessed job queue. 156 | """ 157 | 158 | count = 0 159 | for res in self._pending: 160 | if not res.ready(): 161 | count += 1 162 | 163 | return count 164 | 165 | def __get__(self, instance, owner=None): 166 | if instance is None: 167 | return self 168 | return ScatterGatherCallable( 169 | self._fn, instance, self._executor, self._pending, self._n_workers 170 | ) 171 | 172 | def scatter(self, *args, **kwargs): 173 | """Enqueue a job to be processed by workers. 174 | Spin up workers if necessary. 175 | 176 | Return 177 | ------ 178 | concurrent.futures.Future 179 | """ 180 | if not _PATHOS_AVAILABLE: 181 | raise ValueError( 182 | "pathos is not installed. " 183 | "Install the full package with: pip install lox[multiprocessing]" 184 | ) 185 | 186 | return self._base_callable.scatter(*args, **kwargs) 187 | 188 | def gather(self, *args, **kwargs): 189 | """Block and collect results from prior ``scatter`` calls.""" 190 | if not _PATHOS_AVAILABLE: 191 | raise ValueError( 192 | "pathos is not installed. " 193 | "Install the full package with: pip install lox[multiprocessing]" 194 | ) 195 | 196 | results = self._base_callable.gather(*args, **kwargs) 197 | self._executor = None 198 | return results 199 | 200 | 201 | def process(n_workers) -> Callable[[Callable], ScatterGatherDescriptor]: 202 | """Decorate a function/method to execute in multiple processes. 203 | 204 | Example 205 | ------- 206 | .. doctest:: 207 | :skipif: True 208 | 209 | >>> import lox 210 | >>> 211 | >>> @lox.process(4) # Will operate with a maximum of 4 processes 212 | ... def foo(x, y): 213 | ... return x * y 214 | ... 215 | >>> foo(3, 4) 216 | 12 217 | >>> for i in range(5): 218 | ... foo.scatter(i, i + 1) 219 | ... 220 | >>> # foo is currently being executed in 4 processes 221 | >>> results = foo.gather() 222 | >>> print(results) 223 | [0, 2, 6, 12, 20] 224 | 225 | Parameters 226 | ---------- 227 | n_workers : int 228 | Number of process workers to invoke. Defaults to number of CPU cores. 229 | 230 | Methods 231 | ------- 232 | __call__( *args, **kwargs ) 233 | Vanilla passthrough function execution. Default user function behavior. 234 | 235 | Returns 236 | ------- 237 | Decorated function return type. 238 | Return of decorated function. 239 | __len__() 240 | Returns 241 | ------- 242 | int 243 | job queue length. 244 | scatter( *args, **kwargs ) 245 | Start a job executing ``func( *args, **kwargs )``. 246 | Workers are created and destroyed automatically. 247 | 248 | Returns 249 | ------- 250 | int 251 | Solution's index into the results obtained via ``gather()``. 252 | gather() 253 | Block until all jobs called via ``scatter()`` are complete. 254 | 255 | Returns 256 | ------- 257 | list 258 | Results in the order that scatter was invoked. 259 | """ # noqa: D214,D215,D410,D411 260 | 261 | # Support @process with no arguments. 262 | if callable(n_workers): 263 | return process(os.cpu_count())(n_workers) 264 | 265 | def decorator(fn: Callable) -> ScatterGatherDescriptor: 266 | return ScatterGatherDescriptor(fn, 0 if LOX_DEBUG else n_workers) 267 | 268 | return decorator 269 | 270 | 271 | if __name__ == "__main__": 272 | import doctest 273 | 274 | doctest.testmod() 275 | -------------------------------------------------------------------------------- /lox/worker/thread.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: thread 3 | :synopsis: Easily execute a function or method in multiple threads. 4 | 5 | Still allows the decorated function/method as normal. 6 | 7 | Example 8 | ------- 9 | .. doctest:: 10 | 11 | >>> import lox 12 | >>> 13 | >>> @lox.thread(4) # Will operate with a maximum of 4 threads 14 | ... def foo(x, y): 15 | ... return x * y 16 | ... 17 | >>> foo(3, 4) 18 | 12 19 | >>> for i in range(5): 20 | ... foo.scatter(i, i + 1) 21 | ... 22 | -ignore- 23 | >>> # foo is currently being executed in 4 threads 24 | >>> results = foo.gather() # block until results are ready 25 | >>> print(results) # Results are in the same order as scatter() calls 26 | [0, 2, 6, 12, 20] 27 | """ 28 | 29 | import concurrent.futures 30 | import threading 31 | from typing import Callable 32 | 33 | from ..debug import LOX_DEBUG 34 | 35 | try: 36 | from tqdm import tqdm as TQDM # to avoid argument namespace collisions. 37 | except ModuleNotFoundError: 38 | TQDM = None 39 | 40 | 41 | __all__ = [ 42 | "thread", 43 | ] 44 | 45 | 46 | class ScatterGatherCallable: 47 | def __init__(self, fn, instance, executor, pending, pending_lock): 48 | self._fn = fn 49 | self._instance = instance 50 | self._executor = executor 51 | self._pending = pending 52 | self._pending_lock = pending_lock 53 | 54 | def __call__(self, *args, **kwargs): 55 | if self._instance is not None: 56 | args = (self._instance,) + args 57 | return self._fn(*args, **kwargs) 58 | 59 | def scatter(self, *args, **kwargs): 60 | if self._instance is not None: 61 | args = (self._instance,) + args 62 | with self._pending_lock: 63 | if self._executor: 64 | fut = self._executor.submit(self._fn, *args, **kwargs) 65 | else: 66 | # Execute function in current thread 67 | fut = concurrent.futures.Future() 68 | fut.set_running_or_notify_cancel() 69 | try: 70 | res = self._fn(*args, **kwargs) 71 | except Exception as e: 72 | fut.set_exception(e) 73 | else: 74 | fut.set_result(res) 75 | self._pending.append(fut) 76 | return fut 77 | 78 | def gather(self, *, tqdm=None): 79 | """ 80 | Parameters 81 | ---------- 82 | tqdm : tqdm.tqdm or bool 83 | If ``True``, will internally create and update a default tqdm object. 84 | If ``tqdm``, initialized tqdm object to update with progress. 85 | """ 86 | 87 | pending = [] 88 | with self._pending_lock: 89 | pending[:] = self._pending 90 | self._pending[:] = [] 91 | 92 | if tqdm is not None: 93 | if TQDM is None: 94 | raise ModuleNotFoundError("No module named 'tqdm'") 95 | 96 | if isinstance(tqdm, bool) and tqdm: 97 | tqdm = TQDM(total=len(pending)) 98 | 99 | # We need a mutex for the tqdm object since multiple callbacks 100 | # can be called at the same time via different threads under 101 | # the same parent process. 102 | tqdm_mutex = threading.Lock() 103 | 104 | def tqdm_callback(fut): 105 | with tqdm_mutex: 106 | tqdm.update(1) 107 | 108 | [fut.add_done_callback(tqdm_callback) for fut in pending] 109 | 110 | output = [fut.result() for fut in pending] 111 | 112 | return output 113 | 114 | 115 | class ScatterGatherDescriptor: 116 | def __init__(self, fn, n_workers): 117 | """ 118 | Note: define self._executor in child class before calling this 119 | """ 120 | 121 | self._pending_lock = threading.Lock() 122 | self._fn = fn 123 | self._pending = [] 124 | 125 | if n_workers: 126 | self._executor = concurrent.futures.ThreadPoolExecutor( 127 | max_workers=n_workers 128 | ) 129 | else: 130 | # Disable threading; scatter calls are blocking; useful 131 | # for debugging multithreaded functions. 132 | self._executor = None 133 | 134 | self._base_callable = ScatterGatherCallable( 135 | self._fn, None, self._executor, self._pending, self._pending_lock 136 | ) 137 | 138 | def __call__(self, *args, **kwargs): 139 | """ 140 | Vanilla passthrough function execution. Default user function behavior. 141 | 142 | Returns 143 | ------- 144 | Decorated function return type. 145 | Return of decorated function. 146 | """ 147 | 148 | return self._fn(*args, **kwargs) 149 | 150 | def __len__(self): 151 | """Return length of unprocessed job queue. 152 | 153 | Returns 154 | ------- 155 | Approximate length of unprocessed job queue. 156 | """ 157 | 158 | if self._executor: 159 | return self._executor._work_queue.qsize() 160 | else: 161 | return 0 162 | 163 | def __get__(self, instance, owner=None): 164 | if instance is None: 165 | return self 166 | return ScatterGatherCallable( 167 | self._fn, instance, self._executor, self._pending, self._pending_lock 168 | ) 169 | 170 | def scatter(self, *args, **kwargs): 171 | """Enqueue a job to be processed by workers. 172 | Spin up workers if necessary. 173 | 174 | Return 175 | ------ 176 | concurrent.futures.Future 177 | """ 178 | 179 | return self._base_callable.scatter(*args, **kwargs) 180 | 181 | def gather(self, *args, **kwargs): 182 | """Block and collect results from prior ``scatter`` calls.""" 183 | 184 | return self._base_callable.gather(*args, **kwargs) 185 | 186 | 187 | def thread(n_workers) -> Callable[[Callable], ScatterGatherDescriptor]: 188 | """Decorate a function/method to execute in multiple threads. 189 | 190 | Example: 191 | 192 | .. doctest:: 193 | 194 | >>> import lox 195 | >>> 196 | >>> @lox.thread(4) # Will operate with a maximum of 4 threads 197 | ... def foo(x, y): 198 | ... return x * y 199 | ... 200 | >>> foo(3, 4) 201 | 12 202 | >>> for i in range(5): 203 | ... foo.scatter(i, i + 1) 204 | ... 205 | -ignore- 206 | >>> # foo is currently being executed in 4 threads 207 | >>> results = foo.gather() 208 | >>> print(results) 209 | [0, 2, 6, 12, 20] 210 | 211 | Multiple decorated functions can be chained together, each function drawing 212 | from their own pool of threads. Functions that return tuples will automatically 213 | unpack into the chained function. Positional arguments and keyword arguments 214 | can be passed in as they normally would. 215 | 216 | .. doctest:: 217 | >>> @lox.thread(2) # Will operate with a maximum of 2 threads 218 | ... def bar(x, y): 219 | ... return x + y 220 | ... 221 | 222 | >>> for i in range(5): 223 | ... foo_res = foo.scatter(i, i + 1) 224 | ... bar.scatter(foo_res, 10) # scatter will automatically 225 | ... # unpack the results of foo 226 | ... 227 | >>> 228 | >>> results = bar.gather() 229 | 230 | Parameters 231 | ---------- 232 | max_workers : int 233 | Maximum number of threads to invoke. 234 | When ``lox.thread`` is called without ``()``, the wrapped function 235 | a default number of max_workers is used (50). 236 | If set to 0, scatter calls will be executed in the parent thread. 237 | 238 | Methods 239 | ------- 240 | __call__( *args, **kwargs ) 241 | Vanilla passthrough function execution. Default user function behavior. 242 | 243 | Returns 244 | ------- 245 | Decorated function return type. 246 | Return of decorated function. 247 | 248 | __len__() 249 | Returns 250 | ------- 251 | int 252 | Current job queue length. Number of jobs that are currently waiting 253 | for an available worker. 254 | 255 | scatter( *args, **kwargs) 256 | Start a job executing decorated function ``func( *args, **kwargs )``. 257 | Workers are created and destroyed automatically. 258 | 259 | gather() 260 | Block until all jobs called via ``scatter()`` are complete. 261 | 262 | Returns 263 | ------- 264 | list 265 | Results in the order that scatter was invoked. 266 | """ # noqa: D214,D215,D410,D411 267 | 268 | # Support @thread with no arguments. 269 | if callable(n_workers): 270 | return thread(50)(n_workers) 271 | 272 | def decorator(fn: Callable) -> ScatterGatherDescriptor: 273 | return ScatterGatherDescriptor(fn, 0 if LOX_DEBUG else n_workers) 274 | 275 | return decorator 276 | 277 | 278 | if __name__ == "__main__": 279 | import doctest 280 | 281 | doctest.testmod() 282 | -------------------------------------------------------------------------------- /docs/examples.rst: -------------------------------------------------------------------------------- 1 | ======== 2 | Examples 3 | ======== 4 | 5 | Multithreading Requests 6 | ----------------------- 7 | 8 | A typical usecase for **lox** is the following. Say you wanted to get the content 9 | of websites from a list of URLs. The first naive implementation may look something 10 | like the following. 11 | 12 | .. doctest:: 13 | 14 | >>> import urllib.request 15 | >>> from time import time 16 | >>> urls = ["http://google.com", "http://bing.com", "http://yahoo.com"] 17 | >>> responses = [] 18 | >>> 19 | >>> def get_content(url): 20 | ... res = urllib.request.urlopen(url) 21 | ... return res.read() 22 | ... 23 | >>> 24 | >>> t_start = time() 25 | >>> for url in urls: 26 | ... responses.append(get_content(url)) 27 | ... 28 | >>> t_diff = time() - t_start 29 | >>> print("It took %.3f seconds to get 3 sites" % (t_diff,)) # doctest: +SKIP 30 | It took 2.942 seconds to get 3 sites 31 | 32 | It's nice, simple, and it just works. However, your computer is just idling while 33 | waiting for a network response. With **lox**, you can just decorate the function you 34 | want to add concurrency. We replace the direct calls to the function with ``func.scatter`` which will pass all the ``args`` and ``kwargs`` to the decorated function. Finally, when we need all the function results, we call ``func.gather()`` which will return a list of the outputs of the decorated function. The outputs are guarenteed to be in the same order that the ``scatter`` were called 35 | 36 | .. doctest:: 37 | 38 | >>> import lox 39 | >>> import urllib.request 40 | >>> from time import time 41 | >>> urls = ["http://google.com", "http://bing.com", "http://yahoo.com"] 42 | >>> 43 | >>> @lox.thread 44 | ... def get_content(url): 45 | ... res = urllib.request.urlopen(url) 46 | ... return res.read() 47 | ... 48 | >>> 49 | >>> t_start = time() 50 | >>> for url in urls: 51 | ... get_content.scatter(url) 52 | ... 53 | -ignore- 54 | >>> responses = get_content.gather() 55 | >>> t_diff = time() - t_start 56 | >>> print("It took %.3f seconds to get 3 sites" % (t_diff,)) # doctest: +SKIP 57 | It took 0.928 seconds to get 3 sites 58 | 59 | With minimal modifications, we now have a multithreaded application with 60 | significant performance improvements. 61 | 62 | 63 | Multiprocessing 64 | --------------- 65 | 66 | 67 | .. doctest:: 68 | :skipif: True 69 | 70 | >>> import lox 71 | >>> from time import sleep 72 | >>> 73 | >>> @lox.process(2) 74 | ... def job(x): 75 | ... sleep(1) 76 | ... return 1 77 | ... 78 | >>> 79 | >>> t_start = time() 80 | >>> for i in range(5): 81 | ... res = job(10) 82 | ... 83 | >>> t_diff = time() - t_start 84 | >>> print("Non-parallel took %.3f seconds" % (t_diff,)) # doctest: +SKIP 85 | Non-parallel took 5.007 seconds 86 | >>> 87 | >>> t_start = time() 88 | >>> for i in range(5): 89 | ... job.scatter(10) 90 | ... 91 | >>> res = job.gather() 92 | >>> t_diff = time() - t_start 93 | >>> print("Parallel took %.3f seconds" % (t_diff,)) # doctest: +SKIP 94 | Parallel took 0.062 seconds 95 | 96 | 97 | Obtaining a resource from a pool 98 | -------------------------------- 99 | 100 | Imagine you have 4 GPUs that are part of a data processing pipeline, and the 101 | GPUs perform the task disproportionally faster (or slower!) than the rest of the pipeline. 102 | Below we have many threads fetching and processing data, but they need to share 103 | the 4 GPUs for accelerated processing. 104 | 105 | .. doctest:: 106 | :skipif: True 107 | 108 | >>> import lox 109 | >>> 110 | >>> N_GPUS = 4 111 | >>> gpus = [allocate_gpu(x) for x in range(N_GPUS)] 112 | >>> idx_sem = lox.IndexSemaphore(N_GPUS) 113 | >>> 114 | >>> @lox.thread 115 | ... def process_task(url): 116 | ... data = get_data(url) 117 | ... data = preprocess_data(data) 118 | ... with idx_sem() as idx: # Obtains 0, 1, 2, or 3 119 | ... gpu = gpus[idx] 120 | ... result = gpu.process(data) 121 | ... result = postprocess_data(data) 122 | ... save_file(result) 123 | ... 124 | >>> 125 | >>> urls = [ 126 | ... "http://google.com", 127 | ... ] 128 | >>> for url in urls: 129 | ... process_task.scatter(url) 130 | ... 131 | >>> process_task.gather() 132 | 133 | Block until threads are done 134 | ---------------------------- 135 | 136 | Imagine the following scenario: 137 | 138 | A janitor needs to clean a restroom, but is not allowed to enter until 139 | all people are out of the restroom. How do we implement this? 140 | 141 | The easiest way is to use a **lox.LightSwitch**. The lightswitch pattern 142 | creates a first-in-last-out synchronization mechanism. 143 | The name of the pattern is inspired by people entering a 144 | room in the physical world. The first person to enter the room turns 145 | on the lights; then, when everyone is leaving, the last person to exit 146 | turns the lights off. 147 | 148 | .. doctest:: 149 | :skipif: True 150 | 151 | >>> restroom_occupied = Lock() 152 | >>> restroom = LightSwitch(restroom_occupied) 153 | >>> res = [] 154 | >>> n_people = 5 155 | 156 | A **LightSwitch** is most similar to a semaphore, but it automatically 157 | acquires/releases a provided **Lock** when it's internal counter 158 | increments/decrements from 0. A **LightSwitch** can be acquired multiple times, 159 | but must be released the same amount of times before the **Lock** gets released. 160 | 161 | Here's the janitor's job: 162 | 163 | .. doctest:: 164 | :skipif: True 165 | 166 | >>> @lox.thread(1) 167 | ... def janitor(): 168 | ... with restroom_occupied: # block until the restroom is no longer occupied 169 | ... res.append("j_enter") 170 | ... print("(%0.3f s) Janitor entered the restroom" % (time() - t_start,)) 171 | ... sleep(1) # clean the restroom 172 | ... res.append("j_exit") 173 | ... print("(%0.3f s) Janitor exited the restroom" % (time() - t_start,)) 174 | ... 175 | 176 | Here are the people trying to enter the rest room: 177 | 178 | .. doctest:: 179 | :skipif: True 180 | 181 | >>> @lox.thread(n_people) 182 | ... def people(id): 183 | ... if id == 0: # Get the starting time of execution for display purposes 184 | ... global t_start 185 | ... t_start = time() 186 | ... with restroom: # block if a janitor is in the restroom 187 | ... res.append("p_%d_enter" % (id,)) 188 | ... print( 189 | ... "(%0.3f s) Person %d entered the restroom" 190 | ... % ( 191 | ... time() - t_start, 192 | ... id, 193 | ... ) 194 | ... ) 195 | ... sleep(1) # use the restroom 196 | ... res.append("p_%d_exit" % (id,)) 197 | ... print( 198 | ... "(%0.3f s) Person %d exited the restroom" 199 | ... % ( 200 | ... time() - t_start, 201 | ... id, 202 | ... ) 203 | ... ) 204 | ... 205 | 206 | Lets start these people up: 207 | 208 | .. doctest:: 209 | :skipif: True 210 | 211 | >>> for i in range(n_people): 212 | ... people.scatter(i) # Person i will now attempt to enter the restroom 213 | ... sleep(0.6) # wait for 60% the time a person spends in the restroom 214 | ... if i == 0: # While the first person is in the restroom... 215 | ... janitor_thread.start() # the janitor would like to enter. HOWEVER... 216 | ... print("(%0.3f s) Janitor Dispatched" % (time() - t_start)) 217 | ... 218 | >>> # Wait for all threads to finish 219 | >>> people.gather() 220 | >>> janitor.gather() 221 | 222 | The results will look like:: 223 | 224 | Running Restroom Demo 225 | (0.000 s) Person 0 entered the restroom 226 | (0.061 s) Person 1 entered the restroom 227 | (0.100 s) Person 0 exited the restroom 228 | (0.122 s) Person 2 entered the restroom 229 | (0.162 s) Person 1 exited the restroom 230 | (0.182 s) Person 3 entered the restroom 231 | (0.222 s) Person 2 exited the restroom 232 | (0.243 s) Person 4 entered the restroom 233 | (0.282 s) Person 3 exited the restroom 234 | (0.343 s) Person 4 exited the restroom 235 | (0.343 s) Janitor entered the restroom 236 | (0.443 s) Janitor exited the restroom 237 | 238 | Note that multiple people can be in the restroom. 239 | If people kept using the restroom, the Janitor would never be able 240 | to enter (technically known as thread starvation). 241 | If this is undesired for your application, look at RWLock 242 | 243 | One-Writer-Many-Reader 244 | ---------------------- 245 | 246 | It's common that many threads may be reading from a single resource, but a 247 | single other thread may change the value of that resource. 248 | 249 | If we used a LightSwitch as in the Janitor example above, we can see that the 250 | writer (Janitor) may never get an opporunity to acquire the resource. A 251 | **RWLock** solves this problem by blocking future threads from acquiring the 252 | resource until the writer acquires and subsequently releases the resource. 253 | 254 | 255 | .. doctest:: 256 | :skipif: True 257 | 258 | >>> rwlock = lox.RWLock() 259 | 260 | The janitor task would do something like: 261 | 262 | .. doctest:: 263 | :skipif: True 264 | 265 | >>> with rwlock('w'): 266 | ... # Perform resource write here 267 | ... 268 | 269 | While the people task would look like 270 | 271 | .. doctest:: 272 | :skipif: True 273 | 274 | >>> with rwlock('r'): 275 | ... # Perform resource read here 276 | ... 277 | -------------------------------------------------------------------------------- /tests/worker/test_thread.py: -------------------------------------------------------------------------------- 1 | import logging as log 2 | import threading 3 | from random import random 4 | from time import sleep, time 5 | 6 | import pytest 7 | 8 | import lox 9 | 10 | SLEEP_TIME = 0.01 11 | N_WORKERS = 5 12 | 13 | 14 | @pytest.mark.parametrize( 15 | "n_workers", 16 | [ 17 | 0, 18 | N_WORKERS, 19 | ], 20 | ) 21 | def test_basic_args(n_workers): 22 | in_x = [ 23 | 1, 24 | 2, 25 | 3, 26 | 4, 27 | 5, 28 | 6, 29 | 7, 30 | 8, 31 | 9, 32 | 10, 33 | 11, 34 | 12, 35 | ] 36 | in_y = [ 37 | 13, 38 | 14, 39 | 15, 40 | 16, 41 | 17, 42 | 18, 43 | 19, 44 | 20, 45 | 21, 46 | 22, 47 | 23, 48 | 24, 49 | ] 50 | 51 | @lox.thread(n_workers) # specifying maximum number of threads 52 | def worker_task(x, y): 53 | sleep(SLEEP_TIME) 54 | return x * y 55 | 56 | # Vanilla function execution still works 57 | assert worker_task(2, 5) == 10 58 | 59 | assert len(worker_task) == 0 60 | 61 | for x, y in zip(in_x, in_y): 62 | worker_task.scatter(x, y) 63 | res = worker_task.gather() 64 | 65 | assert len(res) == len(in_x) 66 | 67 | for r, x, y in zip(res, in_x, in_y): 68 | assert (x * y) == r 69 | 70 | 71 | def test_basic_noargs(): 72 | in_x = [ 73 | 1, 74 | 2, 75 | 3, 76 | 4, 77 | 5, 78 | 6, 79 | 7, 80 | 8, 81 | 9, 82 | 10, 83 | 11, 84 | 12, 85 | ] 86 | in_y = [ 87 | 13, 88 | 14, 89 | 15, 90 | 16, 91 | 17, 92 | 18, 93 | 19, 94 | 20, 95 | 21, 96 | 22, 97 | 23, 98 | 24, 99 | ] 100 | 101 | @lox.thread # defaults to a max of 50 threads if none specified 102 | def worker_task(x, y): 103 | sleep(SLEEP_TIME) 104 | return x * y 105 | 106 | # Vanilla function execution still works 107 | assert 10 == worker_task(2, 5) 108 | 109 | assert len(worker_task) == 0 110 | 111 | for x, y in zip(in_x, in_y): 112 | worker_task.scatter(x, y) 113 | res = worker_task.gather() 114 | 115 | assert len(res) == len(in_x) 116 | 117 | for r, x, y in zip(res, in_x, in_y): 118 | assert (x * y) == r 119 | 120 | 121 | class Class1: 122 | def __init__(self, z): 123 | self.z = z 124 | 125 | @lox.thread(2) 126 | def test_method1(self, x, y): 127 | sleep(SLEEP_TIME) 128 | return x * y + self.z 129 | 130 | @lox.thread 131 | def test_method2(self, x, y): 132 | sleep(SLEEP_TIME) 133 | return x * y - self.z 134 | 135 | 136 | def test_method_1(): 137 | in_x = [ 138 | 1, 139 | 2, 140 | 3, 141 | 4, 142 | 5, 143 | 6, 144 | 7, 145 | 8, 146 | 9, 147 | 10, 148 | 11, 149 | 12, 150 | ] 151 | in_y = [ 152 | 13, 153 | 14, 154 | 15, 155 | 16, 156 | 17, 157 | 18, 158 | 19, 159 | 20, 160 | 21, 161 | 22, 162 | 23, 163 | 24, 164 | ] 165 | z1 = 5 166 | z2 = 5 167 | 168 | test_obj_1 = Class1(z1) 169 | test_obj_2 = Class1(z2) 170 | 171 | assert (2 * 5 + z1) == test_obj_1.test_method1(2, 5) 172 | assert (2 * 5 - z1) == test_obj_1.test_method2(2, 5) 173 | 174 | assert (2 * 5 + z2) == test_obj_2.test_method1(2, 5) 175 | assert (2 * 5 - z2) == test_obj_2.test_method2(2, 5) 176 | 177 | for _ in range(2): 178 | for x, y in zip(in_x, in_y): 179 | test_obj_1.test_method1.scatter(x, y) 180 | res = test_obj_1.test_method1.gather() 181 | assert len(res) == len(in_x) 182 | for r, x, y in zip(res, in_x, in_y): 183 | assert (x * y + z1) == r 184 | 185 | for x, y in zip(in_x, in_y): 186 | test_obj_2.test_method1.scatter(x, y) 187 | res = test_obj_2.test_method1.gather() 188 | assert len(res) == len(in_x) 189 | for r, x, y in zip(res, in_x, in_y): 190 | assert (x * y + z2) == r 191 | 192 | for x, y in zip(in_x, in_y): 193 | test_obj_1.test_method2.scatter(x, y) 194 | res = test_obj_1.test_method2.gather() 195 | assert len(res) == len(in_x) 196 | for r, x, y in zip(res, in_x, in_y): 197 | assert (x * y - z1) == r 198 | 199 | for x, y in zip(in_x, in_y): 200 | test_obj_2.test_method2.scatter(x, y) 201 | res = test_obj_2.test_method2.gather() 202 | assert len(res) == len(in_x) 203 | for r, x, y in zip(res, in_x, in_y): 204 | assert (x * y - z2) == r 205 | 206 | 207 | @pytest.fixture 208 | def mock_tqdm(mocker): 209 | return mocker.patch("lox.worker.thread.TQDM") 210 | 211 | 212 | def test_tqdm_bool(mock_tqdm): 213 | 214 | in_x = [ 215 | 1, 216 | 2, 217 | 3, 218 | 4, 219 | 5, 220 | 6, 221 | 7, 222 | 8, 223 | 9, 224 | 10, 225 | 11, 226 | 12, 227 | ] 228 | in_y = [ 229 | 13, 230 | 14, 231 | 15, 232 | 16, 233 | 17, 234 | 18, 235 | 19, 236 | 20, 237 | 21, 238 | 22, 239 | 23, 240 | 24, 241 | ] 242 | 243 | @lox.thread(N_WORKERS) # specifying maximum number of threads 244 | def worker_task(x, y): 245 | sleep(random()) 246 | return x * y 247 | 248 | for x, y in zip(in_x, in_y): 249 | worker_task.scatter(x, y) 250 | res = worker_task.gather(tqdm=True) 251 | 252 | mock_tqdm.assert_called_once_with(total=len(in_x)) 253 | 254 | assert len(res) == len(in_x) 255 | 256 | for r, x, y in zip(res, in_x, in_y): 257 | assert (x * y) == r 258 | 259 | 260 | def test_tqdm_tqdm(mocker): 261 | in_x = [ 262 | 1, 263 | 2, 264 | 3, 265 | 4, 266 | 5, 267 | 6, 268 | 7, 269 | 8, 270 | 9, 271 | 10, 272 | 11, 273 | 12, 274 | ] 275 | in_y = [ 276 | 13, 277 | 14, 278 | 15, 279 | 16, 280 | 17, 281 | 18, 282 | 19, 283 | 20, 284 | 21, 285 | 22, 286 | 23, 287 | 24, 288 | ] 289 | 290 | @lox.thread(N_WORKERS) # specifying maximum number of threads 291 | def worker_task(x, y): 292 | sleep(random()) 293 | return x * y 294 | 295 | pbar = mocker.MagicMock() 296 | for x, y in zip(in_x, in_y): 297 | worker_task.scatter(x, y) 298 | res = worker_task.gather(tqdm=pbar) 299 | 300 | assert pbar.update.call_count == len(in_x) 301 | 302 | assert len(res) == len(in_x) 303 | 304 | for r, x, y in zip(res, in_x, in_y): 305 | assert (x * y) == r 306 | 307 | 308 | @pytest.mark.visual 309 | def test_tqdm_bool_visual(): 310 | """Primarily for visually asserting our tqdm mocks are correct.""" 311 | 312 | in_x = [ 313 | 1, 314 | 2, 315 | 3, 316 | 4, 317 | 5, 318 | 6, 319 | 7, 320 | 8, 321 | 9, 322 | 10, 323 | 11, 324 | 12, 325 | ] 326 | in_y = [ 327 | 13, 328 | 14, 329 | 15, 330 | 16, 331 | 17, 332 | 18, 333 | 19, 334 | 20, 335 | 21, 336 | 22, 337 | 23, 338 | 24, 339 | ] 340 | 341 | @lox.thread(N_WORKERS) # specifying maximum number of threads 342 | def worker_task(x, y): 343 | sleep(random()) 344 | return x * y 345 | 346 | for x, y in zip(in_x, in_y): 347 | worker_task.scatter(x, y) 348 | res = worker_task.gather(tqdm=True) 349 | 350 | assert len(res) == len(in_x) 351 | 352 | for r, x, y in zip(res, in_x, in_y): 353 | assert (x * y) == r 354 | 355 | 356 | def test_non_decorator(): 357 | in_x = [ 358 | 1, 359 | 2, 360 | 3, 361 | 4, 362 | 5, 363 | 6, 364 | 7, 365 | 8, 366 | 9, 367 | 10, 368 | 11, 369 | 12, 370 | ] 371 | in_y = [ 372 | 13, 373 | 14, 374 | 15, 375 | 16, 376 | 17, 377 | 18, 378 | 19, 379 | 20, 380 | 21, 381 | 22, 382 | 23, 383 | 24, 384 | ] 385 | 386 | def worker_task_private(x, y): 387 | sleep(SLEEP_TIME) 388 | return x * y 389 | 390 | worker_task = lox.thread(worker_task_private) 391 | 392 | # Vanilla function execution still works 393 | assert worker_task(2, 5) == 10 394 | 395 | assert len(worker_task) == 0 396 | 397 | for x, y in zip(in_x, in_y): 398 | worker_task.scatter(x, y) 399 | res = worker_task.gather() 400 | 401 | assert len(res) == len(in_x) 402 | 403 | for r, x, y in zip(res, in_x, in_y): 404 | assert (x * y) == r 405 | 406 | 407 | def test_non_decorator_specify_num_workers(): 408 | in_x = [ 409 | 1, 410 | 2, 411 | 3, 412 | 4, 413 | 5, 414 | 6, 415 | 7, 416 | 8, 417 | 9, 418 | 10, 419 | 11, 420 | 12, 421 | ] 422 | in_y = [ 423 | 13, 424 | 14, 425 | 15, 426 | 16, 427 | 17, 428 | 18, 429 | 19, 430 | 20, 431 | 21, 432 | 22, 433 | 23, 434 | 24, 435 | ] 436 | 437 | def worker_task_private(x, y): 438 | sleep(SLEEP_TIME) 439 | return x * y 440 | 441 | worker_task = lox.thread(N_WORKERS)(worker_task_private) 442 | 443 | # Vanilla function execution still works 444 | assert worker_task(2, 5) == 10 445 | 446 | assert len(worker_task) == 0 447 | 448 | for x, y in zip(in_x, in_y): 449 | worker_task.scatter(x, y) 450 | res = worker_task.gather() 451 | 452 | assert len(res) == len(in_x) 453 | 454 | for r, x, y in zip(res, in_x, in_y): 455 | assert (x * y) == r 456 | -------------------------------------------------------------------------------- /lox/queue/announcement.py: -------------------------------------------------------------------------------- 1 | """ 2 | .. module:: Announcement 3 | :synopsis: Many-to-many queue recipients with backlog support for 4 | race condition subscribing or lazy subscribing. 5 | """ 6 | 7 | 8 | import logging as log 9 | import queue 10 | import threading 11 | from collections import deque 12 | 13 | __all__ = [ 14 | "Announcement", 15 | "SubscribeFinalizedError", 16 | ] 17 | 18 | 19 | class SubscribeFinalizedError(Exception): 20 | """Raised when an Announcement.subscribe has been called, 21 | but the Announcement has already been finalized. 22 | """ 23 | 24 | 25 | class Announcement: 26 | """Push to many queues with backlog support. 27 | 28 | Allows the pushing of data to many threads. 29 | 30 | Example 31 | ------- 32 | .. doctest:: 33 | 34 | >>> import lox 35 | >>> ann = lox.Announcement() 36 | >>> foo_q = ann.subscribe() 37 | >>> bar_q = ann.subscribe() 38 | >>> 39 | >>> @lox.thread 40 | ... def foo(): 41 | ... x = foo_q.get() 42 | ... return x 43 | ... 44 | >>> 45 | >>> @lox.thread 46 | ... def bar(): 47 | ... x = bar_q.get() 48 | ... return x**2 49 | ... 50 | >>> 51 | >>> ann.put(5) 52 | >>> foo.scatter() 53 | >>> foo_res = foo.gather() 54 | >>> bar.scatter() 55 | >>> bar_res = bar.gather() 56 | 57 | The backlog allows future (or potentially race-condition) subscribers to 58 | get content put'd before they subscribed. However, the user must be careful 59 | of memory consumption. 60 | """ 61 | 62 | @property 63 | def final( 64 | self, 65 | ): 66 | return self._final 67 | 68 | @final.setter 69 | def final(self, val: bool): 70 | with self.lock: 71 | self._final = val 72 | 73 | @property 74 | def backlog( 75 | self, 76 | ): 77 | """collections.deque: Backlog of queued data. None if not used.""" 78 | 79 | return self._backlog 80 | 81 | @backlog.setter 82 | def backlog(self, val): 83 | self._backlog = val 84 | 85 | def __repr__( 86 | self, 87 | ): 88 | with self.lock: 89 | qs = [str(id(ann.q)) for ann in self.subscribers] 90 | string = super().__repr__() + " with subscriber queues " + ",".join(qs) 91 | return string 92 | 93 | def __init__(self, maxsize=0, backlog=None): 94 | """Create an Announcement object. 95 | 96 | Parameters 97 | ---------- 98 | maxsize : int 99 | Created queue's default maximum size. Defaults to 0 (no size limit). 100 | 101 | backlog : int 102 | Create a log of previously put data. New subscribers get the entire 103 | backlog pushed onto their queue. Memory usage potentially unbounded. 104 | 105 | A value <=0 specifies unbounded backlog size. A value >0 specifies 106 | a circular buffer. 107 | 108 | Defaults to None (backlog not used). 109 | 110 | Note: If used, it is recommended to use the ``finalize()`` method so 111 | that the backlog can be free'd when no longer necessary. 112 | """ 113 | 114 | log.debug("Creating Announcement %X" % id(self)) 115 | 116 | # General Attributes # 117 | self.subscribers = deque() 118 | self.maxsize = maxsize 119 | self.lock = threading.RLock() 120 | self.q = queue.Queue(maxsize=self.maxsize) 121 | self.subscribers.append(self) 122 | 123 | # Backlog objects # 124 | self.backlog_use = False if backlog is None else True 125 | self.final = False 126 | if self.backlog_use: 127 | if backlog <= 0: 128 | backlog = None 129 | self.backlog = deque(maxlen=backlog) 130 | else: 131 | self.backlog = None 132 | 133 | @classmethod 134 | def clone(cls, ann, q: queue.Queue = None): 135 | """Create a new announcement object that shares subscribers and resources 136 | with an existing announcement. 137 | 138 | Only difference from cloned announcement is a new receive queue is created. 139 | 140 | Parameters 141 | ---------- 142 | ann : lox.Announcement 143 | Announcement object to clone from 144 | 145 | q : queue.Queue 146 | Receiving queue. If ``None``, a new one is created. 147 | 148 | Returns 149 | ------- 150 | Announcement 151 | New Announcement object with copied attributes, but new ``q`` 152 | """ 153 | 154 | new_ann = cls() 155 | new_ann.lock = ann.lock 156 | with new_ann.lock: 157 | new_ann.subscribers = ann.subscribers 158 | new_ann.maxsize = ann.maxsize 159 | new_ann.q = queue.Queue(maxsize=ann.maxsize) if q is None else q 160 | new_ann.backlog_use = ann.backlog_use 161 | new_ann.backlog = ann.backlog 162 | return new_ann 163 | 164 | def __len__( 165 | self, 166 | ): 167 | """Get the number of subscribers. 168 | 169 | Returns 170 | ------- 171 | int 172 | Number of subcribers. 173 | """ 174 | 175 | return len(self.subscribers) 176 | 177 | def qsize( 178 | self, 179 | ): 180 | """Return the approximate size of the receive queue. Note, qsize() > 0 181 | doesn’t guarantee that a subsequent get() will not block, nor will 182 | qsize() < maxsize guarantee that put() will not block. 183 | 184 | Returns 185 | ------- 186 | int 187 | approximate size of the receive queue. 188 | """ 189 | 190 | return self.q.qsize() 191 | 192 | def empty( 193 | self, 194 | ): 195 | """Return ``True`` if the receive queue is empty, ``False`` otherwise. 196 | If ``empty()`` returns ``True`` it doesn’t guarantee that a subsequent 197 | call to ``put()`` will not block. Similarly, if ``empty()`` returns 198 | ``False`` it doesn’t guarantee that a subsequent call to ``get()`` 199 | will not block. 200 | 201 | Returns 202 | ------- 203 | bool 204 | ``True`` if the receive queue is currently empty; ``False`` otherwise. 205 | """ 206 | 207 | return self.q.empty() 208 | 209 | def full( 210 | self, 211 | ): 212 | """Return ``True`` if the receive queue is full, ``False`` otherwise. 213 | If ``full()`` returns ``True`` it doesn’t guarantee that a subsequent 214 | call to ``get()`` will not block. Similarly, if ``full()`` returns 215 | ``False`` it doesn’t guarantee that a subsequent call to ``put()`` 216 | will not block. 217 | 218 | Returns 219 | ------- 220 | bool 221 | ``True`` if the receive queue is currently full; ``False`` otherwise. 222 | """ 223 | 224 | return self.q.full() 225 | 226 | def put( 227 | self, 228 | item, 229 | block=True, 230 | timeout=None, 231 | ): 232 | """Put item into all subscribers' queues. 233 | 234 | Parameters 235 | ---------- 236 | item 237 | data to put onto all subscribers' queues 238 | 239 | block : bool 240 | Block until data is put on queues or timeout. 241 | 242 | timeout : float 243 | Wait up to ``timeout`` seconds before raising ``queue.Full``. 244 | Defaults to no timeout. 245 | """ 246 | 247 | with self.lock: 248 | for ann in self.subscribers: 249 | if ann == self: 250 | log.debug("Skipping receiver Announcement %s" % ann) 251 | continue 252 | else: 253 | log.debug("Putting to Announcement %s" % str(ann)) 254 | 255 | ann.q.put(item, block=block, timeout=timeout) 256 | if self.backlog_use and not self.final: 257 | self.backlog.append(item) 258 | 259 | def get(self, block=True, timeout=None): 260 | """Get from the receive queue. 261 | 262 | Parameters 263 | ---------- 264 | block : bool 265 | Block until data is obtained from receive queue or timeout. 266 | 267 | timeout : float 268 | Wait up to ``timeout`` seconds before raising ``queue.Full``. 269 | Defaults to no timeout. 270 | 271 | Returns 272 | ------- 273 | item from receive queue. 274 | 275 | Raises 276 | ------ 277 | queue.Empty 278 | When there are no elements in queue and timeout has been reached. 279 | """ 280 | 281 | return self.q.get(block=block, timeout=timeout) 282 | 283 | def subscribe(self, q=None, maxsize=None, block=True, timeout=None): 284 | """Subscribe to announcements. 285 | 286 | Parameters 287 | ---------- 288 | q : Queue 289 | Existing queue to add to the subscribers' list. 290 | If not provided, a queue is created. 291 | 292 | maxsize : int 293 | Created queue's maximum size. Overrides Announcement's default 294 | maximum size. Ignored if ``q`` is provided. 295 | 296 | block : bool 297 | Block until data from backlog is put on queues or timeout. 298 | 299 | timeout : float 300 | Wait up to ``timeout`` seconds before raising ``queue.Full``. 301 | Defaults to no timeout. 302 | 303 | Returns 304 | ------- 305 | Announcement 306 | object for receiver to ``get`` and ``put`` data from. 307 | """ 308 | 309 | with self.lock: 310 | if self.final: 311 | log.error("%s Subscribe when announcement already finalized" % (self,)) 312 | raise SubscribeFinalizedError 313 | 314 | if q is None: 315 | # Create Queue 316 | if maxsize is None: 317 | maxsize = self.maxsize 318 | q = queue.Queue(maxsize=maxsize) 319 | ann = Announcement.clone(self, q) 320 | self.subscribers.append(ann) 321 | if self.backlog_use: 322 | for x in self.backlog: 323 | q.put(x, block=block, timeout=timeout) 324 | return ann 325 | 326 | def unsubscribe(self, q): 327 | """Remove the queue from queue-list. Will no longer receive announcements. 328 | 329 | Parameters 330 | ---------- 331 | q : Queue 332 | Queue object to remove. 333 | 334 | Raises 335 | ------ 336 | ValueError 337 | ``q`` was not a subscriber. 338 | """ 339 | 340 | with self.lock: 341 | self.subscribers.remove(q) 342 | 343 | def finalize( 344 | self, 345 | ): 346 | """Do not allow any more subscribers. 347 | 348 | Primarily used for memory efficiency if backlog is used. 349 | """ 350 | 351 | with self.lock: 352 | for ann in self.subscribers: 353 | log.debug("%s finalizing" % str(ann)) 354 | ann.final = True 355 | ann.backlog = None 356 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------