├── src └── culsans │ ├── py.typed │ ├── _utils.py │ ├── _exceptions.py │ ├── __init__.py │ ├── _proxies.py │ ├── _protocols.py │ └── _queues.py ├── .git_archival.txt ├── docs ├── changelog.rst ├── license.rst ├── installation.rst ├── index.rst ├── _static │ └── css │ │ └── custom.css ├── conf.py └── api.rst ├── REUSE.toml ├── LICENSES ├── 0BSD.txt ├── ISC.txt ├── PSF-2.0.txt ├── CC0-1.0.txt └── CC-BY-4.0.txt ├── .gitignore ├── tests └── culsans │ ├── test_base.py │ ├── test_mixed.py │ ├── test_asyncio.py │ └── test_queue.py ├── pyproject.toml ├── README.rst └── CHANGELOG.md /src/culsans/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.git_archival.txt: -------------------------------------------------------------------------------- 1 | node: ba009a4be9af28a3bf8466c48d8a226cf4cdd314 2 | node-date: 2025-12-18T02:57:10+04:00 3 | describe-name: 0.10.0-20-gba009a4 4 | -------------------------------------------------------------------------------- /docs/changelog.rst: -------------------------------------------------------------------------------- 1 | .. 2 | SPDX-FileCopyrightText: 2025 Ilya Egorov <0x42005e1f@gmail.com> 3 | SPDX-License-Identifier: CC-BY-4.0 4 | 5 | .. include:: ../CHANGELOG.md 6 | :parser: myst_parser.sphinx_ 7 | -------------------------------------------------------------------------------- /docs/license.rst: -------------------------------------------------------------------------------- 1 | .. 2 | SPDX-FileCopyrightText: 2025 Ilya Egorov <0x42005e1f@gmail.com> 3 | SPDX-License-Identifier: CC-BY-4.0 4 | 5 | License 6 | ======= 7 | 8 | .. include:: ../README.rst 9 | :start-after: .. license-start-marker 10 | :end-before: .. license-end-marker 11 | -------------------------------------------------------------------------------- /REUSE.toml: -------------------------------------------------------------------------------- 1 | version = 1 2 | 3 | # configuration files 4 | [[annotations]] 5 | path = [ 6 | ".github/workflows/python-publish.yml", 7 | ".gitattributes", 8 | ".pre-commit-config.yaml", 9 | ".readthedocs.yaml", 10 | "pyproject.toml", 11 | ] 12 | SPDX-FileCopyrightText = "2025 Ilya Egorov <0x42005e1f@gmail.com>" 13 | SPDX-License-Identifier = "CC0-1.0" 14 | 15 | # computer-generated files 16 | [[annotations]] 17 | path = [ 18 | ".git_archival.txt", 19 | ".gitignore", 20 | "uv.lock", 21 | ] 22 | precedence = "override" 23 | SPDX-FileCopyrightText = "NONE" 24 | SPDX-License-Identifier = "CC0-1.0" 25 | -------------------------------------------------------------------------------- /LICENSES/0BSD.txt: -------------------------------------------------------------------------------- 1 | Copyright (C) YEAR by AUTHOR EMAIL 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted. 4 | 5 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 6 | -------------------------------------------------------------------------------- /src/culsans/_utils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # SPDX-FileCopyrightText: 2025 Ilya Egorov <0x42005e1f@gmail.com> 4 | # SPDX-License-Identifier: ISC 5 | 6 | from __future__ import annotations 7 | 8 | from typing import TYPE_CHECKING 9 | 10 | if TYPE_CHECKING: 11 | import sys 12 | 13 | from typing import Any, TypeVar 14 | 15 | if sys.version_info >= (3, 9): # PEP 585 16 | from collections.abc import Callable 17 | else: 18 | from typing import Callable 19 | 20 | _CallableT = TypeVar("_CallableT", bound=Callable[..., Any]) 21 | 22 | 23 | def copydoc(wrapped: Callable[..., Any]) -> Callable[[_CallableT], _CallableT]: 24 | def decorator(function: _CallableT) -> _CallableT: 25 | function.__doc__ = wrapped.__doc__ 26 | 27 | return function 28 | 29 | return decorator 30 | -------------------------------------------------------------------------------- /LICENSES/ISC.txt: -------------------------------------------------------------------------------- 1 | ISC License: 2 | 3 | Copyright (c) 2004-2010 by Internet Systems Consortium, Inc. ("ISC") 4 | Copyright (c) 1995-2003 by Internet Software Consortium 5 | 6 | Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 9 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. 2 | SPDX-FileCopyrightText: 2025 Ilya Egorov <0x42005e1f@gmail.com> 3 | SPDX-License-Identifier: CC-BY-4.0 4 | 5 | Installation 6 | ============ 7 | 8 | .. include:: ../README.rst 9 | :start-after: .. installation-start-marker 10 | :end-before: .. installation-end-marker 11 | 12 | Third-party distributions 13 | ------------------------- 14 | 15 | Various third-parties provide culsans for their environments. 16 | 17 | piwheels 18 | ^^^^^^^^ 19 | 20 | culsans is also available via the `piwheels `__, a Python package repository which provides pre-compiled packages 22 | for the Raspberry Pi. Installation is similar to that from PyPI, but you will 23 | need to change your pip configuration or explicitly specify the required index 24 | according to the `FAQ `__. 25 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. 2 | SPDX-FileCopyrightText: 2025 Ilya Egorov <0x42005e1f@gmail.com> 3 | SPDX-License-Identifier: CC-BY-4.0 4 | 5 | ======= 6 | culsans 7 | ======= 8 | 9 | .. include:: ../README.rst 10 | :start-after: .. description-start-marker 11 | :end-before: .. description-end-marker 12 | 13 | Usage 14 | ===== 15 | 16 | .. include:: ../README.rst 17 | :start-after: .. usage-start-marker 18 | :end-before: .. usage-end-marker 19 | 20 | Compatibility 21 | ============= 22 | 23 | .. include:: ../README.rst 24 | :start-after: .. compatibility-start-marker 25 | :end-before: .. compatibility-end-marker 26 | 27 | .. toctree:: 28 | :maxdepth: 2 29 | :hidden: 30 | 31 | api 32 | 33 | .. toctree:: 34 | :caption: Quickstart 35 | :maxdepth: 3 36 | :hidden: 37 | 38 | installation 39 | 40 | .. toctree:: 41 | :caption: Meta 42 | :maxdepth: 1 43 | :hidden: 44 | 45 | changelog 46 | license 47 | 48 | .. toctree:: 49 | :caption: Links 50 | :hidden: 51 | 52 | DeepWiki 53 | GitHub 54 | PyPI 55 | -------------------------------------------------------------------------------- /src/culsans/_exceptions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # SPDX-FileCopyrightText: 2024 Ilya Egorov <0x42005e1f@gmail.com> 4 | # SPDX-License-Identifier: ISC 5 | 6 | import asyncio 7 | import queue 8 | import sys 9 | 10 | SyncQueueEmpty = queue.Empty 11 | AsyncQueueEmpty = asyncio.QueueEmpty 12 | 13 | 14 | class QueueEmpty(SyncQueueEmpty, AsyncQueueEmpty): 15 | """ 16 | Raised when non-blocking get/peek with empty queue. 17 | """ 18 | 19 | 20 | SyncQueueFull = queue.Full 21 | AsyncQueueFull = asyncio.QueueFull 22 | 23 | 24 | class QueueFull(SyncQueueFull, AsyncQueueFull): 25 | """ 26 | Raised when non-blocking put with full queue. 27 | """ 28 | 29 | 30 | if sys.version_info >= (3, 13): # python/cpython#96471 31 | SyncQueueShutDown = queue.ShutDown 32 | AsyncQueueShutDown = asyncio.QueueShutDown 33 | 34 | else: 35 | 36 | class SyncQueueShutDown(Exception): 37 | """ 38 | A backport of :class:`queue.ShutDown`. 39 | """ 40 | 41 | class AsyncQueueShutDown(Exception): 42 | """ 43 | A backport of :class:`asyncio.QueueShutDown`. 44 | """ 45 | 46 | 47 | class QueueShutDown(SyncQueueShutDown, AsyncQueueShutDown): 48 | """ 49 | Raised when put/get/peek with shut-down queue. 50 | """ 51 | 52 | 53 | class UnsupportedOperation(ValueError): 54 | """ 55 | Raised when peek/clear with non-peekable/non-clearable queue. 56 | """ 57 | -------------------------------------------------------------------------------- /docs/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | /* SPDX-FileCopyrightText: 2025 Ilya Egorov <0x42005e1f@gmail.com> */ 2 | /* SPDX-License-Identifier: CC0-1.0 */ 3 | 4 | .wy-side-nav-search .switch-menus { 5 | display: none !important; 6 | } 7 | 8 | .wy-menu.wy-menu-vertical a.reference.external::after { 9 | content: "\00a0\f08e"; 10 | display: inline-block; 11 | font-family: FontAwesome; 12 | } 13 | 14 | table.widetable { 15 | width: 100%; 16 | } 17 | 18 | table.widetable th, 19 | table.widetable td { 20 | text-align: center; 21 | } 22 | 23 | div.tab-set { 24 | margin: 1px 0 24px; 25 | } 26 | 27 | div.tab-set > label.tab-label { 28 | padding-top: 0; 29 | } 30 | 31 | div.tab-content > div[class^="highlight-"]:first-child, 32 | div.tab-content > div[class*=" highlight-"]:first-child { 33 | border-top: none; 34 | margin: 0; 35 | } 36 | 37 | div.tab-content table.widetable:first-child, 38 | div.tab-content table.widetable:first-child th { 39 | border-top: none; 40 | } 41 | 42 | dl.py:not(.class):has(dt.py.sig + dt.py.sig) { 43 | display: inline-grid !important; 44 | } 45 | 46 | dl.py:not(.class):has(dt.py.sig + dt.py.sig) > :last-child { 47 | margin-bottom: 0 !important; 48 | } 49 | 50 | dl.py > dt.py.sig:not(:last-of-type) { 51 | padding-bottom: 0 !important; 52 | margin-bottom: 0 !important; 53 | } 54 | 55 | dl.py > dt.py.sig:not(:first-of-type) { 56 | border-top: none !important; 57 | padding-top: 0 !important; 58 | margin-top: 0 !important; 59 | } 60 | 61 | dl.py.property { 62 | display: block !important; 63 | } 64 | 65 | dl.py.data dl.py.attribute span.sig-prename { 66 | display: none !important; 67 | } 68 | -------------------------------------------------------------------------------- /src/culsans/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # SPDX-FileCopyrightText: 2024 Ilya Egorov <0x42005e1f@gmail.com> 4 | # SPDX-License-Identifier: ISC 5 | 6 | """ 7 | Thread-safe async-aware queue for Python 8 | 9 | Mixed sync-async queue, supposed to be used for communicating between classic 10 | synchronous (threaded) code and asynchronous one, between two asynchronous 11 | codes in different threads, and for any other combination that you want. 12 | 13 | If you want to know more, visit https://culsans.readthedocs.io. 14 | """ 15 | 16 | from __future__ import annotations 17 | 18 | __author__: str = "Ilya Egorov <0x42005e1f@gmail.com>" 19 | __version__: str # dynamic 20 | __version_tuple__: tuple[int | str, ...] # dynamic 21 | 22 | from ._exceptions import ( 23 | AsyncQueueEmpty as AsyncQueueEmpty, 24 | AsyncQueueFull as AsyncQueueFull, 25 | AsyncQueueShutDown as AsyncQueueShutDown, 26 | QueueEmpty as QueueEmpty, 27 | QueueFull as QueueFull, 28 | QueueShutDown as QueueShutDown, 29 | SyncQueueEmpty as SyncQueueEmpty, 30 | SyncQueueFull as SyncQueueFull, 31 | SyncQueueShutDown as SyncQueueShutDown, 32 | UnsupportedOperation as UnsupportedOperation, 33 | ) 34 | from ._protocols import ( 35 | AsyncQueue as AsyncQueue, 36 | BaseQueue as BaseQueue, 37 | MixedQueue as MixedQueue, 38 | SyncQueue as SyncQueue, 39 | ) 40 | from ._proxies import ( 41 | AsyncQueueProxy as AsyncQueueProxy, 42 | SyncQueueProxy as SyncQueueProxy, 43 | ) 44 | from ._queues import ( 45 | LifoQueue as LifoQueue, 46 | PriorityQueue as PriorityQueue, 47 | Queue as Queue, 48 | ) 49 | 50 | # prepare for external use 51 | from aiologic import meta # isort: skip 52 | 53 | meta.export(globals()) 54 | meta.export_dynamic(globals(), "__version__", "._version.version") 55 | meta.export_dynamic(globals(), "__version_tuple__", "._version.version_tuple") 56 | 57 | del meta 58 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # SPDX-FileCopyrightText: 2025 Ilya Egorov <0x42005e1f@gmail.com> 4 | # SPDX-License-Identifier: CC0-1.0 5 | 6 | import os 7 | import sys 8 | 9 | from importlib.metadata import version as get_version 10 | 11 | from packaging.version import parse as parse_version 12 | 13 | os.environ["SPHINX_AUTODOC_RELOAD_MODULES"] = "1" 14 | 15 | project = "culsans" 16 | author = "Ilya Egorov" 17 | copyright = "2025 Ilya Egorov" 18 | 19 | v = parse_version(get_version("culsans")) 20 | version = v.base_version 21 | release = v.public 22 | 23 | extensions = [ 24 | "myst_parser", 25 | "sphinx.ext.autodoc", 26 | "sphinx.ext.intersphinx", 27 | "sphinx.ext.napoleon", 28 | "sphinx_copybutton", 29 | "sphinx_inline_tabs", 30 | "sphinx_rtd_theme", 31 | ] 32 | 33 | if sys.version_info >= (3, 11): 34 | extensions.append("sphinxcontrib.autodoc_inherit_overload") 35 | 36 | autodoc_class_signature = "separated" 37 | autodoc_inherit_docstrings = False 38 | autodoc_preserve_defaults = True 39 | autodoc_default_options = { 40 | "exclude-members": "__init_subclass__,__class_getitem__,__weakref__", 41 | "inherited-members": True, 42 | "member-order": "bysource", 43 | "show-inheritance": True, 44 | "special-members": True, 45 | } 46 | 47 | intersphinx_mapping = { 48 | "python": ("https://docs.python.org/3", None), 49 | } 50 | 51 | html_theme = "sphinx_rtd_theme" 52 | html_theme_options = {} 53 | html_static_path = ["_static"] 54 | html_css_files = ["css/custom.css"] 55 | html_context = { 56 | "display_github": True, 57 | "github_user": "x42005e1f", 58 | "github_repo": "culsans", 59 | "github_version": "main", 60 | "conf_py_path": "/docs/", 61 | } 62 | 63 | 64 | def enable_api_index(app, docname, source): 65 | if docname == "api": 66 | source[0] = source[0].replace(":no-index:", "") 67 | 68 | 69 | def setup(app): 70 | app.connect("source-read", enable_api_index) 71 | -------------------------------------------------------------------------------- /LICENSES/PSF-2.0.txt: -------------------------------------------------------------------------------- 1 | PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 2 | 3 | 1. This LICENSE AGREEMENT is between the Python Software Foundation 4 | ("PSF"), and the Individual or Organization ("Licensee") accessing and 5 | otherwise using this software ("Python") in source or binary form and 6 | its associated documentation. 7 | 8 | 2. Subject to the terms and conditions of this License Agreement, PSF hereby 9 | grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, 10 | analyze, test, perform and/or display publicly, prepare derivative works, 11 | distribute, and otherwise use Python alone or in any derivative version, 12 | provided, however, that PSF's License Agreement and PSF's notice of copyright, 13 | i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 14 | 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Python Software Foundation; 15 | All Rights Reserved" are retained in Python alone or in any derivative version 16 | prepared by Licensee. 17 | 18 | 3. In the event Licensee prepares a derivative work that is based on 19 | or incorporates Python or any part thereof, and wants to make 20 | the derivative work available to others as provided herein, then 21 | Licensee hereby agrees to include in any such work a brief summary of 22 | the changes made to Python. 23 | 24 | 4. PSF is making Python available to Licensee on an "AS IS" 25 | basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR 26 | IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND 27 | DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS 28 | FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT 29 | INFRINGE ANY THIRD PARTY RIGHTS. 30 | 31 | 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 32 | FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS 33 | A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, 34 | OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 35 | 36 | 6. This License Agreement will automatically terminate upon a material 37 | breach of its terms and conditions. 38 | 39 | 7. Nothing in this License Agreement shall be deemed to create any 40 | relationship of agency, partnership, or joint venture between PSF and 41 | Licensee. This License Agreement does not grant permission to use PSF 42 | trademarks or trade name in a trademark sense to endorse or promote 43 | products or services of Licensee, or any third party. 44 | 45 | 8. By copying, installing or otherwise using Python, Licensee 46 | agrees to be bound by the terms and conditions of this License 47 | Agreement. 48 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | .. 2 | SPDX-FileCopyrightText: 2025 Ilya Egorov <0x42005e1f@gmail.com> 3 | SPDX-License-Identifier: CC-BY-4.0 4 | 5 | API reference 6 | ============= 7 | 8 | Queues 9 | ------ 10 | 11 | .. autoclass:: culsans.Queue 12 | :members: 13 | :no-inherited-members: 14 | .. autoclass:: culsans.LifoQueue 15 | :members: 16 | :no-inherited-members: 17 | .. autoclass:: culsans.PriorityQueue 18 | :members: 19 | :no-inherited-members: 20 | 21 | Proxies 22 | ------- 23 | 24 | .. autoclass:: culsans.SyncQueueProxy 25 | :members: 26 | :no-inherited-members: 27 | .. autoclass:: culsans.AsyncQueueProxy 28 | :members: 29 | :no-inherited-members: 30 | 31 | Protocols 32 | --------- 33 | 34 | .. autoclass:: culsans.BaseQueue 35 | :members: 36 | :no-inherited-members: 37 | .. autoclass:: culsans.MixedQueue 38 | :members: 39 | :no-inherited-members: 40 | .. autoclass:: culsans.SyncQueue 41 | :members: 42 | :no-inherited-members: 43 | .. autoclass:: culsans.AsyncQueue 44 | :members: 45 | :no-inherited-members: 46 | 47 | Exceptions 48 | ---------- 49 | 50 | .. culsans.QueueEmpty-start-marker 51 | .. py:exception:: culsans.QueueEmpty 52 | :no-index: 53 | 54 | Bases: :exc:`~culsans.SyncQueueEmpty`, :exc:`~culsans.AsyncQueueEmpty` 55 | 56 | Exception raised when non-blocking get/peek is called on a 57 | :class:`~culsans.Queue` object which is empty. 58 | .. culsans.QueueEmpty-end-marker 59 | 60 | .. culsans.SyncQueueEmpty-start-marker 61 | .. py:exception:: culsans.SyncQueueEmpty 62 | :no-index: 63 | 64 | Bases: :exc:`Exception` 65 | 66 | The same as :exc:`queue.Empty`. 67 | .. culsans.SyncQueueEmpty-end-marker 68 | 69 | .. culsans.AsyncQueueEmpty-start-marker 70 | .. py:exception:: culsans.AsyncQueueEmpty 71 | :no-index: 72 | 73 | Bases: :exc:`Exception` 74 | 75 | The same as :exc:`asyncio.QueueEmpty`. 76 | .. culsans.AsyncQueueEmpty-end-marker 77 | 78 | .. culsans.QueueFull-start-marker 79 | .. py:exception:: culsans.QueueFull 80 | :no-index: 81 | 82 | Bases: :exc:`~culsans.SyncQueueFull`, :exc:`~culsans.AsyncQueueFull` 83 | 84 | Exception raised when non-blocking put is called on a :class:`~culsans.Queue` 85 | object which is full. 86 | .. culsans.QueueFull-end-marker 87 | 88 | .. culsans.SyncQueueFull-start-marker 89 | .. py:exception:: culsans.SyncQueueFull 90 | :no-index: 91 | 92 | Bases: :exc:`Exception` 93 | 94 | The same as :exc:`queue.Full`. 95 | .. culsans.SyncQueueFull-end-marker 96 | 97 | .. culsans.AsyncQueueFull-start-marker 98 | .. py:exception:: culsans.AsyncQueueFull 99 | :no-index: 100 | 101 | Bases: :exc:`Exception` 102 | 103 | The same as :exc:`asyncio.QueueFull`. 104 | .. culsans.AsyncQueueFull-end-marker 105 | 106 | .. culsans.QueueShutDown-start-marker 107 | .. py:exception:: culsans.QueueShutDown 108 | :no-index: 109 | 110 | Bases: :exc:`~culsans.SyncQueueShutDown`, :exc:`~culsans.AsyncQueueShutDown` 111 | 112 | Exception raised when put/get/peek is called on a :class:`~culsans.Queue` 113 | object which has been shut down. 114 | .. culsans.QueueShutDown-end-marker 115 | 116 | .. culsans.SyncQueueShutDown-start-marker 117 | .. py:exception:: culsans.SyncQueueShutDown 118 | :no-index: 119 | 120 | Bases: :exc:`Exception` 121 | 122 | The same as :exc:`queue.ShutDown`. 123 | .. culsans.SyncQueueShutDown-end-marker 124 | 125 | .. culsans.AsyncQueueShutDown-start-marker 126 | .. py:exception:: culsans.AsyncQueueShutDown 127 | :no-index: 128 | 129 | Bases: :exc:`Exception` 130 | 131 | The same as :exc:`asyncio.QueueShutDown`. 132 | .. culsans.AsyncQueueShutDown-end-marker 133 | 134 | .. culsans.UnsupportedOperation-start-marker 135 | .. py:exception:: culsans.UnsupportedOperation 136 | :no-index: 137 | 138 | Bases: :exc:`ValueError` 139 | 140 | Exception raised when peek/clear is called on a :class:`~culsans.Queue` 141 | object which is not peekable/clearable. 142 | .. culsans.UnsupportedOperation-end-marker 143 | -------------------------------------------------------------------------------- /src/culsans/_proxies.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # SPDX-FileCopyrightText: 2024 Ilya Egorov <0x42005e1f@gmail.com> 4 | # SPDX-License-Identifier: ISC 5 | 6 | from __future__ import annotations 7 | 8 | from typing import TypeVar 9 | 10 | from ._protocols import AsyncQueue, MixedQueue, SyncQueue 11 | 12 | _T = TypeVar("_T") 13 | 14 | 15 | class SyncQueueProxy(SyncQueue[_T]): 16 | """ 17 | A proxy that implements the :class:`SyncQueue` protocol by wrapping a mixed 18 | queue. 19 | """ 20 | 21 | __slots__ = ( 22 | "__weakref__", 23 | "wrapped", 24 | ) 25 | 26 | wrapped: MixedQueue[_T] 27 | 28 | def __init__(self, wrapped: MixedQueue[_T]) -> None: 29 | self.wrapped = wrapped 30 | 31 | def __repr__(self) -> str: 32 | cls = self.__class__ 33 | cls_repr = f"{cls.__module__}.{cls.__qualname__}" 34 | 35 | return f"{cls_repr}({self.wrapped!r})" 36 | 37 | def peekable(self) -> bool: 38 | return self.wrapped.peekable() 39 | 40 | def clearable(self) -> bool: 41 | return self.wrapped.clearable() 42 | 43 | def qsize(self) -> int: 44 | return self.wrapped.qsize() 45 | 46 | def empty(self) -> bool: 47 | return self.wrapped.empty() 48 | 49 | def full(self) -> bool: 50 | return self.wrapped.full() 51 | 52 | def put( 53 | self, 54 | item: _T, 55 | block: bool = True, 56 | timeout: float | None = None, 57 | ) -> None: 58 | self.wrapped.sync_put(item, block, timeout) 59 | 60 | def put_nowait(self, item: _T) -> None: 61 | self.wrapped.put_nowait(item) 62 | 63 | def get(self, block: bool = True, timeout: float | None = None) -> _T: 64 | return self.wrapped.sync_get(block, timeout) 65 | 66 | def get_nowait(self) -> _T: 67 | return self.wrapped.get_nowait() 68 | 69 | def peek(self, block: bool = True, timeout: float | None = None) -> _T: 70 | return self.wrapped.sync_peek(block, timeout) 71 | 72 | def peek_nowait(self) -> _T: 73 | return self.wrapped.peek_nowait() 74 | 75 | def join(self) -> None: 76 | self.wrapped.sync_join() 77 | 78 | def task_done(self, count: int = 1) -> None: 79 | self.wrapped.task_done(count) 80 | 81 | def shutdown(self, immediate: bool = False) -> None: 82 | self.wrapped.shutdown(immediate) 83 | 84 | def clear(self) -> None: 85 | self.wrapped.clear() 86 | 87 | @property 88 | def unfinished_tasks(self) -> int: 89 | return self.wrapped.unfinished_tasks 90 | 91 | @property 92 | def is_shutdown(self) -> bool: 93 | return self.wrapped.is_shutdown 94 | 95 | @property 96 | def closed(self) -> bool: 97 | return self.wrapped.closed 98 | 99 | @property 100 | def maxsize(self) -> int: 101 | return self.wrapped.maxsize 102 | 103 | @maxsize.setter 104 | def maxsize(self, value: int) -> None: 105 | self.wrapped.maxsize = value 106 | 107 | 108 | class AsyncQueueProxy(AsyncQueue[_T]): 109 | """ 110 | A proxy that implements the :class:`AsyncQueue` protocol by wrapping a 111 | mixed queue. 112 | """ 113 | 114 | __slots__ = ( 115 | "__weakref__", 116 | "wrapped", 117 | ) 118 | 119 | wrapped: MixedQueue[_T] 120 | 121 | def __init__(self, wrapped: MixedQueue[_T]) -> None: 122 | self.wrapped = wrapped 123 | 124 | def __repr__(self) -> str: 125 | cls = self.__class__ 126 | cls_repr = f"{cls.__module__}.{cls.__qualname__}" 127 | 128 | return f"{cls_repr}({self.wrapped!r})" 129 | 130 | def peekable(self) -> bool: 131 | return self.wrapped.peekable() 132 | 133 | def clearable(self) -> bool: 134 | return self.wrapped.clearable() 135 | 136 | def qsize(self) -> int: 137 | return self.wrapped.qsize() 138 | 139 | def empty(self) -> bool: 140 | return self.wrapped.empty() 141 | 142 | def full(self) -> bool: 143 | return self.wrapped.full() 144 | 145 | async def put(self, item: _T) -> None: 146 | await self.wrapped.async_put(item) 147 | 148 | def put_nowait(self, item: _T) -> None: 149 | self.wrapped.put_nowait(item) 150 | 151 | async def get(self) -> _T: 152 | return await self.wrapped.async_get() 153 | 154 | def get_nowait(self) -> _T: 155 | return self.wrapped.get_nowait() 156 | 157 | async def peek(self) -> _T: 158 | return await self.wrapped.async_peek() 159 | 160 | def peek_nowait(self) -> _T: 161 | return self.wrapped.peek_nowait() 162 | 163 | async def join(self) -> None: 164 | await self.wrapped.async_join() 165 | 166 | def task_done(self, count: int = 1) -> None: 167 | self.wrapped.task_done(count) 168 | 169 | def shutdown(self, immediate: bool = False) -> None: 170 | self.wrapped.shutdown(immediate) 171 | 172 | def clear(self) -> None: 173 | self.wrapped.clear() 174 | 175 | @property 176 | def unfinished_tasks(self) -> int: 177 | return self.wrapped.unfinished_tasks 178 | 179 | @property 180 | def is_shutdown(self) -> bool: 181 | return self.wrapped.is_shutdown 182 | 183 | @property 184 | def closed(self) -> bool: 185 | return self.wrapped.closed 186 | 187 | @property 188 | def maxsize(self) -> int: 189 | return self.wrapped.maxsize 190 | 191 | @maxsize.setter 192 | def maxsize(self, value: int) -> None: 193 | self.wrapped.maxsize = value 194 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/python,linux,macos,windows 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=python,linux,macos,windows 3 | 4 | ### Linux ### 5 | *~ 6 | 7 | # temporary files which can be created if a process still has a handle open of a deleted file 8 | .fuse_hidden* 9 | 10 | # KDE directory preferences 11 | .directory 12 | 13 | # Linux trash folder which might appear on any partition or disk 14 | .Trash-* 15 | 16 | # .nfs files are created when an open file is removed but is still being accessed 17 | .nfs* 18 | 19 | ### macOS ### 20 | # General 21 | .DS_Store 22 | .AppleDouble 23 | .LSOverride 24 | 25 | # Icon must end with two \r 26 | Icon 27 | 28 | 29 | # Thumbnails 30 | ._* 31 | 32 | # Files that might appear in the root of a volume 33 | .DocumentRevisions-V100 34 | .fseventsd 35 | .Spotlight-V100 36 | .TemporaryItems 37 | .Trashes 38 | .VolumeIcon.icns 39 | .com.apple.timemachine.donotpresent 40 | 41 | # Directories potentially created on remote AFP share 42 | .AppleDB 43 | .AppleDesktop 44 | Network Trash Folder 45 | Temporary Items 46 | .apdisk 47 | 48 | ### macOS Patch ### 49 | # iCloud generated files 50 | *.icloud 51 | 52 | ### Python ### 53 | # Byte-compiled / optimized / DLL files 54 | __pycache__/ 55 | *.py[cod] 56 | *$py.class 57 | 58 | # C extensions 59 | *.so 60 | 61 | # Distribution / packaging 62 | .Python 63 | build/ 64 | develop-eggs/ 65 | dist/ 66 | downloads/ 67 | eggs/ 68 | .eggs/ 69 | lib/ 70 | lib64/ 71 | parts/ 72 | sdist/ 73 | var/ 74 | wheels/ 75 | share/python-wheels/ 76 | *.egg-info/ 77 | .installed.cfg 78 | *.egg 79 | MANIFEST 80 | 81 | # PyInstaller 82 | # Usually these files are written by a python script from a template 83 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 84 | *.manifest 85 | *.spec 86 | 87 | # Installer logs 88 | pip-log.txt 89 | pip-delete-this-directory.txt 90 | 91 | # Unit test / coverage reports 92 | htmlcov/ 93 | .tox/ 94 | .nox/ 95 | .coverage 96 | .coverage.* 97 | .cache 98 | nosetests.xml 99 | coverage.xml 100 | *.cover 101 | *.py,cover 102 | .hypothesis/ 103 | .pytest_cache/ 104 | cover/ 105 | 106 | # Translations 107 | *.mo 108 | *.pot 109 | 110 | # Django stuff: 111 | *.log 112 | local_settings.py 113 | db.sqlite3 114 | db.sqlite3-journal 115 | 116 | # Flask stuff: 117 | instance/ 118 | .webassets-cache 119 | 120 | # Scrapy stuff: 121 | .scrapy 122 | 123 | # Sphinx documentation 124 | docs/_build/ 125 | 126 | # PyBuilder 127 | .pybuilder/ 128 | target/ 129 | 130 | # Jupyter Notebook 131 | .ipynb_checkpoints 132 | 133 | # IPython 134 | profile_default/ 135 | ipython_config.py 136 | 137 | # pyenv 138 | # For a library or package, you might want to ignore these files since the code is 139 | # intended to run in multiple environments; otherwise, check them in: 140 | # .python-version 141 | 142 | # pipenv 143 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 144 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 145 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 146 | # install all needed dependencies. 147 | #Pipfile.lock 148 | 149 | # poetry 150 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 151 | # This is especially recommended for binary packages to ensure reproducibility, and is more 152 | # commonly ignored for libraries. 153 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 154 | #poetry.lock 155 | 156 | # pdm 157 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 158 | #pdm.lock 159 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 160 | # in version control. 161 | # https://pdm.fming.dev/#use-with-ide 162 | .pdm.toml 163 | 164 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 165 | __pypackages__/ 166 | 167 | # Celery stuff 168 | celerybeat-schedule 169 | celerybeat.pid 170 | 171 | # SageMath parsed files 172 | *.sage.py 173 | 174 | # Environments 175 | .env 176 | .venv 177 | env/ 178 | venv/ 179 | ENV/ 180 | env.bak/ 181 | venv.bak/ 182 | 183 | # Spyder project settings 184 | .spyderproject 185 | .spyproject 186 | 187 | # Rope project settings 188 | .ropeproject 189 | 190 | # mkdocs documentation 191 | /site 192 | 193 | # mypy 194 | .mypy_cache/ 195 | .dmypy.json 196 | dmypy.json 197 | 198 | # Pyre type checker 199 | .pyre/ 200 | 201 | # pytype static type analyzer 202 | .pytype/ 203 | 204 | # Cython debug symbols 205 | cython_debug/ 206 | 207 | # PyCharm 208 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 209 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 210 | # and can be added to the global gitignore or merged into this file. For a more nuclear 211 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 212 | #.idea/ 213 | 214 | ### Python Patch ### 215 | # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration 216 | poetry.toml 217 | 218 | # ruff 219 | .ruff_cache/ 220 | 221 | # LSP config files 222 | pyrightconfig.json 223 | 224 | ### Windows ### 225 | # Windows thumbnail cache files 226 | Thumbs.db 227 | Thumbs.db:encryptable 228 | ehthumbs.db 229 | ehthumbs_vista.db 230 | 231 | # Dump file 232 | *.stackdump 233 | 234 | # Folder config file 235 | [Dd]esktop.ini 236 | 237 | # Recycle Bin used on file shares 238 | $RECYCLE.BIN/ 239 | 240 | # Windows Installer files 241 | *.cab 242 | *.msi 243 | *.msix 244 | *.msm 245 | *.msp 246 | 247 | # Windows shortcuts 248 | *.lnk 249 | 250 | # End of https://www.toptal.com/developers/gitignore/api/python,linux,macos,windows 251 | 252 | # hatch-vcs 253 | /src/*/_version.py 254 | -------------------------------------------------------------------------------- /tests/culsans/test_base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # SPDX-FileCopyrightText: 2024 Ilya Egorov <0x42005e1f@gmail.com> 4 | # SPDX-License-Identifier: 0BSD 5 | 6 | import pytest 7 | 8 | import culsans 9 | 10 | 11 | class _TestQueueBase: 12 | def test_init(self): 13 | assert self.factory().maxsize == 0 14 | assert self.factory(1).maxsize == 1 15 | assert self.factory(maxsize=-1).maxsize == -1 16 | 17 | def test_getters(self): 18 | queue = self.factory() 19 | 20 | assert queue.sync_q.wrapped is queue 21 | assert queue.async_q.wrapped is queue 22 | 23 | assert queue.unfinished_tasks == 0 24 | assert queue.sync_q.unfinished_tasks == 0 25 | assert queue.async_q.unfinished_tasks == 0 26 | 27 | assert not queue.is_shutdown 28 | assert not queue.sync_q.is_shutdown 29 | assert not queue.async_q.is_shutdown 30 | 31 | assert queue.maxsize == 0 32 | assert queue.sync_q.maxsize == 0 33 | assert queue.async_q.maxsize == 0 34 | 35 | with pytest.raises(AttributeError): 36 | queue.nonexistent_attribute # noqa: B018 37 | with pytest.raises(AttributeError): 38 | queue.sync_q.nonexistent_attribute # noqa: B018 39 | with pytest.raises(AttributeError): 40 | queue.async_q.nonexistent_attribute # noqa: B018 41 | 42 | def test_setters(self): 43 | queue = self.factory() 44 | 45 | with pytest.raises(AttributeError): 46 | queue.sync_q = queue.sync_q 47 | with pytest.raises(AttributeError): 48 | queue.async_q = queue.async_q 49 | 50 | with pytest.raises(AttributeError): 51 | queue.unfinished_tasks = queue.unfinished_tasks 52 | with pytest.raises(AttributeError): 53 | queue.sync_q.unfinished_tasks = queue.sync_q.unfinished_tasks 54 | with pytest.raises(AttributeError): 55 | queue.async_q.unfinished_tasks = queue.async_q.unfinished_tasks 56 | 57 | with pytest.raises(AttributeError): 58 | queue.is_shutdown = queue.is_shutdown 59 | with pytest.raises(AttributeError): 60 | queue.sync_q.is_shutdown = queue.sync_q.is_shutdown 61 | with pytest.raises(AttributeError): 62 | queue.async_q.is_shutdown = queue.async_q.is_shutdown 63 | 64 | queue.maxsize = 1 65 | assert queue.maxsize == 1 66 | queue.maxsize = -1 67 | assert queue.maxsize == -1 68 | 69 | queue.sync_q.maxsize = 1 70 | assert queue.maxsize == 1 71 | queue.sync_q.maxsize = -1 72 | assert queue.maxsize == -1 73 | 74 | queue.async_q.maxsize = 1 75 | assert queue.maxsize == 1 76 | queue.async_q.maxsize = -1 77 | assert queue.maxsize == -1 78 | 79 | with pytest.raises(AttributeError): 80 | queue.nonexistent_attribute = 42 81 | with pytest.raises(AttributeError): 82 | queue.sync_q.nonexistent_attribute = 42 83 | with pytest.raises(AttributeError): 84 | queue.async_q.nonexistent_attribute = 42 85 | 86 | def test_deleters(self): 87 | queue = self.factory() 88 | 89 | with pytest.raises(AttributeError): 90 | del queue.sync_q 91 | with pytest.raises(AttributeError): 92 | del queue.async_q 93 | 94 | with pytest.raises(AttributeError): 95 | del queue.unfinished_tasks 96 | with pytest.raises(AttributeError): 97 | del queue.sync_q.unfinished_tasks 98 | with pytest.raises(AttributeError): 99 | del queue.async_q.unfinished_tasks 100 | 101 | with pytest.raises(AttributeError): 102 | del queue.is_shutdown 103 | with pytest.raises(AttributeError): 104 | del queue.sync_q.is_shutdown 105 | with pytest.raises(AttributeError): 106 | del queue.async_q.is_shutdown 107 | 108 | with pytest.raises(AttributeError): 109 | del queue.maxsize 110 | with pytest.raises(AttributeError): 111 | del queue.sync_q.maxsize 112 | with pytest.raises(AttributeError): 113 | del queue.async_q.maxsize 114 | 115 | with pytest.raises(AttributeError): 116 | del queue.nonexistent_attribute 117 | with pytest.raises(AttributeError): 118 | del queue.sync_q.nonexistent_attribute 119 | with pytest.raises(AttributeError): 120 | del queue.async_q.nonexistent_attribute 121 | 122 | def test_unfinished(self): 123 | queue = self.factory() 124 | assert queue.unfinished_tasks == 0 125 | 126 | queue.sync_q.put(1) 127 | assert queue.unfinished_tasks == 1 128 | 129 | queue.sync_q.get() 130 | assert queue.unfinished_tasks == 1 131 | 132 | queue.sync_q.task_done() 133 | assert queue.unfinished_tasks == 0 134 | 135 | def test_shutdown(self): 136 | queue = self.factory() 137 | 138 | queue.sync_q.put(1) 139 | queue.sync_q.task_done() 140 | queue.sync_q.put(2) 141 | 142 | queue.shutdown() 143 | 144 | assert queue._qsize() == 2 145 | assert queue.unfinished_tasks == 1 146 | assert queue.is_shutdown 147 | 148 | queue.shutdown() 149 | 150 | def test_immediate_shutdown(self): 151 | queue = self.factory() 152 | 153 | queue.sync_q.put(1) 154 | queue.sync_q.task_done() 155 | queue.sync_q.put(2) 156 | 157 | queue.shutdown(immediate=True) 158 | 159 | assert queue._qsize() == 0 160 | assert queue.unfinished_tasks == 0 161 | assert queue.is_shutdown 162 | 163 | queue.shutdown(immediate=True) 164 | 165 | 166 | class TestQueue(_TestQueueBase): 167 | factory = culsans.Queue 168 | 169 | 170 | class TestLifoQueue(_TestQueueBase): 171 | factory = culsans.LifoQueue 172 | 173 | 174 | class TestPriorityQueue(_TestQueueBase): 175 | factory = culsans.PriorityQueue 176 | -------------------------------------------------------------------------------- /LICENSES/CC0-1.0.txt: -------------------------------------------------------------------------------- 1 | Creative Commons Legal Code 2 | 3 | CC0 1.0 Universal 4 | 5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN 7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS 10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM 11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED 12 | HEREUNDER. 13 | 14 | Statement of Purpose 15 | 16 | The laws of most jurisdictions throughout the world automatically confer 17 | exclusive Copyright and Related Rights (defined below) upon the creator 18 | and subsequent owner(s) (each and all, an "owner") of an original work of 19 | authorship and/or a database (each, a "Work"). 20 | 21 | Certain owners wish to permanently relinquish those rights to a Work for 22 | the purpose of contributing to a commons of creative, cultural and 23 | scientific works ("Commons") that the public can reliably and without fear 24 | of later claims of infringement build upon, modify, incorporate in other 25 | works, reuse and redistribute as freely as possible in any form whatsoever 26 | and for any purposes, including without limitation commercial purposes. 27 | These owners may contribute to the Commons to promote the ideal of a free 28 | culture and the further production of creative, cultural and scientific 29 | works, or to gain reputation or greater distribution for their Work in 30 | part through the use and efforts of others. 31 | 32 | For these and/or other purposes and motivations, and without any 33 | expectation of additional consideration or compensation, the person 34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she 35 | is an owner of Copyright and Related Rights in the Work, voluntarily 36 | elects to apply CC0 to the Work and publicly distribute the Work under its 37 | terms, with knowledge of his or her Copyright and Related Rights in the 38 | Work and the meaning and intended legal effect of CC0 on those rights. 39 | 40 | 1. Copyright and Related Rights. A Work made available under CC0 may be 41 | protected by copyright and related or neighboring rights ("Copyright and 42 | Related Rights"). Copyright and Related Rights include, but are not 43 | limited to, the following: 44 | 45 | i. the right to reproduce, adapt, distribute, perform, display, 46 | communicate, and translate a Work; 47 | ii. moral rights retained by the original author(s) and/or performer(s); 48 | iii. publicity and privacy rights pertaining to a person's image or 49 | likeness depicted in a Work; 50 | iv. rights protecting against unfair competition in regards to a Work, 51 | subject to the limitations in paragraph 4(a), below; 52 | v. rights protecting the extraction, dissemination, use and reuse of data 53 | in a Work; 54 | vi. database rights (such as those arising under Directive 96/9/EC of the 55 | European Parliament and of the Council of 11 March 1996 on the legal 56 | protection of databases, and under any national implementation 57 | thereof, including any amended or successor version of such 58 | directive); and 59 | vii. other similar, equivalent or corresponding rights throughout the 60 | world based on applicable law or treaty, and any national 61 | implementations thereof. 62 | 63 | 2. Waiver. To the greatest extent permitted by, but not in contravention 64 | of, applicable law, Affirmer hereby overtly, fully, permanently, 65 | irrevocably and unconditionally waives, abandons, and surrenders all of 66 | Affirmer's Copyright and Related Rights and associated claims and causes 67 | of action, whether now known or unknown (including existing as well as 68 | future claims and causes of action), in the Work (i) in all territories 69 | worldwide, (ii) for the maximum duration provided by applicable law or 70 | treaty (including future time extensions), (iii) in any current or future 71 | medium and for any number of copies, and (iv) for any purpose whatsoever, 72 | including without limitation commercial, advertising or promotional 73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each 74 | member of the public at large and to the detriment of Affirmer's heirs and 75 | successors, fully intending that such Waiver shall not be subject to 76 | revocation, rescission, cancellation, termination, or any other legal or 77 | equitable action to disrupt the quiet enjoyment of the Work by the public 78 | as contemplated by Affirmer's express Statement of Purpose. 79 | 80 | 3. Public License Fallback. Should any part of the Waiver for any reason 81 | be judged legally invalid or ineffective under applicable law, then the 82 | Waiver shall be preserved to the maximum extent permitted taking into 83 | account Affirmer's express Statement of Purpose. In addition, to the 84 | extent the Waiver is so judged Affirmer hereby grants to each affected 85 | person a royalty-free, non transferable, non sublicensable, non exclusive, 86 | irrevocable and unconditional license to exercise Affirmer's Copyright and 87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 88 | maximum duration provided by applicable law or treaty (including future 89 | time extensions), (iii) in any current or future medium and for any number 90 | of copies, and (iv) for any purpose whatsoever, including without 91 | limitation commercial, advertising or promotional purposes (the 92 | "License"). The License shall be deemed effective as of the date CC0 was 93 | applied by Affirmer to the Work. Should any part of the License for any 94 | reason be judged legally invalid or ineffective under applicable law, such 95 | partial invalidity or ineffectiveness shall not invalidate the remainder 96 | of the License, and in such case Affirmer hereby affirms that he or she 97 | will not (i) exercise any of his or her remaining Copyright and Related 98 | Rights in the Work or (ii) assert any associated claims and causes of 99 | action with respect to the Work, in either case contrary to Affirmer's 100 | express Statement of Purpose. 101 | 102 | 4. Limitations and Disclaimers. 103 | 104 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 105 | surrendered, licensed or otherwise affected by this document. 106 | b. Affirmer offers the Work as-is and makes no representations or 107 | warranties of any kind concerning the Work, express, implied, 108 | statutory or otherwise, including without limitation warranties of 109 | title, merchantability, fitness for a particular purpose, non 110 | infringement, or the absence of latent or other defects, accuracy, or 111 | the present or absence of errors, whether or not discoverable, all to 112 | the greatest extent permissible under applicable law. 113 | c. Affirmer disclaims responsibility for clearing rights of other persons 114 | that may apply to the Work or any use thereof, including without 115 | limitation any person's Copyright and Related Rights in the Work. 116 | Further, Affirmer disclaims responsibility for obtaining any necessary 117 | consents, permissions or other rights required for any use of the 118 | Work. 119 | d. Affirmer understands and acknowledges that Creative Commons is not a 120 | party to this document and has no duty or obligation with respect to 121 | this CC0 or use of the Work. 122 | -------------------------------------------------------------------------------- /tests/culsans/test_mixed.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # SPDX-FileCopyrightText: 2024 Ilya Egorov <0x42005e1f@gmail.com> 4 | # SPDX-License-Identifier: 0BSD 5 | 6 | import asyncio 7 | 8 | import pytest 9 | 10 | import culsans 11 | 12 | 13 | class TestMixedQueue: 14 | factory = culsans.Queue 15 | 16 | @pytest.mark.asyncio 17 | async def test_sync_put_async_get(self): 18 | queue = self.factory() 19 | loop = asyncio.get_running_loop() 20 | 21 | def sync_run(sync_q): 22 | assert sync_q.empty() 23 | 24 | for i in range(5): 25 | sync_q.put(i) 26 | 27 | async def async_run(async_q): 28 | for i in range(5): 29 | assert await async_q.get() == i 30 | 31 | assert async_q.empty() 32 | 33 | for _ in range(3): 34 | await asyncio.gather( 35 | loop.run_in_executor(None, sync_run, queue.sync_q), 36 | async_run(queue.async_q), 37 | ) 38 | 39 | @pytest.mark.asyncio 40 | async def test_async_put_sync_get(self): 41 | queue = self.factory() 42 | loop = asyncio.get_running_loop() 43 | 44 | def sync_run(sync_q): 45 | for i in range(5): 46 | assert sync_q.get() == i 47 | 48 | assert sync_q.empty() 49 | 50 | async def async_run(async_q): 51 | assert async_q.empty() 52 | 53 | for i in range(5): 54 | await async_q.put(i) 55 | 56 | for _ in range(3): 57 | await asyncio.gather( 58 | loop.run_in_executor(None, sync_run, queue.sync_q), 59 | async_run(queue.async_q), 60 | ) 61 | 62 | @pytest.mark.asyncio 63 | async def test_sync_join_async_done(self): 64 | queue = self.factory() 65 | loop = asyncio.get_running_loop() 66 | 67 | def sync_run(sync_q): 68 | assert sync_q.empty() 69 | 70 | for i in range(5): 71 | sync_q.put(i) 72 | 73 | sync_q.join() 74 | 75 | async def async_run(async_q): 76 | for i in range(5): 77 | assert await async_q.get() == i 78 | 79 | async_q.task_done() 80 | 81 | assert async_q.empty() 82 | 83 | for _ in range(3): 84 | await asyncio.gather( 85 | loop.run_in_executor(None, sync_run, queue.sync_q), 86 | async_run(queue.async_q), 87 | ) 88 | 89 | @pytest.mark.asyncio 90 | async def test_async_join_sync_done(self): 91 | queue = self.factory() 92 | loop = asyncio.get_running_loop() 93 | 94 | def sync_run(sync_q): 95 | for i in range(5): 96 | assert sync_q.get() == i 97 | 98 | sync_q.task_done() 99 | 100 | assert sync_q.empty() 101 | 102 | async def async_run(async_q): 103 | assert async_q.empty() 104 | 105 | for i in range(5): 106 | await async_q.put(i) 107 | 108 | await async_q.join() 109 | 110 | for _ in range(3): 111 | await asyncio.gather( 112 | loop.run_in_executor(None, sync_run, queue.sync_q), 113 | async_run(queue.async_q), 114 | ) 115 | 116 | @pytest.mark.asyncio 117 | async def test_growing_and_shrinking(self): 118 | queue = self.factory(1) 119 | loop = asyncio.get_running_loop() 120 | 121 | queue.async_q.put_nowait(0) 122 | assert queue.async_q.qsize() == 1 123 | 124 | task = loop.run_in_executor(None, queue.sync_q.put, 1) 125 | 126 | while not queue.putting: 127 | await asyncio.sleep(1e-3) 128 | 129 | queue.async_q.maxsize = 2 # growing 130 | 131 | await task 132 | assert queue.async_q.qsize() == 2 133 | 134 | task = loop.run_in_executor(None, queue.sync_q.put, 2) 135 | 136 | while not queue.putting: 137 | await asyncio.sleep(1e-3) 138 | 139 | queue.async_q.maxsize = 1 # shrinking 140 | 141 | assert queue.putting == 1 142 | assert queue.async_q.qsize() == 2 143 | 144 | queue.async_q.get_nowait() 145 | 146 | assert queue.putting == 1 147 | assert queue.async_q.qsize() == 1 148 | 149 | queue.async_q.maxsize = 0 # now the queue size is infinite 150 | 151 | await task 152 | assert queue.async_q.qsize() == 2 153 | 154 | @pytest.mark.asyncio 155 | async def test_peek(self): 156 | queue = self.factory() 157 | 158 | assert queue.sync_q.peekable() 159 | assert queue.async_q.peekable() 160 | 161 | with pytest.raises(culsans.SyncQueueEmpty): 162 | queue.sync_q.peek(block=False) 163 | with pytest.raises(culsans.SyncQueueEmpty): 164 | queue.sync_q.peek_nowait() 165 | with pytest.raises(culsans.AsyncQueueEmpty): 166 | queue.async_q.peek_nowait() 167 | 168 | queue.async_q.put_nowait(42) 169 | 170 | assert queue.sync_q.peek() == 42 171 | assert queue.sync_q.peek_nowait() == 42 172 | assert await queue.async_q.peek() == 42 173 | assert queue.async_q.peek_nowait() == 42 174 | 175 | @pytest.mark.asyncio 176 | async def test_clear(self): 177 | queue = self.factory() 178 | loop = asyncio.get_running_loop() 179 | 180 | for i in range(3): 181 | queue.async_q.put_nowait(i) 182 | 183 | assert queue.unfinished_tasks == 3 184 | assert queue.async_q.qsize() == 3 185 | 186 | task = loop.run_in_executor(None, queue.sync_q.join) 187 | 188 | queue.async_q.clear() 189 | 190 | assert queue.unfinished_tasks == 0 191 | assert queue.async_q.qsize() == 0 192 | 193 | await task 194 | 195 | for i in range(3): 196 | queue.async_q.put_nowait(i) 197 | 198 | assert queue.unfinished_tasks == 3 199 | assert queue.async_q.qsize() == 3 200 | 201 | task = asyncio.create_task(queue.async_q.join()) 202 | 203 | queue.sync_q.clear() 204 | 205 | assert queue.unfinished_tasks == 0 206 | assert queue.async_q.qsize() == 0 207 | 208 | await task 209 | 210 | @pytest.mark.asyncio 211 | async def test_closed(self): 212 | queue = self.factory() 213 | 214 | assert not queue.closed 215 | assert not queue.async_q.closed 216 | assert not queue.sync_q.closed 217 | 218 | queue.close() 219 | 220 | assert queue.closed 221 | assert queue.async_q.closed 222 | assert queue.sync_q.closed 223 | 224 | @pytest.mark.asyncio 225 | async def test_double_closing(self): 226 | queue = self.factory() 227 | 228 | queue.close() 229 | queue.close() 230 | 231 | await queue.wait_closed() 232 | 233 | @pytest.mark.asyncio 234 | async def test_wait_without_closing(self): 235 | queue = self.factory() 236 | 237 | with pytest.raises(RuntimeError): 238 | await queue.wait_closed() 239 | 240 | queue.close() 241 | await queue.wait_closed() 242 | 243 | @pytest.mark.asyncio 244 | async def test_modifying_forbidden_after_closing(self): 245 | queue = self.factory() 246 | queue.close() 247 | 248 | with pytest.raises(culsans.SyncQueueShutDown): 249 | queue.sync_q.put(5) 250 | with pytest.raises(culsans.SyncQueueShutDown): 251 | queue.sync_q.put_nowait(5) 252 | 253 | with pytest.raises(culsans.SyncQueueShutDown): 254 | queue.sync_q.get() 255 | with pytest.raises(culsans.SyncQueueShutDown): 256 | queue.sync_q.get_nowait() 257 | 258 | with pytest.raises(culsans.AsyncQueueShutDown): 259 | await queue.async_q.put(5) 260 | with pytest.raises(culsans.AsyncQueueShutDown): 261 | queue.async_q.put_nowait(5) 262 | 263 | with pytest.raises(culsans.AsyncQueueShutDown): 264 | await queue.async_q.get() 265 | with pytest.raises(culsans.AsyncQueueShutDown): 266 | queue.async_q.get_nowait() 267 | 268 | await queue.wait_closed() 269 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling", "hatch-vcs"] 3 | build-backend = "hatchling.build" 4 | 5 | [dependency-groups] 6 | docs = [ 7 | "docutils>=0.17", 8 | "myst-parser", 9 | "packaging", 10 | "Sphinx", 11 | "sphinx-copybutton", 12 | "sphinx-inline-tabs", 13 | "sphinx-rtd-theme", 14 | "sphinxcontrib.autodoc-inherit-overload; python_version>='3.11'", 15 | ] 16 | 17 | [project] 18 | name = "culsans" 19 | dynamic = ["version"] 20 | description = "Thread-safe async-aware queue for Python" 21 | readme = {file = "README.rst", content-type = "text/x-rst"} 22 | authors = [{name = "Ilya Egorov", email = "0x42005e1f@gmail.com"}] 23 | requires-python = ">=3.8" 24 | classifiers = [ 25 | "Development Status :: 3 - Alpha", 26 | "Topic :: Software Development :: Libraries :: Python Modules", 27 | "License :: OSI Approved :: ISC License (ISCL)", 28 | "Intended Audience :: Developers", 29 | "Typing :: Typed", 30 | "Framework :: AsyncIO", 31 | "Framework :: Trio", 32 | "Framework :: AnyIO", 33 | "Programming Language :: Python :: 3 :: Only", 34 | "Programming Language :: Python :: 3.8", 35 | "Programming Language :: Python :: 3.9", 36 | "Programming Language :: Python :: 3.10", 37 | "Programming Language :: Python :: 3.11", 38 | "Programming Language :: Python :: 3.12", 39 | "Programming Language :: Python :: 3.13", 40 | "Programming Language :: Python :: 3.14", 41 | "Programming Language :: Python :: Free Threading :: 2 - Beta", 42 | "Programming Language :: Python :: Implementation :: CPython", 43 | "Programming Language :: Python :: Implementation :: PyPy", 44 | "Operating System :: OS Independent", 45 | ] 46 | keywords = [ 47 | "anyio", 48 | "async", 49 | "async-await", 50 | "asyncio", 51 | "communication", 52 | "concurrency", 53 | "eventlet", 54 | "gevent", 55 | "greenlet", 56 | "library", 57 | "mypy", 58 | "python", 59 | "queue", 60 | "threading", 61 | "thread-safety", 62 | "trio", 63 | ] 64 | dependencies = [ 65 | "aiologic>=0.16.0,<0.18.0", 66 | "typing-extensions>=4.10.0; python_version<'3.13'", 67 | ] 68 | 69 | [project.urls] 70 | Homepage = "https://github.com/x42005e1f/culsans" 71 | Documentation = "https://culsans.readthedocs.io" 72 | Changelog = "https://culsans.readthedocs.io/latest/changelog.html" 73 | 74 | [tool.codespell] 75 | skip = [ 76 | "tests/culsans/test_asyncio.py", 77 | "tests/culsans/test_queue.py", 78 | ] 79 | 80 | [tool.hatch.build.hooks.vcs] 81 | version-file = "src/culsans/_version.py" 82 | 83 | [tool.hatch.build.targets.sdist] 84 | exclude = [".*"] 85 | 86 | [tool.hatch.version] 87 | source = "vcs" 88 | 89 | [tool.ruff] 90 | fix = true 91 | line-length = 79 92 | preview = true 93 | show-fixes = true 94 | exclude = [ 95 | "tests/culsans/test_asyncio.py", 96 | "tests/culsans/test_queue.py", 97 | ] 98 | 99 | [tool.ruff.lint] 100 | select = [ 101 | # pyflakes 102 | "F", 103 | # pycodestyle 104 | "E", 105 | "W", 106 | # isort 107 | "I", 108 | # pyupgrade 109 | "UP", 110 | # flake8-2020 111 | "YTT", 112 | # flake8-async 113 | "ASYNC", 114 | # flake8-bandit 115 | "S", 116 | # flake8-blind-except 117 | "BLE", 118 | # flake8-bugbear 119 | "B", 120 | # flake8-commas 121 | "COM", 122 | # flake8-comprehensions 123 | "C4", 124 | # flake8-debugger 125 | "T10", 126 | # flake8-errmsg 127 | "EM", 128 | # flake8-executable 129 | "EXE", 130 | # flake8-future-annotations 131 | "FA", 132 | # flake8-implicit-str-concat 133 | "ISC", 134 | # flake8-import-conventions 135 | "ICN", 136 | # flake8-logging 137 | "LOG", 138 | # flake8-logging-format 139 | "G", 140 | # flake8-pie 141 | "PIE", 142 | # flake8-print 143 | "T20", 144 | # flake8-pyi 145 | "PYI", 146 | # flake8-pytest-style 147 | "PT", 148 | # flake8-raise 149 | "RSE", 150 | # flake8-return 151 | "RET", 152 | # flake8-slots 153 | "SLOT", 154 | # flake8-simplify 155 | "SIM", 156 | # flake8-tidy-imports 157 | "TID", 158 | # flake8-type-checking 159 | "TC", 160 | # flake8-todos 161 | "TD", 162 | # flake8-fixme 163 | "FIX", 164 | # pygrep-hooks 165 | "PGH", 166 | # pylint 167 | "PLC", 168 | "PLE", 169 | "PLR", 170 | "PLW", 171 | # tryceratops 172 | "TRY", 173 | # flynt 174 | "FLY", 175 | # perflint 176 | "PERF", 177 | # refurb 178 | "FURB", 179 | # ruff-specific rules 180 | "RUF", 181 | ] 182 | ignore = [ 183 | # flake8-async 184 | "ASYNC105", # trio-sync-call 185 | "ASYNC110", # async-busy-wait 186 | # flake8-bandit 187 | "S101", # assert 188 | # flake8-bugbear 189 | "B901", # return-in-generator 190 | "B905", # zip-without-explicit-strict 191 | "B911", # batched-without-explicit-strict 192 | # flake8-commas 193 | "COM812", # missing-trailing-comma 194 | "COM819", # prohibited-trailing-comma 195 | # flake8-executable 196 | "EXE001", # shebang-not-executable 197 | # flake8-implicit-str-concat 198 | "ISC001", # single-line-implicit-string-concatenation 199 | "ISC002", # multi-line-implicit-string-concatenation 200 | # flake8-pyi 201 | "PYI011", # typed-argument-default-in-stub 202 | "PYI029", # str-or-repr-defined-in-stub 203 | "PYI053", # string-or-bytes-too-long 204 | "PYI054", # numeric-literal-too-long 205 | # flake8-return 206 | "RET501", # unnecessary-return-none 207 | "RET504", # unnecessary-assign 208 | "RET505", # superfluous-else-return 209 | "RET506", # superfluous-else-raise 210 | "RET507", # superfluous-else-continue 211 | "RET508", # superfluous-else-break 212 | # flake8-simplify 213 | "SIM102", # collapsible-if 214 | "SIM105", # suppressible-exception 215 | "SIM108", # if-else-block-instead-of-if-exp 216 | "SIM109", # compare-with-tuple 217 | "SIM114", # if-with-same-arms 218 | "SIM116", # if-else-block-instead-of-dict-lookup 219 | "SIM210", # if-expr-with-true-false 220 | "SIM211", # if-expr-with-false-true 221 | "SIM300", # yoda-conditions 222 | # pylint 223 | "PLC0414", # useless-import-alias 224 | "PLC0415", # import-outside-top-level 225 | "PLC2701", # import-private-name 226 | "PLC2801", # unnecessary-dunder-call 227 | "PLE0604", # invalid-all-object 228 | "PLR0904", # too-many-public-methods 229 | "PLR0911", # too-many-return-statements 230 | "PLR0912", # too-many-branches 231 | "PLR0913", # too-many-arguments 232 | "PLR0914", # too-many-locals 233 | "PLR0915", # too-many-statements 234 | "PLR0916", # too-many-boolean-expressions 235 | "PLR0917", # too-many-positional-arguments 236 | "PLR1702", # too-many-nested-blocks 237 | "PLR1704", # redefined-argument-from-local 238 | "PLR1714", # repeated-equality-comparison 239 | "PLR1730", # if-stmt-min-max 240 | "PLR2004", # magic-value-comparison 241 | "PLR5501", # collapsible-else-if 242 | "PLR6301", # no-self-use 243 | "PLW0603", # global-statement 244 | # tryceratops 245 | "TRY003", # raise-vanilla-args 246 | "TRY301", # raise-within-try 247 | "TRY400", # error-instead-of-exception 248 | # perflint 249 | "PERF203", # try-except-in-loop 250 | # refurb 251 | "FURB101", # read-whole-file 252 | "FURB103", # write-whole-file 253 | "FURB154", # repeated-global 254 | # ruff-specific rules 255 | "RUF029", # unused-async 256 | "RUF052", # used-dummy-variable 257 | ] 258 | fixable = [ 259 | "ALL", 260 | ] 261 | unfixable = [ 262 | # pyflakes 263 | "F401", # unused-import 264 | "F504", # percent-format-extra-named-arguments 265 | "F522", # string-dot-format-extra-named-arguments 266 | "F523", # string-dot-format-extra-positional-arguments 267 | "F601", # multi-value-repeated-key-literal 268 | "F602", # multi-value-repeated-key-variable 269 | "F811", # redefined-while-unused 270 | "F841", # unused-variable 271 | # flake8-bugbear 272 | "B006", # mutable-argument-default 273 | "B007", # unused-loop-control-variable 274 | # flake8-pie 275 | "PIE794", # duplicate-class-field-definition 276 | # flake8-print 277 | "T20", 278 | # flake8-pyi 279 | "PYI010", # non-empty-stub-body 280 | "PYI011", # typed-argument-default-in-stub 281 | "PYI014", # argument-default-in-stub 282 | "PYI015", # assignment-default-in-stub 283 | "PYI021", # docstring-in-stub 284 | "PYI029", # str-or-repr-defined-in-stub 285 | "PYI048", # stub-body-multiple-statements 286 | "PYI051", # redundant-literal-union 287 | # pylint 288 | "PLE4703", # modified-iterating-set 289 | "PLW0128", # redeclared-assigned-name 290 | ] 291 | 292 | [tool.ruff.lint.per-file-ignores] 293 | "benches/**.py" = [ 294 | # flake8-print 295 | "T20", 296 | ] 297 | "benches/bench_all.py" = [ 298 | # flake8-bandit 299 | "S404", # suspicious-subprocess-import 300 | "S603", # subprocess-without-shell-equals-true 301 | ] 302 | 303 | [tool.ruff.lint.isort] 304 | case-sensitive = true 305 | combine-as-imports = true 306 | lines-between-types = 1 307 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | .. 2 | SPDX-FileCopyrightText: 2024 Ilya Egorov <0x42005e1f@gmail.com> 3 | SPDX-License-Identifier: CC-BY-4.0 4 | 5 | ======= 6 | culsans 7 | ======= 8 | 9 | |pypi-dw| |pypi-impl| |pypi-pyv| |pypi-types| 10 | 11 | .. |pypi-dw| image:: https://img.shields.io/pypi/dw/culsans 12 | :target: https://pypistats.org/packages/culsans 13 | :alt: 14 | .. |pypi-impl| image:: https://img.shields.io/pypi/implementation/culsans 15 | :target: #features 16 | :alt: 17 | .. |pypi-pyv| image:: https://img.shields.io/pypi/pyversions/culsans 18 | :target: #features 19 | :alt: 20 | .. |pypi-types| image:: https://img.shields.io/pypi/types/culsans 21 | :target: #features 22 | :alt: 23 | 24 | .. description-start-marker 25 | 26 | Mixed sync-async queue, supposed to be used for communicating between classic 27 | synchronous (threaded) code and asynchronous one, between two asynchronous 28 | codes in different threads, and for any other combination that you want. Based 29 | on the `queue `_ module. Built on 30 | the `aiologic `_ package. Inspired by 31 | the `janus `_ library. 32 | 33 | Like `Culsans god `_, the queue object 34 | from the library has two faces: synchronous and asynchronous interface. Unlike 35 | `Janus library `_, synchronous interface 36 | supports `eventlet `_, `gevent `_, and `threading `_, while asynchronous interface supports `asyncio `_, `curio `_, and `trio `_. 41 | 42 | Synchronous is fully compatible with `standard queue `_, asynchronous one follows `asyncio queue design 44 | `_. 45 | 46 | .. description-end-marker 47 | 48 | Installation 49 | ============ 50 | 51 | .. installation-start-marker 52 | 53 | Install from `PyPI `_ (stable): 54 | 55 | .. code:: console 56 | 57 | pip install culsans 58 | 59 | Or from `GitHub `_ (latest): 60 | 61 | .. code:: console 62 | 63 | pip install git+https://github.com/x42005e1f/culsans.git 64 | 65 | You can also use other package managers, such as `uv `_. 67 | 68 | .. installation-end-marker 69 | 70 | Usage 71 | ===== 72 | 73 | .. usage-start-marker 74 | 75 | Three queues are available: 76 | 77 | * ``Queue`` 78 | * ``LifoQueue`` 79 | * ``PriorityQueue`` 80 | 81 | Each has two properties: ``sync_q`` and ``async_q``. 82 | 83 | Use the first to get synchronous interface and the second to get asynchronous 84 | one. 85 | 86 | Example 87 | ------- 88 | 89 | .. code:: python 90 | 91 | import anyio 92 | import culsans 93 | 94 | 95 | def sync_run(sync_q: culsans.SyncQueue[int]) -> None: 96 | for i in range(100): 97 | sync_q.put(i) 98 | else: 99 | sync_q.join() 100 | 101 | 102 | async def async_run(async_q: culsans.AsyncQueue[int]) -> None: 103 | for i in range(100): 104 | value = await async_q.get() 105 | 106 | assert value == i 107 | 108 | async_q.task_done() 109 | 110 | 111 | async def main() -> None: 112 | queue: culsans.Queue[int] = culsans.Queue() 113 | 114 | async with anyio.create_task_group() as tasks: 115 | tasks.start_soon(anyio.to_thread.run_sync, sync_run, queue.sync_q) 116 | tasks.start_soon(async_run, queue.async_q) 117 | 118 | queue.shutdown() 119 | 120 | 121 | anyio.run(main) 122 | 123 | Extras 124 | ------ 125 | 126 | Both interfaces support some additional features that are not found in the 127 | original queues. 128 | 129 | growing & shrinking 130 | ^^^^^^^^^^^^^^^^^^^ 131 | 132 | You can dynamically change the upperbound limit on the number of items that can 133 | be placed in the queue with ``queue.maxsize = N``. If it increases (growing), 134 | the required number of waiting putters will be woken up. If it decreases 135 | (shrinking), items exceeding the new limit will remain in the queue, but all 136 | putters will be blocked until enough items are retrieved from the queue. And if 137 | *maxsize* is less than or equal to zero, all putters will be woken up. 138 | 139 | .. code:: python 140 | 141 | async with anyio.create_task_group() as tasks: 142 | async_q = culsans.Queue(1).async_q 143 | 144 | for i in range(4): 145 | tasks.start_soon(async_q.put, i) 146 | 147 | await anyio.sleep(1e-3) 148 | assert async_q.qsize() == 1 149 | 150 | async_q.maxsize = 2 # growing 151 | 152 | await anyio.sleep(1e-3) 153 | assert async_q.qsize() == 2 154 | 155 | async_q.maxsize = 1 # shrinking 156 | 157 | await anyio.sleep(1e-3) 158 | assert async_q.qsize() == 2 159 | 160 | async_q.get_nowait() 161 | 162 | await anyio.sleep(1e-3) 163 | assert async_q.qsize() == 1 164 | 165 | async_q.maxsize = 0 # now the queue size is infinite 166 | 167 | await anyio.sleep(1e-3) 168 | assert async_q.qsize() == 3 169 | 170 | peek() & peek_nowait() 171 | ^^^^^^^^^^^^^^^^^^^^^^ 172 | 173 | If you want to check the first item of the queue, but do not want to remove 174 | that item from the queue, you can use the ``peek()`` and ``peek_nowait()`` 175 | methods instead of the ``get()`` and ``get_nowait()`` methods. 176 | 177 | .. code:: python 178 | 179 | sync_q = culsans.Queue().sync_q 180 | 181 | sync_q.put("spam") 182 | 183 | assert sync_q.peekable() 184 | assert sync_q.peek() == "spam" 185 | assert sync_q.peek_nowait() == "spam" 186 | assert sync_q.qsize() == 1 187 | 188 | These methods can be considered an implementation of partial compatibility with 189 | `gevent queues `_. 190 | 191 | clear() 192 | ^^^^^^^ 193 | 194 | In some scenarios it may be necessary to clear the queue. But it is inefficient 195 | to do this through a loop, and it causes additional difficulties when it is 196 | also necessary to ensure that no new items can be added during the clearing 197 | process. For this purpose, there is the atomic ``clear()`` method that clears 198 | the queue most efficiently. 199 | 200 | .. code:: python 201 | 202 | async with anyio.create_task_group() as tasks: 203 | async_q = culsans.Queue(3).async_q 204 | 205 | for i in range(5): 206 | tasks.start_soon(async_q.put, i) 207 | 208 | await anyio.sleep(1e-3) 209 | assert async_q.qsize() == 3 210 | assert async_q.clearable() 211 | 212 | async_q.clear() # clearing 213 | 214 | await anyio.sleep(1e-3) 215 | assert async_q.qsize() == 2 216 | assert async_q.get_nowait() == 3 217 | assert async_q.get_nowait() == 4 218 | 219 | Roughly equivalent to: 220 | 221 | .. code:: python 222 | 223 | def clear(queue): 224 | while True: 225 | try: 226 | queue.get_nowait() 227 | except Empty: 228 | break 229 | else: 230 | queue.task_done() 231 | 232 | Subclasses 233 | ---------- 234 | 235 | You can create your own queues by inheriting from existing queue classes as if 236 | you were using the queue module. For example, this is how you can create an 237 | unordered queue that contains only unique items: 238 | 239 | .. code:: python 240 | 241 | from culsans import Queue 242 | 243 | 244 | class UniqueQueue(Queue): 245 | def _init(self, maxsize): 246 | self.data = set() 247 | 248 | def _qsize(self): 249 | return len(self.data) 250 | 251 | def _isize(self, item): 252 | return 1 253 | 254 | def _put(self, item): 255 | self.data.add(item) 256 | 257 | def _get(self): 258 | return self.data.pop() 259 | 260 | def _peekable(self): 261 | return False 262 | 263 | _peek = None 264 | 265 | def _clearable(self): 266 | return True 267 | 268 | def _clear(self): 269 | self.data.clear() 270 | 271 | .. code:: python 272 | 273 | sync_q = UniqueQueue().sync_q 274 | 275 | sync_q.put_nowait(23) 276 | sync_q.put_nowait(42) 277 | sync_q.put_nowait(23) 278 | 279 | assert sync_q.qsize() == 2 280 | assert sorted(sync_q.get_nowait() for _ in range(2)) == [23, 42] 281 | 282 | All nine of these methods are called in exclusive access mode, so you can 283 | freely create your subclasses without thinking about whether your methods are 284 | thread-safe or not. 285 | 286 | Sequence/flattened queues 287 | ^^^^^^^^^^^^^^^^^^^^^^^^^ 288 | 289 | By default, all items inserted into the queue are considered single-element 290 | items, and checks for maxsize are performed based on this assumption. However, 291 | if you implement queues of sequences such as lists or byte arrays, you may need 292 | to treat items by their actual size so that no insertion can exceed maxsize. To 293 | do this, you can override the ``_isize()`` method (in addition to 294 | ``_qsize()``): 295 | 296 | .. code:: python 297 | 298 | def _isize(self, item): 299 | return len(item) 300 | 301 | This way, the insertion will be blocked until there is enough space in the 302 | queue. But note that since all insertions are performed in FIFO mode, inserting 303 | a large item will prevent the insertion of subsequent small items. 304 | 305 | Checkpoints 306 | ----------- 307 | 308 | Sometimes it is useful when each asynchronous call switches execution to the 309 | next task and checks for cancellation and timeouts. For example, if you want to 310 | distribute CPU usage across all tasks. 311 | 312 | The culsans library adopts aiologic's checkpoints, but unlike it does not 313 | guarantee that there will only be one per asynchronous call, due to design 314 | specifics. 315 | 316 | See the aiologic documentation for details on how to control checkpoints. 317 | 318 | .. usage-end-marker 319 | 320 | Compatibility 321 | ============= 322 | 323 | .. compatibility-start-marker 324 | 325 | If you want to use culsans as a backport of the standard queues to older 326 | versions of Python (for example, if you need the ``shutdown()`` method), you 327 | can replace something like this: 328 | 329 | .. code:: python 330 | 331 | sync_q = queue.Queue() 332 | async_q = asyncio.Queue() 333 | 334 | with this: 335 | 336 | .. code:: python 337 | 338 | sync_q = culsans.Queue().sync_q 339 | async_q = culsans.Queue().async_q 340 | 341 | And if you are using janus in your application and want to switch to culsans, 342 | all you have to do is replace this: 343 | 344 | .. code:: python 345 | 346 | import janus 347 | 348 | with this: 349 | 350 | .. code:: python 351 | 352 | import culsans as janus 353 | 354 | and everything will work! 355 | 356 | .. compatibility-end-marker 357 | 358 | Documentation 359 | ============= 360 | 361 | Read the Docs: https://culsans.readthedocs.io (official) 362 | 363 | DeepWiki: https://deepwiki.com/x42005e1f/culsans (AI generated; lying!) 364 | 365 | Communication channels 366 | ====================== 367 | 368 | GitHub Discussions: https://github.com/x42005e1f/culsans/discussions (ideas, 369 | questions) 370 | 371 | GitHub Issues: https://github.com/x42005e1f/culsans/issues (bug tracker) 372 | 373 | You can also send an email to 0x42005e1f@gmail.com with any feedback. 374 | 375 | Project status 376 | ============== 377 | 378 | The project is developed and maintained by one person in his spare time and is 379 | not a commercial product. The author is not a professional programmer, but has 380 | been programming for over a decade as a hobby with almost no publicity (you may 381 | be able to find some contributions if you try hard, but it will be just a drop 382 | in the ocean). Therefore, if you encounter any misunderstandings, please excuse 383 | him, as he does not have much experience working with people. 384 | 385 | It is published for the simple reason that the author considered it noteworthy 386 | and not too ugly. The topic is quite non-trivial, so although contributions are 387 | not prohibited, they will be very, very difficult if you decide to make them 388 | (except for some very simple ones). The functionality provided is still being 389 | perfected, so the development status is alpha. 390 | 391 | No AI tools are used in the development (nor are IDE tools, for that matter). 392 | The only exception is text translation, since the author is not a native 393 | English speaker, but the texts themselves are not generated. 394 | 395 | What is the goal of the project? To realize the author's vision. Is it worth 396 | trusting what is available now? Well, the choice is yours. But the project `is 397 | already being used `__, so 398 | why not give it a try? 399 | 400 | License 401 | ======= 402 | 403 | .. license-start-marker 404 | 405 | The culsans library is `REUSE-compliant `_ and is offered under multiple licenses: 407 | 408 | * All original source code is licensed under `ISC`_. 409 | * All original test code is licensed under `0BSD`_. 410 | * All documentation is licensed under `CC-BY-4.0`_. 411 | * All configuration is licensed under `CC0-1.0`_. 412 | * Some test code borrowed from `python/cpython `_ is licensed under `PSF-2.0`_. 414 | 415 | For more accurate information, check the individual files. 416 | 417 | .. _ISC: https://choosealicense.com/licenses/isc/ 418 | .. _0BSD: https://choosealicense.com/licenses/0bsd/ 419 | .. _CC-BY-4.0: https://choosealicense.com/licenses/cc-by-4.0/ 420 | .. _CC0-1.0: https://choosealicense.com/licenses/cc0-1.0/ 421 | .. _PSF-2.0: https://docs.python.org/3/license.html 422 | 423 | .. license-end-marker 424 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 5 | 6 | Changelog 7 | ========= 8 | 9 | All notable changes to this project will be documented in this file. 10 | 11 | The format is based on 12 | [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), 13 | and this project adheres to 14 | [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 15 | 16 | Commit messages are consistent with 17 | [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). 18 | 19 | [Unreleased] 20 | ------------ 21 | 22 | ### Added 23 | 24 | - `culsans.__version__` and `culsans.__version_tuple__` as a way to retrieve 25 | the package version at runtime. 26 | - `clearable()` methods and related `culsans.Queue._clearable()` protected 27 | method (for overriding) analogous to those for peek methods, making 28 | implementation of the `clear()` method optional. Along with this, the 29 | `clear()` method is now also used in the implementation of the `shutdown()` 30 | method, when available. 31 | - `count` parameter to the `task_done()` methods to identify that the specified 32 | number of tasks has been completed. Useful for subclasses that implement 33 | flattened queues. 34 | - `_isize()` overridable method, which allows to create subclasses that 35 | implement sequence/flattened queues. Solves 36 | [#9](https://github.com/x42005e1f/culsans/issues/9). 37 | - The proxies can now be weakly referenced. Previously, this was disallowed due 38 | to their limited lifetime (since the corresponding properties return new 39 | objects on each access). This is now allowed for cases where a proxy is used 40 | as a backport to older versions of Python. 41 | - A negative queue length is now a valid value and is handled correctly 42 | (extends subclassing capabilities). 43 | 44 | ### Changed 45 | 46 | - The underlying lock is now reentrant. This differs from the standard queue 47 | approach, but makes it much easier to create subclasses and also makes 48 | `culsans.Queue` somewhat compatible with `sqlalchemy.util.queue.Queue`. 49 | However, the main reason is not this, but to make the following change 50 | efficiently implemented. 51 | - The queues now rely on new `aiologic` safety guarantees when using the 52 | condition variables. Previously, as with `threading.Condition`, a 53 | `KeyboardInterrupt` raised during synchronization on the underlying lock 54 | after notification led to the lock being over-released and, as a result, to a 55 | `RuntimeError`. Now, `aiologic.Condition` is used as a context manager, 56 | thereby including additional checks on the `aiologic` side to ensure that the 57 | current thread owns the lock when releasing it. 58 | - The `unfinished_tasks` property's value now changes according to the queue 59 | size change (the value returned by the `_qsize()` method). This allows to 60 | create flattened queues without introducing related additional methods and 61 | also corrects the behavior for subclasses that may not change the queue size 62 | as a result of insertion. 63 | - The `timeout` parameter's value is now checked and converted at the beginning 64 | of the method call, regardless of the `block` parameter's value. This should 65 | prevent cases where an incorrect value is passed. 66 | - The peekability check is now ensured before each call (after each context 67 | switch), allowing the peek methods to be enabled/disabled dynamically. 68 | - The package now relies on `aiologic.meta.export()` for exports instead of 69 | using its own implementation (similar to `aiologic==0.15.0`), which provides 70 | safer behavior. In particular, queue methods now also update their metadata, 71 | allowing them to be safely referenced during pickling. 72 | - The protocols are now inherited from `typing_extensions.Protocol` on Python 73 | below 3.13, which backports all related fixes and improvements to older 74 | versions of Python. 75 | - The shutdown exceptions are now defined again via backports (on Python below 76 | 3.13), but in a type-friendly manner. 77 | 78 | ### Fixed 79 | 80 | - The sync methods called the checkpoint function regardless of the `block` 81 | parameter's value. Now they do not make the call in the non-blocking case. 82 | - Notifications could be insufficient or excessive when the queue size changed 83 | differently than in the standard behavior. Now such situations are detected 84 | and a different notification mechanism is activated when they are detected. 85 | 86 | [0.10.0] - 2025-11-04 87 | --------------------- 88 | 89 | ### Added 90 | 91 | - `culsans.Queue.waiting` as the third `aiologic`-like property. Useful when 92 | you need to reliably obtain the total number of waiting ones. 93 | - Timeout handling has been improved in line with the latest changes in 94 | `aiologic` (support for very large numbers, additional checking for `NaN`, 95 | use of the clock functions from `aiologic.lowlevel`). This is particularly 96 | relevant for `aiologic>=0.15.0`, which implements safe timeouts, but has 97 | almost no impact on older versions. 98 | - The properties of `culsans.Queue` now return proxies instead of protocols for 99 | type checkers. This allows, for example, accessing the wrapped queue via the 100 | proxy attribute without type errors. 101 | - The proxies are now represented in a readable format, which should make 102 | debugging a little easier. 103 | 104 | ### Changed 105 | 106 | - The priority queues now use stricter bounds for the type variable: collection 107 | elements must be rich comparable (implement `__lt__()` or `__gt__()` method). 108 | This corresponds to recent changes in `typeshed` for the standard queues (see 109 | [python/typeshed#14418](https://github.com/python/typeshed/issues/14418)) and 110 | makes them safer. 111 | 112 | ### Fixed 113 | 114 | - For the underlying lock, `aiologic.lowlevel._thread` was used, which has been 115 | declared deprecated in the latest version of `aiologic`. 116 | - With green checkpoints enabled, the end time was recalculated for the timeout 117 | after rescheduling, which could lead to doubling the actual wait time 118 | (`0.9.0` regression). 119 | 120 | [0.9.0] - 2025-07-16 121 | -------------------- 122 | 123 | ### Changed 124 | 125 | - Checkpoint functions are now always called before queue operations are 126 | performed (previously they were called after). This ensures that all queue 127 | methods behave as expected on cancellations, but it may increase the number 128 | of explicit context switches. 129 | - The build system has been changed from `setuptools` to `uv` + `hatch`. It 130 | keeps the same `pyproject.toml` format, but has better performance, better 131 | logging, and builds cleaner source distributions (without `setup.cfg`). 132 | - The version identifier is now generated dynamically and includes the latest 133 | commit information for development versions, which simplifies bug reporting. 134 | It is also passed to archives generated by GitHub (via `.git_archival.txt`) 135 | and source distributions (via `PKG-INFO`). 136 | 137 | ### Fixed 138 | 139 | - For asynchronous checkpoints, `aiologic.lowlevel.checkpoint()` was used, 140 | which has been declared deprecated in the latest version of `aiologic`. 141 | - With checkpoints enabled (`trio` case by default), the get methods could lose 142 | items on cancellations, and for the put methods, it was impossible to 143 | determine whether the put was successful or not on the same cancellations 144 | (`0.2.1` regression). 145 | 146 | [0.8.0] - 2025-01-19 147 | -------------------- 148 | 149 | ### Added 150 | 151 | - Python 3.8 support. This makes the list of supported versions consistent with 152 | that of `aiologic`. Previously, the lowest supported version was 3.9. 153 | - `culsans.MixedQueue` as a queue protocol that provides both types of blocking 154 | methods via prefixes. Can be used to generalize `culsans.Queue` and its 155 | derivatives by type checkers, as it does not include `janus.Queue`-specific 156 | and other foreign methods and properties/attributes. 157 | 158 | ### Changed 159 | 160 | - Interfaces and type hints have been improved: 161 | + Now `aiologic>=0.13.0` is used (previously `>=0.12.0` was used), which 162 | provides type annotations. This removes mypy-specific type ignores. 163 | + All fields are annotated. A side effect is that the queue subclasses now 164 | define their own slot for storing data. 165 | - The source code tree has been significantly changed for better IDEs support. 166 | The same applies to exports, which are now checker-friendly. 167 | 168 | [0.7.1] - 2024-12-23 169 | -------------------- 170 | 171 | ### Fixed 172 | 173 | - For `culsans.QueueShutDown` on Python below 3.13, backported exceptions were 174 | defined, but they caused name conflicts for type checkers. Now, all queue 175 | shutdown exceptions reference the same class (on Python below 3.13). 176 | 177 | [0.7.0] - 2024-12-23 178 | -------------------- 179 | 180 | ### Added 181 | 182 | - `culsans.QueueEmpty`, `culsans.QueueFull`, and `culsans.QueueShutDown` as 183 | exceptions compatible with any interface (they inherit both types). They are 184 | now also raised instead of specialized exceptions, allowing any type to be 185 | used in try-catch. 186 | - The remaining non-blocking methods and both types of blocking methods to 187 | `culsans.Queue` via prefixes (`sync_` and `async_`). The main reason for this 188 | is to move the entire implementation from the proxy classes to the queue 189 | classes. However, they can also be used as a simpler, shorter style of 190 | working with queues (similar to `aiologic`, but with `sync_` instead of 191 | `green_`). 192 | - `culsans.Queue.putting` and `culsans.Queue.getting` as `aiologic`-like 193 | properties. They can be used to obtain the number of waiting ones for a given 194 | operation type. But note, `culsans.Queue.getting` also includes the number of 195 | peekers. 196 | - `culsans.Queue` now inherits from `culsans.BaseQueue` instead of `Generic`. 197 | This improves type safety and allows introspection at runtime. 198 | - Direct access (without the `_` prefix) to the synchronization primitives 199 | used, to better mimic the `queue` module. 200 | 201 | [0.6.0] - 2024-12-14 202 | -------------------- 203 | 204 | ### Changed 205 | 206 | - The behavior of Janus-specific methods now corresponds to `janus>=2.0.0` 207 | (instead of `1.2.0`). The only change is that after calling 208 | `culsans.Queue.close()`, `*QueueShutDown` is now raised instead of 209 | `RuntimeError`. 210 | 211 | [0.5.0] - 2024-12-14 212 | -------------------- 213 | 214 | ### Added 215 | 216 | - `culsans.Queue.aclose()` method, which replicates the behavior of the 217 | same-named Janus method. This makes the interface compliant with the Janus 218 | API version 1.2.0 (instead of 1.1.0). 219 | 220 | [0.4.0] - 2024-11-17 221 | -------------------- 222 | 223 | ### Added 224 | 225 | - `peekable()` methods and related `culsans.Queue._peekable()` protected method 226 | (for overriding). They simplify non-peekable subclass creation by providing a 227 | unified contract for how to deal with it from the user's side. 228 | - `culsans.UnsupportedOperation` exception, which is raised when attempting to 229 | call any of the peek methods for a non-peekable queue. 230 | 231 | [0.3.0] - 2024-11-08 232 | -------------------- 233 | 234 | ### Added 235 | 236 | - `culsans.Queue.close()` and `culsans.Queue.wait_closed()` methods, 237 | `culsans.Queue.closed` property to implement compatibility with 238 | `janus>=1.1.0`. For the most part, they behave similarly to the originals 239 | (including the exception raised after closing), but semantically they are 240 | closer to the queue shutdown methods from Python 3.13. This differs from the 241 | `janus` behavior, but solves 242 | [aio-libs/janus#237](https://github.com/aio-libs/janus/issues/237). 243 | - `peek()` and `peek_nowait()` methods, related `culsans.Queue._peek()` 244 | protected method (for overriding), as a way to retrieve an item from a queue 245 | without removing it. In a sense, they implement partial compatibility with 246 | the `gevent` queues, but peek/front is also a well-known third type of queue 247 | operation. 248 | - `clear()` method and related `culsans.Queue._clear()` protected method (for 249 | overriding). Atomically clears the queue, ensuring that other threads do not 250 | affect the clearing process, and updates the `unfinished_tasks` property at 251 | the same time. Solves 252 | [aio-libs/janus#645](https://github.com/aio-libs/janus/issues/645). 253 | 254 | [0.2.1] - 2024-11-04 255 | -------------------- 256 | 257 | ### Fixed 258 | 259 | - Mixed use of both types of methods could lead to deadlocks (due to the use of 260 | a shared wait queue for the underlying lock). This could be considered 261 | expected behavior if it did not also affect non-blocking methods (due to the 262 | blocking use of the condition variables). 263 | 264 | [0.2.0] - 2024-11-04 265 | -------------------- 266 | 267 | ### Added 268 | 269 | - `culsans.Queue.maxsize` can now be changed dynamically at runtime (growing & 270 | shrinking). This allows for more complex logic to be implemented, whereby the 271 | maximum queue size is adjusted according to certain external conditions. 272 | Related: 273 | [python/cpython#54319](https://github.com/python/cpython/issues/54319). 274 | 275 | ### Changed 276 | 277 | - The protocols, and therefore the proxies, no longer include the implicit 278 | `__dict__`, thereby preventing unknown attributes from being set. This makes 279 | them safer. 280 | 281 | [0.1.0] - 2024-11-02 282 | -------------------- 283 | 284 | ### Added 285 | 286 | - Janus-like mixed queues and all related API (excluding some Janus-specific 287 | methods that seem redundant on Python 3.13, in favor of new methods such as 288 | `shutdown()`). They use the same patterns as the `queue` module, but rely on 289 | the condition variables from `aiologic`. This makes them 4-8 times faster 290 | than `janus.Queue` in multi-threaded tests, simplifies usage, and expands the 291 | number of supported use cases (multiple event loops, `trio` support, etc.). 292 | 293 | [unreleased]: https://github.com/x42005e1f/culsans/compare/0.10.0...HEAD 294 | [0.10.0]: https://github.com/x42005e1f/culsans/compare/0.9.0...0.10.0 295 | [0.9.0]: https://github.com/x42005e1f/culsans/compare/0.8.0...0.9.0 296 | [0.8.0]: https://github.com/x42005e1f/culsans/compare/0.7.1...0.8.0 297 | [0.7.1]: https://github.com/x42005e1f/culsans/compare/0.7.0...0.7.1 298 | [0.7.0]: https://github.com/x42005e1f/culsans/compare/0.6.0...0.7.0 299 | [0.6.0]: https://github.com/x42005e1f/culsans/compare/0.5.0...0.6.0 300 | [0.5.0]: https://github.com/x42005e1f/culsans/compare/0.4.0...0.5.0 301 | [0.4.0]: https://github.com/x42005e1f/culsans/compare/0.3.0...0.4.0 302 | [0.3.0]: https://github.com/x42005e1f/culsans/compare/0.2.1...0.3.0 303 | [0.2.1]: https://github.com/x42005e1f/culsans/compare/0.2.0...0.2.1 304 | [0.2.0]: https://github.com/x42005e1f/culsans/compare/0.1.0...0.2.0 305 | [0.1.0]: https://github.com/x42005e1f/culsans/releases/tag/0.1.0 306 | -------------------------------------------------------------------------------- /src/culsans/_protocols.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # SPDX-FileCopyrightText: 2024 Ilya Egorov <0x42005e1f@gmail.com> 4 | # SPDX-License-Identifier: ISC 5 | 6 | from __future__ import annotations 7 | 8 | import sys 9 | 10 | from typing import TypeVar 11 | 12 | from ._utils import copydoc 13 | 14 | if sys.version_info >= (3, 13): # various fixes and improvements 15 | from typing import Protocol 16 | else: # typing-extensions>=4.10.0 17 | from typing_extensions import Protocol 18 | 19 | _T = TypeVar("_T") 20 | 21 | 22 | class BaseQueue(Protocol[_T]): 23 | """ 24 | A base queue protocol that includes all except blocking methods. 25 | 26 | Can be useful when you need to make sure that none of the methods will 27 | block (except for the underlying lock). 28 | """ 29 | 30 | __slots__ = () 31 | 32 | def peekable(self) -> bool: 33 | """ 34 | Return :data:`True` if the queue is peekable, :data:`False` otherwise. 35 | """ 36 | ... 37 | 38 | def clearable(self) -> bool: 39 | """ 40 | Return :data:`True` if the queue is clearable, :data:`False` otherwise. 41 | """ 42 | ... 43 | 44 | def qsize(self) -> int: 45 | """ 46 | Return the number of items in the queue. 47 | 48 | This method is provided for compatibility with the standard queues. 49 | Note, ``queue.qsize() > 0`` does not guarantee that subsequent get/peek 50 | calls will not block. Similarly, ``queue.qsize() < queue.maxsize`` does 51 | not guarantee that subsequent put calls will not block. 52 | """ 53 | ... 54 | 55 | def empty(self) -> bool: 56 | """ 57 | Return :data:`True` if the queue is empty, :data:`False` otherwise. 58 | 59 | This method is provided for compatibility with the standard queues. Use 60 | ``queue.qsize() <= 0`` as a direct substitute, but be aware that either 61 | approach risks a race condition where the queue can grow before the 62 | result of :meth:`empty` or :meth:`qsize` can be used. 63 | 64 | To create code that needs to wait for all queued tasks to be completed, 65 | the preferred technique is to use the join methods. 66 | """ 67 | ... 68 | 69 | def full(self) -> bool: 70 | """ 71 | Return :data:`True` if the queue is full, :data:`False` otherwise. 72 | 73 | This method is provided for compatibility with the standard queues. Use 74 | ``queue.qsize() >= queue.maxsize`` as a direct substitute, but be aware 75 | that either approach risks a race condition where the queue can shrink 76 | before the result of :meth:`full` or :meth:`qsize` can be used. 77 | """ 78 | ... 79 | 80 | def put_nowait(self, item: _T) -> None: 81 | """ 82 | Put *item* into the queue without blocking. 83 | 84 | Only put (enqueue) the item if a free slot is immediately available. 85 | 86 | Raises: 87 | QueueFull: 88 | if the queue is full. 89 | QueueShutDown: 90 | if the queue has been shut down. 91 | """ 92 | ... 93 | 94 | def get_nowait(self) -> _T: 95 | """ 96 | Remove and return an item from the queue without blocking. 97 | 98 | Only get (dequeue) an item if one is immediately available. 99 | 100 | Raises: 101 | QueueEmpty: 102 | if the queue is empty. 103 | QueueShutDown: 104 | if the queue has been shut down and is empty, or if the queue has 105 | been shut down immediately. 106 | """ 107 | ... 108 | 109 | def peek_nowait(self) -> _T: 110 | """ 111 | Return an item from the queue without blocking. 112 | 113 | Only peek (front) an item if one is immediately available. 114 | 115 | Raises: 116 | QueueEmpty: 117 | if the queue is empty. 118 | QueueShutDown: 119 | if the queue has been shut down and is empty, or if the queue has 120 | been shut down immediately. 121 | UnsupportedOperation: 122 | if the queue is not peekable. 123 | """ 124 | ... 125 | 126 | def task_done(self, count: int = 1) -> None: 127 | """ 128 | Indicate that a formerly enqueued task is complete. 129 | 130 | Used by queue consumers. For each get call used to fetch a task, 131 | a subsequent call to :meth:`task_done` tells the queue that the 132 | processing on the task is complete. 133 | 134 | If a join call is currently blocking, it will resume when all items 135 | have been processed (meaning that a :meth:`task_done` call was received 136 | for every item that had been put into the queue). 137 | 138 | Args: 139 | count: 140 | Number of completed tasks (useful for flattened queues). 141 | 142 | Raises: 143 | ValueError: 144 | if called more times than there were items placed in the queue. 145 | """ 146 | ... 147 | 148 | def shutdown(self, immediate: bool = False) -> None: 149 | """ 150 | Put the queue into a shutdown mode. 151 | 152 | The queue can no longer grow. Future calls to the put methods raise 153 | :exc:`QueueShutDown`. Currently blocked callers of the put methods will 154 | be unblocked and will raise :exc:`QueueShutDown` in the formerly 155 | blocked thread/task. 156 | 157 | Once the queue is empty, the get/peek methods will also raise 158 | :exc:`QueueShutDown`. 159 | 160 | Args: 161 | immediate: 162 | If set to :data:`True`, the queue is drained to be completely 163 | empty, and :meth:`task_done` is called for each item removed from 164 | the queue (but joiners are unblocked regardless of the number of 165 | unfinished tasks). 166 | """ 167 | ... 168 | 169 | def clear(self) -> None: 170 | """ 171 | Clear all items from the queue atomically. 172 | 173 | Also calls :meth:`task_done` for each removed item. 174 | 175 | Raises: 176 | UnsupportedOperation: 177 | if the queue is not clearable. 178 | """ 179 | ... 180 | 181 | @property 182 | def unfinished_tasks(self) -> int: 183 | """ 184 | The current number of tasks remaining to be processed. 185 | 186 | See the :meth:`task_done` method. 187 | """ 188 | ... 189 | 190 | @property 191 | def is_shutdown(self) -> bool: 192 | """ 193 | A boolean that is :data:`True` if the queue has been shut down, 194 | :data:`False` otherwise. 195 | 196 | See the :meth:`shutdown` method. 197 | """ 198 | ... 199 | 200 | @property 201 | def closed(self) -> bool: 202 | """ 203 | A boolean that is :data:`True` if the queue has been closed, 204 | :data:`False` otherwise. 205 | 206 | This property is provided for compatibility with the Janus queues. Use 207 | :attr:`queue.is_shutdown ` as a direct substitute. 208 | """ 209 | ... 210 | 211 | @property 212 | def maxsize(self) -> int: 213 | """ 214 | The maximum number of items which the queue can hold. 215 | 216 | It can be changed dynamically by setting the attribute. 217 | """ 218 | ... 219 | 220 | @maxsize.setter 221 | def maxsize(self, value: int) -> None: ... 222 | 223 | 224 | class MixedQueue(BaseQueue[_T], Protocol[_T]): 225 | """ 226 | A mixed queue protocol that includes both types of blocking methods via 227 | prefixes. 228 | 229 | Provides specialized proxies via properties. 230 | """ 231 | 232 | __slots__ = () 233 | 234 | def sync_put( 235 | self, 236 | item: _T, 237 | block: bool = True, 238 | timeout: float | None = None, 239 | ) -> None: 240 | """ 241 | Put *item* into the queue. 242 | 243 | Args: 244 | block: 245 | Unless set to :data:`False`, the method will block if necessary 246 | until a free slot is available. Otherwise, ``timeout=0`` is 247 | implied. 248 | timeout: 249 | If set to a non-negative number, the method will block at most 250 | *timeout* seconds and raise the :exc:`QueueFull` exception if no 251 | free slot was available within that time. 252 | 253 | Raises: 254 | QueueFull: 255 | if the queue is full and the timeout has expired. 256 | QueueShutDown: 257 | if the queue has been shut down. 258 | ValueError: 259 | if *timeout* is negative or :data:`NaN `. 260 | """ 261 | ... 262 | 263 | async def async_put(self, item: _T) -> None: 264 | """ 265 | Put *item* into the queue. 266 | 267 | If the queue is full, wait until a free slot is available. 268 | 269 | Raises: 270 | QueueShutDown: 271 | if the queue has been shut down. 272 | """ 273 | ... 274 | 275 | def sync_get( 276 | self, 277 | block: bool = True, 278 | timeout: float | None = None, 279 | ) -> _T: 280 | """ 281 | Remove and return an item from the queue. 282 | 283 | Args: 284 | block: 285 | Unless set to :data:`False`, the method will block if necessary 286 | until an item is available. Otherwise, ``timeout=0`` is implied. 287 | timeout: 288 | If set to a non-negative number, the method will block at most 289 | *timeout* seconds and raise the :exc:`QueueEmpty` exception if no 290 | item was available within that time. 291 | 292 | Raises: 293 | QueueEmpty: 294 | if the queue is empty and the timeout has expired. 295 | QueueShutDown: 296 | if the queue has been shut down and is empty, or if the queue has 297 | been shut down immediately. 298 | ValueError: 299 | if *timeout* is negative or :data:`NaN `. 300 | """ 301 | ... 302 | 303 | async def async_get(self) -> _T: 304 | """ 305 | Remove and return an item from the queue. 306 | 307 | If the queue is empty, wait until an item is available. 308 | 309 | Raises: 310 | QueueShutDown: 311 | if the queue has been shut down and is empty, or if the queue has 312 | been shut down immediately. 313 | """ 314 | ... 315 | 316 | def sync_peek( 317 | self, 318 | block: bool = True, 319 | timeout: float | None = None, 320 | ) -> _T: 321 | """ 322 | Return an item from the queue without removing it. 323 | 324 | Args: 325 | block: 326 | Unless set to :data:`False`, the method will block if necessary 327 | until an item is available. Otherwise, ``timeout=0`` is implied. 328 | timeout: 329 | If set to a non-negative number, the method will block at most 330 | *timeout* seconds and raise the :exc:`QueueEmpty` exception if no 331 | item was available within that time. 332 | 333 | Raises: 334 | QueueEmpty: 335 | if the queue is empty and the timeout has expired. 336 | QueueShutDown: 337 | if the queue has been shut down and is empty, or if the queue has 338 | been shut down immediately. 339 | UnsupportedOperation: 340 | if the queue is not peekable. 341 | ValueError: 342 | if *timeout* is negative or :data:`NaN `. 343 | """ 344 | ... 345 | 346 | async def async_peek(self) -> _T: 347 | """ 348 | Return an item from the queue without removing it. 349 | 350 | If the queue is empty, wait until an item is available. 351 | 352 | Raises: 353 | QueueShutDown: 354 | if the queue has been shut down and is empty, or if the queue has 355 | been shut down immediately. 356 | UnsupportedOperation: 357 | if the queue is not peekable. 358 | """ 359 | ... 360 | 361 | def sync_join(self) -> None: 362 | """ 363 | Block until all items in the queue have been gotten and processed. 364 | 365 | The count of unfinished tasks goes up whenever an item is added to the 366 | queue. The count goes down whenever a consumer calls 367 | :meth:`~BaseQueue.task_done` to indicate that the item was retrieved 368 | and all work on it is complete. 369 | 370 | When the count of unfinished tasks drops to zero, the caller unblocks. 371 | """ 372 | ... 373 | 374 | async def async_join(self) -> None: 375 | """ 376 | Block until all items in the queue have been gotten and processed. 377 | 378 | The count of unfinished tasks goes up whenever an item is added to the 379 | queue. The count goes down whenever a consumer calls 380 | :meth:`~BaseQueue.task_done` to indicate that the item was retrieved 381 | and all work on it is complete. 382 | 383 | When the count of unfinished tasks drops to zero, the caller unblocks. 384 | """ 385 | ... 386 | 387 | @property 388 | def sync_q(self) -> SyncQueue[_T]: 389 | """ 390 | An interface compatible with the standard queues from the :mod:`queue` 391 | module. 392 | """ 393 | ... 394 | 395 | @property 396 | def async_q(self) -> AsyncQueue[_T]: 397 | """ 398 | An interface compatible with the standard queues from the 399 | :mod:`asyncio` module. 400 | """ 401 | ... 402 | 403 | 404 | class SyncQueue(BaseQueue[_T], Protocol[_T]): 405 | """ 406 | A synchronous queue protocol that covers the standard queues' interface 407 | from the :mod:`queue` module. 408 | 409 | Compliant with the Python API version 3.13. 410 | """ 411 | 412 | __slots__ = () 413 | 414 | @copydoc(MixedQueue.sync_put) 415 | def put( 416 | self, 417 | item: _T, 418 | block: bool = True, 419 | timeout: float | None = None, 420 | ) -> None: ... 421 | @copydoc(MixedQueue.sync_get) 422 | def get(self, block: bool = True, timeout: float | None = None) -> _T: ... 423 | @copydoc(MixedQueue.sync_peek) 424 | def peek(self, block: bool = True, timeout: float | None = None) -> _T: ... 425 | @copydoc(MixedQueue.sync_join) 426 | def join(self) -> None: ... 427 | 428 | 429 | class AsyncQueue(BaseQueue[_T], Protocol[_T]): 430 | """ 431 | An asynchronous queue protocol that covers the standard queues' interface 432 | from the :mod:`asyncio` module. 433 | 434 | Compliant with the Python API version 3.13. 435 | """ 436 | 437 | __slots__ = () 438 | 439 | @copydoc(MixedQueue.async_put) 440 | async def put(self, item: _T) -> None: ... 441 | @copydoc(MixedQueue.async_get) 442 | async def get(self) -> _T: ... 443 | @copydoc(MixedQueue.async_peek) 444 | async def peek(self) -> _T: ... 445 | @copydoc(MixedQueue.async_join) 446 | async def join(self) -> None: ... 447 | -------------------------------------------------------------------------------- /LICENSES/CC-BY-4.0.txt: -------------------------------------------------------------------------------- 1 | Creative Commons Attribution 4.0 International 2 | 3 | Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. 4 | 5 | Using Creative Commons Public Licenses 6 | 7 | Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. 8 | 9 | Considerations for licensors: Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. More considerations for licensors. 10 | 11 | Considerations for the public: By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. More considerations for the public. 12 | 13 | Creative Commons Attribution 4.0 International Public License 14 | 15 | By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. 16 | 17 | Section 1 – Definitions. 18 | 19 | a. Adapted Material means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. 20 | 21 | b. Adapter's License means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. 22 | 23 | c. Copyright and Similar Rights means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. 24 | 25 | d. Effective Technological Measures means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. 26 | 27 | e. Exceptions and Limitations means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. 28 | 29 | f. Licensed Material means the artistic or literary work, database, or other material to which the Licensor applied this Public License. 30 | 31 | g. Licensed Rights means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. 32 | 33 | h. Licensor means the individual(s) or entity(ies) granting rights under this Public License. 34 | 35 | i. Share means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. 36 | 37 | j. Sui Generis Database Rights means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. 38 | 39 | k. You means the individual or entity exercising the Licensed Rights under this Public License. Your has a corresponding meaning. 40 | 41 | Section 2 – Scope. 42 | 43 | a. License grant. 44 | 45 | 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: 46 | 47 | A. reproduce and Share the Licensed Material, in whole or in part; and 48 | 49 | B. produce, reproduce, and Share Adapted Material. 50 | 51 | 2. Exceptions and Limitations. For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. 52 | 53 | 3. Term. The term of this Public License is specified in Section 6(a). 54 | 55 | 4. Media and formats; technical modifications allowed. The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. 56 | 57 | 5. Downstream recipients. 58 | 59 | A. Offer from the Licensor – Licensed Material. Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. 60 | 61 | B. No downstream restrictions. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. 62 | 63 | 6. No endorsement. Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). 64 | 65 | b. Other rights. 66 | 67 | 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. 68 | 69 | 2. Patent and trademark rights are not licensed under this Public License. 70 | 71 | 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties. 72 | 73 | Section 3 – License Conditions. 74 | 75 | Your exercise of the Licensed Rights is expressly made subject to the following conditions. 76 | 77 | a. Attribution. 78 | 79 | 1. If You Share the Licensed Material (including in modified form), You must: 80 | 81 | A. retain the following if it is supplied by the Licensor with the Licensed Material: 82 | 83 | i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); 84 | 85 | ii. a copyright notice; 86 | 87 | iii. a notice that refers to this Public License; 88 | 89 | iv. a notice that refers to the disclaimer of warranties; 90 | 91 | v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; 92 | 93 | B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and 94 | 95 | C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. 96 | 97 | 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. 98 | 99 | 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. 100 | 101 | 4. If You Share Adapted Material You produce, the Adapter's License You apply must not prevent recipients of the Adapted Material from complying with this Public License. 102 | 103 | Section 4 – Sui Generis Database Rights. 104 | 105 | Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: 106 | 107 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database; 108 | 109 | b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material; and 110 | 111 | c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. 112 | For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. 113 | 114 | Section 5 – Disclaimer of Warranties and Limitation of Liability. 115 | 116 | a. Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You. 117 | 118 | b. To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You. 119 | 120 | c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. 121 | 122 | Section 6 – Term and Termination. 123 | 124 | a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. 125 | 126 | b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: 127 | 128 | 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or 129 | 130 | 2. upon express reinstatement by the Licensor. 131 | 132 | c. For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. 133 | 134 | d. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. 135 | 136 | e. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. 137 | 138 | Section 7 – Other Terms and Conditions. 139 | 140 | a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. 141 | 142 | b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. 143 | 144 | Section 8 – Interpretation. 145 | 146 | a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. 147 | 148 | b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. 149 | 150 | c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. 151 | 152 | d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. 153 | 154 | Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at creativecommons.org/policies, Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. 155 | 156 | Creative Commons may be contacted at creativecommons.org. 157 | -------------------------------------------------------------------------------- /tests/culsans/test_asyncio.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # SPDX-FileCopyrightText: 2013 Python Software Foundation 4 | # SPDX-FileCopyrightText: 2024 Ilya Egorov <0x42005e1f@gmail.com> 5 | # 6 | # SPDX-License-Identifier: PSF-2.0 7 | 8 | # Brief summary of changes: 9 | # 10 | # * asyncio replaced by culsans 11 | # * added checkpoint support 12 | # * added Python<3.10 support 13 | # * removed GenericAlias tests 14 | # * removed representation tests 15 | # 16 | # Relevant for python/cpython#5892853 17 | 18 | import asyncio 19 | import unittest 20 | 21 | # XXX culsans change: test imports replaced by culsans import 22 | import culsans 23 | 24 | # XXX culsans change: added checkpoint support 25 | try: 26 | from aiologic.lowlevel import async_checkpoint_enabled 27 | except ImportError: # aiologic<0.15.0 28 | from aiologic.lowlevel import asyncio_checkpoints_cvar 29 | 30 | async_checkpoint_enabled = asyncio_checkpoints_cvar.get 31 | 32 | 33 | class QueueBasicTests(unittest.IsolatedAsyncioTestCase): 34 | 35 | async def test_empty(self): 36 | # XXX culsans change: +.async_q 37 | q = culsans.Queue().async_q 38 | self.assertTrue(q.empty()) 39 | await q.put(1) 40 | self.assertFalse(q.empty()) 41 | self.assertEqual(1, await q.get()) 42 | self.assertTrue(q.empty()) 43 | 44 | async def test_full(self): 45 | # XXX culsans change: +.async_q 46 | q = culsans.Queue().async_q 47 | self.assertFalse(q.full()) 48 | 49 | # XXX culsans change: +.async_q 50 | q = culsans.Queue(maxsize=1).async_q 51 | await q.put(1) 52 | self.assertTrue(q.full()) 53 | 54 | async def test_order(self): 55 | # XXX culsans change: +.async_q 56 | q = culsans.Queue().async_q 57 | for i in [1, 3, 2]: 58 | await q.put(i) 59 | 60 | items = [await q.get() for _ in range(3)] 61 | self.assertEqual([1, 3, 2], items) 62 | 63 | async def test_maxsize(self): 64 | # XXX culsans change: +.async_q 65 | q = culsans.Queue(maxsize=2).async_q 66 | self.assertEqual(2, q.maxsize) 67 | have_been_put = [] 68 | 69 | async def putter(): 70 | for i in range(3): 71 | await q.put(i) 72 | have_been_put.append(i) 73 | return True 74 | 75 | t = asyncio.create_task(putter()) 76 | # XXX culsans change: added checkpoint support 77 | for _ in range(4 if async_checkpoint_enabled() else 2): 78 | await asyncio.sleep(0) 79 | 80 | # The putter is blocked after putting two items. 81 | self.assertEqual([0, 1], have_been_put) 82 | self.assertEqual(0, await q.get()) 83 | 84 | # Let the putter resume and put last item. 85 | await asyncio.sleep(0) 86 | self.assertEqual([0, 1, 2], have_been_put) 87 | self.assertEqual(1, await q.get()) 88 | self.assertEqual(2, await q.get()) 89 | 90 | self.assertTrue(t.done()) 91 | self.assertTrue(t.result()) 92 | 93 | 94 | class QueueGetTests(unittest.IsolatedAsyncioTestCase): 95 | 96 | async def test_blocking_get(self): 97 | # XXX culsans change: +.async_q 98 | q = culsans.Queue().async_q 99 | q.put_nowait(1) 100 | 101 | self.assertEqual(1, await q.get()) 102 | 103 | async def test_get_with_putters(self): 104 | # XXX culsans change: +.async_q 105 | q = culsans.Queue(1).async_q 106 | await q.put(1) 107 | 108 | # XXX culsans change: call put() instead of directly inserting future 109 | waiter = asyncio.create_task(q.put(2)) 110 | # XXX culsans change: switch to put() 111 | await asyncio.sleep(0) 112 | 113 | self.assertEqual(1, await q.get()) 114 | # XXX culsans change: switch to put() 115 | await asyncio.sleep(0) 116 | self.assertTrue(waiter.done()) 117 | self.assertIsNone(waiter.result()) 118 | 119 | async def test_blocking_get_wait(self): 120 | loop = asyncio.get_running_loop() 121 | # XXX culsans change: +.async_q 122 | q = culsans.Queue().async_q 123 | started = asyncio.Event() 124 | finished = False 125 | 126 | async def queue_get(): 127 | nonlocal finished 128 | started.set() 129 | res = await q.get() 130 | finished = True 131 | return res 132 | 133 | queue_get_task = asyncio.create_task(queue_get()) 134 | await started.wait() 135 | self.assertFalse(finished) 136 | loop.call_later(0.01, q.put_nowait, 1) 137 | res = await queue_get_task 138 | self.assertTrue(finished) 139 | self.assertEqual(1, res) 140 | 141 | def test_nonblocking_get(self): 142 | # XXX culsans change: +.async_q 143 | q = culsans.Queue().async_q 144 | q.put_nowait(1) 145 | self.assertEqual(1, q.get_nowait()) 146 | 147 | def test_nonblocking_get_exception(self): 148 | # XXX culsans change: +.async_q 149 | q = culsans.Queue().async_q 150 | # XXX culsans change: QueueEmpty -> AsyncQueueEmpty 151 | self.assertRaises(culsans.AsyncQueueEmpty, q.get_nowait) 152 | 153 | async def test_get_cancelled_race(self): 154 | # XXX culsans change: +.async_q 155 | q = culsans.Queue().async_q 156 | 157 | t1 = asyncio.create_task(q.get()) 158 | t2 = asyncio.create_task(q.get()) 159 | 160 | await asyncio.sleep(0) 161 | t1.cancel() 162 | await asyncio.sleep(0) 163 | self.assertTrue(t1.done()) 164 | await q.put('a') 165 | await asyncio.sleep(0) 166 | self.assertEqual('a', await t2) 167 | 168 | async def test_get_with_waiting_putters(self): 169 | # XXX culsans change: +.async_q 170 | q = culsans.Queue(maxsize=1).async_q 171 | asyncio.create_task(q.put('a')) 172 | asyncio.create_task(q.put('b')) 173 | self.assertEqual(await q.get(), 'a') 174 | self.assertEqual(await q.get(), 'b') 175 | 176 | async def test_why_are_getters_waiting(self): 177 | async def consumer(queue, num_expected): 178 | for _ in range(num_expected): 179 | await queue.get() 180 | 181 | async def producer(queue, num_items): 182 | for i in range(num_items): 183 | await queue.put(i) 184 | 185 | producer_num_items = 5 186 | 187 | # XXX culsans change: +.async_q 188 | q = culsans.Queue(1).async_q 189 | # XXX culsans change: added Python<3.10 support 190 | await asyncio.gather( 191 | producer(q, producer_num_items), 192 | consumer(q, producer_num_items), 193 | ) 194 | 195 | async def test_cancelled_getters_not_being_held_in_self_getters(self): 196 | # XXX culsans change: +.async_q 197 | queue = culsans.Queue(maxsize=5).async_q 198 | 199 | # XXX culsans change: +asyncio. 200 | with self.assertRaises(asyncio.TimeoutError): 201 | await asyncio.wait_for(queue.get(), 0.1) 202 | 203 | # XXX culsans change: +.wrapped 204 | self.assertEqual(queue.wrapped.getting, 0) 205 | 206 | 207 | class QueuePutTests(unittest.IsolatedAsyncioTestCase): 208 | 209 | async def test_blocking_put(self): 210 | # XXX culsans change: +.async_q 211 | q = culsans.Queue().async_q 212 | 213 | # No maxsize, won't block. 214 | await q.put(1) 215 | self.assertEqual(1, await q.get()) 216 | 217 | async def test_blocking_put_wait(self): 218 | # XXX culsans change: +.async_q 219 | q = culsans.Queue(maxsize=1).async_q 220 | started = asyncio.Event() 221 | finished = False 222 | 223 | async def queue_put(): 224 | nonlocal finished 225 | started.set() 226 | await q.put(1) 227 | await q.put(2) 228 | finished = True 229 | 230 | loop = asyncio.get_running_loop() 231 | loop.call_later(0.01, q.get_nowait) 232 | queue_put_task = asyncio.create_task(queue_put()) 233 | await started.wait() 234 | self.assertFalse(finished) 235 | await queue_put_task 236 | self.assertTrue(finished) 237 | 238 | def test_nonblocking_put(self): 239 | # XXX culsans change: +.async_q 240 | q = culsans.Queue().async_q 241 | q.put_nowait(1) 242 | self.assertEqual(1, q.get_nowait()) 243 | 244 | async def test_get_cancel_drop_one_pending_reader(self): 245 | # XXX culsans change: +.async_q 246 | q = culsans.Queue().async_q 247 | 248 | reader = asyncio.create_task(q.get()) 249 | 250 | await asyncio.sleep(0) 251 | 252 | q.put_nowait(1) 253 | q.put_nowait(2) 254 | reader.cancel() 255 | 256 | try: 257 | await reader 258 | except asyncio.CancelledError: 259 | # try again 260 | reader = asyncio.create_task(q.get()) 261 | await reader 262 | 263 | result = reader.result() 264 | # if we get 2, it means 1 got dropped! 265 | self.assertEqual(1, result) 266 | 267 | async def test_get_cancel_drop_many_pending_readers(self): 268 | # XXX culsans change: +.async_q 269 | q = culsans.Queue().async_q 270 | 271 | # XXX culsans change: added Python<3.10 support 272 | reader1 = asyncio.create_task(q.get()) 273 | reader2 = asyncio.create_task(q.get()) 274 | reader3 = asyncio.create_task(q.get()) 275 | 276 | await asyncio.sleep(0) 277 | 278 | q.put_nowait(1) 279 | q.put_nowait(2) 280 | reader1.cancel() 281 | 282 | with self.assertRaises(asyncio.CancelledError): 283 | await reader1 284 | 285 | await reader3 286 | 287 | # It is undefined in which order concurrent readers receive results. 288 | self.assertEqual({reader2.result(), reader3.result()}, {1, 2}) 289 | 290 | async def test_put_cancel_drop(self): 291 | q = culsans.Queue(1).async_q 292 | 293 | q.put_nowait(1) 294 | 295 | # putting a second item in the queue has to block (qsize=1) 296 | writer = asyncio.create_task(q.put(2)) 297 | await asyncio.sleep(0) 298 | 299 | value1 = q.get_nowait() 300 | self.assertEqual(value1, 1) 301 | 302 | writer.cancel() 303 | try: 304 | await writer 305 | except asyncio.CancelledError: 306 | # try again 307 | writer = asyncio.create_task(q.put(2)) 308 | await writer 309 | 310 | value2 = q.get_nowait() 311 | self.assertEqual(value2, 2) 312 | self.assertEqual(q.qsize(), 0) 313 | 314 | def test_nonblocking_put_exception(self): 315 | # XXX culsans change: +.async_q 316 | q = culsans.Queue(maxsize=1, ).async_q 317 | q.put_nowait(1) 318 | # XXX culsans change: QueueFull -> AsyncQueueFull 319 | self.assertRaises(culsans.AsyncQueueFull, q.put_nowait, 2) 320 | 321 | async def test_float_maxsize(self): 322 | # XXX culsans change: +.async_q 323 | q = culsans.Queue(maxsize=1.3, ).async_q 324 | q.put_nowait(1) 325 | q.put_nowait(2) 326 | self.assertTrue(q.full()) 327 | # XXX culsans change: QueueFull -> AsyncQueueFull 328 | self.assertRaises(culsans.AsyncQueueFull, q.put_nowait, 3) 329 | 330 | # XXX culsans change: +.async_q 331 | q = culsans.Queue(maxsize=1.3, ).async_q 332 | 333 | await q.put(1) 334 | await q.put(2) 335 | self.assertTrue(q.full()) 336 | 337 | async def test_put_cancelled(self): 338 | q = culsans.Queue().async_q 339 | 340 | async def queue_put(): 341 | await q.put(1) 342 | return True 343 | 344 | t = asyncio.create_task(queue_put()) 345 | 346 | self.assertEqual(1, await q.get()) 347 | self.assertTrue(t.done()) 348 | self.assertTrue(t.result()) 349 | 350 | async def test_put_cancelled_race(self): 351 | # XXX culsans change: +.async_q 352 | q = culsans.Queue(maxsize=1).async_q 353 | 354 | put_a = asyncio.create_task(q.put('a')) 355 | put_b = asyncio.create_task(q.put('b')) 356 | put_c = asyncio.create_task(q.put('X')) 357 | 358 | # XXX culsans change: added checkpoint support 359 | for _ in range(2 if async_checkpoint_enabled() else 1): 360 | await asyncio.sleep(0) 361 | self.assertTrue(put_a.done()) 362 | self.assertFalse(put_b.done()) 363 | 364 | put_c.cancel() 365 | await asyncio.sleep(0) 366 | self.assertTrue(put_c.done()) 367 | self.assertEqual(q.get_nowait(), 'a') 368 | await asyncio.sleep(0) 369 | self.assertEqual(q.get_nowait(), 'b') 370 | 371 | await put_b 372 | 373 | async def test_put_with_waiting_getters(self): 374 | # XXX culsans change: +.async_q 375 | q = culsans.Queue().async_q 376 | t = asyncio.create_task(q.get()) 377 | await asyncio.sleep(0) 378 | await q.put('a') 379 | self.assertEqual(await t, 'a') 380 | 381 | async def test_why_are_putters_waiting(self): 382 | # XXX culsans change: +.async_q 383 | queue = culsans.Queue(2).async_q 384 | 385 | async def putter(item): 386 | await queue.put(item) 387 | 388 | async def getter(): 389 | await asyncio.sleep(0) 390 | num = queue.qsize() 391 | for _ in range(num): 392 | queue.get_nowait() 393 | 394 | # XXX culsans change: added Python<3.10 support 395 | await asyncio.gather( 396 | putter(0), 397 | putter(1), 398 | putter(2), 399 | putter(3), 400 | getter(), 401 | ) 402 | 403 | async def test_cancelled_puts_not_being_held_in_self_putters(self): 404 | # Full queue. 405 | # XXX culsans change: +.async_q 406 | queue = culsans.Queue(maxsize=1).async_q 407 | queue.put_nowait(1) 408 | 409 | # Task waiting for space to put an item in the queue. 410 | put_task = asyncio.create_task(queue.put(1)) 411 | await asyncio.sleep(0) 412 | 413 | # Check that the putter is correctly removed from queue._putters when 414 | # the task is canceled. 415 | # XXX culsans change: +.wrapped 416 | self.assertEqual(queue.wrapped.putting, 1) 417 | put_task.cancel() 418 | with self.assertRaises(asyncio.CancelledError): 419 | await put_task 420 | # XXX culsans change: +.wrapped 421 | self.assertEqual(queue.wrapped.putting, 0) 422 | 423 | async def test_cancelled_put_silence_value_error_exception(self): 424 | # Full Queue. 425 | # XXX culsans change: +.async_q 426 | queue = culsans.Queue(1).async_q 427 | queue.put_nowait(1) 428 | 429 | # Task waiting for space to put a item in the queue. 430 | put_task = asyncio.create_task(queue.put(1)) 431 | await asyncio.sleep(0) 432 | 433 | # get_nowait() remove the future of put_task from queue._putters. 434 | queue.get_nowait() 435 | # When canceled, queue.put is going to remove its future from 436 | # self._putters but it was removed previously by queue.get_nowait(). 437 | put_task.cancel() 438 | 439 | # The ValueError exception triggered by queue._putters.remove(putter) 440 | # inside queue.put should be silenced. 441 | # If the ValueError is silenced we should catch a CancelledError. 442 | with self.assertRaises(asyncio.CancelledError): 443 | await put_task 444 | 445 | 446 | class LifoQueueTests(unittest.IsolatedAsyncioTestCase): 447 | 448 | async def test_order(self): 449 | # XXX culsans change: +.async_q 450 | q = culsans.LifoQueue().async_q 451 | for i in [1, 3, 2]: 452 | await q.put(i) 453 | 454 | items = [await q.get() for _ in range(3)] 455 | self.assertEqual([2, 3, 1], items) 456 | 457 | 458 | class PriorityQueueTests(unittest.IsolatedAsyncioTestCase): 459 | 460 | async def test_order(self): 461 | # XXX culsans change: +.async_q 462 | q = culsans.PriorityQueue().async_q 463 | for i in [1, 3, 2]: 464 | await q.put(i) 465 | 466 | items = [await q.get() for _ in range(3)] 467 | self.assertEqual([1, 2, 3], items) 468 | 469 | 470 | class _QueueJoinTestMixin: 471 | 472 | q_class = None 473 | 474 | def test_task_done_underflow(self): 475 | # XXX culsans change: +.async_q 476 | q = self.q_class().async_q 477 | self.assertRaises(ValueError, q.task_done) 478 | 479 | async def test_task_done(self): 480 | # XXX culsans change: +.async_q 481 | q = self.q_class().async_q 482 | for i in range(100): 483 | q.put_nowait(i) 484 | 485 | accumulator = 0 486 | 487 | # Two workers get items from the queue and call task_done after each. 488 | # Join the queue and assert all items have been processed. 489 | running = True 490 | 491 | async def worker(): 492 | nonlocal accumulator 493 | 494 | while running: 495 | item = await q.get() 496 | accumulator += item 497 | q.task_done() 498 | 499 | # XXX culsans change: added Python<3.10 support 500 | tasks = [asyncio.create_task(worker()) 501 | for index in range(2)] 502 | 503 | await q.join() 504 | self.assertEqual(sum(range(100)), accumulator) 505 | 506 | # close running generators 507 | running = False 508 | for i in range(len(tasks)): 509 | q.put_nowait(0) 510 | # XXX culsans change: added Python<3.10 support 511 | await asyncio.wait(tasks) 512 | 513 | async def test_join_empty_queue(self): 514 | # XXX culsans change: +.async_q 515 | q = self.q_class().async_q 516 | 517 | # Test that a queue join()s successfully, and before anything else 518 | # (done twice for insurance). 519 | 520 | await q.join() 521 | await q.join() 522 | 523 | 524 | class QueueJoinTests(_QueueJoinTestMixin, unittest.IsolatedAsyncioTestCase): 525 | # XXX culsans change: asyncio -> culsans 526 | q_class = culsans.Queue 527 | 528 | 529 | class LifoQueueJoinTests(_QueueJoinTestMixin, unittest.IsolatedAsyncioTestCase): 530 | # XXX culsans change: asyncio -> culsans 531 | q_class = culsans.LifoQueue 532 | 533 | 534 | class PriorityQueueJoinTests(_QueueJoinTestMixin, unittest.IsolatedAsyncioTestCase): 535 | # XXX culsans change: asyncio -> culsans 536 | q_class = culsans.PriorityQueue 537 | 538 | 539 | class _QueueShutdownTestMixin: 540 | q_class = None 541 | 542 | def assertRaisesShutdown(self, msg="Didn't appear to shut-down queue"): 543 | # XXX culsans change: QueueShutDown -> AsyncQueueShutDown 544 | return self.assertRaises(culsans.AsyncQueueShutDown, msg=msg) 545 | 546 | async def test_shutdown_empty(self): 547 | # Test shutting down an empty queue 548 | 549 | # Setup empty queue, and join() and get() tasks 550 | # XXX culsans change: +.async_q 551 | q = self.q_class().async_q 552 | loop = asyncio.get_running_loop() 553 | get_task = loop.create_task(q.get()) 554 | await asyncio.sleep(0) # want get task pending before shutdown 555 | 556 | # Perform shut-down 557 | q.shutdown(immediate=False) # unfinished tasks: 0 -> 0 558 | 559 | self.assertEqual(q.qsize(), 0) 560 | 561 | # Ensure join() task successfully finishes 562 | await q.join() 563 | 564 | # Ensure get() task is finished, and raised ShutDown 565 | await asyncio.sleep(0) 566 | self.assertTrue(get_task.done()) 567 | with self.assertRaisesShutdown(): 568 | await get_task 569 | 570 | # Ensure put() and get() raise ShutDown 571 | with self.assertRaisesShutdown(): 572 | await q.put("data") 573 | with self.assertRaisesShutdown(): 574 | q.put_nowait("data") 575 | 576 | with self.assertRaisesShutdown(): 577 | await q.get() 578 | with self.assertRaisesShutdown(): 579 | q.get_nowait() 580 | 581 | async def test_shutdown_nonempty(self): 582 | # Test shutting down a non-empty queue 583 | 584 | # Setup full queue with 1 item, and join() and put() tasks 585 | # XXX culsans change: +.async_q 586 | q = self.q_class(maxsize=1).async_q 587 | loop = asyncio.get_running_loop() 588 | 589 | q.put_nowait("data") 590 | join_task = loop.create_task(q.join()) 591 | put_task = loop.create_task(q.put("data2")) 592 | 593 | # Ensure put() task is not finished 594 | await asyncio.sleep(0) 595 | self.assertFalse(put_task.done()) 596 | 597 | # Perform shut-down 598 | q.shutdown(immediate=False) # unfinished tasks: 1 -> 1 599 | 600 | self.assertEqual(q.qsize(), 1) 601 | 602 | # Ensure put() task is finished, and raised ShutDown 603 | await asyncio.sleep(0) 604 | self.assertTrue(put_task.done()) 605 | with self.assertRaisesShutdown(): 606 | await put_task 607 | 608 | # Ensure get() succeeds on enqueued item 609 | self.assertEqual(await q.get(), "data") 610 | 611 | # Ensure join() task is not finished 612 | await asyncio.sleep(0) 613 | self.assertFalse(join_task.done()) 614 | 615 | # Ensure put() and get() raise ShutDown 616 | with self.assertRaisesShutdown(): 617 | await q.put("data") 618 | with self.assertRaisesShutdown(): 619 | q.put_nowait("data") 620 | 621 | with self.assertRaisesShutdown(): 622 | await q.get() 623 | with self.assertRaisesShutdown(): 624 | q.get_nowait() 625 | 626 | # Ensure there is 1 unfinished task, and join() task succeeds 627 | q.task_done() 628 | 629 | await asyncio.sleep(0) 630 | self.assertTrue(join_task.done()) 631 | await join_task 632 | 633 | with self.assertRaises( 634 | ValueError, msg="Didn't appear to mark all tasks done" 635 | ): 636 | q.task_done() 637 | 638 | async def test_shutdown_immediate(self): 639 | # Test immediately shutting down a queue 640 | 641 | # Setup queue with 1 item, and a join() task 642 | # XXX culsans change: +.async_q 643 | q = self.q_class().async_q 644 | loop = asyncio.get_running_loop() 645 | q.put_nowait("data") 646 | join_task = loop.create_task(q.join()) 647 | 648 | # Perform shut-down 649 | q.shutdown(immediate=True) # unfinished tasks: 1 -> 0 650 | 651 | self.assertEqual(q.qsize(), 0) 652 | 653 | # Ensure join() task has successfully finished 654 | # XXX culsans change: added checkpoint support 655 | for _ in range(2 if async_checkpoint_enabled() else 1): 656 | await asyncio.sleep(0) 657 | self.assertTrue(join_task.done()) 658 | await join_task 659 | 660 | # Ensure put() and get() raise ShutDown 661 | with self.assertRaisesShutdown(): 662 | await q.put("data") 663 | with self.assertRaisesShutdown(): 664 | q.put_nowait("data") 665 | 666 | with self.assertRaisesShutdown(): 667 | await q.get() 668 | with self.assertRaisesShutdown(): 669 | q.get_nowait() 670 | 671 | # Ensure there are no unfinished tasks 672 | with self.assertRaises( 673 | ValueError, msg="Didn't appear to mark all tasks done" 674 | ): 675 | q.task_done() 676 | 677 | async def test_shutdown_immediate_with_unfinished(self): 678 | # Test immediately shutting down a queue with unfinished tasks 679 | 680 | # Setup queue with 2 items (1 retrieved), and a join() task 681 | # XXX culsans change: +.async_q 682 | q = self.q_class().async_q 683 | loop = asyncio.get_running_loop() 684 | q.put_nowait("data") 685 | q.put_nowait("data") 686 | join_task = loop.create_task(q.join()) 687 | self.assertEqual(await q.get(), "data") 688 | 689 | # Perform shut-down 690 | q.shutdown(immediate=True) # unfinished tasks: 2 -> 1 691 | 692 | self.assertEqual(q.qsize(), 0) 693 | 694 | # Ensure join() task is not finished 695 | await asyncio.sleep(0) 696 | self.assertFalse(join_task.done()) 697 | 698 | # Ensure put() and get() raise ShutDown 699 | with self.assertRaisesShutdown(): 700 | await q.put("data") 701 | with self.assertRaisesShutdown(): 702 | q.put_nowait("data") 703 | 704 | with self.assertRaisesShutdown(): 705 | await q.get() 706 | with self.assertRaisesShutdown(): 707 | q.get_nowait() 708 | 709 | # Ensure there is 1 unfinished task 710 | q.task_done() 711 | with self.assertRaises( 712 | ValueError, msg="Didn't appear to mark all tasks done" 713 | ): 714 | q.task_done() 715 | 716 | # Ensure join() task has successfully finished 717 | await asyncio.sleep(0) 718 | self.assertTrue(join_task.done()) 719 | await join_task 720 | 721 | 722 | class QueueShutdownTests( 723 | _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase 724 | ): 725 | # XXX culsans change: asyncio -> culsans 726 | q_class = culsans.Queue 727 | 728 | 729 | class LifoQueueShutdownTests( 730 | _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase 731 | ): 732 | # XXX culsans change: asyncio -> culsans 733 | q_class = culsans.LifoQueue 734 | 735 | 736 | class PriorityQueueShutdownTests( 737 | _QueueShutdownTestMixin, unittest.IsolatedAsyncioTestCase 738 | ): 739 | # XXX culsans change: asyncio -> culsans 740 | q_class = culsans.PriorityQueue 741 | 742 | 743 | if __name__ == '__main__': 744 | unittest.main() 745 | -------------------------------------------------------------------------------- /tests/culsans/test_queue.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # SPDX-FileCopyrightText: 2002 Python Software Foundation 4 | # SPDX-FileCopyrightText: 2024 Ilya Egorov <0x42005e1f@gmail.com> 5 | # 6 | # SPDX-License-Identifier: PSF-2.0 7 | 8 | # Brief summary of changes: 9 | # 10 | # * queue replaced by culsans 11 | # * removed test dependencies 12 | # * removed C-specific tests 13 | # * removed SimpleQueue tests 14 | # 15 | # Relevant for python/cpython#9017b95 16 | 17 | import threading 18 | import time 19 | import unittest 20 | 21 | # XXX culsans change: test imports replaced by culsans import 22 | import culsans 23 | 24 | QUEUE_SIZE = 5 25 | # XXX culsans change: borrowed from test.support 26 | SHORT_TIMEOUT = 30.0 27 | 28 | def qfull(q): 29 | return q.maxsize > 0 and q.qsize() == q.maxsize 30 | 31 | # XXX culsans change: borrowed from test.support.threading_helper 32 | # SPDX-SnippetBegin 33 | # SPDX-SnippetCopyrightText: 2017 Python Software Foundation 34 | # SPDX-License-Identifier: PSF-2.0 35 | 36 | def join_thread(thread, timeout=None): 37 | """Join a thread. Raise an AssertionError if the thread is still alive 38 | after timeout seconds. 39 | """ 40 | if timeout is None: 41 | timeout = SHORT_TIMEOUT 42 | thread.join(timeout) 43 | if thread.is_alive(): 44 | msg = f"failed to join the thread in {timeout:.1f} seconds" 45 | raise AssertionError(msg) 46 | 47 | # SPDX-SnippetEnd 48 | 49 | # A thread to run a function that unclogs a blocked Queue. 50 | class _TriggerThread(threading.Thread): 51 | def __init__(self, fn, args): 52 | self.fn = fn 53 | self.args = args 54 | self.startedEvent = threading.Event() 55 | threading.Thread.__init__(self) 56 | 57 | def run(self): 58 | # The sleep isn't necessary, but is intended to give the blocking 59 | # function in the main thread a chance at actually blocking before 60 | # we unclog it. But if the sleep is longer than the timeout-based 61 | # tests wait in their blocking functions, those tests will fail. 62 | # So we give them much longer timeout values compared to the 63 | # sleep here (I aimed at 10 seconds for blocking functions -- 64 | # they should never actually wait that long - they should make 65 | # progress as soon as we call self.fn()). 66 | time.sleep(0.1) 67 | self.startedEvent.set() 68 | self.fn(*self.args) 69 | 70 | 71 | # Execute a function that blocks, and in a separate thread, a function that 72 | # triggers the release. Returns the result of the blocking function. Caution: 73 | # block_func must guarantee to block until trigger_func is called, and 74 | # trigger_func must guarantee to change queue state so that block_func can make 75 | # enough progress to return. In particular, a block_func that just raises an 76 | # exception regardless of whether trigger_func is called will lead to 77 | # timing-dependent sporadic failures, and one of those went rarely seen but 78 | # undiagnosed for years. Now block_func must be unexceptional. If block_func 79 | # is supposed to raise an exception, call do_exceptional_blocking_test() 80 | # instead. 81 | 82 | class BlockingTestMixin: 83 | 84 | def do_blocking_test(self, block_func, block_args, trigger_func, trigger_args): 85 | thread = _TriggerThread(trigger_func, trigger_args) 86 | thread.start() 87 | try: 88 | self.result = block_func(*block_args) 89 | # If block_func returned before our thread made the call, we failed! 90 | if not thread.startedEvent.is_set(): 91 | self.fail("blocking function %r appeared not to block" % 92 | block_func) 93 | return self.result 94 | finally: 95 | # XXX culsans change: threading_helper.join_thread -> join_thread 96 | join_thread(thread) # make sure the thread terminates 97 | 98 | # Call this instead if block_func is supposed to raise an exception. 99 | def do_exceptional_blocking_test(self,block_func, block_args, trigger_func, 100 | trigger_args, expected_exception_class): 101 | thread = _TriggerThread(trigger_func, trigger_args) 102 | thread.start() 103 | try: 104 | try: 105 | block_func(*block_args) 106 | except expected_exception_class: 107 | raise 108 | else: 109 | self.fail("expected exception of kind %r" % 110 | expected_exception_class) 111 | finally: 112 | # XXX culsans change: threading_helper.join_thread -> join_thread 113 | join_thread(thread) # make sure the thread terminates 114 | if not thread.startedEvent.is_set(): 115 | self.fail("trigger thread ended but event never set") 116 | 117 | 118 | class BaseQueueTestMixin(BlockingTestMixin): 119 | def setUp(self): 120 | self.cum = 0 121 | self.cumlock = threading.Lock() 122 | 123 | def basic_queue_test(self, q): 124 | if q.qsize(): 125 | raise RuntimeError("Call this function with an empty queue") 126 | self.assertTrue(q.empty()) 127 | self.assertFalse(q.full()) 128 | # I guess we better check things actually queue correctly a little :) 129 | q.put(111) 130 | q.put(333) 131 | q.put(222) 132 | target_order = dict(Queue = [111, 333, 222], 133 | LifoQueue = [222, 333, 111], 134 | PriorityQueue = [111, 222, 333]) 135 | actual_order = [q.get(), q.get(), q.get()] 136 | # XXX culsans change: +.wrapped 137 | self.assertEqual(actual_order, target_order[q.wrapped.__class__.__name__], 138 | "Didn't seem to queue the correct data!") 139 | for i in range(QUEUE_SIZE-1): 140 | q.put(i) 141 | self.assertTrue(q.qsize(), "Queue should not be empty") 142 | self.assertTrue(not qfull(q), "Queue should not be full") 143 | last = 2 * QUEUE_SIZE 144 | full = 3 * 2 * QUEUE_SIZE 145 | q.put(last) 146 | self.assertTrue(qfull(q), "Queue should be full") 147 | self.assertFalse(q.empty()) 148 | self.assertTrue(q.full()) 149 | try: 150 | q.put(full, block=0) 151 | self.fail("Didn't appear to block with a full queue") 152 | # XXX culsans change: Full -> SyncQueueFull 153 | except self.queue.SyncQueueFull: 154 | pass 155 | try: 156 | q.put(full, timeout=0.01) 157 | self.fail("Didn't appear to time-out with a full queue") 158 | # XXX culsans change: Full -> SyncQueueFull 159 | except self.queue.SyncQueueFull: 160 | pass 161 | # Test a blocking put 162 | self.do_blocking_test(q.put, (full,), q.get, ()) 163 | self.do_blocking_test(q.put, (full, True, 10), q.get, ()) 164 | # Empty it 165 | for i in range(QUEUE_SIZE): 166 | q.get() 167 | self.assertTrue(not q.qsize(), "Queue should be empty") 168 | try: 169 | q.get(block=0) 170 | self.fail("Didn't appear to block with an empty queue") 171 | # XXX culsans change: Empty -> SyncQueueEmpty 172 | except self.queue.SyncQueueEmpty: 173 | pass 174 | try: 175 | q.get(timeout=0.01) 176 | self.fail("Didn't appear to time-out with an empty queue") 177 | # XXX culsans change: Empty -> SyncQueueEmpty 178 | except self.queue.SyncQueueEmpty: 179 | pass 180 | # Test a blocking get 181 | self.do_blocking_test(q.get, (), q.put, ('empty',)) 182 | self.do_blocking_test(q.get, (True, 10), q.put, ('empty',)) 183 | 184 | 185 | def worker(self, q): 186 | while True: 187 | x = q.get() 188 | if x < 0: 189 | q.task_done() 190 | return 191 | with self.cumlock: 192 | self.cum += x 193 | q.task_done() 194 | 195 | def queue_join_test(self, q): 196 | self.cum = 0 197 | threads = [] 198 | for i in (0,1): 199 | thread = threading.Thread(target=self.worker, args=(q,)) 200 | thread.start() 201 | threads.append(thread) 202 | for i in range(100): 203 | q.put(i) 204 | q.join() 205 | self.assertEqual(self.cum, sum(range(100)), 206 | "q.join() did not block until all tasks were done") 207 | for i in (0,1): 208 | q.put(-1) # instruct the threads to close 209 | q.join() # verify that you can join twice 210 | for thread in threads: 211 | thread.join() 212 | 213 | def test_queue_task_done(self): 214 | # Test to make sure a queue task completed successfully. 215 | # XXX culsans change: +.sync_q 216 | q = self.type2test().sync_q 217 | try: 218 | q.task_done() 219 | except ValueError: 220 | pass 221 | else: 222 | self.fail("Did not detect task count going negative") 223 | 224 | def test_queue_join(self): 225 | # Test that a queue join()s successfully, and before anything else 226 | # (done twice for insurance). 227 | # XXX culsans change: +.sync_q 228 | q = self.type2test().sync_q 229 | self.queue_join_test(q) 230 | self.queue_join_test(q) 231 | try: 232 | q.task_done() 233 | except ValueError: 234 | pass 235 | else: 236 | self.fail("Did not detect task count going negative") 237 | 238 | def test_basic(self): 239 | # Do it a couple of times on the same queue. 240 | # Done twice to make sure works with same instance reused. 241 | # XXX culsans change: +.sync_q 242 | q = self.type2test(QUEUE_SIZE).sync_q 243 | self.basic_queue_test(q) 244 | self.basic_queue_test(q) 245 | 246 | def test_negative_timeout_raises_exception(self): 247 | # XXX culsans change: +.sync_q 248 | q = self.type2test(QUEUE_SIZE).sync_q 249 | with self.assertRaises(ValueError): 250 | q.put(1, timeout=-1) 251 | with self.assertRaises(ValueError): 252 | q.get(1, timeout=-1) 253 | 254 | def test_nowait(self): 255 | # XXX culsans change: +.sync_q 256 | q = self.type2test(QUEUE_SIZE).sync_q 257 | for i in range(QUEUE_SIZE): 258 | q.put_nowait(1) 259 | # XXX culsans change: Full -> SyncQueueFull 260 | with self.assertRaises(self.queue.SyncQueueFull): 261 | q.put_nowait(1) 262 | 263 | for i in range(QUEUE_SIZE): 264 | q.get_nowait() 265 | # XXX culsans change: Empty -> SyncQueueEmpty 266 | with self.assertRaises(self.queue.SyncQueueEmpty): 267 | q.get_nowait() 268 | 269 | def test_shrinking_queue(self): 270 | # issue 10110 271 | # XXX culsans change: +.sync_q 272 | q = self.type2test(3).sync_q 273 | q.put(1) 274 | q.put(2) 275 | q.put(3) 276 | # XXX culsans change: Full -> SyncQueueFull 277 | with self.assertRaises(self.queue.SyncQueueFull): 278 | q.put_nowait(4) 279 | self.assertEqual(q.qsize(), 3) 280 | q.maxsize = 2 # shrink the queue 281 | # XXX culsans change: Full -> SyncQueueFull 282 | with self.assertRaises(self.queue.SyncQueueFull): 283 | q.put_nowait(4) 284 | 285 | def test_shutdown_empty(self): 286 | # XXX culsans change: +.sync_q 287 | q = self.type2test().sync_q 288 | q.shutdown() 289 | # XXX culsans change: ShutDown -> SyncQueueShutDown 290 | with self.assertRaises(self.queue.SyncQueueShutDown): 291 | q.put("data") 292 | # XXX culsans change: ShutDown -> SyncQueueShutDown 293 | with self.assertRaises(self.queue.SyncQueueShutDown): 294 | q.get() 295 | 296 | def test_shutdown_nonempty(self): 297 | # XXX culsans change: +.sync_q 298 | q = self.type2test().sync_q 299 | q.put("data") 300 | q.shutdown() 301 | q.get() 302 | # XXX culsans change: ShutDown -> SyncQueueShutDown 303 | with self.assertRaises(self.queue.SyncQueueShutDown): 304 | q.get() 305 | 306 | def test_shutdown_immediate(self): 307 | # XXX culsans change: +.sync_q 308 | q = self.type2test().sync_q 309 | q.put("data") 310 | q.shutdown(immediate=True) 311 | # XXX culsans change: ShutDown -> SyncQueueShutDown 312 | with self.assertRaises(self.queue.SyncQueueShutDown): 313 | q.get() 314 | 315 | def test_shutdown_allowed_transitions(self): 316 | # allowed transitions would be from alive via shutdown to immediate 317 | # XXX culsans change: +.sync_q 318 | q = self.type2test().sync_q 319 | self.assertFalse(q.is_shutdown) 320 | 321 | q.shutdown() 322 | self.assertTrue(q.is_shutdown) 323 | 324 | q.shutdown(immediate=True) 325 | self.assertTrue(q.is_shutdown) 326 | 327 | q.shutdown(immediate=False) 328 | 329 | def _shutdown_all_methods_in_one_thread(self, immediate): 330 | # XXX culsans change: +.sync_q 331 | q = self.type2test(2).sync_q 332 | q.put("L") 333 | q.put_nowait("O") 334 | q.shutdown(immediate) 335 | 336 | # XXX culsans change: ShutDown -> SyncQueueShutDown 337 | with self.assertRaises(self.queue.SyncQueueShutDown): 338 | q.put("E") 339 | # XXX culsans change: ShutDown -> SyncQueueShutDown 340 | with self.assertRaises(self.queue.SyncQueueShutDown): 341 | q.put_nowait("W") 342 | if immediate: 343 | # XXX culsans change: ShutDown -> SyncQueueShutDown 344 | with self.assertRaises(self.queue.SyncQueueShutDown): 345 | q.get() 346 | # XXX culsans change: ShutDown -> SyncQueueShutDown 347 | with self.assertRaises(self.queue.SyncQueueShutDown): 348 | q.get_nowait() 349 | with self.assertRaises(ValueError): 350 | q.task_done() 351 | q.join() 352 | else: 353 | self.assertIn(q.get(), "LO") 354 | q.task_done() 355 | self.assertIn(q.get(), "LO") 356 | q.task_done() 357 | q.join() 358 | # on shutdown(immediate=False) 359 | # when queue is empty, should raise ShutDown Exception 360 | # XXX culsans change: ShutDown -> SyncQueueShutDown 361 | with self.assertRaises(self.queue.SyncQueueShutDown): 362 | q.get() # p.get(True) 363 | # XXX culsans change: ShutDown -> SyncQueueShutDown 364 | with self.assertRaises(self.queue.SyncQueueShutDown): 365 | q.get_nowait() # p.get(False) 366 | # XXX culsans change: ShutDown -> SyncQueueShutDown 367 | with self.assertRaises(self.queue.SyncQueueShutDown): 368 | q.get(True, 1.0) 369 | 370 | def test_shutdown_all_methods_in_one_thread(self): 371 | return self._shutdown_all_methods_in_one_thread(False) 372 | 373 | def test_shutdown_immediate_all_methods_in_one_thread(self): 374 | return self._shutdown_all_methods_in_one_thread(True) 375 | 376 | def _write_msg_thread(self, q, n, results, 377 | i_when_exec_shutdown, event_shutdown, 378 | barrier_start): 379 | # All `write_msg_threads` 380 | # put several items into the queue. 381 | for i in range(0, i_when_exec_shutdown//2): 382 | q.put((i, 'LOYD')) 383 | # Wait for the barrier to be complete. 384 | barrier_start.wait() 385 | 386 | for i in range(i_when_exec_shutdown//2, n): 387 | try: 388 | q.put((i, "YDLO")) 389 | # XXX culsans change: ShutDown -> SyncQueueShutDown 390 | except self.queue.SyncQueueShutDown: 391 | results.append(False) 392 | break 393 | 394 | # Trigger queue shutdown. 395 | if i == i_when_exec_shutdown: 396 | # Only one thread should call shutdown(). 397 | if not event_shutdown.is_set(): 398 | event_shutdown.set() 399 | results.append(True) 400 | 401 | def _read_msg_thread(self, q, results, barrier_start): 402 | # Get at least one item. 403 | q.get(True) 404 | q.task_done() 405 | # Wait for the barrier to be complete. 406 | barrier_start.wait() 407 | while True: 408 | try: 409 | q.get(False) 410 | q.task_done() 411 | # XXX culsans change: ShutDown -> SyncQueueShutDown 412 | except self.queue.SyncQueueShutDown: 413 | results.append(True) 414 | break 415 | # XXX culsans change: Empty -> SyncQueueEmpty 416 | except self.queue.SyncQueueEmpty: 417 | pass 418 | 419 | def _shutdown_thread(self, q, results, event_end, immediate): 420 | event_end.wait() 421 | q.shutdown(immediate) 422 | results.append(q.qsize() == 0) 423 | 424 | def _join_thread(self, q, barrier_start): 425 | # Wait for the barrier to be complete. 426 | barrier_start.wait() 427 | q.join() 428 | 429 | def _shutdown_all_methods_in_many_threads(self, immediate): 430 | # Run a 'multi-producers/consumers queue' use case, 431 | # with enough items into the queue. 432 | # When shutdown, all running threads will be joined. 433 | # XXX culsans change: +.sync_q 434 | q = self.type2test().sync_q 435 | ps = [] 436 | res_puts = [] 437 | res_gets = [] 438 | res_shutdown = [] 439 | write_threads = 4 440 | read_threads = 6 441 | join_threads = 2 442 | nb_msgs = 1024*64 443 | nb_msgs_w = nb_msgs // write_threads 444 | when_exec_shutdown = nb_msgs_w // 2 445 | # Use of a Barrier to ensure that 446 | # - all write threads put all their items into the queue, 447 | # - all read thread get at least one item from the queue, 448 | # and keep on running until shutdown. 449 | # The join thread is started only when shutdown is immediate. 450 | nparties = write_threads + read_threads 451 | if immediate: 452 | nparties += join_threads 453 | barrier_start = threading.Barrier(nparties) 454 | ev_exec_shutdown = threading.Event() 455 | lprocs = [ 456 | (self._write_msg_thread, write_threads, (q, nb_msgs_w, res_puts, 457 | when_exec_shutdown, ev_exec_shutdown, 458 | barrier_start)), 459 | (self._read_msg_thread, read_threads, (q, res_gets, barrier_start)), 460 | (self._shutdown_thread, 1, (q, res_shutdown, ev_exec_shutdown, immediate)), 461 | ] 462 | if immediate: 463 | lprocs.append((self._join_thread, join_threads, (q, barrier_start))) 464 | # start all threads. 465 | for func, n, args in lprocs: 466 | for i in range(n): 467 | ps.append(threading.Thread(target=func, args=args)) 468 | ps[-1].start() 469 | for thread in ps: 470 | thread.join() 471 | 472 | self.assertTrue(True in res_puts) 473 | self.assertEqual(res_gets.count(True), read_threads) 474 | if immediate: 475 | self.assertListEqual(res_shutdown, [True]) 476 | self.assertTrue(q.empty()) 477 | 478 | def test_shutdown_all_methods_in_many_threads(self): 479 | return self._shutdown_all_methods_in_many_threads(False) 480 | 481 | def test_shutdown_immediate_all_methods_in_many_threads(self): 482 | return self._shutdown_all_methods_in_many_threads(True) 483 | 484 | def _get(self, q, go, results, shutdown=False): 485 | go.wait() 486 | try: 487 | msg = q.get() 488 | results.append(not shutdown) 489 | return not shutdown 490 | # XXX culsans change: ShutDown -> SyncQueueShutDown 491 | except self.queue.SyncQueueShutDown: 492 | results.append(shutdown) 493 | return shutdown 494 | 495 | def _get_shutdown(self, q, go, results): 496 | return self._get(q, go, results, True) 497 | 498 | def _get_task_done(self, q, go, results): 499 | go.wait() 500 | try: 501 | msg = q.get() 502 | q.task_done() 503 | results.append(True) 504 | return msg 505 | # XXX culsans change: ShutDown -> SyncQueueShutDown 506 | except self.queue.SyncQueueShutDown: 507 | results.append(False) 508 | return False 509 | 510 | def _put(self, q, msg, go, results, shutdown=False): 511 | go.wait() 512 | try: 513 | q.put(msg) 514 | results.append(not shutdown) 515 | return not shutdown 516 | # XXX culsans change: ShutDown -> SyncQueueShutDown 517 | except self.queue.SyncQueueShutDown: 518 | results.append(shutdown) 519 | return shutdown 520 | 521 | def _put_shutdown(self, q, msg, go, results): 522 | return self._put(q, msg, go, results, True) 523 | 524 | def _join(self, q, results, shutdown=False): 525 | try: 526 | q.join() 527 | results.append(not shutdown) 528 | return not shutdown 529 | # XXX culsans change: ShutDown -> SyncQueueShutDown 530 | except self.queue.SyncQueueShutDown: 531 | results.append(shutdown) 532 | return shutdown 533 | 534 | def _join_shutdown(self, q, results): 535 | return self._join(q, results, True) 536 | 537 | def _shutdown_get(self, immediate): 538 | # XXX culsans change: +.sync_q 539 | q = self.type2test(2).sync_q 540 | results = [] 541 | go = threading.Event() 542 | q.put("Y") 543 | q.put("D") 544 | # queue full 545 | 546 | if immediate: 547 | thrds = ( 548 | (self._get_shutdown, (q, go, results)), 549 | (self._get_shutdown, (q, go, results)), 550 | ) 551 | else: 552 | thrds = ( 553 | # on shutdown(immediate=False) 554 | # one of these threads should raise Shutdown 555 | (self._get, (q, go, results)), 556 | (self._get, (q, go, results)), 557 | (self._get, (q, go, results)), 558 | ) 559 | threads = [] 560 | for func, params in thrds: 561 | threads.append(threading.Thread(target=func, args=params)) 562 | threads[-1].start() 563 | q.shutdown(immediate) 564 | go.set() 565 | for t in threads: 566 | t.join() 567 | if immediate: 568 | self.assertListEqual(results, [True, True]) 569 | else: 570 | self.assertListEqual(sorted(results), [False] + [True]*(len(thrds)-1)) 571 | 572 | def test_shutdown_get(self): 573 | return self._shutdown_get(False) 574 | 575 | def test_shutdown_immediate_get(self): 576 | return self._shutdown_get(True) 577 | 578 | def _shutdown_put(self, immediate): 579 | # XXX culsans change: +.sync_q 580 | q = self.type2test(2).sync_q 581 | results = [] 582 | go = threading.Event() 583 | q.put("Y") 584 | q.put("D") 585 | # queue fulled 586 | 587 | thrds = ( 588 | (self._put_shutdown, (q, "E", go, results)), 589 | (self._put_shutdown, (q, "W", go, results)), 590 | ) 591 | threads = [] 592 | for func, params in thrds: 593 | threads.append(threading.Thread(target=func, args=params)) 594 | threads[-1].start() 595 | q.shutdown() 596 | go.set() 597 | for t in threads: 598 | t.join() 599 | 600 | self.assertEqual(results, [True]*len(thrds)) 601 | 602 | def test_shutdown_put(self): 603 | return self._shutdown_put(False) 604 | 605 | def test_shutdown_immediate_put(self): 606 | return self._shutdown_put(True) 607 | 608 | def _shutdown_join(self, immediate): 609 | # XXX culsans change: +.sync_q 610 | q = self.type2test().sync_q 611 | results = [] 612 | q.put("Y") 613 | go = threading.Event() 614 | nb = q.qsize() 615 | 616 | thrds = ( 617 | (self._join, (q, results)), 618 | (self._join, (q, results)), 619 | ) 620 | threads = [] 621 | for func, params in thrds: 622 | threads.append(threading.Thread(target=func, args=params)) 623 | threads[-1].start() 624 | if not immediate: 625 | res = [] 626 | for i in range(nb): 627 | threads.append(threading.Thread(target=self._get_task_done, args=(q, go, res))) 628 | threads[-1].start() 629 | q.shutdown(immediate) 630 | go.set() 631 | for t in threads: 632 | t.join() 633 | 634 | self.assertEqual(results, [True]*len(thrds)) 635 | 636 | def test_shutdown_immediate_join(self): 637 | return self._shutdown_join(True) 638 | 639 | def test_shutdown_join(self): 640 | return self._shutdown_join(False) 641 | 642 | def _shutdown_put_join(self, immediate): 643 | # XXX culsans change: +.sync_q 644 | q = self.type2test(2).sync_q 645 | results = [] 646 | go = threading.Event() 647 | q.put("Y") 648 | # queue not fulled 649 | 650 | thrds = ( 651 | (self._put_shutdown, (q, "E", go, results)), 652 | (self._join, (q, results)), 653 | ) 654 | threads = [] 655 | for func, params in thrds: 656 | threads.append(threading.Thread(target=func, args=params)) 657 | threads[-1].start() 658 | self.assertEqual(q.unfinished_tasks, 1) 659 | 660 | q.shutdown(immediate) 661 | go.set() 662 | 663 | if immediate: 664 | # XXX culsans change: ShutDown -> SyncQueueShutDown 665 | with self.assertRaises(self.queue.SyncQueueShutDown): 666 | q.get_nowait() 667 | else: 668 | result = q.get() 669 | self.assertEqual(result, "Y") 670 | q.task_done() 671 | 672 | for t in threads: 673 | t.join() 674 | 675 | self.assertEqual(results, [True]*len(thrds)) 676 | 677 | def test_shutdown_immediate_put_join(self): 678 | return self._shutdown_put_join(True) 679 | 680 | def test_shutdown_put_join(self): 681 | return self._shutdown_put_join(False) 682 | 683 | def test_shutdown_get_task_done_join(self): 684 | # XXX culsans change: +.sync_q 685 | q = self.type2test(2).sync_q 686 | results = [] 687 | go = threading.Event() 688 | q.put("Y") 689 | q.put("D") 690 | self.assertEqual(q.unfinished_tasks, q.qsize()) 691 | 692 | thrds = ( 693 | (self._get_task_done, (q, go, results)), 694 | (self._get_task_done, (q, go, results)), 695 | (self._join, (q, results)), 696 | (self._join, (q, results)), 697 | ) 698 | threads = [] 699 | for func, params in thrds: 700 | threads.append(threading.Thread(target=func, args=params)) 701 | threads[-1].start() 702 | go.set() 703 | q.shutdown(False) 704 | for t in threads: 705 | t.join() 706 | 707 | self.assertEqual(results, [True]*len(thrds)) 708 | 709 | def test_shutdown_pending_get(self): 710 | def get(): 711 | try: 712 | results.append(q.get()) 713 | except Exception as e: 714 | results.append(e) 715 | 716 | # XXX culsans change: +.sync_q 717 | q = self.type2test().sync_q 718 | results = [] 719 | get_thread = threading.Thread(target=get) 720 | get_thread.start() 721 | q.shutdown(immediate=False) 722 | get_thread.join(timeout=10.0) 723 | self.assertFalse(get_thread.is_alive()) 724 | self.assertEqual(len(results), 1) 725 | # XXX culsans change: ShutDown -> SyncQueueShutDown 726 | self.assertIsInstance(results[0], self.queue.SyncQueueShutDown) 727 | 728 | 729 | class QueueTest(BaseQueueTestMixin): 730 | 731 | def setUp(self): 732 | self.type2test = self.queue.Queue 733 | super().setUp() 734 | 735 | class PyQueueTest(QueueTest, unittest.TestCase): 736 | # XXX culsans change: py_queue -> culsans 737 | queue = culsans 738 | 739 | 740 | class LifoQueueTest(BaseQueueTestMixin): 741 | 742 | def setUp(self): 743 | self.type2test = self.queue.LifoQueue 744 | super().setUp() 745 | 746 | 747 | class PyLifoQueueTest(LifoQueueTest, unittest.TestCase): 748 | # XXX culsans change: py_queue -> culsans 749 | queue = culsans 750 | 751 | 752 | class PriorityQueueTest(BaseQueueTestMixin): 753 | 754 | def setUp(self): 755 | self.type2test = self.queue.PriorityQueue 756 | super().setUp() 757 | 758 | 759 | class PyPriorityQueueTest(PriorityQueueTest, unittest.TestCase): 760 | # XXX culsans change: py_queue -> culsans 761 | queue = culsans 762 | 763 | 764 | # A Queue subclass that can provoke failure at a moment's notice :) 765 | class FailingQueueException(Exception): pass 766 | 767 | 768 | class FailingQueueTest(BlockingTestMixin): 769 | 770 | def setUp(self): 771 | 772 | Queue = self.queue.Queue 773 | 774 | class FailingQueue(Queue): 775 | def __init__(self, *args): 776 | self.fail_next_put = False 777 | self.fail_next_get = False 778 | Queue.__init__(self, *args) 779 | def _put(self, item): 780 | if self.fail_next_put: 781 | self.fail_next_put = False 782 | raise FailingQueueException("You Lose") 783 | return Queue._put(self, item) 784 | def _get(self): 785 | if self.fail_next_get: 786 | self.fail_next_get = False 787 | raise FailingQueueException("You Lose") 788 | return Queue._get(self) 789 | 790 | self.FailingQueue = FailingQueue 791 | 792 | super().setUp() 793 | 794 | def failing_queue_test(self, q): 795 | if q.qsize(): 796 | raise RuntimeError("Call this function with an empty queue") 797 | for i in range(QUEUE_SIZE-1): 798 | q.put(i) 799 | # Test a failing non-blocking put. 800 | # XXX culsans change: +.wrapped 801 | q.wrapped.fail_next_put = True 802 | try: 803 | q.put("oops", block=0) 804 | self.fail("The queue didn't fail when it should have") 805 | except FailingQueueException: 806 | pass 807 | # XXX culsans change: +.wrapped 808 | q.wrapped.fail_next_put = True 809 | try: 810 | q.put("oops", timeout=0.1) 811 | self.fail("The queue didn't fail when it should have") 812 | except FailingQueueException: 813 | pass 814 | q.put("last") 815 | self.assertTrue(qfull(q), "Queue should be full") 816 | # Test a failing blocking put 817 | # XXX culsans change: +.wrapped 818 | q.wrapped.fail_next_put = True 819 | try: 820 | self.do_blocking_test(q.put, ("full",), q.get, ()) 821 | self.fail("The queue didn't fail when it should have") 822 | except FailingQueueException: 823 | pass 824 | # Check the Queue isn't damaged. 825 | # put failed, but get succeeded - re-add 826 | q.put("last") 827 | # Test a failing timeout put 828 | # XXX culsans change: +.wrapped 829 | q.wrapped.fail_next_put = True 830 | try: 831 | self.do_exceptional_blocking_test(q.put, ("full", True, 10), q.get, (), 832 | FailingQueueException) 833 | self.fail("The queue didn't fail when it should have") 834 | except FailingQueueException: 835 | pass 836 | # Check the Queue isn't damaged. 837 | # put failed, but get succeeded - re-add 838 | q.put("last") 839 | self.assertTrue(qfull(q), "Queue should be full") 840 | q.get() 841 | self.assertTrue(not qfull(q), "Queue should not be full") 842 | q.put("last") 843 | self.assertTrue(qfull(q), "Queue should be full") 844 | # Test a blocking put 845 | self.do_blocking_test(q.put, ("full",), q.get, ()) 846 | # Empty it 847 | for i in range(QUEUE_SIZE): 848 | q.get() 849 | self.assertTrue(not q.qsize(), "Queue should be empty") 850 | q.put("first") 851 | # XXX culsans change: +.wrapped 852 | q.wrapped.fail_next_get = True 853 | try: 854 | q.get() 855 | self.fail("The queue didn't fail when it should have") 856 | except FailingQueueException: 857 | pass 858 | self.assertTrue(q.qsize(), "Queue should not be empty") 859 | # XXX culsans change: +.wrapped 860 | q.wrapped.fail_next_get = True 861 | try: 862 | q.get(timeout=0.1) 863 | self.fail("The queue didn't fail when it should have") 864 | except FailingQueueException: 865 | pass 866 | self.assertTrue(q.qsize(), "Queue should not be empty") 867 | q.get() 868 | self.assertTrue(not q.qsize(), "Queue should be empty") 869 | # XXX culsans change: +.wrapped 870 | q.wrapped.fail_next_get = True 871 | try: 872 | self.do_exceptional_blocking_test(q.get, (), q.put, ('empty',), 873 | FailingQueueException) 874 | self.fail("The queue didn't fail when it should have") 875 | except FailingQueueException: 876 | pass 877 | # put succeeded, but get failed. 878 | self.assertTrue(q.qsize(), "Queue should not be empty") 879 | q.get() 880 | self.assertTrue(not q.qsize(), "Queue should be empty") 881 | 882 | def test_failing_queue(self): 883 | 884 | # Test to make sure a queue is functioning correctly. 885 | # Done twice to the same instance. 886 | # XXX culsans change: +.sync_q 887 | q = self.FailingQueue(QUEUE_SIZE).sync_q 888 | self.failing_queue_test(q) 889 | self.failing_queue_test(q) 890 | 891 | 892 | 893 | class PyFailingQueueTest(FailingQueueTest, unittest.TestCase): 894 | # XXX culsans change: py_queue -> culsans 895 | queue = culsans 896 | 897 | 898 | if __name__ == "__main__": 899 | unittest.main() 900 | -------------------------------------------------------------------------------- /src/culsans/_queues.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # SPDX-FileCopyrightText: 2024 Ilya Egorov <0x42005e1f@gmail.com> 4 | # SPDX-License-Identifier: ISC 5 | 6 | from __future__ import annotations 7 | 8 | import sys 9 | 10 | from collections import deque 11 | from heapq import heappop, heappush 12 | from math import inf, isinf, isnan 13 | from typing import TYPE_CHECKING, Protocol, TypeVar, Union 14 | 15 | from aiologic import Condition 16 | from aiologic.lowlevel import ( 17 | async_checkpoint, 18 | async_checkpoint_enabled, 19 | create_thread_rlock, 20 | green_checkpoint, 21 | green_checkpoint_enabled, 22 | green_clock, 23 | ) 24 | 25 | from ._exceptions import ( 26 | QueueEmpty, 27 | QueueFull, 28 | QueueShutDown, 29 | UnsupportedOperation, 30 | ) 31 | from ._protocols import MixedQueue 32 | from ._proxies import AsyncQueueProxy, SyncQueueProxy 33 | 34 | if TYPE_CHECKING: 35 | from typing import Any 36 | 37 | from aiologic.lowlevel import ThreadRLock 38 | 39 | if sys.version_info >= (3, 12): # PEP 698 40 | from typing import override 41 | else: # typing-extensions>=4.5.0 42 | from typing_extensions import override 43 | 44 | 45 | class _SupportsBool(Protocol): 46 | def __bool__(self, /) -> bool: ... 47 | 48 | 49 | class _SupportsLT(Protocol): 50 | def __lt__(self, other: Any, /) -> _SupportsBool: ... 51 | 52 | 53 | class _SupportsGT(Protocol): 54 | def __gt__(self, other: Any, /) -> _SupportsBool: ... 55 | 56 | 57 | _T = TypeVar("_T") 58 | _RichComparableT = TypeVar( 59 | "_RichComparableT", 60 | bound=Union[_SupportsLT, _SupportsGT], 61 | ) 62 | 63 | 64 | class Queue(MixedQueue[_T]): 65 | """ 66 | A mixed sync-async queue that is: 67 | 68 | * :abbr:`MPMC (multi-producer, multi-consumer)` 69 | * :abbr:`FIFO (first-in, first-out)` 70 | 71 | Compliant with the Janus API version 2.0.0. 72 | """ 73 | 74 | __slots__ = ( 75 | "__data", 76 | "__one_get_one_item", 77 | "__one_put_one_item", 78 | "__weakref__", 79 | "_is_shutdown", 80 | "_maxsize", 81 | "_unfinished_tasks", 82 | "all_tasks_done", 83 | "mutex", 84 | "not_empty", 85 | "not_full", 86 | ) 87 | 88 | __data: deque[_T] 89 | 90 | __one_put_one_item: bool 91 | __one_get_one_item: bool 92 | 93 | _maxsize: int 94 | _unfinished_tasks: int 95 | _is_shutdown: bool 96 | 97 | mutex: ThreadRLock 98 | 99 | not_full: Condition[ThreadRLock] 100 | not_empty: Condition[ThreadRLock] 101 | all_tasks_done: Condition[ThreadRLock] 102 | 103 | def __init__(self, maxsize: int = 0) -> None: 104 | """ 105 | Create a queue object with the given maximum size. 106 | 107 | if *maxsize* is <= 0, the queue size is infinite. If it is an integer 108 | greater that 0, then the put methods block when the queue reaches 109 | *maxsize* until an item is removed by the get methods. 110 | """ 111 | 112 | self._maxsize = maxsize 113 | self._unfinished_tasks = 0 114 | self._is_shutdown = False 115 | 116 | self.mutex = mutex = create_thread_rlock() 117 | 118 | self.not_full = Condition(mutex) # putters 119 | self.not_empty = Condition(mutex) # getters 120 | self.all_tasks_done = Condition(mutex) # joiners 121 | 122 | self.__one_put_one_item = True 123 | self.__one_get_one_item = True 124 | 125 | self._init(maxsize) # data 126 | 127 | def peekable(self) -> bool: 128 | with self.mutex: 129 | return self._peekable() 130 | 131 | def clearable(self) -> bool: 132 | with self.mutex: 133 | return self._clearable() 134 | 135 | def qsize(self) -> int: 136 | with self.mutex: 137 | return self._qsize() 138 | 139 | def empty(self) -> bool: 140 | with self.mutex: 141 | return self._qsize() <= 0 142 | 143 | def full(self) -> bool: 144 | with self.mutex: 145 | return 0 < self._maxsize <= self._qsize() 146 | 147 | def sync_put( 148 | self, 149 | item: _T, 150 | block: bool = True, 151 | timeout: float | None = None, 152 | ) -> None: 153 | rescheduled = False 154 | notified = False 155 | deadline = None 156 | 157 | if timeout is not None: 158 | if isinstance(timeout, int): 159 | try: 160 | timeout = float(timeout) 161 | except OverflowError: 162 | timeout = (-1 if timeout < 0 else +1) * inf 163 | 164 | if isnan(timeout): 165 | msg = "'timeout' must be a number (non-NaN)" 166 | raise ValueError(msg) 167 | 168 | if timeout < 0: 169 | msg = "'timeout' must be a non-negative number" 170 | raise ValueError(msg) 171 | 172 | if isinf(timeout): 173 | timeout = None 174 | 175 | if not block: 176 | timeout = 0 177 | 178 | with self.not_full: 179 | try: 180 | while True: 181 | self._check_closing() 182 | 183 | qsize = self._qsize() 184 | isize = self._isize(item) 185 | assert isize >= 0 186 | 187 | if isize != 1: 188 | self.__one_put_one_item = False 189 | 190 | if timeout: 191 | while 0 < self._maxsize <= qsize + isize - 1: 192 | if deadline is None: 193 | deadline = green_clock() + timeout 194 | else: 195 | timeout = deadline - green_clock() 196 | 197 | if timeout <= 0: 198 | raise QueueFull 199 | 200 | notified = self.not_full.wait(timeout) 201 | 202 | self._check_closing() 203 | qsize = self._qsize() 204 | 205 | rescheduled = True 206 | elif timeout is None: 207 | while 0 < self._maxsize <= qsize + isize - 1: 208 | notified = self.not_full.wait() 209 | 210 | self._check_closing() 211 | qsize = self._qsize() 212 | 213 | rescheduled = True 214 | else: 215 | if 0 < self._maxsize <= qsize + isize - 1: 216 | raise QueueFull 217 | 218 | if not block: 219 | break 220 | 221 | if not rescheduled and green_checkpoint_enabled(): 222 | state = self.mutex._release_save() 223 | 224 | try: 225 | green_checkpoint() 226 | finally: 227 | self.mutex._acquire_restore(state) 228 | 229 | rescheduled = True 230 | else: 231 | break 232 | 233 | self._put(item) 234 | 235 | new_qsize = self._qsize() 236 | new_tasks = new_qsize - qsize 237 | assert new_tasks >= 0 238 | 239 | if new_tasks != 1: 240 | self.__one_put_one_item = False 241 | 242 | if new_tasks: 243 | if 0 < new_qsize: 244 | if self.__one_get_one_item: 245 | if qsize < 0: 246 | self.not_empty.notify(new_qsize) 247 | else: 248 | self.not_empty.notify(new_tasks) 249 | else: 250 | self.not_empty.notify() 251 | 252 | self._unfinished_tasks += new_tasks 253 | finally: 254 | if notified and not self.__one_put_one_item: 255 | self.not_full.notify() 256 | 257 | async def async_put(self, item: _T) -> None: 258 | rescheduled = False 259 | notified = False 260 | 261 | with self.not_full: 262 | try: 263 | while True: 264 | self._check_closing() 265 | 266 | qsize = self._qsize() 267 | isize = self._isize(item) 268 | assert isize >= 0 269 | 270 | if isize != 1: 271 | self.__one_put_one_item = False 272 | 273 | while 0 < self._maxsize <= qsize + isize - 1: 274 | notified = await self.not_full 275 | 276 | self._check_closing() 277 | qsize = self._qsize() 278 | 279 | rescheduled = True 280 | 281 | if not rescheduled and async_checkpoint_enabled(): 282 | state = self.mutex._release_save() 283 | 284 | try: 285 | await async_checkpoint() 286 | finally: 287 | self.mutex._acquire_restore(state) 288 | 289 | rescheduled = True 290 | else: 291 | break 292 | 293 | self._put(item) 294 | 295 | new_qsize = self._qsize() 296 | new_tasks = new_qsize - qsize 297 | assert new_tasks >= 0 298 | 299 | if new_tasks != 1: 300 | self.__one_put_one_item = False 301 | 302 | if new_tasks: 303 | if 0 < new_qsize: 304 | if self.__one_get_one_item: 305 | if qsize < 0: 306 | self.not_empty.notify(new_qsize) 307 | else: 308 | self.not_empty.notify(new_tasks) 309 | else: 310 | self.not_empty.notify() 311 | 312 | self._unfinished_tasks += new_tasks 313 | finally: 314 | if notified and not self.__one_put_one_item: 315 | self.not_full.notify() 316 | 317 | def put_nowait(self, item: _T) -> None: 318 | with self.mutex: 319 | self._check_closing() 320 | 321 | qsize = self._qsize() 322 | isize = self._isize(item) 323 | assert isize >= 0 324 | 325 | if isize != 1: 326 | self.__one_put_one_item = False 327 | 328 | if 0 < self._maxsize <= qsize + isize - 1: 329 | raise QueueFull 330 | 331 | self._put(item) 332 | 333 | new_qsize = self._qsize() 334 | new_tasks = new_qsize - qsize 335 | assert new_tasks >= 0 336 | 337 | if new_tasks != 1: 338 | self.__one_put_one_item = False 339 | 340 | if new_tasks: 341 | if 0 < new_qsize: 342 | if self.__one_get_one_item: 343 | if qsize < 0: 344 | self.not_empty.notify(new_qsize) 345 | else: 346 | self.not_empty.notify(new_tasks) 347 | else: 348 | self.not_empty.notify() 349 | 350 | self._unfinished_tasks += new_tasks 351 | 352 | def sync_get(self, block: bool = True, timeout: float | None = None) -> _T: 353 | rescheduled = False 354 | notified = False 355 | deadline = None 356 | 357 | if timeout is not None: 358 | if isinstance(timeout, int): 359 | try: 360 | timeout = float(timeout) 361 | except OverflowError: 362 | timeout = (-1 if timeout < 0 else +1) * inf 363 | 364 | if isnan(timeout): 365 | msg = "'timeout' must be a number (non-NaN)" 366 | raise ValueError(msg) 367 | 368 | if timeout < 0: 369 | msg = "'timeout' must be a non-negative number" 370 | raise ValueError(msg) 371 | 372 | if isinf(timeout): 373 | timeout = None 374 | 375 | if not block: 376 | timeout = 0 377 | 378 | with self.not_empty: 379 | try: 380 | while True: 381 | qsize = self._qsize() 382 | 383 | if timeout: 384 | while qsize <= 0: 385 | self._check_closing() 386 | 387 | if deadline is None: 388 | deadline = green_clock() + timeout 389 | else: 390 | timeout = deadline - green_clock() 391 | 392 | if timeout <= 0: 393 | raise QueueEmpty 394 | 395 | notified = self.not_empty.wait(timeout) 396 | 397 | qsize = self._qsize() 398 | 399 | rescheduled = True 400 | elif timeout is None: 401 | while qsize <= 0: 402 | self._check_closing() 403 | 404 | notified = self.not_empty.wait() 405 | 406 | qsize = self._qsize() 407 | 408 | rescheduled = True 409 | else: 410 | if qsize <= 0: 411 | self._check_closing() 412 | 413 | raise QueueEmpty 414 | 415 | if not block: 416 | break 417 | 418 | if not rescheduled and green_checkpoint_enabled(): 419 | state = self.mutex._release_save() 420 | 421 | try: 422 | green_checkpoint() 423 | finally: 424 | self.mutex._acquire_restore(state) 425 | 426 | rescheduled = True 427 | else: 428 | break 429 | 430 | item = self._get() 431 | 432 | new_qsize = self._qsize() 433 | new_tasks = qsize - new_qsize 434 | assert new_tasks >= 0 435 | 436 | if new_tasks != 1: 437 | self.__one_get_one_item = False 438 | 439 | if 0 < self._maxsize and new_tasks: 440 | if new_qsize < self._maxsize: 441 | if self.__one_put_one_item: 442 | if self._maxsize < qsize: 443 | self.not_full.notify(self._maxsize - new_qsize) 444 | else: 445 | self.not_full.notify(new_tasks) 446 | else: 447 | self.not_full.notify() 448 | 449 | return item 450 | finally: 451 | if notified and not self.__one_get_one_item: 452 | self.not_empty.notify() 453 | 454 | async def async_get(self) -> _T: 455 | rescheduled = False 456 | notified = False 457 | 458 | with self.not_empty: 459 | try: 460 | while True: 461 | qsize = self._qsize() 462 | 463 | while qsize <= 0: 464 | self._check_closing() 465 | 466 | notified = await self.not_empty 467 | 468 | qsize = self._qsize() 469 | 470 | rescheduled = True 471 | 472 | if not rescheduled and async_checkpoint_enabled(): 473 | state = self.mutex._release_save() 474 | 475 | try: 476 | await async_checkpoint() 477 | finally: 478 | self.mutex._acquire_restore(state) 479 | 480 | rescheduled = True 481 | else: 482 | break 483 | 484 | item = self._get() 485 | 486 | new_qsize = self._qsize() 487 | new_tasks = qsize - new_qsize 488 | assert new_tasks >= 0 489 | 490 | if new_tasks != 1: 491 | self.__one_get_one_item = False 492 | 493 | if 0 < self._maxsize and new_tasks: 494 | if new_qsize < self._maxsize: 495 | if self.__one_put_one_item: 496 | if self._maxsize < qsize: 497 | self.not_full.notify(self._maxsize - new_qsize) 498 | else: 499 | self.not_full.notify(new_tasks) 500 | else: 501 | self.not_full.notify() 502 | 503 | return item 504 | finally: 505 | if notified and not self.__one_get_one_item: 506 | self.not_empty.notify() 507 | 508 | def get_nowait(self) -> _T: 509 | with self.mutex: 510 | qsize = self._qsize() 511 | 512 | if qsize <= 0: 513 | self._check_closing() 514 | 515 | raise QueueEmpty 516 | 517 | item = self._get() 518 | 519 | new_qsize = self._qsize() 520 | new_tasks = qsize - new_qsize 521 | assert new_tasks >= 0 522 | 523 | if new_tasks != 1: 524 | self.__one_get_one_item = False 525 | 526 | if 0 < self._maxsize and new_tasks: 527 | if new_qsize < self._maxsize: 528 | if self.__one_put_one_item: 529 | if self._maxsize < qsize: 530 | self.not_full.notify(self._maxsize - new_qsize) 531 | else: 532 | self.not_full.notify(new_tasks) 533 | else: 534 | self.not_full.notify() 535 | 536 | return item 537 | 538 | def sync_peek( 539 | self, 540 | block: bool = True, 541 | timeout: float | None = None, 542 | ) -> _T: 543 | rescheduled = False 544 | notified = False 545 | deadline = None 546 | 547 | if timeout is not None: 548 | if isinstance(timeout, int): 549 | try: 550 | timeout = float(timeout) 551 | except OverflowError: 552 | timeout = (-1 if timeout < 0 else +1) * inf 553 | 554 | if isnan(timeout): 555 | msg = "'timeout' must be a number (non-NaN)" 556 | raise ValueError(msg) 557 | 558 | if timeout < 0: 559 | msg = "'timeout' must be a non-negative number" 560 | raise ValueError(msg) 561 | 562 | if isinf(timeout): 563 | timeout = None 564 | 565 | if not block: 566 | timeout = 0 567 | 568 | with self.not_empty: 569 | try: 570 | while True: 571 | self._check_peekable() 572 | qsize = self._qsize() 573 | 574 | if timeout: 575 | while qsize <= 0: 576 | self._check_closing() 577 | 578 | if deadline is None: 579 | deadline = green_clock() + timeout 580 | else: 581 | timeout = deadline - green_clock() 582 | 583 | if timeout <= 0: 584 | raise QueueEmpty 585 | 586 | notified = self.not_empty.wait(timeout) 587 | 588 | self._check_peekable() 589 | qsize = self._qsize() 590 | 591 | rescheduled = True 592 | elif timeout is None: 593 | while qsize <= 0: 594 | self._check_closing() 595 | 596 | notified = self.not_empty.wait() 597 | 598 | self._check_peekable() 599 | qsize = self._qsize() 600 | 601 | rescheduled = True 602 | else: 603 | if qsize <= 0: 604 | self._check_closing() 605 | 606 | raise QueueEmpty 607 | 608 | if not block: 609 | break 610 | 611 | if not rescheduled and green_checkpoint_enabled(): 612 | state = self.mutex._release_save() 613 | 614 | try: 615 | green_checkpoint() 616 | finally: 617 | self.mutex._acquire_restore(state) 618 | 619 | rescheduled = True 620 | else: 621 | break 622 | 623 | item = self._peek() 624 | assert self._qsize() == qsize 625 | 626 | return item 627 | finally: 628 | if notified: 629 | self.not_empty.notify() 630 | 631 | async def async_peek(self) -> _T: 632 | rescheduled = False 633 | notified = False 634 | 635 | with self.not_empty: 636 | try: 637 | while True: 638 | self._check_peekable() 639 | qsize = self._qsize() 640 | 641 | while qsize <= 0: 642 | self._check_closing() 643 | 644 | notified = await self.not_empty 645 | 646 | self._check_peekable() 647 | qsize = self._qsize() 648 | 649 | rescheduled = True 650 | 651 | if not rescheduled and async_checkpoint_enabled(): 652 | state = self.mutex._release_save() 653 | 654 | try: 655 | await async_checkpoint() 656 | finally: 657 | self.mutex._acquire_restore(state) 658 | 659 | rescheduled = True 660 | else: 661 | break 662 | 663 | item = self._peek() 664 | assert self._qsize() == qsize 665 | 666 | return item 667 | finally: 668 | if notified: 669 | self.not_empty.notify() 670 | 671 | def peek_nowait(self) -> _T: 672 | with self.mutex: 673 | self._check_peekable() 674 | qsize = self._qsize() 675 | 676 | if qsize <= 0: 677 | self._check_closing() 678 | 679 | raise QueueEmpty 680 | 681 | item = self._peek() 682 | assert self._qsize() == qsize 683 | 684 | return item 685 | 686 | def sync_join(self) -> None: 687 | rescheduled = False 688 | 689 | with self.all_tasks_done: 690 | while self._unfinished_tasks: 691 | self.all_tasks_done.wait() 692 | 693 | rescheduled = True 694 | 695 | if not rescheduled: 696 | green_checkpoint() 697 | 698 | async def async_join(self) -> None: 699 | rescheduled = False 700 | 701 | with self.all_tasks_done: 702 | while self._unfinished_tasks: 703 | await self.all_tasks_done 704 | 705 | rescheduled = True 706 | 707 | if not rescheduled: 708 | await async_checkpoint() 709 | 710 | def task_done(self, count: int = 1) -> None: 711 | with self.mutex: 712 | unfinished = self._unfinished_tasks - count 713 | 714 | if unfinished <= 0: 715 | if unfinished < 0: 716 | msg = "task_done() called too many times" 717 | raise ValueError(msg) 718 | 719 | self.all_tasks_done.notify_all() 720 | 721 | self._unfinished_tasks = unfinished 722 | 723 | def shutdown(self, immediate: bool = False) -> None: 724 | with self.mutex: 725 | self._is_shutdown = True 726 | 727 | if immediate: 728 | qsize = self._qsize() 729 | 730 | if self._clearable(): 731 | if qsize > 0: 732 | self._clear() 733 | 734 | new_qsize = self._qsize() 735 | assert new_qsize <= 0 736 | new_tasks = qsize - new_qsize 737 | assert new_tasks >= 0 738 | 739 | self._unfinished_tasks -= new_tasks 740 | 741 | if self._unfinished_tasks < 0: 742 | self._unfinished_tasks = 0 743 | 744 | qsize = new_qsize 745 | else: 746 | while qsize > 0: 747 | self._get() 748 | 749 | new_qsize = self._qsize() 750 | new_tasks = qsize - new_qsize 751 | assert new_tasks >= 0 752 | 753 | self._unfinished_tasks -= new_tasks 754 | 755 | if self._unfinished_tasks < 0: 756 | self._unfinished_tasks = 0 757 | 758 | qsize = new_qsize 759 | 760 | self.all_tasks_done.notify_all() 761 | 762 | self.not_empty.notify_all() 763 | self.not_full.notify_all() 764 | 765 | def close(self) -> None: 766 | """ 767 | Close the queue. 768 | 769 | This method is provided for compatibility with the Janus queues. Use 770 | :meth:`queue.shutdown(immediate=True) ` as a direct 771 | substitute. 772 | """ 773 | 774 | self.shutdown(immediate=True) 775 | 776 | async def wait_closed(self) -> None: 777 | """ 778 | Wait for finishing all pending activities. 779 | 780 | This method is provided for compatibility with the Janus queues. It 781 | actually does nothing. 782 | 783 | Raises: 784 | RuntimeError: 785 | if called for non-closed queue. 786 | """ 787 | 788 | if not self._is_shutdown: 789 | msg = "Waiting for non-closed queue" 790 | raise RuntimeError(msg) 791 | 792 | await async_checkpoint() 793 | 794 | async def aclose(self) -> None: 795 | """ 796 | Shutdown the queue and wait for actual shutting down. 797 | 798 | This method is provided for compatibility with the Janus queues. Use 799 | :meth:`queue.shutdown(immediate=True) ` as a direct 800 | substitute. 801 | """ 802 | 803 | self.close() 804 | await self.wait_closed() 805 | 806 | def clear(self) -> None: 807 | with self.mutex: 808 | self._check_clearable() 809 | qsize = self._qsize() 810 | 811 | self._clear() 812 | 813 | new_qsize = self._qsize() 814 | assert new_qsize <= 0 815 | new_tasks = qsize - new_qsize 816 | assert new_tasks >= 0 817 | 818 | if 0 < self._maxsize and new_tasks: 819 | if new_qsize < self._maxsize: 820 | if self.__one_put_one_item: 821 | if self._maxsize < qsize: 822 | self.not_full.notify(self._maxsize - new_qsize) 823 | else: 824 | self.not_full.notify(new_tasks) 825 | else: 826 | self.not_full.notify() 827 | 828 | self._unfinished_tasks = max(0, self._unfinished_tasks - new_tasks) 829 | 830 | if not self._unfinished_tasks: 831 | self.all_tasks_done.notify_all() 832 | 833 | def _check_closing(self) -> None: 834 | if self._is_shutdown: 835 | raise QueueShutDown 836 | 837 | def _check_peekable(self) -> None: 838 | if not self._peekable(): 839 | msg = "peeking not supported" 840 | raise UnsupportedOperation(msg) 841 | 842 | def _check_clearable(self) -> None: 843 | if not self._clearable(): 844 | msg = "clearing not supported" 845 | raise UnsupportedOperation(msg) 846 | 847 | # Override these methods to implement other queue organizations 848 | # (e.g. stack or priority queue). 849 | # These will only be called with appropriate locks held 850 | 851 | def _init(self, maxsize: int) -> None: 852 | self.__data = deque() 853 | 854 | def _qsize(self) -> int: 855 | return len(self.__data) 856 | 857 | def _isize(self, item: _T) -> int: 858 | return 1 859 | 860 | def _put(self, item: _T) -> None: 861 | self.__data.append(item) 862 | 863 | def _get(self) -> _T: 864 | return self.__data.popleft() 865 | 866 | def _peekable(self) -> bool: 867 | return True 868 | 869 | def _peek(self) -> _T: 870 | return self.__data[0] 871 | 872 | def _clearable(self) -> bool: 873 | return True 874 | 875 | def _clear(self) -> None: 876 | self.__data.clear() 877 | 878 | @property 879 | def sync_q(self) -> SyncQueueProxy[_T]: 880 | return SyncQueueProxy(self) 881 | 882 | @property 883 | def async_q(self) -> AsyncQueueProxy[_T]: 884 | return AsyncQueueProxy(self) 885 | 886 | @property 887 | def putting(self) -> int: 888 | """ 889 | The current number of threads/tasks waiting to put. 890 | 891 | It represents the length of the wait queue and thus changes 892 | immediately. 893 | """ 894 | 895 | return self.not_full.waiting 896 | 897 | @property 898 | def getting(self) -> int: 899 | """ 900 | The current number of threads/tasks waiting to get/peek. 901 | 902 | It represents the length of the wait queue and thus changes 903 | immediately. 904 | """ 905 | 906 | return self.not_empty.waiting 907 | 908 | @property 909 | def waiting(self) -> int: 910 | """ 911 | The current number of threads/tasks waiting to access. 912 | 913 | It is roughly equivalent to the sum of the :attr:`putting` and 914 | :attr:`getting` properties, but is more reliable than the sum in a 915 | multithreaded environment. 916 | """ 917 | 918 | with self.mutex: 919 | # We use the underlying lock to ensure that no thread can increment 920 | # the counters during normal queue operation (since exclusive 921 | # access is required to enter the wait queue). 922 | return self.not_full.waiting + self.not_empty.waiting 923 | 924 | @property 925 | def unfinished_tasks(self) -> int: 926 | return self._unfinished_tasks 927 | 928 | @property 929 | def is_shutdown(self) -> bool: 930 | return self._is_shutdown 931 | 932 | @property 933 | def closed(self) -> bool: 934 | return self._is_shutdown 935 | 936 | @property 937 | def maxsize(self) -> int: 938 | return self._maxsize 939 | 940 | @maxsize.setter 941 | def maxsize(self, value: int) -> None: 942 | with self.mutex: 943 | if 0 < self._maxsize: 944 | if value <= 0: 945 | self.not_full.notify_all() 946 | elif self._maxsize < value: 947 | if self.__one_put_one_item: 948 | self.not_full.notify(value - self._maxsize) 949 | else: 950 | self.not_full.notify() 951 | 952 | self._maxsize = value 953 | 954 | 955 | class LifoQueue(Queue[_T]): 956 | """ 957 | A variant of :class:`Queue` that retrieves most recently added entries 958 | first (:abbr:`LIFO (last-in, first-out)`). 959 | """ 960 | 961 | __slots__ = ("__data",) # noqa: PLW0244 962 | 963 | __data: list[_T] 964 | 965 | @override 966 | def _init(self, maxsize: int) -> None: 967 | self.__data = [] 968 | 969 | @override 970 | def _qsize(self) -> int: 971 | return len(self.__data) 972 | 973 | @override 974 | def _isize(self, item: _T) -> int: 975 | return 1 976 | 977 | @override 978 | def _put(self, item: _T) -> None: 979 | self.__data.append(item) 980 | 981 | @override 982 | def _get(self) -> _T: 983 | return self.__data.pop() 984 | 985 | @override 986 | def _peekable(self) -> bool: 987 | return True 988 | 989 | @override 990 | def _peek(self) -> _T: 991 | return self.__data[-1] 992 | 993 | @override 994 | def _clearable(self) -> bool: 995 | return True 996 | 997 | @override 998 | def _clear(self) -> None: 999 | self.__data.clear() 1000 | 1001 | 1002 | class PriorityQueue(Queue[_RichComparableT]): 1003 | """ 1004 | A variant of :class:`Queue` that retrieves entries in priority order 1005 | (lowest first). 1006 | """ 1007 | 1008 | __slots__ = ("__data",) # noqa: PLW0244 1009 | 1010 | __data: list[_RichComparableT] 1011 | 1012 | @override 1013 | def _init(self, maxsize: int) -> None: 1014 | self.__data = [] 1015 | 1016 | @override 1017 | def _qsize(self) -> int: 1018 | return len(self.__data) 1019 | 1020 | @override 1021 | def _isize(self, item: _RichComparableT) -> int: 1022 | return 1 1023 | 1024 | @override 1025 | def _put(self, item: _RichComparableT) -> None: 1026 | heappush(self.__data, item) 1027 | 1028 | @override 1029 | def _get(self) -> _RichComparableT: 1030 | return heappop(self.__data) 1031 | 1032 | @override 1033 | def _peekable(self) -> bool: 1034 | return True 1035 | 1036 | @override 1037 | def _peek(self) -> _RichComparableT: 1038 | return self.__data[0] 1039 | 1040 | @override 1041 | def _clearable(self) -> bool: 1042 | return True 1043 | 1044 | @override 1045 | def _clear(self) -> None: 1046 | self.__data.clear() 1047 | --------------------------------------------------------------------------------