├── .github └── workflows │ └── publish.yml ├── .gitignore ├── LICENSE ├── README.md ├── pdm.lock ├── pdm.toml ├── pyproject.toml ├── src └── pyqt5_concurrent │ ├── Future.py │ ├── Qt.py │ ├── Task.py │ ├── TaskExecutor.py │ └── __init__.py └── tests ├── __init__.py ├── concurrent_sync_test.py ├── concurrent_test.py ├── future_cancel_test.py ├── priority_test.py ├── qt.py ├── task_error_test.py └── with_test.py /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | 3 | on: workflow_dispatch 4 | 5 | permissions: write-all 6 | 7 | jobs: 8 | pypi-publish: 9 | name: upload release to PyPI 10 | runs-on: ubuntu-latest 11 | permissions: 12 | # IMPORTANT: this permission is mandatory for trusted publishing 13 | id-token: write 14 | steps: 15 | - uses: actions/checkout@v3 16 | 17 | - uses: pdm-project/setup-pdm@v3 18 | 19 | - name: Publish package distributions to PyPI 20 | run: pdm publish 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm-project.org/#use-with-ide 110 | .pdm.toml 111 | .pdm-python 112 | .pdm-build/ 113 | 114 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 115 | __pypackages__/ 116 | 117 | # Celery stuff 118 | celerybeat-schedule 119 | celerybeat.pid 120 | 121 | # SageMath parsed files 122 | *.sage.py 123 | 124 | # Environments 125 | .env 126 | .venv 127 | env/ 128 | venv/ 129 | ENV/ 130 | env.bak/ 131 | venv.bak/ 132 | 133 | # Spyder project settings 134 | .spyderproject 135 | .spyproject 136 | 137 | # Rope project settings 138 | .ropeproject 139 | 140 | # mkdocs documentation 141 | /site 142 | 143 | # mypy 144 | .mypy_cache/ 145 | .dmypy.json 146 | dmypy.json 147 | 148 | # Pyre type checker 149 | .pyre/ 150 | 151 | # pytype static type analyzer 152 | .pytype/ 153 | 154 | # Cython debug symbols 155 | cython_debug/ 156 | 157 | # PyCharm 158 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 159 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 160 | # and can be added to the global gitignore or merged into this file. For a more nuclear 161 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 162 | .idea/ 163 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Ares Connor 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PyQt5-Concurrent 2 | 3 | ### 现已支持Pyside2 PySide6 PyQt5 PyQt6 4 | 5 | ## 简介: 6 | 7 | ​ pyqt5-concurrent是一个基于QThreadPool实现的并发库，主要是对QRunnable的封装，提供了一些易于使用的面向任务的API。简单实现了Future和Task，并支持链式操作，让代码逻辑更流畅。 8 | 9 | ## 为什么需要PyQt5-Concurrent: 10 | 11 | ​ 如果你需要一些可以双向交互，粗粒度的并发，你可以使用QThread，它具有优先级，可运行的事件循环。但是有时候你实现并发可能是细粒度，多次轻量的，再使用QThread会显得有点重（当然你可以使用QOjbect.movetothread()来重复利用一个thread，但这就和size=1的QThreadPool没啥大区别)。 12 | 13 | ​ 这个时候你可能会说，为什么试试QThreadPool+QRunnable呢？这确实是一个好的解决方法。但是QRunnable是一个虚类，你需要重写他的run方法并实例化。比方说你有多个任务，你会不耐烦的发现：你不得不写很多的QRunnable的子类。于是我想到将任务和run解耦，写一个Task模板。QRunnable的构造方法中传入一个目标函数及参数，在run中统一运行，并try catch错误，来实现QtConcurrent那样的面向任务的细粒度并发(pyqt5中并没有封装QtConcurrent库，这是一个高级的面向任务的API库。)。于是这个库就诞生了 14 | 15 | ​ (起初是因为我在给MCSL2写模组广场插件的时候，一页需要异步获取50个图像，任务的重复性很强，不是很适合用QThread，但是子类化QRunnable又太累了（lazy =_=），于是突发奇想，构造了一个初步的Future和TaskExecutor来实现基于任务的并发，之后在群主的建议下分出了并发逻辑构建成库) 16 | 17 | ## 一些例子: 18 | - [示例1] 19 | ```python 20 | import sys 21 | import time 22 | from PyQt5.QtCore import QCoreApplication 23 | from pyqt5_concurrent.TaskExecutor import TaskExecutor 24 | 25 | 26 | app = QCoreApplication(sys.argv) 27 | future = TaskExecutor.run(time.sleep, 3) 28 | future.finished.connect(app.quit) 29 | 30 | print("3s 后退出") 31 | app.exec_() 32 | ``` 33 | 34 | 上图例子中简单介绍了TaskExecutor最基本的用法：TaskExecutor.run(self, target: Callable, *args, **kwargs) -> QFuture。args和kwargs将传入target中，并在线程池中运行。他返回一个future，你可以用他来实现任务成功，任务失败（发生异常），任务结束的回调。 35 | 36 | 37 | 38 | - [示例2] 39 | ```python 40 | import sys 41 | import time 42 | from PyQt5.QtCore import QCoreApplication 43 | from pyqt5_concurrent.TaskExecutor import TaskExecutor 44 | 45 | def work(who, t): 46 | while t > 0: 47 | print(f"{who} - hint") 48 | time.sleep(1) 49 | t -= 1 50 | 51 | app = QCoreApplication(sys.argv) 52 | 53 | TaskExecutor.map(work, [("worker1",3), ("worker2",5)]).then( 54 | onSuccess=app.quit 55 | ) 56 | 57 | app.exec_() 58 | ``` 59 | 60 | 上文中，你将使用一个类似multiprocess.map的方法来简单执行多个任务。这里同时使用了Future的then方法。这是一个可以链式调用的方法，可以设定一些回调。等同于Future.result.connect(cb), 61 | 62 | Future.failed.connect(cb),Future.finished.connect(cb)。但是then返回的是Future本身，因此可以实现链式调用。 63 | 64 | 65 | 66 | - [示例3] 67 | ```python 68 | import sys 69 | import time 70 | from urllib.request import Request,urlopen 71 | from PyQt5.QtCore import QCoreApplication 72 | from pyqt5_concurrent.TaskExecutor import TaskExecutor 73 | 74 | headers = { 75 | 'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 Edg/118.0.0.0" 76 | } 77 | 78 | def work1(url,filename): 79 | req = Request( 80 | method='GET', 81 | url=url, 82 | headers=headers 83 | ) 84 | with open(filename, 'w', encoding='utf-8') as f: 85 | f.write(urlopen(req).read().decode("utf-8")) 86 | 87 | def work2(url): 88 | req = Request( 89 | method='HEAD', 90 | url=url, 91 | headers=headers 92 | ) 93 | return urlopen(req).status 94 | 95 | app = QCoreApplication(sys.argv) 96 | 97 | task1 = TaskExecutor.createTask( 98 | work1, 99 | "https://github.com", 100 | "github.html" 101 | ).then(lambda _:print("success saved page"), lambda e:print("failed",e)) 102 | 103 | task2 = TaskExecutor.createTask( 104 | work2, 105 | "https://www.baidu.com" 106 | ).then(lambda r:print("status",r), lambda e:print("failed",e)) 107 | 108 | TaskExecutor.runTasks([task1, task2]).finished.connect(app.quit) 109 | print("2个任务开始") 110 | 111 | # 你也可以只启动一个: 112 | # task2.runTask().then(app.quit) 113 | # 等于 114 | # TaskExecutor.runTask(task1).then(app.quit) 115 | 116 | # 你也可以等待任务1完成 在执行任务2 117 | # task1.runTask().wait() 或者.synchronize() 118 | # task2.runTask.then(app.quit) 119 | app.exec_() 120 | ``` 121 | 122 | 123 | 124 | - [Task的用法] 125 | 126 | ```python 127 | import sys 128 | import time 129 | 130 | from PyQt5.QtCore import QTimer 131 | from PyQt5.QtWidgets import QApplication 132 | 133 | from pyqt5_concurrent.TaskExecutor import TaskExecutor 134 | from pyqt5_concurrent.Future import QFuture 135 | 136 | WORK_TIME = 10 137 | 138 | # 创建必要的对象 139 | app = QApplication(sys.argv) 140 | futures = [] 141 | 142 | # 记录开始时间 143 | t = time.time() 144 | 145 | # 创建work_time个任务,每个任务sleep 1~work_time 秒 146 | for _ in range(1, WORK_TIME + 1): 147 | futures.append( 148 | TaskExecutor.run( 149 | lambda i: {time.sleep(i), print(f"task_{i} done, waited: {i}s")}, _ 150 | ) 151 | ) # add coroutine tasks 152 | print("task start") 153 | 154 | gathered = QFuture.gather(futures) 155 | gathered.synchronize() # equivalent to: fut.wait() 156 | 157 | print("all tasks done:", time.time() - t, ",expected:", WORK_TIME) 158 | 159 | QTimer.singleShot(1000, app.quit) # close app after 1s 160 | app.exec_() 161 | ``` 162 | 163 | 164 | 165 | - [QFuture.gather以及QFuture.wait()] 166 | 167 | 168 | ```python 169 | # 创建一个带有优先级的任务 170 | TaskExecutor.runWithPriority(print,1,"hello world") 171 | TaskExecutor.createTask(print,"hello world").withPriority(1).runTask() 172 | ``` 173 | 174 | 为任务添加优先级的两种方法（只有在任务等待被调度时，优先级才有意义） 175 | 176 | 177 | 178 | - [UniqueTaskExecutor] 179 | 180 | 0.1.6添加UniqueTaskExecutor 181 | 182 | 它包装了一个非全局的线程池，如其名Unique，不同实例的线程池相互独立，意味着它是独立的执行单元，支持with语句。 183 | 184 | UniqueTaskExecutor的api与TaskExecutor一致，用法请参考后者。 185 | 186 | UniqueTaskExecutor支持设定并行的任务数量。 187 | 188 | UniqueTaskExecutor退出with语句前会自动执行self.threadpool.waitForDone()，并会销毁自己 189 | 190 | ```python 191 | class UniqueTaskExecutor(BaseTaskExecutor): 192 | def __init__(self, workers: int = CPU_COUNTS): 193 | super().__init__(useGlobalThreadPool=False) 194 | self.workers = workers 195 | 196 | ... 197 | 198 | def __enter__(self): 199 | return self 200 | 201 | def __exit__(self, exc_type, exc_val, exc_tb): 202 | self.threadPool.waitForDone() 203 | self.deleteLater() 204 | ``` 205 | 206 | 207 | 208 | ## 鸣谢： 209 | 210 | 1.PyQt5 211 | 212 | 2.[zhiyiYo (之一) (github.com)](https://github.com/zhiyiYo) (提出了一些很好的点子,例如链式调用then) 213 | 214 | 3.[rainzee (rainzee wang) (github.com)](https://github.com/rainzee)(提出了一些很好的点子,例如使用装饰器注册future回调) 215 | 216 | (按照时间先后的顺序) -------------------------------------------------------------------------------- /pdm.lock: -------------------------------------------------------------------------------- 1 | # This file is @generated by PDM. 2 | # It is not intended for manual editing. 3 | 4 | [metadata] 5 | groups = ["default"] 6 | strategy = ["cross_platform"] 7 | lock_version = "4.4" 8 | content_hash = "sha256:889b4ef2c27792c62374b1eb06eb9f9eafaac97314579968f8c80f14cff323d4" 9 | -------------------------------------------------------------------------------- /pdm.toml: -------------------------------------------------------------------------------- 1 | [pypi] 2 | url = "https://mirrors.ustc.edu.cn/pypi/web/simple" 3 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "pyqt5-concurrent" 3 | version = "0.1.6" 4 | description = "QThreadPool based task concurrency library in pyqt5" 5 | authors = [ 6 | {name = "AresConnor", email = "aresconnor867@gmail.com"}, 7 | ] 8 | dependencies = [ 9 | ] 10 | requires-python = ">=3.7,<3.13" 11 | readme = "README.md" 12 | license = {text = "MIT"} 13 | 14 | [build-system] 15 | requires = ["pdm-backend"] 16 | build-backend = "pdm.backend" 17 | -------------------------------------------------------------------------------- /src/pyqt5_concurrent/Future.py: -------------------------------------------------------------------------------- 1 | import enum 2 | from typing import List, Optional, Callable, Iterable, Sized, Tuple, Union 3 | 4 | from .Qt import QObject, Signal, QMutex, QSemaphore, QCoreApplication 5 | 6 | 7 | class FutureError(BaseException): 8 | pass 9 | 10 | 11 | class State(enum.Enum): 12 | PENDING = 0 13 | RUNNING = 1 14 | FAILED = 2 15 | SUCCESS = 3 16 | 17 | 18 | class FutureFailed(FutureError): 19 | def __init__(self, _exception: Optional[BaseException]): 20 | super().__init__() 21 | self.exception = _exception 22 | 23 | def __repr__(self): 24 | return f"FutureFailed({self.exception})" 25 | 26 | def __str__(self): 27 | return f"FutureFailed({self.exception})" 28 | 29 | @property 30 | def original(self): 31 | return self.exception 32 | 33 | 34 | class GatheredFutureFailed(FutureError): 35 | def __init__(self, failures: List[Tuple["QFuture", BaseException]]): 36 | super().__init__() 37 | self.failures = failures 38 | 39 | def __repr__(self): 40 | return f"GatheredFutureFailed({self.failures})" 41 | 42 | def __str__(self): 43 | return f"GatheredFutureFailed({self.failures})" 44 | 45 | def __iter__(self): 46 | return iter(self.failures) 47 | 48 | def __len__(self): 49 | return len(self.failures) 50 | 51 | 52 | class FutureCancelled(FutureError): 53 | def __init__(self): 54 | super().__init__() 55 | 56 | def __repr__(self): 57 | return "FutureCanceled()" 58 | 59 | def __str__(self): 60 | return "FutureCanceled()" 61 | 62 | 63 | class QFuture(QObject): 64 | result = Signal(object) # self 65 | finished = Signal(object) # self 66 | failed = Signal(object) # self 67 | partialDone = Signal(object) # child future 68 | childrenDone = Signal(object) # self 69 | 70 | def __init__(self, semaphore=0): 71 | super().__init__() 72 | self._taskID = -1 73 | self._failedCallback = lambda e: None 74 | self._done = False 75 | self._failed = False 76 | self._result = None 77 | self._exception = None 78 | self._children = [] 79 | self._counter = 0 80 | self._parent = None 81 | self._callback = lambda _: None 82 | self._mutex = QMutex() 83 | self._extra = {} 84 | self._state = State.PENDING # set state by TaskExecutor 85 | self._semaphore = QSemaphore(semaphore) 86 | 87 | def __onChildFinished(self, childFuture: "QFuture") -> None: 88 | self._mutex.lock() 89 | if childFuture.isFailed(): 90 | self._failed = True 91 | self._counter += 1 92 | self.partialDone.emit(childFuture) 93 | try: 94 | idx = getattr(childFuture, "_idx") 95 | self._result[idx] = childFuture._result 96 | self._mutex.unlock() 97 | except AttributeError: 98 | self._mutex.unlock() 99 | raise RuntimeError( 100 | "Invalid child future: please ensure that the child future is created by method 'Future.setChildren'" 101 | ) 102 | if self._counter == len(self._children): 103 | if self._failed: # set failed 104 | fails = [] 105 | for i, child in enumerate(self._children): 106 | e = child.getException() 107 | if isinstance(e, FutureError): 108 | fails.append((self._children[i], e)) 109 | self.setFailed(GatheredFutureFailed(fails)) 110 | else: 111 | self.setResult(self._result) 112 | 113 | def __setChildren(self, children: List["QFuture"]) -> None: 114 | self._children = children 115 | self._result = [None] * len(children) 116 | for i, fut in enumerate(self._children): 117 | setattr(fut, "_idx", i) 118 | fut.childrenDone.connect(self.__onChildFinished) 119 | fut._parent = self 120 | for fut in self._children: # check if child is done 121 | if fut.isDone(): 122 | self.__onChildFinished(fut) 123 | 124 | def unsafeAddChild(self, child: "QFuture") -> None: 125 | """ 126 | use before your wait the parent future 127 | """ 128 | i = len(self._children) 129 | self._children.append(child) 130 | self._result.append(None) 131 | 132 | setattr(child, "_idx", i) 133 | child.childrenDone.connect(self.__onChildFinished) 134 | child._parent = self 135 | 136 | if child.isDone(): 137 | self.__onChildFinished(child) 138 | 139 | def setResult(self, result) -> None: 140 | """ 141 | :param result: The result to set 142 | :return: None 143 | 144 | do not set result in thread pool,or it may not set correctly 145 | please use in main thread,or use signal-slot to set result !!! 146 | """ 147 | if not self._done: 148 | self._result = result 149 | self._done = True 150 | if self._parent: 151 | self.childrenDone.emit(self) 152 | if self._callback: 153 | self._callback(result) 154 | 155 | self._state = State.SUCCESS 156 | self.result.emit(result) 157 | self.finished.emit(self) 158 | else: 159 | raise RuntimeError("Future already done") 160 | # self.deleteLater() # delete this future object 161 | 162 | def setFailed(self, exception) -> None: 163 | """ 164 | :param exception: The exception to set 165 | :return: None 166 | """ 167 | if not self._done: 168 | self._exception = FutureFailed(exception) 169 | self._done = True 170 | self._failed = True 171 | if self._parent: 172 | self.childrenDone.emit(self) 173 | if self._failedCallback: 174 | self._failedCallback(self) 175 | 176 | self._state = State.FAILED 177 | self.failed.emit(self._exception) 178 | self.finished.emit(self) 179 | else: 180 | raise RuntimeError("Future already done") 181 | # self.deleteLater() 182 | 183 | def setCallback( 184 | self, 185 | callback: Callable[ 186 | [ 187 | object, 188 | ], 189 | None, 190 | ], 191 | ) -> None: 192 | self._callback = callback 193 | 194 | def setFailedCallback( 195 | self, 196 | callback: Callable[ 197 | [ 198 | "QFuture", 199 | ], 200 | None, 201 | ], 202 | ) -> None: 203 | self._failedCallback = lambda e: callback(self) 204 | 205 | def then( 206 | self, 207 | onSuccess: Callable, 208 | onFailed: Callable = None, 209 | onFinished: Callable = None, 210 | ) -> "QFuture": 211 | self.result.connect(onSuccess) 212 | if onFailed: 213 | self.failed.connect(onFailed) 214 | if onFinished: 215 | self.finished.connect(onFinished) 216 | return self 217 | 218 | def hasException(self) -> bool: 219 | if self._children: 220 | return any([fut.hasException() for fut in self._children]) 221 | else: 222 | return self._exception is not None 223 | 224 | def hasChildren(self) -> bool: 225 | return bool(self._children) 226 | 227 | def getException(self) -> Optional[BaseException]: 228 | return self._exception 229 | 230 | def setTaskID(self, _id: int) -> None: 231 | if self._taskID != -1: 232 | raise RuntimeError("Task ID can only be set once") 233 | 234 | self._state = State.RUNNING 235 | self._taskID = _id 236 | 237 | def getTaskID(self) -> int: 238 | """ 239 | -1 means that the bound task is pending rather running 240 | """ 241 | return self._taskID 242 | 243 | def getChildren(self) -> List["QFuture"]: 244 | return self._children 245 | 246 | @staticmethod 247 | def gather(futures: {Iterable, Sized}) -> "QFuture": 248 | """ 249 | :param futures: An iterable of Future objects 250 | :return: A Future object that will be done when all futures are done 251 | """ 252 | 253 | future = QFuture() 254 | future.__setChildren(futures) 255 | return future 256 | 257 | @property 258 | def semaphore(self) -> QSemaphore: 259 | return self._semaphore 260 | 261 | @property 262 | def state(self): 263 | """ 264 | if future is not bound to a task (produced by QFuture.gather),its state will skip state.RUNNING (not really running in thread pool) 265 | :return: QFuture state. 266 | """ 267 | return self._state 268 | 269 | def wait(self) -> None: 270 | if self.hasChildren(): 271 | for child in self.getChildren(): 272 | child.wait() 273 | else: 274 | self.semaphore.acquire(1) 275 | QCoreApplication.processEvents() 276 | 277 | def synchronize(self) -> None: 278 | self.wait() 279 | 280 | def isDone(self) -> bool: 281 | return self._done 282 | 283 | def isFailed(self) -> bool: 284 | return self._failed 285 | 286 | def getResult(self) -> Union[object, List[object]]: 287 | return self._result 288 | 289 | def setExtra(self, key, value) -> None: 290 | self._extra[key] = value 291 | 292 | def getExtra(self, key) -> object: 293 | return self._extra.get(key, None) 294 | 295 | def hasExtra(self, key) -> bool: 296 | return key in self._extra 297 | 298 | def __getattr__(self, item): 299 | return self.getExtra(item) 300 | 301 | def __repr__(self): 302 | return f"Future:({self._result})" 303 | 304 | def __str__(self): 305 | return f"Future({self._result})" 306 | 307 | def __eq__(self, other): 308 | return self._result == other.getResult() 309 | -------------------------------------------------------------------------------- /src/pyqt5_concurrent/Qt.py: -------------------------------------------------------------------------------- 1 | import importlib.util 2 | 3 | QT_BINDINGS = None 4 | 5 | qt_bindings = ["PySide6.QtCore", "PyQt6.QtCore", "PySide2.QtCore", "PyQt5.QtCore"] 6 | 7 | 8 | def find_qt_bindings(): 9 | for binding in qt_bindings: 10 | try: 11 | if importlib.util.find_spec(binding) is not None: 12 | return binding 13 | except ModuleNotFoundError: 14 | continue 15 | raise ModuleNotFoundError("No python Qt bindings found.") 16 | 17 | 18 | qt_binding = find_qt_bindings() 19 | QT_BINDINGS = qt_binding.split(".")[0] 20 | 21 | 22 | # import 23 | QThreadPool = importlib.import_module(qt_binding).QThreadPool 24 | QRunnable = importlib.import_module(qt_binding).QRunnable 25 | QSemaphore = importlib.import_module(qt_binding).QSemaphore 26 | QMutex = importlib.import_module(qt_binding).QMutex 27 | QCoreApplication = importlib.import_module(qt_binding).QCoreApplication 28 | QObject = importlib.import_module(qt_binding).QObject 29 | if "PyQt" in qt_binding: 30 | Signal = importlib.import_module(qt_binding).pyqtSignal 31 | else: 32 | Signal = importlib.import_module(qt_binding).Signal 33 | -------------------------------------------------------------------------------- /src/pyqt5_concurrent/Task.py: -------------------------------------------------------------------------------- 1 | import functools 2 | from typing import Callable, Optional 3 | 4 | from .Future import QFuture 5 | from .Qt import QObject, Signal, QRunnable 6 | 7 | 8 | class _Signal(QObject): 9 | finished = Signal(object) 10 | 11 | 12 | class QBaseTask(QRunnable): 13 | def __init__(self, _id: int, future: QFuture, priority): 14 | super().__init__() 15 | self._signal: _Signal = _Signal() # pyqtSignal(object) 16 | self._future: QFuture = future 17 | self._id: int = _id 18 | self._exception: Optional[BaseException] = None 19 | self._semaphore = future.semaphore 20 | self._priority = priority 21 | 22 | @property 23 | def finished(self): 24 | return self._signal.finished 25 | 26 | @property 27 | def signal(self): 28 | return self._signal 29 | 30 | @property 31 | def priority(self): 32 | return self._priority 33 | 34 | @property 35 | def taskID(self): 36 | return self._id 37 | 38 | @property 39 | def future(self): 40 | return self._future 41 | 42 | @property 43 | def state(self): 44 | return self._future.state 45 | 46 | def withPriority(self, priority): 47 | """ 48 | default:0, higher will be handled more quickly. 49 | priority only makes sense when the task is waiting to be scheduled 50 | :param priority: 51 | :return: 52 | """ 53 | self._priority = priority 54 | return self 55 | 56 | def _taskDone(self, **data): 57 | for d in data.items(): 58 | self._future.setExtra(*d) 59 | self._signal.finished.emit(self._future) 60 | self._semaphore.release(1) 61 | 62 | 63 | class QTask(QBaseTask): 64 | def __init__( 65 | self, 66 | _id: int, 67 | future: QFuture, 68 | target: functools.partial, 69 | priority, 70 | executor, 71 | args, 72 | kwargs, 73 | ): 74 | super().__init__(_id=_id, priority=priority, future=future) 75 | self._executor = executor 76 | 77 | self._target = target 78 | self._kwargs = kwargs 79 | self._args = args 80 | 81 | def run(self) -> None: 82 | """ 83 | use QTask.runTask() instead if you know what are you doing. 84 | :return: 85 | """ 86 | try: 87 | self._taskDone(result=self._target(*self._args, **self._kwargs)) 88 | except Exception as exception: 89 | self._taskDone(exception=exception) 90 | 91 | def then( 92 | self, 93 | onSuccess: Callable, 94 | onFailed: Callable = None, 95 | onFinished: Callable = None, 96 | ) -> "QTask": 97 | self._future.then(onSuccess, onFailed, onFinished) 98 | return self 99 | 100 | def runTask(self) -> QFuture: 101 | return self._executor.runTask(self) 102 | -------------------------------------------------------------------------------- /src/pyqt5_concurrent/TaskExecutor.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import os 3 | import warnings 4 | from typing import Dict, List, Callable, Iterable 5 | 6 | from .Future import QFuture, FutureCancelled, State 7 | from .Qt import QThreadPool, QObject 8 | from .Task import QBaseTask, QTask 9 | 10 | CPU_COUNTS = os.cpu_count() 11 | 12 | 13 | class BaseTaskExecutor(QObject): 14 | def __init__(self, useGlobalThreadPool=True): 15 | super().__init__() 16 | self.useGlobalThreadPool = useGlobalThreadPool 17 | if useGlobalThreadPool: 18 | self.threadPool = QThreadPool.globalInstance() 19 | else: 20 | self.threadPool = QThreadPool() 21 | self.taskMap = {} 22 | self.tasks: Dict[int, QBaseTask] = {} 23 | self.taskCounter = 0 24 | 25 | def deleteLater(self) -> None: 26 | if not self.useGlobalThreadPool: 27 | self.threadPool.clear() 28 | self.threadPool.waitForDone() 29 | self.threadPool.deleteLater() 30 | super().deleteLater() 31 | 32 | def _runTask(self, task: QBaseTask) -> QFuture: 33 | future = task._future 34 | future.setTaskID(task.taskID) 35 | future._state = State.RUNNING 36 | task.signal.finished.connect(self._taskDone) 37 | self.threadPool.start(task, priority=task.priority) 38 | return future 39 | 40 | def _createTask(self, target, priority, args, kwargs) -> QTask: 41 | future = QFuture() 42 | task = QTask( 43 | _id=self.taskCounter, 44 | future=future, 45 | target=target if target is functools.partial else functools.partial(target), 46 | priority=priority, 47 | executor=self, 48 | args=args, 49 | kwargs=kwargs, 50 | ) 51 | self.tasks[self.taskCounter] = task 52 | self.taskCounter += 1 53 | return task 54 | 55 | def _asyncRun(self, target: Callable, priority: int, *args, **kwargs) -> QFuture: 56 | task = self._createTask(target, priority, args, kwargs) 57 | return self._runTask(task) 58 | 59 | def _asyncMap( 60 | self, target: Callable, iterable: List[Iterable], priority: int = 0 61 | ) -> QFuture: 62 | futures = [] 63 | for args in iterable: 64 | futures.append(self._asyncRun(target, priority, *args)) 65 | return QFuture.gather(futures) 66 | 67 | def _taskDone(self, fut: QFuture): 68 | """ 69 | need manually set Future.setFailed() or Future.setResult() to be called!!! 70 | """ 71 | self.tasks.pop(fut.getTaskID()) 72 | e = fut.getExtra("exception") 73 | if isinstance(e, Exception): 74 | fut.setFailed(e) 75 | fut._state = State.FAILED 76 | else: 77 | fut.setResult(fut.getExtra("result")) 78 | fut._state = State.SUCCESS 79 | 80 | def _taskCancel(self, fut: QFuture) -> None: 81 | stack: List[QFuture] = [fut] 82 | while stack: 83 | f = stack.pop() 84 | if not f.hasChildren() and not f.isDone(): 85 | self._taskSingleCancel(f) 86 | f.setFailed(FutureCancelled()) 87 | stack.extend(f.getChildren()) 88 | 89 | def _taskSingleCancel(self, fut: QFuture) -> None: 90 | _id = fut.getTaskID() 91 | taskRef: QBaseTask = self.tasks[_id] 92 | if taskRef is not None: 93 | try: 94 | taskRef.setAutoDelete(False) 95 | self.threadPool.tryTake(taskRef) 96 | taskRef.setAutoDelete(True) 97 | except RuntimeError: 98 | print("wrapped C/C++ object of type BaseTask has been deleted") 99 | del taskRef 100 | 101 | def cancelTask(self, fut: QFuture) -> None: 102 | """ 103 | currently, this method can not work properly... 104 | """ 105 | warnings.warn( 106 | "BaseTaskExecutor.cancelTask: currently, this method can not work properly...", 107 | DeprecationWarning, 108 | ) 109 | self._taskCancel(fut) 110 | 111 | @property 112 | def workers(self) -> int: 113 | return self.threadPool.maxThreadCount() 114 | 115 | @workers.setter 116 | def workers(self, workers: int) -> None: 117 | self.threadPool.setMaxThreadCount(workers) 118 | 119 | 120 | class TaskExecutor(BaseTaskExecutor): 121 | _globalInstance = None 122 | 123 | @staticmethod 124 | def globalInstance() -> "TaskExecutor": 125 | if TaskExecutor._globalInstance is None: 126 | TaskExecutor._globalInstance = TaskExecutor() 127 | return TaskExecutor._globalInstance 128 | 129 | @classmethod 130 | def run(cls, target: Callable, *args, **kwargs) -> QFuture: 131 | """ 132 | use the global TaskExecutor instance to avoid task ID conflicts 133 | :param target: 134 | :param args: *arg for target 135 | :param kwargs: **kwargs for target 136 | :return: 137 | """ 138 | return cls.globalInstance()._asyncRun(target, 0, *args, **kwargs) 139 | 140 | @classmethod 141 | def runWithPriority( 142 | cls, target: Callable, priority: int, *args, **kwargs 143 | ) -> QFuture: 144 | """ 145 | use the global TaskExecutor instance to avoid task ID conflicts 146 | :param target: 147 | :param priority: Task priority 148 | :param args: *arg for target 149 | :param kwargs: **kwargs for target 150 | :return: 151 | """ 152 | return cls.globalInstance()._asyncRun(target, priority, *args, **kwargs) 153 | 154 | @classmethod 155 | def map(cls, target: Callable, iter_: Iterable, priority: int = 0) -> QFuture: 156 | """ 157 | a simple wrapper for createTask and runTasks. 158 | 159 | iter_ must be like : [1, 2, 3] for [(1, 2), (3, 4)], 160 | if you need **kwargs in iter_, use createTask instead. 161 | """ 162 | taskList = [] 163 | for args in iter_: 164 | if isinstance(args, tuple): 165 | taskList.append(cls.createTask(target, priority=priority, *args)) 166 | else: 167 | taskList.append(cls.createTask(target, args, priority=priority)) 168 | return cls.runTasks(taskList) 169 | 170 | @classmethod 171 | def createTask(cls, target: Callable, *args, **kwargs) -> QTask: 172 | return cls.globalInstance()._createTask(target, 0, args, kwargs) 173 | 174 | @classmethod 175 | def runTask(cls, task: QTask) -> QFuture: 176 | return cls.globalInstance()._runTask(task) 177 | 178 | @classmethod 179 | def runTasks(cls, tasks: List[QTask]) -> QFuture: 180 | futs = [] 181 | for task in tasks: 182 | futs.append(cls.runTask(task)) 183 | return QFuture.gather(futs) 184 | 185 | 186 | class UniqueTaskExecutor(BaseTaskExecutor): 187 | def __init__(self, workers: int = CPU_COUNTS): 188 | super().__init__(useGlobalThreadPool=False) 189 | self.workers = workers 190 | 191 | def run(self, target: Callable, *args, **kwargs) -> QFuture: 192 | """ 193 | use the global TaskExecutor instance to avoid task ID conflicts 194 | :param target: 195 | :param args: *arg for target 196 | :param kwargs: **kwargs for target 197 | :return: 198 | """ 199 | return self._asyncRun(target, 0, *args, **kwargs) 200 | 201 | def runWithPriority( 202 | self, target: Callable, priority: int, *args, **kwargs 203 | ) -> QFuture: 204 | """ 205 | use the global TaskExecutor instance to avoid task ID conflicts 206 | :param target: 207 | :param priority: Task priority 208 | :param args: *arg for target 209 | :param kwargs: **kwargs for target 210 | :return: 211 | """ 212 | return self._asyncRun(target, priority, *args, **kwargs) 213 | 214 | def map(self, target: Callable, iter_: Iterable, priority: int = 0) -> QFuture: 215 | """ 216 | a simple wrapper for createTask and runTasks. 217 | 218 | iter_ must be like : [1, 2, 3] for [(1, 2), (3, 4)], 219 | if you need **kwargs in iter_, use createTask instead. 220 | """ 221 | taskList = [] 222 | for args in iter_: 223 | if isinstance(args, tuple): 224 | taskList.append(self.createTask(target, priority=priority, *args)) 225 | else: 226 | taskList.append(self.createTask(target, args, priority=priority)) 227 | return self.runTasks(taskList) 228 | 229 | def createTask(self, target: Callable, *args, **kwargs) -> QTask: 230 | return self._createTask(target, 0, args, kwargs) 231 | 232 | def runTask(self, task: QTask) -> QFuture: 233 | return self._runTask(task) 234 | 235 | def runTasks(self, tasks: List[QTask]) -> QFuture: 236 | futs = [] 237 | for task in tasks: 238 | futs.append(self.runTask(task)) 239 | return QFuture.gather(futs) 240 | 241 | def __enter__(self): 242 | return self 243 | 244 | def __exit__(self, exc_type, exc_val, exc_tb): 245 | self.threadPool.waitForDone() 246 | self.deleteLater() 247 | -------------------------------------------------------------------------------- /src/pyqt5_concurrent/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AresConnor/pyqt5-concurrent/0a452d97f497491208f8d495dfd741491f3006ac/src/pyqt5_concurrent/__init__.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AresConnor/pyqt5-concurrent/0a452d97f497491208f8d495dfd741491f3006ac/tests/__init__.py -------------------------------------------------------------------------------- /tests/concurrent_sync_test.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from pyqt5_concurrent.Future import QFuture 4 | from pyqt5_concurrent.TaskExecutor import TaskExecutor 5 | 6 | WORK_TIME = 10 7 | 8 | # 创建必要的对象 9 | # app = QCoreApplication(sys.argv) 10 | futures = [] 11 | 12 | # 记录开始时间 13 | t = time.time() 14 | 15 | # 创建work_time个任务,每个任务sleep 1~work_time 秒 16 | for _ in range(1, WORK_TIME + 1): 17 | futures.append( 18 | TaskExecutor.run( 19 | lambda i: {time.sleep(i), print(f"task_{i} done, waited: {i}s")}, _ 20 | ) 21 | ) # add coroutine tasks 22 | print("task start") 23 | 24 | gathered = QFuture.gather(futures) 25 | gathered.synchronize() # equivalent to: fut.wait() 26 | 27 | print("all tasks done:", time.time() - t, ",expected:", WORK_TIME) 28 | 29 | # QTimer.singleShot(1000, app.quit) # close app after 1s 30 | -------------------------------------------------------------------------------- /tests/concurrent_test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | from urllib.request import Request, urlopen 4 | 5 | from qt import QCoreApplication 6 | from pyqt5_concurrent.TaskExecutor import TaskExecutor 7 | 8 | app = QCoreApplication(sys.argv) 9 | 10 | 11 | def func(i, t): 12 | while t > 0: 13 | print(f"Task_{i} - hint") 14 | time.sleep(1) 15 | t -= 1 16 | 17 | 18 | def savePage(html, path): 19 | with open(path, "w", encoding="utf-8") as f: 20 | f.write(html) 21 | print(f"saved page to path: {path}") 22 | 23 | 24 | def getPage(url): 25 | ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36 Edg/118.0.0.0" 26 | req = Request(method="GET", url=url, headers={"User-Agent": ua}) 27 | return urlopen(req).read().decode("utf-8") 28 | 29 | 30 | print("测试简单的run\n" + "=" * 50) 31 | TaskExecutor.run(func, 114514, t=2).wait() 32 | 33 | print("测试单个Task的链式启动\n" + "=" * 50) 34 | TaskExecutor.createTask(getPage, "https://github.com").then( 35 | onSuccess=lambda r: savePage(r, "github.html"), 36 | onFailed=lambda _: print("failed:", _), 37 | ).runTask().wait() 38 | 39 | print("测试map\n" + "=" * 50) 40 | args = [(0, 3), (1, 5)] 41 | fut = TaskExecutor.map(func, args).wait() 42 | 43 | print("测试异步爬虫\n" + "=" * 50) 44 | task1 = TaskExecutor.createTask(getPage, "https://www.baidu.com").then( 45 | onSuccess=lambda r: savePage(r, "baidu1.html"), 46 | onFailed=lambda _: print("failed:", _), 47 | ) 48 | task2 = TaskExecutor.createTask(getPage, "https://www.baidu.com").then( 49 | onSuccess=lambda r: savePage(r, "baidu2.html"), 50 | onFailed=lambda _: print("failed:", _), 51 | ) 52 | 53 | TaskExecutor.runTasks([task1, task2]).finished.connect(app.quit) 54 | 55 | print("任务开始") 56 | sys.exit(app.exec()) 57 | -------------------------------------------------------------------------------- /tests/future_cancel_test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | from qt import QThread, QCoreApplication, QTimer 4 | from pyqt5_concurrent.TaskExecutor import TaskExecutor 5 | 6 | app = QCoreApplication(sys.argv) 7 | 8 | TIME_TO_SLEEP = 3 9 | TIME_TO_CANCEL = 1 10 | 11 | 12 | # =============================================================================== 13 | def func(t): 14 | print(f"task will work 10s, current Thread:{QThread.currentThread()}") 15 | while t > 0: 16 | print(f"task work time remains {t}s") 17 | time.sleep(0.5) 18 | t -= 0.5 19 | 20 | 21 | def cancelFunc(fut_, beginTime_): 22 | TaskExecutor.globalInstance().cancelTask(fut_) 23 | print(f"task canceled, {time.time() - beginTime_}s elapsed") 24 | 25 | 26 | fut = TaskExecutor.run(func, TIME_TO_SLEEP) 27 | beginTime = time.time() 28 | print("task started") 29 | QTimer.singleShot( 30 | TIME_TO_CANCEL * 1000, lambda: cancelFunc(fut, beginTime) 31 | ) # cancel task after 5s 32 | QTimer.singleShot((TIME_TO_SLEEP + 1) * 1000, app.quit) # close app 33 | print( 34 | "显然,这个测试是失败的,还没研究为什么,TaskExecutor中调用了QThreadPool::cancel()函数,但是任务还是继续执行了" 35 | ) 36 | app.exec() 37 | -------------------------------------------------------------------------------- /tests/priority_test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | 4 | from qt import QCoreApplication 5 | 6 | from pyqt5_concurrent.TaskExecutor import TaskExecutor 7 | 8 | app = QCoreApplication(sys.argv) 9 | 10 | 11 | def task(priority): 12 | print(f"task with {priority} started") 13 | time.sleep(3) 14 | print(f"task with {priority} finished") 15 | 16 | 17 | TaskExecutor.globalInstance().threadPool.setMaxThreadCount(1) 18 | task1 = TaskExecutor.createTask(task, 1).withPriority(1) 19 | task2 = TaskExecutor.createTask(task, 0).withPriority(0) 20 | task3 = TaskExecutor.createTask(task, 2).withPriority(2) 21 | task4 = TaskExecutor.createTask(task, 3).withPriority(3) 22 | gather = TaskExecutor.runTasks([task1, task2, task3, task4]) 23 | gather.finished.connect(app.quit) 24 | 25 | app.exec() 26 | -------------------------------------------------------------------------------- /tests/qt.py: -------------------------------------------------------------------------------- 1 | import importlib.util 2 | 3 | QT_BINDINGS = None 4 | 5 | qt_bindings = ["PySide6.QtCore", "PyQt6.QtCore", "PySide2.QtCore", "PyQt5.QtCore"] 6 | 7 | 8 | def find_qt_bindings(): 9 | for binding in qt_bindings: 10 | try: 11 | if importlib.util.find_spec(binding) is not None: 12 | return binding 13 | except ModuleNotFoundError: 14 | continue 15 | raise ModuleNotFoundError("No python Qt bindings found.") 16 | 17 | 18 | qt_binding = find_qt_bindings() 19 | QT_BINDINGS = qt_binding.split(".")[0] 20 | 21 | 22 | # import 23 | QThreadPool = importlib.import_module(qt_binding).QThreadPool 24 | QRunnable = importlib.import_module(qt_binding).QRunnable 25 | QSemaphore = importlib.import_module(qt_binding).QSemaphore 26 | QMutex = importlib.import_module(qt_binding).QMutex 27 | QCoreApplication = importlib.import_module(qt_binding).QCoreApplication 28 | QObject = importlib.import_module(qt_binding).QObject 29 | QTimer = importlib.import_module(qt_binding).QTimer 30 | QThread = importlib.import_module(qt_binding).QThread 31 | if "PyQt" in qt_binding: 32 | Signal = importlib.import_module(qt_binding).pyqtSignal 33 | else: 34 | Signal = importlib.import_module(qt_binding).Signal 35 | -------------------------------------------------------------------------------- /tests/task_error_test.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | from qt import QCoreApplication 4 | 5 | from pyqt5_concurrent.TaskExecutor import TaskExecutor 6 | 7 | app = QCoreApplication(sys.argv) 8 | 9 | TIME_TO_RAISE = 3 10 | 11 | 12 | def func(t): 13 | print(f"task will raise exception after {t}s") 14 | time.sleep(t) 15 | print("task done") 16 | # raise Exception("test exception") 17 | 18 | 19 | fut = TaskExecutor.run(func, TIME_TO_RAISE) 20 | fut.result.connect( 21 | lambda x: print("result signal:", x) 22 | ) # result 将不会被触发，因为任务抛出了异常 23 | fut.failed.connect(lambda x: print("failed signal:", x)) # failed 信号将会被触发 24 | fut.finished.connect( 25 | lambda x: {print("done signal:", x), app.quit()} 26 | ) # done 信号将会被触发(不管任务是否抛出异常,只要是任务结束了,done信号就会被触发) 27 | 28 | app.exec() 29 | -------------------------------------------------------------------------------- /tests/with_test.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from pyqt5_concurrent.TaskExecutor import UniqueTaskExecutor 4 | 5 | task_num = 10 6 | with UniqueTaskExecutor(4) as executor: 7 | tasks = [] 8 | for i in range(task_num): 9 | executor.run(lambda ident: [print(ident), time.sleep(1)], i) 10 | --------------------------------------------------------------------------------