├── .github └── workflows │ └── publish.yml ├── .gitignore ├── LICENSE ├── README.md ├── assets └── joblib-progress.gif ├── joblib_progress ├── __init__.py └── py.typed ├── pyproject.toml ├── requirements-dev.txt ├── requirements.txt ├── setup.cfg └── setup.py /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 📡 Python 🐍 distributions 📦 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | jobs: 9 | lint: 10 | name: Lint 🧹 and Check 🧐 Python 🐍 package 🗂️ 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Setup 🛠️ Python 🐍 17 | uses: actions/setup-python@v5 18 | with: 19 | python-version: "3.x" 20 | 21 | - name: Install 🛠️ dependencies 📚 22 | run: | 23 | pip install --upgrade pip 24 | pip install --requirement=requirements-dev.txt 25 | 26 | - name: Lint 🧹 with Ruff ⚡️ 27 | run: ruff check --output-format=github . 28 | 29 | build: 30 | name: Build 🏗️ distributions 📦 31 | needs: [lint] 32 | runs-on: ubuntu-latest 33 | 34 | steps: 35 | - uses: actions/checkout@v4 36 | 37 | - name: Setup 🛠️ Python 🐍 38 | uses: actions/setup-python@v5 39 | with: 40 | python-version: "3.x" 41 | 42 | - name: Build 🏗️ a source distribution 🗃️ and a binary wheel 🛞 43 | run: pipx run build --outdir=distributions 44 | 45 | - name: Upload 📤 the built distributions 📦 46 | uses: actions/upload-artifact@v4 47 | with: 48 | name: distributions 49 | path: distributions/ 50 | 51 | publish-to-pypi: 52 | name: Publish 📡 Python 🐍 distributions 📦 to PyPI 🌐 53 | needs: [build] 54 | runs-on: ubuntu-latest 55 | environment: 56 | name: release 57 | url: https://pypi.org/p/joblib-progress 58 | permissions: 59 | id-token: write 60 | 61 | steps: 62 | - name: Download 📥 all the distributions 📦 63 | uses: actions/download-artifact@v4 64 | with: 65 | pattern: distributions 66 | path: distributions/ 67 | merge-multiple: true 68 | 69 | - name: Publish 📡 distributions 📦 to PyPI 🌐 70 | uses: pypa/gh-action-pypi-publish@release/v1 71 | with: 72 | packages-dir: distributions/ 73 | 74 | upload-to-github-release: 75 | name: Upload 📤 Python 🐍 distributions 📦 to GitHub Release 🚀 76 | needs: [publish-to-pypi] 77 | runs-on: ubuntu-latest 78 | permissions: 79 | id-token: write 80 | contents: write 81 | 82 | steps: 83 | - name: Download 📥 all the distributions 📦 84 | uses: actions/download-artifact@v4 85 | with: 86 | pattern: distributions 87 | path: distributions/ 88 | merge-multiple: true 89 | 90 | - name: Sign 🔑 the distributions 📦 with Sigstore 91 | uses: sigstore/gh-action-sigstore-python@v3.0.0 92 | with: 93 | inputs: | 94 | distributions/*.tar.gz 95 | distributions/*.whl 96 | 97 | - name: Create 📂 GitHub Release 🚀 98 | env: 99 | GITHUB_TOKEN: ${{ github.token }} 100 | run: gh release --repo="${{ github.repository }}" create "${{ github.ref_name }}" 101 | 102 | - name: Upload 📤 distributions 📦 and signatures 🔏 to GitHub Release 🚀 103 | env: 104 | GITHUB_TOKEN: ${{ github.token }} 105 | run: gh release --repo="${{ github.repository }}" upload "${{ github.ref_name }}" distributions/** -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.toptal.com/developers/gitignore/api/python 2 | # Edit at https://www.toptal.com/developers/gitignore?templates=python 3 | 4 | ### Python ### 5 | # Byte-compiled / optimized / DLL files 6 | __pycache__/ 7 | *.py[cod] 8 | *$py.class 9 | 10 | # C extensions 11 | *.so 12 | 13 | # Distribution / packaging 14 | .Python 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | wheels/ 27 | share/python-wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .nox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | *.py,cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | cover/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | db.sqlite3 66 | db.sqlite3-journal 67 | 68 | # Flask stuff: 69 | instance/ 70 | .webassets-cache 71 | 72 | # Scrapy stuff: 73 | .scrapy 74 | 75 | # Sphinx documentation 76 | docs/_build/ 77 | 78 | # PyBuilder 79 | .pybuilder/ 80 | target/ 81 | 82 | # Jupyter Notebook 83 | .ipynb_checkpoints 84 | 85 | # IPython 86 | profile_default/ 87 | ipython_config.py 88 | 89 | # pyenv 90 | # For a library or package, you might want to ignore these files since the code is 91 | # intended to run in multiple environments; otherwise, check them in: 92 | # .python-version 93 | 94 | # pipenv 95 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 96 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 97 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 98 | # install all needed dependencies. 99 | #Pipfile.lock 100 | 101 | # poetry 102 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 103 | # This is especially recommended for binary packages to ensure reproducibility, and is more 104 | # commonly ignored for libraries. 105 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 106 | #poetry.lock 107 | 108 | # pdm 109 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 110 | #pdm.lock 111 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 112 | # in version control. 113 | # https://pdm.fming.dev/#use-with-ide 114 | .pdm.toml 115 | 116 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 117 | __pypackages__/ 118 | 119 | # Celery stuff 120 | celerybeat-schedule 121 | celerybeat.pid 122 | 123 | # SageMath parsed files 124 | *.sage.py 125 | 126 | # Environments 127 | .env 128 | .venv 129 | env/ 130 | venv/ 131 | ENV/ 132 | env.bak/ 133 | venv.bak/ 134 | 135 | # Spyder project settings 136 | .spyderproject 137 | .spyproject 138 | 139 | # Rope project settings 140 | .ropeproject 141 | 142 | # mkdocs documentation 143 | /site 144 | 145 | # mypy 146 | .mypy_cache/ 147 | .dmypy.json 148 | dmypy.json 149 | 150 | # Pyre type checker 151 | .pyre/ 152 | 153 | # pytype static type analyzer 154 | .pytype/ 155 | 156 | # Cython debug symbols 157 | cython_debug/ 158 | 159 | # PyCharm 160 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 161 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 162 | # and can be added to the global gitignore or merged into this file. For a more nuclear 163 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 164 | #.idea/ 165 | 166 | # End of https://www.toptal.com/developers/gitignore/api/python -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2022 Jonghwan Hyeon 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # joblib-progress 2 | A contextmanager to track progress of [`joblib`](https://joblib.readthedocs.io) execution using [`rich.progress`](https://rich.readthedocs.io). 3 | [![joblib-progress](https://asciinema.org/a/Ufe9v8MKfxIzMuvlv2IwCk29l.svg)](https://asciinema.org/a/Ufe9v8MKfxIzMuvlv2IwCk29l) 4 | 5 | ## Why 6 | The vanilla `multiprocessing` does not work when an object to multiprocess is not `pickle-able`. The `joblib` solves this, but then its progress is not tracked nicely. This library solves that tracking issue with `joblib`. 7 | 8 | ## Install 9 | ```bash 10 | > pip install joblib-progress 11 | ``` 12 | 13 | ## Usage 14 | ### If you know the number of items 15 | ```python 16 | import time 17 | 18 | from joblib import Parallel, delayed 19 | from joblib_progress import joblib_progress 20 | 21 | 22 | def slow_square(i): 23 | time.sleep(i / 2) 24 | return i ** 2 25 | 26 | with joblib_progress("Calculating square...", total=10): 27 | Parallel(n_jobs=4)(delayed(slow_square)(number) for number in range(10)) 28 | ``` 29 | 30 | ### If you don't know the number of items 31 | ```python 32 | with joblib_progress("Calculating square..."): 33 | Parallel(n_jobs=4)(delayed(slow_square)(number) for number in range(10)) 34 | ``` 35 | 36 | # Acknowledgments 37 | The idea of using `joblib.parallel.BatchCompletionCallBack` is referenced from https://stackoverflow.com/a/58936697/5133167 38 | -------------------------------------------------------------------------------- /assets/joblib-progress.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jonghwanhyeon/joblib-progress/3e2bac96b2b7175f1b12fa0bf406579ff371647c/assets/joblib-progress.gif -------------------------------------------------------------------------------- /joblib_progress/__init__.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | from typing import Optional 3 | 4 | import joblib 5 | from rich.console import Console 6 | from rich.progress import ( 7 | BarColumn, 8 | MofNCompleteColumn, 9 | Progress, 10 | SpinnerColumn, 11 | TaskProgressColumn, 12 | TextColumn, 13 | TimeElapsedColumn, 14 | TimeRemainingColumn, 15 | ) 16 | 17 | __version__ = "1.0.6" 18 | 19 | 20 | @contextlib.contextmanager 21 | def joblib_progress( 22 | description: Optional[str] = None, 23 | total: Optional[int] = None, 24 | console: Optional[Console] = None, 25 | ): 26 | if description is None: 27 | description = "Processing..." 28 | 29 | progress = Progress( 30 | SpinnerColumn(), 31 | TaskProgressColumn(), 32 | TextColumn("[progress.description]{task.description}"), 33 | BarColumn(), 34 | MofNCompleteColumn(), 35 | TimeElapsedColumn(), 36 | "<", 37 | TimeRemainingColumn(), 38 | console=console, 39 | ) 40 | task_id = progress.add_task(f"[cyan]{description}", total=total) 41 | 42 | print_progress = joblib.parallel.Parallel.print_progress 43 | 44 | def update_progress(self: joblib.parallel.Parallel): 45 | progress.update(task_id, completed=self.n_completed_tasks, refresh=True) 46 | return print_progress(self) 47 | 48 | try: 49 | joblib.parallel.Parallel.print_progress = update_progress 50 | progress.start() 51 | 52 | yield progress 53 | finally: 54 | progress.stop() 55 | joblib.parallel.Parallel.print_progress = print_progress 56 | -------------------------------------------------------------------------------- /joblib_progress/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jonghwanhyeon/joblib-progress/3e2bac96b2b7175f1b12fa0bf406579ff371647c/joblib_progress/py.typed -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.black] 6 | line-length = 120 7 | 8 | [tool.isort] 9 | profile = "black" 10 | line_length = 120 -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | --requirement=requirements.txt 2 | ruff -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | joblib 2 | rich -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = joblib-progress 3 | version = attr: joblib_progress.__version__ 4 | description = A contextmanager to track progress of joblib execution 5 | author = Jonghwan Hyeon 6 | author_email = jonghwanhyeon93@gmail.com 7 | url = https://github.com/jonghwanhyeon/joblib-progress 8 | long_description = file: README.md 9 | long_description_content_type = text/markdown 10 | keywords = 11 | joblib 12 | progress 13 | rich 14 | license = MIT 15 | classifiers = 16 | Development Status :: 4 - Beta 17 | License :: OSI Approved :: MIT License 18 | Programming Language :: Python :: 3 19 | Programming Language :: Python :: 3 :: Only 20 | 21 | [options] 22 | install_requires = 23 | joblib 24 | rich 25 | python_requires = >=3.7 26 | packages = find: -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | --------------------------------------------------------------------------------