├── tests ├── __init__.py ├── tests_tk.py ├── tests_gui.py ├── tests_notebook.py ├── tests_rich.py ├── tests_asyncio.py ├── tests_version.py ├── tests_dask.py ├── tests_itertools.py ├── conftest.py ├── tests_concurrent.py ├── tests_contrib.py ├── tests_keras.py ├── py37_asyncio.py ├── tests_contrib_logging.py ├── tests_synchronisation.py ├── tests_pandas.py ├── tests_main.py └── tests_perf.py ├── tqdm ├── __main__.py ├── _main.py ├── _tqdm.py ├── _tqdm_gui.py ├── _tqdm_notebook.py ├── version.py ├── _utils.py ├── contrib │ ├── bells.py │ ├── itertools.py │ ├── utils_worker.py │ ├── __init__.py │ ├── logging.py │ ├── discord.py │ ├── slack.py │ ├── concurrent.py │ └── telegram.py ├── _tqdm_pandas.py ├── completion.sh ├── autonotebook.py ├── auto.py ├── dask.py ├── __init__.py ├── asyncio.py ├── _monitor.py ├── keras.py ├── rich.py ├── gui.py ├── tk.py ├── tqdm.1 └── utils.py ├── logo.png ├── pyproject.toml ├── examples ├── include_no_requirements.py ├── wrapping_generators.py ├── pandas_progress_apply.py ├── async_coroutines.py ├── coroutine_pipe.py ├── tqdm_requests.py ├── simple_examples.py ├── redirect_print.py ├── parallel_bars.py ├── tqdm_wget.py ├── 7zx.py ├── paper.md └── paper.bib ├── setup.py ├── .zenodo.json ├── environment.yml ├── CODE_OF_CONDUCT.md ├── .pre-commit-config.yaml ├── LICENCE ├── tox.ini ├── setup.cfg └── Makefile /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tqdm/__main__.py: -------------------------------------------------------------------------------- 1 | from .cli import main 2 | 3 | main() 4 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tamanna18/tqdm/HEAD/logo.png -------------------------------------------------------------------------------- /tests/tests_tk.py: -------------------------------------------------------------------------------- 1 | """Test `tqdm.tk`.""" 2 | from .tests_tqdm import importorskip 3 | 4 | 5 | def test_tk_import(): 6 | """Test `tqdm.tk` import""" 7 | importorskip('tqdm.tk') 8 | -------------------------------------------------------------------------------- /tests/tests_gui.py: -------------------------------------------------------------------------------- 1 | """Test `tqdm.gui`.""" 2 | from .tests_tqdm import importorskip 3 | 4 | 5 | def test_gui_import(): 6 | """Test `tqdm.gui` import""" 7 | importorskip('tqdm.gui') 8 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.setuptools_scm] 6 | write_to = "tqdm/_dist_ver.py" 7 | write_to_template = "__version__ = '{version}'\n" 8 | -------------------------------------------------------------------------------- /tests/tests_notebook.py: -------------------------------------------------------------------------------- 1 | from tqdm.notebook import tqdm as tqdm_notebook 2 | 3 | 4 | def test_notebook_disabled_description(): 5 | """Test that set_description works for disabled tqdm_notebook""" 6 | with tqdm_notebook(1, disable=True) as t: 7 | t.set_description("description") 8 | -------------------------------------------------------------------------------- /tests/tests_rich.py: -------------------------------------------------------------------------------- 1 | """Test `tqdm.rich`.""" 2 | import sys 3 | 4 | from .tests_tqdm import importorskip, mark 5 | 6 | 7 | @mark.skipif(sys.version_info[:3] < (3, 6, 1), reason="`rich` needs py>=3.6.1") 8 | def test_rich_import(): 9 | """Test `tqdm.rich` import""" 10 | importorskip('tqdm.rich') 11 | -------------------------------------------------------------------------------- /tqdm/_main.py: -------------------------------------------------------------------------------- 1 | from warnings import warn 2 | 3 | from .cli import * # NOQA 4 | from .cli import __all__ # NOQA 5 | from .std import TqdmDeprecationWarning 6 | 7 | warn("This function will be removed in tqdm==5.0.0\n" 8 | "Please use `tqdm.cli.*` instead of `tqdm._main.*`", 9 | TqdmDeprecationWarning, stacklevel=2) 10 | -------------------------------------------------------------------------------- /tqdm/_tqdm.py: -------------------------------------------------------------------------------- 1 | from warnings import warn 2 | 3 | from .std import * # NOQA 4 | from .std import __all__ # NOQA 5 | from .std import TqdmDeprecationWarning 6 | 7 | warn("This function will be removed in tqdm==5.0.0\n" 8 | "Please use `tqdm.std.*` instead of `tqdm._tqdm.*`", 9 | TqdmDeprecationWarning, stacklevel=2) 10 | -------------------------------------------------------------------------------- /tqdm/_tqdm_gui.py: -------------------------------------------------------------------------------- 1 | from warnings import warn 2 | 3 | from .gui import * # NOQA 4 | from .gui import __all__ # NOQA 5 | from .std import TqdmDeprecationWarning 6 | 7 | warn("This function will be removed in tqdm==5.0.0\n" 8 | "Please use `tqdm.gui.*` instead of `tqdm._tqdm_gui.*`", 9 | TqdmDeprecationWarning, stacklevel=2) 10 | -------------------------------------------------------------------------------- /examples/include_no_requirements.py: -------------------------------------------------------------------------------- 1 | # How to import tqdm in any frontend without enforcing it as a dependency 2 | try: 3 | from tqdm.auto import tqdm 4 | except ImportError: 5 | 6 | def tqdm(*args, **kwargs): 7 | if args: 8 | return args[0] 9 | return kwargs.get('iterable', None) 10 | 11 | __all__ = ['tqdm'] 12 | -------------------------------------------------------------------------------- /tqdm/_tqdm_notebook.py: -------------------------------------------------------------------------------- 1 | from warnings import warn 2 | 3 | from .notebook import * # NOQA 4 | from .notebook import __all__ # NOQA 5 | from .std import TqdmDeprecationWarning 6 | 7 | warn("This function will be removed in tqdm==5.0.0\n" 8 | "Please use `tqdm.notebook.*` instead of `tqdm._tqdm_notebook.*`", 9 | TqdmDeprecationWarning, stacklevel=2) 10 | -------------------------------------------------------------------------------- /tests/tests_asyncio.py: -------------------------------------------------------------------------------- 1 | """Tests `tqdm.asyncio` on `python>=3.7`.""" 2 | import sys 3 | 4 | if sys.version_info[:2] > (3, 6): 5 | from .py37_asyncio import * # NOQA, pylint: disable=wildcard-import 6 | else: 7 | from .tests_tqdm import skip 8 | try: 9 | skip("async not supported", allow_module_level=True) 10 | except TypeError: 11 | pass 12 | -------------------------------------------------------------------------------- /tqdm/version.py: -------------------------------------------------------------------------------- 1 | """`tqdm` version detector. Precedence: installed dist, git, 'UNKNOWN'.""" 2 | try: 3 | from ._dist_ver import __version__ 4 | except ImportError: 5 | try: 6 | from setuptools_scm import get_version 7 | __version__ = get_version(root='..', relative_to=__file__) 8 | except (ImportError, LookupError): 9 | __version__ = "UNKNOWN" 10 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | import sys 4 | from os import path 5 | 6 | from setuptools import setup 7 | 8 | src_dir = path.abspath(path.dirname(__file__)) 9 | if sys.argv[1].lower().strip() == 'make': # exec Makefile commands 10 | import pymake 11 | fpath = path.join(src_dir, 'Makefile') 12 | pymake.main(['-f', fpath] + sys.argv[2:]) 13 | # Stop to avoid setup.py raising non-standard command error 14 | sys.exit(0) 15 | 16 | setup(use_scm_version=True) 17 | -------------------------------------------------------------------------------- /examples/wrapping_generators.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from tqdm.contrib import tenumerate, tmap, tzip 4 | 5 | for _ in tenumerate(range(int(1e6)), desc="builtin enumerate"): 6 | pass 7 | 8 | for _ in tenumerate(np.random.random((999, 999)), desc="numpy.ndenumerate"): 9 | pass 10 | 11 | for _ in tzip(np.arange(1e6), np.arange(1e6) + 1, desc="builtin zip"): 12 | pass 13 | 14 | mapped = tmap(lambda x: x + 1, np.arange(1e6), desc="builtin map") 15 | assert (np.arange(1e6) + 1 == list(mapped)).all() 16 | -------------------------------------------------------------------------------- /.zenodo.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "tqdm: A fast, Extensible Progress Bar for Python and CLI", 3 | "keywords": [ 4 | "progressbar", "progressmeter", "progress-bar", "meter", "rate", "eta", 5 | "console", "terminal", "time", "progress", "bar", "gui", "python", 6 | "parallel", "cli", "utilities", "shell", "batch"], 7 | "related_identifiers": [ 8 | {"identifier": "10.21105/joss.01277", "relation": "cites"}], 9 | "contributors": [ 10 | {"name": "tqdm developers", "type": "Other", "affiliation": "tqdm"}] 11 | } 12 | -------------------------------------------------------------------------------- /tests/tests_version.py: -------------------------------------------------------------------------------- 1 | """Test `tqdm.__version__`.""" 2 | import re 3 | from ast import literal_eval 4 | 5 | 6 | def test_version(): 7 | """Test version string""" 8 | from tqdm import __version__ 9 | version_parts = re.split('[.-]', __version__) 10 | if __version__ != "UNKNOWN": 11 | assert 3 <= len(version_parts), "must have at least Major.minor.patch" 12 | assert all( 13 | isinstance(literal_eval(i), int) for i in version_parts[:3] 14 | ), "Version Major.minor.patch must be 3 integers" 15 | -------------------------------------------------------------------------------- /tests/tests_dask.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | from time import sleep 4 | 5 | from .tests_tqdm import importorskip, mark 6 | 7 | pytestmark = mark.slow 8 | 9 | 10 | def test_dask(capsys): 11 | """Test tqdm.dask.TqdmCallback""" 12 | ProgressBar = importorskip('tqdm.dask').TqdmCallback 13 | dask = importorskip('dask') 14 | 15 | schedule = [dask.delayed(sleep)(i / 10) for i in range(5)] 16 | with ProgressBar(desc="computing"): 17 | dask.compute(schedule) 18 | _, err = capsys.readouterr() 19 | assert "computing: " in err 20 | assert '5/5' in err 21 | -------------------------------------------------------------------------------- /tqdm/_utils.py: -------------------------------------------------------------------------------- 1 | from warnings import warn 2 | 3 | from .std import TqdmDeprecationWarning 4 | from .utils import ( # NOQA, pylint: disable=unused-import 5 | CUR_OS, IS_NIX, IS_WIN, RE_ANSI, Comparable, FormatReplace, SimpleTextIOWrapper, _basestring, 6 | _environ_cols_wrapper, _is_ascii, _is_utf, _range, _screen_shape_linux, _screen_shape_tput, 7 | _screen_shape_windows, _screen_shape_wrapper, _supports_unicode, _term_move_up, _unich, 8 | _unicode, colorama) 9 | 10 | warn("This function will be removed in tqdm==5.0.0\n" 11 | "Please use `tqdm.utils.*` instead of `tqdm._utils.*`", 12 | TqdmDeprecationWarning, stacklevel=2) 13 | -------------------------------------------------------------------------------- /tests/tests_itertools.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for `tqdm.contrib.itertools`. 3 | """ 4 | import itertools as it 5 | 6 | from tqdm.contrib.itertools import product 7 | 8 | from .tests_tqdm import StringIO, closing 9 | 10 | 11 | class NoLenIter(object): 12 | def __init__(self, iterable): 13 | self._it = iterable 14 | 15 | def __iter__(self): 16 | for i in self._it: 17 | yield i 18 | 19 | 20 | def test_product(): 21 | """Test contrib.itertools.product""" 22 | with closing(StringIO()) as our_file: 23 | a = range(9) 24 | assert list(product(a, a[::-1], file=our_file)) == list(it.product(a, a[::-1])) 25 | 26 | assert list(product(a, NoLenIter(a), file=our_file)) == list(it.product(a, NoLenIter(a))) 27 | -------------------------------------------------------------------------------- /tqdm/contrib/bells.py: -------------------------------------------------------------------------------- 1 | """ 2 | Even more features than `tqdm.auto` (all the bells & whistles): 3 | 4 | - `tqdm.auto` 5 | - `tqdm.tqdm.pandas` 6 | - `tqdm.contrib.telegram` 7 | + uses `${TQDM_TELEGRAM_TOKEN}` and `${TQDM_TELEGRAM_CHAT_ID}` 8 | - `tqdm.contrib.discord` 9 | + uses `${TQDM_DISCORD_TOKEN}` and `${TQDM_DISCORD_CHANNEL_ID}` 10 | """ 11 | __all__ = ['tqdm', 'trange'] 12 | import warnings 13 | from os import getenv 14 | 15 | if getenv("TQDM_SLACK_TOKEN") and getenv("TQDM_SLACK_CHANNEL"): 16 | from .slack import tqdm, trange 17 | elif getenv("TQDM_TELEGRAM_TOKEN") and getenv("TQDM_TELEGRAM_CHAT_ID"): 18 | from .telegram import tqdm, trange 19 | elif getenv("TQDM_DISCORD_TOKEN") and getenv("TQDM_DISCORD_CHANNEL_ID"): 20 | from .discord import tqdm, trange 21 | else: 22 | from ..auto import tqdm, trange 23 | 24 | with warnings.catch_warnings(): 25 | warnings.simplefilter("ignore", category=FutureWarning) 26 | tqdm.pandas() 27 | -------------------------------------------------------------------------------- /tqdm/contrib/itertools.py: -------------------------------------------------------------------------------- 1 | """ 2 | Thin wrappers around `itertools`. 3 | """ 4 | from __future__ import absolute_import 5 | 6 | import itertools 7 | 8 | from ..auto import tqdm as tqdm_auto 9 | 10 | __author__ = {"github.com/": ["casperdcl"]} 11 | __all__ = ['product'] 12 | 13 | 14 | def product(*iterables, **tqdm_kwargs): 15 | """ 16 | Equivalent of `itertools.product`. 17 | 18 | Parameters 19 | ---------- 20 | tqdm_class : [default: tqdm.auto.tqdm]. 21 | """ 22 | kwargs = tqdm_kwargs.copy() 23 | tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) 24 | try: 25 | lens = list(map(len, iterables)) 26 | except TypeError: 27 | total = None 28 | else: 29 | total = 1 30 | for i in lens: 31 | total *= i 32 | kwargs.setdefault("total", total) 33 | with tqdm_class(**kwargs) as t: 34 | it = itertools.product(*iterables) 35 | for i in it: 36 | yield i 37 | t.update() 38 | -------------------------------------------------------------------------------- /tqdm/_tqdm_pandas.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | __author__ = "github.com/casperdcl" 4 | __all__ = ['tqdm_pandas'] 5 | 6 | 7 | def tqdm_pandas(tclass, **tqdm_kwargs): 8 | """ 9 | Registers the given `tqdm` instance with 10 | `pandas.core.groupby.DataFrameGroupBy.progress_apply`. 11 | """ 12 | from tqdm import TqdmDeprecationWarning 13 | 14 | if isinstance(tclass, type) or (getattr(tclass, '__name__', '').startswith( 15 | 'tqdm_')): # delayed adapter case 16 | TqdmDeprecationWarning( 17 | "Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm, ...)`.", 18 | fp_write=getattr(tqdm_kwargs.get('file', None), 'write', sys.stderr.write)) 19 | tclass.pandas(**tqdm_kwargs) 20 | else: 21 | TqdmDeprecationWarning( 22 | "Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm(...))`.", 23 | fp_write=getattr(tclass.fp, 'write', sys.stderr.write)) 24 | type(tclass).pandas(deprecated_t=tclass) 25 | -------------------------------------------------------------------------------- /tqdm/completion.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | _tqdm(){ 3 | local cur prv 4 | cur="${COMP_WORDS[COMP_CWORD]}" 5 | prv="${COMP_WORDS[COMP_CWORD - 1]}" 6 | 7 | case ${prv} in 8 | --bar_format|--buf_size|--colour|--comppath|--delay|--delim|--desc|--initial|--lock_args|--manpath|--maxinterval|--mininterval|--miniters|--ncols|--nrows|--position|--postfix|--smoothing|--total|--unit|--unit_divisor) 9 | # await user input 10 | ;; 11 | "--log") 12 | COMPREPLY=($(compgen -W 'CRITICAL FATAL ERROR WARN WARNING INFO DEBUG NOTSET' -- ${cur})) 13 | ;; 14 | *) 15 | COMPREPLY=($(compgen -W '--ascii --bar_format --buf_size --bytes --colour --comppath --delay --delim --desc --disable --dynamic_ncols --help --initial --leave --lock_args --log --manpath --maxinterval --mininterval --miniters --ncols --nrows --null --position --postfix --smoothing --tee --total --unit --unit_divisor --unit_scale --update --update_to --version --write_bytes -h -v' -- ${cur})) 16 | ;; 17 | esac 18 | } 19 | complete -F _tqdm tqdm 20 | -------------------------------------------------------------------------------- /examples/pandas_progress_apply.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | 4 | from tqdm.auto import tqdm 5 | 6 | df = pd.DataFrame(np.random.randint(0, 100, (100000, 6))) 7 | 8 | # Register `pandas.progress_apply` and `pandas.Series.map_apply` with `tqdm` 9 | # (can use `tqdm.gui.tqdm`, `tqdm.notebook.tqdm`, optional kwargs, etc.) 10 | tqdm.pandas(desc="my bar!") 11 | 12 | # Now you can use `progress_apply` instead of `apply` 13 | # and `progress_map` instead of `map` 14 | df.progress_apply(lambda x: x**2) 15 | # can also groupby: 16 | # df.groupby(0).progress_apply(lambda x: x**2) 17 | 18 | # -- Source code for `tqdm_pandas` (really simple!) 19 | # def tqdm_pandas(t): 20 | # from pandas.core.frame import DataFrame 21 | # def inner(df, func, *args, **kwargs): 22 | # t.total = groups.size // len(groups) 23 | # def wrapper(*args, **kwargs): 24 | # t.update(1) 25 | # return func(*args, **kwargs) 26 | # result = df.apply(wrapper, *args, **kwargs) 27 | # t.close() 28 | # return result 29 | # DataFrame.progress_apply = inner 30 | -------------------------------------------------------------------------------- /tqdm/autonotebook.py: -------------------------------------------------------------------------------- 1 | """ 2 | Automatically choose between `tqdm.notebook` and `tqdm.std`. 3 | 4 | Usage: 5 | >>> from tqdm.autonotebook import trange, tqdm 6 | >>> for i in trange(10): 7 | ... ... 8 | """ 9 | import sys 10 | from warnings import warn 11 | 12 | try: 13 | get_ipython = sys.modules['IPython'].get_ipython 14 | if 'IPKernelApp' not in get_ipython().config: # pragma: no cover 15 | raise ImportError("console") 16 | from .notebook import WARN_NOIPYW, IProgress 17 | if IProgress is None: 18 | from .std import TqdmWarning 19 | warn(WARN_NOIPYW, TqdmWarning, stacklevel=2) 20 | raise ImportError('ipywidgets') 21 | except Exception: 22 | from .std import tqdm, trange 23 | else: # pragma: no cover 24 | from .notebook import tqdm, trange 25 | from .std import TqdmExperimentalWarning 26 | warn("Using `tqdm.autonotebook.tqdm` in notebook mode." 27 | " Use `tqdm.tqdm` instead to force console mode" 28 | " (e.g. in jupyter console)", TqdmExperimentalWarning, stacklevel=2) 29 | __all__ = ["tqdm", "trange"] 30 | -------------------------------------------------------------------------------- /examples/async_coroutines.py: -------------------------------------------------------------------------------- 1 | """ 2 | Asynchronous examples using `asyncio`, `async` and `await` on `python>=3.7`. 3 | """ 4 | import asyncio 5 | 6 | from tqdm.asyncio import tqdm, trange 7 | 8 | 9 | def count(start=0, step=1): 10 | i = start 11 | while True: 12 | new_start = yield i 13 | if new_start is None: 14 | i += step 15 | else: 16 | i = new_start 17 | 18 | 19 | async def main(): 20 | N = int(1e6) 21 | async for row in tqdm(trange(N, desc="inner"), desc="outer"): 22 | if row >= N: 23 | break 24 | with tqdm(count(), desc="coroutine", total=N + 2) as pbar: 25 | async for row in pbar: 26 | if row == N: 27 | pbar.send(-10) 28 | elif row < 0: 29 | assert row == -9 30 | break 31 | # should be ~1sec rather than ~50s due to async scheduling 32 | for i in tqdm.as_completed([asyncio.sleep(0.01 * i) 33 | for i in range(100, 0, -1)], desc="as_completed"): 34 | await i 35 | 36 | 37 | if __name__ == "__main__": 38 | asyncio.run(main()) 39 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | # development environment 2 | name: tqdm 3 | channels: 4 | - conda-forge 5 | - defaults 6 | dependencies: 7 | # base 8 | - python=3 9 | - pip 10 | - ipykernel 11 | - ipywidgets 12 | - setuptools 13 | - setuptools_scm 14 | - toml 15 | # test env managers 16 | - pre-commit 17 | - tox 18 | - asv 19 | # tests (native) 20 | - pytest 21 | - pytest-cov 22 | - pytest-timeout 23 | - pytest-asyncio # [py>=3.7] 24 | - nbval 25 | - coverage 26 | # extras 27 | - dask # dask 28 | - matplotlib # gui 29 | - numpy # pandas, keras, contrib.tenumerate 30 | - pandas 31 | - tensorflow # keras 32 | - slack-sdk # contrib.slack 33 | - requests # contrib.telegram 34 | - rich # rich 35 | - argopt # `cd wiki && pymake` 36 | - twine # `pymake pypi` 37 | - wheel # `setup.py bdist_wheel` 38 | # `cd docs && pymake` 39 | - mkdocs-material 40 | - pydoc-markdown 41 | - pygments 42 | - pymdown-extensions 43 | - pip: 44 | - py-make >=0.1.0 # `setup.py make/pymake` 45 | - mkdocs-minify-plugin # `cd docs && pymake` 46 | - git+https://github.com/tqdm/jsmin@python3-only#egg=jsmin # `cd docs && pymake` 47 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """Shared pytest config.""" 2 | import sys 3 | 4 | from pytest import fixture 5 | 6 | from tqdm import tqdm 7 | 8 | 9 | @fixture(autouse=True) 10 | def pretest_posttest(): 11 | """Fixture for all tests ensuring environment cleanup""" 12 | try: 13 | sys.setswitchinterval(1) 14 | except AttributeError: 15 | sys.setcheckinterval(100) # deprecated 16 | 17 | if getattr(tqdm, "_instances", False): 18 | n = len(tqdm._instances) 19 | if n: 20 | tqdm._instances.clear() 21 | raise EnvironmentError( 22 | "{0} `tqdm` instances still in existence PRE-test".format(n)) 23 | yield 24 | if getattr(tqdm, "_instances", False): 25 | n = len(tqdm._instances) 26 | if n: 27 | tqdm._instances.clear() 28 | raise EnvironmentError( 29 | "{0} `tqdm` instances still in existence POST-test".format(n)) 30 | 31 | 32 | if sys.version_info[0] > 2: 33 | @fixture 34 | def capsysbin(capsysbinary): 35 | """alias for capsysbinary (py3)""" 36 | return capsysbinary 37 | else: 38 | @fixture 39 | def capsysbin(capsys): 40 | """alias for capsys (py2)""" 41 | return capsys 42 | -------------------------------------------------------------------------------- /tqdm/auto.py: -------------------------------------------------------------------------------- 1 | """ 2 | Enables multiple commonly used features. 3 | 4 | Method resolution order: 5 | 6 | - `tqdm.autonotebook` without import warnings 7 | - `tqdm.asyncio` on Python3.6+ 8 | - `tqdm.std` base class 9 | 10 | Usage: 11 | >>> from tqdm.auto import trange, tqdm 12 | >>> for i in trange(10): 13 | ... ... 14 | """ 15 | import sys 16 | import warnings 17 | 18 | from .std import TqdmExperimentalWarning 19 | 20 | with warnings.catch_warnings(): 21 | warnings.simplefilter("ignore", category=TqdmExperimentalWarning) 22 | from .autonotebook import tqdm as notebook_tqdm 23 | from .autonotebook import trange as notebook_trange 24 | 25 | if sys.version_info[:2] < (3, 6): 26 | tqdm = notebook_tqdm 27 | trange = notebook_trange 28 | else: # Python3.6+ 29 | from .asyncio import tqdm as asyncio_tqdm 30 | from .std import tqdm as std_tqdm 31 | 32 | if notebook_tqdm != std_tqdm: 33 | class tqdm(notebook_tqdm, asyncio_tqdm): # pylint: disable=inconsistent-mro 34 | pass 35 | else: 36 | tqdm = asyncio_tqdm 37 | 38 | def trange(*args, **kwargs): 39 | """ 40 | A shortcut for `tqdm.auto.tqdm(range(*args), **kwargs)`. 41 | """ 42 | return tqdm(range(*args), **kwargs) 43 | 44 | __all__ = ["tqdm", "trange"] 45 | -------------------------------------------------------------------------------- /tqdm/contrib/utils_worker.py: -------------------------------------------------------------------------------- 1 | """ 2 | IO/concurrency helpers for `tqdm.contrib`. 3 | """ 4 | from __future__ import absolute_import 5 | 6 | from collections import deque 7 | from concurrent.futures import ThreadPoolExecutor 8 | 9 | from ..auto import tqdm as tqdm_auto 10 | 11 | __author__ = {"github.com/": ["casperdcl"]} 12 | __all__ = ['MonoWorker'] 13 | 14 | 15 | class MonoWorker(object): 16 | """ 17 | Supports one running task and one waiting task. 18 | The waiting task is the most recent submitted (others are discarded). 19 | """ 20 | def __init__(self): 21 | self.pool = ThreadPoolExecutor(max_workers=1) 22 | self.futures = deque([], 2) 23 | 24 | def submit(self, func, *args, **kwargs): 25 | """`func(*args, **kwargs)` may replace currently waiting task.""" 26 | futures = self.futures 27 | if len(futures) == futures.maxlen: 28 | running = futures.popleft() 29 | if not running.done(): 30 | if len(futures): # clear waiting 31 | waiting = futures.pop() 32 | waiting.cancel() 33 | futures.appendleft(running) # re-insert running 34 | try: 35 | waiting = self.pool.submit(func, *args, **kwargs) 36 | except Exception as e: 37 | tqdm_auto.write(str(e)) 38 | else: 39 | futures.append(waiting) 40 | return waiting 41 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | International law supersedes all codes of conduct, including this one. 2 | Authors of codes of conduct presuming the authority to create new laws 3 | should be brought to justice. 4 | 5 | This being said, if you still insist on wringing advice from us: 6 | Be kind. Be professional. Be friendly. Be respectful. Focus on the project. 7 | [Report inappropriate conduct](https://help.github.com/en/articles/reporting-abuse-or-spam). 8 | 9 | Contributions which fail to conform to these guidelines may be reworded, 10 | censored, and/or reported as abuse without notice. 11 | 12 | ## The long explanation 13 | 14 | We celebrate the fact that all contributions come from different humans with 15 | their own unique attributes, background and history. However we also place a 16 | strong emphasis on neutrality and a simple focus on the project's goals. 17 | Contributions are thus particularly welcome when references to gender, 18 | ethnicity, religion and so on are avoided as much as possible. 19 | This is for inclusiveness, professionality, and also privacy and security. 20 | This likely means that apart from usernames (which frequently unavoidably hint 21 | about the author's background) there should be no personal information included 22 | in contributions. 23 | 24 | Note that we have no contract with contributors. Introducing contributor licence 25 | agreements (CLAs) would allow us to enforce additional rules but would also be a 26 | scary barrier for new contributors. We don't want to be scary. We love you! 27 | -------------------------------------------------------------------------------- /tqdm/dask.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from functools import partial 4 | 5 | from dask.callbacks import Callback 6 | 7 | from .auto import tqdm as tqdm_auto 8 | 9 | __author__ = {"github.com/": ["casperdcl"]} 10 | __all__ = ['TqdmCallback'] 11 | 12 | 13 | class TqdmCallback(Callback): 14 | """Dask callback for task progress.""" 15 | def __init__(self, start=None, pretask=None, tqdm_class=tqdm_auto, 16 | **tqdm_kwargs): 17 | """ 18 | Parameters 19 | ---------- 20 | tqdm_class : optional 21 | `tqdm` class to use for bars [default: `tqdm.auto.tqdm`]. 22 | tqdm_kwargs : optional 23 | Any other arguments used for all bars. 24 | """ 25 | super(TqdmCallback, self).__init__(start=start, pretask=pretask) 26 | if tqdm_kwargs: 27 | tqdm_class = partial(tqdm_class, **tqdm_kwargs) 28 | self.tqdm_class = tqdm_class 29 | 30 | def _start_state(self, _, state): 31 | self.pbar = self.tqdm_class(total=sum( 32 | len(state[k]) for k in ['ready', 'waiting', 'running', 'finished'])) 33 | 34 | def _posttask(self, *_, **__): 35 | self.pbar.update() 36 | 37 | def _finish(self, *_, **__): 38 | self.pbar.close() 39 | 40 | def display(self): 41 | """Displays in the current cell in Notebooks.""" 42 | container = getattr(self.bar, 'container', None) 43 | if container is None: 44 | return 45 | from .notebook import display 46 | display(container) 47 | -------------------------------------------------------------------------------- /examples/coroutine_pipe.py: -------------------------------------------------------------------------------- 1 | """ 2 | Inserting `tqdm` as a "pipe" in a chain of coroutines. 3 | Not to be confused with `asyncio.coroutine`. 4 | """ 5 | from functools import wraps 6 | 7 | from tqdm.auto import tqdm 8 | 9 | 10 | def autonext(func): 11 | @wraps(func) 12 | def inner(*args, **kwargs): 13 | res = func(*args, **kwargs) 14 | next(res) 15 | return res 16 | return inner 17 | 18 | 19 | @autonext 20 | def tqdm_pipe(target, **tqdm_kwargs): 21 | """ 22 | Coroutine chain pipe `send()`ing to `target`. 23 | 24 | This: 25 | >>> r = receiver() 26 | >>> p = producer(r) 27 | >>> next(r) 28 | >>> next(p) 29 | 30 | Becomes: 31 | >>> r = receiver() 32 | >>> t = tqdm.pipe(r) 33 | >>> p = producer(t) 34 | >>> next(r) 35 | >>> next(p) 36 | """ 37 | with tqdm(**tqdm_kwargs) as pbar: 38 | while True: 39 | obj = (yield) 40 | target.send(obj) 41 | pbar.update() 42 | 43 | 44 | def source(target): 45 | for i in ["foo", "bar", "baz", "pythonista", "python", "py"]: 46 | target.send(i) 47 | target.close() 48 | 49 | 50 | @autonext 51 | def grep(pattern, target): 52 | while True: 53 | line = (yield) 54 | if pattern in line: 55 | target.send(line) 56 | 57 | 58 | @autonext 59 | def sink(): 60 | while True: 61 | line = (yield) 62 | tqdm.write(line) 63 | 64 | 65 | if __name__ == "__main__": 66 | source( 67 | tqdm_pipe( 68 | grep('python', 69 | sink()))) 70 | -------------------------------------------------------------------------------- /examples/tqdm_requests.py: -------------------------------------------------------------------------------- 1 | """An example of wrapping manual tqdm updates for `requests.get`. 2 | See also: tqdm_wget.py. 3 | 4 | Usage: 5 | tqdm_requests.py [options] 6 | 7 | Options: 8 | -h, --help 9 | Print this help message and exit 10 | -u URL, --url URL : string, optional 11 | The url to fetch. 12 | [default: https://caspersci.uk.to/matryoshka.zip] 13 | -o FILE, --output FILE : string, optional 14 | The local file path in which to save the url [default: /dev/null]. 15 | """ 16 | 17 | from os import devnull 18 | 19 | import requests 20 | from docopt import docopt 21 | 22 | from tqdm.auto import tqdm 23 | 24 | opts = docopt(__doc__) 25 | 26 | eg_link = opts['--url'] 27 | eg_file = eg_link.replace('/', ' ').split()[-1] 28 | eg_out = opts['--output'].replace("/dev/null", devnull) 29 | 30 | response = requests.get(eg_link, stream=True) 31 | with open(eg_out, "wb") as fout: 32 | with tqdm( 33 | # all optional kwargs 34 | unit='B', unit_scale=True, unit_divisor=1024, miniters=1, 35 | desc=eg_file, total=int(response.headers.get('content-length', 0)) 36 | ) as pbar: 37 | for chunk in response.iter_content(chunk_size=4096): 38 | fout.write(chunk) 39 | pbar.update(len(chunk)) 40 | 41 | # Even simpler progress by wrapping the output file's `write()` 42 | response = requests.get(eg_link, stream=True) 43 | with tqdm.wrapattr( 44 | open(eg_out, "wb"), "write", 45 | unit='B', unit_scale=True, unit_divisor=1024, miniters=1, 46 | desc=eg_file, total=int(response.headers.get('content-length', 0)) 47 | ) as fout: 48 | for chunk in response.iter_content(chunk_size=4096): 49 | fout.write(chunk) 50 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_language_version: 2 | python: python3 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v4.1.0 6 | hooks: 7 | - id: check-added-large-files 8 | - id: check-case-conflict 9 | - id: check-docstring-first 10 | - id: check-executables-have-shebangs 11 | - id: check-toml 12 | - id: check-merge-conflict 13 | - id: check-yaml 14 | - id: debug-statements 15 | - id: end-of-file-fixer 16 | - id: mixed-line-ending 17 | - id: sort-simple-yaml 18 | - id: trailing-whitespace 19 | exclude: ^README.rst$ 20 | - repo: local 21 | hooks: 22 | - id: todo 23 | name: Check TODO 24 | language: pygrep 25 | entry: WIP 26 | args: [-i] 27 | types: [text] 28 | exclude: ^(.pre-commit-config.yaml|.github/workflows/test.yml)$ 29 | - id: pytest 30 | name: pytest quick 31 | language: python 32 | entry: pytest 33 | args: [-qq, --durations=1, -k=not slow] 34 | types: [python] 35 | pass_filenames: false 36 | additional_dependencies: 37 | - numpy 38 | - pandas 39 | - pytest-timeout 40 | - pytest-asyncio 41 | - repo: https://gitlab.com/pycqa/flake8 42 | rev: 3.9.2 43 | hooks: 44 | - id: flake8 45 | args: [-j8] 46 | additional_dependencies: 47 | - flake8-broken-line 48 | - flake8-bugbear 49 | - flake8-comprehensions 50 | - flake8-debugger 51 | - flake8-isort 52 | - flake8-string-format 53 | - flake8-type-annotations 54 | - repo: https://github.com/PyCQA/isort 55 | rev: 5.10.1 56 | hooks: 57 | - id: isort 58 | - repo: https://github.com/kynan/nbstripout 59 | rev: 0.5.0 60 | hooks: 61 | - id: nbstripout 62 | args: [--keep-count, --keep-output] 63 | -------------------------------------------------------------------------------- /examples/simple_examples.py: -------------------------------------------------------------------------------- 1 | """ 2 | # Simple tqdm examples and profiling 3 | 4 | # Benchmark 5 | for i in _range(int(1e8)): 6 | pass 7 | 8 | # Basic demo 9 | import tqdm 10 | for i in tqdm.trange(int(1e8)): 11 | pass 12 | 13 | # Some decorations 14 | import tqdm 15 | for i in tqdm.trange(int(1e8), miniters=int(1e6), ascii=True, 16 | desc="cool", dynamic_ncols=True): 17 | pass 18 | 19 | # Nested bars 20 | from tqdm import trange 21 | for i in trange(10): 22 | for j in trange(int(1e7), leave=False, unit_scale=True): 23 | pass 24 | 25 | # Experimental GUI demo 26 | import tqdm 27 | for i in tqdm.tgrange(int(1e8)): 28 | pass 29 | 30 | # Comparison to https://code.google.com/p/python-progressbar/ 31 | try: 32 | from progressbar.progressbar import ProgressBar 33 | except ImportError: 34 | pass 35 | else: 36 | for i in ProgressBar()(_range(int(1e8))): 37 | pass 38 | 39 | # Dynamic miniters benchmark 40 | from tqdm import trange 41 | for i in trange(int(1e8), miniters=None, mininterval=0.1, smoothing=0): 42 | pass 43 | 44 | # Fixed miniters benchmark 45 | from tqdm import trange 46 | for i in trange(int(1e8), miniters=4500000, mininterval=0.1, smoothing=0): 47 | pass 48 | """ 49 | 50 | import re 51 | from time import sleep 52 | from timeit import timeit 53 | 54 | # Simple demo 55 | from tqdm import trange 56 | 57 | for _ in trange(16, leave=True): 58 | sleep(0.1) 59 | 60 | # Profiling/overhead tests 61 | stmts = filter(None, re.split(r'\n\s*#.*?\n', __doc__)) 62 | for s in stmts: 63 | print(s.replace('import tqdm\n', '')) 64 | print(timeit(stmt='try:\n\t_range = xrange' 65 | '\nexcept:\n\t_range = range\n' + s, number=1), 'seconds') 66 | -------------------------------------------------------------------------------- /examples/redirect_print.py: -------------------------------------------------------------------------------- 1 | """Redirecting writing 2 | 3 | If using a library that can print messages to the console, editing the library 4 | by replacing `print()` with `tqdm.write()` may not be desirable. 5 | In that case, redirecting `sys.stdout` to `tqdm.write()` is an option. 6 | 7 | To redirect `sys.stdout`, create a file-like class that will write 8 | any input string to `tqdm.write()`, and supply the arguments 9 | `file=sys.stdout, dynamic_ncols=True`. 10 | 11 | A reusable canonical example is given below: 12 | """ 13 | from __future__ import print_function 14 | 15 | import contextlib 16 | import sys 17 | from time import sleep 18 | 19 | from tqdm import tqdm 20 | from tqdm.contrib import DummyTqdmFile 21 | 22 | 23 | @contextlib.contextmanager 24 | def std_out_err_redirect_tqdm(): 25 | orig_out_err = sys.stdout, sys.stderr 26 | try: 27 | # sys.stdout = sys.stderr = DummyTqdmFile(orig_out_err[0]) 28 | sys.stdout, sys.stderr = map(DummyTqdmFile, orig_out_err) 29 | yield orig_out_err[0] 30 | # Relay exceptions 31 | except Exception as exc: 32 | raise exc 33 | # Always restore sys.stdout/err if necessary 34 | finally: 35 | sys.stdout, sys.stderr = orig_out_err 36 | 37 | 38 | def some_fun(i): 39 | print("Fee, fi, fo,".split()[i]) 40 | 41 | 42 | # Redirect stdout to tqdm.write() 43 | with std_out_err_redirect_tqdm() as orig_stdout: 44 | # tqdm needs the original stdout 45 | # and dynamic_ncols=True to autodetect console width 46 | for i in tqdm(range(3), file=orig_stdout, dynamic_ncols=True): 47 | # order of the following two lines should not matter 48 | some_fun(i) 49 | sleep(.5) 50 | 51 | # After the `with`, printing is restored 52 | print("Done!") 53 | -------------------------------------------------------------------------------- /tests/tests_concurrent.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for `tqdm.contrib.concurrent`. 3 | """ 4 | from pytest import warns 5 | 6 | from tqdm.contrib.concurrent import process_map, thread_map 7 | 8 | from .tests_tqdm import StringIO, TqdmWarning, closing, importorskip, mark, skip 9 | 10 | 11 | def incr(x): 12 | """Dummy function""" 13 | return x + 1 14 | 15 | 16 | def test_thread_map(): 17 | """Test contrib.concurrent.thread_map""" 18 | with closing(StringIO()) as our_file: 19 | a = range(9) 20 | b = [i + 1 for i in a] 21 | try: 22 | assert thread_map(lambda x: x + 1, a, file=our_file) == b 23 | except ImportError as err: 24 | skip(str(err)) 25 | assert thread_map(incr, a, file=our_file) == b 26 | 27 | 28 | def test_process_map(): 29 | """Test contrib.concurrent.process_map""" 30 | with closing(StringIO()) as our_file: 31 | a = range(9) 32 | b = [i + 1 for i in a] 33 | try: 34 | assert process_map(incr, a, file=our_file) == b 35 | except ImportError as err: 36 | skip(str(err)) 37 | 38 | 39 | @mark.parametrize("iterables,should_warn", [([], False), (['x'], False), ([()], False), 40 | (['x', ()], False), (['x' * 1001], True), 41 | (['x' * 100, ('x',) * 1001], True)]) 42 | def test_chunksize_warning(iterables, should_warn): 43 | """Test contrib.concurrent.process_map chunksize warnings""" 44 | patch = importorskip('unittest.mock').patch 45 | with patch('tqdm.contrib.concurrent._executor_map'): 46 | if should_warn: 47 | warns(TqdmWarning, process_map, incr, *iterables) 48 | else: 49 | process_map(incr, *iterables) 50 | -------------------------------------------------------------------------------- /tqdm/__init__.py: -------------------------------------------------------------------------------- 1 | from ._monitor import TMonitor, TqdmSynchronisationWarning 2 | from ._tqdm_pandas import tqdm_pandas 3 | from .cli import main # TODO: remove in v5.0.0 4 | from .gui import tqdm as tqdm_gui # TODO: remove in v5.0.0 5 | from .gui import trange as tgrange # TODO: remove in v5.0.0 6 | from .std import ( 7 | TqdmDeprecationWarning, TqdmExperimentalWarning, TqdmKeyError, TqdmMonitorWarning, 8 | TqdmTypeError, TqdmWarning, tqdm, trange) 9 | from .version import __version__ 10 | 11 | __all__ = ['tqdm', 'tqdm_gui', 'trange', 'tgrange', 'tqdm_pandas', 12 | 'tqdm_notebook', 'tnrange', 'main', 'TMonitor', 13 | 'TqdmTypeError', 'TqdmKeyError', 14 | 'TqdmWarning', 'TqdmDeprecationWarning', 15 | 'TqdmExperimentalWarning', 16 | 'TqdmMonitorWarning', 'TqdmSynchronisationWarning', 17 | '__version__'] 18 | 19 | 20 | def tqdm_notebook(*args, **kwargs): # pragma: no cover 21 | """See tqdm.notebook.tqdm for full documentation""" 22 | from warnings import warn 23 | 24 | from .notebook import tqdm as _tqdm_notebook 25 | warn("This function will be removed in tqdm==5.0.0\n" 26 | "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`", 27 | TqdmDeprecationWarning, stacklevel=2) 28 | return _tqdm_notebook(*args, **kwargs) 29 | 30 | 31 | def tnrange(*args, **kwargs): # pragma: no cover 32 | """ 33 | A shortcut for `tqdm.notebook.tqdm(xrange(*args), **kwargs)`. 34 | On Python3+, `range` is used instead of `xrange`. 35 | """ 36 | from warnings import warn 37 | 38 | from .notebook import trange as _tnrange 39 | warn("Please use `tqdm.notebook.trange` instead of `tqdm.tnrange`", 40 | TqdmDeprecationWarning, stacklevel=2) 41 | return _tnrange(*args, **kwargs) 42 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | `tqdm` is a product of collaborative work. 2 | Unless otherwise stated, all authors (see commit logs) retain copyright 3 | for their respective work, and release the work under the MIT licence 4 | (text below). 5 | 6 | Exceptions or notable authors are listed below 7 | in reverse chronological order: 8 | 9 | * files: * 10 | MPLv2.0 2015-2021 (c) Casper da Costa-Luis 11 | [casperdcl](https://github.com/casperdcl). 12 | * files: tqdm/_tqdm.py 13 | MIT 2016 (c) [PR #96] on behalf of Google Inc. 14 | * files: tqdm/_tqdm.py setup.py README.rst MANIFEST.in .gitignore 15 | MIT 2013 (c) Noam Yorav-Raphael, original author. 16 | 17 | [PR #96]: https://github.com/tqdm/tqdm/pull/96 18 | 19 | 20 | Mozilla Public Licence (MPL) v. 2.0 - Exhibit A 21 | ----------------------------------------------- 22 | 23 | This Source Code Form is subject to the terms of the 24 | Mozilla Public License, v. 2.0. 25 | If a copy of the MPL was not distributed with this project, 26 | You can obtain one at https://mozilla.org/MPL/2.0/. 27 | 28 | 29 | MIT License (MIT) 30 | ----------------- 31 | 32 | Copyright (c) 2013 noamraph 33 | 34 | Permission is hereby granted, free of charge, to any person obtaining a copy of 35 | this software and associated documentation files (the "Software"), to deal in 36 | the Software without restriction, including without limitation the rights to 37 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 38 | the Software, and to permit persons to whom the Software is furnished to do so, 39 | subject to the following conditions: 40 | 41 | The above copyright notice and this permission notice shall be included in all 42 | copies or substantial portions of the Software. 43 | 44 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 45 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 46 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 47 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 48 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 49 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 50 | -------------------------------------------------------------------------------- /examples/parallel_bars.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import sys 4 | from concurrent.futures import ThreadPoolExecutor 5 | from functools import partial 6 | from multiprocessing import Pool, RLock, freeze_support 7 | from random import random 8 | from threading import RLock as TRLock 9 | from time import sleep 10 | 11 | from tqdm.auto import tqdm, trange 12 | from tqdm.contrib.concurrent import process_map, thread_map 13 | 14 | NUM_SUBITERS = 9 15 | PY2 = sys.version_info[:1] <= (2,) 16 | 17 | 18 | def progresser(n, auto_position=True, write_safe=False, blocking=True, progress=False): 19 | interval = random() * 0.002 / (NUM_SUBITERS - n + 2) # nosec 20 | total = 5000 21 | text = "#{0}, est. {1:<04.2}s".format(n, interval * total) 22 | for _ in trange(total, desc=text, disable=not progress, 23 | lock_args=None if blocking else (False,), 24 | position=None if auto_position else n): 25 | sleep(interval) 26 | # NB: may not clear instances with higher `position` upon completion 27 | # since this worker may not know about other bars #796 28 | if write_safe: 29 | # we think we know about other bars (currently only py3 threading) 30 | if n == 6: 31 | tqdm.write("n == 6 completed") 32 | return n + 1 33 | 34 | 35 | if __name__ == '__main__': 36 | freeze_support() # for Windows support 37 | L = list(range(NUM_SUBITERS))[::-1] 38 | 39 | print("Simple thread mapping") 40 | thread_map(partial(progresser, write_safe=not PY2), L, max_workers=4) 41 | 42 | print("Simple process mapping") 43 | process_map(partial(progresser), L, max_workers=4) 44 | 45 | print("Manual nesting") 46 | for i in trange(16, desc="1"): 47 | for _ in trange(16, desc="2 @ %d" % i, leave=i % 2): 48 | sleep(0.01) 49 | 50 | print("Multi-processing") 51 | tqdm.set_lock(RLock()) 52 | p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) 53 | p.map(partial(progresser, progress=True), L) 54 | 55 | print("Multi-threading") 56 | tqdm.set_lock(TRLock()) 57 | pool_args = {} 58 | if not PY2: 59 | pool_args.update(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) 60 | with ThreadPoolExecutor(**pool_args) as p: 61 | p.map(partial(progresser, progress=True, write_safe=not PY2, blocking=False), L) 62 | -------------------------------------------------------------------------------- /tests/tests_contrib.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for `tqdm.contrib`. 3 | """ 4 | import sys 5 | 6 | import pytest 7 | 8 | from tqdm import tqdm 9 | from tqdm.contrib import tenumerate, tmap, tzip 10 | 11 | from .tests_tqdm import StringIO, closing, importorskip 12 | 13 | 14 | def incr(x): 15 | """Dummy function""" 16 | return x + 1 17 | 18 | 19 | @pytest.mark.parametrize("tqdm_kwargs", [{}, {"tqdm_class": tqdm}]) 20 | def test_enumerate(tqdm_kwargs): 21 | """Test contrib.tenumerate""" 22 | with closing(StringIO()) as our_file: 23 | a = range(9) 24 | assert list(tenumerate(a, file=our_file, **tqdm_kwargs)) == list(enumerate(a)) 25 | assert list(tenumerate(a, 42, file=our_file, **tqdm_kwargs)) == list( 26 | enumerate(a, 42) 27 | ) 28 | with closing(StringIO()) as our_file: 29 | _ = list(tenumerate(iter(a), file=our_file, **tqdm_kwargs)) 30 | assert "100%" not in our_file.getvalue() 31 | with closing(StringIO()) as our_file: 32 | _ = list(tenumerate(iter(a), file=our_file, total=len(a), **tqdm_kwargs)) 33 | assert "100%" in our_file.getvalue() 34 | 35 | 36 | def test_enumerate_numpy(): 37 | """Test contrib.tenumerate(numpy.ndarray)""" 38 | np = importorskip("numpy") 39 | with closing(StringIO()) as our_file: 40 | a = np.random.random((42, 7)) 41 | assert list(tenumerate(a, file=our_file)) == list(np.ndenumerate(a)) 42 | 43 | 44 | @pytest.mark.parametrize("tqdm_kwargs", [{}, {"tqdm_class": tqdm}]) 45 | def test_zip(tqdm_kwargs): 46 | """Test contrib.tzip""" 47 | with closing(StringIO()) as our_file: 48 | a = range(9) 49 | b = [i + 1 for i in a] 50 | if sys.version_info[:1] < (3,): 51 | assert tzip(a, b, file=our_file, **tqdm_kwargs) == zip(a, b) 52 | else: 53 | gen = tzip(a, b, file=our_file, **tqdm_kwargs) 54 | assert gen != list(zip(a, b)) 55 | assert list(gen) == list(zip(a, b)) 56 | 57 | 58 | @pytest.mark.parametrize("tqdm_kwargs", [{}, {"tqdm_class": tqdm}]) 59 | def test_map(tqdm_kwargs): 60 | """Test contrib.tmap""" 61 | with closing(StringIO()) as our_file: 62 | a = range(9) 63 | b = [i + 1 for i in a] 64 | if sys.version_info[:1] < (3,): 65 | assert tmap(lambda x: x + 1, a, file=our_file, **tqdm_kwargs) == map( 66 | incr, a 67 | ) 68 | else: 69 | gen = tmap(lambda x: x + 1, a, file=our_file, **tqdm_kwargs) 70 | assert gen != b 71 | assert list(gen) == b 72 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | # Tox (https://tox.testrun.org/) is a tool for running tests 2 | # in multiple virtualenvs. This configuration file will run the 3 | # test suite on all supported python versions. To use it, "pip install tox" 4 | # and then run "tox" from this directory. 5 | 6 | [tox] 7 | # deprecation warning: py{27,py2,34,35,36} 8 | envlist=py{27,34,35,36,37,38,39,310,py2,py3}{,-tf}{,-keras}, perf, setup.py 9 | isolated_build=True 10 | 11 | [gh-actions] 12 | python= 13 | 2.7: py27 14 | 3.5: py35 15 | 3.6: py36 16 | 3.7: py37 17 | 3.8: py38 18 | 3.9: py39 19 | 3.10: py310 20 | pypy-2.7: pypy2 21 | pypy-3.7: pypy3 22 | [gh-actions:env] 23 | PLATFORM= 24 | ubuntu: tf-keras 25 | 26 | [core] 27 | deps= 28 | pytest 29 | py3{4,5,6}: pytest<7 30 | pytest-cov 31 | pytest-timeout 32 | py3{7,8,9,10}: pytest-asyncio 33 | py3{6,7,8,9,10}: ipywidgets 34 | py3{7,8,9,10}: git+https://github.com/casperdcl/nbval.git@master#egg=nbval 35 | coverage 36 | coveralls 37 | codecov 38 | commands= 39 | - coveralls 40 | codecov -X pycov -e TOXENV 41 | - codacy report -l Python -r coverage.xml --partial 42 | 43 | [testenv] 44 | passenv=TOXENV CI GITHUB_* CODECOV_* COVERALLS_* CODACY_* HOME 45 | deps= 46 | {[core]deps} 47 | cython 48 | dask[delayed] 49 | matplotlib 50 | numpy 51 | pandas 52 | tf: tensorflow!=2.5.0 53 | !py27-keras: keras 54 | py27-keras: keras<2.5 55 | py35-keras: keras<2.7 56 | py27-tf: protobuf<3.18 57 | py3{6,7,8,9,10}: rich 58 | commands= 59 | py3{4,5,6}: pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" -o addopts= -v --tb=short -rxs -W=error --durations=0 --durations-min=0.1 60 | py3{7,8,9,10}: pytest --cov=tqdm --cov-report= tests_notebook.ipynb --nbval --nbval-current-env -W=ignore --nbval-sanitize-with=setup.cfg 61 | py3{7,8,9,10}: pytest --cov=tqdm --cov-report=xml --cov-report=term --cov-append -k "not perf" 62 | {[core]commands} 63 | allowlist_externals=codacy 64 | 65 | [testenv:py{27,py2}{,-tf}{,-keras}] 66 | commands= 67 | pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" -o addopts= -v --tb=short -rxs -W=error --durations=10 68 | {[core]commands} 69 | 70 | # no cython/numpy/pandas 71 | [testenv:py{34,py2,py3}] 72 | deps={[core]deps} 73 | 74 | [testenv:perf] 75 | deps= 76 | pytest 77 | pytest-timeout 78 | pytest-asyncio 79 | commands=pytest -k perf 80 | 81 | [testenv:setup.py] 82 | deps= 83 | docutils 84 | pygments 85 | py-make>=0.1.0 86 | commands= 87 | {envpython} setup.py check --restructuredtext --metadata --strict 88 | {envpython} setup.py make none 89 | -------------------------------------------------------------------------------- /tqdm/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Thin wrappers around common functions. 3 | 4 | Subpackages contain potentially unstable extensions. 5 | """ 6 | import sys 7 | from functools import wraps 8 | 9 | from ..auto import tqdm as tqdm_auto 10 | from ..std import tqdm 11 | from ..utils import ObjectWrapper 12 | 13 | __author__ = {"github.com/": ["casperdcl"]} 14 | __all__ = ['tenumerate', 'tzip', 'tmap'] 15 | 16 | 17 | class DummyTqdmFile(ObjectWrapper): 18 | """Dummy file-like that will write to tqdm""" 19 | 20 | def __init__(self, wrapped): 21 | super(DummyTqdmFile, self).__init__(wrapped) 22 | self._buf = [] 23 | 24 | def write(self, x, nolock=False): 25 | nl = b"\n" if isinstance(x, bytes) else "\n" 26 | pre, sep, post = x.rpartition(nl) 27 | if sep: 28 | blank = type(nl)() 29 | tqdm.write(blank.join(self._buf + [pre, sep]), 30 | end=blank, file=self._wrapped, nolock=nolock) 31 | self._buf = [post] 32 | else: 33 | self._buf.append(x) 34 | 35 | def __del__(self): 36 | if self._buf: 37 | blank = type(self._buf[0])() 38 | try: 39 | tqdm.write(blank.join(self._buf), end=blank, file=self._wrapped) 40 | except (OSError, ValueError): 41 | pass 42 | 43 | 44 | def builtin_iterable(func): 45 | """Wraps `func()` output in a `list()` in py2""" 46 | if sys.version_info[:1] < (3,): 47 | @wraps(func) 48 | def inner(*args, **kwargs): 49 | return list(func(*args, **kwargs)) 50 | return inner 51 | return func 52 | 53 | 54 | def tenumerate(iterable, start=0, total=None, tqdm_class=tqdm_auto, **tqdm_kwargs): 55 | """ 56 | Equivalent of `numpy.ndenumerate` or builtin `enumerate`. 57 | 58 | Parameters 59 | ---------- 60 | tqdm_class : [default: tqdm.auto.tqdm]. 61 | """ 62 | try: 63 | import numpy as np 64 | except ImportError: 65 | pass 66 | else: 67 | if isinstance(iterable, np.ndarray): 68 | return tqdm_class(np.ndenumerate(iterable), total=total or iterable.size, 69 | **tqdm_kwargs) 70 | return enumerate(tqdm_class(iterable, total=total, **tqdm_kwargs), start) 71 | 72 | 73 | @builtin_iterable 74 | def tzip(iter1, *iter2plus, **tqdm_kwargs): 75 | """ 76 | Equivalent of builtin `zip`. 77 | 78 | Parameters 79 | ---------- 80 | tqdm_class : [default: tqdm.auto.tqdm]. 81 | """ 82 | kwargs = tqdm_kwargs.copy() 83 | tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) 84 | for i in zip(tqdm_class(iter1, **kwargs), *iter2plus): 85 | yield i 86 | 87 | 88 | @builtin_iterable 89 | def tmap(function, *sequences, **tqdm_kwargs): 90 | """ 91 | Equivalent of builtin `map`. 92 | 93 | Parameters 94 | ---------- 95 | tqdm_class : [default: tqdm.auto.tqdm]. 96 | """ 97 | for i in tzip(*sequences, **tqdm_kwargs): 98 | yield function(*i) 99 | -------------------------------------------------------------------------------- /tests/tests_keras.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | from .tests_tqdm import importorskip, mark 4 | 5 | pytestmark = mark.slow 6 | 7 | 8 | @mark.filterwarnings("ignore:.*:DeprecationWarning") 9 | def test_keras(capsys): 10 | """Test tqdm.keras.TqdmCallback""" 11 | TqdmCallback = importorskip('tqdm.keras').TqdmCallback 12 | np = importorskip('numpy') 13 | try: 14 | import keras as K 15 | except ImportError: 16 | K = importorskip('tensorflow.keras') 17 | 18 | # 1D autoencoder 19 | dtype = np.float32 20 | model = K.models.Sequential([ 21 | K.layers.InputLayer((1, 1), dtype=dtype), K.layers.Conv1D(1, 1)]) 22 | model.compile("adam", "mse") 23 | x = np.random.rand(100, 1, 1).astype(dtype) 24 | batch_size = 10 25 | batches = len(x) / batch_size 26 | epochs = 5 27 | 28 | # just epoch (no batch) progress 29 | model.fit( 30 | x, 31 | x, 32 | epochs=epochs, 33 | batch_size=batch_size, 34 | verbose=False, 35 | callbacks=[ 36 | TqdmCallback( 37 | epochs, 38 | desc="training", 39 | data_size=len(x), 40 | batch_size=batch_size, 41 | verbose=0)]) 42 | _, res = capsys.readouterr() 43 | assert "training: " in res 44 | assert "{epochs}/{epochs}".format(epochs=epochs) in res 45 | assert "{batches}/{batches}".format(batches=batches) not in res 46 | 47 | # full (epoch and batch) progress 48 | model.fit( 49 | x, 50 | x, 51 | epochs=epochs, 52 | batch_size=batch_size, 53 | verbose=False, 54 | callbacks=[ 55 | TqdmCallback( 56 | epochs, 57 | desc="training", 58 | data_size=len(x), 59 | batch_size=batch_size, 60 | verbose=2)]) 61 | _, res = capsys.readouterr() 62 | assert "training: " in res 63 | assert "{epochs}/{epochs}".format(epochs=epochs) in res 64 | assert "{batches}/{batches}".format(batches=batches) in res 65 | 66 | # auto-detect epochs and batches 67 | model.fit( 68 | x, 69 | x, 70 | epochs=epochs, 71 | batch_size=batch_size, 72 | verbose=False, 73 | callbacks=[TqdmCallback(desc="training", verbose=2)]) 74 | _, res = capsys.readouterr() 75 | assert "training: " in res 76 | assert "{epochs}/{epochs}".format(epochs=epochs) in res 77 | assert "{batches}/{batches}".format(batches=batches) in res 78 | 79 | # continue training (start from epoch != 0) 80 | initial_epoch = 3 81 | model.fit( 82 | x, 83 | x, 84 | initial_epoch=initial_epoch, 85 | epochs=epochs, 86 | batch_size=batch_size, 87 | verbose=False, 88 | callbacks=[TqdmCallback(desc="training", verbose=0, 89 | miniters=1, mininterval=0, maxinterval=0)]) 90 | _, res = capsys.readouterr() 91 | assert "training: " in res 92 | assert "{epochs}/{epochs}".format(epochs=initial_epoch - 1) not in res 93 | assert "{epochs}/{epochs}".format(epochs=epochs) in res 94 | -------------------------------------------------------------------------------- /tqdm/asyncio.py: -------------------------------------------------------------------------------- 1 | """ 2 | Asynchronous progressbar decorator for iterators. 3 | Includes a default `range` iterator printing to `stderr`. 4 | 5 | Usage: 6 | >>> from tqdm.asyncio import trange, tqdm 7 | >>> async for i in trange(10): 8 | ... ... 9 | """ 10 | import asyncio 11 | from sys import version_info 12 | 13 | from .std import tqdm as std_tqdm 14 | 15 | __author__ = {"github.com/": ["casperdcl"]} 16 | __all__ = ['tqdm_asyncio', 'tarange', 'tqdm', 'trange'] 17 | 18 | 19 | class tqdm_asyncio(std_tqdm): 20 | """ 21 | Asynchronous-friendly version of tqdm (Python 3.6+). 22 | """ 23 | def __init__(self, iterable=None, *args, **kwargs): 24 | super(tqdm_asyncio, self).__init__(iterable, *args, **kwargs) 25 | self.iterable_awaitable = False 26 | if iterable is not None: 27 | if hasattr(iterable, "__anext__"): 28 | self.iterable_next = iterable.__anext__ 29 | self.iterable_awaitable = True 30 | elif hasattr(iterable, "__next__"): 31 | self.iterable_next = iterable.__next__ 32 | else: 33 | self.iterable_iterator = iter(iterable) 34 | self.iterable_next = self.iterable_iterator.__next__ 35 | 36 | def __aiter__(self): 37 | return self 38 | 39 | async def __anext__(self): 40 | try: 41 | if self.iterable_awaitable: 42 | res = await self.iterable_next() 43 | else: 44 | res = self.iterable_next() 45 | self.update() 46 | return res 47 | except StopIteration: 48 | self.close() 49 | raise StopAsyncIteration 50 | except BaseException: 51 | self.close() 52 | raise 53 | 54 | def send(self, *args, **kwargs): 55 | return self.iterable.send(*args, **kwargs) 56 | 57 | @classmethod 58 | def as_completed(cls, fs, *, loop=None, timeout=None, total=None, **tqdm_kwargs): 59 | """ 60 | Wrapper for `asyncio.as_completed`. 61 | """ 62 | if total is None: 63 | total = len(fs) 64 | kwargs = {} 65 | if version_info[:2] < (3, 10): 66 | kwargs['loop'] = loop 67 | yield from cls(asyncio.as_completed(fs, timeout=timeout, **kwargs), 68 | total=total, **tqdm_kwargs) 69 | 70 | @classmethod 71 | async def gather(cls, *fs, loop=None, timeout=None, total=None, **tqdm_kwargs): 72 | """ 73 | Wrapper for `asyncio.gather`. 74 | """ 75 | async def wrap_awaitable(i, f): 76 | return i, await f 77 | 78 | ifs = [wrap_awaitable(i, f) for i, f in enumerate(fs)] 79 | res = [await f for f in cls.as_completed(ifs, loop=loop, timeout=timeout, 80 | total=total, **tqdm_kwargs)] 81 | return [i for _, i in sorted(res)] 82 | 83 | 84 | def tarange(*args, **kwargs): 85 | """ 86 | A shortcut for `tqdm.asyncio.tqdm(range(*args), **kwargs)`. 87 | """ 88 | return tqdm_asyncio(range(*args), **kwargs) 89 | 90 | 91 | # Aliases 92 | tqdm = tqdm_asyncio 93 | trange = tarange 94 | -------------------------------------------------------------------------------- /tests/py37_asyncio.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from functools import partial 3 | from sys import platform 4 | from time import time 5 | 6 | from tqdm.asyncio import tarange, tqdm_asyncio 7 | 8 | from .tests_tqdm import StringIO, closing, mark 9 | 10 | tqdm = partial(tqdm_asyncio, miniters=0, mininterval=0) 11 | trange = partial(tarange, miniters=0, mininterval=0) 12 | as_completed = partial(tqdm_asyncio.as_completed, miniters=0, mininterval=0) 13 | gather = partial(tqdm_asyncio.gather, miniters=0, mininterval=0) 14 | 15 | 16 | def count(start=0, step=1): 17 | i = start 18 | while True: 19 | new_start = yield i 20 | if new_start is None: 21 | i += step 22 | else: 23 | i = new_start 24 | 25 | 26 | async def acount(*args, **kwargs): 27 | for i in count(*args, **kwargs): 28 | yield i 29 | 30 | 31 | @mark.asyncio 32 | async def test_break(): 33 | """Test asyncio break""" 34 | pbar = tqdm(count()) 35 | async for _ in pbar: 36 | break 37 | pbar.close() 38 | 39 | 40 | @mark.asyncio 41 | async def test_generators(capsys): 42 | """Test asyncio generators""" 43 | with tqdm(count(), desc="counter") as pbar: 44 | async for i in pbar: 45 | if i >= 8: 46 | break 47 | _, err = capsys.readouterr() 48 | assert '9it' in err 49 | 50 | with tqdm(acount(), desc="async_counter") as pbar: 51 | async for i in pbar: 52 | if i >= 8: 53 | break 54 | _, err = capsys.readouterr() 55 | assert '9it' in err 56 | 57 | 58 | @mark.asyncio 59 | async def test_range(): 60 | """Test asyncio range""" 61 | with closing(StringIO()) as our_file: 62 | async for _ in tqdm(range(9), desc="range", file=our_file): 63 | pass 64 | assert '9/9' in our_file.getvalue() 65 | our_file.seek(0) 66 | our_file.truncate() 67 | 68 | async for _ in trange(9, desc="trange", file=our_file): 69 | pass 70 | assert '9/9' in our_file.getvalue() 71 | 72 | 73 | @mark.asyncio 74 | async def test_nested(): 75 | """Test asyncio nested""" 76 | with closing(StringIO()) as our_file: 77 | async for _ in tqdm(trange(9, desc="inner", file=our_file), 78 | desc="outer", file=our_file): 79 | pass 80 | assert 'inner: 100%' in our_file.getvalue() 81 | assert 'outer: 100%' in our_file.getvalue() 82 | 83 | 84 | @mark.asyncio 85 | async def test_coroutines(): 86 | """Test asyncio coroutine.send""" 87 | with closing(StringIO()) as our_file: 88 | with tqdm(count(), file=our_file) as pbar: 89 | async for i in pbar: 90 | if i == 9: 91 | pbar.send(-10) 92 | elif i < 0: 93 | assert i == -9 94 | break 95 | assert '10it' in our_file.getvalue() 96 | 97 | 98 | @mark.slow 99 | @mark.asyncio 100 | @mark.parametrize("tol", [0.2 if platform.startswith("darwin") else 0.1]) 101 | async def test_as_completed(capsys, tol): 102 | """Test asyncio as_completed""" 103 | for retry in range(3): 104 | t = time() 105 | skew = time() - t 106 | for i in as_completed([asyncio.sleep(0.01 * i) for i in range(30, 0, -1)]): 107 | await i 108 | t = time() - t - 2 * skew 109 | try: 110 | assert 0.3 * (1 - tol) < t < 0.3 * (1 + tol), t 111 | _, err = capsys.readouterr() 112 | assert '30/30' in err 113 | except AssertionError: 114 | if retry == 2: 115 | raise 116 | 117 | 118 | async def double(i): 119 | return i * 2 120 | 121 | 122 | @mark.asyncio 123 | async def test_gather(capsys): 124 | """Test asyncio gather""" 125 | res = await gather(*map(double, range(30))) 126 | _, err = capsys.readouterr() 127 | assert '30/30' in err 128 | assert res == list(range(0, 30 * 2, 2)) 129 | -------------------------------------------------------------------------------- /tqdm/_monitor.py: -------------------------------------------------------------------------------- 1 | import atexit 2 | from threading import Event, Thread, current_thread 3 | from time import time 4 | from warnings import warn 5 | 6 | __all__ = ["TMonitor", "TqdmSynchronisationWarning"] 7 | 8 | 9 | class TqdmSynchronisationWarning(RuntimeWarning): 10 | """tqdm multi-thread/-process errors which may cause incorrect nesting 11 | but otherwise no adverse effects""" 12 | pass 13 | 14 | 15 | class TMonitor(Thread): 16 | """ 17 | Monitoring thread for tqdm bars. 18 | Monitors if tqdm bars are taking too much time to display 19 | and readjusts miniters automatically if necessary. 20 | 21 | Parameters 22 | ---------- 23 | tqdm_cls : class 24 | tqdm class to use (can be core tqdm or a submodule). 25 | sleep_interval : float 26 | Time to sleep between monitoring checks. 27 | """ 28 | _test = {} # internal vars for unit testing 29 | 30 | def __init__(self, tqdm_cls, sleep_interval): 31 | Thread.__init__(self) 32 | self.daemon = True # kill thread when main killed (KeyboardInterrupt) 33 | self.woken = 0 # last time woken up, to sync with monitor 34 | self.tqdm_cls = tqdm_cls 35 | self.sleep_interval = sleep_interval 36 | self._time = self._test.get("time", time) 37 | self.was_killed = self._test.get("Event", Event)() 38 | atexit.register(self.exit) 39 | self.start() 40 | 41 | def exit(self): 42 | self.was_killed.set() 43 | if self is not current_thread(): 44 | self.join() 45 | return self.report() 46 | 47 | def get_instances(self): 48 | # returns a copy of started `tqdm_cls` instances 49 | return [i for i in self.tqdm_cls._instances.copy() 50 | # Avoid race by checking that the instance started 51 | if hasattr(i, 'start_t')] 52 | 53 | def run(self): 54 | cur_t = self._time() 55 | while True: 56 | # After processing and before sleeping, notify that we woke 57 | # Need to be done just before sleeping 58 | self.woken = cur_t 59 | # Sleep some time... 60 | self.was_killed.wait(self.sleep_interval) 61 | # Quit if killed 62 | if self.was_killed.is_set(): 63 | return 64 | # Then monitor! 65 | # Acquire lock (to access _instances) 66 | with self.tqdm_cls.get_lock(): 67 | cur_t = self._time() 68 | # Check tqdm instances are waiting too long to print 69 | instances = self.get_instances() 70 | for instance in instances: 71 | # Check event in loop to reduce blocking time on exit 72 | if self.was_killed.is_set(): 73 | return 74 | # Only if mininterval > 1 (else iterations are just slow) 75 | # and last refresh exceeded maxinterval 76 | if ( 77 | instance.miniters > 1 78 | and (cur_t - instance.last_print_t) >= instance.maxinterval 79 | ): 80 | # force bypassing miniters on next iteration 81 | # (dynamic_miniters adjusts mininterval automatically) 82 | instance.miniters = 1 83 | # Refresh now! (works only for manual tqdm) 84 | instance.refresh(nolock=True) 85 | # Remove accidental long-lived strong reference 86 | del instance 87 | if instances != self.get_instances(): # pragma: nocover 88 | warn("Set changed size during iteration" + 89 | " (see https://github.com/tqdm/tqdm/issues/481)", 90 | TqdmSynchronisationWarning, stacklevel=2) 91 | # Remove accidental long-lived strong references 92 | del instances 93 | 94 | def report(self): 95 | return not self.was_killed.is_set() 96 | -------------------------------------------------------------------------------- /examples/tqdm_wget.py: -------------------------------------------------------------------------------- 1 | """An example of wrapping manual tqdm updates for `urllib` reporthook. 2 | See also: tqdm_requests.py. 3 | 4 | # `urllib.urlretrieve` documentation 5 | > If present, the hook function will be called once 6 | > on establishment of the network connection and once after each block read 7 | > thereafter. The hook will be passed three arguments; a count of blocks 8 | > transferred so far, a block size in bytes, and the total size of the file. 9 | 10 | Usage: 11 | tqdm_wget.py [options] 12 | 13 | Options: 14 | -h, --help 15 | Print this help message and exit 16 | -u URL, --url URL : string, optional 17 | The url to fetch. 18 | [default: https://caspersci.uk.to/matryoshka.zip] 19 | -o FILE, --output FILE : string, optional 20 | The local file path in which to save the url [default: /dev/null]. 21 | """ 22 | 23 | try: 24 | from urllib import request as urllib 25 | except ImportError: # py2 26 | import urllib 27 | from os import devnull 28 | 29 | from docopt import docopt 30 | 31 | from tqdm.auto import tqdm 32 | 33 | 34 | def my_hook(t): 35 | """Wraps tqdm instance. 36 | 37 | Don't forget to close() or __exit__() 38 | the tqdm instance once you're done with it (easiest using `with` syntax). 39 | 40 | Example 41 | ------- 42 | 43 | >>> with tqdm(...) as t: 44 | ... reporthook = my_hook(t) 45 | ... urllib.urlretrieve(..., reporthook=reporthook) 46 | 47 | """ 48 | last_b = [0] 49 | 50 | def update_to(b=1, bsize=1, tsize=None): 51 | """ 52 | b : int, optional 53 | Number of blocks transferred so far [default: 1]. 54 | bsize : int, optional 55 | Size of each block (in tqdm units) [default: 1]. 56 | tsize : int, optional 57 | Total size (in tqdm units). If [default: None] or -1, 58 | remains unchanged. 59 | """ 60 | if tsize not in (None, -1): 61 | t.total = tsize 62 | displayed = t.update((b - last_b[0]) * bsize) 63 | last_b[0] = b 64 | return displayed 65 | 66 | return update_to 67 | 68 | 69 | class TqdmUpTo(tqdm): 70 | """Alternative Class-based version of the above. 71 | 72 | Provides `update_to(n)` which uses `tqdm.update(delta_n)`. 73 | 74 | Inspired by [twine#242](https://github.com/pypa/twine/pull/242), 75 | [here](https://github.com/pypa/twine/commit/42e55e06). 76 | """ 77 | 78 | def update_to(self, b=1, bsize=1, tsize=None): 79 | """ 80 | b : int, optional 81 | Number of blocks transferred so far [default: 1]. 82 | bsize : int, optional 83 | Size of each block (in tqdm units) [default: 1]. 84 | tsize : int, optional 85 | Total size (in tqdm units). If [default: None] remains unchanged. 86 | """ 87 | if tsize is not None: 88 | self.total = tsize 89 | return self.update(b * bsize - self.n) # also sets self.n = b * bsize 90 | 91 | 92 | opts = docopt(__doc__) 93 | 94 | eg_link = opts['--url'] 95 | eg_file = eg_link.replace('/', ' ').split()[-1] 96 | eg_out = opts['--output'].replace("/dev/null", devnull) 97 | # with tqdm(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, 98 | # desc=eg_file) as t: # all optional kwargs 99 | # urllib.urlretrieve(eg_link, filename=eg_out, 100 | # reporthook=my_hook(t), data=None) 101 | with TqdmUpTo(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, 102 | desc=eg_file) as t: # all optional kwargs 103 | urllib.urlretrieve( # nosec 104 | eg_link, filename=eg_out, reporthook=t.update_to, data=None) 105 | t.total = t.n 106 | 107 | # Even simpler progress by wrapping the output file's `write()` 108 | response = urllib.urlopen(eg_link) # nosec 109 | with tqdm.wrapattr(open(eg_out, "wb"), "write", 110 | miniters=1, desc=eg_file, 111 | total=getattr(response, 'length', None)) as fout: 112 | for chunk in response: 113 | fout.write(chunk) 114 | -------------------------------------------------------------------------------- /tqdm/contrib/logging.py: -------------------------------------------------------------------------------- 1 | """ 2 | Helper functionality for interoperability with stdlib `logging`. 3 | """ 4 | from __future__ import absolute_import 5 | 6 | import logging 7 | import sys 8 | from contextlib import contextmanager 9 | 10 | try: 11 | from typing import Iterator, List, Optional, Type # pylint: disable=unused-import 12 | except ImportError: 13 | pass 14 | 15 | from ..std import tqdm as std_tqdm 16 | 17 | 18 | class _TqdmLoggingHandler(logging.StreamHandler): 19 | def __init__( 20 | self, 21 | tqdm_class=std_tqdm # type: Type[std_tqdm] 22 | ): 23 | super(_TqdmLoggingHandler, self).__init__() 24 | self.tqdm_class = tqdm_class 25 | 26 | def emit(self, record): 27 | try: 28 | msg = self.format(record) 29 | self.tqdm_class.write(msg, file=self.stream) 30 | self.flush() 31 | except (KeyboardInterrupt, SystemExit): 32 | raise 33 | except: # noqa pylint: disable=bare-except 34 | self.handleError(record) 35 | 36 | 37 | def _is_console_logging_handler(handler): 38 | return (isinstance(handler, logging.StreamHandler) 39 | and handler.stream in {sys.stdout, sys.stderr}) 40 | 41 | 42 | def _get_first_found_console_logging_handler(handlers): 43 | for handler in handlers: 44 | if _is_console_logging_handler(handler): 45 | return handler 46 | 47 | 48 | @contextmanager 49 | def logging_redirect_tqdm( 50 | loggers=None, # type: Optional[List[logging.Logger]], 51 | tqdm_class=std_tqdm # type: Type[std_tqdm] 52 | ): 53 | # type: (...) -> Iterator[None] 54 | """ 55 | Context manager redirecting console logging to `tqdm.write()`, leaving 56 | other logging handlers (e.g. log files) unaffected. 57 | 58 | Parameters 59 | ---------- 60 | loggers : list, optional 61 | Which handlers to redirect (default: [logging.root]). 62 | tqdm_class : optional 63 | 64 | Example 65 | ------- 66 | ```python 67 | import logging 68 | from tqdm import trange 69 | from tqdm.contrib.logging import logging_redirect_tqdm 70 | 71 | LOG = logging.getLogger(__name__) 72 | 73 | if __name__ == '__main__': 74 | logging.basicConfig(level=logging.INFO) 75 | with logging_redirect_tqdm(): 76 | for i in trange(9): 77 | if i == 4: 78 | LOG.info("console logging redirected to `tqdm.write()`") 79 | # logging restored 80 | ``` 81 | """ 82 | if loggers is None: 83 | loggers = [logging.root] 84 | original_handlers_list = [logger.handlers for logger in loggers] 85 | try: 86 | for logger in loggers: 87 | tqdm_handler = _TqdmLoggingHandler(tqdm_class) 88 | orig_handler = _get_first_found_console_logging_handler(logger.handlers) 89 | if orig_handler is not None: 90 | tqdm_handler.setFormatter(orig_handler.formatter) 91 | tqdm_handler.stream = orig_handler.stream 92 | logger.handlers = [ 93 | handler for handler in logger.handlers 94 | if not _is_console_logging_handler(handler)] + [tqdm_handler] 95 | yield 96 | finally: 97 | for logger, original_handlers in zip(loggers, original_handlers_list): 98 | logger.handlers = original_handlers 99 | 100 | 101 | @contextmanager 102 | def tqdm_logging_redirect( 103 | *args, 104 | # loggers=None, # type: Optional[List[logging.Logger]] 105 | # tqdm=None, # type: Optional[Type[tqdm.tqdm]] 106 | **kwargs 107 | ): 108 | # type: (...) -> Iterator[None] 109 | """ 110 | Convenience shortcut for: 111 | ```python 112 | with tqdm_class(*args, **tqdm_kwargs) as pbar: 113 | with logging_redirect_tqdm(loggers=loggers, tqdm_class=tqdm_class): 114 | yield pbar 115 | ``` 116 | 117 | Parameters 118 | ---------- 119 | tqdm_class : optional, (default: tqdm.std.tqdm). 120 | loggers : optional, list. 121 | **tqdm_kwargs : passed to `tqdm_class`. 122 | """ 123 | tqdm_kwargs = kwargs.copy() 124 | loggers = tqdm_kwargs.pop('loggers', None) 125 | tqdm_class = tqdm_kwargs.pop('tqdm_class', std_tqdm) 126 | with tqdm_class(*args, **tqdm_kwargs) as pbar: 127 | with logging_redirect_tqdm(loggers=loggers, tqdm_class=tqdm_class): 128 | yield pbar 129 | -------------------------------------------------------------------------------- /tqdm/contrib/discord.py: -------------------------------------------------------------------------------- 1 | """ 2 | Sends updates to a Discord bot. 3 | 4 | Usage: 5 | >>> from tqdm.contrib.discord import tqdm, trange 6 | >>> for i in trange(10, token='{token}', channel_id='{channel_id}'): 7 | ... ... 8 | 9 | ![screenshot](https://img.tqdm.ml/screenshot-discord.png) 10 | """ 11 | from __future__ import absolute_import 12 | 13 | import logging 14 | from os import getenv 15 | 16 | try: 17 | from disco.client import Client, ClientConfig 18 | except ImportError: 19 | raise ImportError("Please `pip install disco-py`") 20 | 21 | from ..auto import tqdm as tqdm_auto 22 | from ..utils import _range 23 | from .utils_worker import MonoWorker 24 | 25 | __author__ = {"github.com/": ["casperdcl"]} 26 | __all__ = ['DiscordIO', 'tqdm_discord', 'tdrange', 'tqdm', 'trange'] 27 | 28 | 29 | class DiscordIO(MonoWorker): 30 | """Non-blocking file-like IO using a Discord Bot.""" 31 | def __init__(self, token, channel_id): 32 | """Creates a new message in the given `channel_id`.""" 33 | super(DiscordIO, self).__init__() 34 | config = ClientConfig() 35 | config.token = token 36 | client = Client(config) 37 | self.text = self.__class__.__name__ 38 | try: 39 | self.message = client.api.channels_messages_create(channel_id, self.text) 40 | except Exception as e: 41 | tqdm_auto.write(str(e)) 42 | self.message = None 43 | 44 | def write(self, s): 45 | """Replaces internal `message`'s text with `s`.""" 46 | if not s: 47 | s = "..." 48 | s = s.replace('\r', '').strip() 49 | if s == self.text: 50 | return # skip duplicate message 51 | message = self.message 52 | if message is None: 53 | return 54 | self.text = s 55 | try: 56 | future = self.submit(message.edit, '`' + s + '`') 57 | except Exception as e: 58 | tqdm_auto.write(str(e)) 59 | else: 60 | return future 61 | 62 | 63 | class tqdm_discord(tqdm_auto): 64 | """ 65 | Standard `tqdm.auto.tqdm` but also sends updates to a Discord Bot. 66 | May take a few seconds to create (`__init__`). 67 | 68 | - create a discord bot (not public, no requirement of OAuth2 code 69 | grant, only send message permissions) & invite it to a channel: 70 | 71 | - copy the bot `{token}` & `{channel_id}` and paste below 72 | 73 | >>> from tqdm.contrib.discord import tqdm, trange 74 | >>> for i in tqdm(iterable, token='{token}', channel_id='{channel_id}'): 75 | ... ... 76 | """ 77 | def __init__(self, *args, **kwargs): 78 | """ 79 | Parameters 80 | ---------- 81 | token : str, required. Discord token 82 | [default: ${TQDM_DISCORD_TOKEN}]. 83 | channel_id : int, required. Discord channel ID 84 | [default: ${TQDM_DISCORD_CHANNEL_ID}]. 85 | mininterval : float, optional. 86 | Minimum of [default: 1.5] to avoid rate limit. 87 | 88 | See `tqdm.auto.tqdm.__init__` for other parameters. 89 | """ 90 | if not kwargs.get('disable'): 91 | kwargs = kwargs.copy() 92 | logging.getLogger("HTTPClient").setLevel(logging.WARNING) 93 | self.dio = DiscordIO( 94 | kwargs.pop('token', getenv("TQDM_DISCORD_TOKEN")), 95 | kwargs.pop('channel_id', getenv("TQDM_DISCORD_CHANNEL_ID"))) 96 | kwargs['mininterval'] = max(1.5, kwargs.get('mininterval', 1.5)) 97 | super(tqdm_discord, self).__init__(*args, **kwargs) 98 | 99 | def display(self, **kwargs): 100 | super(tqdm_discord, self).display(**kwargs) 101 | fmt = self.format_dict 102 | if fmt.get('bar_format', None): 103 | fmt['bar_format'] = fmt['bar_format'].replace( 104 | '', '{bar:10u}').replace('{bar}', '{bar:10u}') 105 | else: 106 | fmt['bar_format'] = '{l_bar}{bar:10u}{r_bar}' 107 | self.dio.write(self.format_meter(**fmt)) 108 | 109 | def clear(self, *args, **kwargs): 110 | super(tqdm_discord, self).clear(*args, **kwargs) 111 | if not self.disable: 112 | self.dio.write("") 113 | 114 | 115 | def tdrange(*args, **kwargs): 116 | """ 117 | A shortcut for `tqdm.contrib.discord.tqdm(xrange(*args), **kwargs)`. 118 | On Python3+, `range` is used instead of `xrange`. 119 | """ 120 | return tqdm_discord(_range(*args), **kwargs) 121 | 122 | 123 | # Aliases 124 | tqdm = tqdm_discord 125 | trange = tdrange 126 | -------------------------------------------------------------------------------- /tqdm/contrib/slack.py: -------------------------------------------------------------------------------- 1 | """ 2 | Sends updates to a Slack app. 3 | 4 | Usage: 5 | >>> from tqdm.contrib.slack import tqdm, trange 6 | >>> for i in trange(10, token='{token}', channel='{channel}'): 7 | ... ... 8 | 9 | ![screenshot](https://img.tqdm.ml/screenshot-slack.png) 10 | """ 11 | from __future__ import absolute_import 12 | 13 | import logging 14 | from os import getenv 15 | 16 | try: 17 | from slack_sdk import WebClient 18 | except ImportError: 19 | raise ImportError("Please `pip install slack-sdk`") 20 | 21 | from ..auto import tqdm as tqdm_auto 22 | from ..utils import _range 23 | from .utils_worker import MonoWorker 24 | 25 | __author__ = {"github.com/": ["0x2b3bfa0", "casperdcl"]} 26 | __all__ = ['SlackIO', 'tqdm_slack', 'tsrange', 'tqdm', 'trange'] 27 | 28 | 29 | class SlackIO(MonoWorker): 30 | """Non-blocking file-like IO using a Slack app.""" 31 | def __init__(self, token, channel): 32 | """Creates a new message in the given `channel`.""" 33 | super(SlackIO, self).__init__() 34 | self.client = WebClient(token=token) 35 | self.text = self.__class__.__name__ 36 | try: 37 | self.message = self.client.chat_postMessage(channel=channel, text=self.text) 38 | except Exception as e: 39 | tqdm_auto.write(str(e)) 40 | self.message = None 41 | 42 | def write(self, s): 43 | """Replaces internal `message`'s text with `s`.""" 44 | if not s: 45 | s = "..." 46 | s = s.replace('\r', '').strip() 47 | if s == self.text: 48 | return # skip duplicate message 49 | message = self.message 50 | if message is None: 51 | return 52 | self.text = s 53 | try: 54 | future = self.submit(self.client.chat_update, channel=message['channel'], 55 | ts=message['ts'], text='`' + s + '`') 56 | except Exception as e: 57 | tqdm_auto.write(str(e)) 58 | else: 59 | return future 60 | 61 | 62 | class tqdm_slack(tqdm_auto): 63 | """ 64 | Standard `tqdm.auto.tqdm` but also sends updates to a Slack app. 65 | May take a few seconds to create (`__init__`). 66 | 67 | - create a Slack app with the `chat:write` scope & invite it to a 68 | channel: 69 | - copy the bot `{token}` & `{channel}` and paste below 70 | >>> from tqdm.contrib.slack import tqdm, trange 71 | >>> for i in tqdm(iterable, token='{token}', channel='{channel}'): 72 | ... ... 73 | """ 74 | def __init__(self, *args, **kwargs): 75 | """ 76 | Parameters 77 | ---------- 78 | token : str, required. Slack token 79 | [default: ${TQDM_SLACK_TOKEN}]. 80 | channel : int, required. Slack channel 81 | [default: ${TQDM_SLACK_CHANNEL}]. 82 | mininterval : float, optional. 83 | Minimum of [default: 1.5] to avoid rate limit. 84 | 85 | See `tqdm.auto.tqdm.__init__` for other parameters. 86 | """ 87 | if not kwargs.get('disable'): 88 | kwargs = kwargs.copy() 89 | logging.getLogger("HTTPClient").setLevel(logging.WARNING) 90 | self.sio = SlackIO( 91 | kwargs.pop('token', getenv("TQDM_SLACK_TOKEN")), 92 | kwargs.pop('channel', getenv("TQDM_SLACK_CHANNEL"))) 93 | kwargs['mininterval'] = max(1.5, kwargs.get('mininterval', 1.5)) 94 | super(tqdm_slack, self).__init__(*args, **kwargs) 95 | 96 | def display(self, **kwargs): 97 | super(tqdm_slack, self).display(**kwargs) 98 | fmt = self.format_dict 99 | if fmt.get('bar_format', None): 100 | fmt['bar_format'] = fmt['bar_format'].replace( 101 | '', '`{bar:10}`').replace('{bar}', '`{bar:10u}`') 102 | else: 103 | fmt['bar_format'] = '{l_bar}`{bar:10}`{r_bar}' 104 | if fmt['ascii'] is False: 105 | fmt['ascii'] = [":black_square:", ":small_blue_diamond:", ":large_blue_diamond:", 106 | ":large_blue_square:"] 107 | fmt['ncols'] = 336 108 | self.sio.write(self.format_meter(**fmt)) 109 | 110 | def clear(self, *args, **kwargs): 111 | super(tqdm_slack, self).clear(*args, **kwargs) 112 | if not self.disable: 113 | self.sio.write("") 114 | 115 | 116 | def tsrange(*args, **kwargs): 117 | """ 118 | A shortcut for `tqdm.contrib.slack.tqdm(xrange(*args), **kwargs)`. 119 | On Python3+, `range` is used instead of `xrange`. 120 | """ 121 | return tqdm_slack(_range(*args), **kwargs) 122 | 123 | 124 | # Aliases 125 | tqdm = tqdm_slack 126 | trange = tsrange 127 | -------------------------------------------------------------------------------- /tqdm/keras.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import, division 2 | 3 | from copy import copy 4 | from functools import partial 5 | 6 | from .auto import tqdm as tqdm_auto 7 | 8 | try: 9 | import keras 10 | except (ImportError, AttributeError) as e: 11 | try: 12 | from tensorflow import keras 13 | except ImportError: 14 | raise e 15 | __author__ = {"github.com/": ["casperdcl"]} 16 | __all__ = ['TqdmCallback'] 17 | 18 | 19 | class TqdmCallback(keras.callbacks.Callback): 20 | """Keras callback for epoch and batch progress.""" 21 | @staticmethod 22 | def bar2callback(bar, pop=None, delta=(lambda logs: 1)): 23 | def callback(_, logs=None): 24 | n = delta(logs) 25 | if logs: 26 | if pop: 27 | logs = copy(logs) 28 | [logs.pop(i, 0) for i in pop] 29 | bar.set_postfix(logs, refresh=False) 30 | bar.update(n) 31 | 32 | return callback 33 | 34 | def __init__(self, epochs=None, data_size=None, batch_size=None, verbose=1, 35 | tqdm_class=tqdm_auto, **tqdm_kwargs): 36 | """ 37 | Parameters 38 | ---------- 39 | epochs : int, optional 40 | data_size : int, optional 41 | Number of training pairs. 42 | batch_size : int, optional 43 | Number of training pairs per batch. 44 | verbose : int 45 | 0: epoch, 1: batch (transient), 2: batch. [default: 1]. 46 | Will be set to `0` unless both `data_size` and `batch_size` 47 | are given. 48 | tqdm_class : optional 49 | `tqdm` class to use for bars [default: `tqdm.auto.tqdm`]. 50 | tqdm_kwargs : optional 51 | Any other arguments used for all bars. 52 | """ 53 | if tqdm_kwargs: 54 | tqdm_class = partial(tqdm_class, **tqdm_kwargs) 55 | self.tqdm_class = tqdm_class 56 | self.epoch_bar = tqdm_class(total=epochs, unit='epoch') 57 | self.on_epoch_end = self.bar2callback(self.epoch_bar) 58 | if data_size and batch_size: 59 | self.batches = batches = (data_size + batch_size - 1) // batch_size 60 | else: 61 | self.batches = batches = None 62 | self.verbose = verbose 63 | if verbose == 1: 64 | self.batch_bar = tqdm_class(total=batches, unit='batch', leave=False) 65 | self.on_batch_end = self.bar2callback( 66 | self.batch_bar, pop=['batch', 'size'], 67 | delta=lambda logs: logs.get('size', 1)) 68 | 69 | def on_train_begin(self, *_, **__): 70 | params = self.params.get 71 | auto_total = params('epochs', params('nb_epoch', None)) 72 | if auto_total is not None and auto_total != self.epoch_bar.total: 73 | self.epoch_bar.reset(total=auto_total) 74 | 75 | def on_epoch_begin(self, epoch, *_, **__): 76 | if self.epoch_bar.n < epoch: 77 | ebar = self.epoch_bar 78 | ebar.n = ebar.last_print_n = ebar.initial = epoch 79 | if self.verbose: 80 | params = self.params.get 81 | total = params('samples', params( 82 | 'nb_sample', params('steps', None))) or self.batches 83 | if self.verbose == 2: 84 | if hasattr(self, 'batch_bar'): 85 | self.batch_bar.close() 86 | self.batch_bar = self.tqdm_class( 87 | total=total, unit='batch', leave=True, 88 | unit_scale=1 / (params('batch_size', 1) or 1)) 89 | self.on_batch_end = self.bar2callback( 90 | self.batch_bar, pop=['batch', 'size'], 91 | delta=lambda logs: logs.get('size', 1)) 92 | elif self.verbose == 1: 93 | self.batch_bar.unit_scale = 1 / (params('batch_size', 1) or 1) 94 | self.batch_bar.reset(total=total) 95 | else: 96 | raise KeyError('Unknown verbosity') 97 | 98 | def on_train_end(self, *_, **__): 99 | if self.verbose: 100 | self.batch_bar.close() 101 | self.epoch_bar.close() 102 | 103 | def display(self): 104 | """Displays in the current cell in Notebooks.""" 105 | container = getattr(self.epoch_bar, 'container', None) 106 | if container is None: 107 | return 108 | from .notebook import display 109 | display(container) 110 | batch_bar = getattr(self, 'batch_bar', None) 111 | if batch_bar is not None: 112 | display(batch_bar.container) 113 | 114 | @staticmethod 115 | def _implements_train_batch_hooks(): 116 | return True 117 | 118 | @staticmethod 119 | def _implements_test_batch_hooks(): 120 | return True 121 | 122 | @staticmethod 123 | def _implements_predict_batch_hooks(): 124 | return True 125 | -------------------------------------------------------------------------------- /examples/7zx.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Usage: 3 | 7zx.py [--help | options] ... 4 | 5 | Options: 6 | -h, --help Print this help and exit 7 | -v, --version Print version and exit 8 | -c, --compressed Use compressed (instead of uncompressed) file sizes 9 | -s, --silent Do not print one row per zip file 10 | -y, --yes Assume yes to all queries (for extraction) 11 | -D=, --debug= 12 | Print various types of debugging information. Choices: 13 | CRITICAL|FATAL 14 | ERROR 15 | WARN(ING) 16 | [default: INFO] 17 | DEBUG 18 | NOTSET 19 | -d, --debug-trace Print lots of debugging information (-D NOTSET) 20 | """ 21 | from __future__ import print_function 22 | 23 | import io 24 | import logging 25 | import os 26 | import pty 27 | import re 28 | import subprocess # nosec 29 | 30 | from argopt import argopt 31 | 32 | from tqdm import tqdm 33 | 34 | __author__ = "Casper da Costa-Luis " 35 | __licence__ = "MPLv2.0" 36 | __version__ = "0.2.2" 37 | __license__ = __licence__ 38 | 39 | RE_SCN = re.compile(r"([0-9]+)\s+([0-9]+)\s+(.*)$", flags=re.M) 40 | 41 | 42 | def main(): 43 | args = argopt(__doc__, version=__version__).parse_args() 44 | if args.debug_trace: 45 | args.debug = "NOTSET" 46 | logging.basicConfig(level=getattr(logging, args.debug, logging.INFO), 47 | format='%(levelname)s:%(message)s') 48 | log = logging.getLogger(__name__) 49 | log.debug(args) 50 | 51 | # Get compressed sizes 52 | zips = {} 53 | for fn in args.zipfiles: 54 | info = subprocess.check_output(["7z", "l", fn]).strip() # nosec 55 | finfo = RE_SCN.findall(info) # size|compressed|name 56 | 57 | # builtin test: last line should be total sizes 58 | log.debug(finfo) 59 | totals = map(int, finfo[-1][:2]) 60 | # log.debug(totals) 61 | for s in range(2): # size|compressed totals 62 | totals_s = sum(map(int, (inf[s] for inf in finfo[:-1]))) 63 | if totals_s != totals[s]: 64 | log.warn("%s: individual total %d != 7z total %d", 65 | fn, totals_s, totals[s]) 66 | fcomp = {n: int(c if args.compressed else u) for (u, c, n) in finfo[:-1]} 67 | # log.debug(fcomp) 68 | # zips : {'zipname' : {'filename' : int(size)}} 69 | zips[fn] = fcomp 70 | 71 | # Extract 72 | cmd7zx = ["7z", "x", "-bd"] 73 | if args.yes: 74 | cmd7zx += ["-y"] 75 | log.info("Extracting from %d file(s)", len(zips)) 76 | with tqdm(total=sum(sum(fcomp.values()) for fcomp in zips.values()), 77 | unit="B", unit_scale=True) as tall: 78 | for fn, fcomp in zips.items(): 79 | md, sd = pty.openpty() 80 | ex = subprocess.Popen( # nosec 81 | cmd7zx + [fn], 82 | bufsize=1, 83 | stdout=md, # subprocess.PIPE, 84 | stderr=subprocess.STDOUT) 85 | os.close(sd) 86 | with io.open(md, mode="rU", buffering=1) as m: 87 | with tqdm(total=sum(fcomp.values()), disable=len(zips) < 2, 88 | leave=False, unit="B", unit_scale=True) as t: 89 | if not hasattr(t, "start_t"): # disabled 90 | t.start_t = tall._time() 91 | while True: 92 | try: 93 | l_raw = m.readline() 94 | except IOError: 95 | break 96 | ln = l_raw.strip() 97 | if ln.startswith("Extracting"): 98 | exname = ln[len("Extracting"):].lstrip() 99 | s = fcomp.get(exname, 0) # 0 is likely folders 100 | t.update(s) 101 | tall.update(s) 102 | elif ln: 103 | if not any( 104 | ln.startswith(i) 105 | for i in ("7-Zip ", "p7zip Version ", 106 | "Everything is Ok", "Folders: ", 107 | "Files: ", "Size: ", "Compressed: ")): 108 | if ln.startswith("Processing archive: "): 109 | if not args.silent: 110 | t.write(t.format_interval( 111 | t.start_t - tall.start_t) + ' ' + 112 | ln.replace("Processing archive: ", "")) 113 | else: 114 | t.write(ln) 115 | ex.wait() 116 | 117 | 118 | main.__doc__ = __doc__ 119 | 120 | if __name__ == "__main__": 121 | main() 122 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name=tqdm 3 | url=https://tqdm.github.io 4 | project_urls= 5 | Changelog=https://tqdm.github.io/releases 6 | Source=https://github.com/tqdm/tqdm 7 | Wiki=https://github.com/tqdm/tqdm/wiki 8 | maintainer=tqdm developers 9 | maintainer_email=python.tqdm@gmail.com 10 | license=MPLv2.0, MIT Licences 11 | license_file=LICENCE 12 | description=Fast, Extensible Progress Meter 13 | long_description=file: README.rst 14 | long_description_content_type=text/x-rst 15 | keywords=progressbar, progressmeter, progress, bar, meter, rate, eta, console, terminal, time 16 | platforms=any 17 | provides=tqdm 18 | # Trove classifiers (https://pypi.org/pypi?%3Aaction=list_classifiers) 19 | classifiers= 20 | Development Status :: 5 - Production/Stable 21 | Environment :: Console 22 | Environment :: MacOS X 23 | Environment :: Other Environment 24 | Environment :: Win32 (MS Windows) 25 | Environment :: X11 Applications 26 | Framework :: IPython 27 | Framework :: Jupyter 28 | Intended Audience :: Developers 29 | Intended Audience :: Education 30 | Intended Audience :: End Users/Desktop 31 | Intended Audience :: Other Audience 32 | Intended Audience :: System Administrators 33 | License :: OSI Approved :: MIT License 34 | License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0) 35 | Operating System :: MacOS 36 | Operating System :: MacOS :: MacOS X 37 | Operating System :: Microsoft 38 | Operating System :: Microsoft :: MS-DOS 39 | Operating System :: Microsoft :: Windows 40 | Operating System :: POSIX 41 | Operating System :: POSIX :: BSD 42 | Operating System :: POSIX :: BSD :: FreeBSD 43 | Operating System :: POSIX :: Linux 44 | Operating System :: POSIX :: SunOS/Solaris 45 | Operating System :: Unix 46 | Programming Language :: Python 47 | Programming Language :: Python :: 2 48 | Programming Language :: Python :: 2.7 49 | Programming Language :: Python :: 3 50 | Programming Language :: Python :: 3.5 51 | Programming Language :: Python :: 3.6 52 | Programming Language :: Python :: 3.7 53 | Programming Language :: Python :: 3.8 54 | Programming Language :: Python :: 3.9 55 | Programming Language :: Python :: 3.10 56 | Programming Language :: Python :: Implementation 57 | Programming Language :: Python :: Implementation :: IronPython 58 | Programming Language :: Python :: Implementation :: PyPy 59 | Programming Language :: Unix Shell 60 | Topic :: Desktop Environment 61 | Topic :: Education :: Computer Aided Instruction (CAI) 62 | Topic :: Education :: Testing 63 | Topic :: Office/Business 64 | Topic :: Other/Nonlisted Topic 65 | Topic :: Software Development :: Build Tools 66 | Topic :: Software Development :: Libraries 67 | Topic :: Software Development :: Libraries :: Python Modules 68 | Topic :: Software Development :: Pre-processors 69 | Topic :: Software Development :: User Interfaces 70 | Topic :: System :: Installation/Setup 71 | Topic :: System :: Logging 72 | Topic :: System :: Monitoring 73 | Topic :: System :: Shells 74 | Topic :: Terminals 75 | Topic :: Utilities 76 | [options] 77 | setup_requires=setuptools>=42; setuptools_scm[toml]>=3.4 78 | python_requires=>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* 79 | install_requires= 80 | colorama; platform_system == 'Windows' 81 | importlib_resources; python_version < "3.7" 82 | tests_require=tox 83 | include_package_data=True 84 | packages=find: 85 | [options.extras_require] 86 | dev=py-make>=0.1.0; twine; wheel 87 | slack=slack-sdk 88 | telegram=requests 89 | notebook=ipywidgets>=6 90 | [options.entry_points] 91 | console_scripts= 92 | tqdm=tqdm.cli:main 93 | [options.packages.find] 94 | exclude=benchmarks, tests 95 | 96 | [bdist_wheel] 97 | universal=1 98 | 99 | [flake8] 100 | max_line_length=99 101 | exclude=.asv,.eggs,.tox,.ipynb_checkpoints,build,dist,.git,__pycache__ 102 | 103 | [pydocstyle] 104 | add_ignore=D400,D415 105 | 106 | [yapf] 107 | coalesce_brackets=True 108 | column_limit=99 109 | each_dict_entry_on_separate_line=False 110 | i18n_comment=NOQA 111 | space_between_ending_comma_and_closing_bracket=False 112 | split_before_named_assigns=False 113 | split_before_closing_bracket=False 114 | 115 | [isort] 116 | line_length=99 117 | multi_line_output=4 118 | known_first_party=tqdm,tests 119 | 120 | [tool:pytest] 121 | timeout=30 122 | log_level=INFO 123 | markers= 124 | asyncio 125 | slow 126 | python_files=tests_*.py tests_*.ipynb 127 | testpaths=tests 128 | addopts=-v --tb=short -rxs -W=error --durations=0 --durations-min=0.1 --asyncio-mode=strict 129 | [regex1] 130 | regex: (?<= )[\s\d.]+(it/s|s/it) 131 | replace: ??.??it/s 132 | [regex2] 133 | regex: 00:0[01]<00:0[01] 134 | replace: 00:00<00:00 135 | 136 | [coverage:run] 137 | branch=True 138 | include=tqdm/* 139 | omit= 140 | tqdm/contrib/bells.py 141 | tqdm/contrib/slack.py 142 | tqdm/contrib/discord.py 143 | tqdm/contrib/telegram.py 144 | tqdm/contrib/utils_worker.py 145 | relative_files=True 146 | disable_warnings=include-ignored 147 | [coverage:report] 148 | show_missing=True 149 | -------------------------------------------------------------------------------- /tqdm/contrib/concurrent.py: -------------------------------------------------------------------------------- 1 | """ 2 | Thin wrappers around `concurrent.futures`. 3 | """ 4 | from __future__ import absolute_import 5 | 6 | from contextlib import contextmanager 7 | 8 | from ..auto import tqdm as tqdm_auto 9 | from ..std import TqdmWarning 10 | 11 | try: 12 | from operator import length_hint 13 | except ImportError: 14 | def length_hint(it, default=0): 15 | """Returns `len(it)`, falling back to `default`""" 16 | try: 17 | return len(it) 18 | except TypeError: 19 | return default 20 | try: 21 | from os import cpu_count 22 | except ImportError: 23 | try: 24 | from multiprocessing import cpu_count 25 | except ImportError: 26 | def cpu_count(): 27 | return 4 28 | import sys 29 | 30 | __author__ = {"github.com/": ["casperdcl"]} 31 | __all__ = ['thread_map', 'process_map'] 32 | 33 | 34 | @contextmanager 35 | def ensure_lock(tqdm_class, lock_name=""): 36 | """get (create if necessary) and then restore `tqdm_class`'s lock""" 37 | old_lock = getattr(tqdm_class, '_lock', None) # don't create a new lock 38 | lock = old_lock or tqdm_class.get_lock() # maybe create a new lock 39 | lock = getattr(lock, lock_name, lock) # maybe subtype 40 | tqdm_class.set_lock(lock) 41 | yield lock 42 | if old_lock is None: 43 | del tqdm_class._lock 44 | else: 45 | tqdm_class.set_lock(old_lock) 46 | 47 | 48 | def _executor_map(PoolExecutor, fn, *iterables, **tqdm_kwargs): 49 | """ 50 | Implementation of `thread_map` and `process_map`. 51 | 52 | Parameters 53 | ---------- 54 | tqdm_class : [default: tqdm.auto.tqdm]. 55 | max_workers : [default: min(32, cpu_count() + 4)]. 56 | chunksize : [default: 1]. 57 | lock_name : [default: "":str]. 58 | """ 59 | kwargs = tqdm_kwargs.copy() 60 | if "total" not in kwargs: 61 | kwargs["total"] = length_hint(iterables[0]) 62 | tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) 63 | max_workers = kwargs.pop("max_workers", min(32, cpu_count() + 4)) 64 | chunksize = kwargs.pop("chunksize", 1) 65 | lock_name = kwargs.pop("lock_name", "") 66 | with ensure_lock(tqdm_class, lock_name=lock_name) as lk: 67 | pool_kwargs = {'max_workers': max_workers} 68 | sys_version = sys.version_info[:2] 69 | if sys_version >= (3, 7): 70 | # share lock in case workers are already using `tqdm` 71 | pool_kwargs.update(initializer=tqdm_class.set_lock, initargs=(lk,)) 72 | map_args = {} 73 | if not (3, 0) < sys_version < (3, 5): 74 | map_args.update(chunksize=chunksize) 75 | with PoolExecutor(**pool_kwargs) as ex: 76 | return list(tqdm_class(ex.map(fn, *iterables, **map_args), **kwargs)) 77 | 78 | 79 | def thread_map(fn, *iterables, **tqdm_kwargs): 80 | """ 81 | Equivalent of `list(map(fn, *iterables))` 82 | driven by `concurrent.futures.ThreadPoolExecutor`. 83 | 84 | Parameters 85 | ---------- 86 | tqdm_class : optional 87 | `tqdm` class to use for bars [default: tqdm.auto.tqdm]. 88 | max_workers : int, optional 89 | Maximum number of workers to spawn; passed to 90 | `concurrent.futures.ThreadPoolExecutor.__init__`. 91 | [default: max(32, cpu_count() + 4)]. 92 | """ 93 | from concurrent.futures import ThreadPoolExecutor 94 | return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs) 95 | 96 | 97 | def process_map(fn, *iterables, **tqdm_kwargs): 98 | """ 99 | Equivalent of `list(map(fn, *iterables))` 100 | driven by `concurrent.futures.ProcessPoolExecutor`. 101 | 102 | Parameters 103 | ---------- 104 | tqdm_class : optional 105 | `tqdm` class to use for bars [default: tqdm.auto.tqdm]. 106 | max_workers : int, optional 107 | Maximum number of workers to spawn; passed to 108 | `concurrent.futures.ProcessPoolExecutor.__init__`. 109 | [default: min(32, cpu_count() + 4)]. 110 | chunksize : int, optional 111 | Size of chunks sent to worker processes; passed to 112 | `concurrent.futures.ProcessPoolExecutor.map`. [default: 1]. 113 | lock_name : str, optional 114 | Member of `tqdm_class.get_lock()` to use [default: mp_lock]. 115 | """ 116 | from concurrent.futures import ProcessPoolExecutor 117 | if iterables and "chunksize" not in tqdm_kwargs: 118 | # default `chunksize=1` has poor performance for large iterables 119 | # (most time spent dispatching items to workers). 120 | longest_iterable_len = max(map(length_hint, iterables)) 121 | if longest_iterable_len > 1000: 122 | from warnings import warn 123 | warn("Iterable length %d > 1000 but `chunksize` is not set." 124 | " This may seriously degrade multiprocess performance." 125 | " Set `chunksize=1` or more." % longest_iterable_len, 126 | TqdmWarning, stacklevel=2) 127 | if "lock_name" not in tqdm_kwargs: 128 | tqdm_kwargs = tqdm_kwargs.copy() 129 | tqdm_kwargs["lock_name"] = "mp_lock" 130 | return _executor_map(ProcessPoolExecutor, fn, *iterables, **tqdm_kwargs) 131 | -------------------------------------------------------------------------------- /tqdm/rich.py: -------------------------------------------------------------------------------- 1 | """ 2 | `rich.progress` decorator for iterators. 3 | 4 | Usage: 5 | >>> from tqdm.rich import trange, tqdm 6 | >>> for i in trange(10): 7 | ... ... 8 | """ 9 | from __future__ import absolute_import 10 | 11 | from warnings import warn 12 | 13 | from rich.progress import ( 14 | BarColumn, Progress, ProgressColumn, Text, TimeElapsedColumn, TimeRemainingColumn, filesize) 15 | 16 | from .std import TqdmExperimentalWarning 17 | from .std import tqdm as std_tqdm 18 | from .utils import _range 19 | 20 | __author__ = {"github.com/": ["casperdcl"]} 21 | __all__ = ['tqdm_rich', 'trrange', 'tqdm', 'trange'] 22 | 23 | 24 | class FractionColumn(ProgressColumn): 25 | """Renders completed/total, e.g. '0.5/2.3 G'.""" 26 | def __init__(self, unit_scale=False, unit_divisor=1000): 27 | self.unit_scale = unit_scale 28 | self.unit_divisor = unit_divisor 29 | super().__init__() 30 | 31 | def render(self, task): 32 | """Calculate common unit for completed and total.""" 33 | completed = int(task.completed) 34 | total = int(task.total) 35 | if self.unit_scale: 36 | unit, suffix = filesize.pick_unit_and_suffix( 37 | total, 38 | ["", "K", "M", "G", "T", "P", "E", "Z", "Y"], 39 | self.unit_divisor, 40 | ) 41 | else: 42 | unit, suffix = filesize.pick_unit_and_suffix(total, [""], 1) 43 | precision = 0 if unit == 1 else 1 44 | return Text( 45 | f"{completed/unit:,.{precision}f}/{total/unit:,.{precision}f} {suffix}", 46 | style="progress.download") 47 | 48 | 49 | class RateColumn(ProgressColumn): 50 | """Renders human readable transfer speed.""" 51 | def __init__(self, unit="", unit_scale=False, unit_divisor=1000): 52 | self.unit = unit 53 | self.unit_scale = unit_scale 54 | self.unit_divisor = unit_divisor 55 | super().__init__() 56 | 57 | def render(self, task): 58 | """Show data transfer speed.""" 59 | speed = task.speed 60 | if speed is None: 61 | return Text(f"? {self.unit}/s", style="progress.data.speed") 62 | if self.unit_scale: 63 | unit, suffix = filesize.pick_unit_and_suffix( 64 | speed, 65 | ["", "K", "M", "G", "T", "P", "E", "Z", "Y"], 66 | self.unit_divisor, 67 | ) 68 | else: 69 | unit, suffix = filesize.pick_unit_and_suffix(speed, [""], 1) 70 | precision = 0 if unit == 1 else 1 71 | return Text(f"{speed/unit:,.{precision}f} {suffix}{self.unit}/s", 72 | style="progress.data.speed") 73 | 74 | 75 | class tqdm_rich(std_tqdm): # pragma: no cover 76 | """Experimental rich.progress GUI version of tqdm!""" 77 | # TODO: @classmethod: write()? 78 | def __init__(self, *args, **kwargs): 79 | """ 80 | This class accepts the following parameters *in addition* to 81 | the parameters accepted by `tqdm`. 82 | 83 | Parameters 84 | ---------- 85 | progress : tuple, optional 86 | arguments for `rich.progress.Progress()`. 87 | options : dict, optional 88 | keyword arguments for `rich.progress.Progress()`. 89 | """ 90 | kwargs = kwargs.copy() 91 | kwargs['gui'] = True 92 | # convert disable = None to False 93 | kwargs['disable'] = bool(kwargs.get('disable', False)) 94 | progress = kwargs.pop('progress', None) 95 | options = kwargs.pop('options', {}).copy() 96 | super(tqdm_rich, self).__init__(*args, **kwargs) 97 | 98 | if self.disable: 99 | return 100 | 101 | warn("rich is experimental/alpha", TqdmExperimentalWarning, stacklevel=2) 102 | d = self.format_dict 103 | if progress is None: 104 | progress = ( 105 | "[progress.description]{task.description}" 106 | "[progress.percentage]{task.percentage:>4.0f}%", 107 | BarColumn(bar_width=None), 108 | FractionColumn( 109 | unit_scale=d['unit_scale'], unit_divisor=d['unit_divisor']), 110 | "[", TimeElapsedColumn(), "<", TimeRemainingColumn(), 111 | ",", RateColumn(unit=d['unit'], unit_scale=d['unit_scale'], 112 | unit_divisor=d['unit_divisor']), "]" 113 | ) 114 | options.setdefault('transient', not self.leave) 115 | self._prog = Progress(*progress, **options) 116 | self._prog.__enter__() 117 | self._task_id = self._prog.add_task(self.desc or "", **d) 118 | 119 | def close(self): 120 | if self.disable: 121 | return 122 | super(tqdm_rich, self).close() 123 | self._prog.__exit__(None, None, None) 124 | 125 | def clear(self, *_, **__): 126 | pass 127 | 128 | def display(self, *_, **__): 129 | if not hasattr(self, '_prog'): 130 | return 131 | self._prog.update(self._task_id, completed=self.n, description=self.desc) 132 | 133 | def reset(self, total=None): 134 | """ 135 | Resets to 0 iterations for repeated use. 136 | 137 | Parameters 138 | ---------- 139 | total : int or float, optional. Total to use for the new bar. 140 | """ 141 | if hasattr(self, '_prog'): 142 | self._prog.reset(total=total) 143 | super(tqdm_rich, self).reset(total=total) 144 | 145 | 146 | def trrange(*args, **kwargs): 147 | """ 148 | A shortcut for `tqdm.rich.tqdm(xrange(*args), **kwargs)`. 149 | On Python3+, `range` is used instead of `xrange`. 150 | """ 151 | return tqdm_rich(_range(*args), **kwargs) 152 | 153 | 154 | # Aliases 155 | tqdm = tqdm_rich 156 | trange = trrange 157 | -------------------------------------------------------------------------------- /tqdm/contrib/telegram.py: -------------------------------------------------------------------------------- 1 | """ 2 | Sends updates to a Telegram bot. 3 | 4 | Usage: 5 | >>> from tqdm.contrib.telegram import tqdm, trange 6 | >>> for i in trange(10, token='{token}', chat_id='{chat_id}'): 7 | ... ... 8 | 9 | ![screenshot](https://img.tqdm.ml/screenshot-telegram.gif) 10 | """ 11 | from __future__ import absolute_import 12 | 13 | from os import getenv 14 | from warnings import warn 15 | 16 | from requests import Session 17 | 18 | from ..auto import tqdm as tqdm_auto 19 | from ..std import TqdmWarning 20 | from ..utils import _range 21 | from .utils_worker import MonoWorker 22 | 23 | __author__ = {"github.com/": ["casperdcl"]} 24 | __all__ = ['TelegramIO', 'tqdm_telegram', 'ttgrange', 'tqdm', 'trange'] 25 | 26 | 27 | class TelegramIO(MonoWorker): 28 | """Non-blocking file-like IO using a Telegram Bot.""" 29 | API = 'https://api.telegram.org/bot' 30 | 31 | def __init__(self, token, chat_id): 32 | """Creates a new message in the given `chat_id`.""" 33 | super(TelegramIO, self).__init__() 34 | self.token = token 35 | self.chat_id = chat_id 36 | self.session = Session() 37 | self.text = self.__class__.__name__ 38 | self.message_id 39 | 40 | @property 41 | def message_id(self): 42 | if hasattr(self, '_message_id'): 43 | return self._message_id 44 | try: 45 | res = self.session.post( 46 | self.API + '%s/sendMessage' % self.token, 47 | data={'text': '`' + self.text + '`', 'chat_id': self.chat_id, 48 | 'parse_mode': 'MarkdownV2'}).json() 49 | except Exception as e: 50 | tqdm_auto.write(str(e)) 51 | else: 52 | if res.get('error_code') == 429: 53 | warn("Creation rate limit: try increasing `mininterval`.", 54 | TqdmWarning, stacklevel=2) 55 | else: 56 | self._message_id = res['result']['message_id'] 57 | return self._message_id 58 | 59 | def write(self, s): 60 | """Replaces internal `message_id`'s text with `s`.""" 61 | if not s: 62 | s = "..." 63 | s = s.replace('\r', '').strip() 64 | if s == self.text: 65 | return # avoid duplicate message Bot error 66 | message_id = self.message_id 67 | if message_id is None: 68 | return 69 | self.text = s 70 | try: 71 | future = self.submit( 72 | self.session.post, self.API + '%s/editMessageText' % self.token, 73 | data={'text': '`' + s + '`', 'chat_id': self.chat_id, 74 | 'message_id': message_id, 'parse_mode': 'MarkdownV2'}) 75 | except Exception as e: 76 | tqdm_auto.write(str(e)) 77 | else: 78 | return future 79 | 80 | def delete(self): 81 | """Deletes internal `message_id`.""" 82 | try: 83 | future = self.submit( 84 | self.session.post, self.API + '%s/deleteMessage' % self.token, 85 | data={'chat_id': self.chat_id, 'message_id': self.message_id}) 86 | except Exception as e: 87 | tqdm_auto.write(str(e)) 88 | else: 89 | return future 90 | 91 | 92 | class tqdm_telegram(tqdm_auto): 93 | """ 94 | Standard `tqdm.auto.tqdm` but also sends updates to a Telegram Bot. 95 | May take a few seconds to create (`__init__`). 96 | 97 | - create a bot 98 | - copy its `{token}` 99 | - add the bot to a chat and send it a message such as `/start` 100 | - go to to find out 101 | the `{chat_id}` 102 | - paste the `{token}` & `{chat_id}` below 103 | 104 | >>> from tqdm.contrib.telegram import tqdm, trange 105 | >>> for i in tqdm(iterable, token='{token}', chat_id='{chat_id}'): 106 | ... ... 107 | """ 108 | def __init__(self, *args, **kwargs): 109 | """ 110 | Parameters 111 | ---------- 112 | token : str, required. Telegram token 113 | [default: ${TQDM_TELEGRAM_TOKEN}]. 114 | chat_id : str, required. Telegram chat ID 115 | [default: ${TQDM_TELEGRAM_CHAT_ID}]. 116 | 117 | See `tqdm.auto.tqdm.__init__` for other parameters. 118 | """ 119 | if not kwargs.get('disable'): 120 | kwargs = kwargs.copy() 121 | self.tgio = TelegramIO( 122 | kwargs.pop('token', getenv('TQDM_TELEGRAM_TOKEN')), 123 | kwargs.pop('chat_id', getenv('TQDM_TELEGRAM_CHAT_ID'))) 124 | super(tqdm_telegram, self).__init__(*args, **kwargs) 125 | 126 | def display(self, **kwargs): 127 | super(tqdm_telegram, self).display(**kwargs) 128 | fmt = self.format_dict 129 | if fmt.get('bar_format', None): 130 | fmt['bar_format'] = fmt['bar_format'].replace( 131 | '', '{bar:10u}').replace('{bar}', '{bar:10u}') 132 | else: 133 | fmt['bar_format'] = '{l_bar}{bar:10u}{r_bar}' 134 | self.tgio.write(self.format_meter(**fmt)) 135 | 136 | def clear(self, *args, **kwargs): 137 | super(tqdm_telegram, self).clear(*args, **kwargs) 138 | if not self.disable: 139 | self.tgio.write("") 140 | 141 | def close(self): 142 | if self.disable: 143 | return 144 | super(tqdm_telegram, self).close() 145 | if not (self.leave or (self.leave is None and self.pos == 0)): 146 | self.tgio.delete() 147 | 148 | 149 | def ttgrange(*args, **kwargs): 150 | """ 151 | A shortcut for `tqdm.contrib.telegram.tqdm(xrange(*args), **kwargs)`. 152 | On Python3+, `range` is used instead of `xrange`. 153 | """ 154 | return tqdm_telegram(_range(*args), **kwargs) 155 | 156 | 157 | # Aliases 158 | tqdm = tqdm_telegram 159 | trange = ttgrange 160 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # IMPORTANT: for compatibility with `python setup.py make [alias]`, ensure: 2 | # 1. Every alias is preceded by @[+]make (eg: @make alias) 3 | # 2. A maximum of one @make alias or command per line 4 | # see: https://github.com/tqdm/py-make/issues/1 5 | 6 | .PHONY: 7 | alltests 8 | all 9 | flake8 10 | test 11 | pytest 12 | testsetup 13 | testnb 14 | testcoverage 15 | testperf 16 | testtimer 17 | distclean 18 | coverclean 19 | prebuildclean 20 | clean 21 | toxclean 22 | install_dev 23 | install 24 | build 25 | buildupload 26 | pypi 27 | snap 28 | docker 29 | help 30 | none 31 | run 32 | 33 | help: 34 | @python setup.py make -p 35 | 36 | alltests: 37 | @+make testcoverage 38 | @+make testperf 39 | @+make flake8 40 | @+make testsetup 41 | 42 | all: 43 | @+make alltests 44 | @+make build 45 | 46 | flake8: 47 | @+pre-commit run -a flake8 48 | @+pre-commit run -a nbstripout 49 | 50 | test: 51 | TOX_SKIP_ENV=perf tox --skip-missing-interpreters -p all 52 | tox -e perf 53 | 54 | pytest: 55 | pytest 56 | 57 | testsetup: 58 | @make README.rst 59 | @make tqdm/tqdm.1 60 | @make tqdm/completion.sh 61 | python setup.py check --metadata --restructuredtext --strict 62 | python setup.py make none 63 | 64 | testnb: 65 | pytest tests_notebook.ipynb --nbval --nbval-current-env -W=ignore --nbval-sanitize-with=setup.cfg --cov=tqdm.notebook --cov-report=term 66 | 67 | testcoverage: 68 | @make coverclean 69 | pytest tests_notebook.ipynb --cov=tqdm --cov-report= --nbval --nbval-current-env --nbval-sanitize-with=setup.cfg -W=ignore 70 | pytest -k "not perf" --cov=tqdm --cov-report=xml --cov-report=term --cov-append --cov-fail-under=80 71 | 72 | testperf: 73 | # do not use coverage (which is extremely slow) 74 | pytest -k perf 75 | 76 | testtimer: 77 | pytest 78 | 79 | # another performance test, to check evolution across commits 80 | testasv: 81 | # Test only the last 3 commits (quick test) 82 | asv run -j 8 HEAD~3..HEAD 83 | @make viewasv 84 | 85 | testasvfull: 86 | # Test all the commits since the beginning (full test) 87 | asv run --skip-existing-commits -j 8 v1.0.0..HEAD 88 | @make testasv 89 | 90 | viewasv: 91 | asv publish 92 | asv preview 93 | 94 | tqdm/tqdm.1: .meta/.tqdm.1.md tqdm/cli.py tqdm/std.py 95 | # TODO: add to mkdocs.py 96 | python -m tqdm --help | tail -n+5 |\ 97 | sed -r -e 's/\\/\\\\/g' \ 98 | -e 's/^ (--.*)=<(.*)> : (.*)$$/\n\\\1=*\2*\n: \3./' \ 99 | -e 's/^ (--.*) : (.*)$$/\n\\\1\n: \2./' \ 100 | -e 's/ (-.*, )(--.*) /\n\1\\\2\n: /' |\ 101 | cat "$<" - |\ 102 | pandoc -o "$@" -s -t man 103 | 104 | tqdm/completion.sh: .meta/mkcompletion.py tqdm/std.py tqdm/cli.py 105 | @python .meta/mkcompletion.py 106 | 107 | README.rst: .meta/.readme.rst tqdm/std.py tqdm/cli.py 108 | @python .meta/mkdocs.py 109 | 110 | snapcraft.yaml: .meta/mksnap.py 111 | @python .meta/mksnap.py 112 | 113 | .dockerignore: 114 | @+python -c "fd=open('.dockerignore', 'w'); fd.write('*\n!dist/*.whl\n')" 115 | 116 | Dockerfile: 117 | @+python -c 'fd=open("Dockerfile", "w"); fd.write("FROM python:3.8-alpine\nCOPY dist/*.whl .\nRUN pip install -U $$(ls ./*.whl) && rm ./*.whl\nENTRYPOINT [\"tqdm\"]\n")' 118 | 119 | distclean: 120 | @+make coverclean 121 | @+make prebuildclean 122 | @+make clean 123 | prebuildclean: 124 | @+python -c "import shutil; shutil.rmtree('build', True)" 125 | @+python -c "import shutil; shutil.rmtree('dist', True)" 126 | @+python -c "import shutil; shutil.rmtree('tqdm.egg-info', True)" 127 | @+python -c "import shutil; shutil.rmtree('.eggs', True)" 128 | @+python -c "import os; os.remove('tqdm/_dist_ver.py') if os.path.exists('tqdm/_dist_ver.py') else None" 129 | coverclean: 130 | @+python -c "import os; os.remove('.coverage') if os.path.exists('.coverage') else None" 131 | @+python -c "import os, glob; [os.remove(i) for i in glob.glob('.coverage.*')]" 132 | @+python -c "import shutil; shutil.rmtree('tests/__pycache__', True)" 133 | @+python -c "import shutil; shutil.rmtree('benchmarks/__pycache__', True)" 134 | @+python -c "import shutil; shutil.rmtree('tqdm/__pycache__', True)" 135 | @+python -c "import shutil; shutil.rmtree('tqdm/contrib/__pycache__', True)" 136 | @+python -c "import shutil; shutil.rmtree('tqdm/examples/__pycache__', True)" 137 | clean: 138 | @+python -c "import os, glob; [os.remove(i) for i in glob.glob('*.py[co]')]" 139 | @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tests/*.py[co]')]" 140 | @+python -c "import os, glob; [os.remove(i) for i in glob.glob('benchmarks/*.py[co]')]" 141 | @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/*.py[co]')]" 142 | @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/contrib/*.py[co]')]" 143 | @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/examples/*.py[co]')]" 144 | toxclean: 145 | @+python -c "import shutil; shutil.rmtree('.tox', True)" 146 | 147 | 148 | submodules: 149 | git clone git@github.com:tqdm/tqdm.wiki wiki 150 | git clone git@github.com:tqdm/tqdm.github.io docs 151 | git clone git@github.com:conda-forge/tqdm-feedstock feedstock 152 | cd feedstock && git remote add autotick-bot git@github.com:regro-cf-autotick-bot/tqdm-feedstock 153 | 154 | install: 155 | python setup.py install 156 | install_dev: 157 | python setup.py develop --uninstall 158 | python setup.py develop 159 | install_build: 160 | python -m pip install -r .meta/requirements-dev.txt 161 | install_test: 162 | python -m pip install -r .meta/requirements-test.txt 163 | pre-commit install 164 | 165 | build: 166 | @make prebuildclean 167 | @make testsetup 168 | python setup.py sdist bdist_wheel 169 | # python setup.py bdist_wininst 170 | 171 | pypi: 172 | twine upload dist/* 173 | 174 | buildupload: 175 | @make build 176 | @make pypi 177 | 178 | snap: 179 | @make -B snapcraft.yaml 180 | snapcraft 181 | docker: 182 | @make build 183 | @make .dockerignore 184 | @make Dockerfile 185 | docker build . -t tqdm/tqdm 186 | docker tag tqdm/tqdm:latest tqdm/tqdm:$(shell docker run -i --rm tqdm/tqdm -v) 187 | none: 188 | # used for unit testing 189 | 190 | run: 191 | python -Om tqdm --help 192 | -------------------------------------------------------------------------------- /tqdm/gui.py: -------------------------------------------------------------------------------- 1 | """ 2 | Matplotlib GUI progressbar decorator for iterators. 3 | 4 | Usage: 5 | >>> from tqdm.gui import trange, tqdm 6 | >>> for i in trange(10): 7 | ... ... 8 | """ 9 | # future division is important to divide integers and get as 10 | # a result precise floating numbers (instead of truncated int) 11 | from __future__ import absolute_import, division 12 | 13 | import re 14 | from warnings import warn 15 | 16 | # to inherit from the tqdm class 17 | from .std import TqdmExperimentalWarning 18 | from .std import tqdm as std_tqdm 19 | # import compatibility functions and utilities 20 | from .utils import _range 21 | 22 | __author__ = {"github.com/": ["casperdcl", "lrq3000"]} 23 | __all__ = ['tqdm_gui', 'tgrange', 'tqdm', 'trange'] 24 | 25 | 26 | class tqdm_gui(std_tqdm): # pragma: no cover 27 | """Experimental Matplotlib GUI version of tqdm!""" 28 | # TODO: @classmethod: write() on GUI? 29 | def __init__(self, *args, **kwargs): 30 | from collections import deque 31 | 32 | import matplotlib as mpl 33 | import matplotlib.pyplot as plt 34 | kwargs = kwargs.copy() 35 | kwargs['gui'] = True 36 | colour = kwargs.pop('colour', 'g') 37 | super(tqdm_gui, self).__init__(*args, **kwargs) 38 | 39 | if self.disable: 40 | return 41 | 42 | warn("GUI is experimental/alpha", TqdmExperimentalWarning, stacklevel=2) 43 | self.mpl = mpl 44 | self.plt = plt 45 | 46 | # Remember if external environment uses toolbars 47 | self.toolbar = self.mpl.rcParams['toolbar'] 48 | self.mpl.rcParams['toolbar'] = 'None' 49 | 50 | self.mininterval = max(self.mininterval, 0.5) 51 | self.fig, ax = plt.subplots(figsize=(9, 2.2)) 52 | # self.fig.subplots_adjust(bottom=0.2) 53 | total = self.__len__() # avoids TypeError on None #971 54 | if total is not None: 55 | self.xdata = [] 56 | self.ydata = [] 57 | self.zdata = [] 58 | else: 59 | self.xdata = deque([]) 60 | self.ydata = deque([]) 61 | self.zdata = deque([]) 62 | self.line1, = ax.plot(self.xdata, self.ydata, color='b') 63 | self.line2, = ax.plot(self.xdata, self.zdata, color='k') 64 | ax.set_ylim(0, 0.001) 65 | if total is not None: 66 | ax.set_xlim(0, 100) 67 | ax.set_xlabel("percent") 68 | self.fig.legend((self.line1, self.line2), ("cur", "est"), 69 | loc='center right') 70 | # progressbar 71 | self.hspan = plt.axhspan(0, 0.001, xmin=0, xmax=0, color=colour) 72 | else: 73 | # ax.set_xlim(-60, 0) 74 | ax.set_xlim(0, 60) 75 | ax.invert_xaxis() 76 | ax.set_xlabel("seconds") 77 | ax.legend(("cur", "est"), loc='lower left') 78 | ax.grid() 79 | # ax.set_xlabel('seconds') 80 | ax.set_ylabel((self.unit if self.unit else "it") + "/s") 81 | if self.unit_scale: 82 | plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0)) 83 | ax.yaxis.get_offset_text().set_x(-0.15) 84 | 85 | # Remember if external environment is interactive 86 | self.wasion = plt.isinteractive() 87 | plt.ion() 88 | self.ax = ax 89 | 90 | def close(self): 91 | if self.disable: 92 | return 93 | 94 | self.disable = True 95 | 96 | with self.get_lock(): 97 | self._instances.remove(self) 98 | 99 | # Restore toolbars 100 | self.mpl.rcParams['toolbar'] = self.toolbar 101 | # Return to non-interactive mode 102 | if not self.wasion: 103 | self.plt.ioff() 104 | if self.leave: 105 | self.display() 106 | else: 107 | self.plt.close(self.fig) 108 | 109 | def clear(self, *_, **__): 110 | pass 111 | 112 | def display(self, *_, **__): 113 | n = self.n 114 | cur_t = self._time() 115 | elapsed = cur_t - self.start_t 116 | delta_it = n - self.last_print_n 117 | delta_t = cur_t - self.last_print_t 118 | 119 | # Inline due to multiple calls 120 | total = self.total 121 | xdata = self.xdata 122 | ydata = self.ydata 123 | zdata = self.zdata 124 | ax = self.ax 125 | line1 = self.line1 126 | line2 = self.line2 127 | # instantaneous rate 128 | y = delta_it / delta_t 129 | # overall rate 130 | z = n / elapsed 131 | # update line data 132 | xdata.append(n * 100.0 / total if total else cur_t) 133 | ydata.append(y) 134 | zdata.append(z) 135 | 136 | # Discard old values 137 | # xmin, xmax = ax.get_xlim() 138 | # if (not total) and elapsed > xmin * 1.1: 139 | if (not total) and elapsed > 66: 140 | xdata.popleft() 141 | ydata.popleft() 142 | zdata.popleft() 143 | 144 | ymin, ymax = ax.get_ylim() 145 | if y > ymax or z > ymax: 146 | ymax = 1.1 * y 147 | ax.set_ylim(ymin, ymax) 148 | ax.figure.canvas.draw() 149 | 150 | if total: 151 | line1.set_data(xdata, ydata) 152 | line2.set_data(xdata, zdata) 153 | try: 154 | poly_lims = self.hspan.get_xy() 155 | except AttributeError: 156 | self.hspan = self.plt.axhspan(0, 0.001, xmin=0, xmax=0, color='g') 157 | poly_lims = self.hspan.get_xy() 158 | poly_lims[0, 1] = ymin 159 | poly_lims[1, 1] = ymax 160 | poly_lims[2] = [n / total, ymax] 161 | poly_lims[3] = [poly_lims[2, 0], ymin] 162 | if len(poly_lims) > 4: 163 | poly_lims[4, 1] = ymin 164 | self.hspan.set_xy(poly_lims) 165 | else: 166 | t_ago = [cur_t - i for i in xdata] 167 | line1.set_data(t_ago, ydata) 168 | line2.set_data(t_ago, zdata) 169 | 170 | d = self.format_dict 171 | # remove {bar} 172 | d['bar_format'] = (d['bar_format'] or "{l_bar}{r_bar}").replace( 173 | "{bar}", "") 174 | msg = self.format_meter(**d) 175 | if '' in msg: 176 | msg = "".join(re.split(r'\|?\|?', msg, 1)) 177 | ax.set_title(msg, fontname="DejaVu Sans Mono", fontsize=11) 178 | self.plt.pause(1e-9) 179 | 180 | 181 | def tgrange(*args, **kwargs): 182 | """ 183 | A shortcut for `tqdm.gui.tqdm(xrange(*args), **kwargs)`. 184 | On Python3+, `range` is used instead of `xrange`. 185 | """ 186 | return tqdm_gui(_range(*args), **kwargs) 187 | 188 | 189 | # Aliases 190 | tqdm = tqdm_gui 191 | trange = tgrange 192 | -------------------------------------------------------------------------------- /examples/paper.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: '`tqdm`: A Fast, Extensible Progress Meter for Python and CLI' 3 | tags: 4 | - progressbar 5 | - progressmeter 6 | - progress-bar 7 | - meter 8 | - rate 9 | - eta 10 | - console 11 | - terminal 12 | - time 13 | - progress 14 | - bar 15 | - gui 16 | - python 17 | - parallel 18 | - cli 19 | - utilities 20 | - shell 21 | - batch 22 | authors: 23 | - name: Casper O da Costa-Luis 24 | orcid: 0000-0002-7211-1557 25 | affiliation: 1 26 | affiliations: 27 | - name: "Independent (Non-affiliated)" 28 | index: 1 29 | date: 16 February 2019 30 | bibliography: paper.bib 31 | --- 32 | 33 | # Introduction 34 | 35 | **`tqdm`** is a progress bar library designed to be fast and extensible. It is 36 | written in Python, though ports in other languages are available. `tqdm` means 37 | **progress** in Arabic (*taqadum* [@tqdm-ar]) and is an abbreviation for 38 | **I love you so much** in Spanish (*te quiero demasiado* [@tqdm-es]). 39 | 40 | It is a common programming problem to have iterative operations where progress 41 | monitoring is desirable or advantageous. Including statements within a `for` loop to `print` out the current iteration number is a common strategy. However, there are many improvements which could be made in such a scenario: 42 | 43 | - preventing excessive printing, such as only displaying every $n$^th^ 44 | iteration; 45 | - displaying iteration rate; 46 | - displaying elapsed and estimated completion times, and 47 | - showing all of the above on one continuously updating line. 48 | 49 | Addressing all these issues may well take up more developer time and effort than 50 | the rest of the content of the loop. Any changes to iteration rates or attempts 51 | to re-use the printing logic in a different loop may well result in suboptimal 52 | display rates -- displaying every $n$^th^ iteration may be too (in)frequent -- 53 | requiring manual adjustment of $n$ to fix. 54 | 55 | `tqdm` addresses all of these problems once and for all, taking advantage of 56 | Pythonic patterns to make it a trivial task to add visually appealing, 57 | customisable progress bars without any significant performance degradation even 58 | in the most demanding of scenarios. 59 | 60 | `tqdm` is intended to be used in frontends (giving end users a visual indication 61 | of progress of computations or data transfer). It is also useful for developers 62 | for debugging purposes, both as a profiling tool and also as a way of displaying 63 | logging information of an iterative task (such as error during training of 64 | machine learning algorithms). Due to its ease of use, the library is also an 65 | ideal candidate for inclusion in Python educational courses. For general (not 66 | necessarily Python) purposes, the command-line interface (CLI) mode further 67 | presents a useful tool for CLI users and system administrators monitoring data 68 | flow through pipes. 69 | 70 | # Features 71 | 72 | Exhaustive documentation may be found on the project's [home 73 | page](https://github.com/tqdm/tqdm/#documentation). 74 | 75 | The two basic use cases are within Python code and within a CLI: 76 | 77 | ## Python Iterable Wrapper 78 | 79 | `tqdm`'s primary (and original) use is as a wrapper around Python iterables. A 80 | simple case would be: 81 | 82 | ```python 83 | from tqdm import tqdm 84 | from time import sleep 85 | for i in tqdm(range(100)): 86 |     sleep(0.1) 87 | 100%|#########################################| 100/100 [00:10<00:00,  9.95it/s] 88 | ``` 89 | 90 | Supported features include: 91 | 92 | - Display customisation via arguments such as `desc`, `postfix` and `bar_format` 93 | - Automatic limiting of display updates to avoid slowing down due to excessive 94 | iteration rates [@stdout] 95 | - Automatic detection of console width to fill the display 96 | - Automatic use of Unicode to render smooth-filling progress bars on supported 97 | terminals 98 | - Support for custom rendering frontends, including: 99 | * Command-line interface 100 | * *Jupyter* HTML notebooks 101 | * `matplotlib` 102 | - Support for custom hooks/callbacks, including: 103 | * `pandas` 104 | * `keras` [@keras] 105 | 106 | ## Command-line Interface (CLI) 107 | 108 | A CLI is also provided, where `tqdm` may be used a pipe: 109 | 110 | ```sh 111 | # count lines of text in all *.txt files 112 | $ cat *.txt | wc -l 113 | 1075075 114 | # same but with continuously updating progress information 115 | $ cat *.txt | python3 -m tqdm --unit loc --unit_scale | wc -l 116 | 1.08Mloc [00:07, 142kloc/s] 117 | # same if `total` is known 118 | $ cat *.txt | python3 -m tqdm --unit loc --unit_scale --total 1075075 | wc -l 119 | 100%|#####################################| 1.08/1.08M [00:07<00:00,  142kloc/s] 120 | 1075075 121 | ``` 122 | 123 | # Availability 124 | 125 | The package supports both Python versions 2 and 3, and is available for download 126 | via `conda` [@conda], `pip` [@pypi], `snap` [@snapcraft], `docker` [@docker], 127 | and *Zenodo* [@zenodo]. 128 | Web-based Jupyter interactive demonstrations are also available 129 | [@notebooks;@binder] 130 | 131 | Unit tests are run at least weekly on cloud-based continuous integration 132 | [@travis], with code style and security issues checked on 133 | [Codacy](https://app.codacy.com/project/tqdm/tqdm/dashboard) [@code-review]. 134 | Coverage is reported on [Coveralls](https://coveralls.io/github/tqdm/tqdm) and 135 | [Codecov](https://codecov.io/gh/tqdm/tqdm), and performance is monitored against 136 | regression [@asv]. 137 | 138 | # Impact 139 | 140 | As of January 2019, `tqdm` has accumulated over 20 million downloads 141 | [@pypi-downloads], and 315 thousand code inclusions [@tqdm-results]. Dependants 142 | of `tqdm` include 23 thousand repositories [@tqdm-dependents] and 7 thousand 143 | libraries [@lib-io]. `tqdm` has a SourceRank of 22 [@sourcerank], placing it in 144 | the world's top 20 Python packages as of early 2019 [@sourcerank-descending]. 145 | 146 | The source code of `tqdm` is hosted on GitHub, where it has received over 9 147 | thousand stars [@stars;@stars-hist], and was top trending repository during a 148 | period in December 2015 [@trend-hist]. The documentation has received over 500 149 | thousand hits [@hits], with highest rates during weekdays. Historical reading 150 | rates have also trended upwards at the end of holiday periods. This implies 151 | widespread use in commercial and academic settings. 152 | [OpenHub](https://www.openhub.net/p/tqdm) valuates the work according to the 153 | constructive cost model (COCOMO) as being worth approximately $50,000. 154 | 155 | The library has also been used in several textbooks [@miller;@boxel;@nandy] and 156 | peer-reviewed scientific publications 157 | [@stein;@cook;@madhikar;@palmer;@knight;@moriwaki;@jackson]. 158 | The [`tqdm` wiki](https://github.com/tqdm/tqdm/wiki) also lists other references 159 | in public media. 160 | 161 | # Licence 162 | 163 | `tqdm`'s source code is OSS, and all versions are archived at the DOI 164 | [10.5281/zenodo.595120](https://doi.org/10.5281/zenodo.595120). The primary 165 | maintainer [Casper da Costa-Luis](https://github.com/casperdcl) releases 166 | contributions under the terms of the MPLv2.0, while all other contributions are 167 | released under the terms of the MIT licence [@licence]. 168 | 169 | # References 170 | -------------------------------------------------------------------------------- /tests/tests_contrib_logging.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=missing-module-docstring, missing-class-docstring 2 | # pylint: disable=missing-function-docstring, no-self-use 3 | from __future__ import absolute_import 4 | 5 | import logging 6 | import logging.handlers 7 | import sys 8 | from io import StringIO 9 | 10 | import pytest 11 | 12 | from tqdm import tqdm 13 | from tqdm.contrib.logging import _get_first_found_console_logging_handler 14 | from tqdm.contrib.logging import _TqdmLoggingHandler as TqdmLoggingHandler 15 | from tqdm.contrib.logging import logging_redirect_tqdm, tqdm_logging_redirect 16 | 17 | from .tests_tqdm import importorskip 18 | 19 | LOGGER = logging.getLogger(__name__) 20 | 21 | TEST_LOGGING_FORMATTER = logging.Formatter() 22 | 23 | 24 | class CustomTqdm(tqdm): 25 | messages = [] 26 | 27 | @classmethod 28 | def write(cls, s, **__): # pylint: disable=arguments-differ 29 | CustomTqdm.messages.append(s) 30 | 31 | 32 | class ErrorRaisingTqdm(tqdm): 33 | exception_class = RuntimeError 34 | 35 | @classmethod 36 | def write(cls, s, **__): # pylint: disable=arguments-differ 37 | raise ErrorRaisingTqdm.exception_class('fail fast') 38 | 39 | 40 | class TestTqdmLoggingHandler: 41 | def test_should_call_tqdm_write(self): 42 | CustomTqdm.messages = [] 43 | logger = logging.Logger('test') 44 | logger.handlers = [TqdmLoggingHandler(CustomTqdm)] 45 | logger.info('test') 46 | assert CustomTqdm.messages == ['test'] 47 | 48 | def test_should_call_handle_error_if_exception_was_thrown(self): 49 | patch = importorskip('unittest.mock').patch 50 | logger = logging.Logger('test') 51 | ErrorRaisingTqdm.exception_class = RuntimeError 52 | handler = TqdmLoggingHandler(ErrorRaisingTqdm) 53 | logger.handlers = [handler] 54 | with patch.object(handler, 'handleError') as mock: 55 | logger.info('test') 56 | assert mock.called 57 | 58 | @pytest.mark.parametrize('exception_class', [ 59 | KeyboardInterrupt, 60 | SystemExit 61 | ]) 62 | def test_should_not_swallow_certain_exceptions(self, exception_class): 63 | logger = logging.Logger('test') 64 | ErrorRaisingTqdm.exception_class = exception_class 65 | handler = TqdmLoggingHandler(ErrorRaisingTqdm) 66 | logger.handlers = [handler] 67 | with pytest.raises(exception_class): 68 | logger.info('test') 69 | 70 | 71 | class TestGetFirstFoundConsoleLoggingHandler: 72 | def test_should_return_none_for_no_handlers(self): 73 | assert _get_first_found_console_logging_handler([]) is None 74 | 75 | def test_should_return_none_without_stream_handler(self): 76 | handler = logging.handlers.MemoryHandler(capacity=1) 77 | assert _get_first_found_console_logging_handler([handler]) is None 78 | 79 | def test_should_return_none_for_stream_handler_not_stdout_or_stderr(self): 80 | handler = logging.StreamHandler(StringIO()) 81 | assert _get_first_found_console_logging_handler([handler]) is None 82 | 83 | def test_should_return_stream_handler_if_stream_is_stdout(self): 84 | handler = logging.StreamHandler(sys.stdout) 85 | assert _get_first_found_console_logging_handler([handler]) == handler 86 | 87 | def test_should_return_stream_handler_if_stream_is_stderr(self): 88 | handler = logging.StreamHandler(sys.stderr) 89 | assert _get_first_found_console_logging_handler([handler]) == handler 90 | 91 | 92 | class TestRedirectLoggingToTqdm: 93 | def test_should_add_and_remove_tqdm_handler(self): 94 | logger = logging.Logger('test') 95 | with logging_redirect_tqdm(loggers=[logger]): 96 | assert len(logger.handlers) == 1 97 | assert isinstance(logger.handlers[0], TqdmLoggingHandler) 98 | assert not logger.handlers 99 | 100 | def test_should_remove_and_restore_console_handlers(self): 101 | logger = logging.Logger('test') 102 | stderr_console_handler = logging.StreamHandler(sys.stderr) 103 | stdout_console_handler = logging.StreamHandler(sys.stderr) 104 | logger.handlers = [stderr_console_handler, stdout_console_handler] 105 | with logging_redirect_tqdm(loggers=[logger]): 106 | assert len(logger.handlers) == 1 107 | assert isinstance(logger.handlers[0], TqdmLoggingHandler) 108 | assert logger.handlers == [stderr_console_handler, stdout_console_handler] 109 | 110 | def test_should_inherit_console_logger_formatter(self): 111 | logger = logging.Logger('test') 112 | formatter = logging.Formatter('custom: %(message)s') 113 | console_handler = logging.StreamHandler(sys.stderr) 114 | console_handler.setFormatter(formatter) 115 | logger.handlers = [console_handler] 116 | with logging_redirect_tqdm(loggers=[logger]): 117 | assert logger.handlers[0].formatter == formatter 118 | 119 | def test_should_not_remove_stream_handlers_not_for_stdout_or_stderr(self): 120 | logger = logging.Logger('test') 121 | stream_handler = logging.StreamHandler(StringIO()) 122 | logger.addHandler(stream_handler) 123 | with logging_redirect_tqdm(loggers=[logger]): 124 | assert len(logger.handlers) == 2 125 | assert logger.handlers[0] == stream_handler 126 | assert isinstance(logger.handlers[1], TqdmLoggingHandler) 127 | assert logger.handlers == [stream_handler] 128 | 129 | 130 | class TestTqdmWithLoggingRedirect: 131 | def test_should_add_and_remove_handler_from_root_logger_by_default(self): 132 | original_handlers = list(logging.root.handlers) 133 | with tqdm_logging_redirect(total=1) as pbar: 134 | assert isinstance(logging.root.handlers[-1], TqdmLoggingHandler) 135 | LOGGER.info('test') 136 | pbar.update(1) 137 | assert logging.root.handlers == original_handlers 138 | 139 | def test_should_add_and_remove_handler_from_custom_logger(self): 140 | logger = logging.Logger('test') 141 | with tqdm_logging_redirect(total=1, loggers=[logger]) as pbar: 142 | assert len(logger.handlers) == 1 143 | assert isinstance(logger.handlers[0], TqdmLoggingHandler) 144 | logger.info('test') 145 | pbar.update(1) 146 | assert not logger.handlers 147 | 148 | def test_should_not_fail_with_logger_without_console_handler(self): 149 | logger = logging.Logger('test') 150 | logger.handlers = [] 151 | with tqdm_logging_redirect(total=1, loggers=[logger]): 152 | logger.info('test') 153 | assert not logger.handlers 154 | 155 | def test_should_format_message(self): 156 | logger = logging.Logger('test') 157 | console_handler = logging.StreamHandler(sys.stdout) 158 | console_handler.setFormatter(logging.Formatter( 159 | r'prefix:%(message)s' 160 | )) 161 | logger.handlers = [console_handler] 162 | CustomTqdm.messages = [] 163 | with tqdm_logging_redirect(loggers=[logger], tqdm_class=CustomTqdm): 164 | logger.info('test') 165 | assert CustomTqdm.messages == ['prefix:test'] 166 | 167 | def test_use_root_logger_by_default_and_write_to_custom_tqdm(self): 168 | logger = logging.root 169 | CustomTqdm.messages = [] 170 | with tqdm_logging_redirect(total=1, tqdm_class=CustomTqdm) as pbar: 171 | assert isinstance(pbar, CustomTqdm) 172 | logger.info('test') 173 | assert CustomTqdm.messages == ['test'] 174 | -------------------------------------------------------------------------------- /tqdm/tk.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tkinter GUI progressbar decorator for iterators. 3 | 4 | Usage: 5 | >>> from tqdm.tk import trange, tqdm 6 | >>> for i in trange(10): 7 | ... ... 8 | """ 9 | from __future__ import absolute_import, division 10 | 11 | import re 12 | import sys 13 | from warnings import warn 14 | 15 | try: 16 | import tkinter 17 | import tkinter.ttk as ttk 18 | except ImportError: 19 | import Tkinter as tkinter 20 | import ttk as ttk 21 | 22 | from .std import TqdmExperimentalWarning, TqdmWarning 23 | from .std import tqdm as std_tqdm 24 | from .utils import _range 25 | 26 | __author__ = {"github.com/": ["richardsheridan", "casperdcl"]} 27 | __all__ = ['tqdm_tk', 'ttkrange', 'tqdm', 'trange'] 28 | 29 | 30 | class tqdm_tk(std_tqdm): # pragma: no cover 31 | """ 32 | Experimental Tkinter GUI version of tqdm! 33 | 34 | Note: Window interactivity suffers if `tqdm_tk` is not running within 35 | a Tkinter mainloop and values are generated infrequently. In this case, 36 | consider calling `tqdm_tk.refresh()` frequently in the Tk thread. 37 | """ 38 | 39 | # TODO: @classmethod: write()? 40 | 41 | def __init__(self, *args, **kwargs): 42 | """ 43 | This class accepts the following parameters *in addition* to 44 | the parameters accepted by `tqdm`. 45 | 46 | Parameters 47 | ---------- 48 | grab : bool, optional 49 | Grab the input across all windows of the process. 50 | tk_parent : `tkinter.Wm`, optional 51 | Parent Tk window. 52 | cancel_callback : Callable, optional 53 | Create a cancel button and set `cancel_callback` to be called 54 | when the cancel or window close button is clicked. 55 | """ 56 | kwargs = kwargs.copy() 57 | kwargs['gui'] = True 58 | # convert disable = None to False 59 | kwargs['disable'] = bool(kwargs.get('disable', False)) 60 | self._warn_leave = 'leave' in kwargs 61 | grab = kwargs.pop('grab', False) 62 | tk_parent = kwargs.pop('tk_parent', None) 63 | self._cancel_callback = kwargs.pop('cancel_callback', None) 64 | super(tqdm_tk, self).__init__(*args, **kwargs) 65 | 66 | if self.disable: 67 | return 68 | 69 | if tk_parent is None: # Discover parent widget 70 | try: 71 | tk_parent = tkinter._default_root 72 | except AttributeError: 73 | raise AttributeError( 74 | "`tk_parent` required when using `tkinter.NoDefaultRoot()`") 75 | if tk_parent is None: # use new default root window as display 76 | self._tk_window = tkinter.Tk() 77 | else: # some other windows already exist 78 | self._tk_window = tkinter.Toplevel() 79 | else: 80 | self._tk_window = tkinter.Toplevel(tk_parent) 81 | 82 | warn("GUI is experimental/alpha", TqdmExperimentalWarning, stacklevel=2) 83 | self._tk_dispatching = self._tk_dispatching_helper() 84 | 85 | self._tk_window.protocol("WM_DELETE_WINDOW", self.cancel) 86 | self._tk_window.wm_title(self.desc) 87 | self._tk_window.wm_attributes("-topmost", 1) 88 | self._tk_window.after(0, lambda: self._tk_window.wm_attributes("-topmost", 0)) 89 | self._tk_n_var = tkinter.DoubleVar(self._tk_window, value=0) 90 | self._tk_text_var = tkinter.StringVar(self._tk_window) 91 | pbar_frame = ttk.Frame(self._tk_window, padding=5) 92 | pbar_frame.pack() 93 | _tk_label = ttk.Label(pbar_frame, textvariable=self._tk_text_var, 94 | wraplength=600, anchor="center", justify="center") 95 | _tk_label.pack() 96 | self._tk_pbar = ttk.Progressbar( 97 | pbar_frame, variable=self._tk_n_var, length=450) 98 | if self.total is not None: 99 | self._tk_pbar.configure(maximum=self.total) 100 | else: 101 | self._tk_pbar.configure(mode="indeterminate") 102 | self._tk_pbar.pack() 103 | if self._cancel_callback is not None: 104 | _tk_button = ttk.Button(pbar_frame, text="Cancel", command=self.cancel) 105 | _tk_button.pack() 106 | if grab: 107 | self._tk_window.grab_set() 108 | 109 | def close(self): 110 | if self.disable: 111 | return 112 | 113 | self.disable = True 114 | 115 | with self.get_lock(): 116 | self._instances.remove(self) 117 | 118 | def _close(): 119 | self._tk_window.after('idle', self._tk_window.destroy) 120 | if not self._tk_dispatching: 121 | self._tk_window.update() 122 | 123 | self._tk_window.protocol("WM_DELETE_WINDOW", _close) 124 | 125 | # if leave is set but we are self-dispatching, the left window is 126 | # totally unresponsive unless the user manually dispatches 127 | if not self.leave: 128 | _close() 129 | elif not self._tk_dispatching: 130 | if self._warn_leave: 131 | warn("leave flag ignored if not in tkinter mainloop", 132 | TqdmWarning, stacklevel=2) 133 | _close() 134 | 135 | def clear(self, *_, **__): 136 | pass 137 | 138 | def display(self, *_, **__): 139 | self._tk_n_var.set(self.n) 140 | d = self.format_dict 141 | # remove {bar} 142 | d['bar_format'] = (d['bar_format'] or "{l_bar}{r_bar}").replace( 143 | "{bar}", "") 144 | msg = self.format_meter(**d) 145 | if '' in msg: 146 | msg = "".join(re.split(r'\|?\|?', msg, 1)) 147 | self._tk_text_var.set(msg) 148 | if not self._tk_dispatching: 149 | self._tk_window.update() 150 | 151 | def set_description(self, desc=None, refresh=True): 152 | self.set_description_str(desc, refresh) 153 | 154 | def set_description_str(self, desc=None, refresh=True): 155 | self.desc = desc 156 | if not self.disable: 157 | self._tk_window.wm_title(desc) 158 | if refresh and not self._tk_dispatching: 159 | self._tk_window.update() 160 | 161 | def cancel(self): 162 | """ 163 | `cancel_callback()` followed by `close()` 164 | when close/cancel buttons clicked. 165 | """ 166 | if self._cancel_callback is not None: 167 | self._cancel_callback() 168 | self.close() 169 | 170 | def reset(self, total=None): 171 | """ 172 | Resets to 0 iterations for repeated use. 173 | 174 | Parameters 175 | ---------- 176 | total : int or float, optional. Total to use for the new bar. 177 | """ 178 | if hasattr(self, '_tk_pbar'): 179 | if total is None: 180 | self._tk_pbar.configure(maximum=100, mode="indeterminate") 181 | else: 182 | self._tk_pbar.configure(maximum=total, mode="determinate") 183 | super(tqdm_tk, self).reset(total=total) 184 | 185 | @staticmethod 186 | def _tk_dispatching_helper(): 187 | """determine if Tkinter mainloop is dispatching events""" 188 | codes = {tkinter.mainloop.__code__, tkinter.Misc.mainloop.__code__} 189 | for frame in sys._current_frames().values(): 190 | while frame: 191 | if frame.f_code in codes: 192 | return True 193 | frame = frame.f_back 194 | return False 195 | 196 | 197 | def ttkrange(*args, **kwargs): 198 | """ 199 | A shortcut for `tqdm.tk.tqdm(xrange(*args), **kwargs)`. 200 | On Python3+, `range` is used instead of `xrange`. 201 | """ 202 | return tqdm_tk(_range(*args), **kwargs) 203 | 204 | 205 | # Aliases 206 | tqdm = tqdm_tk 207 | trange = ttkrange 208 | -------------------------------------------------------------------------------- /examples/paper.bib: -------------------------------------------------------------------------------- 1 | @phdthesis{tqdm-ar, 2 | author="Maḥmūd Alī Ġūl", 3 | title="Early Southern Arabian Languages and Classical Arabic Sources: A Critical Examination of Literary and Lexicographical Sources by Comparison with the Inscriptions", 4 | school="{SOAS} University of London", 5 | year="1963" 6 | } 7 | @misc{tqdm-es, 8 | year="2009", 9 | title="¿Lenguaje sms que significa esto?", 10 | url="https://es.answers.yahoo.com/question/index?qid=20090405052137AAF2YBo&guccounter=1", 11 | author="{Yahoo Answers}" 12 | } 13 | @misc{pypi, 14 | year="2019", 15 | author="{Python Package Index ({PyPI})}", 16 | publisher="Python Software Foundation", 17 | title="{tqdm}", 18 | url="https://pypi.org/project/tqdm/" 19 | } 20 | @misc{conda, 21 | author="Anaconda", 22 | year="2019", 23 | title="{tqdm} :: Anaconda Cloud", 24 | url="https://anaconda.org/conda-forge/tqdm" 25 | } 26 | @misc{docker, 27 | year="2019", 28 | author="{Docker Inc.}", 29 | title="{tqdm}/{tqdm} - Docker Hub", 30 | url="https://hub.docker.com/r/tqdm/tqdm" 31 | } 32 | @misc{snapcraft, 33 | year="2019", 34 | author="Snapcraft", 35 | title="Installing {tqdm} for Linux using the Snap Store", 36 | url="https://snapcraft.io/tqdm" 37 | } 38 | @article{zenodo, 39 | year="2019", 40 | author="Casper O. {da Costa-Luis} and {{tqdm} developers}", 41 | title="{tqdm} stable", 42 | publisher="Zenodo", 43 | doi="10.5281/zenodo.595120" 44 | } 45 | @misc{notebooks, 46 | year="2019", 47 | author="{Notebooks {AI}}", 48 | title="{tqdm}", 49 | url="https://notebooks.ai/demo/gh/tqdm/tqdm" 50 | } 51 | @misc{binder, 52 | year="2019", 53 | author="Binder", 54 | title="{tqdm}", 55 | url="https://mybinder.org/v2/gh/tqdm/tqdm/master?filepath=DEMO.ipynb" 56 | } 57 | @misc{stdout, 58 | year="2019", 59 | author="{Stack Overflow}", 60 | title="Why is printing to stdout so slow? Can it be sped up?", 61 | url="https://stackoverflow.com/questions/3857052/why-is-printing-to-stdout-so-slow-can-it-be-sped-up" 62 | } 63 | @misc{pypi-downloads, 64 | year="2019", 65 | author="{Python Packaging Authority ({PyPA})}", 66 | publisher="Python Software Foundation", 67 | title="Analyzing {PyPI} package downloads -- Python Packaging User Guide", 68 | url="https://packaging.python.org/guides/analyzing-pypi-package-downloads/" 69 | } 70 | @misc{keras, 71 | year="2019", 72 | author="Ben", 73 | title="Keras integration with {tqdm} progress bars", 74 | url="https://github.com/bstriner/keras-tqdm" 75 | } 76 | @misc{tqdm-results, 77 | year="2019", 78 | author="GitHub", 79 | title="{tqdm} Code Results", 80 | url="https://github.com/search?q=tqdm&type=Code" 81 | } 82 | @misc{tqdm-dependents, 83 | year="2019", 84 | author="GitHub", 85 | title="{tqdm} dependents", 86 | url="https://github.com/tqdm/tqdm/network/dependents" 87 | } 88 | @misc{lib-io, 89 | year="2019", 90 | author="Libraries.io", 91 | title="{tqdm} on {PyPI}", 92 | url="https://libraries.io/pypi/tqdm" 93 | } 94 | @misc{sourcerank, 95 | year="2019", 96 | author="Libraries.io", 97 | title="SourceRank Breakdown for {tqdm}", 98 | url="https://libraries.io/pypi/tqdm/sourcerank" 99 | } 100 | @misc{sourcerank-descending, 101 | year="2019", 102 | author="Libraries.io", 103 | title="Libraries - The Open Source Discovery Service", 104 | url="https://libraries.io/search?order=desc&platforms=PyPI&sort=rank" 105 | } 106 | @misc{stars, 107 | year="2019", 108 | author="GitHub", 109 | title="{tqdm} Stargazers", 110 | url="https://github.com/tqdm/tqdm/stargazers" 111 | } 112 | @misc{stars-hist, 113 | year="2019", 114 | author="{timqian}", 115 | title="Star history", 116 | url="https://timqian.com/star-history/#tqdm/tqdm" 117 | } 118 | @misc{trend-hist, 119 | year="2018", 120 | month="June", 121 | day="19", 122 | author="Nihey Takizawa", 123 | title="GitHub Trending History", 124 | url="https://github.com/nihey/trending-history/blob/master/histories/Python.md" 125 | } 126 | @misc{hits, 127 | year="2019", 128 | title="{tqdm} hits", 129 | url="https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&a=plot", 130 | author="Casper O. {da Costa-Luis}" 131 | } 132 | @book{miller, 133 | year="2017", 134 | author="Preston Miller and Chapin Bryce", 135 | title="Python Digital Forensics Cookbook: Effective Python recipes for digital investigations", 136 | publisher="Packt Publishing Ltd", 137 | isbn="9781783987474" 138 | } 139 | @book{boxel, 140 | year="2017", 141 | author="Dan {Van Boxel}", 142 | title="Hands-On Deep Learning with TensorFlow", 143 | publisher="Packt Publishing", 144 | isbn="9781787125827" 145 | } 146 | @incollection{nandy, 147 | year="2018", 148 | author="Abhishek Nandy and Manisha Biswas", 149 | title="Reinforcement Learning with Keras, TensorFlow, and ChainerRL", 150 | booktitle="Reinforcement Learning : With Open AI, TensorFlow and Keras Using Python", 151 | publisher="Apress", 152 | isbn="9781484232859", 153 | pages="129--153", 154 | doi="10.1007/978-1-4842-3285-9_5" 155 | } 156 | @journal{stein, 157 | year="2019", 158 | author="Helge S. Stein and Dan Guevarra and Paul F. Newhouse and Edwin Soedarmadji and John M. Gregoire", 159 | title="Machine learning of optical properties of materials -- predicting spectra from images and images from spectra", 160 | journal="Chemical Science", 161 | volume="10", 162 | issue="1", 163 | pages="47--55", 164 | doi="10.1039/C8SC03077D" 165 | } 166 | @journal{cook, 167 | year="2018", 168 | author="Neil J. Cook and Aleks Scholz and Ray Jayawardhana", 169 | title="Very Low-mass Stars and Brown Dwarfs in Upper Scorpius Using Gaia DR1: Mass Function, Disks, and Kinematics", 170 | journal="The Astronomical Journal", 171 | volume="154", 172 | issue="6", 173 | pages="256", 174 | doi="10.3847/1538-3881/aa9751", 175 | url="https://arxiv.org/abs/1710.11625" 176 | } 177 | @journal{madhikar, 178 | year="2018", 179 | author="Pranav Madhikar and Jan Åström and Jan Westerholm and Mikko Karttunen", 180 | title="CellSim3D: GPU accelerated software for simulations of cellular growth and division in three dimensions", 181 | journal="Computer Physics Communications", 182 | volume="232", 183 | pages="206--213", 184 | doi="10.1016/j.cpc.2018.05.024" 185 | } 186 | @journal{palmer, 187 | year="2018", 188 | author="Geraint I. Palmer and Vincent A. Knight and Paul R. Harper and Asyl L. Hawa", 189 | title="Ciw: An open-source discrete event simulation library", 190 | journal="Journal of Simulation", 191 | pages="1--15", 192 | doi="10.1080/17477778.2018.1473909" 193 | } 194 | @journal{knight, 195 | year="2016", 196 | author="Vincent Knight and Owen Campbell and Marc Harper and Karol Langner and James Campbell and Thomas Campbell and Alex Carney and Martin Chorley and Cameron Davidson-Pilon and Kristian Glass and Nikoleta Glynatsi and Tomáš Ehrlich and Martin Jones and Georgios Koutsovoulos and Holly Tibble and Müller Jochen and Geraint Palmer and Piotr Petunov and Paul Slavin and Timothy Standen and Luis Visintini and Karl Molden", 197 | title="An open reproducible framework for the study of the iterated prisoner's dilemma", 198 | journal="Journal of Open Research Software", 199 | volume="4", 200 | doi="10.5334/jors.125", 201 | url="https://arxiv.org/abs/1604.00896", 202 | issn="2049-9647" 203 | } 204 | @article{moriwaki, 205 | title={Mordred: a molecular descriptor calculator}, 206 | author={Moriwaki, Hirotomo and Tian, Yu-Shi and Kawashita, Norihito and Takagi, Tatsuya}, 207 | doi={10.1186/s13321-018-0258-y}, 208 | number={1}, 209 | volume={10}, 210 | month={February}, 211 | year={2018}, 212 | journal={Journal of cheminformatics}, 213 | issn={1758-2946}, 214 | pages={4} 215 | } 216 | @article{jackson, 217 | title={3D for the people: multi-camera motion capture in the field with consumer-grade cameras and open source software}, 218 | author={Jackson, Brandon E and Evangelista, Dennis J and Ray, Dylan D and hedrick, Tyson L}, 219 | doi={10.1242/bio.018713}, 220 | number={9}, 221 | volume={5}, 222 | month={September}, 223 | year={2016}, 224 | journal={Biology open}, 225 | issn={2046-6390}, 226 | pages={1334--1342} 227 | } 228 | @misc{travis, 229 | year="2019", 230 | author="{Travis {CI}}", 231 | title="tqdm/tqdm build status", 232 | url="https://travis-ci.org/tqdm/tqdm" 233 | } 234 | @misc{code-review, 235 | year="2018", 236 | author="Wikipedia", 237 | title="List of tools for code review", 238 | url="https://en.wikipedia.org/wiki/List_of_tools_for_code_review" 239 | } 240 | @misc{asv, 241 | year="2019", 242 | author="{{tqdm} developers}", 243 | title="airspeed velocity", 244 | url="https://tqdm.github.io/tqdm/" 245 | } 246 | @misc{licence, 247 | year="2019", 248 | author="{{tqdm} developers}", 249 | title="{tqdm} Licence", 250 | url="https://github.com/tqdm/tqdm/blob/master/LICENCE", 251 | publisher="GitHub" 252 | } 253 | -------------------------------------------------------------------------------- /tests/tests_synchronisation.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | 3 | import sys 4 | from functools import wraps 5 | from threading import Event 6 | from time import sleep, time 7 | 8 | from tqdm import TMonitor, tqdm, trange 9 | 10 | from .tests_perf import retry_on_except 11 | from .tests_tqdm import StringIO, closing, importorskip, patch_lock, skip 12 | 13 | 14 | class Time(object): 15 | """Fake time class class providing an offset""" 16 | offset = 0 17 | 18 | @classmethod 19 | def reset(cls): 20 | """zeroes internal offset""" 21 | cls.offset = 0 22 | 23 | @classmethod 24 | def time(cls): 25 | """time.time() + offset""" 26 | return time() + cls.offset 27 | 28 | @staticmethod 29 | def sleep(dur): 30 | """identical to time.sleep()""" 31 | sleep(dur) 32 | 33 | @classmethod 34 | def fake_sleep(cls, dur): 35 | """adds `dur` to internal offset""" 36 | cls.offset += dur 37 | sleep(0.000001) # sleep to allow interrupt (instead of pass) 38 | 39 | 40 | def FakeEvent(): 41 | """patched `threading.Event` where `wait()` uses `Time.fake_sleep()`""" 42 | event = Event() # not a class in py2 so can't inherit 43 | 44 | def wait(timeout=None): 45 | """uses Time.fake_sleep""" 46 | if timeout is not None: 47 | Time.fake_sleep(timeout) 48 | return event.is_set() 49 | 50 | event.wait = wait 51 | return event 52 | 53 | 54 | def patch_sleep(func): 55 | """Temporarily makes TMonitor use Time.fake_sleep""" 56 | @wraps(func) 57 | def inner(*args, **kwargs): 58 | """restores TMonitor on completion regardless of Exceptions""" 59 | TMonitor._test["time"] = Time.time 60 | TMonitor._test["Event"] = FakeEvent 61 | if tqdm.monitor: 62 | assert not tqdm.monitor.get_instances() 63 | tqdm.monitor.exit() 64 | del tqdm.monitor 65 | tqdm.monitor = None 66 | try: 67 | return func(*args, **kwargs) 68 | finally: 69 | # Check that class var monitor is deleted if no instance left 70 | tqdm.monitor_interval = 10 71 | if tqdm.monitor: 72 | assert not tqdm.monitor.get_instances() 73 | tqdm.monitor.exit() 74 | del tqdm.monitor 75 | tqdm.monitor = None 76 | TMonitor._test.pop("Event") 77 | TMonitor._test.pop("time") 78 | 79 | return inner 80 | 81 | 82 | def cpu_timify(t, timer=Time): 83 | """Force tqdm to use the specified timer instead of system-wide time""" 84 | t._time = timer.time 85 | t._sleep = timer.fake_sleep 86 | t.start_t = t.last_print_t = t._time() 87 | return timer 88 | 89 | 90 | class FakeTqdm(object): 91 | _instances = set() 92 | get_lock = tqdm.get_lock 93 | 94 | 95 | def incr(x): 96 | return x + 1 97 | 98 | 99 | def incr_bar(x): 100 | with closing(StringIO()) as our_file: 101 | for _ in trange(x, lock_args=(False,), file=our_file): 102 | pass 103 | return incr(x) 104 | 105 | 106 | @patch_sleep 107 | def test_monitor_thread(): 108 | """Test dummy monitoring thread""" 109 | monitor = TMonitor(FakeTqdm, 10) 110 | # Test if alive, then killed 111 | assert monitor.report() 112 | monitor.exit() 113 | assert not monitor.report() 114 | assert not monitor.is_alive() 115 | del monitor 116 | 117 | 118 | @patch_sleep 119 | def test_monitoring_and_cleanup(): 120 | """Test for stalled tqdm instance and monitor deletion""" 121 | # Note: should fix miniters for these tests, else with dynamic_miniters 122 | # it's too complicated to handle with monitoring update and maxinterval... 123 | maxinterval = tqdm.monitor_interval 124 | assert maxinterval == 10 125 | total = 1000 126 | 127 | with closing(StringIO()) as our_file: 128 | with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, 129 | maxinterval=maxinterval) as t: 130 | cpu_timify(t, Time) 131 | # Do a lot of iterations in a small timeframe 132 | # (smaller than monitor interval) 133 | Time.fake_sleep(maxinterval / 10) # monitor won't wake up 134 | t.update(500) 135 | # check that our fixed miniters is still there 136 | assert t.miniters <= 500 # TODO: should really be == 500 137 | # Then do 1 it after monitor interval, so that monitor kicks in 138 | Time.fake_sleep(maxinterval) 139 | t.update(1) 140 | # Wait for the monitor to get out of sleep's loop and update tqdm. 141 | timeend = Time.time() 142 | while not (t.monitor.woken >= timeend and t.miniters == 1): 143 | Time.fake_sleep(1) # Force awake up if it woken too soon 144 | assert t.miniters == 1 # check that monitor corrected miniters 145 | # Note: at this point, there may be a race condition: monitor saved 146 | # current woken time but Time.sleep() happen just before monitor 147 | # sleep. To fix that, either sleep here or increase time in a loop 148 | # to ensure that monitor wakes up at some point. 149 | 150 | # Try again but already at miniters = 1 so nothing will be done 151 | Time.fake_sleep(maxinterval) 152 | t.update(2) 153 | timeend = Time.time() 154 | while t.monitor.woken < timeend: 155 | Time.fake_sleep(1) # Force awake if it woken too soon 156 | # Wait for the monitor to get out of sleep's loop and update 157 | # tqdm 158 | assert t.miniters == 1 # check that monitor corrected miniters 159 | 160 | 161 | @patch_sleep 162 | def test_monitoring_multi(): 163 | """Test on multiple bars, one not needing miniters adjustment""" 164 | # Note: should fix miniters for these tests, else with dynamic_miniters 165 | # it's too complicated to handle with monitoring update and maxinterval... 166 | maxinterval = tqdm.monitor_interval 167 | assert maxinterval == 10 168 | total = 1000 169 | 170 | with closing(StringIO()) as our_file: 171 | with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, 172 | maxinterval=maxinterval) as t1: 173 | # Set high maxinterval for t2 so monitor does not need to adjust it 174 | with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, 175 | maxinterval=1E5) as t2: 176 | cpu_timify(t1, Time) 177 | cpu_timify(t2, Time) 178 | # Do a lot of iterations in a small timeframe 179 | Time.fake_sleep(maxinterval / 10) 180 | t1.update(500) 181 | t2.update(500) 182 | assert t1.miniters <= 500 # TODO: should really be == 500 183 | assert t2.miniters == 500 184 | # Then do 1 it after monitor interval, so that monitor kicks in 185 | Time.fake_sleep(maxinterval) 186 | t1.update(1) 187 | t2.update(1) 188 | # Wait for the monitor to get out of sleep and update tqdm 189 | timeend = Time.time() 190 | while not (t1.monitor.woken >= timeend and t1.miniters == 1): 191 | Time.fake_sleep(1) 192 | assert t1.miniters == 1 # check that monitor corrected miniters 193 | assert t2.miniters == 500 # check that t2 was not adjusted 194 | 195 | 196 | def test_imap(): 197 | """Test multiprocessing.Pool""" 198 | try: 199 | from multiprocessing import Pool 200 | except ImportError as err: 201 | skip(str(err)) 202 | 203 | pool = Pool() 204 | res = list(tqdm(pool.imap(incr, range(100)), disable=True)) 205 | pool.close() 206 | assert res[-1] == 100 207 | 208 | 209 | # py2: locks won't propagate to incr_bar so may cause `AttributeError` 210 | @retry_on_except(n=3 if sys.version_info < (3,) else 1, check_cpu_time=False) 211 | @patch_lock(thread=True) 212 | def test_threadpool(): 213 | """Test concurrent.futures.ThreadPoolExecutor""" 214 | ThreadPoolExecutor = importorskip('concurrent.futures').ThreadPoolExecutor 215 | 216 | with ThreadPoolExecutor(8) as pool: 217 | try: 218 | res = list(tqdm(pool.map(incr_bar, range(100)), disable=True)) 219 | except AttributeError: 220 | if sys.version_info < (3,): 221 | skip("not supported on py2") 222 | else: 223 | raise 224 | assert sum(res) == sum(range(1, 101)) 225 | -------------------------------------------------------------------------------- /tests/tests_pandas.py: -------------------------------------------------------------------------------- 1 | from tqdm import tqdm 2 | 3 | from .tests_tqdm import StringIO, closing, importorskip, mark, skip 4 | 5 | pytestmark = mark.slow 6 | 7 | random = importorskip('numpy.random') 8 | rand = random.rand 9 | randint = random.randint 10 | pd = importorskip('pandas') 11 | 12 | 13 | def test_pandas_setup(): 14 | """Test tqdm.pandas()""" 15 | with closing(StringIO()) as our_file: 16 | tqdm.pandas(file=our_file, leave=True, ascii=True, total=123) 17 | series = pd.Series(randint(0, 50, (100,))) 18 | series.progress_apply(lambda x: x + 10) 19 | res = our_file.getvalue() 20 | assert '100/123' in res 21 | 22 | 23 | def test_pandas_rolling_expanding(): 24 | """Test pandas.(Series|DataFrame).(rolling|expanding)""" 25 | with closing(StringIO()) as our_file: 26 | tqdm.pandas(file=our_file, leave=True, ascii=True) 27 | 28 | series = pd.Series(randint(0, 50, (123,))) 29 | res1 = series.rolling(10).progress_apply(lambda x: 1, raw=True) 30 | res2 = series.rolling(10).apply(lambda x: 1, raw=True) 31 | assert res1.equals(res2) 32 | 33 | res3 = series.expanding(10).progress_apply(lambda x: 2, raw=True) 34 | res4 = series.expanding(10).apply(lambda x: 2, raw=True) 35 | assert res3.equals(res4) 36 | 37 | expects = ['114it'] # 123-10+1 38 | for exres in expects: 39 | our_file.seek(0) 40 | if our_file.getvalue().count(exres) < 2: 41 | our_file.seek(0) 42 | raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( 43 | exres + " at least twice.", our_file.read())) 44 | 45 | 46 | def test_pandas_series(): 47 | """Test pandas.Series.progress_apply and .progress_map""" 48 | with closing(StringIO()) as our_file: 49 | tqdm.pandas(file=our_file, leave=True, ascii=True) 50 | 51 | series = pd.Series(randint(0, 50, (123,))) 52 | res1 = series.progress_apply(lambda x: x + 10) 53 | res2 = series.apply(lambda x: x + 10) 54 | assert res1.equals(res2) 55 | 56 | res3 = series.progress_map(lambda x: x + 10) 57 | res4 = series.map(lambda x: x + 10) 58 | assert res3.equals(res4) 59 | 60 | expects = ['100%', '123/123'] 61 | for exres in expects: 62 | our_file.seek(0) 63 | if our_file.getvalue().count(exres) < 2: 64 | our_file.seek(0) 65 | raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( 66 | exres + " at least twice.", our_file.read())) 67 | 68 | 69 | def test_pandas_data_frame(): 70 | """Test pandas.DataFrame.progress_apply and .progress_applymap""" 71 | with closing(StringIO()) as our_file: 72 | tqdm.pandas(file=our_file, leave=True, ascii=True) 73 | df = pd.DataFrame(randint(0, 50, (100, 200))) 74 | 75 | def task_func(x): 76 | return x + 1 77 | 78 | # applymap 79 | res1 = df.progress_applymap(task_func) 80 | res2 = df.applymap(task_func) 81 | assert res1.equals(res2) 82 | 83 | # apply unhashable 84 | res1 = [] 85 | df.progress_apply(res1.extend) 86 | assert len(res1) == df.size 87 | 88 | # apply 89 | for axis in [0, 1, 'index', 'columns']: 90 | res3 = df.progress_apply(task_func, axis=axis) 91 | res4 = df.apply(task_func, axis=axis) 92 | assert res3.equals(res4) 93 | 94 | our_file.seek(0) 95 | if our_file.read().count('100%') < 3: 96 | our_file.seek(0) 97 | raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( 98 | '100% at least three times', our_file.read())) 99 | 100 | # apply_map, apply axis=0, apply axis=1 101 | expects = ['20000/20000', '200/200', '100/100'] 102 | for exres in expects: 103 | our_file.seek(0) 104 | if our_file.getvalue().count(exres) < 1: 105 | our_file.seek(0) 106 | raise AssertionError("\nExpected:\n{0}\nIn:\n {1}\n".format( 107 | exres + " at least once.", our_file.read())) 108 | 109 | 110 | def test_pandas_groupby_apply(): 111 | """Test pandas.DataFrame.groupby(...).progress_apply""" 112 | with closing(StringIO()) as our_file: 113 | tqdm.pandas(file=our_file, leave=False, ascii=True) 114 | 115 | df = pd.DataFrame(randint(0, 50, (500, 3))) 116 | df.groupby(0).progress_apply(lambda x: None) 117 | 118 | dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc')) 119 | dfs.groupby(['a']).progress_apply(lambda x: None) 120 | 121 | df2 = df = pd.DataFrame({'a': randint(1, 8, 10000), 'b': rand(10000)}) 122 | res1 = df2.groupby("a").apply(max) 123 | res2 = df2.groupby("a").progress_apply(max) 124 | assert res1.equals(res2) 125 | 126 | our_file.seek(0) 127 | 128 | # don't expect final output since no `leave` and 129 | # high dynamic `miniters` 130 | nexres = '100%|##########|' 131 | if nexres in our_file.read(): 132 | our_file.seek(0) 133 | raise AssertionError("\nDid not expect:\n{0}\nIn:{1}\n".format( 134 | nexres, our_file.read())) 135 | 136 | with closing(StringIO()) as our_file: 137 | tqdm.pandas(file=our_file, leave=True, ascii=True) 138 | 139 | dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc')) 140 | dfs.loc[0] = [2, 1, 1] 141 | dfs['d'] = 100 142 | 143 | expects = ['500/500', '1/1', '4/4', '2/2'] 144 | dfs.groupby(dfs.index).progress_apply(lambda x: None) 145 | dfs.groupby('d').progress_apply(lambda x: None) 146 | dfs.groupby(dfs.columns, axis=1).progress_apply(lambda x: None) 147 | dfs.groupby([2, 2, 1, 1], axis=1).progress_apply(lambda x: None) 148 | 149 | our_file.seek(0) 150 | if our_file.read().count('100%') < 4: 151 | our_file.seek(0) 152 | raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( 153 | '100% at least four times', our_file.read())) 154 | 155 | for exres in expects: 156 | our_file.seek(0) 157 | if our_file.getvalue().count(exres) < 1: 158 | our_file.seek(0) 159 | raise AssertionError("\nExpected:\n{0}\nIn:\n {1}\n".format( 160 | exres + " at least once.", our_file.read())) 161 | 162 | 163 | def test_pandas_leave(): 164 | """Test pandas with `leave=True`""" 165 | with closing(StringIO()) as our_file: 166 | df = pd.DataFrame(randint(0, 100, (1000, 6))) 167 | tqdm.pandas(file=our_file, leave=True, ascii=True) 168 | df.groupby(0).progress_apply(lambda x: None) 169 | 170 | our_file.seek(0) 171 | 172 | exres = '100%|##########| 100/100' 173 | if exres not in our_file.read(): 174 | our_file.seek(0) 175 | raise AssertionError("\nExpected:\n{0}\nIn:{1}\n".format( 176 | exres, our_file.read())) 177 | 178 | 179 | def test_pandas_apply_args_deprecation(): 180 | """Test warning info in 181 | `pandas.Dataframe(Series).progress_apply(func, *args)`""" 182 | try: 183 | from tqdm import tqdm_pandas 184 | except ImportError as err: 185 | skip(str(err)) 186 | 187 | with closing(StringIO()) as our_file: 188 | tqdm_pandas(tqdm(file=our_file, leave=False, ascii=True, ncols=20)) 189 | df = pd.DataFrame(randint(0, 50, (500, 3))) 190 | df.progress_apply(lambda x: None, 1) # 1 shall cause a warning 191 | # Check deprecation message 192 | res = our_file.getvalue() 193 | assert all(i in res for i in ( 194 | "TqdmDeprecationWarning", "not supported", 195 | "keyword arguments instead")) 196 | 197 | 198 | def test_pandas_deprecation(): 199 | """Test bar object instance as argument deprecation""" 200 | try: 201 | from tqdm import tqdm_pandas 202 | except ImportError as err: 203 | skip(str(err)) 204 | 205 | with closing(StringIO()) as our_file: 206 | tqdm_pandas(tqdm(file=our_file, leave=False, ascii=True, ncols=20)) 207 | df = pd.DataFrame(randint(0, 50, (500, 3))) 208 | df.groupby(0).progress_apply(lambda x: None) 209 | # Check deprecation message 210 | assert "TqdmDeprecationWarning" in our_file.getvalue() 211 | assert "instead of `tqdm_pandas(tqdm(...))`" in our_file.getvalue() 212 | 213 | with closing(StringIO()) as our_file: 214 | tqdm_pandas(tqdm, file=our_file, leave=False, ascii=True, ncols=20) 215 | df = pd.DataFrame(randint(0, 50, (500, 3))) 216 | df.groupby(0).progress_apply(lambda x: None) 217 | # Check deprecation message 218 | assert "TqdmDeprecationWarning" in our_file.getvalue() 219 | assert "instead of `tqdm_pandas(tqdm, ...)`" in our_file.getvalue() 220 | -------------------------------------------------------------------------------- /tests/tests_main.py: -------------------------------------------------------------------------------- 1 | """Test CLI usage.""" 2 | import logging 3 | import subprocess # nosec 4 | import sys 5 | from functools import wraps 6 | from os import linesep 7 | 8 | from tqdm.cli import TqdmKeyError, TqdmTypeError, main 9 | from tqdm.utils import IS_WIN 10 | 11 | from .tests_tqdm import BytesIO, _range, closing, mark, raises 12 | 13 | 14 | def restore_sys(func): 15 | """Decorates `func(capsysbin)` to save & restore `sys.(stdin|argv)`.""" 16 | @wraps(func) 17 | def inner(capsysbin): 18 | """function requiring capsysbin which may alter `sys.(stdin|argv)`""" 19 | _SYS = sys.stdin, sys.argv 20 | try: 21 | res = func(capsysbin) 22 | finally: 23 | sys.stdin, sys.argv = _SYS 24 | return res 25 | 26 | return inner 27 | 28 | 29 | def norm(bytestr): 30 | """Normalise line endings.""" 31 | return bytestr if linesep == "\n" else bytestr.replace(linesep.encode(), b"\n") 32 | 33 | 34 | @mark.slow 35 | def test_pipes(): 36 | """Test command line pipes""" 37 | ls_out = subprocess.check_output(['ls']) # nosec 38 | ls = subprocess.Popen(['ls'], stdout=subprocess.PIPE) # nosec 39 | res = subprocess.Popen( # nosec 40 | [sys.executable, '-c', 'from tqdm.cli import main; main()'], 41 | stdin=ls.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 42 | out, err = res.communicate() 43 | assert ls.poll() == 0 44 | 45 | # actual test: 46 | assert norm(ls_out) == norm(out) 47 | assert b"it/s" in err 48 | assert b"Error" not in err 49 | 50 | 51 | if sys.version_info[:2] >= (3, 8): 52 | test_pipes = mark.filterwarnings("ignore:unclosed file:ResourceWarning")( 53 | test_pipes) 54 | 55 | 56 | def test_main_import(): 57 | """Test main CLI import""" 58 | N = 123 59 | _SYS = sys.stdin, sys.argv 60 | # test direct import 61 | sys.stdin = [str(i).encode() for i in _range(N)] 62 | sys.argv = ['', '--desc', 'Test CLI import', 63 | '--ascii', 'True', '--unit_scale', 'True'] 64 | try: 65 | import tqdm.__main__ # NOQA, pylint: disable=unused-variable 66 | finally: 67 | sys.stdin, sys.argv = _SYS 68 | 69 | 70 | @restore_sys 71 | def test_main_bytes(capsysbin): 72 | """Test CLI --bytes""" 73 | N = 123 74 | 75 | # test --delim 76 | IN_DATA = '\0'.join(map(str, _range(N))).encode() 77 | with closing(BytesIO()) as sys.stdin: 78 | sys.stdin.write(IN_DATA) 79 | # sys.stdin.write(b'\xff') # TODO 80 | sys.stdin.seek(0) 81 | main(sys.stderr, ['--desc', 'Test CLI delim', '--ascii', 'True', 82 | '--delim', r'\0', '--buf_size', '64']) 83 | out, err = capsysbin.readouterr() 84 | assert out == IN_DATA 85 | assert str(N) + "it" in err.decode("U8") 86 | 87 | # test --bytes 88 | IN_DATA = IN_DATA.replace(b'\0', b'\n') 89 | with closing(BytesIO()) as sys.stdin: 90 | sys.stdin.write(IN_DATA) 91 | sys.stdin.seek(0) 92 | main(sys.stderr, ['--ascii', '--bytes=True', '--unit_scale', 'False']) 93 | out, err = capsysbin.readouterr() 94 | assert out == IN_DATA 95 | assert str(len(IN_DATA)) + "B" in err.decode("U8") 96 | 97 | 98 | @mark.skipif(sys.version_info[0] == 2, reason="no caplog on py2") 99 | def test_main_log(capsysbin, caplog): 100 | """Test CLI --log""" 101 | _SYS = sys.stdin, sys.argv 102 | N = 123 103 | sys.stdin = [(str(i) + '\n').encode() for i in _range(N)] 104 | IN_DATA = b''.join(sys.stdin) 105 | try: 106 | with caplog.at_level(logging.INFO): 107 | main(sys.stderr, ['--log', 'INFO']) 108 | out, err = capsysbin.readouterr() 109 | assert norm(out) == IN_DATA and b"123/123" in err 110 | assert not caplog.record_tuples 111 | with caplog.at_level(logging.DEBUG): 112 | main(sys.stderr, ['--log', 'DEBUG']) 113 | out, err = capsysbin.readouterr() 114 | assert norm(out) == IN_DATA and b"123/123" in err 115 | assert caplog.record_tuples 116 | finally: 117 | sys.stdin, sys.argv = _SYS 118 | 119 | 120 | @restore_sys 121 | def test_main(capsysbin): 122 | """Test misc CLI options""" 123 | N = 123 124 | sys.stdin = [(str(i) + '\n').encode() for i in _range(N)] 125 | IN_DATA = b''.join(sys.stdin) 126 | 127 | # test --tee 128 | main(sys.stderr, ['--mininterval', '0', '--miniters', '1']) 129 | out, err = capsysbin.readouterr() 130 | assert norm(out) == IN_DATA and b"123/123" in err 131 | assert N <= len(err.split(b"\r")) < N + 5 132 | 133 | len_err = len(err) 134 | main(sys.stderr, ['--tee', '--mininterval', '0', '--miniters', '1']) 135 | out, err = capsysbin.readouterr() 136 | assert norm(out) == IN_DATA and b"123/123" in err 137 | # spaces to clear intermediate lines could increase length 138 | assert len_err + len(norm(out)) <= len(err) 139 | 140 | # test --null 141 | main(sys.stderr, ['--null']) 142 | out, err = capsysbin.readouterr() 143 | assert not out and b"123/123" in err 144 | 145 | # test integer --update 146 | main(sys.stderr, ['--update']) 147 | out, err = capsysbin.readouterr() 148 | assert norm(out) == IN_DATA 149 | assert (str(N // 2 * N) + "it").encode() in err, "expected arithmetic sum formula" 150 | 151 | # test integer --update_to 152 | main(sys.stderr, ['--update-to']) 153 | out, err = capsysbin.readouterr() 154 | assert norm(out) == IN_DATA 155 | assert (str(N - 1) + "it").encode() in err 156 | assert (str(N) + "it").encode() not in err 157 | 158 | with closing(BytesIO()) as sys.stdin: 159 | sys.stdin.write(IN_DATA.replace(b'\n', b'D')) 160 | 161 | # test integer --update --delim 162 | sys.stdin.seek(0) 163 | main(sys.stderr, ['--update', '--delim', 'D']) 164 | out, err = capsysbin.readouterr() 165 | assert out == IN_DATA.replace(b'\n', b'D') 166 | assert (str(N // 2 * N) + "it").encode() in err, "expected arithmetic sum" 167 | 168 | # test integer --update_to --delim 169 | sys.stdin.seek(0) 170 | main(sys.stderr, ['--update-to', '--delim', 'D']) 171 | out, err = capsysbin.readouterr() 172 | assert out == IN_DATA.replace(b'\n', b'D') 173 | assert (str(N - 1) + "it").encode() in err 174 | assert (str(N) + "it").encode() not in err 175 | 176 | # test float --update_to 177 | sys.stdin = [(str(i / 2.0) + '\n').encode() for i in _range(N)] 178 | IN_DATA = b''.join(sys.stdin) 179 | main(sys.stderr, ['--update-to']) 180 | out, err = capsysbin.readouterr() 181 | assert norm(out) == IN_DATA 182 | assert (str((N - 1) / 2.0) + "it").encode() in err 183 | assert (str(N / 2.0) + "it").encode() not in err 184 | 185 | 186 | @mark.slow 187 | @mark.skipif(IS_WIN, reason="no manpages on windows") 188 | def test_manpath(tmp_path): 189 | """Test CLI --manpath""" 190 | man = tmp_path / "tqdm.1" 191 | assert not man.exists() 192 | with raises(SystemExit): 193 | main(argv=['--manpath', str(tmp_path)]) 194 | assert man.is_file() 195 | 196 | 197 | @mark.slow 198 | @mark.skipif(IS_WIN, reason="no completion on windows") 199 | def test_comppath(tmp_path): 200 | """Test CLI --comppath""" 201 | man = tmp_path / "tqdm_completion.sh" 202 | assert not man.exists() 203 | with raises(SystemExit): 204 | main(argv=['--comppath', str(tmp_path)]) 205 | assert man.is_file() 206 | 207 | # check most important options appear 208 | script = man.read_text() 209 | opts = {'--help', '--desc', '--total', '--leave', '--ncols', '--ascii', 210 | '--dynamic_ncols', '--position', '--bytes', '--nrows', '--delim', 211 | '--manpath', '--comppath'} 212 | assert all(args in script for args in opts) 213 | 214 | 215 | @restore_sys 216 | def test_exceptions(capsysbin): 217 | """Test CLI Exceptions""" 218 | N = 123 219 | sys.stdin = [str(i) + '\n' for i in _range(N)] 220 | IN_DATA = ''.join(sys.stdin).encode() 221 | 222 | with raises(TqdmKeyError, match="bad_arg_u_ment"): 223 | main(sys.stderr, argv=['-ascii', '-unit_scale', '--bad_arg_u_ment', 'foo']) 224 | out, _ = capsysbin.readouterr() 225 | assert norm(out) == IN_DATA 226 | 227 | with raises(TqdmTypeError, match="invalid_bool_value"): 228 | main(sys.stderr, argv=['-ascii', '-unit_scale', 'invalid_bool_value']) 229 | out, _ = capsysbin.readouterr() 230 | assert norm(out) == IN_DATA 231 | 232 | with raises(TqdmTypeError, match="invalid_int_value"): 233 | main(sys.stderr, argv=['-ascii', '--total', 'invalid_int_value']) 234 | out, _ = capsysbin.readouterr() 235 | assert norm(out) == IN_DATA 236 | 237 | with raises(TqdmKeyError, match="Can only have one of --"): 238 | main(sys.stderr, argv=['--update', '--update_to']) 239 | out, _ = capsysbin.readouterr() 240 | assert norm(out) == IN_DATA 241 | 242 | # test SystemExits 243 | for i in ('-h', '--help', '-v', '--version'): 244 | with raises(SystemExit): 245 | main(argv=[i]) 246 | -------------------------------------------------------------------------------- /tqdm/tqdm.1: -------------------------------------------------------------------------------- 1 | .\" Automatically generated by Pandoc 1.19.2 2 | .\" 3 | .TH "TQDM" "1" "2015\-2021" "tqdm User Manuals" "" 4 | .hy 5 | .SH NAME 6 | .PP 7 | tqdm \- fast, extensible progress bar for Python and CLI 8 | .SH SYNOPSIS 9 | .PP 10 | tqdm [\f[I]options\f[]] 11 | .SH DESCRIPTION 12 | .PP 13 | See . 14 | Can be used as a pipe: 15 | .IP 16 | .nf 17 | \f[C] 18 | $\ #\ count\ lines\ of\ code 19 | $\ cat\ *.py\ |\ tqdm\ |\ wc\ \-l 20 | 327it\ [00:00,\ 981773.38it/s] 21 | 327 22 | 23 | $\ #\ find\ all\ files 24 | $\ find\ .\ \-name\ "*.py"\ |\ tqdm\ |\ wc\ \-l 25 | 432it\ [00:00,\ 833842.30it/s] 26 | 432 27 | 28 | #\ ...\ and\ more\ info 29 | $\ find\ .\ \-name\ \[aq]*.py\[aq]\ \-exec\ wc\ \-l\ \\{}\ \\;\ \\ 30 | \ \ |\ tqdm\ \-\-total\ 432\ \-\-unit\ files\ \-\-desc\ counting\ \\ 31 | \ \ |\ awk\ \[aq]{\ sum\ +=\ $1\ };\ END\ {\ print\ sum\ }\[aq] 32 | counting:\ 100%|█████████|\ 432/432\ [00:00<00:00,\ 794361.83files/s] 33 | 131998 34 | \f[] 35 | .fi 36 | .SH OPTIONS 37 | .TP 38 | .B \-h, \-\-help 39 | Print this help and exit. 40 | .RS 41 | .RE 42 | .TP 43 | .B \-v, \-\-version 44 | Print version and exit. 45 | .RS 46 | .RE 47 | .TP 48 | .B \-\-desc=\f[I]desc\f[] 49 | str, optional. 50 | Prefix for the progressbar. 51 | .RS 52 | .RE 53 | .TP 54 | .B \-\-total=\f[I]total\f[] 55 | int or float, optional. 56 | The number of expected iterations. 57 | If unspecified, len(iterable) is used if possible. 58 | If float("inf") or as a last resort, only basic progress statistics are 59 | displayed (no ETA, no progressbar). 60 | If \f[C]gui\f[] is True and this parameter needs subsequent updating, 61 | specify an initial arbitrary large positive number, e.g. 62 | 9e9. 63 | .RS 64 | .RE 65 | .TP 66 | .B \-\-leave 67 | bool, optional. 68 | If [default: True], keeps all traces of the progressbar upon termination 69 | of iteration. 70 | If \f[C]None\f[], will leave only if \f[C]position\f[] is \f[C]0\f[]. 71 | .RS 72 | .RE 73 | .TP 74 | .B \-\-ncols=\f[I]ncols\f[] 75 | int, optional. 76 | The width of the entire output message. 77 | If specified, dynamically resizes the progressbar to stay within this 78 | bound. 79 | If unspecified, attempts to use environment width. 80 | The fallback is a meter width of 10 and no limit for the counter and 81 | statistics. 82 | If 0, will not print any meter (only stats). 83 | .RS 84 | .RE 85 | .TP 86 | .B \-\-mininterval=\f[I]mininterval\f[] 87 | float, optional. 88 | Minimum progress display update interval [default: 0.1] seconds. 89 | .RS 90 | .RE 91 | .TP 92 | .B \-\-maxinterval=\f[I]maxinterval\f[] 93 | float, optional. 94 | Maximum progress display update interval [default: 10] seconds. 95 | Automatically adjusts \f[C]miniters\f[] to correspond to 96 | \f[C]mininterval\f[] after long display update lag. 97 | Only works if \f[C]dynamic_miniters\f[] or monitor thread is enabled. 98 | .RS 99 | .RE 100 | .TP 101 | .B \-\-miniters=\f[I]miniters\f[] 102 | int or float, optional. 103 | Minimum progress display update interval, in iterations. 104 | If 0 and \f[C]dynamic_miniters\f[], will automatically adjust to equal 105 | \f[C]mininterval\f[] (more CPU efficient, good for tight loops). 106 | If > 0, will skip display of specified number of iterations. 107 | Tweak this and \f[C]mininterval\f[] to get very efficient loops. 108 | If your progress is erratic with both fast and slow iterations (network, 109 | skipping items, etc) you should set miniters=1. 110 | .RS 111 | .RE 112 | .TP 113 | .B \-\-ascii=\f[I]ascii\f[] 114 | bool or str, optional. 115 | If unspecified or False, use unicode (smooth blocks) to fill the meter. 116 | The fallback is to use ASCII characters " 123456789#". 117 | .RS 118 | .RE 119 | .TP 120 | .B \-\-disable 121 | bool, optional. 122 | Whether to disable the entire progressbar wrapper [default: False]. 123 | If set to None, disable on non\-TTY. 124 | .RS 125 | .RE 126 | .TP 127 | .B \-\-unit=\f[I]unit\f[] 128 | str, optional. 129 | String that will be used to define the unit of each iteration [default: 130 | it]. 131 | .RS 132 | .RE 133 | .TP 134 | .B \-\-unit\-scale=\f[I]unit_scale\f[] 135 | bool or int or float, optional. 136 | If 1 or True, the number of iterations will be reduced/scaled 137 | automatically and a metric prefix following the International System of 138 | Units standard will be added (kilo, mega, etc.) [default: False]. 139 | If any other non\-zero number, will scale \f[C]total\f[] and \f[C]n\f[]. 140 | .RS 141 | .RE 142 | .TP 143 | .B \-\-dynamic\-ncols 144 | bool, optional. 145 | If set, constantly alters \f[C]ncols\f[] and \f[C]nrows\f[] to the 146 | environment (allowing for window resizes) [default: False]. 147 | .RS 148 | .RE 149 | .TP 150 | .B \-\-smoothing=\f[I]smoothing\f[] 151 | float, optional. 152 | Exponential moving average smoothing factor for speed estimates (ignored 153 | in GUI mode). 154 | Ranges from 0 (average speed) to 1 (current/instantaneous speed) 155 | [default: 0.3]. 156 | .RS 157 | .RE 158 | .TP 159 | .B \-\-bar\-format=\f[I]bar_format\f[] 160 | str, optional. 161 | Specify a custom bar string formatting. 162 | May impact performance. 163 | [default: \[aq]{l_bar}{bar}{r_bar}\[aq]], where l_bar=\[aq]{desc}: 164 | {percentage:3.0f}%|\[aq] and r_bar=\[aq]| {n_fmt}/{total_fmt} 165 | [{elapsed}<{remaining}, \[aq] \[aq]{rate_fmt}{postfix}]\[aq] Possible 166 | vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, percentage, 167 | elapsed, elapsed_s, ncols, nrows, desc, unit, rate, rate_fmt, 168 | rate_noinv, rate_noinv_fmt, rate_inv, rate_inv_fmt, postfix, 169 | unit_divisor, remaining, remaining_s, eta. 170 | Note that a trailing ": " is automatically removed after {desc} if the 171 | latter is empty. 172 | .RS 173 | .RE 174 | .TP 175 | .B \-\-initial=\f[I]initial\f[] 176 | int or float, optional. 177 | The initial counter value. 178 | Useful when restarting a progress bar [default: 0]. 179 | If using float, consider specifying \f[C]{n:.3f}\f[] or similar in 180 | \f[C]bar_format\f[], or specifying \f[C]unit_scale\f[]. 181 | .RS 182 | .RE 183 | .TP 184 | .B \-\-position=\f[I]position\f[] 185 | int, optional. 186 | Specify the line offset to print this bar (starting from 0) Automatic if 187 | unspecified. 188 | Useful to manage multiple bars at once (eg, from threads). 189 | .RS 190 | .RE 191 | .TP 192 | .B \-\-postfix=\f[I]postfix\f[] 193 | dict or *, optional. 194 | Specify additional stats to display at the end of the bar. 195 | Calls \f[C]set_postfix(**postfix)\f[] if possible (dict). 196 | .RS 197 | .RE 198 | .TP 199 | .B \-\-unit\-divisor=\f[I]unit_divisor\f[] 200 | float, optional. 201 | [default: 1000], ignored unless \f[C]unit_scale\f[] is True. 202 | .RS 203 | .RE 204 | .TP 205 | .B \-\-write\-bytes 206 | bool, optional. 207 | If (default: None) and \f[C]file\f[] is unspecified, bytes will be 208 | written in Python 2. 209 | If \f[C]True\f[] will also write bytes. 210 | In all other cases will default to unicode. 211 | .RS 212 | .RE 213 | .TP 214 | .B \-\-lock\-args=\f[I]lock_args\f[] 215 | tuple, optional. 216 | Passed to \f[C]refresh\f[] for intermediate output (initialisation, 217 | iterating, and updating). 218 | .RS 219 | .RE 220 | .TP 221 | .B \-\-nrows=\f[I]nrows\f[] 222 | int, optional. 223 | The screen height. 224 | If specified, hides nested bars outside this bound. 225 | If unspecified, attempts to use environment height. 226 | The fallback is 20. 227 | .RS 228 | .RE 229 | .TP 230 | .B \-\-colour=\f[I]colour\f[] 231 | str, optional. 232 | Bar colour (e.g. 233 | \[aq]green\[aq], \[aq]#00ff00\[aq]). 234 | .RS 235 | .RE 236 | .TP 237 | .B \-\-delay=\f[I]delay\f[] 238 | float, optional. 239 | Don\[aq]t display until [default: 0] seconds have elapsed. 240 | .RS 241 | .RE 242 | .TP 243 | .B \-\-delim=\f[I]delim\f[] 244 | chr, optional. 245 | Delimiting character [default: \[aq]\\n\[aq]]. 246 | Use \[aq]\\0\[aq] for null. 247 | N.B.: on Windows systems, Python converts \[aq]\\n\[aq] to 248 | \[aq]\\r\\n\[aq]. 249 | .RS 250 | .RE 251 | .TP 252 | .B \-\-buf\-size=\f[I]buf_size\f[] 253 | int, optional. 254 | String buffer size in bytes [default: 256] used when \f[C]delim\f[] is 255 | specified. 256 | .RS 257 | .RE 258 | .TP 259 | .B \-\-bytes 260 | bool, optional. 261 | If true, will count bytes, ignore \f[C]delim\f[], and default 262 | \f[C]unit_scale\f[] to True, \f[C]unit_divisor\f[] to 1024, and 263 | \f[C]unit\f[] to \[aq]B\[aq]. 264 | .RS 265 | .RE 266 | .TP 267 | .B \-\-tee 268 | bool, optional. 269 | If true, passes \f[C]stdin\f[] to both \f[C]stderr\f[] and 270 | \f[C]stdout\f[]. 271 | .RS 272 | .RE 273 | .TP 274 | .B \-\-update 275 | bool, optional. 276 | If true, will treat input as newly elapsed iterations, i.e. 277 | numbers to pass to \f[C]update()\f[]. 278 | Note that this is slow (~2e5 it/s) since every input must be decoded as 279 | a number. 280 | .RS 281 | .RE 282 | .TP 283 | .B \-\-update\-to 284 | bool, optional. 285 | If true, will treat input as total elapsed iterations, i.e. 286 | numbers to assign to \f[C]self.n\f[]. 287 | Note that this is slow (~2e5 it/s) since every input must be decoded as 288 | a number. 289 | .RS 290 | .RE 291 | .TP 292 | .B \-\-null 293 | bool, optional. 294 | If true, will discard input (no stdout). 295 | .RS 296 | .RE 297 | .TP 298 | .B \-\-manpath=\f[I]manpath\f[] 299 | str, optional. 300 | Directory in which to install tqdm man pages. 301 | .RS 302 | .RE 303 | .TP 304 | .B \-\-comppath=\f[I]comppath\f[] 305 | str, optional. 306 | Directory in which to place tqdm completion. 307 | .RS 308 | .RE 309 | .TP 310 | .B \-\-log=\f[I]log\f[] 311 | str, optional. 312 | CRITICAL|FATAL|ERROR|WARN(ING)|[default: \[aq]INFO\[aq]]|DEBUG|NOTSET. 313 | .RS 314 | .RE 315 | .SH AUTHORS 316 | tqdm developers . 317 | -------------------------------------------------------------------------------- /tests/tests_perf.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | import sys 4 | from contextlib import contextmanager 5 | from functools import wraps 6 | from time import sleep, time 7 | 8 | # Use relative/cpu timer to have reliable timings when there is a sudden load 9 | try: 10 | from time import process_time 11 | except ImportError: 12 | from time import clock 13 | process_time = clock 14 | 15 | from tqdm import tqdm, trange 16 | 17 | from .tests_tqdm import _range, importorskip, mark, patch_lock, skip 18 | 19 | pytestmark = mark.slow 20 | 21 | 22 | def cpu_sleep(t): 23 | """Sleep the given amount of cpu time""" 24 | start = process_time() 25 | while (process_time() - start) < t: 26 | pass 27 | 28 | 29 | def checkCpuTime(sleeptime=0.2): 30 | """Check if cpu time works correctly""" 31 | if checkCpuTime.passed: 32 | return True 33 | # First test that sleeping does not consume cputime 34 | start1 = process_time() 35 | sleep(sleeptime) 36 | t1 = process_time() - start1 37 | 38 | # secondly check by comparing to cpusleep (where we actually do something) 39 | start2 = process_time() 40 | cpu_sleep(sleeptime) 41 | t2 = process_time() - start2 42 | 43 | if abs(t1) < 0.0001 and t1 < t2 / 10: 44 | checkCpuTime.passed = True 45 | return True 46 | skip("cpu time not reliable on this machine") 47 | 48 | 49 | checkCpuTime.passed = False 50 | 51 | 52 | @contextmanager 53 | def relative_timer(): 54 | """yields a context timer function which stops ticking on exit""" 55 | start = process_time() 56 | 57 | def elapser(): 58 | return process_time() - start 59 | 60 | yield lambda: elapser() 61 | spent = elapser() 62 | 63 | def elapser(): # NOQA 64 | return spent 65 | 66 | 67 | def retry_on_except(n=3, check_cpu_time=True): 68 | """decroator for retrying `n` times before raising Exceptions""" 69 | def wrapper(func): 70 | """actual decorator""" 71 | @wraps(func) 72 | def test_inner(*args, **kwargs): 73 | """may skip if `check_cpu_time` fails""" 74 | for i in range(1, n + 1): 75 | try: 76 | if check_cpu_time: 77 | checkCpuTime() 78 | func(*args, **kwargs) 79 | except Exception: 80 | if i >= n: 81 | raise 82 | else: 83 | return 84 | return test_inner 85 | return wrapper 86 | 87 | 88 | def simple_progress(iterable=None, total=None, file=sys.stdout, desc='', 89 | leave=False, miniters=1, mininterval=0.1, width=60): 90 | """Simple progress bar reproducing tqdm's major features""" 91 | n = [0] # use a closure 92 | start_t = [time()] 93 | last_n = [0] 94 | last_t = [0] 95 | if iterable is not None: 96 | total = len(iterable) 97 | 98 | def format_interval(t): 99 | mins, s = divmod(int(t), 60) 100 | h, m = divmod(mins, 60) 101 | if h: 102 | return '{0:d}:{1:02d}:{2:02d}'.format(h, m, s) 103 | else: 104 | return '{0:02d}:{1:02d}'.format(m, s) 105 | 106 | def update_and_print(i=1): 107 | n[0] += i 108 | if (n[0] - last_n[0]) >= miniters: 109 | last_n[0] = n[0] 110 | 111 | if (time() - last_t[0]) >= mininterval: 112 | last_t[0] = time() # last_t[0] == current time 113 | 114 | spent = last_t[0] - start_t[0] 115 | spent_fmt = format_interval(spent) 116 | rate = n[0] / spent if spent > 0 else 0 117 | rate_fmt = "%.2fs/it" % (1.0 / rate) if 0.0 < rate < 1.0 else "%.2fit/s" % rate 118 | 119 | frac = n[0] / total 120 | percentage = int(frac * 100) 121 | eta = (total - n[0]) / rate if rate > 0 else 0 122 | eta_fmt = format_interval(eta) 123 | 124 | # full_bar = "#" * int(frac * width) 125 | barfill = " " * int((1.0 - frac) * width) 126 | bar_length, frac_bar_length = divmod(int(frac * width * 10), 10) 127 | full_bar = '#' * bar_length 128 | frac_bar = chr(48 + frac_bar_length) if frac_bar_length else ' ' 129 | 130 | file.write("\r%s %i%%|%s%s%s| %i/%i [%s<%s, %s]" % 131 | (desc, percentage, full_bar, frac_bar, barfill, n[0], 132 | total, spent_fmt, eta_fmt, rate_fmt)) 133 | 134 | if n[0] == total and leave: 135 | file.write("\n") 136 | file.flush() 137 | 138 | def update_and_yield(): 139 | for elt in iterable: 140 | yield elt 141 | update_and_print() 142 | 143 | update_and_print(0) 144 | if iterable is not None: 145 | return update_and_yield() 146 | else: 147 | return update_and_print 148 | 149 | 150 | def assert_performance(thresh, name_left, time_left, name_right, time_right): 151 | """raises if time_left > thresh * time_right""" 152 | if time_left > thresh * time_right: 153 | raise ValueError( 154 | ('{name[0]}: {time[0]:f}, ' 155 | '{name[1]}: {time[1]:f}, ' 156 | 'ratio {ratio:f} > {thresh:f}').format( 157 | name=(name_left, name_right), 158 | time=(time_left, time_right), 159 | ratio=time_left / time_right, thresh=thresh)) 160 | 161 | 162 | @retry_on_except() 163 | def test_iter_basic_overhead(): 164 | """Test overhead of iteration based tqdm""" 165 | total = int(1e6) 166 | 167 | a = 0 168 | with trange(total) as t: 169 | with relative_timer() as time_tqdm: 170 | for i in t: 171 | a += i 172 | assert a == (total ** 2 - total) / 2.0 173 | 174 | a = 0 175 | with relative_timer() as time_bench: 176 | for i in _range(total): 177 | a += i 178 | sys.stdout.write(str(a)) 179 | 180 | assert_performance(3, 'trange', time_tqdm(), 'range', time_bench()) 181 | 182 | 183 | @retry_on_except() 184 | def test_manual_basic_overhead(): 185 | """Test overhead of manual tqdm""" 186 | total = int(1e6) 187 | 188 | with tqdm(total=total * 10, leave=True) as t: 189 | a = 0 190 | with relative_timer() as time_tqdm: 191 | for i in _range(total): 192 | a += i 193 | t.update(10) 194 | 195 | a = 0 196 | with relative_timer() as time_bench: 197 | for i in _range(total): 198 | a += i 199 | sys.stdout.write(str(a)) 200 | 201 | assert_performance(5, 'tqdm', time_tqdm(), 'range', time_bench()) 202 | 203 | 204 | def worker(total, blocking=True): 205 | def incr_bar(x): 206 | for _ in trange(total, lock_args=None if blocking else (False,), 207 | miniters=1, mininterval=0, maxinterval=0): 208 | pass 209 | return x + 1 210 | return incr_bar 211 | 212 | 213 | @retry_on_except() 214 | @patch_lock(thread=True) 215 | def test_lock_args(): 216 | """Test overhead of nonblocking threads""" 217 | ThreadPoolExecutor = importorskip('concurrent.futures').ThreadPoolExecutor 218 | 219 | total = 16 220 | subtotal = 10000 221 | 222 | with ThreadPoolExecutor() as pool: 223 | sys.stderr.write('block ... ') 224 | sys.stderr.flush() 225 | with relative_timer() as time_tqdm: 226 | res = list(pool.map(worker(subtotal, True), range(total))) 227 | assert sum(res) == sum(range(total)) + total 228 | sys.stderr.write('noblock ... ') 229 | sys.stderr.flush() 230 | with relative_timer() as time_noblock: 231 | res = list(pool.map(worker(subtotal, False), range(total))) 232 | assert sum(res) == sum(range(total)) + total 233 | 234 | assert_performance(0.5, 'noblock', time_noblock(), 'tqdm', time_tqdm()) 235 | 236 | 237 | @retry_on_except(10) 238 | def test_iter_overhead_hard(): 239 | """Test overhead of iteration based tqdm (hard)""" 240 | total = int(1e5) 241 | 242 | a = 0 243 | with trange(total, leave=True, miniters=1, 244 | mininterval=0, maxinterval=0) as t: 245 | with relative_timer() as time_tqdm: 246 | for i in t: 247 | a += i 248 | assert a == (total ** 2 - total) / 2.0 249 | 250 | a = 0 251 | with relative_timer() as time_bench: 252 | for i in _range(total): 253 | a += i 254 | sys.stdout.write(("%i" % a) * 40) 255 | 256 | assert_performance(130, 'trange', time_tqdm(), 'range', time_bench()) 257 | 258 | 259 | @retry_on_except(10) 260 | def test_manual_overhead_hard(): 261 | """Test overhead of manual tqdm (hard)""" 262 | total = int(1e5) 263 | 264 | with tqdm(total=total * 10, leave=True, miniters=1, 265 | mininterval=0, maxinterval=0) as t: 266 | a = 0 267 | with relative_timer() as time_tqdm: 268 | for i in _range(total): 269 | a += i 270 | t.update(10) 271 | 272 | a = 0 273 | with relative_timer() as time_bench: 274 | for i in _range(total): 275 | a += i 276 | sys.stdout.write(("%i" % a) * 40) 277 | 278 | assert_performance(130, 'tqdm', time_tqdm(), 'range', time_bench()) 279 | 280 | 281 | @retry_on_except(10) 282 | def test_iter_overhead_simplebar_hard(): 283 | """Test overhead of iteration based tqdm vs simple progress bar (hard)""" 284 | total = int(1e4) 285 | 286 | a = 0 287 | with trange(total, leave=True, miniters=1, 288 | mininterval=0, maxinterval=0) as t: 289 | with relative_timer() as time_tqdm: 290 | for i in t: 291 | a += i 292 | assert a == (total ** 2 - total) / 2.0 293 | 294 | a = 0 295 | s = simple_progress(_range(total), leave=True, 296 | miniters=1, mininterval=0) 297 | with relative_timer() as time_bench: 298 | for i in s: 299 | a += i 300 | 301 | assert_performance(10, 'trange', time_tqdm(), 'simple_progress', time_bench()) 302 | 303 | 304 | @retry_on_except(10) 305 | def test_manual_overhead_simplebar_hard(): 306 | """Test overhead of manual tqdm vs simple progress bar (hard)""" 307 | total = int(1e4) 308 | 309 | with tqdm(total=total * 10, leave=True, miniters=1, 310 | mininterval=0, maxinterval=0) as t: 311 | a = 0 312 | with relative_timer() as time_tqdm: 313 | for i in _range(total): 314 | a += i 315 | t.update(10) 316 | 317 | simplebar_update = simple_progress(total=total * 10, leave=True, 318 | miniters=1, mininterval=0) 319 | a = 0 320 | with relative_timer() as time_bench: 321 | for i in _range(total): 322 | a += i 323 | simplebar_update(10) 324 | 325 | assert_performance(10, 'tqdm', time_tqdm(), 'simple_progress', time_bench()) 326 | -------------------------------------------------------------------------------- /tqdm/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | General helpers required for `tqdm.std`. 3 | """ 4 | import os 5 | import re 6 | import sys 7 | from functools import wraps 8 | from warnings import warn 9 | from weakref import proxy 10 | 11 | # py2/3 compat 12 | try: 13 | _range = xrange 14 | except NameError: 15 | _range = range 16 | 17 | try: 18 | _unich = unichr 19 | except NameError: 20 | _unich = chr 21 | 22 | try: 23 | _unicode = unicode 24 | except NameError: 25 | _unicode = str 26 | 27 | try: 28 | _basestring = basestring 29 | except NameError: 30 | _basestring = str 31 | 32 | CUR_OS = sys.platform 33 | IS_WIN = any(CUR_OS.startswith(i) for i in ['win32', 'cygwin']) 34 | IS_NIX = any(CUR_OS.startswith(i) for i in ['aix', 'linux', 'darwin']) 35 | RE_ANSI = re.compile(r"\x1b\[[;\d]*[A-Za-z]") 36 | 37 | try: 38 | if IS_WIN: 39 | import colorama 40 | else: 41 | raise ImportError 42 | except ImportError: 43 | colorama = None 44 | else: 45 | try: 46 | colorama.init(strip=False) 47 | except TypeError: 48 | colorama.init() 49 | 50 | 51 | class FormatReplace(object): 52 | """ 53 | >>> a = FormatReplace('something') 54 | >>> "{:5d}".format(a) 55 | 'something' 56 | """ # NOQA: P102 57 | def __init__(self, replace=''): 58 | self.replace = replace 59 | self.format_called = 0 60 | 61 | def __format__(self, _): 62 | self.format_called += 1 63 | return self.replace 64 | 65 | 66 | class Comparable(object): 67 | """Assumes child has self._comparable attr/@property""" 68 | def __lt__(self, other): 69 | return self._comparable < other._comparable 70 | 71 | def __le__(self, other): 72 | return (self < other) or (self == other) 73 | 74 | def __eq__(self, other): 75 | return self._comparable == other._comparable 76 | 77 | def __ne__(self, other): 78 | return not self == other 79 | 80 | def __gt__(self, other): 81 | return not self <= other 82 | 83 | def __ge__(self, other): 84 | return not self < other 85 | 86 | 87 | class ObjectWrapper(object): 88 | def __getattr__(self, name): 89 | return getattr(self._wrapped, name) 90 | 91 | def __setattr__(self, name, value): 92 | return setattr(self._wrapped, name, value) 93 | 94 | def wrapper_getattr(self, name): 95 | """Actual `self.getattr` rather than self._wrapped.getattr""" 96 | try: 97 | return object.__getattr__(self, name) 98 | except AttributeError: # py2 99 | return getattr(self, name) 100 | 101 | def wrapper_setattr(self, name, value): 102 | """Actual `self.setattr` rather than self._wrapped.setattr""" 103 | return object.__setattr__(self, name, value) 104 | 105 | def __init__(self, wrapped): 106 | """ 107 | Thin wrapper around a given object 108 | """ 109 | self.wrapper_setattr('_wrapped', wrapped) 110 | 111 | 112 | class SimpleTextIOWrapper(ObjectWrapper): 113 | """ 114 | Change only `.write()` of the wrapped object by encoding the passed 115 | value and passing the result to the wrapped object's `.write()` method. 116 | """ 117 | # pylint: disable=too-few-public-methods 118 | def __init__(self, wrapped, encoding): 119 | super(SimpleTextIOWrapper, self).__init__(wrapped) 120 | self.wrapper_setattr('encoding', encoding) 121 | 122 | def write(self, s): 123 | """ 124 | Encode `s` and pass to the wrapped object's `.write()` method. 125 | """ 126 | return self._wrapped.write(s.encode(self.wrapper_getattr('encoding'))) 127 | 128 | def __eq__(self, other): 129 | return self._wrapped == getattr(other, '_wrapped', other) 130 | 131 | 132 | class DisableOnWriteError(ObjectWrapper): 133 | """ 134 | Disable the given `tqdm_instance` upon `write()` or `flush()` errors. 135 | """ 136 | @staticmethod 137 | def disable_on_exception(tqdm_instance, func): 138 | """ 139 | Quietly set `tqdm_instance.miniters=inf` if `func` raises `errno=5`. 140 | """ 141 | tqdm_instance = proxy(tqdm_instance) 142 | 143 | def inner(*args, **kwargs): 144 | try: 145 | return func(*args, **kwargs) 146 | except OSError as e: 147 | if e.errno != 5: 148 | raise 149 | try: 150 | tqdm_instance.miniters = float('inf') 151 | except ReferenceError: 152 | pass 153 | except ValueError as e: 154 | if 'closed' not in str(e): 155 | raise 156 | try: 157 | tqdm_instance.miniters = float('inf') 158 | except ReferenceError: 159 | pass 160 | return inner 161 | 162 | def __init__(self, wrapped, tqdm_instance): 163 | super(DisableOnWriteError, self).__init__(wrapped) 164 | if hasattr(wrapped, 'write'): 165 | self.wrapper_setattr( 166 | 'write', self.disable_on_exception(tqdm_instance, wrapped.write)) 167 | if hasattr(wrapped, 'flush'): 168 | self.wrapper_setattr( 169 | 'flush', self.disable_on_exception(tqdm_instance, wrapped.flush)) 170 | 171 | def __eq__(self, other): 172 | return self._wrapped == getattr(other, '_wrapped', other) 173 | 174 | 175 | class CallbackIOWrapper(ObjectWrapper): 176 | def __init__(self, callback, stream, method="read"): 177 | """ 178 | Wrap a given `file`-like object's `read()` or `write()` to report 179 | lengths to the given `callback` 180 | """ 181 | super(CallbackIOWrapper, self).__init__(stream) 182 | func = getattr(stream, method) 183 | if method == "write": 184 | @wraps(func) 185 | def write(data, *args, **kwargs): 186 | res = func(data, *args, **kwargs) 187 | callback(len(data)) 188 | return res 189 | self.wrapper_setattr('write', write) 190 | elif method == "read": 191 | @wraps(func) 192 | def read(*args, **kwargs): 193 | data = func(*args, **kwargs) 194 | callback(len(data)) 195 | return data 196 | self.wrapper_setattr('read', read) 197 | else: 198 | raise KeyError("Can only wrap read/write methods") 199 | 200 | 201 | def _is_utf(encoding): 202 | try: 203 | u'\u2588\u2589'.encode(encoding) 204 | except UnicodeEncodeError: 205 | return False 206 | except Exception: 207 | try: 208 | return encoding.lower().startswith('utf-') or ('U8' == encoding) 209 | except Exception: 210 | return False 211 | else: 212 | return True 213 | 214 | 215 | def _supports_unicode(fp): 216 | try: 217 | return _is_utf(fp.encoding) 218 | except AttributeError: 219 | return False 220 | 221 | 222 | def _is_ascii(s): 223 | if isinstance(s, str): 224 | for c in s: 225 | if ord(c) > 255: 226 | return False 227 | return True 228 | return _supports_unicode(s) 229 | 230 | 231 | def _screen_shape_wrapper(): # pragma: no cover 232 | """ 233 | Return a function which returns console dimensions (width, height). 234 | Supported: linux, osx, windows, cygwin. 235 | """ 236 | _screen_shape = None 237 | if IS_WIN: 238 | _screen_shape = _screen_shape_windows 239 | if _screen_shape is None: 240 | _screen_shape = _screen_shape_tput 241 | if IS_NIX: 242 | _screen_shape = _screen_shape_linux 243 | return _screen_shape 244 | 245 | 246 | def _screen_shape_windows(fp): # pragma: no cover 247 | try: 248 | import struct 249 | from ctypes import create_string_buffer, windll 250 | from sys import stdin, stdout 251 | 252 | io_handle = -12 # assume stderr 253 | if fp == stdin: 254 | io_handle = -10 255 | elif fp == stdout: 256 | io_handle = -11 257 | 258 | h = windll.kernel32.GetStdHandle(io_handle) 259 | csbi = create_string_buffer(22) 260 | res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) 261 | if res: 262 | (_bufx, _bufy, _curx, _cury, _wattr, left, top, right, bottom, 263 | _maxx, _maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw) 264 | return right - left, bottom - top # +1 265 | except Exception: # nosec 266 | pass 267 | return None, None 268 | 269 | 270 | def _screen_shape_tput(*_): # pragma: no cover 271 | """cygwin xterm (windows)""" 272 | try: 273 | import shlex 274 | from subprocess import check_call # nosec 275 | return [int(check_call(shlex.split('tput ' + i))) - 1 276 | for i in ('cols', 'lines')] 277 | except Exception: # nosec 278 | pass 279 | return None, None 280 | 281 | 282 | def _screen_shape_linux(fp): # pragma: no cover 283 | 284 | try: 285 | from array import array 286 | from fcntl import ioctl 287 | from termios import TIOCGWINSZ 288 | except ImportError: 289 | return None, None 290 | else: 291 | try: 292 | rows, cols = array('h', ioctl(fp, TIOCGWINSZ, '\0' * 8))[:2] 293 | return cols, rows 294 | except Exception: 295 | try: 296 | return [int(os.environ[i]) - 1 for i in ("COLUMNS", "LINES")] 297 | except (KeyError, ValueError): 298 | return None, None 299 | 300 | 301 | def _environ_cols_wrapper(): # pragma: no cover 302 | """ 303 | Return a function which returns console width. 304 | Supported: linux, osx, windows, cygwin. 305 | """ 306 | warn("Use `_screen_shape_wrapper()(file)[0]` instead of" 307 | " `_environ_cols_wrapper()(file)`", DeprecationWarning, stacklevel=2) 308 | shape = _screen_shape_wrapper() 309 | if not shape: 310 | return None 311 | 312 | @wraps(shape) 313 | def inner(fp): 314 | return shape(fp)[0] 315 | 316 | return inner 317 | 318 | 319 | def _term_move_up(): # pragma: no cover 320 | return '' if (os.name == 'nt') and (colorama is None) else '\x1b[A' 321 | 322 | 323 | try: 324 | # TODO consider using wcswidth third-party package for 0-width characters 325 | from unicodedata import east_asian_width 326 | except ImportError: 327 | _text_width = len 328 | else: 329 | def _text_width(s): 330 | return sum(2 if east_asian_width(ch) in 'FW' else 1 for ch in _unicode(s)) 331 | 332 | 333 | def disp_len(data): 334 | """ 335 | Returns the real on-screen length of a string which may contain 336 | ANSI control codes and wide chars. 337 | """ 338 | return _text_width(RE_ANSI.sub('', data)) 339 | 340 | 341 | def disp_trim(data, length): 342 | """ 343 | Trim a string which may contain ANSI control characters. 344 | """ 345 | if len(data) == disp_len(data): 346 | return data[:length] 347 | 348 | ansi_present = bool(RE_ANSI.search(data)) 349 | while disp_len(data) > length: # carefully delete one char at a time 350 | data = data[:-1] 351 | if ansi_present and bool(RE_ANSI.search(data)): 352 | # assume ANSI reset is required 353 | return data if data.endswith("\033[0m") else data + "\033[0m" 354 | return data 355 | --------------------------------------------------------------------------------