├── neurtu ├── tests │ ├── __init__.py │ ├── test_utils.py │ ├── test_delayed.py │ └── test_benchmark.py ├── __init__.py ├── utils.py ├── metrics.py ├── delayed.py └── base.py ├── .coveragerc ├── examples ├── README.txt ├── numpy_sort_benchmark.py └── logistic_regression_scaling.py ├── setup.cfg ├── .gitignore ├── doc ├── requirements.txt ├── api.rst ├── installation.rst ├── Makefile ├── make.bat ├── index.rst ├── release-notes.rst ├── sphinxext │ └── github_link.py ├── quickstart.rst └── conf.py ├── appveyor.yml ├── .codecov.yml ├── setup.py ├── LICENSE ├── .travis.yml └── README.rst /neurtu/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source=neurtu 3 | omit = *tests*, setup.py 4 | -------------------------------------------------------------------------------- /examples/README.txt: -------------------------------------------------------------------------------- 1 | .. _examples: 2 | 3 | Examples 4 | ======== 5 | 6 | The following examples illustrate neurtu usage 7 | -------------------------------------------------------------------------------- /neurtu/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Benchmark, timeit, memit # noqa 2 | from .delayed import delayed, Delayed # noqa 3 | 4 | __version__ = '0.3.0' 5 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [tool:pytest] 2 | addopts = 3 | --doctest-modules 4 | --ignore=doc/conf.py 5 | --ignore=doc/sphinxext/ 6 | --ignore=examples/ 7 | -rs 8 | [bdist_wheel] 9 | universal=1 10 | 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *pyc 3 | .pytest_cache/ 4 | .cache/ 5 | __pycache__/ 6 | dist/ 7 | build/ 8 | _build/ 9 | generated/ 10 | .coverage 11 | coverage.xml 12 | doc/examples/ 13 | doc/generated_gallery/ 14 | -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | scikit-learn>=0.18 2 | scipy 3 | numpy 4 | pandas 5 | sphinx 6 | pillow 7 | recommonmark 8 | sphinx_rtd_theme 9 | sphinxcontrib-napoleon 10 | sphinx-gallery 11 | pillow 12 | matplotlib 13 | -------------------------------------------------------------------------------- /doc/api.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | .. currentmodule:: neurtu 5 | 6 | 7 | .. autosummary:: 8 | :toctree: ./generated/ 9 | 10 | neurtu.timeit 11 | neurtu.memit 12 | neurtu.Benchmark 13 | neurtu.delayed 14 | 15 | -------------------------------------------------------------------------------- /neurtu/utils.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | # neurtu, BSD 3 clause license 4 | # Authors: Roman Yurchak 5 | 6 | 7 | def import_or_none(path): 8 | """Import a package, return None if it's not installed""" 9 | try: 10 | pkg = importlib.import_module(path) 11 | return pkg 12 | except ImportError: 13 | return None 14 | -------------------------------------------------------------------------------- /neurtu/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # neurtu, BSD 3 clause license 2 | # Authors: Roman Yurchak 3 | 4 | from neurtu.utils import import_or_none 5 | 6 | 7 | def test_import_or_none(): 8 | import os 9 | 10 | os_2 = import_or_none('os') 11 | assert id(os) == id(os_2) 12 | 13 | pkg = import_or_none('not_existing') 14 | assert pkg is None 15 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | build: false 2 | 3 | environment: 4 | 5 | matrix: 6 | - PYTHON_VERSION: "3.6.x" 7 | PYTHON_ARCH: "64" 8 | MINICONDA: C:\Miniconda36-x64 9 | REQUIREMENTS: "pytest numpy pandas" 10 | 11 | 12 | init: 13 | - "ECHO %PYTHON_VERSION% %MINICONDA%" 14 | 15 | install: 16 | - "set PATH=%MINICONDA%;%MINICONDA%\\Scripts;%PATH%" 17 | - conda config --set always_yes yes --set changeps1 no 18 | - conda install %REQUIREMENTS% 19 | - pip install codecov pytest-cov 20 | - pip install -e . 21 | 22 | 23 | test_script: 24 | - pytest -s --cov=neurtu neurtu/ 25 | 26 | on_success: 27 | - codecov 28 | -------------------------------------------------------------------------------- /doc/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | neurtu requires Python 2.7 or 3.4+, it can be installed with, 5 | 6 | .. code:: 7 | 8 | pip install neurtu 9 | 10 | `pandas `_ is an optional (but highly recommended) dependency. 11 | 12 | .. note:: 13 | 14 | the above command will install memory_profiler, shutil (to measure memory use) and tqdm (to make progress bars) mostly for 15 | convinience. However, neurtu does not have any hard depedencies, it you don't need these functionalites, you can install it 16 | with ``pip install --no-deps neurtu`` 17 | -------------------------------------------------------------------------------- /neurtu/tests/test_delayed.py: -------------------------------------------------------------------------------- 1 | # neurtu, BSD 3 clause license 2 | # Authors: Roman Yurchak 3 | import os 4 | 5 | from neurtu import delayed 6 | 7 | 8 | def test_set_env(): 9 | def func(): 10 | return os.environ.get('NEURTU_TEST', None) 11 | 12 | assert func() is None 13 | 14 | assert delayed(func, env={'NEURTU_TEST': 'true'})().compute() == 'true' 15 | 16 | assert func() is None 17 | 18 | 19 | def test_get_args_kwargs(): 20 | def func(pos_arg, key_arg=None): 21 | pass 22 | 23 | delayed_obj = delayed(func)('arg', key_arg='kwarg') 24 | assert delayed_obj.get_args()[0] == 'arg' 25 | assert delayed_obj.get_kwargs()[0] == {'key_arg': 'kwarg'} 26 | -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- 1 | comment: false 2 | 3 | coverage: 4 | status: 5 | project: 6 | default: 7 | # Commits pushed to master should not make the overall 8 | # project coverage decrease by more than 1%: 9 | target: auto 10 | threshold: 3% 11 | patch: 12 | default: 13 | # Be tolerant on slight code coverage diff on PRs to limit 14 | # noisy red coverage status on github PRs. 15 | # Note The coverage stats are still uploaded 16 | # to codecov so that PR reviewers can see uncovered lines 17 | # in the github diff if they install the codecov browser 18 | # extension: 19 | # https://github.com/codecov/browser-extension 20 | target: auto 21 | threshold: 3% 22 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python -msphinx 7 | SPHINXPROJ = neurtu 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | 18 | clean: 19 | rm -rf _build generated/ generated_gallery/ examples/ 20 | 21 | 22 | # Catch-all target: route all unknown targets to Sphinx using the new 23 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 24 | %: Makefile 25 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 26 | -------------------------------------------------------------------------------- /examples/numpy_sort_benchmark.py: -------------------------------------------------------------------------------- 1 | """ 2 | Time complexity of numpy.sort 3 | ============================= 4 | 5 | In this example we will look into the time complexity of :func:`numpy.sort` 6 | 7 | """ 8 | 9 | import numpy as np 10 | from neurtu import timeit, delayed 11 | 12 | 13 | rng = np.random.RandomState(42) 14 | 15 | 16 | df = timeit(delayed(np.sort, tags={'N': N, 'kind': kind})(rng.rand(N), kind=kind) 17 | for N in np.logspace(2, 5, num=5).astype('int') 18 | for kind in ["quicksort", "mergesort", "heapsort"]) 19 | 20 | print(df.to_string()) 21 | 22 | 23 | ############################################################################## 24 | # 25 | # we can use the pandas plotting API (that requires matplotlib) 26 | 27 | ax = df.wall_time.unstack().plot(marker='o') 28 | ax.set_xscale('log') 29 | ax.set_yscale('log') 30 | ax.set_ylabel('Wall time (s)') 31 | ax.set_title('Time complexity of numpy.sort') 32 | -------------------------------------------------------------------------------- /doc/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=python -msphinx 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=neurtu 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The Sphinx module was not found. Make sure you have Sphinx installed, 20 | echo.then set the SPHINXBUILD environment variable to point to the full 21 | echo.path of the 'sphinx-build' executable. Alternatively you may add the 22 | echo.Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. neurtu documentation master file, created by 2 | sphinx-quickstart on Mon Feb 19 17:23:39 2018. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | neurtu 7 | ====== 8 | 9 | Simple performance measurement tool 10 | 11 | 12 | 13 | neurtu is a Python package providing a common interface for multi-metric benchmarks 14 | (including time and memory measurements). It can can be used to estimate time 15 | and space complexity of algorithms, while pandas integration 16 | allows quick analysis and visualization of the results. 17 | 18 | Setting the number of threads at runtime in OpenBlas, and MKL is also supported on Linux 19 | and MacOS. 20 | 21 | *neurtu* means "to measure / evaluate" in Basque language. 22 | 23 | .. toctree:: 24 | :maxdepth: 2 25 | :caption: Contents: 26 | 27 | installation 28 | quickstart 29 | examples/index 30 | api 31 | release-notes.rst 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | from neurtu import __version__ 4 | 5 | 6 | setup( 7 | name='neurtu', 8 | description='A simple benchmarking tool', 9 | long_description=open('README.rst').read(), 10 | version=__version__, 11 | author='Roman Yurchak', 12 | author_email='roman.yurchak@symerio.com', 13 | packages=find_packages(), 14 | url='https://github.com/symerio/neurtu', 15 | install_requires=['memory_profiler', 'psutil', 'tqdm'], 16 | python_requires=">=3.5", 17 | classifiers=['Development Status :: 4 - Beta', 18 | 'Intended Audience :: Science/Research', 19 | 'Intended Audience :: Developers', 20 | 'License :: OSI Approved :: BSD License', 21 | 'Programming Language :: Python', 22 | 'Programming Language :: Python :: 3.5', 23 | 'Programming Language :: Python :: 3.6', 24 | 'Programming Language :: Python :: 3.7', 25 | 'Topic :: Software Development', 26 | 'Operating System :: POSIX', 27 | 'Operating System :: Unix'], 28 | license='BSD') 29 | -------------------------------------------------------------------------------- /doc/release-notes.rst: -------------------------------------------------------------------------------- 1 | Release notes 2 | ============= 3 | 4 | Version 0.3 5 | ----------- 6 | *July 21, 2019* 7 | 8 | API changes 9 | ^^^^^^^^^^^ 10 | 11 | - Functions to set the number of BLAS threads at runtime were removed 12 | in favour of using `threadpoolctl 13 | `_. 14 | 15 | Enhancements 16 | ^^^^^^^^^^^^ 17 | - Add ``get_args`` and ``get_kwargs`` to ``Delayed`` object. 18 | - Better progress bars in Jupyter notebooks with the ``tqdm.auto`` 19 | backend. 20 | 21 | Bug fixes 22 | ^^^^^^^^^ 23 | - Fix progress bar rendering when ``repeat>1``. 24 | - Fix warnings due to ``collection.abc``. 25 | 26 | Version 0.2 27 | ----------- 28 | *August 28, 2018* 29 | 30 | New features 31 | ^^^^^^^^^^^^ 32 | 33 | - Runtime detection of the BLAS used by numpy `#14 `_ 34 | - Ability to set the number of threads in OpenBlas and 35 | MKL BLAS at runtime on Linux. `#15 `_. 36 | 37 | Enhancements 38 | ^^^^^^^^^^^^ 39 | - Better test coverage 40 | - Documentation improvements 41 | - In depth refactoring of the benchmarking code 42 | 43 | API changes 44 | ^^^^^^^^^^^ 45 | - The API of ``timeit``, ``memit``, ``Benchmark`` changed significantly with respect to v0.1 46 | 47 | Version 0.1 48 | ----------- 49 | *March 4, 2018* 50 | 51 | First release, with support for, 52 | 53 | - wall time, cpu time and peak memory measurements 54 | - parametric benchmarks using delayed objects 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Roman Yurchak 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: linux 2 | language: python 3 | sudo: false 4 | 5 | matrix: 6 | include: 7 | - python: 3.6 8 | env: REQUIREMENTS="numpy==1.8.0 pandas==0.20.0rc1" RUN_COVERAGE=true BUILD_DOCS=false INSTALLER="pip" 9 | - python: 3.6 10 | env: REQUIREMENTS="flake8" RUN_FLAKE8=true RUN_COVERAGE=true INSTALLER="pip" 11 | - python: 3.7 12 | env: REQUIREMENTS="numpy pandas tqdm nomkl" RUN_COVERAGE=true BUILD_DOCS=true INSTALLER="conda" 13 | - language: generic 14 | os: osx 15 | python: 3.6 16 | env: PYTHON_VERSION="3.6" REQUIREMENTS="numpy pandas tqdm" RUN_COVERAGE=true INSTALLER="conda" 17 | 18 | install: 19 | - | 20 | if [[ "$INSTALLER" == "conda" ]]; then 21 | if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then 22 | curl -s -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh 23 | else 24 | curl -s -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh 25 | fi 26 | bash miniconda.sh -b -p $HOME/miniconda && rm miniconda.sh 27 | export PATH="$HOME/miniconda/bin:$PATH" 28 | hash -r 29 | conda update conda -y 30 | conda create -y -n neurtu-env ${REQUIREMENTS} pytest-cov pytest==3.7.2 python=${PYTHON_VERSION} 31 | source activate neurtu-env 32 | pip install codecov pytest-cov 33 | else 34 | pip install ${REQUIREMENTS} pytest-cov codecov pytest==3.7.2 35 | fi 36 | - pip install -e . 37 | 38 | script: 39 | - | 40 | if [[ "${RUN_FLAKE8}" == true ]]; then 41 | flake8 --exclude=neurtu/externals/ neurtu/ 42 | fi 43 | - | 44 | if [[ "$INSTALLER" == "conda" ]]; then 45 | source activate neurtu-env 46 | fi 47 | - pytest -s --doctest-modules --cov=neurtu neurtu/ 48 | - | 49 | if [[ "${BUILD_DOCS}" == "true" ]]; then 50 | cd doc/ 51 | pip install -r requirements.txt 52 | make html 53 | fi 54 | 55 | after_success: 56 | - | 57 | if [ ${RUN_COVERAGE} = true ]; then 58 | codecov 59 | fi 60 | 61 | cache: 62 | - directories: 63 | - $HOME/.cache/pip 64 | -------------------------------------------------------------------------------- /examples/logistic_regression_scaling.py: -------------------------------------------------------------------------------- 1 | """ 2 | LogisticRegression scaling in scikit-learn 3 | ========================================== 4 | 5 | In this example we will look into the time and space complexity of 6 | :class:`sklearn.linear_model.LogisticRegression` 7 | 8 | """ 9 | 10 | from collections import OrderedDict 11 | 12 | import numpy as np 13 | from sklearn.linear_model import LogisticRegression 14 | from neurtu import Benchmark, delayed 15 | 16 | 17 | rng = np.random.RandomState(42) 18 | 19 | n_samples, n_features = 50000, 100 20 | 21 | 22 | X = rng.rand(n_samples, n_features) 23 | y = rng.randint(2, size=(n_samples)) 24 | 25 | 26 | def benchmark_cases(): 27 | for N in np.logspace(np.log10(100), np.log10(n_samples), 5).astype('int'): 28 | for solver in ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']: 29 | tags = OrderedDict(N=N, solver=solver) 30 | model = delayed(LogisticRegression, tags=tags)( 31 | solver=solver, random_state=rng) 32 | 33 | yield model.fit(X[:N], y[:N]) 34 | 35 | 36 | bench = Benchmark(wall_time=True, peak_memory=True) 37 | df = bench(benchmark_cases()) 38 | 39 | print(df.tail()) 40 | 41 | 42 | ############################################################################## 43 | # 44 | # The above section will run in approximately 1min, a progress bar will be 45 | # displayed. 46 | # 47 | # We can use the pandas plotting API (that requires matplotlib) to visualize 48 | # the results, 49 | 50 | ax = df.wall_time.unstack().plot(marker='o') 51 | ax.set_xscale('log') 52 | ax.set_yscale('log') 53 | ax.set_ylabel('Wall time (s)') 54 | ax.set_title('Run time scaling for LogisticRegression.fit') 55 | 56 | 57 | ############################################################################## 58 | # 59 | # The solver with the best scalability in this example is "lbfgs". 60 | # 61 | # Similarly the memory scaling is represented below, 62 | 63 | ax = df.peak_memory.unstack().plot(marker='o') 64 | ax.set_xscale('log') 65 | ax.set_yscale('log') 66 | ax.set_ylabel('Peak memory (MB)') 67 | ax.set_title('Peak memory usage for LogisticRegression.fit') 68 | 69 | ############################################################################## 70 | # 71 | # Peak memory usage for "liblinear" and "newton-cg" appear to be significant 72 | # above 10000 samples, while the other solvers 73 | # use less memory than the detection threshold. 74 | # Note that these benchmarks do not account for the memory used by ``X`` and 75 | # ``y`` arrays. 76 | -------------------------------------------------------------------------------- /neurtu/metrics.py: -------------------------------------------------------------------------------- 1 | # neurtu, BSD 3 clause license 2 | # Authors: Roman Yurchak 3 | 4 | from __future__ import division 5 | 6 | import sys 7 | import itertools 8 | import time 9 | import timeit as cpython_timeit 10 | import gc 11 | 12 | 13 | # Timer class copied from ipython 14 | 15 | class Timer(cpython_timeit.Timer): 16 | """Timer class that explicitly uses self.inner 17 | 18 | which is an undocumented implementation detail of CPython, 19 | not shared by PyPy. 20 | """ 21 | # Timer.timeit copied from CPython 3.4.2 22 | def timeit(self, number=cpython_timeit.default_number): 23 | """Time 'number' executions of the main statement. 24 | To be precise, this executes the setup statement once, and 25 | then returns the time it takes to execute the main statement 26 | a number of times, as a float measured in seconds. The 27 | argument is the number of times through the loop, defaulting 28 | to one million. The main statement, the setup statement and 29 | the timer function to be used are passed to the constructor. 30 | """ 31 | it = itertools.repeat(None, number) 32 | gcold = gc.isenabled() 33 | gc.disable() 34 | try: 35 | timing = self.inner(it, self.timer) 36 | finally: 37 | if gcold: 38 | gc.enable() 39 | return timing 40 | 41 | 42 | def measure_wall_time(obj, number=1): 43 | if sys.version_info >= (3, 7): 44 | sys_timer = time.perf_counter_ns 45 | else: 46 | sys_timer = cpython_timeit.default_timer 47 | 48 | timer = Timer(obj.compute, timer=sys_timer) 49 | dt = timer.timeit(number) 50 | 51 | if sys.version_info >= (3, 7): 52 | dt = dt*1e-9 # ns to s 53 | return dt / number 54 | 55 | 56 | def measure_cpu_time(obj, number=1): 57 | try: 58 | import resource 59 | 60 | def timer(): 61 | return resource.getrusage(resource.RUSAGE_SELF).ru_utime 62 | except ImportError: # pragma: no cover 63 | raise ValueError('CPU timer is not available on Windows.') 64 | timer = Timer(obj.compute, timer=timer) 65 | dt = timer.timeit(number) 66 | return dt / number 67 | 68 | 69 | def measure_peak_memory(obj, **kwargs): 70 | from memory_profiler import memory_usage as _memory_usage_profiler 71 | usage = _memory_usage_profiler((obj.compute, (), {}), **kwargs) 72 | # subtract the initial memory usage of the process 73 | usage = [el - usage[0] for el in usage] 74 | return max(usage) 75 | -------------------------------------------------------------------------------- /doc/sphinxext/github_link.py: -------------------------------------------------------------------------------- 1 | # Adapted from scikit learn 2 | 3 | from operator import attrgetter 4 | import inspect 5 | import subprocess 6 | import os 7 | import sys 8 | from functools import partial 9 | 10 | REVISION_CMD = 'git rev-parse --short HEAD' 11 | 12 | 13 | def _get_git_revision(): 14 | try: 15 | revision = subprocess.check_output(REVISION_CMD.split()).strip() 16 | except (subprocess.CalledProcessError, OSError): 17 | print('Failed to execute git to get revision') 18 | return None 19 | return revision.decode('utf-8') 20 | 21 | 22 | def _linkcode_resolve(domain, info, package, url_fmt, revision): 23 | """Determine a link to online source for a class/method/function 24 | 25 | This is called by sphinx.ext.linkcode 26 | 27 | An example with a long-untouched module that everyone has 28 | >>> _linkcode_resolve('py', {'module': 'tty', 29 | ... 'fullname': 'setraw'}, 30 | ... package='tty', 31 | ... url_fmt='http://hg.python.org/cpython/file/' 32 | ... '{revision}/Lib/{package}/{path}#L{lineno}', 33 | ... revision='xxxx') 34 | 'http://hg.python.org/cpython/file/xxxx/Lib/tty/tty.py#L18' 35 | """ 36 | 37 | if revision is None: 38 | return 39 | if domain not in ('py', 'pyx'): 40 | return 41 | if not info.get('module') or not info.get('fullname'): 42 | return 43 | 44 | class_name = info['fullname'].split('.')[0] 45 | if type(class_name) != str: 46 | # Python 2 only 47 | class_name = class_name.encode('utf-8') 48 | module = __import__(info['module'], fromlist=[class_name]) 49 | obj = attrgetter(info['fullname'])(module) 50 | 51 | try: 52 | fn = inspect.getsourcefile(obj) 53 | except Exception: 54 | fn = None 55 | if not fn: 56 | try: 57 | fn = inspect.getsourcefile(sys.modules[obj.__module__]) 58 | except Exception: 59 | fn = None 60 | if not fn: 61 | return 62 | 63 | fn = os.path.relpath(fn, 64 | start=os.path.dirname(__import__(package).__file__)) 65 | try: 66 | lineno = inspect.getsourcelines(obj)[1] 67 | except Exception: 68 | lineno = '' 69 | return url_fmt.format(revision=revision, package=package, 70 | path=fn, lineno=lineno) 71 | 72 | 73 | def make_linkcode_resolve(package, url_fmt): 74 | """Returns a linkcode_resolve function for the given URL format 75 | 76 | revision is a git commit reference (hash or name) 77 | 78 | package is the name of the root module of the package 79 | 80 | url_fmt is along the lines of ('https://github.com/USER/PROJECT/' 81 | 'blob/{revision}/{package}/' 82 | '{path}#L{lineno}') 83 | """ 84 | revision = _get_git_revision() 85 | return partial(_linkcode_resolve, revision=revision, package=package, 86 | url_fmt=url_fmt) 87 | -------------------------------------------------------------------------------- /doc/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ========== 3 | 4 | To illustrate neurtu usage, will will benchmark array sorting in numpy. First, we will 5 | generator of cases, 6 | 7 | .. code:: python 8 | 9 | import numpy as np 10 | import neurtu 11 | 12 | def cases() 13 | rng = np.random.RandomState(42) 14 | 15 | for N in [1000, 10000, 100000]: 16 | X = rng.rand(N) 17 | tags = {'N' : N} 18 | yield neurtu.delayed(X, tags=tags).sort() 19 | 20 | that yields a sequence of delayed calculations, each tagged with the parameters defining individual runs. 21 | 22 | We can evaluate the run time with, 23 | 24 | .. code:: python 25 | 26 | >>> df = neurtu.timeit(cases()) 27 | >>> print(df) 28 | wall_time 29 | N 30 | 1000 0.000014 31 | 10000 0.000134 32 | 100000 0.001474 33 | 34 | which will internally use ``timeit`` module with a sufficient number of evaluation to work around the timer precision 35 | limitations (similarly to IPython's ``%timeit``). It will also display a progress bar for long running benchmarks, 36 | and return the results as a ``pandas.DataFrame`` (if pandas is installed). 37 | 38 | By default, all evaluations are run with ``repeat=1``. If more statistical confidence is required, this value can 39 | be increased, 40 | 41 | .. code:: python 42 | 43 | >>> neurtu.timeit(cases(), repeat=3) 44 | wall_time 45 | mean max std 46 | N 47 | 1000 0.000012 0.000014 0.000002 48 | 10000 0.000116 0.000149 0.000029 49 | 100000 0.001323 0.001714 0.000339 50 | 51 | In this case we will get a frame with a 52 | `pandas.MultiIndex `_ for 53 | columns, where the first level represents the metric name (``wall_time``) and the second -- the aggregation method. 54 | By default ``neurtu.timeit`` is called with ``aggregate=['mean', 'max', 'std']`` methods, as supported 55 | by the `pandas aggregation API `_. To disable, 56 | aggregation and obtains timings for individual runs, use ``aggregate=False``. 57 | See `neurtu.timeit documentation `_ for more details. 58 | 59 | To evaluate the peak memory usage, one can use the ``neurtu.memit`` function with the same API, 60 | 61 | .. code:: python 62 | 63 | >>> neurtu.memit(cases(), repeat=3) 64 | peak_memory 65 | mean max std 66 | N 67 | 10000 0.0 0.0 0.0 68 | 100000 0.0 0.0 0.0 69 | 1000000 0.0 0.0 0.0 70 | 71 | More generally ``neurtu.Benchmark`` supports a wide number of evaluation metrics, 72 | 73 | .. code:: python 74 | 75 | >>> bench = neurtu.Benchmark(wall_time=True, cpu_time=True, peak_memory=True) 76 | >>> bench(cases()) 77 | cpu_time peak_memory wall_time 78 | N 79 | 10000 0.000100 0.0 0.000142 80 | 100000 0.001149 0.0 0.001680 81 | 1000000 0.013677 0.0 0.018347 82 | 83 | including [psutil process metrics](https://psutil.readthedocs.io/en/latest/#psutil.Process). 84 | 85 | For more information see the :ref:`examples`. 86 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | neurtu 2 | ====== 3 | 4 | |pypi| |rdfd| 5 | 6 | |travis| |appveyor| |codecov| 7 | 8 | Simple performance measurement tool 9 | 10 | neurtu is a Python package providing a common interface for multi-metric benchmarks 11 | (including time and memory measurements). It can can be used to estimate time 12 | and space complexity of algorithms, while pandas integration 13 | allows quick analysis and visualization of the results. 14 | 15 | *neurtu* means "to measure / evaluate" in Basque language. 16 | 17 | See the `documentation `_ for more details. 18 | 19 | Installation 20 | ------------ 21 | 22 | neurtu requires 3.5+, it can be installed with, 23 | 24 | .. code:: 25 | 26 | pip install neurtu 27 | 28 | `pandas >=0.20 `_ is an optional (but highly recommended) dependency. 29 | 30 | 31 | Quickstart 32 | ---------- 33 | 34 | To illustrate neurtu usage, will will benchmark array sorting in numpy. First, we will 35 | generator of cases, 36 | 37 | .. code:: python 38 | 39 | import numpy as np 40 | import neurtu 41 | 42 | def cases() 43 | rng = np.random.RandomState(42) 44 | 45 | for N in [1000, 10000, 100000]: 46 | X = rng.rand(N) 47 | tags = {'N' : N} 48 | yield neurtu.delayed(X, tags=tags).sort() 49 | 50 | that yields a sequence of delayed calculations, each tagged with the parameters defining individual runs. 51 | 52 | We can evaluate the run time with, 53 | 54 | .. code:: python 55 | 56 | >>> df = neurtu.timeit(cases()) 57 | >>> print(df) 58 | wall_time 59 | N 60 | 1000 0.000014 61 | 10000 0.000134 62 | 100000 0.001474 63 | 64 | which will internally use ``timeit`` module with a sufficient number of evaluation to work around the timer precision 65 | limitations (similarly to IPython's ``%timeit``). It will also display a progress bar for long running benchmarks, 66 | and return the results as a ``pandas.DataFrame`` (if pandas is installed). 67 | 68 | By default, all evaluations are run with ``repeat=1``. If more statistical confidence is required, this value can 69 | be increased, 70 | 71 | .. code:: python 72 | 73 | >>> neurtu.timeit(cases(), repeat=3) 74 | wall_time 75 | mean max std 76 | N 77 | 1000 0.000012 0.000014 0.000002 78 | 10000 0.000116 0.000149 0.000029 79 | 100000 0.001323 0.001714 0.000339 80 | 81 | In this case we will get a frame with a 82 | `pandas.MultiIndex `_ for 83 | columns, where the first level represents the metric name (``wall_time``) and the second -- the aggregation method. 84 | By default ``neurtu.timeit`` is called with ``aggregate=['mean', 'max', 'std']`` methods, as supported 85 | by the `pandas aggregation API `_. To disable, 86 | aggregation and obtains timings for individual runs, use ``aggregate=False``. 87 | See `neurtu.timeit documentation `_ for more details. 88 | 89 | To evaluate the peak memory usage, one can use the ``neurtu.memit`` function with the same API, 90 | 91 | .. code:: python 92 | 93 | >>> neurtu.memit(cases(), repeat=3) 94 | peak_memory 95 | mean max std 96 | N 97 | 10000 0.0 0.0 0.0 98 | 100000 0.0 0.0 0.0 99 | 1000000 0.0 0.0 0.0 100 | 101 | More generally ``neurtu.Benchmark`` supports a wide number of evaluation metrics, 102 | 103 | .. code:: python 104 | 105 | >>> bench = neurtu.Benchmark(wall_time=True, cpu_time=True, peak_memory=True) 106 | >>> bench(cases()) 107 | cpu_time peak_memory wall_time 108 | N 109 | 10000 0.000100 0.0 0.000142 110 | 100000 0.001149 0.0 0.001680 111 | 1000000 0.013677 0.0 0.018347 112 | 113 | including `psutil process metrics `_. 114 | 115 | For more information see the `documentation `_ and 116 | `examples `_. 117 | 118 | License 119 | ------- 120 | 121 | neurtu is released under the 3-clause BSD license. 122 | 123 | 124 | .. |pypi| image:: https://img.shields.io/pypi/v/neurtu.svg 125 | :target: https://pypi.python.org/pypi/neurtu 126 | 127 | .. |rdfd| image:: https://readthedocs.org/projects/neurtu/badge/?version=latest 128 | :target: http://neurtu.readthedocs.io/ 129 | 130 | .. |travis| image:: https://travis-ci.org/symerio/neurtu.svg?branch=master 131 | :target: https://travis-ci.org/symerio/neurtu 132 | 133 | .. |appveyor| image:: https://ci.appveyor.com/api/projects/status/2i1dx8fi3bue4qwl?svg=true 134 | :target: https://ci.appveyor.com/project/rth/neurtu/branch/master 135 | 136 | .. |codecov| image:: https://codecov.io/gh/symerio/neurtu/branch/master/graph/badge.svg 137 | :target: https://codecov.io/gh/symerio/neurtu 138 | -------------------------------------------------------------------------------- /neurtu/tests/test_benchmark.py: -------------------------------------------------------------------------------- 1 | # neurtu, BSD 3 clause license 2 | # Authors: Roman Yurchak 3 | 4 | from __future__ import division 5 | 6 | import sys 7 | from time import sleep 8 | import pytest 9 | from pytest import approx 10 | 11 | from neurtu import timeit, memit, delayed, Benchmark 12 | from neurtu.utils import import_or_none 13 | 14 | # Timing tests 15 | 16 | 17 | def test_timeit_overhead(): 18 | 19 | dt = 0.2 20 | 21 | res = timeit(delayed(sleep)(dt)) 22 | 23 | # overhead should be less than 500 us 24 | 25 | if sys.platform == 'win32': 26 | # precision of time.time on windows is 16 ms 27 | timer_precision = 25e-3 28 | elif sys.platform == 'darwin': 29 | # for some reason on OS X time.sleep appears to be 30 | # quite inaccurate 31 | timer_precision = 80e-3 32 | else: 33 | timer_precision = 5e-3 34 | 35 | assert res['wall_time'] == approx(dt, abs=timer_precision) 36 | 37 | 38 | def test_wall_user_time(): 39 | pytest.importorskip('resource') 40 | 41 | res = timeit(delayed(sleep)(0), timer='cpu_time') 42 | assert 'cpu_time' in res 43 | 44 | 45 | # Memory based tests 46 | 47 | 48 | def test_memit_overhead(): 49 | res = memit(delayed(sleep)(0.1)) 50 | assert isinstance(res, dict) 51 | 52 | # measurement error is less than 1.0 MB 53 | assert res['peak_memory'] < 1.0 54 | 55 | 56 | def test_memit_array_allocation(): 57 | np = pytest.importorskip('numpy') 58 | 59 | N = 5000 60 | double_size = np.ones(1).nbytes 61 | 62 | def allocate_array(): 63 | X = np.ones((N, N)) 64 | sleep(0.1) 65 | X[:] += 1 66 | 67 | res = memit(delayed(allocate_array)()) 68 | assert res['peak_memory'] == approx(N**2*double_size / 1024**2, 69 | rel=0.05) 70 | 71 | 72 | @pytest.mark.parametrize('repeat', (1, 3)) 73 | @pytest.mark.parametrize('aggregate', [['mean', 'max'], False]) 74 | def test_dataframe_conversion(repeat, aggregate): 75 | 76 | pd = pytest.importorskip('pandas') 77 | 78 | N = 2 79 | 80 | metrics = ['peak_memory', 'wall_time'] 81 | 82 | bench = Benchmark(wall_time=True, peak_memory=True, 83 | repeat=repeat, aggregate=aggregate) 84 | 85 | res = bench(delayed(sleep, tags={'idx': idx})(0.04) for idx in range(N)) 86 | 87 | assert isinstance(res, pd.DataFrame) 88 | 89 | if aggregate: 90 | assert len(res) == N 91 | assert res.index.names == ['idx'] 92 | if repeat > 1: 93 | assert isinstance(res.columns, pd.MultiIndex) 94 | assert set(res.columns.levels[0]) == set(metrics) 95 | assert set(res.columns.levels[1]) == set(aggregate) 96 | else: 97 | assert isinstance(res.columns, pd.Index) 98 | else: 99 | assert len(res) == N*repeat 100 | if repeat > 1: 101 | assert res.index.names == ['idx', 'runid'] 102 | else: 103 | assert res.index.names == ['idx'] 104 | assert isinstance(res.columns, pd.Index) 105 | assert set(res.columns) == set(metrics) 106 | 107 | 108 | # Handling of optional parameters 109 | 110 | def test_repeat(): 111 | 112 | agg = ('mean',) 113 | res = timeit(delayed(sleep)(0), repeat=2, aggregate=agg) 114 | pd = import_or_none('pandas') 115 | 116 | if pd is None: 117 | assert len(res) == 2 118 | else: 119 | assert list(res.columns) == ['wall_time'] 120 | assert list(res.index) == list(agg) 121 | 122 | 123 | @pytest.mark.parametrize('repeat', (1, 2)) 124 | def test_multiple_metrics(repeat): 125 | 126 | bench = Benchmark(wall_time=True, peak_memory=True, 127 | to_dataframe=False, repeat=repeat) 128 | res = bench(delayed(sleep)(0)) 129 | 130 | if repeat == 1: 131 | assert isinstance(res, dict) 132 | else: 133 | assert isinstance(res, list) 134 | len(res) == repeat 135 | assert isinstance(res[0], dict) 136 | res = res[0] 137 | 138 | for metric in ['wall_time', 'peak_memory']: 139 | assert metric in res 140 | assert res[metric] >= 0 141 | 142 | 143 | def test_benchmark_env(): 144 | 145 | res = timeit(delayed(sleep, env={'NEURTU_TEST': 'true'})(0)) 146 | assert 'NEURTU_TEST' in res 147 | assert res['NEURTU_TEST'] == 'true' 148 | 149 | # Parametric benchmark testing 150 | 151 | 152 | @pytest.mark.parametrize('repeat', (1, 2)) 153 | def test_timeit_sequence(repeat): 154 | 155 | res = timeit((delayed(sleep, tags={'idx': idx})(0.1) for idx in range(2)), 156 | repeat=repeat, to_dataframe=False) 157 | assert isinstance(res, list) 158 | for row in res: 159 | assert 'wall_time' in row 160 | assert row['wall_time'] > 0 161 | 162 | 163 | def test_untaged_sequence(): 164 | 165 | with pytest.raises(ValueError) as excinfo: 166 | timeit(delayed(sleep)(0.1) for _ in range(2)) 167 | assert "please provide the tag parameter" in str(excinfo.value) 168 | 169 | with pytest.raises(ValueError) as excinfo: 170 | timeit([delayed(sleep, tags={'a': 1})(0.1), 171 | delayed(sleep, tags={'a': 1})(0.1)]) 172 | assert "but only 1 unique tags were found" in str(excinfo.value) 173 | 174 | 175 | def test_progress_bar(capsys): 176 | timeit((delayed(sleep, tags={'N': idx})(0.1) for idx in range(2)), 177 | repeat=1) 178 | out, err = capsys.readouterr() 179 | out = out + err 180 | assert len(out) == 0 181 | timeit((delayed(sleep, tags={'N': idx})(0.1) for idx in range(2)), 182 | progress_bar=1e-3, repeat=1) 183 | out, err = capsys.readouterr() 184 | out = out + err 185 | assert len(out) > 0 186 | assert '100%' in out 187 | assert '2/2' in out 188 | 189 | 190 | def test_custom_metric(): 191 | with pytest.raises(ValueError) as excinfo: 192 | Benchmark(other_timer=True)(delayed(sleep)(0.1)) 193 | assert 'other_timer=True is not a callable' in str(excinfo.value) 194 | 195 | def custom_metric(obj): 196 | return sum(obj.compute()) 197 | 198 | bench = Benchmark(custom_metric=custom_metric) 199 | res = bench(delayed(range)(3)) 200 | assert res == {'custom_metric': 3} 201 | -------------------------------------------------------------------------------- /neurtu/delayed.py: -------------------------------------------------------------------------------- 1 | # neurtu, BSD 3 clause license 2 | # Authors: Roman Yurchak 3 | import os 4 | 5 | 6 | class Delayed(object): 7 | """Delayed wrapper class 8 | 9 | Parameters 10 | ---------- 11 | obj : object 12 | object, function or Class to delay 13 | func : {str, None} 14 | name of the object (build-in) attribute to call 15 | args: args 16 | positional arguments passed to ``func`` 17 | kwargs: dict 18 | keyword arguments passed to ``func`` 19 | tags: dict 20 | optional tags for the delayed object 21 | env: dict 22 | optional environment variables to set when evaluating the delayed object 23 | """ 24 | def __init__(self, obj, func, args=None, kwargs=None, tags=None, 25 | env=None): 26 | self.__obj = obj 27 | self.__func = func 28 | if args is None: 29 | args = () 30 | self.__args = args 31 | self.__kwargs = kwargs if kwargs is not None else {} 32 | self.__tags = tags if tags is not None else {} 33 | self.__env = env if env is not None else {} 34 | 35 | def __call__(self, *args, **kwargs): 36 | return Delayed(self, '__call__', args, kwargs) 37 | 38 | def __getattr__(self, key): 39 | return Delayed(self, '__getattr__', args=(key,)) 40 | 41 | def __getitem__(self, key): 42 | return Delayed(self, '__getitem__', args=(key,)) 43 | 44 | def _compute(self): 45 | if self.__func is None: 46 | return self.__obj 47 | else: 48 | obj = self.__obj._compute() 49 | args, kwargs = self.__args, self.__kwargs 50 | if self.__func == '__call__': 51 | # this could be an __init__ or a __call__ 52 | return obj(*args, **kwargs) 53 | elif self.__func == '__getattr__': 54 | return obj.__getattribute__(*args, **kwargs) 55 | else: 56 | return getattr(obj, self.__func)(*args, **kwargs) 57 | 58 | def compute(self): 59 | """Evaluate the delayed object""" 60 | 61 | env = self.get_env() 62 | if env: 63 | env_init = os.environ.copy() 64 | try: 65 | os.environ.update(env) 66 | res = self._compute() 67 | finally: 68 | os.environ.clear() 69 | os.environ.update(env_init) 70 | return res 71 | else: 72 | return self._compute() 73 | 74 | def __repr__(self): 75 | """Render a Delayed object""" 76 | parent_repr = self.__obj.__repr__() 77 | 78 | def _str2str(x): 79 | if isinstance(x, str) and not x.strip(): 80 | return "'%s'" % x 81 | else: 82 | return x 83 | 84 | args_repr = ['%s' % _str2str(val) for val in self.__args] 85 | kwargs_repr = ['%s=%s' % (key, _str2str(val)) 86 | for key, val in self.__kwargs.items()] 87 | 88 | args_kwargs_repr = ','.join(args_repr + kwargs_repr) 89 | args_repr = ','.join(args_repr) 90 | kwargs_repr = ','.join(kwargs_repr) 91 | 92 | if self.__func is None: 93 | if self.__tags: 94 | tags_repr = ', tags=%s' % self.__tags 95 | else: 96 | tags_repr = '' 97 | parent_repr = '' % (parent_repr.replace('<', '') 98 | .replace('>', ''), 99 | tags_repr) 100 | return parent_repr 101 | elif self.__func == '__call__': 102 | return (parent_repr[:-1] + '(%s)>' % str(args_kwargs_repr)) 103 | elif self.__func == '__getattr__': 104 | return (parent_repr[:-1] + '.%s>' % args_repr) 105 | elif self.__func == '__getitem__': 106 | return (parent_repr[:-1] + '[%s]>' % args_repr) 107 | else: 108 | return (parent_repr + 109 | ' -> {} args={} kwargs={}' 110 | .format(self.__func, args_repr, kwargs_repr)) 111 | 112 | def get_tags(self): 113 | """Get tags passed at init 114 | 115 | Returns 116 | ------- 117 | tags : dict 118 | a dictionary of tags 119 | """ 120 | if self.__func is None: 121 | return self.__tags 122 | else: 123 | # recursively find the root Delayed object 124 | return self.__obj.get_tags() 125 | 126 | def get_env(self): 127 | """Get environement variables passed at init 128 | 129 | Returns 130 | ------- 131 | env : dict 132 | a dictionary of environement variables 133 | """ 134 | if self.__func is None: 135 | return self.__env 136 | else: 137 | # recursively find the root Delayed object 138 | return self.__obj.get_env() 139 | 140 | def get_args(self): 141 | """Get all arguments passed. 142 | 143 | Returns 144 | ------- 145 | args : list 146 | a list containing all arguments 147 | """ 148 | if self.__func is None: 149 | return list(self.__args) 150 | else: 151 | return list(self.__args) + self.__obj.get_args() 152 | 153 | def get_kwargs(self): 154 | """Get all keyword arguments passed. 155 | 156 | Returns 157 | ------- 158 | kwargs : list of dict 159 | a list containing all keyword arguments 160 | """ 161 | if self.__func is None: 162 | return [self.__kwargs] 163 | else: 164 | return [self.__kwargs] + self.__obj.get_kwargs() 165 | 166 | 167 | def _is_delayed(obj): 168 | """Check that object follows the ``class:neurtu.Delayed`` API 169 | """ 170 | return (hasattr(obj, 'compute') and hasattr(obj, 'get_tags') and 171 | callable(obj.compute) and callable(obj.get_tags)) 172 | 173 | 174 | def delayed(obj, tags=None, env=None): 175 | """Delayed object evaluation 176 | 177 | Parameters 178 | ---------- 179 | obj : object 180 | object or function to wrap 181 | tags : dict 182 | optional tags for the produced delayed object 183 | env: dict 184 | optional environment variables to set when evaluating the delayed object 185 | 186 | Returns 187 | ------- 188 | result : `class:neurtu.Delayed` 189 | a delayed object 190 | 191 | Example 192 | ------- 193 | 194 | >>> x = delayed('some string').split(' ')[::-1] 195 | >>> x 196 | 197 | >>> x.compute() 198 | ['string', 'some'] 199 | 200 | 201 | Using tags 202 | 203 | >>> x = delayed([2, 3], tags={'a': 0}).sum() 204 | >>> x.get_tags() 205 | {'a': 0} 206 | 207 | """ 208 | return Delayed(obj, None, tags=tags, env=env) 209 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # neurtu documentation build configuration file, created by 5 | # sphinx-quickstart on Mon Feb 19 17:23:39 2018. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | # 20 | import os 21 | import sys 22 | 23 | sys.path.insert(0, os.path.abspath('../')) 24 | sys.path.insert(0, os.path.abspath('sphinxext')) 25 | 26 | import neurtu 27 | 28 | from github_link import make_linkcode_resolve 29 | 30 | # -- General configuration ------------------------------------------------ 31 | 32 | # If your documentation needs a minimal Sphinx version, state it here. 33 | # 34 | # needs_sphinx = '1.0' 35 | 36 | # Add any Sphinx extension module names here, as strings. They can be 37 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 38 | # ones. 39 | extensions = [ 40 | 'sphinx.ext.autodoc', 41 | 'sphinx.ext.doctest', 42 | 'sphinx.ext.mathjax', 43 | 'sphinx.ext.autodoc', 44 | 'sphinx.ext.autosummary', 45 | 'sphinxcontrib.napoleon', 46 | 'sphinx_gallery.gen_gallery', 47 | 'sphinx.ext.linkcode', 48 | 'sphinx.ext.intersphinx' 49 | ] 50 | 51 | 52 | autosummary_generate = True 53 | 54 | autodoc_default_flags = ['members', 'inherited-members'] 55 | 56 | 57 | intersphinx_mapping = {'python': ('https://docs.python.org/3.6', None), 58 | 'dask': ('https://dask.pydata.org/en/latest/', None)} 59 | 60 | sphinx_gallery_conf = { 61 | # path to your examples scripts 62 | 'examples_dirs': ['../examples'], 63 | # path where to save gallery generated examples 64 | 'gallery_dirs': ['examples'], 65 | # directory where function granular galleries are stored 66 | 'backreferences_dir': 'generated_gallery', 67 | 68 | # Modules for which function level galleries are created. 69 | 'doc_module': ('neurtu'), 70 | 'filename_pattern': '.*\.py' 71 | } 72 | 73 | # Add any paths that contain templates here, relative to this directory. 74 | templates_path = ['_templates'] 75 | 76 | # The suffix(es) of source filenames. 77 | # You can specify multiple suffix as a list of string: 78 | # 79 | # source_suffix = ['.rst', '.md'] 80 | source_suffix = '.rst' 81 | 82 | # The master toctree document. 83 | master_doc = 'index' 84 | 85 | # General information about the project. 86 | project = 'neurtu' 87 | copyright = '2018, Roman Yurchak' 88 | author = 'Roman Yurchak' 89 | 90 | # The version info for the project you're documenting, acts as replacement for 91 | # |version| and |release|, also used in various other places throughout the 92 | # built documents. 93 | # 94 | # The short X.Y version. 95 | version = neurtu.__version__ 96 | # The full version, including alpha/beta/rc tags. 97 | release = neurtu.__version__ 98 | 99 | # The language for content autogenerated by Sphinx. Refer to documentation 100 | # for a list of supported languages. 101 | # 102 | # This is also used if you do content translation via gettext catalogs. 103 | # Usually you set "language" from the command line for these cases. 104 | language = None 105 | 106 | # List of patterns, relative to source directory, that match files and 107 | # directories to ignore when looking for source files. 108 | # This patterns also effect to html_static_path and html_extra_path 109 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 110 | 111 | # The name of the Pygments (syntax highlighting) style to use. 112 | pygments_style = 'sphinx' 113 | 114 | # If true, `todo` and `todoList` produce output, else they produce nothing. 115 | todo_include_todos = False 116 | 117 | 118 | # -- Options for HTML output ---------------------------------------------- 119 | 120 | # The theme to use for HTML and HTML Help pages. See the documentation for 121 | # a list of builtin themes. 122 | html_theme = 'sphinx_rtd_theme' 123 | 124 | # Theme options are theme-specific and customize the look and feel of a theme 125 | # further. For a list of options available for each theme, see the 126 | # documentation. 127 | # 128 | # html_theme_options = {} 129 | 130 | # Add any paths that contain custom static files (such as style sheets) here, 131 | # relative to this directory. They are copied after the builtin static files, 132 | # so a file named "default.css" will overwrite the builtin "default.css". 133 | html_static_path = ['_static'] 134 | 135 | # Custom sidebar templates, must be a dictionary that maps document names 136 | # to template names. 137 | # 138 | 139 | 140 | # -- Options for HTMLHelp output ------------------------------------------ 141 | 142 | # Output file base name for HTML help builder. 143 | htmlhelp_basename = 'neurtudoc' 144 | 145 | 146 | # -- Options for LaTeX output --------------------------------------------- 147 | 148 | latex_elements = { 149 | # The paper size ('letterpaper' or 'a4paper'). 150 | # 151 | # 'papersize': 'letterpaper', 152 | 153 | # The font size ('10pt', '11pt' or '12pt'). 154 | # 155 | # 'pointsize': '10pt', 156 | 157 | # Additional stuff for the LaTeX preamble. 158 | # 159 | # 'preamble': '', 160 | 161 | # Latex figure (float) alignment 162 | # 163 | # 'figure_align': 'htbp', 164 | } 165 | 166 | # Grouping the document tree into LaTeX files. List of tuples 167 | # (source start file, target name, title, 168 | # author, documentclass [howto, manual, or own class]). 169 | latex_documents = [ 170 | (master_doc, 'neurtu.tex', 'neurtu Documentation', 171 | 'Roman Yurchak', 'manual'), 172 | ] 173 | 174 | 175 | # -- Options for manual page output --------------------------------------- 176 | 177 | # One entry per manual page. List of tuples 178 | # (source start file, name, description, authors, manual section). 179 | man_pages = [ 180 | (master_doc, 'neurtu', 'neurtu Documentation', 181 | [author], 1) 182 | ] 183 | 184 | 185 | # -- Options for Texinfo output ------------------------------------------- 186 | 187 | # Grouping the document tree into Texinfo files. List of tuples 188 | # (source start file, target name, title, author, 189 | # dir menu entry, description, category) 190 | texinfo_documents = [ 191 | (master_doc, 'neurtu', 'neurtu Documentation', 192 | author, 'neurtu', 'One line description of project.', 193 | 'Miscellaneous'), 194 | ] 195 | 196 | # The following is used by sphinx.ext.linkcode to provide links to github 197 | linkcode_resolve = make_linkcode_resolve('neurtu', 198 | u'https://github.com/symerio/' 199 | 'neurtu/blob/{revision}/' 200 | '{package}/{path}#L{lineno}') 201 | 202 | -------------------------------------------------------------------------------- /neurtu/base.py: -------------------------------------------------------------------------------- 1 | # neurtu, BSD 3 clause license 2 | # Authors: Roman Yurchak 3 | 4 | import sys 5 | from collections.abc import Iterable 6 | import timeit as cpython_timeit 7 | import gc 8 | 9 | try: 10 | from tqdm.auto import tqdm 11 | except ImportError: # pragma: no cover 12 | tqdm = None 13 | 14 | 15 | from .delayed import _is_delayed 16 | from .utils import import_or_none 17 | from .metrics import measure_wall_time, measure_cpu_time 18 | from .metrics import measure_peak_memory 19 | 20 | 21 | def _validate_timer_precision(res_mean, func, obj_el, params): 22 | """For timing measurements, increase the number of iterations 23 | if the precision is unsufficient""" 24 | if sys.platform in ['win32', 'darwin']: 25 | timer_threashold = 0.5 26 | else: 27 | timer_threashold = 0.1 28 | if res_mean < timer_threashold: 29 | # if the measured timeing is below the threashold, 30 | # it won't be very accurate. Increase the 31 | # `number` parameter of Timer.timeit to get 32 | # result in the order of 500 ms on Windows, Mac OS 33 | # and 100ms on Linux 34 | 35 | if res_mean == 0.0: 36 | corrected_number = 1000 37 | else: 38 | corrected_number = int(timer_threashold / res_mean) 39 | if corrected_number == 1: 40 | return res_mean 41 | params = params.copy() 42 | params['number'] = corrected_number 43 | gc.collect() 44 | res_mean = func(obj_el, **params) 45 | return res_mean 46 | 47 | 48 | class _ProgressBar(object): 49 | """ Internal progress bar 50 | 51 | Parameters 52 | ---------- 53 | N : int 54 | total number of iterations 55 | delay: {bool, float} 56 | if a number, and tqdm is installed, display the progress bar when the 57 | total benchmark time is expected to be larger than the given number of 58 | seconds. If False, the progress bar is not be displayed. 59 | """ 60 | def __init__(self, N, delay): 61 | self.t0 = cpython_timeit.default_timer() 62 | self.delay = delay 63 | self.N = N 64 | self.idx = 0 65 | self.pbar = None 66 | 67 | def increment(self): 68 | """ 69 | Decide whether to print a progress bar, update it if necessary 70 | """ 71 | self.idx += 1 72 | if tqdm is None or not self.delay: 73 | pass 74 | elif self.pbar is None: 75 | dt = cpython_timeit.default_timer() - self.t0 76 | if dt * self.N / self.idx > self.delay: 77 | self.pbar = tqdm(total=self.N, leave=False) 78 | self.pbar.update(self.idx) 79 | else: 80 | self.pbar.update(1) 81 | 82 | def close(self): 83 | if self.pbar is not None: 84 | self.pbar.close() 85 | 86 | 87 | class Benchmark(object): 88 | """Benchmark calculations 89 | 90 | Parameters 91 | ---------- 92 | wall_time : {bool, dict}, default=None 93 | measure wall time. When a dictionary, it is passed as parameters to the 94 | `func:measure_wall_time` function. Will default to True, unless some 95 | other metric is enabled. 96 | cpu_time : {bool, dict}, default=False 97 | measure CPU time. When a dictionary, it is passed as parameters to the 98 | :func:`measure_cpu_time` function. 99 | peak_memory : {bool, dict}, default=False 100 | measure peak memory usage. When a dictionary, it is passed as parameters 101 | to the :func:`measure_peak_memory` function. 102 | repeat : int, default=1 103 | number of repeated measurements 104 | aggregate : {collection, False}, default=('mean', 'max', 'std') 105 | when repeat > 1, different runs are indexed by the ``runid`` key. 106 | If pandas is installed and aggregate is a collection, aggregate repeated 107 | runs with the provided methods. 108 | to_dataframe : bool, default=None 109 | whether to convert parametric results to a daframe. By default convert to 110 | dataframe is pandas is installed. 111 | progress_bar : {bool, float}, default=5.0 112 | if a number, and tqdm is installed, display the progress bar when the 113 | estimated benchmark time is larger than the given number of seconds. 114 | If False, the progress bar will not be displayed. 115 | **kwargs : dict 116 | custom evaluation metrics of the form ``key=func``, 117 | where ``key`` is the metric name, and the ``func`` is the evaluation 118 | metric that accepts a ``Delayed`` object: ``func(obj)``. 119 | """ 120 | def __init__(self, wall_time=None, cpu_time=False, peak_memory=False, 121 | repeat=1, aggregate=('mean', 'max', 'std'), to_dataframe=None, 122 | progress_bar=5.0, **kwargs): 123 | metrics = {} 124 | for name, params, func in [ 125 | ('wall_time', wall_time, measure_wall_time), 126 | ('cpu_time', cpu_time, measure_cpu_time), 127 | ('peak_memory', peak_memory, measure_peak_memory)]: 128 | if params: 129 | if params is True: 130 | params = {} 131 | else: 132 | params = params.copy() 133 | params['func'] = func 134 | metrics[name] = params 135 | for name, params in kwargs.items(): 136 | if not callable(params): 137 | raise ValueError(('%s=%s is not a callable. Use a callable ' 138 | 'to define a custom metric!') 139 | % (name, params)) 140 | params = {'func': params} 141 | metrics[name] = params 142 | 143 | if not metrics: 144 | # if no metrics were explicitly enabled, measure wall_time 145 | metrics['wall_time'] = {'func': measure_wall_time} 146 | self.metrics = metrics 147 | self.repeat = repeat 148 | self.aggregate = aggregate 149 | self.to_dataframe = to_dataframe 150 | self.progress_bar = progress_bar 151 | 152 | def __call__(self, obj): 153 | """Evaluate metrics on the delayed object 154 | 155 | Parameters 156 | ---------- 157 | obj: :class:`Delayed` or iterable of :class:`Delayed` 158 | a delayed computation or an iterable of delayed computations 159 | """ 160 | 161 | if _is_delayed(obj): 162 | obj = [obj] 163 | if isinstance(obj, list) and len(obj) == 1 and self.repeat == 1: 164 | iterable_input = False 165 | else: 166 | iterable_input = True 167 | 168 | if not isinstance(obj, Iterable): 169 | raise ValueError(('obj=%s must be either a Delayed object or a ' 170 | 'iterable of delayed objects!') % obj) 171 | 172 | # convert the iterable to list 173 | obj = list(obj) 174 | 175 | tags_all = list(set([self._hash_tags_env(el) for el in obj])) 176 | 177 | # check that tags are unique 178 | if len(obj) != len(tags_all): 179 | if len(tags_all) == 1 and tags_all[0] == '': 180 | raise ValueError('When bechmarking a sequence, please provide ' 181 | 'the tag parameter for each delayed object ' 182 | 'to uniquely identify them!') 183 | else: 184 | raise ValueError(('Input sequence has %s delayed objects, ' 185 | 'but only %s unique tags were found!') 186 | % (len(obj), len(tags_all))) 187 | 188 | db = [] 189 | pbar = _ProgressBar( 190 | len(obj)*len(self.metrics)*self.repeat, 191 | self.progress_bar 192 | ) 193 | for runid in range(self.repeat): 194 | for idx, obj_el in enumerate(obj): 195 | res = self._evaluate_single(obj_el, pbar) 196 | if self.repeat > 1: 197 | res['runid'] = runid 198 | db.append(res) 199 | 200 | pbar.close() 201 | 202 | pd = import_or_none('pandas') 203 | 204 | if iterable_input: 205 | if self.to_dataframe is not False and pd is not None: 206 | index = list(obj[0].get_tags().keys()) 207 | if self.repeat > 1: 208 | index.append('runid') 209 | db = pd.DataFrame(db) 210 | if index: 211 | db.set_index(index, inplace=True) 212 | if self.repeat > 1 and self.aggregate: 213 | if index == ['runid']: 214 | # no tags were passed 215 | db = db.agg(self.aggregate) 216 | else: 217 | index.remove('runid') 218 | db = db.groupby(index).agg(self.aggregate) 219 | 220 | return db 221 | else: 222 | return db 223 | else: 224 | return db[0] 225 | 226 | def _hash_tags_env(self, obj): 227 | """Compute a string representation of tags and env of a delayed 228 | object. This is used for duplicates detection.""" 229 | if not _is_delayed(obj): 230 | raise ValueError 231 | tags_el = [] 232 | for key, val in obj.get_tags().items(): 233 | tags_el.append('%s:%s' % (key, val)) 234 | for key, val in obj.get_env().items(): 235 | tags_el.append('%s:%s' % (key, val)) 236 | return '|'.join(tags_el) 237 | 238 | def _evaluate_single(self, obj, pbar): 239 | """Evaluate all metrics a single time""" 240 | row = {} 241 | row.update(obj.get_tags()) 242 | row.update(obj.get_env()) 243 | 244 | for (name, params) in self.metrics.items(): 245 | params = params.copy() 246 | func = params.pop('func') 247 | 248 | gc.collect() 249 | res = func(obj, **params) 250 | 251 | if name in ['wall_time', 'cpu_time']: 252 | res = _validate_timer_precision(res, func, obj, 253 | params) 254 | row[name] = res 255 | pbar.increment() 256 | return row 257 | 258 | 259 | def memit(obj, repeat=1, aggregate=('mean', 'max', 'std'), 260 | interval=0.01, to_dataframe=None, progress_bar=5.0): 261 | """Measure the memory use. 262 | 263 | This is an alias for :class:`Benchmark` with `peak_memory=True)`. 264 | 265 | Parameters 266 | ---------- 267 | repeat : int, default=1 268 | number of repeated measurements 269 | aggregate : {collection, False}, default=('mean', 'max', 'std') 270 | when repeat > 1, different runs are indexed by the ``runid`` key. 271 | If pandas is installed and aggregate is a collection, aggregate repeated 272 | runs with the provided methods. 273 | to_dataframe : bool, default=None 274 | whether to convert parametric results to a daframe. By default convert to 275 | dataframe is pandas is installed. 276 | progress_bar : {bool, float}, default=5.0 277 | if a number, and tqdm is installed, display the progress bar when the 278 | estimated benchmark time is larger than the given number of seconds. 279 | If False, the progress bar will not be displayed. 280 | 281 | Returns 282 | ------- 283 | res : dict, list or pandas.DataFrame 284 | computed memory usage 285 | """ 286 | 287 | return Benchmark(peak_memory={'interval': interval}, 288 | to_dataframe=to_dataframe, 289 | repeat=repeat, aggregate=aggregate, 290 | progress_bar=progress_bar)(obj) 291 | 292 | 293 | def timeit(obj, timer='wall_time', number=1, repeat=1, 294 | aggregate=('mean', 'max', 'std'), to_dataframe=None, 295 | progress_bar=5.0): 296 | """A benchmark decorator 297 | 298 | This is an alias for :class:`Benchmark` with `wall_time=True`. 299 | 300 | Parameters 301 | ---------- 302 | obj : {Delayed, iterable of Delayed} 303 | delayed object to compute, or an iterable of Delayed objects 304 | number : int, default=1 305 | number of runs to pass to ``timeit.Timer`` 306 | repeat : int, default=1 307 | number of repeated measurements 308 | aggregate : {collection, False}, default=('mean', 'max', 'std') 309 | when repeat > 1, different runs are indexed by the ``runid`` key. 310 | If pandas is installed and aggregate is a collection, aggregate repeated 311 | runs with the provided methods. 312 | to_dataframe : bool, default=None 313 | whether to convert parametric results to a daframe. By default convert to 314 | dataframe is pandas is installed. 315 | progress_bar : {bool, float}, default=5.0 316 | if a number, and tqdm is installed, display the progress bar when the 317 | estimated benchmark time is larger than the given number of seconds. 318 | If False, the progress bar will not be displayed. 319 | 320 | Returns 321 | ------- 322 | res : dict, list or pandas.DataFrame 323 | computed timing 324 | """ 325 | 326 | args = {'to_dataframe': to_dataframe, 'progress_bar': progress_bar, 327 | 'repeat': repeat, 'aggregate': aggregate} 328 | 329 | if timer == 'wall_time': 330 | return Benchmark(wall_time={'number': number}, **args)(obj) 331 | elif timer == 'cpu_time': 332 | return Benchmark(cpu_time={'number': number}, **args)(obj) 333 | else: 334 | raise ValueError("timer=%s should be one of 'cpu_time', 'wall_time'" 335 | % timer) 336 | --------------------------------------------------------------------------------