├── docs ├── source │ ├── license.rst │ ├── api.rst │ ├── sum_example.png │ ├── time_example.png │ ├── list_add_example.png │ ├── sum_example_cli.png │ ├── sum_list_example.png │ ├── command_line.rst │ ├── development.rst │ ├── changes.rst │ ├── index.rst │ ├── conf.py │ └── extended.rst ├── Makefile └── make.bat ├── setup.cfg ├── tests ├── test_benchmarkbuilder.py ├── test_benchmark.py ├── test_assertions.py └── test_doc_examples.py ├── .gitignore ├── setup.py ├── README.rst ├── simple_benchmark ├── __main__.py └── __init__.py └── LICENSE /docs/source/license.rst: -------------------------------------------------------------------------------- 1 | License 2 | ======= 3 | 4 | .. literalinclude:: ../../LICENSE 5 | -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- 1 | API Reference 2 | ============= 3 | 4 | .. automodule:: simple_benchmark 5 | :members: -------------------------------------------------------------------------------- /docs/source/sum_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSeifert04/simple_benchmark/HEAD/docs/source/sum_example.png -------------------------------------------------------------------------------- /docs/source/time_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSeifert04/simple_benchmark/HEAD/docs/source/time_example.png -------------------------------------------------------------------------------- /docs/source/list_add_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSeifert04/simple_benchmark/HEAD/docs/source/list_add_example.png -------------------------------------------------------------------------------- /docs/source/sum_example_cli.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSeifert04/simple_benchmark/HEAD/docs/source/sum_example_cli.png -------------------------------------------------------------------------------- /docs/source/sum_list_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MSeifert04/simple_benchmark/HEAD/docs/source/sum_list_example.png -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [build_sphinx] 2 | project = 'simple_benchmark' 3 | 4 | [tool:pytest] 5 | markers = 6 | slow: marks tests as slow 7 | -------------------------------------------------------------------------------- /tests/test_benchmarkbuilder.py: -------------------------------------------------------------------------------- 1 | import simple_benchmark 2 | 3 | 4 | def test_simple(): 5 | bb = simple_benchmark.BenchmarkBuilder() 6 | bb.add_functions([min, max]) 7 | bb.use_random_lists_as_arguments([2, 3, 4]) 8 | bb.run() 9 | -------------------------------------------------------------------------------- /tests/test_benchmark.py: -------------------------------------------------------------------------------- 1 | import simple_benchmark 2 | 3 | import collections 4 | 5 | 6 | def test_simple(): 7 | simple_benchmark.benchmark( 8 | funcs=[min, max], 9 | arguments=collections.OrderedDict([(n, [1]*n) for n in [3, 4, 5, 6]]) 10 | ) 11 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = simple_benchmark 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | set SPHINXPROJ=simple_benchmark 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 20 | echo.installed, then set the SPHINXBUILD environment variable to point 21 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 22 | echo.may add the Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /tests/test_assertions.py: -------------------------------------------------------------------------------- 1 | import simple_benchmark 2 | 3 | import operator 4 | 5 | import numpy as np 6 | import pytest 7 | 8 | 9 | def sort_in_place(l): 10 | l.sort() 11 | return l 12 | 13 | 14 | def test_assert_same_results_work(): 15 | simple_benchmark.assert_same_results( 16 | funcs=[min, np.min], 17 | arguments={2**i: list(range(2**i)) for i in range(2, 5)}, 18 | equality_func=operator.eq 19 | ) 20 | 21 | 22 | def test_assert_same_results_work_when_not_equal(): 23 | with pytest.raises(AssertionError): 24 | simple_benchmark.assert_same_results( 25 | funcs=[min, max], 26 | arguments={2**i: list(range(2**i)) for i in range(2, 5)}, 27 | equality_func=operator.eq 28 | ) 29 | 30 | 31 | def test_assert_not_mutating_input_work(): 32 | simple_benchmark.assert_not_mutating_input( 33 | funcs=[min, np.min], 34 | arguments={2**i: list(range(2**i)) for i in range(2, 5)}, 35 | equality_func=operator.eq 36 | ) 37 | 38 | 39 | def test_assert_not_mutating_input_work_when_modifies(): 40 | with pytest.raises(AssertionError): 41 | simple_benchmark.assert_not_mutating_input( 42 | funcs=[sorted, sort_in_place], 43 | arguments={2**i: list(reversed(range(2**i))) for i in range(2, 5)}, 44 | equality_func=operator.eq 45 | ) 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | # PyCharm-stuff 104 | .idea/ 105 | -------------------------------------------------------------------------------- /docs/source/command_line.rst: -------------------------------------------------------------------------------- 1 | Command Line 2 | ============ 3 | 4 | Using the Command Line 5 | ---------------------- 6 | 7 | .. warning:: 8 | The command line interface is highly experimental. It's very likely to 9 | change its API. 10 | 11 | When you have all optional dependencies installed you can also run 12 | ``simple_benchmark``, in the most basic form it would be:: 13 | 14 | $ python -m simple_benchmark INPUT_FILE OUTPUT_FILE 15 | 16 | Which processes the ``INPUT_FILE`` and writes a plot to ``OUTPUT_FILE``. 17 | 18 | However in order to work correctly the ``INPUT_FILE`` has to fulfill several 19 | criteria: 20 | 21 | - It must be a valid Python file. 22 | - All functions that should be benchmarked have to have a name starting with ``bench_`` 23 | and everything thereafter is used for the label. 24 | - The function generating the arguments for the benchmark has to start with ``args_`` 25 | and everything thereafter is used for the label of the x-axis. 26 | 27 | Also if the benchmarked function has a ``func`` parameter with a default it 28 | will be used to determine the ``alias`` (the displayed name in the table and 29 | plot). 30 | 31 | 32 | Parameters 33 | ---------- 34 | 35 | The first two parameters are the input and output file. However there are a 36 | few more parameters. These can be also seen when running:: 37 | 38 | $ python -m simple_benchmark -h 39 | usage: __main__.py [-h] [-s FIGSIZE] [--time-per-benchmark TIME_PER_BENCHMARK] [-v] [--write-csv] filename out 40 | 41 | Benchmark a file 42 | 43 | positional arguments: 44 | filename the file to run the benchmark on. 45 | out Specifies the output file for the plot 46 | 47 | optional arguments: 48 | -h, --help show this help message and exit 49 | -s FIGSIZE, --figsize FIGSIZE 50 | Specify the output size in inches, needs to be wrapped in quotes on most shells, e.g. "15, 9" (default: 15, 9) 51 | --time-per-benchmark TIME_PER_BENCHMARK 52 | The target time for each individual benchmark in seconds (default: 0.1) 53 | -v, --verbose prints additional information on stdout (default: False) 54 | --write-csv Writes an additional CSV file of the results (default: False) 55 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from setuptools import setup, find_packages 4 | 5 | 6 | package_name = "simple_benchmark" 7 | 8 | optional_dependencies = ["numpy", "matplotlib", "pandas"] 9 | development_dependencies = ["sphinx", "pytest"] 10 | maintainer_dependencies = ["twine"] 11 | 12 | 13 | def readme(): 14 | with open('README.rst') as f: 15 | return f.read() 16 | 17 | 18 | def version(): 19 | with open('{}/__init__.py'.format(package_name)) as f: 20 | for line in f: 21 | if line.startswith('__version__'): 22 | return line.split(r"'")[1] 23 | 24 | 25 | setup(name=package_name, 26 | version=version(), 27 | 28 | description='A simple benchmarking package.', 29 | long_description=readme(), 30 | # Somehow the keywords get lost if I use a list of strings so this is 31 | # just a longish string... 32 | keywords='performance timing timeit', 33 | platforms=["Windows Linux Mac OS-X"], 34 | 35 | classifiers=[ 36 | 'Development Status :: 3 - Alpha', 37 | 'Programming Language :: Python :: 2', 38 | 'Programming Language :: Python :: 2.7', 39 | 'Programming Language :: Python :: 3', 40 | 'Programming Language :: Python :: 3.3', 41 | 'Programming Language :: Python :: 3.4', 42 | 'Programming Language :: Python :: 3.5', 43 | 'Programming Language :: Python :: 3.6', 44 | 'Operating System :: MacOS :: MacOS X', 45 | 'Operating System :: Microsoft :: Windows', 46 | 'Operating System :: POSIX :: Linux', 47 | 'Topic :: Utilities', 48 | 'Topic :: System :: Benchmark' 49 | ], 50 | 51 | license='Apache License Version 2.0', 52 | 53 | url='https://github.com/MSeifert04/simple_benchmark', 54 | 55 | author='Michael Seifert', 56 | author_email='michaelseifert04@yahoo.de', 57 | 58 | packages=find_packages(exclude=['ez_setup']), 59 | 60 | tests_require=["pytest"], 61 | extras_require={ 62 | 'optional': optional_dependencies, 63 | 'development': optional_dependencies + development_dependencies, 64 | 'maintainer': optional_dependencies + development_dependencies + maintainer_dependencies 65 | }, 66 | 67 | include_package_data=True, 68 | zip_safe=False, 69 | ) 70 | -------------------------------------------------------------------------------- /docs/source/development.rst: -------------------------------------------------------------------------------- 1 | Development 2 | =========== 3 | 4 | Prerequisites: 5 | 6 | - Cloned or downloaded source repository. For example ``git clone https://github.com/MSeifert04/simple_benchmark.git``. 7 | - You're in the root directory of the cloned (or downloaded) repository. 8 | - Have an installed Python with pip and setuptools (the following will assume that the Python executable is in your path!). 9 | 10 | Building the package locally 11 | ---------------------------- 12 | 13 | Navigate to the root directory of the repository (the directory where the ``setup.py`` file is) and then run one of 14 | these commands:: 15 | 16 | python setup.py develop 17 | 18 | or:: 19 | 20 | python -m pip install -e . 21 | 22 | In case you want to install all the optional dependencies automatically (**recommended**):: 23 | 24 | python -m pip install -e .[optional] 25 | 26 | 27 | Building the documentation locally 28 | ---------------------------------- 29 | 30 | This requires that the package was installed with all development dependencies:: 31 | 32 | python -m pip install -e .[development] 33 | 34 | Then just run:: 35 | 36 | python setup.py build_sphinx 37 | 38 | The generated HTML documentation should then be available in the 39 | ``./build/sphinx/html`` folder. 40 | 41 | 42 | Running the tests locally 43 | ------------------------- 44 | 45 | This requires that the package was installed with all development dependencies:: 46 | 47 | python -m pip install -e .[development] 48 | 49 | Then use ``pytest``:: 50 | 51 | python -m pytest tests 52 | 53 | Or to exclude the tests marked as slow:: 54 | 55 | python -m pytest tests -m "not slow" 56 | 57 | 58 | Publishing the package to PyPI 59 | ------------------------------ 60 | 61 | .. note:: 62 | This is maintainer-only! 63 | 64 | To install the necessary packages run:: 65 | 66 | python -m pip install -e .[maintainer] 67 | 68 | First clean the repository to avoid outdated artifacts:: 69 | 70 | git clean -dfX 71 | 72 | Then build the source distribution, since it's a very small package without compiled modules, we can omit building 73 | wheels:: 74 | 75 | python setup.py sdist 76 | 77 | Then upload to PyPI:: 78 | 79 | python -m twine upload --repository-url https://upload.pypi.org/legacy/ dist/* 80 | 81 | You will be prompted for the username and password. 82 | -------------------------------------------------------------------------------- /docs/source/changes.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 0.1.0 (not released) 5 | -------------------- 6 | 7 | - Added a command-line command for performing benchmarks on carefully constructed 8 | Python files (experimental) 9 | 10 | - Added the functions ``assert_same_results`` and ``assert_not_mutating_input``. 11 | 12 | - The argument ``time_per_benchmark`` of ``benchmark`` and ``BenchmarkBuilder`` now expects 13 | a ``datetime.timedelta`` instead of a ``float``. 14 | 15 | - Added ``maximum_time`` for ``benchmark`` and ``BenchmarkBuilder`` to control the maximum 16 | time for a single function execution. If exceeded the function will not be not be benchmarked 17 | anymore. 18 | 19 | - Added info-level based logging during benchmark runs. 20 | 21 | 0.0.9 (2019-04-07) 22 | ------------------ 23 | 24 | - Fixed wrong name for optional dependencies in ``extras_require`` of ``setup.py`` 25 | 26 | - Added development documentation. 27 | 28 | 0.0.8 (2019-04-06) 29 | ------------------ 30 | 31 | - Removed ``benchmark_random_list`` and ``benchmark_random_array`` in 32 | favor of the static methods ``use_random_lists_as_arguments`` and 33 | ``use_random_arrays_as_arguments`` on ``BenchmarkBuilder``. 34 | 35 | - Added ``BenchmarkBuilder`` class that provides a decorator-based 36 | construction of a benchmark. 37 | 38 | - Added a title to the plot created by the ``plot`` functions of 39 | ``BenchmarkResult`` that displays some information about the 40 | Python installation and environment. 41 | 42 | 0.0.7 (2018-04-30) 43 | ------------------ 44 | 45 | - Added optional ``estimator`` argument to the benchmark functions. The 46 | ``estimator`` can be used to calculate the reported runtime based on 47 | the individual timings. 48 | 49 | 0.0.6 (2018-04-30) 50 | ------------------ 51 | 52 | - Added ``plot_difference_percentage`` to ``BenchmarkResult`` to plot 53 | percentage differences. 54 | 55 | 0.0.5 (2018-04-22) 56 | ------------------ 57 | 58 | - Print a warning in case multiple functions have the same name 59 | 60 | - Use ``OrderedDict`` to fix issues on older Python versions where ``dict`` 61 | isn't ordered. 62 | 63 | 0.0.4 (2018-04-19) 64 | ------------------ 65 | 66 | - Added ``MultiArgument`` class to provide a way to pass in multiple 67 | arguments to the functions. 68 | 69 | 0.0.3 (2018-04-16) 70 | ------------------ 71 | 72 | - Some bugfixes. 73 | 74 | 0.0.2 (2018-04-16) 75 | ------------------ 76 | 77 | - General restructuring. 78 | 79 | 0.0.1 (2018-02-19) 80 | ------------------ 81 | 82 | - Initial release. -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to simple_benchmark's documentation! 2 | ============================================ 3 | 4 | Installation 5 | ------------ 6 | 7 | Using ``pip``: 8 | 9 | .. code:: 10 | 11 | python -m pip install simple_benchmark 12 | 13 | Or installing the most recent version directly from ``git``: 14 | 15 | .. code:: 16 | 17 | python -m pip install git+https://github.com/MSeifert04/simple_benchmark.git 18 | 19 | To utilize the all features of the library (for example visualization) you need to 20 | install the optional dependencies: 21 | 22 | - `NumPy `_ 23 | - `pandas `_ 24 | - `matplotlib `_ 25 | 26 | Or install them automatically using: 27 | 28 | .. code:: 29 | 30 | python -m pip install simple_benchmark[optional] 31 | 32 | Getting started 33 | --------------- 34 | 35 | Suppose you want to compare how NumPys sum and Pythons sum perform on lists 36 | of different sizes:: 37 | 38 | >>> from simple_benchmark import benchmark 39 | >>> import numpy as np 40 | >>> funcs = [sum, np.sum] 41 | >>> arguments = {i: [1]*i for i in [1, 10, 100, 1000, 10000, 100000]} 42 | >>> argument_name = 'list size' 43 | >>> aliases = {sum: 'Python sum', np.sum: 'NumPy sum'} 44 | >>> b = benchmark(funcs, arguments, argument_name, function_aliases=aliases) 45 | 46 | The result can be visualized with ``pandas`` (needs to be installed):: 47 | 48 | >>> b 49 | Python sum NumPy sum 50 | 1 9.640884e-08 0.000004 51 | 10 1.726930e-07 0.000004 52 | 100 7.935484e-07 0.000008 53 | 1000 7.040000e-06 0.000042 54 | 10000 6.910000e-05 0.000378 55 | 100000 6.899000e-04 0.003941 56 | 57 | Or with ``matplotlib`` (has to be installed too):: 58 | 59 | >>> b.plot() 60 | 61 | >>> # To save the plotted benchmark as PNG file. 62 | >>> import matplotlib.pyplot as plt 63 | >>> plt.savefig('sum_example.png') 64 | 65 | .. image:: ./sum_example.png 66 | 67 | Command-Line interface 68 | ---------------------- 69 | 70 | .. warning:: 71 | The command line interface is highly experimental. It's very likely to 72 | change its API. 73 | 74 | It's an experiment to run it as command-line tool, especially useful if you 75 | want to run it on multiple files and don't want the boilerplate. 76 | 77 | File ``sum.py``:: 78 | 79 | import numpy as np 80 | 81 | def bench_sum(l, func=sum): 82 | return func(l) 83 | 84 | def bench_numpy_sum(l, func=np.sum): 85 | return np.sum(l) 86 | 87 | def args_list_length(): 88 | for i in [1, 10, 100, 1000, 10000, 100000]: 89 | yield i, [1]*i 90 | 91 | Then run:: 92 | 93 | $ python -m simple_benchmark sum.py sum.png 94 | 95 | With a similar result ``sum.png``: 96 | 97 | .. image:: ./sum_example_cli.png 98 | 99 | 100 | .. toctree:: 101 | :maxdepth: 2 102 | :caption: Contents: 103 | 104 | extended 105 | command_line 106 | api 107 | changes 108 | license 109 | development 110 | 111 | 112 | Indices and tables 113 | ================== 114 | 115 | * :ref:`genindex` 116 | * :ref:`modindex` 117 | * :ref:`search` 118 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | simple_benchmark 2 | ================ 3 | 4 | A simple benchmarking package including visualization facilities. 5 | 6 | The goal of this package is to provide a simple way to compare the performance 7 | of different approaches for different inputs and to visualize the result. 8 | 9 | 10 | Documentation 11 | ------------- 12 | 13 | .. image:: https://readthedocs.org/projects/simple-benchmark/badge/?version=stable 14 | :target: http://simple-benchmark.readthedocs.io/en/stable/?badge=stable 15 | :alt: Documentation Status 16 | 17 | 18 | Downloads 19 | --------- 20 | 21 | .. image:: https://img.shields.io/pypi/v/simple_benchmark.svg 22 | :target: https://pypi.python.org/pypi/simple_benchmark 23 | :alt: PyPI Project 24 | 25 | .. image:: https://img.shields.io/github/release/MSeifert04/simple_benchmark.svg 26 | :target: https://github.com/MSeifert04/simple_benchmark/releases 27 | :alt: GitHub Project 28 | 29 | 30 | Installation 31 | ------------ 32 | 33 | Using ``pip``: 34 | 35 | .. code:: 36 | 37 | python -m pip install simple_benchmark 38 | 39 | Or installing the most recent version directly from ``git``: 40 | 41 | .. code:: 42 | 43 | python -m pip install git+https://github.com/MSeifert04/simple_benchmark.git 44 | 45 | To utilize the all features of the library (for example visualization) you need to 46 | install the optional dependencies: 47 | 48 | - `NumPy `_ 49 | - `pandas `_ 50 | - `matplotlib `_ 51 | 52 | Or install them automatically using: 53 | 54 | .. code:: 55 | 56 | python -m pip install simple_benchmark[optional] 57 | 58 | Getting started 59 | --------------- 60 | 61 | Suppose you want to compare how NumPys sum and Pythons sum perform on lists 62 | of different sizes:: 63 | 64 | >>> from simple_benchmark import benchmark 65 | >>> import numpy as np 66 | >>> funcs = [sum, np.sum] 67 | >>> arguments = {i: [1]*i for i in [1, 10, 100, 1000, 10000, 100000]} 68 | >>> argument_name = 'list size' 69 | >>> aliases = {sum: 'Python sum', np.sum: 'NumPy sum'} 70 | >>> b = benchmark(funcs, arguments, argument_name, function_aliases=aliases) 71 | 72 | The result can be visualized with ``pandas`` (needs to be installed):: 73 | 74 | >>> b 75 | Python sum NumPy sum 76 | 1 9.640884e-08 0.000004 77 | 10 1.726930e-07 0.000004 78 | 100 7.935484e-07 0.000008 79 | 1000 7.040000e-06 0.000042 80 | 10000 6.910000e-05 0.000378 81 | 100000 6.899000e-04 0.003941 82 | 83 | Or with ``matplotlib`` (has to be installed too):: 84 | 85 | >>> b.plot() 86 | 87 | To save the plotted benchmark as PNG file:: 88 | 89 | >>> import matplotlib.pyplot as plt 90 | >>> plt.savefig('sum_example.png') 91 | 92 | .. image:: ./docs/source/sum_example.png 93 | 94 | 95 | Command-Line interface 96 | ---------------------- 97 | 98 | .. warning:: 99 | The command line interface is highly experimental. It's very likely to 100 | change its API. 101 | 102 | It's an experiment to run it as command-line tool, especially useful if you 103 | want to run it on multiple files and don't want the boilerplate. 104 | 105 | File ``sum.py``:: 106 | 107 | import numpy as np 108 | 109 | def bench_sum(l, func=sum): # <-- function name needs to start with "bench_" 110 | return func(l) 111 | 112 | def bench_numpy_sum(l, func=np.sum): # <-- using func parameter with the actual function helps 113 | return np.sum(l) 114 | 115 | def args_list_length(): # <-- function providing the argument starts with "args_" 116 | for i in [1, 10, 100, 1000, 10000, 100000]: 117 | yield i, [1] * i 118 | 119 | Then run:: 120 | 121 | $ python -m simple_benchmark sum.py sum.png 122 | 123 | With a similar result: 124 | 125 | .. image:: ./docs/source/sum_example_cli.png 126 | 127 | 128 | Similar packages 129 | ---------------- 130 | 131 | - `perfplot `_ by Nico Schlömer. 132 | -------------------------------------------------------------------------------- /tests/test_doc_examples.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.slow 5 | def test_readme(): 6 | from simple_benchmark import benchmark 7 | import numpy as np 8 | funcs = [sum, np.sum] 9 | arguments = {i: [1] * i for i in [1, 10, 100, 1000, 10000, 100000]} 10 | argument_name = 'list size' 11 | aliases = {sum: 'Python sum', np.sum: 'NumPy sum'} 12 | b = benchmark(funcs, arguments, argument_name, function_aliases=aliases) 13 | b.to_pandas_dataframe() 14 | b.plot() 15 | 16 | 17 | @pytest.mark.slow 18 | def test_extended_benchmarkbuilder(): 19 | from simple_benchmark import BenchmarkBuilder 20 | import math 21 | 22 | bench = BenchmarkBuilder() 23 | 24 | @bench.add_function() 25 | def sum_using_loop(lst): 26 | sum_ = 0 27 | for item in lst: 28 | sum_ += item 29 | return sum_ 30 | 31 | @bench.add_function() 32 | def sum_using_range_loop(lst): 33 | sum_ = 0 34 | for idx in range(len(lst)): 35 | sum_ += lst[idx] 36 | return sum_ 37 | 38 | bench.use_random_lists_as_arguments(sizes=[2**i for i in range(2, 15)]) 39 | 40 | bench.add_functions([sum, math.fsum]) 41 | 42 | b = bench.run() 43 | b.plot() 44 | 45 | 46 | @pytest.mark.slow 47 | def test_extended_multiargument(): 48 | from itertools import starmap 49 | from operator import add 50 | from random import random 51 | 52 | from simple_benchmark import BenchmarkBuilder, MultiArgument 53 | 54 | bench = BenchmarkBuilder() 55 | 56 | @bench.add_function() 57 | def list_addition_zip(list1, list2): 58 | res = [] 59 | for item1, item2 in zip(list1, list2): 60 | res.append(item1 + item2) 61 | return res 62 | 63 | @bench.add_function() 64 | def list_addition_index(list1, list2): 65 | res = [] 66 | for idx in range(len(list1)): 67 | res.append(list1[idx] + list2[idx]) 68 | return res 69 | 70 | @bench.add_function() 71 | def list_addition_map_zip(list1, list2): 72 | return list(starmap(add, zip(list1, list2))) 73 | 74 | @bench.add_arguments(name='list sizes') 75 | def benchmark_arguments(): 76 | for size_exponent in range(2, 15): 77 | size = 2**size_exponent 78 | arguments = MultiArgument([ 79 | [random() for _ in range(size)], 80 | [random() for _ in range(size)]]) 81 | yield size, arguments 82 | 83 | b = bench.run() 84 | b.plot() 85 | 86 | 87 | def test_extended_assert_1(): 88 | import operator 89 | import random 90 | from simple_benchmark import assert_same_results 91 | 92 | funcs = [min, max] # will produce different results 93 | arguments = {2**i: [random.random() for _ in range(2**i)] for i in range(2, 10)} 94 | with pytest.raises(AssertionError): 95 | assert_same_results(funcs, arguments, equality_func=operator.eq) 96 | 97 | 98 | def test_extended_assert_2(): 99 | import operator 100 | import random 101 | from simple_benchmark import assert_not_mutating_input 102 | 103 | def sort(l): 104 | l.sort() # modifies the input 105 | return l 106 | 107 | funcs = [sorted, sort] 108 | arguments = {2**i: [random.random() for _ in range(2**i)] for i in range(2, 10)} 109 | with pytest.raises(AssertionError): 110 | assert_not_mutating_input(funcs, arguments, equality_func=operator.eq) 111 | 112 | 113 | @pytest.mark.slow 114 | def test_extended_time_and_max(): 115 | from simple_benchmark import benchmark 116 | from datetime import timedelta 117 | 118 | def O_n(n): 119 | for i in range(n): 120 | pass 121 | 122 | def O_n_squared(n): 123 | for i in range(n ** 2): 124 | pass 125 | 126 | def O_n_cube(n): 127 | for i in range(n ** 3): 128 | pass 129 | 130 | b = benchmark( 131 | [O_n, O_n_squared, O_n_cube], 132 | {2**i: 2**i for i in range(2, 15)}, 133 | time_per_benchmark=timedelta(milliseconds=500), 134 | maximum_time=timedelta(milliseconds=500) 135 | ) 136 | 137 | b.plot() 138 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Configuration file for the Sphinx documentation builder. 4 | # 5 | # This file does only contain a selection of the most common options. For a 6 | # full list see the documentation: 7 | # http://www.sphinx-doc.org/en/stable/config 8 | 9 | # -- Path setup -------------------------------------------------------------- 10 | 11 | # If extensions (or modules to document with autodoc) are in another directory, 12 | # add these directories to sys.path here. If the directory is relative to the 13 | # documentation root, use os.path.abspath to make it absolute, like shown here. 14 | # 15 | # import os 16 | # import sys 17 | # sys.path.insert(0, os.path.abspath('.')) 18 | 19 | 20 | # -- Project information ----------------------------------------------------- 21 | 22 | project = 'simple_benchmark' 23 | copyright = 'since 2018, Michael Seifert' 24 | author = 'Michael Seifert' 25 | 26 | 27 | def get_version(): 28 | with open('../../{}/__init__.py'.format(project)) as f: 29 | for line in f: 30 | if line.startswith('__version__'): 31 | return line.split(r"'")[1] 32 | 33 | 34 | # The short X.Y version 35 | version = get_version() 36 | # The full version, including alpha/beta/rc tags 37 | release = version 38 | 39 | 40 | # -- General configuration --------------------------------------------------- 41 | 42 | # If your documentation needs a minimal Sphinx version, state it here. 43 | # 44 | # needs_sphinx = '1.0' 45 | 46 | # Add any Sphinx extension module names here, as strings. They can be 47 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 48 | # ones. 49 | extensions = [ 50 | 'sphinx.ext.autodoc', 51 | 'sphinx.ext.intersphinx', 52 | 'sphinx.ext.viewcode', 53 | 'sphinx.ext.napoleon', 54 | 'sphinx.ext.autosummary' 55 | ] 56 | 57 | # Add any paths that contain templates here, relative to this directory. 58 | templates_path = ['_templates'] 59 | 60 | # The suffix(es) of source filenames. 61 | # You can specify multiple suffix as a list of string: 62 | # 63 | # source_suffix = ['.rst', '.md'] 64 | source_suffix = '.rst' 65 | 66 | # The master toctree document. 67 | master_doc = 'index' 68 | 69 | # The language for content autogenerated by Sphinx. Refer to documentation 70 | # for a list of supported languages. 71 | # 72 | # This is also used if you do content translation via gettext catalogs. 73 | # Usually you set "language" from the command line for these cases. 74 | language = None 75 | 76 | # List of patterns, relative to source directory, that match files and 77 | # directories to ignore when looking for source files. 78 | # This pattern also affects html_static_path and html_extra_path . 79 | exclude_patterns = [] 80 | 81 | # The name of the Pygments (syntax highlighting) style to use. 82 | pygments_style = 'sphinx' 83 | 84 | 85 | # -- Options for HTML output ------------------------------------------------- 86 | 87 | # The theme to use for HTML and HTML Help pages. See the documentation for 88 | # a list of builtin themes. 89 | # 90 | html_theme = 'pyramid' 91 | 92 | # Theme options are theme-specific and customize the look and feel of a theme 93 | # further. For a list of options available for each theme, see the 94 | # documentation. 95 | # 96 | # html_theme_options = {} 97 | 98 | # Add any paths that contain custom static files (such as style sheets) here, 99 | # relative to this directory. They are copied after the builtin static files, 100 | # so a file named "default.css" will overwrite the builtin "default.css". 101 | html_static_path = ['_static'] 102 | 103 | # Custom sidebar templates, must be a dictionary that maps document names 104 | # to template names. 105 | # 106 | # The default sidebars (for documents that don't match any pattern) are 107 | # defined by theme itself. Builtin themes are using these templates by 108 | # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', 109 | # 'searchbox.html']``. 110 | # 111 | # html_sidebars = {} 112 | 113 | 114 | # -- Options for HTMLHelp output --------------------------------------------- 115 | 116 | # Output file base name for HTML help builder. 117 | htmlhelp_basename = 'simple_benchmarkdoc' 118 | 119 | 120 | # -- Options for LaTeX output ------------------------------------------------ 121 | 122 | latex_elements = { 123 | # The paper size ('letterpaper' or 'a4paper'). 124 | # 125 | # 'papersize': 'letterpaper', 126 | 127 | # The font size ('10pt', '11pt' or '12pt'). 128 | # 129 | # 'pointsize': '10pt', 130 | 131 | # Additional stuff for the LaTeX preamble. 132 | # 133 | # 'preamble': '', 134 | 135 | # Latex figure (float) alignment 136 | # 137 | # 'figure_align': 'htbp', 138 | } 139 | 140 | # Grouping the document tree into LaTeX files. List of tuples 141 | # (source start file, target name, title, 142 | # author, documentclass [howto, manual, or own class]). 143 | latex_documents = [ 144 | (master_doc, 'simple_benchmark.tex', 'simple\\_benchmark Documentation', 145 | 'Michael Seifert', 'manual'), 146 | ] 147 | 148 | 149 | # -- Options for manual page output ------------------------------------------ 150 | 151 | # One entry per manual page. List of tuples 152 | # (source start file, name, description, authors, manual section). 153 | man_pages = [ 154 | (master_doc, 'simple_benchmark', 'simple_benchmark Documentation', 155 | [author], 1) 156 | ] 157 | 158 | 159 | # -- Options for Texinfo output ---------------------------------------------- 160 | 161 | # Grouping the document tree into Texinfo files. List of tuples 162 | # (source start file, target name, title, author, 163 | # dir menu entry, description, category) 164 | texinfo_documents = [ 165 | (master_doc, 'simple_benchmark', 'simple_benchmark Documentation', 166 | author, 'simple_benchmark', 'One line description of project.', 167 | 'Miscellaneous'), 168 | ] 169 | 170 | 171 | # -- Extension configuration ------------------------------------------------- 172 | 173 | # -- Options for intersphinx extension --------------------------------------- 174 | 175 | # Example configuration for intersphinx: refer to the Python standard library. 176 | intersphinx_mapping = {'https://docs.python.org/3/': None} 177 | -------------------------------------------------------------------------------- /simple_benchmark/__main__.py: -------------------------------------------------------------------------------- 1 | # Licensed under Apache License Version 2.0 - see LICENSE 2 | import argparse 3 | import datetime 4 | import importlib 5 | import importlib.util 6 | import inspect 7 | import pathlib 8 | 9 | import matplotlib.pyplot as plt 10 | 11 | from simple_benchmark import BenchmarkBuilder 12 | 13 | 14 | def _startswith_and_remainder(string, prefix): 15 | """Returns if the string starts with the prefix and the string without the prefix.""" 16 | if string.startswith(prefix): 17 | return True, string[len(prefix):] 18 | else: 19 | return False, '' 20 | 21 | def _import_file(filename, filepath): 22 | """This loads a python module by filepath""" 23 | spec = importlib.util.spec_from_file_location(filename, filepath) 24 | foo = importlib.util.module_from_spec(spec) 25 | spec.loader.exec_module(foo) 26 | return foo 27 | 28 | def _get_version_for_module(module_name): 29 | """Imports the module by its name and tries to get the version. Could fail...""" 30 | module = importlib.import_module(module_name) 31 | return module.__version__ 32 | 33 | 34 | def _hacky_parse_sig(function): 35 | """Extracts a useable alias by inspecting the function signature.""" 36 | sig = inspect.signature(function) 37 | # Yeah, this looks for a parameter 38 | function_parameter = sig.parameters.get('func', None) 39 | if function_parameter: 40 | benchmarked_function = function_parameter.default 41 | if benchmarked_function: 42 | # __module__ will likely contain additional submodules. However 43 | # only the main module is (probably) of interest. 44 | module = benchmarked_function.__module__.split('.')[0] 45 | # Not every function has a __name__ attribute. But the rest of the 46 | # function is hacky too... 47 | name = benchmarked_function.__name__ 48 | try: 49 | return f"{module} {name} ({_get_version_for_module(module)})" 50 | except Exception: 51 | # Something went wrong while determining the version. That's 52 | # okay, just omit it then... 53 | return f"{module} {name}" 54 | 55 | def main(filename, outfilename, figsize, time_per_benchmark, write_csv, verbose): 56 | if verbose: 57 | print("Performing a Benchmark using simple_benchmark") 58 | print("---------------------------------------------") 59 | print("Effective Options:") 60 | print(f"Input-File: {filename}") 61 | print(f"Output-File: {outfilename}") 62 | print(f"Time per individual benchmark: {time_per_benchmark.total_seconds()} seconds") 63 | print(f"Figure size (inches): {figsize}") 64 | print(f"Verbose: {verbose}") 65 | 66 | path = pathlib.Path(filename).absolute() 67 | filename = path.name 68 | 69 | if verbose: 70 | print("") 71 | print("Process file") 72 | print("------------") 73 | 74 | module = _import_file(filename, path) 75 | 76 | b = BenchmarkBuilder(time_per_benchmark) 77 | 78 | for function_name in sorted(dir(module)): 79 | function = getattr(module, function_name) 80 | is_benchmark, benchmark_name = _startswith_and_remainder(function_name, 'bench_') 81 | if is_benchmark: 82 | try: 83 | alias = _hacky_parse_sig(function) 84 | except Exception: 85 | pass 86 | 87 | if not alias: 88 | alias = benchmark_name 89 | 90 | b.add_function(alias=alias)(function) 91 | continue 92 | 93 | is_args, args_name = _startswith_and_remainder(function_name, 'args_') 94 | if is_args: 95 | b.add_arguments(args_name)(function) 96 | continue 97 | 98 | if verbose: 99 | print("successful") 100 | print("") 101 | print("Running Benchmark") 102 | print("-----------------") 103 | print("this may take a while...") 104 | 105 | r = b.run() 106 | 107 | if verbose: 108 | print("successful") 109 | print("") 110 | print("Benchmark Result") 111 | print("----------------") 112 | print(r.to_pandas_dataframe()) 113 | 114 | plt.figure(figsize=figsize) 115 | r.plot() 116 | plt.savefig(outfilename) 117 | 118 | out_file_path = pathlib.Path(outfilename) 119 | 120 | if verbose: 121 | print("") 122 | print(f"Written benchmark plot to {out_file_path.absolute()}") 123 | 124 | if write_csv: 125 | csv_file_path = out_file_path.with_suffix('.csv') 126 | # wtf ... pandas is using %-formatting ... 127 | # well, so %.9f should suppress scientific notation and display 9 128 | # decimals (nanosecond-resolution more is probably not useful anyway). 129 | r.to_pandas_dataframe().to_csv(str(csv_file_path), float_format='%.9f') 130 | print(f"Written CSV to {csv_file_path.absolute()}") 131 | 132 | 133 | if __name__ == '__main__': 134 | parser = argparse.ArgumentParser(description='Benchmark a file', formatter_class=argparse.ArgumentDefaultsHelpFormatter) 135 | parser.add_argument('filename', help='the file to run the benchmark on.') 136 | parser.add_argument('out', help='Specifies the output file for the plot') 137 | parser.add_argument('-s', '--figsize', help='Specify the output size in inches, needs to be wrapped in quotes on most shells, e.g. "15, 9"', default='15, 9') 138 | parser.add_argument('--time-per-benchmark', help='The target time for each individual benchmark in seconds', default='0.1') 139 | parser.add_argument('-v', '--verbose', help='prints additional information on stdout', action="store_true") 140 | parser.add_argument('--write-csv', help='Writes an additional CSV file of the results', action="store_true") 141 | 142 | args = parser.parse_args() 143 | 144 | filename = args.filename 145 | outfilename = args.out 146 | 147 | verbose = args.verbose 148 | figsize = [int(value) for value in args.figsize.split(',')] 149 | time_per_benchmark = datetime.timedelta(seconds=float(args.time_per_benchmark)) 150 | write_csv = args.write_csv 151 | main(filename, outfilename, figsize=figsize, time_per_benchmark=time_per_benchmark, write_csv=write_csv, verbose=verbose) 152 | -------------------------------------------------------------------------------- /docs/source/extended.rst: -------------------------------------------------------------------------------- 1 | Extended examples 2 | ================= 3 | 4 | BenchmarkBuilder 5 | ---------------- 6 | 7 | The :class:`simple_benchmark.BenchmarkBuilder` class can be used to build a benchmark using decorators, essentially 8 | it is just a wrapper around :func:`simple_benchmark.benchmark`. 9 | 10 | For example to compare different approaches to calculate the sum of a list of floats:: 11 | 12 | from simple_benchmark import BenchmarkBuilder 13 | import math 14 | 15 | bench = BenchmarkBuilder() 16 | 17 | @bench.add_function() 18 | def sum_using_loop(lst): 19 | sum_ = 0 20 | for item in lst: 21 | sum_ += item 22 | return sum_ 23 | 24 | @bench.add_function() 25 | def sum_using_range_loop(lst): 26 | sum_ = 0 27 | for idx in range(len(lst)): 28 | sum_ += lst[idx] 29 | return sum_ 30 | 31 | bench.use_random_lists_as_arguments(sizes=[2**i for i in range(2, 15)]) 32 | 33 | bench.add_functions([sum, math.fsum]) 34 | 35 | b = bench.run() 36 | b.plot() 37 | # To save the plotted benchmark as PNG file. 38 | import matplotlib.pyplot as plt 39 | plt.savefig('sum_list_example.png') 40 | 41 | .. image:: ./sum_list_example.png 42 | 43 | MultiArgument 44 | ------------- 45 | 46 | The :py:class:`simple_benchmark.MultiArgument` class can be used to provide multiple arguments to the 47 | functions that should be benchmarked:: 48 | 49 | from itertools import starmap 50 | from operator import add 51 | from random import random 52 | 53 | from simple_benchmark import BenchmarkBuilder, MultiArgument 54 | 55 | bench = BenchmarkBuilder() 56 | 57 | @bench.add_function() 58 | def list_addition_zip(list1, list2): 59 | res = [] 60 | for item1, item2 in zip(list1, list2): 61 | res.append(item1 + item2) 62 | return res 63 | 64 | @bench.add_function() 65 | def list_addition_index(list1, list2): 66 | res = [] 67 | for idx in range(len(list1)): 68 | res.append(list1[idx] + list2[idx]) 69 | return res 70 | 71 | @bench.add_function() 72 | def list_addition_map_zip(list1, list2): 73 | return list(starmap(add, zip(list1, list2))) 74 | 75 | @bench.add_arguments(name='list sizes') 76 | def benchmark_arguments(): 77 | for size_exponent in range(2, 15): 78 | size = 2**size_exponent 79 | arguments = MultiArgument([ 80 | [random() for _ in range(size)], 81 | [random() for _ in range(size)]]) 82 | yield size, arguments 83 | 84 | b = bench.run() 85 | b.plot() 86 | # To save the plotted benchmark as PNG file. 87 | import matplotlib.pyplot as plt 88 | plt.savefig('list_add_example.png') 89 | 90 | .. image:: ./list_add_example.png 91 | 92 | Asserting correctness 93 | --------------------- 94 | 95 | Besides comparing the timings it's also important to assert that the approaches actually 96 | produce the same outcomes and don't modify the input arguments. 97 | 98 | To compare the results there is :py:func:`simple_benchmark.assert_same_results` 99 | (or in case you use BenchmarkBuilder :py:meth:`simple_benchmark.BenchmarkBuilder.assert_same_results`):: 100 | 101 | import operator 102 | import random 103 | from simple_benchmark import assert_same_results 104 | 105 | funcs = [min, max] # will produce different results 106 | arguments = {2**i: [random.random() for _ in range(2**i)] for i in range(2, 10)} 107 | assert_same_results(funcs, arguments, equality_func=operator.eq) 108 | 109 | And to compare that the inputs were not modified :py:func:`simple_benchmark.assert_not_mutating_input` 110 | (or in case you use BenchmarkBuilder :py:meth:`simple_benchmark.BenchmarkBuilder.assert_not_mutating_input`):: 111 | 112 | import operator 113 | import random 114 | from simple_benchmark import assert_not_mutating_input 115 | 116 | def sort(l): 117 | l.sort() # modifies the input 118 | return l 119 | 120 | funcs = [sorted, sort] 121 | arguments = {2**i: [random.random() for _ in range(2**i)] for i in range(2, 10)} 122 | assert_not_mutating_input(funcs, arguments, equality_func=operator.eq) 123 | 124 | Both will produce an :py:class:`AssertionError` if they gave different results or mutate the input arguments. 125 | 126 | Typically the ``equality_func`` will be one of these: 127 | 128 | - :py:func:`operator.eq` will work for most Python objects. 129 | - :py:func:`math.isclose` will work for :py:class:`float` that may be close but not equal. 130 | - ``numpy.array_equal`` will work for element-wise comparison of NumPy arrays. 131 | - ``numpy.allclose`` will work for element-wise comparison of NumPy arrays containing floats that may be close but not equal. 132 | 133 | The :py:func:`simple_benchmark.assert_not_mutating_input` also accepts an optional argument that needs to be used in case 134 | the argument is not trivially copyable. It expects a function that takes the argument as input and should 135 | return a deep-copy of the argument. 136 | 137 | Times for each benchmark 138 | ------------------------ 139 | 140 | The benchmark will run each function on each of the arguments for a certain amount of times. Generally the results will 141 | be more accurate if one increases the number of times the function is executed during each benchmark. But the 142 | benchmark will also take longer. 143 | 144 | To control the time one benchmark should take one can use the ``time_per_benchmark`` argument. This controls how much 145 | time each function will take for each argument. The default is 0.1s (100 milliseconds) but the value is ignored 146 | for calls that either take very short (then it will finish faster) or very slow (because the benchmark tries to do at 147 | least a few calls). 148 | 149 | Another option is to control the maximum time a single function call may take ``maximum_time``. If the first call of this 150 | function exceeds the ``maximum_time`` the function will be excluded from the benchmark from this argument on. 151 | 152 | - To control the quality of the benchmark the ``time_per_benchmark`` can be used. 153 | - To avoid excessive benchmarking times one can use ``maximum_time``. 154 | 155 | An example showing both in action:: 156 | 157 | from simple_benchmark import benchmark 158 | from datetime import timedelta 159 | 160 | def O_n(n): 161 | for i in range(n): 162 | pass 163 | 164 | def O_n_squared(n): 165 | for i in range(n ** 2): 166 | pass 167 | 168 | def O_n_cube(n): 169 | for i in range(n ** 3): 170 | pass 171 | 172 | b = benchmark( 173 | [O_n, O_n_squared, O_n_cube], 174 | {2**i: 2**i for i in range(2, 15)}, 175 | time_per_benchmark=timedelta(milliseconds=500), 176 | maximum_time=timedelta(milliseconds=500) 177 | ) 178 | 179 | b.plot() 180 | # To save the plotted benchmark as PNG file. 181 | import matplotlib.pyplot as plt 182 | plt.savefig('time_example.png') 183 | 184 | .. image:: ./time_example.png 185 | 186 | Examples on StackOverflow 187 | ------------------------- 188 | 189 | In some cases it's probably best to see how it can be used on some real-life problems: 190 | 191 | - `Count the number of non zero values in a numpy array in Numba `_ 192 | - `When numba is effective? `_ 193 | - `Range with repeated consecutive numbers `_ 194 | - `Concatenate tuples using sum() `_ 195 | - `How to retrieve an element from a set without removing it? `_ 196 | - `What exactly is the optimization "functools.partial" is making? `_ 197 | - `Nested lambda statements when sorting lists `_ 198 | - `How to make a flat list out of list of lists? `_ 199 | - `How do you remove duplicates from a list whilst preserving order? `_ 200 | - `Iterating over every two elements in a list `_ 201 | - `Cython - efficiently filtering a typed memoryview `_ 202 | - `Python's sum vs. NumPy's numpy.sum `_ 203 | - `Finding longest run in a list `_ 204 | - `Remove duplicate dict in list in Python `_ 205 | - `How do I find the duplicates in a list and create another list with them? `_ 206 | - `Suppress key addition in collections.defaultdict `_ 207 | - `Numpy first occurrence of value greater than existing value `_ 208 | - `Count the number of times an item occurs in a sequence using recursion Python `_ 209 | - `Converting a series of ints to strings - Why is apply much faster than astype? `_ 210 | 211 | See also `Results for "simple_benchmark" on StackOverflow `_. 212 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright since 2018 Michael Seifert 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /simple_benchmark/__init__.py: -------------------------------------------------------------------------------- 1 | # Licensed under Apache License Version 2.0 - see LICENSE 2 | """ 3 | .. warning:: 4 | This package is under active development. API changes are very likely. 5 | 6 | This package aims to give an easy way to benchmark several functions for 7 | different inputs and provide ways to visualize the benchmark results. 8 | 9 | To utilize the full features (visualization and post-processing) you need to 10 | install the optional dependencies: 11 | 12 | - NumPy 13 | - pandas 14 | - matplotlib 15 | """ 16 | 17 | __version__ = '0.1.0' 18 | 19 | __all__ = [ 20 | 'assert_same_results', 'assert_not_mutating_input', 'benchmark', 21 | 'BenchmarkBuilder', 'BenchmarkResult', 'MultiArgument' 22 | ] 23 | 24 | import collections 25 | import copy 26 | import datetime 27 | import functools 28 | import itertools 29 | import logging 30 | import platform 31 | import pprint 32 | import random 33 | import sys 34 | import timeit 35 | import warnings 36 | 37 | _DEFAULT_ARGUMENT_NAME = '' 38 | _DEFAULT_TIME_PER_BENCHMARK = datetime.timedelta(milliseconds=100) 39 | _DEFAULT_ESTIMATOR = min 40 | _DEFAULT_COPY_FUNC = copy.deepcopy 41 | 42 | _MISSING = object() 43 | _MSG_DECORATOR_FACTORY = ( 44 | 'A decorator factory cannot be applied to a function directly. The decorator factory returns a decorator when ' 45 | 'called so if no arguments should be applied then simply call the decorator factory without arguments.' 46 | ) 47 | _MSG_MISSING_ARGUMENTS = "The BenchmarkBuilder instance is missing arguments for the functions." 48 | 49 | _logger = logging.getLogger(__name__) 50 | _NaN = float('nan') 51 | _TIMEDELTA_ZERO = datetime.timedelta(0) 52 | 53 | 54 | class MultiArgument(tuple): 55 | """Class that behaves like a tuple but signals to the benchmark that it 56 | should pass multiple arguments to the function to benchmark. 57 | """ 58 | pass 59 | 60 | 61 | def _try_importing_matplotlib(): 62 | """Tries to import matplotlib 63 | 64 | Returns 65 | ------- 66 | pyplot : module 67 | The pyplot module from matplotlib 68 | 69 | Raises 70 | ------ 71 | ImportError 72 | In case matplotlib is not installed. 73 | """ 74 | try: 75 | import matplotlib.pyplot as plt 76 | except ImportError: 77 | raise ImportError('simple_benchmark requires matplotlib for the ' 78 | 'plotting functionality.') 79 | return plt 80 | 81 | 82 | def _get_python_bits(): 83 | """Is the current platform 64bit. 84 | 85 | Returns 86 | ------- 87 | result : string 88 | The string '64bit' in case the Python installation uses 64bit or more, otherwise '32bit'. 89 | """ 90 | return '64bit' if sys.maxsize > 2 ** 32 else '32bit' 91 | 92 | 93 | class TimingParams(object): 94 | def __init__(self, repeats, number, stop, timing): 95 | self.repeats = repeats # The number of repeats. 96 | self.number = number # The number of timings in each repetition. 97 | self.stop = stop # Estimate was too slow, function should not be timed. 98 | self.timing = timing # Just informational: The time it took the estimate to run the function. 99 | 100 | 101 | def _estimate_number_of_repeats(func, target_time, maximum_time): 102 | """Estimate the number of repeats for a function so that the benchmark will take a specific time. 103 | 104 | In case the function is very slow or really fast some default values are returned. 105 | 106 | Parameters 107 | ---------- 108 | func : callable 109 | The function to time. Must not have required arguments! 110 | target_time : datetime.timedelta 111 | The amount of time the benchmark should roughly take. 112 | maximum_time : datetime.timedelta or None 113 | If not None it represents the maximum time the first call of the function may take 114 | otherwise a stop will be signalled. 115 | 116 | Returns 117 | ------- 118 | timing_parameter : TimingParams 119 | The parameter used for the actual timings. 120 | """ 121 | # Just for a quick reference: 122 | # One millisecond is 1e-3 123 | # One microsecond is 1e-6 124 | # One nanosecond is 1e-9 125 | single_time = timeit.timeit(func, number=1) 126 | 127 | if maximum_time is not None and single_time > maximum_time.total_seconds(): 128 | return TimingParams(0, 0, stop=True, timing=single_time) 129 | 130 | # Get a more accurate baseline if the function was really fast 131 | if single_time < 1e-6: 132 | single_time = timeit.timeit(func, number=1000) / 1000 133 | if single_time < 1e-5: 134 | single_time = timeit.timeit(func, number=100) / 100 135 | elif single_time < 1e-4: 136 | single_time = timeit.timeit(func, number=10) / 10 137 | 138 | n_repeats = int(target_time.total_seconds() / single_time) 139 | # The timeit execution should be at least 10-100us so that the granularity 140 | # of the timer isn't a limiting factor. 141 | if single_time < 1e-4: 142 | factor = 1e-4 / single_time 143 | return TimingParams(repeats=max(int(n_repeats // factor), 1), number=max(int(factor), 1), stop=False, 144 | timing=single_time) 145 | # Otherwise the number of timings each repeat should be 1. 146 | # However make sure there are at least 3 repeats for each function! 147 | return TimingParams(repeats=max(n_repeats, 3), number=1, stop=False, timing=single_time) 148 | 149 | 150 | def _get_bound_func(func, argument): 151 | """Return a function where the arguments are already bound to the function.""" 152 | if isinstance(argument, MultiArgument): 153 | return functools.partial(func, *argument) 154 | else: 155 | return functools.partial(func, argument) 156 | 157 | 158 | def _get_function_name(func, aliases): 159 | """Returns the associated name of a function.""" 160 | try: 161 | return aliases[func] 162 | except KeyError: 163 | # Has to be a different branch because not every function has a 164 | # __name__ attribute. So we cannot simply use the dictionaries `get` 165 | # with default. 166 | try: 167 | return func.__name__ 168 | except AttributeError: 169 | raise TypeError('function "func" does not have a __name__ attribute. ' 170 | 'Please use "function_aliases" to provide a function name alias.') 171 | 172 | 173 | def assert_same_results(funcs, arguments, equality_func): 174 | """Asserts that all functions return the same result. 175 | 176 | .. versionadded:: 0.1.0 177 | 178 | Parameters 179 | ---------- 180 | funcs : iterable of callables 181 | The functions to check. 182 | arguments : dict 183 | A dictionary containing where the key represents the reported value 184 | (for example an integer representing the list size) as key and the argument 185 | for the functions (for example the list) as value. 186 | In case you want to plot the result it should be sorted and ordered 187 | (e.g. an :py:class:`collections.OrderedDict` or a plain dict if you are 188 | using Python 3.7 or later). 189 | equality_func : callable 190 | The function that determines if the results are equal. This function should 191 | accept two arguments and return a boolean (True if the results should be 192 | considered equal, False if not). 193 | 194 | Raises 195 | ------ 196 | AssertionError 197 | In case any two results are not equal. 198 | """ 199 | funcs = list(funcs) 200 | for arg in arguments.values(): 201 | first_result = _MISSING 202 | for func in funcs: 203 | bound_func = _get_bound_func(func, arg) 204 | result = bound_func() 205 | if first_result is _MISSING: 206 | first_result = result 207 | else: 208 | assert equality_func(first_result, result), (func, first_result, result) 209 | 210 | 211 | def assert_not_mutating_input(funcs, arguments, equality_func, copy_func=_DEFAULT_COPY_FUNC): 212 | """Asserts that none of the functions mutate the arguments. 213 | 214 | .. versionadded:: 0.1.0 215 | 216 | Parameters 217 | ---------- 218 | funcs : iterable of callables 219 | The functions to check. 220 | arguments : dict 221 | A dictionary containing where the key represents the reported value 222 | (for example an integer representing the list size) as key and the argument 223 | for the functions (for example the list) as value. 224 | In case you want to plot the result it should be sorted and ordered 225 | (e.g. an :py:class:`collections.OrderedDict` or a plain dict if you are 226 | using Python 3.7 or later). 227 | equality_func : callable 228 | The function that determines if the results are equal. This function should 229 | accept two arguments and return a boolean (True if the results should be 230 | considered equal, False if not). 231 | copy_func : callable, optional 232 | The function that is used to copy the original argument. 233 | Default is :py:func:`copy.deepcopy`. 234 | 235 | Raises 236 | ------ 237 | AssertionError 238 | In case any two results are not equal. 239 | 240 | Notes 241 | ----- 242 | In case the arguments are :py:class:`MultiArgument` then the copy_func and the 243 | equality_func get these :py:class:`MultiArgument` as single arguments and need 244 | to handle them appropriately. 245 | """ 246 | funcs = list(funcs) 247 | for arg in arguments.values(): 248 | original_arguments = copy_func(arg) 249 | for func in funcs: 250 | bound_func = _get_bound_func(func, arg) 251 | bound_func() 252 | assert equality_func(original_arguments, arg), (func, original_arguments, arg) 253 | 254 | 255 | def benchmark( 256 | funcs, 257 | arguments, 258 | argument_name=_DEFAULT_ARGUMENT_NAME, 259 | warmups=None, 260 | time_per_benchmark=_DEFAULT_TIME_PER_BENCHMARK, 261 | function_aliases=None, 262 | estimator=_DEFAULT_ESTIMATOR, 263 | maximum_time=None): 264 | """Create a benchmark suite for different functions and for different arguments. 265 | 266 | Parameters 267 | ---------- 268 | funcs : iterable of callables 269 | The functions to benchmark. 270 | arguments : dict 271 | A dictionary containing where the key represents the reported value 272 | (for example an integer representing the list size) as key and the argument 273 | for the functions (for example the list) as value. 274 | In case you want to plot the result it should be sorted and ordered 275 | (e.g. an :py:class:`collections.OrderedDict` or a plain dict if you are 276 | using Python 3.7 or later). 277 | argument_name : str, optional 278 | The name of the reported value. For example if the arguments represent 279 | list sizes this could be `"size of the list"`. 280 | Default is an empty string. 281 | warmups : None or iterable of callables, optional 282 | If not None it specifies the callables that need a warmup call 283 | before being timed. That is so, that caches can be filled or 284 | jitters to kick in. 285 | Default is None. 286 | time_per_benchmark : datetime.timedelta, optional 287 | Each benchmark should take approximately this time. 288 | The value is ignored for functions that take very little time or very long. 289 | Default is 0.1 seconds. 290 | 291 | .. versionchanged:: 0.1.0 292 | Now requires a :py:class:`datetime.timedelta` instead of a :py:class:`float`. 293 | function_aliases : None or dict, optional 294 | If not None it should be a dictionary containing the function as key 295 | and the name of the function as value. The value will be used in the 296 | final reports and plots. 297 | Default is None. 298 | estimator : callable, optional 299 | Each function is called with each argument multiple times and each 300 | timing is recorded. The benchmark_estimator (by default :py:func:`min`) 301 | is used to reduce this list of timings to one final value. 302 | The minimum is generally a good way to estimate how fast a function can 303 | run (see also the discussion in :py:meth:`timeit.Timer.repeat`). 304 | Default is :py:func:`min`. 305 | maximum_time : datetime.timedelta or None, optional 306 | If not None it represents the maximum time the first call of the function may take. 307 | If exceeded the benchmark will stop evaluating the function from then on. 308 | Default is None. 309 | 310 | .. versionadded:: 0.1.0 311 | 312 | Returns 313 | ------- 314 | benchmark : BenchmarkResult 315 | The result of the benchmarks. 316 | 317 | See also 318 | -------- 319 | BenchmarkBuilder 320 | """ 321 | if not isinstance(time_per_benchmark, datetime.timedelta): 322 | warnings.warn("Using a number as 'time_per_benchmark' is deprecated since version 0.1.0. " 323 | "Use 'datetime.timedelta(seconds={0})' instead".format(time_per_benchmark), 324 | DeprecationWarning) 325 | time_per_benchmark = datetime.timedelta(seconds=time_per_benchmark) 326 | if time_per_benchmark <= _TIMEDELTA_ZERO: 327 | raise ValueError("'time_per_benchmark' ({}) must be positive.".format(time_per_benchmark)) 328 | if maximum_time is not None and maximum_time <= _TIMEDELTA_ZERO: 329 | raise ValueError("'maximum_time' ({}) must be positive.".format(maximum_time)) 330 | funcs = list(funcs) 331 | warm_up_calls = {func: 0 for func in funcs} 332 | if warmups is not None: 333 | for func in warmups: 334 | warm_up_calls[func] = 1 335 | function_aliases = function_aliases or {} 336 | stopped = set() 337 | 338 | timings = {func: [] for func in funcs} 339 | for arg_name, arg in arguments.items(): 340 | _logger.info("Benchmark for argument: {}".format(arg_name)) 341 | for func, timing_list in timings.items(): 342 | function_name = _get_function_name(func, function_aliases) 343 | _logger.info("Benchmark function: {}".format(function_name)) 344 | if func in stopped: 345 | _logger.info("SKIPPED: Not benchmarking function because a previous run exceeded the maximum time.") 346 | time_per_run = _NaN 347 | else: 348 | bound_func = _get_bound_func(func, arg) 349 | for _ in itertools.repeat(None, times=warm_up_calls[func]): 350 | bound_func() 351 | params = _estimate_number_of_repeats(bound_func, time_per_benchmark, maximum_time) 352 | if params.stop: 353 | _logger.info( 354 | "STOPPED: benchmarking because the first run ({}) exceeded the maximum_time '{}'." 355 | .format(datetime.timedelta(seconds=params.timing), maximum_time)) 356 | stopped.add(func) 357 | time_per_run = _NaN 358 | else: 359 | _logger.info( 360 | "RUN: benchmark with {} x {} runs (repeats & number) for estimated time {}." 361 | .format(params.repeats, params.number, datetime.timedelta(seconds=params.timing))) 362 | # As per the timeit module documentation a very good approximation 363 | # of a timing is found by repeating the benchmark and using the 364 | # minimum. 365 | times = timeit.repeat(bound_func, number=params.number, repeat=params.repeats) 366 | time = estimator(times) 367 | time_per_run = time / params.number 368 | timing_list.append(time_per_run) 369 | 370 | return BenchmarkResult(timings, function_aliases, arguments, argument_name) 371 | 372 | 373 | class BenchmarkResult(object): 374 | """A class holding a benchmarking result that provides additional printing and plotting functions.""" 375 | def __init__(self, timings, function_aliases, arguments, argument_name): 376 | self._timings = timings 377 | self.function_aliases = function_aliases 378 | self._arguments = arguments 379 | self._argument_name = argument_name 380 | 381 | def __str__(self): 382 | """Prints the results as table.""" 383 | try: 384 | return str(self.to_pandas_dataframe()) 385 | except ImportError: 386 | return pprint.pformat({self._function_name(k): v for k, v in self._timings.items()}) 387 | 388 | __repr__ = __str__ 389 | 390 | def _function_name(self, func): 391 | """Returns the function name taking the aliases into account.""" 392 | return _get_function_name(func, self.function_aliases) 393 | 394 | @staticmethod 395 | def _get_title(): 396 | """Returns a string containing some information about Python and the machine.""" 397 | return "{0} {1} {2} ({3} {4})".format( 398 | platform.python_implementation(), platform.python_version(), platform.python_compiler(), 399 | platform.system(), platform.release()) 400 | 401 | def to_pandas_dataframe(self): 402 | """Return the timing results as pandas DataFrame. This is the preferred way of accessing the text form of the 403 | timings. 404 | 405 | Returns 406 | ------- 407 | results : pandas.DataFrame 408 | The timings as DataFrame. 409 | 410 | Warns 411 | ----- 412 | UserWarning 413 | In case multiple functions have the same name. 414 | 415 | Raises 416 | ------ 417 | ImportError 418 | If pandas isn't installed. 419 | """ 420 | try: 421 | import pandas as pd 422 | except ImportError: 423 | raise ImportError('simple_benchmark requires pandas for this method.') 424 | dct = {self._function_name(func): timings for func, timings in self._timings.items()} 425 | if len(dct) != len(self._timings): 426 | warnings.warn('Some timings are not included in the result. Likely ' 427 | 'because multiple functions have the same name. You ' 428 | 'can add an alias to the `function_aliases` mapping ' 429 | 'to avoid this problem.', UserWarning) 430 | 431 | return pd.DataFrame(dct, index=list(self._arguments)) 432 | 433 | def plot_difference_percentage(self, relative_to, ax=None): 434 | """Plot the benchmarks relative to one of the benchmarks with 435 | percentages on the y-axis. 436 | 437 | Parameters 438 | ---------- 439 | relative_to : callable 440 | The benchmarks are plotted relative to the timings of the given 441 | function. 442 | ax : matplotlib.axes.Axes or None, optional 443 | The axes on which to plot. If None plots on the currently active axes. 444 | 445 | Raises 446 | ------ 447 | ImportError 448 | If matplotlib isn't installed. 449 | """ 450 | plt = _try_importing_matplotlib() 451 | ax = ax or plt.gca() 452 | 453 | self.plot(relative_to=relative_to, ax=ax) 454 | 455 | ax.set_yscale('linear') 456 | # Use percentage always including the sign for the y ticks. 457 | ticks = ax.get_yticks() 458 | ax.set_yticklabels(['{:+.1f}%'.format((x-1) * 100) for x in ticks]) 459 | 460 | def plot(self, relative_to=None, ax=None): 461 | """Plot the benchmarks, either relative or absolute. 462 | 463 | Parameters 464 | ---------- 465 | relative_to : callable or None, optional 466 | If None it will plot the absolute timings, otherwise it will use the 467 | given *relative_to* function as reference for the timings. 468 | ax : matplotlib.axes.Axes or None, optional 469 | The axes on which to plot. If None plots on the currently active axes. 470 | 471 | Raises 472 | ------ 473 | ImportError 474 | If matplotlib isn't installed. 475 | """ 476 | plt = _try_importing_matplotlib() 477 | ax = ax or plt.gca() 478 | 479 | x_axis = list(self._arguments) 480 | 481 | for func, timing in self._timings.items(): 482 | label = self._function_name(func) 483 | if relative_to is None: 484 | plot_time = timing 485 | else: 486 | plot_time = [time / ref for time, ref in zip(self._timings[func], self._timings[relative_to])] 487 | ax.plot(x_axis, plot_time, label=label) 488 | 489 | ax.set_title(BenchmarkResult._get_title()) 490 | ax.set_xscale('log') 491 | ax.set_yscale('log') 492 | ax.set_xlabel(self._argument_name) 493 | if relative_to is None: 494 | ax.set_ylabel('time [seconds]') 495 | else: 496 | ax.set_ylabel('time relative to "{}"'.format(self._function_name(relative_to))) 497 | ax.grid(which='both') 498 | ax.legend() 499 | 500 | def plot_both(self, relative_to): 501 | """Plot both the absolute times and the relative time. 502 | 503 | Parameters 504 | ---------- 505 | relative_to : callable or None 506 | If None it will plot the absolute timings, otherwise it will use the 507 | given *relative_to* function as reference for the timings. 508 | 509 | Raises 510 | ------ 511 | ImportError 512 | If matplotlib isn't installed. 513 | """ 514 | plt = _try_importing_matplotlib() 515 | 516 | f, (ax1, ax2) = plt.subplots(2, 1, sharex=True) 517 | self.plot(ax=ax1) 518 | self.plot(ax=ax2, relative_to=relative_to) 519 | 520 | 521 | class BenchmarkBuilder(object): 522 | """A class useful for building benchmarks by adding decorators to the functions instead of collecting them later. 523 | 524 | Parameters 525 | ---------- 526 | time_per_benchmark : datetime.timedelta, optional 527 | Each benchmark should take approximately this time. 528 | The value is ignored for functions that take very little time or very long. 529 | Default is 0.1 seconds. 530 | 531 | .. versionchanged:: 0.1.0 532 | Now requires a :py:class:`datetime.timedelta` instead of a :py:class:`float`. 533 | estimator : callable, optional 534 | Each function is called with each argument multiple times and each 535 | timing is recorded. The benchmark_estimator (by default :py:func:`min`) 536 | is used to reduce this list of timings to one final value. 537 | The minimum is generally a good way to estimate how fast a function can 538 | run (see also the discussion in :py:meth:`timeit.Timer.repeat`). 539 | Default is :py:func:`min`. 540 | maximum_time : datetime.timedelta or None, optional 541 | If not None it represents the maximum time the first call of the function may take. 542 | If exceeded the benchmark will stop evaluating the function from then on. 543 | Default is None. 544 | 545 | .. versionadded:: 0.1.0 546 | 547 | See also 548 | -------- 549 | benchmark 550 | """ 551 | def __init__(self, time_per_benchmark=_DEFAULT_TIME_PER_BENCHMARK, estimator=_DEFAULT_ESTIMATOR, maximum_time=None): 552 | self._funcs = [] 553 | self._arguments = collections.OrderedDict() 554 | self._warmups = [] 555 | self._function_aliases = {} 556 | self._argument_name = _DEFAULT_ARGUMENT_NAME 557 | self._time_per_benchmark = time_per_benchmark 558 | self._estimator = estimator 559 | self._maximum_time = maximum_time 560 | 561 | def add_functions(self, functions): 562 | """Add multiple functions to the benchmark. 563 | 564 | Parameters 565 | ---------- 566 | functions : iterable of callables 567 | The functions to add to the benchmark 568 | """ 569 | self._funcs.extend(functions) 570 | 571 | def add_function(self, warmups=False, alias=None): 572 | """A decorator factory that returns a decorator that can be used to add a function to the benchmark. 573 | 574 | Parameters 575 | ---------- 576 | warmups : bool, optional 577 | If true the function is called once before each benchmark run. 578 | Default is False. 579 | alias : str or None, optional 580 | If None then the displayed function name is the name of the function, otherwise the string is used when 581 | the function is referred to. 582 | Default is None. 583 | 584 | Returns 585 | ------- 586 | decorator : callable 587 | The decorator that adds the function to the benchmark. 588 | 589 | Raises 590 | ------ 591 | TypeError 592 | In case ``name`` is a callable. 593 | """ 594 | if callable(warmups): 595 | raise TypeError(_MSG_DECORATOR_FACTORY) 596 | 597 | def inner(func): 598 | self._funcs.append(func) 599 | if warmups: 600 | self._warmups.append(func) 601 | if alias is not None: 602 | self._function_aliases[func] = alias 603 | return func 604 | 605 | return inner 606 | 607 | def add_arguments(self, name=_DEFAULT_ARGUMENT_NAME): 608 | """A decorator factory that returns a decorator that can be used to add a function that produces the x-axis 609 | values and the associated test data for the benchmark. 610 | 611 | Parameters 612 | ---------- 613 | name : str, optional 614 | The label for the x-axis. 615 | 616 | Returns 617 | ------- 618 | decorator : callable 619 | The decorator that adds the function that produces the x-axis values and the test data to the benchmark. 620 | 621 | Raises 622 | ------ 623 | TypeError 624 | In case ``name`` is a callable. 625 | """ 626 | if callable(name): 627 | raise TypeError(_MSG_DECORATOR_FACTORY) 628 | 629 | def inner(func): 630 | self._arguments = collections.OrderedDict(func()) 631 | self._argument_name = name 632 | return func 633 | 634 | return inner 635 | 636 | def assert_same_results(self, equality_func): 637 | """Asserts that all stored functions return the same result. 638 | 639 | .. versionadded:: 0.1.0 640 | 641 | Parameters 642 | ---------- 643 | equality_func : callable 644 | The function that determines if the results are equal. This function should 645 | accept two arguments and return a boolean (True if the results should be 646 | considered equal, False if not). 647 | 648 | Warns 649 | ----- 650 | UserWarning 651 | In case the instance has no arguments for the functions. 652 | 653 | Raises 654 | ------ 655 | AssertionError 656 | In case any two results are not equal. 657 | """ 658 | if not self._arguments: 659 | warnings.warn(_MSG_MISSING_ARGUMENTS, UserWarning) 660 | return 661 | assert_same_results(self._funcs, self._arguments, equality_func=equality_func) 662 | 663 | def assert_not_mutating_input(self, equality_func, copy_func=_DEFAULT_COPY_FUNC): 664 | """Asserts that none of the stored functions mutate the arguments. 665 | 666 | .. versionadded:: 0.1.0 667 | 668 | Parameters 669 | ---------- 670 | equality_func : callable 671 | The function that determines if the results are equal. This function should 672 | accept two arguments and return a boolean (True if the results should be 673 | considered equal, False if not). 674 | copy_func : callable, optional 675 | The function that is used to copy the original argument. 676 | Default is :py:func:`copy.deepcopy`. 677 | 678 | Warns 679 | ----- 680 | UserWarning 681 | In case the instance has no arguments for the functions. 682 | 683 | Raises 684 | ------ 685 | AssertionError 686 | In case any two results are not equal. 687 | 688 | Notes 689 | ----- 690 | In case the arguments are :py:class:`MultiArgument` then the copy_func and the 691 | equality_func get these :py:class:`MultiArgument` as single arguments and need 692 | to handle them appropriately. 693 | """ 694 | if not self._arguments: 695 | warnings.warn(_MSG_MISSING_ARGUMENTS, UserWarning) 696 | return 697 | assert_not_mutating_input(self._funcs, self._arguments, equality_func=equality_func, copy_func=copy_func) 698 | 699 | def run(self): 700 | """Starts the benchmark. 701 | 702 | Returns 703 | ------- 704 | result : BenchmarkResult 705 | The result of the benchmark. 706 | 707 | Warns 708 | ----- 709 | UserWarning 710 | In case the instance has no arguments for the functions. 711 | 712 | .. versionadded:: 0.1.0 713 | """ 714 | if not self._arguments: 715 | warnings.warn(_MSG_MISSING_ARGUMENTS, UserWarning) 716 | return benchmark( 717 | funcs=self._funcs, 718 | arguments=self._arguments, 719 | argument_name=self._argument_name, 720 | warmups=self._warmups, 721 | time_per_benchmark=self._time_per_benchmark, 722 | function_aliases=self._function_aliases, 723 | estimator=self._estimator, 724 | maximum_time=self._maximum_time 725 | ) 726 | 727 | def use_random_arrays_as_arguments(self, sizes): 728 | """Alternative to :meth:`add_arguments` that provides random arrays of the specified sizes as arguments for the 729 | benchmark. 730 | 731 | Parameters 732 | ---------- 733 | sizes : iterable of int 734 | An iterable containing the sizes for the arrays (should be sorted). 735 | 736 | Raises 737 | ------ 738 | ImportError 739 | If NumPy isn't installed. 740 | """ 741 | try: 742 | import numpy as np 743 | except ImportError: 744 | raise ImportError('simple_benchmark requires NumPy for this function.') 745 | 746 | def provide_random_arrays(): 747 | for size in sizes: 748 | yield size, np.random.random(size) 749 | 750 | self.add_arguments('array size')(provide_random_arrays) 751 | 752 | def use_random_lists_as_arguments(self, sizes): 753 | """Alternative to :meth:`add_arguments` that provides random lists of the specified sizes as arguments for the 754 | benchmark. 755 | 756 | Parameters 757 | ---------- 758 | sizes : iterable of int 759 | An iterable containing the sizes for the lists (should be sorted). 760 | """ 761 | def provide_random_lists(): 762 | random_func = random.random 763 | for size in sizes: 764 | yield size, [random_func() for _ in itertools.repeat(None, times=size)] 765 | 766 | self.add_arguments('list size')(provide_random_lists) 767 | --------------------------------------------------------------------------------