├── ibench ├── __init__.py ├── cmds │ ├── __init__.py │ ├── cmd.py │ ├── configs.py │ └── run.py ├── docker │ ├── __init__.py │ ├── Dockerfile.tpl │ └── build.py ├── configs │ ├── idp201701.py │ ├── idp201700.py │ ├── direct.py │ ├── pip.py │ ├── __init__.py │ ├── conda_py2.py │ └── config.py ├── benchmarks │ ├── eig.py │ ├── det.py │ ├── lu.py │ ├── qr.py │ ├── dot.py │ ├── cholesky.py │ ├── svd.py │ ├── rng.py │ ├── cosine.py │ ├── corr.py │ ├── inv.py │ ├── kmeans.py │ ├── ridge.py │ ├── fft.py │ ├── ridgefit.py │ ├── kmeansfit.py │ ├── lregression.py │ ├── lregressionfit.py │ ├── svm.py │ ├── __init__.py │ ├── blacksch.py │ └── bench.py ├── native │ ├── c │ │ ├── dot.h │ │ ├── inv.h │ │ ├── qr.h │ │ ├── det.h │ │ ├── cholesky.h │ │ ├── bench.h │ │ └── lu.h │ ├── __init__.py │ └── tpl.bench.pyx └── __main__.py ├── requirements.txt ├── setup.cfg ├── conda ├── build.sh ├── build.bat ├── run_test.py └── meta.yaml ├── .gitignore ├── SECURITY.md ├── .travis.yml ├── example └── NERSC │ ├── setup_environment.sh │ ├── run_ibench_hsw.sl │ └── run_ibench_knl_cache.sl ├── tests ├── test_run.py └── test_config.py ├── LICENSE ├── setup.py ├── README.md └── .github └── workflows └── scorecard.yml /ibench/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /ibench/cmds/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /ibench/docker/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /conda/build.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | python setup.py install 6 | -------------------------------------------------------------------------------- /conda/build.bat: -------------------------------------------------------------------------------- 1 | REM Copyright (C) 2016 Intel Corporation 2 | REM 3 | REM SPDX-License-Identifier: MIT 4 | 5 | python setup.py install 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # emacs 2 | *~ 3 | 4 | # python 5 | __pycache__ 6 | *.egg-info 7 | *.pyc 8 | .cache 9 | /.project 10 | /x 11 | results 12 | Dockerfile.ibench.shared.ubuntu 13 | -------------------------------------------------------------------------------- /conda/run_test.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import subprocess 6 | 7 | subprocess.run('python -m pytest tests', shell=True, check=True) 8 | -------------------------------------------------------------------------------- /ibench/configs/idp201701.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from .pip import Pip 6 | 7 | class IDP201701(Pip): 8 | _python_path = '/miniconda3/envs/idp2017.0.1/bin/python' 9 | -------------------------------------------------------------------------------- /ibench/configs/idp201700.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from .pip import Pip 6 | 7 | class IDP201700(Pip): 8 | _python_path = '/miniconda3/envs/idp2017.0.0/bin/python' 9 | 10 | -------------------------------------------------------------------------------- /ibench/configs/direct.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import sys 6 | 7 | from .config import Config 8 | 9 | class Direct(Config): 10 | '''Run with same python interpreter as invoked the script''' 11 | _python_path = sys.executable 12 | -------------------------------------------------------------------------------- /ibench/configs/pip.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from .config import Config 6 | from ..docker.build import build as dbuild 7 | 8 | class Pip(Config): 9 | _docker = 'rscohn2/ibench.shared.ubuntu' 10 | 11 | def build(self): 12 | dbuild([{'os_name': 'ubuntu', 'config': 'shared'}]) 13 | 14 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. 3 | 4 | ## Reporting a Vulnerability 5 | Please report any security vulnerabilities in this project utilizing the guidelines [here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). -------------------------------------------------------------------------------- /ibench/cmds/cmd.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from __future__ import print_function 6 | import sys 7 | 8 | class Cmd: 9 | def execute(): 10 | pass 11 | 12 | def parse_args(): 13 | pass 14 | 15 | def _cmd_error(self,message): 16 | print('error: %s' % message, file=sys.stderr) 17 | sys.exit(1) 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | sudo: required 4 | 5 | services: 6 | - docker 7 | 8 | python: 9 | - 2.7 10 | - 3.5 11 | 12 | before_install: 13 | - sudo apt-get -qq update 14 | - sudo apt-get install -y numactl 15 | 16 | install: 17 | - pip install --upgrade pip 18 | - pip install -r requirements.txt 19 | - pip install -e . 20 | 21 | script: 22 | - python -m pytest -s tests 23 | -------------------------------------------------------------------------------- /ibench/configs/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from . import conda_py2 6 | from . import direct 7 | from . import idp201700 8 | from . import idp201701 9 | from . import pip 10 | 11 | configs = { 12 | 'conda_py2': conda_py2.Conda_py2, 13 | 'direct': direct.Direct, 14 | 'idp201700': idp201700.IDP201700, 15 | 'idp201701': idp201701.IDP201701, 16 | 'pip': pip.Pip, 17 | } 18 | -------------------------------------------------------------------------------- /example/NERSC/setup_environment.sh: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | # set up script on NERSC Cori machine 6 | module load python/3.5-anaconda 7 | if [[ ! -d $HOME/.conda ]]; then 8 | mkdir $HOME/.conda 9 | fi 10 | 11 | conda create -n wrapped_ibench -c intel -y python=3.6 hugetlbfs scipy 12 | source $HOME/.conda/envs/wrapped_ibench/bin/activate wrapped_ibench 13 | cd ../../ 14 | python setup.py install 15 | -------------------------------------------------------------------------------- /ibench/configs/conda_py2.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import subprocess 6 | import sys 7 | 8 | from .config import Config 9 | 10 | class Conda_py2(Config): 11 | '''py2 conda environment''' 12 | def build(self): 13 | subprocess.call('conda env remove -y -q -n ibench.py2', shell=True) 14 | subprocess.check_call('conda create -y -q -n ibench.py2 python=2 jinja2 scipy pytest', shell=True) 15 | -------------------------------------------------------------------------------- /ibench/benchmarks/eig.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2017 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import scipy 7 | 8 | from .bench import Bench 9 | 10 | class Eig(Bench): 11 | sizes = {'large': 10000, 'small': 5000, 'tiny': 1000,'test': 2} 12 | 13 | def _ops(self, n): 14 | return (4./3.)*n*n*n*1e-9 15 | 16 | def _make_args(self, n): 17 | self._A = np.asarray(np.random.rand(n,n), dtype=self._dtype) 18 | 19 | def _compute(self): 20 | scipy.linalg.eig(self._A, overwrite_a=True, check_finite=False) 21 | -------------------------------------------------------------------------------- /ibench/benchmarks/det.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2017 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import scipy 7 | 8 | from .bench import Bench 9 | 10 | class Det(Bench): 11 | sizes = {'large': 30000, 'small': 15000, 'tiny': 5000, 'test': 2} 12 | 13 | def _ops(self, n): 14 | return 2./3.*n*n*n*1e-9 15 | 16 | def _make_args(self, n): 17 | self._A = np.asfortranarray(np.random.rand(n,n), dtype=self._dtype) 18 | 19 | def _compute(self): 20 | scipy.linalg.det(self._A, overwrite_a=True, check_finite=False) 21 | -------------------------------------------------------------------------------- /ibench/benchmarks/lu.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import scipy 7 | 8 | from .bench import Bench 9 | 10 | class Lu(Bench): 11 | sizes = {'large': 20000, 'small': 8000, 'tiny': 5000, 'test': 2} 12 | 13 | def _ops(self, n): 14 | return 2./3.*n*n*n*1e-9 15 | 16 | def _make_args(self, n): 17 | self._A = np.asfortranarray(np.asarray(np.random.rand(n,n), dtype=self._dtype)) 18 | 19 | def _compute(self): 20 | scipy.linalg.lu(a=self._A, overwrite_a=True, check_finite=False) 21 | -------------------------------------------------------------------------------- /ibench/benchmarks/qr.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2017 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import scipy 7 | 8 | from .bench import Bench 9 | 10 | 11 | class Qr(Bench): 12 | sizes = {'large': 10000, 'small': 5000, 'tiny': 1000, 'test': 2} 13 | 14 | def _ops(self, n): 15 | return (4./3.)*n*n*n*1e-9 16 | 17 | def _make_args(self, n): 18 | self._A = np.asfortranarray(np.random.rand(n, n), dtype=self._dtype) 19 | 20 | def _compute(self): 21 | scipy.linalg.qr(self._A, overwrite_a=True, check_finite=False, 22 | mode='raw') 23 | -------------------------------------------------------------------------------- /ibench/benchmarks/dot.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2017 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | 7 | from .bench import Bench 8 | 9 | class Dot(Bench): 10 | sizes = {'large': 10000, 'small': 5000, 'tiny': 1000, 'test': 2} 11 | 12 | def _ops(self, n): 13 | return 2E-9 * n*n*n 14 | 15 | def _make_args(self, n): 16 | self._A = np.asarray(np.random.rand(n, n), dtype=self._dtype) 17 | self._B = np.asarray(np.random.rand(n, n), dtype=self._dtype) 18 | self._C = np.asarray(np.random.rand(n, n), dtype=self._dtype) 19 | 20 | def _compute(self): 21 | self._A.dot(self._B, out=self._C) 22 | -------------------------------------------------------------------------------- /ibench/benchmarks/cholesky.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2017 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import scipy.linalg 7 | 8 | from .bench import Bench 9 | 10 | 11 | class Cholesky(Bench): 12 | sizes = {'large': 40000, 'small': 10000, 'tiny': 2000, 'test': 2} 13 | 14 | def _ops(self, n): 15 | return n*n*n/3.0*1e-9 16 | 17 | def _make_args(self, n): 18 | self._A = np.asarray(np.random.rand(n, n), dtype=self._dtype) 19 | self._A = np.asfortranarray(self._A.T @ self._A + n * np.eye(n)) 20 | 21 | def _compute(self): 22 | scipy.linalg.cholesky(self._A, lower=False, overwrite_a=False, 23 | check_finite=False) 24 | -------------------------------------------------------------------------------- /ibench/benchmarks/svd.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2017 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import scipy.linalg 7 | 8 | from .bench import Bench 9 | 10 | 11 | class Svd(Bench): 12 | sizes = {'large': 10000, 'small': 5000, 'tiny': 1000, 'test': 2} 13 | 14 | def _ops(self, n): 15 | return (4./3.)*n*n*n*1e-9 16 | 17 | def _make_args(self, n): 18 | self._A = np.asfortranarray(np.random.rand(n, n), dtype=self._dtype) 19 | 20 | def _compute(self): 21 | # We specify overwrite_a=False here because once the input array 22 | # is overwritten, dgesdd might decide to terminate early 23 | scipy.linalg.svd(self._A, overwrite_a=False, check_finite=False) 24 | -------------------------------------------------------------------------------- /conda/meta.yaml: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | package: 6 | name: ibench 7 | version: 0.1rc 8 | 9 | build: 10 | number: 0 11 | 12 | source: 13 | git_tag: 0.1rc 14 | git_url: https://github.com/IntelPython/ibench.git 15 | 16 | requirements: 17 | build: 18 | - cython 19 | - jinja2 20 | - numpy 21 | - python 22 | - scipy 23 | - setuptools 24 | run: 25 | - jinja2 26 | - python 27 | - numpy 28 | - scipy 29 | 30 | test: 31 | source_files: 32 | - tests 33 | requires: 34 | - pytest 35 | 36 | about: 37 | home: http://github.com/IntelPython/ibench 38 | license: MIT 39 | licene_file: LICENSE 40 | summary: 'Benchmarking for numerical python workloads' 41 | -------------------------------------------------------------------------------- /ibench/benchmarks/rng.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | 7 | from .bench import Bench 8 | 9 | try: 10 | import numpy.random_intel as rnd 11 | except: 12 | import numpy.random as rnd 13 | 14 | class Rng(Bench): 15 | 16 | sizes = {'large': 20000, 'small': 5000, 'tiny': 100, 'test': 2} 17 | 18 | def _ops(self, n): 19 | # TODO: Needs a more accurate count 20 | return 2E-9 * n 21 | 22 | def _sample_uniform(self, rs, sz): 23 | rs.uniform(-1, 1, size=sz) 24 | 25 | def _make_args(self, n): 26 | self._rs = rnd.RandomState(123) 27 | # rnd.RandomState(123, brng='MT19937') with the Intel variant in the future 28 | self._size = n 29 | 30 | def _compute(self): 31 | self._sample_uniform(self._rs, (self._size * 100, 1000)) 32 | -------------------------------------------------------------------------------- /ibench/benchmarks/cosine.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import sklearn 7 | import multiprocessing 8 | from numpy.random import rand 9 | from sklearn.metrics.pairwise import pairwise_distances 10 | 11 | from .bench import Bench 12 | 13 | if sklearn.__version__ == '0.18.2': 14 | sklearn.utils.validation._assert_all_finite = lambda X: None 15 | 16 | class Cosine(Bench): 17 | """ 18 | Benchmark for Cosine Distance from Scikit-learn 19 | Attempts to utilize parallelism for larger datasets 20 | """ 21 | sizes = {'large': 50000, 'small': 30000, 'tiny': 10000, 'test': 10} 22 | 23 | def _ops(self, n): 24 | return 2E-9 * n 25 | 26 | def _make_args(self, n): 27 | p = int(n/10) 28 | self._X = rand(p,n) 29 | 30 | def _compute(self): 31 | self._cor_dist = pairwise_distances(self._X, metric='cosine', n_jobs=-1) 32 | -------------------------------------------------------------------------------- /tests/test_run.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import subprocess 6 | 7 | def test_run_plugin(): 8 | subprocess.check_call('IBENCH_PLUGINS="os sys" python -m ibench run -b cholesky --size test --file foo', shell=True) 9 | 10 | def test_run(): 11 | subprocess.check_call('python -m ibench run', shell=True) 12 | 13 | def test_run_simple(): 14 | subprocess.check_call('python -m ibench run -b cholesky --size test --file foo', shell=True) 15 | 16 | def test_run_sizes(): 17 | subprocess.check_call('python -m ibench run -b fft --size test --file foo', shell=True) 18 | subprocess.check_call('python -m ibench run -b fft --size small --file foo', shell=True) 19 | subprocess.check_call('python -m ibench run -b fft --size large --file foo', shell=True) 20 | 21 | def test_run_groups(): 22 | subprocess.check_call('python -m ibench run -b linalg --size test --file foo', shell=True) 23 | -------------------------------------------------------------------------------- /ibench/benchmarks/corr.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import sklearn 7 | import multiprocessing 8 | from numpy.random import rand 9 | from sklearn.metrics.pairwise import pairwise_distances 10 | 11 | from .bench import Bench 12 | 13 | if sklearn.__version__ == '0.18.2': 14 | sklearn.utils.validation._assert_all_finite = lambda X: None 15 | 16 | class Corr(Bench): 17 | """ 18 | Benchmark for Correlation Distance from Scikit-learn 19 | Attempts to utilize parallelism for larger datasets 20 | """ 21 | sizes = {'large': 50000, 'small': 15000, 'tiny': 10000, 'test': 10} 22 | 23 | def _ops(self, n): 24 | return 2E-9 * n 25 | 26 | def _make_args(self, n): 27 | p = int(n/10) 28 | self._X = rand(p,n) 29 | 30 | def _compute(self): 31 | self._cor_dist = pairwise_distances(self._X, metric='correlation', n_jobs=-1) 32 | -------------------------------------------------------------------------------- /ibench/native/c/dot.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: MIT 5 | */ 6 | 7 | #include "bench.h" 8 | 9 | class dot_C : public Bench { 10 | public: 11 | dot_C() { 12 | a_mat = 0; 13 | b_mat = 0; 14 | r_mat = 0; 15 | } 16 | 17 | ~dot_C() { 18 | if (a_mat) 19 | mkl_free(a_mat); 20 | if (b_mat) 21 | mkl_free(b_mat); 22 | if (r_mat) 23 | mkl_free(r_mat); 24 | } 25 | 26 | void make_args(int size) { 27 | m = n = k = size; 28 | 29 | a_mat = make_random_mat(m*k); 30 | b_mat = make_random_mat(k*n); 31 | r_mat = make_random_mat(m*n); 32 | } 33 | 34 | void compute() { 35 | double alpha = 1.0; 36 | double beta = 0.0; 37 | 38 | cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, 39 | n, n, k, alpha, a_mat, k, b_mat, n, beta, r_mat, n); 40 | } 41 | 42 | private: 43 | double *a_mat, *b_mat, *r_mat; 44 | int m,n,k; 45 | }; 46 | -------------------------------------------------------------------------------- /ibench/benchmarks/inv.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2017 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import scipy 7 | 8 | from .bench import Bench 9 | 10 | class Inv(Bench): 11 | sizes = {'large': 25000, 'small': 10000, 'tiny': 2000, 'test': 2} 12 | 13 | def _ops(self, n): 14 | # scipy is getrf getri 15 | return 2.*n*n*n*1e-9 16 | # numpy calls gesv 17 | # lu + triangular solve 18 | # TRF + TRS 19 | # 2/3 n^3 + 2 n^3 = 8/3 n^3 20 | # return 8./3.*N*N*N*1e-9 21 | 22 | def _make_args(self, n): 23 | self._A = np.asfortranarray(np.random.rand(n,n), dtype=self._dtype) 24 | 25 | def _compute(self): 26 | # yes, we overwrite the input here without refreshing it, 27 | # but because (A**-1)**-1 = A, there shouldn't be any big problems 28 | # w.r.t. early termination of inverse 29 | scipy.linalg.inv(self._A, overwrite_a=True, check_finite=False) 30 | -------------------------------------------------------------------------------- /ibench/benchmarks/kmeans.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import sklearn 7 | import multiprocessing 8 | from numpy.random import rand 9 | from sklearn.cluster import KMeans 10 | 11 | from .bench import Bench 12 | 13 | if sklearn.__version__ == '0.18.2': 14 | sklearn.utils.validation._assert_all_finite = lambda X: None 15 | 16 | class Kmeans(Bench): 17 | """ 18 | Benchmark for Kmeans Training from Scikit-learn 19 | Attempts to utilize parallelism for larger datasets 20 | """ 21 | sizes = {'large': 1000000, 'small': 100000, 'tiny': 10000, 'test': 1000} 22 | 23 | def _ops(self, n): 24 | return 2E-9 * n*n*n 25 | 26 | def _make_args(self, n): 27 | p = int(np.log(n)+100) 28 | self._X = rand(n,p) 29 | self._kmeans = KMeans(n_clusters=10, n_jobs=int(-1), n_init=1) 30 | 31 | def _compute(self): 32 | self._trained_model = self._kmeans.fit(self._X) 33 | -------------------------------------------------------------------------------- /ibench/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import argparse 6 | import os 7 | import sys 8 | 9 | from .cmds import run 10 | from .cmds import configs 11 | 12 | cmds = [run, configs] 13 | 14 | def parse_args(): 15 | """ 16 | Does the initial argument scope and parsing 17 | by bringing in run+configs and matching them 18 | with the typed arguments. 19 | """ 20 | parser = argparse.ArgumentParser("ibench") 21 | parser.set_defaults(func=None) 22 | subparsers = parser.add_subparsers() 23 | for cmd in cmds: 24 | cmd.add_parser(subparsers) 25 | return parser.parse_args() 26 | 27 | 28 | # Load plugins first so it can modify everything that follows 29 | if 'IBENCH_PLUGINS' in os.environ: 30 | for plugin in os.environ['IBENCH_PLUGINS'].split(' '): 31 | __import__(plugin) 32 | 33 | 34 | # Parse arguments and intiate run 35 | args = parse_args() 36 | if args.func: 37 | args.func(args) 38 | -------------------------------------------------------------------------------- /ibench/native/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016, 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | # python loads libraries with RTLD_LOCAL, but MKL requires RTLD_GLOBAL 6 | # pre-load MKL with RTLD_GLOBAL before loading the native extension 7 | import ctypes 8 | try: 9 | ctypes.CDLL('libmkl_rt.so', ctypes.RTLD_GLOBAL) 10 | except OSError: 11 | raise ImportError 12 | 13 | from ibench.benchmarks import benchmarks, benchmark_groups 14 | from . import det 15 | from . import dot 16 | from . import inv 17 | from . import lu 18 | from . import cholesky 19 | from . import qr 20 | 21 | local_benchmarks = { 22 | 'dot_native': dot.Dot_native, 23 | 'det_native': det.Det_native, 24 | 'inv_native': inv.Inv_native, 25 | 'lu_native': lu.Lu_native, 26 | 'cholesky_native': cholesky.Cholesky_native, 27 | 'qr_native': qr.Qr_native 28 | } 29 | 30 | # add to the list of benchmark options 31 | benchmarks.update(local_benchmarks) 32 | benchmark_groups['native'] = local_benchmarks 33 | -------------------------------------------------------------------------------- /ibench/benchmarks/ridge.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import sklearn 7 | import multiprocessing 8 | from numpy.random import rand 9 | from sklearn import linear_model 10 | 11 | from .bench import Bench 12 | 13 | if sklearn.__version__ == '0.18.2': 14 | sklearn.utils.validation._assert_all_finite = lambda X: None 15 | 16 | class Ridge(Bench): 17 | """ 18 | Benchmark for Ridge Regression Training from Scikit-learn 19 | Attempts to utilize parallelism for larger datasets 20 | """ 21 | sizes = {'large': 1000000, 'small': 800000, 'tiny': 100000, 'test': 1000} 22 | 23 | def _ops(self, n): 24 | return 2E-9 * n*n*n 25 | 26 | def _make_args(self, n): 27 | p = int(np.log(n)+100) 28 | self._X = rand(n,p) 29 | self._y = rand(n) 30 | self._regr = linear_model.Ridge() 31 | 32 | def _compute(self): 33 | self._trained_model = self._regr.fit(self._X,self._y) 34 | -------------------------------------------------------------------------------- /example/NERSC/run_ibench_hsw.sl: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | 3 | # Copyright (C) 2016-2018 Intel Corporation 4 | # 5 | # SPDX-License-Identifier: MIT 6 | 7 | #SBATCH -N 1 8 | #SBATCH -p regular 9 | #SBATCH -t 00:45:00 10 | #SBATCH -C haswell 11 | 12 | export NUM_OF_THREADS=$(grep 'model name' /proc/cpuinfo | wc -l) 13 | export OMP_NUM_THREADS=$(( $NUM_OF_THREADS )) 14 | export MKL_NUM_THREADS=$(( $NUM_OF_THREADS )) 15 | export KMP_HW_SUBSET=${OMP_NUM_THREADS}c,1t 16 | export HPL_LARGEPAGE=1 17 | export KMP_BLOCKTIME=800 18 | export TEST=all 19 | export SIZE=large 20 | export OUTPUT_DIR="." 21 | 22 | module load python/3.5-anaconda 23 | source $HOME/.conda/envs/wrapped_ibench/bin/activate wrapped_ibench 24 | 25 | # Make sure that the transparent huge page is enabled for best performance 26 | module load craype-hugepages2M 27 | 28 | #### This is a script for running the benchmark 29 | srun -N 1 python -m ibench run -b $TEST --size $SIZE --file \ 30 | $OUTPUT_DIR/${TEST}_${SIZE}_hsw_$(date '+%Y-%m-%d_%H:%M:%S').log 31 | -------------------------------------------------------------------------------- /ibench/benchmarks/fft.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import math 6 | import numpy as np 7 | import scipy.fftpack 8 | 9 | from .bench import Bench 10 | 11 | class Fft(Bench): 12 | sizes = {'large': 1000000, 'small': 1000000, 'tiny': 52000, 'test': 100} 13 | # If you change the value of runs, change native.cpp as well 14 | _runs = 1000 15 | 16 | def _ops(self, n): 17 | # This is not an actual flop count; it is simply a convenient scaling, 18 | # based on the fact that the radix-2 Cooley-Tukey algorithm asymptotically 19 | # requires 5 N log2(N) floating-point operations. 20 | # http://www.fftw.org/speed/method.html 21 | return self._runs*5*n*math.log(n,2)*1e-9 22 | 23 | def _make_args(self, n): 24 | self._A = np.asarray(np.random.rand(n), dtype=np.complex128) 25 | 26 | def _compute(self): 27 | for i in range(self._runs): 28 | scipy.fftpack.fft(self._A, overwrite_x = True) 29 | -------------------------------------------------------------------------------- /ibench/native/tpl.bench.pyx: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | # distutils: language = c++ 6 | 7 | from ibench.benchmarks.{{bench}} import {{Bench}} 8 | 9 | # Expose the C++ class 10 | cdef extern from '../ibench/native/c/{{bench}}.h': 11 | cdef cppclass {{bench}}_C: 12 | {{bench}}_C() except + 13 | void make_args(int) except + 14 | void compute() except + 15 | 16 | # Wrap the c++ class in an extension class 17 | # extension class cannot inherit from python classes 18 | cdef class Wrapper: 19 | cdef {{bench}}_C c_class 20 | def make_args(self,n): 21 | self.c_class.make_args(n) 22 | def compute(self): 23 | self.c_class.compute() 24 | 25 | # Inherit from python bench with methods specific to native 26 | class {{Bench}}_native({{Bench}}): 27 | def _make_args(self, n): 28 | self._wrapper = Wrapper() 29 | self._wrapper.make_args(n) 30 | 31 | def _compute(self): 32 | self._wrapper.compute() 33 | -------------------------------------------------------------------------------- /ibench/benchmarks/ridgefit.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import sklearn 7 | import multiprocessing 8 | from numpy.random import rand 9 | from sklearn import linear_model 10 | 11 | from .bench import Bench 12 | 13 | if sklearn.__version__ == '0.18.2': 14 | sklearn.utils.validation._assert_all_finite = lambda X: None 15 | 16 | class Ridgefit(Bench): 17 | """ 18 | Benchmark for Ridge Regression Prediction from Scikit-learn 19 | Attempts to utilize parallelism for larger datasets 20 | """ 21 | sizes = {'large': 1000000, 'small': 800000, 'tiny': 100000, 'test': 1000} 22 | 23 | def _ops(self, n): 24 | return 2E-9 * n*n*n 25 | 26 | def _make_args(self, n): 27 | p = int(np.log(n)+100) 28 | self._X = rand(n,p) 29 | self._y = rand(n) 30 | self._regr = linear_model.Ridge() 31 | self._regr.fit(self._X,self._y) 32 | 33 | def _compute(self): 34 | self._regr.predict(self._X) 35 | -------------------------------------------------------------------------------- /ibench/benchmarks/kmeansfit.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import sklearn 7 | import multiprocessing 8 | from numpy.random import rand 9 | from sklearn.cluster import KMeans 10 | 11 | from .bench import Bench 12 | 13 | if sklearn.__version__ == '0.18.2': 14 | sklearn.utils.validation._assert_all_finite = lambda X: None 15 | 16 | class Kmeansfit(Bench): 17 | """ 18 | Benchmark for Kmeans Prediction from Scikit-learn 19 | Attempts to utilize parallelism for larger datasets 20 | """ 21 | sizes = {'large': 1000000, 'small': 100000, 'tiny': 10000, 'test': 1000} 22 | 23 | def _ops(self, n): 24 | return 2E-9 * n*n*n 25 | 26 | def _make_args(self, n): 27 | p = int(np.log(n)+100) 28 | self._X = rand(n,p) 29 | self._kmeans = KMeans(n_clusters=10, n_jobs=int(-1), n_init=1) 30 | self._trained_model = self._kmeans.fit(self._X) 31 | 32 | def _compute(self): 33 | self._predicted = self._kmeans.fit_predict(self._X) 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016-2019 Intel Corporation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /ibench/benchmarks/lregression.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import sklearn 7 | import multiprocessing 8 | from numpy.random import rand 9 | from sklearn import linear_model 10 | 11 | from .bench import Bench 12 | 13 | if sklearn.__version__ == '0.18.2': 14 | sklearn.utils.validation._assert_all_finite = lambda X: None 15 | 16 | class Lregression(Bench): 17 | """ 18 | Benchmark for Linear Regression Training from Scikit-learn 19 | Attempts to utilize parallelism for larger datasets 20 | """ 21 | sizes = {'large': 1000000, 'small': 800000, 'tiny': 100000, 'test': 1000} 22 | 23 | def _ops(self, n): 24 | return 2E-9 * n*n*n 25 | 26 | def _make_args(self, n): 27 | p = int(np.log(n)+100) 28 | self._X = rand(n,p) 29 | self._y = rand(n) 30 | if n < 8000: 31 | self._regr = linear_model.LinearRegression() 32 | else: 33 | self._regr = linear_model.LinearRegression(n_jobs=-1) 34 | 35 | 36 | def _compute(self): 37 | self._trained_model = self._regr.fit(self._X,self._y) 38 | -------------------------------------------------------------------------------- /ibench/benchmarks/lregressionfit.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import sklearn 6 | import numpy as np 7 | import multiprocessing 8 | from numpy.random import rand 9 | from sklearn import linear_model 10 | 11 | from .bench import Bench 12 | 13 | if sklearn.__version__ == '0.18.2': 14 | sklearn.utils.validation._assert_all_finite = lambda X: None 15 | 16 | class Lregressionfit(Bench): 17 | """ 18 | Benchmark for Linear Regression Prediction from Scikit-learn 19 | Attempts to utilize parallelism for larger datasets 20 | """ 21 | sizes = {'large': 1000000, 'small': 800000, 'tiny': 100000, 'test': 1000} 22 | 23 | def _ops(self, n): 24 | return 2E-9 * n*n*n 25 | 26 | def _make_args(self, n): 27 | p = int(np.log(n)+100) 28 | self._X = rand(n,p) 29 | self._y = rand(n) 30 | if n < 8000: 31 | self._regr = linear_model.LinearRegression() 32 | else: 33 | self._regr = linear_model.LinearRegression(n_jobs=-1) 34 | 35 | self._regr.fit(self._X,self._y) 36 | 37 | def _compute(self): 38 | self._trained_model = self._regr.predict(self._X) 39 | -------------------------------------------------------------------------------- /example/NERSC/run_ibench_knl_cache.sl: -------------------------------------------------------------------------------- 1 | #!/bin/bash -l 2 | 3 | # Copyright (C) 2016-2018 Intel Corporation 4 | # 5 | # SPDX-License-Identifier: MIT 6 | 7 | #SBATCH -N 1 8 | #SBATCH -p regular 9 | #SBATCH -t 00:45:00 10 | #SBATCH -C knl,quad,cache 11 | 12 | # specify threading settings 13 | export KMP_AFFINITY=granularity=fine,compact 14 | export NUM_OF_THREADS=$(grep 'model name' /proc/cpuinfo | wc -l) 15 | export OMP_NUM_THREADS=$(( $NUM_OF_THREADS / 4 )) 16 | export MKL_NUM_THREADS=$(( $NUM_OF_THREADS / 4 )) 17 | export KMP_HW_SUBSET=${OMP_NUM_THREADS}c,1t 18 | export HPL_LARGEPAGE=1 19 | export KMP_BLOCKTIME=800 20 | export TEST=all 21 | export SIZE=large 22 | export OUTPUT_DIR="." 23 | 24 | # load the python module on Cori 25 | module load python/3.5-anaconda 26 | 27 | # activate the relevant Conda environment 28 | source $HOME/.conda/envs/wrapped_ibench/bin/activate wrapped_ibench 29 | 30 | # make sure that the Cray transparent huge page module is loaded for the best performance 31 | module load craype-hugepages2M 32 | 33 | # run the benchmark and specify the location and name of the log file 34 | srun -N 1 python -m ibench run -b $TEST --size $SIZE --file \ 35 | $OUTPUT_DIR/${TEST}_${SIZE}_$(date '+%Y-%m-%d_%H:%M:%S').log 36 | -------------------------------------------------------------------------------- /ibench/native/c/inv.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: MIT 5 | */ 6 | 7 | #include "bench.h" 8 | 9 | class inv_C : public Bench { 10 | public: 11 | inv_C() { 12 | x_mat = 0; 13 | r_mat = 0; 14 | ipiv = 0; 15 | } 16 | 17 | ~inv_C() { 18 | if (r_mat) 19 | mkl_free(r_mat); 20 | if (ipiv) 21 | mkl_free(ipiv); 22 | if (x_mat) 23 | mkl_free(x_mat); 24 | } 25 | 26 | void make_args(int size) { 27 | N = size; 28 | M = size; 29 | LDA = size; 30 | int mat_size = M*N, mn_min = min(M, N); 31 | 32 | assert(M == N); 33 | 34 | /* input matrix */ 35 | x_mat = make_random_mat(mat_size); 36 | 37 | /* list of pivots */ 38 | ipiv = (int *) mkl_malloc(mn_min * sizeof(int), 64); 39 | assert(ipiv); 40 | 41 | /* matrix for result */ 42 | r_mat = make_random_mat(mat_size); 43 | } 44 | 45 | void compute() { 46 | /* compute pivoted lu decomposition */ 47 | int info = LAPACKE_dgetrf(LAPACK_ROW_MAJOR, M, N, r_mat, LDA, ipiv); 48 | assert(info == 0); 49 | 50 | info = LAPACKE_dgetri(LAPACK_ROW_MAJOR, N, r_mat, LDA, ipiv); 51 | assert(info == 0); 52 | } 53 | 54 | private: 55 | double *x_mat, *r_mat; 56 | int *ipiv; 57 | int N,M,LDA; 58 | }; 59 | -------------------------------------------------------------------------------- /ibench/native/c/qr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2019 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: MIT 5 | */ 6 | 7 | #include "bench.h" 8 | 9 | class qr_C : public Bench { 10 | public: 11 | qr_C(); 12 | ~qr_C(); 13 | void make_args(int size); 14 | void compute(); 15 | 16 | private: 17 | double *x_mat, *r_mat, *tau_vec; 18 | int n,lda; 19 | }; 20 | 21 | 22 | qr_C::qr_C() { 23 | x_mat = r_mat = tau_vec = 0; 24 | } 25 | 26 | void 27 | qr_C::make_args(int size) { 28 | n = lda = size; 29 | 30 | int mat_size = n*n; 31 | 32 | /* input matrix */ 33 | x_mat = make_random_mat(mat_size); 34 | 35 | /* upper triangular output matrix */ 36 | r_mat = make_mat(mat_size); 37 | memset(r_mat, 0, mat_size * sizeof(*r_mat)); 38 | 39 | /* tau */ 40 | tau_vec = make_mat(n); 41 | } 42 | 43 | void qr_C::compute() { 44 | /* compute qr decomposition */ 45 | int info = LAPACKE_dgeqrf(LAPACK_COL_MAJOR, n, n, x_mat, lda, tau_vec); 46 | assert(info == 0); 47 | 48 | /* numpy computes upper triangular part of A even when mode='raw' */ 49 | for (int i = 0; i < n; i++) { 50 | memcpy(&r_mat[i*n], &x_mat[i*n], (i+1) * sizeof(*r_mat)); 51 | } 52 | 53 | } 54 | 55 | qr_C::~qr_C() { 56 | if (x_mat) mkl_free(x_mat); 57 | if (r_mat) mkl_free(r_mat); 58 | if (tau_vec) mkl_free(tau_vec); 59 | } 60 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import subprocess 6 | 7 | def test_auto_size(): 8 | subprocess.check_call('python -m ibench configs --build pip', shell=True) 9 | subprocess.check_call('python -m ibench configs --size auto --cpu xeon --run idp201700 --run-args "-b dot"', shell=True) 10 | 11 | def test_py2(): 12 | # only run if conda is present 13 | ret = subprocess.call('which conda',shell=True) 14 | if ret == 0: 15 | subprocess.check_call('python -m ibench configs --build conda_py2', shell=True) 16 | 17 | def test_direct(): 18 | subprocess.check_call('python -m ibench configs --threads 1 --cpu xeon --run direct --run-args "-b cholesky fft det inv lu svd qr dot --size test"', shell=True) 19 | 20 | def test_docker_build(): 21 | subprocess.check_call('python -m ibench configs --build pip', shell=True) 22 | 23 | def test_pip(): 24 | subprocess.check_call('python -m ibench configs --build pip', shell=True) 25 | subprocess.check_call('python -m ibench configs --threads 1 --cpu xeon --run pip --run-args "-b det --size test"', shell=True) 26 | 27 | def test_idp(): 28 | subprocess.check_call('python -m ibench configs --build pip', shell=True) 29 | subprocess.check_call('python -m ibench configs --threads 1 --cpu xeon --run idp201700 idp201701 --run-args "-b det --size test"', shell=True) 30 | 31 | -------------------------------------------------------------------------------- /ibench/native/c/det.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: MIT 5 | */ 6 | 7 | #include "bench.h" 8 | 9 | class det_C : public Bench { 10 | public: 11 | det_C(); 12 | ~det_C(); 13 | void make_args(int size); 14 | void compute(); 15 | private: 16 | double *x_mat, *r_mat; 17 | int *ipiv; 18 | int n,m,lda; 19 | int mn_min; 20 | double result; 21 | }; 22 | 23 | det_C::det_C() { 24 | r_mat = 0; 25 | ipiv = 0; 26 | x_mat = 0; 27 | } 28 | 29 | void 30 | det_C::make_args(int size) { 31 | n = size; 32 | m = size; 33 | mn_min = min(m, n); 34 | lda = size; 35 | int mat_size = m*n; 36 | assert(m == n); 37 | 38 | /* input matrix */ 39 | x_mat = make_random_mat(mat_size); 40 | 41 | /* list of pivots */ 42 | ipiv = (int *) mkl_malloc(mn_min * sizeof(int), 64); 43 | assert(ipiv); 44 | 45 | /* matrix for result */ 46 | r_mat = make_random_mat(mat_size); 47 | } 48 | 49 | void det_C::compute() { 50 | /* compute pivoted lu decomposition */ 51 | int info = LAPACKE_dgetrf(LAPACK_COL_MAJOR, m, n, r_mat, lda, ipiv); 52 | assert(info == 0); 53 | 54 | double t = 1.0; 55 | int i,j; 56 | for(i=0, j=0; i < mn_min; i++, j+= lda+1) { 57 | t *= (ipiv[i]==i) ? r_mat[j] : -r_mat[j]; 58 | } 59 | result = t; 60 | } 61 | 62 | det_C::~det_C() { 63 | if (r_mat) mkl_free(r_mat); 64 | if (ipiv) mkl_free(ipiv); 65 | if (x_mat) mkl_free(x_mat); 66 | } 67 | -------------------------------------------------------------------------------- /ibench/native/c/cholesky.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2019 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: MIT 5 | */ 6 | 7 | #include "bench.h" 8 | 9 | class cholesky_C : public Bench { 10 | public: 11 | cholesky_C(); 12 | ~cholesky_C(); 13 | void make_args(int size); 14 | void compute(); 15 | 16 | private: 17 | double *x_mat, *r_mat; 18 | int n,lda; 19 | }; 20 | 21 | 22 | cholesky_C::cholesky_C() { 23 | x_mat = r_mat = 0; 24 | } 25 | 26 | void 27 | cholesky_C::make_args(int size) { 28 | n = lda = size; 29 | 30 | int mat_size = n*n; 31 | int r_size = mat_size; 32 | 33 | /* input matrix */ 34 | x_mat = make_random_mat(mat_size); 35 | 36 | /* matrix for result */ 37 | r_mat = make_mat(r_size); 38 | memset(r_mat, 0, r_size * sizeof(*r_mat)); 39 | // Set r_mat to identity matrix as in python bench 40 | for (int i = 0; i < n; i++) r_mat[i*n+i] = 1; 41 | cblas_dsyrk(CblasColMajor, CblasUpper, CblasNoTrans, n, n, 1.0, x_mat, lda, n, r_mat, lda); 42 | 43 | // we now have r_mat = x_mat * x_mat' + n * np.eye(n) 44 | // copy back into x_mat 45 | mkl_domatcopy('R', 'N', n, n, 1.0, r_mat, n, x_mat, n); 46 | } 47 | 48 | void cholesky_C::compute() { 49 | /* compute cholesky decomposition */ 50 | int info = LAPACKE_dpotrf(LAPACK_COL_MAJOR, 'U', n, r_mat, lda); 51 | assert(info == 0); 52 | 53 | } 54 | 55 | cholesky_C::~cholesky_C() { 56 | if (r_mat) mkl_free(r_mat); 57 | if (x_mat) mkl_free(x_mat); 58 | } 59 | -------------------------------------------------------------------------------- /ibench/native/c/bench.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: MIT 5 | */ 6 | 7 | #include 8 | 9 | using namespace std; 10 | 11 | 12 | #include "assert.h" 13 | #include "stdlib.h" 14 | 15 | #if defined(__INTEL_LLVM_COMPILER) 16 | 17 | #include "mkl.h" 18 | 19 | class Random{ 20 | private: 21 | enum {SEED = 77777}; 22 | static constexpr double const d_zero = 0.0, d_one = 1.0; 23 | VSLStreamStatePtr stream; 24 | 25 | public: 26 | Random() { 27 | int err = vslNewStream(&stream, VSL_BRNG_MT19937, SEED); 28 | assert(err == VSL_STATUS_OK); 29 | } 30 | ~Random() { 31 | int err = vslDeleteStream(&stream); 32 | assert(err == VSL_STATUS_OK); 33 | } 34 | void init_mat(double* mat, int size) { 35 | int err = vdRngGaussian(VSL_RNG_METHOD_GAUSSIAN_ICDF, stream, size, mat, d_zero, d_one); 36 | assert(err == VSL_STATUS_OK); 37 | } 38 | }; 39 | 40 | #else 41 | 42 | #include "lapacke.h" 43 | #include "cblas.h" 44 | 45 | class Random { 46 | public: 47 | void init_mat(double* mat, int size) { 48 | } 49 | }; 50 | 51 | static void* mkl_malloc(int size, int align) { 52 | return malloc(size); 53 | } 54 | 55 | static void mkl_free(void*p) { 56 | free(p); 57 | } 58 | #endif 59 | 60 | 61 | class Bench { 62 | private: 63 | Random random; 64 | public: 65 | 66 | double* make_random_mat(int size) { 67 | double* mat = make_mat(size); 68 | random.init_mat(mat,size); 69 | return mat; 70 | } 71 | 72 | double* make_mat(int mat_size) { 73 | double *mat = (double *) mkl_malloc(mat_size * sizeof(double), 64); 74 | assert(mat); 75 | return mat; 76 | } 77 | 78 | virtual void compute()=0; 79 | }; 80 | -------------------------------------------------------------------------------- /ibench/benchmarks/svm.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | import sklearn, sklearn.utils 7 | import sklearn.svm as svm 8 | from sklearn.datasets import make_classification 9 | from sklearn.metrics import accuracy_score 10 | 11 | from .bench import Bench 12 | 13 | sklearn._ASSUME_FINITE = True 14 | 15 | if sklearn.__version__ == '0.18.2': 16 | sklearn.utils.validation._assert_all_finite = lambda X: None 17 | 18 | features = [10, 50, 100, 200, 400, 800, 1000, 2000] 19 | vectors = [1000, 2000, 4000, 10000, 20000] 20 | 21 | class Svm(Bench): 22 | """ 23 | Benchmark for Ridge Regression Prediction from Scikit-learn 24 | Attempts to utilize parallelism for larger datasets 25 | """ 26 | sizes = {'large': 5, 'small': 3, 'tiny': 2, 'test': 1} 27 | 28 | def _gen_datasets(self, features, vectors, classes, dest='data'): 29 | """Generate classification datasets in binary .npy files 30 | features: a list of feature lengths to test 31 | vectors: a list of sample lengths to test 32 | classes: number of classes (2 for binary classification dataset) 33 | """ 34 | self._X, self._y = make_classification(n_samples=vectors, n_features=features, n_informative=features, n_redundant=0, n_classes=classes, random_state=0) 35 | return self._X, self._y 36 | 37 | def _ops(self, n): 38 | return 2E-9 * n 39 | 40 | def _make_args(self, n): 41 | self._X, self._y = self._gen_datasets(features[n-1],vectors[n-1],2) 42 | self._clf = svm.SVC(C=0.01, kernel='linear', max_iter=10000, tol=1e-16, shrinking=True) 43 | 44 | def _compute(self): 45 | self._clf.fit(self._X, self._y) 46 | -------------------------------------------------------------------------------- /ibench/benchmarks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from . import cholesky 6 | from . import det 7 | from . import dot 8 | from . import fft 9 | from . import inv 10 | from . import lu 11 | from . import qr 12 | from . import svd 13 | from . import blacksch 14 | from . import rng 15 | from . import lregression 16 | from . import lregressionfit 17 | from . import ridge 18 | from . import ridgefit 19 | from . import cosine 20 | from . import corr 21 | from . import svm 22 | from . import kmeans 23 | from . import kmeansfit 24 | from . import eig 25 | 26 | benchmarks = { 27 | 'cholesky': cholesky.Cholesky, 28 | 'det': det.Det, 29 | 'dot': dot.Dot, 30 | 'fft': fft.Fft, 31 | 'inv': inv.Inv, 32 | 'lu': lu.Lu, 33 | 'qr': qr.Qr, 34 | 'eig': eig.Eig, 35 | 'svd': svd.Svd, 36 | 'blacksch':blacksch.Blacksch, 37 | 'rng':rng.Rng, 38 | 'lregression':lregression.Lregression, 39 | 'lregressionfit': lregressionfit.Lregressionfit, 40 | 'ridge':ridge.Ridge, 41 | 'ridgefit':ridgefit.Ridgefit, 42 | 'cosine':cosine.Cosine, 43 | 'corr':corr.Corr, 44 | 'svm':svm.Svm, 45 | 'kmeans':kmeans.Kmeans, 46 | 'kmeansfit':kmeansfit.Kmeansfit 47 | } 48 | 49 | benchmark_groups = { 50 | 'linalg': ['cholesky', 'det', 'dot', 'inv', 'lu', 'qr', 'svd', 'eig'], 51 | 'all': list(benchmarks.keys()), 52 | 'sklearn': ['lregression', 'lregressionfit', 'ridge', 'ridgefit', 53 | 'cosine','corr','svm','kmeans','kmeansfit'], 54 | 'bench2018': ['fft', 'lu', 'dot', 'cholesky', 'qr', 'eig', 'blacksch', 55 | 'rng', 'lregression', 'lregressionfit', 'ridge', 56 | 'ridgefit','cosine','corr','svm','kmeans','kmeansfit'] 57 | } 58 | 59 | # Try to get native benchmarks 60 | try: 61 | from .. import native 62 | except ImportError: 63 | pass 64 | -------------------------------------------------------------------------------- /ibench/benchmarks/blacksch.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import numpy as np 6 | from numpy import log, exp 7 | from .bench import Bench 8 | 9 | try: 10 | import numpy.random_intel as rnd 11 | except: 12 | import numpy.random as rnd 13 | try: 14 | from numpy import erf 15 | except: 16 | from scipy.special import erf 17 | 18 | try: 19 | from numpy import invsqrt 20 | numpy_ver += "-invsqrt" 21 | except: 22 | invsqrt = lambda x: 1.0/np.sqrt(x) 23 | 24 | try: 25 | xrange 26 | except NameError: 27 | xrange = range 28 | 29 | SEED = 7777777 30 | S0L = 10.0 31 | S0H = 50.0 32 | XL = 10.0 33 | XH = 50.0 34 | TL = 1.0 35 | TH = 2.0 36 | RISK_FREE = 0.1 37 | VOLATILITY = 0.2 38 | TEST_ARRAY_LENGTH = 1024 39 | 40 | class Blacksch(Bench): 41 | sizes = {'large': 200000000, 'small': 100000000, 'tiny': 1000000, 'test': 100} 42 | 43 | def _gen_data(self, nopt): 44 | return ( 45 | rnd.uniform(S0L, S0H, nopt), 46 | rnd.uniform(XL, XH, nopt), 47 | rnd.uniform(TL, TH, nopt), 48 | ) 49 | 50 | def _black_scholes (self, nopt, price, strike, t, rate, vol ): 51 | mr = -rate 52 | sig_sig_two = vol * vol * 2 53 | 54 | P = price 55 | S = strike 56 | T = t 57 | 58 | a = log(P / S) 59 | b = T * mr 60 | 61 | z = T * sig_sig_two 62 | c = 0.25 * z 63 | y = invsqrt(z) 64 | 65 | w1 = (a - b + c) * y 66 | w2 = (a - b - c) * y 67 | 68 | d1 = 0.5 + 0.5 * erf(w1) 69 | d2 = 0.5 + 0.5 * erf(w2) 70 | 71 | Se = exp(b) * S 72 | 73 | call = P * d1 - Se * d2 74 | put = call - P + Se 75 | 76 | return (call, put) 77 | 78 | def _ops(self, n): 79 | # TODO: Need better ops count here 80 | return 2E-9 * n*11 81 | 82 | def _make_args(self, n): 83 | self._nopt=n 84 | self._price, self._strike, self._t = self._gen_data(self._nopt) 85 | self._call = np.zeros(self._nopt, dtype=np.float64) 86 | self._put = -np.ones(self._nopt, dtype=np.float64) 87 | 88 | def _compute(self): 89 | self._black_scholes(self._nopt, self._price, self._strike, self._t, RISK_FREE, VOLATILITY) 90 | 91 | -------------------------------------------------------------------------------- /ibench/configs/config.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from __future__ import print_function 6 | import datetime 7 | import os 8 | import subprocess 9 | import sys 10 | 11 | class Config: 12 | _docker = None 13 | _python_path = '/usr/bin/python' 14 | _affinity = None 15 | _numactl = None 16 | _mt_size = 'large' 17 | _st_size = 'small' 18 | 19 | def __init__(self, args): 20 | self.name = self.__class__.__name__ 21 | self.args = args 22 | if self.args.cpu == 'xeon': 23 | self._affinity = 'compact' 24 | self._numactl = '--interleave=all' 25 | 26 | def _add_docker_env(self, key, value): 27 | if self._docker: 28 | c = ' -e' 29 | else: 30 | c = '' 31 | c += ' %s=%s' % (key,value) 32 | return c 33 | 34 | def _log(self, message): 35 | if self.args.quiet: 36 | return 37 | print('%s: %s' % (self.name,message), file=sys.stderr) 38 | 39 | def build(self): 40 | pass 41 | 42 | def size(self, threads): 43 | if self.args.size != 'auto': 44 | return self.args.size 45 | return self._mt_size if threads > 1 else self._st_size 46 | 47 | def run(self, threads): 48 | cmd = '' 49 | if self._docker: 50 | cmd += 'docker run' 51 | if self.args.editable: 52 | cmd += ' -v %s:/ibench-master' % os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 53 | if threads > 0: 54 | cmd += self._add_docker_env('OMP_NUM_THREADS', threads) 55 | if self._affinity: 56 | cmd += self._add_docker_env('KMP_AFFINITY', self._affinity) 57 | if self._docker: 58 | cmd += ' --rm %s' % self._docker 59 | if self._numactl: 60 | cmd += ' numactl %s' % self._numactl 61 | cmd += ' %s' % self._python_path 62 | cmd += ' -m ibench run' 63 | cmd += ' --name %s' % self.name 64 | cmd += ' --size %s' % self.size(threads) 65 | cmd += ' %s' % self.args.run_args 66 | self._log(cmd) 67 | if not self.args.dry_run: 68 | time = datetime.datetime.now() 69 | date = time.strftime('%Y-%m-%d-%H-%M-%S') 70 | try: 71 | os.mkdir('results') 72 | except OSError: 73 | pass 74 | filename = 'results/bench-%s.json' % date 75 | with open(filename,'w') as stdoutfp: 76 | subprocess.call(cmd,stdout=stdoutfp,shell=True) 77 | -------------------------------------------------------------------------------- /ibench/benchmarks/bench.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from __future__ import print_function 6 | import gc 7 | import numpy as np 8 | import sys 9 | import time 10 | 11 | class Bench: 12 | _dtype = np.double 13 | 14 | def __init__(self, cmd): 15 | self.name = self.__class__.__name__ 16 | self._cmd = cmd 17 | 18 | def _log(self, message): 19 | if self._cmd.args.quiet: 20 | return 21 | print('%s: %s' % (self.name,message), file=sys.stderr) 22 | 23 | def _run(self, n): 24 | self._make_args(n) 25 | 26 | # Disable garbage collector 27 | gcold = gc.isenabled() 28 | gc.disable() 29 | 30 | times = [] 31 | for i in range(self._cmd.args.runs): 32 | t_start = time.time() 33 | self._compute() 34 | elapsed = time.time() - t_start 35 | times.append(elapsed) 36 | 37 | if gcold: 38 | gc.enable() 39 | 40 | return times 41 | 42 | def measure(self, n, args): 43 | self._log('') 44 | self._log(' N = %d' % n) 45 | 46 | times = self._run(n) 47 | 48 | ops = self._ops(n) 49 | if args.gflops is False: 50 | for elapsed in times: 51 | self._log(' elapsed %f' % (elapsed)) 52 | 53 | else: 54 | for elapsed in times: 55 | self._log(' elapsed %f gflops %f' % (elapsed,ops/elapsed)) 56 | 57 | self._summarize(n, times, args) 58 | 59 | def _summarize(self, n, times, args): 60 | t = np.asarray(times) 61 | median = np.median(t) 62 | ops = self._ops(n) 63 | if args.gflops is False: 64 | self._cmd.results['runs'].append({'name': self.name, 65 | 'N': n, 66 | 'times': times, 67 | 'stats': {'min': np.amin(t), 68 | 'max': np.max(t), 69 | 'median': median}}) 70 | else: 71 | self._log(' gflops %f' % (ops/median)) 72 | self._cmd.results['runs'].append({'name': self.name, 73 | 'N': n, 74 | 'gflops': ops/median, 75 | 'ops': ops, 76 | 'times': times, 77 | 'stats': {'min': np.amin(t), 78 | 'max': np.max(t), 79 | 'median': median}}) 80 | -------------------------------------------------------------------------------- /ibench/native/c/lu.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2016 Intel Corporation 3 | * 4 | * SPDX-License-Identifier: MIT 5 | */ 6 | 7 | #include "bench.h" 8 | 9 | class lu_C : public Bench { 10 | public: 11 | lu_C(); 12 | ~lu_C(); 13 | void make_args(int size); 14 | void compute(); 15 | 16 | private: 17 | int *ipiv; 18 | double *x_mat, *r_mat, *l_mat, *u_mat, *p_mat; 19 | int m,n,lda,mn_min; 20 | int l_size, u_size, p_size; 21 | }; 22 | 23 | 24 | lu_C::lu_C() { 25 | x_mat = r_mat = l_mat = u_mat = p_mat = 0; 26 | } 27 | 28 | void 29 | lu_C::make_args(int size) { 30 | m = n = lda = size; 31 | 32 | int mat_size = m*n; 33 | int r_size = mat_size, 34 | 35 | mn_min = min(m, n); 36 | l_size = m*mn_min; 37 | u_size = mn_min*n; 38 | p_size = m*m; 39 | 40 | /* input matrix */ 41 | x_mat = make_random_mat(mat_size); 42 | 43 | /* list of pivots */ 44 | ipiv = (int *) mkl_malloc(mn_min * sizeof(int), 64); 45 | assert(ipiv); 46 | 47 | /* matrix for result */ 48 | r_mat = make_random_mat(r_size); 49 | 50 | /* lower-triangular matrix */ 51 | l_mat = make_random_mat(l_size); 52 | 53 | /* upper triangular matrix */ 54 | u_mat = make_random_mat(u_size); 55 | 56 | /* permutation matrix */ 57 | p_mat = make_random_mat(p_size); 58 | 59 | mkl_domatcopy('R', 'T', m, n, 1.0, x_mat, n, r_mat, m); 60 | lda = m + n - lda; 61 | } 62 | 63 | void lu_C::compute() { 64 | /* compute pivoted lu decomposition */ 65 | int info = LAPACKE_dgetrf(LAPACK_COL_MAJOR, m, n, r_mat, lda, ipiv); 66 | assert(info == 0); 67 | 68 | int ld_l = m; 69 | int ld_u = mn_min; 70 | int ld_p = m; 71 | memset(l_mat, 0, l_size * sizeof(double)); 72 | memset(u_mat, 0, u_size * sizeof(double)); 73 | 74 | /* extract L and U matrix elements from r_mat */ 75 | #pragma ivdep 76 | for(int i = 0; i < m; i++) { 77 | #pragma ivdep 78 | for(int j = 0; j < n; j++){ 79 | if (j < mn_min) { 80 | if(i == j) { 81 | l_mat[j * ld_l + i] = 1.0; 82 | } else if (i > j) { 83 | l_mat[j * ld_l + i] = r_mat[j * lda + i]; 84 | } 85 | } 86 | if (i < mn_min) { 87 | if(i <= j) 88 | u_mat[j * ld_u + i] = r_mat[j * lda + i]; 89 | } 90 | } 91 | } 92 | 93 | /* make a diagonal matrix (m,m) */ 94 | memset(p_mat, 0, p_size * sizeof(double)); 95 | for(int i = 0; i < m; i++) p_mat[i*(m + 1)] = 1.0; 96 | 97 | info = LAPACKE_dlaswp(LAPACK_COL_MAJOR, m, p_mat, m, 1, mn_min, ipiv, -1); 98 | assert(info == 0); 99 | } 100 | 101 | lu_C::~lu_C() { 102 | if (l_mat) mkl_free(l_mat); 103 | if (u_mat) mkl_free(u_mat); 104 | if (r_mat) mkl_free(r_mat); 105 | if (p_mat) mkl_free(p_mat); 106 | 107 | if (ipiv) mkl_free(ipiv); 108 | if (x_mat) mkl_free(x_mat); 109 | } 110 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016, 2018 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from __future__ import print_function 6 | from setuptools import setup 7 | 8 | 9 | def build_native(): 10 | '''Return cythonized extensions for native benchmarks''' 11 | try: 12 | # use icx if it is available 13 | icpx = subprocess.check_output('which icpx',shell=True).decode('utf-8') 14 | except: 15 | icpx = None 16 | extra_args = [] 17 | else: 18 | print('Using icpx: %s' % icpx) 19 | os.environ['CC'] = icpx 20 | os.environ['CXX'] = os.environ['CC'] 21 | extra_args = ['-qmkl'] 22 | 23 | if not 'CXX' in os.environ: 24 | print('icpx not detected, and CXX is not set. Skipping building native benchmarks.') 25 | print('If you want to build native benchmarks, specify a compiler in the CXX ' 26 | 'environment variable.') 27 | return 28 | 29 | try: 30 | os.mkdir('pyx') 31 | except OSError: 32 | pass 33 | 34 | def make_bench(name): 35 | tpl_env = Environment(loader=FileSystemLoader('ibench/native')) 36 | with open('pyx/%s.pyx' % name,'w') as pyxf: 37 | pyxf.write(tpl_env.get_template('tpl.bench.pyx').render({'bench': name, 'Bench': name.capitalize()})) 38 | return Extension(name='ibench.native.%s' % name, 39 | extra_compile_args=extra_args, 40 | extra_link_args=extra_args, 41 | sources=['pyx/%s.pyx' % name]) 42 | 43 | return cythonize([make_bench(i) for i in ['det', 'dot', 'inv', 'lu', 'cholesky', 'qr']]) 44 | 45 | 46 | packages = ['ibench','ibench/docker','ibench/cmds','ibench/configs','ibench/benchmarks'] 47 | 48 | 49 | try: 50 | from Cython.Build import cythonize 51 | from jinja2 import FileSystemLoader 52 | from jinja2 import Environment 53 | import os 54 | from setuptools import Extension 55 | import subprocess 56 | import sys 57 | except ImportError: 58 | print('Cython not found. Skipping building native benchmarks.') 59 | extensions = None 60 | else: 61 | # Cython and Jinja2 found in build environment. Look for compilers 62 | extensions = build_native() 63 | if extensions: 64 | packages.append('ibench/native') 65 | 66 | 67 | setup(name='ibench', 68 | version='0.1rc', 69 | description='Benchmarking for scientific python', 70 | url='http://github.com/intelpython/ibench', 71 | download_url='http://github.com/intelpython/ibench/tarball/0.1rc', 72 | author='Robert Cohn', 73 | author_email='Robert.S.Cohn@intel.com', 74 | license='MIT', 75 | packages=packages, 76 | install_requires=['jinja2','numpy','scipy','scikit-learn'], 77 | ext_modules=extensions, 78 | package_data={'ibench': ['docker/Dockerfile.tpl']}, 79 | zip_safe=False) 80 | -------------------------------------------------------------------------------- /ibench/cmds/configs.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2017 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import argparse 6 | 7 | 8 | from .cmd import Cmd 9 | from ..configs import configs 10 | 11 | 12 | def add_parser(subparsers): 13 | parser = subparsers.add_parser('configs') 14 | parser.set_defaults(func=Configs) 15 | parser.add_argument('--run-args', 16 | default='', 17 | help='arguments to pass to run command') 18 | parser.add_argument('--cpu', 19 | choices=['xeon','phi','core'], 20 | default='xeon', 21 | help='cpu executing the benchmarks') 22 | parser.add_argument('--dry-run', 23 | default=False, 24 | action='store_true', 25 | help="Emit commands, but do not run them") 26 | parser.add_argument('--editable', 27 | default=True, 28 | action='store_true', 29 | help="Install ibench editable in docker image so we can change it") 30 | parser.add_argument('-q', 31 | '--quiet', 32 | default=False, 33 | action='store_true', 34 | help="Logging") 35 | parser.add_argument('--build', 36 | default=[], 37 | choices=configs.keys(), 38 | nargs='+', 39 | help='Configs to build') 40 | parser.add_argument('--run', 41 | default=[], 42 | choices=configs.keys(), 43 | nargs='+', 44 | help='Configs to run') 45 | parser.add_argument('--size', 46 | default='test', 47 | choices=['auto','small','large','tiny','test'], 48 | help='Size of problem. auto adjusts size so test will finish in a few minutes') 49 | parser.add_argument('--threads', 50 | default=None, 51 | nargs='+', 52 | type=int, 53 | help="Number of threads to use") 54 | 55 | class Configs(Cmd): 56 | 57 | def __init__(self, args): 58 | if args.threads: 59 | self._threads = args.threads 60 | elif args.cpu == 'xeon': 61 | self._threads = [1,32] 62 | elif args.cpu == 'phi': 63 | self._threads = [1,64] 64 | elif args.cpu == 'core': 65 | self._threads = [1,2] 66 | 67 | for config_name in args.build: 68 | config = configs[config_name](args) 69 | config.build() 70 | for config_name in args.run: 71 | config = configs[config_name](args) 72 | for threads in self._threads: 73 | config.run(threads) 74 | -------------------------------------------------------------------------------- /ibench/docker/Dockerfile.tpl: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | {% if os_name == "centos" %} 6 | 7 | {% elif os_name == "ubuntu" %} 8 | FROM ubuntu:16.04 9 | ENV DEBIAN_FRONTEND=noninteractive 10 | RUN apt-get update && apt install -y \ 11 | libatlas3-base \ 12 | libblas3 \ 13 | liblapack3 \ 14 | libopenblas-base \ 15 | numactl \ 16 | python \ 17 | python-pip \ 18 | unzip \ 19 | wget \ 20 | && pip install --upgrade pip 21 | {% endif %} 22 | 23 | ARG MINICONDA_PACKAGE=Miniconda3-4.1.11-Linux-x86_64.sh 24 | 25 | RUN wget --quiet https://github.com/rscohn2/ibench/archive/master.zip \ 26 | && unzip master.zip \ 27 | && rm master.zip 28 | 29 | {% if config == "shared" %} 30 | 31 | RUN pip install numpy scipy \ 32 | && pip install -e ibench-master 33 | 34 | # create all the anaconda installations 35 | RUN wget -q https://repo.continuum.io/miniconda/$MINICONDA_PACKAGE 36 | RUN chmod +x $MINICONDA_PACKAGE 37 | ARG MINICONDA=/miniconda3 38 | RUN ./$MINICONDA_PACKAGE -b -p $MINICONDA 39 | RUN rm $MINICONDA_PACKAGE \ 40 | && $MINICONDA/bin/conda update -y -q conda 41 | ENV ACCEPT_INTEL_PYTHON_EULA=yes 42 | RUN $MINICONDA/bin/conda create -y -q -n idp2017.0.0 -c intel intelpython3_core=2017.0.0 43 | RUN $MINICONDA/envs/idp2017.0.0/bin/pip install -e ibench-master 44 | RUN $MINICONDA/bin/conda create -y -q -n idp2017.0.1 -c intel intelpython3_core=2017.0.1 45 | RUN $MINICONDA/envs/idp2017.0.1/bin/pip install -e ibench-master \ 46 | && $MINICONDA/bin/conda create -y -q -n anaconda scipy \ 47 | && $MINICONDA/envs/anaconda/bin/pip install -e ibench-master 48 | 49 | {% else %} 50 | 51 | RUN apt-get install -y \ 52 | python3-numpy \ 53 | python3-scipy \ 54 | pip install -e ibench-master 55 | 56 | {% if config == "sys_atlas" %} 57 | RUN update-alternatives --set libblas.so.3 /usr/lib/atlas-base/atlas/libblas.so.3 \ 58 | && update-alternatives --set liblapack.so.3 /usr/lib/atlas-base/atlas/liblapack.so.3 59 | {% elif config == "sys_openblas" %} 60 | RUN update-alternatives --set libblas.so.3 /usr/lib/openblas-base/libblas.so.3 \ 61 | && update-alternatives --set liblapack.so.3 /usr/lib/openblas-base/liblapack.so.3 62 | {% elif config == "sys_reference" %} 63 | RUN update-alternatives --set libblas.so.3 /usr/lib/libblas/libblas.so.3 \ 64 | && update-alternatives --set liblapack.so.3 /usr/lib/lapack/liblapack.so.3 65 | {% endif %} 66 | {% endif %} 67 | 68 | MAINTAINER Robert Cohn 69 | LABEL org.label-schema.build-date="{{build_date}}" \ 70 | org.label-schema.name="IDP Python benchmarking container" \ 71 | org.label-schema.description="Container used for benchmarking scipy stack and related components" \ 72 | org.label-schema.url="https://www.intel.com/content/www/us/en/developer/tools/oneapi/distribution-for-python.html" \ 73 | org.label-schema.vcs-ref="{{vcs_ref}}" \ 74 | org.label-schema.vcs-url="https://github.com/rscohn2/IDP-bench-docker" \ 75 | org.label-schema.vendor="Intel" \ 76 | org.label-schema.schema-version="1.0" 77 | -------------------------------------------------------------------------------- /ibench/docker/build.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from __future__ import print_function 6 | 7 | import argparse 8 | import datetime 9 | from jinja2 import PackageLoader 10 | from jinja2 import Environment 11 | import os 12 | import pkg_resources 13 | import subprocess 14 | import sys 15 | 16 | 17 | def get_proxies(): 18 | proxies = '' 19 | for var in ['http_proxy','https_proxy','no_proxy']: 20 | if var in os.environ: 21 | proxies += ' --build-arg %s=%s' % (var,os.environ[var]) 22 | return proxies 23 | 24 | def docker_build(envs, publish): 25 | proxies = get_proxies() 26 | for env in envs: 27 | dockerfile = dockerfileName(env) 28 | repo = 'rscohn2/%s' % repoName(env) 29 | tags = '-t %s' % repo 30 | command = 'docker build %s %s --file %s .' % (proxies,tags,dockerfile) 31 | print(command) 32 | subprocess.check_call(command, shell=True) 33 | if publish: 34 | subprocess.check_call('docker login -u $DOCKER_USER -p $DOCKER_PASSWORD',shell=True) 35 | subprocess.check_call('docker push %s' % repo,shell=True) 36 | 37 | 38 | def repoName(env): 39 | return 'ibench.%s.%s' % (env['config'],env['os_name']) 40 | 41 | def dockerfileName(env): 42 | return 'Dockerfile.%s' % repoName(env) 43 | 44 | def gen_dockerfiles(envs): 45 | tpl_env = Environment(loader=PackageLoader('ibench', 'docker')) 46 | for env in envs: 47 | with open(dockerfileName(env),'w') as df: 48 | df.write(tpl_env.get_template('Dockerfile.tpl').render(env)) 49 | 50 | #os_default = ['centos','ubuntu'] 51 | os_default = ['ubuntu'] 52 | config_default = ['sys_openblas', 'sys_atlas', 'sys_reference', 'pip'] 53 | 54 | def parseArgs(): 55 | argParser = argparse.ArgumentParser(description='Build Dockerfiles and images for benchmarking', 56 | formatter_class=argparse.RawDescriptionHelpFormatter) 57 | argParser.add_argument('--publish', default=False, action='store_true', help='publish on dockerhub') 58 | argParser.add_argument('--os', default=None, nargs='+', 59 | help='operating system for docker image. Default: %s' % os_default) 60 | argParser.add_argument('--config', default=None, nargs='+', 61 | help='python config. Default: %s' % config_default) 62 | args = argParser.parse_args() 63 | if not args.os: 64 | args.os = os_default 65 | if not args.config: 66 | args.config = config_default 67 | return args 68 | 69 | def genEnvs(args): 70 | envs = [] 71 | build_date = datetime.datetime.now().strftime('%c') 72 | vcs_ref = subprocess.check_output('git rev-parse --short HEAD',shell=True).strip() 73 | for os in args.os: 74 | for config in args.config: 75 | envs.append({'os_name': os, 'config': config, 'build_date': build_date, 'vcs_ref': vcs_ref}) 76 | return envs 77 | 78 | def build(envs, publish=False): 79 | gen_dockerfiles(envs) 80 | docker_build(envs, publish) 81 | 82 | if __name__ == '__main__': 83 | args = parseArgs() 84 | envs = genEnvs(args) 85 | build(envs, args.publish) 86 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ibench 2 | 3 | Benchmarks for Intel Distribution for Python. 4 | 5 | ## Install 6 | ```bash 7 | # Since you want to benchmark numpy/scipy, manually install the one you want 8 | # instead of letting pip install one 9 | # cython is needed to build native extensions in ibench_native 10 | conda install scipy cython scikit-learn 11 | pip install -v --upgrade . 12 | ``` 13 | 14 | ### Native versions 15 | If `icc` and `cython` are available during the build, they will be used 16 | to build native benchmarks. To specify a different compiler, specify one 17 | in the environment variable `CXX`. 18 | 19 | ## Run 20 | ```bash 21 | # basic command 22 | python -m ibench run -b all --size large --runs 3 --file all.out 23 | ``` 24 | 25 | ### Specifying benchmarks 26 | - To run one or multiple benchmarks, pass the `-b BENCHMARKS...` option. 27 | Benchmarks can be specified individually, or in predefined groups 28 | (e.g. `native` contains all native benchmarks). 29 | - To specify the problem size, use the `--size` option. This selects 30 | from a list of predefined problem sizes. 31 | 32 | ### Configuring output 33 | ibench supports both JSON (default) and CSV output with collection of 34 | environment information. When using CSV format, ibench will prefix each 35 | line of environment information with a comment character (default `@`). 36 | Use the following arguments to the `run` subparser to configure its output: 37 | - `-f,--format FORMAT` - use the specified format (`json` or `csv`) 38 | - `--no-get-env-info` - don't collect environment or machine info 39 | - `--env-info-prefix` - prefix character to use before environment info 40 | 41 | ## Running benchmarks by domain 42 | 43 | ### Linear Algebra 44 | - To run python benchmarks: `python -m ibench run -b linalg --size large --runs 3 --file linalg.out` 45 | - To run native benchmarks*: `python -m ibench run -b native --size large --runs 3 --file native.out` 46 | 47 | \* Currently, native benchmarks are only available for `det`, `dot`, `lu`, and `inv`. 48 | 49 | ### Machine Learning 50 | - To run python benchmarks: `python -m ibench run -b sklearn --size large --runs 3 --file sklearn.out` 51 | - For comparable python and native benchmarks, see [scikit-learn_bench](https://github.com/IntelPython/scikit-learn_bench). 52 | 53 | ### Fast Fourier Transforms 54 | - To run python benchmarks: `python -m ibench run -b fft --size large --runs 3 --file fft.out` 55 | - For comparable python and native benchmarks, see [fft_benchmark](https://github.com/IntelPython/fft_benchmark). 56 | 57 | ### Random Number Generation 58 | - To run python benchmarks: `python -m ibench run -b rng --size large --runs 3 --file rng.out` 59 | - For comparable python and native benchmarks, see [optimizations_bench](https://github.com/IntelPython/optimizations_bench#random-number-generation). 60 | 61 | ### Black-Scholes Formula 62 | - To run python Black-Scholes benchmark: `python -m ibench run -b blacksch --size large --runs 3 --file blacksch.out` 63 | - For comparable python and native benchmarks, see [BlackScholes_bench](https://github.com/IntelPython/BlackScholes_bench). 64 | 65 | ### UMath 66 | - See [optimizations_bench](https://github.com/IntelPython/optimizations_bench#random-number-generation). 67 | 68 | ## Help 69 | ```bash 70 | python -m ibench --help 71 | python -m ibench run --help 72 | ``` 73 | -------------------------------------------------------------------------------- /.github/workflows/scorecard.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. They are provided 2 | # by a third-party and are governed by separate terms of service, privacy 3 | # policy, and support documentation. 4 | 5 | name: Scorecard supply-chain security 6 | on: 7 | # For Branch-Protection check. Only the default branch is supported. See 8 | # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection 9 | branch_protection_rule: 10 | # To guarantee Maintained check is occasionally updated. See 11 | # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained 12 | schedule: 13 | - cron: '45 12 * * 5' 14 | push: 15 | branches: [ "master" ] 16 | 17 | # Declare default permissions as read only. 18 | permissions: read-all 19 | 20 | jobs: 21 | analysis: 22 | name: Scorecard analysis 23 | runs-on: ubuntu-latest 24 | permissions: 25 | # Needed to upload the results to code-scanning dashboard. 26 | security-events: write 27 | # Needed to publish results and get a badge (see publish_results below). 28 | id-token: write 29 | # Uncomment the permissions below if installing in a private repository. 30 | # contents: read 31 | # actions: read 32 | 33 | steps: 34 | - name: "Checkout code" 35 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 36 | with: 37 | persist-credentials: false 38 | 39 | - name: "Run analysis" 40 | uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 41 | with: 42 | results_file: results.sarif 43 | results_format: sarif 44 | # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: 45 | # - you want to enable the Branch-Protection check on a *public* repository, or 46 | # - you are installing Scorecard on a *private* repository 47 | # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. 48 | # repo_token: ${{ secrets.SCORECARD_TOKEN }} 49 | 50 | # Public repositories: 51 | # - Publish results to OpenSSF REST API for easy access by consumers 52 | # - Allows the repository to include the Scorecard badge. 53 | # - See https://github.com/ossf/scorecard-action#publishing-results. 54 | # For private repositories: 55 | # - `publish_results` will always be set to `false`, regardless 56 | # of the value entered here. 57 | publish_results: true 58 | 59 | # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF 60 | # format to the repository Actions tab. 61 | # - name: "Upload artifact" 62 | # uses: actions/upload-artifact@97a0fba1372883ab732affbe8f94b823f91727db # v3.pre.node20 63 | # with: 64 | # name: SARIF file 65 | # path: results.sarif 66 | # retention-days: 5 67 | 68 | # # Upload the results to GitHub's code scanning dashboard (optional). 69 | # # Commenting out will disable upload of results to your repo's Code Scanning dashboard 70 | # - name: "Upload to code-scanning" 71 | # uses: github/codeql-action/upload-sarif@v3 72 | # with: 73 | # sarif_file: results.sarif 74 | -------------------------------------------------------------------------------- /ibench/cmds/run.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2016-2019 Intel Corporation 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from __future__ import print_function 6 | 7 | import csv 8 | import datetime 9 | import json 10 | import os 11 | import platform 12 | import subprocess 13 | import sys 14 | 15 | from ..benchmarks import benchmarks 16 | from ..benchmarks import benchmark_groups 17 | from .cmd import Cmd 18 | 19 | 20 | def capture_multiline_output(command): 21 | try: 22 | out = subprocess.check_output(command, shell=True, 23 | stderr=subprocess.STDOUT) 24 | return out.decode().split('\n') 25 | except subprocess.CalledProcessError: 26 | return '' 27 | 28 | 29 | def add_parser(subparsers): 30 | parser = subparsers.add_parser('run') 31 | parser.add_argument('-b', '--benchmarks', 32 | choices=list(benchmarks.keys()) 33 | + list(benchmark_groups.keys()), 34 | nargs='+', 35 | help='Benchmarks to run') 36 | parser.add_argument('--file', 37 | help='Write results to instead of stdout') 38 | parser.add_argument('--name', default='noname', 39 | help='Descriptive name of run to include in results') 40 | parser.add_argument('--size', 41 | choices=['large', 'small', 'tiny', 'test'], 42 | default='test', 43 | help='Size of workload to run. In general, use large ' 44 | 'for multicore, small for single thread, and ' 45 | 'test for debugging') 46 | parser.add_argument('-q', '--quiet', default=False, action='store_true', 47 | help='Suppress logging outputs') 48 | parser.add_argument('--runs', default=3, type=int, help='Number of runs') 49 | parser.add_argument('--gflops', default=False, 50 | help='Turn on approximate gflops for test saturation') 51 | parser.add_argument('--no-get-env-info', dest='get_env_info', 52 | action='store_false', default=True, 53 | help='Skip getting environment information') 54 | parser.add_argument('-f', '--format', choices=['json', 'csv'], 55 | default='json', help='Output format') 56 | parser.add_argument('--env-info-prefix', default='@', 57 | help='Character to put in front of each line of env ' 58 | 'info when using csv output') 59 | parser.set_defaults(func=Run) 60 | 61 | 62 | class Run(Cmd): 63 | results = {} 64 | 65 | def __init__(self, args): 66 | '''Run a set of benchmarks''' 67 | self.args = args 68 | self._parse_bench() 69 | self._add_configuration() 70 | for bench_name in self._bmarks: 71 | bench = benchmarks[bench_name](self) 72 | n = bench.sizes[self.args.size] 73 | bench.measure(n, args) 74 | del bench 75 | self._write_output() 76 | 77 | def _parse_bench(self): 78 | default_bench = ['dot'] 79 | self._bmarks = [] 80 | 81 | bstack = self.args.benchmarks 82 | if bstack is None: 83 | bstack = default_bench 84 | 85 | for bs in bstack: 86 | # expand groups 87 | if bs in benchmark_groups: 88 | bstack.extend(benchmark_groups[bs]) 89 | elif bs in benchmarks: 90 | self._bmarks.append(bs) 91 | else: 92 | print('Unknown benchmark: ', 93 | bs, 94 | ' must be in: ', 95 | list(benchmarks.keys()) + list(benchmark_groups.keys()), 96 | file=sys.stderr) 97 | sys.exit(1) 98 | 99 | def _set_from_environ(self, key): 100 | self.results[key] = os.environ[key] if key in os.environ else 'not set' 101 | 102 | def _add_configuration(self): 103 | results = self.results 104 | time = datetime.datetime.now() 105 | results['name'] = self.args.name 106 | results['date'] = time.strftime('%Y-%m-%d-%H-%M-%S') 107 | self._set_from_environ('KMP_AFFINITY') 108 | self._set_from_environ('OMP_NUM_THREADS') 109 | self._set_from_environ('MKL_NUM_THREADS') 110 | 111 | if self.args.get_env_info: 112 | results['host'] = platform.node() 113 | results['lscpu'] = capture_multiline_output('lscpu') 114 | results['numactl'] = capture_multiline_output('numactl --show') 115 | results['pip list'] = capture_multiline_output('/usr/bin/pip list') 116 | results['conda'] = capture_multiline_output('conda list') 117 | 118 | results['runs'] = [] 119 | 120 | def _write_output(self): 121 | filename = self.args.file 122 | fh = open(filename, 'w') if filename else sys.stdout 123 | 124 | if self.args.format == 'json': 125 | json.dump(self.results, fh, indent=2) 126 | elif self.args.format == 'csv': 127 | for env_key in self.results.keys() - {'name', 'date', 'runs'}: 128 | 129 | # Output environment information in an easily cleanable way 130 | # We will prefix each line with something like "@ " 131 | env_prefix = self.args.env_info_prefix 132 | 133 | env_info = self.results[env_key] 134 | if type(env_info) is str: 135 | fh.write('{0}{0} {1}={2}\n'.format(env_prefix, env_key, 136 | env_info)) 137 | else: 138 | fh.write('{0}{0} {1}=\n'.format(env_prefix, env_key)) 139 | for line in [env_key + ":"] + env_info: 140 | fh.write('{0} {1}\n'.format(env_prefix, line)) 141 | 142 | # Now, for the actual CSV output... 143 | writer = csv.writer(fh, delimiter=',') 144 | writer.writerow(['Prefix', 'Function', 'Size', 'Time']) 145 | 146 | all_runs_output = [self.results['name']] 147 | 148 | for bench in self.results['runs']: 149 | bench_output = all_runs_output + [bench['name'], bench['N']] 150 | for time in bench['times']: 151 | time_output = bench_output + [time] 152 | writer.writerow(time_output) 153 | 154 | if filename: 155 | fh.close() 156 | --------------------------------------------------------------------------------