├── lib ├── benchpark │ ├── __init__.py │ ├── test │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── spec_semantics.py │ │ ├── paths.py │ │ └── system.py │ ├── cmd │ │ ├── bootstrap.py │ │ ├── configure.py │ │ └── experiment.py │ ├── openmpsystem.py │ ├── error.py │ ├── debug.py │ ├── mpi.py │ ├── cudasystem.py │ ├── rocmsystem.py │ ├── openmp.py │ ├── cuda.py │ ├── rocm.py │ └── paths.py └── scripts │ ├── env-collect-branch-tips.py │ └── experiment-build-info.py ├── .codespellignore ├── setup-env.sh ├── var ├── exp_repo │ └── repo.yaml └── sys_repo │ └── repo.yaml ├── docs ├── Tree.png ├── No-mpi.png ├── requirements.txt ├── graph-and-tree.png ├── flux_jobs_a_output.png ├── _static │ ├── images │ │ ├── dryruns.png │ │ ├── workflow_fig.png │ │ ├── dray-diffspecs.png │ │ ├── shared-nonshared.png │ │ ├── quicksilver-diffspecs.png │ │ ├── kripke_cuda_weak_raw_exc.png │ │ ├── kripke_cuda_strong_raw_exc.png │ │ ├── kripke_cuda_strong_raw_inc.png │ │ ├── kripke_cuda_strong_raw_exc-2.png │ │ ├── kripke_cuda_throughput_raw_exc.png │ │ └── kripke_cuda_strong_percentage_exc.png │ ├── main.js │ ├── qr-codes │ │ ├── benchpark-docs-qrcode.png │ │ └── benchpark-github-qrcode.png │ ├── slides │ │ ├── HPCIC25-Tutorial-Slides.pdf │ │ ├── eScience25-Tutorial-Slides.pdf │ │ └── ReproduciblePerfAnalysis-HPDC25-Tutorial-Slides.pdf │ └── css │ │ └── custom.css ├── kripke_mpi_strong_raw_exc.png ├── finished_experiments_example.png ├── tutorial │ ├── HPCIC25-Tutorial-Slide-Preview.jpg │ ├── eScience25-Tutorial-Slide-Preview.jpg │ └── ReproduciblePerfAnalysis-HPDC25-Tutorial-Slide-Preview.jpg ├── software.yaml ├── benchmark-list.rst ├── analyze-experiment.rst ├── system-list.rst ├── Makefile ├── make.bat ├── experiments_root_structure.rst ├── why-benchpark.rst ├── getting-started.rst ├── showBuild.rst ├── build-experiment.rst ├── examples │ └── compare_experiment_builds │ │ └── compareExperimentBuilds.sh ├── diffPackageCommits.rst ├── index.rst ├── conf.py ├── run-binary.rst ├── diffExperimentSpecs.rst ├── set-of-experiments.rst ├── generate-sys-defs-list.py ├── create-mirror.rst ├── add-a-benchmark.rst ├── run-experiment.rst ├── benchpark-workflow.rst ├── diffBuildSpecs.rst └── for-the-impatient.rst ├── systems ├── repo.yaml ├── csc-lumi │ └── externals │ │ └── libsci │ │ └── 02-rocmcc-packages.yaml ├── common │ └── spack_includes.yaml ├── all_hardware_descriptions │ ├── x86_64 │ │ └── hardware_description.yaml │ ├── Penguin-haswell-OmniPath │ │ └── hardware_description.yaml │ ├── AWS_PCluster-zen-EFA │ │ └── hardware_description.yaml │ ├── AWS_Tutorial-sapphirerapids-EFA │ │ └── hardware_description.yaml │ ├── DELL-cascadelake-Infiniband │ │ └── hardware_description.yaml │ ├── Fujitsu-A64FX-TofuD │ │ └── hardware_description.yaml │ ├── Supermicro-icelake-OmniPath │ │ └── hardware_description.yaml │ ├── HPECray-zen2-Slingshot │ │ └── hardware_description.yaml │ ├── DELL-sapphirerapids-H100-Infiniband │ │ └── hardware_description.yaml │ ├── DELL-sapphirerapids-OmniPath │ │ └── hardware_description.yaml │ ├── Penguin-icelake-OmniPath │ │ └── hardware_description.yaml │ ├── HPECray-neoverse-H100-Slingshot │ │ └── hardware_description.yaml │ ├── Atos-rome-A100-Infiniband │ │ └── hardware_description.yaml │ ├── HPECray-haswell-P100-Infiniband │ │ └── hardware_description.yaml │ ├── IBM-power9-V100-Infiniband │ │ └── hardware_description.yaml │ ├── HPECray-zen4-MI300A-Slingshot │ │ └── hardware_description.yaml │ ├── HPECray-zen3-A100-Slingshot │ │ └── hardware_description.yaml │ └── HPECray-zen3-MI250X-Slingshot │ │ └── hardware_description.yaml └── generic-x86 │ └── system.py ├── experiments ├── repo.yaml ├── ad │ └── experiment.py ├── commbench │ └── experiment.py ├── gpcnet │ └── experiment.py ├── smb │ └── experiment.py ├── phloem │ └── experiment.py ├── stream │ └── experiment.py ├── md-test │ └── experiment.py ├── salmon-tddft │ └── experiment.py ├── babelstream │ └── experiment.py ├── saxpy │ └── experiment.py ├── qws │ └── experiment.py ├── genesis │ └── experiment.py └── hpcg │ └── experiment.py ├── .github ├── workflows │ ├── requirements │ │ ├── docstrfmt.toml │ │ ├── style.txt │ │ └── docs.txt │ ├── nightly.yml │ ├── license.yml │ ├── diffpackages.yml │ ├── label.yml │ ├── style.yml │ ├── coverage.yml │ └── docs.yml ├── dependabot.yml ├── utils │ └── dryrun.sh ├── labeler.yml ├── ISSUE_TEMPLATE │ ├── benchmark-status.md │ └── bug_report.md └── pull_request_template.md ├── modifiers ├── modifier_repo.yaml └── affinity │ └── parse_affinity_log.py ├── pytest.ini ├── .gitignore ├── CMakeLists.txt ├── repo ├── repo.yaml ├── md-test │ └── application.py ├── blt │ └── package.py ├── kokkos │ └── package.py ├── hpl │ └── application.py ├── essl │ └── package.py ├── gromacs │ ├── gmxDetectSimd-cmake-3.14.patch │ └── gmxDetectCpu-cmake-3.14.patch ├── lapack-xl │ └── package.py ├── saxpy │ ├── config.hh.in │ ├── application.py │ └── CMakeLists.txt ├── camp │ └── package.py ├── raja │ └── package.py ├── spectrum-mpi │ └── package.py ├── hpcc │ └── application.py ├── umpire │ └── package.py ├── enzyme │ └── package.py ├── osu-micro-benchmarks │ ├── application.py │ └── package.py ├── ad │ ├── application.py │ └── package.py ├── raja-perf │ └── application.py ├── chai │ └── package.py ├── phloem │ ├── package.py │ └── application.py ├── stream │ └── package.py ├── cuda │ └── package.py ├── gpcnet │ └── application.py ├── commbench │ ├── application.py │ └── package.py ├── branson │ ├── application.py │ └── package.py ├── hpcg │ └── package.py ├── lammps │ └── package.py ├── cray-mpich │ └── package.py ├── smb │ ├── package.py │ └── application.py ├── mfem │ └── package.py ├── affinity │ └── package.py ├── salmon-tddft │ └── fjmpi.patch └── cray-mpich-gtl │ └── package.py ├── spack.yaml ├── codecov.yml ├── .gitlab ├── utils │ ├── cancel-slurm.sh │ ├── cancel-flux.sh │ ├── rules.yml │ ├── machine_checks.yml │ └── run-experiment.sh └── bin │ └── exit-codes ├── common-resources └── execute_experiment.tpl ├── checkout-versions.yaml ├── remote-urls.yaml ├── .envrc ├── requirements.txt ├── .flake8 ├── COPYRIGHT ├── CTestConfig.cmake ├── CTestGitlab.cmake ├── bin ├── benchpark-python └── benchpark ├── NOTICE ├── pyproject.toml └── CITATION.cff /lib/benchpark/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/benchpark/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/benchpark/test/conftest.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.codespellignore: -------------------------------------------------------------------------------- 1 | cachable 2 | parma -------------------------------------------------------------------------------- /setup-env.sh: -------------------------------------------------------------------------------- 1 | export PATH=${PWD}/bin:$PATH -------------------------------------------------------------------------------- /var/exp_repo/repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: builtin 3 | -------------------------------------------------------------------------------- /var/sys_repo/repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: sysbuiltin 3 | -------------------------------------------------------------------------------- /docs/Tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/Tree.png -------------------------------------------------------------------------------- /systems/repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: sysbuiltin 3 | subdirectory: '' 4 | -------------------------------------------------------------------------------- /docs/No-mpi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/No-mpi.png -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | Sphinx 2 | sphinx-rtd-theme 3 | sphinxcontrib-programoutput -------------------------------------------------------------------------------- /experiments/repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: builtin 3 | subdirectory: '' 4 | -------------------------------------------------------------------------------- /.github/workflows/requirements/docstrfmt.toml: -------------------------------------------------------------------------------- 1 | [tool.docstrfmt] 2 | line-length = 88 3 | -------------------------------------------------------------------------------- /modifiers/modifier_repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: benchpark 3 | subdirectory: '' 4 | -------------------------------------------------------------------------------- /docs/graph-and-tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/graph-and-tree.png -------------------------------------------------------------------------------- /docs/flux_jobs_a_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/flux_jobs_a_output.png -------------------------------------------------------------------------------- /docs/_static/images/dryruns.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/images/dryruns.png -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | # content of pytest.ini 2 | [pytest] 3 | testpaths = lib/benchpark/test 4 | python_files = *.py 5 | -------------------------------------------------------------------------------- /docs/kripke_mpi_strong_raw_exc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/kripke_mpi_strong_raw_exc.png -------------------------------------------------------------------------------- /docs/_static/images/workflow_fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/images/workflow_fig.png -------------------------------------------------------------------------------- /docs/_static/main.js: -------------------------------------------------------------------------------- 1 | $(document).ready( function () { 2 | $('table.datatable').DataTable({paging: false}); 3 | } ); 4 | -------------------------------------------------------------------------------- /docs/_static/images/dray-diffspecs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/images/dray-diffspecs.png -------------------------------------------------------------------------------- /docs/finished_experiments_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/finished_experiments_example.png -------------------------------------------------------------------------------- /docs/_static/images/shared-nonshared.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/images/shared-nonshared.png -------------------------------------------------------------------------------- /docs/_static/images/quicksilver-diffspecs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/images/quicksilver-diffspecs.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *~ 3 | \#*\# 4 | __pycache__ 5 | _build 6 | spack.lock 7 | configure.yaml 8 | .spack-env 9 | .env 10 | .DS_Store 11 | -------------------------------------------------------------------------------- /docs/_static/images/kripke_cuda_weak_raw_exc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/images/kripke_cuda_weak_raw_exc.png -------------------------------------------------------------------------------- /docs/_static/qr-codes/benchpark-docs-qrcode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/qr-codes/benchpark-docs-qrcode.png -------------------------------------------------------------------------------- /docs/_static/qr-codes/benchpark-github-qrcode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/qr-codes/benchpark-github-qrcode.png -------------------------------------------------------------------------------- /docs/_static/slides/HPCIC25-Tutorial-Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/slides/HPCIC25-Tutorial-Slides.pdf -------------------------------------------------------------------------------- /docs/tutorial/HPCIC25-Tutorial-Slide-Preview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/tutorial/HPCIC25-Tutorial-Slide-Preview.jpg -------------------------------------------------------------------------------- /docs/_static/images/kripke_cuda_strong_raw_exc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/images/kripke_cuda_strong_raw_exc.png -------------------------------------------------------------------------------- /docs/_static/images/kripke_cuda_strong_raw_inc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/images/kripke_cuda_strong_raw_inc.png -------------------------------------------------------------------------------- /docs/_static/slides/eScience25-Tutorial-Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/slides/eScience25-Tutorial-Slides.pdf -------------------------------------------------------------------------------- /docs/tutorial/eScience25-Tutorial-Slide-Preview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/tutorial/eScience25-Tutorial-Slide-Preview.jpg -------------------------------------------------------------------------------- /docs/_static/images/kripke_cuda_strong_raw_exc-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/images/kripke_cuda_strong_raw_exc-2.png -------------------------------------------------------------------------------- /docs/_static/images/kripke_cuda_throughput_raw_exc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/images/kripke_cuda_throughput_raw_exc.png -------------------------------------------------------------------------------- /.github/workflows/requirements/style.txt: -------------------------------------------------------------------------------- 1 | black==25.12.0 2 | flake8==7.3.0 3 | isort[colors]==7.0.0 4 | codespell==2.4.1 5 | yamlfix==1.19.0 6 | docstrfmt==2.0.0 7 | -------------------------------------------------------------------------------- /docs/_static/images/kripke_cuda_strong_percentage_exc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/images/kripke_cuda_strong_percentage_exc.png -------------------------------------------------------------------------------- /docs/_static/slides/ReproduciblePerfAnalysis-HPDC25-Tutorial-Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/_static/slides/ReproduciblePerfAnalysis-HPDC25-Tutorial-Slides.pdf -------------------------------------------------------------------------------- /docs/tutorial/ReproduciblePerfAnalysis-HPDC25-Tutorial-Slide-Preview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/llnl/benchpark/HEAD/docs/tutorial/ReproduciblePerfAnalysis-HPDC25-Tutorial-Slide-Preview.jpg -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(PythonTestProject LANGUAGES NONE) 3 | ENABLE_TESTING() 4 | INCLUDE(CTest) 5 | find_package(Python3 REQUIRED) 6 | 7 | add_test(NAME Gitlab 8 | COMMAND ${CMAKE_COMMAND} -E $ENV{EXIT_CODE}) 9 | -------------------------------------------------------------------------------- /systems/csc-lumi/externals/libsci/02-rocmcc-packages.yaml: -------------------------------------------------------------------------------- 1 | packages: 2 | cray-libsci: 3 | - spec: cray-libsci@23.09.1.1%rocmcc 4 | prefix: /opt/cray/pe/libsci/23.09.1.1/AMD/5.0/x86_64 5 | modules: [cray-libsci/23.09.1.1] 6 | buildable: false 7 | -------------------------------------------------------------------------------- /repo/repo.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | repo: 6 | namespace: benchpark 7 | subdirectory: '' 8 | -------------------------------------------------------------------------------- /systems/common/spack_includes.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | packages: 6 | blt: 7 | require: 8 | - one_of: ['%gcc', '%clang'] 9 | -------------------------------------------------------------------------------- /spack.yaml: -------------------------------------------------------------------------------- 1 | # This is a Spack Environment file. 2 | # 3 | # It describes a set of packages to be installed, along with 4 | # configuration settings. 5 | spack: 6 | # add package specs to the `specs` list 7 | specs: [py-pyyaml, py-pytest, py-codespell] 8 | view: true 9 | concretizer: 10 | unify: true 11 | -------------------------------------------------------------------------------- /docs/software.yaml: -------------------------------------------------------------------------------- 1 | # This is a Spack Environment file. 2 | # 3 | # It describes a set of packages to be installed, along with 4 | # configuration settings. 5 | spack: 6 | # add package specs to the `specs` list 7 | specs: [py-sphinx, py-sphinx-rtd-theme] 8 | view: true 9 | concretizer: 10 | unify: true 11 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | round: down 3 | range: 40...100 4 | comment: 5 | layout: header, diff, changes, tree 6 | behavior: new 7 | require_changes: false 8 | require_base: false 9 | require_head: true 10 | after_n_builds: 10 # Number of SEPARATE codecov uploads expected, e.g. "Upload coverage to Codecov" jobs 11 | -------------------------------------------------------------------------------- /lib/benchpark/cmd/bootstrap.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | def setup_parser(root_parser): 8 | pass 9 | 10 | 11 | def command(args): 12 | pass 13 | -------------------------------------------------------------------------------- /.gitlab/utils/cancel-slurm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | 4 | NO_CLEAN=false 5 | if [[ "$1" == "--no-clean" ]]; then 6 | NO_CLEAN=true 7 | fi 8 | 9 | export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A) 10 | ([[ -n "${JOBID}" ]] && scancel ${JOBID} || exit 0) 11 | 12 | if ! $NO_CLEAN; then 13 | rm -rf $CUSTOM_CI_BUILDS_DIR 14 | fi 15 | -------------------------------------------------------------------------------- /common-resources/execute_experiment.tpl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | 7 | {allocation_directives} 8 | 9 | cd {experiment_run_dir} 10 | 11 | {pre_exec} 12 | {command} 13 | {post_exec} 14 | -------------------------------------------------------------------------------- /.gitlab/utils/cancel-flux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | 4 | NO_CLEAN=false 5 | if [[ "$1" == "--no-clean" ]]; then 6 | NO_CLEAN=true 7 | fi 8 | 9 | export URI=$(flux jobs -o "{id} {name}" | grep ${ALLOC_NAME}${GPUMODE} | awk '{print $1}') 10 | ([[ -n "${URI}" ]] && flux cancel ${URI} || exit 0) 11 | 12 | if ! $NO_CLEAN; then 13 | rm -rf $CUSTOM_CI_BUILDS_DIR 14 | fi 15 | -------------------------------------------------------------------------------- /checkout-versions.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | versions: 6 | ramble: 6b4ecac # develop on 5/06/2025 (newer than 0.6.0 release) 7 | spack: a85ec51 # Dec. 1 2025 v1.1.0 8 | spack-packages: d13c881 # Dec. 3 2025 9 | -------------------------------------------------------------------------------- /remote-urls.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | urls: 6 | ramble: https://github.com/GoogleCloudPlatform/ramble.git 7 | spack: https://github.com/spack/spack.git 8 | spack-packages: https://github.com/spack/spack-packages.git 9 | -------------------------------------------------------------------------------- /lib/benchpark/test/spec_semantics.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2024 Lawrence Livermore National Security, LLC and other 2 | # Spack Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | from benchpark.spec import Spec 7 | 8 | 9 | def test_spec_hashing_and_eq(): 10 | x = Spec("+x") 11 | y = Spec("+y") 12 | 13 | items = set([x]) 14 | assert y not in items 15 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: / 5 | schedule: 6 | interval: daily 7 | - package-ecosystem: pip 8 | directory: /.github/workflows/requirements 9 | schedule: 10 | interval: weekly 11 | ignore: 12 | # setuptools releases new versions almost daily 13 | - dependency-name: setuptools 14 | update-types: [version-update:semver-patch] 15 | -------------------------------------------------------------------------------- /repo/md-test/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.app.builtin.md_test import MdTest as MdTestBase 9 | from ramble.appkit import * 10 | 11 | 12 | class MdTest(MdTestBase): 13 | 14 | tags = ['synthetic','i-o','large-scale','mpi','c'] 15 | -------------------------------------------------------------------------------- /lib/benchpark/openmpsystem.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import provides 7 | 8 | 9 | class OpenMPCPUOnlySystem: 10 | provides("openmp") 11 | 12 | name = "openmp" 13 | 14 | def system_specific_variables(self, system): 15 | return {} 16 | -------------------------------------------------------------------------------- /lib/benchpark/test/paths.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import pathlib 7 | 8 | import benchpark.paths 9 | 10 | 11 | def test_benchpark_root(pytestconfig): 12 | expected_path = pathlib.Path(pytestconfig.inipath).resolve().parent 13 | assert benchpark.paths.benchpark_root == expected_path 14 | -------------------------------------------------------------------------------- /.github/workflows/nightly.yml: -------------------------------------------------------------------------------- 1 | name: nightly 2 | on: 3 | schedule: 4 | - cron: 0 14 * * * 5 | #------------------------------------------------------------------------ 6 | # Execute a nightly CI run to verify benchmarks continue to build & run 7 | # sucessfully. Will also update the CI buildcache to minimize build 8 | # times in PRs. 9 | #------------------------------------------------------------------------ 10 | jobs: 11 | run: 12 | uses: ./.github/workflows/run.yml 13 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------ 2 | # Load Development Spack Environment (If Spack is installed.) 3 | # 4 | # Run 'direnv allow' from within the cloned repository to automatically 5 | # load the spack environment when you enter the directory. 6 | #------------------------------------------------------------------------ 7 | if type spack &>/dev/null; then 8 | . $SPACK_ROOT/share/spack/setup-env.sh 9 | spack env activate -d . 10 | fi 11 | -------------------------------------------------------------------------------- /docs/benchmark-list.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ############################ 8 | Benchmarks and Experiments 9 | ############################ 10 | 11 | .. csv-table:: Current Benchpark tags by Benchmark and tag groups. 12 | :file: benchmark-list.csv 13 | :header-rows: 1 14 | :align: left 15 | -------------------------------------------------------------------------------- /repo/blt/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | import re 8 | 9 | from spack.package import * 10 | from spack_repo.builtin.packages.blt.package import Blt as BuiltinBlt 11 | 12 | 13 | class Blt(BuiltinBlt): 14 | 15 | version("0.7.0", sha256="df8720a9cba1199d21f1d32649cebb9dddf95aa61bc3ac23f6c8a3c6b6083528") -------------------------------------------------------------------------------- /.github/workflows/license.yml: -------------------------------------------------------------------------------- 1 | name: License Checks 2 | on: 3 | workflow_call: 4 | jobs: 5 | verify-license: 6 | runs-on: ubuntu-24.04 7 | steps: 8 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 9 | - name: Set up Python 3.11 10 | uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 11 | with: 12 | python-version: '3.11' 13 | - name: Verify license headers 14 | run: |- 15 | python .github/workflows/bin/license verify 16 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pyyaml 2 | # The remaining requirements are from Ramble 3 | pytest 4 | flake8 5 | google-cloud-storage # for gcs fetch test 6 | google-api-core # for gcs fetch error .exceptions 7 | coverage 8 | pytest-cov 9 | pre-commit 10 | graphlib-backport;python_version<"3.9" 11 | urllib3==1.26.19;python_version<="3.6" 12 | protobuf;python_version>"3.6" 13 | protobuf==3.19.5;python_version<="3.6" 14 | pyarrow==14.0.1;python_version<="3.6" 15 | google-cloud-bigquery 16 | tqdm 17 | deprecation 18 | deepdiff 19 | xmltodict 20 | -------------------------------------------------------------------------------- /lib/benchpark/error.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | #: at what level we should write stack traces or short error messages 7 | #: this is module-scoped because it needs to be set very early 8 | debug = 0 9 | 10 | 11 | class BenchparkError(Exception): 12 | """This is the superclass for all Benchpark errors. 13 | Subclasses can be found in the modules they have to do with. 14 | """ 15 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | # -*- conf -*- 2 | # flake8 settings for thicket 3 | # 4 | # These are the flake8 settings recommended by Black 5 | # See https://github.com/psf/black 6 | # 7 | [flake8] 8 | max-line-length = 88 9 | select = C,E,F,W,B,B950 10 | ignore = E501,W503,E203 11 | 12 | per-file-ignores = 13 | modifiers/*/modifier.py:F403,F405 14 | legacy/modifiers/*/modifier.py:F403,F405 15 | 16 | builtins = IPython 17 | exclude = 18 | .eggs, 19 | .git, 20 | .hg, 21 | .mypy_cache, 22 | .tox, 23 | .venv, 24 | _build, 25 | buck-out, 26 | build, 27 | dist, 28 | repo 29 | -------------------------------------------------------------------------------- /repo/kokkos/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.kokkos.package import Kokkos as BuiltinKokkos 8 | 9 | 10 | class Kokkos(BuiltinKokkos): 11 | flag_handler = build_system_flags 12 | 13 | def setup_build_environment(self, env): 14 | if "+cuda" in self.spec: 15 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 16 | -------------------------------------------------------------------------------- /repo/hpl/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.app.builtin.hpl import Hpl as HplBase 9 | from ramble.appkit import * 10 | 11 | 12 | class Hpl(HplBase): 13 | 14 | tags = ['synthetic', 15 | 'blas','solver','dense-linear-algebra', 16 | 'large-scale', 17 | 'mpi','network-collectives','network-point-to-point', 18 | 'c'] 19 | -------------------------------------------------------------------------------- /.github/workflows/diffpackages.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_call: 3 | inputs: 4 | modified_repo_dirs: 5 | required: true 6 | type: string 7 | jobs: 8 | diffPackages: 9 | runs-on: ubuntu-24.04 10 | steps: 11 | - name: Checkout repository 12 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 13 | - name: Add needed Python libs 14 | run: | 15 | pip install -r ./requirements.txt 16 | - name: Check Packages Against Spack 17 | run: |- 18 | ./bin/benchpark-python lib/scripts/diffPackageCommits.py --packages "${{ inputs.modified_repo_dirs }}" 19 | -------------------------------------------------------------------------------- /.gitlab/bin/exit-codes: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | 6 | 7 | def main(results_file): 8 | with open(results_file, "r") as f: 9 | data = json.load(f) 10 | 11 | res = 0 12 | 13 | for experiment in data["experiments"]: 14 | for context in experiment["CONTEXTS"]: 15 | for fom in context["foms"]: 16 | if fom["origin"] == "exit-code": 17 | if int(fom["value"]) != 0: 18 | res += 1 19 | 20 | return res 21 | 22 | if __name__ == "__main__": 23 | f = sys.argv[1] # path to results.latest.json 24 | sys.exit(main(f)) 25 | -------------------------------------------------------------------------------- /repo/essl/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.essl.package import Essl as BuiltinEssl 8 | 9 | 10 | class Essl(BuiltinEssl): 11 | @property 12 | def lapack_libs(self): 13 | essl_libs = super(Essl, self).lapack_libs 14 | 15 | essl_libs += find_libraries( 16 | ["libessl", "libessl6464", "libesslsmp", "libesslsmp6464"], root=self.prefix.lib64, shared=True 17 | ) 18 | return essl_libs 19 | -------------------------------------------------------------------------------- /repo/gromacs/gmxDetectSimd-cmake-3.14.patch: -------------------------------------------------------------------------------- 1 | --- a/cmake/gmxDetectSimd.cmake 2 | +++ b/cmake/gmxDetectSimd.cmake 3 | @@ -77,7 +77,7 @@ function(gmx_suggest_simd _suggested_simd) 4 | else() 5 | set(GMX_TARGET_X86_VALUE 0) 6 | endif() 7 | - set(_compile_definitions "${GCC_INLINE_ASM_DEFINE} -I${CMAKE_SOURCE_DIR}/src -DGMX_CPUINFO_STANDALONE ${GMX_STDLIB_CXX_FLAGS} -DGMX_TARGET_X86=${GMX_TARGET_X86_VALUE}") 8 | + set(_compile_definitions ${GCC_INLINE_ASM_DEFINE} -I${CMAKE_SOURCE_DIR}/src -DGMX_CPUINFO_STANDALONE ${GMX_STDLIB_CXX_FLAGS} -DGMX_TARGET_X86=${GMX_TARGET_X86_VALUE}) 9 | 10 | # Prepare a default suggestion 11 | set(OUTPUT_SIMD "None") 12 | -------------------------------------------------------------------------------- /.github/workflows/label.yml: -------------------------------------------------------------------------------- 1 | #----------------------------------------------------------------------- 2 | # DO NOT modify unless you really know what you are doing. 3 | # 4 | # See https://stackoverflow.com/a/74959635 for more info. 5 | # Talk to @alecbcs if you have questions/are not sure of a change's 6 | # possible impact to security. 7 | #----------------------------------------------------------------------- 8 | name: label 9 | on: 10 | pull_request_target: 11 | branches: [develop] 12 | jobs: 13 | pr: 14 | runs-on: ubuntu-24.04 15 | permissions: 16 | contents: read 17 | pull-requests: write 18 | steps: 19 | - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b 20 | -------------------------------------------------------------------------------- /docs/analyze-experiment.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ###################### 8 | Experiment pass/fail 9 | ###################### 10 | 11 | Once the experiments completed running, the command: 12 | 13 | :: 14 | 15 | ramble --workspace-dir . workspace analyze 16 | 17 | can be used to analyze figures of merit and evaluate `success/failure 18 | `_ of 19 | the experiments. Ramble generates a file with summary of the results in ``$workspace``. 20 | -------------------------------------------------------------------------------- /.github/workflows/requirements/docs.txt: -------------------------------------------------------------------------------- 1 | # docs 2 | sphinx==8.2.3 3 | sphinx-rtd-theme==3.0.2 4 | codespell==2.4.1 5 | pyyaml==6.0.3 6 | pandas==2.3.3 7 | sphinxcontrib-programoutput==0.18 8 | # benchpark analyze -h command 9 | llnl-thicket 10 | matplotlib 11 | # The remaining requirements are from Ramble 12 | pytest 13 | flake8 14 | google-cloud-storage # for gcs fetch test 15 | google-api-core # for gcs fetch error .exceptions 16 | coverage 17 | pre-commit 18 | graphlib-backport;python_version<"3.9" 19 | urllib3==1.26.18;python_version<="3.6" 20 | protobuf;python_version>"3.6" 21 | protobuf==3.19.4;python_version<="3.6" 22 | pyarrow==3.0.0;python_version<="3.6" 23 | google-cloud-bigquery 24 | tqdm 25 | deprecation 26 | deepdiff -------------------------------------------------------------------------------- /repo/lapack-xl/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class LapackXl(Package): 10 | 11 | provides("lapack") 12 | 13 | @property 14 | def lapack_libs(self): 15 | libname = ["liblapack", "libblas"] 16 | return find_libraries(libname, root=self.prefix.lib, shared=False, recursive=False) 17 | 18 | @property 19 | def libs(self): 20 | libname = ["liblapack", "libblas"] 21 | return find_libraries(libname, root=self.prefix.lib, shared=False, recursive=False) 22 | -------------------------------------------------------------------------------- /.github/utils/dryrun.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | #!/bin/bash 7 | 8 | benchmark_spec="$1" 9 | system_spec="$2" 10 | 11 | timestamp=$(date +%s) 12 | benchmark="b-$timestamp" 13 | system="s-$timestamp" 14 | ./bin/benchpark system init --dest=$system $system_spec 15 | ./bin/benchpark experiment init --dest=$benchmark $system $benchmark_spec 16 | ./bin/benchpark setup ./$system/$benchmark workspace/ 17 | . workspace/setup.sh 18 | ramble \ 19 | --workspace-dir "workspace/$system/$benchmark/workspace" \ 20 | --disable-logger \ 21 | workspace setup --dry-run -------------------------------------------------------------------------------- /docs/system-list.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ####################### 8 | System Specifications 9 | ####################### 10 | 11 | The table below provides a directory of information for systems that have been specified 12 | in Benchpark. The column headers in the table below are available for use as the 13 | ``system`` parameter in ``benchpark setup``. 14 | 15 | .. csv-table:: Current System Specifications in Benchpark. 16 | :class: datatable 17 | :file: current-system-definitions.csv 18 | :header-rows: 1 19 | :align: left 20 | -------------------------------------------------------------------------------- /repo/saxpy/config.hh.in: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | // Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: Apache-2.0 5 | 6 | #ifndef SAXPY_CONFIG_HH 7 | #define SAXPY_CONFIG_HH 8 | 9 | #define SAXPY_COMPILER_ID "@CMAKE_C_COMPILER_ID@" 10 | #define SAXPY_COMPILER_VERSION "@CMAKE_C_COMPILER_VERSION@" 11 | #define SAXPY_CMAKE_C_FLAGS "@CMAKE_C_FLAGS@" 12 | 13 | // Version information 14 | #define SAXPY_MAJOR_VERSION @SAXPY_MAJOR_VERSION@ 15 | #define SAXPY_MINOR_VERSION @SAXPY_MINOR_VERSION@ 16 | #define SAXPY_PATCH_VERSION @SAXPY_PATCH_VERSION@ 17 | #define SAXPY_VERSION "@SAXPY_VERSION@" 18 | 19 | #endif // SAXPY_CONFIG_HH 20 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/x86_64/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: x86_64 7 | integrator: 8 | vendor: 9 | name: 10 | processor: 11 | vendor: 12 | name: 13 | ISA: x86_64 14 | uArch: 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: 22 | name: 23 | systems-tested: 24 | any: 25 | os: 26 | scheduler: 27 | compiler: 28 | runtime: 29 | mpi: 30 | top500-system-instances: [null] 31 | -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- 1 | application: 2 | - changed-files: 3 | - any-glob-to-any-file: [repo/**] 4 | ci: 5 | - changed-files: 6 | - any-glob-to-any-file: [.github/**] 7 | dependencies: 8 | - changed-files: 9 | - any-glob-to-any-file: 10 | - .github/worklflows/requirements/** 11 | - pyproject.toml 12 | - requirements.txt 13 | docs: 14 | - changed-files: 15 | - any-glob-to-any-file: [docs/**, README.rst, taxonomy.yaml] 16 | experiment: 17 | - changed-files: 18 | - any-glob-to-any-file: [experiments/**, var/exp_repo/**] 19 | feature: 20 | - changed-files: 21 | - any-glob-to-any-file: [bin/**, lib/**, modifiers/**] 22 | system: 23 | - changed-files: 24 | - any-glob-to-any-file: [systems/**] 25 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Intellectual Property Notice 2 | ------------------------------ 3 | 4 | Benchpark is licensed under the Apache License, Version 2.0 (LICENSE-APACHE 5 | or http://www.apache.org/licenses/LICENSE-2.0). 6 | 7 | Copyrights and patents in the Benchpark project are retained by contributors. 8 | No copyright assignment is required to contribute to Benchpark. 9 | 10 | Benchpark is derived from Ramble and Spack. 11 | Ramble: https://github.com/GoogleCloudPlatform/ramble 12 | Spack: https://github.com/spack/spack 13 | 14 | SPDX usage 15 | ------------ 16 | 17 | Individual files contain SPDX tags instead of the full license text. 18 | This enables machine processing of license information based on the SPDX 19 | License Identifiers that are available here: https://spdx.org/licenses/ 20 | -------------------------------------------------------------------------------- /lib/benchpark/debug.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | import sys 8 | from inspect import currentframe, getframeinfo 9 | 10 | DEBUG = False 11 | 12 | 13 | def get_linenumber(): 14 | cf = currentframe() 15 | return cf.f_back.f_back.f_lineno 16 | 17 | 18 | def get_filename(): 19 | cf = currentframe() 20 | return os.path.basename(getframeinfo(cf.f_back).filename) 21 | 22 | 23 | def debug_print(message): 24 | if DEBUG: 25 | print( 26 | "(debug) " + get_filename() + "::" + str(get_linenumber()) + " " + message, 27 | file=sys.stderr, 28 | ) 29 | -------------------------------------------------------------------------------- /repo/camp/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.camp.package import Camp as BuiltinCamp 8 | 9 | 10 | class Camp(BuiltinCamp): 11 | 12 | def setup_build_environment(self, env): 13 | super().setup_build_environment(env) 14 | if "+cuda" in self.spec: 15 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 16 | 17 | def setup_run_environment(self, env): 18 | super().setup_run_environment(env) 19 | if "+cuda" in self.spec: 20 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 21 | -------------------------------------------------------------------------------- /repo/raja/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.raja.package import Raja as BuiltinRaja 8 | 9 | 10 | class Raja(BuiltinRaja): 11 | 12 | def setup_build_environment(self, env): 13 | super().setup_build_environment(env) 14 | if "+cuda" in self.spec: 15 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 16 | 17 | def setup_run_environment(self, env): 18 | super().setup_run_environment(env) 19 | if "+cuda" in self.spec: 20 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 21 | -------------------------------------------------------------------------------- /repo/gromacs/gmxDetectCpu-cmake-3.14.patch: -------------------------------------------------------------------------------- 1 | --- a/cmake/gmxDetectCpu.cmake 2 | +++ b/cmake/gmxDetectCpu.cmake 3 | @@ -83,7 +83,7 @@ function(gmx_run_cpu_detection TYPE) 4 | set(GCC_INLINE_ASM_DEFINE "-DGMX_X86_GCC_INLINE_ASM=0") 5 | endif() 6 | 7 | - set(_compile_definitions "${GCC_INLINE_ASM_DEFINE} -I${PROJECT_SOURCE_DIR}/src -DGMX_CPUINFO_STANDALONE ${GMX_STDLIB_CXX_FLAGS} -DGMX_TARGET_X86=${GMX_TARGET_X86_VALUE}") 8 | + set(_compile_definitions ${GCC_INLINE_ASM_DEFINE} -I${PROJECT_SOURCE_DIR}/src -DGMX_CPUINFO_STANDALONE ${GMX_STDLIB_CXX_FLAGS} -DGMX_TARGET_X86=${GMX_TARGET_X86_VALUE}) 9 | set(LINK_LIBRARIES "${GMX_STDLIB_LIBRARIES}") 10 | try_compile(CPU_DETECTION_COMPILED 11 | "${PROJECT_BINARY_DIR}" 12 | 13 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/Penguin-haswell-OmniPath/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: Penguin-haswell-OmniPath 7 | integrator: 8 | vendor: Penguin 9 | name: 10 | processor: 11 | vendor: Intel 12 | name: Xeon-E5-2695v4 13 | ISA: x86_64 14 | uArch: haswell 15 | interconnect: 16 | vendor: CornelisNetworks 17 | name: OmniPath 18 | systems-tested: 19 | llnl-rzgenie: 20 | os: TOSS 21 | scheduler: slurm 22 | compiler: gcc 23 | runtime: 24 | mpi: mvapich 25 | installation-year: 2017 26 | top500-system-instances: 27 | -------------------------------------------------------------------------------- /lib/scripts/env-collect-branch-tips.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import sys 4 | 5 | import spack.environment as ev 6 | from spack.fetch_strategy import GitFetchStrategy 7 | 8 | 9 | def main(): 10 | destination = sys.argv[1] 11 | 12 | e = ev.active_environment() 13 | for _, spec in e.concretized_specs(): 14 | df = spec.package.stage[0].default_fetcher 15 | if not df.cachable and isinstance(df, GitFetchStrategy): 16 | df.get_full_repo = True 17 | pkg_dst = os.path.join(destination, spec.name) 18 | if not os.path.exists(pkg_dst): 19 | spec.package.stage.fetch() 20 | shutil.move(spec.package.stage.source_path, pkg_dst) 21 | print(f"{spec.name}") 22 | 23 | 24 | if __name__ == "__main__": 25 | main() 26 | -------------------------------------------------------------------------------- /lib/benchpark/mpi.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | from benchpark.directives import requires, variant 8 | from benchpark.experiment import ExperimentHelper 9 | 10 | 11 | class MpiOnlyExperiment: 12 | requires("mpi") 13 | variant("mpi", default=True, description="Run with MPI only") 14 | 15 | def __init__(self): 16 | super().__init__() 17 | if self.spec.variants["mpi"][0]: 18 | self.device_type = "cpu" 19 | self.programming_models.append("mpi") 20 | 21 | class Helper(ExperimentHelper): 22 | def get_helper_name_prefix(self): 23 | return "mpi" if self.spec.satisfies("+mpi") else "" 24 | -------------------------------------------------------------------------------- /repo/spectrum-mpi/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.spectrum_mpi.package import SpectrumMpi as BuiltinSM 8 | 9 | 10 | class SpectrumMpi(BuiltinSM): 11 | @property 12 | def libs(self): 13 | libnames = [ 14 | 'mpi_ibm', 15 | 'mpi_ibm_mpifh', 16 | 'mpiprofilesupport', 17 | 'mpi_ibm_usempi', 18 | ] 19 | libs = list('lib' + x for x in libnames) 20 | return find_libraries( 21 | libs, 22 | self.spec.prefix, 23 | shared=True, 24 | recursive=True 25 | ) 26 | -------------------------------------------------------------------------------- /CTestConfig.cmake: -------------------------------------------------------------------------------- 1 | ## This file should be placed in the root directory of your project. 2 | ## Then modify the CMakeLists.txt file in the root directory of your 3 | ## project to incorporate the testing dashboard. 4 | ## 5 | ## The following are required to submit to the CDash dashboard: 6 | #ENABLE_TESTING() 7 | #INCLUDE(CTest) 8 | 9 | set(CTEST_PROJECT_NAME benchpark) 10 | set(CTEST_NIGHTLY_START_TIME 01:00:00 UTC) 11 | 12 | if(CMAKE_VERSION VERSION_GREATER 3.14) 13 | set(CTEST_SUBMIT_URL https://my.cdash.org/submit.php?project=Benchpark) 14 | else() 15 | set(CTEST_DROP_METHOD "https") 16 | set(CTEST_DROP_SITE "my.cdash.org") 17 | set(CTEST_DROP_LOCATION "/submit.php?project=Benchpark") 18 | endif() 19 | 20 | set(CTEST_DROP_SITE_CDASH TRUE) 21 | 22 | # CDash token expected in ENV 23 | set(_auth_token $ENV{CDASH_AUTH_TOKEN}) 24 | -------------------------------------------------------------------------------- /repo/hpcc/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.app.builtin.hpcc import Hpcc as HpccBase 9 | from ramble.appkit import * 10 | 11 | 12 | class Hpcc(HpccBase): 13 | 14 | tags = ['synthetic', 15 | 'blas','solver','dense-linear-algebra','fft', 16 | 'large-scale', 17 | 'high-fp','high-memory-bandwidth', 18 | 'regular-memory-access','irregular-memory-access', 19 | 'mpi','network-collectives','network-point-to-point', 20 | 'network-bandwidth-bound','network-bisection-bandwidth-bound', 21 | 'network-latency-bound', 22 | 'c','fortran'] 23 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | systemconfigs: 16 | ./generate-sys-defs-list.py 17 | 18 | tags: 19 | ./generate-benchmark-list.py 20 | 21 | .PHONY: help systemconfigs tags Makefile 22 | 23 | # Catch-all target: route all unknown targets to Sphinx using the new 24 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 25 | %: Makefile 26 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 27 | -------------------------------------------------------------------------------- /.gitlab/utils/rules.yml: -------------------------------------------------------------------------------- 1 | .on_host_template: 2 | rules: 3 | - if: $CI_PIPELINE_SOURCE == "schedule" || $ADVANCED_JOB == "ON" && $CI_COMMIT_BRANCH 4 | != "main" && $CI_COMMIT_BRANCH != "develop" && $ALL_TARGETS != "ON" 5 | when: never 6 | - if: $CI_JOB_NAME =~ /release_resources|cleanup_pipeline_dir/ && $CI_PIPELINE_SOURCE 7 | != "schedule" 8 | when: always 9 | - when: on_success 10 | - changes: 11 | - .gitlab-ci.yml 12 | - .gitlab/** 13 | - bin/** 14 | - experiments/$BENCHMARK/** 15 | - systems/$ARCHCONFIG/** 16 | - repo/$BENCHMARK/** 17 | - modifiers/** 18 | - var/** 19 | - lib/** 20 | .nightly_rules: 21 | rules: 22 | - if: $CI_PIPELINE_SOURCE == "schedule" 23 | when: always 24 | - if: $CI_PIPELINE_SOURCE != "schedule" 25 | when: never 26 | -------------------------------------------------------------------------------- /CTestGitlab.cmake: -------------------------------------------------------------------------------- 1 | set(CTEST_SOURCE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}") 2 | set(CTEST_BINARY_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}") 3 | set(CTEST_OUTPUT_ON_FAILURE ON) 4 | set(CTEST_CMAKE_GENERATOR "Unix Makefiles") 5 | 6 | set(CTEST_UPDATE_COMMAND "git") 7 | set(CTEST_UPDATE_VERSION_ONLY 1) 8 | 9 | set(CTEST_SITE ${SITE}) 10 | set(CTEST_BUILD_NAME ${BUILD_NAME}) 11 | 12 | # Separate dashboards for Nightly testing and PRs 13 | if ("${TEST_TYPE}" STREQUAL "Nightly") 14 | ctest_start("Nightly" GROUP "${DASHBOARD_NAME}") 15 | else() 16 | ctest_start("Continuous" GROUP "${DASHBOARD_NAME}") 17 | endif() 18 | 19 | ctest_update() 20 | ctest_configure() 21 | ctest_test(INCLUDE "Gitlab") 22 | 23 | # Submit results 24 | ctest_submit( 25 | PARTS Update Test # 'Configure' and 'Build' not uploaded 26 | HTTPHEADER "Authorization: Bearer ${_auth_token}" 27 | ) 28 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/AWS_PCluster-zen-EFA/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: AWS_PCluster-zen-EFA 7 | integrator: 8 | vendor: AWS 9 | name: ParallelCluster 10 | processor: 11 | vendor: AMD 12 | name: EPYC-Zen 13 | ISA: x86_64 14 | uArch: zen 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: AWS 22 | name: EFA 23 | systems-tested: 24 | aws-pcluster: 25 | os: alinux2 26 | scheduler: slurm 27 | compiler: gcc 28 | runtime: 29 | mpi: openmpi 30 | instance-types: 31 | -------------------------------------------------------------------------------- /lib/benchpark/cudasystem.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import provides 7 | 8 | 9 | class CudaSystem: 10 | provides("cuda") 11 | 12 | name = "cuda" 13 | 14 | def verify(self, system): 15 | assert "cuda" in system.variants 16 | assert "gtl" in system.variants 17 | assert hasattr(system, "cuda_arch") 18 | assert hasattr(system, "cuda_version") 19 | assert hasattr(system, "gtl_flag") 20 | 21 | def system_specific_variables(self, system): 22 | return { 23 | "cuda_arch": system.cuda_arch, 24 | "cuda_version": system.cuda_version, 25 | "gtl_flag": system.gtl_flag, 26 | } 27 | -------------------------------------------------------------------------------- /lib/benchpark/rocmsystem.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import provides 7 | 8 | 9 | class ROCmSystem: 10 | provides("rocm") 11 | 12 | name = "rocm" 13 | 14 | def verify(self, system): 15 | assert "rocm" in system.variants 16 | assert "gtl" in system.variants 17 | assert hasattr(system, "rocm_arch") 18 | assert hasattr(system, "rocm_version") 19 | assert hasattr(system, "gtl_flag") 20 | 21 | def system_specific_variables(self, system): 22 | return { 23 | "rocm_arch": system.rocm_arch, 24 | "rocm_version": system.rocm_version, 25 | "gtl_flag": system.gtl_flag, 26 | } 27 | -------------------------------------------------------------------------------- /repo/umpire/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.umpire.package import Umpire as BuiltinUmpire 8 | 9 | 10 | class Umpire(BuiltinUmpire): 11 | 12 | depends_on("fmt@9.1: cxxstd=17", when="@2024.02.0: %oneapi") 13 | 14 | def setup_build_environment(self, env): 15 | super().setup_build_environment(env) 16 | if "+cuda" in self.spec: 17 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 18 | 19 | def setup_run_environment(self, env): 20 | super().setup_run_environment(env) 21 | if "+cuda" in self.spec: 22 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 23 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/AWS_Tutorial-sapphirerapids-EFA/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: AWS_Tutorial-sapphirerapids-EFA 7 | integrator: 8 | vendor: AWS 9 | name: EKS 10 | processor: 11 | vendor: Intel 12 | name: Xeon Sapphire Rapids 13 | ISA: x86_64 14 | uArch: SapphireRapids 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: AWS 22 | name: EFA 23 | systems-tested: 24 | aws-pcluster: 25 | os: ubuntu-22.04 26 | scheduler: flux 27 | compiler: gcc 28 | runtime: 29 | mpi: mpich 30 | instance-types: https://aws.amazon.com/ec2/instance-types/c7i/ 31 | -------------------------------------------------------------------------------- /bin/benchpark-python: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 4 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 5 | # 6 | # SPDX-License-Identifier: Apache-2.0 7 | 8 | # 9 | # benchpark-python 10 | # 11 | # If you want to write your own executable Python script that uses Benchpark 12 | # modules, on Mac OS or maybe some others, you may be able to do it like 13 | # this: 14 | # 15 | # #!/usr/bin/env benchpark-python 16 | # 17 | # This is compatible across platforms. 18 | # 19 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 20 | export PYTHONPATH="${SCRIPT_DIR}/../lib":$PYTHONPATH 21 | # Check if the first argument is -i 22 | if [ "$1" = "-i" ]; then 23 | # Shift the arguments to remove the first one (-i) 24 | shift 25 | exec python3 -i "$@" 26 | else 27 | exec python3 "$@" 28 | fi 29 | -------------------------------------------------------------------------------- /repo/enzyme/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class Enzyme(CMakePackage): 10 | """Enzyme""" 11 | 12 | git = "https://github.com/jandrej/Enzyme.git" 13 | 14 | tags = ["enzyme"] 15 | 16 | license("Apache-2.0") 17 | 18 | maintainers("jandrej") 19 | 20 | version("patch-1", branch="patch-1") 21 | 22 | depends_on("c", type="build") 23 | depends_on("cxx", type="build") 24 | depends_on("fortran", type="build") 25 | 26 | depends_on("llvm@18:") 27 | 28 | root_cmakelists_dir = "enzyme" 29 | 30 | def cmake_args(self): 31 | cmake_options = [ 32 | f"-DLLVM_DIR={self.spec['llvm'].prefix}", 33 | ] 34 | 35 | return cmake_options 36 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/DELL-cascadelake-Infiniband/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: DELL-cascadelake-InfiniBand 7 | integrator: 8 | vendor: DELL 9 | name: 10 | processor: 11 | vendor: Intel 12 | name: Xeon6248R 13 | ISA: x86_64 14 | uArch: cascadelake 15 | accelerator: 16 | vendor: NVIDIA 17 | name: A100 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: Mellanox 22 | name: HDR100InfiniBand 23 | systems-tested: 24 | tamu-grace: 25 | os: 26 | scheduler: slurm 27 | compiler: gcc 28 | runtime: gpu 29 | mpi: intelmpi 30 | top500-system-instances: 31 | Grace: 32 | benchpark_system: 33 | top500: 34 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/Fujitsu-A64FX-TofuD/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: Fujitsu-A64FX-TofuD 7 | integrator: 8 | vendor: Fujitsu 9 | name: FX1000 10 | processor: 11 | vendor: Fujitsu 12 | name: A64FX 13 | ISA: Armv8.2-A-SVE 14 | uArch: aarch64 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: Fujitsu 22 | name: TofuInterconnectD 23 | systems-tested: 24 | rccs-fugaku: 25 | os: RHEL 26 | scheduler: pjm 27 | compiler: fj 28 | runtime: 29 | mpi: fujitsu-mpi 30 | top500-system-instances: 31 | Fugaku: 32 | benchpark_system: riken-fugaku 33 | top500: 34 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/Supermicro-icelake-OmniPath/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: Supermicro-icelake-OmniPath 7 | integrator: 8 | vendor: Supermicro 9 | name: 10 | processor: 11 | vendor: Intel 12 | name: XeonPlatinum8276L 13 | ISA: x86_64 14 | uArch: icelake 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: CornelisNetworks 22 | name: OmniPath 23 | systems-tested: 24 | llnl-cluster: 25 | os: TOSS 26 | scheduler: flux 27 | compiler: gcc 28 | runtime: 29 | mpi: mvapich 30 | top500-system-instances: 31 | Ruby: 32 | benchpark_system: llnl-cluster 33 | top500: 34 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/HPECray-zen2-Slingshot/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: HPECray-zen2-Slingshot 7 | integrator: 8 | vendor: HPECray 9 | name: 10 | processor: 11 | vendor: AMD 12 | name: EPYC-7742 13 | ISA: x86_64 14 | uArch: zen2 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: HPECray 22 | name: Slingshot 23 | systems-tested: 24 | cscs-eiger: 25 | os: HPECrayOS 26 | scheduler: slurm 27 | compiler: gcc 28 | runtime: 29 | mpi: cray-mpich 30 | installation-year: 2017 31 | top500-system-instances: 32 | Eiger: 33 | benchpark_system: cscs-eiger 34 | top500: 35 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/DELL-sapphirerapids-H100-Infiniband/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: DELL-sapphirerapids-H100-Infiniband 7 | integrator: 8 | vendor: DELLEMC 9 | name: PowerEdge 10 | processor: 11 | vendor: Intel 12 | name: XeonPlatinum8480 13 | ISA: x86_64 14 | uArch: sapphirerapids 15 | accelerator: 16 | vendor: NVIDIA 17 | name: H100 18 | ISA: PTX 19 | uArch: sm_90 20 | interconnect: 21 | vendor: Mellanox 22 | name: HDR100InfiniBand 23 | systems-tested: 24 | llnl-cluster: 25 | os: TOSS 26 | scheduler: slurm 27 | compiler: gcc 28 | runtime: 29 | mpi: mvapich 30 | top500-system-instances: 31 | Dane: 32 | benchpark_system: llnl-matrix 33 | top500: 34 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/DELL-sapphirerapids-OmniPath/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: DELL-sapphirerapids-OmniPath 7 | integrator: 8 | vendor: DELLEMC 9 | name: PowerEdge 10 | processor: 11 | vendor: Intel 12 | name: XeonPlatinum8480 13 | ISA: x86_64 14 | uArch: sapphirerapids 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: CornelisNetworks 22 | name: OmniPath 23 | systems-tested: 24 | llnl-cluster: 25 | os: TOSS 26 | scheduler: flux 27 | compiler: gcc 28 | runtime: 29 | mpi: mvapich 30 | top500-system-instances: 31 | Dane: 32 | benchpark_system: llnl-cluster 33 | top500: 34 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/Penguin-icelake-OmniPath/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: Penguin-icelake-OmniPath 7 | integrator: 8 | vendor: PenguinComputing 9 | name: RelionCluster 10 | processor: 11 | vendor: Intel 12 | name: XeonPlatinum924248C 13 | ISA: x86_64 14 | uArch: icelake 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: CornelisNetworks 22 | name: OmniPath 23 | systems-tested: 24 | llnl-cluster: 25 | os: TOSS 26 | scheduler: flux 27 | compiler: gcc 28 | runtime: 29 | mpi: mvapich 30 | top500-system-instances: 31 | Magma: 32 | benchpark_system: llnl-cluster 33 | top500: 34 | -------------------------------------------------------------------------------- /.github/workflows/style.yml: -------------------------------------------------------------------------------- 1 | name: Linting & Style Checks 2 | on: 3 | workflow_call: 4 | jobs: 5 | lint: 6 | runs-on: ubuntu-24.04 7 | steps: 8 | - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 9 | - name: Set up Python 3.11 10 | uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 11 | with: 12 | python-version: '3.11' 13 | cache: pip 14 | cache-dependency-path: .github/workflows/requirements/style.txt 15 | - name: Install Python dependencies 16 | run: | 17 | pip install -r .github/workflows/requirements/style.txt 18 | - name: Lint and Format Check 19 | run: |- 20 | black --diff --check . 21 | codespell --ignore-words=.codespellignore 22 | isort --check-only . 23 | flake8 . 24 | yamlfix --check --verbose . 25 | docstrfmt -p .github/workflows/requirements/docstrfmt.toml --check --verbose --ignore-cache docs/ 26 | -------------------------------------------------------------------------------- /repo/osu-micro-benchmarks/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.app.builtin.osu_micro_benchmarks import ( 9 | OsuMicroBenchmarks as OsuMicroBenchmarksBase, 10 | ) 11 | from ramble.appkit import * 12 | 13 | 14 | class OsuMicroBenchmarks(OsuMicroBenchmarksBase): 15 | 16 | tags = ['synthetic', 17 | 'large-scale','multi-node','single-node', 18 | 'atomics','managed-memory', 19 | 'mpi','openshmem','upc','upc++','nccl','rccl', 20 | 'network-bandwidth-bound','network-bisection-bandwidth-bound', 21 | 'network-collectives','network-latency-bound', 22 | 'network-multi-threaded','network-nonblocking-collectives', 23 | 'network-onesided','network-point-to-point', 24 | 'c','java','python','openacc'] 25 | -------------------------------------------------------------------------------- /lib/benchpark/openmp.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | from benchpark.directives import requires, variant 8 | from benchpark.experiment import ExperimentHelper 9 | 10 | 11 | class OpenMPExperiment: 12 | requires("openmp", when="+openmp") 13 | variant("openmp", default=False, description="Build and run with OpenMP") 14 | 15 | def __init__(self): 16 | super().__init__() 17 | if self.spec.variants["openmp"][0]: 18 | self.device_type = "cpu" 19 | self.programming_models.append("openmp") 20 | 21 | class Helper(ExperimentHelper): 22 | def get_helper_name_prefix(self): 23 | return "openmp" if self.spec.satisfies("+openmp") else "" 24 | 25 | def get_spack_variants(self): 26 | return "+openmp" if self.spec.satisfies("+openmp") else "~openmp" 27 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/HPECray-neoverse-H100-Slingshot/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: HPECray-neoverse-H100-Slingshot 7 | integrator: 8 | vendor: HPECray 9 | name: EX254n 10 | processor: 11 | vendor: NVIDIA 12 | name: Grace 13 | ISA: Armv9 14 | uArch: neoverse 15 | accelerator: 16 | vendor: NVIDIA 17 | name: H100 18 | ISA: PTX 19 | uArch: sm_90 20 | interconnect: 21 | vendor: HPECray 22 | name: Slingshot11 23 | systems-tested: 24 | lanl-venado: 25 | os: HPECrayOS 26 | scheduler: slurm 27 | compiler: cce 28 | runtime: cuda 29 | mpi: cray-mpich 30 | top500-system-instances: 31 | Venado: 32 | benchpark_system: lanl-venado 33 | top500: 34 | -------------------------------------------------------------------------------- /repo/ad/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Ad(ExecutableApplication): 12 | """AD benchmark""" 13 | name = "ad" 14 | 15 | tags = ['mpi','c','c++','automatic-differentiation','compiler-transformation'] 16 | 17 | executable('c_interface_test', 'c_interface_test', use_mpi=True) 18 | 19 | workload('ad', executables=['c_interface_test']) 20 | 21 | #figure_of_merit('Photons per Second', 22 | # log_file='{experiment_run_dir}/{experiment_name}.out', 23 | # fom_regex=r'Photons Per Second \(FOM\):\s+(?P[0-9]+\.[0-9]*([0-9]*)?e\+[0-9]*)', 24 | # group_name='fom', units='photons') 25 | 26 | #success_criteria('pass', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out') 27 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/Atos-rome-A100-Infiniband/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: Atos-zen2-A100-Infiniband 7 | integrator: 8 | vendor: Atos 9 | name: XH2000 10 | processor: 11 | vendor: AMD 12 | name: EPYC-Zen2 13 | ISA: x86_64 14 | uArch: zen2 15 | accelerator: 16 | vendor: NVIDIA 17 | name: A100 18 | ISA: PTX 19 | uArch: sm_80 20 | interconnect: 21 | vendor: Mellanox 22 | name: Infiniband 23 | systems-tested: 24 | jsc-juwels-booster: 25 | os: 26 | scheduler: slurm 27 | compiler: gcc 28 | runtime: cuda 29 | mpi: openmpi 30 | installation-year: 2020 31 | top500-system-instances: 32 | JUWELS-Booster: 33 | benchpark_system: jsc-juwels 34 | top500: 35 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/benchmark-status.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Benchmark status 3 | about: Per-Benchmark tasks 4 | title: '' 5 | labels: experiment 6 | assignees: rfhaque 7 | 8 | --- 9 | 10 | ## Components 11 | 1. [ ] Source code, Build system: Choose one: Make, CMake, Autotools 12 | 2. [ ] Source code released 13 | 3. [ ] Spack package.py 14 | 4. [ ] Ramble application.py 15 | 5. [ ] Benchpark experiment.py 16 | 6. [ ] +cuda experiment.py 17 | 7. [ ] +rocm experiment.py 18 | 8. [ ] Benchmark dependencies: List here 19 | 9. [ ] small test size problem for CI 20 | 10. [ ] Scaling (Weak, Strong, or both) 21 | 11. [ ] Large scale problem size defined 22 | 12. [ ] CI running on CPUs/CDash 23 | 13. [ ] CI running on NVIDIA GPUs/CDash 24 | 14. [ ] CI running on AMD GPUs/CDash 25 | 15. [ ] +single_memory experiment.py 26 | 16. [ ] +caliper experiment.py 27 | 17. [ ] CPU-only performance analysis 28 | 18. [ ] CUDA performance analysis 29 | 19. [ ] ROCm performance analysis 30 | 20. [ ] Documentation (build, run instructions, performance demo) 31 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/HPECray-haswell-P100-Infiniband/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: HPECray-haswell-P100-Infiniband 7 | integrator: 8 | vendor: HPECray 9 | name: 10 | processor: 11 | vendor: Intel 12 | name: Xeon-E5-2650v3 13 | ISA: x86_64 14 | uArch: haswell 15 | accelerator: 16 | vendor: NVIDIA 17 | name: P100 18 | ISA: PTX 19 | uArch: sm_60 20 | interconnect: 21 | vendor: HPECray 22 | name: Aries 23 | systems-tested: 24 | cscs-daint: 25 | os: HPECrayOS 26 | scheduler: slurm 27 | compiler: cce 28 | runtime: cuda 29 | mpi: cray-mpich 30 | installation-year: 2017 31 | top500-system-instances: 32 | Piz Daint: 33 | benchpark_system: cscs-daint 34 | top500: 35 | -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Run a Set of Tests to Collect Code Coverage for Benchpark 2 | on: 3 | workflow_call: 4 | secrets: 5 | BENCHPARK_CODECOV_TOKEN: 6 | required: true 7 | jobs: 8 | run_unit_tests: 9 | runs-on: ubuntu-24.04 10 | steps: 11 | - name: Checkout Benchpark 12 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 13 | - name: Add needed Python libs 14 | run: | 15 | pip install -r ./requirements.txt 16 | - name: Run unit tests with coverage 17 | run: | 18 | ./bin/benchpark unit-test --cov=./ --cov-branch --cov-report=xml --durations=20 -ra 19 | - name: Upload coverage to Codecov 20 | uses: codecov/codecov-action@v4 21 | with: 22 | token: ${{ secrets.BENCHPARK_CODECOV_TOKEN }} 23 | directory: ./coverage/reports 24 | files: /home/runner/work/benchpark/benchpark/coverage.xml 25 | flags: unittests 26 | verbose: true 27 | fail_ci_if_error: true 28 | -------------------------------------------------------------------------------- /repo/ad/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class Ad(CMakePackage): 10 | """AD Benchmark""" 11 | 12 | git = "git@github.com:jandrej/dfem-enzyme-compat.git" 13 | 14 | tags = ["benchmark"] 15 | 16 | license("Apache-2.0") 17 | 18 | maintainers("jandrej") 19 | 20 | version("main", branch="main") 21 | 22 | depends_on("c", type="build") 23 | depends_on("cxx", type="build") 24 | depends_on("fortran", type="build") 25 | 26 | depends_on("enzyme") 27 | 28 | def cmake_args(self): 29 | cmake_options = [ 30 | f"-DENZYME_DIR={self.spec['enzyme'].prefix}", 31 | ] 32 | 33 | return cmake_options 34 | 35 | def install(self, spec, prefix): 36 | mkdir(prefix.bin) 37 | install(join_path(self.build_directory, "src", "c_interface_test"), prefix.bin) 38 | -------------------------------------------------------------------------------- /docs/experiments_root_structure.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | For each ``experiment`` (``benchmark`` x ``ProgrammingModel`` x ), Ramble sets up the 8 | following ``workspace`` directory structure to build and run the experiment: 9 | 10 | :: 11 | 12 | experiments_root/ 13 | ramble/ 14 | spack/ 15 | / 16 | / 17 | / 18 | workspace/ 19 | configs/ 20 | (everything from system_def_dir) 21 | (everything from experiment_def_dir) 22 | experiments/ 23 | / 24 | / 25 | __ 26 | execute_experiment 27 | -------------------------------------------------------------------------------- /lib/benchpark/cuda.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | from benchpark.directives import requires, variant 8 | from benchpark.experiment import ExperimentHelper 9 | 10 | 11 | class CudaExperiment: 12 | requires("cuda", when="+cuda") 13 | variant("cuda", default=False, description="Build and run with CUDA") 14 | 15 | def __init__(self): 16 | super().__init__() 17 | if self.spec.variants["cuda"][0]: 18 | self.device_type = "gpu" 19 | self.programming_models.append("cuda") 20 | 21 | class Helper(ExperimentHelper): 22 | def get_helper_name_prefix(self): 23 | return "cuda" if self.spec.satisfies("+cuda") else "" 24 | 25 | def get_spack_variants(self): 26 | return ( 27 | "+cuda cuda_arch={cuda_arch}" 28 | if self.spec.satisfies("+cuda") 29 | else "~cuda" 30 | ) 31 | -------------------------------------------------------------------------------- /lib/benchpark/cmd/configure.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | from pathlib import Path 8 | 9 | import yaml 10 | 11 | import benchpark.paths 12 | 13 | 14 | def setup_parser(root_parser): 15 | root_parser.add_argument( 16 | "-bl", 17 | "--bootstrap-location", 18 | default=None, 19 | help="Set the path to the bootstrap location", 20 | ) 21 | 22 | 23 | def command(args): 24 | data = {} 25 | 26 | if args.bootstrap_location: 27 | loc = os.path.expandvars(os.path.expanduser(args.bootstrap_location)) 28 | bl = str(Path(loc).resolve()).rstrip("/") + "/.benchpark/" 29 | data["bootstrap"] = { 30 | "location": bl, 31 | } 32 | 33 | print(f"Writing configuration to {benchpark.paths.benchpark_config}") 34 | with open(benchpark.paths.benchpark_config, "w") as yaml_file: 35 | yaml.safe_dump(data, yaml_file) 36 | -------------------------------------------------------------------------------- /lib/benchpark/rocm.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | from benchpark.directives import requires, variant 8 | from benchpark.experiment import ExperimentHelper 9 | 10 | 11 | class ROCmExperiment: 12 | requires("rocm", when="+rocm") 13 | variant("rocm", default=False, description="Build and run with ROCm") 14 | 15 | def __init__(self): 16 | super().__init__() 17 | if self.spec.variants["rocm"][0]: 18 | self.device_type = "gpu" 19 | self.programming_models.append("rocm") 20 | 21 | class Helper(ExperimentHelper): 22 | def get_helper_name_prefix(self): 23 | return "rocm" if self.spec.satisfies("+rocm") else "" 24 | 25 | def get_spack_variants(self): 26 | return ( 27 | "+rocm amdgpu_target={rocm_arch}" 28 | if self.spec.satisfies("+rocm") 29 | else "~rocm" 30 | ) 31 | -------------------------------------------------------------------------------- /repo/saxpy/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Saxpy(ExecutableApplication): 12 | """saxpy benchmark""" 13 | name = "saxpy" 14 | 15 | tags = ['synthetic','single-node','high-memory-bandwidth', 16 | 'regular-memory-access', 17 | 'c++'] 18 | 19 | executable('p', 'saxpy -n {n}', use_mpi=True) 20 | 21 | workload('problem', executables=['p']) 22 | 23 | workload_variable('n', default='1024', description='problem size', workloads=['problem']) 24 | 25 | figure_of_merit('Kernel {num} size', fom_regex=r'Kernel done \((?P[0-9]+)\): (?P[0-9]+)', group_name='size', units='') 26 | figure_of_merit("success", fom_regex=r'(?PKernel done)', group_name='done', units='') 27 | 28 | success_criteria('pass', mode='string', match=r'Kernel done', file='{experiment_run_dir}/{experiment_name}.out') 29 | -------------------------------------------------------------------------------- /docs/why-benchpark.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ################ 8 | Why Benchpark? 9 | ################ 10 | 11 | This is an overview of features that make Benchpark different from other benchmarking 12 | suites. 13 | 14 | *********************************************************** 15 | Automation of benchmark building, running, and evaluation 16 | *********************************************************** 17 | 18 | Benchpark is a continuous benchmarking suite, or a fully automated mechanism for 19 | evaluating HPC benchmark performance on specified HPC systems. 20 | 21 | ***************************** 22 | Reproducibly run benchmarks 23 | ***************************** 24 | 25 | ***************************************** 26 | Collaborative maintenance of benchmarks 27 | ***************************************** 28 | 29 | Due to benchpark being open-source, 30 | 31 | ******************** 32 | System definitions 33 | ******************** 34 | -------------------------------------------------------------------------------- /experiments/ad/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers, variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | 10 | 11 | class Ad(Experiment, MpiOnlyExperiment): 12 | variant( 13 | "workload", 14 | default="ad", 15 | description="ad", 16 | ) 17 | 18 | variant( 19 | "version", 20 | default="main", 21 | description="app version", 22 | ) 23 | 24 | maintainers("rfhaque") 25 | 26 | def compute_applications_section(self): 27 | self.add_experiment_variable("n_ranks", 1, True) 28 | self.add_experiment_variable("n_threads_per_proc", 1, True) 29 | 30 | self.set_required_variables( 31 | n_resources="{n_ranks}", process_problem_size="", total_problem_size="" 32 | ) 33 | 34 | def compute_package_section(self): 35 | self.add_package_spec(self.name, [f"ad{self.determine_version()}"]) 36 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/IBM-power9-V100-Infiniband/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: IBM-power9-V100-Infiniband 7 | integrator: 8 | vendor: IBM 9 | name: AC922 10 | processor: 11 | vendor: IBM 12 | name: POWER9 13 | ISA: ppc64le 14 | uArch: power9 15 | accelerator: 16 | vendor: NVIDIA 17 | name: V100 18 | ISA: PTX 19 | uArch: sm_70 20 | interconnect: 21 | vendor: Mellanox 22 | name: EDR-Infiniband 23 | systems-tested: 24 | llnl-sierra: 25 | os: RHEL 26 | scheduler: lsf 27 | compiler: xl 28 | runtime: cuda 29 | mpi: spectrum-mpi 30 | top500-system-instances: 31 | Summit: 32 | benchpark_system: 33 | top500: 34 | Sierra: 35 | benchpark_system: llnl-sierra 36 | top500: 37 | Lassen: 38 | benchpark_system: llnl-sierra 39 | top500: 40 | -------------------------------------------------------------------------------- /repo/raja-perf/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class RajaPerf(ExecutableApplication): 12 | """RAJA Performance suite""" 13 | name = "raja-perf" 14 | 15 | tags = ['asc','single-node','sub-node','structured-grid', 16 | 'atomics','simd','vectorization','register-pressure', 17 | 'high-memory-bandwidth','regular-memory-access', 18 | 'mpi','network-point-to-point','network-latency-bound', 19 | 'c++','raja','sycl','builtin-caliper'] 20 | 21 | executable('run', 'raja-perf.exe --size {process_problem_size} -atsc ${CALI_CONFIG_MODE} -atcc ${OTHER_CALI_CONFIG}', use_mpi=True) 22 | 23 | workload('suite', executables=['run']) 24 | 25 | figure_of_merit('All tests pass', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'(?PDONE)!!!...', group_name='tpass', units='') 26 | 27 | success_criteria('pass', mode='string', match=r'DONE!!!....', file='{experiment_run_dir}/{experiment_name}.out') 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Log files** 24 | If applicable, add log files to help explain your problem. 25 | 26 | **Supercomputer (please complete the following information):** 27 | - system: which of the systems in benchpark/systems you are using [e.g., El Capitan] 28 | - system parameters: what system parameters did you use to instantiate the system 29 | - experiment: which of the experiments in benchpark/experiments are you using [e.g., amg2023] 30 | - experiment parameters: what experiment parameters did you use to instantiate the experiment 31 | 32 | **Python environment, etc. (please complete the following information):** 33 | - Python version 34 | 35 | **Additional context** 36 | Add any other context about the problem here. 37 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | This work was produced under the auspices of the U.S. Department of 2 | Energy by Lawrence Livermore National Laboratory under Contract 3 | DE-AC52-07NA27344. 4 | 5 | This work was prepared as an account of work sponsored by an agency of 6 | the United States Government. Neither the United States Government nor 7 | Lawrence Livermore National Security, LLC, nor any of their employees 8 | makes any warranty, expressed or implied, or assumes any legal liability 9 | or responsibility for the accuracy, completeness, or usefulness of any 10 | information, apparatus, product, or process disclosed, or represents that 11 | its use would not infringe privately owned rights. 12 | 13 | Reference herein to any specific commercial product, process, or service 14 | by trade name, trademark, manufacturer, or otherwise does not necessarily 15 | constitute or imply its endorsement, recommendation, or favoring by the 16 | United States Government or Lawrence Livermore National Security, LLC. 17 | 18 | The views and opinions of authors expressed herein do not necessarily 19 | state or reflect those of the United States Government or Lawrence 20 | Livermore National Security, LLC, and shall not be used for advertising 21 | or product endorsement purposes. 22 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/HPECray-zen4-MI300A-Slingshot/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: HPECray-zen4-MI300A-Slingshot 7 | integrator: 8 | vendor: HPECray 9 | name: EX255a 10 | processor: 11 | vendor: AMD 12 | name: EPYC-Zen4 13 | ISA: x86_64 14 | uArch: zen4 15 | accelerator: 16 | vendor: AMD 17 | name: MI300A 18 | ISA: GCN 19 | uArch: gfx942 20 | gpumode: [SPX, TPX, CPX] 21 | interconnect: 22 | vendor: HPECray 23 | name: Slingshot11 24 | systems-tested: 25 | llnl-elcapitan: 26 | os: TOSS 27 | scheduler: flux 28 | compiler: cce 29 | runtime: rocm 30 | mpi: cray-mpich 31 | top500-system-instances: 32 | El Capitan: 33 | benchpark_system: llnl-elcapitan 34 | top500: 35 | Tuolumne: 36 | benchpark_system: 37 | top500: 38 | rzAdams: 39 | benchpark_system: 40 | top500: 41 | -------------------------------------------------------------------------------- /docs/getting-started.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ################# 8 | Getting Started 9 | ################# 10 | 11 | ************** 12 | Installation 13 | ************** 14 | 15 | Git is needed to clone Benchpark, and Python 3.8+ is needed to run Benchpark: 16 | 17 | :: 18 | 19 | git clone https://github.com/LLNL/benchpark.git 20 | 21 | To add the executable to your path, source the setup script: 22 | 23 | :: 24 | 25 | cd benchpark 26 | . setup-env.sh 27 | 28 | Now, to check the version you can run: 29 | 30 | :: 31 | 32 | benchpark --version 33 | 34 | ********************** 35 | System Prerequisites 36 | ********************** 37 | 38 | Once Benchpark is available on your system, its python dependencies can be installed 39 | using the ``requirements.txt`` file included in the root directory of Benchpark. 40 | 41 | To install this, you can use: 42 | 43 | :: 44 | 45 | pip install -r requirements.txt 46 | 47 | Now you are ready to look at the benchmarks and systems available in Benchpark, and 48 | determine your workflow as described in :doc:`benchpark-commands`. 49 | -------------------------------------------------------------------------------- /.gitlab/utils/machine_checks.yml: -------------------------------------------------------------------------------- 1 | .machine-check: 2 | tags: [shell, oslic] 3 | variables: 4 | GIT_STRATEGY: none 5 | script: 6 | - | 7 | if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]] 8 | then 9 | echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m" 10 | curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \ 11 | --header 'Content-Type: application/json' \ 12 | --header "authorization: Bearer ${GITHUB_TOKEN}" \ 13 | --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }" 14 | exit 1 15 | fi 16 | tioga-up-check: 17 | stage: machine-checks 18 | variables: 19 | CI_MACHINE: tioga 20 | extends: [.machine-check] 21 | tuolumne-up-check: 22 | stage: machine-checks 23 | variables: 24 | CI_MACHINE: tuolumne 25 | extends: [.machine-check] 26 | dane-up-check: 27 | stage: machine-checks 28 | variables: 29 | CI_MACHINE: dane 30 | extends: [.machine-check] 31 | matrix-up-check: 32 | stage: machine-checks 33 | variables: 34 | CI_MACHINE: matrix 35 | extends: [.machine-check] 36 | -------------------------------------------------------------------------------- /lib/benchpark/paths.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | import pathlib 8 | 9 | import yaml 10 | 11 | 12 | def _source_location() -> pathlib.Path: 13 | """Return the location of the project source files directory.""" 14 | path_to_this_file = __file__ 15 | return pathlib.Path(path_to_this_file).resolve().parents[2] 16 | 17 | 18 | benchpark_root = _source_location() 19 | lib_path = benchpark_root / "lib" / "benchpark" 20 | test_path = lib_path / "test" 21 | 22 | benchpark_config = benchpark_root / "configure.yaml" 23 | 24 | if benchpark_config.exists(): 25 | with benchpark_config.open("r") as f: 26 | config = yaml.safe_load(f) 27 | benchpark_home = config["bootstrap"]["location"] 28 | else: 29 | benchpark_home = "~/.benchpark" 30 | benchpark_home = pathlib.Path(os.path.expanduser(benchpark_home)) 31 | 32 | 33 | global_ramble_path = benchpark_home / "ramble" 34 | global_spack_path = benchpark_home / "spack" 35 | hardware_descriptions = benchpark_root / "systems" / "all_hardware_descriptions" 36 | checkout_versions = benchpark_root / "checkout-versions.yaml" 37 | remote_urls = benchpark_root / "remote-urls.yaml" 38 | -------------------------------------------------------------------------------- /repo/chai/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.chai.package import Chai as BuiltinChai 8 | 9 | 10 | class Chai(BuiltinChai): 11 | variant("single_memory", default=False, description="Enable single memory space model") 12 | 13 | conflicts("+single_memory", when="~rocm") 14 | 15 | def initconfig_hardware_entries(self): 16 | spec = self.spec 17 | entries = super().initconfig_hardware_entries() 18 | 19 | if spec.satisfies("+single_memory"): 20 | entries.append(cmake_cache_option("CHAI_THIN_GPU_ALLOCATE", True)) 21 | entries.append(cmake_cache_option("CHAI_DISABLE_RM", True)) 22 | 23 | return entries 24 | 25 | def setup_build_environment(self, env): 26 | super().setup_build_environment(env) 27 | if "+cuda" in self.spec: 28 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 29 | 30 | def setup_run_environment(self, env): 31 | super().setup_run_environment(env) 32 | if "+cuda" in self.spec: 33 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 34 | -------------------------------------------------------------------------------- /repo/phloem/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class Phloem(MakefilePackage): 10 | tags = [] 11 | 12 | url = "https://github.com/LLNL/phloem/archive/refs/tags/v1.4.5.tar.gz" 13 | git = "https://github.com/LLNL/phloem" 14 | 15 | maintainers("knox10") 16 | 17 | version("master", branch="master") 18 | version("1.4.5", tag="v1.4.5") 19 | 20 | variant("mpi", default=False, description="Build with MPI support") 21 | 22 | depends_on("c", type="build") 23 | depends_on("cxx", type="build") 24 | depends_on("fortran", type="build") 25 | 26 | depends_on("mpi", when="+mpi") 27 | 28 | def install(self, spec, prefix): 29 | mkdir(prefix.bin) 30 | mkdir(prefix.doc) 31 | install("mpigraph-1.6/mpiBench/mpiBench", prefix.bin) 32 | install("sqmr-1.1.0/sqmr", prefix.bin) 33 | install("mpigraph-1.6/mpiGraph/mpiGraph", prefix.bin) 34 | install("presta-1.3.0/com", prefix.bin) 35 | install("presta-1.3.0/bw.message.sizes", prefix.bin) 36 | install("presta-1.3.0/latency.message.sizes", prefix.bin) 37 | install("README", prefix.doc) 38 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/HPECray-zen3-A100-Slingshot/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: HPECray-zen3-A100-Slingshot 7 | integrator: 8 | vendor: HPECray 9 | name: EX235a 10 | processor: 11 | vendor: AMD 12 | name: EPYC-Zen3 13 | ISA: x86_64 14 | uArch: zen3 15 | accelerator: 16 | vendor: NVIDIA 17 | name: A100 18 | ISA: PTX 19 | uArch: sm_80 20 | interconnect: 21 | vendor: HPECray 22 | name: Slingshot11 23 | systems-tested: 24 | csc-lumi: 25 | os: HPECrayOS 26 | scheduler: slurm 27 | compiler: cce 28 | runtime: rocm 29 | mpi: cray-mpich 30 | installation-year: 2023 31 | llnl-elcapitan: 32 | os: TOSS 33 | scheduler: flux 34 | compiler: cce 35 | runtime: rocm 36 | mpi: cray-mpich 37 | top500-system-instances: 38 | Frontier: 39 | benchpark_system: 40 | top500: 41 | Lumi: 42 | benchpark_system: csc-lumi 43 | top500: 44 | Tioga: 45 | benchpark_system: llnl-elcapitan 46 | top500: 47 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/HPECray-zen3-MI250X-Slingshot/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: HPECray-zen3-MI250X-Slingshot 7 | integrator: 8 | vendor: HPECray 9 | name: EX235a 10 | processor: 11 | vendor: AMD 12 | name: EPYC-Zen3 13 | ISA: x86_64 14 | uArch: zen3 15 | accelerator: 16 | vendor: AMD 17 | name: MI250X 18 | ISA: GCN 19 | uArch: gfx90a 20 | interconnect: 21 | vendor: HPECray 22 | name: Slingshot11 23 | systems-tested: 24 | csc-lumi: 25 | os: HPECrayOS 26 | scheduler: slurm 27 | compiler: cce 28 | runtime: rocm 29 | mpi: cray-mpich 30 | installation-year: 2023 31 | llnl-elcapitan: 32 | os: TOSS 33 | scheduler: flux 34 | compiler: cce 35 | runtime: rocm 36 | mpi: cray-mpich 37 | top500-system-instances: 38 | Frontier: 39 | benchpark_system: 40 | top500: 41 | Lumi: 42 | benchpark_system: csc-lumi 43 | top500: 44 | Tioga: 45 | benchpark_system: llnl-elcapitan 46 | top500: 47 | -------------------------------------------------------------------------------- /docs/showBuild.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ############ 8 | Show Build 9 | ############ 10 | 11 | If you build an experiment with Benchpark, you can see exactly how the experiment was 12 | built. 13 | 14 | ``` bin/benchpark experiment init --dest=def-raja-perf raja-perf bin/benchpark system 15 | init --dest=def-ruby llnl-cluster cluster=ruby compiler=gcc bin/benchpark setup 16 | def-ruby/def-raja-perf/ workspace/ . `pwd`/workspace/setup.sh ramble --workspace-dir 17 | `pwd`/workspace/def-raja-perf/def-ruby/workspace workspace setup ``` 18 | 19 | You will now be able to `benchpark show-build dump`, a command that will dump a log of 20 | how Spack built the experiment: 21 | 22 | ``` bin/benchpark show-build dump 23 | /usr/workspace/scheibel/oslic/benchpark2/workspace/def-raja-perf/def-ruby/workspace/ 24 | build-test/ ``` 25 | 26 | for this example, it will add the following artifacts to `build-test/` 27 | 28 | 1. `build-raja-perf.log`: a full log of all build commands and their output 29 | 2. `extracted-commands.txt`: [1] but with most output filtered out (generally much 30 | easier to understand what commands to run to do the build) 31 | 3. `spack-build-env.txt`: the environment variables set at build time 32 | -------------------------------------------------------------------------------- /lib/scripts/experiment-build-info.py: -------------------------------------------------------------------------------- 1 | import json 2 | import sys 3 | 4 | import spack.environment as ev 5 | 6 | 7 | def main(): 8 | x = ev.active_environment() 9 | y = list(x.concrete_roots()) 10 | # There may be multiple roots in the env. Assume that the experiment of 11 | # interest is the largest dag (every other root should just be an attempt 12 | # to constrain dependencies of this experiment). 13 | z = max(y, key=lambda i: sum(1 for _ in i.traverse())) 14 | built_specs = list() 15 | for dep in z.traverse(deptype="link"): 16 | if dep.external: 17 | continue 18 | if "runtime" in getattr(dep.package, "tags", []): 19 | continue 20 | built_specs.append(dep) 21 | 22 | urls = {} 23 | for spec in built_specs: 24 | x = spec.package.stage[0].default_fetcher 25 | if hasattr(x, "url"): 26 | urls[spec.name] = { 27 | "url": x.url, 28 | "details": spec.package.versions[spec.version], 29 | } 30 | else: 31 | raise Exception(f"Unexpected: {spec.name} has no url attribute") 32 | 33 | result = { 34 | "root": z.name, 35 | "tree": z.tree(), 36 | "info": [(w.name, w.package.install_env_path) for w in built_specs], 37 | "urls": urls, 38 | } 39 | json.dump(result, sys.stdout) 40 | 41 | 42 | if __name__ == "__main__": 43 | main() 44 | -------------------------------------------------------------------------------- /docs/build-experiment.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ##################### 8 | Build an Experiment 9 | ##################### 10 | 11 | ``benchpark setup`` has created the directory structure for your experiment. The next 12 | step is setting up the Ramble workspace and building the code: 13 | 14 | :: 15 | 16 | cd ///workspace 17 | ramble --workspace-dir . workspace setup 18 | 19 | Ramble will build the source code and set up the following workspace directory 20 | structure: 21 | 22 | :: 23 | 24 | experiments_root/ 25 | ramble/ 26 | spack/ 27 | / 28 | / 29 | workspace/ 30 | configs/ 31 | (everything from system_def_dir) 32 | (everything from experiment_def_dir) 33 | experiments/ 34 | / 35 | / 36 | __ 37 | execute_experiment 38 | 39 | If you edit any of the files, see :doc:`FAQ` to determine whether you need to re-do any 40 | of the previous steps. 41 | 42 | Once that is complete, you are ready for :doc:`run-experiment`. 43 | -------------------------------------------------------------------------------- /repo/stream/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class Stream(CMakePackage): 10 | """The STREAM benchmark is a simple synthetic benchmark program that 11 | measures sustainable memory bandwidth (in MB/s) and the corresponding 12 | computation rate for simple vector kernels. 13 | 14 | This package builds a fork of the official code with Caliper support, 15 | a CMake build system, and the ability to configure settings 16 | (array size, iterations, offset) at runtime via the command line.""" 17 | 18 | homepage = "https://www.cs.virginia.edu/stream/ref.html" 19 | git = "https://github.com/daboehme/STREAM.git" 20 | 21 | version("5.10-caliper", git="https://github.com/daboehme/STREAM.git", 22 | branch="caliper-benchpark") 23 | 24 | variant("caliper", default=False, description="Enable Caliper/Adiak support") 25 | 26 | requires("@5.10-caliper", when="+caliper") 27 | 28 | depends_on("c", type="build") 29 | depends_on("fortran", type="build") 30 | 31 | depends_on("caliper", when="+caliper") 32 | depends_on("adiak@0.4:", when="+caliper") 33 | 34 | def cmake_args(self): 35 | args = [ 36 | self.define_from_variant("STREAM_ENABLE_CALIPER", "caliper") 37 | ] 38 | 39 | return args 40 | -------------------------------------------------------------------------------- /repo/cuda/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import pathlib 7 | 8 | import llnl.util.tty as tty 9 | import spack_repo.builtin.packages.cuda.package 10 | from llnl.util.filesystem import find_headers 11 | from spack.package import * 12 | 13 | 14 | class Cuda(spack_repo.builtin.packages.cuda.package.Cuda): 15 | # Layout of hpc-sdk puts some headers in sibling directories: 16 | # cuda compiler in /opt/nvidia/hpc_sdk/Linux_aarch64/24.7/cuda/12.5 17 | # cufft in /opt/nvidia/hpc_sdk/Linux_aarch64/24.7/math_libs/12.5 18 | # In this case, we assume that the external prefix is set to the first path 19 | variant("im-hpc-sdk", default=False) 20 | 21 | @property 22 | def headers(self): 23 | home = getattr(self.spec.package, "home") 24 | headers = find_headers("*", root=home.include, recursive=True) 25 | 26 | if self.spec.satisfies("+im-hpc-sdk"): 27 | prefix = pathlib.Path(self.prefix) 28 | version_component = prefix.name # 12.5 29 | split_point = prefix.parent.parent 30 | cufft_base = split_point / "math_libs" / version_component 31 | #tty.debug(f"<---- {prefix}\n\t{split_point}\n\t{cufft_base}") 32 | headers = headers + find_headers("cufft", root=str(cufft_base), recursive=True) 33 | 34 | return headers 35 | -------------------------------------------------------------------------------- /systems/generic-x86/system.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers 7 | from benchpark.openmpsystem import OpenMPCPUOnlySystem 8 | from benchpark.system import System 9 | 10 | 11 | class GenericX86(System): 12 | """This is the generic system class for an x86 system, gcc compiler, mpi. 13 | It can be easily copied and modified to model other systems.""" 14 | 15 | maintainers("slabasan") 16 | 17 | def __init__(self, spec): 18 | super().__init__(spec) 19 | self.programming_models = [OpenMPCPUOnlySystem()] 20 | 21 | self.scheduler = "mpi" 22 | setattr(self, "sys_cores_per_node", 1) 23 | 24 | def compute_software_section(self): 25 | """This is somewhat vestigial, and maybe deleted later. The experiments 26 | will fail if these variables are not defined though, so for now 27 | they are still generated (but with more-generic values). 28 | """ 29 | return { 30 | "software": { 31 | "packages": { 32 | "default-compiler": {"pkg_spec": "gcc"}, 33 | "compiler-gcc": {"pkg_spec": "gcc"}, 34 | "default-mpi": {"pkg_spec": "openmpi"}, 35 | "blas": {"pkg_spec": "openblas"}, 36 | "lapack": {"pkg_spec": "openblas"}, 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /repo/gpcnet/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Gpcnet(ExecutableApplication): 12 | """GPCNet benchmark""" 13 | name = "GPCNet" 14 | 15 | executable('p1', 'network_test', use_mpi=True) 16 | executable('p2', 'network_load_test', use_mpi=True) 17 | workload('network_test', executables=['p1']) 18 | workload('network_load_test', executables=['p2']) 19 | 20 | figure_of_merit('Avg Multiple Allreduce', 21 | log_file='{experiment_run_dir}/{experiment_name}.out', 22 | fom_regex=r'\|\s+Multiple Allreduce \([0-9]* B\)\s+\|\s+(?P[0-9]+\.[0-9]*)', 23 | group_name='fom', units='MiB/sec') 24 | figure_of_merit('Avg RR Two-sided Lat', 25 | log_file='{experiment_run_dir}/{experiment_name}.out', 26 | fom_regex=r'\|\s+RR Two-sided Lat \([0-9]* B\)\s+\|\s+(?P[0-9]+\.[0-9]*)', 27 | group_name='fom', units='MiB/sec') 28 | figure_of_merit('Avg RR Get Lat', 29 | log_file='{experiment_run_dir}/{experiment_name}.out', 30 | fom_regex=r'\|\s+RR Get Lat \([0-9]* B\)\s+\|\s+(?P[0-9]+\.[0-9]*)', 31 | group_name='fom', units='MiB/sec') 32 | success_criteria('pass', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out') 33 | -------------------------------------------------------------------------------- /experiments/commbench/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.cuda import CudaExperiment 7 | from benchpark.directives import maintainers, variant 8 | from benchpark.experiment import Experiment 9 | from benchpark.rocm import ROCmExperiment 10 | 11 | 12 | class Commbench( 13 | Experiment, 14 | CudaExperiment, 15 | ROCmExperiment, 16 | ): 17 | variant("workload", description="workload name", default="basic") 18 | maintainers("arhag23") 19 | 20 | def compute_applications_section(self): 21 | self.add_experiment_variable("n_ranks", 2, True) 22 | 23 | if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): 24 | lib = "mpi" 25 | else: 26 | lib = "mpi" 27 | 28 | input_variables = { 29 | "size": "100000000", 30 | "lib": f"{lib}", 31 | "pattern": "broadcast", 32 | } 33 | 34 | for k, v in input_variables.items(): 35 | self.add_experiment_variable(k, v, True) 36 | 37 | self.set_required_variables( 38 | n_resources="{n_ranks}", 39 | process_problem_size="{size}", 40 | total_problem_size="{size}*{n_ranks}", 41 | ) 42 | 43 | def compute_package_section(self): 44 | spack_specs = [] 45 | spack_specs = " ".join(spack_specs) 46 | self.add_package_spec(self.name, [f"commbench {spack_specs}"]) 47 | -------------------------------------------------------------------------------- /bin/benchpark: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 4 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 5 | # 6 | # SPDX-License-Identifier: Apache-2.0 7 | 8 | import os 9 | import pathlib 10 | import subprocess 11 | import sys 12 | import uuid 13 | 14 | 15 | def main(): 16 | if sys.version_info[0] < 3 or sys.version_info[1] < 8: 17 | raise Exception("\n\nERR: Must be using Python 3.8 or later!\n") 18 | basedir = pathlib.Path(__file__).resolve().parents[1] 19 | main_py = basedir / "lib" / "main.py" 20 | 21 | if os.environ.get("BENCHPARK_RUN_COVERAGE"): 22 | # Create a unique coverage data file name 23 | coverage_dir = ( 24 | basedir / f"coverage-data-{os.environ.get('BENCHPARK_RUN_COVERAGE')}" 25 | ) 26 | coverage_dir.mkdir(exist_ok=True, parents=True) 27 | unique_id = uuid.uuid4().hex 28 | coverage_file = coverage_dir / f".coverage.{unique_id}" 29 | # Run the code with coverage, writing to the unique file 30 | subprocess.run( 31 | [ 32 | sys.executable, 33 | "-m", 34 | "coverage", 35 | "run", 36 | f"--data-file={coverage_file}", 37 | "--source=lib", 38 | main_py, 39 | ] 40 | + sys.argv[1:], 41 | check=True, 42 | ) 43 | else: 44 | subprocess.run([sys.executable, main_py] + sys.argv[1:], check=True) 45 | 46 | 47 | if __name__ == "__main__": 48 | main() 49 | -------------------------------------------------------------------------------- /experiments/gpcnet/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers, variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | 10 | 11 | class Gpcnet(Experiment, MpiOnlyExperiment): 12 | variant( 13 | "workload", 14 | default="network_test", 15 | values=("network_test", "network_load_test"), 16 | description="network_test or network_load_test", 17 | ) 18 | 19 | variant( 20 | "version", 21 | default="master", 22 | values=("master", "latest"), 23 | description="app version", 24 | ) 25 | 26 | maintainers("rfhaque") 27 | 28 | def compute_applications_section(self): 29 | # TODO: Replace with conflicts clause 30 | self.add_experiment_variable( 31 | "n_ranks", "{n_nodes}*{sys_cores_per_node}//2", True 32 | ) 33 | if self.spec.satisfies("workload=network_test"): 34 | self.add_experiment_variable("n_nodes", ["2", "4"]) 35 | elif self.spec.satisfies("workload=network_load_test"): 36 | self.add_experiment_variable("n_nodes", "10") 37 | 38 | self.set_required_variables( 39 | n_resources="{n_ranks}", process_problem_size="", total_problem_size="" 40 | ) 41 | 42 | def compute_package_section(self): 43 | self.add_package_spec(self.name, [f"gpcnet{self.determine_version()}"]) 44 | -------------------------------------------------------------------------------- /docs/examples/compare_experiment_builds/compareExperimentBuilds.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Run from benchpark root '. docs/examples/compare_experiment_builds/compareExperimentBuilds.sh' 3 | 4 | compilers=("gcc12" "intel") 5 | scaling=("weak") 6 | . setup-env.sh 7 | rm -rf daneGCC 8 | rm -rf daneIntel 9 | rm -rf quicksilvergcc* 10 | benchpark system init --dest=daneGCC llnl-cluster cluster=dane compiler=gcc 11 | benchpark system init --dest=daneIntel llnl-cluster cluster=dane compiler=intel 12 | # Set up all experiments 13 | for runNum in {1..3} 14 | do 15 | for scale in ${scaling[@]} 16 | do 17 | for i in ${compilers[@]} 18 | do 19 | echo $i $scale 20 | # Setup specific experiment 21 | benchpark experiment init --dest=quicksilver$i$scale$runNum quicksilver +$scale +openmp caliper=mpi 22 | if [ "$i" == "gcc12" ]; then 23 | benchpark setup daneGCC/quicksilver$i$scale$runNum workspace 24 | . workspace/setup.sh 25 | ramble --workspace-dir workspace/quicksilver$i$scale$runNum/daneGCC/workspace workspace setup 26 | ramble --workspace-dir workspace/quicksilver$i$scale$runNum/daneGCC/workspace on 27 | else 28 | benchpark setup daneIntel/quicksilver$i$scale$runNum workspace 29 | . workspace/setup.sh 30 | ramble --workspace-dir workspace/quicksilver$i$scale$runNum/daneIntel/workspace workspace setup 31 | ramble --workspace-dir workspace/quicksilver$i$scale$runNum/daneIntel/workspace on 32 | fi 33 | done 34 | done 35 | done 36 | -------------------------------------------------------------------------------- /repo/commbench/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | from ramble.expander import Expander 10 | 11 | 12 | class Commbench(ExecutableApplication): 13 | name = "commbench" 14 | 15 | maintainers("arhag23") 16 | 17 | software_spec("commbench", pkg_spec="commbench", package_manager="spack*") 18 | 19 | executable("execute-bench", "CommBench --use {lib} --pattern {pat}", use_mpi=True) 20 | # executable("execute-test", "mpiexec -n {n_ranks} CommBench --use {lib} --pattern {pat} --validate", use_mpi=True) 21 | executable("execute-test", "CommBench --use {lib} --pattern {pat} --validate", use_mpi=True) 22 | # executable("execute-test3", "mpiexec -n {n_ranks} CommBench --use {lib} --pattern {pat} --validate") 23 | # executable("execute-test4", "mpiexec CommBench --use {lib} --pattern {pat} --validate", use_mpi=True) 24 | 25 | workload("basic", executables=["execute-test"]) 26 | 27 | workload_variable("lib", default="mpi", values=["mpi", "ipc_get", "ipc_put", "xccl"], description="Communication library to benchmark", workload="basic") 28 | workload_variable("pat", default="p2p", values=["p2p", "gather", "scatter", "broadcast", "alltoall", "allgather"], description="Communication pattern", workload="basic") 29 | # workload_variable("mpi_command", default="mpirun -n {n_ranks}", description="mpi command", workload="basic") 30 | 31 | # success_criteria("pass", "string", match="[\s\S.]*PASSED![\s\S.]*") 32 | -------------------------------------------------------------------------------- /repo/osu-micro-benchmarks/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.osu_micro_benchmarks.package import ( 8 | OsuMicroBenchmarks as BuiltinOsu, 9 | ) 10 | 11 | 12 | class OsuMicroBenchmarks(BuiltinOsu, ROCmPackage): 13 | 14 | depends_on("cray-mpich+gtl", when="+rocm") 15 | 16 | def configure_args(self): 17 | args = super().configure_args() 18 | if self.spec.satisfies("+rocm"): 19 | args.extend([f"LDFLAGS={self.spec['mpi'].libs.ld_flags}"]) 20 | print(self.spec['mpi']) 21 | new_args = list() 22 | for x in args: 23 | if "NVCCFLAGS" in x and self.spec.satisfies("%intel-oneapi-compilers"): 24 | new_args.append(x + " -allow-unsupported-compiler") 25 | else: 26 | new_args.append(x) 27 | return new_args 28 | 29 | def setup_run_environment(self, env): 30 | mpidir = join_path(self.prefix.libexec, "osu-micro-benchmarks", "mpi") 31 | env.prepend_path("PATH", join_path(mpidir, "startup")) 32 | env.prepend_path("PATH", join_path(mpidir, "pt2pt")) 33 | env.prepend_path("PATH", join_path(mpidir, "one-sided")) 34 | env.prepend_path("PATH", join_path(mpidir, "collective")) 35 | if self.spec.satisfies("+rocm"): 36 | if 'gtl_flags' in self.spec['mpi'].extra_attributes: 37 | env.prepend_path("LOCAL_RANK", self.spec['mpi'].extra_attributes['gtl_flags']) 38 | -------------------------------------------------------------------------------- /experiments/smb/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers, variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | 10 | 11 | class Smb(Experiment, MpiOnlyExperiment): 12 | variant( 13 | "workload", 14 | default="mpi_overhead", 15 | values=("mpi_overhead", "msgrate", "rma_mt"), 16 | description="workload", 17 | ) 18 | 19 | variant( 20 | "version", 21 | default="1.1", 22 | values=("master", "latest", "1.1"), 23 | description="app version", 24 | ) 25 | 26 | maintainers("nhanford") 27 | 28 | def compute_applications_section(self): 29 | if self.spec.satisfies("workload=mpi_overhead"): 30 | self.add_experiment_variable("n_ranks", "2") 31 | elif self.spec.satisfies("workload=msgrate") or self.spec.satisfies( 32 | "workload=rma_mt" 33 | ): 34 | self.add_experiment_variable("n_nodes", "1") 35 | self.add_experiment_variable("n_ranks", "{n_nodes}*{sys_cores_per_node}") 36 | 37 | self.set_required_variables( 38 | n_resources="{n_ranks}", process_problem_size="", total_problem_size="" 39 | ) 40 | 41 | def compute_package_section(self): 42 | spec_string = f"smb{self.determine_version()}" 43 | if self.spec.satisfies("workload=rma_mt"): 44 | spec_string += "+rma" 45 | self.add_package_spec(self.name, [spec_string]) 46 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name="benchpark" 3 | description="An open collaborative repository for cross site benchmarking environments" 4 | readme = "README.rst" 5 | requires-python = ">=3.8" 6 | dynamic = ["version"] 7 | 8 | [tool.setuptools] 9 | packages = [ 10 | "lib", 11 | ] 12 | 13 | [tool.black] 14 | line-length = 88 15 | color = true 16 | target-version = ['py38', 'py39', 'py310', 'py311'] 17 | include = '\.pyi?$|bin\/benchpark$' 18 | exclude = ''' 19 | /( 20 | \.eggs 21 | | \.git 22 | | \.hg 23 | | \.mypy_cache 24 | | \.tox 25 | | \.venv 26 | | _build 27 | | buck-out 28 | | build 29 | | dist 30 | | repo 31 | )/ 32 | ''' 33 | force-exclude = ''' 34 | /( 35 | '/lib/benchpark/test_repo/(/.*)?$' 36 | | /lib/benchpark/test_repo(/.*)?$ 37 | | lib/benchpark/test_repo 38 | | '/bin/benchpark-python' 39 | | \.github 40 | )/ 41 | ''' 42 | 43 | [tool.isort] 44 | profile = "black" 45 | skip_gitignore = true 46 | color_output = true 47 | skip_glob = [ 48 | "lib/benchpark/test_repo/**" 49 | ] 50 | 51 | [tool.flake8] 52 | exclude = [ 53 | "lib/benchpark/test_repo/**" 54 | ] 55 | skip_glob = [ 56 | "lib/benchpark/test_repo/**" 57 | ] 58 | force-exclude = [ 59 | "./lib/benchpark/test_repo/**" 60 | ] 61 | per-file-ignores = """ 62 | ./lib/benchpark/test_repo/** 63 | """ 64 | 65 | [tool.codespell] 66 | skip = './docs/_build,./docs/_static,./taxonomy.yaml,./repo/**/*.patch' 67 | ignore-words-list = 'fom' 68 | 69 | [tool.yamlfix] 70 | line-length = 88 71 | allow_duplicate_keys = true 72 | explicit_start = false 73 | 74 | [project.optional-dependencies] 75 | analyze = [ 76 | "llnl-hatchet==2024.1.3", 77 | "llnl-thicket[plotting]==2025.1.0", 78 | "matplotlib" 79 | ] -------------------------------------------------------------------------------- /repo/branson/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Branson(ExecutableApplication): 12 | """Branson benchmark""" 13 | name = "branson" 14 | 15 | tags = ['asc','montecarlo','particles', 16 | 'high-branching','irregular-memory-access', 17 | 'mpi','c++'] 18 | 19 | executable('setup_experiment', 20 | template=[ 21 | 'cp {branson}/inputs/* {experiment_run_dir}/.', 22 | 'sed -i "s|250000000|{num_particles}|g" {experiment_run_dir}/{input_file}' 23 | ]) 24 | 25 | executable('p', '{branson}/bin/BRANSON {experiment_run_dir}/{input_file}', use_mpi=True) 26 | 27 | workload('branson', executables=['setup_experiment','p']) 28 | 29 | workload_variable('input_file', default='3D_hohlraum_multi_node.xml', 30 | description='input file name', 31 | workloads=['branson']) 32 | 33 | 34 | workload_variable('num_particles', default='250000000', 35 | description='procs on node', 36 | workloads=['branson']) 37 | 38 | 39 | figure_of_merit('Photons per Second', 40 | log_file='{experiment_run_dir}/{experiment_name}.out', 41 | fom_regex=r'Photons Per Second \(FOM\):\s+(?P[0-9]+\.[0-9]*([0-9]*)?e\+[0-9]*)', 42 | group_name='fom', units='photons') 43 | 44 | success_criteria('pass', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out') 45 | -------------------------------------------------------------------------------- /docs/diffPackageCommits.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ######################### 8 | Compare Package Commits 9 | ######################### 10 | 11 | ``lib/scripts/diffPackageCommits.py`` 12 | 13 | Compare a commit of a Spack package in Benchpark/repo with the package upstreamed to 14 | Spack. If the comparison of the ``package.py`` in ``benchpark/repo/`` is identical to 15 | the ``package.py`` in ``spack/var/spack/repos/builtin/packages/``, ``package.py`` in 16 | ``benchpark/repo/`` can be safely deleted without changing how the benchmark is built; 17 | this scenario occurs if ``package.py`` has been upstreamed to Spack. This script runs in 18 | the benchpark CI and will fail if a package should be deleted in benchpark. 19 | 20 | ******************************** 21 | Example: amg2023 and raja-perf 22 | ******************************** 23 | 24 | In this example, we made ``benchpark/repo/amg2023/package.py`` the same as the spack 25 | ``amg2023/package.py`` and equivalently ``benchpark/repo/raja-perf/package.py`` the same 26 | as spack ``raja-perf/package.py``. 27 | 28 | .. code-block:: console 29 | 30 | $ benchpark-python diffPackageCommits.py --packages amg2023 raja-perf 31 | 32 | Comparing benchpark packages to packages in spack develop 33 | amg2023 34 | No differences found. Please delete 'benchpark/repo/amg2023/package.py' (use spack upstream) 35 | 0 different lines 36 | raja-perf 37 | No differences found. Please delete 'benchpark/repo/raja-perf/package.py' (use spack upstream) 38 | 0 different lines 39 | -------------------------------------------------------------------------------- /experiments/phloem/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers, variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | 10 | 11 | class Phloem(Experiment, MpiOnlyExperiment): 12 | variant( 13 | "workload", 14 | default="sqmr", 15 | values=("sqmr", "mpiBench", "mpiGraph"), 16 | description="sqmr, mpiBench, or mpiGraph", 17 | ) 18 | 19 | variant( 20 | "version", 21 | default="1.4.5", 22 | values=("master", "latest", "1.4.5"), 23 | description="app version", 24 | ) 25 | 26 | maintainers("nhanford") 27 | 28 | def compute_applications_section(self): 29 | if self.spec.satisfies("workload=sqmr"): 30 | self.add_experiment_variable( 31 | "n_ranks", "{num_cores}*{num_nbors}+{num_cores}" 32 | ) 33 | self.add_experiment_variable("num_cores", "4") 34 | self.add_experiment_variable("num_nbors", "{num_cores}") 35 | elif self.spec.satisfies("workload=mpiBench"): 36 | self.add_experiment_variable("n_ranks", "2") 37 | elif self.spec.satisfies("workload=mpiGraph"): 38 | self.add_experiment_variable("n_ranks", "2") 39 | 40 | self.set_required_variables( 41 | n_resources="{n_ranks}", process_problem_size="", total_problem_size="" 42 | ) 43 | 44 | def compute_package_section(self): 45 | self.add_package_spec(self.name, [f"phloem{self.determine_version()}"]) 46 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | .. include:: ../README.rst 8 | 9 | .. toctree:: 10 | :maxdepth: 1 11 | :caption: Basics 12 | 13 | for-the-impatient 14 | getting-started 15 | benchpark-commands 16 | benchpark-workflow 17 | FAQ 18 | 19 | .. toctree:: 20 | :maxdepth: 1 21 | :caption: Catalogue 22 | 23 | system-list 24 | benchmark-list 25 | 26 | .. toctree:: 27 | :maxdepth: 1 28 | :caption: Tutorials 29 | 30 | tutorial-101 31 | tutorial-llnl 32 | comparing-experiment-builds 33 | 34 | .. toctree:: 35 | :maxdepth: 1 36 | :caption: Using Benchpark 37 | 38 | benchpark-setup 39 | build-experiment 40 | run-experiment 41 | analyze-experiment 42 | benchpark-analyze 43 | modifiers 44 | set-of-experiments 45 | run-binary 46 | create-mirror 47 | io-benchmarking 48 | 49 | .. toctree:: 50 | :maxdepth: 1 51 | :caption: Contributing 52 | 53 | add-a-system-config 54 | add-a-benchmark 55 | add-an-experiment 56 | testing-your-contribution 57 | update-a-system-config 58 | 59 | .. toctree:: 60 | :maxdepth: 1 61 | :caption: Developer Docs 62 | 63 | developer-guide 64 | ci-developer-guide 65 | 66 | .. toctree:: 67 | :maxdepth: 1 68 | :caption: Benchpark Utils 69 | 70 | diffBuildSpecs 71 | diffExperimentBuilds 72 | diffExperimentSpecs 73 | diffPackageCommits 74 | diffSystemSpecs 75 | showBuild 76 | 77 | .. toctree:: 78 | :maxdepth: 2 79 | :caption: API Docs 80 | 81 | Benchpark API Docs 82 | -------------------------------------------------------------------------------- /repo/hpcg/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | import platform 8 | 9 | from spack.package import * 10 | 11 | 12 | class Hpcg(CMakePackage): 13 | """HPCG is a software package that performs a fixed number of multigrid 14 | preconditioned (using a symmetric Gauss-Seidel smoother) conjugate gradient 15 | (PCG) iterations using double precision (64 bit) floating point values.""" 16 | 17 | #homepage = "https://www.hpcg-benchmark.org" 18 | #url = "https://www.hpcg-benchmark.org/downloads/hpcg-3.1.tar.gz" 19 | git = "https://github.com/daboehme/hpcg.git" 20 | 21 | version("develop", branch="master") 22 | #version("3.1", sha256="33a434e716b79e59e745f77ff72639c32623e7f928eeb7977655ffcaade0f4a4") 23 | version("caliper", branch="caliper-support") 24 | 25 | variant("openmp", default=True, description="Enable OpenMP support") 26 | variant("caliper", default=False, description="Enable Caliper support") 27 | 28 | depends_on("c", type="build") 29 | depends_on("cxx", type="build") 30 | depends_on("fortran", type="build") 31 | 32 | depends_on("mpi@1.1:") 33 | depends_on("caliper", when="+caliper") 34 | depends_on("adiak", when="+caliper") 35 | 36 | def cmake_args(self): 37 | build_targets = ["all", "docs"] 38 | install_targets = ["install", "docs"] 39 | args = [ 40 | "-DHPCG_ENABLE_MPI=TRUE", 41 | self.define_from_variant("HPCG_ENABLE_CALIPER", "caliper"), 42 | self.define_from_variant("HPCG_ENABLE_OPENMP", "openmp"), 43 | ] 44 | 45 | return args 46 | -------------------------------------------------------------------------------- /experiments/stream/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.caliper import Caliper 7 | from benchpark.directives import maintainers, variant 8 | from benchpark.experiment import Experiment 9 | from benchpark.mpi import MpiOnlyExperiment 10 | 11 | 12 | class Stream( 13 | Experiment, 14 | MpiOnlyExperiment, 15 | Caliper, 16 | ): 17 | variant( 18 | "workload", 19 | default="stream", 20 | description="stream", 21 | ) 22 | 23 | variant( 24 | "version", 25 | default="5.10-caliper", 26 | values=("develop", "latest", "5.10-caliper"), 27 | description="app version", 28 | ) 29 | 30 | maintainers("daboehme", "rfhaque") 31 | 32 | def compute_applications_section(self): 33 | 34 | array_size = {"s": 650000000} 35 | 36 | self.add_experiment_variable("processes_per_node", "1", named=True) 37 | self.add_experiment_variable("n", "35", False) 38 | self.add_experiment_variable("o", "0", False) 39 | self.add_experiment_variable("n_ranks", 1, True) 40 | self.add_experiment_variable( 41 | "n_threads_per_proc", [16, 32], named=True, matrixed=True 42 | ) 43 | 44 | for pk, pv in array_size.items(): 45 | self.add_experiment_variable(pk, pv, True) 46 | 47 | self.set_required_variables( 48 | n_resources="{n_ranks}", 49 | process_problem_size="{n}/{n_ranks}", 50 | total_problem_size="{n}", 51 | ) 52 | 53 | def compute_package_section(self): 54 | self.add_package_spec(self.name, [f"stream{self.determine_version()}"]) 55 | -------------------------------------------------------------------------------- /repo/phloem/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Phloem(ExecutableApplication): 12 | """Phloem benchmark""" 13 | name = "Phloem" 14 | 15 | tags = ['synthetic', 16 | 'large-scale','multi-node','single-node', 17 | 'c','mpi','network-point-to-point', 18 | 'network-bandwidth-bound','network-latency-bound', 19 | 'network-collectives','network-nonblocking-collectives'] 20 | 21 | executable('p1', 'sqmr --num_cores={num_cores} --num_nbors={num_nbors}', use_mpi=True) 22 | executable('p2', 'mpiBench', use_mpi=True) 23 | executable('p3', 'mpiGraph', use_mpi=True) 24 | #executable('p4', 'com', use_mpi=True) 25 | workload('sqmr', executables=['p1']) 26 | workload('mpiBench', executables=['p2']) 27 | workload('mpiGraph', executables=['p3']) 28 | #workload('com', executables=['p4']) 29 | 30 | workload_variable('num_cores', default='1', 31 | description='Number of MPI ranks on the core node, correlates to number of cores on one compute node.', 32 | workloads=['sqmr']) 33 | 34 | workload_variable('num_nbors', default='1', 35 | description='Number of distinct neighbors to each rank on the core node.', 36 | workloads=['sqmr']) 37 | #TODO: Determine FOMs 38 | # figure_of_merit('TBD', 39 | # log_file='{experiment_run_dir}/{experiment_name}.out', 40 | # fom_regex=r'.*', 41 | # group_name='fom', units='usec') 42 | success_criteria('pass', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out') 43 | -------------------------------------------------------------------------------- /repo/lammps/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.lammps.package import Lammps as BuiltinLammps 8 | 9 | 10 | class Lammps(BuiltinLammps): 11 | 12 | variant("pace", default=False, description="Enable the ML PACE module") 13 | depends_on("pace", when="+pace") 14 | 15 | depends_on("kokkos+openmp cxxstd=17", when="+openmp") 16 | depends_on("kokkos+wrapper", when="+cuda") 17 | 18 | flag_handler = build_system_flags 19 | 20 | def setup_run_environment(self, env): 21 | super().setup_run_environment(env) 22 | 23 | if self.compiler.extra_rpaths: 24 | for rpath in self.compiler.extra_rpaths: 25 | env.prepend_path("LD_LIBRARY_PATH", rpath) 26 | 27 | def setup_build_environment(self, env): 28 | super().setup_build_environment(env) 29 | 30 | if "+cuda" in self.spec: 31 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 32 | 33 | if "+mpi" in self.spec: 34 | if self.spec["mpi"].extra_attributes and "ldflags" in self.spec["mpi"].extra_attributes: 35 | env.append_flags("LDFLAGS", self.spec["mpi"].extra_attributes["ldflags"]) 36 | 37 | def cmake_args(self): 38 | args = super().cmake_args() 39 | args.append(f"-DMPI_CXX_LINK_FLAGS='{self.spec['mpi'].libs.ld_flags}'") 40 | args.append(f"-DMPI_C_COMPILER='{self.spec['mpi'].mpicc}'") 41 | args.append(f"-DMPI_CXX_COMPILER={self.spec['mpi'].mpicxx}") 42 | if "+pace" in self.spec: 43 | args.append(f"-DPKG_ML-PACE=ON") 44 | 45 | return args 46 | 47 | def install(self, spec, prefix): 48 | super().install(spec, prefix) 49 | mkdirp(prefix.src) 50 | install_tree(self.stage.source_path, prefix.src) 51 | -------------------------------------------------------------------------------- /repo/cray-mpich/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.cray_mpich.package import CrayMpich as BuiltinCM 8 | 9 | 10 | class CrayMpich(BuiltinCM): 11 | 12 | variant("gtl", default=True, description="enable GPU-aware mode") 13 | 14 | @property 15 | def libs(self): 16 | libs = super().libs 17 | 18 | if self.spec.satisfies("+gtl"): 19 | ld_flags = self.spec.extra_attributes["ldflags"] 20 | gtl_lib_prefix = self.spec.extra_attributes["gtl_lib_path"] 21 | # gtl_libs, if set, must be a single string. You can pass multiple 22 | # libs by adding a space between each 23 | gtl_libs = self.spec.extra_attributes["gtl_libs"].split() 24 | libs += find_libraries(gtl_libs, root=gtl_lib_prefix, recursive=True) 25 | 26 | return libs 27 | 28 | def setup_run_environment(self, env): 29 | 30 | super().setup_run_environment(env) 31 | 32 | if self.spec.satisfies("+gtl"): 33 | env.set("MPICH_GPU_SUPPORT_ENABLED", "1") 34 | env.prepend_path("LD_LIBRARY_PATH", self.spec.extra_attributes["gtl_lib_path"]) 35 | else: 36 | env.set("MPICH_GPU_SUPPORT_ENABLED", "0") 37 | gtl_path = self.spec.extra_attributes.get("gtl_lib_path", "") 38 | if gtl_path: 39 | env.prepend_path("LD_LIBRARY_PATH", gtl_path) 40 | 41 | def cmake_args(self): 42 | args = super().cmake_args(self) 43 | 44 | if self.spec.satisfies("+gtl"): 45 | # Link GTL for MPICH GPU-aware 46 | args.append(self.define("CMAKE_EXE_LINKER_FLAGS", self.spec['mpi'].libs.ld_flags)) 47 | 48 | return args 49 | -------------------------------------------------------------------------------- /repo/smb/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | import inspect 6 | 7 | import llnl.util.filesystem as fs 8 | from spack.package import * 9 | 10 | 11 | class Smb(MakefilePackage): 12 | tags = [] 13 | 14 | url = "https://github.com/sandialabs/SMB/archive/refs/tags/1.1.tar.gz" 15 | git = "https://github.com/sandialabs/SMB" 16 | 17 | maintainers("knox10") 18 | 19 | version("master", branch="master") 20 | version("1.1", tag="1.1") 21 | 22 | variant("mpi", default=False, description="Build with MPI support") 23 | variant("rma", default=False, description="Build RMA-MT variant") 24 | 25 | depends_on("c", type="build") 26 | 27 | depends_on("mpi", when="+mpi") 28 | 29 | build_directory = ["src/mpi_overhead"] 30 | 31 | def edit(self, spec, prefix): 32 | if "+rma" in spec: 33 | makefile = FileFilter("src/rma_mt_mpi/Makefile") 34 | makefile.filter('CC=cc', "CC = {0}".format(spec["mpi"].mpicc)) 35 | #TODO: add shm variant 36 | def build(self, spec, prefix): 37 | if "+rma" in spec: 38 | self.build_directory.append("src/rma_mt_mpi") 39 | else: 40 | self.build_directory.append("src/msgrate") 41 | 42 | for path in self.build_directory: 43 | with fs.working_dir(path): 44 | make() 45 | def install(self, spec, prefix): 46 | mkdir(prefix.bin) 47 | mkdir(prefix.doc) 48 | install("src/mpi_overhead/mpi_overhead", prefix.bin) 49 | install("src/mpi_overhead/README", prefix.doc) 50 | if "+rma" in spec: 51 | install("src/rma_mt_mpi/msgrate", prefix.bin) 52 | else: 53 | install("src/msgrate/msgrate", prefix.bin) 54 | install("src/msgrate/README", prefix.doc) 55 | -------------------------------------------------------------------------------- /lib/benchpark/test/system.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import pytest 7 | 8 | import benchpark.spec 9 | 10 | 11 | def test_system_compute_variables_section(monkeypatch): 12 | sys_spec = benchpark.spec.SystemSpec("llnl-elcapitan cluster=tuolumne").concretize() 13 | system = sys_spec.system 14 | 15 | vars_section = system.compute_variables_section() 16 | 17 | assert vars_section == { 18 | "variables": { 19 | "timeout": "120", 20 | "scheduler": "flux", 21 | "sys_cores_per_node": 84, 22 | "n_ranks": 2**64 - 1, 23 | "n_nodes": 2**64 - 1, 24 | "batch_submit": "placeholder", 25 | "mpi_command": "placeholder", 26 | "sys_cores_os_reserved_per_node": 12, 27 | "sys_cores_os_reserved_per_node_list": [ 28 | 0, 29 | 8, 30 | 16, 31 | 24, 32 | 32, 33 | 40, 34 | 48, 35 | 56, 36 | 64, 37 | 72, 38 | 80, 39 | 88, 40 | ], 41 | "sys_gpus_per_node": 4, 42 | "sys_mem_per_node_GB": 512, 43 | "rocm_arch": "gfx942", 44 | "rocm_version": "6.4.0", 45 | "gtl_flag": True, 46 | "gpu_factor": 1, 47 | "extra_batch_opts": "--setattr=gpumode=SPX\n--conf=resource.rediscover=true", 48 | } 49 | } 50 | 51 | 52 | def test_system_timeout(): 53 | with pytest.raises(ValueError, match="is unsatisfiable for the selected queue"): 54 | sys_spec = benchpark.spec.SystemSpec( 55 | "llnl-elcapitan cluster=tioga queue=pdebug timeout=9999" 56 | ).concretize() 57 | sys_spec.system.compute_variables_section() 58 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | # -- Project information ----------------------------------------------------- 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 8 | 9 | import os 10 | import subprocess 11 | import sys 12 | 13 | subprocess.call( 14 | [ 15 | "make", 16 | "systemconfigs", 17 | ] 18 | ) 19 | 20 | subprocess.call( 21 | [ 22 | "make", 23 | "tags", 24 | ] 25 | ) 26 | 27 | project = "Benchpark" 28 | copyright = "2023, LLNS LLC" 29 | author = "Olga Pearce, Alec Scott, Peter Scheibel, Greg Becker, Riyaz Haque, and Nathan Hanford" 30 | 31 | # -- General configuration --------------------------------------------------- 32 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 33 | 34 | extensions = [ 35 | "sphinx_rtd_theme", 36 | "sphinxcontrib.programoutput", 37 | "sphinx.ext.autodoc", 38 | "sphinx.ext.napoleon", 39 | ] 40 | 41 | sys.path.insert(0, os.path.abspath("../lib")) 42 | 43 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".spack-env"] 44 | 45 | # -- Options for HTML output ------------------------------------------------- 46 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 47 | 48 | html_static_path = ["_static"] 49 | html_js_files = [ 50 | "https://cdn.datatables.net/1.10.23/js/jquery.dataTables.min.js", 51 | "main.js", 52 | ] 53 | html_css_files = [ 54 | ( 55 | "https://cdn.datatables.net/1.10.23/css/jquery.dataTables.min.css", 56 | {"priority": 800}, 57 | ), 58 | ("css/custom.css", {"priority": 999}), 59 | ] 60 | html_logo = "_static/images/benchpark-dark.svg" 61 | html_theme = "sphinx_rtd_theme" 62 | html_theme_options = {"logo_only": True} 63 | pygments_style = "sphinx" 64 | -------------------------------------------------------------------------------- /docs/run-binary.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ############## 8 | Run a Binary 9 | ############## 10 | 11 | If you have a pre-built binary of your application, you can use it in your Benchpark 12 | experiment using the ``user-managed`` Ramble-defined package manager (`see docs 13 | `_). 14 | When initializing your experiment, provide the path to the binary using ``prepend_path`` 15 | which will add the binary path to ``PATH``, and specify ``user-managed`` as the package 16 | manager. System setup does not change. 17 | 18 | Example running the ``osu-micro-benchmarks`` workload ``osu_latency`` on the ``ruby`` 19 | system: 20 | 21 | :: 22 | 23 | benchpark system init --dest=ruby llnl-cluster cluster=ruby 24 | benchpark experiment init --dest=osumb ruby osu-micro-benchmarks \ 25 | package_manager="user-managed" \ 26 | workload="osu_latency" \ 27 | prepend_path="/usr/myuser/osu-micro-benchmarks/mpi/pt2pt" 28 | benchpark setup ./ruby/osumb/ osumb-ruby/ 29 | # Follow Ramble execution instructions ... 30 | 31 | This will execute using the ``osu_latency`` binary located at 32 | ``osu-micro-benchmarks/mpi/pt2pt/osu_latency``. 33 | 34 | Or for example, if we have a build ``kripke`` with spack on ``dane`` and then used that 35 | binary (``bin/kripke.exe``): 36 | 37 | :: 38 | 39 | benchpark system init --dest=dane llnl-cluster cluster=dane 40 | benchpark experiment init --dest=kripke dane kripke \ 41 | package_manager="user-managed" \ 42 | prepend_path="/usr/myuser/benchpark/wkp/spack/opt/spack/linux-rhel8-sapphirerapids/oneapi-2023.2.1/kripke-develop-ehvoc6dzdprgm3lhaghh7uoiqsc5xcf6/bin" 43 | benchpark setup ./dane/kripke/ kripke-dane/ 44 | # Follow Ramble execution instructions ... 45 | 46 | Using the spack built binary. 47 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | - [ ] Replace with: A short description of the change, including motivation and context. 4 | - [ ] Replace with: A list of any dependencies. 5 | - [ ] Replace with: Link(s) to relevant [issue(s)](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) 6 | - [ ] Complete the checklist for a relevant section(s) below 7 | - [ ] Delete sections below that are not relevant to this PR 8 | 9 | ## Adding/modifying a system (docs: [Adding a System](https://software.llnl.gov/benchpark/add-a-system-config.html)) 10 | 11 | - [ ] Add/modify `systems/system_name/system.py` file 12 | - [ ] Add/modify `systems/all_hardware_descriptions/hardware_name/hardware_description.yaml` which will appear in the [docs catalogue](https://software.llnl.gov/benchpark/system-list.html) 13 | 14 | ## Adding/modifying a benchmark (docs: [Adding a Benchmark](https://software.llnl.gov/benchpark/add-a-benchmark.html)) 15 | 16 | - [ ] If modifying the source code of a benchmark: create, self-assign, and link here a follow up issue with a link to the PR in the benchmark repo. 17 | - [ ] If package.py upstreamed to Spack is insufficient, add/modify `repo/benchmark_name/package.py` plus: create, self-assign, and link here a follow up issue with a link to the PR in the Spack repo. 18 | - [ ] If application.py upstreamed to Ramble is insufficient, add/modify `repo/benchmark_name/application.py` plus: create, self-assign, and link here a follow up issue with a link to the PR in the Ramble repo. 19 | - [ ] Tags in Ramble's `application.py` or in `repo/benchmark_name/application.py` will appear in the [docs catalogue](https://software.llnl.gov/benchpark/benchmark-list.html) 20 | - [ ] Add/modify an `experiments/benchmark_name/experiment.py` to define an experiment 21 | 22 | ## Adding/modifying core functionality, CI, or documentation: 23 | 24 | - [ ] Update docs 25 | - [ ] Update `.github/workflows` and `.gitlab/tests` unit tests (if needed) 26 | -------------------------------------------------------------------------------- /docs/diffExperimentSpecs.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ########################## 8 | Compare Experiment Specs 9 | ########################## 10 | 11 | ``lib/scripts/diffExperimentSpecs.py`` 12 | 13 | Compare two experiment specs `.yaml` files generated by ``benchpark experiment init``. 14 | This script enables the user to compare changes to a ``experiment.py`` between commits 15 | by comparing ``.yaml`` files generated from ``benchpark experiment init ...``. This 16 | script is helpful when it is unclear if changes made to ``experiment.py`` will affect 17 | the resulting experiment spec. 18 | 19 | Stages that occur during this script: 20 | - ``benchpark experiment init`` 21 | 22 | ********************************** 23 | Example: Change n_ranks in Saxpy 24 | ********************************** 25 | 26 | In this example, we modify the ``saxpy+openmp`` experiment to see how the script 27 | identifies the differences. On the branch ``myBranch``, we change 28 | ``self.add_experiment_variable("n_ranks", "8")`` to 29 | ``self.add_experiment_variable("n_ranks", "4")``. 30 | 31 | .. code-block:: console 32 | 33 | $ benchpark-python diffExperimentSpecs.py -n myBranch -e saxpy+openmp 34 | 35 | saxpy+openmp 36 | saxpy/openmp/ramble.yaml 37 | The YAML files benchpark-old/saxpy/openmp/ramble.yaml and benchpark-new/saxpy/openmp/ramble.yaml are different. Here are the differences: 38 | {'values_changed': {"root['ramble']['applications']['saxpy']['workloads']['problem']['experiments']['saxpy_problem_single_node_openmp_caliper_none_{n_nodes}_{n_threads_per_proc}_{n}']['variables']['n_ranks']": {'new_value': 4, 39 | 'old_value': 8}}} 40 | -------------------------------------------------------------------------------- /docs/set-of-experiments.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | #################### 8 | Set of Experiments 9 | #################### 10 | 11 | You may want to use the same experiments_root directory when setting up multiple 12 | experiments: 13 | 14 | - Benchpark only stores one copy of Spack/Ramble per experiments directory, and does not 15 | repeatedly download them when reusing the experiments directory. 16 | - Since each experiment in the experiments directory shares the Spack/Ramble instances, 17 | the edits/updates you make to Spack packages will apply to all experiments in your 18 | experiments directory. 19 | 20 | To use the same experiments directory when setting up multiple experiments, instruct 21 | benchpark to set them up in the same experiments_root: 22 | 23 | :: 24 | 25 | benchpark setup ${System1}/${Benchmark1}/${ProgrammingModel1} /output/path/to/experiments_root 26 | benchpark setup ${System1}/${Benchmark1}/${ProgrammingModel2} /output/path/to/experiments_root 27 | benchpark setup ${System1}/${Benchmark2}/${ProgrammingModel2} /output/path/to/experiments_root 28 | 29 | This will result in the following directory structure: 30 | 31 | :: 32 | 33 | experiments_root/ 34 | ramble/ 35 | spack/ 36 | ${Benchmark1}/ 37 | ${ProgrammingModel1}/ 38 | ${System1}/ 39 | workspace/ 40 | ${ProgrammingModel2}/ 41 | ${System2}/ 42 | workspace/ 43 | ${Benchmark2}/ 44 | ${ProgrammingModel2}/ 45 | ${System1}/ 46 | workspace/ 47 | 48 | Note that there is a single clone of Ramble, and a single clone of Spack, which all of 49 | the ``experiments`` use. Each ``experiment`` (``Benchmark/ProgrammingModel`` x 50 | ``system`` combination) has its own ``Ramble workspace``, where this specific 51 | ``experiment`` will be compiled and run. 52 | -------------------------------------------------------------------------------- /repo/mfem/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | import shutil 8 | import sys 9 | 10 | from spack.package import * 11 | from spack_repo.builtin.packages.mfem.package import Mfem as BuiltinMfem 12 | 13 | 14 | class Mfem(BuiltinMfem): 15 | 16 | variant("caliper", default=False, description="Build Caliper support") 17 | 18 | depends_on("caliper", when="+caliper") 19 | depends_on("adiak", when="+caliper") 20 | depends_on("hypre+shared", when="+mpi") 21 | 22 | requires("+caliper", when="^hypre+caliper") 23 | 24 | def configure(self, spec, prefix): 25 | if spec.satisfies('%oneapi'): 26 | spec.compiler_flags["cxxflags"] = [flag for flag in spec.compiler_flags["cxxflags"] if not flag.startswith('-O')] 27 | spec.compiler_flags["cxxflags"].append("-O2") 28 | spec.compiler_flags["cxxflags"].append("-fp-speculation=safe") 29 | super().configure(spec, prefix) 30 | 31 | def setup_build_environment(self, env): 32 | if "+cuda" in self.spec: 33 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 34 | if "+mpi" in self.spec: 35 | if self.spec["mpi"].extra_attributes and "ldflags" in self.spec["mpi"].extra_attributes: 36 | env.append_flags("LDFLAGS", self.spec["mpi"].extra_attributes["ldflags"]) 37 | 38 | def get_make_config_options(self, spec, prefix): 39 | def yes_no(varstr): 40 | return "YES" if varstr in self.spec else "NO" 41 | 42 | options = super().get_make_config_options(spec, prefix) 43 | options.append("MFEM_USE_CALIPER=%s" % yes_no("+caliper")) 44 | if "+caliper" in self.spec: 45 | options.append("CALIPER_DIR=%s" % self.spec["caliper"].prefix) 46 | options.append("MFEM_USE_ADIAK=%s" % yes_no("+adiak")) 47 | options.append("ADIAK_DIR=%s" % self.spec["adiak"].prefix) 48 | 49 | return options 50 | -------------------------------------------------------------------------------- /repo/affinity/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class Affinity(CMakePackage, CudaPackage, ROCmPackage): 10 | """Simple applications for determining Linux thread and gpu affinity.""" 11 | 12 | homepage = "https://github.com/bcumming/affinity" 13 | git = "https://github.com/bcumming/affinity.git" 14 | version("master", branch="master") 15 | 16 | maintainers("bcumming", "nhanford") 17 | 18 | license("BSD-3-Clause", checked_by="nhanford") 19 | 20 | variant("mpi", default=False, description="Build MPI support") 21 | variant("rocm", default=False, description="Build ROCm Support") 22 | 23 | depends_on("c", type="build") 24 | depends_on("cxx", type="build") 25 | 26 | depends_on("mpi", when="+mpi") 27 | depends_on("hip", when="+rocm") 28 | depends_on("cuda", when="+cuda") 29 | 30 | def cmake_args(self): 31 | spec = self.spec 32 | args = [] 33 | 34 | if '+mpi' in spec: 35 | args.append('-DCMAKE_CXX_COMPILER={0}'.format(spec["mpi"].mpicxx)) 36 | args.append("-DMPI_CXX_LINK_FLAGS={0}".format(spec["mpi"].libs.ld_flags)) 37 | args.append("-DAFFINITY_MPI=on") 38 | 39 | if '+cuda' in spec: 40 | args.append('-DCMAKE_CUDA_HOST_COMPILER={0}'.format(spec["mpi"].mpicxx)) 41 | args.append('-DCMAKE_CUDA_COMPILER={0}'.format(spec["cuda"].prefix + "/bin/nvcc")) 42 | args.append("-DAFFINITY_GPU_BACKEND=cuda") 43 | 44 | if '+rocm' in spec: 45 | args.append("-DROCM_PATH={0}".format(spec['hip'].prefix)) 46 | args.append("-DAFFINITY_GPU_BACKEND=rocm") 47 | rocm_archs = spec.variants["amdgpu_target"].value 48 | if "none" not in rocm_archs: 49 | args.append("-DHIP_HIPCC_FLAGS=--amdgpu-target={0}".format(",".join(rocm_archs))) 50 | args.append("-DCMAKE_HIP_ARCHITECTURES={0}".format(rocm_archs)) 51 | 52 | return args 53 | -------------------------------------------------------------------------------- /experiments/md-test/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers, variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | from benchpark.scaling import Scaling, ScalingMode 10 | 11 | 12 | class MdTest( 13 | Experiment, 14 | MpiOnlyExperiment, 15 | Scaling(ScalingMode.Strong), 16 | ): 17 | variant( 18 | "workload", 19 | default="multi-file", 20 | description="base md-test or other problem", 21 | ) 22 | 23 | variant( 24 | "version", 25 | default="1.9.3", 26 | description="app version", 27 | ) 28 | 29 | maintainers("rfhaque") 30 | 31 | def compute_applications_section(self): 32 | 33 | num_resources = {"n_ranks": 1} 34 | 35 | self.add_experiment_variable("num-objects", "1000", True) 36 | self.add_experiment_variable("iterations", "10", True) 37 | 38 | if self.spec.satisfies("exec_mode=test"): 39 | for pk, pv in num_resources.items(): 40 | self.add_experiment_variable(pk, pv, True) 41 | 42 | self.register_scaling_config( 43 | { 44 | ScalingMode.Strong: { 45 | "n_ranks": lambda var, itr, dim, scaling_factor: var.val(dim) 46 | * scaling_factor, 47 | "num-objects": lambda var, itr, dim, scaling_factor: var.val(dim), 48 | }, 49 | } 50 | ) 51 | 52 | self.set_required_variables( 53 | n_resources="{n_ranks}", 54 | process_problem_size="{num-objects}/{n_ranks}", 55 | total_problem_size="{num-objects}", 56 | ) 57 | 58 | def compute_package_section(self): 59 | self.add_package_spec( 60 | "ior", 61 | [ 62 | "ior@3.3.0", 63 | ], 64 | ) 65 | self.add_package_spec(self.name, [f"mdtest{self.determine_version()}"]) 66 | -------------------------------------------------------------------------------- /experiments/salmon-tddft/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | from benchpark.openmp import OpenMPExperiment 10 | 11 | 12 | class SalmonTddft(Experiment, MpiOnlyExperiment, OpenMPExperiment): 13 | variant( 14 | "workload", 15 | default="gs", 16 | values=("gs", "rt"), 17 | multi=True, 18 | description="salmon-tddft", 19 | ) 20 | 21 | variant( 22 | "version", 23 | default="2.0.0", 24 | description="app version", 25 | ) 26 | 27 | def compute_applications_section(self): 28 | self.add_experiment_variable("experiment_setup", "") 29 | 30 | if self.spec.satisfies("workload=gs"): 31 | self.add_experiment_variable("exercise", "exercise_01_C2H2_gs") 32 | self.add_experiment_variable("inp_file", "C2H2_gs.inp") 33 | elif self.spec.satisfies("workload=rt"): 34 | self.add_experiment_variable("exercise", "exercise_03_C2H2_rt") 35 | self.add_experiment_variable("inp_file", "C2H2_rt_pulse.inp") 36 | self.add_experiment_variable( 37 | "restart_data", 38 | "../../gs/salmon_{n_nodes}_{n_ranks}_{n_threads}/data_for_restart/", 39 | ) 40 | 41 | if self.spec.satisfies("+openmp"): 42 | self.add_experiment_variable("omp_num_threads", ["12"]) 43 | self.add_experiment_variable("n_ranks", "{processes_per_node} * {n_nodes}") 44 | self.add_experiment_variable("processes_per_node", ["4"]) 45 | self.add_experiment_variable("n_nodes", ["1"], True) 46 | 47 | self.set_required_variables( 48 | n_resources="{n_ranks}", process_problem_size="", total_problem_size="" 49 | ) 50 | 51 | def compute_package_section(self): 52 | self.add_package_spec(self.name, [f"salmon-tddft{self.determine_version()}"]) 53 | -------------------------------------------------------------------------------- /docs/generate-sys-defs-list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import glob 4 | 5 | import pandas as pd 6 | import yaml 7 | 8 | 9 | def main(): 10 | sysconfig_yaml_files = glob.glob( 11 | "../systems/all_hardware_descriptions/**/hardware_description.yaml", 12 | recursive=True, 13 | ) 14 | 15 | df_list = [] 16 | for f in sysconfig_yaml_files: 17 | with open(f, "r") as stream: 18 | try: 19 | data = yaml.safe_load(stream) 20 | except yaml.YAMLError as exc: 21 | print(exc) 22 | 23 | tmp_df = pd.json_normalize(data, max_level=2) 24 | df_list.append(tmp_df) 25 | 26 | df = pd.concat(df_list) 27 | 28 | systested_columns_to_merge = [col for col in df.columns if "systems-tested" in col] 29 | top500_cols_to_merge = [ 30 | col for col in df.columns if "top500-system-instances" in col 31 | ] 32 | 33 | def merge_dicts(row, merge_cols): 34 | combined_dict = {} 35 | for col in merge_cols: 36 | if isinstance(row[col], dict): # Check if the value is a dictionary 37 | combined_dict.update( 38 | {col.split(".")[-1]: row[col]} 39 | ) # Merge the dictionary 40 | return combined_dict 41 | 42 | df["systems-tested"] = df.apply( 43 | lambda row: merge_dicts(row, systested_columns_to_merge), axis=1 44 | ) 45 | df["top500-system-instances"] = df.apply( 46 | lambda row: merge_dicts(row, top500_cols_to_merge), axis=1 47 | ) 48 | df = df.drop(columns=systested_columns_to_merge + top500_cols_to_merge) 49 | 50 | # Remove system_definition from all field names 51 | # (e.g., system_definition.system-tested.description) 52 | df.columns = df.columns.str.removeprefix("system_definition.") 53 | 54 | # Replace NaN with empty string 55 | df.fillna("", inplace=True) 56 | 57 | # Set index to be field names 58 | df.set_index("name", inplace=True) 59 | 60 | # Write out current system definitions to CSV format 61 | df.to_csv("current-system-definitions.csv") 62 | 63 | 64 | if __name__ == "__main__": 65 | main() 66 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Build & Deploy docs site to GitHub Pages 2 | on: 3 | workflow_call: 4 | jobs: 5 | build: 6 | runs-on: ubuntu-24.04 7 | steps: 8 | - name: Checkout 9 | uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 10 | - name: Setup Python 11 | uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 12 | with: 13 | python-version: '3.11' 14 | cache: pip 15 | cache-dependency-path: .github/workflows/requirements/docs.txt 16 | - name: Setup GitHub Pages 17 | id: pages 18 | uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b 19 | - name: Install Sphinx and Theme via Pip 20 | run: | 21 | pip install -r .github/workflows/requirements/docs.txt 22 | # create a dummy workspace to get a working ramble 23 | bin/benchpark system init --dest=generic-x86 generic-x86 24 | bin/benchpark experiment init --dest=saxpy generic-x86 saxpy 25 | bin/benchpark setup generic-x86/saxpy /tmp/workspace 26 | # this is gross and we should better setup ramble for people or make it a Spack package 27 | pip install -r /tmp/workspace/ramble/requirements.txt 28 | - name: Build with sphinx 29 | run: | 30 | cd docs 31 | make html WORKSPACE_PATH=/tmp/workspace 32 | - name: Check for Typos using Codespell 33 | run: | 34 | codespell --ignore-words=.codespellignore 35 | - name: Upload artifact 36 | uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b 37 | if: github.ref == 'refs/heads/develop' 38 | with: 39 | path: ./docs/_build/html 40 | deploy: 41 | needs: build 42 | if: github.ref == 'refs/heads/develop' 43 | permissions: 44 | pages: write 45 | id-token: write 46 | environment: 47 | name: github-pages 48 | url: ${{ steps.deployment.outputs.page_url }} 49 | runs-on: ubuntu-24.04 50 | steps: 51 | - name: Deploy to GitHub Pages 52 | id: deployment 53 | uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e 54 | -------------------------------------------------------------------------------- /repo/saxpy/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | cmake_minimum_required(VERSION 3.23) 7 | 8 | option(USE_OPNEMP "Use OpenMP" OFF) 9 | option(USE_CUDA "Use CUDA" OFF) 10 | option(USE_HIP "Use HIP" OFF) 11 | option(USE_CALIPER "Enable Caliper" FALSE) 12 | 13 | if(USE_CUDA) 14 | project(saxpy VERSION 1.0.0 LANGUAGES CXX CUDA) 15 | else() 16 | project(saxpy VERSION 1.0.0 LANGUAGES CXX) 17 | endif() 18 | 19 | set(SOURCES saxpy.cc) 20 | set(SAXPY_DEPENDENCIES ) 21 | 22 | add_executable(${PROJECT_NAME} ${SOURCES}) 23 | 24 | find_package(MPI REQUIRED) 25 | list(APPEND SAXPY_DEPENDENCIES 26 | MPI::MPI_CXX) 27 | 28 | if(USE_OPENMP) 29 | target_compile_options(${PROJECT_NAME} PRIVATE -DUSE_OPENMP) 30 | find_package(OpenMP REQUIRED) 31 | list(APPEND SAXPY_DEPENDENCIES 32 | OpenMP::OpenMP_CXX) 33 | endif() 34 | 35 | if(USE_HIP) 36 | include_directories(${ROCM_PATH}/include) 37 | target_compile_options(${PROJECT_NAME} PRIVATE -x hip -DUSE_HIP --offload-arch=${ROCM_ARCH}) 38 | list(APPEND SAXPY_DEPENDENCIES 39 | "-L${ROCM_PATH}/lib -lamdhip64") 40 | endif() 41 | 42 | if(USE_CUDA) 43 | set_source_files_properties(saxpy.cc PROPERTIES LANGUAGE "CUDA") 44 | set(CMAKE_CUDA_STANDARD_REQUIRED TRUE) 45 | set(CMAKE_CXX_STANDARD_REQUIRED TRUE) 46 | set(CMAKE_CXX_EXTENSIONS OFF) 47 | set(CMAKE_CUDA_FLAGS "-Xcompiler -DUSE_CUDA") 48 | set_target_properties(${PROJECT_NAME} PROPERTIES CUDA_SEPARABLE_COMPILATION ON) 49 | endif() 50 | 51 | if (USE_CALIPER) 52 | find_package(caliper REQUIRED) 53 | list(APPEND SAXPY_DEPENDENCIES 54 | caliper) 55 | find_package(adiak REQUIRED) 56 | list(APPEND SAXPY_DEPENDENCIES 57 | adiak::adiak) 58 | target_compile_options(${PROJECT_NAME} PRIVATE -DUSE_CALIPER) 59 | endif() 60 | 61 | configure_file( 62 | "${CMAKE_SOURCE_DIR}/config.hh.in" 63 | "${CMAKE_CURRENT_SOURCE_DIR}/config.hh" 64 | ) 65 | 66 | target_link_libraries(${PROJECT_NAME} PRIVATE ${SAXPY_DEPENDENCIES}) 67 | 68 | install(TARGETS ${PROJECT_NAME} 69 | DESTINATION bin 70 | ) 71 | -------------------------------------------------------------------------------- /experiments/babelstream/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.caliper import Caliper 7 | from benchpark.cuda import CudaExperiment 8 | from benchpark.directives import maintainers, variant 9 | from benchpark.experiment import Experiment 10 | from benchpark.openmp import OpenMPExperiment 11 | from benchpark.rocm import ROCmExperiment 12 | 13 | 14 | class Babelstream( 15 | Experiment, 16 | Caliper, 17 | CudaExperiment, 18 | ROCmExperiment, 19 | OpenMPExperiment, 20 | ): 21 | variant( 22 | "workload", 23 | default="babelstream", 24 | description="babelstream", 25 | ) 26 | 27 | variant( 28 | "version", 29 | default="caliper", 30 | values=("develop", "latest", "5.0", "caliper"), 31 | description="app version", 32 | ) 33 | 34 | maintainers("daboehme") 35 | 36 | def compute_applications_section(self): 37 | 38 | self.add_experiment_variable("processes_per_node", "1", True) 39 | self.add_experiment_variable("n", "35", False) 40 | self.add_experiment_variable("o", "0", False) 41 | n_resources = 1 42 | 43 | if self.spec.satisfies("+cuda"): 44 | self.add_experiment_variable("execute", "cuda-stream", False) 45 | 46 | elif self.spec.satisfies("+rocm"): 47 | self.add_experiment_variable("execute", "hip-stream", False) 48 | 49 | else: 50 | self.add_experiment_variable("n_ranks", n_resources, True) 51 | self.add_experiment_variable("execute", "omp-stream", False) 52 | 53 | if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): 54 | self.add_experiment_variable("n_gpus", n_resources, True) 55 | 56 | self.set_required_variables( 57 | n_resources=f"{n_resources}", 58 | process_problem_size="{n}/" + str(n_resources), 59 | total_problem_size="{n}", 60 | ) 61 | 62 | def compute_package_section(self): 63 | self.add_package_spec(self.name, [f"babelstream{self.determine_version()}"]) 64 | -------------------------------------------------------------------------------- /docs/create-mirror.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ############### 8 | System Mirror 9 | ############### 10 | 11 | If you build a benchmark on a networked system, you can use `benchpark mirror` to create 12 | a directory that bundles all necessary resources to install and run that benchmark on 13 | another system, such that the destination system does not need network access. 14 | 15 | On the networked system, if you created/built the benchmark with: 16 | 17 | :: 18 | 19 | benchpark system init --dest=def-ruby llnl-cluster cluster=ruby compiler=gcc 20 | benchpark experiment init --dest=def-raja-perf def-ruby raja-perf 21 | benchpark setup def-ruby/def-raja-perf/ workspace/ 22 | . `pwd`/workspace/setup.sh 23 | ramble --workspace-dir `pwd`/workspace/def-raja-perf/def-ruby/workspace workspace setup 24 | 25 | You can then create a directory that bundles all the resources needed to build that 26 | benchmark with: 27 | 28 | :: 29 | 30 | benchpark mirror create `pwd`/workspace/def-raja-perf/def-ruby/workspace/ test-benchpark-mirror/ 31 | 32 | You can copy `test-bencpark-mirror/` to another system, and on that system, within that 33 | directory you can do: 34 | 35 | :: 36 | 37 | python3 -m venv mirror-env && . mirror-env/bin/activate 38 | pip install --no-index --find-links=pip-cache pip-cache/* 39 | bash first-time.sh 40 | . `pwd`/setup.sh 41 | ramble --workspace-dir `pwd`/def-raja-perf/def-ruby/workspace/ workspace setup 42 | 43 | this will install the benchmark on the new system, and also configure Ramble to use 44 | mirror resources that were bundled in `test-benchmark-mirror/` (so it does not need 45 | internet access to build the benchmark). 46 | 47 | ************* 48 | Limitations 49 | ************* 50 | 51 | For now, benchpark can only create mirrors that are useful for destination systems that 52 | match the host system in terms of: 53 | 54 | - available compilers 55 | - provided external software 56 | 57 | Also, `benchpark mirror` can only create mirrors for benchmarks that have been built on 58 | the source system. 59 | -------------------------------------------------------------------------------- /docs/add-a-benchmark.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | #################### 8 | Adding a Benchmark 9 | #################### 10 | 11 | This guide is intended for application developers who want to add a benchmark so that it 12 | can be run with Benchpark. 13 | 14 | ************************ 15 | Create a New Benchmark 16 | ************************ 17 | 18 | The following system-independent specification is required for each ${Benchmark1}: 19 | 20 | - ``package.py`` is a Spack specification that defines how to build and install 21 | ${Benchmark1}. See the `Spack Package Creation Tutorial 22 | `_ to learn 23 | how to create and test a Spack ``package.py``. If your benchmark already exists in 24 | Spack, benchpark will use that version of the ``package.py``, unless you define a 25 | version in ``benchpark/repo/``. 26 | - ``application.py`` is a Ramble specification that defines the ${Benchmark1} input and 27 | parameters. See the `Ramble Application Definition Developers Guide 28 | `_ to 29 | learn how to create a Ramble ``application.py``. We recommend testing your 30 | ``application.py`` using the `Ramble Usage While Developing 31 | `_ 32 | instructions. If your benchmark already exists in Ramble, benchpark will use that 33 | version of the ``application.py``, unless you define a version in ``benchpark/repo/``. 34 | 35 | Again, by default Benchpark will use ${Benchmark1} specifications (``application.py`` 36 | and ``package.py``) provided in the Spack and Ramble upstream repositories. Overwrite 37 | the upstream definitions by adding the ``application.py`` and/or ``package.py`` to 38 | ``benchpark/repo/${Benchmark1}``, see :doc:`FAQ` for details. 39 | 40 | After satisfying the above prerequisites, in order to use your benchmark in Benchpark, 41 | you will need to create an experiment as described in :doc:`add-an-experiment`. 42 | -------------------------------------------------------------------------------- /docs/run-experiment.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ################### 8 | Run an Experiment 9 | ################### 10 | 11 | To run all of the experiments in the workspace: 12 | 13 | :: 14 | 15 | ramble --workspace-dir . on 16 | 17 | An output file is generated for each experiment in its unique directory: 18 | 19 | :: 20 | 21 | $workspace 22 | | └── experiments 23 | | └── amg2023 24 | | └── problem1 25 | | ├── amg2023_cuda11.8.0_problem1_1_8_2_2_2_10_10_10 26 | | │   ├── execute_experiment 27 | | │   ├── amg2023_cuda11.8.0_problem1_1_8_2_2_2_10_10_10.out 28 | | │   └── ... 29 | | ├── amg2023_cuda11.8.0_problem1_2_4_2_2_2_10_10_10 30 | | │ ├── execute_experiment 31 | | │   ├── amg2023_cuda11.8.0_problem1_2_4_2_2_2_10_10_10.out 32 | | │ └── ... 33 | | ├── amg2023_cuda11.8.0_problem1_1_8_2_2_2_20_20_20 34 | | │   ├── execute_experiment 35 | | │   ├── amg2023_cuda11.8.0_problem1_1_8_2_2_2_20_20_20.out 36 | | │   └── ... 37 | | └── amg2023_cuda11.8.0_problem1_2_4_2_2_2_20_20_20 38 | | ├── execute_experiment 39 | |    ├── amg2023_cuda11.8.0_problem1_2_4_2_2_2_20_20_20.out 40 | | └── ... 41 | 42 | An experiment instance can also be executed individually by directly invoking its 43 | ``execute_experiment`` script (e.g., 44 | ``$workspace/experiments/amg2023/problem1/amg2023_cuda11.8.0_problem1_1_8_2_2_2_10_10_10/execute_experiment``). 45 | 46 | Note that re-running the experiment may overwrite any existing output files in the 47 | directory. Further, if the benchmark has restart capability, existing output may alter 48 | the experiments benchpark would run in the second run. Generally, we would advise the 49 | user to remove the ``$workspace/experiments`` directory before re-running the 50 | experiments using ``ramble --workspace-dir . on``. 51 | 52 | Once you have run your experiment you can try :doc:`analyze-experiment`. 53 | -------------------------------------------------------------------------------- /docs/benchpark-workflow.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | #################### 8 | Benchpark Workflow 9 | #################### 10 | 11 | Follow the workflow diagram to determine if you can run existing configurations or if 12 | you will need to add or edit any Benchpark files. 13 | 14 | .. image:: /_static/images/workflow_fig.png 15 | 16 | ***************************************************** 17 | A. Run Existing Systems, Benchmarks and Experiments 18 | ***************************************************** 19 | 20 | If you are running on an existing ``system``, and the ``benchmark`` and ``experiment`` 21 | are already configured, you can proceed directly to the Running Benchpark Steps, 22 | starting with :doc:`benchpark-setup` for: 23 | 24 | - Setup / Run / Analyze 25 | 26 | ************************* 27 | B. System Specification 28 | ************************* 29 | 30 | A system specification defines the hardware, scheduling system, compilers, and any 31 | external libraries that might exist on a system. 32 | 33 | If you are running on a new system that has not been defined in Benchpark yet, proceed 34 | to :doc:`add-a-system-config` for the following: 35 | 36 | - Find a similar ``system`` 37 | - Add or edit a ``system`` 38 | 39 | **************************** 40 | C. Benchmark Specification 41 | **************************** 42 | 43 | A benchmark specification defines the application build and run behavior, utilizing 44 | spack for build and dependency management. 45 | 46 | If you are creating a new benchmark that has not been defined in Benchpark yet, proceed 47 | to :doc:`add-a-benchmark` for the following: 48 | 49 | - Add a ``benchmark`` 50 | 51 | ***************************** 52 | D. Experiment Specification 53 | ***************************** 54 | 55 | An experiment specification defines application parameters for experiments that require 56 | one or more application runs, such as single-node, scaling, or throughput runs. 57 | 58 | If you are adding experiments to a new or existing benchmark, proceed to 59 | :doc:`add-an-experiment` for the following: 60 | 61 | - Add/edit an ``experiment`` 62 | -------------------------------------------------------------------------------- /repo/smb/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Smb(ExecutableApplication): 12 | """Sandia microbenchmarks""" 13 | name = "Sandia microbenchmarks" 14 | 15 | executable('p1', 'mpi_overhead', use_mpi=True) 16 | executable('p2', 'msgrate -n {ppn}', use_mpi=True) 17 | 18 | workload('mpi_overhead', executables=['p1']) 19 | workload('msgrate', executables=['p2']) 20 | workload('rma_mt', executables=['p2']) 21 | 22 | workload_variable('ppn', default='1', 23 | description='Number of procs per node', 24 | workloads=['msgrate', 'rma_mt']) 25 | 26 | figure_of_merit('single direction', 27 | log_file='{experiment_run_dir}/{experiment_name}.out', 28 | fom_regex=r'single direction:\s+(?P[0-9]+\.[0-9]*)', 29 | group_name='fom', units='') 30 | #TODO:fix this FOM. Not sure what's causing it to not detect 31 | figure_of_merit('overhead', 32 | log_file='{experiment_run_dir}/{experiment_name}.out', 33 | fom_regex=r'(?:[0-9]+\.?[0-9]* +){4}(?P[0-9]+\.[0-9]*)', 34 | #fom_regex=r'avail\(%\)(?:\s|\t)*\n\s*(?:[0-9]+\.*[0-9]*\s*){4}(?P[0-9]+\.*[0-9]*)', 35 | group_name='fom', units='') 36 | 37 | figure_of_merit('pair based', 38 | log_file='{experiment_run_dir}/{experiment_name}.out', 39 | fom_regex=r'pair-based:\s+(?P[0-9]+\.[0-9]*)', 40 | group_name='fom', units='') 41 | 42 | figure_of_merit('pre-post', 43 | log_file='{experiment_run_dir}/{experiment_name}.out', 44 | fom_regex=r'\s*pre-post:\s+(?P[0-9]+\.[0-9]*)', 45 | group_name='fom', units='') 46 | 47 | figure_of_merit('all-start', 48 | log_file='{experiment_run_dir}/{experiment_name}.out', 49 | fom_regex=r'\s*all-start:\s+(?P[0-9]+\.[0-9]*)', 50 | group_name='fom', units='') 51 | #success_criteria('pass', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out') 52 | -------------------------------------------------------------------------------- /experiments/saxpy/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | from benchpark.caliper import Caliper 8 | from benchpark.cuda import CudaExperiment 9 | from benchpark.directives import maintainers, variant 10 | from benchpark.experiment import Experiment 11 | from benchpark.mpi import MpiOnlyExperiment 12 | from benchpark.openmp import OpenMPExperiment 13 | from benchpark.rocm import ROCmExperiment 14 | 15 | 16 | class Saxpy( 17 | Experiment, 18 | MpiOnlyExperiment, 19 | OpenMPExperiment, 20 | CudaExperiment, 21 | ROCmExperiment, 22 | Caliper, 23 | ): 24 | variant( 25 | "workload", 26 | default="problem", 27 | description="problem", 28 | ) 29 | 30 | variant( 31 | "version", 32 | default="1.0.0", 33 | description="app version", 34 | ) 35 | 36 | maintainers("rfhaque") 37 | 38 | def compute_applications_section(self): 39 | # GPU tests include some smaller sizes 40 | n = ["512", "1024"] 41 | if self.spec.satisfies("+openmp"): 42 | self.add_experiment_variable("n_nodes", ["1", "2"], named=True) 43 | # resource_count is the number of resources used for this experiment: 44 | self.add_experiment_variable("resource_count", "8") 45 | self.add_experiment_variable( 46 | "n_threads_per_proc", ["2", "4"], named=True, matrixed=True 47 | ) 48 | else: 49 | n = ["128", "256"] + n 50 | # resource_count is the number of resources used for this experiment: 51 | self.add_experiment_variable("resource_count", "1") 52 | 53 | self.add_experiment_variable("n", n, named=True, matrixed=True) 54 | 55 | self.set_required_variables( 56 | n_resources="{resource_count}", 57 | process_problem_size="{n}/{n_resources}", 58 | total_problem_size="{n}", 59 | ) 60 | 61 | if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): 62 | self.add_experiment_variable("n_gpus", "{n_resources}", True) 63 | else: 64 | self.add_experiment_variable("n_ranks", "{n_resources}", True) 65 | 66 | def compute_package_section(self): 67 | self.add_package_spec(self.name, [f"saxpy{self.determine_version()}"]) 68 | -------------------------------------------------------------------------------- /docs/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | a { 2 | color: #6a984e 3 | } 4 | 5 | a:hover { 6 | color: #7fa866 7 | } 8 | 9 | a:visited { 10 | color: #9b59b6 11 | } 12 | 13 | .wy-nav-content { 14 | max-width: 1200px !important; 15 | } 16 | 17 | .wy-side-nav-search, .wy-nav-top { 18 | background: #7fa866 19 | } 20 | 21 | .wy-nav .wy-menu-vertical header { 22 | color: #7fa866 23 | } 24 | 25 | .wy-menu-vertical a { 26 | color:#d9d9d9 27 | } 28 | 29 | .wy-nav .wy-menu-vertical a:hover { 30 | background-color: #7fa866 31 | } 32 | 33 | .wy-menu-vertical header, .wy-menu-vertical p.caption { 34 | color: #a1ca7b 35 | } 36 | 37 | .wy-nav-top { 38 | background: #7fa866 39 | } 40 | 41 | .rst-content .wy-alert-neutral.admonition-todo a, .rst-content .wy-alert-neutral.admonition a, .rst-content .wy-alert-neutral.attention a, .rst-content .wy-alert-neutral.caution a, .rst-content .wy-alert-neutral.danger a, .rst-content .wy-alert-neutral.error a, .rst-content .wy-alert-neutral.hint a, .rst-content .wy-alert-neutral.important a, .rst-content .wy-alert-neutral.note a, .rst-content .wy-alert-neutral.seealso a, .rst-content .wy-alert-neutral.tip a, .rst-content .wy-alert-neutral.warning a, .wy-alert.wy-alert-neutral a { 42 | color: #7fa866 43 | } 44 | 45 | .wy-tray-container li.wy-tray-item-info { 46 | background: #7fa866 47 | } 48 | 49 | .btn-info { 50 | background-color: #7fa866 !important 51 | } 52 | 53 | .btn-link { 54 | color: #7fa866; 55 | } 56 | 57 | .wy-dropdown-menu > dd > a:hover { 58 | background: #7fa866 59 | } 60 | 61 | .wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a:hover { 62 | background: #7fa866 63 | } 64 | 65 | .wy-inline-validate.wy-inline-validate-info .wy-input-context { 66 | color: #7fa866 67 | } 68 | 69 | .wy-text-info { 70 | color:#7fa866 !important 71 | } 72 | 73 | .wy-side-nav-search { 74 | background-color: #7fa866 75 | } 76 | 77 | .wy-side-nav-search input[type=text] { 78 | border-color: #7fa866 79 | } 80 | 81 | .wy-side-nav-search img { 82 | background-color: #7fa866 83 | } 84 | .dataTables_wrapper .dataTables_filter { 85 | float: unset; 86 | text-align: right; 87 | } 88 | 89 | html.writer-html4 .rst-content dl:not(.docutils) > dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) > dt { 90 | color: #7fa866 91 | } 92 | 93 | .rst-content div[class^="highlight"] pre { 94 | max-height: 600px; 95 | } 96 | -------------------------------------------------------------------------------- /experiments/qws/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.caliper import Caliper 7 | from benchpark.directives import maintainers, variant 8 | from benchpark.experiment import Experiment 9 | from benchpark.mpi import MpiOnlyExperiment 10 | from benchpark.openmp import OpenMPExperiment 11 | 12 | 13 | class Qws(Experiment, MpiOnlyExperiment, OpenMPExperiment, Caliper): 14 | 15 | variant( 16 | "workload", 17 | default="qws", 18 | description="qws", 19 | ) 20 | 21 | variant( 22 | "version", 23 | default="master", 24 | description="app version", 25 | ) 26 | 27 | maintainers("jdomke", "SBA0486") 28 | 29 | def compute_applications_section(self): 30 | 31 | self.add_experiment_variable("experiment_setup", "") 32 | self.add_experiment_variable("lx", "32") 33 | self.add_experiment_variable("ly", "6") 34 | self.add_experiment_variable("lz", "4") 35 | self.add_experiment_variable("lt", "3") 36 | self.add_experiment_variable("px", "1") 37 | self.add_experiment_variable("py", "1") 38 | self.add_experiment_variable("pz", "1") 39 | self.add_experiment_variable("pt", "1") 40 | self.add_experiment_variable("tol_outer", "-1") 41 | self.add_experiment_variable("tol_inner", "-1") 42 | self.add_experiment_variable("maxiter_plus1_outer", "6") 43 | self.add_experiment_variable("maxiter_inner", "50") 44 | 45 | if self.spec.satisfies("+openmp"): 46 | self.add_experiment_variable("n_nodes", ["1"], True) 47 | self.add_experiment_variable("processes_per_node", ["1"]) 48 | self.add_experiment_variable("n_ranks", "{processes_per_node} * {n_nodes}") 49 | self.add_experiment_variable("omp_num_threads", ["48"]) 50 | self.add_experiment_variable("arch", "OpenMP") 51 | else: 52 | self.add_experiment_variable("n_nodes", ["1"], True) 53 | 54 | self.set_required_variables( 55 | n_resources="{n_ranks}", 56 | process_problem_size="{lx}*{ly}*{lz}/{n_ranks}", 57 | total_problem_size="{lx}*{ly}*{lz}", 58 | ) 59 | 60 | def compute_package_section(self): 61 | self.add_package_spec(self.name, [f"qws{self.determine_version()}"]) 62 | -------------------------------------------------------------------------------- /modifiers/affinity/parse_affinity_log.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import argparse 7 | import json 8 | import re 9 | from pathlib import Path 10 | 11 | 12 | def parse_affinity(affinity_log_file, mode): 13 | with open(affinity_log_file, "r") as f: 14 | lines = f.readlines() 15 | 16 | affinity_map = {"gpus": [], "cores": []} 17 | 18 | current_rank = None 19 | current_node = None 20 | 21 | for line in lines: 22 | # Match rank and node 23 | rank_match = re.match(r"rank\s+(\d+)\s+@\s+(\S+)", line) 24 | if rank_match: 25 | current_rank = int(rank_match.group(1)) 26 | current_node = rank_match.group(2) 27 | continue 28 | 29 | # Match CPU affinity (cores) 30 | core_match = re.match(r"\s*cores\s*:\s*(\d+)", line) 31 | if core_match and current_rank is not None: 32 | affinity_map["cores"].append( 33 | { 34 | "rank": current_rank, 35 | "node": current_node, 36 | "cores": int(core_match.group(1)), 37 | } 38 | ) 39 | continue 40 | 41 | if mode == "cuda" or mode == "rocm": 42 | # Match GPU affinity 43 | gpu_match = re.match(r"\s*gpu\s*\d+\s*:\s*(GPU-\S+)", line) 44 | if gpu_match and current_rank is not None: 45 | affinity_map["gpus"].append( 46 | { 47 | "rank": current_rank, 48 | "node": current_node, 49 | "gpu_id": gpu_match.group(1), 50 | } 51 | ) 52 | continue 53 | 54 | # Save to JSON file 55 | output_file = Path(affinity_log_file).with_suffix(".json") 56 | with open(output_file, "w") as json_file: 57 | json.dump(affinity_map, json_file) 58 | 59 | 60 | if __name__ == "__main__": 61 | parser = argparse.ArgumentParser(description="affinity output to JSON") 62 | parser.add_argument("affinity_log_file", type=str, help="affinity log file (text)") 63 | parser.add_argument( 64 | "mode", choices=["mpi", "cuda", "rocm"], help="Mode: 'mpi', 'cuda' or 'rocm'" 65 | ) 66 | 67 | args = parser.parse_args() 68 | parse_affinity(args.affinity_log_file, args.mode) 69 | -------------------------------------------------------------------------------- /repo/branson/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.boost.package import Boost 8 | 9 | 10 | class Branson(CMakePackage): 11 | """Branson's purpose is to study different algorithms for parallel Monte 12 | Carlo transport. Currently it contains particle passing and mesh passing 13 | methods for domain decomposition.""" 14 | 15 | homepage = "https://github.com/lanl/branson" 16 | url = "https://github.com/lanl/branson/archive/0.82.tar.gz" 17 | git = "https://github.com/lanl/branson.git" 18 | 19 | tags = ["proxy-app"] 20 | 21 | license("MIT") 22 | 23 | version("develop", branch="develop") 24 | 25 | version("1.01", sha256="90208eaec4f6d64a4fd81cd838e30b5e7207246cb7f407e482965f23bbcee848") 26 | version( 27 | "0.82", 28 | sha256="7d83d41d0c7ab9c1c906a902165af31182da4604dd0b69aec28d709fe4d7a6ec", 29 | preferred=True, 30 | ) 31 | version("0.81", sha256="493f720904791f06b49ff48c17a681532c6a4d9fa59636522cf3f9700e77efe4") 32 | version("0.8", sha256="85ffee110f89be00c37798700508b66b0d15de1d98c54328b6d02a9eb2cf1cb8") 33 | 34 | depends_on("c", type="build") 35 | depends_on("cxx", type="build") 36 | 37 | #depends_on("mpi") 38 | depends_on("mpi@2:") 39 | 40 | # TODO: replace this with an explicit list of components of Boost, 41 | # for instance depends_on('boost +filesystem') 42 | # See https://github.com/spack/spack/pull/22303 for reference 43 | depends_on(Boost.with_default_variants, when="@:0.81") 44 | depends_on("metis") 45 | depends_on("parmetis", when="@:0.81") 46 | 47 | root_cmakelists_dir = "src" 48 | 49 | def cmake_args(self): 50 | spec = self.spec 51 | args = [] 52 | #args.append("--enable-mpi") 53 | args.append(f"-DCMAKE_C_COMPILER={spec['mpi'].mpicc}") 54 | args.append(f"-DCMAKE_CXX_COMPILER={spec['mpi'].mpicxx}") 55 | args.append(f"-DCMAKE_Fortran_COMPILER={spec['mpi'].mpifc}") 56 | return args 57 | 58 | def install(self, spec, prefix): 59 | mkdir(prefix.bin) 60 | mkdir(prefix.doc) 61 | install(join_path(self.build_directory, "BRANSON"), prefix.bin) 62 | install("LICENSE.md", prefix.doc) 63 | install("README.md", prefix.doc) 64 | install_tree("inputs", prefix.inputs) 65 | -------------------------------------------------------------------------------- /repo/salmon-tddft/fjmpi.patch: -------------------------------------------------------------------------------- 1 | --- spack-src.org/CMakeLists.txt 2020-07-21 12:08:53.000000000 +0900 2 | +++ spack-src/CMakeLists.txt 2020-12-23 11:32:57.147128677 +0900 3 | @@ -38,6 +38,7 @@ 4 | option_set(USE_SCALAPACK "Use ScaLAPACK Library" OFF) 5 | option_set(USE_EIGENEXA "Use EigenExa Library" OFF) 6 | option_set(USE_LIBXC "Use Libxc library" OFF) 7 | +option_set(USE_FJMPI "Use FUJITSU MPI" OFF) 8 | 9 | ## Optimization for stencil compitations 10 | option_set(USE_OPT_DOMAIN_IS_POW2 "Enable whether the optimization assumes that a stencil domain size is power of two" OFF) 11 | --- spack-src.org/src/config.h.in 2020-07-21 12:08:53.000000000 +0900 12 | +++ spack-src/src/config.h.in 2020-12-23 11:48:48.384061907 +0900 13 | @@ -15,6 +15,7 @@ 14 | #cmakedefine SYSTEM_HAS_PATH_MAX_IN_LINUX_LIMITS_H 15 | 16 | #cmakedefine USE_MPI 17 | +#cmakedefine USE_FJMPI 18 | #cmakedefine USE_SCALAPACK 19 | #cmakedefine USE_EIGENEXA 20 | #cmakedefine USE_LIBXC 21 | diff -ru spack-src.org/src/parallel/init_communicator.f90 spack-src/src/parallel/init_communicator.f90 22 | --- spack-src.org/src/parallel/init_communicator.f90 2020-07-21 12:08:53.000000000 +0900 23 | +++ spack-src/src/parallel/init_communicator.f90 2020-12-23 11:50:58.997384260 +0900 24 | @@ -13,6 +13,7 @@ 25 | ! See the License for the specific language governing permissions and 26 | ! limitations under the License. 27 | ! 28 | +#include "config.h" 29 | MODULE init_communicator 30 | implicit none 31 | 32 | @@ -35,7 +36,7 @@ 33 | integer :: i1,i2,i3,i4,i5,ix,iy,iz,nl 34 | integer,allocatable :: iranklists(:) 35 | 36 | -#ifdef __FUJITSU 37 | +#ifdef USE_FJMPI 38 | integer :: iret 39 | #endif 40 | 41 | @@ -59,7 +60,7 @@ 42 | 0:nproc_k-1)) 43 | 44 | ! communicator r,o,k,ro,ko 45 | -#ifdef __FUJITSU 46 | +#ifdef USE_FJMPI 47 | call tofu_network_oriented_mapping(iret) 48 | if (iret < 0) then 49 | if (comm_is_root(info%id_rko)) then 50 | @@ -106,7 +107,7 @@ 51 | 52 | if (nl /= info%isize_rko-1) & 53 | stop '[FATAL ERROR] init_communicator_dft' 54 | -#ifdef __FUJITSU 55 | +#ifdef USE_FJMPI 56 | end if 57 | #endif 58 | 59 | @@ -234,7 +235,7 @@ 60 | info%icomm_xy = comm_create_group_byid(comm, iranklists(1:nl)) 61 | call comm_get_groupinfo(info%icomm_xy, info%id_xy, info%isize_xy) 62 | 63 | -#ifdef __FUJITSU 64 | +#ifdef USE_FJMPI 65 | contains 66 | subroutine tofu_network_oriented_mapping(iret) 67 | use mpi_ext 68 | -------------------------------------------------------------------------------- /repo/cray-mpich-gtl/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | import os.path 8 | import stat 9 | 10 | from spack.package import * 11 | from spack.package import LinkTree 12 | from spack_repo.builtin.packages.mpich.package import MpichEnvironmentModifications 13 | 14 | 15 | class CrayMpichGtl(MpichEnvironmentModifications, BundlePackage): 16 | 17 | version("1.0.0") 18 | 19 | depends_on("cray-mpich", type="build") 20 | provides("mpi@3") 21 | 22 | @property 23 | def libs(self): 24 | return self.spec["cray-mpich"].libs 25 | 26 | def setup_dependent_build_environment(self, env, dependent_spec): 27 | if dependent_spec.satisfies("+rocm"): 28 | env.set("SPACK_GTL", "hip") 29 | elif dependent_spec.satisfies("+cuda"): 30 | env.set("SPACK_GTL", "cuda") 31 | 32 | def setup_dependent_run_environment(self, env, dependent_spec): 33 | env.set("MPICH_GPU_SUPPORT_ENABLED", "1") 34 | 35 | def setup_run_environment(self, env: EnvironmentModifications) -> None: 36 | self.setup_mpi_wrapper_variables(env) 37 | 38 | def setup_dependent_package(self, module, dependent_spec): 39 | MpichEnvironmentModifications.setup_dependent_package( 40 | self, module, dependent_spec 41 | ) 42 | 43 | def install(self, spec, prefix): 44 | dep = spec["cray-mpich"] 45 | for subdir in os.listdir(dep.prefix): 46 | if subdir == "bin": 47 | continue 48 | link_tree = LinkTree(os.path.join(dep.prefix, subdir)) 49 | link_tree.merge(os.path.join(self.prefix, subdir)) 50 | 51 | mkdir(self.prefix.bin) 52 | for target in os.listdir(dep.prefix.bin): 53 | if target in ["mpicc", "mpicxx", "mpif90", "mpif77"]: 54 | fpath = os.path.join(self.prefix.bin, target) 55 | with open(fpath, "w") as f: 56 | f.write( 57 | f"""\ 58 | #!/bin/bash 59 | 60 | if [[ "$SPACK_GTL" == "hip" ]]; then 61 | addlibs="-lmpi_gtl_hsa" 62 | elif [[ "$SPACK_GTL" == "cuda" ]]; then 63 | addlibs="-lmpi_gtl_cuda" 64 | else 65 | addlibs="" 66 | fi 67 | 68 | {dep.prefix.bin}/{target} "$addlibs" "$@" 69 | """ 70 | ) 71 | st = os.stat(fpath) 72 | os.chmod(fpath, st.st_mode | stat.S_IEXEC) 73 | -------------------------------------------------------------------------------- /.gitlab/utils/run-experiment.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # Activate Virtual Environment 5 | . /usr/workspace/benchpark-dev/benchpark-venv/$SYS_TYPE/bin/activate 6 | 7 | echo "./bin/benchpark system init --dest=${HOST} ${ARCHCONFIG} $([ "$HOST" != "matrix" ] && echo "cluster=$HOST") $SYSTEM_ARGS" 8 | echo "./bin/benchpark experiment init --dest=${BENCHMARK} ${HOST} ${BENCHMARK} ${VARIANT}" 9 | echo "./bin/benchpark setup ${HOST}/${BENCHMARK} wkp/" 10 | echo ". wkp/setup.sh" 11 | echo "cd ./wkp/${HOST}/${BENCHMARK}/workspace/" 12 | echo "ramble --disable-logger --workspace-dir . workspace setup" 13 | echo "ramble --disable-logger --workspace-dir . on --executor '{execute_experiment}' --where '{n_nodes} == 1'" 14 | echo "ramble --disable-logger --workspace-dir . workspace analyze --format json yaml text" 15 | 16 | # Ensure proper bootstrap location configured 17 | ./bin/benchpark configure --bootstrap-location $CUSTOM_CI_BUILDS_DIR 18 | 19 | # Initialize System 20 | ./bin/benchpark system init --dest=${HOST} ${ARCHCONFIG} $([ "$HOST" != "matrix" ] && echo "cluster=$HOST") $SYSTEM_ARGS 21 | 22 | # Initialize Experiment 23 | BV="" 24 | if [[ -n "$BENCHMARK_VERSION" ]]; then 25 | BV="version=$BENCHMARK_VERSION" 26 | fi 27 | ./bin/benchpark experiment init --dest=${BENCHMARK} ${HOST} ${BENCHMARK} ${VARIANT} ${BV} 28 | 29 | # Build Workspace 30 | ./bin/benchpark setup ${HOST}/${BENCHMARK} wkp/ 31 | 32 | # Setup Ramble & Spack 33 | . wkp/setup.sh 34 | 35 | # Setup Workspace 36 | cd ./wkp/${HOST}/${BENCHMARK}/workspace/ 37 | 38 | ramble --disable-logger --workspace-dir . workspace setup 39 | 40 | # Using flux on dane (srun called in "ramble on") 41 | if [ "$HOST" == "dane" ] && \ 42 | # Nightly testing still using slurm 43 | [ $CI_PIPELINE_SOURCE != "schedule" ]; then 44 | find . -type f -name execute_experiment -exec sed -i 's/\bsrun\b/flux run --exclusive/g' {} + 45 | fi 46 | 47 | # Runs experiments where n_nodes == 1, and Print Log 48 | ramble --disable-logger --workspace-dir . on --executor '{execute_experiment}' --where '{n_nodes} == 1' 49 | find experiments/ -type f -name "*.out" -exec cat {} + 50 | 51 | # Analyze Experiments 52 | ramble --disable-logger --workspace-dir . workspace analyze --format json yaml text 53 | 54 | cd - 55 | 56 | # Test 'benchpark analyze' 57 | if [[ "$TEST_ANALYZE" == "true" ]]; then 58 | ./bin/benchpark analyze --workspace-dir ./wkp/${HOST}/${BENCHMARK}/workspace/ 59 | fi 60 | 61 | # Check Experiment Exit Codes 62 | python ./.gitlab/bin/exit-codes ./wkp/${HOST}/${BENCHMARK}/workspace/results.latest.json -------------------------------------------------------------------------------- /lib/benchpark/cmd/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # Copyright 2013-2023 Spack Project Developers. 5 | # 6 | # SPDX-License-Identifier: Apache-2.0 7 | 8 | import os 9 | import pickle 10 | import shutil 11 | import sys 12 | 13 | import benchpark.spec 14 | from benchpark.error import BenchparkError 15 | 16 | 17 | def experiment_init(args): 18 | experiment_spec = benchpark.spec.ExperimentSpec(" ".join(args.spec)).concretize() 19 | experiment = experiment_spec.experiment 20 | 21 | if not os.path.exists(args.system): 22 | raise BenchparkError( 23 | f"System '{args.system}' does not exist. Run 'benchpark system init --dest {args.system} ...'." 24 | ) 25 | system_file = os.path.join(args.system, "system.pkl") 26 | with open(system_file, "rb") as f: 27 | system_spec = pickle.load(f) 28 | 29 | experiment.system_spec = system_spec 30 | 31 | if args.dest: 32 | experiment_component = args.dest 33 | else: 34 | experiment_component = experiment_spec.name 35 | 36 | destdir = os.path.join(args.system, experiment_component) 37 | 38 | try: 39 | os.makedirs(destdir) 40 | experiment.write_ramble_dict(f"{destdir}/ramble.yaml") 41 | print( 42 | f"Run `benchpark setup {destdir} ` to generate Ramble workspace" 43 | ) 44 | except FileExistsError: 45 | print(f"Abort: experiment description dir already exists ({destdir})") 46 | sys.exit(1) 47 | except Exception: 48 | # If there was a failure, remove any partially-generated resources 49 | shutil.rmtree(destdir) 50 | raise 51 | 52 | 53 | def setup_parser(root_parser): 54 | system_subparser = root_parser.add_subparsers( 55 | dest="experiment_subcommand", required=True 56 | ) 57 | 58 | init_parser = system_subparser.add_parser("init") 59 | init_parser.add_argument("--dest", help="Place all system files here directly") 60 | init_parser.add_argument( 61 | "system", 62 | help="The system this experiment will run on (directory generated by 'benchpark system init')", 63 | ) 64 | 65 | init_parser.add_argument("spec", nargs="+", help="Experiment spec") 66 | 67 | 68 | def command(args): 69 | actions = { 70 | "init": experiment_init, 71 | } 72 | if args.experiment_subcommand in actions: 73 | actions[args.experiment_subcommand](args) 74 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | # If you are referencing Benchpark in a publication, please cite the paper 2 | # described here. 3 | # 4 | # Here's the raw citation: 5 | # 6 | # Olga Pearce, Alec Scott, Gregory Becker, Riyaz Haque, Nathan Hanford, Stephanie Brink, 7 | # Doug Jacobsen, Heidi Poxon, Jens Domke, and Todd Gamblin. 2023. 8 | # Towards Collaborative Continuous Benchmarking for HPC. 9 | # In Workshops of The International Conference on High Performance Computing, 10 | # Network, Storage, and Analysis (SC-W 2023), November 12–17, 2023, Denver, CO, USA. 11 | # ACM, New York, NY, USA, 9 pages. 12 | # 13 | # And here's the CITATION.cff format: 14 | 15 | cff-version: 1.2.0 16 | title: "Benchpark" 17 | message: "If you are referencing Benchpark in a publication, please cite the paper below." 18 | repository-code: "https://github.com/llnl/benchpark" 19 | preferred-citation: 20 | type: conference-paper 21 | doi: 10.1145/3624062.3624135 22 | url: https://doi.org/10.1145/3624062.3624135 23 | authors: 24 | - family-names: "Pearce" 25 | given-names: "Olga" 26 | - family-names: "Scott" 27 | given-names: "Alec" 28 | - family-names: "Becker" 29 | given-names: "Gregory" 30 | - family-names: "Haque" 31 | given-names: "Riyaz" 32 | - family-names: "Hanford" 33 | given-names: "Nathan" 34 | - family-names: "Brink" 35 | given-names: "Stephanie" 36 | - family-names: "Jacobsen" 37 | given-names: "Doug" 38 | - family-names: "Poxon" 39 | given-names: "Heidi" 40 | - family-names: "Domke" 41 | given-names: "Jens" 42 | - family-names: "Gamblin" 43 | given-names: "Todd" 44 | title: "Towards Collaborative Continuous Benchmarking for HPC" 45 | conference: 46 | name: "Supercomputing 2023 (SC’23)" 47 | city: "Denver" 48 | region: "Colorado" 49 | country: "US" 50 | date-start: 2023-11-12 51 | date-end: 2023-11-17 52 | month: 11 53 | year: 2023 54 | notes: LLNL-CONF-853344 55 | authors: 56 | - family-names: "Pearce" 57 | given-names: "Olga" 58 | - family-names: "Scott" 59 | given-names: "Alec" 60 | - family-names: "Becker" 61 | given-names: "Gregory" 62 | - family-names: "Haque" 63 | given-names: "Riyaz" 64 | - family-names: "Hanford" 65 | given-names: "Nathan" 66 | - family-names: "Brink" 67 | given-names: "Stephanie" 68 | - family-names: "Jacobsen" 69 | given-names: "Doug" 70 | - family-names: "Poxon" 71 | given-names: "Heidi" 72 | - family-names: "Domke" 73 | given-names: "Jens" 74 | - family-names: "Gamblin" 75 | given-names: "Todd" 76 | -------------------------------------------------------------------------------- /docs/diffBuildSpecs.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ##################### 8 | Compare Build Specs 9 | ##################### 10 | 11 | ``lib/scripts/diffBuildSpecs.py`` 12 | 13 | Compare two experiment build specs (dependence graphs with versions of dependencies), 14 | generated by Spack for specified packages+options. This script enables the user to 15 | compare packages and versions of two spack builds. Given two yaml package specs (use 16 | ``spack spec --yaml``), the script outputs which components of the packages are 17 | different. 18 | 19 | .. note:: 20 | 21 | If you are trying to compare benchpark benchmarks, try using 22 | ``diffExperimentBuilds.py``. 23 | 24 | ******************************** 25 | Example: dray with/without MPI 26 | ******************************** 27 | 28 | In this example, we see the difference of ``dray`` built with and without ``mpi``. The 29 | difference between the specs ``dray+mpi`` and ``dray~mpi`` is indicated in the output by 30 | ``-> [openmpi]`` and the console output highlights the package differences in red. 31 | 32 | .. code-block:: console 33 | 34 | $ spack spec --yaml dray+mpi > dray-mpi.yaml 35 | $ spack spec --yaml dray~mpi > dray-nompi.yaml 36 | $ spack-python lib/scripts/diffBuildSpecs.py ./dray-mpi.yaml ./dray-nompi.yaml 37 | 38 | .. figure:: /_static/images/dray-diffspecs.png 39 | :align: center 40 | 41 | (Image output is truncated) 42 | 43 | .. note:: 44 | 45 | ``spack-python`` is required to import the spack libraries needed for this script. 46 | It will automatically be added to your ``$PATH`` when you run ``benchpark setup 47 | ...``. 48 | 49 | ***************************** 50 | Example: Installed packages 51 | ***************************** 52 | 53 | If you want to compare already installed packages, provide the hashes 54 | 55 | .. code-block:: console 56 | 57 | $ spack find -L quicksilver 58 | 59 | -- linux-rhel8-sapphirerapids / gcc@12.1.1 ---------------------- 60 | fubnce7wzgjxhkim2cylijt4cbpfhxi6 quicksilver@master 61 | 62 | -- linux-rhel8-sapphirerapids / intel@2021.6.0-classic ---------- 63 | qwev4yodp2joikf2oxvlo224ksjcqve3 quicksilver@master 64 | ==> 2 installed packages 65 | 66 | $ spack-python lib/scripts/diffBuildSpecs.py quicksilver/fubnce7wzgjxhkim2cylijt4cbpfhxi6 quicksilver/qwev4yodp2joikf2oxvlo224ksjcqve3 67 | 68 | .. figure:: /_static/images/quicksilver-diffspecs.png 69 | :align: center 70 | -------------------------------------------------------------------------------- /experiments/genesis/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers, variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | from benchpark.openmp import OpenMPExperiment 10 | 11 | 12 | class Genesis(Experiment, MpiOnlyExperiment, OpenMPExperiment): 13 | 14 | variant( 15 | "workload", 16 | default="DHFR", 17 | values=("DHFR", "ApoA1", "UUN", "cryoEM"), 18 | description="genesis", 19 | ) 20 | 21 | variant( 22 | "version", 23 | default="2.1.6", 24 | values=("2.1.6", "main"), 25 | description="app version", 26 | ) 27 | 28 | maintainers("jdomke", "SBA0486") 29 | 30 | def compute_applications_section(self): 31 | if self.spec.satisfies("exec_mode=test"): 32 | self.add_experiment_variable("n_nodes", ["1"], True) 33 | # Must be exec_mode=perf 34 | else: 35 | self.add_experiment_variable("n_nodes", ["2"], True) 36 | 37 | self.add_experiment_variable("experiment_setup", "") 38 | self.add_experiment_variable("lx", "32") 39 | self.add_experiment_variable("ly", "6") 40 | self.add_experiment_variable("lz", "4") 41 | self.add_experiment_variable("lt", "3") 42 | self.add_experiment_variable("px", "1") 43 | self.add_experiment_variable("py", "1") 44 | self.add_experiment_variable("pz", "1") 45 | self.add_experiment_variable("pt", "1") 46 | self.add_experiment_variable("tol_outer", "-1") 47 | self.add_experiment_variable("tol_inner", "-1") 48 | self.add_experiment_variable("maxiter_plus1_outer", "6") 49 | self.add_experiment_variable("maxiter_inner", "50") 50 | 51 | if self.spec.satisfies("+openmp"): 52 | self.add_experiment_variable("processes_per_node", ["8"]) 53 | self.add_experiment_variable("n_ranks", "{processes_per_node} * {n_nodes}") 54 | self.add_experiment_variable("omp_num_threads", ["12"]) 55 | self.add_experiment_variable("arch", "OpenMP") 56 | 57 | self.set_required_variables( 58 | n_resources="{n_ranks}", 59 | process_problem_size="{lx}*{ly}*{lz}/{n_ranks}", 60 | total_problem_size="{lx}*{ly}*{lz}", 61 | ) 62 | 63 | def compute_package_section(self): 64 | self.add_package_spec(self.name, [f"genesis{self.determine_version()}"]) 65 | -------------------------------------------------------------------------------- /experiments/hpcg/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.caliper import Caliper 7 | from benchpark.directives import maintainers, variant 8 | from benchpark.experiment import Experiment 9 | from benchpark.mpi import MpiOnlyExperiment 10 | from benchpark.openmp import OpenMPExperiment 11 | from benchpark.scaling import Scaling, ScalingMode 12 | 13 | 14 | class Hpcg( 15 | Experiment, 16 | MpiOnlyExperiment, 17 | OpenMPExperiment, 18 | Scaling(ScalingMode.Strong, ScalingMode.Weak), 19 | Caliper, 20 | ): 21 | 22 | variant( 23 | "workload", 24 | default="standard", 25 | description="workload to run", 26 | ) 27 | variant( 28 | "version", 29 | default="caliper", 30 | values=("3.1", "develop", "caliper"), 31 | description="app version", 32 | ) 33 | 34 | maintainers("pearce8") 35 | 36 | def compute_applications_section(self): 37 | if self.spec.satisfies("exec_mode=test"): 38 | self.add_experiment_variable( 39 | "problem_sizes", {"mx": 104, "my": 104, "mz": 104}, True 40 | ) 41 | self.add_experiment_variable("num_procs", 1, True) 42 | 43 | self.register_scaling_config( 44 | { 45 | ScalingMode.Strong: { 46 | "num_procs": lambda var, itr, dim, scaling_factor: var.val(dim) 47 | * scaling_factor, 48 | "problem_sizes": lambda var, itr, dim, scaling_factor: var.val(dim), 49 | }, 50 | ScalingMode.Weak: { 51 | "num_procs": lambda var, itr, dim, scaling_factor: var.val(dim) 52 | * scaling_factor, 53 | "problem_sizes": lambda var, itr, dim, scaling_factor: var.val(dim) 54 | * scaling_factor, 55 | }, 56 | } 57 | ) 58 | 59 | self.add_experiment_variable("n_ranks", "{num_procs}", True) 60 | self.add_experiment_variable("n_threads_per_proc", 1, True) 61 | self.add_experiment_variable("matrix_size", "{mx} {my} {mz}", False) 62 | 63 | self.add_experiment_variable("iterations", "60", False) 64 | 65 | self.set_required_variables( 66 | n_resources="{n_ranks}", 67 | process_problem_size="{mx}*{my}*{mz}/{n_ranks}", 68 | total_problem_size="{mx}*{my}*{mz}", 69 | ) 70 | 71 | def compute_package_section(self): 72 | self.add_package_spec(self.name, [f"hpcg{self.determine_version()}"]) 73 | -------------------------------------------------------------------------------- /docs/for-the-impatient.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | ################### 8 | For the Impatient 9 | ################### 10 | 11 | You need git and Python 3.8+: 12 | 13 | :: 14 | 15 | git clone https://github.com/LLNL/benchpark.git 16 | cd benchpark 17 | . setup-env.sh 18 | benchpark --version 19 | 20 | python3 -m venv my-env 21 | . my-env/bin/activate 22 | 23 | pip install -r requirements.txt 24 | 25 | ******************** 26 | Set up a Workspace 27 | ******************** 28 | 29 | :: 30 | 31 | benchpark system init --dest= compiler= 32 | benchpark experiment init +/~ = 33 | benchpark setup / 34 | 35 | where: 36 | 37 | - ````: amg2023 | saxpy | etc. (predefined choices in :doc:`benchmark-list`) 38 | - ````: Cts | Tioga | etc. (predefined systems in :doc:`system-list`) 39 | 40 | ``benchpark setup`` will output instructions to follow: 41 | 42 | :: 43 | 44 | . /setup.sh 45 | 46 | ********************* 47 | Build an Experiment 48 | ********************* 49 | 50 | :: 51 | 52 | cd ///workspace 53 | ramble --workspace-dir . workspace setup 54 | 55 | ******************** 56 | Run the Experiment 57 | ******************** 58 | 59 | To run all of the experiments in the workspace: 60 | 61 | :: 62 | 63 | ramble --workspace-dir . on 64 | 65 | To run a single experiment in the workspace, invoke the ``execute_experiment`` script 66 | for the specific experiment (e.g., 67 | ``$workspace/experiments/amg2023/problem1/amg2023_cuda11.8.0_problem1_1_8_2_2_2_10_10_10/execute_experiment``). 68 | 69 | ********************** 70 | Experiment pass/fail 71 | ********************** 72 | 73 | Once the experiments completed running, the command: 74 | 75 | :: 76 | 77 | ramble --workspace-dir . workspace analyze 78 | 79 | can be used to analyze figures of merit and evaluate `success/failure 80 | `_ of 81 | the experiments. Ramble generates a file with summary of the results in ``$workspace``. 82 | 83 | If the benchmark you are running is instrumented with `Caliper 84 | `_, you can use the Caliper modifier (see 85 | :doc:`modifiers`) to collect detailed measurements you can later analyze with `Thicket 86 | `_. 87 | -------------------------------------------------------------------------------- /repo/commbench/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class Commbench(CMakePackage, CudaPackage, ROCmPackage): 10 | """FIXME: Put a proper description of your package here.""" 11 | 12 | # FIXME: Add a proper url for your package's homepage here. 13 | homepage = "https://github.com/simongdg/CommBench" 14 | # url = "https://github.com/arhag23/CommBench/archive/cmake.tar.gz" 15 | git = "https://github.com/simongdg/CommBench.git" 16 | 17 | version("0.1.0", commit="efed373") 18 | 19 | # FIXME: Add a list of GitHub accounts to 20 | # notify when the package is updated. 21 | maintainers("simongdg") 22 | 23 | # FIXME: Add the SPDX identifier of the project's license below. 24 | # See https://spdx.org/licenses/ for a list. Upon manually verifying 25 | # the license, set checked_by to your Github username. 26 | license("Apache-2.0", checked_by="simongdg") 27 | 28 | version("0.1.0") 29 | variant("cuda", default=False, description="Enable CUDA support") 30 | variant("rocm", default=False, description="Enable HIP support") 31 | variant("caliper", default=True, description="Enable Caliper profiling") 32 | 33 | # FIXME: Add dependencies if required. 34 | depends_on("c", type="build") 35 | depends_on("cxx", type="build") 36 | 37 | depends_on("cmake@3.24:", type="build") 38 | 39 | depends_on("mpi") 40 | 41 | depends_on("rccl", when="+rocm") 42 | depends_on("caliper", when="+caliper") 43 | 44 | for arch in ("none", "50", "60", "70", "80", "90"): 45 | depends_on(f"nccl cuda_arch={arch}", when=f"cuda_arch={arch}") 46 | 47 | def cmake_args(self): 48 | args = [] 49 | # args.append("--debug-trycompile") 50 | if self.spec.satisfies("+rocm"): 51 | args.append("-DUSE_HIP=ON") 52 | args.append("-DUSE_XCCL=ON") 53 | elif self.spec.satisfies("+cuda"): 54 | args.append("-DUSE_CUDA=ON") 55 | args.append("-DUSE_XCCL=ON") 56 | 57 | cuda_arch = self.spec.variants["cuda_arch"].value[0] 58 | if cuda_arch is not None: 59 | args.append(f"-DCMAKE_CUDA_ARCHITECTURES={cuda_arch}") 60 | 61 | mpi_spec = self.spec["mpi"] 62 | 63 | if mpi_spec.satisfies("+gtl"): 64 | gtl_path = mpi_spec.extra_attributes["gtl_path"] 65 | args.append("-DUSE_GTL=ON") 66 | args.append(f"-DGTL_PATH={gtl_path}") 67 | 68 | if self.spec.satisfies("+caliper"): 69 | args.append("-DUSE_CALIPER=ON") 70 | 71 | return args 72 | --------------------------------------------------------------------------------