├── .codespellignore ├── .envrc ├── .flake8 ├── .github ├── ISSUE_TEMPLATE │ ├── benchmark-status.md │ └── bug_report.md ├── dependabot.yml ├── labeler.yml ├── pull_request_template.md ├── utils │ ├── dryrun.sh │ └── dryruns.py └── workflows │ ├── bin │ └── license │ ├── ci.yml │ ├── coverage.yml │ ├── diffpackages.yml │ ├── docs.yml │ ├── label.yml │ ├── license.yml │ ├── nightly.yml │ ├── requirements │ ├── docs.txt │ ├── docstrfmt.toml │ └── style.txt │ ├── run.yml │ └── style.yml ├── .gitignore ├── .gitlab-ci.yml ├── .gitlab ├── bin │ └── exit-codes ├── tests │ ├── nightly.yml │ ├── shared_flux_clusters.yml │ └── shared_slurm_clusters.yml └── utils │ ├── cancel-flux.sh │ ├── cancel-slurm.sh │ ├── machine_checks.yml │ ├── rules.yml │ ├── run-experiment.sh │ └── status.yml ├── CITATION.cff ├── CMakeLists.txt ├── COPYRIGHT ├── CTestConfig.cmake ├── CTestGitlab.cmake ├── LICENSE ├── NOTICE ├── README.rst ├── bin ├── benchpark └── benchpark-python ├── checkout-versions.yaml ├── codecov.yml ├── common-resources └── execute_experiment.tpl ├── docs ├── FAQ.rst ├── Makefile ├── No-mpi.png ├── Tree.png ├── _static │ ├── css │ │ └── custom.css │ ├── images │ │ ├── benchpark-dark.svg │ │ ├── benchpark-icon-background.svg │ │ ├── benchpark-icon.svg │ │ ├── benchpark-light.svg │ │ ├── dray-diffspecs.png │ │ ├── dryruns.png │ │ ├── kripke_cuda_strong_percentage_exc.png │ │ ├── kripke_cuda_strong_raw_exc-2.png │ │ ├── kripke_cuda_strong_raw_exc.png │ │ ├── kripke_cuda_strong_raw_inc.png │ │ ├── kripke_cuda_throughput_raw_exc.png │ │ ├── kripke_cuda_weak_raw_exc.png │ │ ├── quicksilver-diffspecs.png │ │ ├── shared-nonshared.png │ │ └── workflow_fig.png │ ├── main.js │ ├── qr-codes │ │ ├── benchpark-docs-qrcode.png │ │ └── benchpark-github-qrcode.png │ └── slides │ │ ├── HPCIC25-Tutorial-Slides.pdf │ │ ├── ReproduciblePerfAnalysis-HPDC25-Tutorial-Slides.pdf │ │ └── eScience25-Tutorial-Slides.pdf ├── add-a-benchmark.rst ├── add-a-system-config.rst ├── add-an-experiment.rst ├── analyze-experiment.rst ├── benchmark-list.rst ├── benchpark-analyze.rst ├── benchpark-commands.rst ├── benchpark-setup.rst ├── benchpark-workflow.rst ├── build-experiment.rst ├── ci-developer-guide.rst ├── comparing-experiment-builds.rst ├── conf.py ├── create-mirror.rst ├── developer-guide.rst ├── diffBuildSpecs.rst ├── diffExperimentBuilds.rst ├── diffExperimentSpecs.rst ├── diffPackageCommits.rst ├── diffSystemSpecs.rst ├── examples │ └── compare_experiment_builds │ │ └── compareExperimentBuilds.sh ├── experiments_root_structure.rst ├── finished_experiments_example.png ├── flux_jobs_a_output.png ├── for-the-impatient.rst ├── generate-benchmark-list.py ├── generate-sys-defs-list.py ├── getting-started.rst ├── graph-and-tree.png ├── index.rst ├── kripke_mpi_strong_raw_exc.png ├── make.bat ├── modifiers.rst ├── requirements.txt ├── run-binary.rst ├── run-experiment.rst ├── set-of-experiments.rst ├── showBuild.rst ├── software.yaml ├── system-list.rst ├── system │ └── fugaku.rst ├── testing-your-contribution.rst ├── tutorial-101.rst ├── tutorial-llnl.rst ├── tutorial │ ├── HPCIC25-Tutorial-Slide-Preview.jpg │ ├── ReproduciblePerfAnalysis-HPDC25-Tutorial-Slide-Preview.jpg │ └── eScience25-Tutorial-Slide-Preview.jpg └── why-benchpark.rst ├── experiments ├── ad │ └── experiment.py ├── amg2023 │ └── experiment.py ├── babelstream │ └── experiment.py ├── commbench │ └── experiment.py ├── genesis │ └── experiment.py ├── gpcnet │ └── experiment.py ├── gromacs │ └── experiment.py ├── hpcg │ └── experiment.py ├── hpl │ └── experiment.py ├── ior │ └── experiment.py ├── kripke │ └── experiment.py ├── laghos │ └── experiment.py ├── lammps │ └── experiment.py ├── md-test │ └── experiment.py ├── osu-micro-benchmarks │ └── experiment.py ├── phloem │ └── experiment.py ├── quicksilver │ └── experiment.py ├── qws │ └── experiment.py ├── raja-perf │ └── experiment.py ├── remhos │ └── experiment.py ├── repo.yaml ├── salmon-tddft │ └── experiment.py ├── saxpy │ └── experiment.py ├── smb │ └── experiment.py ├── sparta-snl │ └── experiment.py └── stream │ └── experiment.py ├── lib ├── benchpark │ ├── __init__.py │ ├── accounting.py │ ├── caliper.py │ ├── cmd │ │ ├── analyze.py │ │ ├── audit.py │ │ ├── bootstrap.py │ │ ├── experiment.py │ │ ├── info.py │ │ ├── list.py │ │ ├── mirror.py │ │ ├── setup.py │ │ ├── show_build.py │ │ ├── system.py │ │ └── unit_test.py │ ├── cuda.py │ ├── cudasystem.py │ ├── debug.py │ ├── directives.py │ ├── error.py │ ├── experiment.py │ ├── mpi.py │ ├── openmp.py │ ├── openmpsystem.py │ ├── paths.py │ ├── repo.py │ ├── rocm.py │ ├── rocmsystem.py │ ├── runtime.py │ ├── scaling.py │ ├── spec.py │ ├── system.py │ ├── test │ │ ├── __init__.py │ │ ├── caliper.py │ │ ├── commands.py │ │ ├── conftest.py │ │ ├── experiment.py │ │ ├── paths.py │ │ ├── spec_semantics.py │ │ ├── spec_syntax.py │ │ └── system.py │ ├── variables.py │ └── variant.py ├── main.py └── scripts │ ├── diffBuildSpecs.py │ ├── diffExperimentBuilds.py │ ├── diffExperimentSpecs.py │ ├── diffPackageCommits.py │ ├── diffSystemSpecs.py │ ├── env-collect-branch-tips.py │ └── experiment-build-info.py ├── modifiers ├── affinity │ ├── modifier.py │ └── parse_affinity_log.py ├── allocation │ └── modifier.py ├── caliper │ └── modifier.py ├── hwloc │ ├── modifier.py │ └── parse_hwloc_output.py └── modifier_repo.yaml ├── pyproject.toml ├── pytest.ini ├── remote-urls.yaml ├── repo ├── ad │ ├── application.py │ └── package.py ├── affinity │ └── package.py ├── amg2023 │ ├── application.py │ └── package.py ├── babelstream │ ├── application.py │ └── package.py ├── blt │ └── package.py ├── branson │ ├── application.py │ └── package.py ├── caliper │ ├── for_aarch64.patch │ ├── package.py │ └── sampler-service-missing-libunwind-include-dir.patch ├── camp │ └── package.py ├── chai │ └── package.py ├── commbench │ ├── application.py │ └── package.py ├── cray-mpich │ └── package.py ├── cuda │ └── package.py ├── enzyme │ └── package.py ├── essl │ └── package.py ├── genesis │ ├── application.py │ └── package.py ├── gpcnet │ └── application.py ├── gromacs │ ├── application.py │ ├── gmxDetectCpu-cmake-3.14.patch │ ├── gmxDetectSimd-cmake-3.14.patch │ └── package.py ├── hipsycl │ └── package.py ├── hpcc │ └── application.py ├── hpcg │ ├── application.py │ └── package.py ├── hpl │ ├── application.py │ └── package.py ├── hypre │ └── package.py ├── ior │ └── application.py ├── kokkos │ └── package.py ├── kripke │ ├── application.py │ └── package.py ├── laghos │ ├── application.py │ └── package.py ├── lammps │ ├── application.py │ └── package.py ├── lapack-xl │ └── package.py ├── md-test │ └── application.py ├── mfem │ └── package.py ├── osu-micro-benchmarks │ ├── application.py │ └── package.py ├── phloem │ ├── application.py │ └── package.py ├── quicksilver │ ├── application.py │ └── package.py ├── qws │ ├── application.py │ └── package.py ├── raja-perf │ ├── application.py │ └── package.py ├── raja │ └── package.py ├── remhos │ ├── application.py │ └── package.py ├── repo.yaml ├── salmon-tddft │ ├── application.py │ ├── cmakefix.patch │ ├── fjmpi.patch │ ├── package.py │ └── v2.0.libxc-5.0.patch ├── saxpy │ ├── CMakeLists.txt │ ├── application.py │ ├── config.hh.in │ ├── package.py │ └── saxpy.cc ├── smb │ ├── application.py │ └── package.py ├── sparta-snl │ ├── application.py │ └── package.py ├── spectrum-mpi │ └── package.py ├── stream │ ├── application.py │ └── package.py └── umpire │ └── package.py ├── requirements.txt ├── setup-env.sh ├── spack.yaml ├── systems ├── all_hardware_descriptions │ ├── AWS_PCluster-zen-EFA │ │ └── hardware_description.yaml │ ├── AWS_Tutorial-sapphirerapids-EFA │ │ └── hardware_description.yaml │ ├── Atos-rome-A100-Infiniband │ │ └── hardware_description.yaml │ ├── DELL-cascadelake-Infiniband │ │ └── hardware_description.yaml │ ├── DELL-sapphirerapids-H100-Infiniband │ │ └── hardware_description.yaml │ ├── DELL-sapphirerapids-OmniPath │ │ └── hardware_description.yaml │ ├── Fujitsu-A64FX-TofuD │ │ └── hardware_description.yaml │ ├── HPECray-haswell-P100-Infiniband │ │ └── hardware_description.yaml │ ├── HPECray-neoverse-H100-Slingshot │ │ └── hardware_description.yaml │ ├── HPECray-zen2-Slingshot │ │ └── hardware_description.yaml │ ├── HPECray-zen3-MI250X-Slingshot │ │ └── hardware_description.yaml │ ├── HPECray-zen4-MI300A-Slingshot │ │ └── hardware_description.yaml │ ├── IBM-power9-V100-Infiniband │ │ └── hardware_description.yaml │ ├── Penguin-haswell-OmniPath │ │ └── hardware_description.yaml │ ├── Penguin-icelake-OmniPath │ │ └── hardware_description.yaml │ ├── Supermicro-icelake-OmniPath │ │ └── hardware_description.yaml │ └── x86_64 │ │ └── hardware_description.yaml ├── aws-pcluster │ └── system.py ├── aws-tutorial │ └── system.py ├── common │ └── spack_includes.yaml ├── csc-lumi │ ├── externals │ │ └── libsci │ │ │ └── 02-rocmcc-packages.yaml │ └── system.py ├── cscs-daint │ └── system.py ├── cscs-eiger │ └── system.py ├── generic-x86 │ └── system.py ├── jsc-juwels │ └── system.py ├── lanl-venado │ └── system.py ├── llnl-cluster │ └── system.py ├── llnl-elcapitan │ └── system.py ├── llnl-matrix │ └── system.py ├── llnl-sierra │ └── system.py ├── repo.yaml ├── riken-fugaku │ └── system.py └── snl-eldorado │ └── system.py ├── taxonomy.yaml └── var ├── exp_repo └── repo.yaml └── sys_repo └── repo.yaml /.codespellignore: -------------------------------------------------------------------------------- 1 | cachable 2 | parma -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------ 2 | # Load Development Spack Environment (If Spack is installed.) 3 | # 4 | # Run 'direnv allow' from within the cloned repository to automatically 5 | # load the spack environment when you enter the directory. 6 | #------------------------------------------------------------------------ 7 | if type spack &>/dev/null; then 8 | . $SPACK_ROOT/share/spack/setup-env.sh 9 | spack env activate -d . 10 | fi 11 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | # -*- conf -*- 2 | # flake8 settings for thicket 3 | # 4 | # These are the flake8 settings recommended by Black 5 | # See https://github.com/psf/black 6 | # 7 | [flake8] 8 | max-line-length = 88 9 | select = C,E,F,W,B,B950 10 | ignore = E501,W503,E203 11 | 12 | per-file-ignores = 13 | modifiers/*/modifier.py:F403,F405 14 | legacy/modifiers/*/modifier.py:F403,F405 15 | 16 | builtins = IPython 17 | exclude = 18 | .eggs, 19 | .git, 20 | .hg, 21 | .mypy_cache, 22 | .tox, 23 | .venv, 24 | _build, 25 | buck-out, 26 | build, 27 | dist, 28 | repo 29 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/benchmark-status.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Benchmark status 3 | about: Per-Benchmark tasks 4 | title: '' 5 | labels: experiment 6 | assignees: rfhaque 7 | 8 | --- 9 | 10 | ## Components 11 | 1. [ ] Source code, Build system: Choose one: Make, CMake, Autotools 12 | 2. [ ] Source code released 13 | 3. [ ] Spack package.py 14 | 4. [ ] Ramble application.py 15 | 5. [ ] Benchpark experiment.py 16 | 6. [ ] +cuda experiment.py 17 | 7. [ ] +rocm experiment.py 18 | 8. [ ] Benchmark dependencies: List here 19 | 9. [ ] small test size problem for CI 20 | 10. [ ] Scaling (Weak, Strong, or both) 21 | 11. [ ] Large scale problem size defined 22 | 12. [ ] CI running on CPUs/CDash 23 | 13. [ ] CI running on NVIDIA GPUs/CDash 24 | 14. [ ] CI running on AMD GPUs/CDash 25 | 15. [ ] +single_memory experiment.py 26 | 16. [ ] +caliper experiment.py 27 | 17. [ ] CPU-only performance analysis 28 | 18. [ ] CUDA performance analysis 29 | 19. [ ] ROCm performance analysis 30 | 20. [ ] Documentation (build, run instructions, performance demo) 31 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Log files** 24 | If applicable, add log files to help explain your problem. 25 | 26 | **Supercomputer (please complete the following information):** 27 | - system: which of the systems in benchpark/systems you are using [e.g., El Capitan] 28 | - system parameters: what system parameters did you use to instantiate the system 29 | - experiment: which of the experiments in benchpark/experiments are you using [e.g., amg2023] 30 | - experiment parameters: what experiment parameters did you use to instantiate the experiment 31 | 32 | **Python environment, etc. (please complete the following information):** 33 | - Python version 34 | 35 | **Additional context** 36 | Add any other context about the problem here. 37 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: / 5 | schedule: 6 | interval: daily 7 | - package-ecosystem: pip 8 | directory: /.github/workflows/requirements 9 | schedule: 10 | interval: weekly 11 | ignore: 12 | # setuptools releases new versions almost daily 13 | - dependency-name: setuptools 14 | update-types: [version-update:semver-patch] 15 | -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- 1 | application: 2 | - changed-files: 3 | - any-glob-to-any-file: [repo/**] 4 | ci: 5 | - changed-files: 6 | - any-glob-to-any-file: [.github/**] 7 | dependencies: 8 | - changed-files: 9 | - any-glob-to-any-file: 10 | - .github/worklflows/requirements/** 11 | - pyproject.toml 12 | - requirements.txt 13 | docs: 14 | - changed-files: 15 | - any-glob-to-any-file: [docs/**, README.rst, taxonomy.yaml] 16 | experiment: 17 | - changed-files: 18 | - any-glob-to-any-file: [experiments/**, var/exp_repo/**] 19 | feature: 20 | - changed-files: 21 | - any-glob-to-any-file: [bin/**, lib/**, modifiers/**] 22 | system: 23 | - changed-files: 24 | - any-glob-to-any-file: [systems/**] 25 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | - [ ] Replace with: A short description of the change, including motivation and context. 4 | - [ ] Replace with: A list of any dependencies. 5 | - [ ] Replace with: Link(s) to relevant [issue(s)](https://docs.github.com/en/issues/tracking-your-work-with-issues/using-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) 6 | - [ ] Complete the checklist for a relevant section(s) below 7 | - [ ] Delete sections below that are not relevant to this PR 8 | 9 | ## Adding/modifying a system (docs: [Adding a System](https://software.llnl.gov/benchpark/add-a-system-config.html)) 10 | 11 | - [ ] Add/modify `systems/system_name/system.py` file 12 | - [ ] Add/modify `systems/all_hardware_descriptions/hardware_name/hardware_description.yaml` which will appear in the [docs catalogue](https://software.llnl.gov/benchpark/system-list.html) 13 | 14 | ## Adding/modifying a benchmark (docs: [Adding a Benchmark](https://software.llnl.gov/benchpark/add-a-benchmark.html)) 15 | 16 | - [ ] If modifying the source code of a benchmark: create, self-assign, and link here a follow up issue with a link to the PR in the benchmark repo. 17 | - [ ] If package.py upstreamed to Spack is insufficient, add/modify `repo/benchmark_name/package.py` plus: create, self-assign, and link here a follow up issue with a link to the PR in the Spack repo. 18 | - [ ] If application.py upstreamed to Ramble is insufficient, add/modify `repo/benchmark_name/application.py` plus: create, self-assign, and link here a follow up issue with a link to the PR in the Ramble repo. 19 | - [ ] Tags in Ramble's `application.py` or in `repo/benchmark_name/application.py` will appear in the [docs catalogue](https://software.llnl.gov/benchpark/benchmark-list.html) 20 | - [ ] Add/modify an `experiments/benchmark_name/experiment.py` to define an experiment 21 | 22 | ## Adding/modifying core functionality, CI, or documentation: 23 | 24 | - [ ] Update docs 25 | - [ ] Update `.github/workflows` and `.gitlab/tests` unit tests (if needed) 26 | -------------------------------------------------------------------------------- /.github/utils/dryrun.sh: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | #!/bin/bash 7 | 8 | benchmark_spec="$1" 9 | system_spec="$2" 10 | 11 | timestamp=$(date +%s) 12 | benchmark="b-$timestamp" 13 | system="s-$timestamp" 14 | ./bin/benchpark system init --dest=$system $system_spec 15 | ./bin/benchpark experiment init --dest=$benchmark --system=$system $benchmark_spec 16 | ./bin/benchpark setup ./$benchmark workspace/ 17 | . workspace/setup.sh 18 | ramble \ 19 | --workspace-dir "workspace/$benchmark/$system/workspace" \ 20 | --disable-logger \ 21 | workspace setup --dry-run -------------------------------------------------------------------------------- /.github/workflows/coverage.yml: -------------------------------------------------------------------------------- 1 | name: Run a Set of Tests to Collect Code Coverage for Benchpark 2 | on: 3 | workflow_call: 4 | secrets: 5 | BENCHPARK_CODECOV_TOKEN: 6 | required: true 7 | jobs: 8 | run_unit_tests: 9 | runs-on: ubuntu-24.04 10 | steps: 11 | - name: Checkout Benchpark 12 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 13 | - name: Add needed Python libs 14 | run: | 15 | pip install -r ./requirements.txt 16 | - name: Run unit tests with coverage 17 | run: | 18 | ./bin/benchpark unit-test --cov=./ --cov-branch --cov-report=xml --durations=20 -ra 19 | - name: Upload coverage to Codecov 20 | uses: codecov/codecov-action@v4 21 | with: 22 | token: ${{ secrets.BENCHPARK_CODECOV_TOKEN }} 23 | directory: ./coverage/reports 24 | files: /home/runner/work/benchpark/benchpark/coverage.xml 25 | flags: unittests 26 | verbose: true 27 | fail_ci_if_error: true 28 | -------------------------------------------------------------------------------- /.github/workflows/diffpackages.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_call: 3 | inputs: 4 | modified_repo_dirs: 5 | required: true 6 | type: string 7 | jobs: 8 | diffPackages: 9 | runs-on: ubuntu-24.04 10 | steps: 11 | - name: Checkout repository 12 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 13 | - name: Add needed Python libs 14 | run: | 15 | pip install -r ./requirements.txt 16 | - name: Check Packages Against Spack 17 | run: |- 18 | ./bin/benchpark-python lib/scripts/diffPackageCommits.py --packages "${{ inputs.modified_repo_dirs }}" 19 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: Build & Deploy docs site to GitHub Pages 2 | on: 3 | workflow_call: 4 | jobs: 5 | build: 6 | runs-on: ubuntu-24.04 7 | steps: 8 | - name: Checkout 9 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 10 | - name: Setup Python 11 | uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c 12 | with: 13 | python-version: '3.11' 14 | cache: pip 15 | cache-dependency-path: .github/workflows/requirements/docs.txt 16 | - name: Setup GitHub Pages 17 | id: pages 18 | uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b 19 | - name: Install Sphinx and Theme via Pip 20 | run: | 21 | pip install -r .github/workflows/requirements/docs.txt 22 | # create a dummy workspace to get a working ramble 23 | bin/benchpark system init --dest=generic-x86 generic-x86 24 | bin/benchpark experiment init --dest=saxpy --system=generic-x86 saxpy 25 | bin/benchpark setup saxpy /tmp/workspace 26 | # this is gross and we should better setup ramble for people or make it a Spack package 27 | pip install -r /tmp/workspace/ramble/requirements.txt 28 | - name: Build with sphinx 29 | run: | 30 | cd docs 31 | make html WORKSPACE_PATH=/tmp/workspace 32 | - name: Check for Typos using Codespell 33 | run: | 34 | codespell --ignore-words=.codespellignore 35 | - name: Upload artifact 36 | uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b 37 | if: github.ref == 'refs/heads/develop' 38 | with: 39 | path: ./docs/_build/html 40 | deploy: 41 | needs: build 42 | if: github.ref == 'refs/heads/develop' 43 | permissions: 44 | pages: write 45 | id-token: write 46 | environment: 47 | name: github-pages 48 | url: ${{ steps.deployment.outputs.page_url }} 49 | runs-on: ubuntu-24.04 50 | steps: 51 | - name: Deploy to GitHub Pages 52 | id: deployment 53 | uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e 54 | -------------------------------------------------------------------------------- /.github/workflows/label.yml: -------------------------------------------------------------------------------- 1 | #----------------------------------------------------------------------- 2 | # DO NOT modify unless you really know what you are doing. 3 | # 4 | # See https://stackoverflow.com/a/74959635 for more info. 5 | # Talk to @alecbcs if you have questions/are not sure of a change's 6 | # possible impact to security. 7 | #----------------------------------------------------------------------- 8 | name: label 9 | on: 10 | pull_request_target: 11 | branches: [develop] 12 | jobs: 13 | pr: 14 | runs-on: ubuntu-24.04 15 | permissions: 16 | contents: read 17 | pull-requests: write 18 | steps: 19 | - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b 20 | -------------------------------------------------------------------------------- /.github/workflows/license.yml: -------------------------------------------------------------------------------- 1 | name: License Checks 2 | on: 3 | workflow_call: 4 | jobs: 5 | verify-license: 6 | runs-on: ubuntu-24.04 7 | steps: 8 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 9 | - name: Set up Python 3.11 10 | uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c 11 | with: 12 | python-version: '3.11' 13 | - name: Verify license headers 14 | run: |- 15 | python .github/workflows/bin/license verify 16 | -------------------------------------------------------------------------------- /.github/workflows/nightly.yml: -------------------------------------------------------------------------------- 1 | name: nightly 2 | on: 3 | schedule: 4 | - cron: 0 14 * * * 5 | #------------------------------------------------------------------------ 6 | # Execute a nightly CI run to verify benchmarks continue to build & run 7 | # sucessfully. Will also update the CI buildcache to minimize build 8 | # times in PRs. 9 | #------------------------------------------------------------------------ 10 | jobs: 11 | run: 12 | uses: ./.github/workflows/run.yml 13 | -------------------------------------------------------------------------------- /.github/workflows/requirements/docs.txt: -------------------------------------------------------------------------------- 1 | # docs 2 | sphinx==8.2.3 3 | sphinx-rtd-theme==3.0.2 4 | codespell==2.4.1 5 | pyyaml==6.0.3 6 | pandas==2.3.3 7 | sphinxcontrib-programoutput==0.18 8 | # benchpark analyze -h command 9 | llnl-thicket 10 | matplotlib 11 | # The remaining requirements are from Ramble 12 | pytest 13 | flake8 14 | google-cloud-storage # for gcs fetch test 15 | google-api-core # for gcs fetch error .exceptions 16 | coverage 17 | pre-commit 18 | graphlib-backport;python_version<"3.9" 19 | urllib3==1.26.18;python_version<="3.6" 20 | protobuf;python_version>"3.6" 21 | protobuf==3.19.4;python_version<="3.6" 22 | pyarrow==3.0.0;python_version<="3.6" 23 | google-cloud-bigquery 24 | tqdm 25 | deprecation 26 | deepdiff -------------------------------------------------------------------------------- /.github/workflows/requirements/docstrfmt.toml: -------------------------------------------------------------------------------- 1 | [tool.docstrfmt] 2 | line-length = 88 3 | -------------------------------------------------------------------------------- /.github/workflows/requirements/style.txt: -------------------------------------------------------------------------------- 1 | black==25.9.0 2 | flake8==7.3.0 3 | isort[colors]==7.0.0 4 | codespell==2.4.1 5 | yamlfix==1.19.0 6 | docstrfmt==1.11.1 7 | -------------------------------------------------------------------------------- /.github/workflows/style.yml: -------------------------------------------------------------------------------- 1 | name: Linting & Style Checks 2 | on: 3 | workflow_call: 4 | jobs: 5 | lint: 6 | runs-on: ubuntu-24.04 7 | steps: 8 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 9 | - name: Set up Python 3.11 10 | uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c 11 | with: 12 | python-version: '3.11' 13 | cache: pip 14 | cache-dependency-path: .github/workflows/requirements/style.txt 15 | - name: Install Python dependencies 16 | run: | 17 | pip install -r .github/workflows/requirements/style.txt 18 | - name: Lint and Format Check 19 | run: |- 20 | black --diff --check . 21 | codespell --ignore-words=.codespellignore 22 | isort --check-only . 23 | flake8 . 24 | yamlfix --check --verbose . 25 | docstrfmt -p .github/workflows/requirements/docstrfmt.toml --check --verbose docs/ 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *~ 3 | \#*\# 4 | __pycache__ 5 | _build 6 | spack.lock 7 | .spack-env 8 | .env 9 | .DS_Store 10 | -------------------------------------------------------------------------------- /.gitlab/bin/exit-codes: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import json 4 | import sys 5 | 6 | 7 | def main(results_file): 8 | with open(results_file, "r") as f: 9 | data = json.load(f) 10 | 11 | res = 0 12 | 13 | for experiment in data["experiments"]: 14 | for context in experiment["CONTEXTS"]: 15 | for fom in context["foms"]: 16 | if fom["origin"] == "exit-code": 17 | if int(fom["value"]) != 0: 18 | res += 1 19 | 20 | return res 21 | 22 | if __name__ == "__main__": 23 | f = sys.argv[1] # path to results.latest.json 24 | sys.exit(main(f)) 25 | -------------------------------------------------------------------------------- /.gitlab/utils/cancel-flux.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | 4 | NO_CLEAN=false 5 | if [[ "$1" == "--no-clean" ]]; then 6 | NO_CLEAN=true 7 | fi 8 | 9 | export URI=$(flux jobs -o "{id} {name}" | grep ${ALLOC_NAME}${GPUMODE} | awk '{print $1}') 10 | ([[ -n "${URI}" ]] && flux cancel ${URI} || exit 0) 11 | 12 | if ! $NO_CLEAN; then 13 | rm -rf $CUSTOM_CI_BUILDS_DIR 14 | fi 15 | -------------------------------------------------------------------------------- /.gitlab/utils/cancel-slurm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | 4 | NO_CLEAN=false 5 | if [[ "$1" == "--no-clean" ]]; then 6 | NO_CLEAN=true 7 | fi 8 | 9 | export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A) 10 | ([[ -n "${JOBID}" ]] && scancel ${JOBID} || exit 0) 11 | 12 | if ! $NO_CLEAN; then 13 | rm -rf $CUSTOM_CI_BUILDS_DIR 14 | fi 15 | -------------------------------------------------------------------------------- /.gitlab/utils/machine_checks.yml: -------------------------------------------------------------------------------- 1 | .machine-check: 2 | tags: [shell, oslic] 3 | variables: 4 | GIT_STRATEGY: none 5 | script: 6 | - | 7 | if [[ $(jq '.[env.CI_MACHINE].total_nodes_up' /usr/global/tools/lorenz/data/loginnodeStatus) == 0 ]] 8 | then 9 | echo -e "\e[31mNo node available on ${CI_MACHINE}\e[0m" 10 | curl --url "https://api.github.com/repos/${GITHUB_PROJECT_ORG}/${GITHUB_PROJECT_NAME}/statuses/${CI_COMMIT_SHA}" \ 11 | --header 'Content-Type: application/json' \ 12 | --header "authorization: Bearer ${GITHUB_TOKEN}" \ 13 | --data "{ \"state\": \"failure\", \"target_url\": \"${CI_PIPELINE_URL}\", \"description\": \"GitLab ${CI_MACHINE} down\", \"context\": \"ci/gitlab/${CI_MACHINE}\" }" 14 | exit 1 15 | fi 16 | tioga-up-check: 17 | stage: machine-checks 18 | variables: 19 | CI_MACHINE: tioga 20 | extends: [.machine-check] 21 | tuolumne-up-check: 22 | stage: machine-checks 23 | variables: 24 | CI_MACHINE: tuolumne 25 | extends: [.machine-check] 26 | dane-up-check: 27 | stage: machine-checks 28 | variables: 29 | CI_MACHINE: dane 30 | extends: [.machine-check] 31 | matrix-up-check: 32 | stage: machine-checks 33 | variables: 34 | CI_MACHINE: matrix 35 | extends: [.machine-check] 36 | -------------------------------------------------------------------------------- /.gitlab/utils/rules.yml: -------------------------------------------------------------------------------- 1 | .on_host_template: 2 | rules: 3 | - if: $CI_PIPELINE_SOURCE == "schedule" || $ADVANCED_JOB == "ON" && $CI_COMMIT_BRANCH 4 | != "main" && $CI_COMMIT_BRANCH != "develop" && $ALL_TARGETS != "ON" 5 | when: never 6 | - if: $CI_JOB_NAME =~ /release_resources|cleanup_pipeline_dir/ && $CI_PIPELINE_SOURCE 7 | != "schedule" 8 | when: always 9 | - when: on_success 10 | - changes: 11 | - .gitlab-ci.yml 12 | - .gitlab/** 13 | - bin/** 14 | - experiments/$BENCHMARK/** 15 | - systems/$ARCHCONFIG/** 16 | - repo/$BENCHMARK/** 17 | - modifiers/** 18 | - var/** 19 | - lib/** 20 | .nightly_rules: 21 | rules: 22 | - if: $CI_PIPELINE_SOURCE == "schedule" 23 | when: always 24 | - if: $CI_PIPELINE_SOURCE != "schedule" 25 | when: never 26 | -------------------------------------------------------------------------------- /.gitlab/utils/run-experiment.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # Activate Virtual Environment 5 | . /usr/workspace/benchpark-dev/benchpark-venv/$SYS_TYPE/bin/activate 6 | 7 | echo "./bin/benchpark system init --dest=${HOST} ${ARCHCONFIG} $([ "$HOST" != "matrix" ] && echo "cluster=$HOST") $SYSTEM_ARGS" 8 | echo "./bin/benchpark experiment init --dest=${BENCHMARK} --system=${HOST} ${BENCHMARK} ${VARIANT}" 9 | echo "./bin/benchpark setup ${BENCHMARK} wkp/" 10 | echo ". wkp/setup.sh" 11 | echo "cd ./wkp/${BENCHMARK}/${HOST}/workspace/" 12 | echo "ramble --disable-logger --workspace-dir . workspace setup" 13 | echo "ramble --disable-logger --workspace-dir . on --executor '{execute_experiment}' --where '{n_nodes} == 1'" 14 | echo "ramble --disable-logger --workspace-dir . workspace analyze --format json yaml text" 15 | 16 | # Initialize System 17 | ./bin/benchpark system init --dest=${HOST} ${ARCHCONFIG} $([ "$HOST" != "matrix" ] && echo "cluster=$HOST") $SYSTEM_ARGS 18 | 19 | # Initialize Experiment 20 | BV="" 21 | if [[ -n "$BENCHMARK_VERSION" ]]; then 22 | BV="version=$BENCHMARK_VERSION" 23 | fi 24 | ./bin/benchpark experiment init --dest=${BENCHMARK} --system=${HOST} ${BENCHMARK} ${VARIANT} ${BV} 25 | 26 | # Build Workspace 27 | ./bin/benchpark setup ${BENCHMARK} wkp/ 28 | 29 | # Setup Ramble & Spack 30 | . wkp/setup.sh 31 | 32 | # Setup Workspace 33 | cd ./wkp/${BENCHMARK}/${HOST}/workspace/ 34 | 35 | ramble --disable-logger --workspace-dir . workspace setup 36 | 37 | # Using flux on dane (srun called in "ramble on") 38 | if [ "$HOST" == "dane" ] && \ 39 | # Nightly testing still using slurm 40 | [ $CI_PIPELINE_SOURCE != "schedule" ]; then 41 | find . -type f -name execute_experiment -exec sed -i 's/\bsrun\b/flux run --exclusive/g' {} + 42 | fi 43 | 44 | # Runs experiments where n_nodes == 1, and Print Log 45 | ramble --disable-logger --workspace-dir . on --executor '{execute_experiment}' --where '{n_nodes} == 1' 46 | find experiments/ -type f -name "*.out" -exec cat {} + 47 | 48 | # Analyze Experiments 49 | ramble --disable-logger --workspace-dir . workspace analyze --format json yaml text 50 | 51 | cd - 52 | 53 | # Test 'benchpark analyze' 54 | if [[ "$TEST_ANALYZE" == "true" ]]; then 55 | ./bin/benchpark analyze --workspace-dir ./wkp/${BENCHMARK}/${HOST}/workspace/ 56 | fi 57 | 58 | # Check Experiment Exit Codes 59 | python ./.gitlab/bin/exit-codes ./wkp/${BENCHMARK}/${HOST}/workspace/results.latest.json -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | # If you are referencing Benchpark in a publication, please cite the paper 2 | # described here. 3 | # 4 | # Here's the raw citation: 5 | # 6 | # Olga Pearce, Alec Scott, Gregory Becker, Riyaz Haque, Nathan Hanford, Stephanie Brink, 7 | # Doug Jacobsen, Heidi Poxon, Jens Domke, and Todd Gamblin. 2023. 8 | # Towards Collaborative Continuous Benchmarking for HPC. 9 | # In Workshops of The International Conference on High Performance Computing, 10 | # Network, Storage, and Analysis (SC-W 2023), November 12–17, 2023, Denver, CO, USA. 11 | # ACM, New York, NY, USA, 9 pages. 12 | # 13 | # And here's the CITATION.cff format: 14 | 15 | cff-version: 1.2.0 16 | title: "Benchpark" 17 | message: "If you are referencing Benchpark in a publication, please cite the paper below." 18 | repository-code: "https://github.com/llnl/benchpark" 19 | preferred-citation: 20 | type: conference-paper 21 | doi: 10.1145/3624062.3624135 22 | url: https://doi.org/10.1145/3624062.3624135 23 | authors: 24 | - family-names: "Pearce" 25 | given-names: "Olga" 26 | - family-names: "Scott" 27 | given-names: "Alec" 28 | - family-names: "Becker" 29 | given-names: "Gregory" 30 | - family-names: "Haque" 31 | given-names: "Riyaz" 32 | - family-names: "Hanford" 33 | given-names: "Nathan" 34 | - family-names: "Brink" 35 | given-names: "Stephanie" 36 | - family-names: "Jacobsen" 37 | given-names: "Doug" 38 | - family-names: "Poxon" 39 | given-names: "Heidi" 40 | - family-names: "Domke" 41 | given-names: "Jens" 42 | - family-names: "Gamblin" 43 | given-names: "Todd" 44 | title: "Towards Collaborative Continuous Benchmarking for HPC" 45 | conference: 46 | name: "Supercomputing 2023 (SC’23)" 47 | city: "Denver" 48 | region: "Colorado" 49 | country: "US" 50 | date-start: 2023-11-12 51 | date-end: 2023-11-17 52 | month: 11 53 | year: 2023 54 | notes: LLNL-CONF-853344 55 | authors: 56 | - family-names: "Pearce" 57 | given-names: "Olga" 58 | - family-names: "Scott" 59 | given-names: "Alec" 60 | - family-names: "Becker" 61 | given-names: "Gregory" 62 | - family-names: "Haque" 63 | given-names: "Riyaz" 64 | - family-names: "Hanford" 65 | given-names: "Nathan" 66 | - family-names: "Brink" 67 | given-names: "Stephanie" 68 | - family-names: "Jacobsen" 69 | given-names: "Doug" 70 | - family-names: "Poxon" 71 | given-names: "Heidi" 72 | - family-names: "Domke" 73 | given-names: "Jens" 74 | - family-names: "Gamblin" 75 | given-names: "Todd" 76 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.10) 2 | project(PythonTestProject LANGUAGES NONE) 3 | ENABLE_TESTING() 4 | INCLUDE(CTest) 5 | find_package(Python3 REQUIRED) 6 | 7 | add_test(NAME Gitlab 8 | COMMAND ${CMAKE_COMMAND} -E $ENV{EXIT_CODE}) 9 | -------------------------------------------------------------------------------- /COPYRIGHT: -------------------------------------------------------------------------------- 1 | Intellectual Property Notice 2 | ------------------------------ 3 | 4 | Benchpark is licensed under the Apache License, Version 2.0 (LICENSE-APACHE 5 | or http://www.apache.org/licenses/LICENSE-2.0). 6 | 7 | Copyrights and patents in the Benchpark project are retained by contributors. 8 | No copyright assignment is required to contribute to Benchpark. 9 | 10 | Benchpark is derived from Ramble and Spack. 11 | Ramble: https://github.com/GoogleCloudPlatform/ramble 12 | Spack: https://github.com/spack/spack 13 | 14 | SPDX usage 15 | ------------ 16 | 17 | Individual files contain SPDX tags instead of the full license text. 18 | This enables machine processing of license information based on the SPDX 19 | License Identifiers that are available here: https://spdx.org/licenses/ 20 | -------------------------------------------------------------------------------- /CTestConfig.cmake: -------------------------------------------------------------------------------- 1 | ## This file should be placed in the root directory of your project. 2 | ## Then modify the CMakeLists.txt file in the root directory of your 3 | ## project to incorporate the testing dashboard. 4 | ## 5 | ## The following are required to submit to the CDash dashboard: 6 | #ENABLE_TESTING() 7 | #INCLUDE(CTest) 8 | 9 | set(CTEST_PROJECT_NAME benchpark) 10 | set(CTEST_NIGHTLY_START_TIME 01:00:00 UTC) 11 | 12 | if(CMAKE_VERSION VERSION_GREATER 3.14) 13 | set(CTEST_SUBMIT_URL https://my.cdash.org/submit.php?project=Benchpark) 14 | else() 15 | set(CTEST_DROP_METHOD "https") 16 | set(CTEST_DROP_SITE "my.cdash.org") 17 | set(CTEST_DROP_LOCATION "/submit.php?project=Benchpark") 18 | endif() 19 | 20 | set(CTEST_DROP_SITE_CDASH TRUE) 21 | 22 | # CDash token expected in ENV 23 | set(_auth_token $ENV{CDASH_AUTH_TOKEN}) 24 | -------------------------------------------------------------------------------- /CTestGitlab.cmake: -------------------------------------------------------------------------------- 1 | set(CTEST_SOURCE_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}") 2 | set(CTEST_BINARY_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}") 3 | set(CTEST_OUTPUT_ON_FAILURE ON) 4 | set(CTEST_CMAKE_GENERATOR "Unix Makefiles") 5 | 6 | set(CTEST_UPDATE_COMMAND "git") 7 | set(CTEST_UPDATE_VERSION_ONLY 1) 8 | 9 | set(CTEST_SITE ${SITE}) 10 | set(CTEST_BUILD_NAME ${BUILD_NAME}) 11 | 12 | # Separate dashboards for Nightly testing and PRs 13 | if ("${TEST_TYPE}" STREQUAL "Nightly") 14 | ctest_start("Nightly" GROUP "${DASHBOARD_NAME}") 15 | else() 16 | ctest_start("Continuous" GROUP "${DASHBOARD_NAME}") 17 | endif() 18 | 19 | ctest_update() 20 | ctest_configure() 21 | ctest_test(INCLUDE "Gitlab") 22 | 23 | # Submit results 24 | ctest_submit( 25 | PARTS Update Test # 'Configure' and 'Build' not uploaded 26 | HTTPHEADER "Authorization: Bearer ${_auth_token}" 27 | ) 28 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | This work was produced under the auspices of the U.S. Department of 2 | Energy by Lawrence Livermore National Laboratory under Contract 3 | DE-AC52-07NA27344. 4 | 5 | This work was prepared as an account of work sponsored by an agency of 6 | the United States Government. Neither the United States Government nor 7 | Lawrence Livermore National Security, LLC, nor any of their employees 8 | makes any warranty, expressed or implied, or assumes any legal liability 9 | or responsibility for the accuracy, completeness, or usefulness of any 10 | information, apparatus, product, or process disclosed, or represents that 11 | its use would not infringe privately owned rights. 12 | 13 | Reference herein to any specific commercial product, process, or service 14 | by trade name, trademark, manufacturer, or otherwise does not necessarily 15 | constitute or imply its endorsement, recommendation, or favoring by the 16 | United States Government or Lawrence Livermore National Security, LLC. 17 | 18 | The views and opinions of authors expressed herein do not necessarily 19 | state or reflect those of the United States Government or Lawrence 20 | Livermore National Security, LLC, and shall not be used for advertising 21 | or product endorsement purposes. 22 | -------------------------------------------------------------------------------- /bin/benchpark: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 4 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 5 | # 6 | # SPDX-License-Identifier: Apache-2.0 7 | 8 | import os 9 | import pathlib 10 | import subprocess 11 | import sys 12 | import uuid 13 | 14 | 15 | def main(): 16 | if sys.version_info[0] < 3 or sys.version_info[1] < 8: 17 | raise Exception("\n\nERR: Must be using Python 3.8 or later!\n") 18 | basedir = pathlib.Path(__file__).resolve().parents[1] 19 | main_py = basedir / "lib" / "main.py" 20 | 21 | if os.environ.get("BENCHPARK_RUN_COVERAGE"): 22 | # Create a unique coverage data file name 23 | coverage_dir = ( 24 | basedir / f"coverage-data-{os.environ.get('BENCHPARK_RUN_COVERAGE')}" 25 | ) 26 | coverage_dir.mkdir(exist_ok=True, parents=True) 27 | unique_id = uuid.uuid4().hex 28 | coverage_file = coverage_dir / f".coverage.{unique_id}" 29 | # Run the code with coverage, writing to the unique file 30 | subprocess.run( 31 | [ 32 | sys.executable, 33 | "-m", 34 | "coverage", 35 | "run", 36 | f"--data-file={coverage_file}", 37 | "--source=lib", 38 | main_py, 39 | ] 40 | + sys.argv[1:], 41 | check=True, 42 | ) 43 | else: 44 | subprocess.run([sys.executable, main_py] + sys.argv[1:], check=True) 45 | 46 | 47 | if __name__ == "__main__": 48 | main() 49 | -------------------------------------------------------------------------------- /bin/benchpark-python: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 4 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 5 | # 6 | # SPDX-License-Identifier: Apache-2.0 7 | 8 | # 9 | # benchpark-python 10 | # 11 | # If you want to write your own executable Python script that uses Benchpark 12 | # modules, on Mac OS or maybe some others, you may be able to do it like 13 | # this: 14 | # 15 | # #!/usr/bin/env benchpark-python 16 | # 17 | # This is compatible across platforms. 18 | # 19 | SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) 20 | export PYTHONPATH="${SCRIPT_DIR}/../lib":$PYTHONPATH 21 | # Check if the first argument is -i 22 | if [ "$1" = "-i" ]; then 23 | # Shift the arguments to remove the first one (-i) 24 | shift 25 | exec python3 -i "$@" 26 | else 27 | exec python3 "$@" 28 | fi 29 | -------------------------------------------------------------------------------- /checkout-versions.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | versions: 6 | ramble: 6b4ecac # develop on 5/06/2025 (newer than 0.6.0 release) 7 | spack: 7eeef1f 8 | spack-packages: a1941c9 9 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | round: down 3 | range: 40...100 4 | comment: 5 | layout: header, diff, changes, tree 6 | behavior: new 7 | require_changes: false 8 | require_base: false 9 | require_head: true 10 | after_n_builds: 10 # Number of SEPARATE codecov uploads expected, e.g. "Upload coverage to Codecov" jobs 11 | -------------------------------------------------------------------------------- /common-resources/execute_experiment.tpl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | 7 | {allocation_directives} 8 | 9 | cd {experiment_run_dir} 10 | 11 | {pre_exec} 12 | {command} 13 | {post_exec} 14 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | systemconfigs: 16 | ./generate-sys-defs-list.py 17 | 18 | tags: 19 | ./generate-benchmark-list.py 20 | 21 | .PHONY: help systemconfigs tags Makefile 22 | 23 | # Catch-all target: route all unknown targets to Sphinx using the new 24 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 25 | %: Makefile 26 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 27 | -------------------------------------------------------------------------------- /docs/No-mpi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/No-mpi.png -------------------------------------------------------------------------------- /docs/Tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/Tree.png -------------------------------------------------------------------------------- /docs/_static/css/custom.css: -------------------------------------------------------------------------------- 1 | a { 2 | color: #6a984e 3 | } 4 | 5 | a:hover { 6 | color: #7fa866 7 | } 8 | 9 | a:visited { 10 | color: #9b59b6 11 | } 12 | 13 | .wy-nav-content { 14 | max-width: 1200px !important; 15 | } 16 | 17 | .wy-side-nav-search, .wy-nav-top { 18 | background: #7fa866 19 | } 20 | 21 | .wy-nav .wy-menu-vertical header { 22 | color: #7fa866 23 | } 24 | 25 | .wy-menu-vertical a { 26 | color:#d9d9d9 27 | } 28 | 29 | .wy-nav .wy-menu-vertical a:hover { 30 | background-color: #7fa866 31 | } 32 | 33 | .wy-menu-vertical header, .wy-menu-vertical p.caption { 34 | color: #a1ca7b 35 | } 36 | 37 | .wy-nav-top { 38 | background: #7fa866 39 | } 40 | 41 | .rst-content .wy-alert-neutral.admonition-todo a, .rst-content .wy-alert-neutral.admonition a, .rst-content .wy-alert-neutral.attention a, .rst-content .wy-alert-neutral.caution a, .rst-content .wy-alert-neutral.danger a, .rst-content .wy-alert-neutral.error a, .rst-content .wy-alert-neutral.hint a, .rst-content .wy-alert-neutral.important a, .rst-content .wy-alert-neutral.note a, .rst-content .wy-alert-neutral.seealso a, .rst-content .wy-alert-neutral.tip a, .rst-content .wy-alert-neutral.warning a, .wy-alert.wy-alert-neutral a { 42 | color: #7fa866 43 | } 44 | 45 | .wy-tray-container li.wy-tray-item-info { 46 | background: #7fa866 47 | } 48 | 49 | .btn-info { 50 | background-color: #7fa866 !important 51 | } 52 | 53 | .btn-link { 54 | color: #7fa866; 55 | } 56 | 57 | .wy-dropdown-menu > dd > a:hover { 58 | background: #7fa866 59 | } 60 | 61 | .wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a:hover { 62 | background: #7fa866 63 | } 64 | 65 | .wy-inline-validate.wy-inline-validate-info .wy-input-context { 66 | color: #7fa866 67 | } 68 | 69 | .wy-text-info { 70 | color:#7fa866 !important 71 | } 72 | 73 | .wy-side-nav-search { 74 | background-color: #7fa866 75 | } 76 | 77 | .wy-side-nav-search input[type=text] { 78 | border-color: #7fa866 79 | } 80 | 81 | .wy-side-nav-search img { 82 | background-color: #7fa866 83 | } 84 | .dataTables_wrapper .dataTables_filter { 85 | float: unset; 86 | text-align: right; 87 | } 88 | 89 | html.writer-html4 .rst-content dl:not(.docutils) > dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) > dt { 90 | color: #7fa866 91 | } 92 | 93 | .rst-content div[class^="highlight"] pre { 94 | max-height: 600px; 95 | } 96 | -------------------------------------------------------------------------------- /docs/_static/images/dray-diffspecs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/images/dray-diffspecs.png -------------------------------------------------------------------------------- /docs/_static/images/dryruns.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/images/dryruns.png -------------------------------------------------------------------------------- /docs/_static/images/kripke_cuda_strong_percentage_exc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/images/kripke_cuda_strong_percentage_exc.png -------------------------------------------------------------------------------- /docs/_static/images/kripke_cuda_strong_raw_exc-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/images/kripke_cuda_strong_raw_exc-2.png -------------------------------------------------------------------------------- /docs/_static/images/kripke_cuda_strong_raw_exc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/images/kripke_cuda_strong_raw_exc.png -------------------------------------------------------------------------------- /docs/_static/images/kripke_cuda_strong_raw_inc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/images/kripke_cuda_strong_raw_inc.png -------------------------------------------------------------------------------- /docs/_static/images/kripke_cuda_throughput_raw_exc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/images/kripke_cuda_throughput_raw_exc.png -------------------------------------------------------------------------------- /docs/_static/images/kripke_cuda_weak_raw_exc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/images/kripke_cuda_weak_raw_exc.png -------------------------------------------------------------------------------- /docs/_static/images/quicksilver-diffspecs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/images/quicksilver-diffspecs.png -------------------------------------------------------------------------------- /docs/_static/images/shared-nonshared.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/images/shared-nonshared.png -------------------------------------------------------------------------------- /docs/_static/images/workflow_fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/images/workflow_fig.png -------------------------------------------------------------------------------- /docs/_static/main.js: -------------------------------------------------------------------------------- 1 | $(document).ready( function () { 2 | $('table.datatable').DataTable({paging: false}); 3 | } ); 4 | -------------------------------------------------------------------------------- /docs/_static/qr-codes/benchpark-docs-qrcode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/qr-codes/benchpark-docs-qrcode.png -------------------------------------------------------------------------------- /docs/_static/qr-codes/benchpark-github-qrcode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/qr-codes/benchpark-github-qrcode.png -------------------------------------------------------------------------------- /docs/_static/slides/HPCIC25-Tutorial-Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/slides/HPCIC25-Tutorial-Slides.pdf -------------------------------------------------------------------------------- /docs/_static/slides/ReproduciblePerfAnalysis-HPDC25-Tutorial-Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/slides/ReproduciblePerfAnalysis-HPDC25-Tutorial-Slides.pdf -------------------------------------------------------------------------------- /docs/_static/slides/eScience25-Tutorial-Slides.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/_static/slides/eScience25-Tutorial-Slides.pdf -------------------------------------------------------------------------------- /docs/add-a-benchmark.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Adding a Benchmark 8 | ================== 9 | 10 | This guide is intended for application developers who want to add a benchmark so that it 11 | can be run with Benchpark. 12 | 13 | Create a New Benchmark 14 | ---------------------- 15 | 16 | The following system-independent specification is required for each ${Benchmark1}: 17 | 18 | - ``package.py`` is a Spack specification that defines how to build and install 19 | ${Benchmark1}. See the `Spack Package Creation Tutorial 20 | `_ to learn 21 | how to create and test a Spack ``package.py``. If your benchmark already exists in 22 | Spack, benchpark will use that version of the ``package.py``, unless you define a 23 | version in ``benchpark/repo/``. 24 | - ``application.py`` is a Ramble specification that defines the ${Benchmark1} input and 25 | parameters. See the `Ramble Application Definition Developers Guide 26 | `_ to 27 | learn how to create a Ramble ``application.py``. We recommend testing your 28 | ``application.py`` using the `Ramble Usage While Developing 29 | `_ 30 | instructions. If your benchmark already exists in Ramble, benchpark will use that 31 | version of the ``application.py``, unless you define a version in ``benchpark/repo/``. 32 | 33 | Again, by default Benchpark will use ${Benchmark1} specifications (``application.py`` 34 | and ``package.py``) provided in the Spack and Ramble upstream repositories. Overwrite 35 | the upstream definitions by adding the ``application.py`` and/or ``package.py`` to 36 | ``benchpark/repo/${Benchmark1}``, see :doc:`FAQ` for details. 37 | 38 | After satisfying the above prerequisites, in order to use your benchmark in Benchpark, 39 | you will need to create an experiment as described in :doc:`add-an-experiment`. 40 | -------------------------------------------------------------------------------- /docs/analyze-experiment.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Experiment pass/fail 8 | ==================== 9 | 10 | Once the experiments completed running, the command: 11 | 12 | :: 13 | 14 | ramble --workspace-dir . workspace analyze 15 | 16 | can be used to analyze figures of merit and evaluate `success/failure 17 | `_ of 18 | the experiments. Ramble generates a file with summary of the results in ``$workspace``. 19 | -------------------------------------------------------------------------------- /docs/benchmark-list.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Benchmarks and Experiments 8 | ========================== 9 | 10 | .. csv-table:: Current Benchpark tags by Benchmark and tag groups. 11 | :file: benchmark-list.csv 12 | :header-rows: 1 13 | :align: left 14 | -------------------------------------------------------------------------------- /docs/benchpark-workflow.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Benchpark Workflow 8 | ================== 9 | 10 | Follow the workflow diagram to determine if you can run existing configurations or if 11 | you will need to add or edit any Benchpark files. 12 | 13 | .. image:: /_static/images/workflow_fig.png 14 | 15 | A. Run Existing Systems, Benchmarks and Experiments 16 | --------------------------------------------------- 17 | 18 | If you are running on an existing ``system``, and the ``benchmark`` and ``experiment`` 19 | are already configured, you can proceed directly to the Running Benchpark Steps, 20 | starting with :doc:`benchpark-setup` for: 21 | 22 | - Setup / Run / Analyze 23 | 24 | B. System Specification 25 | ----------------------- 26 | 27 | A system specification defines the hardware, scheduling system, compilers, and any 28 | external libraries that might exist on a system. 29 | 30 | If you are running on a new system that has not been defined in Benchpark yet, proceed 31 | to :doc:`add-a-system-config` for the following: 32 | 33 | - Find a similar ``system`` 34 | - Add or edit a ``system`` 35 | 36 | C. Benchmark Specification 37 | -------------------------- 38 | 39 | A benchmark specification defines the application build and run behavior, utilizing 40 | spack for build and dependency management. 41 | 42 | If you are creating a new benchmark that has not been defined in Benchpark yet, proceed 43 | to :doc:`add-a-benchmark` for the following: 44 | 45 | - Add a ``benchmark`` 46 | 47 | D. Experiment Specification 48 | --------------------------- 49 | 50 | An experiment specification defines application parameters for experiments that require 51 | one or more application runs, such as single-node, scaling, or throughput runs. 52 | 53 | If you are adding experiments to a new or existing benchmark, proceed to 54 | :doc:`add-an-experiment` for the following: 55 | 56 | - Add/edit an ``experiment`` 57 | -------------------------------------------------------------------------------- /docs/build-experiment.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Build an Experiment 8 | =================== 9 | 10 | ``benchpark setup`` has created the directory structure for your experiment. The next 11 | step is setting up the Ramble workspace and building the code: 12 | 13 | :: 14 | 15 | cd ///workspace 16 | ramble --workspace-dir . workspace setup 17 | 18 | Ramble will build the source code and set up the following workspace directory 19 | structure: 20 | 21 | :: 22 | 23 | experiments_root/ 24 | ramble/ 25 | spack/ 26 | / 27 | / 28 | workspace/ 29 | configs/ 30 | (everything from system_def_dir) 31 | (everything from experiment_def_dir) 32 | experiments/ 33 | / 34 | / 35 | __ 36 | execute_experiment 37 | 38 | If you edit any of the files, see :doc:`FAQ` to determine whether you need to re-do any 39 | of the previous steps. 40 | 41 | Once that is complete, you are ready for :doc:`run-experiment`. 42 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | # -- Project information ----------------------------------------------------- 7 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 8 | 9 | import os 10 | import subprocess 11 | import sys 12 | 13 | subprocess.call( 14 | [ 15 | "make", 16 | "systemconfigs", 17 | ] 18 | ) 19 | 20 | subprocess.call( 21 | [ 22 | "make", 23 | "tags", 24 | ] 25 | ) 26 | 27 | project = "Benchpark" 28 | copyright = "2023, LLNS LLC" 29 | author = "Olga Pearce, Alec Scott, Peter Scheibel, Greg Becker, Riyaz Haque, and Nathan Hanford" 30 | 31 | # -- General configuration --------------------------------------------------- 32 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 33 | 34 | extensions = [ 35 | "sphinx_rtd_theme", 36 | "sphinxcontrib.programoutput", 37 | "sphinx.ext.autodoc", 38 | "sphinx.ext.napoleon", 39 | ] 40 | 41 | sys.path.insert(0, os.path.abspath("../lib")) 42 | 43 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".spack-env"] 44 | 45 | # -- Options for HTML output ------------------------------------------------- 46 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 47 | 48 | html_static_path = ["_static"] 49 | html_js_files = [ 50 | "https://cdn.datatables.net/1.10.23/js/jquery.dataTables.min.js", 51 | "main.js", 52 | ] 53 | html_css_files = [ 54 | ( 55 | "https://cdn.datatables.net/1.10.23/css/jquery.dataTables.min.css", 56 | {"priority": 800}, 57 | ), 58 | ("css/custom.css", {"priority": 999}), 59 | ] 60 | html_logo = "_static/images/benchpark-dark.svg" 61 | html_theme = "sphinx_rtd_theme" 62 | html_theme_options = {"logo_only": True} 63 | pygments_style = "sphinx" 64 | -------------------------------------------------------------------------------- /docs/create-mirror.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | System Mirror 8 | ============= 9 | 10 | If you build a benchmark on a networked system, you can use `benchpark mirror` to create 11 | a directory that bundles all necessary resources to install and run that benchmark on 12 | another system, such that the destination system does not need network access. 13 | 14 | On the networked system, if you created/built the benchmark with: 15 | 16 | :: 17 | 18 | benchpark system init --dest=def-ruby llnl-cluster cluster=ruby compiler=gcc 19 | benchpark experiment init --dest=def-raja-perf --system=def-ruby raja-perf 20 | benchpark setup def-raja-perf/ workspace/ 21 | . `pwd`/workspace/setup.sh 22 | ramble --workspace-dir `pwd`/workspace/def-raja-perf/def-ruby/workspace workspace setup 23 | 24 | You can then create a directory that bundles all the resources needed to build that 25 | benchmark with: 26 | 27 | :: 28 | 29 | benchpark mirror create `pwd`/workspace/def-raja-perf/def-ruby/workspace/ test-benchpark-mirror/ 30 | 31 | You can copy `test-bencpark-mirror/` to another system, and on that system, within that 32 | directory you can do: 33 | 34 | :: 35 | 36 | python3 -m venv mirror-env && . mirror-env/bin/activate 37 | pip install --no-index --find-links=pip-cache pip-cache/* 38 | bash first-time.sh 39 | . `pwd`/setup.sh 40 | ramble --workspace-dir `pwd`/def-raja-perf/def-ruby/workspace/ workspace setup 41 | 42 | this will install the benchmark on the new system, and also configure Ramble to use 43 | mirror resources that were bundled in `test-benchmark-mirror/` (so it does not need 44 | internet access to build the benchmark). 45 | 46 | Limitations 47 | ----------- 48 | 49 | For now, benchpark can only create mirrors that are useful for destination systems that 50 | match the host system in terms of: 51 | 52 | - available compilers 53 | - provided external software 54 | 55 | Also, `benchpark mirror` can only create mirrors for benchmarks that have been built on 56 | the source system. 57 | -------------------------------------------------------------------------------- /docs/developer-guide.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Developer Guide 8 | =============== 9 | 10 | This guide is intended for people who want to work on Benchpark itself. 11 | 12 | Overview 13 | -------- 14 | 15 | Benchpark is designed with several roles in mind: 16 | 17 | 1. **Users**, who want to install, run, and analyze performance of HPC benchmarks 18 | 2. **Application Developers**, who want to share their benchmarks. 19 | 3. **Procurement Teams**, who curate workload representation, evaluate and monitor 20 | system progress at HPC centers. 21 | 4. **HPC Vendors**, who understand the curated workload of HPC centers, propose systems. 22 | 5. **Benchpark Developers**, who work on Benchpark, add new features, and try to make 23 | the jobs of benchmark developers and users easier. 24 | 25 | This gets us to the key concepts in Benchpark's software design: 26 | 27 | - Specs: expressions for describing experiments and compute systems 28 | - Packages: Python modules that build benchmarks according to a spec. 29 | 30 | Directory Structure 31 | ------------------- 32 | 33 | So that you can familiarize yourself with the project, we will start with a high-level 34 | view of Benchpark's directory structure: 35 | 36 | .. code-block:: none 37 | 38 | benchpark/ 39 | bin/ 40 | benchpark <- main benchpark executable 41 | benchpark-python <- execute python scripts using benchpark library 42 | 43 | docs/ <- source for this documentation 44 | 45 | experiments/ <- experiment specs 46 | 47 | lib/ 48 | benchpark/ <- benchpark library 49 | scripts/ <- scripts for common benchpark use cases 50 | 51 | modifiers/ <- modifier definitions 52 | 53 | repo/ <- benchmarks are defined here 54 | **/application.py <- ramble application spec 55 | **/package.py <- spack package spec 56 | 57 | systems/ <- system specs 58 | 59 | Updating Documentation 60 | ---------------------- 61 | 62 | To build the documentation, requirements can be easily installed from 63 | ``.github/workflows/requirements/docs.txt``, using: 64 | 65 | .. code-block:: bash 66 | 67 | pip install -r .github/workflows/requirements/docs.txt 68 | 69 | This requires ``python>=3.11`` (or try an earlier version of sphinx). After updating the 70 | documentation, render the pages with the following: 71 | 72 | .. code-block:: bash 73 | 74 | cd docs 75 | make html 76 | 77 | Then, open ``_build/html/index.html`` in a browser to view the rendered documentation. 78 | -------------------------------------------------------------------------------- /docs/diffBuildSpecs.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Compare Build Specs 8 | =================== 9 | 10 | ``lib/scripts/diffBuildSpecs.py`` 11 | 12 | Compare two experiment build specs (dependence graphs with versions of dependencies), 13 | generated by Spack for specified packages+options. This script enables the user to 14 | compare packages and versions of two spack builds. Given two yaml package specs (use 15 | ``spack spec --yaml``), the script outputs which components of the packages are 16 | different. 17 | 18 | .. note:: 19 | 20 | If you are trying to compare benchpark benchmarks, try using 21 | ``diffExperimentBuilds.py``. 22 | 23 | Example: dray with/without MPI 24 | ------------------------------ 25 | 26 | In this example, we see the difference of ``dray`` built with and without ``mpi``. The 27 | difference between the specs ``dray+mpi`` and ``dray~mpi`` is indicated in the output by 28 | ``-> [openmpi]`` and the console output highlights the package differences in red. 29 | 30 | .. code-block:: console 31 | 32 | $ spack spec --yaml dray+mpi > dray-mpi.yaml 33 | $ spack spec --yaml dray~mpi > dray-nompi.yaml 34 | $ spack-python lib/scripts/diffBuildSpecs.py ./dray-mpi.yaml ./dray-nompi.yaml 35 | 36 | .. figure:: /_static/images/dray-diffspecs.png 37 | :align: center 38 | 39 | (Image output is truncated) 40 | 41 | .. note:: 42 | 43 | ``spack-python`` is required to import the spack libraries needed for this script. 44 | It will automatically be added to your ``$PATH`` when you run ``benchpark setup 45 | ...``. 46 | 47 | Example: Installed packages 48 | --------------------------- 49 | 50 | If you want to compare already installed packages, provide the hashes 51 | 52 | .. code-block:: console 53 | 54 | $ spack find -L quicksilver 55 | 56 | -- linux-rhel8-sapphirerapids / gcc@12.1.1 ---------------------- 57 | fubnce7wzgjxhkim2cylijt4cbpfhxi6 quicksilver@master 58 | 59 | -- linux-rhel8-sapphirerapids / intel@2021.6.0-classic ---------- 60 | qwev4yodp2joikf2oxvlo224ksjcqve3 quicksilver@master 61 | ==> 2 installed packages 62 | 63 | $ spack-python lib/scripts/diffBuildSpecs.py quicksilver/fubnce7wzgjxhkim2cylijt4cbpfhxi6 quicksilver/qwev4yodp2joikf2oxvlo224ksjcqve3 64 | 65 | .. figure:: /_static/images/quicksilver-diffspecs.png 66 | :align: center 67 | -------------------------------------------------------------------------------- /docs/diffExperimentSpecs.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Compare Experiment Specs 8 | ======================== 9 | 10 | ``lib/scripts/diffExperimentSpecs.py`` 11 | 12 | Compare two experiment specs `.yaml` files generated by ``benchpark experiment init``. 13 | This script enables the user to compare changes to a ``experiment.py`` between commits 14 | by comparing ``.yaml`` files generated from ``benchpark experiment init ...``. This 15 | script is helpful when it is unclear if changes made to ``experiment.py`` will affect 16 | the resulting experiment spec. 17 | 18 | Stages that occur during this script: 19 | - ``benchpark experiment init`` 20 | 21 | Example: Change n_ranks in Saxpy 22 | -------------------------------- 23 | 24 | In this example, we modify the ``saxpy+openmp`` experiment to see how the script 25 | identifies the differences. On the branch ``myBranch``, we change 26 | ``self.add_experiment_variable("n_ranks", "8")`` to 27 | ``self.add_experiment_variable("n_ranks", "4")``. 28 | 29 | .. code-block:: console 30 | 31 | $ benchpark-python diffExperimentSpecs.py -n myBranch -e saxpy+openmp 32 | 33 | saxpy+openmp 34 | saxpy/openmp/ramble.yaml 35 | The YAML files benchpark-old/saxpy/openmp/ramble.yaml and benchpark-new/saxpy/openmp/ramble.yaml are different. Here are the differences: 36 | {'values_changed': {"root['ramble']['applications']['saxpy']['workloads']['problem']['experiments']['saxpy_problem_single_node_openmp_caliper_none_{n_nodes}_{n_threads_per_proc}_{n}']['variables']['n_ranks']": {'new_value': 4, 37 | 'old_value': 8}}} 38 | -------------------------------------------------------------------------------- /docs/diffPackageCommits.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Compare Package Commits 8 | ======================= 9 | 10 | ``lib/scripts/diffPackageCommits.py`` 11 | 12 | Compare a commit of a Spack package in Benchpark/repo with the package upstreamed to 13 | Spack. If the comparison of the ``package.py`` in ``benchpark/repo/`` is identical to 14 | the ``package.py`` in ``spack/var/spack/repos/builtin/packages/``, ``package.py`` in 15 | ``benchpark/repo/`` can be safely deleted without changing how the benchmark is built; 16 | this scenario occurs if ``package.py`` has been upstreamed to Spack. This script runs in 17 | the benchpark CI and will fail if a package should be deleted in benchpark. 18 | 19 | Example: amg2023 and raja-perf 20 | ------------------------------ 21 | 22 | In this example, we made ``benchpark/repo/amg2023/package.py`` the same as the spack 23 | ``amg2023/package.py`` and equivalently ``benchpark/repo/raja-perf/package.py`` the same 24 | as spack ``raja-perf/package.py``. 25 | 26 | .. code-block:: console 27 | 28 | $ benchpark-python diffPackageCommits.py --packages amg2023 raja-perf 29 | 30 | Comparing benchpark packages to packages in spack develop 31 | amg2023 32 | No differences found. Please delete 'benchpark/repo/amg2023/package.py' (use spack upstream) 33 | 0 different lines 34 | raja-perf 35 | No differences found. Please delete 'benchpark/repo/raja-perf/package.py' (use spack upstream) 36 | 0 different lines 37 | -------------------------------------------------------------------------------- /docs/examples/compare_experiment_builds/compareExperimentBuilds.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Run from benchpark root '. docs/examples/compare_experiment_builds/compareExperimentBuilds.sh' 3 | 4 | compilers=("gcc12" "intel") 5 | scaling=("weak") 6 | . setup-env.sh 7 | rm -rf daneGCC 8 | rm -rf daneIntel 9 | rm -rf quicksilvergcc* 10 | benchpark system init --dest=daneGCC llnl-cluster cluster=dane compiler=gcc 11 | benchpark system init --dest=daneIntel llnl-cluster cluster=dane compiler=intel 12 | # Set up all experiments 13 | for runNum in {1..3} 14 | do 15 | for scale in ${scaling[@]} 16 | do 17 | for i in ${compilers[@]} 18 | do 19 | echo $i $scale 20 | # Setup specific experiment 21 | benchpark experiment init --dest=quicksilver$i$scale$runNum quicksilver +$scale +openmp caliper=mpi 22 | if [ "$i" == "gcc12" ]; then 23 | benchpark setup quicksilver$i$scale$runNum workspace 24 | . workspace/setup.sh 25 | ramble --workspace-dir workspace/quicksilver$i$scale$runNum/daneGCC/workspace workspace setup 26 | ramble --workspace-dir workspace/quicksilver$i$scale$runNum/daneGCC/workspace on 27 | else 28 | benchpark setup quicksilver$i$scale$runNum workspace 29 | . workspace/setup.sh 30 | ramble --workspace-dir workspace/quicksilver$i$scale$runNum/daneIntel/workspace workspace setup 31 | ramble --workspace-dir workspace/quicksilver$i$scale$runNum/daneIntel/workspace on 32 | fi 33 | done 34 | done 35 | done 36 | -------------------------------------------------------------------------------- /docs/experiments_root_structure.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | For each ``experiment`` (``benchmark`` x ``ProgrammingModel`` x ), Ramble sets up the 8 | following ``workspace`` directory structure to build and run the experiment: 9 | 10 | :: 11 | 12 | experiments_root/ 13 | ramble/ 14 | spack/ 15 | / 16 | / 17 | / 18 | workspace/ 19 | configs/ 20 | (everything from system_def_dir) 21 | (everything from experiment_def_dir) 22 | experiments/ 23 | / 24 | / 25 | __ 26 | execute_experiment 27 | -------------------------------------------------------------------------------- /docs/finished_experiments_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/finished_experiments_example.png -------------------------------------------------------------------------------- /docs/flux_jobs_a_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/flux_jobs_a_output.png -------------------------------------------------------------------------------- /docs/for-the-impatient.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | For the Impatient 8 | ================= 9 | 10 | You need git and Python 3.8+: 11 | 12 | :: 13 | 14 | git clone https://github.com/LLNL/benchpark.git 15 | cd benchpark 16 | . setup-env.sh 17 | benchpark --version 18 | 19 | python3 -m venv my-env 20 | . my-env/bin/activate 21 | 22 | pip install -r requirements.txt 23 | 24 | Set up a Workspace 25 | ------------------ 26 | 27 | :: 28 | 29 | benchpark system init --dest= compiler= 30 | benchpark experiment init --dest= +/~ = --system= 31 | benchpark setup 32 | 33 | where: 34 | 35 | - ````: amg2023 | saxpy | etc. (predefined choices in :doc:`benchmark-list`) 36 | - ````: Cts | Tioga | etc. (predefined systems in :doc:`system-list`) 37 | 38 | ``benchpark setup`` will output instructions to follow: 39 | 40 | :: 41 | 42 | . /setup.sh 43 | 44 | Build an Experiment 45 | ------------------- 46 | 47 | :: 48 | 49 | cd ///workspace 50 | ramble --workspace-dir . workspace setup 51 | 52 | Run the Experiment 53 | ------------------ 54 | 55 | To run all of the experiments in the workspace: 56 | 57 | :: 58 | 59 | ramble --workspace-dir . on 60 | 61 | To run a single experiment in the workspace, invoke the ``execute_experiment`` script 62 | for the specific experiment (e.g., 63 | ``$workspace/experiments/amg2023/problem1/amg2023_cuda11.8.0_problem1_1_8_2_2_2_10_10_10/execute_experiment``). 64 | 65 | Experiment pass/fail 66 | -------------------- 67 | 68 | Once the experiments completed running, the command: 69 | 70 | :: 71 | 72 | ramble --workspace-dir . workspace analyze 73 | 74 | can be used to analyze figures of merit and evaluate `success/failure 75 | `_ of 76 | the experiments. Ramble generates a file with summary of the results in ``$workspace``. 77 | 78 | If the benchmark you are running is instrumented with `Caliper 79 | `_, you can use the Caliper modifier (see 80 | :doc:`modifiers`) to collect detailed measurements you can later analyze with `Thicket 81 | `_. 82 | -------------------------------------------------------------------------------- /docs/generate-sys-defs-list.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import glob 4 | 5 | import pandas as pd 6 | import yaml 7 | 8 | 9 | def main(): 10 | sysconfig_yaml_files = glob.glob( 11 | "../systems/all_hardware_descriptions/**/hardware_description.yaml", 12 | recursive=True, 13 | ) 14 | 15 | df_list = [] 16 | for f in sysconfig_yaml_files: 17 | with open(f, "r") as stream: 18 | try: 19 | data = yaml.safe_load(stream) 20 | except yaml.YAMLError as exc: 21 | print(exc) 22 | 23 | tmp_df = pd.json_normalize(data, max_level=2) 24 | df_list.append(tmp_df) 25 | 26 | df = pd.concat(df_list) 27 | 28 | systested_columns_to_merge = [col for col in df.columns if "systems-tested" in col] 29 | top500_cols_to_merge = [ 30 | col for col in df.columns if "top500-system-instances" in col 31 | ] 32 | 33 | def merge_dicts(row, merge_cols): 34 | combined_dict = {} 35 | for col in merge_cols: 36 | if isinstance(row[col], dict): # Check if the value is a dictionary 37 | combined_dict.update( 38 | {col.split(".")[-1]: row[col]} 39 | ) # Merge the dictionary 40 | return combined_dict 41 | 42 | df["systems-tested"] = df.apply( 43 | lambda row: merge_dicts(row, systested_columns_to_merge), axis=1 44 | ) 45 | df["top500-system-instances"] = df.apply( 46 | lambda row: merge_dicts(row, top500_cols_to_merge), axis=1 47 | ) 48 | df = df.drop(columns=systested_columns_to_merge + top500_cols_to_merge) 49 | 50 | # Remove system_definition from all field names 51 | # (e.g., system_definition.system-tested.description) 52 | df.columns = df.columns.str.removeprefix("system_definition.") 53 | 54 | # Replace NaN with empty string 55 | df.fillna("", inplace=True) 56 | 57 | # Set index to be field names 58 | df.set_index("name", inplace=True) 59 | 60 | # Write out current system definitions to CSV format 61 | df.to_csv("current-system-definitions.csv") 62 | 63 | 64 | if __name__ == "__main__": 65 | main() 66 | -------------------------------------------------------------------------------- /docs/getting-started.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Getting Started 8 | =============== 9 | 10 | Installation 11 | ------------ 12 | 13 | Git is needed to clone Benchpark, and Python 3.8+ is needed to run Benchpark: 14 | 15 | :: 16 | 17 | git clone https://github.com/LLNL/benchpark.git 18 | 19 | To add the executable to your path, source the setup script: 20 | 21 | :: 22 | 23 | cd benchpark 24 | . setup-env.sh 25 | 26 | Now, to check the version you can run: 27 | 28 | :: 29 | 30 | benchpark --version 31 | 32 | System Prerequisites 33 | -------------------- 34 | 35 | Once Benchpark is available on your system, its python dependencies can be installed 36 | using the ``requirements.txt`` file included in the root directory of Benchpark. 37 | 38 | To install this, you can use: 39 | 40 | :: 41 | 42 | pip install -r requirements.txt 43 | 44 | Now you are ready to look at the benchmarks and systems available in Benchpark, and 45 | determine your workflow as described in :doc:`benchpark-commands`. 46 | -------------------------------------------------------------------------------- /docs/graph-and-tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/graph-and-tree.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | .. include:: ../README.rst 8 | 9 | .. toctree:: 10 | :maxdepth: 1 11 | :caption: Basics 12 | 13 | for-the-impatient 14 | getting-started 15 | benchpark-commands 16 | benchpark-workflow 17 | FAQ 18 | 19 | .. toctree:: 20 | :maxdepth: 1 21 | :caption: Catalogue 22 | 23 | system-list 24 | benchmark-list 25 | 26 | .. toctree:: 27 | :maxdepth: 1 28 | :caption: Tutorials 29 | 30 | tutorial-101 31 | tutorial-llnl 32 | comparing-experiment-builds 33 | 34 | .. toctree:: 35 | :maxdepth: 1 36 | :caption: Using Benchpark 37 | 38 | benchpark-setup 39 | build-experiment 40 | run-experiment 41 | analyze-experiment 42 | benchpark-analyze 43 | modifiers 44 | set-of-experiments 45 | run-binary 46 | create-mirror 47 | 48 | .. toctree:: 49 | :maxdepth: 1 50 | :caption: Contributing 51 | 52 | add-a-system-config 53 | add-a-benchmark 54 | add-an-experiment 55 | testing-your-contribution 56 | update-a-system-config 57 | 58 | .. toctree:: 59 | :maxdepth: 1 60 | :caption: Developer Docs 61 | 62 | developer-guide 63 | ci-developer-guide 64 | 65 | .. toctree:: 66 | :maxdepth: 1 67 | :caption: Benchpark Utils 68 | 69 | diffBuildSpecs 70 | diffExperimentBuilds 71 | diffExperimentSpecs 72 | diffPackageCommits 73 | diffSystemSpecs 74 | showBuild 75 | 76 | .. toctree:: 77 | :maxdepth: 2 78 | :caption: API Docs 79 | 80 | Benchpark API Docs 81 | -------------------------------------------------------------------------------- /docs/kripke_mpi_strong_raw_exc.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/kripke_mpi_strong_raw_exc.png -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | Sphinx 2 | sphinx-rtd-theme 3 | sphinxcontrib-programoutput -------------------------------------------------------------------------------- /docs/run-binary.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Run a Binary 8 | ============ 9 | 10 | If you have a pre-built binary of your application, you can use it in your Benchpark 11 | experiment using the ``user-managed`` Ramble-defined package manager (`see docs 12 | `_). 13 | When initializing your experiment, provide the path to the binary using ``prepend_path`` 14 | which will add the binary path to ``PATH``, and specify ``user-managed`` as the package 15 | manager. System setup does not change. 16 | 17 | Example running the ``osu-micro-benchmarks`` workload ``osu_latency`` on the ``ruby`` 18 | system: 19 | 20 | :: 21 | 22 | benchpark system init --dest=ruby llnl-cluster cluster=ruby 23 | benchpark experiment init --dest=osumb --system=ruby osu-micro-benchmarks \ 24 | package_manager="user-managed" \ 25 | workload="osu_latency" \ 26 | prepend_path="/usr/myuser/osu-micro-benchmarks/mpi/pt2pt" 27 | benchpark setup ./osumb/ osumb-ruby/ 28 | # Follow Ramble execution instructions ... 29 | 30 | This will execute using the ``osu_latency`` binary located at 31 | ``osu-micro-benchmarks/mpi/pt2pt/osu_latency``. 32 | 33 | Or for example, if we have a build ``kripke`` with spack on ``dane`` and then used that 34 | binary (``bin/kripke.exe``): 35 | 36 | :: 37 | 38 | benchpark system init --dest=dane llnl-cluster cluster=dane 39 | benchpark experiment init --dest=kripke --system=dane kripke \ 40 | package_manager="user-managed" \ 41 | prepend_path="/usr/myuser/benchpark/wkp/spack/opt/spack/linux-rhel8-sapphirerapids/oneapi-2023.2.1/kripke-develop-ehvoc6dzdprgm3lhaghh7uoiqsc5xcf6/bin" 42 | benchpark setup ./kripke/ kripke-dane/ 43 | # Follow Ramble execution instructions ... 44 | 45 | Using the spack built binary. 46 | -------------------------------------------------------------------------------- /docs/run-experiment.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Run an Experiment 8 | ================= 9 | 10 | To run all of the experiments in the workspace: 11 | 12 | :: 13 | 14 | ramble --workspace-dir . on 15 | 16 | An output file is generated for each experiment in its unique directory: 17 | 18 | :: 19 | 20 | $workspace 21 | | └── experiments 22 | | └── amg2023 23 | | └── problem1 24 | | ├── amg2023_cuda11.8.0_problem1_1_8_2_2_2_10_10_10 25 | | │   ├── execute_experiment 26 | | │   ├── amg2023_cuda11.8.0_problem1_1_8_2_2_2_10_10_10.out 27 | | │   └── ... 28 | | ├── amg2023_cuda11.8.0_problem1_2_4_2_2_2_10_10_10 29 | | │ ├── execute_experiment 30 | | │   ├── amg2023_cuda11.8.0_problem1_2_4_2_2_2_10_10_10.out 31 | | │ └── ... 32 | | ├── amg2023_cuda11.8.0_problem1_1_8_2_2_2_20_20_20 33 | | │   ├── execute_experiment 34 | | │   ├── amg2023_cuda11.8.0_problem1_1_8_2_2_2_20_20_20.out 35 | | │   └── ... 36 | | └── amg2023_cuda11.8.0_problem1_2_4_2_2_2_20_20_20 37 | | ├── execute_experiment 38 | |    ├── amg2023_cuda11.8.0_problem1_2_4_2_2_2_20_20_20.out 39 | | └── ... 40 | 41 | An experiment instance can also be executed individually by directly invoking its 42 | ``execute_experiment`` script (e.g., 43 | ``$workspace/experiments/amg2023/problem1/amg2023_cuda11.8.0_problem1_1_8_2_2_2_10_10_10/execute_experiment``). 44 | 45 | Note that re-running the experiment may overwrite any existing output files in the 46 | directory. Further, if the benchmark has restart capability, existing output may alter 47 | the experiments benchpark would run in the second run. Generally, we would advise the 48 | user to remove the ``$workspace/experiments`` directory before re-running the 49 | experiments using ``ramble --workspace-dir . on``. 50 | 51 | Once you have run your experiment you can try :doc:`analyze-experiment`. 52 | -------------------------------------------------------------------------------- /docs/set-of-experiments.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Set of Experiments 8 | ================== 9 | 10 | You may want to use the same experiments_root directory when setting up multiple 11 | experiments: 12 | 13 | - Benchpark only stores one copy of Spack/Ramble per experiments directory, and does not 14 | repeatedly download them when reusing the experiments directory. 15 | - Since each experiment in the experiments directory shares the Spack/Ramble instances, 16 | the edits/updates you make to Spack packages will apply to all experiments in your 17 | experiments directory. 18 | 19 | To use the same experiments directory when setting up multiple experiments, instruct 20 | benchpark to set them up in the same experiments_root: 21 | 22 | :: 23 | 24 | benchpark setup ${Benchmark1}/${ProgrammingModel1} /output/path/to/experiments_root 25 | benchpark setup ${Benchmark1}/${ProgrammingModel2} /output/path/to/experiments_root 26 | benchpark setup ${Benchmark2}/${ProgrammingModel2} /output/path/to/experiments_root 27 | 28 | This will result in the following directory structure: 29 | 30 | :: 31 | 32 | experiments_root/ 33 | ramble/ 34 | spack/ 35 | ${Benchmark1}/ 36 | ${ProgrammingModel1}/ 37 | ${System1}/ 38 | workspace/ 39 | ${ProgrammingModel2}/ 40 | ${System2}/ 41 | workspace/ 42 | ${Benchmark2}/ 43 | ${ProgrammingModel2}/ 44 | ${System1}/ 45 | workspace/ 46 | 47 | Note that there is a single clone of Ramble, and a single clone of Spack, which all of 48 | the ``experiments`` use. Each ``experiment`` (``Benchmark/ProgrammingModel`` x 49 | ``system`` combination) has its own ``Ramble workspace``, where this specific 50 | ``experiment`` will be compiled and run. 51 | -------------------------------------------------------------------------------- /docs/showBuild.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Show Build 8 | ========== 9 | 10 | If you build an experiment with Benchpark, you can see exactly how the experiment was 11 | built. 12 | 13 | ``` bin/benchpark experiment init --dest=def-raja-perf raja-perf bin/benchpark system 14 | init --dest=def-ruby llnl-cluster cluster=ruby compiler=gcc bin/benchpark setup 15 | def-raja-perf/ workspace/ . `pwd`/workspace/setup.sh ramble --workspace-dir 16 | `pwd`/workspace/def-raja-perf/def-ruby/workspace workspace setup ``` 17 | 18 | You will now be able to `benchpark show-build dump`, a command that will dump a log of 19 | how Spack built the experiment: 20 | 21 | ``` bin/benchpark show-build dump 22 | /usr/workspace/scheibel/oslic/benchpark2/workspace/def-raja-perf/def-ruby/workspace/ 23 | build-test/ ``` 24 | 25 | for this example, it will add the following artifacts to `build-test/` 26 | 27 | 1. `build-raja-perf.log`: a full log of all build commands and their output 28 | 2. `extracted-commands.txt`: [1] but with most output filtered out (generally much 29 | easier to understand what commands to run to do the build) 30 | 3. `spack-build-env.txt`: the environment variables set at build time 31 | -------------------------------------------------------------------------------- /docs/software.yaml: -------------------------------------------------------------------------------- 1 | # This is a Spack Environment file. 2 | # 3 | # It describes a set of packages to be installed, along with 4 | # configuration settings. 5 | spack: 6 | # add package specs to the `specs` list 7 | specs: [py-sphinx, py-sphinx-rtd-theme] 8 | view: true 9 | concretizer: 10 | unify: true 11 | -------------------------------------------------------------------------------- /docs/system-list.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | System Specifications 8 | ===================== 9 | 10 | The table below provides a directory of information for systems that have been specified 11 | in Benchpark. The column headers in the table below are available for use as the 12 | ``system`` parameter in ``benchpark setup``. 13 | 14 | .. csv-table:: Current System Specifications in Benchpark. 15 | :class: datatable 16 | :file: current-system-definitions.csv 17 | :header-rows: 1 18 | :align: left 19 | -------------------------------------------------------------------------------- /docs/tutorial/HPCIC25-Tutorial-Slide-Preview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/tutorial/HPCIC25-Tutorial-Slide-Preview.jpg -------------------------------------------------------------------------------- /docs/tutorial/ReproduciblePerfAnalysis-HPDC25-Tutorial-Slide-Preview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/tutorial/ReproduciblePerfAnalysis-HPDC25-Tutorial-Slide-Preview.jpg -------------------------------------------------------------------------------- /docs/tutorial/eScience25-Tutorial-Slide-Preview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/docs/tutorial/eScience25-Tutorial-Slide-Preview.jpg -------------------------------------------------------------------------------- /docs/why-benchpark.rst: -------------------------------------------------------------------------------- 1 | .. 2 | Copyright 2023 Lawrence Livermore National Security, LLC and other 3 | Benchpark Project Developers. See the top-level COPYRIGHT file for details. 4 | 5 | SPDX-License-Identifier: Apache-2.0 6 | 7 | Why Benchpark? 8 | ============== 9 | 10 | This is an overview of features that make Benchpark different from other benchmarking 11 | suites. 12 | 13 | Automation of benchmark building, running, and evaluation 14 | --------------------------------------------------------- 15 | 16 | Benchpark is a continuous benchmarking suite, or a fully automated mechanism for 17 | evaluating HPC benchmark performance on specified HPC systems. 18 | 19 | Reproducibly run benchmarks 20 | --------------------------- 21 | 22 | Collaborative maintenance of benchmarks 23 | --------------------------------------- 24 | 25 | Due to benchpark being open-source, 26 | 27 | System definitions 28 | ------------------ 29 | -------------------------------------------------------------------------------- /experiments/ad/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers, variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | 10 | 11 | class Ad(Experiment, MpiOnlyExperiment): 12 | variant( 13 | "workload", 14 | default="ad", 15 | description="ad", 16 | ) 17 | 18 | variant( 19 | "version", 20 | default="main", 21 | description="app version", 22 | ) 23 | 24 | maintainers("rfhaque") 25 | 26 | def compute_applications_section(self): 27 | self.add_experiment_variable("n_ranks", 1, True) 28 | self.add_experiment_variable("n_threads_per_proc", 1, True) 29 | 30 | self.set_required_variables( 31 | n_resources="{n_ranks}", process_problem_size="", total_problem_size="" 32 | ) 33 | 34 | def compute_package_section(self): 35 | self.add_package_spec(self.name, [f"ad{self.determine_version()}"]) 36 | -------------------------------------------------------------------------------- /experiments/babelstream/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.caliper import Caliper 7 | from benchpark.cuda import CudaExperiment 8 | from benchpark.directives import maintainers, variant 9 | from benchpark.experiment import Experiment 10 | from benchpark.openmp import OpenMPExperiment 11 | from benchpark.rocm import ROCmExperiment 12 | 13 | 14 | class Babelstream( 15 | Experiment, 16 | Caliper, 17 | CudaExperiment, 18 | ROCmExperiment, 19 | OpenMPExperiment, 20 | ): 21 | variant( 22 | "workload", 23 | default="babelstream", 24 | description="babelstream", 25 | ) 26 | 27 | variant( 28 | "version", 29 | default="caliper", 30 | values=("develop", "latest", "5.0", "caliper"), 31 | description="app version", 32 | ) 33 | 34 | maintainers("daboehme") 35 | 36 | def compute_applications_section(self): 37 | 38 | self.add_experiment_variable("processes_per_node", "1", True) 39 | self.add_experiment_variable("n", "35", False) 40 | self.add_experiment_variable("o", "0", False) 41 | n_resources = 1 42 | 43 | if self.spec.satisfies("+cuda"): 44 | self.add_experiment_variable("execute", "cuda-stream", False) 45 | 46 | elif self.spec.satisfies("+rocm"): 47 | self.add_experiment_variable("execute", "hip-stream", False) 48 | 49 | else: 50 | self.add_experiment_variable("n_ranks", n_resources, True) 51 | self.add_experiment_variable("execute", "omp-stream", False) 52 | 53 | if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): 54 | self.add_experiment_variable("n_gpus", n_resources, True) 55 | 56 | self.set_required_variables( 57 | n_resources=f"{n_resources}", 58 | process_problem_size="{n}/" + str(n_resources), 59 | total_problem_size="{n}", 60 | ) 61 | 62 | def compute_package_section(self): 63 | self.add_package_spec(self.name, [f"babelstream{self.determine_version()}"]) 64 | -------------------------------------------------------------------------------- /experiments/commbench/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.cuda import CudaExperiment 7 | from benchpark.directives import maintainers, variant 8 | from benchpark.experiment import Experiment 9 | from benchpark.rocm import ROCmExperiment 10 | 11 | 12 | class Commbench( 13 | Experiment, 14 | CudaExperiment, 15 | ROCmExperiment, 16 | ): 17 | variant("workload", description="workload name", default="basic") 18 | maintainers("arhag23") 19 | 20 | def compute_applications_section(self): 21 | self.add_experiment_variable("n_ranks", 2, True) 22 | 23 | if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): 24 | lib = "mpi" 25 | else: 26 | lib = "mpi" 27 | 28 | input_variables = { 29 | "size": "100000000", 30 | "lib": f"{lib}", 31 | "pattern": "broadcast", 32 | } 33 | 34 | for k, v in input_variables.items(): 35 | self.add_experiment_variable(k, v, True) 36 | 37 | self.set_required_variables( 38 | n_resources="{n_ranks}", 39 | process_problem_size="{size}", 40 | total_problem_size="{size}*{n_ranks}", 41 | ) 42 | 43 | def compute_package_section(self): 44 | spack_specs = [] 45 | spack_specs = " ".join(spack_specs) 46 | self.add_package_spec(self.name, [f"commbench {spack_specs}"]) 47 | -------------------------------------------------------------------------------- /experiments/genesis/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers, variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | from benchpark.openmp import OpenMPExperiment 10 | 11 | 12 | class Genesis(Experiment, MpiOnlyExperiment, OpenMPExperiment): 13 | 14 | variant( 15 | "workload", 16 | default="DHFR", 17 | values=("DHFR", "ApoA1", "UUN", "cryoEM"), 18 | description="genesis", 19 | ) 20 | 21 | variant( 22 | "version", 23 | default="main", 24 | description="app version", 25 | ) 26 | 27 | maintainers("jdomke", "SBA0486") 28 | 29 | def compute_applications_section(self): 30 | 31 | self.add_experiment_variable("experiment_setup", "") 32 | self.add_experiment_variable("lx", "32") 33 | self.add_experiment_variable("ly", "6") 34 | self.add_experiment_variable("lz", "4") 35 | self.add_experiment_variable("lt", "3") 36 | self.add_experiment_variable("px", "1") 37 | self.add_experiment_variable("py", "1") 38 | self.add_experiment_variable("pz", "1") 39 | self.add_experiment_variable("pt", "1") 40 | self.add_experiment_variable("tol_outer", "-1") 41 | self.add_experiment_variable("tol_inner", "-1") 42 | self.add_experiment_variable("maxiter_plus1_outer", "6") 43 | self.add_experiment_variable("maxiter_inner", "50") 44 | 45 | if self.spec.satisfies("+openmp"): 46 | self.add_experiment_variable("n_nodes", ["2"], True) 47 | self.add_experiment_variable("processes_per_node", ["4"]) 48 | self.add_experiment_variable("n_ranks", "{processes_per_node} * {n_nodes}") 49 | self.add_experiment_variable("omp_num_threads", ["12"]) 50 | self.add_experiment_variable("arch", "OpenMP") 51 | else: 52 | self.add_experiment_variable("n_nodes", ["2"], True) 53 | 54 | self.set_required_variables( 55 | n_resources="{n_ranks}", 56 | process_problem_size="{lx}*{ly}*{lz}/{n_ranks}", 57 | total_problem_size="{lx}*{ly}*{lz}", 58 | ) 59 | 60 | def compute_package_section(self): 61 | self.add_package_spec(self.name, [f"genesis{self.determine_version()}"]) 62 | -------------------------------------------------------------------------------- /experiments/gpcnet/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers, variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | 10 | 11 | class Gpcnet(Experiment, MpiOnlyExperiment): 12 | variant( 13 | "workload", 14 | default="network_test", 15 | values=("network_test", "network_load_test"), 16 | description="network_test or network_load_test", 17 | ) 18 | 19 | variant( 20 | "version", 21 | default="master", 22 | values=("master", "latest"), 23 | description="app version", 24 | ) 25 | 26 | maintainers("rfhaque") 27 | 28 | def compute_applications_section(self): 29 | # TODO: Replace with conflicts clause 30 | self.add_experiment_variable( 31 | "n_ranks", "{n_nodes}*{sys_cores_per_node}//2", True 32 | ) 33 | if self.spec.satisfies("workload=network_test"): 34 | self.add_experiment_variable("n_nodes", ["2", "4"]) 35 | elif self.spec.satisfies("workload=network_load_test"): 36 | self.add_experiment_variable("n_nodes", "10") 37 | 38 | self.set_required_variables( 39 | n_resources="{n_ranks}", process_problem_size="", total_problem_size="" 40 | ) 41 | 42 | def compute_package_section(self): 43 | self.add_package_spec(self.name, [f"gpcnet{self.determine_version()}"]) 44 | -------------------------------------------------------------------------------- /experiments/hpcg/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.caliper import Caliper 7 | from benchpark.directives import maintainers, variant 8 | from benchpark.experiment import Experiment 9 | from benchpark.mpi import MpiOnlyExperiment 10 | from benchpark.openmp import OpenMPExperiment 11 | from benchpark.scaling import Scaling, ScalingMode 12 | 13 | 14 | class Hpcg( 15 | Experiment, 16 | MpiOnlyExperiment, 17 | OpenMPExperiment, 18 | Scaling(ScalingMode.Strong, ScalingMode.Weak), 19 | Caliper, 20 | ): 21 | 22 | variant( 23 | "workload", 24 | default="standard", 25 | description="workload to run", 26 | ) 27 | variant( 28 | "version", 29 | default="caliper", 30 | values=("3.1", "develop", "caliper"), 31 | description="app version", 32 | ) 33 | 34 | maintainers("pearce8") 35 | 36 | def compute_applications_section(self): 37 | if self.spec.satisfies("exec_mode=test"): 38 | self.add_experiment_variable( 39 | "problem_sizes", {"mx": 104, "my": 104, "mz": 104}, True 40 | ) 41 | self.add_experiment_variable("num_procs", 1, True) 42 | 43 | self.register_scaling_config( 44 | { 45 | ScalingMode.Strong: { 46 | "num_procs": lambda var, itr, dim, scaling_factor: var.val(dim) 47 | * scaling_factor, 48 | "problem_sizes": lambda var, itr, dim, scaling_factor: var.val(dim), 49 | }, 50 | ScalingMode.Weak: { 51 | "num_procs": lambda var, itr, dim, scaling_factor: var.val(dim) 52 | * scaling_factor, 53 | "problem_sizes": lambda var, itr, dim, scaling_factor: var.val(dim) 54 | * scaling_factor, 55 | }, 56 | } 57 | ) 58 | 59 | self.add_experiment_variable("n_ranks", "{num_procs}", True) 60 | self.add_experiment_variable("n_threads_per_proc", 1, True) 61 | self.add_experiment_variable("matrix_size", "{mx} {my} {mz}", False) 62 | 63 | self.add_experiment_variable("iterations", "60", False) 64 | 65 | self.set_required_variables( 66 | n_resources="{n_ranks}", 67 | process_problem_size="{mx}*{my}*{mz}/{n_ranks}", 68 | total_problem_size="{mx}*{my}*{mz}", 69 | ) 70 | 71 | def compute_package_section(self): 72 | self.add_package_spec(self.name, [f"hpcg{self.determine_version()}"]) 73 | -------------------------------------------------------------------------------- /experiments/md-test/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers, variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | from benchpark.scaling import Scaling, ScalingMode 10 | 11 | 12 | class MdTest( 13 | Experiment, 14 | MpiOnlyExperiment, 15 | Scaling(ScalingMode.Strong), 16 | ): 17 | variant( 18 | "workload", 19 | default="multi-file", 20 | description="base md-test or other problem", 21 | ) 22 | 23 | variant( 24 | "version", 25 | default="1.9.3", 26 | description="app version", 27 | ) 28 | 29 | maintainers("rfhaque") 30 | 31 | def compute_applications_section(self): 32 | 33 | num_resources = {"n_ranks": 1} 34 | 35 | self.add_experiment_variable("num-objects", "1000", True) 36 | self.add_experiment_variable("iterations", "10", True) 37 | 38 | if self.spec.satisfies("exec_mode=test"): 39 | for pk, pv in num_resources.items(): 40 | self.add_experiment_variable(pk, pv, True) 41 | 42 | self.register_scaling_config( 43 | { 44 | ScalingMode.Strong: { 45 | "n_ranks": lambda var, itr, dim, scaling_factor: var.val(dim) 46 | * scaling_factor, 47 | "num-objects": lambda var, itr, dim, scaling_factor: var.val(dim), 48 | }, 49 | } 50 | ) 51 | 52 | self.set_required_variables( 53 | n_resources="{n_ranks}", 54 | process_problem_size="{num-objects}/{n_ranks}", 55 | total_problem_size="{num-objects}", 56 | ) 57 | 58 | def compute_package_section(self): 59 | self.add_package_spec( 60 | "ior", 61 | [ 62 | "ior@3.3.0", 63 | ], 64 | ) 65 | self.add_package_spec(self.name, [f"mdtest{self.determine_version()}"]) 66 | -------------------------------------------------------------------------------- /experiments/phloem/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers, variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | 10 | 11 | class Phloem(Experiment, MpiOnlyExperiment): 12 | variant( 13 | "workload", 14 | default="sqmr", 15 | values=("sqmr", "mpiBench", "mpiGraph"), 16 | description="sqmr, mpiBench, or mpiGraph", 17 | ) 18 | 19 | variant( 20 | "version", 21 | default="1.4.5", 22 | values=("master", "latest", "1.4.5"), 23 | description="app version", 24 | ) 25 | 26 | maintainers("nhanford") 27 | 28 | def compute_applications_section(self): 29 | if self.spec.satisfies("workload=sqmr"): 30 | self.add_experiment_variable( 31 | "n_ranks", "{num_cores}*{num_nbors}+{num_cores}" 32 | ) 33 | self.add_experiment_variable("num_cores", "4") 34 | self.add_experiment_variable("num_nbors", "{num_cores}") 35 | elif self.spec.satisfies("workload=mpiBench"): 36 | self.add_experiment_variable("n_ranks", "2") 37 | elif self.spec.satisfies("workload=mpiGraph"): 38 | self.add_experiment_variable("n_ranks", "2") 39 | 40 | self.set_required_variables( 41 | n_resources="{n_ranks}", process_problem_size="", total_problem_size="" 42 | ) 43 | 44 | def compute_package_section(self): 45 | self.add_package_spec(self.name, [f"phloem{self.determine_version()}"]) 46 | -------------------------------------------------------------------------------- /experiments/qws/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.caliper import Caliper 7 | from benchpark.directives import maintainers, variant 8 | from benchpark.experiment import Experiment 9 | from benchpark.mpi import MpiOnlyExperiment 10 | from benchpark.openmp import OpenMPExperiment 11 | 12 | 13 | class Qws(Experiment, MpiOnlyExperiment, OpenMPExperiment, Caliper): 14 | 15 | variant( 16 | "workload", 17 | default="qws", 18 | description="qws", 19 | ) 20 | 21 | variant( 22 | "version", 23 | default="master", 24 | description="app version", 25 | ) 26 | 27 | maintainers("jdomke", "SBA0486") 28 | 29 | def compute_applications_section(self): 30 | 31 | self.add_experiment_variable("experiment_setup", "") 32 | self.add_experiment_variable("lx", "32") 33 | self.add_experiment_variable("ly", "6") 34 | self.add_experiment_variable("lz", "4") 35 | self.add_experiment_variable("lt", "3") 36 | self.add_experiment_variable("px", "1") 37 | self.add_experiment_variable("py", "1") 38 | self.add_experiment_variable("pz", "1") 39 | self.add_experiment_variable("pt", "1") 40 | self.add_experiment_variable("tol_outer", "-1") 41 | self.add_experiment_variable("tol_inner", "-1") 42 | self.add_experiment_variable("maxiter_plus1_outer", "6") 43 | self.add_experiment_variable("maxiter_inner", "50") 44 | 45 | if self.spec.satisfies("+openmp"): 46 | self.add_experiment_variable("n_nodes", ["1"], True) 47 | self.add_experiment_variable("processes_per_node", ["1"]) 48 | self.add_experiment_variable("n_ranks", "{processes_per_node} * {n_nodes}") 49 | self.add_experiment_variable("omp_num_threads", ["48"]) 50 | self.add_experiment_variable("arch", "OpenMP") 51 | else: 52 | self.add_experiment_variable("n_nodes", ["1"], True) 53 | 54 | self.set_required_variables( 55 | n_resources="{n_ranks}", 56 | process_problem_size="{lx}*{ly}*{lz}/{n_ranks}", 57 | total_problem_size="{lx}*{ly}*{lz}", 58 | ) 59 | 60 | def compute_package_section(self): 61 | self.add_package_spec(self.name, [f"qws{self.determine_version()}"]) 62 | -------------------------------------------------------------------------------- /experiments/repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: builtin 3 | subdirectory: '' 4 | -------------------------------------------------------------------------------- /experiments/salmon-tddft/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | from benchpark.openmp import OpenMPExperiment 10 | 11 | 12 | class SalmonTddft(Experiment, MpiOnlyExperiment, OpenMPExperiment): 13 | variant( 14 | "workload", 15 | default="gs", 16 | values=("gs", "rt"), 17 | multi=True, 18 | description="salmon-tddft", 19 | ) 20 | 21 | variant( 22 | "version", 23 | default="2.0.0", 24 | description="app version", 25 | ) 26 | 27 | def compute_applications_section(self): 28 | self.add_experiment_variable("experiment_setup", "") 29 | 30 | if self.spec.satisfies("workload=gs"): 31 | self.add_experiment_variable("exercise", "exercise_01_C2H2_gs") 32 | self.add_experiment_variable("inp_file", "C2H2_gs.inp") 33 | elif self.spec.satisfies("workload=rt"): 34 | self.add_experiment_variable("exercise", "exercise_03_C2H2_rt") 35 | self.add_experiment_variable("inp_file", "C2H2_rt_pulse.inp") 36 | self.add_experiment_variable( 37 | "restart_data", 38 | "../../gs/salmon_{n_nodes}_{n_ranks}_{n_threads}/data_for_restart/", 39 | ) 40 | 41 | if self.spec.satisfies("+openmp"): 42 | self.add_experiment_variable("omp_num_threads", ["12"]) 43 | self.add_experiment_variable("n_ranks", "{processes_per_node} * {n_nodes}") 44 | self.add_experiment_variable("processes_per_node", ["4"]) 45 | self.add_experiment_variable("n_nodes", ["1"], True) 46 | 47 | self.set_required_variables( 48 | n_resources="{n_ranks}", process_problem_size="", total_problem_size="" 49 | ) 50 | 51 | def compute_package_section(self): 52 | self.add_package_spec(self.name, [f"salmon-tddft{self.determine_version()}"]) 53 | -------------------------------------------------------------------------------- /experiments/saxpy/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | from benchpark.caliper import Caliper 8 | from benchpark.cuda import CudaExperiment 9 | from benchpark.directives import maintainers, variant 10 | from benchpark.experiment import Experiment 11 | from benchpark.mpi import MpiOnlyExperiment 12 | from benchpark.openmp import OpenMPExperiment 13 | from benchpark.rocm import ROCmExperiment 14 | 15 | 16 | class Saxpy( 17 | Experiment, 18 | MpiOnlyExperiment, 19 | OpenMPExperiment, 20 | CudaExperiment, 21 | ROCmExperiment, 22 | Caliper, 23 | ): 24 | variant( 25 | "workload", 26 | default="problem", 27 | description="problem", 28 | ) 29 | 30 | variant( 31 | "version", 32 | default="1.0.0", 33 | description="app version", 34 | ) 35 | 36 | maintainers("rfhaque") 37 | 38 | def compute_applications_section(self): 39 | # GPU tests include some smaller sizes 40 | n = ["512", "1024"] 41 | if self.spec.satisfies("+openmp"): 42 | self.add_experiment_variable("n_nodes", ["1", "2"], named=True) 43 | # resource_count is the number of resources used for this experiment: 44 | self.add_experiment_variable("resource_count", "8") 45 | self.add_experiment_variable( 46 | "n_threads_per_proc", ["2", "4"], named=True, matrixed=True 47 | ) 48 | else: 49 | n = ["128", "256"] + n 50 | # resource_count is the number of resources used for this experiment: 51 | self.add_experiment_variable("resource_count", "1") 52 | 53 | self.add_experiment_variable("n", n, named=True, matrixed=True) 54 | 55 | self.set_required_variables( 56 | n_resources="{resource_count}", 57 | process_problem_size="{n}/{n_resources}", 58 | total_problem_size="{n}", 59 | ) 60 | 61 | if self.spec.satisfies("+cuda") or self.spec.satisfies("+rocm"): 62 | self.add_experiment_variable("n_gpus", "{n_resources}", True) 63 | else: 64 | self.add_experiment_variable("n_ranks", "{n_resources}", True) 65 | 66 | def compute_package_section(self): 67 | self.add_package_spec(self.name, [f"saxpy{self.determine_version()}"]) 68 | -------------------------------------------------------------------------------- /experiments/smb/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers, variant 7 | from benchpark.experiment import Experiment 8 | from benchpark.mpi import MpiOnlyExperiment 9 | 10 | 11 | class Smb(Experiment, MpiOnlyExperiment): 12 | variant( 13 | "workload", 14 | default="mpi_overhead", 15 | values=("mpi_overhead", "msgrate", "rma_mt"), 16 | description="workload", 17 | ) 18 | 19 | variant( 20 | "version", 21 | default="1.1", 22 | values=("master", "latest", "1.1"), 23 | description="app version", 24 | ) 25 | 26 | maintainers("nhanford") 27 | 28 | def compute_applications_section(self): 29 | if self.spec.satisfies("workload=mpi_overhead"): 30 | self.add_experiment_variable("n_ranks", "2") 31 | elif self.spec.satisfies("workload=msgrate") or self.spec.satisfies( 32 | "workload=rma_mt" 33 | ): 34 | self.add_experiment_variable("n_nodes", "1") 35 | self.add_experiment_variable("n_ranks", "{n_nodes}*{sys_cores_per_node}") 36 | 37 | self.set_required_variables( 38 | n_resources="{n_ranks}", process_problem_size="", total_problem_size="" 39 | ) 40 | 41 | def compute_package_section(self): 42 | spec_string = f"smb{self.determine_version()}" 43 | if self.spec.satisfies("workload=rma_mt"): 44 | spec_string += "+rma" 45 | self.add_package_spec(self.name, [spec_string]) 46 | -------------------------------------------------------------------------------- /experiments/stream/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.caliper import Caliper 7 | from benchpark.directives import maintainers, variant 8 | from benchpark.experiment import Experiment 9 | from benchpark.mpi import MpiOnlyExperiment 10 | 11 | 12 | class Stream( 13 | Experiment, 14 | MpiOnlyExperiment, 15 | Caliper, 16 | ): 17 | variant( 18 | "workload", 19 | default="stream", 20 | description="stream", 21 | ) 22 | 23 | variant( 24 | "version", 25 | default="5.10-caliper", 26 | values=("develop", "latest", "5.10-caliper"), 27 | description="app version", 28 | ) 29 | 30 | maintainers("daboehme", "rfhaque") 31 | 32 | def compute_applications_section(self): 33 | 34 | array_size = {"s": 650000000} 35 | 36 | self.add_experiment_variable("processes_per_node", "1", named=True) 37 | self.add_experiment_variable("n", "35", False) 38 | self.add_experiment_variable("o", "0", False) 39 | self.add_experiment_variable("n_ranks", 1, True) 40 | self.add_experiment_variable( 41 | "n_threads_per_proc", [16, 32], named=True, matrixed=True 42 | ) 43 | 44 | for pk, pv in array_size.items(): 45 | self.add_experiment_variable(pk, pv, True) 46 | 47 | self.set_required_variables( 48 | n_resources="{n_ranks}", 49 | process_problem_size="{n}/{n_ranks}", 50 | total_problem_size="{n}", 51 | ) 52 | 53 | def compute_package_section(self): 54 | self.add_package_spec(self.name, [f"stream{self.determine_version()}"]) 55 | -------------------------------------------------------------------------------- /lib/benchpark/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/lib/benchpark/__init__.py -------------------------------------------------------------------------------- /lib/benchpark/cmd/bootstrap.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import benchpark.paths 7 | from benchpark.runtime import RuntimeResources 8 | 9 | 10 | def setup_parser(subparser): 11 | pass 12 | 13 | 14 | def command(args): 15 | bootstrapper = RuntimeResources(benchpark.paths.benchpark_home) 16 | bootstrapper.bootstrap() 17 | -------------------------------------------------------------------------------- /lib/benchpark/cmd/experiment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # Copyright 2013-2023 Spack Project Developers. 5 | # 6 | # SPDX-License-Identifier: Apache-2.0 7 | 8 | import os 9 | import pickle 10 | import shutil 11 | import sys 12 | 13 | import benchpark.spec 14 | from benchpark.error import BenchparkError 15 | 16 | 17 | def experiment_init(args): 18 | experiment_spec = benchpark.spec.ExperimentSpec(" ".join(args.spec)).concretize() 19 | experiment = experiment_spec.experiment 20 | 21 | if args.system: 22 | if not os.path.exists(args.system): 23 | raise BenchparkError( 24 | f"System '{args.system}' does not exist. Run 'benchpark system init --dest {args.system} ...'." 25 | ) 26 | system_file = os.path.join(args.system, "system.pkl") 27 | with open(system_file, "rb") as f: 28 | system_spec = pickle.load(f) 29 | 30 | experiment.system_spec = system_spec 31 | 32 | if args.basedir: 33 | base = args.basedir 34 | expdir = str(hash(experiment_spec)) 35 | destdir = os.path.join(base, expdir) 36 | elif args.dest: 37 | destdir = args.dest 38 | else: 39 | raise ValueError("Must specify one of: --dest, --basedir") 40 | 41 | try: 42 | os.makedirs(destdir) 43 | experiment.write_ramble_dict(f"{destdir}/ramble.yaml") 44 | except FileExistsError: 45 | print(f"Abort: experiment description dir already exists ({destdir})") 46 | sys.exit(1) 47 | except Exception: 48 | # If there was a failure, remove any partially-generated resources 49 | shutil.rmtree(destdir) 50 | raise 51 | 52 | 53 | def setup_parser(root_parser): 54 | system_subparser = root_parser.add_subparsers( 55 | dest="experiment_subcommand", required=True 56 | ) 57 | 58 | init_parser = system_subparser.add_parser("init") 59 | init_parser.add_argument("--dest", help="Place all system files here directly") 60 | init_parser.add_argument( 61 | "--basedir", help="Generate a system dir under this, and place all files there" 62 | ) 63 | init_parser.add_argument( 64 | "--system", 65 | help="The system this experiment will run on (directory generated by 'benchpark system init')", 66 | required=True, 67 | ) 68 | 69 | init_parser.add_argument("spec", nargs="+", help="Experiment spec") 70 | 71 | 72 | def command(args): 73 | actions = { 74 | "init": experiment_init, 75 | } 76 | if args.experiment_subcommand in actions: 77 | actions[args.experiment_subcommand](args) 78 | -------------------------------------------------------------------------------- /lib/benchpark/cuda.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | from benchpark.directives import requires, variant 8 | from benchpark.experiment import ExperimentHelper 9 | 10 | 11 | class CudaExperiment: 12 | requires("cuda", when="+cuda") 13 | variant("cuda", default=False, description="Build and run with CUDA") 14 | 15 | def __init__(self): 16 | super().__init__() 17 | if self.spec.variants["cuda"][0]: 18 | self.device_type = "gpu" 19 | self.programming_models.append("cuda") 20 | 21 | class Helper(ExperimentHelper): 22 | def get_helper_name_prefix(self): 23 | return "cuda" if self.spec.satisfies("+cuda") else "" 24 | 25 | def get_spack_variants(self): 26 | return ( 27 | "+cuda cuda_arch={cuda_arch}" 28 | if self.spec.satisfies("+cuda") 29 | else "~cuda" 30 | ) 31 | -------------------------------------------------------------------------------- /lib/benchpark/cudasystem.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import provides 7 | 8 | 9 | class CudaSystem: 10 | provides("cuda") 11 | 12 | name = "cuda" 13 | 14 | def verify(self, system): 15 | assert "cuda" in system.variants 16 | assert "gtl" in system.variants 17 | assert hasattr(system, "cuda_arch") 18 | assert hasattr(system, "cuda_version") 19 | assert hasattr(system, "gtl_flag") 20 | 21 | def system_specific_variables(self, system): 22 | return { 23 | "cuda_arch": system.cuda_arch, 24 | "cuda_version": system.cuda_version, 25 | "gtl_flag": system.gtl_flag, 26 | } 27 | -------------------------------------------------------------------------------- /lib/benchpark/debug.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | import sys 8 | from inspect import currentframe, getframeinfo 9 | 10 | DEBUG = False 11 | 12 | 13 | def get_linenumber(): 14 | cf = currentframe() 15 | return cf.f_back.f_back.f_lineno 16 | 17 | 18 | def get_filename(): 19 | cf = currentframe() 20 | return os.path.basename(getframeinfo(cf.f_back).filename) 21 | 22 | 23 | def debug_print(message): 24 | if DEBUG: 25 | print( 26 | "(debug) " + get_filename() + "::" + str(get_linenumber()) + " " + message, 27 | file=sys.stderr, 28 | ) 29 | -------------------------------------------------------------------------------- /lib/benchpark/error.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | #: at what level we should write stack traces or short error messages 7 | #: this is module-scoped because it needs to be set very early 8 | debug = 0 9 | 10 | 11 | class BenchparkError(Exception): 12 | """This is the superclass for all Benchpark errors. 13 | Subclasses can be found in the modules they have to do with. 14 | """ 15 | -------------------------------------------------------------------------------- /lib/benchpark/mpi.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | from benchpark.directives import requires, variant 8 | from benchpark.experiment import ExperimentHelper 9 | 10 | 11 | class MpiOnlyExperiment: 12 | requires("mpi") 13 | variant("mpi", default=True, description="Run with MPI only") 14 | 15 | def __init__(self): 16 | super().__init__() 17 | if self.spec.variants["mpi"][0]: 18 | self.device_type = "cpu" 19 | self.programming_models.append("mpi") 20 | 21 | class Helper(ExperimentHelper): 22 | def get_helper_name_prefix(self): 23 | return "mpi" if self.spec.satisfies("+mpi") else "" 24 | -------------------------------------------------------------------------------- /lib/benchpark/openmp.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | from benchpark.directives import requires, variant 8 | from benchpark.experiment import ExperimentHelper 9 | 10 | 11 | class OpenMPExperiment: 12 | requires("openmp", when="+openmp") 13 | variant("openmp", default=False, description="Build and run with OpenMP") 14 | 15 | def __init__(self): 16 | super().__init__() 17 | if self.spec.variants["openmp"][0]: 18 | self.device_type = "cpu" 19 | self.programming_models.append("openmp") 20 | 21 | class Helper(ExperimentHelper): 22 | def get_helper_name_prefix(self): 23 | return "openmp" if self.spec.satisfies("+openmp") else "" 24 | 25 | def get_spack_variants(self): 26 | return "+openmp" if self.spec.satisfies("+openmp") else "~openmp" 27 | -------------------------------------------------------------------------------- /lib/benchpark/openmpsystem.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import provides 7 | 8 | 9 | class OpenMPCPUOnlySystem: 10 | provides("openmp") 11 | 12 | name = "openmp" 13 | 14 | def system_specific_variables(self, system): 15 | return {} 16 | -------------------------------------------------------------------------------- /lib/benchpark/paths.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | import pathlib 8 | 9 | 10 | def _source_location() -> pathlib.Path: 11 | """Return the location of the project source files directory.""" 12 | path_to_this_file = __file__ 13 | return pathlib.Path(path_to_this_file).resolve().parents[2] 14 | 15 | 16 | benchpark_root = _source_location() 17 | lib_path = benchpark_root / "lib" / "benchpark" 18 | test_path = lib_path / "test" 19 | 20 | benchpark_home = pathlib.Path( 21 | os.getenv("BENCHPARK_HOME", default=os.path.expanduser("~/.benchpark")) 22 | ) 23 | 24 | global_ramble_path = benchpark_home / "ramble" 25 | global_spack_path = benchpark_home / "spack" 26 | hardware_descriptions = benchpark_root / "systems" / "all_hardware_descriptions" 27 | checkout_versions = benchpark_root / "checkout-versions.yaml" 28 | remote_urls = benchpark_root / "remote-urls.yaml" 29 | -------------------------------------------------------------------------------- /lib/benchpark/rocm.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | 7 | from benchpark.directives import requires, variant 8 | from benchpark.experiment import ExperimentHelper 9 | 10 | 11 | class ROCmExperiment: 12 | requires("rocm", when="+rocm") 13 | variant("rocm", default=False, description="Build and run with ROCm") 14 | 15 | def __init__(self): 16 | super().__init__() 17 | if self.spec.variants["rocm"][0]: 18 | self.device_type = "gpu" 19 | self.programming_models.append("rocm") 20 | 21 | class Helper(ExperimentHelper): 22 | def get_helper_name_prefix(self): 23 | return "rocm" if self.spec.satisfies("+rocm") else "" 24 | 25 | def get_spack_variants(self): 26 | return ( 27 | "+rocm amdgpu_target={rocm_arch}" 28 | if self.spec.satisfies("+rocm") 29 | else "~rocm" 30 | ) 31 | -------------------------------------------------------------------------------- /lib/benchpark/rocmsystem.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import provides 7 | 8 | 9 | class ROCmSystem: 10 | provides("rocm") 11 | 12 | name = "rocm" 13 | 14 | def verify(self, system): 15 | assert "rocm" in system.variants 16 | assert "gtl" in system.variants 17 | assert hasattr(system, "rocm_arch") 18 | assert hasattr(system, "rocm_version") 19 | assert hasattr(system, "gtl_flag") 20 | 21 | def system_specific_variables(self, system): 22 | return { 23 | "rocm_arch": system.rocm_arch, 24 | "rocm_version": system.rocm_version, 25 | "gtl_flag": system.gtl_flag, 26 | } 27 | -------------------------------------------------------------------------------- /lib/benchpark/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/lib/benchpark/test/__init__.py -------------------------------------------------------------------------------- /lib/benchpark/test/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/LLNL/benchpark/809d497d86cb299286db8776ff8dfdc57b4a2df2/lib/benchpark/test/conftest.py -------------------------------------------------------------------------------- /lib/benchpark/test/paths.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import pathlib 7 | 8 | import benchpark.paths 9 | 10 | 11 | def test_benchpark_root(pytestconfig): 12 | expected_path = pathlib.Path(pytestconfig.inipath).resolve().parent 13 | assert benchpark.paths.benchpark_root == expected_path 14 | -------------------------------------------------------------------------------- /lib/benchpark/test/spec_semantics.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2024 Lawrence Livermore National Security, LLC and other 2 | # Spack Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: (Apache-2.0 OR MIT) 5 | 6 | from benchpark.spec import Spec 7 | 8 | 9 | def test_spec_hashing_and_eq(): 10 | x = Spec("+x") 11 | y = Spec("+y") 12 | 13 | items = set([x]) 14 | assert y not in items 15 | -------------------------------------------------------------------------------- /lib/benchpark/test/system.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import pytest 7 | 8 | import benchpark.spec 9 | 10 | 11 | def test_system_compute_variables_section(monkeypatch): 12 | sys_spec = benchpark.spec.SystemSpec("llnl-elcapitan cluster=tuolumne").concretize() 13 | system = sys_spec.system 14 | 15 | vars_section = system.compute_variables_section() 16 | 17 | assert vars_section == { 18 | "variables": { 19 | "timeout": "120", 20 | "scheduler": "flux", 21 | "sys_cores_per_node": 84, 22 | "n_ranks": 2**64 - 1, 23 | "n_nodes": 2**64 - 1, 24 | "batch_submit": "placeholder", 25 | "mpi_command": "placeholder", 26 | "sys_cores_os_reserved_per_node": 12, 27 | "sys_cores_os_reserved_per_node_list": [ 28 | 0, 29 | 8, 30 | 16, 31 | 24, 32 | 32, 33 | 40, 34 | 48, 35 | 56, 36 | 64, 37 | 72, 38 | 80, 39 | 88, 40 | ], 41 | "sys_gpus_per_node": 4, 42 | "sys_mem_per_node_GB": 512, 43 | "rocm_arch": "gfx942", 44 | "rocm_version": "6.4.0", 45 | "gtl_flag": True, 46 | "gpu_factor": 1, 47 | "extra_batch_opts": "--setattr=gpumode=SPX\n--conf=resource.rediscover=true", 48 | } 49 | } 50 | 51 | 52 | def test_system_timeout(): 53 | with pytest.raises(ValueError, match="is unsatisfiable for the selected queue"): 54 | sys_spec = benchpark.spec.SystemSpec( 55 | "llnl-elcapitan cluster=tioga queue=pdebug timeout=9999" 56 | ).concretize() 57 | sys_spec.system.compute_variables_section() 58 | -------------------------------------------------------------------------------- /lib/scripts/env-collect-branch-tips.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import sys 4 | 5 | import spack.environment as ev 6 | from spack.fetch_strategy import GitFetchStrategy 7 | 8 | 9 | def main(): 10 | destination = sys.argv[1] 11 | 12 | e = ev.active_environment() 13 | for _, spec in e.concretized_specs(): 14 | df = spec.package.stage[0].default_fetcher 15 | if not df.cachable and isinstance(df, GitFetchStrategy): 16 | df.get_full_repo = True 17 | pkg_dst = os.path.join(destination, spec.name) 18 | if not os.path.exists(pkg_dst): 19 | spec.package.stage.fetch() 20 | shutil.move(spec.package.stage.source_path, pkg_dst) 21 | print(f"{spec.name}") 22 | 23 | 24 | if __name__ == "__main__": 25 | main() 26 | -------------------------------------------------------------------------------- /lib/scripts/experiment-build-info.py: -------------------------------------------------------------------------------- 1 | import json 2 | import sys 3 | 4 | import spack.environment as ev 5 | 6 | 7 | def main(): 8 | x = ev.active_environment() 9 | y = list(x.concrete_roots()) 10 | # There may be multiple roots in the env. Assume that the experiment of 11 | # interest is the largest dag (every other root should just be an attempt 12 | # to constrain dependencies of this experiment). 13 | z = max(y, key=lambda i: sum(1 for _ in i.traverse())) 14 | built_specs = list() 15 | for dep in z.traverse(deptype="link"): 16 | if dep.external: 17 | continue 18 | if "runtime" in getattr(dep.package, "tags", []): 19 | continue 20 | built_specs.append(dep) 21 | 22 | urls = {} 23 | for spec in built_specs: 24 | x = spec.package.stage[0].default_fetcher 25 | if hasattr(x, "url"): 26 | urls[spec.name] = { 27 | "url": x.url, 28 | "details": spec.package.versions[spec.version], 29 | } 30 | else: 31 | raise Exception(f"Unexpected: {spec.name} has no url attribute") 32 | 33 | result = { 34 | "root": z.name, 35 | "tree": z.tree(), 36 | "info": [(w.name, w.package.install_env_path) for w in built_specs], 37 | "urls": urls, 38 | } 39 | json.dump(result, sys.stdout) 40 | 41 | 42 | if __name__ == "__main__": 43 | main() 44 | -------------------------------------------------------------------------------- /modifiers/affinity/parse_affinity_log.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import argparse 7 | import json 8 | import re 9 | from pathlib import Path 10 | 11 | 12 | def parse_affinity(affinity_log_file, mode): 13 | with open(affinity_log_file, "r") as f: 14 | lines = f.readlines() 15 | 16 | affinity_map = {"gpus": [], "cores": []} 17 | 18 | current_rank = None 19 | current_node = None 20 | 21 | for line in lines: 22 | # Match rank and node 23 | rank_match = re.match(r"rank\s+(\d+)\s+@\s+(\S+)", line) 24 | if rank_match: 25 | current_rank = int(rank_match.group(1)) 26 | current_node = rank_match.group(2) 27 | continue 28 | 29 | # Match CPU affinity (cores) 30 | core_match = re.match(r"\s*cores\s*:\s*(\d+)", line) 31 | if core_match and current_rank is not None: 32 | affinity_map["cores"].append( 33 | { 34 | "rank": current_rank, 35 | "node": current_node, 36 | "cores": int(core_match.group(1)), 37 | } 38 | ) 39 | continue 40 | 41 | if mode == "cuda" or mode == "rocm": 42 | # Match GPU affinity 43 | gpu_match = re.match(r"\s*gpu\s*\d+\s*:\s*(GPU-\S+)", line) 44 | if gpu_match and current_rank is not None: 45 | affinity_map["gpus"].append( 46 | { 47 | "rank": current_rank, 48 | "node": current_node, 49 | "gpu_id": gpu_match.group(1), 50 | } 51 | ) 52 | continue 53 | 54 | # Save to JSON file 55 | output_file = Path(affinity_log_file).with_suffix(".json") 56 | with open(output_file, "w") as json_file: 57 | json.dump(affinity_map, json_file) 58 | 59 | 60 | if __name__ == "__main__": 61 | parser = argparse.ArgumentParser(description="affinity output to JSON") 62 | parser.add_argument("affinity_log_file", type=str, help="affinity log file (text)") 63 | parser.add_argument( 64 | "mode", choices=["mpi", "cuda", "rocm"], help="Mode: 'mpi', 'cuda' or 'rocm'" 65 | ) 66 | 67 | args = parser.parse_args() 68 | parse_affinity(args.affinity_log_file, args.mode) 69 | -------------------------------------------------------------------------------- /modifiers/modifier_repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: benchpark 3 | subdirectory: '' 4 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name="benchpark" 3 | description="An open collaborative repository for cross site benchmarking environments" 4 | readme = "README.rst" 5 | requires-python = ">=3.8" 6 | dynamic = ["version"] 7 | 8 | [tool.setuptools] 9 | packages = [ 10 | "lib", 11 | ] 12 | 13 | [tool.black] 14 | line-length = 88 15 | color = true 16 | target-version = ['py38', 'py39', 'py310', 'py311'] 17 | include = '\.pyi?$|bin\/benchpark$' 18 | exclude = ''' 19 | /( 20 | \.eggs 21 | | \.git 22 | | \.hg 23 | | \.mypy_cache 24 | | \.tox 25 | | \.venv 26 | | _build 27 | | buck-out 28 | | build 29 | | dist 30 | | repo 31 | )/ 32 | ''' 33 | force-exclude = ''' 34 | /( 35 | '/lib/benchpark/test_repo/(/.*)?$' 36 | | /lib/benchpark/test_repo(/.*)?$ 37 | | lib/benchpark/test_repo 38 | | '/bin/benchpark-python' 39 | | \.github 40 | )/ 41 | ''' 42 | 43 | [tool.isort] 44 | profile = "black" 45 | skip_gitignore = true 46 | color_output = true 47 | skip_glob = [ 48 | "lib/benchpark/test_repo/**" 49 | ] 50 | 51 | [tool.flake8] 52 | exclude = [ 53 | "lib/benchpark/test_repo/**" 54 | ] 55 | skip_glob = [ 56 | "lib/benchpark/test_repo/**" 57 | ] 58 | force-exclude = [ 59 | "./lib/benchpark/test_repo/**" 60 | ] 61 | per-file-ignores = """ 62 | ./lib/benchpark/test_repo/** 63 | """ 64 | 65 | [tool.codespell] 66 | skip = './docs/_build,./docs/_static,./taxonomy.yaml,./repo/**/*.patch' 67 | ignore-words-list = 'fom' 68 | 69 | [tool.yamlfix] 70 | line-length = 88 71 | allow_duplicate_keys = true 72 | explicit_start = false 73 | 74 | [project.optional-dependencies] 75 | analyze = [ 76 | "llnl-thicket[plotting]", 77 | "matplotlib" 78 | ] -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | # content of pytest.ini 2 | [pytest] 3 | testpaths = lib/benchpark/test 4 | python_files = *.py 5 | -------------------------------------------------------------------------------- /remote-urls.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | urls: 6 | ramble: https://github.com/GoogleCloudPlatform/ramble.git 7 | spack: https://github.com/spack/spack.git 8 | spack-packages: https://github.com/spack/spack-packages.git 9 | -------------------------------------------------------------------------------- /repo/ad/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Ad(ExecutableApplication): 12 | """AD benchmark""" 13 | name = "ad" 14 | 15 | tags = ['mpi','c','c++','automatic-differentiation','compiler-transformation'] 16 | 17 | executable('c_interface_test', 'c_interface_test', use_mpi=True) 18 | 19 | workload('ad', executables=['c_interface_test']) 20 | 21 | #figure_of_merit('Photons per Second', 22 | # log_file='{experiment_run_dir}/{experiment_name}.out', 23 | # fom_regex=r'Photons Per Second \(FOM\):\s+(?P[0-9]+\.[0-9]*([0-9]*)?e\+[0-9]*)', 24 | # group_name='fom', units='photons') 25 | 26 | #success_criteria('pass', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out') 27 | -------------------------------------------------------------------------------- /repo/ad/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class Ad(CMakePackage): 10 | """AD Benchmark""" 11 | 12 | git = "git@github.com:jandrej/dfem-enzyme-compat.git" 13 | 14 | tags = ["benchmark"] 15 | 16 | license("Apache-2.0") 17 | 18 | maintainers("jandrej") 19 | 20 | version("main", branch="main") 21 | 22 | depends_on("c", type="build") 23 | depends_on("cxx", type="build") 24 | depends_on("fortran", type="build") 25 | 26 | depends_on("enzyme") 27 | 28 | def cmake_args(self): 29 | cmake_options = [ 30 | f"-DENZYME_DIR={self.spec['enzyme'].prefix}", 31 | ] 32 | 33 | return cmake_options 34 | 35 | def install(self, spec, prefix): 36 | mkdir(prefix.bin) 37 | install(join_path(self.build_directory, "src", "c_interface_test"), prefix.bin) 38 | -------------------------------------------------------------------------------- /repo/affinity/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class Affinity(CMakePackage, CudaPackage, ROCmPackage): 10 | """Simple applications for determining Linux thread and gpu affinity.""" 11 | 12 | homepage = "https://github.com/bcumming/affinity" 13 | git = "https://github.com/bcumming/affinity.git" 14 | version("master", branch="master") 15 | 16 | maintainers("bcumming", "nhanford") 17 | 18 | license("BSD-3-Clause", checked_by="nhanford") 19 | 20 | variant("mpi", default=False, description="Build MPI support") 21 | variant("rocm", default=False, description="Build ROCm Support") 22 | 23 | depends_on("c", type="build") 24 | depends_on("cxx", type="build") 25 | 26 | depends_on("mpi", when="+mpi") 27 | depends_on("hip", when="+rocm") 28 | depends_on("cuda", when="+cuda") 29 | 30 | def cmake_args(self): 31 | spec = self.spec 32 | args = [] 33 | 34 | if '+mpi' in spec: 35 | args.append('-DCMAKE_CXX_COMPILER={0}'.format(spec["mpi"].mpicxx)) 36 | args.append("-DMPI_CXX_LINK_FLAGS={0}".format(spec["mpi"].libs.ld_flags)) 37 | args.append("-DAFFINITY_MPI=on") 38 | 39 | if '+cuda' in spec: 40 | args.append('-DCMAKE_CUDA_HOST_COMPILER={0}'.format(spec["mpi"].mpicxx)) 41 | args.append('-DCMAKE_CUDA_COMPILER={0}'.format(spec["cuda"].prefix + "/bin/nvcc")) 42 | args.append("-DAFFINITY_GPU_BACKEND=cuda") 43 | 44 | if '+rocm' in spec: 45 | args.append("-DROCM_PATH={0}".format(spec['hip'].prefix)) 46 | args.append("-DAFFINITY_GPU_BACKEND=rocm") 47 | rocm_archs = spec.variants["amdgpu_target"].value 48 | if "none" not in rocm_archs: 49 | args.append("-DHIP_HIPCC_FLAGS=--amdgpu-target={0}".format(",".join(rocm_archs))) 50 | args.append("-DCMAKE_HIP_ARCHITECTURES={0}".format(rocm_archs)) 51 | 52 | return args 53 | -------------------------------------------------------------------------------- /repo/blt/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | import re 8 | 9 | from spack.package import * 10 | from spack_repo.builtin.packages.blt.package import Blt as BuiltinBlt 11 | 12 | 13 | class Blt(BuiltinBlt): 14 | 15 | version("0.7.0", sha256="df8720a9cba1199d21f1d32649cebb9dddf95aa61bc3ac23f6c8a3c6b6083528") -------------------------------------------------------------------------------- /repo/branson/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Branson(ExecutableApplication): 12 | """Branson benchmark""" 13 | name = "branson" 14 | 15 | tags = ['asc','montecarlo','particles', 16 | 'high-branching','irregular-memory-access', 17 | 'mpi','c++'] 18 | 19 | executable('setup_experiment', 20 | template=[ 21 | 'cp {branson}/inputs/* {experiment_run_dir}/.', 22 | 'sed -i "s|250000000|{num_particles}|g" {experiment_run_dir}/{input_file}' 23 | ]) 24 | 25 | executable('p', '{branson}/bin/BRANSON {experiment_run_dir}/{input_file}', use_mpi=True) 26 | 27 | workload('branson', executables=['setup_experiment','p']) 28 | 29 | workload_variable('input_file', default='3D_hohlraum_multi_node.xml', 30 | description='input file name', 31 | workloads=['branson']) 32 | 33 | 34 | workload_variable('num_particles', default='250000000', 35 | description='procs on node', 36 | workloads=['branson']) 37 | 38 | 39 | figure_of_merit('Photons per Second', 40 | log_file='{experiment_run_dir}/{experiment_name}.out', 41 | fom_regex=r'Photons Per Second \(FOM\):\s+(?P[0-9]+\.[0-9]*([0-9]*)?e\+[0-9]*)', 42 | group_name='fom', units='photons') 43 | 44 | success_criteria('pass', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out') 45 | -------------------------------------------------------------------------------- /repo/branson/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.boost.package import Boost 8 | 9 | 10 | class Branson(CMakePackage): 11 | """Branson's purpose is to study different algorithms for parallel Monte 12 | Carlo transport. Currently it contains particle passing and mesh passing 13 | methods for domain decomposition.""" 14 | 15 | homepage = "https://github.com/lanl/branson" 16 | url = "https://github.com/lanl/branson/archive/0.82.tar.gz" 17 | git = "https://github.com/lanl/branson.git" 18 | 19 | tags = ["proxy-app"] 20 | 21 | license("MIT") 22 | 23 | version("develop", branch="develop") 24 | 25 | version("1.01", sha256="90208eaec4f6d64a4fd81cd838e30b5e7207246cb7f407e482965f23bbcee848") 26 | version( 27 | "0.82", 28 | sha256="7d83d41d0c7ab9c1c906a902165af31182da4604dd0b69aec28d709fe4d7a6ec", 29 | preferred=True, 30 | ) 31 | version("0.81", sha256="493f720904791f06b49ff48c17a681532c6a4d9fa59636522cf3f9700e77efe4") 32 | version("0.8", sha256="85ffee110f89be00c37798700508b66b0d15de1d98c54328b6d02a9eb2cf1cb8") 33 | 34 | depends_on("c", type="build") 35 | depends_on("cxx", type="build") 36 | 37 | #depends_on("mpi") 38 | depends_on("mpi@2:") 39 | 40 | # TODO: replace this with an explicit list of components of Boost, 41 | # for instance depends_on('boost +filesystem') 42 | # See https://github.com/spack/spack/pull/22303 for reference 43 | depends_on(Boost.with_default_variants, when="@:0.81") 44 | depends_on("metis") 45 | depends_on("parmetis", when="@:0.81") 46 | 47 | root_cmakelists_dir = "src" 48 | 49 | def cmake_args(self): 50 | spec = self.spec 51 | args = [] 52 | #args.append("--enable-mpi") 53 | args.append(f"-DCMAKE_C_COMPILER={spec['mpi'].mpicc}") 54 | args.append(f"-DCMAKE_CXX_COMPILER={spec['mpi'].mpicxx}") 55 | args.append(f"-DCMAKE_Fortran_COMPILER={spec['mpi'].mpifc}") 56 | return args 57 | 58 | def install(self, spec, prefix): 59 | mkdir(prefix.bin) 60 | mkdir(prefix.doc) 61 | install(join_path(self.build_directory, "BRANSON"), prefix.bin) 62 | install("LICENSE.md", prefix.doc) 63 | install("README.md", prefix.doc) 64 | install_tree("inputs", prefix.inputs) 65 | -------------------------------------------------------------------------------- /repo/caliper/for_aarch64.patch: -------------------------------------------------------------------------------- 1 | --- spack-src/src/services/callpath/Callpath.cpp.bak 2020-10-28 14:38:19.668122844 +0900 2 | +++ spack-src/src/services/callpath/Callpath.cpp 2020-10-28 15:03:12.258061188 +0900 3 | @@ -63,7 +63,7 @@ 4 | unw_context_t unw_ctx; 5 | unw_cursor_t unw_cursor; 6 | 7 | - unw_getcontext(&unw_ctx); 8 | + unw_getcontext(unw_ctx); 9 | 10 | if (unw_init_local(&unw_cursor, &unw_ctx) < 0) { 11 | Log(0).stream() << "callpath: unable to init libunwind cursor" << endl; 12 | -------------------------------------------------------------------------------- /repo/caliper/sampler-service-missing-libunwind-include-dir.patch: -------------------------------------------------------------------------------- 1 | diff -ruN spack-src/src/services/sampler/CMakeLists.txt spack-src-patched/src/services/sampler/CMakeLists.txt 2 | --- spack-src/src/services/sampler/CMakeLists.txt 2022-11-30 13:52:42.000000000 -0500 3 | +++ spack-src-patched/src/services/sampler/CMakeLists.txt 2023-05-04 20:43:47.240310306 -0400 4 | @@ -17,6 +17,10 @@ 5 | 6 | include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) 7 | 8 | +if (CALIPER_HAVE_LIBUNWIND) 9 | + include_directories(${LIBUNWIND_INCLUDE_DIRS}) 10 | +endif() 11 | + 12 | add_library(caliper-sampler OBJECT ${CALIPER_SAMPLER_SOURCES}) 13 | 14 | add_service_objlib("caliper-sampler") 15 | -------------------------------------------------------------------------------- /repo/camp/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.camp.package import Camp as BuiltinCamp 8 | 9 | 10 | class Camp(BuiltinCamp): 11 | 12 | def setup_build_environment(self, env): 13 | super().setup_build_environment(env) 14 | if "+cuda" in self.spec: 15 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 16 | 17 | def setup_run_environment(self, env): 18 | super().setup_run_environment(env) 19 | if "+cuda" in self.spec: 20 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 21 | -------------------------------------------------------------------------------- /repo/chai/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.chai.package import Chai as BuiltinChai 8 | 9 | 10 | class Chai(BuiltinChai): 11 | variant("single_memory", default=False, description="Enable single memory space model") 12 | 13 | conflicts("+single_memory", when="~rocm") 14 | 15 | def initconfig_hardware_entries(self): 16 | spec = self.spec 17 | entries = super().initconfig_hardware_entries() 18 | 19 | if spec.satisfies("+single_memory"): 20 | entries.append(cmake_cache_option("CHAI_THIN_GPU_ALLOCATE", True)) 21 | entries.append(cmake_cache_option("CHAI_DISABLE_RM", True)) 22 | 23 | return entries 24 | 25 | def setup_build_environment(self, env): 26 | super().setup_build_environment(env) 27 | if "+cuda" in self.spec: 28 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 29 | 30 | def setup_run_environment(self, env): 31 | super().setup_run_environment(env) 32 | if "+cuda" in self.spec: 33 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 34 | -------------------------------------------------------------------------------- /repo/commbench/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | from ramble.expander import Expander 10 | 11 | 12 | class Commbench(ExecutableApplication): 13 | name = "commbench" 14 | 15 | maintainers("arhag23") 16 | 17 | software_spec("commbench", pkg_spec="commbench", package_manager="spack*") 18 | 19 | executable("execute-bench", "CommBench --use {lib} --pattern {pat}", use_mpi=True) 20 | # executable("execute-test", "mpiexec -n {n_ranks} CommBench --use {lib} --pattern {pat} --validate", use_mpi=True) 21 | executable("execute-test", "CommBench --use {lib} --pattern {pat} --validate", use_mpi=True) 22 | # executable("execute-test3", "mpiexec -n {n_ranks} CommBench --use {lib} --pattern {pat} --validate") 23 | # executable("execute-test4", "mpiexec CommBench --use {lib} --pattern {pat} --validate", use_mpi=True) 24 | 25 | workload("basic", executables=["execute-test"]) 26 | 27 | workload_variable("lib", default="mpi", values=["mpi", "ipc_get", "ipc_put", "xccl"], description="Communication library to benchmark", workload="basic") 28 | workload_variable("pat", default="p2p", values=["p2p", "gather", "scatter", "broadcast", "alltoall", "allgather"], description="Communication pattern", workload="basic") 29 | # workload_variable("mpi_command", default="mpirun -n {n_ranks}", description="mpi command", workload="basic") 30 | 31 | # success_criteria("pass", "string", match="[\s\S.]*PASSED![\s\S.]*") 32 | -------------------------------------------------------------------------------- /repo/commbench/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class Commbench(CMakePackage, CudaPackage, ROCmPackage): 10 | """FIXME: Put a proper description of your package here.""" 11 | 12 | # FIXME: Add a proper url for your package's homepage here. 13 | homepage = "https://github.com/simongdg/CommBench" 14 | # url = "https://github.com/arhag23/CommBench/archive/cmake.tar.gz" 15 | git = "https://github.com/simongdg/CommBench.git" 16 | 17 | version("0.1.0", commit="efed373") 18 | 19 | # FIXME: Add a list of GitHub accounts to 20 | # notify when the package is updated. 21 | maintainers("simongdg") 22 | 23 | # FIXME: Add the SPDX identifier of the project's license below. 24 | # See https://spdx.org/licenses/ for a list. Upon manually verifying 25 | # the license, set checked_by to your Github username. 26 | license("Apache-2.0", checked_by="simongdg") 27 | 28 | version("0.1.0") 29 | variant("cuda", default=False, description="Enable CUDA support") 30 | variant("rocm", default=False, description="Enable HIP support") 31 | variant("caliper", default=True, description="Enable Caliper profiling") 32 | 33 | # FIXME: Add dependencies if required. 34 | depends_on("c", type="build") 35 | depends_on("cxx", type="build") 36 | 37 | depends_on("cmake@3.24:", type="build") 38 | 39 | depends_on("mpi") 40 | 41 | depends_on("rccl", when="+rocm") 42 | depends_on("caliper", when="+caliper") 43 | 44 | for arch in ("none", "50", "60", "70", "80", "90"): 45 | depends_on(f"nccl cuda_arch={arch}", when=f"cuda_arch={arch}") 46 | 47 | def cmake_args(self): 48 | args = [] 49 | # args.append("--debug-trycompile") 50 | if self.spec.satisfies("+rocm"): 51 | args.append("-DUSE_HIP=ON") 52 | args.append("-DUSE_XCCL=ON") 53 | elif self.spec.satisfies("+cuda"): 54 | args.append("-DUSE_CUDA=ON") 55 | args.append("-DUSE_XCCL=ON") 56 | 57 | cuda_arch = self.spec.variants["cuda_arch"].value[0] 58 | if cuda_arch is not None: 59 | args.append(f"-DCMAKE_CUDA_ARCHITECTURES={cuda_arch}") 60 | 61 | mpi_spec = self.spec["mpi"] 62 | 63 | if mpi_spec.satisfies("+gtl"): 64 | gtl_path = mpi_spec.extra_attributes["gtl_path"] 65 | args.append("-DUSE_GTL=ON") 66 | args.append(f"-DGTL_PATH={gtl_path}") 67 | 68 | if self.spec.satisfies("+caliper"): 69 | args.append("-DUSE_CALIPER=ON") 70 | 71 | return args 72 | -------------------------------------------------------------------------------- /repo/cray-mpich/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.cray_mpich.package import CrayMpich as BuiltinCM 8 | 9 | 10 | class CrayMpich(BuiltinCM): 11 | 12 | variant("gtl", default=True, description="enable GPU-aware mode") 13 | 14 | @property 15 | def libs(self): 16 | libs = super().libs 17 | 18 | if self.spec.satisfies("+gtl"): 19 | gtl_lib_prefix = self.spec.extra_attributes["gtl_lib_path"] 20 | # gtl_libs, if set, must be a single string. You can pass multiple 21 | # libs by adding a space between each 22 | gtl_libs = self.spec.extra_attributes["gtl_libs"].split() 23 | libs += find_libraries(gtl_libs, root=gtl_lib_prefix, recursive=True) 24 | 25 | return libs 26 | 27 | def setup_run_environment(self, env): 28 | 29 | super().setup_run_environment(env) 30 | 31 | if self.spec.satisfies("+gtl"): 32 | env.set("MPICH_GPU_SUPPORT_ENABLED", "1") 33 | env.prepend_path("LD_LIBRARY_PATH", self.spec.extra_attributes["gtl_lib_path"]) 34 | else: 35 | env.set("MPICH_GPU_SUPPORT_ENABLED", "0") 36 | gtl_path = self.spec.extra_attributes.get("gtl_lib_path", "") 37 | if gtl_path: 38 | env.prepend_path("LD_LIBRARY_PATH", gtl_path) 39 | 40 | def cmake_args(self): 41 | args = super().cmake_args(self) 42 | 43 | if self.spec.satisfies("+gtl"): 44 | # Link GTL for MPICH GPU-aware 45 | args.append(self.define("CMAKE_EXE_LINKER_FLAGS", self.spec['mpi'].libs.ld_flags)) 46 | 47 | return args 48 | -------------------------------------------------------------------------------- /repo/cuda/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import pathlib 7 | 8 | import llnl.util.tty as tty 9 | import spack_repo.builtin.packages.cuda.package 10 | from llnl.util.filesystem import find_headers 11 | from spack.package import * 12 | 13 | 14 | class Cuda(spack_repo.builtin.packages.cuda.package.Cuda): 15 | # Layout of hpc-sdk puts some headers in sibling directories: 16 | # cuda compiler in /opt/nvidia/hpc_sdk/Linux_aarch64/24.7/cuda/12.5 17 | # cufft in /opt/nvidia/hpc_sdk/Linux_aarch64/24.7/math_libs/12.5 18 | # In this case, we assume that the external prefix is set to the first path 19 | variant("im-hpc-sdk", default=False) 20 | 21 | @property 22 | def headers(self): 23 | home = getattr(self.spec.package, "home") 24 | headers = find_headers("*", root=home.include, recursive=True) 25 | 26 | if self.spec.satisfies("+im-hpc-sdk"): 27 | prefix = pathlib.Path(self.prefix) 28 | version_component = prefix.name # 12.5 29 | split_point = prefix.parent.parent 30 | cufft_base = split_point / "math_libs" / version_component 31 | #tty.debug(f"<---- {prefix}\n\t{split_point}\n\t{cufft_base}") 32 | headers = headers + find_headers("cufft", root=str(cufft_base), recursive=True) 33 | 34 | return headers 35 | -------------------------------------------------------------------------------- /repo/enzyme/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class Enzyme(CMakePackage): 10 | """Enzyme""" 11 | 12 | git = "https://github.com/jandrej/Enzyme.git" 13 | 14 | tags = ["enzyme"] 15 | 16 | license("Apache-2.0") 17 | 18 | maintainers("jandrej") 19 | 20 | version("patch-1", branch="patch-1") 21 | 22 | depends_on("c", type="build") 23 | depends_on("cxx", type="build") 24 | depends_on("fortran", type="build") 25 | 26 | depends_on("llvm@18:") 27 | 28 | root_cmakelists_dir = "enzyme" 29 | 30 | def cmake_args(self): 31 | cmake_options = [ 32 | f"-DLLVM_DIR={self.spec['llvm'].prefix}", 33 | ] 34 | 35 | return cmake_options 36 | -------------------------------------------------------------------------------- /repo/essl/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.essl.package import Essl as BuiltinEssl 8 | 9 | 10 | class Essl(BuiltinEssl): 11 | @property 12 | def lapack_libs(self): 13 | essl_libs = super(Essl, self).lapack_libs 14 | 15 | essl_libs += find_libraries( 16 | ["libessl", "libessl6464", "libesslsmp", "libesslsmp6464"], root=self.prefix.lib64, shared=True 17 | ) 18 | return essl_libs 19 | -------------------------------------------------------------------------------- /repo/gpcnet/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Gpcnet(ExecutableApplication): 12 | """GPCNet benchmark""" 13 | name = "GPCNet" 14 | 15 | executable('p1', 'network_test', use_mpi=True) 16 | executable('p2', 'network_load_test', use_mpi=True) 17 | workload('network_test', executables=['p1']) 18 | workload('network_load_test', executables=['p2']) 19 | 20 | figure_of_merit('Avg Multiple Allreduce', 21 | log_file='{experiment_run_dir}/{experiment_name}.out', 22 | fom_regex=r'\|\s+Multiple Allreduce \([0-9]* B\)\s+\|\s+(?P[0-9]+\.[0-9]*)', 23 | group_name='fom', units='MiB/sec') 24 | figure_of_merit('Avg RR Two-sided Lat', 25 | log_file='{experiment_run_dir}/{experiment_name}.out', 26 | fom_regex=r'\|\s+RR Two-sided Lat \([0-9]* B\)\s+\|\s+(?P[0-9]+\.[0-9]*)', 27 | group_name='fom', units='MiB/sec') 28 | figure_of_merit('Avg RR Get Lat', 29 | log_file='{experiment_run_dir}/{experiment_name}.out', 30 | fom_regex=r'\|\s+RR Get Lat \([0-9]* B\)\s+\|\s+(?P[0-9]+\.[0-9]*)', 31 | group_name='fom', units='MiB/sec') 32 | success_criteria('pass', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out') 33 | -------------------------------------------------------------------------------- /repo/gromacs/gmxDetectCpu-cmake-3.14.patch: -------------------------------------------------------------------------------- 1 | --- a/cmake/gmxDetectCpu.cmake 2 | +++ b/cmake/gmxDetectCpu.cmake 3 | @@ -83,7 +83,7 @@ function(gmx_run_cpu_detection TYPE) 4 | set(GCC_INLINE_ASM_DEFINE "-DGMX_X86_GCC_INLINE_ASM=0") 5 | endif() 6 | 7 | - set(_compile_definitions "${GCC_INLINE_ASM_DEFINE} -I${PROJECT_SOURCE_DIR}/src -DGMX_CPUINFO_STANDALONE ${GMX_STDLIB_CXX_FLAGS} -DGMX_TARGET_X86=${GMX_TARGET_X86_VALUE}") 8 | + set(_compile_definitions ${GCC_INLINE_ASM_DEFINE} -I${PROJECT_SOURCE_DIR}/src -DGMX_CPUINFO_STANDALONE ${GMX_STDLIB_CXX_FLAGS} -DGMX_TARGET_X86=${GMX_TARGET_X86_VALUE}) 9 | set(LINK_LIBRARIES "${GMX_STDLIB_LIBRARIES}") 10 | try_compile(CPU_DETECTION_COMPILED 11 | "${PROJECT_BINARY_DIR}" 12 | 13 | -------------------------------------------------------------------------------- /repo/gromacs/gmxDetectSimd-cmake-3.14.patch: -------------------------------------------------------------------------------- 1 | --- a/cmake/gmxDetectSimd.cmake 2 | +++ b/cmake/gmxDetectSimd.cmake 3 | @@ -77,7 +77,7 @@ function(gmx_suggest_simd _suggested_simd) 4 | else() 5 | set(GMX_TARGET_X86_VALUE 0) 6 | endif() 7 | - set(_compile_definitions "${GCC_INLINE_ASM_DEFINE} -I${CMAKE_SOURCE_DIR}/src -DGMX_CPUINFO_STANDALONE ${GMX_STDLIB_CXX_FLAGS} -DGMX_TARGET_X86=${GMX_TARGET_X86_VALUE}") 8 | + set(_compile_definitions ${GCC_INLINE_ASM_DEFINE} -I${CMAKE_SOURCE_DIR}/src -DGMX_CPUINFO_STANDALONE ${GMX_STDLIB_CXX_FLAGS} -DGMX_TARGET_X86=${GMX_TARGET_X86_VALUE}) 9 | 10 | # Prepare a default suggestion 11 | set(OUTPUT_SIMD "None") 12 | -------------------------------------------------------------------------------- /repo/hpcc/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.app.builtin.hpcc import Hpcc as HpccBase 9 | from ramble.appkit import * 10 | 11 | 12 | class Hpcc(HpccBase): 13 | 14 | tags = ['synthetic', 15 | 'blas','solver','dense-linear-algebra','fft', 16 | 'large-scale', 17 | 'high-fp','high-memory-bandwidth', 18 | 'regular-memory-access','irregular-memory-access', 19 | 'mpi','network-collectives','network-point-to-point', 20 | 'network-bandwidth-bound','network-bisection-bandwidth-bound', 21 | 'network-latency-bound', 22 | 'c','fortran'] 23 | -------------------------------------------------------------------------------- /repo/hpcg/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | import platform 8 | 9 | from spack.package import * 10 | 11 | 12 | class Hpcg(CMakePackage): 13 | """HPCG is a software package that performs a fixed number of multigrid 14 | preconditioned (using a symmetric Gauss-Seidel smoother) conjugate gradient 15 | (PCG) iterations using double precision (64 bit) floating point values.""" 16 | 17 | #homepage = "https://www.hpcg-benchmark.org" 18 | #url = "https://www.hpcg-benchmark.org/downloads/hpcg-3.1.tar.gz" 19 | git = "https://github.com/daboehme/hpcg.git" 20 | 21 | version("develop", branch="master") 22 | #version("3.1", sha256="33a434e716b79e59e745f77ff72639c32623e7f928eeb7977655ffcaade0f4a4") 23 | version("caliper", branch="caliper-support") 24 | 25 | variant("openmp", default=True, description="Enable OpenMP support") 26 | variant("caliper", default=False, description="Enable Caliper support") 27 | 28 | depends_on("c", type="build") 29 | depends_on("cxx", type="build") 30 | depends_on("fortran", type="build") 31 | 32 | depends_on("mpi@1.1:") 33 | depends_on("caliper", when="+caliper") 34 | depends_on("adiak", when="+caliper") 35 | 36 | def cmake_args(self): 37 | build_targets = ["all", "docs"] 38 | install_targets = ["install", "docs"] 39 | args = [ 40 | "-DHPCG_ENABLE_MPI=TRUE", 41 | self.define_from_variant("HPCG_ENABLE_CALIPER", "caliper"), 42 | self.define_from_variant("HPCG_ENABLE_OPENMP", "openmp"), 43 | ] 44 | 45 | return args 46 | -------------------------------------------------------------------------------- /repo/hpl/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.app.builtin.hpl import Hpl as HplBase 9 | from ramble.appkit import * 10 | 11 | 12 | class Hpl(HplBase): 13 | 14 | tags = ['synthetic', 15 | 'blas','solver','dense-linear-algebra', 16 | 'large-scale', 17 | 'mpi','network-collectives','network-point-to-point', 18 | 'c'] 19 | -------------------------------------------------------------------------------- /repo/hypre/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import os 7 | 8 | from spack.package import * 9 | from spack_repo.builtin.packages.hypre.package import Hypre as BuiltinHypre 10 | 11 | 12 | class Hypre(BuiltinHypre): 13 | requires("+rocm", when="^rocblas") 14 | requires("+rocm", when="^rocsolver") 15 | 16 | compiler_to_cpe_name = { 17 | "cce": "cray", 18 | "gcc": "gnu", 19 | } 20 | 21 | def setup_build_environment(self, env): 22 | if self.spec.satisfies('%oneapi'): 23 | compiler_version = self.spec.compiler.version 24 | 25 | # Convert to integer for comparison (e.g., 2023.1.0 -> 2023) 26 | major_version = int(str(compiler_version).split('.')[0]) 27 | 28 | if major_version < 2023: 29 | raise RuntimeError( 30 | "This package requires Intel oneAPI compiler version 2023 or newer. " 31 | "Detected version: {0}".format(compiler_version) 32 | ) 33 | 34 | super().setup_build_environment(env) 35 | 36 | spec = self.spec 37 | if "+mpi" in spec: 38 | if "+fortran" in spec: 39 | env.set("FC", spec["mpi"].mpifc) 40 | if spec["mpi"].extra_attributes and "ldflags" in spec["mpi"].extra_attributes: 41 | env.append_flags("LDFLAGS", spec["mpi"].extra_attributes["ldflags"]) 42 | if spec["lapack"].satisfies("rocsolver"): 43 | rocm_rpath_flag = f"-Wl,-rpath,{os.path.dirname(spec['lapack'].prefix)}/lib" 44 | env.append_flags("LDFLAGS", rocm_rpath_flag) 45 | if spec["lapack"].satisfies("cray-libsci"): 46 | libsci_name = "sci_" 47 | libsci_name += self.compiler_to_cpe_name[spec.compiler.name] 48 | if spec.satisfies("+mpi"): 49 | libsci_name += "_mpi" 50 | if spec.satisfies("+openmp"): 51 | libsci_name += "_mp" 52 | env.append_flags("LDFLAGS", f"-L{spec['lapack'].prefix}/lib -l{libsci_name}") 53 | if "+cuda" in self.spec: 54 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") -------------------------------------------------------------------------------- /repo/kokkos/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.kokkos.package import Kokkos as BuiltinKokkos 8 | 9 | 10 | class Kokkos(BuiltinKokkos): 11 | flag_handler = build_system_flags 12 | 13 | def setup_build_environment(self, env): 14 | if "+cuda" in self.spec: 15 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 16 | -------------------------------------------------------------------------------- /repo/laghos/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Laghos(ExecutableApplication): 12 | """Laghos benchmark""" 13 | name = "laghos" 14 | 15 | tags = ['asc','engineering','hypre','solver','mfem','cfd','large-scale', 16 | 'multi-node','single-node','mpi','c++','high-order','hydrodynamics', 17 | 'explicit-timestepping','finite-element','time-dependent','ode', 18 | 'full-assembly','partial-assembly', 19 | 'lagrangian','spatial-discretization','unstructured-grid', 20 | 'network-latency-bound','network-collectives','unstructured-grid'] 21 | 22 | executable('prob', 'laghos -p {problem} -m {mesh} -rs {rs} -rp {rp} -ms {ms} -d {device}', use_mpi=True) 23 | 24 | workload('triplept', executables=['prob']) 25 | 26 | workload_variable('mesh', default='{laghos}/data/box01_hex.mesh', 27 | description='mesh file', 28 | workloads=['triplept']) 29 | 30 | workload_variable('problem', default='3', 31 | description='problem number', 32 | workloads=['triplept']) 33 | 34 | workload_variable('rs', default='2', 35 | description='number of serial refinements', 36 | workloads=['triplept']) 37 | 38 | workload_variable('rp', default='0', 39 | description='number of parallel refinements', 40 | workloads=['triplept']) 41 | 42 | workload_variable('ms', default='250', 43 | description='max number of steps', 44 | workloads=['triplept']) 45 | workload_variable('device', default='cpu', 46 | description='cpu or cuda', 47 | workloads=['triplept']) 48 | 49 | figure_of_merit('Major kernels total time', 50 | log_file='{experiment_run_dir}/{experiment_name}.out', 51 | fom_regex=r'Major kernels total time \(seconds\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', 52 | group_name='fom', units='seconds') 53 | 54 | success_criteria('pass', mode='string', match=r'Major kernels total time', file='{experiment_run_dir}/{experiment_name}.out') 55 | -------------------------------------------------------------------------------- /repo/lammps/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.lammps.package import Lammps as BuiltinLammps 8 | 9 | 10 | class Lammps(BuiltinLammps): 11 | 12 | variant("pace", default=False, description="Enable the ML PACE module") 13 | depends_on("pace", when="+pace") 14 | 15 | depends_on("kokkos+openmp cxxstd=17", when="+openmp") 16 | depends_on("kokkos+wrapper", when="+cuda") 17 | 18 | flag_handler = build_system_flags 19 | 20 | def setup_run_environment(self, env): 21 | super().setup_run_environment(env) 22 | 23 | if self.compiler.extra_rpaths: 24 | for rpath in self.compiler.extra_rpaths: 25 | env.prepend_path("LD_LIBRARY_PATH", rpath) 26 | 27 | def setup_build_environment(self, env): 28 | super().setup_build_environment(env) 29 | 30 | if "+cuda" in self.spec: 31 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 32 | 33 | if "+mpi" in self.spec: 34 | if self.spec["mpi"].extra_attributes and "ldflags" in self.spec["mpi"].extra_attributes: 35 | env.append_flags("LDFLAGS", self.spec["mpi"].extra_attributes["ldflags"]) 36 | 37 | def cmake_args(self): 38 | args = super().cmake_args() 39 | args.append(f"-DMPI_CXX_LINK_FLAGS='{self.spec['mpi'].libs.ld_flags}'") 40 | args.append(f"-DMPI_C_COMPILER='{self.spec['mpi'].mpicc}'") 41 | args.append(f"-DMPI_CXX_COMPILER={self.spec['mpi'].mpicxx}") 42 | if "+pace" in self.spec: 43 | args.append(f"-DPKG_ML-PACE=ON") 44 | 45 | return args 46 | 47 | def install(self, spec, prefix): 48 | super().install(spec, prefix) 49 | mkdirp(prefix.src) 50 | install_tree(self.stage.source_path, prefix.src) 51 | -------------------------------------------------------------------------------- /repo/lapack-xl/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class LapackXl(Package): 10 | 11 | provides("lapack") 12 | 13 | @property 14 | def lapack_libs(self): 15 | libname = ["liblapack", "libblas"] 16 | return find_libraries(libname, root=self.prefix.lib, shared=False, recursive=False) 17 | 18 | @property 19 | def libs(self): 20 | libname = ["liblapack", "libblas"] 21 | return find_libraries(libname, root=self.prefix.lib, shared=False, recursive=False) 22 | -------------------------------------------------------------------------------- /repo/md-test/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.app.builtin.md_test import MdTest as MdTestBase 9 | from ramble.appkit import * 10 | 11 | 12 | class MdTest(MdTestBase): 13 | 14 | tags = ['synthetic','i-o','large-scale','mpi','c'] 15 | -------------------------------------------------------------------------------- /repo/osu-micro-benchmarks/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.app.builtin.osu_micro_benchmarks import ( 9 | OsuMicroBenchmarks as OsuMicroBenchmarksBase, 10 | ) 11 | from ramble.appkit import * 12 | 13 | 14 | class OsuMicroBenchmarks(OsuMicroBenchmarksBase): 15 | 16 | tags = ['synthetic', 17 | 'large-scale','multi-node','single-node', 18 | 'atomics','managed-memory', 19 | 'mpi','openshmem','upc','upc++','nccl','rccl', 20 | 'network-bandwidth-bound','network-bisection-bandwidth-bound', 21 | 'network-collectives','network-latency-bound', 22 | 'network-multi-threaded','network-nonblocking-collectives', 23 | 'network-onesided','network-point-to-point', 24 | 'c','java','python','openacc'] 25 | -------------------------------------------------------------------------------- /repo/osu-micro-benchmarks/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.osu_micro_benchmarks.package import ( 8 | OsuMicroBenchmarks as BuiltinOsu, 9 | ) 10 | 11 | 12 | class OsuMicroBenchmarks(BuiltinOsu, ROCmPackage): 13 | 14 | depends_on("cray-mpich+gtl", when="+rocm") 15 | 16 | def configure_args(self): 17 | args = super().configure_args() 18 | if self.spec.satisfies("+rocm"): 19 | args.extend([f"LDFLAGS={self.spec['mpi'].libs.ld_flags}"]) 20 | print(self.spec['mpi']) 21 | new_args = list() 22 | for x in args: 23 | if "NVCCFLAGS" in x and self.spec.satisfies("%intel-oneapi-compilers"): 24 | new_args.append(x + " -allow-unsupported-compiler") 25 | else: 26 | new_args.append(x) 27 | return new_args 28 | 29 | def setup_run_environment(self, env): 30 | mpidir = join_path(self.prefix.libexec, "osu-micro-benchmarks", "mpi") 31 | env.prepend_path("PATH", join_path(mpidir, "startup")) 32 | env.prepend_path("PATH", join_path(mpidir, "pt2pt")) 33 | env.prepend_path("PATH", join_path(mpidir, "one-sided")) 34 | env.prepend_path("PATH", join_path(mpidir, "collective")) 35 | if self.spec.satisfies("+rocm"): 36 | if 'gtl_flags' in self.spec['mpi'].extra_attributes: 37 | env.prepend_path("LOCAL_RANK", self.spec['mpi'].extra_attributes['gtl_flags']) 38 | -------------------------------------------------------------------------------- /repo/phloem/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Phloem(ExecutableApplication): 12 | """Phloem benchmark""" 13 | name = "Phloem" 14 | 15 | tags = ['synthetic', 16 | 'large-scale','multi-node','single-node', 17 | 'c','mpi','network-point-to-point', 18 | 'network-bandwidth-bound','network-latency-bound', 19 | 'network-collectives','network-nonblocking-collectives'] 20 | 21 | executable('p1', 'sqmr --num_cores={num_cores} --num_nbors={num_nbors}', use_mpi=True) 22 | executable('p2', 'mpiBench', use_mpi=True) 23 | executable('p3', 'mpiGraph', use_mpi=True) 24 | #executable('p4', 'com', use_mpi=True) 25 | workload('sqmr', executables=['p1']) 26 | workload('mpiBench', executables=['p2']) 27 | workload('mpiGraph', executables=['p3']) 28 | #workload('com', executables=['p4']) 29 | 30 | workload_variable('num_cores', default='1', 31 | description='Number of MPI ranks on the core node, correlates to number of cores on one compute node.', 32 | workloads=['sqmr']) 33 | 34 | workload_variable('num_nbors', default='1', 35 | description='Number of distinct neighbors to each rank on the core node.', 36 | workloads=['sqmr']) 37 | #TODO: Determine FOMs 38 | # figure_of_merit('TBD', 39 | # log_file='{experiment_run_dir}/{experiment_name}.out', 40 | # fom_regex=r'.*', 41 | # group_name='fom', units='usec') 42 | success_criteria('pass', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out') 43 | -------------------------------------------------------------------------------- /repo/phloem/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class Phloem(MakefilePackage): 10 | tags = [] 11 | 12 | url = "https://github.com/LLNL/phloem/archive/refs/tags/v1.4.5.tar.gz" 13 | git = "https://github.com/LLNL/phloem" 14 | 15 | maintainers("knox10") 16 | 17 | version("master", branch="master") 18 | version("1.4.5", tag="v1.4.5") 19 | 20 | variant("mpi", default=False, description="Build with MPI support") 21 | 22 | depends_on("c", type="build") 23 | depends_on("cxx", type="build") 24 | depends_on("fortran", type="build") 25 | 26 | depends_on("mpi", when="+mpi") 27 | 28 | def install(self, spec, prefix): 29 | mkdir(prefix.bin) 30 | mkdir(prefix.doc) 31 | install("mpigraph-1.6/mpiBench/mpiBench", prefix.bin) 32 | install("sqmr-1.1.0/sqmr", prefix.bin) 33 | install("mpigraph-1.6/mpiGraph/mpiGraph", prefix.bin) 34 | install("presta-1.3.0/com", prefix.bin) 35 | install("presta-1.3.0/bw.message.sizes", prefix.bin) 36 | install("presta-1.3.0/latency.message.sizes", prefix.bin) 37 | install("README", prefix.doc) 38 | -------------------------------------------------------------------------------- /repo/raja-perf/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class RajaPerf(ExecutableApplication): 12 | """RAJA Performance suite""" 13 | name = "raja-perf" 14 | 15 | tags = ['asc','single-node','sub-node','structured-grid', 16 | 'atomics','simd','vectorization','register-pressure', 17 | 'high-memory-bandwidth','regular-memory-access', 18 | 'mpi','network-point-to-point','network-latency-bound', 19 | 'c++','raja','sycl','builtin-caliper'] 20 | 21 | executable('run', 'raja-perf.exe --size {process_problem_size} -atsc ${CALI_CONFIG_MODE} -atcc ${OTHER_CALI_CONFIG}', use_mpi=True) 22 | 23 | workload('suite', executables=['run']) 24 | 25 | figure_of_merit('All tests pass', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'(?PDONE)!!!...', group_name='tpass', units='') 26 | 27 | success_criteria('pass', mode='string', match=r'DONE!!!....', file='{experiment_run_dir}/{experiment_name}.out') 28 | -------------------------------------------------------------------------------- /repo/raja/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.raja.package import Raja as BuiltinRaja 8 | 9 | 10 | class Raja(BuiltinRaja): 11 | 12 | def setup_build_environment(self, env): 13 | super().setup_build_environment(env) 14 | if "+cuda" in self.spec: 15 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 16 | 17 | def setup_run_environment(self, env): 18 | super().setup_run_environment(env) 19 | if "+cuda" in self.spec: 20 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 21 | -------------------------------------------------------------------------------- /repo/repo.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | repo: 6 | namespace: benchpark 7 | subdirectory: '' 8 | -------------------------------------------------------------------------------- /repo/salmon-tddft/fjmpi.patch: -------------------------------------------------------------------------------- 1 | --- spack-src.org/CMakeLists.txt 2020-07-21 12:08:53.000000000 +0900 2 | +++ spack-src/CMakeLists.txt 2020-12-23 11:32:57.147128677 +0900 3 | @@ -38,6 +38,7 @@ 4 | option_set(USE_SCALAPACK "Use ScaLAPACK Library" OFF) 5 | option_set(USE_EIGENEXA "Use EigenExa Library" OFF) 6 | option_set(USE_LIBXC "Use Libxc library" OFF) 7 | +option_set(USE_FJMPI "Use FUJITSU MPI" OFF) 8 | 9 | ## Optimization for stencil compitations 10 | option_set(USE_OPT_DOMAIN_IS_POW2 "Enable whether the optimization assumes that a stencil domain size is power of two" OFF) 11 | --- spack-src.org/src/config.h.in 2020-07-21 12:08:53.000000000 +0900 12 | +++ spack-src/src/config.h.in 2020-12-23 11:48:48.384061907 +0900 13 | @@ -15,6 +15,7 @@ 14 | #cmakedefine SYSTEM_HAS_PATH_MAX_IN_LINUX_LIMITS_H 15 | 16 | #cmakedefine USE_MPI 17 | +#cmakedefine USE_FJMPI 18 | #cmakedefine USE_SCALAPACK 19 | #cmakedefine USE_EIGENEXA 20 | #cmakedefine USE_LIBXC 21 | diff -ru spack-src.org/src/parallel/init_communicator.f90 spack-src/src/parallel/init_communicator.f90 22 | --- spack-src.org/src/parallel/init_communicator.f90 2020-07-21 12:08:53.000000000 +0900 23 | +++ spack-src/src/parallel/init_communicator.f90 2020-12-23 11:50:58.997384260 +0900 24 | @@ -13,6 +13,7 @@ 25 | ! See the License for the specific language governing permissions and 26 | ! limitations under the License. 27 | ! 28 | +#include "config.h" 29 | MODULE init_communicator 30 | implicit none 31 | 32 | @@ -35,7 +36,7 @@ 33 | integer :: i1,i2,i3,i4,i5,ix,iy,iz,nl 34 | integer,allocatable :: iranklists(:) 35 | 36 | -#ifdef __FUJITSU 37 | +#ifdef USE_FJMPI 38 | integer :: iret 39 | #endif 40 | 41 | @@ -59,7 +60,7 @@ 42 | 0:nproc_k-1)) 43 | 44 | ! communicator r,o,k,ro,ko 45 | -#ifdef __FUJITSU 46 | +#ifdef USE_FJMPI 47 | call tofu_network_oriented_mapping(iret) 48 | if (iret < 0) then 49 | if (comm_is_root(info%id_rko)) then 50 | @@ -106,7 +107,7 @@ 51 | 52 | if (nl /= info%isize_rko-1) & 53 | stop '[FATAL ERROR] init_communicator_dft' 54 | -#ifdef __FUJITSU 55 | +#ifdef USE_FJMPI 56 | end if 57 | #endif 58 | 59 | @@ -234,7 +235,7 @@ 60 | info%icomm_xy = comm_create_group_byid(comm, iranklists(1:nl)) 61 | call comm_get_groupinfo(info%icomm_xy, info%id_xy, info%isize_xy) 62 | 63 | -#ifdef __FUJITSU 64 | +#ifdef USE_FJMPI 65 | contains 66 | subroutine tofu_network_oriented_mapping(iret) 67 | use mpi_ext 68 | -------------------------------------------------------------------------------- /repo/saxpy/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | cmake_minimum_required(VERSION 3.23) 7 | 8 | option(USE_OPNEMP "Use OpenMP" OFF) 9 | option(USE_CUDA "Use CUDA" OFF) 10 | option(USE_HIP "Use HIP" OFF) 11 | option(USE_CALIPER "Enable Caliper" FALSE) 12 | 13 | if(USE_CUDA) 14 | project(saxpy VERSION 1.0.0 LANGUAGES CXX CUDA) 15 | else() 16 | project(saxpy VERSION 1.0.0 LANGUAGES CXX) 17 | endif() 18 | 19 | set(SOURCES saxpy.cc) 20 | set(SAXPY_DEPENDENCIES ) 21 | 22 | add_executable(${PROJECT_NAME} ${SOURCES}) 23 | 24 | find_package(MPI REQUIRED) 25 | list(APPEND SAXPY_DEPENDENCIES 26 | MPI::MPI_CXX) 27 | 28 | if(USE_OPENMP) 29 | target_compile_options(${PROJECT_NAME} PRIVATE -DUSE_OPENMP) 30 | find_package(OpenMP REQUIRED) 31 | list(APPEND SAXPY_DEPENDENCIES 32 | OpenMP::OpenMP_CXX) 33 | endif() 34 | 35 | if(USE_HIP) 36 | include_directories(${ROCM_PATH}/include) 37 | target_compile_options(${PROJECT_NAME} PRIVATE -x hip -DUSE_HIP --offload-arch=${ROCM_ARCH}) 38 | list(APPEND SAXPY_DEPENDENCIES 39 | "-L${ROCM_PATH}/lib -lamdhip64") 40 | endif() 41 | 42 | if(USE_CUDA) 43 | set_source_files_properties(saxpy.cc PROPERTIES LANGUAGE "CUDA") 44 | set(CMAKE_CUDA_STANDARD_REQUIRED TRUE) 45 | set(CMAKE_CXX_STANDARD_REQUIRED TRUE) 46 | set(CMAKE_CXX_EXTENSIONS OFF) 47 | set(CMAKE_CUDA_FLAGS "-Xcompiler -DUSE_CUDA") 48 | set_target_properties(${PROJECT_NAME} PROPERTIES CUDA_SEPARABLE_COMPILATION ON) 49 | endif() 50 | 51 | if (USE_CALIPER) 52 | find_package(caliper REQUIRED) 53 | list(APPEND SAXPY_DEPENDENCIES 54 | caliper) 55 | find_package(adiak REQUIRED) 56 | list(APPEND SAXPY_DEPENDENCIES 57 | adiak::adiak) 58 | target_compile_options(${PROJECT_NAME} PRIVATE -DUSE_CALIPER) 59 | endif() 60 | 61 | configure_file( 62 | "${CMAKE_SOURCE_DIR}/config.hh.in" 63 | "${CMAKE_CURRENT_SOURCE_DIR}/config.hh" 64 | ) 65 | 66 | target_link_libraries(${PROJECT_NAME} PRIVATE ${SAXPY_DEPENDENCIES}) 67 | 68 | install(TARGETS ${PROJECT_NAME} 69 | DESTINATION bin 70 | ) 71 | -------------------------------------------------------------------------------- /repo/saxpy/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Saxpy(ExecutableApplication): 12 | """saxpy benchmark""" 13 | name = "saxpy" 14 | 15 | tags = ['synthetic','single-node','high-memory-bandwidth', 16 | 'regular-memory-access', 17 | 'c++'] 18 | 19 | executable('p', 'saxpy -n {n}', use_mpi=True) 20 | 21 | workload('problem', executables=['p']) 22 | 23 | workload_variable('n', default='1024', description='problem size', workloads=['problem']) 24 | 25 | figure_of_merit('Kernel {num} size', fom_regex=r'Kernel done \((?P[0-9]+)\): (?P[0-9]+)', group_name='size', units='') 26 | figure_of_merit("success", fom_regex=r'(?PKernel done)', group_name='done', units='') 27 | 28 | success_criteria('pass', mode='string', match=r'Kernel done', file='{experiment_run_dir}/{experiment_name}.out') 29 | -------------------------------------------------------------------------------- /repo/saxpy/config.hh.in: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | // Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | // 4 | // SPDX-License-Identifier: Apache-2.0 5 | 6 | #ifndef SAXPY_CONFIG_HH 7 | #define SAXPY_CONFIG_HH 8 | 9 | #define SAXPY_COMPILER_ID "@CMAKE_C_COMPILER_ID@" 10 | #define SAXPY_COMPILER_VERSION "@CMAKE_C_COMPILER_VERSION@" 11 | #define SAXPY_CMAKE_C_FLAGS "@CMAKE_C_FLAGS@" 12 | 13 | // Version information 14 | #define SAXPY_MAJOR_VERSION @SAXPY_MAJOR_VERSION@ 15 | #define SAXPY_MINOR_VERSION @SAXPY_MINOR_VERSION@ 16 | #define SAXPY_PATCH_VERSION @SAXPY_PATCH_VERSION@ 17 | #define SAXPY_VERSION "@SAXPY_VERSION@" 18 | 19 | #endif // SAXPY_CONFIG_HH 20 | -------------------------------------------------------------------------------- /repo/smb/application.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | import sys 7 | 8 | from ramble.appkit import * 9 | 10 | 11 | class Smb(ExecutableApplication): 12 | """Sandia microbenchmarks""" 13 | name = "Sandia microbenchmarks" 14 | 15 | executable('p1', 'mpi_overhead', use_mpi=True) 16 | executable('p2', 'msgrate -n {ppn}', use_mpi=True) 17 | 18 | workload('mpi_overhead', executables=['p1']) 19 | workload('msgrate', executables=['p2']) 20 | workload('rma_mt', executables=['p2']) 21 | 22 | workload_variable('ppn', default='1', 23 | description='Number of procs per node', 24 | workloads=['msgrate', 'rma_mt']) 25 | 26 | figure_of_merit('single direction', 27 | log_file='{experiment_run_dir}/{experiment_name}.out', 28 | fom_regex=r'single direction:\s+(?P[0-9]+\.[0-9]*)', 29 | group_name='fom', units='') 30 | #TODO:fix this FOM. Not sure what's causing it to not detect 31 | figure_of_merit('overhead', 32 | log_file='{experiment_run_dir}/{experiment_name}.out', 33 | fom_regex=r'(?:[0-9]+\.?[0-9]* +){4}(?P[0-9]+\.[0-9]*)', 34 | #fom_regex=r'avail\(%\)(?:\s|\t)*\n\s*(?:[0-9]+\.*[0-9]*\s*){4}(?P[0-9]+\.*[0-9]*)', 35 | group_name='fom', units='') 36 | 37 | figure_of_merit('pair based', 38 | log_file='{experiment_run_dir}/{experiment_name}.out', 39 | fom_regex=r'pair-based:\s+(?P[0-9]+\.[0-9]*)', 40 | group_name='fom', units='') 41 | 42 | figure_of_merit('pre-post', 43 | log_file='{experiment_run_dir}/{experiment_name}.out', 44 | fom_regex=r'\s*pre-post:\s+(?P[0-9]+\.[0-9]*)', 45 | group_name='fom', units='') 46 | 47 | figure_of_merit('all-start', 48 | log_file='{experiment_run_dir}/{experiment_name}.out', 49 | fom_regex=r'\s*all-start:\s+(?P[0-9]+\.[0-9]*)', 50 | group_name='fom', units='') 51 | #success_criteria('pass', mode='string', match=r'.*', file='{experiment_run_dir}/{experiment_name}.out') 52 | -------------------------------------------------------------------------------- /repo/smb/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | import inspect 6 | 7 | import llnl.util.filesystem as fs 8 | from spack.package import * 9 | 10 | 11 | class Smb(MakefilePackage): 12 | tags = [] 13 | 14 | url = "https://github.com/sandialabs/SMB/archive/refs/tags/1.1.tar.gz" 15 | git = "https://github.com/sandialabs/SMB" 16 | 17 | maintainers("knox10") 18 | 19 | version("master", branch="master") 20 | version("1.1", tag="1.1") 21 | 22 | variant("mpi", default=False, description="Build with MPI support") 23 | variant("rma", default=False, description="Build RMA-MT variant") 24 | 25 | depends_on("c", type="build") 26 | 27 | depends_on("mpi", when="+mpi") 28 | 29 | build_directory = ["src/mpi_overhead"] 30 | 31 | def edit(self, spec, prefix): 32 | if "+rma" in spec: 33 | makefile = FileFilter("src/rma_mt_mpi/Makefile") 34 | makefile.filter('CC=cc', "CC = {0}".format(spec["mpi"].mpicc)) 35 | #TODO: add shm variant 36 | def build(self, spec, prefix): 37 | if "+rma" in spec: 38 | self.build_directory.append("src/rma_mt_mpi") 39 | else: 40 | self.build_directory.append("src/msgrate") 41 | 42 | for path in self.build_directory: 43 | with fs.working_dir(path): 44 | make() 45 | def install(self, spec, prefix): 46 | mkdir(prefix.bin) 47 | mkdir(prefix.doc) 48 | install("src/mpi_overhead/mpi_overhead", prefix.bin) 49 | install("src/mpi_overhead/README", prefix.doc) 50 | if "+rma" in spec: 51 | install("src/rma_mt_mpi/msgrate", prefix.bin) 52 | else: 53 | install("src/msgrate/msgrate", prefix.bin) 54 | install("src/msgrate/README", prefix.doc) 55 | -------------------------------------------------------------------------------- /repo/spectrum-mpi/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.spectrum_mpi.package import SpectrumMpi as BuiltinSM 8 | 9 | 10 | class SpectrumMpi(BuiltinSM): 11 | @property 12 | def libs(self): 13 | libnames = [ 14 | 'mpi_ibm', 15 | 'mpi_ibm_mpifh', 16 | 'mpiprofilesupport', 17 | 'mpi_ibm_usempi', 18 | ] 19 | libs = list('lib' + x for x in libnames) 20 | return find_libraries( 21 | libs, 22 | self.spec.prefix, 23 | shared=True, 24 | recursive=True 25 | ) 26 | -------------------------------------------------------------------------------- /repo/stream/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | 8 | 9 | class Stream(CMakePackage): 10 | """The STREAM benchmark is a simple synthetic benchmark program that 11 | measures sustainable memory bandwidth (in MB/s) and the corresponding 12 | computation rate for simple vector kernels. 13 | 14 | This package builds a fork of the official code with Caliper support, 15 | a CMake build system, and the ability to configure settings 16 | (array size, iterations, offset) at runtime via the command line.""" 17 | 18 | homepage = "https://www.cs.virginia.edu/stream/ref.html" 19 | git = "https://github.com/daboehme/STREAM.git" 20 | 21 | version("5.10-caliper", git="https://github.com/daboehme/STREAM.git", 22 | branch="caliper-benchpark") 23 | 24 | variant("caliper", default=False, description="Enable Caliper/Adiak support") 25 | 26 | requires("@5.10-caliper", when="+caliper") 27 | 28 | depends_on("c", type="build") 29 | depends_on("fortran", type="build") 30 | 31 | depends_on("caliper", when="+caliper") 32 | depends_on("adiak@0.4:", when="+caliper") 33 | 34 | def cmake_args(self): 35 | args = [ 36 | self.define_from_variant("STREAM_ENABLE_CALIPER", "caliper") 37 | ] 38 | 39 | return args 40 | -------------------------------------------------------------------------------- /repo/umpire/package.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from spack.package import * 7 | from spack_repo.builtin.packages.umpire.package import Umpire as BuiltinUmpire 8 | 9 | 10 | class Umpire(BuiltinUmpire): 11 | 12 | depends_on("fmt@9.1: cxxstd=17", when="@2024.02.0: %oneapi") 13 | 14 | def setup_build_environment(self, env): 15 | super().setup_build_environment(env) 16 | if "+cuda" in self.spec: 17 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 18 | 19 | def setup_run_environment(self, env): 20 | super().setup_run_environment(env) 21 | if "+cuda" in self.spec: 22 | env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") 23 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pyyaml 2 | # The remaining requirements are from Ramble 3 | pytest 4 | flake8 5 | google-cloud-storage # for gcs fetch test 6 | google-api-core # for gcs fetch error .exceptions 7 | coverage 8 | pytest-cov 9 | pre-commit 10 | graphlib-backport;python_version<"3.9" 11 | urllib3==1.26.19;python_version<="3.6" 12 | protobuf;python_version>"3.6" 13 | protobuf==3.19.5;python_version<="3.6" 14 | pyarrow==14.0.1;python_version<="3.6" 15 | google-cloud-bigquery 16 | tqdm 17 | deprecation 18 | deepdiff 19 | xmltodict 20 | -------------------------------------------------------------------------------- /setup-env.sh: -------------------------------------------------------------------------------- 1 | export PATH=${PWD}/bin:$PATH -------------------------------------------------------------------------------- /spack.yaml: -------------------------------------------------------------------------------- 1 | # This is a Spack Environment file. 2 | # 3 | # It describes a set of packages to be installed, along with 4 | # configuration settings. 5 | spack: 6 | # add package specs to the `specs` list 7 | specs: [py-pyyaml, py-pytest, py-codespell] 8 | view: true 9 | concretizer: 10 | unify: true 11 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/AWS_PCluster-zen-EFA/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: AWS_PCluster-zen-EFA 7 | integrator: 8 | vendor: AWS 9 | name: ParallelCluster 10 | processor: 11 | vendor: AMD 12 | name: EPYC-Zen 13 | ISA: x86_64 14 | uArch: zen 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: AWS 22 | name: EFA 23 | systems-tested: 24 | aws-pcluster: 25 | os: alinux2 26 | scheduler: slurm 27 | compiler: gcc 28 | runtime: 29 | mpi: openmpi 30 | instance-types: 31 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/AWS_Tutorial-sapphirerapids-EFA/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: AWS_Tutorial-sapphirerapids-EFA 7 | integrator: 8 | vendor: AWS 9 | name: EKS 10 | processor: 11 | vendor: Intel 12 | name: Xeon Sapphire Rapids 13 | ISA: x86_64 14 | uArch: SapphireRapids 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: AWS 22 | name: EFA 23 | systems-tested: 24 | aws-pcluster: 25 | os: ubuntu-22.04 26 | scheduler: flux 27 | compiler: gcc 28 | runtime: 29 | mpi: mpich 30 | instance-types: https://aws.amazon.com/ec2/instance-types/c7i/ 31 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/Atos-rome-A100-Infiniband/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: Atos-zen2-A100-Infiniband 7 | integrator: 8 | vendor: Atos 9 | name: XH2000 10 | processor: 11 | vendor: AMD 12 | name: EPYC-Zen2 13 | ISA: x86_64 14 | uArch: zen2 15 | accelerator: 16 | vendor: NVIDIA 17 | name: A100 18 | ISA: PTX 19 | uArch: sm_80 20 | interconnect: 21 | vendor: Mellanox 22 | name: Infiniband 23 | systems-tested: 24 | jsc-juwels-booster: 25 | os: 26 | scheduler: slurm 27 | compiler: gcc 28 | runtime: cuda 29 | mpi: openmpi 30 | installation-year: 2020 31 | top500-system-instances: 32 | JUWELS-Booster: 33 | benchpark_system: jsc-juwels 34 | top500: 35 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/DELL-cascadelake-Infiniband/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: DELL-cascadelake-InfiniBand 7 | integrator: 8 | vendor: DELL 9 | name: 10 | processor: 11 | vendor: Intel 12 | name: Xeon6248R 13 | ISA: x86_64 14 | uArch: cascadelake 15 | accelerator: 16 | vendor: NVIDIA 17 | name: A100 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: Mellanox 22 | name: HDR100InfiniBand 23 | systems-tested: 24 | tamu-grace: 25 | os: 26 | scheduler: slurm 27 | compiler: gcc 28 | runtime: gpu 29 | mpi: intelmpi 30 | top500-system-instances: 31 | Grace: 32 | benchpark_system: 33 | top500: 34 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/DELL-sapphirerapids-H100-Infiniband/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: DELL-sapphirerapids-H100-Infiniband 7 | integrator: 8 | vendor: DELLEMC 9 | name: PowerEdge 10 | processor: 11 | vendor: Intel 12 | name: XeonPlatinum8480 13 | ISA: x86_64 14 | uArch: sapphirerapids 15 | accelerator: 16 | vendor: NVIDIA 17 | name: H100 18 | ISA: PTX 19 | uArch: sm_90 20 | interconnect: 21 | vendor: Mellanox 22 | name: HDR100InfiniBand 23 | systems-tested: 24 | llnl-cluster: 25 | os: TOSS 26 | scheduler: slurm 27 | compiler: gcc 28 | runtime: 29 | mpi: mvapich 30 | top500-system-instances: 31 | Dane: 32 | benchpark_system: llnl-matrix 33 | top500: 34 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/DELL-sapphirerapids-OmniPath/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: DELL-sapphirerapids-OmniPath 7 | integrator: 8 | vendor: DELLEMC 9 | name: PowerEdge 10 | processor: 11 | vendor: Intel 12 | name: XeonPlatinum8480 13 | ISA: x86_64 14 | uArch: sapphirerapids 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: CornelisNetworks 22 | name: OmniPath 23 | systems-tested: 24 | llnl-cluster: 25 | os: TOSS 26 | scheduler: flux 27 | compiler: gcc 28 | runtime: 29 | mpi: mvapich 30 | top500-system-instances: 31 | Dane: 32 | benchpark_system: llnl-cluster 33 | top500: 34 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/Fujitsu-A64FX-TofuD/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: Fujitsu-A64FX-TofuD 7 | integrator: 8 | vendor: Fujitsu 9 | name: FX1000 10 | processor: 11 | vendor: Fujitsu 12 | name: A64FX 13 | ISA: Armv8.2-A-SVE 14 | uArch: aarch64 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: Fujitsu 22 | name: TofuInterconnectD 23 | systems-tested: 24 | rccs-fugaku: 25 | os: RHEL 26 | scheduler: pjm 27 | compiler: fj 28 | runtime: 29 | mpi: fujitsu-mpi 30 | top500-system-instances: 31 | Fugaku: 32 | benchpark_system: riken-fugaku 33 | top500: 34 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/HPECray-haswell-P100-Infiniband/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: HPECray-haswell-P100-Infiniband 7 | integrator: 8 | vendor: HPECray 9 | name: 10 | processor: 11 | vendor: Intel 12 | name: Xeon-E5-2650v3 13 | ISA: x86_64 14 | uArch: haswell 15 | accelerator: 16 | vendor: NVIDIA 17 | name: P100 18 | ISA: PTX 19 | uArch: sm_60 20 | interconnect: 21 | vendor: HPECray 22 | name: Aries 23 | systems-tested: 24 | cscs-daint: 25 | os: HPECrayOS 26 | scheduler: slurm 27 | compiler: cce 28 | runtime: cuda 29 | mpi: cray-mpich 30 | installation-year: 2017 31 | top500-system-instances: 32 | Piz Daint: 33 | benchpark_system: cscs-daint 34 | top500: 35 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/HPECray-neoverse-H100-Slingshot/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: HPECray-neoverse-H100-Slingshot 7 | integrator: 8 | vendor: HPECray 9 | name: EX254n 10 | processor: 11 | vendor: NVIDIA 12 | name: Grace 13 | ISA: Armv9 14 | uArch: neoverse 15 | accelerator: 16 | vendor: NVIDIA 17 | name: H100 18 | ISA: PTX 19 | uArch: sm_90 20 | interconnect: 21 | vendor: HPECray 22 | name: Slingshot11 23 | systems-tested: 24 | lanl-venado: 25 | os: HPECrayOS 26 | scheduler: slurm 27 | compiler: cce 28 | runtime: cuda 29 | mpi: cray-mpich 30 | top500-system-instances: 31 | Venado: 32 | benchpark_system: lanl-venado 33 | top500: 34 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/HPECray-zen2-Slingshot/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: HPECray-zen2-Slingshot 7 | integrator: 8 | vendor: HPECray 9 | name: 10 | processor: 11 | vendor: AMD 12 | name: EPYC-7742 13 | ISA: x86_64 14 | uArch: zen2 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: HPECray 22 | name: Slingshot 23 | systems-tested: 24 | cscs-eiger: 25 | os: HPECrayOS 26 | scheduler: slurm 27 | compiler: gcc 28 | runtime: 29 | mpi: cray-mpich 30 | installation-year: 2017 31 | top500-system-instances: 32 | Eiger: 33 | benchpark_system: cscs-eiger 34 | top500: 35 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/HPECray-zen3-MI250X-Slingshot/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: HPECray-zen3-MI250X-Slingshot 7 | integrator: 8 | vendor: HPECray 9 | name: EX235a 10 | processor: 11 | vendor: AMD 12 | name: EPYC-Zen3 13 | ISA: x86_64 14 | uArch: zen3 15 | accelerator: 16 | vendor: AMD 17 | name: MI250X 18 | ISA: GCN 19 | uArch: gfx90a 20 | interconnect: 21 | vendor: HPECray 22 | name: Slingshot11 23 | systems-tested: 24 | csc-lumi: 25 | os: HPECrayOS 26 | scheduler: slurm 27 | compiler: cce 28 | runtime: rocm 29 | mpi: cray-mpich 30 | installation-year: 2023 31 | llnl-elcapitan: 32 | os: TOSS 33 | scheduler: flux 34 | compiler: cce 35 | runtime: rocm 36 | mpi: cray-mpich 37 | top500-system-instances: 38 | Frontier: 39 | benchpark_system: 40 | top500: 41 | Lumi: 42 | benchpark_system: csc-lumi 43 | top500: 44 | Tioga: 45 | benchpark_system: llnl-elcapitan 46 | top500: 47 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/HPECray-zen4-MI300A-Slingshot/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: HPECray-zen4-MI300A-Slingshot 7 | integrator: 8 | vendor: HPECray 9 | name: EX255a 10 | processor: 11 | vendor: AMD 12 | name: EPYC-Zen4 13 | ISA: x86_64 14 | uArch: zen4 15 | accelerator: 16 | vendor: AMD 17 | name: MI300A 18 | ISA: GCN 19 | uArch: gfx942 20 | gpumode: [SPX, TPX, CPX] 21 | interconnect: 22 | vendor: HPECray 23 | name: Slingshot11 24 | systems-tested: 25 | llnl-elcapitan: 26 | os: TOSS 27 | scheduler: flux 28 | compiler: cce 29 | runtime: rocm 30 | mpi: cray-mpich 31 | top500-system-instances: 32 | El Capitan: 33 | benchpark_system: llnl-elcapitan 34 | top500: 35 | Tuolumne: 36 | benchpark_system: 37 | top500: 38 | rzAdams: 39 | benchpark_system: 40 | top500: 41 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/IBM-power9-V100-Infiniband/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: IBM-power9-V100-Infiniband 7 | integrator: 8 | vendor: IBM 9 | name: AC922 10 | processor: 11 | vendor: IBM 12 | name: POWER9 13 | ISA: ppc64le 14 | uArch: power9 15 | accelerator: 16 | vendor: NVIDIA 17 | name: V100 18 | ISA: PTX 19 | uArch: sm_70 20 | interconnect: 21 | vendor: Mellanox 22 | name: EDR-Infiniband 23 | systems-tested: 24 | llnl-sierra: 25 | os: RHEL 26 | scheduler: lsf 27 | compiler: xl 28 | runtime: cuda 29 | mpi: spectrum-mpi 30 | top500-system-instances: 31 | Summit: 32 | benchpark_system: 33 | top500: 34 | Sierra: 35 | benchpark_system: llnl-sierra 36 | top500: 37 | Lassen: 38 | benchpark_system: llnl-sierra 39 | top500: 40 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/Penguin-haswell-OmniPath/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: Penguin-haswell-OmniPath 7 | integrator: 8 | vendor: Penguin 9 | name: 10 | processor: 11 | vendor: Intel 12 | name: Xeon-E5-2695v4 13 | ISA: x86_64 14 | uArch: haswell 15 | interconnect: 16 | vendor: CornelisNetworks 17 | name: OmniPath 18 | systems-tested: 19 | llnl-rzgenie: 20 | os: TOSS 21 | scheduler: slurm 22 | compiler: gcc 23 | runtime: 24 | mpi: mvapich 25 | installation-year: 2017 26 | top500-system-instances: 27 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/Penguin-icelake-OmniPath/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: Penguin-icelake-OmniPath 7 | integrator: 8 | vendor: PenguinComputing 9 | name: RelionCluster 10 | processor: 11 | vendor: Intel 12 | name: XeonPlatinum924248C 13 | ISA: x86_64 14 | uArch: icelake 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: CornelisNetworks 22 | name: OmniPath 23 | systems-tested: 24 | llnl-cluster: 25 | os: TOSS 26 | scheduler: flux 27 | compiler: gcc 28 | runtime: 29 | mpi: mvapich 30 | top500-system-instances: 31 | Magma: 32 | benchpark_system: llnl-cluster 33 | top500: 34 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/Supermicro-icelake-OmniPath/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: Supermicro-icelake-OmniPath 7 | integrator: 8 | vendor: Supermicro 9 | name: 10 | processor: 11 | vendor: Intel 12 | name: XeonPlatinum8276L 13 | ISA: x86_64 14 | uArch: icelake 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: CornelisNetworks 22 | name: OmniPath 23 | systems-tested: 24 | llnl-cluster: 25 | os: TOSS 26 | scheduler: flux 27 | compiler: gcc 28 | runtime: 29 | mpi: mvapich 30 | top500-system-instances: 31 | Ruby: 32 | benchpark_system: llnl-cluster 33 | top500: 34 | -------------------------------------------------------------------------------- /systems/all_hardware_descriptions/x86_64/hardware_description.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | system_definition: 6 | name: x86_64 7 | integrator: 8 | vendor: 9 | name: 10 | processor: 11 | vendor: 12 | name: 13 | ISA: x86_64 14 | uArch: 15 | accelerator: 16 | vendor: 17 | name: 18 | ISA: 19 | uArch: 20 | interconnect: 21 | vendor: 22 | name: 23 | systems-tested: 24 | any: 25 | os: 26 | scheduler: 27 | compiler: 28 | runtime: 29 | mpi: 30 | top500-system-instances: [null] 31 | -------------------------------------------------------------------------------- /systems/common/spack_includes.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | packages: 6 | blt: 7 | require: 8 | - one_of: ['%gcc', '%clang'] 9 | -------------------------------------------------------------------------------- /systems/csc-lumi/externals/libsci/02-rocmcc-packages.yaml: -------------------------------------------------------------------------------- 1 | packages: 2 | cray-libsci: 3 | - spec: cray-libsci@23.09.1.1%rocmcc 4 | prefix: /opt/cray/pe/libsci/23.09.1.1/AMD/5.0/x86_64 5 | modules: [cray-libsci/23.09.1.1] 6 | buildable: false 7 | -------------------------------------------------------------------------------- /systems/generic-x86/system.py: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Lawrence Livermore National Security, LLC and other 2 | # Benchpark Project Developers. See the top-level COPYRIGHT file for details. 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | from benchpark.directives import maintainers 7 | from benchpark.openmpsystem import OpenMPCPUOnlySystem 8 | from benchpark.system import System 9 | 10 | 11 | class GenericX86(System): 12 | """This is the generic system class for an x86 system, gcc compiler, mpi. 13 | It can be easily copied and modified to model other systems.""" 14 | 15 | maintainers("slabasan") 16 | 17 | def __init__(self, spec): 18 | super().__init__(spec) 19 | self.programming_models = [OpenMPCPUOnlySystem()] 20 | 21 | self.scheduler = "mpi" 22 | setattr(self, "sys_cores_per_node", 1) 23 | 24 | def compute_software_section(self): 25 | """This is somewhat vestigial, and maybe deleted later. The experiments 26 | will fail if these variables are not defined though, so for now 27 | they are still generated (but with more-generic values). 28 | """ 29 | return { 30 | "software": { 31 | "packages": { 32 | "default-compiler": {"pkg_spec": "gcc"}, 33 | "compiler-gcc": {"pkg_spec": "gcc"}, 34 | "default-mpi": {"pkg_spec": "openmpi"}, 35 | "blas": {"pkg_spec": "openblas"}, 36 | "lapack": {"pkg_spec": "openblas"}, 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /systems/repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: sysbuiltin 3 | subdirectory: '' 4 | -------------------------------------------------------------------------------- /var/exp_repo/repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: builtin 3 | -------------------------------------------------------------------------------- /var/sys_repo/repo.yaml: -------------------------------------------------------------------------------- 1 | repo: 2 | namespace: sysbuiltin 3 | --------------------------------------------------------------------------------