├── .github ├── pull_request_template.md └── workflows │ ├── docker_build.yml │ ├── python3.yml │ └── singularity_build.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── docs ├── README.md ├── building_blocks.md ├── citation.md ├── getting_started.md ├── misc_api.md ├── primitives.md ├── recipes.md ├── tutorial.md └── workflows.md ├── hpccm.sh ├── hpccm ├── Stage.py ├── __init__.py ├── base_object.py ├── building_blocks │ ├── __init__.py │ ├── amgx.py │ ├── apt_get.py │ ├── arm_allinea_studio.py │ ├── base.py │ ├── boost.py │ ├── catalyst.py │ ├── cgns.py │ ├── charm.py │ ├── cmake.py │ ├── conda.py │ ├── doca_ofed.py │ ├── fftw.py │ ├── gdrcopy.py │ ├── generic_autotools.py │ ├── generic_build.py │ ├── generic_cmake.py │ ├── gnu.py │ ├── hdf5.py │ ├── hpcx.py │ ├── intel_mpi.py │ ├── intel_psxe.py │ ├── intel_psxe_runtime.py │ ├── julia.py │ ├── knem.py │ ├── kokkos.py │ ├── libsim.py │ ├── llvm.py │ ├── magma.py │ ├── mkl.py │ ├── mlnx_ofed.py │ ├── mpich.py │ ├── multi_ofed.py │ ├── mvapich2.py │ ├── mvapich2_gdr.py │ ├── nccl.py │ ├── netcdf.py │ ├── nsight_compute.py │ ├── nsight_systems.py │ ├── nvhpc.py │ ├── nvshmem.py │ ├── ofed.py │ ├── openblas.py │ ├── openmpi.py │ ├── packages.py │ ├── pgi.py │ ├── pip.py │ ├── pmix.py │ ├── pnetcdf.py │ ├── python.py │ ├── rdma_core.py │ ├── scif.py │ ├── sensei.py │ ├── slurm_pmi2.py │ ├── ucx.py │ ├── xpmem.py │ └── yum.py ├── cli.py ├── common.py ├── config.py ├── primitives │ ├── __init__.py │ ├── baseimage.py │ ├── blob.py │ ├── comment.py │ ├── copy.py │ ├── environment.py │ ├── label.py │ ├── raw.py │ ├── runscript.py │ ├── shell.py │ ├── user.py │ └── workdir.py ├── recipe.py ├── templates │ ├── CMakeBuild.py │ ├── ConfigureMake.py │ ├── __init__.py │ ├── annotate.py │ ├── downloader.py │ ├── envvars.py │ ├── git.py │ ├── ldconfig.py │ ├── rm.py │ ├── sed.py │ ├── tar.py │ ├── wget.py │ └── zipfile.py ├── toolchain.py └── version.py ├── pydocmd.yml ├── recipes ├── easybuild.py ├── examples │ ├── basic.py │ ├── cloverleaf.py │ ├── gnu-devel.py │ ├── multistage.py │ ├── scif.py │ ├── script.py │ └── userargs.py ├── gromacs │ └── gromacs.py ├── hpcbase-gnu-mvapich2.py ├── hpcbase-gnu-openmpi.py ├── hpcbase-nvhpc-mvapich2.py ├── hpcbase-nvhpc-openmpi.py ├── hpccm │ ├── Dockerfile │ ├── README.md │ ├── Singularity.def │ └── bootstrap.py ├── jupyter │ ├── .gitignore │ ├── README.md │ ├── environment.yml │ ├── jupyter.py │ ├── notebook.ipynb │ └── requirements.txt ├── lammps │ └── lammps.py ├── milc │ └── milc.py ├── mpi_bandwidth.py ├── nvhpc.py ├── osu_benchmarks │ ├── README.md │ ├── common.py │ ├── entrypoint.sh │ └── osu_benchmarks.py └── spack.py ├── runtests.sh ├── setup.py └── test ├── bad_recipe.py ├── docker.blob ├── global_vars_recipe.py ├── helpers.py ├── include1.py ├── include2.py ├── include3.py ├── singularity.blob ├── test_CMakeBuild.py ├── test_ConfigureMake.py ├── test_Stage.py ├── test_amgx.py ├── test_annotate.py ├── test_apt_get.py ├── test_arm_allinea_studio.py ├── test_baseimage.py ├── test_bb_base.py ├── test_blob.py ├── test_boost.py ├── test_catalyst.py ├── test_cgns.py ├── test_charm.py ├── test_cli.py ├── test_cmake.py ├── test_comment.py ├── test_conda.py ├── test_config.py ├── test_copy.py ├── test_doca_ofed.py ├── test_downloader.py ├── test_environment.py ├── test_envvars.py ├── test_fftw.py ├── test_gdrcopy.py ├── test_generic_autotools.py ├── test_generic_build.py ├── test_generic_cmake.py ├── test_git.py ├── test_global_vars.py ├── test_gnu.py ├── test_hdf5.py ├── test_hpcx.py ├── test_intel_mpi.py ├── test_intel_psxe.py ├── test_intel_psxe_runtime.py ├── test_julia.py ├── test_knem.py ├── test_kokkos.py ├── test_label.py ├── test_ldconfig.py ├── test_libsim.py ├── test_llvm.py ├── test_magma.py ├── test_mkl.py ├── test_mlnx_ofed.py ├── test_mpich.py ├── test_multi_ofed.py ├── test_mvapich2.py ├── test_mvapich2_gdr.py ├── test_nccl.py ├── test_netcdf.py ├── test_nsight_compute.py ├── test_nsight_systems.py ├── test_nvhpc.py ├── test_nvshmem.py ├── test_ofed.py ├── test_openblas.py ├── test_openmpi.py ├── test_packages.py ├── test_pgi.py ├── test_pip.py ├── test_pmix.py ├── test_pnetcdf.py ├── test_python.py ├── test_raw.py ├── test_rdma_core.py ├── test_recipe.py ├── test_rm.py ├── test_runscript.py ├── test_scif.py ├── test_sed.py ├── test_sensei.py ├── test_shell.py ├── test_slurm_pmi2.py ├── test_tar.py ├── test_toolchain.py ├── test_ucx.py ├── test_user.py ├── test_wget.py ├── test_workdir.py ├── test_xpmem.py ├── test_yum.py └── test_zipfile.py /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Pull Request Description 2 | 3 | ## Author Checklist 4 | * [ ] Updated documentation (`pydocmd generate`) if any docstrings have been modified 5 | * [ ] Passes all unit tests 6 | -------------------------------------------------------------------------------- /.github/workflows/docker_build.yml: -------------------------------------------------------------------------------- 1 | name: Docker Build 2 | 3 | on: 4 | schedule: 5 | - cron: '0 12 * * 3' # Weekly 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | continue-on-error: true 11 | strategy: 12 | matrix: 13 | include: 14 | #- recipe: recipes/easybuild.py 15 | # userargs: none=none 16 | # context: . 17 | - recipe: recipes/examples/basic.py 18 | userargs: none=none 19 | context: . 20 | - recipe: recipes/examples/cloverleaf.py 21 | userargs: none=none 22 | context: . 23 | - recipe: recipes/examples/multistage.py 24 | userargs: none=none 25 | context: . 26 | #- recipe: recipes/examples/scif.py 27 | # userargs: none=none 28 | # context: . 29 | #- recipe: recipes/examples/userargs.py 30 | # userargs: cuda=11.0 ompi=4.0.5 31 | # context: . 32 | #- recipe: recipes/gromacs/gromacs.py 33 | # userargs: none=none 34 | # context: . 35 | - recipe: recipes/hpcbase-gnu-mvapich2.py 36 | userargs: none=none 37 | context: . 38 | #- recipe: recipes/hpcbase-gnu-mvapich2.py 39 | # userargs: centos=true 40 | # context: . 41 | - recipe: recipes/hpcbase-gnu-openmpi.py 42 | userargs: none=none 43 | context: . 44 | #- recipe: recipes/hpcbase-gnu-openmpi.py 45 | # userargs: centos=true 46 | # context: . 47 | #- recipe: recipes/hpcbase-nvhpc-mvapich2.py 48 | # userargs: nvhpc_eula_accept=yes 49 | # context: . 50 | #- recipe: recipes/hpcbase-nvhpc-mvapich2.py 51 | # userargs: nvhpc_eula_accept=yes centos=true 52 | # context: . 53 | #- recipe: recipes/hpcbase-nvhpc-openmpi.py 54 | # userargs: nvhpc_eula_accept=yes 55 | # context: . 56 | #- recipe: recipes/hpcbase-nvhpc-openmpi.py 57 | # userargs: nvhpc_eula_accept=yes centos=true 58 | # context: . 59 | - recipe: recipes/hpccm/bootstrap.py 60 | userargs: none=none 61 | context: . 62 | - recipe: recipes/lammps/lammps.py 63 | userargs: none=none 64 | context: . 65 | #- recipe: recipes/milc/milc.py 66 | # userargs: none=none 67 | # context: . 68 | #- recipe: recipes/mpi_bandwidth.py 69 | # userargs: none=none 70 | # context: . 71 | #- recipe: recipes/osu_benchmarks/osu_benchmarks.py 72 | # userargs: none=none 73 | # context: recipes/osu_benchmarks 74 | - recipe: recipes/spack.py 75 | userargs: none=none 76 | context: . 77 | 78 | steps: 79 | - uses: actions/checkout@v2 80 | 81 | - name: Set up Python 82 | uses: actions/setup-python@v5 83 | 84 | - name: Install dependencies 85 | run: | 86 | python3 -m pip install --upgrade pip 87 | pip3 install six archspec packaging 88 | 89 | - name: Build container 90 | run: | 91 | ./hpccm.sh --recipe ${{ matrix.recipe }} --userarg ${{ matrix.userargs }} > Dockerfile 92 | sudo docker build -f Dockerfile ${{ matrix.context }} 93 | -------------------------------------------------------------------------------- /.github/workflows/python3.yml: -------------------------------------------------------------------------------- 1 | name: Python 3 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ${{ matrix.os }} 8 | strategy: 9 | matrix: 10 | os: [ubuntu-latest, macos-latest, windows-latest] 11 | python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | 16 | - name: Set up Python 17 | uses: actions/setup-python@v2 18 | with: 19 | python-version: ${{ matrix.python-version }} 20 | 21 | - name: Install dependencies 22 | run: | 23 | if [ "${{ matrix.python-version }}" = "3.5" ]; then 24 | python -m pip install --upgrade "pip < 21.0" 25 | else 26 | python -m pip install --upgrade pip 27 | fi 28 | pip install six archspec packaging 29 | shell: bash 30 | 31 | - name: Run unit tests 32 | run: bash ./runtests.sh 33 | -------------------------------------------------------------------------------- /.github/workflows/singularity_build.yml: -------------------------------------------------------------------------------- 1 | name: Singularity Build 2 | 3 | on: 4 | schedule: 5 | - cron: '0 13 * * 3' # Weekly 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | continue-on-error: true 11 | strategy: 12 | matrix: 13 | include: 14 | #- recipe: recipes/easybuild.py 15 | # userargs: none=none 16 | # context: . 17 | - recipe: recipes/examples/basic.py 18 | userargs: none=none 19 | context: . 20 | - recipe: recipes/examples/cloverleaf.py 21 | userargs: none=none 22 | context: . 23 | - recipe: recipes/examples/multistage.py 24 | userargs: none=none 25 | context: . 26 | #- recipe: recipes/examples/scif.py 27 | # userargs: none=none 28 | # context: . 29 | #- recipe: recipes/examples/userargs.py 30 | # userargs: cuda=11.0 ompi=4.0.5 31 | # context: . 32 | #- recipe: recipes/gromacs/gromacs.py 33 | # userargs: none=none 34 | # context: . 35 | - recipe: recipes/hpcbase-gnu-mvapich2.py 36 | userargs: none=none 37 | context: . 38 | #- recipe: recipes/hpcbase-gnu-mvapich2.py 39 | # userargs: centos=true 40 | # context: . 41 | - recipe: recipes/hpcbase-gnu-openmpi.py 42 | userargs: none=none 43 | context: . 44 | #- recipe: recipes/hpcbase-gnu-openmpi.py 45 | # userargs: centos=true 46 | # context: . 47 | #- recipe: recipes/hpcbase-nvhpc-mvapich2.py 48 | # userargs: nvhpc_eula_accept=yes 49 | # context: . 50 | #- recipe: recipes/hpcbase-nvhpc-mvapich2.py 51 | # userargs: nvhpc_eula_accept=yes centos=true 52 | # context: . 53 | #- recipe: recipes/hpcbase-nvhpc-openmpi.py 54 | # userargs: nvhpc_eula_accept=yes 55 | # context: . 56 | #- recipe: recipes/hpcbase-nvhpc-openmpi.py 57 | # userargs: nvhpc_eula_accept=yes centos=true 58 | # context: . 59 | - recipe: recipes/hpccm/bootstrap.py 60 | userargs: none=none 61 | context: . 62 | - recipe: recipes/lammps/lammps.py 63 | userargs: none=none 64 | context: . 65 | #- recipe: recipes/milc/milc.py 66 | # userargs: none=none 67 | # context: . 68 | #- recipe: recipes/mpi_bandwidth.py 69 | # userargs: none=none 70 | # context: . 71 | #- recipe: recipes/osu_benchmarks/osu_benchmarks.py 72 | # userargs: none=none 73 | # context: recipes/osu_benchmarks 74 | - recipe: recipes/spack.py 75 | userargs: none=none 76 | context: . 77 | 78 | steps: 79 | - uses: actions/checkout@v2 80 | 81 | - name: Set up Python 82 | uses: actions/setup-python@v5 83 | 84 | - name: Install dependencies 85 | run: | 86 | python3 -m pip install --upgrade pip 87 | pip3 install six archspec packaging 88 | 89 | - name: Install Singularity 90 | run: | 91 | # Dependencies 92 | sudo apt-get update -y 93 | sudo apt-get install -y build-essential libseccomp-dev pkg-config squashfs-tools cryptsetup 94 | # Singularity 95 | wget https://github.com/sylabs/singularity/releases/download/v3.7.4/singularity-3.7.4.tar.gz 96 | tar -xzf singularity-3.7.4.tar.gz 97 | cd singularity 98 | ./mconfig 99 | cd ./builddir 100 | make 101 | sudo make install 102 | 103 | - name: Build container 104 | run: | 105 | ./hpccm.sh --format singularity --singularity-version 3.7.4 --recipe ${{ matrix.recipe }} --userarg ${{ matrix.userargs }} > Singularity.def 106 | sudo singularity build image.sif Singularity.def 107 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | hpccm.egg-info 2 | __pycache__ 3 | *.pyc 4 | .coverage 5 | Dockerfile* 6 | Singularity* 7 | build/ 8 | dist/ 9 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CHANGELOG.md 2 | include LICENSE 3 | include README.md 4 | include test/bad_recipe.py 5 | include test/docker.blob 6 | include test/global_vars_recipe.py 7 | include test/helpers.py 8 | include test/include1.py 9 | include test/include2.py 10 | include test/include3.py 11 | include test/singularity.blob 12 | graft docs 13 | graft recipes 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Python 3](https://github.com/NVIDIA/hpc-container-maker/workflows/Python%203/badge.svg)](https://github.com/NVIDIA/hpc-container-maker/actions?query=workflow%3A%22Python+3%22) 2 | [![Conda](https://img.shields.io/conda/dn/conda-forge/hpccm?label=Conda%20downloads)](https://anaconda.org/conda-forge/hpccm) 3 | [![PyPI - Downloads](https://img.shields.io/pypi/dm/hpccm?label=PyPI%20downloads)](https://pypi.org/project/hpccm/) 4 | [![License](https://img.shields.io/github/license/NVIDIA/hpc-container-maker)](https://github.com/NVIDIA/hpc-container-maker/blob/master/LICENSE) 5 | 6 | # HPC Container Maker 7 | 8 | HPC Container Maker (HPCCM - pronounced H-P-see-M) is an open source 9 | tool to make it easier to generate container specification files. 10 | 11 | - [Documentation](/docs) 12 | - [Getting Started](/docs/getting_started.md) 13 | - [Tutorial](/docs/tutorial.md) 14 | - [Recipes](/docs/recipes.md) 15 | - [Workflows](/docs/workflows.md) 16 | - [API: Building Blocks](/docs/building_blocks.md) 17 | - [API: Primitives](/docs/primitives.md) 18 | - [API: Miscellaneous](/docs/misc_api.md) 19 | - [Examples](/recipes/) 20 | - [Citation](/docs/citation.md) 21 | - [License](/LICENSE) 22 | 23 | ## Overview 24 | 25 | HPC Container Maker generates Dockerfiles or Singularity definition 26 | files from a high level Python recipe. HPCCM recipes have some 27 | distinct advantages over "native" container specification formats. 28 | 29 | 1. A library of HPC [building blocks](/docs/building_blocks.md) that 30 | separate the choice of what to include in a container image from 31 | the details of how it's done. The building blocks transparently 32 | provide the latest component and container best practices. 33 | 34 | 2. Python provides increased flexibility over static container 35 | specification formats. Python-based recipes can branch, validate 36 | user input, etc. - the same recipe can generate multiple container 37 | specifications. 38 | 39 | 3. Generate either Dockerfiles or Singularity definition files from 40 | the same recipe. 41 | 42 | ## Additional Resources 43 | 44 | - [Making Containers Easier With HPC Container Maker (paper)](https://github.com/HPCSYSPROS/Workshop18/blob/master/Making_Containers_Easier_with_HPC_Container_Maker/ws_hpcsysp103.pdf), presented at the [HPC Systems Professionals Workshop at SC18](/docs/citation.md) 45 | - [Overview presentation at SC18 (video)](http://on-demand.gputechconf.com/supercomputing/2018/video/sc1843-making-containers-easier-hpc-container-maker.html) 46 | - [Making Containers Easier with HPC Container Maker (webinar)](https://www.nvidia.com/content/webinar-portal/src/webinar-portal.html?D2C=1802760&isSocialSharing=Y&partnerref=emailShareFromGateway) 47 | - [ADMIN Magazine article](http://www.admin-magazine.com/HPC/Articles/HPC-Container-Maker) by Jeff Layton 48 | - [NVIDIA Developer Blog](https://devblogs.nvidia.com/making-containers-easier-with-hpc-container-maker/) by Scott McMillan 49 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | [![Python 3](https://github.com/NVIDIA/hpc-container-maker/workflows/Python%203/badge.svg)](https://github.com/NVIDIA/hpc-container-maker/actions?query=workflow%3A%22Python+3%22) 2 | [![Conda](https://img.shields.io/conda/dn/conda-forge/hpccm?label=Conda%20downloads)](https://anaconda.org/conda-forge/hpccm) 3 | [![PyPI - Downloads](https://img.shields.io/pypi/dm/hpccm?label=PyPI%20downloads)](https://pypi.org/project/hpccm/) 4 | [![License](https://img.shields.io/github/license/NVIDIA/hpc-container-maker)](https://github.com/NVIDIA/hpc-container-maker/blob/master/LICENSE) 5 | 6 | # HPC Container Maker 7 | 8 | HPC Container Maker (HPCCM - pronounced H-P-see-M) is an open source 9 | tool to make it easier to generate container specification files. 10 | 11 | - [Documentation](/docs) 12 | - [Getting Started](/docs/getting_started.md) 13 | - [Tutorial](/docs/tutorial.md) 14 | - [Recipes](/docs/recipes.md) 15 | - [Workflows](/docs/workflows.md) 16 | - [API: Building Blocks](/docs/building_blocks.md) 17 | - [API: Primitives](/docs/primitives.md) 18 | - [API: Miscellaneous](/docs/misc_api.md) 19 | - [Examples](/recipes/) 20 | - [Citation](/docs/citation.md) 21 | - [License](/LICENSE) 22 | 23 | ## Overview 24 | 25 | HPC Container Maker generates Dockerfiles or Singularity definition 26 | files from a high level Python recipe. HPCCM recipes have some 27 | distinct advantages over "native" container specification formats. 28 | 29 | 1. A library of HPC [building blocks](/docs/building_blocks.md) that 30 | separate the choice of what to include in a container image from 31 | the details of how it's done. The building blocks transparently 32 | provide the latest component and container best practices. 33 | 34 | 2. Python provides increased flexibility over static container 35 | specification formats. Python-based recipes can branch, validate 36 | user input, etc. - the same recipe can generate multiple container 37 | specifications. 38 | 39 | 3. Generate either Dockerfiles or Singularity definition files from 40 | the same recipe. 41 | 42 | ## Additional Resources 43 | 44 | - [Making Containers Easier With HPC Container Maker (paper)](https://github.com/HPCSYSPROS/Workshop18/blob/master/Making_Containers_Easier_with_HPC_Container_Maker/ws_hpcsysp103.pdf), presented at the [HPC Systems Professionals Workshop at SC18](/docs/citation.md) 45 | - [Overview presentation at SC18 (video)](http://on-demand.gputechconf.com/supercomputing/2018/video/sc1843-making-containers-easier-hpc-container-maker.html) 46 | - [Making Containers Easier with HPC Container Maker (webinar)](https://www.nvidia.com/content/webinar-portal/src/webinar-portal.html?D2C=1802760&isSocialSharing=Y&partnerref=emailShareFromGateway) 47 | - [ADMIN Magazine article](http://www.admin-magazine.com/HPC/Articles/HPC-Container-Maker) by Jeff Layton 48 | - [NVIDIA Developer Blog](https://devblogs.nvidia.com/making-containers-easier-with-hpc-container-maker/) by Scott McMillan 49 | -------------------------------------------------------------------------------- /docs/citation.md: -------------------------------------------------------------------------------- 1 | # Citation 2 | 3 | ``` 4 | @InProceedings{McMillanHPCSYSPROS18, 5 | author = {McMillan, Scott}, 6 | title = {Making Containers Easier with HPC Container Maker}, 7 | url = {https://github.com/HPCSYSPROS/Workshop18/tree/master/Making_Containers_Easier_with_HPC_Container_Maker}, 8 | booktitle = {In HPCSYSPROS18: HPC System Professionals Workshop}, 9 | year = {2018}, 10 | month = {November}, 11 | address = {Dallas, TX}, 12 | } 13 | ``` 14 | -------------------------------------------------------------------------------- /docs/getting_started.md: -------------------------------------------------------------------------------- 1 | # Getting Started with HPC Container Maker 2 | 3 | ## Installation 4 | 5 | HPCCM can be installed from [PyPi](https://pypi.org/project/hpccm/) 6 | or [Conda](https://anaconda.org/conda-forge/hpccm). 7 | 8 | ``` 9 | $ sudo pip install hpccm 10 | ``` 11 | 12 | ``` 13 | $ conda install -c conda-forge hpccm 14 | ``` 15 | 16 | ## Using HPCCM 17 | 18 | HPCCM can be used as a Python module or via the `hpccm` command line 19 | interface. 20 | 21 | ### Python Module 22 | 23 | The HPCCM Python module is a set of routines to generate container 24 | specification files. 25 | 26 | ```python 27 | #!/usr/bin/env python 28 | 29 | import hpccm 30 | 31 | # Set to 'docker' to generate a Dockerfile or set to 'singularity' to 32 | # generate a Singularity definition file 33 | hpccm.config.set_container_format('docker') 34 | 35 | print(hpccm.primitives.baseimage(image='centos:7')) 36 | print(hpccm.building_blocks.gnu()) 37 | ``` 38 | 39 | The Python module provides the most flexibility, but you are 40 | responsible for managing input and output. 41 | 42 | ### Command Line Tool 43 | 44 | The `hpccm` command line tool processes HPCCM recipe files to generate 45 | container specification files. 46 | 47 | ```python 48 | Stage0 += baseimage(image='centos:7') 49 | Stage0 += gnu() 50 | ``` 51 | 52 | The command line tool manages input and output for you. 53 | 54 | ``` 55 | $ hpccm --recipe --format docker 56 | ``` 57 | 58 | ``` 59 | $ hpccm --recipe --format singularity 60 | ``` 61 | 62 | HPCCM recipes are Python scripts, so you can incorporate Python code in 63 | the recipe. 64 | 65 | ### Building Container Images 66 | 67 | The HPCCM output is the container specification, so save the output to 68 | a file. By convention, the container specification files are named 69 | `Dockerfile` or `Singularity.def` for Docker and Singularity, 70 | respectively. To generate a container image, use your preferred 71 | container image builder. 72 | 73 | Using [Docker](https://docs.docker.com/engine/reference/commandline/build/): 74 | 75 | ``` 76 | $ hpccm --recipe --format docker > Dockerfile 77 | $ sudo docker build -t -f Dockerfile . 78 | ``` 79 | 80 | Using [Singularity](https://www.sylabs.io/guides/latest/user-guide/build_a_container.html): 81 | 82 | ``` 83 | $ hpccm --recipe --format singularity > Singularity.def 84 | $ sudo singularity build Singularity.def 85 | ``` 86 | 87 | Other container builders that understand Dockerfiles or Singularity 88 | definition files may also be used. 89 | 90 | A bash script can also be generated from a recipe: 91 | 92 | ``` 93 | $ hpccm --recipe --format bash > script.sh 94 | ``` 95 | 96 | ## Next Steps 97 | 98 | Go through the [tutorial](/docs/tutorial.md) for some more in depth 99 | examples. 100 | 101 | A few [simple examples](/recipes/examples) are included to illustrate 102 | some of the basic concepts. 103 | 104 | Several [example recipes](/recipes) are also included. 105 | 106 | The "hpcbase" recipes provide representative HPC development 107 | environments including a compiler, MPI library, and common HPC 108 | libraries. These recipes are a great starting point for building an 109 | HPC application container. Choose the recipe with the compiler / MPI 110 | library combination that best matches the application requirements and 111 | add the application specific build instructions. 112 | 113 | A few complete application examples are also provided, including 114 | [GROMACS](/recipes/gromacs/gromacs.py), [MILC](/recipes/milc/milc.py), 115 | and [MPI Bandwidth](/recipes/mpi_bandwidth.py). 116 | 117 | Read the rest of the [documentation](/docs) for more information on 118 | creating recipes, customizing building block behavior, the API 119 | reference, and more. 120 | -------------------------------------------------------------------------------- /docs/workflows.md: -------------------------------------------------------------------------------- 1 | # HPC Container Maker Workflows 2 | 3 | HPC Container Maker enables three workflows for creating HPC 4 | application container images. 5 | 6 | ## Application Recipes 7 | 8 | The [MPI Bandwidth example](/recipes/mpi_bandwidth.py) shows how the 9 | entire specification of an application container image can be 10 | expressed as a HPCCM recipe. Examples are also included for 11 | [GROMACS](/recipes/gromacs/gromacs.py) and 12 | [MILC](/recipes/milc/milc.py). 13 | 14 | The GROMACS and MILC recipes demonstrate how to use [multi-stage 15 | builds](/docs/recipes.md#multi-stage-recipes) to minimize the 16 | size of the resulting container image. The first stage includes the 17 | required building blocks and GROMACS source code and then builds the 18 | application binary. Build artifacts such as the application source 19 | code, compiler, and development versions of the software toolchain are 20 | part of the image at this stage. Since only the final application 21 | binary and its runtime dependencies are needed when deploying the 22 | container image, the container image size can potentially be reduced 23 | significantly. The second stage uses a runtime version of the CUDA 24 | base image and copies the application binary and the required runtime 25 | dependencies from the first stage to generate a significantly smaller 26 | container image. HPCCM building blocks provide a method to easily 27 | copy the runtime from the first stage into the second stage. 28 | 29 | This workflow is the most portable since the HPCCM recipe can be used 30 | to generate either a Dockerfile or Singularity definition file, and is 31 | the easiest to maintain. 32 | 33 | ## Base Image Generation 34 | 35 | On a bare metal system using environment modules, the typical workflow 36 | is to first setup the necessary software environment by loading the 37 | appropriate modules. With the proper software environment loaded, an 38 | application can then be built or run. 39 | 40 | This "base" software environment is analogous to the base image used 41 | when building a container image. A base image with the proper set of 42 | HPC software components present can be used as the starting point for 43 | building an application container image. 44 | 45 | HPCCM includes [base recipes](/recipes) for all combinations of the 46 | Ubuntu and CentOS Linux distributions, the GNU and PGI compilers, and 47 | the OpenMPI and MVAPICH2 MPI libraries, plus commonly used software 48 | components such as the Mellanox OpenFabrics Enterprise Distribution, 49 | Python, FFTW, and HDF5. The provided base recipe can be easily 50 | customized to change component versions or add or subtract building 51 | blocks. 52 | 53 | For example: 54 | 55 | ``` 56 | $ wget https://raw.githubusercontent.com/NVIDIA/hpc-container-maker/master/recipes/hpcbase-gnu-openmpi.py 57 | $ hpccm --recipe hpcbase-gnu-openmpi.py > Dockerfile 58 | $ sudo docker build -t hpcbase -f Dockerfile . 59 | ``` 60 | 61 | The `hpcbase` image, with the software development environment, can now 62 | be used in a Dockerfile or Singularity definition file as the base image 63 | for building an application container image. 64 | 65 | ``` 66 | FROM hpcbase 67 | 68 | # Application build instructions 69 | ... 70 | ``` 71 | 72 | ``` 73 | BootStrap: docker 74 | From: hpcbase 75 | 76 | # Application build instructions 77 | ... 78 | ``` 79 | 80 | ## Template Generation 81 | 82 | Instead of going through the intermediate step of building a base 83 | image with the necessary HPC software components, the container 84 | specification file produced by HPCCM can be extended directly to 85 | include the application build instructions. In other words, HPCCM 86 | generates the boilerplate container specification syntax to install 87 | the necessary HPC software components with the application specific 88 | content appended by the user. The drawback to this approach is that 89 | it may be cumbersome to incorporate future HPCCM building block 90 | improvements. 91 | -------------------------------------------------------------------------------- /hpccm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # add the path where this script resides to the search path for python modules 18 | wd=$(readlink -e $(dirname $0)) 19 | export PYTHONPATH=${wd}:$PYTHONPATH 20 | 21 | python3 -m hpccm.cli $* 22 | -------------------------------------------------------------------------------- /hpccm/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import absolute_import 16 | 17 | from hpccm.version import __version__ 18 | 19 | from hpccm.base_object import base_object 20 | 21 | from hpccm.common import cpu_arch 22 | from hpccm.common import container_type 23 | from hpccm.common import linux_distro 24 | 25 | from hpccm.Stage import Stage 26 | from hpccm.recipe import include 27 | from hpccm.recipe import recipe 28 | from hpccm.toolchain import toolchain 29 | 30 | import hpccm.building_blocks 31 | import hpccm.templates 32 | import hpccm.primitives 33 | 34 | # Templates 35 | # For backwards compatibility with recipes that use "hpccm.git()", etc. 36 | from hpccm.templates.ConfigureMake import ConfigureMake 37 | from hpccm.templates.git import git 38 | from hpccm.templates.rm import rm 39 | from hpccm.templates.sed import sed 40 | from hpccm.templates.tar import tar 41 | from hpccm.templates.wget import wget 42 | -------------------------------------------------------------------------------- /hpccm/base_object.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Object base class""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | class base_object(object): 24 | """Object base class 25 | 26 | This base class is necessary for MRO inheritance to work. The 27 | built-in `object` class initializer does not accept keyword 28 | arguments, so it must be called like `super().__init__()`. 29 | However, for building blocks to pass keyword arguments to 30 | templates, the template initializers must be called like 31 | `super().__init__(**kwargs)`. There must be a base class at the 32 | bottom that only inherits from `object` and does not call 33 | `super()` with keyword arguments. 34 | 35 | """ 36 | 37 | def __init__(self, **kwargs): 38 | """Initialize base class""" 39 | super(base_object, self).__init__() 40 | -------------------------------------------------------------------------------- /hpccm/building_blocks/base.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Building block base class""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import hpccm.base_object 24 | 25 | class bb_instructions(hpccm.base_object): 26 | """Base class for building block instructions.""" 27 | 28 | def __init__(self, **kwargs): 29 | """Initialize building block instruction class""" 30 | 31 | super(bb_instructions, self).__init__(**kwargs) 32 | 33 | self.__instructions_bb = [] 34 | 35 | def __iadd__(self, instruction): 36 | """Add the instruction to the list of instructions. Allows "+=" 37 | syntax.""" 38 | 39 | if isinstance(instruction, list): 40 | self.__instructions_bb.extend(instruction) 41 | else: 42 | self.__instructions_bb.append(instruction) 43 | return self 44 | 45 | def __getitem__(self, key): 46 | """Return the specified element from the list of instructions""" 47 | return self.__instructions_bb[key] 48 | 49 | def __len__(self): 50 | """Return the size of the list of instructions""" 51 | return len(self.__instructions_bb) 52 | 53 | def __str__(self): 54 | """String representation of the building block""" 55 | return '\n'.join(str(x) for x in self.__instructions_bb if str(x)) 56 | 57 | class bb_base(bb_instructions): 58 | """Base class for building blocks.""" 59 | 60 | def __init__(self, **kwargs): 61 | super(bb_base, self).__init__(**kwargs) 62 | 63 | # Runtime instructions are kept in a separate list from the 64 | # "regular" instructions 65 | self.rt = bb_instructions() 66 | -------------------------------------------------------------------------------- /hpccm/building_blocks/knem.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | # pylint: disable=too-many-instance-attributes 17 | 18 | """KNEM building block""" 19 | 20 | from __future__ import absolute_import 21 | from __future__ import unicode_literals 22 | from __future__ import print_function 23 | 24 | import posixpath 25 | 26 | import hpccm.templates.envvars 27 | 28 | from hpccm.building_blocks.base import bb_base 29 | from hpccm.building_blocks.generic_build import generic_build 30 | from hpccm.building_blocks.packages import packages 31 | from hpccm.primitives.comment import comment 32 | 33 | class knem(bb_base, hpccm.templates.envvars): 34 | """The `knem` building block install the headers from the 35 | [KNEM](http://knem.gforge.inria.fr) component. 36 | 37 | # Parameters 38 | 39 | annotate: Boolean flag to specify whether to include annotations 40 | (labels). The default is False. 41 | 42 | environment: Boolean flag to specify whether the environment 43 | (`CPATH`) should be modified to include knem. The default is True. 44 | 45 | ospackages: List of OS packages to install prior to installing. 46 | The default values are `ca-certificates` and `git`. 47 | 48 | prefix: The top level install location. The default value is 49 | `/usr/local/knem`. 50 | 51 | version: The version of KNEM source to download. The default 52 | value is `1.1.4`. 53 | 54 | # Examples 55 | 56 | ```python 57 | knem(prefix='/opt/knem/1.1.3', version='1.1.3') 58 | ``` 59 | 60 | """ 61 | 62 | def __init__(self, **kwargs): 63 | """Initialize building block""" 64 | 65 | super(knem, self).__init__(**kwargs) 66 | 67 | self.__ospackages = kwargs.pop('ospackages', 68 | ['ca-certificates', 'git']) 69 | self.__prefix = kwargs.pop('prefix', '/usr/local/knem') 70 | self.__repository = kwargs.pop('repository', 71 | 'https://gitlab.inria.fr/knem/knem.git') 72 | self.__version = kwargs.pop('version', '1.1.4') 73 | 74 | # Setup the environment variables 75 | self.environment_variables['CPATH'] = '{}:$CPATH'.format( 76 | posixpath.join(self.__prefix, 'include')) 77 | 78 | # Setup build configuration 79 | self.__bb = generic_build( 80 | base_annotation=self.__class__.__name__, 81 | branch='knem-{}'.format(self.__version), 82 | comment=False, 83 | devel_environment=self.environment_variables, 84 | install=['mkdir -p {}/include'.format(self.__prefix), 85 | 'cp common/*.h {}/include'.format(self.__prefix)], 86 | runtime_environment=self.environment_variables, 87 | prefix=self.__prefix, 88 | repository=self.__repository, 89 | **kwargs) 90 | 91 | # Container instructions 92 | self += comment('KNEM version {}'.format(self.__version)) 93 | self += packages(ospackages=self.__ospackages) 94 | self += self.__bb 95 | 96 | def runtime(self, _from='0'): 97 | """Generate the set of instructions to install the runtime specific 98 | components from a build in a previous stage. 99 | 100 | # Examples 101 | 102 | ```python 103 | k = knem(...) 104 | Stage0 += k 105 | Stage1 += k.runtime() 106 | ``` 107 | """ 108 | self.rt += comment('KNEM') 109 | self.rt += self.__bb.runtime(_from=_from) 110 | return str(self.rt) 111 | -------------------------------------------------------------------------------- /hpccm/cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # pylint: disable=invalid-name 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import archspec.cpu 24 | import argparse 25 | import logging 26 | 27 | import hpccm 28 | from hpccm.version import __version__ 29 | 30 | class KeyValue(argparse.Action): # pylint: disable=too-few-public-methods 31 | """Extend argparse to handle key value pair options""" 32 | def __init__(self, option_strings, dest, nargs=None, **kwargs): 33 | """Initializing custom action, i.e., call the base class init""" 34 | super(KeyValue, self).__init__(option_strings, dest, nargs, **kwargs) 35 | 36 | def __call__(self, parser, namespace, values, option_string=None): 37 | """Process key value pair arguments""" 38 | d = {} 39 | for kv in values: 40 | key, value = kv.split('=') 41 | d[key] = value 42 | setattr(namespace, self.dest, d) 43 | 44 | def main(): # pragma: no cover 45 | parser = argparse.ArgumentParser(description='HPC Container Maker') 46 | parser.add_argument('--cpu-target', type=str, default=None, 47 | choices=[a for a in sorted(archspec.cpu.TARGETS)], 48 | help='cpu microarchitecture optimization target') 49 | parser.add_argument('--format', type=str, default='docker', 50 | choices=[i.name.lower() for i in hpccm.container_type], 51 | help='select output format') 52 | parser.add_argument('--print-exceptions', action='store_true', 53 | default=False, 54 | help='print exceptions (stack traces)') 55 | parser.add_argument('--recipe', required=True, 56 | help='generate a container spec for the RECIPE file') 57 | parser.add_argument('--single-stage', action='store_true', default=False, 58 | help='only process the first stage of a multi-stage ' + 59 | 'recipe') 60 | parser.add_argument('--singularity-version', type=str, default='2.6', 61 | help='set Singularity definition file format version') 62 | parser.add_argument('--userarg', action=KeyValue, metavar='key=value', 63 | nargs='+', help='specify user parameters') 64 | parser.add_argument('--version', action='version', version=__version__) 65 | parser.add_argument('--working-directory', '--wd', type=str, 66 | default='/var/tmp', 67 | help='set container working directory') 68 | args = parser.parse_args() 69 | 70 | # configure logger 71 | logging.basicConfig(format='%(levelname)s: %(message)s') 72 | 73 | recipe = hpccm.recipe(args.recipe, 74 | cpu_target=args.cpu_target, 75 | ctype=hpccm.container_type[args.format.upper()], 76 | raise_exceptions=args.print_exceptions, 77 | single_stage=args.single_stage, 78 | singularity_version=args.singularity_version, 79 | userarg=args.userarg, 80 | working_directory=args.working_directory) 81 | print(recipe) 82 | 83 | if __name__ == "__main__": # pragma: no cover 84 | main() 85 | -------------------------------------------------------------------------------- /hpccm/common.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Common stuff used by multiple parts of HPC Container Maker""" 18 | 19 | from enum import Enum 20 | 21 | class container_type(Enum): 22 | """Supported container types""" 23 | DOCKER = 1 24 | SINGULARITY = 2 25 | BASH = 3 26 | 27 | class cpu_arch(Enum): 28 | """Supported CPU architectures""" 29 | X86_64 = 1 30 | AARCH64 = 2 31 | PPC64LE = 3 32 | 33 | class linux_distro(Enum): 34 | """Supported Linux distribution types""" 35 | UBUNTU = 1 36 | CENTOS = 2 37 | RHEL = 2 38 | REDHAT = 2 39 | ROCKYLINUX = 2 40 | -------------------------------------------------------------------------------- /hpccm/primitives/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import absolute_import 16 | 17 | __all__ = ['baseimage', 'blob', 'comment', 'copy', 'environment', 'label', 18 | 'raw', 'runscript', 'shell', 'user', 'workdir'] 19 | 20 | from hpccm.primitives.baseimage import baseimage 21 | from hpccm.primitives.blob import blob 22 | from hpccm.primitives.comment import comment 23 | from hpccm.primitives.copy import copy 24 | from hpccm.primitives.environment import environment 25 | from hpccm.primitives.label import label 26 | from hpccm.primitives.raw import raw 27 | from hpccm.primitives.runscript import runscript 28 | from hpccm.primitives.shell import shell 29 | from hpccm.primitives.user import user 30 | from hpccm.primitives.workdir import workdir 31 | -------------------------------------------------------------------------------- /hpccm/primitives/blob.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Blob primitive""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import logging # pylint: disable=unused-import 24 | 25 | import hpccm.config 26 | 27 | from hpccm.common import container_type 28 | 29 | class blob(object): 30 | """The `blob` primitive inserts a file, without modification, into the 31 | corresponding place in the container specification file. If a 32 | relative path is specified, the path is relative to current 33 | directory. 34 | 35 | Generally, the blob should be functionally equivalent for each 36 | container format. 37 | 38 | Wherever possible, the blob primitive should be avoided and other, 39 | more portable, operations should be used instead. 40 | 41 | # Parameters 42 | 43 | docker: Path to the file containing the Dockerfile blob (Docker 44 | specific). 45 | 46 | singularity: Path to the file containing the Singularity blob 47 | (Singularity specific). 48 | 49 | # Example 50 | 51 | ```python 52 | blob(docker='path/to/foo.docker', singularity='path/to/foo.singularity') 53 | ``` 54 | """ 55 | 56 | def __init__(self, **kwargs): 57 | """Initialize primitive""" 58 | 59 | #super(blob, self).__init__() 60 | 61 | self.__docker = kwargs.get('docker', {}) # Docker specific 62 | self.__singularity = kwargs.get('singularity', {}) # Singularity 63 | # specific 64 | 65 | def __str__(self): 66 | """String representation of the primitive""" 67 | if hpccm.config.g_ctype == container_type.DOCKER: 68 | return self.__read_blob(self.__docker) 69 | if hpccm.config.g_ctype == container_type.SINGULARITY: 70 | return self.__read_blob(self.__singularity) 71 | elif hpccm.config.g_ctype == container_type.BASH: 72 | return '' 73 | else: 74 | raise RuntimeError('Unknown container type') 75 | 76 | def __read_blob(self, path): 77 | """Read the blob from a file""" 78 | 79 | b = '' 80 | try: 81 | if path: 82 | with open(path, 'r') as f: 83 | b = f.read() 84 | else: 85 | logging.warning('Blob file not specified') 86 | except IOError: 87 | logging.error('Error opening blob {}'.format(path)) 88 | 89 | return b 90 | -------------------------------------------------------------------------------- /hpccm/primitives/comment.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Comment primitive""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import logging # pylint: disable=unused-import 24 | import re 25 | import textwrap 26 | 27 | import hpccm.config 28 | 29 | from hpccm.common import container_type 30 | 31 | class comment(object): 32 | """The `comment` primitive inserts a comment into the corresponding 33 | place in the container specification file. 34 | 35 | # Parameters 36 | 37 | _app: String containing the 38 | [SCI-F](https://www.sylabs.io/guides/2.6/user-guide/reproducible_scif_apps.html) 39 | identifier. This also causes the comment to be enclosed in a 40 | Singularity block to named `%apphelp` (Singularity specific). 41 | 42 | reformat: Boolean flag to specify whether the comment string 43 | should be wrapped to fit into lines not exceeding 80 characters. 44 | The default is True. 45 | 46 | # Examples 47 | 48 | ```python 49 | comment('libfoo version X.Y') 50 | ``` 51 | 52 | """ 53 | 54 | def __init__(self, *args, **kwargs): 55 | """Initialize primitive""" 56 | 57 | #super(comment, self).__init__() 58 | 59 | try: 60 | self.__string = args[0] 61 | except IndexError: 62 | self.__string = '' 63 | 64 | self._app = kwargs.get('_app', False) # Singularity specific 65 | self.__reformat = kwargs.get('reformat', True) 66 | 67 | def __str__(self): 68 | """String representation of the primitive""" 69 | if self.__string: 70 | # Comments are universal (so far...) 71 | if (self._app and 72 | hpccm.config.g_ctype == container_type.SINGULARITY): 73 | return '%apphelp {0}\n{1}'.format(self._app, 74 | self.__string) 75 | 76 | if self.__reformat: 77 | # Wrap comments 78 | return textwrap.fill(self.__string, initial_indent='# ', 79 | subsequent_indent='# ', width=70) 80 | else: 81 | # Just prepend but otherwise apply no formatting 82 | return re.sub('^', '# ', self.__string, flags=re.MULTILINE) 83 | else: 84 | return '' 85 | 86 | def merge(self, lst, _app=None): 87 | """Merge one or more instances of the primitive into a single 88 | instance. Due to conflicts or option differences the merged 89 | primitive may not be exact merger. 90 | 91 | """ 92 | 93 | if not lst: # pragma: nocover 94 | raise RuntimeError('no items provided to merge') 95 | 96 | s = [] 97 | for item in lst: 98 | if not item.__class__.__name__ == 'comment': # pragma: nocover 99 | logging.warning('item is not the correct type, skipping...') 100 | continue 101 | 102 | s.append(item._comment__string) 103 | 104 | return comment('\n'.join(s), reformat=False, _app=_app) 105 | -------------------------------------------------------------------------------- /hpccm/primitives/label.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Label primitive""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import logging # pylint: disable=unused-import 24 | 25 | import hpccm.config 26 | 27 | from hpccm.common import container_type 28 | 29 | class label(object): 30 | """The `label` primitive sets container metadata. 31 | 32 | # Parameters 33 | 34 | _app: String containing the [SCI-F](https://www.sylabs.io/guides/2.6/user-guide/reproducible_scif_apps.html) 35 | identifier. This also causes the Singularity block to named 36 | `%applabels` rather than `%labels` (Singularity specific). 37 | 38 | metadata: A dictionary of key / value pairs. The default is an 39 | empty dictionary. 40 | 41 | # Examples 42 | 43 | ```python 44 | label(metadata={'maintainer': 'jane@doe'}) 45 | ``` 46 | """ 47 | 48 | def __init__(self, **kwargs): 49 | """Initialize primitive""" 50 | 51 | #super(label, self).__init__() 52 | 53 | self._app = kwargs.get('_app', '') # Singularity specific 54 | self.__metadata = kwargs.get('metadata', {}) 55 | 56 | def __str__(self): 57 | """String representation of the primitive""" 58 | 59 | if self.__metadata: 60 | if hpccm.config.g_ctype == container_type.DOCKER: 61 | if self._app: 62 | logging.warning('The Singularity specific %app.. syntax ' 63 | 'was requested. Docker does not have an ' 64 | 'equivalent: using regular LABEL!') 65 | 66 | # Format: 67 | # LABEL K1=V1 \ 68 | # K2=V2 \ 69 | # K3=V3 70 | keyvals = [] 71 | for key, val in sorted(self.__metadata.items()): 72 | keyvals.append('{0}={1}'.format(key, val)) 73 | 74 | l = ['LABEL {}'.format(keyvals[0])] 75 | l.extend([' {}'.format(x) for x in keyvals[1:]]) 76 | return ' \\\n'.join(l) 77 | elif hpccm.config.g_ctype == container_type.SINGULARITY: 78 | # Format: 79 | # %labels 80 | # K1 V1 81 | # K2 V2 82 | # K3 V3 83 | keyvals = [] 84 | for key, val in sorted(self.__metadata.items()): 85 | keyvals.append('{0} {1}'.format(key, val)) 86 | 87 | if self._app: 88 | l = ['%applabels {0}'.format(self._app)] 89 | else: 90 | l = ['%labels'] 91 | l.extend([' {}'.format(x) for x in keyvals]) 92 | return '\n'.join(l) 93 | elif hpccm.config.g_ctype == container_type.BASH: 94 | logging.warning('label primitive does not map into bash') 95 | return '' 96 | else: 97 | raise RuntimeError('Unknown container type') 98 | else: 99 | return '' 100 | 101 | def merge(self, lst, _app=None): 102 | """Merge one or more instances of the primitive into a single 103 | instance. Due to conflicts or option differences the merged 104 | primitive may not be exact. 105 | 106 | """ 107 | 108 | if not lst: # pragma: nocover 109 | raise RuntimeError('no items provided to merge') 110 | 111 | labels = {} 112 | for item in lst: 113 | if not item.__class__.__name__ == 'label': # pragma: nocover 114 | logging.warning('item is not the correct type, skipping...') 115 | continue 116 | 117 | labels.update(item._label__metadata) 118 | 119 | return label(metadata=labels, _app=_app) 120 | -------------------------------------------------------------------------------- /hpccm/primitives/raw.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Raw primitive""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import hpccm.config 24 | 25 | from hpccm.common import container_type 26 | 27 | class raw(object): 28 | """The `raw` primitive inserts the specified string, without 29 | modification, into the corresponding place in the container 30 | specification file. 31 | 32 | Generally, the string should be functionally equivalent for each 33 | container format. 34 | 35 | Wherever possible, the raw primitive should be avoided and other, 36 | more portable, primitives should be used instead. 37 | 38 | # Parameters 39 | 40 | docker: String containing the Dockerfile instruction (Docker 41 | specific). 42 | 43 | singularity: String containing the Singularity instruction 44 | (Singularity specific). 45 | 46 | # Examples 47 | 48 | ```python 49 | raw(docker='COPY --from=0 /usr/local/openmpi /usr/local/openmpi', 50 | singularity='# no equivalent to --from') 51 | ``` 52 | 53 | """ 54 | 55 | def __init__(self, **kwargs): 56 | """Raw primitive""" 57 | 58 | #super(raw, self).__init__() 59 | 60 | self.__docker = kwargs.get('docker', '') # Docker specific 61 | self.__singularity = kwargs.get('singularity', '') # Singularity 62 | # specific 63 | 64 | def __str__(self): 65 | """String representation of the primitive""" 66 | if hpccm.config.g_ctype == container_type.DOCKER: 67 | return str(self.__docker) 68 | elif hpccm.config.g_ctype == container_type.SINGULARITY: 69 | return str(self.__singularity) 70 | elif hpccm.config.g_ctype == container_type.BASH: 71 | return '' 72 | else: 73 | raise RuntimeError('Unknown container type') 74 | -------------------------------------------------------------------------------- /hpccm/primitives/user.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """User primitive""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import logging # pylint: disable=unused-import 24 | 25 | import hpccm.config 26 | 27 | from hpccm.common import container_type 28 | 29 | class user(object): 30 | """The `user` primitive sets the user name to use for any subsequent 31 | steps. 32 | 33 | This primitive is the null operation for Singularity. 34 | 35 | # Parameters 36 | 37 | user: The user name to use. The default is an empty string. 38 | 39 | # Examples 40 | 41 | ```python 42 | user(user='ncognito') 43 | ``` 44 | """ 45 | 46 | def __init__(self, **kwargs): 47 | """Initialize primitive""" 48 | 49 | self.user = kwargs.get('user', '') 50 | 51 | def __str__(self): 52 | """String representation of the primitive""" 53 | if self.user: 54 | if hpccm.config.g_ctype == container_type.DOCKER: 55 | return 'USER {}'.format(self.user) 56 | elif hpccm.config.g_ctype == container_type.SINGULARITY: 57 | return '' 58 | elif hpccm.config.g_ctype == container_type.BASH: 59 | return '' 60 | else: 61 | raise RuntimeError('Unknown container type') 62 | else: 63 | logging.error('No user specified') 64 | return '' 65 | -------------------------------------------------------------------------------- /hpccm/primitives/workdir.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Working directory primitive""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import logging # pylint: disable=unused-import 24 | 25 | import hpccm.config 26 | 27 | from hpccm.common import container_type 28 | from hpccm.primitives.shell import shell 29 | 30 | class workdir(object): 31 | """The `workdir` primitive sets the working directory for any 32 | subsequent operations. As a side effect, if the directory does 33 | not exist, it is created. 34 | 35 | # Parameters 36 | 37 | directory: The directory path. 38 | 39 | # Examples 40 | 41 | ```python 42 | workdir(directory='/path/to/directory') 43 | ``` 44 | """ 45 | 46 | def __init__(self, **kwargs): 47 | """Initialize primitive""" 48 | 49 | #super(workdir, self).__init__() 50 | 51 | self.directory = kwargs.get('directory', '') 52 | 53 | def __str__(self): 54 | """String representation of the primitive""" 55 | if self.directory: 56 | if hpccm.config.g_ctype == container_type.DOCKER: 57 | return 'WORKDIR {}'.format(self.directory) 58 | elif hpccm.config.g_ctype == container_type.SINGULARITY: 59 | s = shell(commands=['mkdir -p {}'.format(self.directory), 60 | 'cd {}'.format(self.directory)]) 61 | return str(s) 62 | elif hpccm.config.g_ctype == container_type.BASH: 63 | logging.warning('workdir primitive does not map into bash') 64 | return '' 65 | else: 66 | raise RuntimeError('Unknown container type') 67 | else: 68 | logging.error('No directory specified') 69 | return '' 70 | -------------------------------------------------------------------------------- /hpccm/templates/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import absolute_import 16 | 17 | from hpccm.templates.CMakeBuild import CMakeBuild 18 | from hpccm.templates.ConfigureMake import ConfigureMake 19 | from hpccm.templates.annotate import annotate 20 | from hpccm.templates.downloader import downloader 21 | from hpccm.templates.envvars import envvars 22 | from hpccm.templates.git import git 23 | from hpccm.templates.ldconfig import ldconfig 24 | from hpccm.templates.rm import rm 25 | from hpccm.templates.sed import sed 26 | from hpccm.templates.tar import tar 27 | from hpccm.templates.wget import wget 28 | from hpccm.templates.zipfile import zipfile 29 | -------------------------------------------------------------------------------- /hpccm/templates/annotate.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """annotate template""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | from six import string_types 23 | from six.moves import shlex_quote 24 | 25 | import hpccm.base_object 26 | 27 | class annotate(hpccm.base_object): 28 | """Template for setting annotations""" 29 | 30 | def __init__(self, **kwargs): 31 | """Initialize template""" 32 | 33 | super(annotate, self).__init__(**kwargs) 34 | 35 | self.annotate = kwargs.get('annotate', False) 36 | self.base_annotation = kwargs.get('base_annotation', True) 37 | self.__labels = {} 38 | 39 | def add_annotation(self, key, value): 40 | if isinstance(self.base_annotation, string_types): 41 | key = 'hpccm.' + self.base_annotation + '.' + key 42 | elif self.base_annotation: 43 | key = 'hpccm.' + self.__class__.__name__ + '.' + key 44 | 45 | self.__labels[key] = shlex_quote(str(value)) 46 | 47 | def annotate_step(self): 48 | """Return dictionary of annotations""" 49 | 50 | if self.annotate: 51 | return self.__labels 52 | else: 53 | return {} 54 | -------------------------------------------------------------------------------- /hpccm/templates/envvars.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """environment variables template""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import hpccm.base_object 24 | 25 | class envvars(hpccm.base_object): 26 | """Template for setting environment variables""" 27 | 28 | def __init__(self, **kwargs): 29 | """Initialize template""" 30 | 31 | super(envvars, self).__init__(**kwargs) 32 | 33 | self.environment = kwargs.get('environment', True) 34 | self.environment_variables = {} 35 | # Use only if the runtime environment is incompatible with the 36 | # non-runtime environment, e.g., PATH contains different 37 | # values. Otherwise, try to use the filtering options. 38 | self.runtime_environment_variables = {} 39 | 40 | def environment_step(self, include_only=None, exclude=None, runtime=False): 41 | """Return dictionary of environment variables""" 42 | 43 | if runtime: 44 | e = self.runtime_environment_variables 45 | else: 46 | e = self.environment_variables 47 | 48 | if self.environment: 49 | if include_only: 50 | return {x: e[x] for x in e if x in include_only} 51 | elif exclude: 52 | return {x: e[x] for x in e if x not in exclude} 53 | else: 54 | return e 55 | else: 56 | return {} 57 | -------------------------------------------------------------------------------- /hpccm/templates/ldconfig.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """ldconfig template""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import logging # pylint: disable=unused-import 24 | import posixpath 25 | 26 | import hpccm.base_object 27 | 28 | class ldconfig(hpccm.base_object): 29 | """Template for manipulating the dynamic linker""" 30 | 31 | def __init__(self, **kwargs): 32 | """Initialize template""" 33 | 34 | super(ldconfig, self).__init__(**kwargs) 35 | 36 | self.ldconfig = kwargs.get('ldconfig', False) 37 | 38 | def ldcache_step(self, conf='hpccm.conf', directory=None): 39 | """Add a directory to the dynamic linker cache""" 40 | 41 | if not directory: 42 | logging.error('directory is not defined') 43 | return '' 44 | 45 | return 'echo "{0}" >> {1} && ldconfig'.format( 46 | directory, posixpath.join('/etc/ld.so.conf.d', conf)) 47 | -------------------------------------------------------------------------------- /hpccm/templates/rm.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """rm template""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import logging # pylint: disable=unused-import 24 | 25 | import hpccm.base_object 26 | 27 | class rm(hpccm.base_object): 28 | """Template for cleaning up files and directories""" 29 | 30 | def __init__(self, **kwargs): 31 | """Initialize template""" 32 | 33 | super(rm, self).__init__(**kwargs) 34 | 35 | def cleanup_step(self, items=None): 36 | """Cleanup files and directories""" 37 | 38 | if not items: 39 | logging.error('items are not defined') 40 | return '' 41 | 42 | return 'rm -rf {}'.format(' '.join(items)) 43 | -------------------------------------------------------------------------------- /hpccm/templates/sed.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """sed template""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | from six.moves import shlex_quote 23 | 24 | import logging # pylint: disable=unused-import 25 | 26 | import hpccm.base_object 27 | 28 | class sed(hpccm.base_object): 29 | """sed template""" 30 | 31 | def __init__(self, **kwargs): 32 | """Initialize sed template""" 33 | 34 | super(sed, self).__init__(**kwargs) 35 | 36 | self.sed_opts = kwargs.get('opts', []) 37 | 38 | def sed_step(self, file=None, in_place=True, patterns=[]): 39 | """Generate sed command line string""" 40 | 41 | if not file: 42 | logging.error('file is not defined') 43 | return '' 44 | 45 | if not patterns: 46 | logging.error('patterns is not defined') 47 | return '' 48 | 49 | # Copy so not to modify the member variable 50 | opts = list(self.sed_opts) 51 | 52 | if in_place: 53 | opts.append('-i') 54 | 55 | opt_string = ' '.join(opts) 56 | 57 | quoted_patterns = ['-e {}'.format(shlex_quote(patterns[0]))] 58 | quoted_patterns.extend(' -e {}'.format(shlex_quote(x)) for x in patterns[1:]) 59 | quoted_pattern_string = ' \\\n'.join(quoted_patterns) 60 | 61 | return 'sed {0} {1} {2}'.format(opt_string, quoted_pattern_string, file) 62 | -------------------------------------------------------------------------------- /hpccm/templates/tar.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """tar template""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import logging # pylint: disable=unused-import 24 | import re 25 | 26 | import hpccm.base_object 27 | 28 | class tar(hpccm.base_object): 29 | """tar template""" 30 | 31 | def __init__(self, **kwargs): 32 | """Initialize tar template""" 33 | 34 | super(tar, self).__init__(**kwargs) 35 | 36 | def untar_step(self, tarball=None, directory=None, args=None): 37 | """Generate untar command line string""" 38 | 39 | if not tarball: 40 | logging.error('tarball is not defined') 41 | return '' 42 | 43 | opts = ['-x', '-f {}'.format(tarball)] 44 | if directory: 45 | opts.append('-C {}'.format(directory)) 46 | 47 | if re.search(r'\.tar\.bz2$', tarball): 48 | opts.append('-j') 49 | elif re.search(r'\.tar\.gz$', tarball): 50 | opts.append('-z') 51 | elif re.search(r'\.tgz$', tarball): 52 | opts.append('-z') 53 | elif re.search(r'\.tar\.xz$', tarball): 54 | opts.append('-J') 55 | elif re.search(r'\.txz$', tarball): 56 | opts.append('-J') 57 | elif re.search(r'\.tbz$', tarball): 58 | opts.append('-j') 59 | elif re.search(r'\.tar$', tarball): 60 | pass 61 | else: 62 | logging.warning('File type not recognized, trying anyway...') 63 | 64 | if args: 65 | opts.extend(args) 66 | 67 | if directory: 68 | return 'mkdir -p {0} && tar {1}'.format(directory, ' '.join(opts)) 69 | else: 70 | return 'tar {}'.format(' '.join(opts)) 71 | -------------------------------------------------------------------------------- /hpccm/templates/wget.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """wget template""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import logging # pylint: disable=unused-import 24 | 25 | import hpccm.base_object 26 | 27 | class wget(hpccm.base_object): 28 | """wget template""" 29 | 30 | def __init__(self, **kwargs): 31 | """Initialize wget template""" 32 | 33 | super(wget, self).__init__(**kwargs) 34 | 35 | self.wget_opts = kwargs.get('opts', ['-q', '-nc', 36 | '--no-check-certificate']) 37 | 38 | def download_step(self, outfile=None, referer=None, url=None, 39 | directory='/tmp'): 40 | """Generate wget command line string""" 41 | 42 | if not url: 43 | logging.error('url is not defined') 44 | return '' 45 | 46 | # Copy so not to modify the member variable 47 | opts = self.wget_opts 48 | 49 | if outfile: 50 | opts.append('-O {}'.format(outfile)) 51 | 52 | if referer: 53 | opts.append('--referer {}'.format(referer)) 54 | 55 | opt_string = ' '.join(self.wget_opts) 56 | 57 | # Add annotation if the caller inherits from the annotate template 58 | if callable(getattr(self, 'add_annotation', None)): 59 | self.add_annotation('url', url) 60 | 61 | # Ensure the directory exists 62 | return 'mkdir -p {1} && wget {0} -P {1} {2}'.format(opt_string, 63 | directory, url) 64 | -------------------------------------------------------------------------------- /hpccm/templates/zipfile.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """zipfile template""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import logging # pylint: disable=unused-import 24 | 25 | import hpccm.base_object 26 | 27 | class zipfile(hpccm.base_object): 28 | """zipfile template""" 29 | 30 | def __init__(self, **kwargs): 31 | """Initialize zip template""" 32 | 33 | super(zipfile, self).__init__(**kwargs) 34 | 35 | def unzip_step(self, zipfile=None, directory=None): 36 | """Generate zip command line string""" 37 | 38 | if not zipfile: 39 | logging.error('zip file is not defined') 40 | return '' 41 | 42 | if directory: 43 | return 'mkdir -p {0} && unzip -d {0} {1}'.format(directory, 44 | zipfile) 45 | else: 46 | return 'unzip {}'.format(zipfile) 47 | -------------------------------------------------------------------------------- /hpccm/toolchain.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Build toolchain""" 18 | 19 | from __future__ import absolute_import 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | class toolchain(object): 24 | """Class for the build toolchain. Attributes map to the commonly used 25 | environment variables, e.g, CC is the C compiler, CXX is the 26 | C++ compiler.""" 27 | 28 | __attrs__ = ['CC', 'CFLAGS', 'CPPFLAGS', 'CUDA_HOME', 'CXX', 29 | 'CXXFLAGS', 'F77', 'F90', 'FC', 'FCFLAGS', 'FFLAGS', 30 | 'FLIBS', 'LDFLAGS', 'LD_LIBRARY_PATH', 'LIBS'] 31 | 32 | def __init__(self, **kwargs): 33 | """Initialize toolchain""" 34 | 35 | self.CC = kwargs.get('CC') 36 | self.CFLAGS = kwargs.get('CFLAGS') 37 | self.CPPFLAGS = kwargs.get('CPPFLAGS') 38 | self.CUDA_HOME = kwargs.get('CUDA_HOME') 39 | self.CXX = kwargs.get('CXX') 40 | self.CXXFLAGS = kwargs.get('CXXFLAGS') 41 | self.F77 = kwargs.get('F77') 42 | self.F90 = kwargs.get('F90') 43 | self.FC = kwargs.get('FC') 44 | self.FCFLAGS = kwargs.get('FCFLAGS') 45 | self.FFLAGS = kwargs.get('FFLAGS') 46 | self.FLIBS = kwargs.get('FLIBS') 47 | self.LDFLAGS = kwargs.get('LDFLAGS') 48 | self.LD_LIBRARY_PATH = kwargs.get('LD_LIBRARY_PATH') 49 | self.LIBS = kwargs.get('LIBS') 50 | 51 | def __copy__(self): 52 | """Copy all the attributes even if __dict__ only returns the pairs 53 | with non-null values.""" 54 | cls = self.__class__ 55 | result = cls.__new__(cls) 56 | for key in self.__attrs__: 57 | val = getattr(self, key) 58 | setattr(result, key, val if val else None) 59 | return result 60 | 61 | def __deepcopy__(self, memo): 62 | """Copy all the attributes even if __dict__ only returns the pairs 63 | with non-null values.""" 64 | result = self.__copy__() 65 | memo[id(self)] = result 66 | return result 67 | 68 | @property 69 | def __dict__(self): 70 | """Return only those attributes that have non-null values. This 71 | enables usage like 'environment(variables=var(toolchain))'""" 72 | return {key: getattr(self, key) for key in self.__attrs__ 73 | if getattr(self, key)} 74 | -------------------------------------------------------------------------------- /hpccm/version.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from __future__ import absolute_import 16 | 17 | __version__ = '25.3.0' 18 | -------------------------------------------------------------------------------- /pydocmd.yml: -------------------------------------------------------------------------------- 1 | # Configuration file for pydocmd 2 | # Run "pydocmd generate" to generate updated documentation 3 | site_name: "HPC Container Maker" 4 | 5 | generate: 6 | - building_blocks.md: 7 | - hpccm.building_blocks.amgx+ 8 | - hpccm.building_blocks.apt_get+ 9 | - hpccm.building_blocks.arm_allinea_studio+ 10 | - hpccm.building_blocks.boost+ 11 | - hpccm.building_blocks.catalyst+ 12 | - hpccm.building_blocks.cgns+ 13 | - hpccm.building_blocks.charm+ 14 | - hpccm.building_blocks.cmake+ 15 | - hpccm.building_blocks.conda+ 16 | - hpccm.building_blocks.doca_ofed+ 17 | - hpccm.building_blocks.fftw+ 18 | - hpccm.building_blocks.gdrcopy+ 19 | - hpccm.building_blocks.generic_autotools+ 20 | - hpccm.building_blocks.generic_build+ 21 | - hpccm.building_blocks.generic_cmake+ 22 | - hpccm.building_blocks.gnu+ 23 | - hpccm.building_blocks.hdf5+ 24 | - hpccm.building_blocks.hpcx+ 25 | - hpccm.building_blocks.intel_mpi+ 26 | - hpccm.building_blocks.intel_psxe+ 27 | - hpccm.building_blocks.intel_psxe_runtime+ 28 | - hpccm.building_blocks.julia+ 29 | - hpccm.building_blocks.knem+ 30 | - hpccm.building_blocks.kokkos+ 31 | - hpccm.building_blocks.libsim+ 32 | - hpccm.building_blocks.llvm+ 33 | - hpccm.building_blocks.magma+ 34 | - hpccm.building_blocks.mkl+ 35 | - hpccm.building_blocks.mlnx_ofed+ 36 | - hpccm.building_blocks.mpich+ 37 | - hpccm.building_blocks.multi_ofed+ 38 | - hpccm.building_blocks.mvapich2+ 39 | - hpccm.building_blocks.mvapich2_gdr+ 40 | - hpccm.building_blocks.nccl+ 41 | - hpccm.building_blocks.netcdf+ 42 | - hpccm.building_blocks.nsight_compute+ 43 | - hpccm.building_blocks.nsight_systems+ 44 | - hpccm.building_blocks.nvhpc+ 45 | - hpccm.building_blocks.nvshmem+ 46 | - hpccm.building_blocks.ofed+ 47 | - hpccm.building_blocks.openblas+ 48 | - hpccm.building_blocks.openmpi+ 49 | - hpccm.building_blocks.packages+ 50 | - hpccm.building_blocks.pgi+ 51 | - hpccm.building_blocks.pip+ 52 | - hpccm.building_blocks.pmix+ 53 | - hpccm.building_blocks.pnetcdf+ 54 | - hpccm.building_blocks.python+ 55 | - hpccm.building_blocks.rdma_core+ 56 | - hpccm.building_blocks.scif+ 57 | - hpccm.building_blocks.sensei+ 58 | - hpccm.building_blocks.slurm_pmi2+ 59 | - hpccm.building_blocks.ucx+ 60 | - hpccm.building_blocks.xpmem+ 61 | - hpccm.building_blocks.yum+ 62 | - primitives.md: 63 | - hpccm.primitives.baseimage 64 | - hpccm.primitives.blob 65 | - hpccm.primitives.comment 66 | - hpccm.primitives.copy 67 | - hpccm.primitives.environment 68 | - hpccm.primitives.label 69 | - hpccm.primitives.raw 70 | - hpccm.primitives.runscript 71 | - hpccm.primitives.shell 72 | - hpccm.primitives.user 73 | - hpccm.primitives.workdir 74 | - misc_api.md: 75 | - hpccm.config+ 76 | - hpccm.recipe+ 77 | - hpccm.Stage+ 78 | 79 | pages: 80 | - Home: README.md << README.md 81 | #- Getting Started: getting_started.md << docs/getting_started.md 82 | #- Tutorial: tutorial.md << docs/tutorial.md 83 | - Building Blocks: building_blocks.md 84 | - Primitives: primitives.md 85 | - Miscellaneous APIs: misc_api.md 86 | #- Citation: citation.md << docs/citation.md 87 | #- License: license << LICENSE 88 | 89 | headers: markdown 90 | 91 | gens_dir: docs 92 | site_dir: site 93 | -------------------------------------------------------------------------------- /recipes/easybuild.py: -------------------------------------------------------------------------------- 1 | """EasyBuild container (https://github.com/easybuilders/easybuild) 2 | Set the user argument 'easyconfig' to specify the EasyConfig to 3 | build. Otherwise, it will just build a base EasyBuild container 4 | image. 5 | 6 | Sample workflow: 7 | $ hpccm.py --recipe recipes/easybuild.py --userarg easyconfig=GROMACS-2019.3-fosscuda-2019b.eb > Dockerfile.gromacs.eb 8 | $ docker build -t gromacs.eb -f Dockerfile.gromacs.eb . 9 | $ nvidia-docker run --rm -ti gromacs.eb bash -l 10 | container:/tmp> module load GROMACS 11 | ... 12 | """ 13 | # pylint: disable=invalid-name, undefined-variable, used-before-assignment 14 | import os 15 | 16 | Stage0 += comment(__doc__, reformat=False) 17 | 18 | Stage0 += baseimage(image='rockylinux:8') 19 | 20 | Stage0 += shell(commands=['yum update -y rocky-release', 21 | 'rm -rf /var/cache/yum/*']) 22 | 23 | # Base dependencies 24 | Stage0 += python(python3=False) 25 | Stage0 += gnu() 26 | Stage0 += ofed() 27 | Stage0 += packages(epel=True, powertools=True, 28 | yum=['bzip2', 'diffutils', 'file', 'git', 'gzip', 29 | 'libtool', 'Lmod', 'make', 'openssh-clients', 30 | 'openssl-devel', 'patch', 'rsh', 'tar', 'unzip', 31 | 'which', 'xz']) 32 | 33 | # Setup and install EasyBuild 34 | Stage0 += pip(packages=['easybuild'], pip='pip2') 35 | Stage0 += shell(commands=['useradd -m easybuild', 36 | 'mkdir -p /opt/easybuild', 37 | 'chown easybuild:easybuild /opt/easybuild']) 38 | 39 | # Module environment 40 | Stage0 += environment(variables={'MODULEPATH': '/opt/easybuild/modules/all:/home/easybuild/.local/easybuild/modules/all:$MODULEPATH'}) 41 | 42 | easyconfig = USERARG.get('easyconfig', None) 43 | if easyconfig: 44 | # If the easyconfig is a file in the local build context, inject it 45 | # into the container image 46 | if os.path.isfile(easyconfig): 47 | Stage0 += copy(src=easyconfig, dest='/home/easybuild') 48 | 49 | Stage0 += shell(commands=['runuser easybuild -l -c "eb {} -r --installpath /opt/easybuild"'.format(easyconfig)]) 50 | -------------------------------------------------------------------------------- /recipes/examples/basic.py: -------------------------------------------------------------------------------- 1 | """This example demonstrates recipe basics. 2 | 3 | Usage: 4 | $ hpccm.py --recipe recipes/examples/basic.py --format docker 5 | # hpccm.py --recipe recipes/examples/basic.py --format singularity 6 | """ 7 | 8 | # Choose a base image 9 | Stage0 += baseimage(image='ubuntu:16.04') 10 | 11 | # Install GNU compilers (upstream) 12 | Stage0 += apt_get(ospackages=['gcc', 'g++', 'gfortran']) 13 | -------------------------------------------------------------------------------- /recipes/examples/cloverleaf.py: -------------------------------------------------------------------------------- 1 | """ 2 | Demonstrate how to include a recipe in another recipe 3 | """ 4 | 5 | # the gnu-devel recipe file must be in the same directory 6 | hpccm.include('gnu-devel.py') 7 | 8 | Stage0 += generic_build(branch='v1.3', 9 | build=['make COMPILER=GNU'], 10 | install=['cp clover_leaf /usr/local/bin'], 11 | repository='https://github.com/UK-MAC/CloverLeaf_Serial.git') 12 | -------------------------------------------------------------------------------- /recipes/examples/gnu-devel.py: -------------------------------------------------------------------------------- 1 | """ 2 | Basic development container image 3 | """ 4 | 5 | Stage0 += baseimage(image='ubuntu:18.04') 6 | 7 | # GNU compilers 8 | compiler = gnu() 9 | Stage0 += compiler 10 | 11 | # Additional development tools 12 | Stage0 += packages(ospackages=['autoconf', 'autoconf-archive', 'automake', 13 | 'bzip2', 'ca-certificates', 'cmake', 'git', 14 | 'gzip', 'libtool', 'make', 'patch', 'xz-utils']) 15 | -------------------------------------------------------------------------------- /recipes/examples/multistage.py: -------------------------------------------------------------------------------- 1 | """This example demonstrates how to specify a multi-stage recipe. 2 | 3 | Usage: 4 | $ hpccm.py --recipe recipes/examples/multistage.py 5 | """ 6 | 7 | ###### 8 | # Devel stage 9 | ###### 10 | 11 | # Devel stage base image 12 | Stage0 += baseimage(image='nvcr.io/nvidia/cuda:9.0-devel-ubuntu16.04', 13 | _as='devel') 14 | 15 | # Install compilers (upstream) 16 | Stage0 += gnu() 17 | 18 | # Build FFTW using all default options 19 | Stage0 += fftw() 20 | 21 | ###### 22 | # Runtime stage 23 | ###### 24 | 25 | # Runtime stage base image 26 | Stage1 += baseimage(image='nvcr.io/nvidia/cuda:9.0-runtime-ubuntu16.04') 27 | 28 | # Copy the compiler runtime and FFTW from the devel stage and setup the 29 | # runtime environment. 30 | # The _from option is not necessary, but increases clarity. 31 | Stage1 += Stage0.runtime(_from='devel') 32 | -------------------------------------------------------------------------------- /recipes/examples/scif.py: -------------------------------------------------------------------------------- 1 | """ 2 | Build the CUDA-STREAM benchmark for multiple CUDA compute capabilities. 3 | 4 | Make each build available as a SCI-F application. 5 | """ 6 | Stage0 += baseimage(image='nvcr.io/nvidia/cuda:9.1-devel-centos7', _as='devel') 7 | 8 | # Install the GNU compiler 9 | Stage0 += gnu(fortran=False) 10 | 11 | # Install SCI-F 12 | Stage0 += pip(packages=['scif'], upgrade=True) 13 | 14 | # Download a single copy of the source code 15 | Stage0 += packages(ospackages=['ca-certificates', 'git']) 16 | Stage0 += shell(commands=['cd /var/tmp', 17 | 'git clone --depth=1 https://github.com/bcumming/cuda-stream.git cuda-stream']) 18 | 19 | # Build CUDA-STREAM as a SCI-F application for each CUDA compute capability 20 | for cc in ['35', '60', '70']: 21 | binpath = '/scif/apps/cc{}/bin'.format(cc) 22 | 23 | stream = scif(name='cc{}'.format(cc)) 24 | stream += comment('CUDA-STREAM built for CUDA compute capability {}'.format(cc)) 25 | stream += shell(commands=['nvcc -std=c++11 -ccbin=g++ -gencode arch=compute_{0},code=\\"sm_{0},compute_{0}\\" -o {1}/stream /var/tmp/cuda-stream/stream.cu'.format(cc, binpath)]) 26 | stream += environment(variables={'PATH': '{}:$PATH'.format(binpath)}) 27 | stream += label(metadata={'COMPUTE_CAPABILITY': cc}) 28 | stream += shell(commands=['stream'], _test=True) 29 | stream += runscript(commands=['stream']) 30 | 31 | Stage0 += stream 32 | 33 | # Runtime stage 34 | Stage1 += baseimage(image='nvcr.io/nvidia/cuda:9.1-base-centos7') 35 | 36 | # Install SCI-F 37 | Stage1 += pip(packages=['scif'], upgrade=True) 38 | 39 | # Install runtime components from the first stage 40 | Stage1 += Stage0.runtime() 41 | -------------------------------------------------------------------------------- /recipes/examples/script.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """This example demonstrates importing hpccm in a Python script. 4 | 5 | Rather than using the hpccm command line tool to convert a recipe into 6 | a container specification, import hpccm directly. 7 | 8 | The script is responsible for handling user arguments, managing 9 | layers, and printing the output. 10 | 11 | Note: HPCCM must be installed via pip prior to using this script. 12 | 13 | Usage: 14 | $ python recipes/examples/script.py --help 15 | $ python recipes/examples/script.py --linux ubuntu 16 | $ python recipes/examples/script.py --format singularity 17 | 18 | """ 19 | 20 | from __future__ import unicode_literals 21 | from __future__ import print_function 22 | 23 | import argparse 24 | import hpccm 25 | 26 | ### Parse command line arguments 27 | parser = argparse.ArgumentParser(description='HPCCM example') 28 | parser.add_argument('--compiler', type=str, default='gnu', 29 | choices=['gnu', 'llvm', 'nvhpc'], 30 | help='Compiler choice (default: gnu)') 31 | parser.add_argument('--format', type=str, default='docker', 32 | choices=['docker', 'singularity'], 33 | help='Container specification format (default: docker)') 34 | parser.add_argument('--linux', type=str, default='centos', 35 | choices=['centos', 'ubuntu'], 36 | help='Linux distribution choice (default: centos)') 37 | parser.add_argument('--nvhpc_eula_accept', action='store_true', 38 | default=False, 39 | help='Accept PGI EULA (default: false)') 40 | args = parser.parse_args() 41 | 42 | ### Create Stage 43 | Stage0 = hpccm.Stage() 44 | 45 | ### Linux distribution 46 | if args.linux == 'centos': 47 | Stage0 += hpccm.primitives.baseimage(image='centos:7') 48 | elif args.linux == 'ubuntu': 49 | Stage0 += hpccm.primitives.baseimage(image='ubuntu:16.04') 50 | 51 | ### Compiler 52 | if args.compiler == 'gnu': 53 | Stage0 += hpccm.building_blocks.gnu() 54 | elif args.compiler == 'llvm': 55 | Stage0 += hpccm.building_blocks.llvm() 56 | elif args.compiler == 'nvhpc': 57 | if not args.nvhpc_eula_accept: 58 | print('EULA not accepted. To accept, use "--nvhpc_eula_accept".\n' 59 | 'See NVIDIA EULA at https://docs.nvidia.com/hpc-sdk/eula') 60 | exit(1) 61 | Stage0 += hpccm.building_blocks.nvhpc(eula=args.nvhpc_eula_accept) 62 | 63 | ### Set container specification output format 64 | hpccm.config.set_container_format(args.format) 65 | 66 | ### Output container specification 67 | print(Stage0) 68 | -------------------------------------------------------------------------------- /recipes/examples/userargs.py: -------------------------------------------------------------------------------- 1 | """This example demonstrates user arguments. 2 | 3 | The CUDA and OpenMPI versions can be specified on the command line. If 4 | they are not, then reasonable defaults are used. 5 | 6 | Note: no validation is performed on the user supplied information. 7 | 8 | Usage: 9 | $ hpccm.py --recipe recipes/examples/userargs.py --userarg cuda=9.0 ompi=2.1.2 10 | """ 11 | 12 | # Set the image tag based on the specified version (default to 9.1) 13 | cuda_version = USERARG.get('cuda', '9.1') 14 | image = 'nvcr.io/nvidia/cuda:{}-devel-ubuntu16.04'.format(cuda_version) 15 | 16 | Stage0 += baseimage(image=image) 17 | 18 | # Set the OpenMPI version based on the specified version (default to 3.0.0) 19 | ompi_version = USERARG.get('ompi', '3.0.0') 20 | Stage0 += openmpi(infiniband=False, version=ompi_version) 21 | -------------------------------------------------------------------------------- /recipes/gromacs/gromacs.py: -------------------------------------------------------------------------------- 1 | r""" 2 | GROMACS 2020 3 | 4 | Contents: 5 | Ubuntu 16.04 6 | CUDA version 10.1 7 | GNU compilers (upstream) 8 | OFED (upstream) 9 | OpenMPI version 3.1.4 10 | """ 11 | 12 | gromacs_version = USERARG.get('gromacs', '2020') 13 | 14 | Stage0 += comment(__doc__.strip(), reformat=False) 15 | Stage0 += baseimage(image='nvcr.io/nvidia/cuda:10.1-devel-ubuntu16.04', 16 | _as='build') 17 | 18 | Stage0 += python(python3=False) 19 | 20 | Stage0 += gnu(fortran=False) 21 | 22 | Stage0 += cmake(eula=True) 23 | 24 | Stage0 += ofed() 25 | 26 | Stage0 += openmpi(version='3.1.4') 27 | 28 | Stage0 += generic_cmake(cmake_opts=['-D CMAKE_BUILD_TYPE=Release', 29 | '-D CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda', 30 | '-D GMX_BUILD_OWN_FFTW=ON', 31 | '-D GMX_GPU=ON', 32 | '-D GMX_MPI=ON', 33 | '-D GMX_OPENMP=ON', 34 | '-D GMX_PREFER_STATIC_LIBS=ON', 35 | '-D MPIEXEC_PREFLAGS=--allow-run-as-root'], 36 | prefix='/usr/local/gromacs', 37 | url='http://ftp.gromacs.org/pub/gromacs/gromacs-{}.tar.gz'.format(gromacs_version)) 38 | 39 | Stage0 += environment(variables={'PATH': '$PATH:/usr/local/gromacs/bin'}) 40 | 41 | Stage0 += label(metadata={'gromacs.version': gromacs_version}) 42 | 43 | ###### 44 | # Runtime image stage 45 | ###### 46 | Stage1 += baseimage(image='nvcr.io/nvidia/cuda:10.1-base-ubuntu16.04') 47 | 48 | Stage1 += packages(ospackages=['cuda-cufft-10-1']) 49 | 50 | Stage1 += Stage0.runtime(_from='build') 51 | 52 | Stage1 += environment(variables={'PATH': '$PATH:/usr/local/gromacs/bin'}) 53 | 54 | Stage1 += label(metadata={'gromacs.version': gromacs_version}) 55 | -------------------------------------------------------------------------------- /recipes/hpcbase-gnu-mvapich2.py: -------------------------------------------------------------------------------- 1 | """ 2 | HPC Base image 3 | 4 | Contents: 5 | CUDA version 10.1 6 | FFTW version 3.3.8 7 | GNU compilers (upstream) 8 | HDF5 version 1.10.5 9 | Mellanox OFED version 4.6-1.0.1.1 10 | MVAPICH version 2.3.2 11 | Python 2 and 3 (upstream) 12 | """ 13 | # pylint: disable=invalid-name, undefined-variable, used-before-assignment 14 | 15 | # Choose between either Ubuntu 16.04 (default) or CentOS 7 16 | # Add '--userarg centos=true' to the command line to select CentOS 17 | devel_image = 'nvcr.io/nvidia/cuda:10.1-devel-ubuntu16.04' 18 | runtime_image = 'nvcr.io/nvidia/cuda:10.1-runtime-ubuntu16.04' 19 | if USERARG.get('centos', False): 20 | devel_image = 'nvcr.io/nvidia/cuda:10.1-devel-centos7' 21 | runtime_image = 'nvcr.io/nvidia/cuda:10.1-runtime-centos7' 22 | 23 | ###### 24 | # Devel stage 25 | ###### 26 | 27 | Stage0 += comment(__doc__, reformat=False) 28 | 29 | Stage0 += baseimage(image=devel_image, _as='devel') 30 | 31 | # Python 32 | Stage0 += python() 33 | 34 | # GNU compilers 35 | compiler = gnu() 36 | Stage0 += compiler 37 | 38 | # Mellanox OFED 39 | Stage0 += mlnx_ofed(version='4.6-1.0.1.1') 40 | 41 | # MVAPICH2 42 | Stage0 += mvapich2(version='2.3.2', toolchain=compiler.toolchain) 43 | 44 | # FFTW 45 | Stage0 += fftw(version='3.3.8', mpi=True, toolchain=compiler.toolchain) 46 | 47 | # HDF5 48 | Stage0 += hdf5(version='1.10.5', toolchain=compiler.toolchain) 49 | 50 | ###### 51 | # Runtime image 52 | ###### 53 | 54 | Stage1 += baseimage(image=runtime_image) 55 | 56 | Stage1 += Stage0.runtime(_from='devel') 57 | -------------------------------------------------------------------------------- /recipes/hpcbase-gnu-openmpi.py: -------------------------------------------------------------------------------- 1 | """ 2 | HPC Base image 3 | 4 | Contents: 5 | CUDA version 10.1 6 | FFTW version 3.3.8 7 | GNU compilers (upstream) 8 | HDF5 version 1.10.5 9 | Mellanox OFED version 4.6-1.0.1.1 10 | OpenMPI version 4.0.2 11 | Python 2 and 3 (upstream) 12 | """ 13 | # pylint: disable=invalid-name, undefined-variable, used-before-assignment 14 | 15 | # Choose between either Ubuntu 16.04 (default) or CentOS 7 16 | # Add '--userarg centos=true' to the command line to select CentOS 17 | devel_image = 'nvcr.io/nvidia/cuda:10.1-devel-ubuntu16.04' 18 | runtime_image = 'nvcr.io/nvidia/cuda:10.1-runtime-ubuntu16.04' 19 | if USERARG.get('centos', False): 20 | devel_image = 'nvcr.io/nvidia/cuda:10.1-devel-centos7' 21 | runtime_image = 'nvcr.io/nvidia/cuda:10.1-runtime-centos7' 22 | 23 | ###### 24 | # Devel stage 25 | ###### 26 | 27 | Stage0 += comment(__doc__, reformat=False) 28 | 29 | Stage0 += baseimage(image=devel_image, _as='devel') 30 | 31 | # Python 32 | Stage0 += python() 33 | 34 | # GNU compilers 35 | compiler = gnu() 36 | Stage0 += compiler 37 | 38 | # Mellanox OFED 39 | Stage0 += mlnx_ofed(version='4.6-1.0.1.1') 40 | 41 | # OpenMPI 42 | Stage0 += openmpi(version='4.0.2', toolchain=compiler.toolchain) 43 | 44 | # FFTW 45 | Stage0 += fftw(version='3.3.8', mpi=True, toolchain=compiler.toolchain) 46 | 47 | # HDF5 48 | Stage0 += hdf5(version='1.10.5', toolchain=compiler.toolchain) 49 | 50 | ###### 51 | # Runtime image 52 | ###### 53 | 54 | Stage1 += baseimage(image=runtime_image) 55 | 56 | Stage1 += Stage0.runtime(_from='devel') 57 | -------------------------------------------------------------------------------- /recipes/hpcbase-nvhpc-mvapich2.py: -------------------------------------------------------------------------------- 1 | """ 2 | HPC Base image 3 | 4 | Contents: 5 | FFTW version 3.3.8 6 | HDF5 version 1.10.6 7 | Mellanox OFED version 5.0-2.1.8.0 8 | MVAPICH version 2.3.3 9 | NVIDIA HPC SDK version 20.7 10 | Python 2 and 3 (upstream) 11 | """ 12 | # pylint: disable=invalid-name, undefined-variable, used-before-assignment 13 | 14 | # The NVIDIA HPC SDK End-User License Agreement must be accepted. 15 | # https://docs.nvidia.com/hpc-sdk/eula 16 | nvhpc_eula=False 17 | if USERARG.get('nvhpc_eula_accept', False): 18 | nvhpc_eula=True 19 | else: 20 | raise RuntimeError('NVIDIA HPC SDK EULA not accepted. To accept, use "--userarg nvhpc_eula_accept=yes"\nSee NVIDIA HPC SDK EULA at https://docs.nvidia.com/hpc-sdk/eula') 21 | 22 | # Choose between either Ubuntu 18.04 (default) or RockyLinux 8 23 | # Add '--userarg rockylinux=true' to the command line to select RockyLinux 24 | image = 'ubuntu:18.04' 25 | if USERARG.get('rockylinux', False): 26 | image = 'rockylinux:8' 27 | 28 | ###### 29 | # Devel stage 30 | ###### 31 | 32 | Stage0 += comment(__doc__, reformat=False) 33 | 34 | Stage0 += baseimage(image=image, _as='devel') 35 | 36 | # Python 37 | Stage0 += python() 38 | 39 | # NVIDIA HPC SDK 40 | compiler = nvhpc(eula=nvhpc_eula, mpi=False, redist=['compilers/lib/*'], 41 | version='20.7') 42 | Stage0 += compiler 43 | 44 | # Mellanox OFED 45 | Stage0 += mlnx_ofed(version='5.0-2.1.8.0') 46 | 47 | # MVAPICH2 48 | Stage0 += mvapich2(cuda=False, version='2.3.3', toolchain=compiler.toolchain) 49 | 50 | # FFTW 51 | Stage0 += fftw(version='3.3.8', mpi=True, toolchain=compiler.toolchain) 52 | 53 | # HDF5 54 | Stage0 += hdf5(version='1.10.6', toolchain=compiler.toolchain) 55 | 56 | # nvidia-container-runtime 57 | Stage0 += environment(variables={ 58 | 'NVIDIA_VISIBLE_DEVICES': 'all', 59 | 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility', 60 | 'NVIDIA_REQUIRE_CUDA': '"cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411"'}) 61 | 62 | ###### 63 | # Runtime image 64 | ###### 65 | 66 | Stage1 += baseimage(image=image) 67 | 68 | Stage1 += Stage0.runtime(_from='devel') 69 | 70 | # nvidia-container-runtime 71 | Stage0 += environment(variables={ 72 | 'NVIDIA_VISIBLE_DEVICES': 'all', 73 | 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility', 74 | 'NVIDIA_REQUIRE_CUDA': '"cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411"'}) 75 | -------------------------------------------------------------------------------- /recipes/hpcbase-nvhpc-openmpi.py: -------------------------------------------------------------------------------- 1 | """ 2 | HPC Base image 3 | 4 | Contents: 5 | FFTW version 3.3.8 6 | HDF5 version 1.10.6 7 | Mellanox OFED version 5.0-2.1.8.0 8 | NVIDIA HPC SDK version 20.7 9 | OpenMPI version 4.0.4 10 | Python 2 and 3 (upstream) 11 | """ 12 | # pylint: disable=invalid-name, undefined-variable, used-before-assignment 13 | 14 | # The NVIDIA HPC SDK End-User License Agreement must be accepted. 15 | # https://docs.nvidia.com/hpc-sdk/eula 16 | nvhpc_eula=False 17 | if USERARG.get('nvhpc_eula_accept', False): 18 | nvhpc_eula=True 19 | else: 20 | raise RuntimeError('NVIDIA HPC SDK EULA not accepted. To accept, use "--userarg nvhpc_eula_accept=yes"\nSee NVIDIA HPC SDK EULA at https://docs.nvidia.com/hpc-sdk/eula') 21 | 22 | # Choose between either Ubuntu 18.04 (default) or RockyLinux 8 23 | # Add '--userarg rockylinux=true' to the command line to select RockyLinux 24 | image = 'ubuntu:18.04' 25 | if USERARG.get('rockylinux', False): 26 | image = 'rockylinux:8' 27 | 28 | ###### 29 | # Devel stage 30 | ###### 31 | 32 | Stage0 += comment(__doc__, reformat=False) 33 | 34 | Stage0 += baseimage(image=image, _as='devel') 35 | 36 | # Python 37 | Stage0 += python() 38 | 39 | # NVIDIA HPC SDK 40 | compiler = nvhpc(eula=nvhpc_eula, mpi=False, redist=['compilers/lib/*'], 41 | version='20.7') 42 | compiler.toolchain.CUDA_HOME = '/opt/nvidia/hpc_sdk/Linux_x86_64/20.7/cuda' 43 | Stage0 += compiler 44 | 45 | # Mellanox OFED 46 | Stage0 += mlnx_ofed(version='5.0-2.1.8.0') 47 | 48 | # OpenMPI 49 | Stage0 += openmpi(version='4.0.4', toolchain=compiler.toolchain) 50 | 51 | # FFTW 52 | Stage0 += fftw(version='3.3.8', mpi=True, toolchain=compiler.toolchain) 53 | 54 | # HDF5 55 | Stage0 += hdf5(version='1.10.6', toolchain=compiler.toolchain) 56 | 57 | # nvidia-container-runtime 58 | Stage0 += environment(variables={ 59 | 'NVIDIA_VISIBLE_DEVICES': 'all', 60 | 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility', 61 | 'NVIDIA_REQUIRE_CUDA': '"cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411"'}) 62 | 63 | ###### 64 | # Runtime image 65 | ###### 66 | 67 | Stage1 += baseimage(image=image) 68 | 69 | Stage1 += Stage0.runtime(_from='devel') 70 | 71 | # nvidia-container-runtime 72 | Stage0 += environment(variables={ 73 | 'NVIDIA_VISIBLE_DEVICES': 'all', 74 | 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility', 75 | 'NVIDIA_REQUIRE_CUDA': '"cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411"'}) 76 | -------------------------------------------------------------------------------- /recipes/hpccm/Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Recipe to bootstrap HPC Container Maker (HPCCM) as a container 3 | # 4 | # Docker: 5 | # $ sudo docker build -t hpccm -f Dockerfile . 6 | # $ sudo docker run --rm -v $(pwd):/recipes hpccm --recipe /recipes/... 7 | # 8 | # Singularity: 9 | # $ sudo singularity build hpccm.simg Singularity.def 10 | # $ singularity exec hpccm.simg --recipe ... 11 | # 12 | 13 | FROM python:3-slim 14 | 15 | RUN pip install --no-cache-dir hpccm 16 | 17 | ENTRYPOINT ["hpccm"] 18 | 19 | 20 | -------------------------------------------------------------------------------- /recipes/hpccm/README.md: -------------------------------------------------------------------------------- 1 | # Running HPC Container Maker from a Container 2 | 3 | Please refer to the [Getting Started](/docs/getting_started.md) guide 4 | for more information on installing HPC Container Maker. 5 | 6 | If installing HPCCM on the host is not an option, then the 7 | files in this directory may be used to build a HPCCM container image. 8 | The HPCCM recipe used to generate the Dockerfile and Singularity 9 | definition file is also included. 10 | 11 | ## Docker 12 | 13 | Use the provided Dockerfile to build a HPCCM container image. 14 | 15 | ``` 16 | $ sudo docker build -t hpccm -f Dockerfile . 17 | $ sudo docker run -rm -v $(pwd):/recipes hpccm --recipe /recipes/... 18 | ``` 19 | 20 | ## Singularity 21 | 22 | Use the provided Singularity definition file to build a HPCCM container 23 | image. 24 | 25 | ``` 26 | $ sudo singularity build hpccm.simg Singularity.def 27 | $ ./hpccm.simg --recipe ... 28 | ``` 29 | -------------------------------------------------------------------------------- /recipes/hpccm/Singularity.def: -------------------------------------------------------------------------------- 1 | # 2 | # Recipe to bootstrap HPC Container Maker (HPCCM) as a container 3 | # 4 | # Docker: 5 | # $ sudo docker build -t hpccm -f Dockerfile . 6 | # $ sudo docker run --rm -v $(pwd):/recipes hpccm --recipe /recipes/... 7 | # 8 | # Singularity: 9 | # $ sudo singularity build hpccm.simg Singularity.def 10 | # $ ./hpccm.simg --recipe ... 11 | # 12 | 13 | BootStrap: docker 14 | From: python:3-slim 15 | 16 | %post 17 | pip install --no-cache-dir hpccm 18 | 19 | %runscript 20 | exec hpccm $@ 21 | 22 | 23 | -------------------------------------------------------------------------------- /recipes/hpccm/bootstrap.py: -------------------------------------------------------------------------------- 1 | """ 2 | Recipe for a HPC Container Maker (HPCCM) container image 3 | 4 | Docker: 5 | $ sudo docker build -t hpccm -f Dockerfile . 6 | $ sudo docker run --rm -v $(pwd):/recipes hpccm --recipe /recipes/... 7 | 8 | Singularity: 9 | $ sudo singularity build hpccm.simg Singularity.def 10 | $ ./hpccm.simg --recipe ... 11 | """ 12 | from hpccm.common import container_type 13 | 14 | Stage0 += comment(__doc__, reformat=False) 15 | 16 | Stage0 += baseimage(image='python:3-slim', _distro='ubuntu', _docker_env=False) 17 | 18 | Stage0 += shell(commands=['pip install --no-cache-dir hpccm'], chdir=False) 19 | 20 | if hpccm.config.g_ctype == container_type.DOCKER: 21 | # Docker automatically passes through command line arguments 22 | Stage0 += runscript(commands=['hpccm']) 23 | elif hpccm.config.g_ctype == container_type.SINGULARITY: 24 | # Singularity does not automatically pass through command line arguments 25 | Stage0 += runscript(commands=['hpccm $@']) 26 | -------------------------------------------------------------------------------- /recipes/jupyter/.gitignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | Singularity.def 3 | -------------------------------------------------------------------------------- /recipes/jupyter/README.md: -------------------------------------------------------------------------------- 1 | # Containerizing Jupyter Notebooks 2 | 3 | This example demonstrates how to generate a container for a Jupyter 4 | notebook. 5 | 6 | The `jupyter.py` script outputs a container specification file that 7 | can be built into a container image. Use `--help` to list all the 8 | script command line options. 9 | 10 | The example includes a very simple Jupyter notebook, Conda environment 11 | (`environment.yml`), and `requirements.txt` for illustrative purposes. 12 | 13 | ## Docker 14 | 15 | Using pip: 16 | 17 | ``` 18 | $ jupyter.py --notebook notebook.ipynb --requirements requirements.txt > Dockerfile 19 | ``` 20 | 21 | Using Anaconda: 22 | 23 | ``` 24 | $ jupyter.py --packager anaconda --notebook notebook.ipynb --environment environment.yml > Dockerfile 25 | ``` 26 | 27 | Once the Dockerfile has been generated, the steps to build and run 28 | the container are the same. 29 | 30 | ``` 31 | $ sudo docker build -t jupyter:example -f Dockerfile . 32 | $ sudo docker run --rm -p 8888:8888 jupyter:example 33 | ``` 34 | 35 | ## Singularity 36 | 37 | Using pip: 38 | 39 | ``` 40 | $ jupyter.py --notebook notebook.ipynb --requirements requirements.txt --format singularity > Singularity.def 41 | ``` 42 | 43 | Using Anaconda: 44 | ``` 45 | $ jupyter.py --packager anaconda --notebook notebook.ipynb --environment.yml --format singularity > Singularity.def 46 | ``` 47 | 48 | Once the Singularity definition file has been generated the steps to 49 | build and run the container are the same. 50 | 51 | ``` 52 | $ sudo singularity build jupyter-example.sif Singularity.def 53 | $ singularity run jupyter-example.sif 54 | ``` 55 | -------------------------------------------------------------------------------- /recipes/jupyter/environment.yml: -------------------------------------------------------------------------------- 1 | channels: 2 | - conda-forge 3 | dependencies: 4 | - conda-forge::hpccm 5 | -------------------------------------------------------------------------------- /recipes/jupyter/jupyter.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import absolute_import 4 | from __future__ import unicode_literals 5 | from __future__ import print_function 6 | 7 | import argparse 8 | import hpccm 9 | import yaml 10 | 11 | if __name__ == '__main__': 12 | parser = argparse.ArgumentParser( 13 | description='Jupyter notebook container generator') 14 | parser.add_argument('--environment', type=str, 15 | help='Conda environment file') 16 | parser.add_argument('--format', type=str, default='docker', 17 | choices=['docker', 'singularity'], 18 | help='Container specification format (default: docker)') 19 | parser.add_argument('--image', type=str, default='ubuntu:18.04', 20 | help='Base container image (default: ubuntu:18.04)') 21 | parser.add_argument('--notebook', required=True, type=str, 22 | help='Jupyter notebook file') 23 | parser.add_argument('--packager', type=str, default='pip', 24 | choices=['anaconda', 'pip'], 25 | help='Python package manager (default: pip)') 26 | parser.add_argument('--requirements', type=str, 27 | help='pip requirements file') 28 | args = parser.parse_args() 29 | 30 | ### Create Stage 31 | stage = hpccm.Stage() 32 | 33 | ### Base image 34 | stage += hpccm.primitives.baseimage(image=args.image, _docker_env=False) 35 | 36 | ### Install Python and Jupyter (and requirements / environment) 37 | if args.packager == 'pip': 38 | stage += hpccm.building_blocks.python(python2=False) 39 | stage += hpccm.building_blocks.pip(packages=['ipython', 'jupyter'], 40 | pip='pip3', 41 | requirements=args.requirements) 42 | elif args.packager == 'anaconda': 43 | stage += hpccm.building_blocks.conda(environment=args.environment, 44 | eula=True, 45 | packages=['ipython', 'jupyter']) 46 | 47 | ### Make the port accessible (Docker only) 48 | stage += hpccm.primitives.raw(docker='EXPOSE 8888') 49 | 50 | ### Add the notebook itself 51 | stage += hpccm.primitives.copy(src=args.notebook, dest='/notebook/', 52 | _mkdir=True) 53 | 54 | ### Run Jupyter 55 | if args.packager == 'pip': 56 | stage += hpccm.primitives.runscript( 57 | commands=['jupyter notebook --no-browser --ip 0.0.0.0 --notebook-dir /notebook --allow-root']) 58 | elif args.packager == 'anaconda': 59 | env = 'base' 60 | if args.environment: 61 | with open(args.environment) as e: 62 | y = yaml.safe_load(e) 63 | env = y.get('name', 'base') 64 | 65 | stage += hpccm.primitives.shell(commands=[ 66 | 'echo "#!/bin/bash\\nsource /usr/local/anaconda/bin/activate {}\\njupyter notebook --ip 0.0.0.0 --no-browser --notebook-dir /notebook --allow-root" > /usr/local/bin/entrypoint.sh'.format(env), 67 | 'chmod a+x /usr/local/bin/entrypoint.sh']) 68 | stage += hpccm.primitives.runscript( 69 | commands=['/usr/local/bin/entrypoint.sh']) 70 | 71 | ### Set container specification output format 72 | hpccm.config.set_container_format(args.format) 73 | 74 | ### Output container specification 75 | print(stage) 76 | 77 | -------------------------------------------------------------------------------- /recipes/jupyter/notebook.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Example Jupyter notebook container\n", 8 | "\n", 9 | "Hello World!" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": null, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "!hpccm --version" 19 | ] 20 | } 21 | ], 22 | "metadata": { 23 | "kernelspec": { 24 | "display_name": "Python 3", 25 | "language": "python", 26 | "name": "python3" 27 | }, 28 | "language_info": { 29 | "codemirror_mode": { 30 | "name": "ipython", 31 | "version": 3 32 | }, 33 | "file_extension": ".py", 34 | "mimetype": "text/x-python", 35 | "name": "python", 36 | "nbconvert_exporter": "python", 37 | "pygments_lexer": "ipython3", 38 | "version": "3.6.7" 39 | } 40 | }, 41 | "nbformat": 4, 42 | "nbformat_minor": 2 43 | } 44 | -------------------------------------------------------------------------------- /recipes/jupyter/requirements.txt: -------------------------------------------------------------------------------- 1 | hpccm 2 | -------------------------------------------------------------------------------- /recipes/milc/milc.py: -------------------------------------------------------------------------------- 1 | """ 2 | MILC 7.8.1 3 | 4 | Contents: 5 | Ubuntu 16.04 6 | CUDA version 11.2 7 | GNU compilers (upstream) 8 | OFED (upstream) 9 | OpenMPI version 3.1.4 10 | QUDA version 0.8.0 11 | """ 12 | # pylint: disable=invalid-name, undefined-variable, used-before-assignment 13 | # pylama: ignore=E0602 14 | 15 | gpu_arch = USERARG.get('GPU_ARCH', 'sm_60') 16 | 17 | # add docstring to Dockerfile 18 | Stage0 += comment(__doc__.strip(), reformat=False) 19 | 20 | ############################################################################### 21 | # Devel stage 22 | ############################################################################### 23 | Stage0 += baseimage(image='nvcr.io/nvidia/cuda:11.2.0-devel-ubuntu18.04', 24 | _as='devel') 25 | 26 | Stage0 += gnu() 27 | Stage0 += cmake(eula=True) 28 | Stage0 += ofed() 29 | Stage0 += openmpi(version='3.1.4') 30 | 31 | # build QUDA 32 | Stage0 += packages(ospackages=['ca-certificates', 'git']) 33 | Stage0 += generic_cmake(branch='develop', 34 | cmake_opts=['-D CMAKE_BUILD_TYPE=RELEASE', 35 | '-D QUDA_DIRAC_CLOVER=ON', 36 | '-D QUDA_DIRAC_DOMAIN_WALL=ON', 37 | '-D QUDA_DIRAC_STAGGERED=ON', 38 | '-D QUDA_DIRAC_TWISTED_CLOVER=ON', 39 | '-D QUDA_DIRAC_TWISTED_MASS=ON', 40 | '-D QUDA_DIRAC_WILSON=ON', 41 | '-D QUDA_FORCE_GAUGE=ON', 42 | '-D QUDA_FORCE_HISQ=ON', 43 | '-D QUDA_GPU_ARCH={}'.format(gpu_arch), 44 | '-D QUDA_INTERFACE_MILC=ON', 45 | '-D QUDA_INTERFACE_QDP=ON', 46 | '-D QUDA_LINK_HISQ=ON', 47 | '-D QUDA_MPI=ON'], 48 | install=False, 49 | ldconfig=True, 50 | postinstall=['cp -a /var/tmp/quda/build/* /usr/local/quda'], 51 | preconfigure=['mkdir -p /usr/local/quda'], 52 | prefix='/usr/local/quda', 53 | repository='https://github.com/lattice/quda.git') 54 | 55 | # build MILC 56 | Stage0 += generic_build(branch='develop', 57 | build=['cp Makefile ks_imp_rhmc', 58 | 'cd ks_imp_rhmc', 59 | 'make -j 1 su3_rhmd_hisq \ 60 | CC=/usr/local/openmpi/bin/mpicc \ 61 | LD=/usr/local/openmpi/bin/mpicxx \ 62 | QUDA_HOME=/usr/local/quda \ 63 | WANTQUDA=true \ 64 | WANT_GPU=true \ 65 | WANT_CL_BCG_GPU=true \ 66 | WANT_FN_CG_GPU=true \ 67 | WANT_FL_GPU=true \ 68 | WANT_FF_GPU=true \ 69 | WANT_GF_GPU=true \ 70 | MPP=true \ 71 | PRECISION=2 \ 72 | WANTQIO=""'], 73 | install=['mkdir -p /usr/local/milc/bin', 74 | 'cp /var/tmp/milc_qcd/ks_imp_rhmc/su3_rhmd_hisq /usr/local/milc/bin'], 75 | prefix='/usr/local/milc', 76 | repository='https://github.com/milc-qcd/milc_qcd') 77 | Stage0 += environment(variables={'PATH': '/usr/local/milc/bin:$PATH'}) 78 | 79 | ############################################################################### 80 | # Release stage 81 | ############################################################################### 82 | Stage1 += baseimage(image='nvcr.io/nvidia/cuda:11.2.0-base-ubuntu18.04') 83 | 84 | Stage1 += packages(ospackages=['libcublas-11-2']) 85 | 86 | Stage1 += Stage0.runtime() 87 | 88 | Stage1 += environment(variables={'PATH': '/usr/local/milc/bin:$PATH'}) 89 | -------------------------------------------------------------------------------- /recipes/mpi_bandwidth.py: -------------------------------------------------------------------------------- 1 | """ 2 | MPI Bandwidth 3 | 4 | Contents: 5 | CentOS 7 6 | GNU compilers (upstream) 7 | Mellanox OFED 8 | OpenMPI 9 | PMI2 (SLURM) 10 | UCX 11 | 12 | Building: 13 | 1. Docker to Singularity 14 | $ hpccm --recipe mpi_bandwidth.py > Dockerfile 15 | $ sudo docker build -t mpi_bw -f Dockerfile . 16 | $ singularity build mpi_bw.sif docker-daemon://mpi_bw:latest 17 | 18 | 2. Singularity 19 | $ hpccm --recipe mpi_bandwidth.py --format singularity --singularity-version=3.2 > Singularity.def 20 | $ sudo singularity build mpi_bw.sif Singularity.def 21 | 22 | Running with Singularity: 23 | 1. Using a compatible host MPI runtime 24 | $ mpirun -n 2 singularity run mpi_bw.sif mpi_bandwidth 25 | 26 | 2. Using the MPI runtime inside the container 27 | $ singularity run mpi_bw.sif mpirun -n 2 -H node1:1,node2:1 --launch-agent "singularity exec \$SINGULARITY_CONTAINER orted" mpi_bandwidth 28 | 29 | 3. Using SLURM srun 30 | $ srun -n 2 --mpi=pmi2 singularity run mpi_bw.sif mpi_bandwidth 31 | """ 32 | 33 | Stage0 += comment(__doc__, reformat=False) 34 | 35 | # CentOS base image 36 | Stage0 += baseimage(image='centos:7', _as='build') 37 | 38 | # GNU compilers 39 | Stage0 += gnu(fortran=False) 40 | 41 | # Mellanox OFED 42 | Stage0 += mlnx_ofed() 43 | 44 | # UCX 45 | Stage0 += ucx(cuda=False) 46 | 47 | # PMI2 48 | Stage0 += slurm_pmi2() 49 | 50 | # OpenMPI (use UCX instead of IB directly) 51 | Stage0 += openmpi(cuda=False, infiniband=False, pmi='/usr/local/slurm-pmi2', 52 | ucx='/usr/local/ucx') 53 | 54 | # MPI Bandwidth 55 | Stage0 += shell(commands=[ 56 | 'wget -q -nc --no-check-certificate -P /var/tmp https://hpc-tutorials.llnl.gov/mpi/examples/mpi_bandwidth.c', 57 | 'mpicc -o /usr/local/bin/mpi_bandwidth /var/tmp/mpi_bandwidth.c']) 58 | 59 | ### Runtime distributable stage 60 | Stage1 += baseimage(image='centos:7') 61 | Stage1 += Stage0.runtime() 62 | Stage1 += copy(_from='build', src='/usr/local/bin/mpi_bandwidth', 63 | dest='/usr/local/bin/mpi_bandwidth') 64 | -------------------------------------------------------------------------------- /recipes/nvhpc.py: -------------------------------------------------------------------------------- 1 | """ 2 | Recipe for a NVIDIA HPC SDK container and typical development environment. 3 | 4 | $ hpccm --recipe nvhpc.py --userarg eula=yes 5 | 6 | $ hpccm --recipe nvhpc.py --userarg eula=yes cuda_multi=no arch=x86_64 7 | """ 8 | from packaging.version import Version 9 | import platform 10 | 11 | # Verify correct version of HPCCM is used 12 | if Version(hpccm.__version__) < Version('21.7.0'): 13 | raise RuntimeError('requires HPCCM version 21.7.0 or later') 14 | 15 | arch = USERARG.get('arch', platform.machine()) 16 | image = USERARG.get('image', 'ubuntu:20.04') 17 | version = USERARG.get('version', '21.7') 18 | release = '20' + version[:2] 19 | default_cuda = '11.4' 20 | 21 | cuda_multi=True 22 | if USERARG.get('cuda_multi', False) in [0, '0', 'no', 'false']: 23 | cuda_multi=False 24 | 25 | # The NVIDIA HPC SDK End-User License Agreement must be accepted. 26 | # https://docs.nvidia.com/hpc-sdk/eula 27 | eula=False 28 | if USERARG.get('eula', False) in [1, '1', 'yes', 'true']: 29 | eula=True 30 | else: 31 | raise RuntimeError('NVIDIA HPC SDK EULA not accepted. To accept, use "--userarg eula=yes"\nSee NVIDIA HPC SDK EULA at https://docs.nvidia.com/hpc-sdk/eula') 32 | 33 | if cuda_multi is True: 34 | # CUDA 10.2, 11.0, 11.4 35 | cuda_driver = '"cuda>=10.2"' 36 | else: 37 | # CUDA 11.4 only, but enhanced forward compatibility means 11.2 is 38 | # sufficient 39 | cuda_driver = '"cuda>=11.2"' 40 | 41 | Stage0 += baseimage(image=image, _arch=arch) 42 | 43 | # Typical development environment 44 | common_packages = ['automake', 'autoconf', 'autoconf-archive', 'binutils', 45 | 'bzip2', 'ca-certificates', 'cmake', 'diffutils', 'file', 46 | 'gdb', 'git', 'gzip', 'libtool', 'make', 'patch', 'tar', 47 | 'vim', 'wget'] 48 | Stage0 += packages(apt=common_packages + ['libaec-dev', 'libncursesw5', 49 | 'libsz2', 'libtinfo5', 'lmod', 50 | 'xz-utils', 'zlib1g-dev'], 51 | epel=True, 52 | yum=common_packages + ['Lmod', 'libaec-devel', 'xz', 53 | 'zlib-devel']) 54 | 55 | # Mellanox OFED 56 | Stage0 += mlnx_ofed(version='5.2-2.2.0.0') 57 | 58 | # NVIDIA HPC SDK 59 | Stage0 += nvhpc(cuda=default_cuda, cuda_multi=cuda_multi, eula=eula, 60 | _hpcx=True, mpi=False, version=version) 61 | 62 | # Container metadata 63 | Stage0 += environment(variables={'HPCSDK_VERSION': version, 64 | 'HPCSDK_RELEASE': release}) 65 | Stage0 += label( 66 | metadata={'com.nvidia.hpcsdk.version': '"{}"'.format(version), 67 | 'com.nvidia.hpcsdk.release': '"{}"'.format(release)}) 68 | 69 | # nvidia-container-runtime and enroot 70 | Stage0 += environment(variables={ 71 | 'MELLANOX_VISIBLE_DEVICES': 'all', # enroot 72 | 'NVIDIA_VISIBLE_DEVICES': 'all', 73 | 'NVIDIA_DRIVER_CAPABILITIES': 'compute,utility', 74 | 'NVIDIA_REQUIRE_CUDA': cuda_driver}) 75 | -------------------------------------------------------------------------------- /recipes/osu_benchmarks/README.md: -------------------------------------------------------------------------------- 1 | # OSU Micro-Benchmarks 2 | 3 | This example demonstrates how to build a container that is portable 4 | with respect to the host OFED version. PMI is also included to 5 | simplify running the container on multiple nodes. 6 | 7 | The `common.py` recipe builds a generic software environment that 8 | includes multiple versions of OFED and UCX. The entrypoint script 9 | determines the best match to the host at runtime and configures the 10 | appropriate software environment. 11 | 12 | The `osu_benchmarks.py` recipe builds the [OSU 13 | Micro-Benchmarks](http://mvapich.cse.ohio-state.edu/benchmarks/) on 14 | top of this environment. 15 | 16 | ## Build 17 | 18 | ``` 19 | $ hpccm --recipe osu_benchmarks.py > Dockerfile 20 | $ sudo docker build -t osu_benchmarks -f Dockerfile . 21 | $ singularity build osu_benchmarks.sif docker-daemon://osu_benchmarks:latest 22 | ``` 23 | 24 | ## Run 25 | 26 | ``` 27 | $ srun -N 2 -n 2 --mpi=pmix singularity run --nv osu_benchmarks.sif get_local_rank osu_bw 28 | ``` 29 | 30 | On some systems `--mpi=pmi2` may be more appropriate. 31 | 32 | Note: Setting the 33 | [`UCX_TLS`](https://github.com/openucx/ucx/wiki/UCX-environment-parameters) 34 | environment variable may be necessary in some cases. 35 | -------------------------------------------------------------------------------- /recipes/osu_benchmarks/common.py: -------------------------------------------------------------------------------- 1 | # Generic recipe to build a OFED+UCX+MPI+CUDA container environment 2 | # that supports both OFED 4.x and 5.x. 3 | 4 | # Development stage 5 | Stage0 += baseimage(image='nvcr.io/nvidia/cuda:11.0-devel-ubuntu18.04', 6 | _as='devel') 7 | 8 | # Compiler 9 | Stage0 += gnu() 10 | 11 | # Communication libraries 12 | Stage0 += gdrcopy(ldconfig=True) 13 | Stage0 += knem(ldconfig=True) 14 | 15 | # Mellanox legacy OFED support 16 | mlnx_versions=['4.2-1.5.1.0', '4.3-1.0.1.0', '4.4-1.0.0.0', '4.5-1.0.1.0', 17 | '4.6-1.0.1.1', '4.7-3.2.9.0', '4.9-0.1.7.0'] 18 | Stage0 += multi_ofed(inbox=False, mlnx_versions=mlnx_versions, 19 | prefix="/usr/local/ofed", symlink=False) 20 | 21 | # RDMA-core based OFED support 22 | Stage0 += mlnx_ofed(version="5.2-2.2.0.0", symlink=False) 23 | 24 | # UCX default - RDMA-core based OFED 25 | Stage0 += ucx(version='1.10.0', cuda=True, 26 | gdrcopy='/usr/local/gdrcopy', knem='/usr/local/knem', 27 | disable_static=True, enable_mt=True) 28 | 29 | # UCX - Mellanox legacy support 30 | Stage0 += ucx(version='1.10.0', 31 | build_environment={ 32 | "LD_LIBRARY_PATH": "/usr/local/ofed/4.6-1.0.1.1/lib:${LD_LIBRARY_PATH}"}, 33 | cuda=True, environment=False, gdrcopy='/usr/local/gdrcopy', 34 | knem='/usr/local/knem', prefix='/usr/local/ucx-mlnx-legacy', 35 | disable_static=True, enable_mt=True, 36 | with_verbs='/usr/local/ofed/4.6-1.0.1.1/usr', 37 | with_rdmacm='/usr/local/ofed/4.6-1.0.1.1/usr') 38 | 39 | # Symlink legacy UCX into legacy OFED versions 40 | Stage0 += shell(commands=[ 41 | 'ln -s /usr/local/ucx-mlnx-legacy/{1}/* /usr/local/ofed/{0}/usr/{1}'.format(version, directory) for version in mlnx_versions for directory in ['bin', 'lib']]) 42 | 43 | # PMI2 support 44 | Stage0 += slurm_pmi2(prefix="/usr/local/pmi", version='20.11.9') 45 | 46 | # OpenMPI 47 | Stage0 += openmpi(cuda=True, infiniband=False, ldconfig=True, ucx=True, 48 | version='4.0.5', 49 | disable_oshmem=True, disable_static=True, 50 | enable_mca_no_build='btl-uct', with_slurm=False, 51 | with_pmi='/usr/local/pmi') 52 | 53 | # Deployment stage 54 | Stage1 += baseimage(image='nvcr.io/nvidia/cuda:11.0-base-ubuntu18.04') 55 | Stage1 += Stage0.runtime(_from='devel') 56 | 57 | # Allow running MPI as root 58 | Stage1 += environment(variables={'OMPI_ALLOW_RUN_AS_ROOT': '1', 59 | 'OMPI_ALLOW_RUN_AS_ROOT_CONFIRM': '1'}) 60 | 61 | # Entrypoint 62 | Stage1 += copy(src='entrypoint.sh', dest='/usr/local/bin/entrypoint.sh') 63 | Stage1 += runscript(commands=['/usr/local/bin/entrypoint.sh']) 64 | 65 | # Performance and compatibility tuning 66 | Stage1 += environment(variables={'CUDA_CACHE_DISABLE': '1', 67 | 'MELLANOX_VISIBLE_DEVICES': 'all', # enroot 68 | 'OMPI_MCA_pml': 'ucx'}) 69 | -------------------------------------------------------------------------------- /recipes/osu_benchmarks/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | if [[ "$(find /usr /.singularity.d -name libcuda.so.1 2>/dev/null) " == " " || "$(ls /dev/nvidiactl 2>/dev/null) " == " " ]]; then 5 | echo 6 | echo "WARNING: The NVIDIA Driver was not detected. GPU functionality will not be available." 7 | if [[ -d /.singularity.d ]]; then 8 | echo "Use 'singularity run --nv' to start this container; see" 9 | echo "https://sylabs.io/guides/3.5/user-guide/gpu.html" 10 | else 11 | echo "Use 'docker run --gpus all' to start this container; see" 12 | echo "https://github.com/NVIDIA/nvidia-docker/wiki/Installation-(Native-GPU-Support)" 13 | fi 14 | echo 15 | fi 16 | 17 | if [ ! -d /dev/infiniband ]; then 18 | echo "WARNING: No InfiniBand devices detected." 19 | echo " Multi-node communication performance may be reduced." 20 | echo 21 | fi 22 | 23 | DETECTED_MOFED=$(cat /sys/module/mlx5_core/version 2>/dev/null || true) 24 | if [ -n "${DETECTED_MOFED}" ]; then 25 | if [[ ${DETECTED_MOFED} =~ ^5 ]]; then 26 | # do nothing, use default config 27 | echo "Detected MOFED ${DETECTED_MOFED}." 28 | elif [[ ${DETECTED_MOFED} =~ ^4 ]]; then 29 | case "${DETECTED_MOFED}" in 30 | 4.2*) 31 | UPDATE_MOFED="4.2-1.5.1.0" 32 | ;; 33 | 4.3*) 34 | UPDATE_MOFED="4.3-1.0.1.0" 35 | ;; 36 | 4.4*) 37 | UPDATE_MOFED="4.4-1.0.0.0" 38 | ;; 39 | 4.5*) 40 | UPDATE_MOFED="4.5-1.0.1.0" 41 | ;; 42 | 4.6*) 43 | UPDATE_MOFED="4.6-1.0.1.1" 44 | ;; 45 | 4.7*) 46 | UPDATE_MOFED="4.7-3.2.9.0" 47 | ;; 48 | 4.9*) 49 | UPDATE_MOFED="4.9-0.1.7.0" 50 | ;; 51 | *) 52 | echo "ERROR: Detected MOFED driver ${DETECTED_MOFED}, but this container does not support it." 53 | echo " Use of RDMA for multi-node communication will be unreliable." 54 | sleep 2 55 | ;; 56 | esac 57 | 58 | if [ -n "${UPDATE_MOFED}" ]; then 59 | echo "NOTE: Detected MOFED driver ${DETECTED_MOFED}; version automatically updated." 60 | export PATH=/usr/local/ofed/${UPDATE_MOFED}/usr/bin:$PATH 61 | export LD_LIBRARY_PATH=/usr/local/ofed/${UPDATE_MOFED}/usr/lib:/usr/local/ofed/${UPDATE_MOFED}/usr/lib/libibverbs:$LD_LIBRARY_PATH 62 | fi 63 | else 64 | echo "ERROR: Detected MOFED driver ${DETECTED_MOFED}, but this container does not support it." 65 | echo " Use of RDMA for multi-node communication will be unreliable." 66 | sleep 2 67 | fi 68 | else 69 | echo "NOTE: MOFED driver for multi-node communication was not detected." 70 | echo " Multi-node communication performance may be reduced." 71 | fi 72 | 73 | DETECTED_NVPEERMEM=$(cat /sys/module/nv_peer_mem/version 2>/dev/null || true) 74 | if [[ "${DETECTED_MOFED} " != " " && "${DETECTED_NVPEERMEM} " == " " ]]; then 75 | echo 76 | echo "NOTE: MOFED driver was detected, but nv_peer_mem driver was not detected." 77 | echo " Multi-node communication performance may be reduced." 78 | fi 79 | 80 | # set PMIx client configuration to match the server 81 | # enroot already handles this, so only do this under singularity 82 | # https://github.com/NVIDIA/enroot/blob/master/conf/hooks/extra/50-slurm-pmi.sh 83 | if [[ -z "${SLURM_MPI_TYPE-}" || "${SLURM_MPI_TYPE}" == pmix* ]]; then 84 | if [ -d /.singularity.d ]; then 85 | echo "Configuring PMIX" 86 | 87 | if [ -n "${PMIX_PTL_MODULE-}" ] && [ -z "${PMIX_MCA_ptl-}" ]; then 88 | export PMIX_MCA_ptl=${PMIX_PTL_MODULE} 89 | fi 90 | if [ -n "${PMIX_SECURITY_MODE-}" ] && [ -z "${PMIX_MCA_psec-}" ]; then 91 | export PMIX_MCA_psec=${PMIX_SECURITY_MODE} 92 | fi 93 | if [ -n "${PMIX_GDS_MODULE-}" ] && [ -z "${PMIX_MCA_gds-}" ]; then 94 | export PMIX_MCA_gds=${PMIX_GDS_MODULE} 95 | fi 96 | fi 97 | fi 98 | 99 | echo 100 | 101 | if [[ $# -eq 0 ]]; then 102 | exec "/bin/bash" 103 | else 104 | exec "$@" 105 | fi 106 | -------------------------------------------------------------------------------- /recipes/osu_benchmarks/osu_benchmarks.py: -------------------------------------------------------------------------------- 1 | # Use the generic OFED+UCX+OpenMPI recipe 2 | hpccm.include('common.py') 3 | 4 | # Build the OSU Micro-Benchmarks in the development stage 5 | Stage0 += generic_autotools( 6 | build_environment={'CC': 'mpicc', 'CXX': 'mpicxx'}, 7 | enable_cuda=True, 8 | prefix='/usr/local/osu', 9 | url='http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.7.tar.gz', 10 | with_cuda='/usr/local/cuda') 11 | 12 | # Copy the OSU Micro-Benchmark binaries into the deployment stage 13 | Stage1 += copy(_from='devel', src='/usr/local/osu', dest='/usr/local/osu') 14 | 15 | # Add the OSU Micro-Benchmarks to the default PATH 16 | base_path = '/usr/local/osu/libexec/osu-micro-benchmarks' 17 | Stage1 += environment(variables={'PATH': '{0}:{0}/mpi/collective:{0}/mpi/one-sided:{0}/mpi/pt2pt:{0}/mpi/startup:$PATH'.format(base_path)}) 18 | -------------------------------------------------------------------------------- /recipes/spack.py: -------------------------------------------------------------------------------- 1 | """Spack container (https://github.com/spack/spack) 2 | Set the user argument 'package' to specify the Spack package to 3 | install. Otherwise, it will just build a base Spack container 4 | image. 5 | 6 | Sample workflow: 7 | $ hpccm.py --recipe recipes/spack.py --userarg package="gromacs@2018.2 +cuda" > Dockerfile.gromacs.spack 8 | $ docker build -t gromacs.spack -f Dockerfile.gromacs.spack . 9 | $ nvidia-docker run --rm -ti gromacs.spack bash -l 10 | container:/> spack load gromacs 11 | """ 12 | # pylint: disable=invalid-name, undefined-variable, used-before-assignment 13 | from hpccm.templates.git import git 14 | 15 | spack_branch = 'develop' 16 | 17 | Stage0 += comment(__doc__, reformat=False) 18 | 19 | Stage0 += baseimage(image='ubuntu:16.04') 20 | 21 | # Base dependencies 22 | Stage0 += python() 23 | Stage0 += gnu() 24 | 25 | # Additional dependencies 26 | ospackages = ['autoconf', 'build-essential', 'bzip2', 'ca-certificates', 27 | 'coreutils', 'curl', 'environment-modules', 'git', 'gzip', 28 | 'libssl-dev', 'make', 'openssh-client', 'patch', 'pkg-config', 29 | 'tcl', 'tar', 'unzip', 'zlib1g'] 30 | Stage0 += apt_get(ospackages=ospackages) 31 | 32 | # Setup and install Spack 33 | Stage0 += shell(commands=[ 34 | git().clone_step(repository='https://github.com/spack/spack', 35 | branch=spack_branch, path='/opt'), 36 | 'ln -s /opt/spack/share/spack/setup-env.sh /etc/profile.d/spack.sh', 37 | 'ln -s /opt/spack/share/spack/spack-completion.bash /etc/profile.d']) 38 | Stage0 += environment(variables={'PATH': '/opt/spack/bin:$PATH', 39 | 'FORCE_UNSAFE_CONFIGURE': '1'}) 40 | 41 | spack_package = USERARG.get('package', None) 42 | if spack_package: 43 | Stage0 += shell(commands=['spack install {}'.format(spack_package), 44 | 'spack clean --all']) 45 | -------------------------------------------------------------------------------- /runtests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | COVERAGE=false 4 | COVERAGE_TOOL=coverage 5 | PYTHON=python 6 | TEST="" 7 | 8 | while [ "$1" != "" ]; do 9 | case $1 in 10 | --coverage ) 11 | COVERAGE=true 12 | ;; 13 | --python2 ) 14 | COVERAGE_TOOL=coverage2 15 | PYTHON=python2 16 | ;; 17 | --python3 ) 18 | COVERAGE_TOOL=coverage3 19 | PYTHON=python3 20 | ;; 21 | --verbose ) 22 | VERBOSE="-v" 23 | ;; 24 | --test=?*) 25 | TEST=${1#*=} 26 | ;; 27 | * ) 28 | ;; 29 | esac 30 | shift 31 | done 32 | 33 | if ${COVERAGE}; then 34 | ${COVERAGE_TOOL} run --source=hpccm -m unittest discover -s test ${VERBOSE} 35 | ${COVERAGE_TOOL} report -m 36 | elif [ -n "${TEST}" ]; then 37 | echo "Test: ${TEST}" 38 | ${PYTHON} -m unittest discover ${VERBOSE} -s test -p "*${TEST}*.py" 39 | else 40 | ${PYTHON} -m unittest discover -s test ${VERBOSE} 41 | fi 42 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """Setup script for hpccm.""" 4 | 5 | import os 6 | 7 | from setuptools import find_packages 8 | from setuptools import setup 9 | 10 | here = os.path.abspath(os.path.dirname(__file__)) 11 | 12 | # Get the package version from hpccm/version.py 13 | version = {} 14 | with open(os.path.join(here, 'hpccm', 'version.py')) as fp: 15 | exec(fp.read(), version) 16 | 17 | # Get the long description from the README file 18 | with open(os.path.join(here, 'README.md')) as fp: 19 | long_description = fp.read() 20 | 21 | setup( 22 | name='hpccm', 23 | version=version['__version__'], 24 | description='HPC Container Maker', 25 | long_description=long_description, 26 | long_description_content_type='text/markdown', 27 | maintainer='Scott McMillan', 28 | maintainer_email='smcmillan@nvidia.com', 29 | license='Apache License Version 2.0', 30 | url='https://github.com/NVIDIA/hpc-container-maker', 31 | packages=find_packages(), 32 | classifiers=[ 33 | "License :: OSI Approved :: Apache Software License", 34 | "Programming Language :: Python :: 2", 35 | "Programming Language :: Python :: 2.7", 36 | "Programming Language :: Python :: 3", 37 | "Programming Language :: Python :: 3.4", 38 | "Programming Language :: Python :: 3.5", 39 | "Programming Language :: Python :: 3.6" 40 | ], 41 | # Make hpccm.cli.main available from the command line as `hpccm`. 42 | install_requires=['archspec', "enum34; python_version < '3.4'", 43 | 'packaging', 'six'], 44 | entry_points={ 45 | 'console_scripts': [ 46 | 'hpccm=hpccm.cli:main']}) 47 | -------------------------------------------------------------------------------- /test/bad_recipe.py: -------------------------------------------------------------------------------- 1 | """This is bad python""" 2 | 3 | Stage0 += baseimage() 4 | 5 | # missing parenthesis 6 | Stage0 += apt_get(['gcc'] 7 | 8 | Stage0 += apt_get(['make']) 9 | -------------------------------------------------------------------------------- /test/docker.blob: -------------------------------------------------------------------------------- 1 | COPY foo bar 2 | RUN bar 3 | -------------------------------------------------------------------------------- /test/global_vars_recipe.py: -------------------------------------------------------------------------------- 1 | from hpccm.common import container_type 2 | 3 | if hpccm.config.g_ctype != container_type.SINGULARITY: 4 | raise Exception("Global variable g_ctype not set correctly!") 5 | -------------------------------------------------------------------------------- /test/include1.py: -------------------------------------------------------------------------------- 1 | Stage0 += baseimage(image='ubuntu:16.04') 2 | -------------------------------------------------------------------------------- /test/include2.py: -------------------------------------------------------------------------------- 1 | include('../test/include1.py') 2 | compiler = gnu() 3 | -------------------------------------------------------------------------------- /test/include3.py: -------------------------------------------------------------------------------- 1 | include('include2.py') 2 | Stage0 += compiler 3 | -------------------------------------------------------------------------------- /test/singularity.blob: -------------------------------------------------------------------------------- 1 | %files 2 | foo bar 3 | 4 | %post 5 | bar 6 | -------------------------------------------------------------------------------- /test/test_annotate.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the annotate module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from hpccm.templates.annotate import annotate 26 | 27 | class Test_annotate(unittest.TestCase): 28 | def setUp(self): 29 | """Disable logging output messages""" 30 | logging.disable(logging.ERROR) 31 | 32 | def test_no_annotations(self): 33 | """No variables specified""" 34 | a = annotate() 35 | 36 | self.assertDictEqual(a.annotate_step(), {}) 37 | 38 | def test_no_base(self): 39 | """Basic annotations""" 40 | a = annotate(annotate=True, base_annotation=False) 41 | a.add_annotation('A', 'a') 42 | a.add_annotation('one', 1) 43 | 44 | self.assertDictEqual(a.annotate_step(), {'A': 'a', 'one': '1'}) 45 | 46 | def test_default_base(self): 47 | """Basic annotations""" 48 | a = annotate(annotate=True, base_annotation=True) 49 | a.add_annotation('A', 'a') 50 | a.add_annotation('one', 1) 51 | 52 | self.assertDictEqual(a.annotate_step(), {'hpccm.annotate.A': 'a', 53 | 'hpccm.annotate.one': '1'}) 54 | 55 | def test_custom_base(self): 56 | """Basic annotations""" 57 | a = annotate(annotate=True, base_annotation='foo') 58 | a.add_annotation('A', 'a') 59 | a.add_annotation('one', 1) 60 | 61 | self.assertDictEqual(a.annotate_step(), {'hpccm.foo.A': 'a', 62 | 'hpccm.foo.one': '1'}) 63 | 64 | def test_no_annotations(self): 65 | """Basic annotations""" 66 | a = annotate(annotate=False) 67 | a.add_annotation('A', 'a') 68 | a.add_annotation('one', 1) 69 | 70 | self.assertDictEqual(a.annotate_step(), {}) 71 | 72 | -------------------------------------------------------------------------------- /test/test_apt_get.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the apt_get module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import docker, ubuntu 26 | 27 | from hpccm.building_blocks.apt_get import apt_get 28 | 29 | class Test_apt_get(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @ubuntu 35 | @docker 36 | def test_basic(self): 37 | """Basic apt_get""" 38 | a = apt_get(ospackages=['gcc', 'g++', 'gfortran']) 39 | self.assertEqual(str(a), 40 | r'''RUN apt-get update -y && \ 41 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 42 | g++ \ 43 | gcc \ 44 | gfortran && \ 45 | rm -rf /var/lib/apt/lists/*''') 46 | 47 | @ubuntu 48 | @docker 49 | def test_add_repo_signed_by(self): 50 | """Add repo and key, using the signed-by method rather than apt-key""" 51 | a = apt_get(keys=['https://www.example.com/key.pub'], 52 | ospackages=['example'], 53 | repositories=['deb [signed-by=/usr/share/keyrings/key.gpg] http://www.example.com all main']) 54 | self.assertEqual(str(a), 55 | r'''RUN mkdir -p /usr/share/keyrings && \ 56 | rm -f /usr/share/keyrings/key.gpg && \ 57 | wget -qO - https://www.example.com/key.pub | gpg --dearmor -o /usr/share/keyrings/key.gpg && \ 58 | echo "deb [signed-by=/usr/share/keyrings/key.gpg] http://www.example.com all main" >> /etc/apt/sources.list.d/hpccm.list && \ 59 | apt-get update -y && \ 60 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 61 | example && \ 62 | rm -rf /var/lib/apt/lists/*''') 63 | 64 | @ubuntu 65 | @docker 66 | def test_download(self): 67 | """Download parameter""" 68 | a = apt_get(download=True, download_directory='/tmp/download', 69 | ospackages=['libibverbs1']) 70 | self.assertEqual(str(a), 71 | r'''RUN apt-get update -y && \ 72 | mkdir -m 777 -p /tmp/download && cd /tmp/download && \ 73 | DEBIAN_FRONTEND=noninteractive apt-get download -y --no-install-recommends \ 74 | libibverbs1 && \ 75 | rm -rf /var/lib/apt/lists/*''') 76 | 77 | @ubuntu 78 | @docker 79 | def test_extract(self): 80 | """Extract parameter""" 81 | a = apt_get(download=True, extract='/usr/local/ofed', 82 | ospackages=['libibverbs1']) 83 | self.assertEqual(str(a), 84 | r'''RUN apt-get update -y && \ 85 | mkdir -m 777 -p /var/tmp/apt_get_download && cd /var/tmp/apt_get_download && \ 86 | DEBIAN_FRONTEND=noninteractive apt-get download -y --no-install-recommends \ 87 | libibverbs1 && \ 88 | mkdir -p /usr/local/ofed && \ 89 | find /var/tmp/apt_get_download -regextype posix-extended -type f -regex "/var/tmp/apt_get_download/(libibverbs1).*deb" -exec dpkg --extract {} /usr/local/ofed \; && \ 90 | rm -rf /var/tmp/apt_get_download && \ 91 | rm -rf /var/lib/apt/lists/*''') 92 | -------------------------------------------------------------------------------- /test/test_bb_base.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods, bad-continuation 16 | 17 | """Test cases for the building block base class""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import centos, docker, ubuntu 26 | 27 | from hpccm.building_blocks.base import bb_base 28 | from hpccm.primitives import shell 29 | 30 | class Test_bb_base(unittest.TestCase): 31 | def setUp(self): 32 | """Disable logging output messages""" 33 | logging.disable(logging.ERROR) 34 | 35 | @ubuntu 36 | @docker 37 | def test_defaults(self): 38 | """Default building block base class""" 39 | b = bb_base() 40 | self.assertEqual(str(b), '') 41 | 42 | @ubuntu 43 | @docker 44 | def test_instruction_manipulations(self): 45 | """Instruction manipulations""" 46 | b = bb_base() 47 | 48 | # Append instructions 49 | b += shell(commands=['echo a']) 50 | # Append directly to "private" class variable (not recommended) 51 | b._bb_instructions__instructions_bb.append(shell(commands=['echo b'])) 52 | self.assertEqual(len(b), 2) 53 | self.assertEqual(str(b), 'RUN echo a\nRUN echo b') 54 | 55 | # Direct element access 56 | self.assertEqual(str(b[0]), 'RUN echo a') 57 | self.assertEqual(str(b[1]), 'RUN echo b') 58 | 59 | # Iterators 60 | i = iter(b) 61 | self.assertEqual(str(next(i)), 'RUN echo a') 62 | self.assertEqual(str(next(i)), 'RUN echo b') 63 | 64 | # Insertion, using "private" class variable (not recommended) 65 | b._bb_instructions__instructions_bb.insert(0, shell(commands=['echo c'])) 66 | self.assertEqual(len(b), 3) 67 | self.assertEqual(str(b), 'RUN echo c\nRUN echo a\nRUN echo b') 68 | 69 | # Deletion (not allowed) 70 | with self.assertRaises(TypeError): 71 | del(b[1]) 72 | 73 | # Deletion via "private" class variable (not recommended) 74 | del(b._bb_instructions__instructions_bb[1]) 75 | self.assertEqual(len(b), 2) 76 | self.assertEqual(str(b), 'RUN echo c\nRUN echo b') 77 | 78 | @ubuntu 79 | @docker 80 | def test_runtime_instructions(self): 81 | """Instruction manipulations""" 82 | b = bb_base() 83 | 84 | # A non-runtime instruction that should not appear 85 | b += shell(commands=['echo a']) 86 | 87 | b.rt += shell(commands=['echo r1']) 88 | b.rt += shell(commands=['echo r2']) 89 | self.assertEqual(len(b.rt), 2) 90 | self.assertEqual(str(b.rt), 'RUN echo r1\nRUN echo r2') 91 | -------------------------------------------------------------------------------- /test/test_blob.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods, bad-continuation 16 | 17 | """Test cases for the blob module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import os 24 | import unittest 25 | 26 | from helpers import bash, docker, invalid_ctype, singularity 27 | 28 | from hpccm.primitives.blob import blob 29 | 30 | class Test_blob(unittest.TestCase): 31 | def setUp(self): 32 | """Disable logging output messages""" 33 | logging.disable(logging.ERROR) 34 | 35 | @docker 36 | def test_empty(self): 37 | """No blob specified""" 38 | b = blob() 39 | self.assertEqual(str(b), '') 40 | 41 | @invalid_ctype 42 | def test_invalid_ctype(self): 43 | """Invalid container type specified""" 44 | b = blob() 45 | with self.assertRaises(RuntimeError): 46 | str(b) 47 | 48 | @docker 49 | def test_invalid_files(self): 50 | """Invalid file""" 51 | b = blob(docker='/path/to/nonexistent/file') 52 | self.assertEqual(str(b), '') 53 | 54 | @docker 55 | def test_docker_only_docker(self): 56 | """Only Docker blob specified""" 57 | path = os.path.dirname(__file__) 58 | b = blob(docker=os.path.join(path, 'docker.blob')) 59 | self.assertEqual(str(b), 'COPY foo bar\nRUN bar\n') 60 | 61 | @singularity 62 | def test_docker_only_singularity(self): 63 | """Only Docker blob specified""" 64 | path = os.path.dirname(__file__) 65 | b = blob(docker=os.path.join(path, 'docker.blob')) 66 | self.assertEqual(str(b), '') 67 | 68 | @docker 69 | def test_singularity_only_docker(self): 70 | """Only Singularity blob specified""" 71 | path = os.path.dirname(__file__) 72 | b = blob(singularity=os.path.join(path, 'singularity.blob')) 73 | self.assertEqual(str(b), '') 74 | 75 | @singularity 76 | def test_singularity_only_singularity(self): 77 | """Only Singularity blob specified""" 78 | path = os.path.dirname(__file__) 79 | b = blob(singularity=os.path.join(path, 'singularity.blob')) 80 | self.assertEqual(str(b), 81 | '''%files 82 | foo bar 83 | 84 | %post 85 | bar 86 | ''') 87 | 88 | @docker 89 | def test_all_docker(self): 90 | """Both Docker and Singularity blobs specified""" 91 | path = os.path.dirname(__file__) 92 | b = blob(docker=os.path.join(path, 'docker.blob'), 93 | singularity=os.path.join(path, 'singularity.blob')) 94 | self.assertEqual(str(b), 'COPY foo bar\nRUN bar\n') 95 | 96 | @singularity 97 | def test_all_singularity(self): 98 | """Both Docker and Singularity blobs specified""" 99 | path = os.path.dirname(__file__) 100 | b = blob(docker=os.path.join(path, 'docker.blob'), 101 | singularity=os.path.join(path, 'singularity.blob')) 102 | self.assertEqual(str(b), 103 | '''%files 104 | foo bar 105 | 106 | %post 107 | bar 108 | ''') 109 | 110 | @bash 111 | def test_all_bash(self): 112 | """Both Docker and Singularity blobs specified""" 113 | path = os.path.dirname(__file__) 114 | b = blob(docker=os.path.join(path, 'docker.blob'), 115 | singularity=os.path.join(path, 'singularity.blob')) 116 | self.assertEqual(str(b), '') 117 | -------------------------------------------------------------------------------- /test/test_cgns.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods, bad-continuation 16 | 17 | """Test cases for the cgns module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import centos, docker, ubuntu 26 | 27 | from hpccm.building_blocks.cgns import cgns 28 | 29 | class Test_cgns(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @ubuntu 35 | @docker 36 | def test_defaults_ubuntu(self): 37 | """Default cgns building block""" 38 | c = cgns() 39 | self.assertEqual(str(c), 40 | r'''# CGNS version 4.1.2 41 | RUN apt-get update -y && \ 42 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 43 | file \ 44 | make \ 45 | wget \ 46 | zlib1g-dev && \ 47 | rm -rf /var/lib/apt/lists/* 48 | RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/CGNS/CGNS/archive/v4.1.2.tar.gz && \ 49 | mkdir -p /var/tmp && tar -x -f /var/tmp/v4.1.2.tar.gz -C /var/tmp -z && \ 50 | cd /var/tmp/CGNS-4.1.2/src && FLIBS='-Wl,--no-as-needed -ldl' LIBS='-Wl,--no-as-needed -ldl' ./configure --prefix=/usr/local/cgns --with-hdf5=/usr/local/hdf5 --with-zlib && \ 51 | make -j$(nproc) && \ 52 | make -j$(nproc) install && \ 53 | rm -rf /var/tmp/CGNS-4.1.2/src /var/tmp/v4.1.2.tar.gz''') 54 | 55 | @centos 56 | @docker 57 | def test_defaults_centos(self): 58 | """Default cgns building block""" 59 | c = cgns() 60 | self.assertEqual(str(c), 61 | r'''# CGNS version 4.1.2 62 | RUN yum install -y \ 63 | bzip2 \ 64 | file \ 65 | make \ 66 | wget \ 67 | zlib-devel && \ 68 | rm -rf /var/cache/yum/* 69 | RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/CGNS/CGNS/archive/v4.1.2.tar.gz && \ 70 | mkdir -p /var/tmp && tar -x -f /var/tmp/v4.1.2.tar.gz -C /var/tmp -z && \ 71 | cd /var/tmp/CGNS-4.1.2/src && FLIBS='-Wl,--no-as-needed -ldl' LIBS='-Wl,--no-as-needed -ldl' ./configure --prefix=/usr/local/cgns --with-hdf5=/usr/local/hdf5 --with-zlib && \ 72 | make -j$(nproc) && \ 73 | make -j$(nproc) install && \ 74 | rm -rf /var/tmp/CGNS-4.1.2/src /var/tmp/v4.1.2.tar.gz''') 75 | 76 | @ubuntu 77 | @docker 78 | def test_runtime(self): 79 | """Runtime""" 80 | c = cgns() 81 | r = c.runtime() 82 | self.assertEqual(r, 83 | r'''# CGNS 84 | RUN apt-get update -y && \ 85 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 86 | zlib1g && \ 87 | rm -rf /var/lib/apt/lists/* 88 | COPY --from=0 /usr/local/cgns /usr/local/cgns''') 89 | -------------------------------------------------------------------------------- /test/test_cli.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the cli module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import argparse 23 | import logging # pylint: disable=unused-import 24 | import unittest 25 | 26 | from hpccm.cli import KeyValue 27 | 28 | class Test_cli(unittest.TestCase): 29 | def setUp(self): 30 | """Disable logging output messages""" 31 | logging.disable(logging.ERROR) 32 | 33 | def test_argparse_userarg(self): 34 | parser = argparse.ArgumentParser() 35 | parser.add_argument('--userarg', action=KeyValue, nargs='+') 36 | args = parser.parse_args(['--userarg', 'a=b', 'c=d', 'e="f f f"']) 37 | self.assertEqual(args.userarg['a'], 'b') 38 | self.assertEqual(args.userarg['c'], 'd') 39 | self.assertEqual(args.userarg['e'], '"f f f"') 40 | -------------------------------------------------------------------------------- /test/test_comment.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the comment module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import docker, invalid_ctype, singularity 26 | 27 | from hpccm.primitives.comment import comment 28 | 29 | class Test_comment(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @docker 35 | def test_empty(self): 36 | """No comment string specified""" 37 | c = comment() 38 | self.assertEqual(str(c), '') 39 | 40 | @docker 41 | def test_empty_noreformat(self): 42 | """No comment string specified, reformatting disabled""" 43 | c = comment(reformat=False) 44 | self.assertEqual(str(c), '') 45 | 46 | @invalid_ctype 47 | def test_invalid_ctype(self): 48 | """Invalid container type specified 49 | Assumes default comment format.""" 50 | c = comment('foo') 51 | self.assertEqual(str(c), '# foo') 52 | 53 | @docker 54 | def test_comment_docker(self): 55 | """Comment string specified""" 56 | c = comment('foo') 57 | self.assertEqual(str(c), '# foo') 58 | 59 | @singularity 60 | def test_comment_singularity(self): 61 | """Comment string specified""" 62 | c = comment('foo') 63 | self.assertEqual(str(c), '# foo') 64 | 65 | @docker 66 | def test_noreformat(self): 67 | """Disable reformatting""" 68 | c = comment('foo\nbar', reformat=False) 69 | self.assertEqual(str(c), '# foo\n# bar') 70 | 71 | @docker 72 | def test_wrap(self): 73 | """Comment wrapping""" 74 | c = comment('foo\nbar') 75 | self.assertEqual(str(c), '# foo bar') 76 | 77 | @docker 78 | def test_merge_docker(self): 79 | """Comment merge""" 80 | c = [] 81 | c.append(comment('a')) 82 | c.append(comment('b')) 83 | merged = c[0].merge(c) 84 | self.assertEqual(str(merged), '# a\n# b') 85 | 86 | @singularity 87 | def test_merge_singularity(self): 88 | """Comment merge""" 89 | c = [] 90 | c.append(comment('a')) 91 | c.append(comment('b')) 92 | merged = c[0].merge(c) 93 | self.assertEqual(str(merged), '# a\n# b') 94 | 95 | apphelp = c[0].merge(c, _app='foo') 96 | self.assertEqual(str(apphelp), '%apphelp foo\na\nb') 97 | -------------------------------------------------------------------------------- /test/test_envvars.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the envvars module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from hpccm.templates.envvars import envvars 26 | 27 | class Test_envvars(unittest.TestCase): 28 | def setUp(self): 29 | """Disable logging output messages""" 30 | logging.disable(logging.ERROR) 31 | 32 | def test_no_variables(self): 33 | """No variables specified""" 34 | e = envvars() 35 | 36 | self.assertDictEqual(e.environment_step(), {}) 37 | 38 | def test_basic(self): 39 | """Basic envvars""" 40 | d = {'A': 'a', 'B': 'b', 'one': 1} 41 | e = envvars() 42 | e.environment_variables = d 43 | 44 | self.assertDictEqual(e.environment_step(), d) 45 | 46 | e.environment = False 47 | self.assertDictEqual(e.environment_step(), {}) 48 | 49 | def test_include_exclude(self): 50 | """Include / exclude keys""" 51 | d = {'A': 'a', 'B': 'b', 'one': 1} 52 | e = envvars() 53 | e.environment_variables = d 54 | 55 | self.assertDictEqual(e.environment_step(exclude=['B']), 56 | {'A': 'a', 'one': 1}) 57 | 58 | self.assertDictEqual(e.environment_step(include_only=['A', 'one']), 59 | {'A': 'a', 'one': 1}) 60 | 61 | def test_runtime(self): 62 | """Runtime environment variables""" 63 | d = {'A': 'a', 'B': 'b', 'one': 1} 64 | r = {'A': 'alpha', 'B': 'b'} 65 | e = envvars() 66 | e.environment_variables = d 67 | e.runtime_environment_variables = r 68 | 69 | self.assertDictEqual(e.environment_step(runtime=True), 70 | {'A': 'alpha', 'B': 'b'}) 71 | -------------------------------------------------------------------------------- /test/test_global_vars.py: -------------------------------------------------------------------------------- 1 | 2 | from __future__ import unicode_literals 3 | from __future__ import print_function 4 | 5 | import logging # pylint: disable=unused-import 6 | import unittest 7 | import os 8 | 9 | from helpers import docker, ubuntu 10 | 11 | from hpccm.common import container_type 12 | from hpccm.recipe import recipe 13 | 14 | class Test_global_vars(unittest.TestCase): 15 | def test_global_vars(self): 16 | """Global variables""" 17 | path = os.path.dirname(__file__) 18 | rf = os.path.join(path, 'global_vars_recipe.py') 19 | try: 20 | recipe(rf, ctype=container_type.SINGULARITY, raise_exceptions=True) 21 | except Exception as e: 22 | self.fail(e) 23 | -------------------------------------------------------------------------------- /test/test_knem.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods, bad-continuation 16 | 17 | """Test cases for the knem module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import centos, docker, ubuntu 26 | 27 | from hpccm.building_blocks.knem import knem 28 | 29 | class Test_knem(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @ubuntu 35 | @docker 36 | def test_defaults_ubuntu(self): 37 | """Default knem building block""" 38 | k = knem() 39 | self.assertEqual(str(k), 40 | r'''# KNEM version 1.1.4 41 | RUN apt-get update -y && \ 42 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 43 | ca-certificates \ 44 | git && \ 45 | rm -rf /var/lib/apt/lists/* 46 | RUN mkdir -p /var/tmp && cd /var/tmp && git clone --depth=1 --branch knem-1.1.4 https://gitlab.inria.fr/knem/knem.git knem && cd - && \ 47 | mkdir -p /usr/local/knem && \ 48 | cd /var/tmp/knem && \ 49 | mkdir -p /usr/local/knem/include && \ 50 | cp common/*.h /usr/local/knem/include && \ 51 | rm -rf /var/tmp/knem 52 | ENV CPATH=/usr/local/knem/include:$CPATH''') 53 | 54 | @centos 55 | @docker 56 | def test_defaults_centos(self): 57 | """Default knem building block""" 58 | k = knem() 59 | self.assertEqual(str(k), 60 | r'''# KNEM version 1.1.4 61 | RUN yum install -y \ 62 | ca-certificates \ 63 | git && \ 64 | rm -rf /var/cache/yum/* 65 | RUN mkdir -p /var/tmp && cd /var/tmp && git clone --depth=1 --branch knem-1.1.4 https://gitlab.inria.fr/knem/knem.git knem && cd - && \ 66 | mkdir -p /usr/local/knem && \ 67 | cd /var/tmp/knem && \ 68 | mkdir -p /usr/local/knem/include && \ 69 | cp common/*.h /usr/local/knem/include && \ 70 | rm -rf /var/tmp/knem 71 | ENV CPATH=/usr/local/knem/include:$CPATH''') 72 | 73 | @ubuntu 74 | @docker 75 | def test_runtime(self): 76 | """Runtime""" 77 | k = knem() 78 | r = k.runtime() 79 | self.assertEqual(r, 80 | r'''# KNEM 81 | COPY --from=0 /usr/local/knem /usr/local/knem 82 | ENV CPATH=/usr/local/knem/include:$CPATH''') 83 | -------------------------------------------------------------------------------- /test/test_label.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods, bad-continuation 16 | 17 | """Test cases for the label module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import bash, docker, invalid_ctype, singularity 26 | 27 | from hpccm.primitives.label import label 28 | 29 | class Test_label(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @docker 35 | def test_empty(self): 36 | """No label specified""" 37 | l = label() 38 | self.assertEqual(str(l), '') 39 | 40 | @invalid_ctype 41 | def test_invalid_ctype(self): 42 | """Invalid container type specified""" 43 | l = label(metadata={'A': 'B'}) 44 | with self.assertRaises(RuntimeError): 45 | str(l) 46 | 47 | @docker 48 | def test_single_docker(self): 49 | """Single label specified""" 50 | l = label(metadata={'A': 'B'}) 51 | self.assertEqual(str(l), 'LABEL A=B') 52 | 53 | @singularity 54 | def test_single_singularity(self): 55 | """Single label specified""" 56 | l = label(metadata={'A': 'B'}) 57 | self.assertEqual(str(l), '%labels\n A B') 58 | 59 | @bash 60 | def test_single_bash(self): 61 | """Single label specified""" 62 | l = label(metadata={'A': 'B'}) 63 | self.assertEqual(str(l), '') 64 | 65 | @docker 66 | def test_multiple_docker(self): 67 | """Multiple labels specified""" 68 | l = label(metadata={'ONE': 1, 'TWO': 2, 'THREE': 3}) 69 | self.assertEqual(str(l), 70 | '''LABEL ONE=1 \\ 71 | THREE=3 \\ 72 | TWO=2''') 73 | 74 | @singularity 75 | def test_multiple_singularity(self): 76 | """Multiple labels specified""" 77 | l = label(metadata={'ONE': 1, 'TWO': 2, 'THREE': 3}) 78 | self.assertEqual(str(l), 79 | '''%labels 80 | ONE 1 81 | THREE 3 82 | TWO 2''') 83 | 84 | @singularity 85 | def test_applabel_multiple_singularity(self): 86 | """Multiple app-specific labels specified""" 87 | l = label(metadata={'ONE': 1, 'TWO': 2, 'THREE': 3}, _app='foo') 88 | self.assertEqual(str(l), 89 | '''%applabels foo 90 | ONE 1 91 | THREE 3 92 | TWO 2''') 93 | 94 | @docker 95 | def test_applabels_docker(self): 96 | """applabels not implemented in Docker""" 97 | l = label(metadata={'ONE': 1, 'TWO': 2, 'THREE': 3}, _app='foo') 98 | self.assertEqual(str(l), 99 | '''LABEL ONE=1 \\ 100 | THREE=3 \\ 101 | TWO=2''') 102 | 103 | @docker 104 | def test_merge_docker(self): 105 | """merge primitives""" 106 | l = [] 107 | l.append(label(metadata={'ONE': 1, 'TWO': 2})) 108 | l.append(label(metadata={'THREE': 3})) 109 | merged = l[0].merge(l) 110 | self.assertEqual(str(merged), 111 | '''LABEL ONE=1 \\ 112 | THREE=3 \\ 113 | TWO=2''') 114 | 115 | l.append(label(metadata={'ONE': 'uno'})) 116 | key_overwrite = l[0].merge(l) 117 | self.assertEqual(str(key_overwrite), 118 | '''LABEL ONE=uno \\ 119 | THREE=3 \\ 120 | TWO=2''') 121 | 122 | @singularity 123 | def test_merge_singularity(self): 124 | """merge primitives""" 125 | l = [] 126 | l.append(label(metadata={'ONE': 1, 'TWO': 2})) 127 | l.append(label(metadata={'THREE': 3})) 128 | merged = l[0].merge(l) 129 | self.assertEqual(str(merged), 130 | '''%labels 131 | ONE 1 132 | THREE 3 133 | TWO 2''') 134 | 135 | l.append(label(metadata={'ONE': 'uno'})) 136 | key_overwrite = l[0].merge(l) 137 | self.assertEqual(str(key_overwrite), 138 | '''%labels 139 | ONE uno 140 | THREE 3 141 | TWO 2''') 142 | -------------------------------------------------------------------------------- /test/test_ldconfig.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the ldconfig module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from hpccm.templates.ldconfig import ldconfig 26 | 27 | class Test_ldconfig(unittest.TestCase): 28 | def setUp(self): 29 | """Disable logging output messages""" 30 | logging.disable(logging.ERROR) 31 | 32 | def test_no_directory(self): 33 | """No directory specified""" 34 | l = ldconfig() 35 | 36 | self.assertEqual(l.ldcache_step(directory=None), '') 37 | 38 | def test_basic(self): 39 | """Basic ldconfig""" 40 | l = ldconfig() 41 | 42 | self.assertEqual(l.ldcache_step(directory='/usr/local/lib'), 43 | 'echo "/usr/local/lib" >> /etc/ld.so.conf.d/hpccm.conf && ldconfig') 44 | -------------------------------------------------------------------------------- /test/test_magma.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods, bad-continuation 16 | 17 | """Test cases for the magma module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import centos, docker, ubuntu 26 | 27 | from hpccm.building_blocks.magma import magma 28 | from hpccm.toolchain import toolchain 29 | 30 | class Test_sensei(unittest.TestCase): 31 | def setUp(self): 32 | """Disable logging output messages""" 33 | logging.disable(logging.ERROR) 34 | 35 | @ubuntu 36 | @docker 37 | def test_defaults_ubuntu(self): 38 | """Default magma building block""" 39 | m = magma() 40 | self.assertEqual(str(m), 41 | r'''# MAGMA version 2.5.3 42 | RUN apt-get update -y && \ 43 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 44 | tar \ 45 | wget && \ 46 | rm -rf /var/lib/apt/lists/* 47 | RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp http://icl.utk.edu/projectsfiles/magma/downloads/magma-2.5.3.tar.gz && \ 48 | mkdir -p /var/tmp && tar -x -f /var/tmp/magma-2.5.3.tar.gz -C /var/tmp -z && \ 49 | mkdir -p /var/tmp/magma-2.5.3/build && cd /var/tmp/magma-2.5.3/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/magma -DGPU_TARGET="Pascal Volta Turing" /var/tmp/magma-2.5.3 && \ 50 | cmake --build /var/tmp/magma-2.5.3/build --target all -- -j$(nproc) && \ 51 | cmake --build /var/tmp/magma-2.5.3/build --target install -- -j$(nproc) && \ 52 | rm -rf /var/tmp/magma-2.5.3 /var/tmp/magma-2.5.3.tar.gz 53 | ENV CPATH=/usr/local/magma/include:$CPATH \ 54 | LD_LIBRARY_PATH=/usr/local/magma/lib:$LD_LIBRARY_PATH \ 55 | LIBRARY_PATH=/usr/local/magma/lib:$LIBRARY_PATH''') 56 | 57 | @centos 58 | @docker 59 | def test_defaults_centos(self): 60 | """Default magma building block""" 61 | m = magma() 62 | self.assertEqual(str(m), 63 | r'''# MAGMA version 2.5.3 64 | RUN yum install -y \ 65 | tar \ 66 | wget && \ 67 | rm -rf /var/cache/yum/* 68 | RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp http://icl.utk.edu/projectsfiles/magma/downloads/magma-2.5.3.tar.gz && \ 69 | mkdir -p /var/tmp && tar -x -f /var/tmp/magma-2.5.3.tar.gz -C /var/tmp -z && \ 70 | mkdir -p /var/tmp/magma-2.5.3/build && cd /var/tmp/magma-2.5.3/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/magma -DGPU_TARGET="Pascal Volta Turing" /var/tmp/magma-2.5.3 && \ 71 | cmake --build /var/tmp/magma-2.5.3/build --target all -- -j$(nproc) && \ 72 | cmake --build /var/tmp/magma-2.5.3/build --target install -- -j$(nproc) && \ 73 | rm -rf /var/tmp/magma-2.5.3 /var/tmp/magma-2.5.3.tar.gz 74 | ENV CPATH=/usr/local/magma/include:$CPATH \ 75 | LD_LIBRARY_PATH=/usr/local/magma/lib:$LD_LIBRARY_PATH \ 76 | LIBRARY_PATH=/usr/local/magma/lib:$LIBRARY_PATH''') 77 | 78 | @ubuntu 79 | @docker 80 | def test_runtime(self): 81 | """Runtime""" 82 | m = magma() 83 | r = m.runtime() 84 | self.assertEqual(r, 85 | r'''# MAGMA 86 | COPY --from=0 /usr/local/magma /usr/local/magma 87 | ENV CPATH=/usr/local/magma/include:$CPATH \ 88 | LD_LIBRARY_PATH=/usr/local/magma/lib:$LD_LIBRARY_PATH \ 89 | LIBRARY_PATH=/usr/local/magma/lib:$LIBRARY_PATH''') 90 | -------------------------------------------------------------------------------- /test/test_packages.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the packages module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import centos, docker, invalid_distro, ubuntu 26 | 27 | from hpccm.building_blocks.packages import packages 28 | 29 | class Test_packages(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @ubuntu 35 | @docker 36 | def test_basic_ubuntu(self): 37 | """Basic packages""" 38 | p = packages(ospackages=['gcc', 'g++', 'gfortran']) 39 | self.assertEqual(str(p), 40 | r'''RUN apt-get update -y && \ 41 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 42 | g++ \ 43 | gcc \ 44 | gfortran && \ 45 | rm -rf /var/lib/apt/lists/*''') 46 | 47 | @centos 48 | @docker 49 | def test_basic_centos(self): 50 | """Basic packages""" 51 | p = packages(ospackages=['gcc', 'gcc-c++', 'gcc-fortran']) 52 | self.assertEqual(str(p), 53 | r'''RUN yum install -y \ 54 | gcc \ 55 | gcc-c++ \ 56 | gcc-fortran && \ 57 | rm -rf /var/cache/yum/*''') 58 | 59 | @invalid_distro 60 | def test_invalid_distro(self): 61 | """Invalid package type specified""" 62 | with self.assertRaises(RuntimeError): 63 | packages(ospackages=['gcc', 'g++', 'gfortran']) 64 | -------------------------------------------------------------------------------- /test/test_pmix.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods, bad-continuation 16 | 17 | """Test cases for the pmix module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import centos, docker, ubuntu, x86_64 26 | 27 | from hpccm.building_blocks.pmix import pmix 28 | 29 | class Test_pmix(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @x86_64 35 | @ubuntu 36 | @docker 37 | def test_defaults_ubuntu(self): 38 | """Default pmix building block""" 39 | p = pmix() 40 | self.assertEqual(str(p), 41 | r'''# PMIX version 4.1.2 42 | RUN apt-get update -y && \ 43 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 44 | file \ 45 | hwloc \ 46 | libevent-dev \ 47 | make \ 48 | tar \ 49 | wget && \ 50 | rm -rf /var/lib/apt/lists/* 51 | RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/openpmix/openpmix/releases/download/v4.1.2/pmix-4.1.2.tar.gz && \ 52 | mkdir -p /var/tmp && tar -x -f /var/tmp/pmix-4.1.2.tar.gz -C /var/tmp -z && \ 53 | cd /var/tmp/pmix-4.1.2 && ./configure --prefix=/usr/local/pmix && \ 54 | make -j$(nproc) && \ 55 | make -j$(nproc) install && \ 56 | rm -rf /var/tmp/pmix-4.1.2 /var/tmp/pmix-4.1.2.tar.gz 57 | ENV CPATH=/usr/local/pmix/include:$CPATH \ 58 | LD_LIBRARY_PATH=/usr/local/pmix/lib:$LD_LIBRARY_PATH \ 59 | PATH=/usr/local/pmix/bin:$PATH''') 60 | 61 | @x86_64 62 | @centos 63 | @docker 64 | def test_ldconfig(self): 65 | """ldconfig option""" 66 | p = pmix(ldconfig=True, version='3.1.4') 67 | self.assertEqual(str(p), 68 | r'''# PMIX version 3.1.4 69 | RUN yum install -y \ 70 | file \ 71 | hwloc \ 72 | libevent-devel \ 73 | make \ 74 | tar \ 75 | wget && \ 76 | rm -rf /var/cache/yum/* 77 | RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/openpmix/openpmix/releases/download/v3.1.4/pmix-3.1.4.tar.gz && \ 78 | mkdir -p /var/tmp && tar -x -f /var/tmp/pmix-3.1.4.tar.gz -C /var/tmp -z && \ 79 | cd /var/tmp/pmix-3.1.4 && ./configure --prefix=/usr/local/pmix && \ 80 | make -j$(nproc) && \ 81 | make -j$(nproc) install && \ 82 | echo "/usr/local/pmix/lib" >> /etc/ld.so.conf.d/hpccm.conf && ldconfig && \ 83 | rm -rf /var/tmp/pmix-3.1.4 /var/tmp/pmix-3.1.4.tar.gz 84 | ENV CPATH=/usr/local/pmix/include:$CPATH \ 85 | PATH=/usr/local/pmix/bin:$PATH''') 86 | 87 | @x86_64 88 | @ubuntu 89 | @docker 90 | def test_runtime(self): 91 | """Runtime""" 92 | p = pmix() 93 | r = p.runtime() 94 | self.assertEqual(r, 95 | r'''# PMIX 96 | RUN apt-get update -y && \ 97 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 98 | libevent-2.* \ 99 | libevent-pthreads-2.* && \ 100 | rm -rf /var/lib/apt/lists/* 101 | COPY --from=0 /usr/local/pmix /usr/local/pmix 102 | ENV CPATH=/usr/local/pmix/include:$CPATH \ 103 | LD_LIBRARY_PATH=/usr/local/pmix/lib:$LD_LIBRARY_PATH \ 104 | PATH=/usr/local/pmix/bin:$PATH''') 105 | -------------------------------------------------------------------------------- /test/test_pnetcdf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods, bad-continuation 16 | 17 | """Test cases for the pnetcdf module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import centos, docker, ubuntu 26 | 27 | from hpccm.building_blocks.pnetcdf import pnetcdf 28 | 29 | class Test_pnetcdf(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @ubuntu 35 | @docker 36 | def test_defaults(self): 37 | """Default pnetcdf building block""" 38 | p = pnetcdf() 39 | self.assertEqual(str(p), 40 | r'''# PnetCDF version 1.12.1 41 | RUN apt-get update -y && \ 42 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 43 | file \ 44 | m4 \ 45 | make \ 46 | perl \ 47 | tar \ 48 | wget && \ 49 | rm -rf /var/lib/apt/lists/* 50 | RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://parallel-netcdf.github.io/Release/pnetcdf-1.12.1.tar.gz && \ 51 | mkdir -p /var/tmp && tar -x -f /var/tmp/pnetcdf-1.12.1.tar.gz -C /var/tmp -z && \ 52 | cd /var/tmp/pnetcdf-1.12.1 && CC=mpicc CXX=mpicxx F77=mpif77 F90=mpif90 FC=mpifort ./configure --prefix=/usr/local/pnetcdf --enable-shared && \ 53 | cd /var/tmp/pnetcdf-1.12.1 && \ 54 | sed -i -e 's#pic_flag=""#pic_flag=" -fpic -DPIC"#' -e 's#wl=""#wl="-Wl,"#' libtool && \ 55 | make -j$(nproc) && \ 56 | make -j$(nproc) install && \ 57 | rm -rf /var/tmp/pnetcdf-1.12.1 /var/tmp/pnetcdf-1.12.1.tar.gz 58 | ENV CPATH=/usr/local/pnetcdf/include:$CPATH \ 59 | LD_LIBRARY_PATH=/usr/local/pnetcdf/lib:$LD_LIBRARY_PATH \ 60 | LIBRARY_PATH=/usr/local/pnetcdf/lib:$LIBRARY_PATH \ 61 | PATH=/usr/local/pnetcdf/bin:$PATH''') 62 | 63 | @ubuntu 64 | @docker 65 | def test_ldconfig(self): 66 | """ldconfig option""" 67 | p = pnetcdf(ldconfig=True, version='1.10.0') 68 | self.assertEqual(str(p), 69 | r'''# PnetCDF version 1.10.0 70 | RUN apt-get update -y && \ 71 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 72 | file \ 73 | m4 \ 74 | make \ 75 | perl \ 76 | tar \ 77 | wget && \ 78 | rm -rf /var/lib/apt/lists/* 79 | RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://parallel-netcdf.github.io/Release/parallel-netcdf-1.10.0.tar.gz && \ 80 | mkdir -p /var/tmp && tar -x -f /var/tmp/parallel-netcdf-1.10.0.tar.gz -C /var/tmp -z && \ 81 | cd /var/tmp/parallel-netcdf-1.10.0 && CC=mpicc CXX=mpicxx F77=mpif77 F90=mpif90 FC=mpifort ./configure --prefix=/usr/local/pnetcdf --enable-shared && \ 82 | cd /var/tmp/parallel-netcdf-1.10.0 && \ 83 | sed -i -e 's#pic_flag=""#pic_flag=" -fpic -DPIC"#' -e 's#wl=""#wl="-Wl,"#' libtool && \ 84 | make -j$(nproc) && \ 85 | make -j$(nproc) install && \ 86 | echo "/usr/local/pnetcdf/lib" >> /etc/ld.so.conf.d/hpccm.conf && ldconfig && \ 87 | rm -rf /var/tmp/parallel-netcdf-1.10.0 /var/tmp/parallel-netcdf-1.10.0.tar.gz 88 | ENV CPATH=/usr/local/pnetcdf/include:$CPATH \ 89 | LIBRARY_PATH=/usr/local/pnetcdf/lib:$LIBRARY_PATH \ 90 | PATH=/usr/local/pnetcdf/bin:$PATH''') 91 | 92 | @ubuntu 93 | @docker 94 | def test_runtime(self): 95 | """Runtime""" 96 | p = pnetcdf() 97 | r = p.runtime() 98 | self.assertEqual(r, 99 | r'''# PnetCDF 100 | RUN apt-get update -y && \ 101 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 102 | libatomic1 && \ 103 | rm -rf /var/lib/apt/lists/* 104 | COPY --from=0 /usr/local/pnetcdf /usr/local/pnetcdf 105 | ENV CPATH=/usr/local/pnetcdf/include:$CPATH \ 106 | LD_LIBRARY_PATH=/usr/local/pnetcdf/lib:$LD_LIBRARY_PATH \ 107 | LIBRARY_PATH=/usr/local/pnetcdf/lib:$LIBRARY_PATH \ 108 | PATH=/usr/local/pnetcdf/bin:$PATH''') 109 | -------------------------------------------------------------------------------- /test/test_python.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods, bad-continuation 16 | 17 | """Test cases for the python module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import centos, centos8, docker, ubuntu, ubuntu22 26 | 27 | from hpccm.building_blocks.python import python 28 | 29 | class Test_python(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @ubuntu 35 | @docker 36 | def test_defaults_ubuntu(self): 37 | """Default python building block""" 38 | p = python() 39 | self.assertEqual(str(p), 40 | r'''# Python 41 | RUN apt-get update -y && \ 42 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 43 | python \ 44 | python3 && \ 45 | rm -rf /var/lib/apt/lists/*''') 46 | 47 | @ubuntu22 48 | @docker 49 | def test_defaults_ubuntu22(self): 50 | """Default python building block""" 51 | p = python() 52 | self.assertEqual(str(p), 53 | r'''# Python 54 | RUN apt-get update -y && \ 55 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 56 | python2 \ 57 | python3 && \ 58 | rm -rf /var/lib/apt/lists/*''') 59 | 60 | @centos 61 | @docker 62 | def test_defaults_centos(self): 63 | """Default python building block""" 64 | p = python() 65 | self.assertEqual(str(p), 66 | r'''# Python 67 | RUN yum install -y \ 68 | python2 \ 69 | python3 && \ 70 | rm -rf /var/cache/yum/*''') 71 | 72 | @centos8 73 | @docker 74 | def test_defaults_alternatives(self): 75 | """Default python building block""" 76 | p = python(alternatives=True) 77 | self.assertEqual(str(p), 78 | r'''# Python 79 | RUN yum install -y \ 80 | python2 \ 81 | python3 && \ 82 | rm -rf /var/cache/yum/* 83 | RUN alternatives --set python /usr/bin/python2''') 84 | 85 | @ubuntu 86 | @docker 87 | def test_devel(self): 88 | """devel option""" 89 | p = python(devel=True) 90 | self.assertEqual(str(p), 91 | r'''# Python 92 | RUN apt-get update -y && \ 93 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 94 | python \ 95 | python-dev \ 96 | python3 \ 97 | python3-dev && \ 98 | rm -rf /var/lib/apt/lists/*''') 99 | 100 | @ubuntu 101 | @docker 102 | def test_runtime(self): 103 | """Runtime""" 104 | p = python() 105 | r = p.runtime() 106 | self.assertEqual(r, 107 | r'''# Python 108 | RUN apt-get update -y && \ 109 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 110 | python \ 111 | python3 && \ 112 | rm -rf /var/lib/apt/lists/*''') 113 | -------------------------------------------------------------------------------- /test/test_raw.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the raw module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import bash, docker, invalid_ctype, singularity 26 | 27 | from hpccm.primitives.raw import raw 28 | 29 | class Test_raw(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @docker 35 | def test_empty(self): 36 | """No raw strings specified""" 37 | r = raw() 38 | self.assertEqual(str(r), '') 39 | 40 | @invalid_ctype 41 | def test_invalid_ctype(self): 42 | """Invalid container type specified""" 43 | r = raw(docker='RAW') 44 | with self.assertRaises(RuntimeError): 45 | str(r) 46 | 47 | @docker 48 | def test_docker_only_docker(self): 49 | """Only Docker string specified""" 50 | r = raw(docker='RAW string') 51 | self.assertEqual(str(r), 'RAW string') 52 | 53 | @singularity 54 | def test_docker_only_singularity(self): 55 | """Only Docker string specified""" 56 | r = raw(docker='RAW string') 57 | self.assertEqual(str(r), '') 58 | 59 | @docker 60 | def test_singularity_only_docker(self): 61 | """Only Singularity string specified""" 62 | r = raw(singularity='%raw\n string') 63 | self.assertEqual(str(r), '') 64 | 65 | @singularity 66 | def test_singularity_only_singularity(self): 67 | """Only Singularity string specified""" 68 | r = raw(singularity='%raw\n string') 69 | self.assertEqual(str(r), '%raw\n string') 70 | 71 | @docker 72 | def test_all_docker(self): 73 | """Both Docker and Singularity strings specified""" 74 | r = raw(docker='RAW string', singularity='%raw\n string') 75 | self.assertEqual(str(r), 'RAW string') 76 | 77 | @singularity 78 | def test_all_singularity(self): 79 | """Both Docker and Singularity strings specified""" 80 | r = raw(docker='RAW string', singularity='%raw\n string') 81 | self.assertEqual(str(r), '%raw\n string') 82 | 83 | @bash 84 | def test_all_bash(self): 85 | """Both Docker and Singularity strings specified""" 86 | r = raw(docker='RAW string', singularity='%raw\n string') 87 | self.assertEqual(str(r), '') 88 | -------------------------------------------------------------------------------- /test/test_rm.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the rm module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from hpccm.templates.rm import rm 26 | 27 | class Test_rm(unittest.TestCase): 28 | def setUp(self): 29 | """Disable logging output messages""" 30 | logging.disable(logging.ERROR) 31 | 32 | def test_missing_files(self): 33 | """Missing files""" 34 | r = rm() 35 | self.assertEqual(r.cleanup_step(), '') 36 | 37 | def test_basic(self): 38 | """Basic file removal""" 39 | r = rm() 40 | 41 | self.assertEqual(r.cleanup_step(items=['a', 'b', 'c']), 42 | 'rm -rf a b c') 43 | -------------------------------------------------------------------------------- /test/test_runscript.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the runscript module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import bash, docker, invalid_ctype, singularity 26 | 27 | from hpccm.primitives.runscript import runscript 28 | 29 | class Test_runscript(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @docker 35 | def test_empty(self): 36 | """No commands specified""" 37 | s = runscript() 38 | self.assertEqual(str(s), '') 39 | 40 | @invalid_ctype 41 | def test_invalid_ctype(self): 42 | """Invalid container type specified""" 43 | s = runscript(commands=['a']) 44 | with self.assertRaises(RuntimeError): 45 | str(s) 46 | 47 | @docker 48 | def test_single_docker(self): 49 | """Single command specified""" 50 | cmd = ['z arg1 arg2'] 51 | s = runscript(commands=cmd) 52 | self.assertEqual(str(s), 'ENTRYPOINT ["z", "arg1", "arg2"]') 53 | 54 | @singularity 55 | def test_single_singularity(self): 56 | """Single command specified""" 57 | cmd = ['z'] 58 | s = runscript(commands=cmd) 59 | self.assertEqual(str(s), '%runscript\n exec z "$@"') 60 | 61 | @docker 62 | def test_multiple_docker(self): 63 | """List of commands specified""" 64 | cmds = ['a', 'b', 'c'] 65 | s = runscript(commands=cmds) 66 | self.assertEqual(str(s), 'ENTRYPOINT ["a"]') 67 | 68 | @singularity 69 | def test_multiple_singularity(self): 70 | """List of commands specified""" 71 | cmds = ['a arga', 'b argb', 'c'] 72 | s = runscript(commands=cmds) 73 | self.assertEqual(str(s), '%runscript\n a arga\n b argb\n exec c') 74 | 75 | @singularity 76 | def test_apprun_multiple_singularity(self): 77 | """List of commands specified""" 78 | cmds = ['a', 'b', 'c'] 79 | s = runscript(commands=cmds, _app='foo') 80 | self.assertEqual(str(s), '%apprun foo\n a\n b\n exec c') 81 | 82 | @docker 83 | def test_apprun_docker(self): 84 | """apprun not implemented in Docker""" 85 | cmds = ['a', 'b', 'c'] 86 | s = runscript(commands=cmds, _app='foo') 87 | self.assertEqual(str(s), 'ENTRYPOINT ["a"]') 88 | 89 | @singularity 90 | def test_multiple_noexec_singularity(self): 91 | """exec option""" 92 | cmds = ['a', 'b', 'c'] 93 | s = runscript(commands=cmds, _exec=False) 94 | self.assertEqual(str(s), '%runscript\n a\n b\n c') 95 | 96 | @docker 97 | def test_merge_docker(self): 98 | """merge primitives""" 99 | r = [] 100 | r.append(runscript(commands=['a', 'b'])) 101 | r.append(runscript(commands=['c'])) 102 | merged = r[0].merge(r) 103 | self.assertEqual(str(merged), 'ENTRYPOINT ["a"]') 104 | 105 | @singularity 106 | def test_merge_singularity(self): 107 | """merge primitives""" 108 | r = [] 109 | r.append(runscript(commands=['a', 'b'])) 110 | r.append(runscript(commands=['c'])) 111 | merged = r[0].merge(r) 112 | self.assertEqual(str(merged), '%runscript\n a\n b\n exec c') 113 | 114 | @bash 115 | def test_bash(self): 116 | """Single command specified""" 117 | cmd = ['z'] 118 | s = runscript(commands=cmd) 119 | self.assertEqual(str(s), '') 120 | -------------------------------------------------------------------------------- /test/test_sed.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the sed module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from hpccm.templates.sed import sed 26 | 27 | class Test_sed(unittest.TestCase): 28 | def setUp(self): 29 | """Disable logging output messages""" 30 | logging.disable(logging.ERROR) 31 | 32 | def test_basic(self): 33 | """Basic sed""" 34 | s = sed() 35 | self.assertEqual(s.sed_step(file='foo', 36 | patterns=[r's/a/A/g', 37 | r's/FOO = BAR/FOO = BAZ/g']), 38 | r'''sed -i -e s/a/A/g \ 39 | -e 's/FOO = BAR/FOO = BAZ/g' foo''') 40 | 41 | def test_nofile(self): 42 | """No file specified""" 43 | s = sed() 44 | self.assertEqual(s.sed_step(patterns=[r's/a/A/g']), '') 45 | 46 | def test_nopatterns(self): 47 | """No patterns specified""" 48 | s = sed() 49 | self.assertEqual(s.sed_step(file='foo'), '') 50 | -------------------------------------------------------------------------------- /test/test_sensei.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods, bad-continuation 16 | 17 | """Test cases for the sensei module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import centos, docker, ubuntu 26 | 27 | from hpccm.building_blocks.sensei import sensei 28 | from hpccm.toolchain import toolchain 29 | 30 | class Test_sensei(unittest.TestCase): 31 | def setUp(self): 32 | """Disable logging output messages""" 33 | logging.disable(logging.ERROR) 34 | 35 | @ubuntu 36 | @docker 37 | def test_defaults_ubuntu(self): 38 | """Default sensei building block""" 39 | s = sensei(libsim='/usr/local/visit', vtk='/usr/local/visit/third-party/vtk/6.1.0/linux-x86_64_gcc-5.4/lib/cmake/vtk-6.1') 40 | self.assertEqual(str(s), 41 | r'''# SENSEI version v2.1.1 42 | RUN apt-get update -y && \ 43 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 44 | ca-certificates \ 45 | git \ 46 | make && \ 47 | rm -rf /var/lib/apt/lists/* 48 | RUN mkdir -p /var/tmp && cd /var/tmp && git clone --depth=1 --branch v2.1.1 https://gitlab.kitware.com/sensei/sensei.git sensei && cd - && \ 49 | mkdir -p /var/tmp/sensei/build && cd /var/tmp/sensei/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/sensei -DENABLE_SENSEI=ON -DENABLE_LIBSIM=ON -DLIBSIM_DIR=/usr/local/visit -DENABLE_PARALLEL3D=OFF -DENABLE_OSCILLATORS=OFF -DVTK_DIR=/usr/local/visit/third-party/vtk/6.1.0/linux-x86_64_gcc-5.4/lib/cmake/vtk-6.1 /var/tmp/sensei && \ 50 | cmake --build /var/tmp/sensei/build --target all -- -j$(nproc) && \ 51 | cmake --build /var/tmp/sensei/build --target install -- -j$(nproc) && \ 52 | rm -rf /var/tmp/sensei''') 53 | 54 | @centos 55 | @docker 56 | def test_defaults_centos(self): 57 | """Default sensei building block""" 58 | s = sensei(catalyst='/usr/local/catalyst') 59 | self.assertEqual(str(s), 60 | r'''# SENSEI version v2.1.1 61 | RUN yum install -y \ 62 | ca-certificates \ 63 | git \ 64 | make && \ 65 | rm -rf /var/cache/yum/* 66 | RUN mkdir -p /var/tmp && cd /var/tmp && git clone --depth=1 --branch v2.1.1 https://gitlab.kitware.com/sensei/sensei.git sensei && cd - && \ 67 | mkdir -p /var/tmp/sensei/build && cd /var/tmp/sensei/build && cmake -DCMAKE_INSTALL_PREFIX=/usr/local/sensei -DENABLE_SENSEI=ON -DENABLE_CATALYST=ON -DParaView_DIR=/usr/local/catalyst -DENABLE_PARALLEL3D=OFF -DENABLE_OSCILLATORS=OFF /var/tmp/sensei && \ 68 | cmake --build /var/tmp/sensei/build --target all -- -j$(nproc) && \ 69 | cmake --build /var/tmp/sensei/build --target install -- -j$(nproc) && \ 70 | rm -rf /var/tmp/sensei''') 71 | 72 | @ubuntu 73 | @docker 74 | def test_runtime(self): 75 | """Runtime""" 76 | s = sensei() 77 | r = s.runtime() 78 | self.assertEqual(r, 79 | r'''# SENSEI 80 | COPY --from=0 /usr/local/sensei /usr/local/sensei''') 81 | -------------------------------------------------------------------------------- /test/test_slurm_pmi2.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods, bad-continuation 16 | 17 | """Test cases for the slurm_pmi2 module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import centos, docker, ubuntu, x86_64 26 | 27 | from hpccm.building_blocks.slurm_pmi2 import slurm_pmi2 28 | 29 | class Test_slurm_pmi2(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @x86_64 35 | @ubuntu 36 | @docker 37 | def test_defaults_ubuntu(self): 38 | """Default slurm_pmi2 building block""" 39 | p = slurm_pmi2() 40 | self.assertEqual(str(p), 41 | r'''# SLURM PMI2 version 21.08.8 42 | RUN apt-get update -y && \ 43 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 44 | bzip2 \ 45 | file \ 46 | make \ 47 | perl \ 48 | tar \ 49 | wget && \ 50 | rm -rf /var/lib/apt/lists/* 51 | RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://download.schedmd.com/slurm/slurm-21.08.8.tar.bz2 && \ 52 | mkdir -p /var/tmp && tar -x -f /var/tmp/slurm-21.08.8.tar.bz2 -C /var/tmp -j && \ 53 | cd /var/tmp/slurm-21.08.8 && ./configure --prefix=/usr/local/slurm-pmi2 && \ 54 | cd /var/tmp/slurm-21.08.8 && \ 55 | make -C contribs/pmi2 install && \ 56 | rm -rf /var/tmp/slurm-21.08.8 /var/tmp/slurm-21.08.8.tar.bz2''') 57 | 58 | @x86_64 59 | @ubuntu 60 | @docker 61 | def test_ldconfig(self): 62 | """ldconfig option""" 63 | p = slurm_pmi2(ldconfig=True, version='20.02.7') 64 | self.assertEqual(str(p), 65 | r'''# SLURM PMI2 version 20.02.7 66 | RUN apt-get update -y && \ 67 | DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ 68 | bzip2 \ 69 | file \ 70 | make \ 71 | perl \ 72 | tar \ 73 | wget && \ 74 | rm -rf /var/lib/apt/lists/* 75 | RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://download.schedmd.com/slurm/slurm-20.02.7.tar.bz2 && \ 76 | mkdir -p /var/tmp && tar -x -f /var/tmp/slurm-20.02.7.tar.bz2 -C /var/tmp -j && \ 77 | cd /var/tmp/slurm-20.02.7 && ./configure --prefix=/usr/local/slurm-pmi2 && \ 78 | cd /var/tmp/slurm-20.02.7 && \ 79 | make -C contribs/pmi2 install && \ 80 | echo "/usr/local/slurm-pmi2/lib" >> /etc/ld.so.conf.d/hpccm.conf && ldconfig && \ 81 | rm -rf /var/tmp/slurm-20.02.7 /var/tmp/slurm-20.02.7.tar.bz2''') 82 | 83 | @x86_64 84 | @ubuntu 85 | @docker 86 | def test_runtime(self): 87 | """Runtime""" 88 | p = slurm_pmi2() 89 | r = p.runtime() 90 | self.assertEqual(r, 91 | r'''# SLURM PMI2 92 | COPY --from=0 /usr/local/slurm-pmi2 /usr/local/slurm-pmi2''') 93 | -------------------------------------------------------------------------------- /test/test_tar.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the tar module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from hpccm.templates.tar import tar 26 | 27 | class Test_tar(unittest.TestCase): 28 | def setUp(self): 29 | """Disable logging output messages""" 30 | logging.disable(logging.ERROR) 31 | 32 | def test_missing_tarball(self): 33 | """Missing tarball option""" 34 | t = tar() 35 | self.assertEqual(t.untar_step(), '') 36 | 37 | def test_filetypes(self): 38 | t = tar() 39 | 40 | self.assertEqual(t.untar_step(tarball='foo.tar.bz2'), 41 | 'tar -x -f foo.tar.bz2 -j') 42 | 43 | self.assertEqual(t.untar_step(tarball='foo.tar.gz'), 44 | 'tar -x -f foo.tar.gz -z') 45 | 46 | self.assertEqual(t.untar_step(tarball='foo.tar.xz'), 47 | 'tar -x -f foo.tar.xz -J') 48 | 49 | self.assertEqual(t.untar_step(tarball='foo.txz'), 50 | 'tar -x -f foo.txz -J') 51 | 52 | self.assertEqual(t.untar_step(tarball='foo.tgz'), 53 | 'tar -x -f foo.tgz -z') 54 | 55 | self.assertEqual(t.untar_step(tarball='foo.tar'), 56 | 'tar -x -f foo.tar') 57 | 58 | self.assertEqual(t.untar_step(tarball='foo.unknown'), 59 | 'tar -x -f foo.unknown') 60 | 61 | def test_directory(self): 62 | """Directory specified""" 63 | t = tar() 64 | self.assertEqual(t.untar_step(tarball='foo.tgz', directory='bar'), 65 | 'mkdir -p bar && tar -x -f foo.tgz -C bar -z') 66 | 67 | def test_args(self): 68 | """Argument given""" 69 | t = tar() 70 | self.assertEqual(t.untar_step(tarball="foo.tar.gz", 71 | args=["--strip-components=1"]), 72 | 'tar -x -f foo.tar.gz -z --strip-components=1') 73 | -------------------------------------------------------------------------------- /test/test_toolchain.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the toolchain module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | from copy import copy, deepcopy 23 | import logging # pylint: disable=unused-import 24 | import unittest 25 | 26 | from hpccm.toolchain import toolchain 27 | 28 | class Test_toolchain(unittest.TestCase): 29 | def setUp(self): 30 | """Disable logging output messages""" 31 | logging.disable(logging.ERROR) 32 | 33 | def test_creation(self): 34 | """Toolchain creation""" 35 | t = toolchain(CC='gcc', CXX='g++', FC='gfortran') 36 | self.assertEqual(t.CC, 'gcc') 37 | self.assertEqual(t.CXX, 'g++') 38 | self.assertEqual(t.FC, 'gfortran') 39 | 40 | def test_modification(self): 41 | """Toolchain modification""" 42 | t = toolchain(CC='gcc', CXX='g++', FC='gfortran') 43 | t.CC = 'mygcc' 44 | self.assertEqual(t.CC, 'mygcc') 45 | self.assertEqual(t.CXX, 'g++') 46 | self.assertEqual(t.FC, 'gfortran') 47 | 48 | def test_copy(self): 49 | """Toolchain copies""" 50 | t = toolchain(CC='gcc', CXX='g++', FC='gfortran') 51 | r = t # ref 52 | c = copy(t) 53 | d = deepcopy(t) 54 | t.CC = 'mygcc' 55 | c.CC = 'cc' 56 | self.assertEqual(t.CC, 'mygcc') 57 | self.assertEqual(r.CC, 'mygcc') 58 | self.assertEqual(c.CC, 'cc') 59 | self.assertEqual(d.CC, 'gcc') 60 | 61 | def test_vars(self): 62 | """Toolchain dictionaries""" 63 | t = toolchain(CC='gcc', CXX='g++', FC='gfortran') 64 | v = vars(t) 65 | self.assertDictEqual(v, {'CC': 'gcc', 'CXX': 'g++', 'FC': 'gfortran'}) 66 | 67 | def test_unknown_keys(self): 68 | """Toolchain unknown keys""" 69 | t = toolchain(FOO='bar') 70 | with self.assertRaises(AttributeError): 71 | f = t.FOO 72 | -------------------------------------------------------------------------------- /test/test_user.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods, bad-continuation 16 | 17 | """Test cases for the user module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import bash, docker, invalid_ctype, singularity 26 | 27 | from hpccm.primitives.user import user 28 | 29 | class Test_user(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @docker 35 | def test_empty(self): 36 | """No user specified""" 37 | u = user() 38 | self.assertEqual(str(u), '') 39 | 40 | @invalid_ctype 41 | def test_invalid_ctype(self): 42 | """Invalid container type specified""" 43 | u = user(user='root') 44 | with self.assertRaises(RuntimeError): 45 | str(u) 46 | 47 | @docker 48 | def test_docker(self): 49 | """User specified""" 50 | u = user(user='root') 51 | self.assertEqual(str(u), 'USER root') 52 | 53 | @singularity 54 | def test_singularity(self): 55 | """User specified""" 56 | u = user(user='root') 57 | self.assertEqual(str(u), '') 58 | 59 | @bash 60 | def test_bash(self): 61 | """User specified""" 62 | u = user(user='root') 63 | self.assertEqual(str(u), '') 64 | -------------------------------------------------------------------------------- /test/test_wget.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the wget module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from hpccm.templates.wget import wget 26 | 27 | class Test_wget(unittest.TestCase): 28 | def setUp(self): 29 | """Disable logging output messages""" 30 | logging.disable(logging.ERROR) 31 | 32 | def test_missing_url(self): 33 | """Missing url option""" 34 | w = wget() 35 | self.assertEqual(w.download_step(), '') 36 | 37 | def test_basic(self): 38 | """Basic wget""" 39 | w = wget() 40 | self.assertEqual(w.download_step(url='http://mysite.com/foo.tgz'), 41 | 'mkdir -p /tmp && wget -q -nc --no-check-certificate -P /tmp http://mysite.com/foo.tgz') 42 | 43 | def test_referer(self): 44 | """wget with referer""" 45 | w = wget() 46 | self.assertEqual(w.download_step(url='http://mysite.com/foo.tgz', 47 | referer='http://mysite.com/foo.html'), 48 | 'mkdir -p /tmp && wget -q -nc --no-check-certificate --referer http://mysite.com/foo.html -P /tmp http://mysite.com/foo.tgz') 49 | 50 | def test_directory(self): 51 | """wget with non-default output directory""" 52 | w = wget() 53 | self.assertEqual(w.download_step(url='http://mysite.com/foo.tgz', 54 | directory='/scratch'), 55 | 'mkdir -p /scratch && wget -q -nc --no-check-certificate -P /scratch http://mysite.com/foo.tgz') 56 | 57 | def test_outfile(self): 58 | """wget with non-default output file""" 59 | w = wget() 60 | self.assertEqual(w.download_step(url='http://mysite.com/foo.tgz', 61 | outfile='bar.tgz'), 62 | 'mkdir -p /tmp && wget -q -nc --no-check-certificate -O bar.tgz -P /tmp http://mysite.com/foo.tgz') 63 | 64 | def test_opts(self): 65 | """wget with non-default command line options""" 66 | w = wget(opts=['-fast']) 67 | self.assertEqual(w.download_step(url='http://mysite.com/foo.tgz'), 68 | 'mkdir -p /tmp && wget -fast -P /tmp http://mysite.com/foo.tgz') 69 | -------------------------------------------------------------------------------- /test/test_workdir.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the workdir module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from helpers import bash, docker, invalid_ctype, singularity 26 | 27 | from hpccm.primitives.workdir import workdir 28 | 29 | class Test_workdir(unittest.TestCase): 30 | def setUp(self): 31 | """Disable logging output messages""" 32 | logging.disable(logging.ERROR) 33 | 34 | @docker 35 | def test_empty(self): 36 | """No workdir specified""" 37 | w = workdir() 38 | self.assertEqual(str(w), '') 39 | 40 | @invalid_ctype 41 | def test_invalid_ctype(self): 42 | """Invalid container type specified""" 43 | w = workdir(directory='foo') 44 | with self.assertRaises(RuntimeError): 45 | str(w) 46 | 47 | @docker 48 | def test_dir_docker(self): 49 | """Working directory specified""" 50 | w = workdir(directory='foo') 51 | self.assertEqual(str(w), 'WORKDIR foo') 52 | 53 | @singularity 54 | def test_dir_singularity(self): 55 | """Working directory specified""" 56 | w = workdir(directory='foo') 57 | self.assertEqual(str(w), '%post\n cd /\n mkdir -p foo\n cd foo') 58 | 59 | @bash 60 | def test_dir_bash(self): 61 | """Working directory specified""" 62 | w = workdir(directory='foo') 63 | self.assertEqual(str(w), '') 64 | -------------------------------------------------------------------------------- /test/test_zipfile.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # pylint: disable=invalid-name, too-few-public-methods 16 | 17 | """Test cases for the zip module""" 18 | 19 | from __future__ import unicode_literals 20 | from __future__ import print_function 21 | 22 | import logging # pylint: disable=unused-import 23 | import unittest 24 | 25 | from hpccm.templates.zipfile import zipfile 26 | 27 | class Test_zipfile(unittest.TestCase): 28 | def setUp(self): 29 | """Disable logging output messages""" 30 | logging.disable(logging.ERROR) 31 | 32 | def test_missing_zipfile(self): 33 | """Missing zipfile option""" 34 | z = zipfile() 35 | self.assertEqual(z.unzip_step(), '') 36 | 37 | def test_filetypes(self): 38 | z = zipfile() 39 | self.assertEqual(z.unzip_step('foo.zip'), 'unzip foo.zip') 40 | 41 | def test_directory(self): 42 | """Directory specified""" 43 | z = zipfile() 44 | self.assertEqual(z.unzip_step('foo.zip', 'bar'), 45 | 'mkdir -p bar && unzip -d bar foo.zip') 46 | --------------------------------------------------------------------------------