├── .cmake-format.py ├── .gitignore ├── CMakeLists.txt ├── README.md ├── amrex ├── CMakeLists.txt └── sundials │ ├── CMakeLists.txt │ ├── README.md │ ├── amrex_sundials_advection_diffusion.cpp │ └── amrex_sundials_advection_diffusion.h ├── cmake ├── FindAMReX.cmake ├── FindBLASPP.cmake ├── FindHYPRE.cmake ├── FindLAPACKPP.cmake ├── FindMAGMA.cmake ├── FindMETIS.cmake ├── FindMFEM.cmake ├── FindPETSc.cmake ├── FindPLASMA.cmake ├── FindSLATE.cmake ├── FindSUNDIALS.cmake ├── FindSUPERLUDIST.cmake └── XsdkAddTest.cmake ├── dealii ├── CMakeLists.txt ├── README.md ├── petsc_trilinos │ ├── CMakeLists.txt │ ├── README.md │ └── petsc_trilinos.cc ├── precice │ ├── CMakeLists.txt │ ├── README.md │ ├── boundary_condition.cc │ ├── clean │ ├── laplace_problem.cc │ ├── precice-config.xml │ └── runtargets.sh └── sundials │ ├── CMakeLists.txt │ ├── README.md │ └── sundials.cc ├── heffte ├── CMakeLists.txt ├── README.md └── heffte_example_gpu.cpp ├── hypre ├── CMakeLists.txt ├── COPYRIGHT ├── LICENSE-APACHE ├── LICENSE-MIT ├── NOTICE ├── README.md ├── ij_laplacian.c ├── vis.c └── vis │ └── glvis-ij-laplacian.sh ├── libensemble ├── LICENSE ├── README.md ├── persistent_aposmm.py ├── persistent_aposmm_alloc.py └── test_persistent_aposmm_tao.py ├── mfem ├── CMakeLists.txt ├── README.md ├── data │ ├── README.md │ ├── amr-quad.mesh │ ├── beam-hex.mesh │ ├── disc-nurbs.mesh │ ├── periodic-cube.mesh │ ├── periodic-hexagon.mesh │ ├── periodic-segment.mesh │ ├── periodic-square.mesh │ ├── pumi │ │ ├── geom │ │ │ ├── Kova.dmg │ │ │ └── Kova.x_t │ │ └── parallel │ │ │ └── KovaLinear │ │ │ ├── Kova-linear-10k_2p0.smb │ │ │ └── Kova-linear-10k_2p1.smb │ └── star.mesh ├── ginkgo │ ├── CMakeLists.txt │ ├── GINKGO-LICENSE │ ├── README.md │ └── mfem_ex22_gko.cpp ├── hiop │ ├── CMakeLists.txt │ ├── README.md │ └── adv.cpp ├── hypre-superlu │ ├── CMakeLists.txt │ ├── README.md │ └── convdiff.cpp ├── hypre │ ├── CMakeLists.txt │ ├── README.md │ └── magnetic-diffusion.cpp ├── petsc │ ├── CMakeLists.txt │ ├── README.md │ └── obstacle.cpp ├── pumi │ ├── CMakeLists.txt │ ├── README.md │ └── adapt.cpp ├── strumpack │ ├── CMakeLists.txt │ ├── README.md │ └── diffusion-eigen.cpp └── sundials │ ├── CMakeLists.txt │ ├── README.md │ ├── advection.cpp │ └── transient-heat.cpp ├── petsc ├── CMakeLists.txt ├── LICENSE ├── README.md ├── ex19.c ├── makefile └── output │ ├── ex19_1.testout │ ├── ex19_cuda_1.out │ ├── ex19_hip.out │ ├── ex19_hypre.out │ └── ex19_superlu.out ├── plasma ├── CMakeLists.txt ├── README.md └── ex1solve.cpp ├── strumpack ├── CMakeLists.txt ├── README.md └── sparse.cpp ├── sundials ├── CMakeLists.txt ├── LICENSE ├── NOTICE ├── README.md ├── ark_brusselator1D_FEM_sludist.cpp ├── cv_bruss_batched_magma.cpp └── cv_petsc_ex7.c ├── tasmanian ├── CMakeLists.txt ├── README.md └── example_unstructured_grid.cpp └── trilinos ├── CMakeLists.txt ├── LICENSE ├── README.md └── SimpleSolve_WithParameters.cpp /.cmake-format.py: -------------------------------------------------------------------------------- 1 | # -*- Python -*- 2 | 3 | with section("format"): 4 | 5 | # How wide to allow formatted cmake files 6 | line_width = 100 7 | 8 | # How many spaces to tab for indent 9 | tab_size = 4 10 | 11 | # If true, separate flow control names from their parentheses with a space 12 | separate_ctrl_name_with_space = False 13 | 14 | # If true, separate function names from parentheses with a space 15 | separate_fn_name_with_space = False 16 | 17 | # If a statement is wrapped to more than one line, than dangle the closing 18 | # parenthesis on its own line. 19 | dangle_parens = True 20 | 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | /builddir 3 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.21) 2 | project( 3 | xsdk-examples 4 | DESCRIPTION "xSDK Examples" 5 | LANGUAGES CXX C 6 | ) 7 | 8 | set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) 9 | include(CTest) 10 | include(FindPackageHandleStandardArgs) 11 | include(XsdkAddTest) 12 | 13 | # build options 14 | option(ENABLE_CUDA "Enable CUDA" OFF) 15 | option(ENABLE_HIP "Enable HIP" OFF) 16 | 17 | option(ENABLE_AMREX "Enable AMReX" ON) 18 | set(AMREX_DIR 19 | "${AMREX_DIR}" 20 | CACHE PATH "Path to AMReX installation directory" 21 | ) 22 | 23 | option(ENABLE_DEAL_II "Enable deal.II" ON) 24 | set(DEAL_II_DIR 25 | "${DEAL_II_DIR}" 26 | CACHE PATH "Path to deal.II installation directory" 27 | ) 28 | 29 | option(ENABLE_GINKGO "Enable Ginkgo" ON) 30 | set(Ginkgo_DIR 31 | "${Ginkgo_DIR}" 32 | CACHE PATH "Path to Ginkgo installation directory" 33 | ) 34 | 35 | option(ENABLE_HEFFTE "Enable heFFTe" OFF) 36 | set(HEFFTE_DIR 37 | "${HEFFTE_DIR}" 38 | CACHE PATH "Path to the heFFTe installation directory" 39 | ) 40 | 41 | option(ENABLE_HIOP "Enable HiOp" ON) 42 | set(HIOP_DIR 43 | "${HIOP_DIR}" 44 | CACHE PATH "Path to HiOp installation directory" 45 | ) 46 | 47 | option(ENABLE_HYPRE "Enable hypre" ON) 48 | set(HYPRE_DIR 49 | "${HYPRE_DIR}" 50 | CACHE PATH "Path to hypre installation directory" 51 | ) 52 | 53 | option(ENABLE_MAGMA "Enable MAGMA" OFF) 54 | set(MAGMA_DIR 55 | "${MAGMA_DIR}" 56 | CACHE PATH "Path to MAGMA installation directory" 57 | ) 58 | 59 | option(ENABLE_MFEM "Enable MFEM" ON) 60 | set(MFEM_DIR 61 | "${MFEM_DIR}" 62 | CACHE PATH "Path to MFEM installation directory" 63 | ) 64 | 65 | option(ENABLE_PETSC "Enable PETSc" ON) 66 | set(PETSc_DIR 67 | "${PETSc_DIR}" 68 | CACHE PATH "Path to PETSc installation directory" 69 | ) 70 | 71 | option(ENABLE_PLASMA "Enable PLASMA" ON) 72 | set(PLASMA_DIR 73 | "${PLASMA_DIR}" 74 | CACHE PATH "Path to PLASMA installation directory" 75 | ) 76 | 77 | option(ENABLE_PRECICE "Enable preCICE" ON) 78 | set(PRECICE_DIR 79 | "${PRECICE_DIR}" 80 | CACHE PATH "Path to preCICE installation directory" 81 | ) 82 | 83 | option(ENABLE_PUMI "Enable PUMI" ON) 84 | set(PUMI_DIR 85 | "${PUMI_DIR}" 86 | CACHE PATH "Path to PUMI installation directory" 87 | ) 88 | 89 | option(ENABLE_SUNDIALS "Enable SUNDIALS" ON) 90 | set(SUNDIALS_DIR 91 | "${SUNDIALS_DIR}" 92 | CACHE PATH "Path to SUNDIALS installation directory" 93 | ) 94 | 95 | option(ENABLE_SUPERLU "Enable SuperLU" ON) 96 | set(SUPERLUDIST_DIR 97 | "${SUPERLUDIST_DIR}" 98 | CACHE PATH "Path to SuperLU_DIST installation directory" 99 | ) 100 | 101 | option(ENABLE_STRUMPACK "Enable STRUMPACK" OFF) 102 | set(STRUMPACK_DIR 103 | "${STRUMPACK_DIR}" 104 | CACHE PATH "Path to STRUMPACK installation directory" 105 | ) 106 | 107 | option(ENABLE_TASMANIAN "Enable Tasmanian" OFF) 108 | set(TASMANIAN_DIR 109 | "${TASMANIAN_DIR}" 110 | CACHE PATH "Path to the Tasmanian installation directory" 111 | ) 112 | 113 | option(ENABLE_TRILINOS "Enable TRILINOS" OFF) 114 | set(TRILINOS_DIR 115 | "${Trilinos_DIR}" 116 | CACHE PATH "Path to Trilinos installation directory" 117 | ) 118 | 119 | set(METIS_DIR 120 | "${METIS_DIR}" 121 | CACHE PATH "Path to Metis installation directory" 122 | ) 123 | 124 | # Don't skip the full RPATH for the build tree 125 | set(CMAKE_SKIP_BUILD_RPATH FALSE) 126 | 127 | # When building, don't use the install RPATH already (but later on when 128 | # installing) 129 | set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) 130 | set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_FULL_LIBDIR}") 131 | set(CMAKE_INSTALL_NAME_DIR "${CMAKE_INSTALL_FULL_LIBDIR}") 132 | 133 | # Add the automatically determined parts of the RPATH which point to directories 134 | # outside the build tree to the install RPATH 135 | set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) 136 | 137 | # The RPATH to be used when installing, but only if it's not a system directory 138 | list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_FULL_LIBDIR}" isSystemDir) 139 | if("${isSystemDir}" STREQUAL "-1") 140 | set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_FULL_LIBDIR}") 141 | endif() 142 | 143 | # check for MPI 144 | find_package(MPI REQUIRED) 145 | 146 | # check for OpenMP 147 | find_package(OpenMP) 148 | 149 | # compiler options 150 | if(NOT DEFINED CMAKE_CXX_STANDARD) 151 | set(CMAKE_CXX_STANDARD 14) 152 | set(CMAKE_CXX_STANDARD_REQUIRED TRUE) 153 | set(CMAKE_CXX_EXTENSIONS OFF) 154 | endif() 155 | 156 | # setup CUDA 157 | if(ENABLE_CUDA) 158 | enable_language(CUDA) 159 | set(CMAKE_CUDA_HOST_COMPILER ${CMAKE_CXX_COMPILER}) 160 | find_package(CUDAToolkit REQUIRED) 161 | endif() 162 | 163 | if(ENABLE_HIP) 164 | enable_language(HIP) 165 | find_package(hip REQUIRED) 166 | find_package(hipsparse REQUIRED) 167 | find_package(hiprand REQUIRED) 168 | find_package(rocrand REQUIRED) 169 | find_package(rocprim REQUIRED) 170 | find_package(rocsparse REQUIRED) 171 | find_package(rocsolver REQUIRED) 172 | endif() 173 | 174 | # check for AMReX 175 | if(ENABLE_AMREX) 176 | find_package(AMReX REQUIRED) 177 | endif() 178 | 179 | # check for deal.II 180 | if(ENABLE_DEAL_II) 181 | # deal.II resets CMAKE_CUDA_ARCHITECTURES in its CMake config file, so we 182 | # need to save it here and restore it below. 183 | set(_cuda_archs "${CMAKE_CUDA_ARCHITECTURES}") 184 | find_package(deal.II REQUIRED 185 | HINTS ${deal.II_DIR} ${DEAL_II_DIR} $ENV{DEAL_II_DIR} 186 | ) 187 | set(CMAKE_CUDA_ARCHITECTURES "${_cuda_archs}") 188 | endif() 189 | 190 | if(ENABLE_HEFFTE) 191 | find_package(Heffte 2.2 PATHS "${HEFFTE_DIR}" REQUIRED) 192 | endif() 193 | 194 | # check for HiOp 195 | if(ENABLE_HIOP) 196 | find_package(HiOp REQUIRED) 197 | endif() 198 | 199 | # check for hypre 200 | if(ENABLE_HYPRE) 201 | find_package(HYPRE REQUIRED) 202 | endif() 203 | 204 | # check for MAGMA 205 | if(ENABLE_MAGMA) 206 | find_package(MAGMA REQUIRED) 207 | endif() 208 | 209 | # check for MFEM 210 | if(ENABLE_MFEM) 211 | find_package(ZLIB REQUIRED) 212 | find_package(MFEM REQUIRED) 213 | if(ENABLE_GINKGO) 214 | find_package(Ginkgo REQUIRED) 215 | endif() 216 | endif() 217 | 218 | # check for PETSC 219 | if(ENABLE_PETSC) 220 | find_package(PETSc REQUIRED) 221 | endif() 222 | 223 | # check for PLASMA 224 | if(ENABLE_PLASMA) 225 | find_package(PLASMA REQUIRED) 226 | find_package(BLASPP REQUIRED) 227 | find_package(LAPACKPP REQUIRED) 228 | find_package(SLATE REQUIRED) 229 | endif() 230 | 231 | # check for preCICE 232 | if(ENABLE_PRECICE) 233 | find_package(precice 2 REQUIRED 234 | HINTS ${PRECICE_DIR} $ENV{PRECICE_DIR} 235 | ) 236 | endif() 237 | 238 | # check for PUMI 239 | if(ENABLE_PUMI) 240 | # not really needed: MFEM links with PUMI if it is built with it 241 | # find_package(SCOREC REQUIRED) 242 | endif() 243 | 244 | # check for SUNDIALS 245 | if(ENABLE_SUNDIALS) 246 | find_package(SUNDIALS REQUIRED) 247 | endif() 248 | 249 | # check for SuperLU DIST 250 | if(ENABLE_SUPERLU) 251 | find_package(SUPERLUDIST REQUIRED) 252 | endif() 253 | 254 | # check for STRUMPACK 255 | if(ENABLE_STRUMPACK) 256 | find_package(STRUMPACK REQUIRED) 257 | endif() 258 | 259 | if(ENABLE_TASMANIAN) 260 | find_package(Tasmanian 7.5 PATHS "${TASMANIAN_DIR}" REQUIRED) 261 | endif() 262 | 263 | # check for math 264 | find_library(MATH_LIBRARY NAMES m) 265 | 266 | # check for metis 267 | find_package(METIS) 268 | 269 | # example subdirectories 270 | if(ENABLE_AMREX) 271 | add_subdirectory(amrex) 272 | endif() 273 | if(ENABLE_DEAL_II) 274 | add_subdirectory(dealii) 275 | endif() 276 | if(ENABLE_HEFFTE) 277 | add_subdirectory(heffte) 278 | endif() 279 | if(ENABLE_HYPRE) 280 | add_subdirectory(hypre) 281 | endif() 282 | if(ENABLE_MFEM) 283 | add_subdirectory(mfem) 284 | endif() 285 | if(ENABLE_PETSC) 286 | add_subdirectory(petsc) 287 | endif() 288 | if(ENABLE_PLASMA) 289 | add_subdirectory(plasma) 290 | endif() 291 | if(ENABLE_STRUMPACK) 292 | add_subdirectory(strumpack) 293 | endif() 294 | if(ENABLE_SUNDIALS) 295 | add_subdirectory(sundials) 296 | endif() 297 | if(ENABLE_TASMANIAN) 298 | add_subdirectory(tasmanian) 299 | endif() 300 | if(ENABLE_TRILINOS) 301 | add_subdirectory(trilinos) 302 | endif() 303 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # xSDK Examples v0.4.0 2 | 3 | The example codes provided here demonstrate the use of of various xSDK libraries in tandem to solve problems of 4 | interest. Each of the library folders has one or more examples codes that are built of that library 5 | and utilize code integrations with other xSDK libraries. Running these example codes and 6 | examining the output is a good way to better understand how these libraries can work together. The 7 | code samples are a good place to start for new projects. More details about the examples can be found 8 | in the README.md files in the library subfolders. For more information on the xSDK see . 9 | 10 | ## Example Summary 11 | 12 | These examples were tested and verified against xsdk@0.8.0. 13 | 14 | | Example | Libraries | Description | GPUs | 15 | |:------------------------------------------------------|:-------------------------|:--------------------------------------------------|:---------------| 16 | | amrex/sundials/amrex_sundials_advection_diffusion.cpp | AMReX+SUNDIALS | 2D Advection-diffusion problem | ![cuda] | 17 | | dealii/petsc_trilinos/petsc_trilinos.cpp | deal.II+PETSc/Trilinos | Poisson problem using MPI and AMG preconditioners | | 18 | | dealii/precice/laplace_problem.cc | deal.II+preCICE | Coupling of Laplace problem with external b.c. | | 19 | | dealii/sundials/sundials.cpp | deal.II+SUNDIALS | Nonlinear, minimal surface problem | | 20 | | heffte/heffte_example_gpu.cpp | heFFTe+MAGMA | 3D FFT transform using the GPU | ![cuda] ![hip] | 21 | | hypre/ij_laplacian.c | HYPRE+SuperLU_Dist | 2D Laplacian problem | | 22 | | libensemble/test_persistent_aposmm_tao.py | libEnsemble+PETSc | 2D constrained optimization problem | | 23 | | mfem/ginkgo/mfem_ex22_gko.cpp | MFEM+Ginkgo | 3D damped harmonic oscillator with Ginkgo solver | ![cuda] ![hip] | 24 | | mfem/hiop/adv.cpp | MFEM+HiOp | Time-dependent advection | | 25 | | mfem/hypre/magnetic-diffusion.cpp | MFEM+HYPRE | Steady state magnetic diffusion problem | ![cuda] ![hip] | 26 | | mfem/hypre-superlu/convdiff.cpp | MFEM+HYPRE+SuperLU_Dist | 2D steady state convective diffusion | | 27 | | mfem/petsc/obstacle.cpp | MFEM+PETSc | Membrane obstacle problem (min energy functional) | | 28 | | mfem/pumi/adapt.cpp | MFEM+PUMI | Adaptive mesh refinement for a diffusion problem | | 29 | | mfem/strumpack/diffusion-eigen.cpp | MFEM+STRUMPACK+HYPRE | Diffusion eigenvalue problem | | 30 | | mfem/sundials/transient-heat.cpp | MFEM+SUNDIALS | 2D transient nonlinear heat conduction | | 31 | | mfem/sundials/advection.cpp | MFEM+SUNDIALS | 2D time-dependent advection | ![cuda] | 32 | | petsc/ex19.c | PETSc | 2D nonlinear driven cavity problem | ![cuda] ![hip] | 33 | | petsc/ex19.c | PETSc+HYPRE | 2D nonlinear driven cavity problem | ![cuda] | 34 | | petsc/ex19.c | PETSc+SuperLU_Dist | 2D nonlinear driven cavity problem | | 35 | | plasma/ex1solve.c | PLASMA+SLATE+BLASPP | Linear system direct solution | ![cuda] | 36 | | sundials/ark_brusselator1D_FEM_sludist.cpp | SUNDIALS+SuperLU_Dist | 1D nonlinear time-dependent PDE solution | | 37 | | sundials/cv_petsc_ex7.c | SUNDIALS+PETSc | 2D nonlinear time-dependent PDE solution | | 38 | | sundials/cv_bruss_batched_magma.cpp | SUNDIALS+MAGMA | Batch of 0D chemical kinetics ODEs | ![cuda] ![hip] | 39 | | tasmanian/example_unstructured_grid.cpp | Tasmanian+MAGMA | Constructs a sparse grid model from random data | ![cuda] ![hip] | 40 | | trilinos/SimpleSolve_WithParameters.cpp | Trilinos+SuperLU_Dist | Small linear system direct solution | | 41 | | strumpack/sparse.cpp | STRUMPACK+ButterflyPACK | 3D Poisson problem with STRUMPACK preconditioner | | 42 | 43 | 44 | ## Installing the Examples 45 | 46 | The examples can be installed along with the xSDK utilizing the Spack package. 47 | 48 | ``` 49 | spack install xsdk-examples 50 | ``` 51 | 52 | To install with CUDA support, 53 | 54 | ``` 55 | spack install xsdk-examples+cuda cuda_arch= 56 | ``` 57 | 58 | Since `xsdk-examples` depends on the `xsdk` Spack package, Spack will also install `xsdk`. In some cases, it may be easier to install the `xsdk` package (separately) following https://xsdk.info/download/ prior to the `xsdk-examples` package. 59 | 60 | Alternatively the examples can be built and installed with CMake directly: 61 | 62 | ``` 63 | git clone https://github.com/xsdk-project/xsdk-examples 64 | cmake -DCMAKE_PREFIX_PATH=/path/to/libraries -DENABLE_CUDA= -DENABLE_HIP= -S xsdk-examples/ -B xsdk-examples/builddir 65 | cd xsdk-examples/builddir 66 | make 67 | make install 68 | ``` 69 | 70 | Note, that to build with HIP support CMake must be used directly. 71 | 72 | ## Running and Testing 73 | 74 | xsdk-examples is setup to use `ctest`. Each example in the repository is tested with at least a set of default options. If CMake is used to build xsdk-examples, the tests can be run from the build directory (`builddir` above): 75 | ``` 76 | ctest . 77 | ``` 78 | or 79 | ``` 80 | make test 81 | ``` 82 | Details on how to run each example code manually (and with different options) can be found in each example folder's README.md file. 83 | 84 | 85 | [cuda]: https://img.shields.io/badge/-cuda-brightgreen?style=flat "CUDA" 86 | [hip]: https://img.shields.io/badge/-hip-red?style=flat "HIP" 87 | -------------------------------------------------------------------------------- /amrex/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # AMReX+SUNDIALS requires amrex@22.04: and sundials@6.2.0: 2 | if(ENABLE_AMREX AND ENABLE_SUNDIALS) 3 | if(("${AMReX_RELEASE_NUMBER}" VERSION_GREATER_EQUAL "220400") AND ("${SUNDIALS_VERSION}" VERSION_GREATER_EQUAL 6.2.0)) 4 | add_subdirectory(sundials) 5 | else() 6 | message(STATUS "SKIPPED AMReX+SUNDIALS example because AMReX version is ${AMReX_RELEASE_NUMBER} and SUNDIALS version is ${SUNDIALS_VERSION} (need 22.04: and 6.2.0:)") 7 | endif() 8 | endif() 9 | -------------------------------------------------------------------------------- /amrex/sundials/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if(ENABLE_CUDA) 2 | set(CMAKE_CUDA_HOST_LINK_LAUNCHER ${CMAKE_CXX_COMPILER}) 3 | set_source_files_properties(amrex_sundials_advection_diffusion.cpp PROPERTIES LANGUAGE CUDA) 4 | endif() 5 | # Due to some linking issues when HIP is enabled, we skip this example in that 6 | # case; for details, see the following GitHub issue and PR: 7 | # https://github.com/xsdk-project/xsdk-examples/issues/50 8 | # https://github.com/xsdk-project/xsdk-examples/pull/52 9 | if (ENABLE_HIP) 10 | return() 11 | # set_source_files_properties(amrex_sundials_advection_diffusion.cpp PROPERTIES LANGUAGE HIP) 12 | endif() 13 | 14 | add_executable(amrex_sundials_advection_diffusion amrex_sundials_advection_diffusion.cpp) 15 | 16 | target_link_libraries(amrex_sundials_advection_diffusion PRIVATE XSDK::AMReX XSDK::SUNDIALS MPI::MPI_CXX) 17 | if(ENABLE_CUDA) 18 | set_target_properties(amrex_sundials_advection_diffusion 19 | PROPERTIES CUDA_SEPARABLE_COMPILATION ON 20 | ) # This adds -dc 21 | endif() 22 | 23 | xsdk_add_test( 24 | NAME AMREX-amrex_sundials_advection_diffusion 25 | MPI_NPROCS 1 26 | COMMAND 27 | $ n_cell=64 tfinal=10 28 | ) 29 | 30 | install(TARGETS amrex_sundials_advection_diffusion RUNTIME DESTINATION bin) 31 | -------------------------------------------------------------------------------- /amrex/sundials/README.md: -------------------------------------------------------------------------------- 1 | # AMReX + SUNDIALS examples 2 | 3 | Example codes demonstrating the use of [AMReX](https://amrex-codes.github.io/) 4 | and [SUNDIALS](https://computing.llnl.gov/projects/sundials). 5 | 6 | ## Advection-Diffusion Example 7 | 8 | This is an example of a scalar-valued advection-diffusion problem for chemical 9 | transport. The governing equation is: 10 | 11 | $$ \frac{\partial u}{\partial t} + \vec{a} \cdot \nabla u - \nabla \cdot ( D \nabla u ) = 0 $$ 12 | 13 | where $u = u(t,x,y)$ is the chemical concentration, $\vec{a}$ is the advection 14 | vector, and $D$ is a diagonal matrix of diffusion coefficients. The problem is 15 | solved on the square domain $[-1, 1]^2$ with periodic boundary conditions and 16 | the initial condition is 17 | 18 | $$ u(0,x,y) = u_0(x,y) = \frac{10}{\sqrt{2\pi}} e^{-50 (x^2 + y^2)} $$ 19 | 20 | ## Problem Options 21 | 22 | The problem inputs are listed below and may be specified on the command line 23 | e.g., `./amrex_sundials_advection_diffusion help=1` or by supplying an input 24 | file of values e.g., `./amrex_sundials_advection_diffusion inputs` where 25 | `inputs` is a text file with `option = value` lines. 26 | 27 | | Option | Type | Description | Default | 28 | |----------------------|--------|----------------------------------------------------|----------| 29 | | `help` | `int` | print input options and exit (1) | 0 | 30 | | `n_cell` | `int` | number of cells on each side of the square domain | 128 | 31 | | `max_grid_size` | `int` | max size of boxes in box array | 64 | 32 | | `plot_int` | `int` | enable (1) or disable (0) plots | 0 | 33 | | `arkode_order` | `int` | ARKStep method order | 4 | 34 | | `nls_max_iter` | `int` | maximum number of nonlinear iterations | 3 | 35 | | `ls_max_iter` | `int` | maximum number of linear iterations | 5 | 36 | | `rhs_adv` | `int` | advection: implicit (0) or explicit (1) | 1 | 37 | | `rtol` | `Real` | relative tolerance | 1e-4 | 38 | | `atol` | `Real` | absolute tolerance | 1e-9 | 39 | | `fixed_dt` | `Real` | use a fixed time step size (if `fixed_dt` > 0.0) | -1.0 | 40 | | `tfinal` | `Real` | final integration time | 1e4 | 41 | | `dtout` | `Real` | output frequency | `tfinal` | 42 | | `max_steps` | `int` | maximum number of steps between outputs | 10000 | 43 | | `advCoeffx` | `Real` | advection speed in the x-direction | 5.0e-4 | 44 | | `advCoeffy` | `Real` | advection speed in the y-direction | 2.5e-4 | 45 | | `diffCoeffx` | `Real` | diffusion coefficient in the x-direction | 1.0e-6 | 46 | | `diffCoeffy` | `Real` | diffusion coefficient in the y-direction | 1.0e-6 | 47 | | `use_preconditioner` | `int` | use preconditioning (1) or not (0) | 0 | 48 | 49 | If preconditioning is enabled, then additional options may be set (see AMReX 50 | documentation of the `MLMG` solver for descriptions): 51 | 52 | | Option | Type | Default | 53 | |-----------------------------|--------|---------| 54 | | `mlmg.agglomeration` | `int` | 1 | 55 | | `mlmg.consolidation` | `int` | 1 | 56 | | `mlmg.max_coarsening_level` | `int` | 1000 | 57 | | `mlmg.linop_maxorder` | `int` | 2 | 58 | | `mlmg.max_iter` | `int` | 1000 | 59 | | `mlmg.max_fmg_iter` | `int` | 1000 | 60 | | `mlmg.mg_fixed_iter` | `int` | 0 | 61 | | `mlmg.verbose` | `int` | 0 | 62 | | `mlmg.bottom_verbose` | `int` | 0 | 63 | | `mlmg.use_hypre` | `int` | 1 | 64 | | `mlmg.hypre_interface` | `int` | 3 | 65 | | `mlmg.use_petsc` | `int` | 0 | 66 | | `mlmg.tol_rel` | `Real` | 1.0e-6 | 67 | | `mlmg.tol_abs` | `Real` | 1.0e-6 | 68 | -------------------------------------------------------------------------------- /amrex/sundials/amrex_sundials_advection_diffusion.h: -------------------------------------------------------------------------------- 1 | /* ----------------------------------------------------------------------------- 2 | * AMReX + SUNDIALS xSDK 2D Advection-Diffusion example code 3 | * 4 | * Based on Hands-on Lessons with SUNDIALS + AMReX from the Argonne Training 5 | * Program in Extreme-Scale Computing (ATPESC) written by (alphabetical): 6 | * David Gardner (gardner48@llnl.gov) 7 | * John Loffeld (loffeld1@llnl.gov) 8 | * Daniel Reynolds (reynolds@smu.edu) 9 | * Donald Willcox (dewillcox@lbl.gov) 10 | * ---------------------------------------------------------------------------*/ 11 | 12 | #ifndef ADVECTION_DIFFUSION_H 13 | #define ADVECTION_DIFFUSION_H 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | // ------------------------ 28 | // User defined structures 29 | // ------------------------ 30 | 31 | // Structure for problem options 32 | struct ProblemOpt 33 | { 34 | int help = 0; // Print help message and exit (1) 35 | int plot_int = -1; // Enable (>0) or disable (<0) output files 36 | int arkode_order = 4; // ARKODE method order 37 | amrex::Real rtol = 1.0e-4; // Relative tolerance 38 | amrex::Real atol = 1.0e-9; // Absolute tolerance 39 | amrex::Real fixed_dt = -1.0; // Fixed time step size (-1 = adaptive) 40 | amrex::Real tfinal = 1.0e4; // Final integration time 41 | amrex::Real dtout = 1.0e4; // Output frequency 42 | int max_steps = 10000; // Max steps between outputs 43 | int nls_max_iter = 3; // Max number of nonlinear iterations 44 | int ls_max_iter = 5; // Max number of linear iterations 45 | int rhs_adv = 1; // Implicit (0) or Explicit (1) advection 46 | int use_preconditioner = 0; // Enable (>0) preconditioner 47 | }; 48 | 49 | 50 | // User-defined structure passed through SUNDIALS to RHS functions 51 | struct ProblemData 52 | { 53 | // Grid options 54 | int n_cell = 128; // Number of cells on each side of a square domain. 55 | int max_grid_size = 64; // The domain split into boxes of size max_grid_size 56 | 57 | // AMReX grid data structures 58 | amrex::Geometry* geom = nullptr; 59 | amrex::BoxArray* grid = nullptr; 60 | amrex::DistributionMapping* dmap = nullptr; 61 | 62 | // AMReX MLMG preconditioner data and parameters 63 | amrex::MultiFab* acoef = nullptr; 64 | amrex::MultiFab* bcoef = nullptr; 65 | 66 | // MLMG Preconditioner options 67 | int mg_agglomeration = 1; 68 | int mg_consolidation = 1; 69 | int mg_max_coarsening_level = 1000; 70 | int mg_linop_maxorder = 2; 71 | int mg_max_iter = 1000; 72 | int mg_max_fmg_iter = 1000; 73 | int mg_fixed_iter = 0; 74 | int mg_verbose = 0; 75 | int mg_bottom_verbose = 0; 76 | int mg_use_hypre = 0; 77 | int mg_hypre_interface = 3; 78 | int mg_use_petsc = 0; 79 | amrex::Real mg_tol_rel = 1.0e-6; 80 | amrex::Real mg_tol_abs = 1.0e-6; 81 | 82 | // Advection coefficients 83 | amrex::Real advCoeffx = 5.0e-4; 84 | amrex::Real advCoeffy = 2.5e-4; 85 | 86 | // Diffusion coefficients 87 | amrex::Real diffCoeffx = 1.0e-6; 88 | amrex::Real diffCoeffy = 1.0e-6; 89 | 90 | // Scratch space for flux computation 91 | amrex::Array* flux = nullptr; 92 | }; 93 | 94 | // ----------------------- 95 | // User defined functions 96 | // ----------------------- 97 | 98 | // Read the problem inputs, print help info, and problem setup 99 | int ParseInputs(ProblemOpt& prob_opt, 100 | ProblemData& prob_data); 101 | 102 | // Print help information message and exit 103 | void PrintHelp(); 104 | 105 | // Print the problem setup 106 | void PrintSetup(ProblemOpt& prob_opt, 107 | ProblemData& prob_data); 108 | 109 | // Decompose the problem in space 110 | void SetUpGeometry(amrex::BoxArray& ba, 111 | amrex::Geometry& geom, 112 | ProblemData& prob_data); 113 | 114 | // Set the problem initial state 115 | void FillInitConds2D(amrex::MultiFab& sol, 116 | const amrex::Geometry& geom); 117 | 118 | // Setup the time integrator and advance the solution with SUNDIALS 119 | void ComputeSolution(N_Vector nv_sol, 120 | ProblemOpt* prob_opt, 121 | ProblemData* prob_data); 122 | 123 | // --------------------------------------------- 124 | // User-supplied ODE RHS functions for SUNDIALS 125 | // --------------------------------------------- 126 | 127 | // Advection RHS function 128 | int ComputeRhsAdv(amrex::Real t, 129 | N_Vector nv_sol, 130 | N_Vector nv_rhs, 131 | void* data); 132 | 133 | // Diffusion RHS function 134 | int ComputeRhsDiff(amrex::Real t, 135 | N_Vector nv_sol, 136 | N_Vector nv_rhs, 137 | void* data); 138 | 139 | // Advection-Diffusion RHS function 140 | int ComputeRhsAdvDiff(amrex::Real t, 141 | N_Vector nv_sol, 142 | N_Vector nv_rhs, 143 | void* data); 144 | 145 | // ----------------------------------------------- 146 | // Utility functions to compute ODE RHS functions 147 | // ----------------------------------------------- 148 | 149 | // Advective portion of ODE RHS 150 | void ComputeAdvectionUpwind(amrex::MultiFab& sol, 151 | amrex::MultiFab& advection, 152 | amrex::Geometry& geom, 153 | amrex::Real advCoeffx, 154 | amrex::Real advCoeffy); 155 | 156 | // Diffusive portion of ODE RHS 157 | void ComputeDiffusion(amrex::MultiFab& sol, 158 | amrex::MultiFab& diff_mf, 159 | amrex::MultiFab& fx_mf, 160 | amrex::MultiFab& fy_mf, 161 | amrex::Geometry& geom, 162 | amrex::Real diffCoeffx, 163 | amrex::Real diffCoeffy); 164 | 165 | // Utility functions for computing diffusion 166 | void ComputeDiffFlux(amrex::MultiFab& sol, 167 | amrex::MultiFab& fx, 168 | amrex::MultiFab& fy, 169 | amrex::Geometry& geom, 170 | amrex::Real diffCoeffx, 171 | amrex::Real diffCoeffy); 172 | 173 | void ComputeDivergence(amrex::MultiFab& div, 174 | amrex::MultiFab& fx, 175 | amrex::MultiFab& fy, 176 | amrex::Geometry& geom); 177 | 178 | // --------------------------------------------------- 179 | // User-supplied preconditioner function for SUNDIALS 180 | // --------------------------------------------------- 181 | 182 | int precondition_solve(amrex::Real tn, 183 | N_Vector u, 184 | N_Vector fu, 185 | N_Vector r, 186 | N_Vector z, 187 | amrex::Real gamma, 188 | amrex::Real delta, 189 | int lr, 190 | void *user_data); 191 | 192 | #endif 193 | -------------------------------------------------------------------------------- /cmake/FindAMReX.cmake: -------------------------------------------------------------------------------- 1 | find_package(AMReX REQUIRED COMPONENTS 2 | HINTS ${AMREX_DIR} $ENV{AMREX_DIR} ${CMAKE_PREFIX_PATH} 3 | NO_DEFAULT_PATH) 4 | 5 | if(NOT TARGET XSDK::AMReX) 6 | add_library(XSDK_AMREX INTERFACE) 7 | target_link_libraries(XSDK_AMREX INTERFACE AMReX::amrex) 8 | if(ENABLE_HIP) 9 | target_link_libraries(XSDK_AMREX INTERFACE hip::amdhip64) 10 | target_link_options(XSDK_AMREX INTERFACE "-fgpu-rdc") 11 | endif() 12 | add_library(XSDK::AMReX ALIAS XSDK_AMREX) 13 | endif() 14 | 15 | -------------------------------------------------------------------------------- /cmake/FindBLASPP.cmake: -------------------------------------------------------------------------------- 1 | # find the BLAS++ include path 2 | find_path(BLASPP_INCLUDE_DIR blas.hh 3 | NAMES blas.hh 4 | HINTS ${BLASPP_DIR} $ENV{BLASPP_DIR} ${CMAKE_PREFIX_PATH} 5 | PATH_SUFFIXES include 6 | NO_DEFAULT_PATH 7 | DOC "Directory with BLASPP header" 8 | ) 9 | 10 | # find the main BLAS++ library 11 | find_library(BLASPP_LIBRARIES 12 | NAMES blaspp 13 | HINTS ${BLASPP_DIR} $ENV{BLASPP_DIR} ${CMAKE_PREFIX_PATH} 14 | PATH_SUFFIXES lib lib64 15 | NO_DEFAULT_PATH 16 | DOC "The BLAS++ library." 17 | ) 18 | 19 | find_package_handle_standard_args(BLASPP 20 | REQUIRED_VARS 21 | BLASPP_LIBRARIES 22 | BLASPP_INCLUDE_DIR 23 | VERSION_VAR 24 | BLASPP_VERSION 25 | ) 26 | 27 | # Create target for BLAS++ 28 | if(BLASPP_FOUND) 29 | 30 | if(NOT TARGET XSDK::BLASPP) 31 | add_library(XSDK::BLASPP INTERFACE IMPORTED) 32 | endif() 33 | 34 | message(STATUS "Created XSDK::BLASPP target") 35 | message(STATUS " INTERFACE_INCLUDE_DIRECTORIES: ${BLASPP_INCLUDE_DIR}") 36 | message(STATUS " INTERFACE_LINK_LIBRARIES: ${BLASPP_LIBRARIES}") 37 | 38 | set_target_properties(XSDK::BLASPP PROPERTIES 39 | INTERFACE_INCLUDE_DIRECTORIES "${BLASPP_INCLUDE_DIR}" 40 | INTERFACE_LINK_LIBRARIES "${BLASPP_LIBRARIES}") 41 | endif() 42 | -------------------------------------------------------------------------------- /cmake/FindHYPRE.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------- 2 | # SUNDIALS Copyright Start 3 | # Copyright (c) 2002-2020, Lawrence Livermore National Security 4 | # and Southern Methodist University. 5 | # All rights reserved. 6 | # 7 | # See the top-level LICENSE and NOTICE files for details. 8 | # 9 | # SPDX-License-Identifier: BSD-3-Clause 10 | # SUNDIALS Copyright End 11 | # --------------------------------------------------------------- 12 | # HYPRE find module that creates an imported target for HYPRE. 13 | # The target is XSDK::HYPRE. 14 | # 15 | # This module also defines variables, but it is best to use 16 | # the defined target to ensure includes and compile/link 17 | # options are correctly passed to consumers. 18 | # 19 | # HYPRE_FOUND - system has HYPRE library 20 | # HYPRE_LIBRARY - the HYPRE library 21 | # HYPRE_INCLUDE_DIR - the HYPRE include path 22 | # HYPRE_LIBRARIES - all of the libraries needed for HYPRE 23 | # --------------------------------------------------------------- 24 | 25 | ### find include dir 26 | find_path(HYPRE_INCLUDE_DIR 27 | NAMES HYPRE.h hypre.h 28 | HINTS ${HYPRE_DIR} $ENV{HYPRE_DIR} ${CMAKE_PREFIX_PATH} 29 | PATH_SUFFIXES include 30 | NO_DEFAULT_PATH 31 | DOC "Directory with hypre header.") 32 | 33 | ### find library 34 | find_library(HYPRE_LIBRARY 35 | NAMES HYPRE hypre 36 | HINTS ${HYPRE_DIR} $ENV{HYPRE_DIR} ${CMAKE_PREFIX_PATH} 37 | PATH_SUFFIXES lib lib64 38 | NO_DEFAULT_PATH 39 | DOC "The hypre library.") 40 | 41 | list(FIND HYPRE_LIBRARIES ${HYPRE_LIBRARY} _idx) 42 | if (_idx EQUAL -1) 43 | set(HYPRE_LIBRARIES "${HYPRE_LIBRARY};${HYPRE_LIBRARIES}" CACHE STRING "" FORCE) 44 | endif () 45 | 46 | # set package variables including HYPRE_FOUND 47 | find_package_handle_standard_args(HYPRE 48 | REQUIRED_VARS 49 | HYPRE_LIBRARY 50 | HYPRE_LIBRARIES 51 | HYPRE_INCLUDE_DIR 52 | ) 53 | 54 | # Create target for HYPRE 55 | if(HYPRE_FOUND) 56 | 57 | if(NOT TARGET XSDK::HYPRE) 58 | add_library(XSDK::HYPRE UNKNOWN IMPORTED) 59 | endif() 60 | 61 | set_target_properties(XSDK::HYPRE PROPERTIES 62 | INTERFACE_INCLUDE_DIRECTORIES "${HYPRE_INCLUDE_DIR}" 63 | INTERFACE_LINK_LIBRARIES "${HYPRE_LIBRARIES}" 64 | IMPORTED_LOCATION "${HYPRE_LIBRARY}") 65 | 66 | if(ENABLE_HIP) 67 | target_link_libraries(XSDK::HYPRE INTERFACE roc::rocsparse roc::rocrand) 68 | endif() 69 | 70 | endif() 71 | -------------------------------------------------------------------------------- /cmake/FindLAPACKPP.cmake: -------------------------------------------------------------------------------- 1 | # find the LAPACK++ include path 2 | find_path(LAPACKPP_INCLUDE_DIR lapack.hh 3 | NAMES lapack.hh 4 | HINTS ${LAPACKPP_DIR} $ENV{LAPACKPP_DIR} ${CMAKE_PREFIX_PATH} 5 | PATH_SUFFIXES include 6 | NO_DEFAULT_PATH 7 | DOC "Directory with LAPACKPP header" 8 | ) 9 | 10 | # find the main LAPACK++ library 11 | find_library(LAPACKPP_LIBRARIES 12 | NAMES lapackpp 13 | HINTS ${LAPACKPP_DIR} $ENV{LAPACKPP_DIR} ${CMAKE_PREFIX_PATH} 14 | PATH_SUFFIXES lib lib64 15 | NO_DEFAULT_PATH 16 | DOC "The LAPACK++ library." 17 | ) 18 | 19 | find_package_handle_standard_args(LAPACKPP 20 | REQUIRED_VARS 21 | LAPACKPP_LIBRARIES 22 | LAPACKPP_INCLUDE_DIR 23 | VERSION_VAR 24 | LAPACKPP_VERSION 25 | ) 26 | 27 | # Create target for LAPACK++ 28 | if(LAPACKPP_FOUND) 29 | 30 | if(NOT TARGET XSDK::LAPACKPP) 31 | add_library(XSDK::LAPACKPP INTERFACE IMPORTED) 32 | endif() 33 | 34 | message(STATUS "Created XSDK::LAPACKPP target") 35 | message(STATUS " INTERFACE_INCLUDE_DIRECTORIES: ${LAPACKPP_INCLUDE_DIR}") 36 | message(STATUS " INTERFACE_LINK_LIBRARIES: ${LAPACKPP_LIBRARIES}") 37 | 38 | set_target_properties(XSDK::LAPACKPP PROPERTIES 39 | INTERFACE_INCLUDE_DIRECTORIES "${LAPACKPP_INCLUDE_DIR}" 40 | INTERFACE_LINK_LIBRARIES "${LAPACKPP_LIBRARIES}") 41 | 42 | endif() 43 | -------------------------------------------------------------------------------- /cmake/FindMAGMA.cmake: -------------------------------------------------------------------------------- 1 | # SUNDIALS Copyright Start 2 | # Copyright (c) 2002-2021, Lawrence Livermore National Security 3 | # and Southern Methodist University. 4 | # All rights reserved. 5 | # 6 | # See the top-level LICENSE and NOTICE files for details. 7 | # 8 | # SPDX-License-Identifier: BSD-3-Clause 9 | # SUNDIALS Copyright End 10 | # ----------------------------------------------------------------------------- 11 | # Find module that locates the MAGMA linear algebra library. 12 | # ----------------------------------------------------------------------------- 13 | 14 | # find the MAGMA include path 15 | find_path(MAGMA_INCLUDE_DIR magma_v2.h 16 | NAMES magma_v2.h 17 | HINTS ${MAGMA_DIR} $ENV{MAGMA_DIR} ${CMAKE_PREFIX_PATH} 18 | PATH_SUFFIXES include 19 | NO_DEFAULT_PATH 20 | DOC "Directory with MAGMA header" 21 | ) 22 | 23 | # find the main MAGMA library 24 | find_library(MAGMA_LIBRARY 25 | NAMES magma 26 | HINTS ${MAGMA_DIR} $ENV{MAGMA_DIR} ${CMAKE_PREFIX_PATH} 27 | PATH_SUFFIXES lib lib64 28 | NO_DEFAULT_PATH 29 | DOC "The MAGMA library.") 30 | 31 | # Find the optional sparse component 32 | if("SPARSE" IN_LIST MAGMA_FIND_COMPONENTS) 33 | set(_sparse_required MAGMA_SPARSE_LIBRARY) 34 | find_library(MAGMA_SPARSE_LIBRARY 35 | NAMES magma_sparse 36 | HINTS ${MAGMA_DIR} $ENV{MAGMA_DIR} ${CMAKE_PREFIX_PATH} 37 | PATH_SUFFIXES lib lib64 38 | NO_DEFAULT_PATH 39 | DOC "The MAGMA sparse library.") 40 | else() 41 | set(_sparse_required ) 42 | endif() 43 | 44 | # Determine MAGMA version and libraries it depends on 45 | if(MAGMA_LIBRARY AND MAGMA_INCLUDE_DIR) 46 | get_filename_component(libdir ${MAGMA_LIBRARY} DIRECTORY) 47 | find_file(MAGMA_PKG_CONFIG_PATH magma.pc PATHS "${libdir}/pkgconfig") 48 | 49 | if(MAGMA_PKG_CONFIG_PATH) 50 | file(STRINGS ${MAGMA_PKG_CONFIG_PATH} _version_string REGEX "Version: [0-9].[0-9].[0-9]") 51 | string(REGEX MATCHALL "[0-9]" _version_full "${_version_string}") 52 | 53 | list(GET _version_full 0 _version_major) 54 | list(GET _version_full 1 _version_minor) 55 | list(GET _version_full 2 _version_patch) 56 | 57 | set(MAGMA_VERSION "${_version_major}.${_version_minor}.${_version_patch}") 58 | 59 | file(STRINGS ${MAGMA_PKG_CONFIG_PATH} _libraries_string REGEX "Libs:.*") 60 | string(REPLACE " " ";" _libraries_list ${_libraries_string}) 61 | list(SUBLIST _libraries_list 1 -1 _libraries_list) # remove 'Libs:' part 62 | 63 | set(_interface_libraires ) 64 | foreach(lib ${_libraries_list}) 65 | if(NOT (lib STREQUAL "-lmagma" OR lib STREQUAL "-lmagma_sparse" OR lib STREQUAL "-L\${libdir}" OR lib STREQUAL "") ) 66 | string(REGEX REPLACE "^-l" "" lib ${lib}) 67 | list(APPEND _interface_libraires ${lib}) 68 | endif() 69 | endforeach() 70 | endif() 71 | endif() 72 | 73 | set(MAGMA_LIBRARIES "${MAGMA_LIBRARY};${_interface_libraires}") 74 | 75 | find_package_handle_standard_args(MAGMA 76 | REQUIRED_VARS 77 | MAGMA_LIBRARY 78 | MAGMA_LIBRARIES 79 | MAGMA_INCLUDE_DIR 80 | ${_sparse_required} 81 | VERSION_VAR 82 | MAGMA_VERSION 83 | ) 84 | 85 | # Create target for MAGMA 86 | if(MAGMA_FOUND) 87 | 88 | if(NOT TARGET XSDK::MAGMA) 89 | add_library(XSDK::MAGMA UNKNOWN IMPORTED) 90 | endif() 91 | 92 | set_target_properties(XSDK::MAGMA PROPERTIES 93 | INTERFACE_INCLUDE_DIRECTORIES "${MAGMA_INCLUDE_DIR}" 94 | INTERFACE_LINK_LIBRARIES "${_interface_libraires}" 95 | IMPORTED_LOCATION "${MAGMA_LIBRARY}") 96 | 97 | if(MAGMA_SPARSE_LIBRARY) 98 | if(NOT TARGET XSDK::MAGMA_SPARSE) 99 | add_library(XSDK::MAGMA_SPARSE UNKNOWN IMPORTED) 100 | endif() 101 | 102 | set_target_properties(XSDK::MAGMA_SPARSE PROPERTIES 103 | INTERFACE_INCLUDE_DIRECTORIES "${MAGMA_INCLUDE_DIR}" 104 | INTERFACE_LINK_LIBRARIES "${MAGMA_LIBRARY};${_interface_libraires}" 105 | IMPORTED_LOCATION "${MAGMA_SPARSE_LIBRARY}") 106 | endif() 107 | 108 | endif() 109 | -------------------------------------------------------------------------------- /cmake/FindMETIS.cmake: -------------------------------------------------------------------------------- 1 | 2 | find_path(METIS_INCLUDE_DIRS metis.h 3 | HINTS ${METIS_DIR} $ENV{METIS_DIR} ${CMAKE_PREFIX_PATH} 4 | PATH_SUFFIXES include 5 | NO_DEFAULT_PATH 6 | DOC "The directory with the Metis header file.") 7 | 8 | find_library(METIS_LIBRARY metis 9 | HINTS ${METIS_DIR} $ENV{METIS_DIR} ${CMAKE_PREFIX_PATH} 10 | PATH_SUFFIXES lib 11 | NO_DEFAULT_PATH 12 | DOC "The Metis library.") 13 | 14 | find_library(ZLIB_LIBRARY z 15 | HINTS ${ZLIB_LIBRARY_DIR} $ENV{ZLIB_LIBRARY_DIR} ${CMAKE_PREFIX_PATH} 16 | PATH_SUFFIXES lib 17 | NO_DEFAULT_PATH 18 | DOC "The zlib library.") 19 | 20 | # set package variables including METIS_FOUND 21 | find_package_handle_standard_args(METIS 22 | REQUIRED_VARS 23 | METIS_LIBRARY 24 | ZLIB_LIBRARY 25 | ) 26 | 27 | # Create target for METIS 28 | if(METIS_FOUND) 29 | 30 | if(NOT TARGET METIS) 31 | add_library(METIS UNKNOWN IMPORTED) 32 | endif() 33 | 34 | set_target_properties(METIS PROPERTIES 35 | INTERFACE_INCLUDE_DIRECTORIES "${METIS_INCLUDE_DIRS}" 36 | INTERFACE_LINK_LIBRARIES "${ZLIB_LIBRARY}" 37 | IMPORTED_LOCATION "${METIS_LIBRARY}") 38 | 39 | endif() 40 | -------------------------------------------------------------------------------- /cmake/FindMFEM.cmake: -------------------------------------------------------------------------------- 1 | # Find MFEM 2 | # Find the MFEM header files 3 | find_path(MFEM_INCLUDE_DIRS mfem.hpp 4 | HINTS ${MFEM_DIR} $ENV{MFEM_DIR} ${CMAKE_PREFIX_PATH} 5 | PATH_SUFFIXES include 6 | NO_DEFAULT_PATH 7 | DOC "Directory with MFEM header.") 8 | 9 | # Find the MFEM library 10 | find_library(MFEM_LIBRARY mfem 11 | HINTS ${MFEM_DIR} $ENV{MFEM_DIR} ${CMAKE_PREFIX_PATH} 12 | PATH_SUFFIXES lib 13 | NO_DEFAULT_PATH 14 | DOC "The MFEM library.") 15 | 16 | # Find config.mk 17 | find_file(MFEM_CONFIG_MK config.mk 18 | HINTS ${MFEM_DIR} $ENV{MFEM_DIR} ${CMAKE_PREFIX_PATH} 19 | PATH_SUFFIXES share/mfem config 20 | REQUIRED 21 | NO_DEFAULT_PATH 22 | DOC "MFEM's config.mk file.") 23 | 24 | # From config.mk, read the values of MFEM_TPLFLAGS and MFEM_EXT_LIBS 25 | file(STRINGS "${MFEM_CONFIG_MK}" MFEM_TPLFLAGS_LINE 26 | REGEX "^MFEM_TPLFLAGS * = .*$") 27 | string(REGEX REPLACE "^MFEM_TPLFLAGS * = *" "" 28 | MFEM_TPLFLAGS "${MFEM_TPLFLAGS_LINE}") 29 | file(STRINGS "${MFEM_CONFIG_MK}" MFEM_EXT_LIBS_LINE 30 | REGEX "^MFEM_EXT_LIBS * = .*$") 31 | string(REGEX REPLACE "^MFEM_EXT_LIBS * = *" "" 32 | MFEM_EXT_LIBS "${MFEM_EXT_LIBS_LINE}") 33 | # message(STATUS "MFEM_TPLFLAGS: ${MFEM_TPLFLAGS}") 34 | # message(STATUS "MFEM_EXT_LIBS: ${MFEM_EXT_LIBS}") 35 | 36 | # set package variables including MFEM_FOUND 37 | find_package_handle_standard_args(MFEM 38 | REQUIRED_VARS 39 | MFEM_LIBRARY 40 | MFEM_INCLUDE_DIRS 41 | ) 42 | 43 | # Create target for MFEM 44 | if(MFEM_FOUND) 45 | 46 | if(NOT TARGET XSDK::MFEM) 47 | add_library(XSDK::MFEM INTERFACE IMPORTED) 48 | endif() 49 | 50 | if(ENABLE_CUDA) 51 | string(REPLACE "-Xlinker=" "-Wl," MFEM_EXT_LIBS "${MFEM_EXT_LIBS}") 52 | endif() 53 | separate_arguments(MFEM_TPLFLAGS_LIST UNIX_COMMAND "${MFEM_TPLFLAGS}") 54 | separate_arguments(MFEM_EXT_LIBS_LIST UNIX_COMMAND "${MFEM_EXT_LIBS}") 55 | # message(STATUS "MFEM_TPLFLAGS_LIST: ${MFEM_TPLFLAGS_LIST}") 56 | # message(STATUS "MFEM_EXT_LIBS_LIST: ${MFEM_EXT_LIBS_LIST}") 57 | set_target_properties(XSDK::MFEM PROPERTIES 58 | INTERFACE_INCLUDE_DIRECTORIES "${MFEM_INCLUDE_DIRS}" 59 | INTERFACE_COMPILE_OPTIONS "${MFEM_TPLFLAGS_LIST}" 60 | INTERFACE_LINK_LIBRARIES "${MFEM_LIBRARY};${MFEM_EXT_LIBS_LIST}") 61 | 62 | # Assuming MPI build of MFEM: 63 | target_link_libraries(XSDK::MFEM INTERFACE MPI::MPI_C) 64 | 65 | if(ENABLE_CUDA) 66 | target_link_libraries(XSDK::MFEM INTERFACE CUDA::cudart CUDA::cusparse) 67 | endif() 68 | if(ENABLE_HIP) 69 | target_link_libraries(XSDK::MFEM INTERFACE hip::amdhip64 roc::hipsparse) 70 | endif() 71 | 72 | endif() 73 | -------------------------------------------------------------------------------- /cmake/FindPLASMA.cmake: -------------------------------------------------------------------------------- 1 | # find the PLASMA include path 2 | find_path(PLASMA_INCLUDE_DIR plasma.h 3 | NAMES plasma.h 4 | HINTS ${PLASMA_DIR} $ENV{PLASMA_DIR} ${CMAKE_PREFIX_PATH} 5 | PATH_SUFFIXES include 6 | NO_DEFAULT_PATH 7 | DOC "Directory with PLASMA header" 8 | ) 9 | 10 | # find the main PLASMA library 11 | find_library(PLASMA_LIBRARY 12 | NAMES plasma 13 | HINTS ${PLASMA_DIR} $ENV{PLASMA_DIR} ${CMAKE_PREFIX_PATH} 14 | PATH_SUFFIXES lib lib64 15 | NO_DEFAULT_PATH 16 | DOC "The PLASMA library." 17 | ) 18 | 19 | find_library(PLASMA_CORE_BLAS_LIBRARY 20 | NAMES plasma_core_blas 21 | HINTS ${PLASMA_DIR} $ENV{PLASMA_DIR} ${CMAKE_PREFIX_PATH} 22 | PATH_SUFFIXES lib lib64 23 | NO_DEFAULT_PATH 24 | DOC "The PLASMA core blas library." 25 | ) 26 | 27 | set(PLASMA_LIBRARIES "${PLASMA_LIBRARY};${PLASMA_CORE_BLAS_LIBRARY}") 28 | 29 | find_package_handle_standard_args(PLASMA 30 | REQUIRED_VARS 31 | PLASMA_LIBRARIES 32 | PLASMA_INCLUDE_DIR 33 | VERSION_VAR 34 | PLASMA_VERSION 35 | ) 36 | 37 | # Create target for PLASMA 38 | if(PLASMA_FOUND) 39 | if(NOT TARGET XSDK::PLASMA) 40 | add_library(XSDK::PLASMA INTERFACE IMPORTED) 41 | endif() 42 | 43 | message(STATUS "Created XSDK::PLASMA target") 44 | message(STATUS " INTERFACE_INCLUDE_DIRECTORIES: ${PLASMA_INCLUDE_DIR}") 45 | message(STATUS " INTERFACE_LINK_LIBRARIES: ${PLASMA_LIBRARIES}") 46 | 47 | set_target_properties(XSDK::PLASMA PROPERTIES 48 | INTERFACE_INCLUDE_DIRECTORIES "${PLASMA_INCLUDE_DIR}" 49 | INTERFACE_LINK_LIBRARIES "${PLASMA_LIBRARIES}") 50 | endif() 51 | -------------------------------------------------------------------------------- /cmake/FindSLATE.cmake: -------------------------------------------------------------------------------- 1 | # find the SLATE include path 2 | find_path(SLATE_INCLUDE_DIR slate.hh 3 | NAMES slate/slate.hh 4 | HINTS ${SLATE_DIR} $ENV{SLATE_DIR} ${CMAKE_PREFIX_PATH} 5 | PATH_SUFFIXES include 6 | NO_DEFAULT_PATH 7 | DOC "Directory with SLATE header" 8 | ) 9 | 10 | # find the main SLATE library 11 | find_library(SLATE_LIBRARIES 12 | NAMES slate 13 | HINTS ${SLATE_DIR} $ENV{SLATE_DIR} ${CMAKE_PREFIX_PATH} 14 | PATH_SUFFIXES lib lib64 15 | NO_DEFAULT_PATH 16 | DOC "The SLATE library." 17 | ) 18 | 19 | find_package_handle_standard_args(SLATE 20 | REQUIRED_VARS 21 | SLATE_LIBRARIES 22 | SLATE_INCLUDE_DIR 23 | VERSION_VAR 24 | SLATE_VERSION 25 | ) 26 | 27 | # Create target for SLATE 28 | if(SLATE_FOUND) 29 | 30 | if(NOT TARGET XSDK::SLATE) 31 | add_library(XSDK::SLATE INTERFACE IMPORTED) 32 | endif() 33 | 34 | message(STATUS "Created XSDK::SLATE target") 35 | message(STATUS " INTERFACE_INCLUDE_DIRECTORIES: ${SLATE_INCLUDE_DIR}") 36 | message(STATUS " INTERFACE_LINK_LIBRARIES: ${SLATE_LIBRARIES}") 37 | 38 | set_target_properties(XSDK::SLATE PROPERTIES 39 | INTERFACE_INCLUDE_DIRECTORIES "${SLATE_INCLUDE_DIR}" 40 | INTERFACE_LINK_LIBRARIES "${SLATE_LIBRARIES}") 41 | 42 | endif() 43 | -------------------------------------------------------------------------------- /cmake/FindSUNDIALS.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------- 2 | # Programmer: Cody J. Balos, David J. Gardner @ LLNL 3 | # --------------------------------------------------------------- 4 | # SUNDIALS Copyright Start 5 | # Copyright (c) 2002-2019, Lawrence Livermore National Security 6 | # and Southern Methodist University. 7 | # All rights reserved. 8 | # 9 | # See the top-level LICENSE and NOTICE files for details. 10 | # 11 | # SPDX-License-Identifier: BSD-3-Clause 12 | # SUNDIALS Copyright End 13 | # --------------------------------------------------------------- 14 | 15 | # determine SUNDIALS components needed 16 | if(NOT SUNDIALS_FIND_COMPONENTS) 17 | set(SUNDIALS_FIND_COMPONENTS 18 | "arkode" 19 | "cvodes" 20 | "nvecserial" 21 | "nvecparallel" 22 | "nvecmpiplusx") 23 | endif() 24 | 25 | if(ENABLE_CUDA) 26 | list(APPEND SUNDIALS_FIND_COMPONENTS "nveccuda") 27 | endif() 28 | if(ENABLE_MFEM) 29 | list(APPEND SUNDIALS_FIND_COMPONENTS "kinsol") 30 | endif() 31 | if(ENABLE_MAGMA) 32 | list(APPEND SUNDIALS_FIND_COMPONENTS "sunmatrixmagmadense") 33 | list(APPEND SUNDIALS_FIND_COMPONENTS "sunlinsolmagmadense") 34 | endif() 35 | if(ENABLE_PETSC) 36 | list(APPEND SUNDIALS_FIND_COMPONENTS "nvecpetsc") 37 | list(APPEND SUNDIALS_FIND_COMPONENTS "sunnonlinsolpetscsnes") 38 | endif() 39 | if(ENABLE_SUPERLU) 40 | list(APPEND SUNDIALS_FIND_COMPONENTS "sunmatrixslunrloc") 41 | list(APPEND SUNDIALS_FIND_COMPONENTS "sunlinsolsuperludist") 42 | endif() 43 | 44 | find_package(SUNDIALS REQUIRED COMPONENTS ${SUNDIALS_FIND_COMPONENTS} 45 | HINTS ${SUNDIALS_DIR} $ENV{SUNDIALS_DIR} ${CMAKE_PREFIX_PATH} 46 | NO_DEFAULT_PATH) 47 | 48 | if(NOT TARGET XSDK::SUNDIALS) 49 | add_library(XSDK_SUNDIALS INTERFACE) 50 | foreach(_component ${SUNDIALS_FIND_COMPONENTS}) 51 | target_link_libraries(XSDK_SUNDIALS INTERFACE SUNDIALS::${_component}) 52 | endforeach() 53 | add_library(XSDK::SUNDIALS ALIAS XSDK_SUNDIALS) 54 | endif() 55 | -------------------------------------------------------------------------------- /cmake/FindSUPERLUDIST.cmake: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------- 2 | # SUNDIALS Copyright Start 3 | # Copyright (c) 2002-2020, Lawrence Livermore National Security 4 | # and Southern Methodist University. 5 | # All rights reserved. 6 | # 7 | # See the top-level LICENSE and NOTICE files for details. 8 | # 9 | # SPDX-License-Identifier: BSD-3-Clause 10 | # SUNDIALS Copyright End 11 | # --------------------------------------------------------------- 12 | # SuperLUDIST find module that creates an imported target for 13 | # SuperLU_DIST. The target is XSDK::SUPERLU. 14 | # 15 | # This module also defines variables, but it is best to use 16 | # the defined target to ensure includes and compile/link 17 | # options are correctly passed to consumers. 18 | # 19 | # SUPERLUDIST_FOUND - system has SuperLU_DIST library 20 | # SUPERLUDIST_INCLUDE_DIR - Directory with the SuperLU_DIST header 21 | # SUPERLUDIST_LIBRARY - the SuperLU_DIST library 22 | # --------------------------------------------------------------- 23 | 24 | ### find include dir 25 | find_path(SUPERLUDIST_INCLUDE_DIR superlu_defs.h 26 | HINTS ${SUPERLU_DIR} $ENV{SUPERLU_DIR} ${CMAKE_PREFIX_PATH} 27 | PATH_SUFFIXES include 28 | NO_DEFAULT_PATH 29 | DOC "Directory with the SuperLU DIST header") 30 | 31 | ### find library 32 | find_library(SUPERLUDIST_LIBRARY superlu_dist 33 | HINTS ${SUPERLU_DIR} $ENV{SUPERLU_DIR} ${CMAKE_PREFIX_PATH} 34 | PATH_SUFFIXES lib lib64 35 | NO_DEFAULT_PATH 36 | DOC "The SuperLU DIST library") 37 | 38 | # set package variables including SUPERLUDIST_FOUND 39 | find_package_handle_standard_args(SUPERLUDIST 40 | REQUIRED_VARS 41 | SUPERLUDIST_LIBRARY 42 | SUPERLUDIST_INCLUDE_DIR 43 | ) 44 | 45 | # Create target for SuperLU_DIST 46 | if(SUPERLUDIST_FOUND AND (NOT TARGET XSDK::SUPERLU)) 47 | add_library(XSDK::SUPERLU UNKNOWN IMPORTED) 48 | set_target_properties(XSDK::SUPERLU PROPERTIES 49 | INTERFACE_INCLUDE_DIRECTORIES "${SUPERLUDIST_INCLUDE_DIR}" 50 | INTERFACE_LINK_LIBRARIES "${SUPERLUDIST_LIBRARIES}" 51 | IMPORTED_LOCATION "${SUPERLUDIST_LIBRARY}") 52 | endif() 53 | -------------------------------------------------------------------------------- /cmake/XsdkAddTest.cmake: -------------------------------------------------------------------------------- 1 | macro(xsdk_add_test) 2 | 3 | set(options) 4 | 5 | # MPI_NPROCS = number of mpi tasks to use in parallel tests 6 | set(oneValueArgs "NAME" "MPI_NPROCS") 7 | 8 | set(multiValueArgs "COMMAND" "ENVIRONMENT") 9 | 10 | # parse inputs and create variables SUNDIALS_ADD_TEST_ 11 | cmake_parse_arguments(xsdk_add_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) 12 | 13 | if(xsdk_add_test_MPI_NPROCS) 14 | add_test(NAME ${xsdk_add_test_NAME} 15 | COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${xsdk_add_test_MPI_NPROCS} 16 | ${MPIEXEC_PREFLAGS} ${xsdk_add_test_COMMAND} ${MPIEXEC_POSTFLAGS} 17 | ) 18 | else() 19 | add_test(NAME ${xsdk_add_test_NAME} COMMAND ${xsdk_add_test_COMMAND}) 20 | endif() 21 | 22 | if(xsdk_add_test_ENVIRONMENT) 23 | set_tests_properties( 24 | ${xsdk_add_test_NAME} PROPERTIES ENVIRONMENT ${xsdk_add_test_ENVIRONMENT} 25 | ) 26 | endif() 27 | 28 | endmacro(xsdk_add_test) 29 | -------------------------------------------------------------------------------- /dealii/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if(ENABLE_PETSC OR ENABLE_TRILINOS) 2 | add_subdirectory(petsc_trilinos) 3 | endif() 4 | if(ENABLE_PRECICE) 5 | add_subdirectory(precice) 6 | endif() 7 | if(ENABLE_SUNDIALS) 8 | add_subdirectory(sundials) 9 | endif() 10 | -------------------------------------------------------------------------------- /dealii/README.md: -------------------------------------------------------------------------------- 1 | # deal.II examples 2 | Example codes demonstrating the use of deal.II using other XSDK packages. 3 | -------------------------------------------------------------------------------- /dealii/petsc_trilinos/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ## 2 | # CMake script for the petsc_trilinos tutorial program: 3 | ## 4 | 5 | # Set the name of the project and target: 6 | set(TARGET "petsc_trilinos") 7 | 8 | # Declare all source files the target consists of. Here, this is only 9 | # the one step-X.cc file, but as you expand your project you may wish 10 | # to add other source files as well. If your project becomes much larger, 11 | # you may want to either replace the following statement by something like 12 | # file(GLOB_RECURSE TARGET_SRC "source/*.cc") 13 | # file(GLOB_RECURSE TARGET_INC "include/*.h") 14 | # set(TARGET_SRC ${TARGET_SRC} ${TARGET_INC}) 15 | # or switch altogether to the large project CMakeLists.txt file discussed 16 | # in the "CMake in user projects" page accessible from the "User info" 17 | # page of the documentation. 18 | set(TARGET_SRC 19 | ${TARGET}.cc 20 | ) 21 | 22 | # At least in some cases, deal.II needs to use the MPI wrapper: 23 | if (MPI_CXX_COMPILER) 24 | set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER}) 25 | endif() 26 | 27 | # Usually, you will not need to modify anything beyond this point... 28 | 29 | cmake_minimum_required(VERSION 3.13.4) 30 | 31 | # 32 | # Are all dependencies fulfilled? 33 | # 34 | if(NOT ((DEAL_II_WITH_PETSC AND NOT DEAL_II_PETSC_WITH_COMPLEX) OR DEAL_II_WITH_TRILINOS) OR NOT DEAL_II_WITH_P4EST) # keep in one line 35 | message(FATAL_ERROR " 36 | Error! This tutorial requires a deal.II library that was configured with the following options: 37 | DEAL_II_WITH_PETSC = ON 38 | DEAL_II_PETSC_WITH_COMPLEX = OFF 39 | DEAL_II_WITH_P4EST = ON 40 | or 41 | DEAL_II_WITH_TRILINOS = ON 42 | DEAL_II_WITH_P4EST = ON 43 | However, the deal.II library found at ${DEAL_II_PATH} was configured with these options: 44 | DEAL_II_WITH_PETSC = ${DEAL_II_WITH_PETSC} 45 | DEAL_II_PETSC_WITH_COMPLEX = ${DEAL_II_PETSC_WITH_COMPLEX} 46 | DEAL_II_WITH_P4EST = ${DEAL_II_WITH_P4EST} 47 | DEAL_II_WITH_TRILINOS = ${DEAL_II_WITH_TRILINOS} 48 | This conflicts with the requirements. 49 | One or both of the aforementioned combinations of prerequisites are not met by your installation, but at least one is required for this tutorial step." 50 | ) 51 | endif() 52 | 53 | deal_ii_initialize_cached_variables() 54 | set(CLEAN_UP_FILES *.log *.gmv *.gnuplot *.gpl *.eps *.pov *.vtk *.ucd *.d2 *.vtu *.pvtu) 55 | add_executable(${TARGET} ${TARGET_SRC}) 56 | deal_ii_setup_target(${TARGET}) 57 | 58 | if (NOT ENABLE_CUDA) 59 | # This example fails when using Spack-built `xsdk` with CUDA enabled, so it 60 | # is disabled for now. 61 | xsdk_add_test( 62 | NAME 63 | dealii-${TARGET} 64 | MPI_NPROCS 65 | 1 66 | COMMAND 67 | $ 68 | ) 69 | endif() 70 | -------------------------------------------------------------------------------- /dealii/petsc_trilinos/README.md: -------------------------------------------------------------------------------- 1 | # deal.II-Trilinos and deal.II-PETSc example 2 | 3 | This is essentially deal.II's step-40 example, see 4 | https://www.dealii.org/current/doxygen/deal.II/step_40.html for documentation. 5 | -------------------------------------------------------------------------------- /dealii/precice/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | # At least in some cases, deal.II needs to use the MPI wrapper: 3 | if (MPI_CXX_COMPILER) 4 | set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER}) 5 | endif() 6 | 7 | # build Laplace problem, which requires both deal.II and preCICE 8 | add_executable(laplace_problem laplace_problem.cc) 9 | DEAL_II_SETUP_TARGET(laplace_problem) 10 | target_link_libraries(laplace_problem precice::precice) 11 | 12 | # build the boundary condition, which only requires preCICE 13 | add_executable(boundary_condition boundary_condition.cc) 14 | target_link_libraries(boundary_condition precice::precice) 15 | 16 | # Copy the shell script 'runtargets.sh' and config file 'precice-config.xml' 17 | # from the source tree to the build tree so that the example can be run there. 18 | add_custom_command( 19 | OUTPUT runtargets.sh precice-config.xml 20 | COMMAND ${CMAKE_COMMAND} -E copy 21 | ${CMAKE_CURRENT_SOURCE_DIR}/runtargets.sh 22 | ${CMAKE_CURRENT_SOURCE_DIR}/precice-config.xml 23 | ${CMAKE_CURRENT_BINARY_DIR} 24 | COMMENT "Copying the deal.II + preCICE run script and config ..." 25 | ) 26 | add_custom_target(copy_precice_data ALL 27 | DEPENDS runtargets.sh precice-config.xml 28 | ) 29 | 30 | xsdk_add_test( 31 | NAME 32 | "deal.II-preCICE-laplace_problem" 33 | COMMAND 34 | runtargets.sh 35 | ) 36 | -------------------------------------------------------------------------------- /dealii/precice/README.md: -------------------------------------------------------------------------------- 1 | # deal.II-preCICE example 2 | 3 | A simple example on how to use deal.II with preCICE: a Laplace equation implemented in deal.II is coupled to a stand-alone C++-only boundary condition by calling preCICE in both codes. This example is taken from the [deal.II code gallery](https://github.com/dealii/code-gallery/tree/master/coupled_laplace_problem) and was contributed there by @davidscn. 4 | 5 | - `boundary_condition`: Boundary condition calling preCICE for coupling 6 | - `laplace_problem.cc`: Laplace equation implemented in deal.II and calling preCICE for coupling 7 | - `precice-config.xml`: preCICE configuration file, read by both codes 8 | - `clean`: Script to clean temporary, result, and log files 9 | 10 | To run the example, run both codes from the level where `precice-config.xml` is. 11 | 12 | ```txt 13 | ./laplace_problem & ./boundary_condition 14 | ``` 15 | -------------------------------------------------------------------------------- /dealii/precice/boundary_condition.cc: -------------------------------------------------------------------------------- 1 | // This program does not use any deal.II functionality and depends only on 2 | // preCICE and the standard libraries. 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | // The program computes a time-varying parabolic boundary condition, which is 9 | // passed to preCICE and serves as Dirichlet boundary condition for the other 10 | // coupling participant. 11 | 12 | // Function to generate boundary values in each time step 13 | void 14 | define_boundary_values(std::vector &boundary_data, 15 | const double time, 16 | const double end_time) 17 | { 18 | // Scale the current time value 19 | const double relative_time = time / end_time; 20 | // Define the amplitude. Values run from -0.5 to 0.5 21 | const double amplitude = (relative_time - 0.5); 22 | // Specify the actual data we want to pass to the other participant. Here, we 23 | // choose a parabola with boundary values 2 in order to enforce continuity 24 | // to adjacent boundaries. 25 | const double n_elements = boundary_data.size(); 26 | const double right_zero = boundary_data.size() - 1; 27 | const double left_zero = 0; 28 | const double offset = 2; 29 | for (uint i = 0; i < n_elements; ++i) 30 | boundary_data[i] = 31 | -amplitude * ((i - left_zero) * (i - right_zero)) + offset; 32 | } 33 | 34 | 35 | int 36 | main() 37 | { 38 | std::cout << "Boundary participant: starting... \n"; 39 | 40 | // Configuration 41 | const std::string configFileName("precice-config.xml"); 42 | const std::string solverName("boundary-participant"); 43 | const std::string meshName("boundary-mesh"); 44 | const std::string dataWriteName("boundary-data"); 45 | 46 | // Adjust to MPI rank and size for parallel computation 47 | const int commRank = 0; 48 | const int commSize = 1; 49 | 50 | precice::SolverInterface precice(solverName, 51 | configFileName, 52 | commRank, 53 | commSize); 54 | 55 | const int meshID = precice.getMeshID(meshName); 56 | const int dimensions = precice.getDimensions(); 57 | const int numberOfVertices = 6; 58 | 59 | const int dataID = precice.getDataID(dataWriteName, meshID); 60 | 61 | // Set up data structures 62 | std::vector writeData(numberOfVertices); 63 | std::vector vertexIDs(numberOfVertices); 64 | std::vector vertices(numberOfVertices * dimensions); 65 | 66 | // Define a boundary mesh 67 | std::cout << "Boundary participant: defining boundary mesh \n"; 68 | const double length = 2; 69 | const double xCoord = 1; 70 | const double deltaY = length / (numberOfVertices - 1); 71 | for (int i = 0; i < numberOfVertices; ++i) 72 | for (int j = 0; j < dimensions; ++j) 73 | { 74 | const unsigned int index = dimensions * i + j; 75 | // The x-coordinate is always 1, i.e., the boundary is parallel to the 76 | // y-axis. The y-coordinate is descending from 1 to -1. 77 | if (j == 0) 78 | vertices[index] = xCoord; 79 | else 80 | vertices[index] = 1 - deltaY * i; 81 | } 82 | 83 | // Pass the vertices to preCICE 84 | precice.setMeshVertices(meshID, 85 | numberOfVertices, 86 | vertices.data(), 87 | vertexIDs.data()); 88 | 89 | // initialize the Solverinterface 90 | double dt = precice.initialize(); 91 | 92 | // Start time loop 93 | const double end_time = 1; 94 | double time = 0; 95 | while (precice.isCouplingOngoing()) 96 | { 97 | // Generate new boundary data 98 | define_boundary_values(writeData, time, end_time); 99 | 100 | { 101 | std::cout << "Boundary participant: writing coupling data \n"; 102 | precice.writeBlockScalarData(dataID, 103 | numberOfVertices, 104 | vertexIDs.data(), 105 | writeData.data()); 106 | } 107 | 108 | dt = precice.advance(dt); 109 | std::cout << "Boundary participant: advancing in time\n"; 110 | 111 | time += dt; 112 | } 113 | 114 | std::cout << "Boundary participant: closing...\n"; 115 | 116 | return 0; 117 | } 118 | -------------------------------------------------------------------------------- /dealii/precice/clean: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | cd ${0%/*} || exit 1 # Run from this directory 3 | 4 | echo "Cleaning..." 5 | 6 | # Remove vtk result files 7 | rm -fv solution-*.vtk 8 | 9 | # Remove the preCICE-related log files 10 | echo "Deleting the preCICE log files..." 11 | rm -fv \ 12 | precice-*.log \ 13 | precice-*-events.json 14 | 15 | rm -rfv precice-run 16 | 17 | echo "Cleaning complete!" 18 | #------------------------------------------------------------------------------ 19 | -------------------------------------------------------------------------------- /dealii/precice/precice-config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /dealii/precice/runtargets.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # Invoke with 4 | # runtargets.sh path/to/targets 5 | 6 | ./laplace_problem & 7 | PID=$! 8 | 9 | ./boundary_condition 10 | RET=$? 11 | 12 | set -e 13 | wait $PID 14 | exit $RET 15 | -------------------------------------------------------------------------------- /dealii/sundials/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | ## 2 | # CMake script for the sundials tutorial program: 3 | ## 4 | 5 | # Set the name of the project and target: 6 | set(TARGET "sundials") 7 | 8 | # Declare all source files the target consists of. Here, this is only 9 | # the one step-X.cc file, but as you expand your project you may wish 10 | # to add other source files as well. If your project becomes much larger, 11 | # you may want to either replace the following statement by something like 12 | # file(GLOB_RECURSE TARGET_SRC "source/*.cc") 13 | # file(GLOB_RECURSE TARGET_INC "include/*.h") 14 | # set(TARGET_SRC ${TARGET_SRC} ${TARGET_INC}) 15 | # or switch altogether to the large project CMakeLists.txt file discussed 16 | # in the "CMake in user projects" page accessible from the "User info" 17 | # page of the documentation. 18 | set(TARGET_SRC 19 | ${TARGET}.cc 20 | ) 21 | 22 | # At least in some cases, deal.II needs to use the MPI wrapper: 23 | if (MPI_CXX_COMPILER) 24 | set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER}) 25 | endif() 26 | 27 | # Usually, you will not need to modify anything beyond this point... 28 | 29 | cmake_minimum_required(VERSION 3.13.4) 30 | 31 | if(NOT DEAL_II_WITH_SUNDIALS) # keep in one line 32 | message(FATAL_ERROR " 33 | Error! This tutorial requires a deal.II library that was configured with the following options: 34 | DEAL_II_WITH_SUNDIALS = ON 35 | However, the deal.II library found at ${DEAL_II_PATH} was configured with these options: 36 | DEAL_II_WITH_SUNDIALS = ${DEAL_II_WITH_SUNDIALS} 37 | This conflicts with the requirements." 38 | ) 39 | endif() 40 | 41 | deal_ii_initialize_cached_variables() 42 | add_executable(${TARGET} ${TARGET_SRC}) 43 | deal_ii_setup_target(${TARGET}) 44 | 45 | xsdk_add_test( 46 | NAME 47 | dealii-${TARGET} 48 | MPI_NPROCS 49 | 1 50 | COMMAND 51 | $ 52 | ) 53 | -------------------------------------------------------------------------------- /dealii/sundials/README.md: -------------------------------------------------------------------------------- 1 | # deal.II-Sundials example 2 | 3 | This is essentially deal.II's step-77 example, see 4 | https://www.dealii.org/current/doxygen/deal.II/step_77.html for documentation. 5 | -------------------------------------------------------------------------------- /heffte/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(heffte_example heffte_example_gpu.cpp) 2 | target_link_libraries(heffte_example Heffte::Heffte) 3 | 4 | xsdk_add_test(NAME heFFTe-MAGMA MPI_NPROCS 2 COMMAND $) 5 | -------------------------------------------------------------------------------- /heffte/README.md: -------------------------------------------------------------------------------- 1 | # heFFTe example 2 | 3 | Usage: 4 | ``` 5 | mpirun -np 2 heffte_example 6 | ``` 7 | 8 | Computes a 3D FFT transform on data of size 4x4x4 using the GPU backend, 9 | the data is distributed across two MPI ranks. 10 | Two transforms are computed, forward and backward, and the result of the 11 | forward transform is scaled by the full factor `1/4^3 = 1/64`. 12 | Thus, the two transforms are mathematical inverses, which is verified 13 | with a simple check in the end. 14 | 15 | #### MAGMA integration 16 | 17 | If MAGMA integration is enabled within heFFTe, then the MAGMA linear 18 | algebra methods are used in place of the internal heFFTe kernels. 19 | Most notably, the scaling BLAS method `xscal()` is implemented 20 | much more efficiently in MAGMA compared to the reference kernels 21 | in heFFTe. 22 | 23 | MAGMA is enabled at compile time and is automatically used without 24 | and additional runtime switches or options. 25 | -------------------------------------------------------------------------------- /heffte/heffte_example_gpu.cpp: -------------------------------------------------------------------------------- 1 | #include "heffte.h" 2 | 3 | #ifdef Heffte_ENABLE_GPU 4 | 5 | // There are multiple ways to select the GPU tag 6 | 7 | // Using the default_backend trait with the tag::gpu for the location 8 | using backend_tag = heffte::backend::default_backend::type; 9 | 10 | void compute_dft(MPI_Comm comm){ 11 | 12 | // wrapper around MPI_Comm_rank() and MPI_Comm_size(), using this is optional 13 | int const my_rank = heffte::mpi::comm_rank(comm); 14 | int const num_ranks = heffte::mpi::comm_size(comm); 15 | 16 | if (num_ranks != 2){ 17 | if (my_rank == 0) std::cout << " heffte_example_cuda is set to 2 mpi ranks, exiting \n"; 18 | return; 19 | } 20 | 21 | // define the domain split between two boxes 22 | heffte::box3d<> const left_box = {{0, 0, 0}, {3, 3, 1}}; 23 | heffte::box3d<> const right_box = {{0, 0, 2}, {3, 3, 3}}; 24 | 25 | // the box associated with this MPI rank 26 | heffte::box3d<> const my_box = (my_rank == 0) ? left_box : right_box; 27 | 28 | if (heffte::gpu::device_count() > 1){ 29 | // on a multi-gpu system, distribute the devices across the mpi ranks 30 | heffte::gpu::device_set(heffte::mpi::comm_rank(comm) % heffte::gpu::device_count()); 31 | } 32 | 33 | heffte::plan_options options = heffte::default_options(); 34 | options.use_gpu_aware = false; 35 | 36 | // define the heffte class and the input and output geometry 37 | // heffte::plan_options can be specified just as in the backend::fftw 38 | heffte::fft3d fft(my_box, my_box, comm, options); 39 | 40 | // create some input on the CPU 41 | std::vector> input(fft.size_inbox()); 42 | std::iota(input.begin(), input.end(), 0); // put some data in the input 43 | 44 | // load the input into the GPU memory 45 | // this is equivalent to cudaMalloc() followed by cudaMemcpy() 46 | // the destructor of heffte::gpu::vector will call cudaFree() 47 | heffte::gpu::vector> gpu_input = heffte::gpu::transfer().load(input); 48 | 49 | // allocate memory on the device for the output 50 | heffte::gpu::vector> gpu_output(fft.size_outbox()); 51 | 52 | // allocate scratch space, this is using the public type alias buffer_container 53 | // and for the cufft backend this is heffte::gpu::vector 54 | // for the CPU backends (fftw and mkl) the buffer_container is std::vector 55 | heffte::fft3d::buffer_container> workspace(fft.size_workspace()); 56 | static_assert(std::is_same::value, 57 | "the containers for the output and workspace have different types"); 58 | 59 | // perform forward fft using arrays and the user-created workspace 60 | // if heFFTe has been compiled with MAGMA capabilities, the scaling below will automatically use MAGMA 61 | fft.forward(gpu_input.data(), gpu_output.data(), workspace.data(), heffte::scale::full); 62 | 63 | // optional step, free the workspace since the inverse will use the vector API 64 | workspace = heffte::gpu::vector>(); 65 | 66 | // compute the inverse FFT transform using the container API 67 | heffte::gpu::vector> gpu_inverse = fft.backward(gpu_output); 68 | 69 | // move the result back to the CPU for comparison purposes 70 | std::vector> inverse = heffte::gpu::transfer::unload(gpu_inverse); 71 | 72 | // compute the error between the input and the inverse 73 | double err = 0.0; 74 | for(size_t i=0; i 17 | -dslu_th 18 | 50 19 | ) 20 | 21 | install(TARGETS ij_laplacian RUNTIME DESTINATION bin) 22 | 23 | # Copy glvis helper script and directory 24 | file(COPY vis DESTINATION .) 25 | install( 26 | DIRECTORY vis 27 | DESTINATION share/xsdk-examples/hypre 28 | PATTERN 29 | vis/* 30 | PERMISSIONS 31 | OWNER_EXECUTE 32 | OWNER_WRITE 33 | OWNER_READ 34 | GROUP_EXECUTE 35 | GROUP_READ 36 | WORLD_READ 37 | WORLD_EXECUTE 38 | ) 39 | else() 40 | message( 41 | STATUS 42 | "SKIPPED HYPRE-ij_laplacian example. Requires ENABLE_SUPERLU=ON, ENABLE_CUDA=OFF, and ENABLE_HIP=OFF but got ${ENABLE_SUPERLU}, ${ENABLE_CUDA}, and ${ENABLE_HIP}" 43 | ) 44 | endif() 45 | -------------------------------------------------------------------------------- /hypre/COPYRIGHT: -------------------------------------------------------------------------------- 1 | 2 | Intellectual Property Notice 3 | ---------------------------- 4 | 5 | HYPRE is licensed under the Apache License, Version 2.0 (see LICENSE-APACHE or 6 | http://www.apache.org/licenses/LICENSE-2.0) or the MIT license (see LICENSE-MIT 7 | or http://opensource.org/licenses/MIT), at your option. 8 | 9 | Copyrights and patents in the HYPRE project are retained by contributors. 10 | No copyright assignment is required to contribute to HYPRE. 11 | 12 | HYPRE's original open source license was LGPL-2.1. Consent from contributors 13 | was received and documented before relicensing to Apache-2.0/MIT. 14 | 15 | 16 | SPDX usage 17 | ---------- 18 | 19 | Individual files contain SPDX tags instead of the full license text. 20 | This enables machine processing of license information based on the SPDX 21 | License Identifiers that are available here: https://spdx.org/licenses/ 22 | 23 | Files that are dual-licensed as Apache-2.0 OR MIT contain the following 24 | text in the license header: 25 | 26 | SPDX-License-Identifier: (Apache-2.0 OR MIT) 27 | 28 | 29 | External Software 30 | ----------------- 31 | 32 | External software in hypre is covered by various permissive licenses. A summary 33 | listing follows. See the license included with the software for full details. 34 | 35 | Directory: src/blas 36 | License: University of Tennessee 37 | 38 | Directory: src/lapack 39 | License: University of Tennessee 40 | 41 | -------------------------------------------------------------------------------- /hypre/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright 1998-2019 Lawrence Livermore National Security, LLC and other 2 | HYPRE Project Developers. See the top-level COPYRIGHT file for details. 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a 5 | copy of this software and associated documentation files (the "Software"), 6 | to deal in the Software without restriction, including without limitation 7 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 | and/or sell copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in 12 | all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 | DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /hypre/NOTICE: -------------------------------------------------------------------------------- 1 | This work was produced under the auspices of the U.S. Department of Energy by 2 | Lawrence Livermore National Laboratory under Contract DE-AC52-07NA27344. 3 | 4 | This work was prepared as an account of work sponsored by an agency of 5 | the United States Government. Neither the United States Government nor 6 | Lawrence Livermore National Security, LLC, nor any of their employees 7 | makes any warranty, expressed or implied, or assumes any legal liability 8 | or responsibility for the accuracy, completeness, or usefulness of any 9 | information, apparatus, product, or process disclosed, or represents that 10 | its use would not infringe privately owned rights. 11 | 12 | Reference herein to any specific commercial product, process, or service 13 | by trade name, trademark, manufacturer, or otherwise does not necessarily 14 | constitute or imply its endorsement, recommendation, or favoring by the 15 | United States Government or Lawrence Livermore National Security, LLC. 16 | 17 | The views and opinions of authors expressed herein do not necessarily 18 | state or reflect those of the United States Government or Lawrence 19 | Livermore National Security, LLC, and shall not be used for advertising 20 | or product endorsement purposes. 21 | -------------------------------------------------------------------------------- /hypre/README.md: -------------------------------------------------------------------------------- 1 | 7 | # HYPRE examples 8 | 9 | Example codes demonstrating the use of [HYPRE](https://computing.llnl.gov/projects/hypre-scalable-linear-solvers-multigrid-methods) using other XSDK packages (SuperLU_DIST). 10 | 11 | ## HYPRE + SuperLU_DIST 12 | 13 | The `ij_laplacian.c` example demonstrates HYPRE's Linear-Algebraic (IJ) interface 14 | for solving the 2-D Laplacian problem with zero boundary conditions on an n x n grid. 15 | The number of unknowns is N=n^2. 16 | The standard 5-point stencil is used, and we solve for the interior nodes only. 17 | 18 | Available solvers are AMG (default), PCG, PCG with AMG or Parasails 19 | preconditioners, flexible GMRES with AMG. 20 | Within HYPRE's AMG V-cycle there is an option to use a 21 | distributed LU factorization from 22 | SuperLU_DIST as the coarse level solver. 23 | 24 | ### Usage 25 | 26 | **Sample run**: `mpirun -np 4 ij_laplacian -dslu_th 50` 27 | 28 | - This run solves a system corresponding to a discretization 29 | of the Laplace equation -Delta u = 1 with zero boundary 30 | conditions on a 33 x 33 grid, using HYPRE's BoomerAMG with SuperLU_DIST's 31 | distributed LU factorization for the coarse level solver where `50` 32 | is a maximum number of coarse level degrees of freedom using 4 MPI tasks. 33 | - The output of the example is various information regarding the 34 | solver and solver performance. 35 | 36 | By specifying the command line parameter `-dslu_th` to be 37 | the maximum coarse level number of degrees of freedom, the 38 | coarse level solver within BoomerAMG will be changed from the 39 | default Gaussian Elimination, to a sparse LU decomposition and 40 | triangular solution from SuperLU_DIST. 41 | 42 | Optionally running with `-vis` command-line option, the solution is saved for GLVis visualization (see `vis/glvis-ij-laplacian.sh` for example usage of GLVis visualization). 43 | 44 | To see the full list of command-line options, run: `ij_laplacian -help` 45 | 46 | ## License 47 | HYPRE is distributed under the terms of both the MIT license and the Apache 48 | License (Version 2.0). Users may choose either license, at their option. 49 | 50 | All new contributions must be made under both the MIT and Apache-2.0 licenses. 51 | 52 | See [LICENSE-MIT](./LICENSE-MIT), [LICENSE-APACHE](./LICENSE-APACHE), 53 | [COPYRIGHT](./COPYRIGHT), and [NOTICE](./NOTICE) for details. 54 | 55 | SPDX-License-Identifier: (Apache-2.0 OR MIT) 56 | 57 | LLNL-CODE-778117 58 | -------------------------------------------------------------------------------- /hypre/vis/glvis-ij-laplacian.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright 1998-2019 Lawrence Livermore National Security, LLC and other 3 | # HYPRE Project Developers. See the top-level COPYRIGHT file for details. 4 | # 5 | # SPDX-License-Identifier: (Apache-2.0 OR MIT) 6 | 7 | 8 | ex=ij-laplacian 9 | dir=`basename \`pwd\`` 10 | keys=Aaamc 11 | 12 | if [ "$dir" = "vis" ]; then 13 | dir=. 14 | mesh=$ex.mesh 15 | sol=$ex.sol 16 | else 17 | dir=vis 18 | mesh=vis/$ex.mesh 19 | sol=vis/$ex.sol 20 | fi 21 | 22 | if [ ! -e $mesh ] 23 | then 24 | echo "Can't find visualization data for $ex!" 25 | exit 26 | fi 27 | 28 | echo "FiniteElementSpace" > $sol 29 | echo "FiniteElementCollection: H1_2D_P1" >> $sol 30 | echo "VDim: 1" >> $sol 31 | echo "Ordering: 0" >> $sol 32 | echo "" >> $sol 33 | find $dir -name "$ex.sol.??????" | sort | xargs cat >> $sol 34 | 35 | glvis -m $mesh -g $sol -k $keys 36 | -------------------------------------------------------------------------------- /libensemble/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018, UChicago Argonne, LLC 2 | 3 | All Rights Reserved 4 | Software Name: LibEnsemble 5 | By: Argonne National Laboratory 6 | OPEN SOURCE LICENSE (BSD) 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are met: 10 | 11 | 1 Redistributions of source code must retain the above copyright notice, this 12 | list of conditions and the following disclaimer. 13 | 14 | 2 Redistributions in binary form must reproduce the above copyright notice, 15 | this list of conditions and the following disclaimer in the documentation 16 | and/or other materials provided with the distribution. 17 | 18 | 3 Neither the name of the copyright holder nor the names of its contributors 19 | may be used to endorse or promote products derived from this software without 20 | specific prior written permission. 21 | 22 | 23 | ******************************************************************************** 24 | DISCLAIMER 25 | 26 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 27 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 29 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 30 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 32 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 33 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 34 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 35 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 36 | ******************************************************************************** 37 | -------------------------------------------------------------------------------- /libensemble/README.md: -------------------------------------------------------------------------------- 1 | ## libEnsemble with PETSc/TAO 2 | 3 | This example script demonstrates interoperability between libEnsemble and 4 | PETSc/TAO; another xSDK package. 5 | 6 | #### Note: The supplied scripts work with libEnsemble v0.5.2 (As used in xSDK release v0.5.0). 7 | 8 | This script uses libEnsemble to optimize a two-dimensional algebraic function, the [six-hump camel](https://www.sfu.ca/~ssurjano/camel6.html) function, over a bound-constrained domain. 9 | 10 | The optimization is governed by the [APOSMM](https://www.mcs.anl.gov/~jlarson/APOSMM/) generating function, which performs multistart local optimization. 11 | 12 | The local optimization instances are governed by a quasi-Newton method in 13 | PETSc/TAO, [BLMVM](https://www.mcs.anl.gov/petsc/petsc-current/docs/manualpages/Tao/TAOBLMVM.html). 14 | 15 | ### Usage 16 | 17 | Execute via one of the following commands (e.g. 3 workers). 18 | 19 | Using MPI (mpi4py): 20 | 21 | mpiexec -np 4 python3 test_persistent_aposmm_tao.py 22 | 23 | Using multiprocessing: 24 | 25 | python3 test_persistent_aposmm_tao.py --nworkers 3 --comms local 26 | 27 | An exit code of zero indicates the test has passed. -------------------------------------------------------------------------------- /libensemble/persistent_aposmm_alloc.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from libensemble.alloc_funcs.support import avail_worker_ids, sim_work, gen_work 4 | 5 | 6 | def persistent_aposmm_alloc(W, H, sim_specs, gen_specs, alloc_specs, persis_info): 7 | """ 8 | This allocation function will give simulation work if possible, but 9 | otherwise start a persistent APOSMM generator. If all points requested by 10 | the persistent generator have been returned from the simulation evaluation, 11 | then this information is given back to the persistent generator. 12 | 13 | This function assumes that one persistent APOSMM will be started and never 14 | stopped (until some exit_criterion is satisfied). 15 | 16 | .. seealso:: 17 | `test_6-hump_camel_persistent_aposmm.py `_ 18 | """ 19 | 20 | Work = {} 21 | if persis_info.get('first_call', True): 22 | assert np.all(H['given']), "Initial points in H have never been given." 23 | assert np.all(H['given_back']), "Initial points in H have never been given_back." 24 | assert np.all(H['returned']), "Initial points in H have never been returned." 25 | persis_info['fields_to_give_back'] = ['f'] + [n[0] for n in gen_specs['out']] 26 | 27 | if 'grad' in [n[0] for n in sim_specs['out']]: 28 | persis_info['fields_to_give_back'] += ['grad'] 29 | 30 | if 'fvec' in [n[0] for n in sim_specs['out']]: 31 | persis_info['fields_to_give_back'] += ['fvec'] 32 | 33 | persis_info['samples_in_H0'] = sum(H['local_pt'] == 0) 34 | persis_info['next_to_give'] = len(H) # 35 | persis_info['first_call'] = False 36 | 37 | # If any persistent worker's calculated values have returned, give them back. 38 | for i in avail_worker_ids(W, persistent=True): 39 | if persis_info.get('sample_done') or sum(H['returned']) >= gen_specs['initial_sample_size'] + persis_info['samples_in_H0']: 40 | # Don't return if the initial sample is not complete 41 | persis_info['sample_done'] = True 42 | 43 | returned_but_not_given = np.logical_and(H['returned'], ~H['given_back']) 44 | if np.any(returned_but_not_given): 45 | inds_to_give = np.where(returned_but_not_given)[0] 46 | 47 | gen_work(Work, i, persis_info['fields_to_give_back'], 48 | np.atleast_1d(inds_to_give), persis_info[i], persistent=True) 49 | 50 | H['given_back'][inds_to_give] = True 51 | 52 | for i in avail_worker_ids(W, persistent=False): 53 | if persis_info['next_to_give'] < len(H): 54 | # perform sim evaluations (if they exist in History). 55 | sim_work(Work, i, sim_specs['in'], np.atleast_1d(persis_info['next_to_give']), persis_info[i]) 56 | persis_info['next_to_give'] += 1 57 | 58 | elif persis_info.get('gen_started') is None: 59 | # Finally, call a persistent generator as there is nothing else to do. 60 | persis_info['gen_started'] = True 61 | 62 | gen_work(Work, i, gen_specs['in'], range(len(H)), persis_info[i], 63 | persistent=True) 64 | 65 | return Work, persis_info 66 | -------------------------------------------------------------------------------- /libensemble/test_persistent_aposmm_tao.py: -------------------------------------------------------------------------------- 1 | # """ 2 | # Runs libEnsemble on the 6-hump camel problem. Documented here: 3 | # https://www.sfu.ca/~ssurjano/camel6.html 4 | # 5 | # Execute via one of the following commands (e.g. 3 workers): 6 | # mpiexec -np 4 python3 test_persistent_aposmm_tao.py 7 | # python3 test_persistent_aposmm_tao.py --nworkers 3 --comms local 8 | # python3 test_persistent_aposmm_tao.py --nworkers 3 --comms tcp 9 | # 10 | # The number of concurrent evaluations of the objective function will be 4-1=3. 11 | # """ 12 | 13 | # Do not change these lines - they are parsed by run-tests.sh 14 | # TESTSUITE_COMMS: local 15 | # TESTSUITE_NPROCS: 4 16 | 17 | import sys 18 | import numpy as np 19 | 20 | # Import libEnsemble items for this test 21 | from libensemble.libE import libE 22 | from math import gamma, pi, sqrt 23 | from libensemble.sim_funcs.six_hump_camel import six_hump_camel as sim_f 24 | from persistent_aposmm import aposmm as gen_f 25 | from persistent_aposmm_alloc import persistent_aposmm_alloc as alloc_f 26 | from libensemble.tests.regression_tests.common import parse_args, save_libE_output, per_worker_stream 27 | from libensemble.tests.regression_tests.support import six_hump_camel_minima as minima 28 | from time import time 29 | 30 | nworkers, is_master, libE_specs, _ = parse_args() 31 | 32 | if is_master: 33 | start_time = time() 34 | 35 | if nworkers < 2: 36 | sys.exit("Cannot run with a persistent worker if only one worker -- aborting...") 37 | 38 | n = 2 39 | sim_specs = {'sim_f': sim_f, 40 | 'in': ['x'], 41 | 'out': [('f', float), ('grad', float, n)]} 42 | 43 | gen_out = [('x', float, n), ('x_on_cube', float, n), ('sim_id', int), 44 | ('local_min', bool), ('local_pt', bool)] 45 | 46 | gen_specs = {'gen_f': gen_f, 47 | 'in': [], 48 | 'out': gen_out, 49 | 'initial_sample_size': 100, 50 | 'sample_points': np.round(minima, 1), 51 | 'localopt_method': 'blmvm', 52 | 'rk_const': 0.5*((gamma(1+(n/2))*5)**(1/n))/sqrt(pi), 53 | 'grtol': 1e-4, 54 | 'gatol': 1e-4, 55 | 'dist_to_bound_multiple': 0.5, 56 | 'max_active_runs': 6, 57 | 'lb': np.array([-3, -2]), 58 | 'ub': np.array([3, 2]) 59 | } 60 | 61 | alloc_specs = {'alloc_f': alloc_f, 'out': [('given_back', bool)], 'user': {}} 62 | 63 | persis_info = per_worker_stream({}, nworkers + 1) 64 | 65 | exit_criteria = {'sim_max': 1000} 66 | 67 | # Perform the run 68 | H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, persis_info, 69 | alloc_specs, libE_specs) 70 | 71 | if is_master: 72 | print('[Manager]:', H[np.where(H['local_min'])]['x']) 73 | print('[Manager]: Time taken =', time() - start_time, flush=True) 74 | 75 | tol = 1e-5 76 | for m in minima: 77 | # The minima are known on this test problem. 78 | # We use their values to test APOSMM has identified all minima 79 | print(np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)), flush=True) 80 | assert np.min(np.sum((H[H['local_min']]['x'] - m)**2, 1)) < tol 81 | 82 | save_libE_output(H, persis_info, __file__, nworkers) 83 | -------------------------------------------------------------------------------- /mfem/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # All MFEM dependencies 2 | set(MFEM_ALL_DEPS XSDK::MFEM) 3 | 4 | # Ginkgo is included at the upper level when MFEM is enabled 5 | if(ENABLE_GINKGO) 6 | add_subdirectory(ginkgo) 7 | endif() 8 | if(ENABLE_HIOP) 9 | add_subdirectory(hiop) 10 | endif() 11 | if(ENABLE_HYPRE) 12 | add_subdirectory(hypre) 13 | endif() 14 | if(ENABLE_HYPRE AND ENABLE_STRUMPACK) 15 | add_subdirectory(strumpack) 16 | endif() 17 | if(ENABLE_HYPRE AND ENABLE_SUNDIALS) 18 | add_subdirectory(sundials) 19 | endif() 20 | if(ENABLE_HYPRE AND ENABLE_SUPERLU) 21 | add_subdirectory(hypre-superlu) 22 | endif() 23 | if(ENABLE_PETSC) 24 | add_subdirectory(petsc) 25 | endif() 26 | if(ENABLE_PUMI) 27 | add_subdirectory(pumi) 28 | endif() 29 | 30 | # Copy the data directory from the source tree to the build tree so that the examples can be run 31 | # from in the build tree. 32 | add_custom_command( 33 | OUTPUT data_is_copied 34 | COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/data data 35 | COMMAND ${CMAKE_COMMAND} -E touch data_is_copied 36 | COMMENT "Copying the MFEM data directory ..." 37 | ) 38 | add_custom_target(copy_mfem_data ALL DEPENDS data_is_copied) 39 | 40 | # Install 'data' in the install prefix, next to 'bin': 41 | install(DIRECTORY data DESTINATION ".") 42 | -------------------------------------------------------------------------------- /mfem/README.md: -------------------------------------------------------------------------------- 1 | # MFEM examples 2 | Example codes demonstrating the use of MFEM using other XSDK packages. 3 | -------------------------------------------------------------------------------- /mfem/data/README.md: -------------------------------------------------------------------------------- 1 | # data 2 | 3 | This folder contains mesh files used by the MFEM example programs. 4 | -------------------------------------------------------------------------------- /mfem/data/amr-quad.mesh: -------------------------------------------------------------------------------- 1 | MFEM NC mesh v1.0 2 | 3 | # NCMesh supported geometry types: 4 | # SEGMENT = 1 5 | # TRIANGLE = 2 6 | # SQUARE = 3 7 | # TETRAHEDRON = 4 8 | # CUBE = 5 9 | # PRISM = 6 10 | 11 | dimension 12 | 2 13 | 14 | # rank attr geom ref_type nodes/children 15 | elements 16 | 37 17 | -1 1 3 3 1 10 19 28 18 | -1 1 3 3 2 3 4 9 19 | 0 1 3 0 0 9 21 12 20 | 0 1 3 0 9 4 10 21 21 | -1 1 3 3 5 6 7 8 22 | 0 1 3 0 21 22 37 25 23 | 0 1 3 0 22 10 23 37 24 | 0 1 3 0 37 23 8 24 25 | 0 1 3 0 25 37 24 11 26 | 0 1 3 0 12 21 11 7 27 | -1 1 3 3 11 12 13 14 28 | 0 1 3 0 4 13 26 10 29 | 0 1 3 0 13 1 14 26 30 | 0 1 3 0 26 14 5 15 31 | -1 1 3 3 15 16 17 18 32 | 0 1 3 0 10 27 38 23 33 | 0 1 3 0 27 26 28 38 34 | 0 1 3 0 38 28 15 29 35 | 0 1 3 0 23 38 29 8 36 | -1 1 3 3 20 25 26 27 37 | -1 1 3 3 21 22 23 24 38 | 0 1 3 0 8 29 39 33 39 | 0 1 3 0 29 15 31 39 40 | 0 1 3 0 39 31 30 32 41 | 0 1 3 0 33 39 32 18 42 | 0 1 3 0 15 5 16 30 43 | 0 1 3 0 30 16 3 17 44 | 0 1 3 0 18 30 17 6 45 | -1 1 3 3 29 30 35 36 46 | 0 1 3 0 7 11 34 20 47 | -1 1 3 3 31 32 33 34 48 | 0 1 3 0 11 24 40 35 49 | 0 1 3 0 24 8 33 40 50 | 0 1 3 0 40 33 18 36 51 | 0 1 3 0 35 40 36 34 52 | 0 1 3 0 34 18 6 19 53 | 0 1 3 0 20 34 19 2 54 | 55 | # attr geom nodes 56 | boundary 57 | 16 58 | 1 1 0 9 59 | 4 1 12 0 60 | 1 1 9 4 61 | 4 1 7 12 62 | 1 1 4 13 63 | 1 1 13 1 64 | 2 1 1 14 65 | 2 1 14 5 66 | 2 1 5 16 67 | 2 1 16 3 68 | 3 1 3 17 69 | 3 1 17 6 70 | 4 1 20 7 71 | 3 1 6 19 72 | 3 1 19 2 73 | 4 1 2 20 74 | 75 | # vert_id p1 p2 76 | vertex_parents 77 | 37 78 | 4 0 1 79 | 5 1 3 80 | 6 2 3 81 | 7 0 2 82 | 8 4 6 83 | 9 0 4 84 | 10 4 8 85 | 11 7 8 86 | 12 0 7 87 | 13 1 4 88 | 14 1 5 89 | 15 5 8 90 | 16 3 5 91 | 17 3 6 92 | 18 6 8 93 | 19 2 6 94 | 20 2 7 95 | 21 9 11 96 | 22 10 21 97 | 23 8 10 98 | 24 8 11 99 | 25 11 21 100 | 26 13 15 101 | 27 10 26 102 | 28 15 26 103 | 29 8 15 104 | 30 15 17 105 | 31 15 30 106 | 32 18 30 107 | 33 8 18 108 | 34 11 19 109 | 35 11 34 110 | 36 18 34 111 | 37 22 24 112 | 38 27 29 113 | 39 29 32 114 | 40 24 36 115 | 116 | # top-level node coordinates 117 | coordinates 118 | 4 119 | 2 120 | 0 0 121 | 1 0 122 | 0 1 123 | 1 1 124 | 125 | mfem_mesh_end 126 | -------------------------------------------------------------------------------- /mfem/data/beam-hex.mesh: -------------------------------------------------------------------------------- 1 | MFEM mesh v1.0 2 | 3 | # 4 | # MFEM Geometry Types (see mesh/geom.hpp): 5 | # 6 | # POINT = 0 7 | # SEGMENT = 1 8 | # TRIANGLE = 2 9 | # SQUARE = 3 10 | # TETRAHEDRON = 4 11 | # CUBE = 5 12 | # 13 | 14 | dimension 15 | 3 16 | 17 | elements 18 | 8 19 | 1 5 0 1 10 9 18 19 28 27 20 | 1 5 1 2 11 10 19 20 29 28 21 | 1 5 2 3 12 11 20 21 30 29 22 | 1 5 3 4 13 12 21 22 31 30 23 | 2 5 4 5 14 13 22 23 32 31 24 | 2 5 5 6 15 14 23 24 33 32 25 | 2 5 6 7 16 15 24 25 34 33 26 | 2 5 7 8 17 16 25 26 35 34 27 | 28 | boundary 29 | 34 30 | 3 3 9 10 1 0 31 | 3 3 0 1 19 18 32 | 3 3 10 9 27 28 33 | 1 3 9 0 18 27 34 | 3 3 18 19 28 27 35 | 3 3 10 11 2 1 36 | 3 3 1 2 20 19 37 | 3 3 11 10 28 29 38 | 3 3 19 20 29 28 39 | 3 3 11 12 3 2 40 | 3 3 2 3 21 20 41 | 3 3 12 11 29 30 42 | 3 3 20 21 30 29 43 | 3 3 12 13 4 3 44 | 3 3 3 4 22 21 45 | 3 3 13 12 30 31 46 | 3 3 21 22 31 30 47 | 3 3 13 14 5 4 48 | 3 3 4 5 23 22 49 | 3 3 14 13 31 32 50 | 3 3 22 23 32 31 51 | 3 3 14 15 6 5 52 | 3 3 5 6 24 23 53 | 3 3 15 14 32 33 54 | 3 3 23 24 33 32 55 | 3 3 15 16 7 6 56 | 3 3 6 7 25 24 57 | 3 3 16 15 33 34 58 | 3 3 24 25 34 33 59 | 3 3 16 17 8 7 60 | 3 3 7 8 26 25 61 | 2 3 8 17 35 26 62 | 3 3 17 16 34 35 63 | 3 3 25 26 35 34 64 | 65 | vertices 66 | 36 67 | 3 68 | 0 0 0 69 | 1 0 0 70 | 2 0 0 71 | 3 0 0 72 | 4 0 0 73 | 5 0 0 74 | 6 0 0 75 | 7 0 0 76 | 8 0 0 77 | 0 1 0 78 | 1 1 0 79 | 2 1 0 80 | 3 1 0 81 | 4 1 0 82 | 5 1 0 83 | 6 1 0 84 | 7 1 0 85 | 8 1 0 86 | 0 0 1 87 | 1 0 1 88 | 2 0 1 89 | 3 0 1 90 | 4 0 1 91 | 5 0 1 92 | 6 0 1 93 | 7 0 1 94 | 8 0 1 95 | 0 1 1 96 | 1 1 1 97 | 2 1 1 98 | 3 1 1 99 | 4 1 1 100 | 5 1 1 101 | 6 1 1 102 | 7 1 1 103 | 8 1 1 104 | -------------------------------------------------------------------------------- /mfem/data/disc-nurbs.mesh: -------------------------------------------------------------------------------- 1 | MFEM NURBS mesh v1.0 2 | 3 | # 4 | # MFEM Geometry Types (see mesh/geom.hpp): 5 | # 6 | # SEGMENT = 1 7 | # SQUARE = 3 8 | # CUBE = 5 9 | # 10 | 11 | dimension 12 | 2 13 | 14 | elements 15 | 5 16 | 1 3 4 5 6 7 17 | 1 3 0 1 5 4 18 | 1 3 1 2 6 5 19 | 1 3 3 7 6 2 20 | 1 3 0 4 7 3 21 | 22 | boundary 23 | 4 24 | 1 1 0 1 25 | 1 1 2 3 26 | 1 1 1 2 27 | 1 1 3 0 28 | 29 | edges 30 | 12 31 | 0 0 1 32 | 0 4 5 33 | 0 7 6 34 | 0 3 2 35 | 1 1 2 36 | 1 5 6 37 | 1 4 7 38 | 1 0 3 39 | 2 0 4 40 | 2 1 5 41 | 2 2 6 42 | 2 3 7 43 | 44 | vertices 45 | 8 46 | 47 | knotvectors 48 | 3 49 | 2 3 0 0 0 1 1 1 50 | 2 3 0 0 0 1 1 1 51 | 2 3 0 0 0 1 1 1 52 | 53 | weights 54 | 1 1 1 1 1 1 1 1 55 | 56 | 0.7071067811865475244 57 | 1 58 | 1 59 | 0.7071067811865475244 60 | 61 | 0.7071067811865475244 62 | 1 63 | 1 64 | 0.7071067811865475244 65 | 66 | 1 67 | 1 68 | 1 69 | 1 70 | 71 | 1 72 | 0.8535533905932737622 73 | 0.8535533905932737622 74 | 0.8535533905932737622 75 | 0.8535533905932737622 76 | 77 | FiniteElementSpace 78 | FiniteElementCollection: NURBS2 79 | VDim: 2 80 | Ordering: 1 81 | 82 | -2 -2 83 | 2 -2 84 | 2 2 85 | -2 2 86 | -1 -1 87 | 1 -1 88 | 1 1 89 | -1 1 90 | 91 | 0 -4 92 | 0 -1 93 | 0 1 94 | 0 4 95 | 96 | 4 0 97 | 1 0 98 | -1 0 99 | -4 0 100 | 101 | -1.5 -1.5 102 | 1.5 -1.5 103 | 1.5 1.5 104 | -1.5 1.5 105 | 106 | 0 0 107 | 0 -2.5 108 | 2.5 0 109 | 0 2.5 110 | -2.5 0 111 | -------------------------------------------------------------------------------- /mfem/data/periodic-cube.mesh: -------------------------------------------------------------------------------- 1 | MFEM mesh v1.0 2 | 3 | # 4 | # MFEM Geometry Types (see mesh/geom.hpp): 5 | # 6 | # POINT = 0 7 | # SEGMENT = 1 8 | # TRIANGLE = 2 9 | # SQUARE = 3 10 | # TETRAHEDRON = 4 11 | # CUBE = 5 12 | # 13 | 14 | dimension 15 | 3 16 | 17 | # format: ... 18 | elements 19 | 27 20 | 1 5 0 1 4 3 9 10 13 12 21 | 1 5 1 2 5 4 10 11 14 13 22 | 1 5 2 0 3 5 11 9 12 14 23 | 1 5 3 4 7 6 12 13 16 15 24 | 1 5 4 5 8 7 13 14 17 16 25 | 1 5 5 3 6 8 14 12 15 17 26 | 1 5 6 7 1 0 15 16 10 9 27 | 1 5 7 8 2 1 16 17 11 10 28 | 1 5 8 6 0 2 17 15 9 11 29 | 1 5 9 10 13 12 18 19 22 21 30 | 1 5 10 11 14 13 19 20 23 22 31 | 1 5 11 9 12 14 20 18 21 23 32 | 1 5 12 13 16 15 21 22 25 24 33 | 1 5 13 14 17 16 22 23 26 25 34 | 1 5 14 12 15 17 23 21 24 26 35 | 1 5 15 16 10 9 24 25 19 18 36 | 1 5 16 17 11 10 25 26 20 19 37 | 1 5 17 15 9 11 26 24 18 20 38 | 1 5 18 19 22 21 0 1 4 3 39 | 1 5 19 20 23 22 1 2 5 4 40 | 1 5 20 18 21 23 2 0 3 5 41 | 1 5 21 22 25 24 3 4 7 6 42 | 1 5 22 23 26 25 4 5 8 7 43 | 1 5 23 21 24 26 5 3 6 8 44 | 1 5 24 25 19 18 6 7 1 0 45 | 1 5 25 26 20 19 7 8 2 1 46 | 1 5 26 24 18 20 8 6 0 2 47 | 48 | boundary 49 | 54 50 | 1 3 0 1 4 3 51 | 1 3 1 2 5 4 52 | 1 3 2 0 3 5 53 | 1 3 3 4 7 6 54 | 1 3 4 5 8 7 55 | 1 3 5 3 6 8 56 | 1 3 6 7 1 0 57 | 1 3 7 8 2 1 58 | 1 3 8 6 0 2 59 | 6 3 4 1 0 3 60 | 6 3 5 2 1 4 61 | 6 3 3 0 2 5 62 | 6 3 7 4 3 6 63 | 6 3 8 5 4 7 64 | 6 3 6 3 5 8 65 | 6 3 1 7 6 0 66 | 6 3 2 8 7 1 67 | 6 3 0 6 8 2 68 | 2 3 0 3 12 9 69 | 2 3 3 6 15 12 70 | 2 3 6 0 9 15 71 | 2 3 9 12 21 18 72 | 2 3 12 15 24 21 73 | 2 3 15 9 18 24 74 | 2 3 18 21 3 0 75 | 2 3 21 24 6 3 76 | 2 3 24 18 0 6 77 | 5 3 12 3 0 9 78 | 5 3 15 6 3 12 79 | 5 3 9 0 6 15 80 | 5 3 21 12 9 18 81 | 5 3 24 15 12 21 82 | 5 3 18 9 15 24 83 | 5 3 3 21 18 0 84 | 5 3 6 24 21 3 85 | 5 3 0 18 24 6 86 | 3 3 0 1 10 9 87 | 3 3 1 2 11 10 88 | 3 3 2 0 9 11 89 | 3 3 9 10 19 18 90 | 3 3 10 11 20 19 91 | 3 3 11 9 18 20 92 | 3 3 18 19 1 0 93 | 3 3 19 20 2 1 94 | 3 3 20 18 0 2 95 | 4 3 10 1 0 9 96 | 4 3 11 2 1 10 97 | 4 3 9 0 2 11 98 | 4 3 19 10 9 18 99 | 4 3 20 11 10 19 100 | 4 3 18 9 11 20 101 | 4 3 1 19 18 0 102 | 4 3 2 20 19 1 103 | 4 3 0 18 20 2 104 | 105 | vertices 106 | 27 107 | 108 | nodes 109 | FiniteElementSpace 110 | FiniteElementCollection: L2_T1_3D_P1 111 | VDim: 3 112 | Ordering: 1 113 | 114 | -1.000000 -1.000000 -1.000000 115 | -0.333333 -1.000000 -1.000000 116 | -1.000000 -0.333333 -1.000000 117 | -0.333333 -0.333333 -1.000000 118 | -1.000000 -1.000000 -0.333333 119 | -0.333333 -1.000000 -0.333333 120 | -1.000000 -0.333333 -0.333333 121 | -0.333333 -0.333333 -0.333333 122 | -0.333333 -1.000000 -1.000000 123 | 0.333333 -1.000000 -1.000000 124 | -0.333333 -0.333333 -1.000000 125 | 0.333333 -0.333333 -1.000000 126 | -0.333333 -1.000000 -0.333333 127 | 0.333333 -1.000000 -0.333333 128 | -0.333333 -0.333333 -0.333333 129 | 0.333333 -0.333333 -0.333333 130 | 0.333333 -1.000000 -1.000000 131 | 1.000000 -1.000000 -1.000000 132 | 0.333333 -0.333333 -1.000000 133 | 1.000000 -0.333333 -1.000000 134 | 0.333333 -1.000000 -0.333333 135 | 1.000000 -1.000000 -0.333333 136 | 0.333333 -0.333333 -0.333333 137 | 1.000000 -0.333333 -0.333333 138 | -1.000000 -0.333333 -1.000000 139 | -0.333333 -0.333333 -1.000000 140 | -1.000000 0.333333 -1.000000 141 | -0.333333 0.333333 -1.000000 142 | -1.000000 -0.333333 -0.333333 143 | -0.333333 -0.333333 -0.333333 144 | -1.000000 0.333333 -0.333333 145 | -0.333333 0.333333 -0.333333 146 | -0.333333 -0.333333 -1.000000 147 | 0.333333 -0.333333 -1.000000 148 | -0.333333 0.333333 -1.000000 149 | 0.333333 0.333333 -1.000000 150 | -0.333333 -0.333333 -0.333333 151 | 0.333333 -0.333333 -0.333333 152 | -0.333333 0.333333 -0.333333 153 | 0.333333 0.333333 -0.333333 154 | 0.333333 -0.333333 -1.000000 155 | 1.000000 -0.333333 -1.000000 156 | 0.333333 0.333333 -1.000000 157 | 1.000000 0.333333 -1.000000 158 | 0.333333 -0.333333 -0.333333 159 | 1.000000 -0.333333 -0.333333 160 | 0.333333 0.333333 -0.333333 161 | 1.000000 0.333333 -0.333333 162 | -1.000000 0.333333 -1.000000 163 | -0.333333 0.333333 -1.000000 164 | -1.000000 1.000000 -1.000000 165 | -0.333333 1.000000 -1.000000 166 | -1.000000 0.333333 -0.333333 167 | -0.333333 0.333333 -0.333333 168 | -1.000000 1.000000 -0.333333 169 | -0.333333 1.000000 -0.333333 170 | -0.333333 0.333333 -1.000000 171 | 0.333333 0.333333 -1.000000 172 | -0.333333 1.000000 -1.000000 173 | 0.333333 1.000000 -1.000000 174 | -0.333333 0.333333 -0.333333 175 | 0.333333 0.333333 -0.333333 176 | -0.333333 1.000000 -0.333333 177 | 0.333333 1.000000 -0.333333 178 | 0.333333 0.333333 -1.000000 179 | 1.000000 0.333333 -1.000000 180 | 0.333333 1.000000 -1.000000 181 | 1.000000 1.000000 -1.000000 182 | 0.333333 0.333333 -0.333333 183 | 1.000000 0.333333 -0.333333 184 | 0.333333 1.000000 -0.333333 185 | 1.000000 1.000000 -0.333333 186 | -1.000000 -1.000000 -0.333333 187 | -0.333333 -1.000000 -0.333333 188 | -1.000000 -0.333333 -0.333333 189 | -0.333333 -0.333333 -0.333333 190 | -1.000000 -1.000000 0.333333 191 | -0.333333 -1.000000 0.333333 192 | -1.000000 -0.333333 0.333333 193 | -0.333333 -0.333333 0.333333 194 | -0.333333 -1.000000 -0.333333 195 | 0.333333 -1.000000 -0.333333 196 | -0.333333 -0.333333 -0.333333 197 | 0.333333 -0.333333 -0.333333 198 | -0.333333 -1.000000 0.333333 199 | 0.333333 -1.000000 0.333333 200 | -0.333333 -0.333333 0.333333 201 | 0.333333 -0.333333 0.333333 202 | 0.333333 -1.000000 -0.333333 203 | 1.000000 -1.000000 -0.333333 204 | 0.333333 -0.333333 -0.333333 205 | 1.000000 -0.333333 -0.333333 206 | 0.333333 -1.000000 0.333333 207 | 1.000000 -1.000000 0.333333 208 | 0.333333 -0.333333 0.333333 209 | 1.000000 -0.333333 0.333333 210 | -1.000000 -0.333333 -0.333333 211 | -0.333333 -0.333333 -0.333333 212 | -1.000000 0.333333 -0.333333 213 | -0.333333 0.333333 -0.333333 214 | -1.000000 -0.333333 0.333333 215 | -0.333333 -0.333333 0.333333 216 | -1.000000 0.333333 0.333333 217 | -0.333333 0.333333 0.333333 218 | -0.333333 -0.333333 -0.333333 219 | 0.333333 -0.333333 -0.333333 220 | -0.333333 0.333333 -0.333333 221 | 0.333333 0.333333 -0.333333 222 | -0.333333 -0.333333 0.333333 223 | 0.333333 -0.333333 0.333333 224 | -0.333333 0.333333 0.333333 225 | 0.333333 0.333333 0.333333 226 | 0.333333 -0.333333 -0.333333 227 | 1.000000 -0.333333 -0.333333 228 | 0.333333 0.333333 -0.333333 229 | 1.000000 0.333333 -0.333333 230 | 0.333333 -0.333333 0.333333 231 | 1.000000 -0.333333 0.333333 232 | 0.333333 0.333333 0.333333 233 | 1.000000 0.333333 0.333333 234 | -1.000000 0.333333 -0.333333 235 | -0.333333 0.333333 -0.333333 236 | -1.000000 1.000000 -0.333333 237 | -0.333333 1.000000 -0.333333 238 | -1.000000 0.333333 0.333333 239 | -0.333333 0.333333 0.333333 240 | -1.000000 1.000000 0.333333 241 | -0.333333 1.000000 0.333333 242 | -0.333333 0.333333 -0.333333 243 | 0.333333 0.333333 -0.333333 244 | -0.333333 1.000000 -0.333333 245 | 0.333333 1.000000 -0.333333 246 | -0.333333 0.333333 0.333333 247 | 0.333333 0.333333 0.333333 248 | -0.333333 1.000000 0.333333 249 | 0.333333 1.000000 0.333333 250 | 0.333333 0.333333 -0.333333 251 | 1.000000 0.333333 -0.333333 252 | 0.333333 1.000000 -0.333333 253 | 1.000000 1.000000 -0.333333 254 | 0.333333 0.333333 0.333333 255 | 1.000000 0.333333 0.333333 256 | 0.333333 1.000000 0.333333 257 | 1.000000 1.000000 0.333333 258 | -1.000000 -1.000000 0.333333 259 | -0.333333 -1.000000 0.333333 260 | -1.000000 -0.333333 0.333333 261 | -0.333333 -0.333333 0.333333 262 | -1.000000 -1.000000 1.000000 263 | -0.333333 -1.000000 1.000000 264 | -1.000000 -0.333333 1.000000 265 | -0.333333 -0.333333 1.000000 266 | -0.333333 -1.000000 0.333333 267 | 0.333333 -1.000000 0.333333 268 | -0.333333 -0.333333 0.333333 269 | 0.333333 -0.333333 0.333333 270 | -0.333333 -1.000000 1.000000 271 | 0.333333 -1.000000 1.000000 272 | -0.333333 -0.333333 1.000000 273 | 0.333333 -0.333333 1.000000 274 | 0.333333 -1.000000 0.333333 275 | 1.000000 -1.000000 0.333333 276 | 0.333333 -0.333333 0.333333 277 | 1.000000 -0.333333 0.333333 278 | 0.333333 -1.000000 1.000000 279 | 1.000000 -1.000000 1.000000 280 | 0.333333 -0.333333 1.000000 281 | 1.000000 -0.333333 1.000000 282 | -1.000000 -0.333333 0.333333 283 | -0.333333 -0.333333 0.333333 284 | -1.000000 0.333333 0.333333 285 | -0.333333 0.333333 0.333333 286 | -1.000000 -0.333333 1.000000 287 | -0.333333 -0.333333 1.000000 288 | -1.000000 0.333333 1.000000 289 | -0.333333 0.333333 1.000000 290 | -0.333333 -0.333333 0.333333 291 | 0.333333 -0.333333 0.333333 292 | -0.333333 0.333333 0.333333 293 | 0.333333 0.333333 0.333333 294 | -0.333333 -0.333333 1.000000 295 | 0.333333 -0.333333 1.000000 296 | -0.333333 0.333333 1.000000 297 | 0.333333 0.333333 1.000000 298 | 0.333333 -0.333333 0.333333 299 | 1.000000 -0.333333 0.333333 300 | 0.333333 0.333333 0.333333 301 | 1.000000 0.333333 0.333333 302 | 0.333333 -0.333333 1.000000 303 | 1.000000 -0.333333 1.000000 304 | 0.333333 0.333333 1.000000 305 | 1.000000 0.333333 1.000000 306 | -1.000000 0.333333 0.333333 307 | -0.333333 0.333333 0.333333 308 | -1.000000 1.000000 0.333333 309 | -0.333333 1.000000 0.333333 310 | -1.000000 0.333333 1.000000 311 | -0.333333 0.333333 1.000000 312 | -1.000000 1.000000 1.000000 313 | -0.333333 1.000000 1.000000 314 | -0.333333 0.333333 0.333333 315 | 0.333333 0.333333 0.333333 316 | -0.333333 1.000000 0.333333 317 | 0.333333 1.000000 0.333333 318 | -0.333333 0.333333 1.000000 319 | 0.333333 0.333333 1.000000 320 | -0.333333 1.000000 1.000000 321 | 0.333333 1.000000 1.000000 322 | 0.333333 0.333333 0.333333 323 | 1.000000 0.333333 0.333333 324 | 0.333333 1.000000 0.333333 325 | 1.000000 1.000000 0.333333 326 | 0.333333 0.333333 1.000000 327 | 1.000000 0.333333 1.000000 328 | 0.333333 1.000000 1.000000 329 | 1.000000 1.000000 1.000000 -------------------------------------------------------------------------------- /mfem/data/periodic-hexagon.mesh: -------------------------------------------------------------------------------- 1 | MFEM mesh v1.0 2 | 3 | # 4 | # MFEM Geometry Types (see mesh/geom.hpp): 5 | # 6 | # POINT = 0 7 | # SEGMENT = 1 8 | # TRIANGLE = 2 9 | # SQUARE = 3 10 | # TETRAHEDRON = 4 11 | # CUBE = 5 12 | # 13 | 14 | dimension 15 | 2 16 | 17 | # format: ... 18 | elements 19 | 12 20 | 21 | 1 3 0 2 8 5 22 | 1 3 2 1 3 8 23 | 1 3 5 8 6 11 24 | 1 3 8 3 0 6 25 | 26 | 2 3 0 4 9 6 27 | 2 3 4 1 2 9 28 | 2 3 6 9 7 11 29 | 2 3 9 2 0 7 30 | 31 | 3 3 0 3 10 7 32 | 3 3 3 1 4 10 33 | 3 3 7 10 5 11 34 | 3 3 10 4 0 5 35 | 36 | boundary 37 | 0 38 | 39 | vertices 40 | 12 41 | 42 | nodes 43 | FiniteElementSpace 44 | FiniteElementCollection: L2_T1_2D_P1 45 | VDim: 2 46 | Ordering: 1 47 | 48 | -0.50 -0.8660254037844386 49 | 0.00 -0.8660254037844386 50 | -0.25 -0.4330127018922193 51 | 0.25 -0.4330127018922193 52 | 53 | 0.00 -0.8660254037844386 54 | 0.50 -0.8660254037844386 55 | 0.25 -0.4330127018922193 56 | 0.75 -0.4330127018922193 57 | 58 | -0.25 -0.4330127018922193 59 | 0.25 -0.4330127018922193 60 | 0.00 0.0 61 | 0.50 0.0 62 | 63 | 0.25 -0.4330127018922193 64 | 0.75 -0.4330127018922193 65 | 0.50 0.0 66 | 1.00 0.0 67 | 68 | 1.00 0.0 69 | 0.75 0.4330127018922193 70 | 0.50 0.0 71 | 0.25 0.4330127018922193 72 | 73 | 0.75 0.4330127018922193 74 | 0.50 0.8660254037844386 75 | 0.25 0.4330127018922193 76 | 0.00 0.8660254037844386 77 | 78 | 0.50 0.0 79 | 0.25 0.4330127018922193 80 | 0.00 0.0 81 | -0.25 0.4330127018922193 82 | 83 | 0.25 0.4330127018922193 84 | 0.00 0.8660254037844386 85 | -0.25 0.4330127018922193 86 | -0.50 0.8660254037844386 87 | 88 | -0.50 0.8660254037844386 89 | -0.75 0.4330127018922193 90 | -0.25 0.4330127018922193 91 | -0.50 0.0 92 | 93 | -0.75 0.4330127018922193 94 | -1.00 0.0 95 | -0.50 0.0 96 | -0.75 -0.4330127018922193 97 | 98 | -0.25 0.4330127018922193 99 | -0.50 0.0 100 | 0.00 0.0 101 | -0.25 -0.4330127018922193 102 | 103 | -0.50 0.0 104 | -0.75 -0.4330127018922193 105 | -0.25 -0.4330127018922193 106 | -0.50 -0.8660254037844386 107 | -------------------------------------------------------------------------------- /mfem/data/periodic-segment.mesh: -------------------------------------------------------------------------------- 1 | MFEM mesh v1.0 2 | 3 | # 4 | # MFEM Geometry Types (see mesh/geom.hpp): 5 | # 6 | # POINT = 0 7 | # SEGMENT = 1 8 | # TRIANGLE = 2 9 | # SQUARE = 3 10 | # TETRAHEDRON = 4 11 | # CUBE = 5 12 | # 13 | 14 | dimension 15 | 1 16 | 17 | # format: ... 18 | elements 19 | 4 20 | 1 1 0 1 21 | 1 1 1 2 22 | 1 1 2 3 23 | 1 1 3 0 24 | 25 | boundary 26 | 0 27 | 28 | vertices 29 | 4 30 | 31 | nodes 32 | FiniteElementSpace 33 | FiniteElementCollection: L2_T1_1D_P1 34 | VDim: 1 35 | Ordering: 1 36 | 37 | 0 0.25 38 | 0.25 0.5 39 | 0.5 0.75 40 | 0.75 1 41 | -------------------------------------------------------------------------------- /mfem/data/periodic-square.mesh: -------------------------------------------------------------------------------- 1 | MFEM mesh v1.0 2 | 3 | # 4 | # MFEM Geometry Types (see mesh/geom.hpp): 5 | # 6 | # POINT = 0 7 | # SEGMENT = 1 8 | # TRIANGLE = 2 9 | # SQUARE = 3 10 | # TETRAHEDRON = 4 11 | # CUBE = 5 12 | # 13 | 14 | dimension 15 | 2 16 | 17 | # format: ... 18 | elements 19 | 9 20 | 1 3 0 1 4 3 21 | 2 3 1 2 5 4 22 | 3 3 2 0 3 5 23 | 4 3 3 4 7 6 24 | 5 3 4 5 8 7 25 | 6 3 5 3 6 8 26 | 7 3 6 7 1 0 27 | 8 3 7 8 2 1 28 | 9 3 8 6 0 2 29 | 30 | boundary 31 | 0 32 | 33 | vertices 34 | 9 35 | 36 | nodes 37 | FiniteElementSpace 38 | FiniteElementCollection: L2_T1_2D_P1 39 | VDim: 2 40 | Ordering: 1 41 | 42 | -1 -1 43 | -0.333333333 -1 44 | -1 -0.333333333 45 | -0.333333333 -0.333333333 46 | 47 | -0.333333333 -1 48 | +0.333333333 -1 49 | -0.333333333 -0.333333333 50 | +0.333333333 -0.333333333 51 | 52 | +0.333333333 -1 53 | +1 -1 54 | +0.333333333 -0.333333333 55 | +1 -0.333333333 56 | 57 | -1 -0.333333333 58 | -0.333333333 -0.333333333 59 | -1 +0.333333333 60 | -0.333333333 +0.333333333 61 | 62 | -0.333333333 -0.333333333 63 | +0.333333333 -0.333333333 64 | -0.333333333 +0.333333333 65 | +0.333333333 +0.333333333 66 | 67 | +0.333333333 -0.333333333 68 | +1 -0.333333333 69 | +0.333333333 +0.333333333 70 | +1 +0.333333333 71 | 72 | -1 +0.333333333 73 | -0.333333333 +0.333333333 74 | -1 +1 75 | -0.333333333 +1 76 | 77 | -0.333333333 +0.333333333 78 | +0.333333333 +0.333333333 79 | -0.333333333 +1 80 | +0.333333333 +1 81 | 82 | +0.333333333 +0.333333333 83 | +1 +0.333333333 84 | +0.333333333 +1 85 | +1 +1 86 | -------------------------------------------------------------------------------- /mfem/data/pumi/geom/Kova.dmg: -------------------------------------------------------------------------------- 1 | 1 8 18 12 2 | 0 0 0 3 | 0 0 0 4 | 25 0 0 0 5 | 20 0 0 0 6 | 15 0 0 0 7 | 10 0 0 0 8 | 5 0 0 0 9 | 2 0 0 0 10 | 72 0 0 0 11 | 74 0 0 0 12 | 76 0 0 0 13 | 78 0 0 0 14 | 80 0 0 0 15 | 82 0 0 0 16 | 101 5 80 17 | 100 10 78 18 | 99 15 76 19 | 98 20 74 20 | 97 25 72 21 | 102 2 82 22 | 30 25 2 23 | 26 20 25 24 | 21 15 20 25 | 16 10 15 26 | 11 5 10 27 | 6 2 5 28 | 60 72 82 29 | 62 74 72 30 | 64 76 74 31 | 66 78 76 32 | 68 80 78 33 | 70 82 80 34 | 114 1 35 | 4 36 | 70 0 37 | 101 0 38 | 6 0 39 | 102 0 40 | 112 1 41 | 4 42 | 68 0 43 | 100 0 44 | 11 0 45 | 101 0 46 | 110 1 47 | 4 48 | 66 0 49 | 99 0 50 | 16 0 51 | 100 0 52 | 108 1 53 | 4 54 | 64 0 55 | 98 0 56 | 21 0 57 | 99 0 58 | 106 1 59 | 4 60 | 62 0 61 | 97 0 62 | 26 0 63 | 98 0 64 | 104 1 65 | 4 66 | 60 0 67 | 102 0 68 | 30 0 69 | 97 0 70 | 34 1 71 | 6 72 | 6 0 73 | 11 0 74 | 16 0 75 | 21 0 76 | 26 0 77 | 30 0 78 | 56 1 79 | 6 80 | 70 0 81 | 60 0 82 | 62 0 83 | 64 0 84 | 66 0 85 | 68 0 86 | 128 1 87 | 8 88 | 114 0 89 | 112 0 90 | 110 0 91 | 108 0 92 | 106 0 93 | 104 0 94 | 34 0 95 | 56 0 96 | -------------------------------------------------------------------------------- /mfem/data/pumi/geom/Kova.x_t: -------------------------------------------------------------------------------- 1 | **ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz************************** 2 | **PARASOLID !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~0123456789************************** 3 | **PART1; 4 | MC=unknown; 5 | MC_MODEL=unknown; 6 | MC_ID=unknown; 7 | OS=unknown; 8 | OS_RELEASE=unknown; 9 | FRU=sdl_parasolid_customer_support; 10 | APPL=parasolid_acceptance_tests; 11 | SITE=unknown; 12 | USER=unknown; 13 | FORMAT=text; 14 | GUISE=transmit; 15 | KEY=/users/zaided/core-sim/build/meshes/curved/Kova_nat.x_t; 16 | FILE=/users/zaided/core-sim/build/meshes/curved/Kova_nat.x_t; 17 | DATE=unknown; 18 | **PART2; 19 | SCH=SCH_2600217_25001; 20 | USFLD_SIZE=1; 21 | **PART3; 22 | **END_OF_HEADER***************************************************************** 23 | T51 : TRANSMIT FILE created by modeller version 260021723 SCH_2600217_25001_1300 24 | 6189 1 12 29 CCCCCCCCCCCCCDI5 owner1040 0 CCCCCCCCCA16 index_map_offset0 0 1 dA9 25 | index_map82 0 A17 node_id_index_map82 0 A20 schema_embedding_map82 0 A5 child12 26 | 0 A14 lowest_node_id0 0 1 dZ1 223 2 3 0 0 0 0 1e3 1e-8 0 0 0 1 0 1 1 4 5 6 7 8 27 | 9 10 0 0 0 0 0 0 0 81 255 1 2 188 11 1 12 0 0 0 13 0 70 11 CI9 list_type0 0 1 u 28 | I10 notransmit0 0 1 lCCCDCCDI12 finger_index0 0 1 dI12 finger_block1012 0 CZ3 0 29 | 4 T1 0 0 5 20 1 14 14 0 13 255 4 127 0 1 0 15 0 0 16 0 0 50 255 5 126 0 15 17 0 30 | 0 +-.5 .860998942748213 1 1 161837843240675e-30 0 -161837843240675e-30 1 0 0 30 31 | 255 6 120 0 18 19 0 0 +-.5 .860998942748213 1 0 0 -1 0 29 255 7 91 0 20 21 0 .1 32 | 390010572517868 1.5 0 0 19 8 CCCCCCCA5 owner12 0 Z8 1 0 1 16 0 22 V0 0 16 255 9 33 | 101 23 ?24 0 25 19 0 0 1 0 18 255 10 25 0 26 0 27 28 ?1 0 17 255 26 0 29 30 31 34 | 10 32 33 0 34 +0 18 27 20 0 31 10 35 36 ?1 0 29 28 27 0 10 36 37 .1390010572517 35 | 868 1.5 1 0 29 36 22 0 27 38 28 1 1.5 1 0 29 37 96 0 39 28 40 -.5 .8609989427482 36 | 13 0 0 18 39 82 0 41 42 0 37 ?1 0 29 40 95 0 42 37 43 -.5 .174989414696048 0 0 1 37 | 8 42 80 0 44 45 39 40 ?1 0 29 43 94 0 45 40 46 .174989414696047 -.5 0 0 18 45 78 38 | 0 47 48 42 43 ?1 0 29 46 93 0 48 43 21 1 -.5 0 0 18 48 76 0 49 50 45 46 ?1 0 29 39 | 21 92 0 50 46 7 1 1.5 0 0 18 50 74 0 51 20 48 21 ?1 0 17 51 0 52 53 54 50 55 56 40 | 0 57 +0 18 20 72 0 58 59 50 7 ?1 0 17 58 0 60 34 57 20 61 62 0 63 +0 18 59 2 0 41 | 30 64 20 65 ?1 0 17 30 0 29 66 26 59 67 68 0 69 +0 18 64 5 0 70 71 59 72 ?1 0 2 42 | 9 65 4 0 59 0 72 -.5 .860998942748213 1 0 29 72 7 0 64 65 73 -.5 .17498941469604 43 | 8 1 0 29 73 12 0 71 72 38 .174989414696047 -.5 1 0 18 71 10 0 74 35 64 73 ?1 0 2 44 | 9 38 17 0 35 73 36 1 -.5 1 0 18 35 15 0 75 27 71 38 ?1 0 17 75 0 29 31 74 35 76 45 | 77 0 78 +0 15 255 29 33 0 66 79 0 0 17 31 0 29 26 75 27 80 81 0 53 +0 17 74 0 2 46 | 9 75 66 71 70 82 0 83 +0 17 76 0 84 85 78 71 75 77 0 0 -0 16 77 16 86 ?75 81 82 47 | 87 0 0 1 0 17 78 0 84 76 49 35 54 88 0 80 -0 15 84 109 0 49 89 0 0 17 49 0 84 7 48 | 8 85 48 90 91 0 54 +0 17 54 0 52 51 80 48 78 88 0 55 +0 16 88 99 92 ?54 25 93 94 49 | 0 0 1 0 17 80 0 52 54 53 35 31 81 0 0 -0 15 52 107 0 51 95 0 0 17 53 0 52 80 51 50 | 27 57 93 0 32 -0 16 81 21 96 ?31 33 77 97 0 0 1 0 81 1 96 156 98 81 0 0 99 23 1 51 | 00 0 16 33 26 101 ?26 68 81 102 0 0 1 0 30 97 24 0 81 87 102 0 +1 -.5 1 -1110223 52 | 024625156e-31 1 0 0 30 87 19 0 77 103 97 0 +.1749894146960485 -.5 1 1 0 0 0 30 1 53 | 02 29 0 33 97 104 0 +1 1.5 1 -1 0 0 0 31 255 104 32 0 68 102 105 0 +.13900105725 54 | 1787 .860998942748213 1 0 0 1 1 0 0 .639001057251787 0 16 68 30 106 ?30 18 33 10 55 | 4 0 0 1 0 30 105 90 0 107 104 108 0 +-.5 .860998942748213 0 161837843240675e-30 56 | -1 0 0 16 107 70 109 ?110 111 0 105 0 0 1 0 31 108 89 0 111 105 112 0 +.1749894 57 | 146960485 .1749894146960485 0 0 0 1 1 0 0 .674989414696048 0 16 111 68 113 ?47 9 58 | 1 107 108 0 0 1 0 30 112 88 0 91 108 114 0 +.1749894146960485 -.5 0 1 0 0 0 16 9 59 | 1 66 115 ?49 56 111 112 0 0 1 0 30 114 87 0 56 112 116 0 +1 -.5 0 -1110223024625 60 | 156e-31 1 0 0 16 56 64 117 ?51 62 91 114 0 0 1 0 30 116 86 0 62 114 118 0 +1 1.5 61 | 0 -1 0 0 0 16 62 62 119 ?58 120 56 116 0 0 1 0 31 118 85 0 120 116 121 0 +.1390 62 | 01057251787 .860998942748213 0 0 0 1 1 0 0 .639001057251787 0 16 120 60 99 ?41 1 63 | 22 62 118 0 0 1 0 30 121 115 0 123 118 124 0 +.1390010572517868 1.5 1 0 0 -1 0 1 64 | 6 123 97 125 ?63 93 18 121 0 0 1 0 30 124 116 0 93 121 94 0 +1 1.5 1 0 0 -1 0 16 65 | 93 98 126 ?57 88 123 124 0 0 1 0 30 94 117 0 88 124 127 0 +1 -.5 1 0 0 -1 0 30 66 | 127 118 0 25 94 19 0 +.174989414696047 -.5 1 0 0 -1 0 16 25 100 128 ?85 9 88 12 67 | 7 0 0 1 0 30 19 119 0 9 127 6 0 +-.5 .174989414696048 1 0 0 -1 0 81 1 128 162 98 68 | 25 0 0 129 106 130 0 17 85 0 84 49 76 45 83 25 0 90 +0 17 83 0 131 70 47 71 85 69 | 25 0 76 -0 17 90 0 132 44 55 45 49 91 0 0 -0 15 132 59 0 133 134 0 0 17 44 0 13 70 | 2 133 90 42 47 111 0 24 -0 17 55 0 132 90 61 48 51 56 0 0 -0 17 61 0 132 55 135 71 | 50 58 62 0 0 -0 17 135 0 132 61 133 20 41 120 0 0 -0 17 133 0 132 135 44 39 110 72 | 107 0 0 -0 17 41 0 136 69 63 39 135 120 0 137 +0 15 136 103 0 41 138 0 0 17 69 73 | 0 136 67 41 59 137 18 0 139 -0 17 63 0 136 41 67 20 34 123 0 135 +0 17 137 0 14 74 | 0 110 139 39 69 18 0 133 +0 15 140 113 0 110 15 0 0 17 110 0 140 141 137 42 133 75 | 107 0 0 +0 17 139 0 140 137 141 59 66 122 0 0 -0 16 18 102 142 ?137 123 68 6 0 76 | 0 1 0 81 1 142 166 98 18 0 0 113 109 143 0 80 255 1 98 144 145 9000 1 1 1 1 1 1 77 | 1 1 0 FFFFTFTFFFFFFF3 0 81 1 113 165 98 111 0 0 86 142 146 0 81 1 109 167 98 10 78 | 7 0 0 142 125 147 0 84 255 32 143 e0133d0a274ad6dae35817e003cac7de0 81 1 125 168 79 | 98 123 0 0 109 119 148 0 84 32 147 74f22b21f039e3a6ac7931fe8932a5520 81 1 119 1 80 | 69 98 62 0 0 125 149 150 0 84 32 148 acd61ea4c8be62b78e9852176f30a4550 81 1 149 81 | 192 98 15 151 0 119 152 153 0 84 32 150 8d0eff17801a7eb54b3eb74f8c5221200 14 25 82 | 5 15 114 149 ?154 0 140 4 5 -0 0 154 0 22 0 81 1 151 191 155 15 0 149 0 156 0 0 83 | 81 1 152 200 98 138 157 0 149 158 159 0 84 32 153 415f8090dedd9057aae31f62f0b53 84 | 4e90 14 138 104 152 ?79 160 136 4 161 -0 0 79 160 22 0 81 1 157 199 155 138 0 15 85 | 2 156 162 0 0 81 1 158 204 98 95 162 0 152 163 164 0 84 32 159 9a7c21dd752b6ae7d 86 | 38d08b5eca460b60 14 95 108 158 ?160 89 52 4 165 -0 0 160 89 22 0 81 1 162 203 15 87 | 5 95 0 158 157 166 0 0 81 1 163 212 98 89 167 0 158 168 169 0 84 32 164 a652734a 88 | 899d6b9de0ab3374074f52830 14 89 110 163 ?95 154 84 4 170 -0 0 95 154 22 0 81 1 1 89 | 67 211 155 89 0 163 166 171 0 0 81 1 168 216 98 160 171 0 163 172 173 0 84 32 16 90 | 9 a65a3edd3b0754bed4b0abe972cef0d90 14 160 106 168 ?138 95 60 4 174 -0 0 138 95 91 | 22 0 81 1 171 215 155 160 0 168 167 175 0 0 81 1 172 217 98 79 166 0 168 176 17 92 | 7 0 84 32 173 d1f66d1574e353d3605efa9a8be928270 14 79 34 172 ?134 138 29 4 178 + 93 | 0 0 134 138 22 0 81 1 166 207 155 79 0 172 162 167 0 0 81 1 176 222 98 154 156 0 94 | 172 179 180 0 84 32 177 2eda16169305e8b87f034319cc9663cb0 14 154 112 176 ?89 15 95 | 131 4 17 -0 0 89 15 22 0 81 1 156 195 155 154 0 176 151 157 0 0 81 1 179 223 98 96 | 134 175 0 176 0 181 0 84 32 180 567582872ae46d7afc7f71f141028fe60 14 134 56 179 97 | ?0 79 132 4 182 -0 0 0 79 22 0 81 1 175 220 155 134 0 179 171 0 0 0 84 32 181 5 98 | 9eef558892982058fc012b04ce30e510 80 1 155 0 183 8001 0 0 0 0 3 5 0 0 0 FFFFTFTFF 99 | FFFFF2 0 79 255 15 183 SDL/TYSA_COLOUR0 50 182 84 0 134 178 161 0 +.174989414696 100 | 0485 .1749894146960485 0 0 0 1 1 0 0 0 13 22 3 0 0 0 0 0 0 8 15 0 50 178 35 0 79 101 | 0 182 0 +.1749894146960485 .1749894146960485 1 0 0 1 1 0 0 0 51 255 161 121 0 1 102 | 38 182 174 0 -.139001057251787 .860998942748213 1 0 0 -1 .639001057251787 -1 0 0 103 | 0 50 174 122 0 160 161 165 0 +1 1.5 1 0 -1 0 0 0 -1 0 50 165 123 0 95 174 170 0 104 | +1 -.5 1 -1 -1110223024625156e-31 0 1110223024625156e-31 -1 0 0 50 170 124 0 89 105 | 165 17 0 +.1749894146960485 -.5 1 0 1 0 0 0 1 0 51 17 125 0 154 170 5 0 -.17498 106 | 94146960485 .1749894146960485 1 0 0 -1 .674989414696048 -1 0 0 0 15 131 111 0 47 107 | 154 0 0 17 47 0 131 83 24 45 44 111 0 85 +0 17 24 0 131 47 70 42 141 9 0 110 +0 108 | 17 70 0 131 24 83 64 74 82 0 66 -0 17 141 0 140 139 110 64 24 9 0 0 -0 16 82 11 109 | 184 ?74 77 122 103 0 0 1 0 17 66 0 29 74 30 64 139 122 0 141 +0 16 122 6 129 ?6 110 | 6 82 120 185 0 0 1 0 81 1 129 161 98 122 0 0 126 128 186 0 30 185 9 0 122 0 103 111 | 0 +-.5 .860998942748213 1 161837843240675e-30 -1 0 0 31 103 14 0 82 185 87 0 +. 112 | 1749894146960485 .1749894146960485 1 0 0 1 1 0 0 .674989414696048 0 81 1 126 160 113 | 98 93 0 0 184 129 187 0 84 32 186 f26c7a3c61e3aedc6bb1e6f1b0e4e2240 81 1 184 15 114 | 9 98 82 0 0 101 126 188 0 84 32 187 0e305e88c1bed3a69ebc0ebee326188e0 81 1 101 1 115 | 58 98 33 0 0 23 184 189 0 84 32 188 04acbae21e24b9d7c8cb705326a5bdce0 81 1 23 15 116 | 7 98 9 0 0 96 101 190 0 84 32 189 cc269817b9e21a66d21d60effd84665b0 84 32 190 69 117 | e9732e7b92342dff88348ced3b60f50 15 60 105 0 58 160 0 0 81 1 86 164 98 77 0 0 106 118 | 113 191 0 84 32 146 0ac5d0ee37213df988bfcfe7a89236e70 81 1 106 163 98 68 0 0 12 119 | 8 86 192 0 84 32 191 f3b3cf913878a42afe5099f9154b08020 84 32 192 6490dfa2b2ce409 120 | 9bfb82f12aa2942a30 79 15 145 UGS/ObjectState0 17 67 0 136 63 69 10 30 68 0 0 -0 121 | 17 34 0 60 32 58 10 63 123 0 67 -0 17 32 0 60 57 34 27 26 33 0 0 -0 17 57 0 60 122 | 58 32 50 53 93 0 61 +0 84 32 130 23f2d0d1982a7ae57e28eb97e0618c7c0 81 1 99 155 123 | 98 120 0 0 92 96 193 0 81 1 92 154 98 88 0 0 115 99 194 0 84 32 193 aeb3c617f9e 124 | fd5cd4594fdbb1fa4d92f0 81 1 115 153 98 91 0 0 117 92 195 0 84 32 194 182f297ced6 125 | 08d62d8b0e28d582b5faf0 81 1 117 152 98 56 0 0 0 115 196 0 84 32 195 d8f45e66bdc1 126 | 570ee4c522614ea3a99b0 84 32 196 3a86e5ca064430cb660d916fb8c3cd0d0 84 32 100 6ce3 127 | 7655f732f67dad146e77456099620 19 16 128 0 1 0 8 4 S0 0 74 4 CI16 index_map_offse 128 | t0 0 1 dCCZ20 14 5 0 0 179 197 12 175 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 81 2 197 129 | 178 198 1 0 12 0 0 199 200 0 81 1 12 179 201 1 197 2 0 0 202 0 80 1 201 98 203 130 | 9000 0 0 0 0 3 5 0 0 0 FFTFFFFFFFFFFF1 0 82 255 1 202 13 0 79 7 203 body_id0 80 131 | 2 198 204 205 8004 0 0 0 0 3 5 0 0 0 FFTFFFFFFFFFFF2 3 0 83 255 1 199 7830.64 0 132 | 84 8 200 Kg/Cu M 0 79 16 205 SDL/TYSA_DENSITY0 80 1 11 201 206 9000 0 0 0 0 3 5 133 | 0 0 0 FFTFFFFFFFFFFF1 0 82 1 13 129 0 79 24 206 UG2/BODY_COLOR_ATTRIBUTE0 1 0 134 | -------------------------------------------------------------------------------- /mfem/data/pumi/parallel/KovaLinear/Kova-linear-10k_2p0.smb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xsdk-project/xsdk-examples/b68cb6234b104511371dd6ef880bc8d322dd86fe/mfem/data/pumi/parallel/KovaLinear/Kova-linear-10k_2p0.smb -------------------------------------------------------------------------------- /mfem/data/pumi/parallel/KovaLinear/Kova-linear-10k_2p1.smb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/xsdk-project/xsdk-examples/b68cb6234b104511371dd6ef880bc8d322dd86fe/mfem/data/pumi/parallel/KovaLinear/Kova-linear-10k_2p1.smb -------------------------------------------------------------------------------- /mfem/data/star.mesh: -------------------------------------------------------------------------------- 1 | MFEM mesh v1.0 2 | 3 | # 4 | # MFEM Geometry Types (see mesh/geom.hpp): 5 | # 6 | # POINT = 0 7 | # SEGMENT = 1 8 | # TRIANGLE = 2 9 | # SQUARE = 3 10 | # TETRAHEDRON = 4 11 | # CUBE = 5 12 | # 13 | 14 | dimension 15 | 2 16 | 17 | elements 18 | 20 19 | 1 3 0 11 26 14 20 | 1 3 0 14 27 17 21 | 1 3 0 17 28 20 22 | 1 3 0 20 29 23 23 | 1 3 0 23 30 11 24 | 1 3 11 1 12 26 25 | 1 3 26 12 3 13 26 | 1 3 14 26 13 2 27 | 1 3 14 2 15 27 28 | 1 3 27 15 5 16 29 | 1 3 17 27 16 4 30 | 1 3 17 4 18 28 31 | 1 3 28 18 7 19 32 | 1 3 20 28 19 6 33 | 1 3 20 6 21 29 34 | 1 3 29 21 9 22 35 | 1 3 23 29 22 8 36 | 1 3 23 8 24 30 37 | 1 3 30 24 10 25 38 | 1 3 11 30 25 1 39 | 40 | boundary 41 | 20 42 | 1 1 13 2 43 | 1 1 12 3 44 | 1 1 16 4 45 | 1 1 15 5 46 | 1 1 19 6 47 | 1 1 18 7 48 | 1 1 22 8 49 | 1 1 21 9 50 | 1 1 25 1 51 | 1 1 24 10 52 | 1 1 3 13 53 | 1 1 1 12 54 | 1 1 5 16 55 | 1 1 2 15 56 | 1 1 7 19 57 | 1 1 4 18 58 | 1 1 9 22 59 | 1 1 6 21 60 | 1 1 10 25 61 | 1 1 8 24 62 | 63 | vertices 64 | 31 65 | 2 66 | 0 0 67 | 1 0 68 | 0.309017 0.951057 69 | 1.30902 0.951057 70 | -0.809017 0.587785 71 | -0.5 1.53884 72 | -0.809017 -0.587785 73 | -1.61803 0 74 | 0.309017 -0.951057 75 | -0.5 -1.53884 76 | 1.30902 -0.951057 77 | 0.5 0 78 | 1.15451 0.475529 79 | 0.809019 0.951057 80 | 0.154508 0.475529 81 | -0.0954915 1.24495 82 | -0.654508 1.06331 83 | -0.404508 0.293893 84 | -1.21352 0.293893 85 | -1.21352 -0.293892 86 | -0.404508 -0.293893 87 | -0.654508 -1.06331 88 | -0.0954915 -1.24495 89 | 0.154508 -0.475529 90 | 0.809019 -0.951057 91 | 1.15451 -0.475529 92 | 0.654509 0.475529 93 | -0.25 0.769421 94 | -0.809016 0 95 | -0.25 -0.76942 96 | 0.654509 -0.475529 97 | -------------------------------------------------------------------------------- /mfem/ginkgo/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (ENABLE_CUDA) 2 | set_source_files_properties(mfem_ex22_gko.cpp PROPERTIES LANGUAGE CUDA) 3 | endif() 4 | if (ENABLE_HIP) 5 | set_source_files_properties(mfem_ex22_gko.cpp PROPERTIES LANGUAGE HIP) 6 | endif() 7 | add_executable(mfem_ex22_gko mfem_ex22_gko.cpp) 8 | target_link_libraries(mfem_ex22_gko PRIVATE ${MFEM_ALL_DEPS}) 9 | 10 | xsdk_add_test(NAME MFEM-mfem_ex22_gko COMMAND $ --no-partial-assembly) 11 | if(ENABLE_CUDA) 12 | xsdk_add_test( 13 | NAME MFEM-mfem_ex22_gko--gpu 14 | COMMAND $ 15 | --partial-assembly --device cuda) 16 | endif() 17 | if(ENABLE_HIP) 18 | xsdk_add_test( 19 | NAME MFEM-mfem_ex22_gko--gpu 20 | COMMAND $ 21 | --partial-assembly --device hip) 22 | endif() 23 | 24 | install(TARGETS mfem_ex22_gko RUNTIME DESTINATION bin) 25 | -------------------------------------------------------------------------------- /mfem/ginkgo/GINKGO-LICENSE: -------------------------------------------------------------------------------- 1 | /************************************************************* 2 | Copyright (c) 2017-2023, the Ginkgo authors 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions 7 | are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright 13 | notice, this list of conditions and the following disclaimer in the 14 | documentation and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 21 | IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 | TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 23 | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | *************************************************************/ 32 | -------------------------------------------------------------------------------- /mfem/ginkgo/README.md: -------------------------------------------------------------------------------- 1 | # MFEM-Ginkgo Example 2 | 3 | This example modifies MFEM's `example 22` to demonstrate the use of Ginkgo from 4 | within MFEM. The code can solve three variations of a damped harmonic oscillator. 5 | MFEM handles the discretization, linear operator (assembled as a `SparseMatrix` or 6 | applied directly by MFEM within Ginkgo as a matrix-free operator), right hand side 7 | vector, and preconditioner; Ginkgo's GMRES solver is used in place of MFEM's. 8 | 9 | 10 | 11 | Useful non-default options: 12 | | Flag | Meaning | 13 | |:----------------------| :-------------------------------------------------| 14 | | -m [file] | Mesh file to use. | 15 | | -d cuda | Use the MFEM cuda backend and Ginkgo CudaExecutor | 16 | | -d hip | Use the MFEM hip backend and Ginkgo HipExecutor | 17 | | -no-pa, --no-partial-assembly | Don't use partial assembly, but build the full SparseMatrix for the system | 18 | | -no-gko, --no-ginkgo | Don't use Ginkgo (for easy comparison with "all-MFEM" case) | 19 | 20 | ## More about the MFEM-Ginkgo integration 21 | 22 | Modifying MFEM code to use Ginkgo via MFEM's wrappers is simple. First, we create a 23 | `GinkgoExecutor` wrapper (for Ginkgo's `Executor` class). The example shows both ways 24 | to create a `GinkgoExecutor`: if MFEM is using CUDA or HIP, we 25 | match the device configuration of MFEM. Otherwise, we explicitly create a `Reference` 26 | executor for the CPU. 27 | 28 | ``` 29 | Ginkgo::GinkgoExecutor *exec; 30 | // device is the MFEM Device configuration 31 | if (device.Allows(Backend::HIP_MASK) || device.Allows(Backend::CUDA_MASK)) 32 | exec = new Ginkgo::GinkgoExecutor(device); 33 | else 34 | exec = new Ginkgo::GinkgoExecutor(Ginkgo::GinkgoExecutor::REFERENCE); 35 | ``` 36 | 37 | The Ginkgo wrappers in MFEM provide mix-and-match interoperability: 38 | Ginkgo solvers can use MFEM preconditioners, and MFEM can use Ginkgo preconditioners. 39 | To allow Ginkgo to use the MFEM block diagonal preconditioner, `BDP`, we create 40 | a wrapped object: 41 | 42 | ``` 43 | Ginkgo::MFEMPreconditioner gko_precond(exec, BDP); 44 | ``` 45 | 46 | Then, we use this preconditioner when creating the Ginkgo GMRES solver: 47 | 48 | ``` 49 | Ginkgo::GMRESSolver gmres(exec, gko_precond, 50); // 50 is GMRES restart value 50 | ``` 51 | 52 | From this point, the configuration and application of the GMRES solver proceed with 53 | code identical to that for the MFEM GMRES solver: 54 | 55 | ``` 56 | // Finish configuring the solver 57 | gmres.SetOperator(*A.Ptr()); 58 | gmres.SetRelTol(1e-12); 59 | gmres.SetMaxIter(1000); 60 | gmres.SetPrintLevel(1); 61 | // Apply the solver with rhs B and solution U 62 | gmres.Mult(B, U); 63 | ``` 64 | 65 | The `Operator`, `A`, can be a `SparseMatrix` or a matrix-free operator. For `SparseMatrix` 66 | and `Vector` objects, MFEM and Ginkgo operate on the same data, whether on host or device, 67 | without unnecessary copying between the two libraries. 68 | 69 | An example of MFEM using a Ginkgo preconditioner can be found in MFEM's Ginkgo-enabled example 1, 70 | (`mfem/examples/ginkgo/ex1.cpp`). 71 | For more about Ginkgo, see the Ginkgo [documentation](https://ginkgo-project.github.io/ginkgo/doc/develop/). 72 | -------------------------------------------------------------------------------- /mfem/hiop/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (ENABLE_CUDA) 2 | set_source_files_properties(adv.cpp PROPERTIES LANGUAGE CUDA) 3 | endif() 4 | if (ENABLE_HIP) 5 | set_source_files_properties(adv.cpp PROPERTIES LANGUAGE HIP) 6 | endif() 7 | add_executable(adv adv.cpp) 8 | target_link_libraries(adv PRIVATE ${MFEM_ALL_DEPS}) 9 | if (NOT ENABLE_CUDA AND NOT ENABLE_HIP) 10 | # This test does not seem to work with HYPRE built with CUDA or HIP. 11 | xsdk_add_test( 12 | NAME 13 | MFEM-HIOP-adv 14 | MPI_NPROCS 15 | 4 16 | COMMAND 17 | $ 18 | -no-vis 19 | ) 20 | endif() 21 | install(TARGETS adv RUNTIME DESTINATION bin) 22 | -------------------------------------------------------------------------------- /mfem/hiop/README.md: -------------------------------------------------------------------------------- 1 | # MFEM-HiOp example 2 | 3 | This example, `adv.cpp`, illustrates the integration of HiOp with MFEM and it is 4 | based on the parallel HiOp-extended version of example 9 from MFEM. 5 | 6 | For a description of the problem being solved, see the documentation in the 7 | beginning of the source file. 8 | 9 | Sample runs: 10 | ``` 11 | mpirun -np 4 ./adv 12 | mpirun -np 4 ./adv -m ../data/periodic-segment.mesh -rs 3 -p 0 -o 2 -dt 0.002 13 | mpirun -np 4 ./adv -m ../data/disc-nurbs.mesh -p 1 -rs 2 -dt 0.005 -tf 9 14 | mpirun -np 4 ./adv -m ../data/periodic-square.mesh -p 3 -rs 3 -dt 0.0025 -tf 9 15 | mpirun -np 4 ./adv -m ../data/periodic-cube.mesh -p 0 -rs 2 -o 2 -dt 0.02 -tf 8 16 | ``` 17 | 18 | For a full list of options, see 19 | ``` 20 | ./adv -h 21 | ``` 22 | -------------------------------------------------------------------------------- /mfem/hypre-superlu/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (ENABLE_CUDA) 2 | set_source_files_properties(convdiff.cpp PROPERTIES LANGUAGE CUDA) 3 | endif() 4 | if (ENABLE_HIP) 5 | set_source_files_properties(convdiff.cpp PROPERTIES LANGUAGE HIP) 6 | endif() 7 | add_executable(convdiff convdiff.cpp) 8 | target_link_libraries(convdiff PRIVATE ${MFEM_ALL_DEPS}) 9 | 10 | xsdk_add_test( 11 | NAME 12 | MFEM-convdiff--hypre-boomeramg 13 | MPI_NPROCS 14 | 4 15 | COMMAND 16 | $ 17 | --no-superlu 18 | --velocity 19 | 100.0 20 | ) 21 | xsdk_add_test( 22 | NAME 23 | MFEM-convdiff--superlu 24 | MPI_NPROCS 25 | 4 26 | COMMAND 27 | $ 28 | --superlu 29 | --velocity 30 | 100.0 31 | ) 32 | 33 | install(TARGETS convdiff RUNTIME DESTINATION bin) 34 | -------------------------------------------------------------------------------- /mfem/hypre-superlu/README.md: -------------------------------------------------------------------------------- 1 | # MFEM-hypre-superlu example 2 | 3 | This example code utilizes MFEM to discretize a Convection-Diffusion equation. This problem is 4 | models the steady-state concentration of a material as it diffuses and is carried through a flowing 5 | medium. From the command line the user can vary the velocity of the flow field. The linear system 6 | that arises from this discretization can optionally be solved utilizing HYPRE's 7 | BoomerAMG solver or the SuperLU_Dist solver. 8 | 9 | This example can be used to show the advantages and disadvantages of HYPRE's iterative solvers, 10 | and SuperLU's direct solvers. At lower velocities the HYPRE solver/preconditioner is 11 | sufficient to solve the system and does so very quickly. At higher velocities the BoomerAMG solver 12 | is no longer able to handle the increasingly ill-conditioned system and a direct solver is 13 | required. For these high velocities selecting the SuperLU solver will allow the system to 14 | be solved. 15 | 16 | This example is built to run in parallel, so launch it with mpirun and your desired options: 17 | ``` 18 | mpirun -np 4 ./convdiff --velocity 100.0 --no-superlu 19 | mpirun -np 4 ./convdiff --velocity 100.0 --superlu 20 | ``` 21 | 22 | Available options: 23 | | Flag | Meaning | 24 | |:----------------------| :-----------------------------------------------------| 25 | | --help | Print a help message and exit. | 26 | | --refine n | Number of times to uniformly refine the initial mesh. | 27 | | --order n | Set the polynomial order of the discretization. | 28 | | --velocity n | Velocity of the flow field. | 29 | | --superlu | Use the SuperLU_Disy direct solver. | 30 | | --no-superlu | Use the GMRES + HYPRE BoomerAMG solver. (default) | 31 | | --slu-colperm n | Set the SuperLU Column permutation algorithm: | 32 | | | 0 - natural | 33 | | | 1 - mmd-ata | 34 | | | 2 - mmd_at_plus_a | 35 | | | 3 - colamd | 36 | | | 4 - metis_at_plus_a (default) | 37 | | | 5 - parmetis | 38 | | | 6 - zoltan | 39 | | --slu-rowperm n | Set the SuperLU Row permutation algorithm: | 40 | | | 0 - NoRowPerm | 41 | | | 1 - LargeDiag (default) | 42 | | | 2 - MyPermR | 43 | | --one-matrix | Solve with one matrix. (default) | 44 | | --two-matrix | Solve with two different matrices. | 45 | | --one-matrix | Solve with one rhs. (default) | 46 | | --two-matrix | Solve with two different rhs. | 47 | | --visit | Output VisIt files for visualation of the solution. | 48 | | --no-visit | Do not output VisIt files. (default) | 49 | -------------------------------------------------------------------------------- /mfem/hypre-superlu/convdiff.cpp: -------------------------------------------------------------------------------- 1 | // MFEM Convection-Diffusion Example with SuperLU 2 | 3 | #include "mfem.hpp" 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace std; 9 | using namespace mfem; 10 | 11 | double SourceField(const Vector &x); 12 | 13 | int main(int argc, char *argv[]) 14 | { 15 | // 1. Initialize MPI. 16 | int num_procs, myid; 17 | MPI_Init(&argc, &argv); 18 | MPI_Comm_size(MPI_COMM_WORLD, &num_procs); 19 | MPI_Comm_rank(MPI_COMM_WORLD, &myid); 20 | 21 | // 2. Parse command-line options. 22 | int ref_levels = 0; 23 | int order = 1; 24 | double velocity = 100.0; 25 | bool visit = false; 26 | bool slu_solver = false; 27 | int slu_colperm = 4; 28 | int slu_rowperm = 1; 29 | int slu_iterref = 2; 30 | bool two_matrix = false; 31 | bool two_rhs = false; 32 | 33 | OptionsParser args(argc, argv); 34 | args.AddOption(&ref_levels, "-r", "--refine", 35 | "Number of times to refine the mesh uniformly in parallel."); 36 | args.AddOption(&order, "-o", "--order", 37 | "Finite element order (polynomial degree) or -1 for" 38 | " isoparametric space."); 39 | args.AddOption(&velocity, "-vel", "--velocity", 40 | "Constant velocity in x that the fluid is flowing with."); 41 | args.AddOption(&visit, "-v", "--visit", "-nov", "--no-visit", 42 | "Enable VisIt visualization."); 43 | args.AddOption(&slu_solver, "-slu", "--superlu", "-no-slu", "--no-superlu", 44 | "Use the SuperLU Solver."); 45 | args.AddOption(&slu_colperm, "-cp", "--slu-colperm", 46 | "Set the SuperLU Column permutation algorithm: 0-natural, " 47 | "1-mmd-ata, 2-mmd_at_plus_a, 3-colamd, 4-metis_at_plus_a, 5-parmetis, 6-zoltan"); 48 | args.AddOption(&slu_rowperm, "-rp", "--slu-rowperm", 49 | "Set the SuperLU Row permutation algorithm: 0-NoRowPerm, " 50 | "1-LargeDiag, 2-MyPermR"); 51 | args.AddOption(&two_matrix, "-2mat", "--two-matrix", "-1mat", "--one-matrix", 52 | "Solve with 1 or two different matrices."); 53 | args.AddOption(&two_rhs, "-2rhs", "--two-rhs", "-1rhs", "--one-rhs", 54 | "Solve with 1 or two different rhs."); 55 | args.Parse(); 56 | if (!args.Good()) 57 | { 58 | if (myid == 0) 59 | { 60 | args.PrintUsage(cout); 61 | } 62 | MPI_Finalize(); 63 | return 1; 64 | } 65 | if (myid == 0) 66 | { 67 | args.PrintOptions(cout); 68 | } 69 | 70 | // 3. Read the (serial) mesh from the given mesh file on all processors. We 71 | // can handle triangular, quadrilateral, tetrahedral, hexahedral, surface 72 | // and volume meshes with the same code. 73 | Mesh mesh = Mesh::MakeCartesian2D(100, 100, Element::QUADRILATERAL, 74 | true, 1.0, 1.0); 75 | int dim = mesh.Dimension(); 76 | 77 | // 5. Define a parallel mesh by a partitioning of the serial mesh. Refine 78 | // this mesh further in parallel to increase the resolution (1 time by 79 | // default, or specified on the command line with -rp). Once the parallel 80 | // mesh is defined, the serial mesh can be deleted. 81 | ParMesh *pmesh = new ParMesh(MPI_COMM_WORLD, mesh); 82 | mesh.Clear(); 83 | for (int lev = 0; lev < ref_levels; lev++) 84 | { 85 | pmesh->UniformRefinement(); 86 | } 87 | 88 | // 6. Define a parallel finite element space on the parallel mesh. Here we 89 | // use continuous Lagrange finite elements of the specified order. If 90 | // order < 1, we instead use an isoparametric/isogeometric space. 91 | FiniteElementCollection *fec; 92 | if (order > 0) 93 | { 94 | fec = new H1_FECollection(order, dim); 95 | } 96 | else if (pmesh->GetNodes()) 97 | { 98 | fec = pmesh->GetNodes()->OwnFEC(); 99 | } 100 | else 101 | { 102 | fec = new H1_FECollection(order = 1, dim); 103 | } 104 | ParFiniteElementSpace *fespace = new ParFiniteElementSpace(pmesh, fec); 105 | HYPRE_Int size = fespace->GlobalTrueVSize(); 106 | if (myid == 0) 107 | { 108 | cout << "Number of unknowns: " << size << endl; 109 | } 110 | 111 | // 7. Set up the parallel bilinear form representing the convection-diffusion 112 | // system. 113 | Array ess_tdof_list; 114 | if (pmesh->bdr_attributes.Size()) 115 | { 116 | Array ess_bdr(pmesh->bdr_attributes.Max()); 117 | ess_bdr = 1; 118 | fespace->GetEssentialTrueDofs(ess_bdr, ess_tdof_list); 119 | } 120 | 121 | ConstantCoefficient diffcoef(1.0); 122 | Vector V(dim); 123 | V = 0.0; 124 | V[0] = velocity; 125 | VectorConstantCoefficient velocitycoef(V); 126 | ParBilinearForm *cd = new ParBilinearForm(fespace); 127 | cd->AddDomainIntegrator(new ConvectionIntegrator(velocitycoef)); 128 | cd->AddDomainIntegrator(new DiffusionIntegrator(diffcoef)); 129 | cd->Assemble(); 130 | 131 | FunctionCoefficient source(SourceField); 132 | ParLinearForm *b = new ParLinearForm(fespace); 133 | b->AddDomainIntegrator(new DomainLFIntegrator(source)); 134 | b->Assemble(); 135 | 136 | HypreParMatrix CD, CD2; 137 | ParGridFunction x(fespace); 138 | x = 0.0; 139 | Vector B, X; 140 | cd->FormLinearSystem(ess_tdof_list, x, *b, CD, X, B); 141 | 142 | // 8. Define and configure the solver. We will use HYPRE with BoomerAMG as 143 | // the preconditioner, or use SuperLU which will handle the full solve in 144 | // one go. 145 | Solver *solver = NULL; 146 | HypreBoomerAMG *amg = NULL; 147 | HypreGMRES *gmres = NULL; 148 | SuperLUSolver *superlu = NULL; 149 | Operator *SLUCD = NULL; 150 | 151 | if (!slu_solver) 152 | { 153 | amg = new HypreBoomerAMG(CD); 154 | amg->SetPrintLevel(1); 155 | gmres = new HypreGMRES(CD); 156 | gmres->SetPrintLevel(1); 157 | gmres->SetTol(1e-12); 158 | gmres->SetMaxIter(200); 159 | gmres->SetPreconditioner(*amg); 160 | solver = gmres; 161 | } 162 | else 163 | { 164 | CD.HostRead(); 165 | SLUCD = new SuperLURowLocMatrix(CD); 166 | CD.HypreRead(); 167 | superlu = new SuperLUSolver(MPI_COMM_WORLD); 168 | superlu->SetPrintStatistics(true); 169 | superlu->SetSymmetricPattern(false); 170 | 171 | if (slu_colperm == 0) 172 | { 173 | superlu->SetColumnPermutation(superlu::NATURAL); 174 | } 175 | else if (slu_colperm == 1) 176 | { 177 | superlu->SetColumnPermutation(superlu::MMD_ATA); 178 | } 179 | else if (slu_colperm == 2) 180 | { 181 | superlu->SetColumnPermutation(superlu::MMD_AT_PLUS_A); 182 | } 183 | else if (slu_colperm == 3) 184 | { 185 | superlu->SetColumnPermutation(superlu::COLAMD); 186 | } 187 | else if (slu_colperm == 4) 188 | { 189 | superlu->SetColumnPermutation(superlu::METIS_AT_PLUS_A); 190 | } 191 | else if (slu_colperm == 5) 192 | { 193 | superlu->SetColumnPermutation(superlu::PARMETIS); 194 | } 195 | else if (slu_colperm == 6) 196 | { 197 | superlu->SetColumnPermutation(superlu::ZOLTAN); 198 | } 199 | 200 | if (slu_rowperm == 0) 201 | { 202 | superlu->SetRowPermutation(superlu::NOROWPERM); 203 | } 204 | else if (slu_rowperm == 1) 205 | { 206 | superlu->SetRowPermutation(superlu::LargeDiag_MC64); 207 | } 208 | else if (slu_rowperm == 2) 209 | { 210 | superlu->SetRowPermutation(superlu::MY_PERMR); 211 | } 212 | 213 | if (slu_iterref == 0) 214 | { 215 | superlu->SetIterativeRefine(superlu::NOREFINE); 216 | } 217 | else if (slu_iterref == 1) 218 | { 219 | superlu->SetIterativeRefine(superlu::SLU_SINGLE); 220 | } 221 | else if (slu_iterref == 2) 222 | { 223 | superlu->SetIterativeRefine(superlu::SLU_DOUBLE); 224 | } 225 | else if (slu_iterref == 3) 226 | { 227 | superlu->SetIterativeRefine(superlu::SLU_EXTRA); 228 | } 229 | 230 | superlu->SetOperator(*SLUCD); 231 | solver = superlu; 232 | } 233 | 234 | // 9. Complete the solve and recover the concentration in the grid function 235 | tic(); 236 | solver->Mult(B, X); 237 | if (myid == 0) 238 | { 239 | cout << "Time required for first solve: " << toc() << " (s)" << endl; 240 | } 241 | Vector R(B); // R = B 242 | CD.Mult(1.0, X, -1.0, R); // R = CD X - B 243 | double res_final = sqrt(InnerProduct(pmesh->GetComm(), R, R)); 244 | if (myid == 0) 245 | { 246 | cout << "Final L2 norm of residual: " << res_final << endl << endl; 247 | } 248 | if (two_rhs) 249 | { 250 | X = 0.0; 251 | tic(); 252 | solver->Mult(B, X); 253 | if (myid == 0) 254 | { 255 | cout << "Time required for second solve (new rhs): " << toc() << " (s)" << endl; 256 | } 257 | Vector R(B); // R = B 258 | CD.Mult(1.0, X, -1.0, R); // R = CD X - B 259 | res_final = sqrt(InnerProduct(pmesh->GetComm(), R, R)); 260 | if (myid == 0) 261 | { 262 | cout << "Final L2 norm of residual: " << res_final << endl << endl; 263 | } 264 | } 265 | 266 | // 9b. Complete the solve a second time with another matrix to show off the saved 267 | // permutation work in SuperLU. 268 | if (two_matrix) 269 | { 270 | X = 0.0; 271 | if (!slu_solver) 272 | { 273 | delete amg; 274 | delete gmres; 275 | amg = new HypreBoomerAMG(CD); 276 | amg->SetPrintLevel(1); 277 | gmres = new HypreGMRES(CD); 278 | gmres->SetPrintLevel(1); 279 | gmres->SetTol(1e-12); 280 | gmres->SetMaxIter(200); 281 | gmres->SetPreconditioner(*amg); 282 | solver = gmres; 283 | } 284 | else 285 | { 286 | delete SLUCD; 287 | CD.HostRead(); 288 | SLUCD = new SuperLURowLocMatrix(CD); 289 | CD.HypreRead(); 290 | superlu->SetOperator(*SLUCD); 291 | } 292 | tic(); 293 | solver->Mult(B, X); 294 | if (myid == 0) 295 | { 296 | cout << "Time required for second matrix (same sparsity): " << toc() << " (s)" << endl; 297 | } 298 | R = B; 299 | CD.Mult(1.0, X, -1.0, R); // R = CD X - B 300 | res_final = sqrt(InnerProduct(pmesh->GetComm(), R, R)); 301 | if (myid == 0) 302 | { 303 | cout << "Final L2 norm of residual: " << res_final << endl; 304 | } 305 | } 306 | cd->RecoverFEMSolution(X, *b, x); 307 | 308 | // 10. Dump the concentration values out to a visit file 309 | if (visit) 310 | { 311 | VisItDataCollection visit_dc("dump", pmesh); 312 | visit_dc.RegisterField("concentration", &x); 313 | visit_dc.Save(); 314 | } 315 | 316 | // 11. Free the used memory. 317 | delete cd; 318 | delete b; 319 | 320 | delete solver; 321 | if (slu_solver) 322 | { 323 | delete SLUCD; 324 | } 325 | else 326 | { 327 | delete amg; 328 | } 329 | 330 | delete fespace; 331 | if (order > 0) 332 | { 333 | delete fec; 334 | } 335 | delete pmesh; 336 | 337 | MPI_Finalize(); 338 | 339 | return 0; 340 | } 341 | 342 | 343 | // This will represent a disc of constant rate input at (0.5, 0.5) 344 | double SourceField(const Vector &x) 345 | { 346 | double R = 0.0; 347 | if (abs(x[0] - 0.5) < 0.05 && abs(x[1] - 0.5) < 0.05) 348 | { 349 | R = 1.0; 350 | } 351 | 352 | return R; 353 | } 354 | -------------------------------------------------------------------------------- /mfem/hypre/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(example magnetic-diffusion) 2 | 3 | if (ENABLE_CUDA) 4 | set_source_files_properties(${example}.cpp PROPERTIES LANGUAGE CUDA) 5 | endif() 6 | if (ENABLE_HIP) 7 | set_source_files_properties(${example}.cpp PROPERTIES LANGUAGE HIP) 8 | endif() 9 | add_executable(${example} ${example}.cpp) 10 | target_link_libraries(${example} PRIVATE ${MFEM_ALL_DEPS}) 11 | 12 | xsdk_add_test( 13 | NAME 14 | MFEM-magnetic-diffusion--cpu 15 | MPI_NPROCS 16 | 4 17 | COMMAND 18 | $ 19 | -m 20 | ../data/star.mesh 21 | -no-vis 22 | ) 23 | # TODO: add the other CPU sample runs from README.md 24 | if(ENABLE_CUDA) 25 | xsdk_add_test( 26 | NAME 27 | MFEM-magnetic-diffusion--gpu 28 | MPI_NPROCS 29 | 4 30 | COMMAND 31 | $ 32 | -m 33 | ../data/star.mesh 34 | -pa 35 | -d 36 | cuda 37 | -no-vis 38 | ) 39 | # TODO: add the other GPU sample runs from README.md 40 | endif() 41 | if(ENABLE_HIP) 42 | xsdk_add_test( 43 | NAME 44 | MFEM-magnetic-diffusion--gpu 45 | MPI_NPROCS 46 | 4 47 | COMMAND 48 | $ 49 | -m 50 | ../data/star.mesh 51 | -pa 52 | -d 53 | hip 54 | -no-vis 55 | ) 56 | # TODO: add the other GPU sample runs from README.md 57 | endif() 58 | 59 | install(TARGETS ${example} RUNTIME DESTINATION bin) 60 | -------------------------------------------------------------------------------- /mfem/hypre/README.md: -------------------------------------------------------------------------------- 1 | # MFEM-HYPRE example 2 | 3 | This example demonstrates the integration of MFEM with HYPRE on both CPUs and 4 | GPUs. 5 | 6 | This example code utilizes MFEM to discretize a steady state magnetic diffusion 7 | problem, curl curl E + E = f, with suitable boundary condition. The problem is 8 | discretized using H(curl)-conforming finite elements of arbitrary specified 9 | order on any given mesh. The discretized problem is then solved using PCG with 10 | HYPRE's AMS preconditioner. 11 | 12 | Sample CPU runs: 13 | ``` 14 | mpirun -np 4 ./magnetic-diffusion -m ../data/star.mesh 15 | mpirun -np 4 ./magnetic-diffusion -m ../data/beam-hex.mesh 16 | mpirun -np 4 ./magnetic-diffusion -m ../data/beam-hex.mesh -o 2 -pa 17 | ``` 18 | 19 | Sample GPU runs, replace `` with `cuda` or `hip`: 20 | ``` 21 | mpirun -np 4 ./magnetic-diffusion -m ../data/star.mesh -pa -d 22 | mpirun -np 4 ./magnetic-diffusion -m ../data/star.mesh -no-pa -d 23 | mpirun -np 4 ./magnetic-diffusion -m ../data/beam-hex.mesh -pa -d 24 | ``` 25 | 26 | For a full list of options, see 27 | ``` 28 | ./magnetic-diffusion -h 29 | ``` 30 | -------------------------------------------------------------------------------- /mfem/petsc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (ENABLE_CUDA) 2 | set_source_files_properties(obstacle.cpp PROPERTIES LANGUAGE CUDA) 3 | endif() 4 | if (ENABLE_HIP) 5 | set_source_files_properties(obstacle.cpp PROPERTIES LANGUAGE HIP) 6 | endif() 7 | add_executable(obstacle obstacle.cpp) 8 | 9 | target_link_libraries(obstacle PRIVATE ${MFEM_ALL_DEPS}) 10 | 11 | # This is a serial example linked with MPI, so launch it with 'mpirun' or 12 | # equivalent to avoid warnings on some platforms. 13 | xsdk_add_test( 14 | NAME 15 | MFEM-obstacle 16 | MPI_NPROCS 17 | 1 18 | COMMAND 19 | $ 20 | --order 21 | 2 22 | ) 23 | 24 | install(TARGETS obstacle RUNTIME DESTINATION bin) 25 | -------------------------------------------------------------------------------- /mfem/petsc/README.md: -------------------------------------------------------------------------------- 1 | # MFEM-PETSc example 2 | This example solves the classical obstacle problem which models an edge clamped 3 | elastic membrane pulled over a rigid obstacle. MFEM is used to discretize the 4 | underlying Poisson equation and PETSC-TAO is used the solve to optimization 5 | problem. This problem also demonstrates the how MFEM and PETSc can share the 6 | data from vectors of each type. 7 | 8 | This example is built to run in serial, so launch it wth your desired options: 9 | ``` 10 | ./obstacle --order 2 11 | ``` 12 | 13 | Useful non-default options: 14 | | Flag | Meaning | 15 | |:----------------------| :-----------------------------------------------------| 16 | | --order n | Set the polynomial order of the discretization. | 17 | | --visit | Output VisIt files for visualation of the solution. | 18 | -------------------------------------------------------------------------------- /mfem/pumi/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (ENABLE_CUDA) 2 | set_source_files_properties(adapt.cpp PROPERTIES LANGUAGE CUDA) 3 | endif() 4 | if (ENABLE_HIP) 5 | set_source_files_properties(adapt.cpp PROPERTIES LANGUAGE HIP) 6 | endif() 7 | add_executable(adapt adapt.cpp) 8 | target_link_libraries(adapt PRIVATE ${MFEM_ALL_DEPS}) 9 | xsdk_add_test( 10 | NAME 11 | MFEM-PUMI-adapt 12 | MPI_NPROCS 13 | 2 14 | COMMAND 15 | $ 16 | -no-vis 17 | ) 18 | install(TARGETS adapt RUNTIME DESTINATION bin) 19 | -------------------------------------------------------------------------------- /mfem/pumi/README.md: -------------------------------------------------------------------------------- 1 | # MFEM-PUMI example 2 | 3 | This example, `adapt.cpp`, illustrates the integration of PUMI with MFEM and it 4 | is based on the parallel PUMI-based version of example 6 from MFEM. 5 | 6 | For a description of the problem being solved, see the documentation in the 7 | beginning of the source file. 8 | 9 | Sample runs: 10 | ``` 11 | mpirun -np 2 ./adapt 12 | ``` 13 | 14 | For a full list of options, see 15 | ``` 16 | ./adapt -h 17 | ``` 18 | -------------------------------------------------------------------------------- /mfem/strumpack/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (ENABLE_CUDA) 2 | set_source_files_properties(diffusion-eigen.cpp PROPERTIES LANGUAGE CUDA) 3 | endif() 4 | if (ENABLE_HIP) 5 | set_source_files_properties(diffusion-eigen.cpp PROPERTIES LANGUAGE HIP) 6 | endif() 7 | add_executable(diffusion-eigen diffusion-eigen.cpp) 8 | target_link_libraries(diffusion-eigen PRIVATE ${MFEM_ALL_DEPS}) 9 | 10 | xsdk_add_test( 11 | NAME 12 | MFEM-diffusion-eigen--strumpack 13 | MPI_NPROCS 14 | 4 15 | COMMAND 16 | $ 17 | -m 18 | ../data/star.mesh 19 | -ls 20 | strumpack 21 | ) 22 | if(ENABLE_SUPERLU) 23 | xsdk_add_test( 24 | NAME 25 | MFEM-diffusion-eigen--superlu 26 | MPI_NPROCS 27 | 4 28 | COMMAND 29 | $ 30 | -m 31 | ../data/star.mesh 32 | -ls 33 | superlu 34 | ) 35 | endif() 36 | xsdk_add_test( 37 | NAME 38 | MFEM-diffusion-eigen--hypre-boomeramg 39 | MPI_NPROCS 40 | 4 41 | COMMAND 42 | $ 43 | -m 44 | ../data/star.mesh 45 | -ls 46 | hypre 47 | ) 48 | 49 | install(TARGETS diffusion-eigen RUNTIME DESTINATION bin) 50 | -------------------------------------------------------------------------------- /mfem/strumpack/README.md: -------------------------------------------------------------------------------- 1 | # MFEM-STRUMPACK-SuperLU-HYPRE example 2 | 3 | This example code utilizes MFEM to discretize a diffusion eigenvalue problem 4 | -Delta u = lambda u, with homogeneous Dirichlet boundary conditions. The problem 5 | is discretized using continuous finite elements of arbitrary specified order on 6 | any given mesh. The discretized eigenvalue problem is solved using HYPRE's 7 | LOBPCG eigenvalue solver with linear system preconditioner/solver using 8 | STRUMPACK, SuperLU, or HYPRE's BoomerAMG. 9 | 10 | This example is built to run in parallel, so launch it with mpirun and your 11 | desired options, e.g. using STRUMPACK: 12 | ``` 13 | mpirun -np 4 ./diffusion-eigen -m ../data/star.mesh -ls strumpack 14 | ``` 15 | SuperLU: 16 | ``` 17 | mpirun -np 4 ./diffusion-eigen -m ../data/star.mesh -ls superlu 18 | ``` 19 | or HYPRE: 20 | ``` 21 | mpirun -np 4 ./diffusion-eigen -m ../data/star.mesh -ls hypre 22 | ``` 23 | 24 | For a full list of options, see 25 | ``` 26 | ./diffusion-eigen -h 27 | ``` 28 | Note that when using STRUMPACK (the default) command-line parameters are also 29 | passed to STRUMPACK to support STRUMPACK-specifiic parameters. 30 | -------------------------------------------------------------------------------- /mfem/sundials/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | if (ENABLE_CUDA) 2 | set_source_files_properties(transient-heat.cpp PROPERTIES LANGUAGE CUDA) 3 | endif() 4 | if (ENABLE_HIP) 5 | set_source_files_properties(transient-heat.cpp PROPERTIES LANGUAGE HIP) 6 | endif() 7 | add_executable(transient-heat transient-heat.cpp) 8 | target_link_libraries(transient-heat PRIVATE ${MFEM_ALL_DEPS}) 9 | if(NOT ENABLE_CUDA AND NOT ENABLE_HIP) 10 | # This example fails (as of 2023/05/08) when HYPRE is built with CUDA or 11 | # HIP, so it is disabled in that case for now. 12 | xsdk_add_test( 13 | NAME 14 | MFEM-transient-heat 15 | MPI_NPROCS 16 | 4 17 | COMMAND 18 | $ 19 | --mesh 20 | ../data/star.mesh 21 | --kappa 22 | 0.5 23 | --alpha 24 | 0.01 25 | --ode-solver 26 | 8 27 | --no-visualization 28 | ) 29 | endif() 30 | install(TARGETS transient-heat RUNTIME DESTINATION bin) 31 | 32 | if (ENABLE_CUDA) 33 | set_source_files_properties(advection.cpp PROPERTIES LANGUAGE CUDA) 34 | endif() 35 | if (ENABLE_HIP) 36 | set_source_files_properties(advection.cpp PROPERTIES LANGUAGE HIP) 37 | endif() 38 | add_executable(advection advection.cpp) 39 | target_link_libraries(advection PRIVATE ${MFEM_ALL_DEPS}) 40 | if(NOT ENABLE_CUDA AND NOT ENABLE_HIP) 41 | # This example fails (as of 2023/05/08) when CUDA or HIP is enabled, so we 42 | # disable it here for now. 43 | xsdk_add_test( 44 | NAME 45 | MFEM-advection--cpu 46 | MPI_NPROCS 47 | 4 48 | COMMAND 49 | $ 50 | --device 51 | cpu 52 | --partial-assembly 53 | --ode-solver 54 | 8 55 | --no-visualization 56 | ) 57 | endif() 58 | if(ENABLE_CUDA) 59 | xsdk_add_test( 60 | NAME 61 | MFEM-advection--gpu 62 | MPI_NPROCS 63 | 4 64 | COMMAND 65 | $ 66 | --device 67 | cuda 68 | --partial-assembly 69 | --ode-solver 70 | 8 71 | --no-visualization 72 | ) 73 | endif() 74 | if (ENABLE_HIP) 75 | # MFEM+SUNDIALS+HIP requires MFEM PR #3596 which as of 2023/05/08 is under 76 | # review. 77 | endif() 78 | install(TARGETS advection RUNTIME DESTINATION bin) 79 | -------------------------------------------------------------------------------- /mfem/sundials/README.md: -------------------------------------------------------------------------------- 1 | # MFEM-SUNDIALS examples 2 | 3 | ## transient-heat.cpp 4 | 5 | This example solves a time dependent heat equation with a temperature dependent 6 | conductivity. MFEM is used for the spatial discretization and SUNDIALS is 7 | used for the ODE time integration. This problem demonstrates the MFEM 8 | integrations with the SUNDIALS CVODE and ARKODE solvers which give MFEM 9 | access to an array of advance ODE solvers. 10 | 11 | This example is built to run in parallel, so launch it with mpirun and your desired options: 12 | ``` 13 | mpirun -np 4 ./transient-heat --kappa 0.5 --alpha 0.01 --ode-solver 8 14 | ``` 15 | 16 | Useful non-default options: 17 | | Flag | Meaning | 18 | |:----------------------| :-----------------------------------------------------| 19 | | --order n | Set the polynomial order of the discretization. | 20 | | --kappa n | Set up the conductivity model C(T) = kappa + alpha T. | 21 | | --alpha n | Set up the conductivity model C(T) = kappa + alpha T. | 22 | | --ode-solver n | Pick the ODE solver used for the time integration. | 23 | | .............. | 1 - MFEM (Forward Euler). | 24 | | .............. | 2 - MFEM (RK2). | 25 | | .............. | 3 - MFEM (RK3 SSP). | 26 | | .............. | 4 - MFEM (RK4). | 27 | | .............. | 5 - MFEM (Backward Euler). | 28 | | .............. | 6 - MFEM (SDIRK 2). | 29 | | .............. | 7 - MFEM (SDIRK 3). | 30 | | .............. | 8 - CVODE (implicit Adams). | 31 | | .............. | 9 - CVODE (implicit BDF). | 32 | | .............. | 10 - ARKODE (default explicit). | 33 | | .............. | 11 - ARKODE (explicit Fehlberg-6-4-5). | 34 | | .............. | 12 - ARKODE (default impicit). | 35 | | --mesh s | Mesh file to use. | 36 | 37 | 38 | ## advection.cpp 39 | 40 | This example code solves the time-dependent advection equation. MFEM is used for the 41 | spatial discretization and SUNDIALS is used for the ODE time integration. This problem 42 | demonstrates MFEM integration with the SUNDIALS CVODE and ARKODE solvers for CUDA. 43 | 44 | This example is built to run in parallel, so launch it with mpirun and your desired options: 45 | ``` 46 | mpirun -np 4 ./advection --device cuda --partial-assembly --ode-solver 8 47 | mpirun -np 4 ./advection --device cpu --partial-assembly --ode-solver 8 48 | ``` 49 | 50 | Useful non-default options: 51 | | Flag | Meaning | 52 | |:----------------------| :-----------------------------------------------------| 53 | | --device s | Device configuration string. Choose cpu or cuda. | 54 | | --order n | Set the polynomial order of the discretization. | 55 | | --partial-assembly | Enable Partial Assembly. | 56 | | --element-assembly | Enable Element Assembly. | 57 | | --full-assembly | Enable Full Assembly. | 58 | | --ode-solver n | Pick the ODE solver used for the time integration. | 59 | | .............. | 1 - MFEM (Forward Euler). | 60 | | .............. | 2 - MFEM (RK2). | 61 | | .............. | 3 - MFEM (RK3 SSP). | 62 | | .............. | 4 - MFEM (RK4). | 63 | | .............. | 6 - MFEM (RK6). | 64 | | .............. | 7 - CVODE (implicit Adams). | 65 | | .............. | 8 - ARKODE (default 4th order explicit). | 66 | | .............. | 9 - ARKODE (RK8). | 67 | | --mesh s | Mesh file to use. | 68 | -------------------------------------------------------------------------------- /petsc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(ex19 ex19.c) 2 | target_link_libraries(ex19 PRIVATE XSDK::PETSc MPI::MPI_C) 3 | if (TARGET OpenMP::OpenMP_C) 4 | target_link_libraries(ex19 PRIVATE OpenMP::OpenMP_C) 5 | endif() 6 | 7 | # The makefile to run examples requires rm command 8 | find_program(RM rm) 9 | 10 | # If PETSc was found without setting PETSc_DIR, e.g. using CMAKE_PREFIX_PATH 11 | if(NOT PETSc_DIR) 12 | set(PETSc_DIR ${PC_PETSc_PREFIX}) 13 | endif() 14 | # Handle cases like Flux where MPIEXEC_EXECUTABLE can be "flux;run" 15 | string(REPLACE ";" " " MPIEXEC_EXECUTABLE "${MPIEXEC_EXECUTABLE}") 16 | 17 | # We use the makefile to launch the tests 18 | configure_file(makefile ${CMAKE_CURRENT_BINARY_DIR}/makefile-run) 19 | file(COPY output DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/output) 20 | 21 | if(MATH_LIBRARY) 22 | target_link_libraries(ex19 PRIVATE ${MATH_LIBRARY}) 23 | endif() 24 | 25 | xsdk_add_test( 26 | NAME 27 | PETSc-ex19_1 28 | COMMAND 29 | make 30 | -f 31 | makefile-run 32 | runex19 33 | ENVIRONMENT 34 | PETSC_DIR=${PETSc_DIR} 35 | ) 36 | if(ENABLE_CUDA) 37 | xsdk_add_test( 38 | NAME 39 | PETSc-ex19_cuda 40 | COMMAND 41 | make 42 | -f 43 | makefile-run 44 | runex19_cuda 45 | ENVIRONMENT 46 | PETSC_DIR=${PETSc_DIR} 47 | ) 48 | endif() 49 | if(ENABLE_HIP) 50 | xsdk_add_test( 51 | NAME 52 | PETSc-ex19_hip 53 | COMMAND 54 | make 55 | -f 56 | makefile-run 57 | runex19_hip 58 | ENVIRONMENT 59 | PETSC_DIR=${PETSc_DIR} 60 | ) 61 | endif() 62 | if(ENABLE_HYPRE) 63 | if(ENABLE_CUDA) 64 | xsdk_add_test( 65 | NAME 66 | PETSc-ex19_hypre_cuda 67 | COMMAND 68 | make 69 | -f 70 | makefile-run 71 | runex19_hypre_cuda 72 | ENVIRONMENT 73 | PETSC_DIR=${PETSc_DIR} 74 | ) 75 | else() 76 | xsdk_add_test( 77 | NAME 78 | PETSc-ex19_hypre 79 | COMMAND 80 | make 81 | -f 82 | makefile-run 83 | runex19_hypre 84 | ENVIRONMENT 85 | PETSC_DIR=${PETSc_DIR} 86 | ) 87 | endif() 88 | endif() 89 | if(ENABLE_SUPERLU) 90 | xsdk_add_test( 91 | NAME 92 | PETSc-ex19_superlu_dist 93 | COMMAND 94 | make 95 | -f 96 | makefile-run 97 | runex19_superlu_dist 98 | ENVIRONMENT 99 | PETSC_DIR=${PETSc_DIR} 100 | ) 101 | endif() 102 | 103 | install(FILES output/ex19_1.testout output/ex19_cuda_1.out output/ex19_hypre.out 104 | output/ex19_hip.out output/ex19_superlu.out DESTINATION output 105 | ) 106 | install(TARGETS ex19 RUNTIME DESTINATION bin) 107 | -------------------------------------------------------------------------------- /petsc/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 1991-2014, UChicago Argonne, LLC and the PETSc Development Team 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above copyright notice, this 10 | list of conditions and the following disclaimer in the documentation and/or 11 | other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 20 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | 24 | 25 | This license DOES NOT apply to any software that may be obtained via the 26 | --download-package option of the PETSc configuration. Each of those 27 | packages are covered by their own licenses. 28 | -------------------------------------------------------------------------------- /petsc/README.md: -------------------------------------------------------------------------------- 1 | # PETSc-HYPRE-SuperLU-ML example 2 | This example sets up and solves a 2D driven cavity problem on the unit square. A finite 3 | difference method with a 5 point stencil is used to discretize the velocity-vorticity 4 | formulation and create operators and vectors in the PETSc framework. The PETSc 5 | SNES (Scalable Nonlinear Equations Solvers) components are then used to solve the 6 | nonlinear system that arises. For each iteration of the nonlinear solution, a system 7 | a linear equations is solved with either HYPRE or SuperLU, ML from Trilinos. 8 | 9 | This example is built to run in parallel, so launch it with mpirun and your desired options. 10 | 11 | To run with Hypre: 12 | ``` 13 | mpirun -np 2 ./ex19 -da_refine 3 -snes_monitor_short -pc_type hypre 14 | ``` 15 | To run with SuperLU: 16 | ``` 17 | mpirun -np 2 ./ex19 -da_grid_x 20 -da_grid_y 20 -pc_type lu -pc_factor_mat_solver_type superlu_dist 18 | ``` 19 | To run with ML from Trilinos: 20 | ``` 21 | mpirun -np 2 ./ex19 -da_refine 3 -snes_monitor_short -pc_type ml 22 | ``` 23 | 24 | Useful non-default options: 25 | | Flag | Meaning | 26 | |:----------------------| :-----------------------------------------------------| 27 | | -pctype [type] | Set the pc solver type for example: hypre, ml, lu. | 28 | | -da_refine n | Set the number of times to refine the mesh. | 29 | -------------------------------------------------------------------------------- /petsc/makefile: -------------------------------------------------------------------------------- 1 | CFLAGS = 2 | FFLAGS = 3 | CPPFLAGS = 4 | FPPFLAGS = 5 | 6 | include @PETSc_DIR@/lib/petsc/conf/variables 7 | include @PETSc_DIR@/lib/petsc/conf/rules 8 | OUTPUT_DIR = @CMAKE_CURRENT_SOURCE_DIR@/output 9 | DIFF = @PETSc_DIR@/lib/petsc/bin/petscdiff 10 | 11 | #------------------------------------------------------------------------- 12 | 13 | runex19: 14 | @@MPIEXEC_EXECUTABLE@ @MPIEXEC_NUMPROC_FLAG@ 2 ./ex19 -da_refine 3 -pc_type mg -ksp_type fgmres > ex19_1.tmp 2>&1; \ 15 | if $(DIFF) $(OUTPUT_DIR)/ex19_1.testout ex19_1.tmp; then \ 16 | echo "C/C++ example src/snes/tutorials/ex19 run successfully with 2 MPI processes"; \ 17 | else printf "$(PWD)\nPossible problem with ex19, diffs above\n=========================================\n"; exit 1; fi; \ 18 | @RM@ -f ex19_1.tmp 19 | runex19_cuda: 20 | @@MPIEXEC_EXECUTABLE@ @MPIEXEC_NUMPROC_FLAG@ 1 ./ex19 -snes_monitor -dm_mat_type seqaijcusparse -dm_vec_type seqcuda -pc_type gamg -ksp_monitor -mg_levels_ksp_max_it 3 > ex19_1.tmp 2>&1; \ 21 | if $(DIFF) $(OUTPUT_DIR)/ex19_cuda_1.out ex19_1.tmp; then \ 22 | echo "C/C++ example src/snes/tutorials/ex19 run successfully with cuda"; \ 23 | else printf "$(PWD)\nPossible problem with ex19 running with cuda, diffs above\n=========================================\n"; exit 1; fi; \ 24 | @RM@ -f ex19_1.tmp 25 | 26 | runex19_hip: 27 | @@MPIEXEC_EXECUTABLE@ @MPIEXEC_NUMPROC_FLAG@ 1 ./ex19 -dm_vec_type hip -dm_mat_type aijhipsparse -pc_type none -ksp_type fgmres -snes_monitor_short -snes_rtol 1.e-5 > ex19_1.tmp 2>&1; \ 28 | if $(DIFF) $(OUTPUT_DIR)/ex19_hip.out ex19_1.tmp; then \ 29 | echo "C/C++ example src/snes/tutorials/ex19 run successfully with hip"; \ 30 | else printf "$(PWD)\nPossible problem with ex19 running with hip, diffs above\n=========================================\n";\ exit 1; fi; \ 31 | @RM@ -f ex19_1.tmp 32 | 33 | runex19_hypre: 34 | @@MPIEXEC_EXECUTABLE@ @MPIEXEC_NUMPROC_FLAG@ 2 ./ex19 -da_refine 3 -snes_monitor_short -pc_type hypre > ex19_1.tmp 2>&1; \ 35 | if $(DIFF) $(OUTPUT_DIR)/ex19_hypre.out ex19_1.tmp; then \ 36 | echo "C/C++ example src/snes/examples/tutorials/ex19 run successfully with hypre"; \ 37 | else printf "$(PWD)\nPossible problem with ex19 running with hypre, diffs above\n=========================================\n"; exit 1; fi; \ 38 | @RM@ -f ex19_1.tmp 39 | runex19_hypre_cuda: 40 | @@MPIEXEC_EXECUTABLE@ @MPIEXEC_NUMPROC_FLAG@ 2 ./ex19 -dm_vec_type cuda -dm_mat_type aijcusparse -da_refine 3 -snes_monitor_short -ksp_norm_type unpreconditioned -pc_type hypre > ex19_1.tmp 2>&1; \ 41 | if $(DIFF) $(OUTPUT_DIR)/ex19_hypre.out ex19_1.tmp; then \ 42 | echo "C/C++ example src/snes/tutorials/ex19 run successfully with hypre/cuda"; \ 43 | else printf "$(PWD)\nPossible problem with ex19 running with hypre, diffs above\n=========================================\n"; exit 1; fi; \ 44 | @RM@ -f ex19_1.tmp 45 | runex19_superlu_dist: 46 | @@MPIEXEC_EXECUTABLE@ @MPIEXEC_NUMPROC_FLAG@ 2 ./ex19 -da_grid_x 20 -da_grid_y 20 -pc_type lu -pc_factor_mat_solver_type superlu_dist > ex19.tmp 2>&1; \ 47 | if $(DIFF) $(OUTPUT_DIR)/ex19_superlu.out ex19.tmp; then \ 48 | echo "C/C++ example src/snes/examples/tutorials/ex19 run successfully with superlu_dist"; \ 49 | else printf "$(PWD)\nPossible problem with ex19 running with superlu_dist, diffs above\n=========================================\n"; exit 1; fi; \ 50 | @RM@ -f ex19.tmp 51 | 52 | # include @PETSc_DIR@/lib/petsc/conf/test 53 | -------------------------------------------------------------------------------- /petsc/output/ex19_1.testout: -------------------------------------------------------------------------------- 1 | lid velocity = 0.0016, prandtl # = 1., grashof # = 1. 2 | Number of SNES iterations = 2 3 | -------------------------------------------------------------------------------- /petsc/output/ex19_cuda_1.out: -------------------------------------------------------------------------------- 1 | lid velocity = 0.0625, prandtl # = 1., grashof # = 1. 2 | 0 SNES Function norm 2.391552133017e-01 3 | 0 KSP Residual norm 2.928487269734e-01 4 | 1 KSP Residual norm 1.876489580142e-02 5 | 2 KSP Residual norm 3.291394847944e-03 6 | 3 KSP Residual norm 2.456493072124e-04 7 | 4 KSP Residual norm 1.161647147715e-05 8 | 5 KSP Residual norm 1.285648407621e-06 9 | 1 SNES Function norm 6.846805706142e-05 10 | 0 KSP Residual norm 2.292783790384e-05 11 | 1 KSP Residual norm 2.100673631699e-06 12 | 2 KSP Residual norm 2.121341386147e-07 13 | 3 KSP Residual norm 2.455932678957e-08 14 | 4 KSP Residual norm 1.753095730744e-09 15 | 5 KSP Residual norm 7.489214418904e-11 16 | 2 SNES Function norm 2.103908447865e-10 17 | Number of SNES iterations = 2 18 | -------------------------------------------------------------------------------- /petsc/output/ex19_hip.out: -------------------------------------------------------------------------------- 1 | lid velocity = 0.0625, prandtl # = 1., grashof # = 1. 2 | 0 SNES Function norm 0.239155 3 | 1 SNES Function norm 6.82338e-05 4 | 2 SNES Function norm 4.36339e-08 5 | Number of SNES iterations = 2 6 | -------------------------------------------------------------------------------- /petsc/output/ex19_hypre.out: -------------------------------------------------------------------------------- 1 | lid velocity = 0.0016, prandtl # = 1., grashof # = 1. 2 | 0 SNES Function norm 0.0406612 3 | 1 SNES Function norm 4.12227e-06 4 | 2 SNES Function norm 6.098e-11 5 | Number of SNES iterations = 2 6 | -------------------------------------------------------------------------------- /petsc/output/ex19_superlu.out: -------------------------------------------------------------------------------- 1 | lid velocity = 0.0025, prandtl # = 1., grashof # = 1. 2 | Number of SNES iterations = 2 3 | -------------------------------------------------------------------------------- /plasma/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # enforce OpenMP requirement; OpenMP search results should be cached at 2 | # this point 3 | find_package(OpenMP REQUIRED) 4 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") 5 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 6 | 7 | add_executable(ex1solve ex1solve.cpp) 8 | if(NOT ENABLE_CUDA) 9 | target_compile_definitions(ex1solve PRIVATE SLATE_NO_CUDA) 10 | endif() 11 | target_compile_definitions(ex1solve PRIVATE SLATE_NO_HIP) 12 | 13 | target_link_libraries( 14 | ex1solve PRIVATE XSDK::PLASMA XSDK::SLATE XSDK::LAPACKPP XSDK::BLASPP MPI::MPI_CXX 15 | ) 16 | if(MATH_LIBRARY) 17 | target_link_libraries(ex1solve PRIVATE ${MATH_LIBRARY}) 18 | endif() 19 | if(ENABLE_CUDA) 20 | target_link_libraries(ex1solve PRIVATE CUDA::cudart) 21 | endif() 22 | 23 | xsdk_add_test( 24 | NAME 25 | PLASMA-ex1solve 26 | COMMAND 27 | env 28 | OMP_NUM_THREADS=1 29 | $ 30 | --n=1000 31 | ) 32 | 33 | install(TARGETS ex1solve RUNTIME DESTINATION bin) 34 | -------------------------------------------------------------------------------- /plasma/README.md: -------------------------------------------------------------------------------- 1 | # PLASMA examples 2 | Example codes demonstrating the use of PLASMA using other XSDK packages. 3 | 4 | ## Using SLATE and its compute layers 5 | `ex1solve` solves a system of linear equations by using Level 3 BLAS from 6 | SLATE's BLAS++ instead of PLASMA's internal interface. The example accepts an 7 | optional command line parameter to specify linear system size like so: 8 | ``` 9 | ./ex1solve [--n=1000] 10 | ``` 11 | If the size is not specified then 1000 is used. 12 | -------------------------------------------------------------------------------- /plasma/ex1solve.cpp: -------------------------------------------------------------------------------- 1 | /* ex1solve.cpp */ 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | 17 | static double 18 | wtime() { 19 | struct timeval tv; 20 | gettimeofday(&tv, NULL); 21 | return tv.tv_sec + 1e-6 * tv.tv_usec; 22 | } 23 | 24 | 25 | static inline blas::Op 26 | t_p2bpp(plasma_enum_t trans) { 27 | return PlasmaNoTrans == trans ? blas::Op::NoTrans : (PlasmaTrans == trans ? blas::Op::Trans : blas::Op::ConjTrans); 28 | } 29 | 30 | static inline blas::Side 31 | s_p2bpp(plasma_enum_t side) { 32 | return PlasmaLeft == side ? blas::Side::Left : blas::Side::Right; 33 | } 34 | 35 | static inline blas::Uplo 36 | u_p2bpp(plasma_enum_t uplo) { 37 | return PlasmaUpper == uplo ? blas::Uplo::Upper : blas::Uplo::Lower; 38 | } 39 | 40 | static inline blas::Diag 41 | d_p2bpp(plasma_enum_t diag) { 42 | return PlasmaUnit == diag ? blas::Diag::Unit : blas::Diag::NonUnit; 43 | } 44 | 45 | static std::atomic Counter_Gemm; 46 | 47 | extern "C" void 48 | plasma_core_dgemm(plasma_enum_t transa, plasma_enum_t transb, int m, int n, int k, double alpha, const double *A, int lda, const double *B, int ldb, double beta, double *C, int ldc) { 49 | ++Counter_Gemm; 50 | blas::gemm(blas::Layout::ColMajor, t_p2bpp(transa), t_p2bpp(transb), m, n, k, alpha, A, lda, B, ldb, beta, C, ldc); 51 | } 52 | 53 | static std::atomic Counter_Trsm; 54 | 55 | extern "C" void 56 | plasma_core_dtrsm(plasma_enum_t side, plasma_enum_t uplo, plasma_enum_t trans, plasma_enum_t diag, int m, int n, double alpha, const double *A, int lda, double *B, int ldb) { 57 | ++Counter_Trsm; 58 | blas::trsm(blas::Layout::ColMajor, s_p2bpp(side), u_p2bpp(uplo), t_p2bpp(trans), d_p2bpp(diag), m, n, alpha, A, lda, B, ldb); 59 | } 60 | 61 | void 62 | drndset(int m, int n, double *A, int A_ld, int seed) { 63 | double rcp = 1.0-1.0/RAND_MAX; 64 | 65 | srand(seed); 66 | 67 | for (int j = 0; j < n; ++j) 68 | for (int i = 0; i < m; ++i) 69 | A[i + j * A_ld] = rand() * rcp; 70 | } 71 | 72 | int 73 | main(int argc, char *argv[]) { 74 | int n, nrhs, nb, ib, A_ld, B_ld, *piv, info; 75 | double *A, *B; 76 | 77 | n = 1000; 78 | nrhs = 1; 79 | 80 | for (int i = 1; i < argc && argv[i]; ++i) 81 | if (strncmp("--n=", argv[i], 2+1+1) != 0 || sscanf(argv[i]+2+1+1, "%d", &n) <= 0 || n < 1) 82 | n = 1000; 83 | 84 | A_ld = n; 85 | B_ld = n; 86 | 87 | A = (double *)malloc( sizeof *A * A_ld * n ); 88 | B = (double *)malloc( sizeof *B * B_ld * nrhs ); 89 | piv = (int *)malloc( sizeof *piv * n ); 90 | 91 | drndset(n, n, A, A_ld, 1313); 92 | drndset(n, nrhs, B, B_ld, 1313); 93 | 94 | plasma_init(); 95 | 96 | plasma_get(PlasmaNb, &nb); 97 | plasma_get(PlasmaIb, &ib); 98 | 99 | Counter_Gemm = 0; 100 | Counter_Trsm = 0; 101 | double t = -wtime(); 102 | info = plasma_dgesv(n, nrhs, A, A_ld, piv, B, B_ld); 103 | t += wtime(); 104 | int cntgemm = Counter_Gemm; 105 | int cnttrsm = Counter_Trsm; 106 | 107 | plasma_finalize(); 108 | 109 | free(piv); 110 | free(B); 111 | free(A); 112 | 113 | printf("n=%d nrhs=%d t=%g gflop/s=%g", n, nrhs, t, lapack::Gflop::gesv(n, nrhs) / t); 114 | printf(" nb=%d ib=%d gemm=%d trsm=%d", nb, ib, cnttrsm, cntgemm); 115 | printf("\n"); 116 | 117 | return 0; 118 | } 119 | -------------------------------------------------------------------------------- /strumpack/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(sparse sparse.cpp) 2 | 3 | target_link_libraries(sparse PRIVATE STRUMPACK::strumpack) 4 | if(MATH_LIBRARY) 5 | target_link_libraries(sparse PRIVATE ${MATH_LIBRARY}) 6 | endif() 7 | 8 | if (NOT ENABLE_CUDA) 9 | # This example randomly fails with strumpack@7.0.1 and slate@2022.07.00, 10 | # i.e. xsdk@0.8.0, with CUDA enabled, so it is disabled for now. 11 | # See https://github.com/xsdk-project/xsdk-examples/issues/47 12 | xsdk_add_test( 13 | NAME 14 | STRUMPACK-sparse 15 | MPI_NPROCS 16 | 4 17 | COMMAND 18 | $ 19 | 24 20 | --sp_compression 21 | hodlr 22 | --hodlr_butterfly_levels 23 | 10 24 | ENVIRONMENT 25 | OMP_NUM_THREADS=4 26 | ) 27 | endif() 28 | 29 | install(TARGETS sparse RUNTIME DESTINATION bin) 30 | -------------------------------------------------------------------------------- /strumpack/README.md: -------------------------------------------------------------------------------- 1 | 24 | # STRUMPACK examples 25 | 26 | Example codes demonstrating the use of [STRUMPACK](https://portal.nersc.gov/project/sparse/strumpack/) using other XSDK packages ([ButterflyPACK](https://github.com/liuyangzhuan/ButterflyPACK)). 27 | 28 | ## STRUMPACK + ButterflyPACK 29 | 30 | The `sparse.cpp` example demonstrates STRUMPACK's algebraic sparse 31 | direct solver and preconditioners for solving the 3-D Laplacian 32 | problem with zero boundary conditions on an n x n x n grid. The 33 | number of unknowns is N=n^3. The standard 7-point stencil is used, 34 | and we solve for the interior nodes only. 35 | 36 | STRUMPACK implements multifrontal sparse LU factorization, with the 37 | option of compression of the larger frontal matrices. Compression 38 | algorithms include HODLR (Hierarchically Off-Diagonal Low Rank) and 39 | HODBF (Hierarchically Off-Diagonal Butterfly), BLR (Block Low Rank), 40 | HSS (Hierarchically Semi Separable), lossy or lossless. 41 | 42 | After factorization, linear system can be solved using forward and 43 | backward substitution with the lower and upper triangular factors 44 | respectively. Without compression, the solver behaves as a sparse 45 | direct method. The sparse direct solver still uses iterative 46 | refinement, but typically only needs a single iteration. When 47 | compression is enabled, the LU factorization is only approximate, and 48 | the solver is used as a preconditioner for GMRES (or BiCGStab). 49 | 50 | 51 | ### Usage 52 | 53 | **Sample run**: `OMP_NUM_THREADS=4 mpirun -np 4 ./sparse 100 --sp_compression hodlr --hodlr_butterfly_levels 10` 54 | 55 | - This run solves a system corresponding to a discretization 56 | of the Laplace equation -Delta u = f with zero boundary 57 | conditions on a 100 x 100 x 100 grid, using STRUMPACK's 58 | distributed LU factorization, and with both HODBF compression 59 | for the largest fronts (dense sub-blocks in the sparse factors). 60 | - The output of the example is various information regarding the 61 | solver and solver performance. 62 | 63 | Options for the compression algorithm include `none` (direct solver), 64 | `blr`, `lossy`/`lossless` (needs the ZFP library), `hodlr` (needs 65 | ButterflyPACK), `blr_hodlr` (needs ButterflyPACK). `blr_hodlr` 66 | combines both BLR (on medium sized blocks) and HODLR (on the largest 67 | blocks). 68 | 69 | The thresholds for using the compression schemes can be set using 70 | `--sp_compression_min_sep_size 1000` or, for specific formats 71 | ``--sp_blr_min_sep_size 1000``. The compression tolerance can be 72 | tuned using `--blr_rel_tol 1e-6` or `--hodlr_rel_tol 1e-6` 73 | 74 | 75 | To see the full list of command-line options, run: `./sparse --help` 76 | 77 | For more information on how to tune the preconditioners, see 78 | [here](https://portal.nersc.gov/project/sparse/strumpack/master/prec.html). 79 | 80 | ## License 81 | STRUMPACK -- STRUctured Matrix PACKage, Copyright (c) 2014-2022, The 82 | Regents of the University of California, through Lawrence Berkeley 83 | National Laboratory (subject to receipt of any required approvals from 84 | the U.S. Dept. of Energy). All rights reserved. 85 | 86 | If you have questions about your rights to use or distribute this 87 | software, please contact Berkeley Lab's Technology Transfer Department 88 | at TTD@lbl.gov. 89 | 90 | NOTICE. This software is owned by the U.S. Department of Energy. As 91 | such, the U.S. Government has been granted for itself and others 92 | acting on its behalf a paid-up, nonexclusive, irrevocable, worldwide 93 | license in the Software to reproduce, prepare derivative works, and 94 | perform publicly and display publicly. Beginning five (5) years after 95 | the date permission to assert copyright is obtained from the 96 | U.S. Department of Energy, and subject to any subsequent five (5) year 97 | renewals, the U.S. Government is granted for itself and others acting 98 | on its behalf a paid-up, nonexclusive, irrevocable, worldwide license 99 | in the Software to reproduce, prepare derivative works, distribute 100 | copies to the public, perform publicly and display publicly, and to 101 | permit others to do so 102 | -------------------------------------------------------------------------------- /strumpack/sparse.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * STRUMPACK -- STRUctured Matrix PACKage, Copyright (c) 2014-2022, 3 | * The Regents of the University of California, through Lawrence 4 | * Berkeley National Laboratory (subject to receipt of any required 5 | * approvals from the U.S. Dept. of Energy). All rights reserved. 6 | * 7 | * If you have questions about your rights to use or distribute this 8 | * software, please contact Berkeley Lab's Technology Transfer 9 | * Department at TTD@lbl.gov. 10 | * 11 | * NOTICE. This software is owned by the U.S. Department of Energy. As 12 | * such, the U.S. Government has been granted for itself and others 13 | * acting on its behalf a paid-up, nonexclusive, irrevocable, 14 | * worldwide license in the Software to reproduce, prepare derivative 15 | * works, and perform publicly and display publicly. Beginning five 16 | * (5) years after the date permission to assert copyright is obtained 17 | * from the U.S. Department of Energy, and subject to any subsequent 18 | * five (5) year renewals, the U.S. Government is granted for itself 19 | * and others acting on its behalf a paid-up, nonexclusive, 20 | * irrevocable, worldwide license in the Software to reproduce, 21 | * prepare derivative works, distribute copies to the public, perform 22 | * publicly and display publicly, and to permit others to do so. 23 | * 24 | * Developers: Pieter Ghysels, and others. 25 | * (Lawrence Berkeley National Lab, Computational Research 26 | * Division). 27 | * 28 | */ 29 | #include 30 | #include "StrumpackSparseSolverMPIDist.hpp" 31 | #include "sparse/CSRMatrix.hpp" 32 | #include "misc/TaskTimer.hpp" 33 | 34 | typedef double scalar; 35 | // typedef int64_t integer; // to use 64 bit integers 36 | typedef int integer; 37 | 38 | using namespace strumpack; 39 | 40 | int main(int argc, char* argv[]) { 41 | int thread_level, myrank; 42 | MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &thread_level); 43 | MPI_Comm_rank(MPI_COMM_WORLD, &myrank); 44 | if (thread_level != MPI_THREAD_MULTIPLE && myrank == 0) 45 | std::cout << "MPI implementation does not support MPI_THREAD_MULTIPLE" 46 | << std::endl; 47 | { 48 | int n = 30, nrhs = 1; 49 | if (argc > 1) n = atoi(argv[1]); // get grid size 50 | else std::cout << "# please provide grid size" << std::endl; 51 | // get number of right-hand sides 52 | // if (argc > 2) nrhs = std::max(1, atoi(argv[2])); 53 | if (!myrank) 54 | std::cout << "solving 3D " << n << "^3 Poisson problem" 55 | << " with " << nrhs << " right hand sides" << std::endl; 56 | 57 | // Create the main solver object, using an MPI communicator. 58 | StrumpackSparseSolverMPIDist spss(MPI_COMM_WORLD); 59 | 60 | // The matching phase finds a column permutation that maximizes 61 | // the diagonal elements. Since the 3D Poisson problem is already 62 | // diagonally dominant, we can disable this matching. 63 | spss.options().set_matching(MatchingJob::NONE); 64 | 65 | // A fill reducing ordering ordering is a symmtric permutation of 66 | // the matrix that minimizes the fill in the sparse triangular 67 | // factors. Since the problem here is defined on a regular mesh, 68 | // we can use a simple geometric nested dissection algorithm (see 69 | // also below spss.reorder(n, n, n) where the mesh dimensions are 70 | // specified). For general sparse matrices, use any other option, 71 | // such as the default ReorderingStrategy::METIS, or the parallel 72 | // ReorderingStrategy::PARMETIS (if STRUMPACK was configured with 73 | // PARMETIS support). 74 | spss.options().set_reordering_method(ReorderingStrategy::GEOMETRIC); 75 | spss.options().set_from_command_line(argc, argv); 76 | 77 | // construct a sparse matrix from a simple 7 point stencil, with 78 | // zero boundary conditions 79 | CSRMatrix A; 80 | if (!myrank) { 81 | int n2 = n * n; 82 | int N = n * n2; 83 | int nnz = 7 * N - 6 * n2; 84 | A = CSRMatrix(N, nnz); 85 | integer* col_ptr = A.ptr(); 86 | integer* row_ind = A.ind(); 87 | scalar* val = A.val(); 88 | 89 | nnz = 0; 90 | col_ptr[0] = 0; 91 | for (integer xdim=0; xdim 0) { val[nnz] = -1.0; row_ind[nnz++] = ind-1; } // left 98 | if (zdim < n-1){ val[nnz] = -1.0; row_ind[nnz++] = ind+1; } // right 99 | if (ydim > 0) { val[nnz] = -1.0; row_ind[nnz++] = ind-n; } // front 100 | if (ydim < n-1){ val[nnz] = -1.0; row_ind[nnz++] = ind+n; } // back 101 | if (xdim > 0) { val[nnz] = -1.0; row_ind[nnz++] = ind-n2; } // up 102 | if (xdim < n-1){ val[nnz] = -1.0; row_ind[nnz++] = ind+n2; } // down 103 | col_ptr[ind+1] = nnz; 104 | } 105 | A.set_symm_sparse(); 106 | } 107 | // This scatters the sparse matrix A from the root over all the 108 | // ranks, using a 1D block row distribution, see 109 | // https://portal.nersc.gov/project/sparse/strumpack/master/sparse_example_usage.html#autotoc_md9 110 | CSRMatrixMPI Adist(&A, MPI_COMM_WORLD, true); 111 | // delete sequential sparse matrix (on the root) 112 | A = CSRMatrix(); 113 | 114 | auto n_local = Adist.local_rows(); 115 | DenseMatrix b(n_local, nrhs), x(n_local, nrhs), 116 | x_exact(n_local, nrhs); 117 | 118 | // construct a random exact solution 119 | x_exact.random(); 120 | 121 | // compute a right hand-side corresponding to exact solution 122 | // x_exact 123 | Adist.spmv(x_exact, b); 124 | 125 | 126 | // One can also directly pass the CSR rowptr, colind, and value 127 | // arrays, see 128 | // https://portal.nersc.gov/project/sparse/strumpack/master/classstrumpack_1_1SparseSolver.html 129 | spss.set_matrix(Adist); 130 | 131 | // For geometric nested dissection, the the mesh dimensions n x n 132 | // x n (and separator width if not 1) need to be provided. For 133 | // other fill-reducing orderings, such as the default 134 | // ReorderingStrategy::GEOMETRIC, just call spss.reorder(); 135 | spss.reorder(n, n, n); 136 | 137 | // the actual numerical factorization phase. If reorder() was not 138 | // already called, it will be called by factor internally. 139 | spss.factor(); 140 | 141 | // solve a linear system Ax=b for x. If factor was not already 142 | // called, then it will be called by solve internally. 143 | spss.solve(b, x); 144 | 145 | auto scaled_res = Adist.max_scaled_residual(x, b); 146 | x.scaled_add(-1., x_exact); 147 | auto relerr = x.normF() / x_exact.normF(); 148 | if (!myrank) { 149 | std::cout << "# COMPONENTWISE SCALED RESIDUAL = " 150 | << scaled_res << std::endl; 151 | std::cout << "# relative error = ||x-x_exact||_F/||x_exact||_F = " 152 | << relerr << std::endl; 153 | } 154 | } 155 | scalapack::Cblacs_exit(1); 156 | MPI_Finalize(); 157 | return 0; 158 | } 159 | -------------------------------------------------------------------------------- /sundials/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # SUNDIALS Copyright Start Copyright (c) 2002-2019, Lawrence Livermore National Security and 2 | # Southern Methodist University. All rights reserved. 3 | # 4 | # See the top-level LICENSE and NOTICE files for details. 5 | # 6 | # SPDX-License-Identifier: BSD-3-Clause SUNDIALS Copyright End 7 | # --------------------------------------------------------------- 8 | 9 | # add the executables 10 | if(ENABLE_PETSC) 11 | add_executable(cv_petsc_ex7 cv_petsc_ex7.c) 12 | target_link_libraries( 13 | cv_petsc_ex7 PRIVATE XSDK::SUNDIALS XSDK::PETSc MPI::MPI_C 14 | ) 15 | if(MATH_LIBRARY) 16 | target_link_libraries(cv_petsc_ex7 PRIVATE ${MATH_LIBRARY}) 17 | endif() 18 | if (TARGET OpenMP::OpenMP_C) 19 | target_link_libraries(cv_petsc_ex7 PRIVATE OpenMP::OpenMP_C) 20 | endif() 21 | xsdk_add_test(NAME SUNDIALS-cv_petsc_ex7_1 COMMAND $ MPI_NPROCS 4) 22 | xsdk_add_test( 23 | NAME 24 | SUNDIALS-cv_petsc_ex7_2 25 | COMMAND 26 | $ 27 | --snes_type 28 | anderson 29 | MPI_NPROCS 30 | 4 31 | ) 32 | install(TARGETS cv_petsc_ex7 RUNTIME DESTINATION bin) 33 | endif() 34 | if(ENABLE_SUPERLU) 35 | set(tgt ark_brusselator1D_FEM_sludist) 36 | if (ENABLE_HIP) 37 | set_source_files_properties(${tgt}.cpp PROPERTIES LANGUAGE HIP) 38 | endif() 39 | add_executable(${tgt} ark_brusselator1D_FEM_sludist.cpp) 40 | target_link_libraries(${tgt} PRIVATE XSDK::SUNDIALS XSDK::SUPERLU MPI::MPI_C) 41 | if(MATH_LIBRARY) 42 | target_link_libraries(ark_brusselator1D_FEM_sludist PRIVATE ${MATH_LIBRARY}) 43 | endif() 44 | xsdk_add_test(NAME SUNDIALS-${tgt} COMMAND $ MPI_NPROCS 1) 45 | install(TARGETS ${tgt} RUNTIME DESTINATION bin) 46 | endif() 47 | if(ENABLE_MAGMA) 48 | if(ENABLE_CUDA) 49 | set_source_files_properties(cv_bruss_batched_magma.cpp PROPERTIES LANGUAGE CUDA) 50 | elseif(ENABLE_HIP) 51 | set_source_files_properties(cv_bruss_batched_magma.cpp PROPERTIES LANGUAGE HIP) 52 | endif() 53 | add_executable(cv_bruss_batched_magma cv_bruss_batched_magma.cpp) 54 | target_link_libraries(cv_bruss_batched_magma PRIVATE XSDK::SUNDIALS XSDK::MAGMA) 55 | if(MATH_LIBRARY) 56 | target_link_libraries(cv_bruss_batched_magma PRIVATE ${MATH_LIBRARY}) 57 | endif() 58 | xsdk_add_test( 59 | NAME SUNDIALS-cv_bruss_batched_magma COMMAND $ 60 | ) 61 | install(TARGETS cv_bruss_batched_magma RUNTIME DESTINATION bin) 62 | endif() 63 | -------------------------------------------------------------------------------- /sundials/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2002-2019, Lawrence Livermore National Security and Southern Methodist University. 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /sundials/NOTICE: -------------------------------------------------------------------------------- 1 | This work was produced under the auspices of the U.S. Department of 2 | Energy by Lawrence Livermore National Laboratory under Contract 3 | DE-AC52-07NA27344. 4 | 5 | This work was prepared as an account of work sponsored by an agency of 6 | the United States Government. Neither the United States Government nor 7 | Lawrence Livermore National Security, LLC, nor any of their employees 8 | makes any warranty, expressed or implied, or assumes any legal liability 9 | or responsibility for the accuracy, completeness, or usefulness of any 10 | information, apparatus, product, or process disclosed, or represents that 11 | its use would not infringe privately owned rights. 12 | 13 | Reference herein to any specific commercial product, process, or service 14 | by trade name, trademark, manufacturer, or otherwise does not necessarily 15 | constitute or imply its endorsement, recommendation, or favoring by the 16 | United States Government or Lawrence Livermore National Security, LLC. 17 | 18 | The views and opinions of authors expressed herein do not necessarily 19 | state or reflect those of the United States Government or Lawrence 20 | Livermore National Security, LLC, and shall not be used for advertising 21 | or product endorsement purposes. -------------------------------------------------------------------------------- /sundials/README.md: -------------------------------------------------------------------------------- 1 | # SUNDIALS examples 2 | 3 | Example codes demonstrating the use of 4 | [SUNDIALS](https://computing.llnl.gov/projects/sundials) using other XSDK 5 | packages. 6 | 7 | ## SUNDIALS + SuperLU 8 | 9 | The `ark_brusselator1D_FEM_sludist.cpp` example simulates a brusselator problem 10 | from chemical kinetics. This program solves the problem with the diagonally 11 | implicit Runge--Kutta method from the SUNDIALS ARKode time integrator package. 12 | It uses the SUNDIALS serial vector for the solution data, the SUNDIALS Newton 13 | nonlinear solver to solve a nonlinear system at every time step, and the 14 | [SuperLU_DIST](https://github.com/xiaoyeli/superlu_dist) parallel sparse-direct 15 | linear solver to solve the resulting linear system. Jacobian data is stored in 16 | a SuperLU_DIST SuperMatrix. 100 outputs are printed to an output file at equal time 17 | intervals, and run statistics are printed to stdout at the end. 18 | 19 | **Usage** 20 | 21 | ``` 22 | mpirun -np 1 ./ark_brusselator1D_FEM_sludist 23 | ``` 24 | 25 | ## SUNDIALS + PETSc 26 | 27 | The ``cv_petsc_ex7.c`` example solves a nonlinear, time-dependent PDE in 2d. The 28 | example is based on the [PETSc](https://www.mcs.anl.gov/petsc/) TS ex7.c, but 29 | uses the BDF method from the SUNDIALS CVODE time integrator package. It 30 | interfaces with the PETSc Vec for the solution data, the PETSc SNES nonlinear 31 | solvers to solve a nonlinear system at every time step, an the PETSc KSP linear 32 | solvers to solve the resulting linear systems. Output, nonlinear/linear solver 33 | and vector options can be controlled using the typical PETSc runtime arguments. 34 | 35 | **Usage** 36 | 37 | To run with the default options: 38 | 39 | ``` 40 | ./cv_petsc_ex7 41 | ``` 42 | 43 | To monitor the KSP linear solver progress: 44 | 45 | ``` 46 | ./cv_petsc_ex7 -ksp_monitor 47 | ``` 48 | 49 | To use the Anderson method from SNES instead of the default Newton line search: 50 | 51 | ``` 52 | ./cv_petsc_ex7 -snes_type anderson 53 | ``` 54 | 55 | To view all the options available: 56 | 57 | ``` 58 | ./cv_petsc_ex7 -h 59 | ``` 60 | 61 | ## SUNDIALS + MAGMA 62 | 63 | The ``cvRoberts_blockdiag_magma.cpp`` example solves a group of independent ODEs 64 | simulating a problem from chemical kinetics. The example groups the independent ODEs together 65 | to form a larger system which is evolved with the CVODE implicit BDF time integration 66 | method. The linear system that arises is block-diagonal and there is no coupling 67 | between the blocks. Thus, a batched LU method from the [MAGMA](https://icl.utk.edu/magma/) linear algebra library 68 | is utilized. This example is CUDA-enabled. 69 | 70 | To run: 71 | 72 | ``` 73 | ./cvRoberts_blockdiag_magma [number of groups] 74 | ``` 75 | 76 | 77 | ## License 78 | 79 | SUNDIALS is released under the BSD 3-clause license. See the 80 | [LICENSE](./LICENSE) and [NOTICE](./NOTICE) files for details. All new 81 | contributions must be made under the BSD 3-clause license. 82 | 83 | **Please Note** If you are using SUNDIALS with any third party libraries linked 84 | in (e.g., LAPACK, KLU, SuperLU_MT, PETSc, or *hypre*), be sure to review the 85 | respective license of the package as that license may have more restrictive 86 | terms than the SUNDIALS license. 87 | 88 | ```text 89 | SPDX-License-Identifier: BSD-3-Clause 90 | 91 | LLNL-CODE-667205 (ARKODE) 92 | UCRL-CODE-155951 (CVODE) 93 | UCRL-CODE-155950 (CVODES) 94 | UCRL-CODE-155952 (IDA) 95 | UCRL-CODE-237203 (IDAS) 96 | LLNL-CODE-665877 (KINSOL) 97 | ``` 98 | -------------------------------------------------------------------------------- /tasmanian/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(tasmanian_magma example_unstructured_grid.cpp) 2 | target_link_libraries(tasmanian_magma Tasmanian::Tasmanian) 3 | 4 | xsdk_add_test(NAME tasmanian COMMAND tasmanian_magma) 5 | -------------------------------------------------------------------------------- /tasmanian/README.md: -------------------------------------------------------------------------------- 1 | # Tasmanian example 2 | 3 | Usage: 4 | ``` 5 | ./tasmanian_magma 6 | ``` 7 | 8 | Constructs a surrogate model for a simple reference model using random 9 | samples. 10 | The surrogate uses a spars grid basis, but the samples are not aligned 11 | to the grid and the basis coefficients are computed using 12 | a least-squares approximation. 13 | A linear algebra engine is needed, either BLAS or the GPU native 14 | cuBLAS/rocBLAS or the out-of-core methods implemented in MAGMA. 15 | MAGMA is chosen by default, but the other two can be used as fallback. 16 | 17 | #### MAGMA integration 18 | 19 | The example is chosen to be small for reference purposes; 20 | however, in practice, the size of the random data can easily exceed 21 | the memory capacity of the GPU. 22 | MAGMA offers out-of-core methods for QR factorization, which can be 23 | used to solve the least-squares problem while leveraging 24 | the computational capacity of the GPU but without the memory limitations. 25 | In comparison, using CPU BLAS will be slow and cuBLAS/rocBLAS will 26 | soon hit the memory limit of the GPU device. 27 | -------------------------------------------------------------------------------- /tasmanian/example_unstructured_grid.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "Tasmanian.hpp" 3 | #include 4 | 5 | using namespace std; 6 | 7 | void sparse_grids_example_11(){ 8 | cout << "\n---------------------------------------------------------------------------------------------------\n"; 9 | cout << std::scientific; cout.precision(4); 10 | cout << "Construction of a sparse grid surrogate but using unstructured data\n\n"; 11 | 12 | int const num_inputs = 2; 13 | 14 | // using random points to test the error 15 | int const num_test_points = 1000; 16 | std::vector test_points(num_test_points * num_inputs); 17 | std::minstd_rand park_miller(42); 18 | std::uniform_real_distribution domain(-1.0, 1.0); 19 | for(auto &t : test_points) t = domain(park_miller); 20 | 21 | // computes the error between the gird surrogate model and the actual model 22 | // using the test points, finds the largest absolute error 23 | auto get_error = [&](TasGrid::TasmanianSparseGrid const &grid, 24 | std::function model)-> 25 | double{ 26 | std::vector grid_result; 27 | grid.evaluateBatch(test_points, grid_result); 28 | double err = 0.0; 29 | for(int i=0; i 39 | void{ y[0] = std::exp(-x[0]*x[0] -x[1]-x[1]); }; 40 | 41 | auto grid = TasGrid::makeGlobalGrid(num_inputs, 1, 4, TasGrid::type_level, TasGrid::rule_clenshawcurtis); 42 | 43 | // generate random data for the inputs, and compute the corresponding outputs 44 | int const num_data_points = 2000; 45 | std::vector data_input(num_inputs * num_data_points); 46 | std::vector data_output(num_data_points); 47 | 48 | for(auto &d : data_input) d = domain(park_miller); 49 | for(int i=0; i 72 | ) 73 | 74 | install(TARGETS SimpleSolve_WithParameters RUNTIME DESTINATION bin) 75 | endif() 76 | -------------------------------------------------------------------------------- /trilinos/LICENSE: -------------------------------------------------------------------------------- 1 | The Trilinos project is a collection of open-source packages licensed 2 | individually under multiple open-source licenses. Licensing terms are 3 | available at the Trilinos website: 4 | 5 | https://trilinos.org/download/license/ 6 | 7 | For information about the software license for a particular package, 8 | see package-specific documentation. 9 | -------------------------------------------------------------------------------- /trilinos/README.md: -------------------------------------------------------------------------------- 1 | # Trilinos examples 2 | 3 | Example codes demonstrating the use of 4 | [Trilinos](https://trilinos.github.io) using other XSDK 5 | packages. 6 | 7 | ## Trilinos + SuperLU 8 | 9 | 'SimpleSolve_WithParameters.cpp` is an example of sparse linear system direct solvers 10 | allocated to single MPI rank. The code uses Amesos2 (interface for sparse 11 | direct solver libraries) APIs to call 12 | [SuperLU_DIST](https://github.com/xiaoyeli/superlu_dist). 13 | 14 | The source code ptovides the examples of solver parameters. 15 | Teuchos::ParameterList superludist_params = amesos2_params.sublist("SuperLU_DIST"); 16 | superludist_params.set("Trans","No","Whether to solve with A^T"); 17 | superludist_params.set("npcol",1,"Number of Processor Columns"); 18 | superludist_params.set("nprow",1,"Number of Processor Rows"); 19 | superludist_params.set("ColPerm","NATURAL","Use 'natural' ordering of columns"); 20 | 21 | 22 | **Usage** 23 | 24 | ``` 25 | mpirun -np 1 ./SimpleSolve_WithParameters 26 | ``` 27 | -------------------------------------------------------------------------------- /trilinos/SimpleSolve_WithParameters.cpp: -------------------------------------------------------------------------------- 1 | // @HEADER 2 | // 3 | // *********************************************************************** 4 | // 5 | // Amesos2: Templated Direct Sparse Solver Package 6 | // Copyright 2011 Sandia Corporation 7 | // 8 | // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, 9 | // the U.S. Government retains certain rights in this software. 10 | // 11 | // Redistribution and use in source and binary forms, with or without 12 | // modification, are permitted provided that the following conditions are 13 | // met: 14 | // 15 | // 1. Redistributions of source code must retain the above copyright 16 | // notice, this list of conditions and the following disclaimer. 17 | // 18 | // 2. Redistributions in binary form must reproduce the above copyright 19 | // notice, this list of conditions and the following disclaimer in the 20 | // documentation and/or other materials provided with the distribution. 21 | // 22 | // 3. Neither the name of the Corporation nor the names of the 23 | // contributors may be used to endorse or promote products derived from 24 | // this software without specific prior written permission. 25 | // 26 | // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY 27 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 | // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 | // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE 30 | // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 31 | // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 32 | // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 33 | // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 34 | // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 35 | // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 36 | // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 | // 38 | // Questions? Contact Michael A. Heroux (maherou@sandia.gov) 39 | // 40 | // *********************************************************************** 41 | // 42 | // @HEADER 43 | 44 | /** 45 | \file SimpleSolve_WithParameters.cpp 46 | \author Eric Bavier 47 | \date Sat Jul 17 14:07:36 2010 48 | 49 | \brief Small example of using a Teuchos::ParameterList to specify Amesos2 50 | solver parameters. 51 | */ 52 | 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | 59 | #include 60 | #include 61 | #include 62 | #include 63 | 64 | #include "Amesos2.hpp" 65 | #include "Amesos2_Version.hpp" 66 | 67 | 68 | int main(int argc, char *argv[]) { 69 | Tpetra::ScopeGuard tpetraScope(&argc,&argv); 70 | typedef double Scalar; 71 | 72 | typedef double Scalar; 73 | typedef Tpetra::Map<>::local_ordinal_type LO; 74 | typedef Tpetra::Map<>::global_ordinal_type GO; 75 | 76 | typedef Tpetra::CrsMatrix MAT; 77 | typedef Tpetra::MultiVector MV; 78 | 79 | using Tpetra::global_size_t; 80 | using Teuchos::tuple; 81 | using Teuchos::RCP; 82 | using Teuchos::rcp; 83 | 84 | Teuchos::RCP > comm = Tpetra::getDefaultComm(); 85 | RCP fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); 86 | 87 | 88 | size_t myRank = comm->getRank(); 89 | 90 | if( myRank == 0 ) *fos << Amesos2::version() << std::endl << std::endl; 91 | 92 | const size_t numVectors = 1; 93 | 94 | // create a Map 95 | global_size_t nrows = 6; 96 | RCP > map = rcp( new Tpetra::Map(nrows,0,comm) ); 97 | RCP A = rcp( new MAT(map,3) ); // max of three entries in a row 98 | 99 | /* 100 | * We will solve a system with a known solution, for which we will be using 101 | * the following matrix: 102 | * 103 | * [ [ 7, 2, 0, -3, 0, 0 ] 104 | * [ 0, 8, 0, 0, -1, 0 ] 105 | * [ -3, 0, 1, 0, 0, 0 ] 106 | * [ 0, 0, 0, 5, 0, -2 ] 107 | * [ -1, 0, 0, 0, 4, 0 ] 108 | * [ 0, 0, 0, 0, 0, 6 ] ] 109 | * 110 | * And we will solve with A^T 111 | */ 112 | // Construct matrix 113 | if( myRank == 0 ){ 114 | A->insertGlobalValues(0,tuple(0,1,3),tuple(7,2,-3)); 115 | A->insertGlobalValues(1,tuple(1,4),tuple(8,-1)); 116 | A->insertGlobalValues(2,tuple(0,2),tuple(-3,1)); 117 | A->insertGlobalValues(3,tuple(3,5),tuple(5,-2)); 118 | A->insertGlobalValues(4,tuple(0,4),tuple(-1,4)); 119 | A->insertGlobalValues(5,tuple(5),tuple(6)); 120 | } 121 | A->fillComplete(); 122 | 123 | // Create random X 124 | RCP X = rcp(new MV(map,numVectors)); 125 | X->randomize(); 126 | 127 | /* Create B 128 | * 129 | * Use RHS: 130 | * 131 | * [[-7] 132 | * [18] 133 | * [ 3] 134 | * [17] 135 | * [18] 136 | * [28]] 137 | */ 138 | RCP B = rcp(new MV(map,numVectors)); 139 | int data[6] = {-7,18,3,17,18,28}; 140 | for( int i = 0; i < 6; ++i ){ 141 | if( B->getMap()->isNodeGlobalElement(i) ){ 142 | B->replaceGlobalValue(i,0,data[i]); 143 | } 144 | } 145 | 146 | // Check first whether SuperLU is supported 147 | if( Amesos2::query("Superludist") ){ 148 | 149 | // Constructor from Factory 150 | RCP > solver = Amesos2::create("Superludist", A, X, B); 151 | 152 | // Create a Teuchos::ParameterList to hold solver parameters 153 | Teuchos::ParameterList amesos2_params("Amesos2"); 154 | Teuchos::ParameterList superludist_params = amesos2_params.sublist("SuperLU_DIST"); 155 | superludist_params.set("Trans","No","Whether to solve with A^T"); 156 | superludist_params.set("npcol",1,"Number of Processor Columns"); 157 | superludist_params.set("nprow",1,"Number of Processor Rows"); 158 | superludist_params.set("ColPerm","NATURAL","Use 'natural' ordering of columns"); 159 | 160 | solver->setParameters( Teuchos::rcpFromRef(amesos2_params) ); 161 | solver->symbolicFactorization().numericFactorization().solve(); 162 | 163 | /* Print the solution 164 | * 165 | * Should be: 166 | * 167 | * [[1] 168 | * [2] 169 | * [3] 170 | * [4] 171 | * [5] 172 | * [6]] 173 | */ 174 | X->describe(*fos,Teuchos::VERB_EXTREME); 175 | } else { 176 | *fos << "SuperLU_Dist solver not enable. Exiting..." << std::endl; 177 | } 178 | 179 | // We are done. 180 | return EXIT_SUCCESS; 181 | } 182 | --------------------------------------------------------------------------------