├── COPYING
├── COPYING.LESSER
├── README.md
├── cpp
├── CMakeLists.txt
├── cmake
│ ├── modules
│ │ └── FindUFCx.cmake
│ ├── post-install
│ │ └── CMakeLists.txt
│ └── templates
│ │ ├── CUDOLFINXConfig.cmake.in
│ │ ├── cmake_uninstall.cmake.in
│ │ ├── cudolfinx.conf.in
│ │ └── cudolfinx.pc.in
└── cudolfinx
│ ├── CMakeLists.txt
│ ├── common
│ ├── CMakeLists.txt
│ ├── CUDA.cpp
│ ├── CUDA.h
│ ├── CUDAStore.h
│ └── version.h.in
│ ├── cudolfinx.h
│ ├── fem
│ ├── CMakeLists.txt
│ ├── CUDAAssembler.cpp
│ ├── CUDAAssembler.h
│ ├── CUDACoefficient.h
│ ├── CUDADirichletBC.h
│ ├── CUDADofMap.cpp
│ ├── CUDADofMap.h
│ ├── CUDAForm.h
│ ├── CUDAFormCoefficients.h
│ ├── CUDAFormConstants.h
│ ├── CUDAFormIntegral.cpp
│ ├── CUDAFormIntegral.h
│ └── petsc.h
│ ├── la
│ ├── CMakeLists.txt
│ ├── CUDAMatrix.cpp
│ ├── CUDAMatrix.h
│ ├── CUDASeqMatrix.cpp
│ ├── CUDASeqMatrix.h
│ ├── CUDAVector.cpp
│ ├── CUDAVector.h
│ ├── petsc.cpp
│ └── petsc.h
│ └── mesh
│ ├── CMakeLists.txt
│ ├── CUDAMesh.h
│ ├── CUDAMeshEntities.h
│ ├── util.cpp
│ └── util.h
├── docker
├── Dockerfile.end-user
└── Dockerfile.test-env
├── python
├── CMakeLists.txt
├── README.md
├── build-requirements.txt
├── cudolfinx
│ ├── __init__.py
│ ├── assemble.py
│ ├── bcs.py
│ ├── context.py
│ ├── form.py
│ ├── jit.py
│ ├── la.py
│ ├── mesh.py
│ └── wrappers
│ │ ├── caster_petsc.h
│ │ ├── cudolfinx.cpp
│ │ ├── fem.cpp
│ │ └── petsc.cpp
├── examples
│ ├── poisson.py
│ └── poisson_sum_factorization.py
├── pyproject.toml
└── test
│ ├── test_cuda_assembly.py
│ └── test_multigpu_assembly.py
└── spack
├── packages
├── cuda-dolfinx
│ └── package.py
└── py-cuda-dolfinx
│ └── package.py
└── repo.yaml
/COPYING.LESSER:
--------------------------------------------------------------------------------
1 | GNU LESSER GENERAL PUBLIC LICENSE
2 | Version 3, 29 June 2007
3 |
4 | Copyright (C) 2007 Free Software Foundation, Inc.
5 | Everyone is permitted to copy and distribute verbatim copies
6 | of this license document, but changing it is not allowed.
7 |
8 |
9 | This version of the GNU Lesser General Public License incorporates
10 | the terms and conditions of version 3 of the GNU General Public
11 | License, supplemented by the additional permissions listed below.
12 |
13 | 0. Additional Definitions.
14 |
15 | As used herein, "this License" refers to version 3 of the GNU Lesser
16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU
17 | General Public License.
18 |
19 | "The Library" refers to a covered work governed by this License,
20 | other than an Application or a Combined Work as defined below.
21 |
22 | An "Application" is any work that makes use of an interface provided
23 | by the Library, but which is not otherwise based on the Library.
24 | Defining a subclass of a class defined by the Library is deemed a mode
25 | of using an interface provided by the Library.
26 |
27 | A "Combined Work" is a work produced by combining or linking an
28 | Application with the Library. The particular version of the Library
29 | with which the Combined Work was made is also called the "Linked
30 | Version".
31 |
32 | The "Minimal Corresponding Source" for a Combined Work means the
33 | Corresponding Source for the Combined Work, excluding any source code
34 | for portions of the Combined Work that, considered in isolation, are
35 | based on the Application, and not on the Linked Version.
36 |
37 | The "Corresponding Application Code" for a Combined Work means the
38 | object code and/or source code for the Application, including any data
39 | and utility programs needed for reproducing the Combined Work from the
40 | Application, but excluding the System Libraries of the Combined Work.
41 |
42 | 1. Exception to Section 3 of the GNU GPL.
43 |
44 | You may convey a covered work under sections 3 and 4 of this License
45 | without being bound by section 3 of the GNU GPL.
46 |
47 | 2. Conveying Modified Versions.
48 |
49 | If you modify a copy of the Library, and, in your modifications, a
50 | facility refers to a function or data to be supplied by an Application
51 | that uses the facility (other than as an argument passed when the
52 | facility is invoked), then you may convey a copy of the modified
53 | version:
54 |
55 | a) under this License, provided that you make a good faith effort to
56 | ensure that, in the event an Application does not supply the
57 | function or data, the facility still operates, and performs
58 | whatever part of its purpose remains meaningful, or
59 |
60 | b) under the GNU GPL, with none of the additional permissions of
61 | this License applicable to that copy.
62 |
63 | 3. Object Code Incorporating Material from Library Header Files.
64 |
65 | The object code form of an Application may incorporate material from
66 | a header file that is part of the Library. You may convey such object
67 | code under terms of your choice, provided that, if the incorporated
68 | material is not limited to numerical parameters, data structure
69 | layouts and accessors, or small macros, inline functions and templates
70 | (ten or fewer lines in length), you do both of the following:
71 |
72 | a) Give prominent notice with each copy of the object code that the
73 | Library is used in it and that the Library and its use are
74 | covered by this License.
75 |
76 | b) Accompany the object code with a copy of the GNU GPL and this license
77 | document.
78 |
79 | 4. Combined Works.
80 |
81 | You may convey a Combined Work under terms of your choice that,
82 | taken together, effectively do not restrict modification of the
83 | portions of the Library contained in the Combined Work and reverse
84 | engineering for debugging such modifications, if you also do each of
85 | the following:
86 |
87 | a) Give prominent notice with each copy of the Combined Work that
88 | the Library is used in it and that the Library and its use are
89 | covered by this License.
90 |
91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license
92 | document.
93 |
94 | c) For a Combined Work that displays copyright notices during
95 | execution, include the copyright notice for the Library among
96 | these notices, as well as a reference directing the user to the
97 | copies of the GNU GPL and this license document.
98 |
99 | d) Do one of the following:
100 |
101 | 0) Convey the Minimal Corresponding Source under the terms of this
102 | License, and the Corresponding Application Code in a form
103 | suitable for, and under terms that permit, the user to
104 | recombine or relink the Application with a modified version of
105 | the Linked Version to produce a modified Combined Work, in the
106 | manner specified by section 6 of the GNU GPL for conveying
107 | Corresponding Source.
108 |
109 | 1) Use a suitable shared library mechanism for linking with the
110 | Library. A suitable mechanism is one that (a) uses at run time
111 | a copy of the Library already present on the user's computer
112 | system, and (b) will operate properly with a modified version
113 | of the Library that is interface-compatible with the Linked
114 | Version.
115 |
116 | e) Provide Installation Information, but only if you would otherwise
117 | be required to provide such information under section 6 of the
118 | GNU GPL, and only to the extent that such information is
119 | necessary to install and execute a modified version of the
120 | Combined Work produced by recombining or relinking the
121 | Application with a modified version of the Linked Version. (If
122 | you use option 4d0, the Installation Information must accompany
123 | the Minimal Corresponding Source and Corresponding Application
124 | Code. If you use option 4d1, you must provide the Installation
125 | Information in the manner specified by section 6 of the GNU GPL
126 | for conveying Corresponding Source.)
127 |
128 | 5. Combined Libraries.
129 |
130 | You may place library facilities that are a work based on the
131 | Library side by side in a single library together with other library
132 | facilities that are not Applications and are not covered by this
133 | License, and convey such a combined library under terms of your
134 | choice, if you do both of the following:
135 |
136 | a) Accompany the combined library with a copy of the same work based
137 | on the Library, uncombined with any other library facilities,
138 | conveyed under the terms of this License.
139 |
140 | b) Give prominent notice with the combined library that part of it
141 | is a work based on the Library, and explaining where to find the
142 | accompanying uncombined form of the same work.
143 |
144 | 6. Revised Versions of the GNU Lesser General Public License.
145 |
146 | The Free Software Foundation may publish revised and/or new versions
147 | of the GNU Lesser General Public License from time to time. Such new
148 | versions will be similar in spirit to the present version, but may
149 | differ in detail to address new problems or concerns.
150 |
151 | Each version is given a distinguishing version number. If the
152 | Library as you received it specifies that a certain numbered version
153 | of the GNU Lesser General Public License "or any later version"
154 | applies to it, you have the option of following the terms and
155 | conditions either of that published version or of any later version
156 | published by the Free Software Foundation. If the Library as you
157 | received it does not specify a version number of the GNU Lesser
158 | General Public License, you may choose any version of the GNU Lesser
159 | General Public License ever published by the Free Software Foundation.
160 |
161 | If the Library as you received it specifies that a proxy can decide
162 | whether future versions of the GNU Lesser General Public License shall
163 | apply, that proxy's public statement of acceptance of any version is
164 | permanent authorization for you to choose that version for the
165 | Library.
166 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # About
2 |
3 | This repository is an add-on extension to the DOLFINx library providing CUDA accelerated assembly routines. It complements the CUDA linear solvers in PETSc to enable fully GPU-accelerated DOLFINx codes. It is designed to enable GPU acceleration for existing codes with minimal changes.
4 |
5 | # Basic Usage
6 |
7 | ```
8 | import cudolfinx as cufem
9 |
10 | # given UFL forms A and L representing a stiffness matrix and right-hand-side
11 | cuda_A = cufem.form(A)
12 | cuda_L = cufem.form(L)
13 | asm = cufem.CUDAAssembler()
14 | # returns a custom type CUDAMatrix
15 | mat = asm.assemble_matrix(cuda_A)
16 | mat.assemble()
17 | # get PETSc matrix
18 | petsc_mat = mat.mat()
19 | # returns a custom type CUDAVector
20 | vec = asm.assemble_vector(cuda_L)
21 | #get PETSc vector
22 | petsc_vec = vec.vector()
23 | ```
24 |
25 | # Dependencies
26 |
27 | - dolfinx 0.9.0
28 | - PETSc with CUDA support
29 | - CUDA Toolkit 12.x
30 |
31 | # Installation
32 |
33 | There are three ways to do the install, in increasing order of difficulty. Currently, it is not possible to use `cudolfinx` with the existing Conda and Docker distributions of `dolfinx`, because these force installation of PETSc without CUDA support. Consequently, installing `cudolfinx` requires a custom modification to the `dolfinx` dependency stack that has CUDA-enabled PETSc.
34 |
35 | ## Docker
36 |
37 | Using Docker is by far the easiest approach.
38 |
39 | ```
40 | docker run --gpus all -it benpachev/cudolfinx:v0.9.0-cuda12.6
41 | ```
42 | You may experience errors with the prebuilt container due to CUDA Toolkit or MPI version mismatch between the host and container. In this case, the Dockerfiles in `docker/` can be modified to use a different CUDA Toolkit version or MPI version to build a container that will work with your system. Note that on HPC systems, Docker is not available, but Docker containers can be converted to Apptainer/Singularity containers.
43 |
44 | ```
45 | apptainer pull docker://benpachev/cudolfinx:v0.9.0-cuda12.6
46 | apptainer run --nv cudolfinx_v0.9.0-cuda12.6.sif
47 | ```
48 |
49 | ## Spack
50 |
51 | Spack is a management tool for HPC software, and allows for an extreme amount of flexibility in compilation of code and dependency selection. It has somewhat of a learning curve, and typically doesn't work out of the box without some manual configuration. However, it can be a good choice for HPC systems without Apptainer installed, or if more control over the compilation process and dependencies is desired. To install with Spack:
52 |
53 | ```
54 | git clone https://github.com/spack/spack.git
55 | . spack/share/spack/setup-env.sh
56 | spack env create cudolfinx-env
57 | spacktivate cudolfinx-env
58 | git clone https://github.com/bpachev/cuda-dolfinx.git
59 | spack repo add cuda-dolfinx/spack
60 | spack add cuda-dolfinx py-cuda-dolfinx
61 | spack install
62 | ```
63 |
64 | If this leads to errors, it is likely due to either (a) Spack is unable to find a suitable compiler or properly configure your existing compiler (b) Spack is trying to build a poorly supported low-level package from source. To resolve (a), you can usually do `spack compiler add`. Especially on HPC systems, [additional configuration](https://spack-tutorial.readthedocs.io/en/latest/tutorial_configuration.html#compiler-configuration) is usually needed. To solve (b), you will often need to [force Spack to use existing](https://spack-tutorial.readthedocs.io/en/latest/tutorial_configuration.html#external-packages) low-level software on your system instead of trying to install it from source.
65 |
66 | ## From Source
67 |
68 | The difficult part about the install is the dependencies. The Dockerfiles under `docker/` provide a template for how to install the dependencies on Debian-based systems (and using Docker is by far the easiest way to get a development environment). Once that is taken care of, the installation of `cuda-dolfinx` itself is simple.
69 |
70 | ### C++ Core
71 | ```
72 | cd cpp
73 | mkdir build
74 | cmake .. -DCUDOLFINX_SKIP_BUILD_TESTS=YES
75 | make install
76 | ```
77 |
78 | ### Python Bindings:
79 | ```
80 | cd python
81 | pip --check-build-dependencies --no-build-isolation .
82 | ```
83 |
84 | For help with installing or using the library, feel free to contact me at benjaminpachev@gmail.com.
85 |
--------------------------------------------------------------------------------
/cpp/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | # ------------------------------------------------------------------------------
2 | # Top level CMakeLists.txt file for DOLFINx
3 | cmake_minimum_required(VERSION 3.19)
4 |
5 | # ------------------------------------------------------------------------------
6 | # Set project name and version number
7 | project(CUDOLFINX VERSION "0.9.0")
8 |
9 | set(DOXYGEN_CUDOLFINX_VERSION
10 | ${CUDOLFINX_VERSION}
11 | CACHE STRING "Version for Doxygen" FORCE
12 | )
13 |
14 | # ------------------------------------------------------------------------------
15 | # Use C++20
16 | set(CMAKE_CXX_STANDARD 20)
17 |
18 | # Require C++20
19 | set(CMAKE_CXX_STANDARD_REQUIRED ON)
20 |
21 | # Do not enable compler-specific extensions
22 | set(CMAKE_CXX_EXTENSIONS OFF)
23 |
24 | # ------------------------------------------------------------------------------
25 | # Get GIT changeset, if available
26 | find_program(GIT_FOUND git)
27 |
28 | if(GIT_FOUND)
29 | # Get the commit hash of the working branch
30 | execute_process(
31 | COMMAND git rev-parse HEAD
32 | WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
33 | OUTPUT_VARIABLE GIT_COMMIT_HASH
34 | OUTPUT_STRIP_TRAILING_WHITESPACE
35 | )
36 | else()
37 | set(GIT_COMMIT_HASH "unknown")
38 | endif()
39 |
40 | # ------------------------------------------------------------------------------
41 | # General configuration
42 |
43 | # Set location of our FindFoo.cmake modules
44 | set(CMAKE_MODULE_PATH "${CUDOLFINX_SOURCE_DIR}/cmake/modules")
45 |
46 | # Make sure CMake uses the correct DOLFINConfig.cmake for tests and demos
47 | set(CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${CMAKE_CURRENT_BINARY_DIR}/cudolfinx)
48 |
49 | # ------------------------------------------------------------------------------
50 | # Configurable options for how we want to build
51 | include(FeatureSummary)
52 |
53 | option(BUILD_SHARED_LIBS "Build CUDOLFINx with shared libraries." ON)
54 | add_feature_info(
55 | BUILD_SHARED_LIBS BUILD_SHARED_LIBS "Build CUDOLFINx with shared libraries."
56 | )
57 |
58 | option(CUDOLFINX_SKIP_BUILD_TESTS
59 | "Skip build tests for testing usability of dependency packages." OFF
60 | )
61 | add_feature_info(
62 | CUDOLFINX_SKIP_BUILD_TESTS CUDOLFINX_SKIP_BUILD_TESTS
63 | "Skip build tests for testing usability of dependency packages."
64 | )
65 |
66 | # Add shared library paths so shared libs in non-system paths are found
67 | option(CMAKE_INSTALL_RPATH_USE_LINK_PATH
68 | "Add paths to linker search and installed rpath." ON
69 | )
70 | add_feature_info(
71 | CMAKE_INSTALL_RPATH_USE_LINK_PATH CMAKE_INSTALL_RPATH_USE_LINK_PATH
72 | "Add paths to linker search and installed rpath."
73 | )
74 |
75 | # Control UFCx discovery
76 | option(
77 | CUDOLFINX_UFCX_PYTHON
78 | "Enable UFCx discovery using Python. Disable if UFCx should be found using CMake."
79 | ON
80 | )
81 | add_feature_info(
82 | CUDOLFINX_UFCX_PYTHON
83 | CUDOLFINX_UFCX_PYTHON
84 | "Enable UFCx discovery using Python. Disable if UFCx should be found using a CMake config file."
85 | )
86 |
87 | # ------------------------------------------------------------------------------
88 | # Enable or disable optional packages
89 |
90 |
91 | if(CUDOLFINX_ENABLE_PETSC)
92 | set(_REQUIRE_PETSC
93 | TRUE
94 | CACHE BOOL "Is PETSc REQUIRED?"
95 | )
96 | else()
97 | set(_REQUIRE_PETSC
98 | FALSE
99 | CACHE BOOL "Is PETSc REQUIRED?"
100 | )
101 | endif()
102 |
103 | option(CUDOLFINX_ENABLE_PETSC "Compile with support for PETSc." ON)
104 | set_package_properties(
105 | PETSc PROPERTIES
106 | TYPE RECOMMENDED
107 | DESCRIPTION "Portable, Extensible Toolkit for Scientific Computation"
108 | URL "https://petsc.org/"
109 | PURPOSE "Linear and nonlinear solvers"
110 | )
111 |
112 |
113 | # ------------------------------------------------------------------------------
114 | # Check for MPI
115 | find_package(MPI 3 REQUIRED)
116 |
117 | # ------------------------------------------------------------------------------
118 | # Compiler flags
119 |
120 | # Default build type (can be overridden by user)
121 | if(NOT CMAKE_BUILD_TYPE)
122 | set(CMAKE_BUILD_TYPE
123 | "RelWithDebInfo"
124 | CACHE
125 | STRING
126 | "Choose the type of build, options are: Debug Developer MinSizeRel Release RelWithDebInfo."
127 | FORCE
128 | )
129 | endif()
130 |
131 | # Check for some compiler flags
132 | include(CheckCXXCompilerFlag)
133 | check_cxx_compiler_flag(-pipe HAVE_PIPE)
134 |
135 | if(HAVE_PIPE)
136 | list(APPEND CUDOLFINX_CXX_DEVELOPER_FLAGS -pipe)
137 | endif()
138 |
139 | # Add some strict compiler checks
140 | check_cxx_compiler_flag("-Wall -Werror -Wextra -pedantic" HAVE_PEDANTIC)
141 |
142 | if(HAVE_PEDANTIC)
143 | list(APPEND CUDOLFINX_CXX_DEVELOPER_FLAGS -Wall;-Werror;-Wextra;-pedantic)
144 | endif()
145 |
146 | # Debug flags
147 | check_cxx_compiler_flag(-g HAVE_DEBUG)
148 |
149 | if(HAVE_DEBUG)
150 | list(APPEND CUDOLFINX_CXX_DEVELOPER_FLAGS -g)
151 | endif()
152 |
153 | # Optimisation
154 | check_cxx_compiler_flag(-O2 HAVE_O2_OPTIMISATION)
155 |
156 | if(HAVE_O2_OPTIMISATION)
157 | list(APPEND CUDOLFINX_CXX_DEVELOPER_FLAGS -O2)
158 | endif()
159 |
160 | # ------------------------------------------------------------------------------
161 | # Find required packages
162 |
163 | # pugixml
164 | find_package(pugixml REQUIRED)
165 |
166 | # Note: When updating Boost version, also update CUDOLFINXCongif.cmake.in
167 | if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT)
168 | set(Boost_NO_SYSTEM_PATHS on)
169 | endif()
170 |
171 | set(Boost_USE_MULTITHREADED $ENV{BOOST_USE_MULTITHREADED})
172 | set(Boost_VERBOSE TRUE)
173 | find_package(Boost 1.70 REQUIRED timer)
174 | set_package_properties(
175 | Boost PROPERTIES
176 | TYPE REQUIRED
177 | DESCRIPTION "Boost C++ libraries"
178 | URL "http://www.boost.org"
179 | )
180 |
181 | # Use Python for detecting UFCx and Basix
182 | find_package(
183 | Python3
184 | COMPONENTS Interpreter
185 | QUIET
186 | )
187 |
188 | # Check for Basix Note: Basix may be installed as a standalone C++ library, or
189 | # in the Basix Python module tree
190 | if(Python3_Interpreter_FOUND)
191 | message(STATUS "Checking for basix hints with ${Python3_EXECUTABLE}")
192 | execute_process(
193 | COMMAND
194 | ${Python3_EXECUTABLE} -c
195 | "import basix, os, sys; sys.stdout.write(os.path.dirname(basix.__file__))"
196 | OUTPUT_VARIABLE BASIX_PY_DIR
197 | RESULT_VARIABLE BASIX_PY_COMMAND_RESULT
198 | ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE
199 | )
200 |
201 | if(BASIX_PY_DIR)
202 | message(STATUS "Adding ${BASIX_PY_DIR} to Basix search hints")
203 |
204 | # Basix installed from manylinux wheel
205 | if(IS_DIRECTORY ${BASIX_PY_DIR}/../fenics_basix.libs)
206 | set(CMAKE_INSTALL_RPATH ${BASIX_PY_DIR}/../fenics_basix.libs)
207 | endif()
208 | endif()
209 | endif()
210 |
211 | find_package(Basix 0.8 REQUIRED CONFIG HINTS ${BASIX_PY_DIR})
212 | set_package_properties(
213 | basix PROPERTIES
214 | TYPE REQUIRED
215 | DESCRIPTION "FEniCS tabulation library"
216 | URL "https://github.com/fenics/basix"
217 | )
218 |
219 | find_package(DOLFINX 0.8 REQUIRED CONFIG)
220 | set_package_properties(
221 | DOLFINX PROPERTIES
222 | TYPE REQUIRED
223 | DESCRIPTION "Dynamic Object-oriented Library for FINite element computation"
224 | URL "https://github.com/fenics/basix"
225 | )
226 |
227 | # Check for HDF5
228 | set(HDF5_PREFER_PARALLEL TRUE)
229 | set(HDF5_FIND_DEBUG TRUE)
230 | find_package(HDF5 REQUIRED COMPONENTS C)
231 |
232 | if(NOT HDF5_IS_PARALLEL)
233 | message(
234 | FATAL_ERROR
235 | "Found serial HDF5 build, MPI HDF5 build required, try setting HDF5_DIR or HDF5_ROOT"
236 | )
237 | endif()
238 |
239 | set_package_properties(
240 | HDF5 PROPERTIES
241 | TYPE REQUIRED
242 | DESCRIPTION "Hierarchical Data Format 5 (HDF5)"
243 | URL "https://www.hdfgroup.org/HDF5"
244 | )
245 |
246 | # Check for UFC Note: we use the case (ufcx vs UFCx) elsewhere to determine by
247 | # which method UFCx was found
248 | if(NOT CUDOLFINX_UFCX_PYTHON)
249 | # Check in CONFIG mode, i.e. look for installed ufcxConfig.cmake
250 | find_package(ufcx 0.8 REQUIRED CONFIG)
251 | else()
252 | # Check in MODULE mode (using FindUFCX.cmake)
253 | find_package(
254 | Python3
255 | COMPONENTS Interpreter
256 | REQUIRED
257 | )
258 | find_package(UFCx 0.8 REQUIRED MODULE)
259 | endif()
260 |
261 | set_package_properties(
262 | UFCx PROPERTIES
263 | TYPE REQUIRED
264 | DESCRIPTION "Interface for form-compilers (part of FFCx)"
265 | URL "https://github.com/fenics/ffcx"
266 | )
267 |
268 | find_package(CUDAToolkit REQUIRED)
269 |
270 | set_package_properties(CUDAToolkit PROPERTIES TYPE OPTIONAL
271 | DESCRIPTION "Parallel computing platform for GPUs"
272 | URL "https://developer.nvidia.com/cuda-toolkit"
273 | PURPOSE "Enables GPU-accelerated computing"
274 | )
275 |
276 |
277 | # ------------------------------------------------------------------------------
278 | # Find optional packages
279 |
280 | if(CUDOLFINX_ENABLE_PETSC)
281 | find_package(PkgConfig REQUIRED)
282 | set(ENV{PKG_CONFIG_PATH}
283 | "$ENV{PETSC_DIR}/$ENV{PETSC_ARCH}/lib/pkgconfig:$ENV{PETSC_DIR}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}"
284 | )
285 | if(_REQUIRE_PETSC)
286 | pkg_search_module(PETSC REQUIRED IMPORTED_TARGET PETSc>=3.15 petsc>=3.15)
287 | else()
288 | pkg_search_module(PETSC OPTIONAL IMPORTED_TARGET PETSc>=3.15 petsc>=3.15)
289 | endif()
290 |
291 | # Setting for FeatureSummary
292 | if(PETSC_FOUND)
293 | message(
294 | STATUS "Found PETSc version ${PETSC_VERSION}, prefix: ${PETSC_PREFIX}"
295 | )
296 | set_property(GLOBAL APPEND PROPERTY PACKAGES_FOUND PETSc)
297 | else()
298 | set_property(GLOBAL APPEND PROPERTY PACKAGES_NOT_FOUND PETSc)
299 | endif()
300 | endif()
301 |
302 | # ------------------------------------------------------------------------------
303 | # Print summary of found and not found optional packages
304 | feature_summary(WHAT ALL)
305 |
306 |
307 |
308 | # ------------------------------------------------------------------------------
309 | # Installation of DOLFINx library
310 | add_subdirectory(cudolfinx)
311 |
312 | # ------------------------------------------------------------------------------
313 | # Generate and install helper file cudolfinx.conf
314 |
315 | # FIXME: Can CMake provide the library path name variable?
316 | if(APPLE)
317 | set(OS_LIBRARY_PATH_NAME "DYLD_LIBRARY_PATH")
318 | else()
319 | set(OS_LIBRARY_PATH_NAME "LD_LIBRARY_PATH")
320 | endif()
321 |
322 | # FIXME: not cross-platform compatible Create and install cudolfinx.conf file
323 | configure_file(
324 | ${CUDOLFINX_SOURCE_DIR}/cmake/templates/cudolfinx.conf.in
325 | ${CMAKE_BINARY_DIR}/cudolfinx.conf @ONLY
326 | )
327 | install(
328 | FILES ${CMAKE_BINARY_DIR}/cudolfinx.conf
329 | DESTINATION ${CMAKE_INSTALL_LIBDIR}/cudolfinx
330 | COMPONENT Development
331 | )
332 |
333 | # ------------------------------------------------------------------------------
334 | # Add "make uninstall" target
335 | configure_file(
336 | "${CUDOLFINX_SOURCE_DIR}/cmake/templates/cmake_uninstall.cmake.in"
337 | "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" IMMEDIATE @ONLY
338 | )
339 |
340 | add_custom_target(
341 | uninstall "${CMAKE_COMMAND}" -P
342 | "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake"
343 | )
344 |
345 | # ------------------------------------------------------------------------------
346 | # Print post-install message
347 | add_subdirectory(cmake/post-install)
348 |
349 | # ------------------------------------------------------------------------------
350 |
--------------------------------------------------------------------------------
/cpp/cmake/modules/FindUFCx.cmake:
--------------------------------------------------------------------------------
1 | #=============================================================================
2 | # - Try to find UFCx by interrogating the Python module FFCx
3 | # Once done this will define
4 | #
5 | # UFCX_FOUND - system has UFCx
6 | # UFCX_INCLUDE_DIRS - include directories for UFCx
7 | # UFCX_SIGNATURE - signature for UFCx
8 | # UFCX_VERSION - version for UFCx
9 | #
10 | #=============================================================================
11 | # Copyright (C) 2010-2021 Johannes Ring and Garth N. Wells
12 | # All rights reserved.
13 | #
14 | # Redistribution and use in source and binary forms, with or without
15 | # modification, are permitted provided that the following conditions
16 | # are met:
17 | #
18 | # 1. Redistributions of source code must retain the above copyright
19 | # notice, this list of conditions and the following disclaimer.
20 | # 2. Redistributions in binary form must reproduce the above copyright
21 | # notice, this list of conditions and the following disclaimer in
22 | # the documentation and/or other materials provided with the
23 | # distribution.
24 | #
25 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
26 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
27 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
28 | # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
29 | # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
30 | # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
31 | # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
32 | # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
33 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
35 | # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36 | # POSSIBILITY OF SUCH DAMAGE.
37 | #=============================================================================
38 |
39 | message(
40 | STATUS
41 | "Asking Python module FFCx for location of UFC... (Python executable: ${Python3_EXECUTABLE})"
42 | )
43 |
44 | # Get include path
45 | execute_process(
46 | COMMAND
47 | ${Python3_EXECUTABLE} -c
48 | "import ffcx.codegeneration, sys; sys.stdout.write(ffcx.codegeneration.get_include_path())"
49 | OUTPUT_VARIABLE UFCX_INCLUDE_DIR
50 | )
51 |
52 | # Get ufcx.h version
53 | if(UFCX_INCLUDE_DIR)
54 | set(UFCX_INCLUDE_DIRS
55 | ${UFCX_INCLUDE_DIR}
56 | CACHE STRING "Where to find ufcx.h"
57 | )
58 | execute_process(
59 | COMMAND ${Python3_EXECUTABLE} -c
60 | "import ffcx, sys; sys.stdout.write(ffcx.__version__)"
61 | OUTPUT_VARIABLE UFCX_VERSION
62 | )
63 | endif()
64 |
65 | # Compute hash of ufcx.h
66 | find_file(_UFCX_HEADER "ufcx.h" ${UFCX_INCLUDE_DIR})
67 | if(_UFCX_HEADER)
68 | file(SHA1 ${_UFCX_HEADER} UFCX_SIGNATURE)
69 | endif()
70 |
71 | mark_as_advanced(UFCX_VERSION UFCX_INCLUDE_DIRS UFCX_SIGNATURE)
72 | find_package_handle_standard_args(
73 | UFCx
74 | REQUIRED_VARS UFCX_INCLUDE_DIRS UFCX_SIGNATURE UFCX_VERSION
75 | VERSION_VAR UFCX_VERSION HANDLE_VERSION_RANGE REASON_FAILURE_MESSAGE
76 | "UFCx could not be found."
77 | )
78 |
--------------------------------------------------------------------------------
/cpp/cmake/post-install/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | install(
2 | CODE "MESSAGE(
3 | \"----------------------------------------------------------------------------
4 | CUDOLFINx has now been installed in
5 |
6 | ${CMAKE_INSTALL_PREFIX}
7 |
8 |
9 | Don't forget to update your environment variables. This can be done
10 | easily using the helper file 'cudolfinx.conf' which sets the appropriate
11 | variables (for users of the Bash shell).
12 |
13 | To update your environment variables, run the following command:
14 |
15 | source ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/cudolfinx/cudolfinx.conf
16 |
17 | ----------------------------------------------------------------------------\")"
18 | )
19 |
--------------------------------------------------------------------------------
/cpp/cmake/templates/CUDOLFINXConfig.cmake.in:
--------------------------------------------------------------------------------
1 | # * Build details for CUDOLFINx: CUDA extension for DOLFINX
2 | #
3 | # This file has been automatically generated.
4 |
5 | # FIXME: Check that naming conforms to CMake standards
6 |
7 | @PACKAGE_INIT@
8 | include(CMakeFindDependencyMacro)
9 |
10 | find_dependency(MPI REQUIRED)
11 | find_dependency(pugixml)
12 |
13 | # Check for Boost
14 | if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT)
15 | set(Boost_NO_SYSTEM_PATHS on)
16 | endif()
17 | set(Boost_USE_MULTITHREADED $ENV{BOOST_USE_MULTITHREADED})
18 | set(Boost_VERBOSE TRUE)
19 | find_dependency(Boost 1.70 REQUIRED COMPONENTS timer filesystem)
20 |
21 | if(@ufcx_FOUND@)
22 | find_dependency(ufcx)
23 | endif()
24 |
25 | # Basix
26 | find_package(Python3 COMPONENTS Interpreter)
27 | if(Python3_Interpreter_FOUND)
28 | execute_process(
29 | COMMAND
30 | ${Python3_EXECUTABLE} -c
31 | "import basix, os, sys; sys.stdout.write(os.path.dirname(basix.__file__))"
32 | OUTPUT_VARIABLE BASIX_PY_DIR
33 | RESULT_VARIABLE BASIX_PY_COMMAND_RESULT
34 | ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE
35 | )
36 | endif()
37 | if(BASIX_PY_DIR)
38 | message(STATUS "Adding ${BASIX_PY_DIR} to Basix search hints")
39 | endif()
40 | find_dependency(Basix CONFIG HINTS ${BASIX_PY_DIR})
41 |
42 | # HDF5
43 | if(NOT TARGET hdf5::hdf5)
44 | set(HDF5_PREFER_PARALLEL TRUE)
45 | set(HDF5_FIND_DEBUG TRUE)
46 | find_dependency(HDF5 COMPONENTS C)
47 | if(HDF5_FOUND AND NOT HDF5_IS_PARALLEL)
48 | message(FATAL_ERROR "Found serial HDF5 build, MPI HDF5 build required")
49 | endif()
50 | endif()
51 |
52 | if(@PETSC_FOUND@)
53 | if(NOT TARGET PkgConfig::PETSC)
54 | find_package(PkgConfig REQUIRED)
55 | set(ENV{PKG_CONFIG_PATH}
56 | "$ENV{PETSC_DIR}/$ENV{PETSC_ARCH}/lib/pkgconfig:$ENV{PETSC_DIR}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}"
57 | )
58 | pkg_search_module(PETSC REQUIRED IMPORTED_TARGET PETSc petsc)
59 | endif()
60 | endif()
61 |
62 | if(@SLEPC_FOUND@)
63 | if(NOT TARGET PkgConfig::SLEPC)
64 | find_package(PkgConfig REQUIRED)
65 | set(ENV{PKG_CONFIG_PATH}
66 | "$ENV{SLEPC_DIR}/$ENV{PETSC_ARCH}/lib/pkgconfig:$ENV{SLEPC_DIR}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}"
67 | )
68 | set(ENV{PKG_CONFIG_PATH}
69 | "$ENV{PETSC_DIR}/$ENV{PETSC_ARCH}/lib/pkgconfig:$ENV{PETSC_DIR}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}"
70 | )
71 | set(ENV{PKG_CONFIG_PATH}
72 | "$ENV{PETSC_DIR}/$ENV{PETSC_ARCH}:$ENV{PETSC_DIR}:$ENV{PKG_CONFIG_PATH}"
73 | )
74 | pkg_search_module(SLEPC REQUIRED IMPORTED_TARGET SLEPc slepc)
75 | endif()
76 | endif()
77 |
78 | if(NOT TARGET cudolfinx)
79 | include("${CMAKE_CURRENT_LIST_DIR}/CUDOLFINXTargets.cmake")
80 | endif()
81 |
82 | check_required_components(CUDOLFINX)
83 |
--------------------------------------------------------------------------------
/cpp/cmake/templates/cmake_uninstall.cmake.in:
--------------------------------------------------------------------------------
1 | if(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt")
2 | message(
3 | FATAL_ERROR
4 | "Cannot find install manifest: \"@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt\""
5 | )
6 | endif()
7 |
8 | file(READ "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt" files)
9 | string(REGEX REPLACE "\n" ";" files "${files}")
10 | foreach(file ${files})
11 | message(STATUS "Uninstalling \"$ENV{DESTDIR}${file}\"")
12 | if(EXISTS "$ENV{DESTDIR}${file}")
13 | exec_program(
14 | "@CMAKE_COMMAND@" ARGS
15 | "-E remove \"$ENV{DESTDIR}${file}\""
16 | OUTPUT_VARIABLE rm_out
17 | RETURN_VALUE rm_retval
18 | )
19 | if(NOT "${rm_retval}" STREQUAL 0)
20 | message(FATAL_ERROR "Problem when removing \"$ENV{DESTDIR}${file}\"")
21 | endif()
22 | else()
23 | message(STATUS "File \"$ENV{DESTDIR}${file}\" does not exist.")
24 | endif()
25 | endforeach()
26 |
--------------------------------------------------------------------------------
/cpp/cmake/templates/cudolfinx.conf.in:
--------------------------------------------------------------------------------
1 | # Helper file for setting non-default CUDOLFINx environment variables
2 |
3 | # Common Unix variables
4 | export @OS_LIBRARY_PATH_NAME@=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@:$@OS_LIBRARY_PATH_NAME@
5 | export PATH=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_BINDIR@:$PATH
6 | export PKG_CONFIG_PATH=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/pkgconfig:$PKG_CONFIG_PATH
7 | export CMAKE_PREFIX_PATH=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@/cmake:$CMAKE_PREFIX_PATH
8 |
9 | # Special macOS variables
10 | export DYLD_FRAMEWORK_PATH=/opt/local/Library/Frameworks:$DYLD_FRAMEWORK_PATH
11 |
--------------------------------------------------------------------------------
/cpp/cmake/templates/cudolfinx.pc.in:
--------------------------------------------------------------------------------
1 | # pkg-config configuration for CUDOLFINx
2 | prefix=@CMAKE_INSTALL_PREFIX@
3 | exec_prefix=@CMAKE_INSTALL_PREFIX@
4 | libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@
5 | includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
6 | compiler=@CMAKE_CXX_COMPILER@
7 | definitions=@PKG_DEFINITIONS@
8 | extlibs=@CUDOLFINX_EXT_LIBS@
9 |
10 | Name: CUDOLFINx
11 | Description: CUDA extension for DOLFINX
12 | Version: @CUDOLFINX_VERSION@
13 | Requires: @PKG_REQUIRES@
14 | Conflicts:
15 | Libs: @PKG_LINKFLAGS@ -L${libdir} -lcudolfinx
16 | Cflags: @PKG_CXXFLAGS@ -DCUDOLFINX_VERSION=\"@CUDOLFINX_VERSION@\" ${definitions} -I${includedir} @PKG_INCLUDES@
17 |
--------------------------------------------------------------------------------
/cpp/cudolfinx/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | # ------------------------------------------------------------------------------
2 | include(GNUInstallDirs)
3 |
4 | # ------------------------------------------------------------------------------
5 | # Declare the library (target)
6 | add_library(cudolfinx)
7 |
8 | # ------------------------------------------------------------------------------
9 | # Add source files to the target
10 | set(CUDOLFINX_DIRS
11 | common
12 | fem
13 | la
14 | mesh
15 | )
16 |
17 | # Add source to dolfinx target, and get sets of header files
18 | foreach(DIR ${CUDOLFINX_DIRS})
19 | add_subdirectory(${DIR})
20 | endforeach()
21 |
22 | # Set target include location (for build and installed)
23 | target_include_directories(
24 | cudolfinx
25 | PUBLIC
26 | $
27 | "$"
28 | )
29 |
30 | # ------------------------------------------------------------------------------
31 | # Configure the common/version.h file
32 | configure_file(
33 | ${CMAKE_CURRENT_SOURCE_DIR}/common/version.h.in common/version.h @ONLY
34 | )
35 |
36 | # ------------------------------------------------------------------------------
37 | # Set target properties
38 | set_target_properties(
39 | cudolfinx
40 | PROPERTIES VERSION ${CUDOLFINX_VERSION}
41 | SOVERSION ${CUDOLFINX_VERSION_MAJOR}.${CUDOLFINX_VERSION_MINOR}
42 | )
43 |
44 | # Add git revision flag to the one affected file
45 | #set_source_files_properties(
46 | # common/defines.cpp
47 | # PROPERTIES
48 | # COMPILE_DEFINITIONS
49 | # "UFCX_SIGNATURE=\"${UFCX_SIGNATURE}\";CUDOLFINX_GIT_COMMIT_HASH=\"${GIT_COMMIT_HASH}\""
50 | #)
51 |
52 | # ------------------------------------------------------------------------------
53 | # Set compiler options and definitions
54 |
55 | # Set 'Developer' build type flags
56 | target_compile_options(
57 | cudolfinx PRIVATE $<$:${CUDOLFINX_CXX_DEVELOPER_FLAGS}>
58 | )
59 |
60 | # Add version to definitions (public)
61 | target_compile_definitions(cudolfinx PUBLIC CUDOLFINX_VERSION="${CUDOLFINX_VERSION}")
62 |
63 | # ------------------------------------------------------------------------------
64 | # Add include directories and libraries of required packages
65 |
66 | # UFCx
67 | if(TARGET ufcx::ufcx)
68 | target_link_libraries(cudolfinx PUBLIC ufcx::ufcx)
69 | else()
70 | target_include_directories(cudolfinx SYSTEM PUBLIC ${UFCX_INCLUDE_DIRS})
71 | endif()
72 |
73 | # Basix
74 | target_link_libraries(cudolfinx PUBLIC Basix::basix)
75 |
76 | # Boost
77 | target_link_libraries(cudolfinx PUBLIC Boost::headers)
78 | target_link_libraries(cudolfinx PUBLIC Boost::timer)
79 |
80 | # MPI
81 | target_link_libraries(cudolfinx PUBLIC MPI::MPI_CXX)
82 |
83 | # HDF5
84 | target_link_libraries(cudolfinx PUBLIC hdf5::hdf5)
85 |
86 | # CUDA Toolkit
87 | target_link_libraries(cudolfinx PRIVATE CUDA::cuda_driver CUDA::nvrtc CUDA::cupti)
88 | target_include_directories(cudolfinx SYSTEM PRIVATE ${CUDAToolkit_INCLUDE_DIRS})
89 |
90 | # Dolfinx
91 | target_link_libraries(cudolfinx PUBLIC dolfinx)
92 |
93 | # ------------------------------------------------------------------------------
94 | # Optional packages
95 |
96 | # PETSc
97 | if(CUDOLFINX_ENABLE_PETSC AND PETSC_FOUND)
98 | target_link_libraries(cudolfinx PUBLIC PkgConfig::PETSC)
99 | target_compile_definitions(cudolfinx PUBLIC HAS_PETSC)
100 | endif()
101 |
102 |
103 | # ------------------------------------------------------------------------------
104 | # Install cudolfinx library and header files
105 | install(
106 | TARGETS cudolfinx
107 | EXPORT CUDOLFINXTargets
108 | RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT RuntimeExecutables
109 | LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT RuntimeLibraries
110 | ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT Development
111 | )
112 |
113 | # Generate CUDOLFINXTargets.cmake
114 | install(EXPORT CUDOLFINXTargets DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/cudolfinx)
115 |
116 | # Install the header files
117 | install(
118 | FILES cudolfinx.h
119 | DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
120 | COMPONENT Development
121 | )
122 |
123 | foreach(DIR ${CUDOLFINX_DIRS})
124 | install(
125 | FILES ${HEADERS_${DIR}}
126 | DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/cudolfinx/${DIR}
127 | COMPONENT Development
128 | )
129 | endforeach()
130 |
131 | install(
132 | FILES ${CMAKE_CURRENT_BINARY_DIR}/common/version.h
133 | DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/cudolfinx/common
134 | COMPONENT Development
135 | )
136 |
137 | # ------------------------------------------------------------------------------
138 | # Generate CMake config files (CUDOLFINXConfig{,Version}.cmake)
139 | include(CMakePackageConfigHelpers)
140 | write_basic_package_version_file(
141 | ${CMAKE_BINARY_DIR}/cudolfinx/CUDOLFINXConfigVersion.cmake
142 | VERSION ${CUDOLFINX_VERSION}
143 | COMPATIBILITY AnyNewerVersion
144 | )
145 |
146 | configure_package_config_file(
147 | ${CUDOLFINX_SOURCE_DIR}/cmake/templates/CUDOLFINXConfig.cmake.in
148 | ${CMAKE_BINARY_DIR}/cudolfinx/CUDOLFINXConfig.cmake
149 | INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/cudolfinx
150 | )
151 |
152 | # Install CMake helper files
153 | install(
154 | FILES ${CMAKE_BINARY_DIR}/cudolfinx/CUDOLFINXConfig.cmake
155 | ${CMAKE_BINARY_DIR}/cudolfinx/CUDOLFINXConfigVersion.cmake
156 | DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/cudolfinx
157 | COMPONENT Development
158 | )
159 |
160 | # ------------------------------------------------------------------------------
161 | # Generate pkg-config file and install it
162 |
163 | # Define packages that should be required by pkg-config file
164 | set(PKG_REQUIRES "")
165 |
166 | # Get link libraries and includes
167 | get_target_property(
168 | PKGCONFIG_CUDOLFINX_TARGET_LINK_LIBRARIES cudolfinx INTERFACE_LINK_LIBRARIES
169 | )
170 | get_target_property(
171 | PKGCONFIG_CUDOLFINX_INCLUDE_DIRECTORIES cudolfinx
172 | INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
173 | )
174 |
175 | # Add imported targets to lists for creating pkg-config file
176 | set(PKGCONFIG_CUDOLFINX_LIBS)
177 |
178 | foreach(_target ${PKGCONFIG_CUDOLFINX_TARGET_LINK_LIBRARIES})
179 | if("${_target}" MATCHES "^[^<>]+$") # Skip "$", which we get with
180 | # static libs
181 | if("${_target}" MATCHES "^.*::.*$")
182 | # Get include paths
183 | get_target_property(_inc_dirs ${_target} INTERFACE_INCLUDE_DIRECTORIES)
184 |
185 | if(_inc_dirs)
186 | list(APPEND PKGCONFIG_CUDOLFINX_INCLUDE_DIRECTORIES ${_inc_dirs})
187 | endif()
188 |
189 | # Get libraries
190 | get_target_property(_libs ${_target} INTERFACE_LINK_LIBRARIES)
191 |
192 | if(_libs)
193 | list(APPEND PKGCONFIG_CUDOLFINX_LIBS ${_libs})
194 | endif()
195 |
196 | else()
197 | # 'regular' libs, i.e. not imported targets
198 | list(APPEND PKGCONFIG_CUDOLFINX_LIBS ${_target})
199 | endif()
200 |
201 | # Special handling for compiled Boost imported targets
202 | if(("${_target}" MATCHES "^.*Boost::.*$") AND NOT "${_target}" STREQUAL
203 | "Boost::headers"
204 | )
205 | get_target_property(_libs ${_target} IMPORTED_LOCATION_RELEASE)
206 |
207 | if(_libs)
208 | list(APPEND PKGCONFIG_CUDOLFINX_LIBS ${_libs})
209 | endif()
210 | endif()
211 | endif()
212 | endforeach()
213 |
214 | # Join include lists and remove duplicates
215 | list(REMOVE_DUPLICATES PKGCONFIG_CUDOLFINX_INCLUDE_DIRECTORIES)
216 | list(REMOVE_DUPLICATES PKGCONFIG_CUDOLFINX_LIBS)
217 |
218 | # Convert include dirs to -I form
219 | foreach(_inc_dir ${PKGCONFIG_CUDOLFINX_INCLUDE_DIRECTORIES})
220 | set(PKG_INCLUDES "-I${_inc_dir} ${PKG_INCLUDES}")
221 | endforeach()
222 |
223 | # Get cudolfinx definitions
224 | get_target_property(
225 | PKG_CUDOLFINX_DEFINITIONS cudolfinx INTERFACE_COMPILE_DEFINITIONS
226 | )
227 | set(PKG_DEFINITIONS)
228 |
229 | foreach(_def ${PKG_DOLFINX_DEFINITIONS})
230 | set(PKG_DEFINITIONS "${PKG_DEFINITIONS} -D${_def}")
231 | endforeach()
232 |
233 | # Get basix definitions (this is required to propagate Basix definition to the
234 | # pkg-config file, in the future Basix should create its own basix.pc file, see
235 | # https://github.com/FEniCS/basix/issues/204)
236 | get_target_property(
237 | PKG_BASIX_DEFINITIONS Basix::basix INTERFACE_COMPILE_DEFINITIONS
238 | )
239 |
240 | foreach(_def ${PKG_BASIX_DEFINITIONS})
241 | set(PKG_DEFINITIONS "${PKG_DEFINITIONS} -D${_def}")
242 | endforeach()
243 |
244 | # Convert compiler flags and definitions into space separated strings
245 | string(REPLACE ";" " " PKG_CXXFLAGS "${CMAKE_CXX_FLAGS}")
246 | string(REPLACE ";" " " PKG_LINKFLAGS "${CMAKE_EXE_LINKER_FLAGS}")
247 |
248 | # Convert libraries to -L -l form
249 | foreach(_lib ${PKGCONFIG_CUDOLFINX_LIBS})
250 | # Add -Wl,option directives
251 | if("${_lib}" MATCHES "-Wl,[^ ]*")
252 | set(PKG_LINKFLAGS "${_lib} ${PKG_LINKFLAGS}")
253 | else()
254 | get_filename_component(_path ${_lib} DIRECTORY)
255 | get_filename_component(_name ${_lib} NAME_WE)
256 | string(REPLACE "lib" "" _name "${_name}")
257 |
258 | # Add libraries that matches the form -L -l
259 | if(NOT "${_path}" STREQUAL "")
260 | set(PKG_LINKFLAGS "-L${_path} -l${_name} ${PKG_LINKFLAGS}")
261 | endif()
262 | endif()
263 | endforeach()
264 |
265 | # Remove duplicated link flags
266 | separate_arguments(PKG_LINKFLAGS)
267 | list(REMOVE_DUPLICATES PKG_LINKFLAGS)
268 | string(REPLACE ";" " " PKG_LINKFLAGS "${PKG_LINKFLAGS}")
269 |
270 | # Add additional link flags
271 | foreach(_linkflag ${CUDOLFINX_LINK_FLAGS})
272 | set(PKG_LINKFLAGS "${PKG_LINKFLAGS} ${_linkflag}")
273 | endforeach()
274 |
275 | # Boost include dir (used as pkg-config variable)
276 | get_target_property(
277 | BOOST_INCLUDE_DIR Boost::headers INTERFACE_INCLUDE_DIRECTORIES
278 | )
279 |
280 | # Configure and install pkg-config file
281 | configure_file(
282 | ${CUDOLFINX_SOURCE_DIR}/cmake/templates/cudolfinx.pc.in
283 | ${CMAKE_BINARY_DIR}/cudolfinx.pc @ONLY
284 | )
285 | install(
286 | FILES ${CMAKE_BINARY_DIR}/cudolfinx.pc
287 | DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig
288 | COMPONENT Development
289 | )
290 |
291 | # ------------------------------------------------------------------------------
292 |
--------------------------------------------------------------------------------
/cpp/cudolfinx/common/CMakeLists.txt:
--------------------------------------------------------------------------------
1 | set(HEADERS_common
2 | ${CMAKE_CURRENT_SOURCE_DIR}/CUDA.h
3 | ${CMAKE_CURRENT_SOURCE_DIR}/CUDAStore.h
4 | PARENT_SCOPE
5 | )
6 |
7 | target_sources(
8 | cudolfinx
9 | PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/CUDA.cpp
10 | )
11 |
--------------------------------------------------------------------------------
/cpp/cudolfinx/common/CUDA.h:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2024 Benjamin Pachev, James D. Trotter
2 | //
3 | // This file is part of cuDOLFINX
4 | //
5 | // SPDX-License-Identifier: LGPL-3.0-or-later
6 |
7 | #pragma once
8 |
9 | #include
10 | #include
11 | #include
12 | #include
13 |
14 | namespace dolfinx
15 | {
16 |
17 | namespace CUDA
18 | {
19 | class Module;
20 | class Kernel;
21 |
22 | /// This class is a wrapper around a CUDA device context
23 | class Context
24 | {
25 | public:
26 | /// Create a CUDA device context
27 | Context();
28 |
29 | /// Destructor
30 | ~Context();
31 |
32 | /// Copy constructor
33 | /// @param[in] context The object to be copied
34 | Context(const Context& context) = delete;
35 |
36 | /// Move constructor
37 | /// @param[in] context The object to be moved
38 | Context(Context&& context) = delete;
39 |
40 | /// Assignment operator
41 | /// @param[in] context The object to assign from
42 | Context& operator=(const Context& context) = delete;
43 |
44 | /// Move assignment operator
45 | /// @param[in] context The object to assign from
46 | Context& operator=(Context&& context) = delete;
47 |
48 | /// Return underlying CUDA device
49 | const CUdevice& device() const;
50 |
51 | /// Return underlying CUDA context
52 | CUcontext& context();
53 |
54 | private:
55 | CUdevice _device;
56 | CUcontext _context;
57 | };
58 |
59 | /// This class is a wrapper around a module, which is obtained by
60 | /// compiling PTX assembly to CUDA device code.
61 | class Module
62 | {
63 | public:
64 | /// Create an empty module
65 | Module();
66 |
67 | /// Create a module
68 | Module(
69 | const CUDA::Context& cuda_context,
70 | const std::string& ptx,
71 | CUjit_target target,
72 | int num_module_load_options,
73 | CUjit_option* module_load_options,
74 | void** module_load_option_values,
75 | bool verbose,
76 | bool debug);
77 |
78 | /// Destructor
79 | ~Module();
80 |
81 | /// Copy constructor
82 | /// @param[in] module The object to be copied
83 | Module(const Module& module) = delete;
84 |
85 | /// Move constructor
86 | /// @param[in] module The object to be moved
87 | Module(Module&& module);
88 |
89 | /// Assignment operator
90 | /// @param[in] module The object to assign from
91 | Module& operator=(const Module& module) = delete;
92 |
93 | /// Move assignment operator
94 | /// @param[in] module The object to assign from
95 | Module& operator=(Module&& module);
96 |
97 | /// Get a device-side function from a loaded module
98 | CUfunction get_device_function(
99 | const std::string& device_function_name) const;
100 |
101 | /// Get info log for a loaded module
102 | const char* info_log() const {
103 | return _info_log; }
104 |
105 | /// Get error log for a loaded module
106 | const char* error_log() const {
107 | return _error_log; }
108 |
109 | private:
110 | /// Handle to the CUDA module
111 | CUmodule _module;
112 |
113 | /// Size of the buffer for informational log messages
114 | size_t _info_log_size;
115 |
116 | /// Informational log messages related to loading the module
117 | char* _info_log;
118 |
119 | /// Size of the buffer for error log messages
120 | size_t _error_log_size;
121 |
122 | /// Error log messages related to loading the module
123 | char* _error_log;
124 | };
125 |
126 | /// Use the NVIDIA CUDA Runtime Compilation (nvrtc) library to compile
127 | /// device-side code for a given CUDA program.
128 | std::string compile_cuda_cpp_to_ptx(
129 | const char* program_name,
130 | int num_program_headers,
131 | const char** program_headers,
132 | const char** program_include_names,
133 | int num_compile_options,
134 | const char** compile_options,
135 | const char* program_src,
136 | const char* cudasrcdir,
137 | bool verbose);
138 |
139 | void safeMemAlloc(CUdeviceptr* dptr, size_t bytesize);
140 | void safeMemcpyDtoH(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount);
141 | void safeMemcpyHtoD(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount);
142 | void safeDeviceGetAttribute(int * res, CUdevice_attribute attr, CUdevice dev);
143 | void safeCtxSynchronize();
144 | void safeStreamCreate(CUstream* streamptr, unsigned int flags);
145 |
146 | template void safeVectorCreate(CUdeviceptr* dptr, std::vector arr) {
147 | size_t bytesize = sizeof(T) * arr.size();
148 | safeMemAlloc(dptr, bytesize);
149 | safeMemcpyHtoD(*dptr, (void *)arr.data(), bytesize);
150 | }
151 |
152 | CUjit_target get_cujit_target(const Context& cuda_context);
153 |
154 | } // namespace CUDA
155 |
156 |
157 | } // namespace dolfinx
158 |
--------------------------------------------------------------------------------
/cpp/cudolfinx/common/CUDAStore.h:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2024 Benjamin Pachev, James D. Trotter
2 | //
3 | // This file is part of cuDOLFINX
4 | //
5 | // SPDX-License-Identifier: LGPL-3.0-or-later
6 |
7 | #pragma once
8 | #include
9 | #include