├── README.md ├── CMakeLists.txt ├── .gitignore ├── LICENSE ├── .clang-format └── main.cpp /README.md: -------------------------------------------------------------------------------- 1 | # neighbourhood-creation 2 | Code for investigating the performance of MPI_Graph_create 3 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # This file was generated by running 2 | # 3 | # python cmake/scripts/generate-cmakefiles from dolfinx/cpp 4 | # 5 | cmake_minimum_required(VERSION 3.19) 6 | 7 | set(PROJECT_NAME neighbor_bench) 8 | project(${PROJECT_NAME} LANGUAGES C CXX) 9 | 10 | # Set C++20 standard 11 | set(CMAKE_CXX_STANDARD 20) 12 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 13 | set(CMAKE_CXX_EXTENSIONS OFF) 14 | 15 | # Add CXX flags 16 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -march=native -mtune=native -O3 -DNDEBUG") 17 | 18 | find_package(DOLFINX REQUIRED) 19 | 20 | set(CMAKE_INCLUDE_CURRENT_DIR ON) 21 | 22 | add_executable(${PROJECT_NAME} main.cpp) 23 | target_link_libraries(${PROJECT_NAME} dolfinx) 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.o 3 | *.ko 4 | *.obj 5 | *.elf 6 | 7 | # Compiled Static libraries 8 | *.a 9 | *.lib 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dll 14 | *.dylib 15 | 16 | # Executables 17 | *.exe 18 | *.out 19 | *.app 20 | 21 | # CMake build directory 22 | build/ 23 | 24 | # Visual Studio files 25 | *.sln 26 | *.vcxproj 27 | *.vcxproj.filters 28 | *.pdb 29 | *.ipdb 30 | *.obj 31 | *.idb 32 | *.pch 33 | *.sdf 34 | *.opensdf 35 | *.tlog 36 | *.log 37 | *.vspscc 38 | *.vssscc 39 | *.psess 40 | *.vsp 41 | *.vspx 42 | *.user 43 | *.userosscache 44 | *.suo 45 | *.cache 46 | *.ncb 47 | *.clw 48 | *.ilk 49 | *.mdmp 50 | *.map 51 | *.exp 52 | 53 | # Xcode files 54 | *.pbxproj 55 | *.xcworkspace 56 | *.xcuserstate 57 | project.xcworkspace/ 58 | xcuserdata/ 59 | 60 | # Eclipse files 61 | .settings/ 62 | .project 63 | .cproject 64 | 65 | # Sublime Text files 66 | *.sublime-workspace 67 | 68 | # Qt Creator files 69 | *.pro.user 70 | *.pro.user.* 71 | 72 | # Other files 73 | *.swp 74 | *~ 75 | .DS_Store 76 | 77 | *build/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Igor Baratta 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | # BasedOnStyle: LLVM 4 | AccessModifierOffset: -2 5 | AlignAfterOpenBracket: Align 6 | AlignConsecutiveAssignments: false 7 | AlignConsecutiveDeclarations: false 8 | AlignEscapedNewlinesLeft: false 9 | AlignOperands: true 10 | AlignTrailingComments: true 11 | AllowAllParametersOfDeclarationOnNextLine: true 12 | AllowShortBlocksOnASingleLine: false 13 | AllowShortCaseLabelsOnASingleLine: false 14 | AllowShortFunctionsOnASingleLine: All 15 | AllowShortIfStatementsOnASingleLine: false 16 | AllowShortLoopsOnASingleLine: false 17 | AlwaysBreakAfterDefinitionReturnType: None 18 | AlwaysBreakAfterReturnType: None 19 | AlwaysBreakBeforeMultilineStrings: false 20 | AlwaysBreakTemplateDeclarations: true 21 | BinPackArguments: true 22 | BinPackParameters: true 23 | BraceWrapping: 24 | AfterClass: false 25 | AfterControlStatement: false 26 | AfterEnum: false 27 | AfterFunction: false 28 | AfterNamespace: false 29 | AfterObjCDeclaration: false 30 | AfterStruct: false 31 | AfterUnion: false 32 | BeforeCatch: false 33 | BeforeElse: false 34 | IndentBraces: false 35 | BreakBeforeBinaryOperators: All 36 | BreakBeforeBraces: Allman 37 | BreakBeforeTernaryOperators: true 38 | BreakConstructorInitializersBeforeComma: false 39 | BreakAfterJavaFieldAnnotations: false 40 | BreakStringLiterals: true 41 | ColumnLimit: 80 42 | CommentPragmas: '^ IWYU pragma:' 43 | ConstructorInitializerAllOnOneLineOrOnePerLine: false 44 | ConstructorInitializerIndentWidth: 4 45 | ContinuationIndentWidth: 4 46 | Cpp11BracedListStyle: true 47 | DerivePointerAlignment: false 48 | DisableFormat: false 49 | ExperimentalAutoDetectBinPacking: false 50 | ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] 51 | IncludeCategories: 52 | - Regex: '^"(llvm|llvm-c|clang|clang-c)/' 53 | Priority: 2 54 | - Regex: '^(<|"(gtest|isl|json)/)' 55 | Priority: 3 56 | - Regex: '.*' 57 | Priority: 1 58 | IncludeIsMainRegex: '$' 59 | IndentCaseLabels: false 60 | IndentWidth: 2 61 | IndentWrappedFunctionNames: false 62 | JavaScriptQuotes: Leave 63 | JavaScriptWrapImports: true 64 | KeepEmptyLinesAtTheStartOfBlocks: true 65 | MacroBlockBegin: '' 66 | MacroBlockEnd: '' 67 | MaxEmptyLinesToKeep: 1 68 | NamespaceIndentation: None 69 | ObjCBlockIndentWidth: 2 70 | ObjCSpaceAfterProperty: false 71 | ObjCSpaceBeforeProtocolList: true 72 | PenaltyBreakBeforeFirstCallParameter: 19 73 | PenaltyBreakComment: 300 74 | PenaltyBreakFirstLessLess: 120 75 | PenaltyBreakString: 1000 76 | PenaltyExcessCharacter: 1000000 77 | PenaltyReturnTypeOnItsOwnLine: 60 78 | PointerAlignment: Left 79 | ReflowComments: true 80 | SortIncludes: true 81 | SpaceAfterCStyleCast: false 82 | SpaceAfterTemplateKeyword: true 83 | SpaceBeforeAssignmentOperators: true 84 | SpaceBeforeParens: ControlStatements 85 | SpaceInEmptyParentheses: false 86 | SpacesBeforeTrailingComments: 1 87 | SpacesInAngles: false 88 | SpacesInContainerLiterals: true 89 | SpacesInCStyleCastParentheses: false 90 | SpacesInParentheses: false 91 | SpacesInSquareBrackets: false 92 | Standard: Cpp11 93 | TabWidth: 8 94 | UseTab: Never 95 | ... -------------------------------------------------------------------------------- /main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | int main(int argc, char* argv[]) 10 | { 11 | dolfinx::init_logging(argc, argv); 12 | MPI_Init(&argc, &argv); 13 | { 14 | MPI_Comm comm = MPI_COMM_WORLD; 15 | 16 | dolfinx::common::TimeLogger logger; 17 | 18 | // get number of neighbors from command line 19 | int rank = dolfinx::MPI::rank(comm); 20 | int size = dolfinx::MPI::size(comm); 21 | 22 | // Find power of 3 closest to size 23 | std::size_t n = 1; 24 | while (n * n * n < std::size_t(100 * size)) 25 | n++; 26 | 27 | const std::array, 2> p = {{{0, 0, 0}, {1, 1, 1}}}; 28 | dolfinx::mesh::CellType cell_type = dolfinx::mesh::CellType::hexahedron; 29 | std::array N = {n, n, n}; 30 | 31 | // create a mesh 32 | auto mesh = dolfinx::mesh::create_box(comm, p, N, cell_type); 33 | 34 | // get index map 35 | std::shared_ptr map = mesh.topology()->index_map(0); 36 | const std::vector& dest = map->dest(); 37 | 38 | // ------------------------------------------------------------ 39 | // Create a communicator using MPI_Dist_graph_create_adjacent 40 | // ------------------------------------------------------------ 41 | MPI_Barrier(comm); 42 | { 43 | const std::vector& src = map->src(); 44 | for (int i = 0; i < 100; i++) 45 | { 46 | dolfinx::common::Timer timer0("x0 MPI_Dist_graph_create_adjacent"); 47 | MPI_Comm comm_dist_graph_adjacent; 48 | int err = MPI_Dist_graph_create_adjacent( 49 | comm, src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), 50 | dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, 51 | &comm_dist_graph_adjacent); 52 | dolfinx::MPI::check_error(comm, err); 53 | timer0.stop(); 54 | MPI_Comm_free(&comm_dist_graph_adjacent); 55 | } 56 | } 57 | 58 | // ------------------------------------------------------------ 59 | // Compute graph edges using non-blocking collectives and create 60 | // a communicator using MPI_Dist_graph_create_adjacent 61 | // ------------------------------------------------------------ 62 | MPI_Barrier(comm); 63 | { 64 | const std::vector& src = map->src(); 65 | std::vector recv_sizes(src.size(), 0); 66 | std::vector send_sizes(dest.size(), 0); 67 | for (int i = 0; i < 100; i++) 68 | { 69 | dolfinx::common::Timer timer0("x1 alltoall + graph_create_adjacent"); 70 | MPI_Comm comm_dist_graph_adjacent; 71 | int err = MPI_Dist_graph_create_adjacent( 72 | comm, src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), 73 | dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, 74 | &comm_dist_graph_adjacent); 75 | dolfinx::MPI::check_error(comm, err); 76 | 77 | // Exchange number of indices to send/receive from each rank 78 | send_sizes.reserve(1); 79 | recv_sizes.reserve(1); 80 | err = MPI_Neighbor_alltoall(send_sizes.data(), 1, MPI_INT, 81 | recv_sizes.data(), 1, MPI_INT, 82 | comm_dist_graph_adjacent); 83 | dolfinx::MPI::check_error(comm, err); 84 | 85 | timer0.stop(); 86 | MPI_Comm_free(&comm_dist_graph_adjacent); 87 | } 88 | } 89 | 90 | // ------------------------------------------------------------ 91 | // Compute graph edges using non-blocking nbx and create a communicator 92 | // using MPI_Dist_graph_create_adjacent 93 | // ------------------------------------------------------------ 94 | MPI_Barrier(comm); 95 | { 96 | for (int i = 0; i < 100; i++) 97 | { 98 | dolfinx::common::Timer timer0("x2 NBX + graph_create_adjacent"); 99 | MPI_Comm comm_dist_graph_adjacent; 100 | const std::vector src 101 | = dolfinx::MPI::compute_graph_edges_nbx(comm, dest); 102 | int err = MPI_Dist_graph_create_adjacent( 103 | comm, src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), 104 | dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, 105 | &comm_dist_graph_adjacent); 106 | dolfinx::MPI::check_error(comm, err); 107 | timer0.stop(); 108 | MPI_Comm_free(&comm_dist_graph_adjacent); 109 | } 110 | } 111 | 112 | // ------------------------------------------------------------ 113 | // Create a communicator using MPI_Dist_graph_create 114 | // ------------------------------------------------------------ 115 | MPI_Barrier(comm); 116 | std::vector src{rank}; 117 | std::vector degrees{static_cast(dest.size())}; 118 | { 119 | for (int i = 0; i < 100; i++) 120 | { 121 | MPI_Comm comm_dist_graph; 122 | dolfinx::common::Timer timer1("x3 MPI_Dist_graph_create"); 123 | int err = MPI_Dist_graph_create( 124 | MPI_COMM_WORLD, src.size(), src.data(), degrees.data(), dest.data(), 125 | MPI_UNWEIGHTED, MPI_INFO_NULL, 0, &comm_dist_graph); 126 | dolfinx::MPI::check_error(comm, err); 127 | timer1.stop(); 128 | MPI_Comm_free(&comm_dist_graph); 129 | } 130 | } 131 | 132 | // ------------------------------------------------------------ 133 | // Check whether the two communicators are the same 134 | // ------------------------------------------------------------ 135 | { 136 | MPI_Comm comm_dist_graph; 137 | MPI_Comm comm_dist_graph_adjacent; 138 | { 139 | const std::vector src 140 | = dolfinx::MPI::compute_graph_edges_nbx(comm, dest); 141 | int err = MPI_Dist_graph_create_adjacent( 142 | comm, src.size(), src.data(), MPI_UNWEIGHTED, dest.size(), 143 | dest.data(), MPI_UNWEIGHTED, MPI_INFO_NULL, false, 144 | &comm_dist_graph_adjacent); 145 | dolfinx::MPI::check_error(comm, err); 146 | } 147 | { 148 | int err = MPI_Dist_graph_create( 149 | MPI_COMM_WORLD, src.size(), src.data(), degrees.data(), dest.data(), 150 | MPI_UNWEIGHTED, MPI_INFO_NULL, 0, &comm_dist_graph); 151 | dolfinx::MPI::check_error(comm, err); 152 | } 153 | 154 | // Check whether the two communicators are the same 155 | int same = 0; 156 | MPI_Comm_compare(comm_dist_graph, comm_dist_graph_adjacent, &same); 157 | if (same == MPI_UNEQUAL) 158 | { 159 | std::cout << "Communicators are not the same" << std::endl; 160 | return 1; 161 | } 162 | 163 | // get neighbors and compare 164 | int indegree, outdegree, weighted; 165 | MPI_Dist_graph_neighbors_count(comm_dist_graph, &indegree, &outdegree, 166 | &weighted); 167 | 168 | std::vector sources(indegree); 169 | std::vector sourceweights(indegree); 170 | std::vector destinations(outdegree); 171 | std::vector destweights(outdegree); 172 | MPI_Dist_graph_neighbors(comm_dist_graph, indegree, sources.data(), 173 | sourceweights.data(), outdegree, 174 | destinations.data(), destweights.data()); 175 | 176 | int indegree_adj, outdegree_adj, weighted_adj; 177 | MPI_Dist_graph_neighbors_count(comm_dist_graph_adjacent, &indegree_adj, 178 | &outdegree_adj, &weighted_adj); 179 | 180 | std::vector sources_adj(indegree_adj); 181 | std::vector sourceweights_adj(indegree_adj); 182 | std::vector destinations_adj(outdegree_adj); 183 | std::vector destweights_adj(outdegree_adj); 184 | MPI_Dist_graph_neighbors(comm_dist_graph_adjacent, indegree_adj, 185 | sources_adj.data(), sourceweights_adj.data(), 186 | outdegree_adj, destinations_adj.data(), 187 | destweights_adj.data()); 188 | 189 | if (indegree != indegree_adj) 190 | { 191 | std::cout << "indegree != indegree_adj" << std::endl; 192 | return 1; 193 | } 194 | 195 | if (outdegree != outdegree_adj) 196 | { 197 | std::cout << "outdegree != outdegree_adj" << std::endl; 198 | return 1; 199 | } 200 | 201 | std::sort(sources.begin(), sources.end()); 202 | std::sort(sources_adj.begin(), sources_adj.end()); 203 | if (sources != sources_adj) 204 | { 205 | std::cout << "sources != sources_adj" << std::endl; 206 | return 1; 207 | } 208 | // Free communicators 209 | MPI_Comm_free(&comm_dist_graph); 210 | MPI_Comm_free(&comm_dist_graph_adjacent); 211 | } 212 | 213 | dolfinx::list_timings(comm, {dolfinx::TimingType::wall}, 214 | dolfinx::Table::Reduction::max); 215 | } 216 | MPI_Finalize(); 217 | } --------------------------------------------------------------------------------