├── LICENSE.md ├── Makefile ├── README ├── coloring.cpp ├── coloring.hpp ├── compare.cpp ├── compare.hpp ├── converters ├── convert.cpp ├── dimacs.cpp ├── dimacs.hpp ├── matrix-market.cpp ├── matrix-market.hpp ├── metis.cpp ├── metis.hpp ├── shards.cpp ├── shards.hpp ├── simple.cpp ├── simple.hpp ├── simple2.cpp ├── snap.cpp └── snap.hpp ├── distgraph.cpp ├── distgraph.hpp ├── edge.hpp ├── generators ├── gen-io.cpp ├── nkit-ba.cpp ├── nkit-ba.hpp ├── nkit-crg.cpp ├── nkit-crg.hpp ├── nkit-er.cpp ├── nkit-er.hpp ├── nkit-hg.cpp └── nkit-hg.hpp ├── graph.hpp ├── locks.hpp ├── louvain.cpp ├── louvain.hpp ├── main.cpp ├── parallel-converters ├── parallel-converter.cpp ├── parallel-shards.cpp └── parallel-shards.hpp ├── rebuild.cpp ├── rebuild.hpp ├── utils.cpp └── utils.hpp /LICENSE.md: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, Battelle Memorial Institute 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # change to CC for Cray systems 2 | CXX = mpicxx 3 | 4 | OPTFLAGS = -g -O3 -fopenmp -DPRINT_DIST_STATS -DDONT_CREATE_DIAG_FILES #-DDEBUG_PRINTF -DCHECK_COLORING_CONFLICTS 5 | # use export ASAN_OPTIONS=verbosity=1 to check ASAN output 6 | SNTFLAGS = -std=c++11 -fopenmp -fsanitize=address -O1 -fno-omit-frame-pointer 7 | CXXFLAGS = -std=c++11 $(OPTFLAGS) -DABS_MOD_PER_ITER #-DUSE_MPI_COLLECTIVES #-DUSE_32_BIT_GRAPH #-DDEBUG_PRINTF 8 | 9 | ENABLE_HPCLINK=0 10 | ifeq ($(ENABLE_HPCLINK),1) 11 | HPCLINKER = /projects/Tools/hpctoolkit/pkgs-theta/hpctoolkit-2018-08-09/bin/hpclink 12 | # Static LLVM OpenMP 13 | OMPT= -Wl,-rpath=/projects/Tools/hpctoolkit/pkgs-theta/llvm-openmp/lib -L/projects/Tools/hpctoolkit/pkgs-theta/llvm-openmp/lib -lomp 14 | LDFLAGS = $(OMPT) -lm 15 | endif 16 | 17 | ENABLE_NETWORKIT=0 18 | ifeq ($(ENABLE_NETWORKIT),1) 19 | NETWORKIT_DIR = $(HOME)/builds/networkit 20 | CXXFLAGS += -std=c++14 -I$(NETWORKIT_DIR)/include 21 | NOBJFILES = generators/nkit-er.o generators/nkit-crg.o generators/nkit-ba.o generators/nkit-hg.o generators/gen-io.o 22 | NTARGET = $(BIN)/graphGenerator 23 | LDFLAGS = -L$(NETWORKIT_DIR)/lib -Wl,-rpath=$(NETWORKIT_DIR)/lib -lnetworkit -L$(NETWORKIT_DIR)/lib64 -Wl,-rpath=$(NETWORKIT_DIR)/lib -ltlx 24 | endif 25 | 26 | GOBJFILES = main.o rebuild.o distgraph.o louvain.o coloring.o compare.o 27 | FOBJFILES = converters/convert.o converters/matrix-market.o converters/dimacs.o converters/metis.o converters/simple2.o converters/simple.o converters/snap.o converters/shards.o utils.o 28 | POBJFILES = parallel-converters/parallel-converter.o parallel-converters/parallel-shards.o utils.o 29 | ALLOBJFILES = $(GOBJFILES) $(FOBJFILES) $(NOBJFILES) $(POBJFILES) 30 | 31 | BIN = bin 32 | 33 | GTARGET = $(BIN)/graphClustering 34 | FTARGET = $(BIN)/fileConvert 35 | PTARGET = $(BIN)/parallelFileConvert 36 | 37 | ALLTARGETS = $(GTARGET) $(FTARGET) $(NTARGET) $(PTARGET) 38 | 39 | all: bindir $(ALLTARGETS) 40 | 41 | bindir: $(BIN) 42 | 43 | $(BIN): 44 | mkdir -p $(BIN) 45 | 46 | %.o: %.cpp 47 | $(CXX) $(CXXFLAGS) -c -o $@ $^ 48 | 49 | $(GTARGET): $(GOBJFILES) 50 | $(CXX) $^ $(OPTFLAGS) -o $@ -lstdc++ 51 | 52 | $(FTARGET): $(FOBJFILES) 53 | $(CXX) $^ $(OPTFLAGS) -o $@ $(LDFLAGS) 54 | 55 | $(NTARGET): utils.o $(NOBJFILES) 56 | $(CXX) $^ $(OPTFLAGS) -o $@ $(LDFLAGS) 57 | 58 | $(PTARGET): $(POBJFILES) 59 | $(CXX) $^ $(OPTFLAGS) -o $@ $(LDFLAGS) 60 | 61 | .PHONY: bindir clean 62 | 63 | clean: 64 | rm -rf *~ $(ALLOBJFILES) $(ALLTARGETS) $(BIN) dat.out.* check.out.* 65 | -------------------------------------------------------------------------------- /coloring.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #ifndef __COLORING_H 51 | #define __COLORING_H 52 | 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | #include 60 | #include 61 | #include 62 | 63 | #include 64 | #include 65 | 66 | #include "graph.hpp" 67 | #include "distgraph.hpp" 68 | 69 | #ifndef MAX_COVG 70 | #define MAX_COVG (70) 71 | #endif 72 | 73 | 74 | #ifdef USE_32_BIT_GRAPH 75 | typedef int32_t ColorElem; 76 | #else 77 | typedef int64_t ColorElem; 78 | #endif 79 | 80 | const int ColoringSizeTag = 6; 81 | const int ColoringDataTag = 7; 82 | 83 | typedef std::unordered_set ColoredVertexSet; 84 | typedef std::vector ColorVector; 85 | 86 | ColorElem distColoringMultiHashMinMax(const int me, const int nprocs, const DistGraph &dg, ColorVector &vertexColor, const ColorElem nHash, const int target_percent, const bool singleIteration); 87 | 88 | static unsigned int hash(unsigned int a, unsigned int seed); 89 | 90 | void distColoringIteration(const int me, const DistGraph &dg, ColorVector &vertexColor, ColoredVertexSet &remoteColoredVertices, const ColorElem nHash, const ColorElem nextColor, const unsigned int seed); 91 | 92 | void setUpGhostVertices(const int me, const int nprocs, const DistGraph &dg, std::vector &ghostVertices, std::vector &ghostSizes); 93 | 94 | void sendColoredRemoteVertices(const int me, const int nprocs, const DistGraph &dg, ColoredVertexSet &remoteColoredVertices, const ColorVector &vertexColor, const std::vector &ghostVertices, const std::vector &ghostSizes); 95 | 96 | GraphElem countUnassigned(const ColorVector &vertexColor); 97 | 98 | GraphElem distCheckColoring(const int me, const int nprocs, const DistGraph &dg, const ColorVector &vertexColor, const std::vector &ghostVertices, const std::vector &ghostSizes); 99 | 100 | #endif 101 | -------------------------------------------------------------------------------- /compare.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #include "compare.hpp" 52 | 53 | extern std::ofstream ofs; 54 | 55 | //Only a single process performs the cluster comparisons 56 | //Assume that clusters have been numbered in a contiguous manner 57 | //Assume that C1 is the truth data 58 | void compare_communities(std::vector const& C1, 59 | std::vector const& C2) { 60 | 61 | // number of vertices of two sets must match 62 | const GraphElem N1 = C1.size(); 63 | const GraphElem N2 = C2.size(); 64 | 65 | assert(N1>0 && N2>0); 66 | assert(N1 == N2); 67 | 68 | int nT; 69 | #pragma omp parallel 70 | { 71 | nT = omp_get_num_threads(); 72 | } 73 | 74 | //Compute number of communities in each set: 75 | GraphElem nC1=-1; 76 | #pragma omp parallel for reduction(max: nC1) 77 | for(GraphElem i = 0; i < N1; i++) { 78 | if (C1[i] > nC1) { 79 | nC1 = C1[i]; 80 | } 81 | } 82 | 83 | nC1++; // nC1 contains index, so need to +1 84 | 85 | GraphElem nC2=-1; 86 | #pragma omp parallel for reduction(max: nC2) 87 | for(GraphElem i = 0; i < N2; i++) { 88 | if (C2[i] > nC2) { 89 | nC2 = C2[i]; 90 | } 91 | } 92 | 93 | nC2++; 94 | 95 | #if defined(DONT_CREATE_DIAG_FILES) 96 | std::cout << "Number of unique communities in C1 = " << nC1 << 97 | ", and C2 = " << nC2 << std::endl; 98 | #else 99 | ofs << "Number of unique communities in C1 = " << nC1 << 100 | ", and C2 = " << nC2 << std::endl; 101 | #endif 102 | 103 | //////////STEP 1: Create a CSR-like datastructure for communities in C1 104 | GraphElem* commPtr1 = new GraphElem[nC1+1]; 105 | GraphElem* commIndex1 = new GraphElem[N1]; 106 | GraphElem* commAdded1 = new GraphElem[nC1]; 107 | GraphElem* clusterDist1 = new GraphElem[nC1]; //For Gini coefficient 108 | 109 | // Initialization 110 | #pragma omp parallel for 111 | for(GraphElem i = 0; i < nC1; i++) { 112 | commPtr1[i] = 0; 113 | commAdded1[i] = 0; 114 | } 115 | commPtr1[nC1] = 0; 116 | 117 | // Count the size of each community 118 | #pragma omp parallel for 119 | for(GraphElem i = 0; i < N1; i++) { 120 | #pragma omp atomic update 121 | commPtr1[C1[i]+1]++; 122 | } 123 | 124 | #pragma omp parallel for 125 | for(GraphElem i = 0; i < nC1; i++) { 126 | clusterDist1[i] = commPtr1[i+1]; //Zeroth position is not valid 127 | } 128 | 129 | //Prefix sum: 130 | for(GraphElem i=0; i const& C1, std::vector const& C2); 57 | double compute_gini_coeff(GraphElem *colorSize, int numColors); 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /converters/convert.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #include 51 | #include 52 | #include 53 | 54 | #include 55 | #include 56 | #include 57 | 58 | #include 59 | #include 60 | #include 61 | #include 62 | 63 | #include "../graph.hpp" 64 | #include "../utils.hpp" 65 | 66 | #include "dimacs.hpp" 67 | #include "matrix-market.hpp" 68 | #include "metis.hpp" 69 | #include "simple.hpp" 70 | #include "snap.hpp" 71 | #include "shards.hpp" 72 | 73 | static std::string inputFileName, outputFileName, shardedFileArgs; 74 | 75 | static bool matrixMarketFormat = false; 76 | 77 | static bool dimacsFormat = false; 78 | static bool dimacs_directed = false; 79 | 80 | static bool metisFormat = false; 81 | static bool simpleFormat = false; 82 | static bool simpleFormat2 = false; 83 | static bool snapFormat = false; 84 | static bool shardedFormat = false; 85 | 86 | static bool output = false; 87 | static bool indexOneBased = false; 88 | 89 | // this option will override whatever 90 | // weights there are in the file, and 91 | // make weights 1.0 for ALL edges 92 | static bool makeWeightsOne = false; 93 | static bool randomEdgeWeight = false; 94 | static bool origEdgeWeight = false; 95 | 96 | static void parseCommandLine(const int argc, char * const argv[]); 97 | 98 | int main(int argc, char *argv[]) 99 | { 100 | parseCommandLine(argc, argv); 101 | 102 | // Only the following formats supported for now 103 | assert(dimacsFormat || metisFormat || simpleFormat || matrixMarketFormat|| simpleFormat2 || snapFormat || shardedFormat); 104 | 105 | Graph *g = NULL; 106 | 107 | rusage rus; 108 | 109 | getrusage(RUSAGE_SELF, &rus); 110 | std::cout << "Memory used: " << (static_cast(rus.ru_maxrss) * 1024.0) / 1048576.0 << 111 | std::endl; 112 | 113 | if (dimacsFormat) 114 | { 115 | if (dimacs_directed) { 116 | if (randomEdgeWeight) 117 | loadDimacsFile(g, inputFileName, RND_WEIGHT); 118 | else if (makeWeightsOne) 119 | loadDimacsFile(g, inputFileName, ONE_WEIGHT); 120 | else if (origEdgeWeight) 121 | loadDimacsFile(g, inputFileName, ORG_WEIGHT); 122 | else 123 | loadDimacsFile(g, inputFileName, ABS_WEIGHT); 124 | } 125 | else { 126 | if (randomEdgeWeight) 127 | loadDimacsFileUn(g, inputFileName, RND_WEIGHT); 128 | else if (makeWeightsOne) 129 | loadDimacsFileUn(g, inputFileName, ONE_WEIGHT); 130 | else if (origEdgeWeight) 131 | loadDimacsFileUn(g, inputFileName, ORG_WEIGHT); 132 | else 133 | loadDimacsFileUn(g, inputFileName, ABS_WEIGHT); 134 | } 135 | } 136 | else if (metisFormat) { 137 | if (randomEdgeWeight) 138 | loadMetisFile(g, inputFileName, RND_WEIGHT); 139 | else if (makeWeightsOne) 140 | loadMetisFile(g, inputFileName, ONE_WEIGHT); 141 | else if (origEdgeWeight) 142 | loadMetisFile(g, inputFileName, ORG_WEIGHT); 143 | else 144 | loadMetisFile(g, inputFileName, ABS_WEIGHT); 145 | } 146 | else if (simpleFormat) { 147 | if (randomEdgeWeight) 148 | loadSimpleFile(g, inputFileName, indexOneBased, RND_WEIGHT); 149 | else if (makeWeightsOne) 150 | loadSimpleFile(g, inputFileName, indexOneBased, ONE_WEIGHT); 151 | else if (origEdgeWeight) 152 | loadSimpleFile(g, inputFileName, indexOneBased, ORG_WEIGHT); 153 | else 154 | loadSimpleFile(g, inputFileName, indexOneBased, ABS_WEIGHT); 155 | } 156 | else if (matrixMarketFormat) { 157 | if (randomEdgeWeight) 158 | loadMatrixMarketFile(g, inputFileName, RND_WEIGHT); 159 | else if (makeWeightsOne) 160 | loadMatrixMarketFile(g, inputFileName, ONE_WEIGHT); 161 | else if (origEdgeWeight) 162 | loadMatrixMarketFile(g, inputFileName, ORG_WEIGHT); 163 | else 164 | loadMatrixMarketFile(g, inputFileName, ABS_WEIGHT); 165 | } 166 | else if (simpleFormat2) { 167 | if (randomEdgeWeight) 168 | loadSimpleFileUn(g, inputFileName, RND_WEIGHT); 169 | else if (makeWeightsOne) 170 | loadSimpleFileUn(g, inputFileName, ONE_WEIGHT); 171 | else if (origEdgeWeight) 172 | loadSimpleFileUn(g, inputFileName, ORG_WEIGHT); 173 | else 174 | loadSimpleFileUn(g, inputFileName, ABS_WEIGHT); 175 | } 176 | else if (snapFormat) { 177 | // For SNAP format files, weights are not read 178 | // from the file, only source/target 179 | if (randomEdgeWeight) 180 | loadSNAPFile(g, inputFileName, RND_WEIGHT); 181 | //else if (makeWeightsOne) 182 | // loadSNAPFile(g, inputFileName, ONE_WEIGHT); 183 | //else if (origEdgeWeight) 184 | // loadSNAPFile(g, inputFileName, ORG_WEIGHT); 185 | else 186 | loadSNAPFile(g, inputFileName, ONE_WEIGHT); 187 | } 188 | else if (shardedFormat) { 189 | // fetch arguments, expecting 3 190 | std::stringstream ss(shardedFileArgs); 191 | std::string s; 192 | std::vector args; 193 | 194 | while (std::getline(ss, s, ' ')) { 195 | args.push_back(s); 196 | } 197 | if (args.size() != 3) { 198 | std::cerr << "For sharded files, expecting (in this order): " 199 | << std::endl; 200 | exit(EXIT_FAILURE); 201 | } 202 | 203 | GraphElem startChunk = std::stoi(args[0]); 204 | GraphElem endChunk = std::stoi(args[1]); 205 | GraphElem shardCount = std::stoi(args[2]); 206 | 207 | args.clear(); 208 | 209 | if (randomEdgeWeight) 210 | loadFileShards(g, inputFileName, startChunk, endChunk, indexOneBased, RND_WEIGHT, shardCount); 211 | else if (makeWeightsOne) 212 | loadFileShards(g, inputFileName, startChunk, endChunk, indexOneBased, ONE_WEIGHT, shardCount); 213 | else if (origEdgeWeight) 214 | loadFileShards(g, inputFileName, startChunk, endChunk, indexOneBased, ORG_WEIGHT, shardCount); 215 | else 216 | loadFileShards(g, inputFileName, startChunk, endChunk, indexOneBased, ABS_WEIGHT, shardCount); 217 | } 218 | else { 219 | std::cerr << "Format not specified correctly!" << std::endl; 220 | if (g) 221 | delete g; 222 | return 1; 223 | } 224 | 225 | getrusage(RUSAGE_SELF, &rus); 226 | std::cout << "Loaded graph from file: " << inputFileName << std::endl; 227 | std::cout << "Memory used: " << 228 | (static_cast(rus.ru_maxrss) * 1024.0) / 1048576.0 229 | << std::endl; 230 | 231 | double t0, t1; 232 | 233 | t0 = mytimer(); 234 | 235 | std::ofstream ofs(outputFileName.c_str(), std::ofstream::out | std::ofstream::binary | 236 | std::ofstream::trunc); 237 | if (!ofs) { 238 | std::cerr << "Error opening output file: " << outputFileName << std::endl; 239 | exit(EXIT_FAILURE); 240 | } 241 | 242 | GraphElem nv, ne; 243 | 244 | nv = g->getNumVertices(); 245 | ne = g->getNumEdges(); 246 | 247 | ofs.write(reinterpret_cast(&nv), sizeof(GraphElem)); 248 | ofs.write(reinterpret_cast(&ne), sizeof(GraphElem)); 249 | 250 | ofs.write(reinterpret_cast(&g->edgeListIndexes[0]), (nv+1)*sizeof(GraphElem)); 251 | 252 | for (GraphElem v = 0; v < nv; v++) { 253 | GraphElem e0, e1; 254 | 255 | g->getEdgeRangeForVertex(v, e0, e1); 256 | 257 | for (GraphElem j = e0; j < e1; j++) { 258 | const Edge &edge = g->getEdge(j); 259 | ofs.write(reinterpret_cast(&edge), sizeof(Edge)); 260 | } 261 | } 262 | 263 | ofs.close(); 264 | delete g; 265 | 266 | t1 = mytimer(); 267 | 268 | std::cout << "Time writing binary file: " << (t1 - t0) << std::endl; 269 | 270 | return 0; 271 | } // main 272 | 273 | void parseCommandLine(const int argc, char * const argv[]) 274 | { 275 | int ret; 276 | 277 | while ((ret = getopt(argc, argv, "f:o:md:uesnrix:zw")) != -1) { 278 | switch (ret) { 279 | case 'f': 280 | inputFileName.assign(optarg); 281 | break; 282 | case 'o': 283 | outputFileName.assign(optarg); 284 | break; 285 | case 'm': 286 | matrixMarketFormat = true; 287 | break; 288 | case 'd': 289 | dimacsFormat = true; 290 | dimacs_directed = (atoi(optarg) == 1 ? true : false); 291 | break; 292 | case 'e': 293 | metisFormat = true; 294 | break; 295 | case 's': 296 | simpleFormat = true; 297 | break; 298 | case 'n': 299 | snapFormat = true; 300 | break; 301 | case 'u': 302 | simpleFormat2 = true; 303 | break; 304 | case 'r': 305 | randomEdgeWeight = true; 306 | break; 307 | case 'i': 308 | origEdgeWeight = true; 309 | break; 310 | case 'x': 311 | shardedFormat = true; 312 | shardedFileArgs.assign(optarg); 313 | break; 314 | case 'z': 315 | indexOneBased = true; 316 | break; 317 | case 'w': 318 | makeWeightsOne = true; 319 | break; 320 | default: 321 | assert(0 && "Should not reach here!!"); 322 | break; 323 | } 324 | } 325 | 326 | if ((matrixMarketFormat || dimacsFormat || metisFormat || simpleFormat 327 | || simpleFormat2 || snapFormat || shardedFormat) == false) { 328 | std::cerr << "Must select a file format for the input file!" << std::endl; 329 | exit(EXIT_FAILURE); 330 | } 331 | const bool fileFormat[] = { matrixMarketFormat, dimacsFormat, metisFormat, 332 | simpleFormat, simpleFormat2, snapFormat, shardedFormat}; 333 | const int numFormats = sizeof(fileFormat) / sizeof(fileFormat[0]); 334 | 335 | int numTrue = 0; 336 | for (int i = 0; i < numFormats; i++) 337 | if (numTrue == 0 && fileFormat[i]) 338 | numTrue++; 339 | else if (numTrue > 0 && fileFormat[i]) { 340 | std::cerr << "Must select a SINGLE file format for the input file!" << std::endl; 341 | exit(EXIT_FAILURE); 342 | } 343 | 344 | if (inputFileName.empty()) { 345 | std::cerr << "Must specify an input file name with -f" << std::endl; 346 | exit(EXIT_FAILURE); 347 | } 348 | 349 | if (outputFileName.empty()) { 350 | std::cerr << "Must specify an output file name with -o" << std::endl; 351 | exit(EXIT_FAILURE); 352 | } 353 | } // parseCommandLine 354 | -------------------------------------------------------------------------------- /converters/dimacs.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #include 51 | 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | 58 | #include "dimacs.hpp" 59 | 60 | /* Assuming `a u v w` format for both directed/undirected */ 61 | 62 | // reading directed files, edges stored twice 63 | void loadDimacsFile(Graph *&g, const std::string &fileName, Weight_t wtype) 64 | { 65 | std::ifstream ifs; 66 | 67 | ifs.open(fileName.c_str()); 68 | if (!ifs) { 69 | std::cerr << "Error opening Dimacs format file: " << fileName << std::endl; 70 | exit(EXIT_FAILURE); 71 | } 72 | 73 | std::string line; 74 | char comment; 75 | 76 | do { 77 | std::getline(ifs, line); 78 | comment = line[0]; 79 | } while (comment == 'c'); 80 | 81 | GraphElem numVertices, numEdges, value = 0; 82 | { 83 | std::string p, sp; 84 | 85 | std::istringstream iss(line); 86 | 87 | iss >> p >> sp >> numVertices >> numEdges; 88 | if (!iss || iss.fail() || p[0] != 'p') { 89 | std::cerr << "Error parsing DIMACS format: " << line << std::endl; 90 | std::cerr << "p: " << p << ", p[0]: " << p[0] << ", sp: " << sp << 91 | ", numVertices: " << numVertices << ", numEdges: " << numEdges << std::endl; 92 | exit(EXIT_FAILURE); 93 | } 94 | } 95 | 96 | std::cout << "Loading (undirected) DIMACS file: " << fileName << ", initial numvertices: " << numVertices << 97 | ", numEdges: " << numEdges << std::endl; 98 | 99 | double t0 = mytimer(); 100 | 101 | std::vector edgeCount(numVertices + 1); 102 | std::set edgeSets; 103 | 104 | // populate edge set 105 | for (GraphElem i = 0; i < numEdges; i++) { 106 | 107 | std::string label; 108 | GraphElem source, dest; 109 | GraphWeight weight; 110 | 111 | ifs >> label >> source >> dest >> weight; 112 | 113 | if (wtype == ONE_WEIGHT) 114 | weight = 1.0; 115 | 116 | if (wtype == RND_WEIGHT) 117 | weight = genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 118 | 119 | if (wtype == ABS_WEIGHT) 120 | weight = std::fabs(weight); 121 | 122 | if (!ifs || ifs.fail()) { 123 | std::cerr << "Error parsing DIMACS edge" << std::endl; 124 | exit(EXIT_FAILURE); 125 | } 126 | 127 | edgeSets.insert({source-1, dest-1, weight}); 128 | edgeSets.insert({dest-1, source-1, weight}); 129 | } 130 | 131 | ifs.close(); 132 | 133 | double t1 = mytimer(); 134 | std::cout << "Time taken to read the file: " << (t1-t0) << " secs." << std::endl; 135 | 136 | for (auto it = edgeSets.begin(); it != edgeSets.end(); ++it) 137 | edgeCount[it->i_+1]++; 138 | 139 | std::vector edgeList(edgeSets.begin(), edgeSets.end()); 140 | 141 | // adjust numEdges and edgeList size 142 | numEdges = edgeSets.size(); 143 | std::cout << "Adjusted numEdges after removing duplicates: " << numEdges << std::endl; 144 | 145 | // create graph data structure 146 | g = new Graph(numVertices, numEdges); 147 | processGraphData(*g, edgeCount, edgeList, numVertices, numEdges); 148 | } // loadDimacsFile 149 | 150 | // reading undirected files 151 | void loadDimacsFileUn(Graph *&g, const std::string &fileName, Weight_t wtype) 152 | { 153 | std::ifstream ifs; 154 | 155 | ifs.open(fileName.c_str()); 156 | if (!ifs) { 157 | std::cerr << "Error opening Dimacs format file: " << fileName << std::endl; 158 | exit(EXIT_FAILURE); 159 | } 160 | 161 | std::string line; 162 | char comment; 163 | 164 | do { 165 | std::getline(ifs, line); 166 | comment = line[0]; 167 | } while (comment == 'c'); 168 | 169 | GraphElem numVertices, numEdges, value = 0; 170 | { 171 | std::string p, sp; 172 | 173 | std::istringstream iss(line); 174 | 175 | iss >> p >> sp >> numVertices >> numEdges; 176 | if (!iss || iss.fail() || p[0] != 'p') { 177 | std::cerr << "Error parsing DIMACS format: " << line << std::endl; 178 | std::cerr << "p: " << p << ", p[0]: " << p[0] << ", sp: " << sp << 179 | ", numVertices: " << numVertices << ", numEdges: " << numEdges << std::endl; 180 | exit(EXIT_FAILURE); 181 | } 182 | } 183 | 184 | std::cout << "Loading (undirected) DIMACS file: " << fileName << ", initial numvertices: " << numVertices << 185 | ", numEdges: " << numEdges << std::endl; 186 | 187 | double t0 = mytimer(); 188 | 189 | std::vector edgeCount(numVertices + 1); 190 | std::set edgeSets; 191 | 192 | // populate edge set 193 | for (GraphElem i = 0; i < numEdges; i++) { 194 | std::string label; 195 | GraphElem source, dest; 196 | GraphWeight weight; 197 | 198 | ifs >> label >> source >> dest >> weight; 199 | 200 | if (!ifs || ifs.fail()) { 201 | std::cerr << "Error parsing DIMACS edge" << std::endl; 202 | exit(EXIT_FAILURE); 203 | } 204 | 205 | if (wtype == ONE_WEIGHT) 206 | weight = 1.0; 207 | 208 | if (wtype == RND_WEIGHT) 209 | weight = genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 210 | 211 | if (wtype == ABS_WEIGHT) 212 | weight = std::fabs(weight); 213 | 214 | edgeSets.insert({source-1, dest-1, weight}); 215 | } 216 | 217 | ifs.close(); 218 | 219 | double t1 = mytimer(); 220 | std::cout << "Time taken to read the file: " << (t1-t0) << " secs." << std::endl; 221 | 222 | for (auto it = edgeSets.begin(); it != edgeSets.end(); ++it) 223 | edgeCount[it->i_+1]++; 224 | 225 | std::vector edgeList(edgeSets.begin(), edgeSets.end()); 226 | 227 | // adjust numEdges and edgeList size 228 | numEdges = edgeSets.size(); 229 | std::cout << "Adjusted numEdges after removing duplicates: " << numEdges << std::endl; 230 | 231 | // create graph data structure 232 | g = new Graph(numVertices, numEdges); 233 | processGraphData(*g, edgeCount, edgeList, numVertices, numEdges); 234 | } // loadDimacsFileUn 235 | -------------------------------------------------------------------------------- /converters/dimacs.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #ifndef __DIMACS_H 51 | #define __DIMACS_H 52 | 53 | #include 54 | 55 | #include "../utils.hpp" 56 | #include "../graph.hpp" 57 | 58 | void loadDimacsFileUn(Graph *&g, const std::string &fileName, Weight_t wtype = ABS_WEIGHT); 59 | void loadDimacsFile(Graph *&g, const std::string &fileName, Weight_t wtype = ABS_WEIGHT); 60 | 61 | #endif // __DIMACS_H 62 | -------------------------------------------------------------------------------- /converters/matrix-market.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #include 51 | #include 52 | #include 53 | 54 | #include 55 | #include 56 | #include 57 | #include 58 | 59 | #include "matrix-market.hpp" 60 | 61 | /// Note: Trying out the new random number generator with static seed generator, 62 | // instead of first generating the seed (based on input filename) and then passing 63 | // it to the random number generator... 64 | 65 | void loadMatrixMarketFile(Graph *&g, const std::string &fileName, Weight_t wtype) 66 | { 67 | std::ifstream ifs; 68 | 69 | ifs.open(fileName.c_str()); 70 | if (!ifs) { 71 | std::cerr << "Error opening Matrix Market format file: " << fileName << std::endl; 72 | exit(EXIT_FAILURE); 73 | } 74 | 75 | std::string line; 76 | bool isComplex = false, isPattern = false, isSymmetric = false, isGeneral = false; 77 | 78 | std::getline(ifs, line); 79 | { 80 | std::string ls[5]; 81 | 82 | std::istringstream iss(line); 83 | iss >> ls[0] >> ls[1] >> ls[2] >> ls[3] >> ls[4]; 84 | 85 | if (!iss) { 86 | std::cerr << "Error parsing matrix market file: " << fileName << std::endl; 87 | exit(EXIT_FAILURE); 88 | } 89 | 90 | std::cout << "Processing: " << ls[0] << " " << ls[1] << " " << ls[2] << " " << ls[3] << 91 | " " << ls[4] << std::endl; 92 | for (int i = 1; i < 5; i++) 93 | std::transform(ls[i].begin(), ls[i].end(), ls[i].begin(), toupper); 94 | 95 | if (ls[0] != "%%MatrixMarket") { 96 | std::cerr << "Error in the first line of: " << fileName << " " << ls[0] << 97 | ", expected: %%MatrixMarket" << std::endl; 98 | exit(EXIT_FAILURE); 99 | } 100 | 101 | if (ls[1] != "MATRIX") { 102 | std::cerr << "The object type should be matrix for file: " << fileName << std::endl; 103 | exit(EXIT_FAILURE); 104 | } 105 | 106 | if (ls[2] != "COORDINATE") { 107 | std::cerr << "The object type should be coordinate for file: " << fileName << std::endl; 108 | exit(EXIT_FAILURE); 109 | } 110 | 111 | if (ls[3] == "COMPLEX") { 112 | std::cout << "Warning will only process the real part" << std::endl; 113 | isComplex = true; 114 | } 115 | 116 | if (ls[3] == "PATTERN") { 117 | std::cout << "Note: matrix type is pattern. All weights will be set to 1.0 unless -r is passed" << 118 | std::endl; 119 | isPattern = true; 120 | } 121 | 122 | if (ls[4] == "GENERAL") 123 | isGeneral = true; 124 | else if (ls[4] == "SYMMETRIC") { 125 | std::cout << "Note: matrix type is symmetric" << std::endl; 126 | isSymmetric = true; 127 | } 128 | } 129 | 130 | if (!isSymmetric && !isGeneral) { 131 | std::cerr << "Error: matrix market type should be SYMMETRIC or GENERAL for file: " << fileName << 132 | std::endl; 133 | exit(EXIT_FAILURE); 134 | } 135 | 136 | do { 137 | std::getline(ifs, line); 138 | } while (line[0] == '%'); 139 | 140 | GraphElem numVertices, numEdges; 141 | GraphElem ns, nt, ne; 142 | 143 | std::istringstream iss(line); 144 | 145 | iss >> ns >> nt >> ne; 146 | if (!iss || iss.fail()) { 147 | std::cerr << "Error parsing Matrix Market format: " << line << std::endl; 148 | exit(EXIT_FAILURE); 149 | } 150 | 151 | numVertices = ns; 152 | numEdges = ne; 153 | 154 | std::cout << "Loading Matrix Market file: " << fileName << ", numvertices: " << numVertices << 155 | ", numEdges: " << numEdges << std::endl; 156 | 157 | std::string crd; 158 | GraphElem source, dest; 159 | GraphWeight weight = 1.0; 160 | std::vector edgeCount(numVertices+1); 161 | std::vector edgeList; 162 | 163 | // weights will be converted to positive numbers 164 | if (isSymmetric) { 165 | 166 | for (GraphElem i = 0; i < numEdges; i++) { 167 | 168 | std::getline(ifs, crd); 169 | std::istringstream iss(crd); 170 | 171 | if (isPattern) 172 | iss >> source >> dest; 173 | else { 174 | iss >> source >> dest >> weight; 175 | if (wtype == ABS_WEIGHT) 176 | weight = std::fabs(weight); 177 | } 178 | 179 | source--; // Matrix market has 1-based indexing 180 | dest--; 181 | 182 | assert((source >= 0) && (source < numVertices)); 183 | assert((dest >= 0) && (dest < numVertices)); 184 | 185 | if (wtype == ONE_WEIGHT) 186 | weight = 1.0; 187 | 188 | if (wtype == RND_WEIGHT) 189 | weight = genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 190 | 191 | if (source != dest) { 192 | edgeList.emplace_back(source, dest, weight); 193 | edgeList.emplace_back(dest, source, weight); 194 | edgeCount[source+1]++; 195 | edgeCount[dest+1]++; 196 | } 197 | else { 198 | edgeList.emplace_back(source, dest, weight); 199 | edgeCount[source+1]++; 200 | } 201 | } 202 | } 203 | else { // if General type (directed) 204 | 205 | for (GraphElem i = 0; i < numEdges; i++) { 206 | 207 | std::getline(ifs, crd); 208 | std::istringstream iss(crd); 209 | 210 | if (isPattern) 211 | iss >> source >> dest; 212 | else { 213 | iss >> source >> dest >> weight; 214 | if (wtype == ABS_WEIGHT) 215 | weight = std::fabs(weight); 216 | } 217 | 218 | source--; // Matrix market has 1-based indexing 219 | dest--; 220 | 221 | assert((source >= 0) && (source < numVertices)); 222 | assert((dest >= 0) && (dest < numVertices)); 223 | 224 | if (wtype == ONE_WEIGHT) 225 | weight = 1.0; 226 | 227 | if (wtype == RND_WEIGHT) 228 | weight = genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 229 | 230 | edgeList.emplace_back(source, dest, weight); 231 | edgeCount[source+1]++; 232 | } 233 | } 234 | 235 | ifs.close(); 236 | 237 | numEdges = edgeList.size(); 238 | 239 | g = new Graph(numVertices, numEdges); 240 | processGraphData(*g, edgeCount, edgeList, numVertices, numEdges); 241 | } // loadMatrixMarketFile 242 | -------------------------------------------------------------------------------- /converters/matrix-market.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov), 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov), 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #ifndef __MATRIX_MARKET_H 51 | #define __MATRIX_MARKET_H 52 | 53 | #include 54 | 55 | #include "../utils.hpp" 56 | #include "../graph.hpp" 57 | 58 | void loadMatrixMarketFile(Graph *&g, const std::string &fileName, Weight_t wtype = ABS_WEIGHT); 59 | 60 | #endif // __MATRIX_MARKET_H 61 | -------------------------------------------------------------------------------- /converters/metis.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #include 51 | #include 52 | #include 53 | 54 | #include "metis.hpp" 55 | 56 | // This function will most likely be purged from future versions of Vite 57 | 58 | void loadMetisFile(Graph *&g, const std::string &fileName, Weight_t wtype) 59 | { 60 | std::ifstream ifs; 61 | 62 | ifs.open(fileName.c_str()); 63 | if (!ifs) { 64 | std::cerr << "Error opening Metis format file: " << fileName << std::endl; 65 | exit(EXIT_FAILURE); 66 | } 67 | 68 | std::string line; 69 | char comment; 70 | 71 | do { 72 | std::getline(ifs, line); 73 | comment = line[0]; 74 | } while (comment == '%'); 75 | 76 | GraphElem numVertices, numEdges, value = 0L; 77 | { 78 | std::istringstream iss(line); 79 | 80 | iss >> numVertices >> numEdges; 81 | if (!iss.eof()) 82 | iss >> value; 83 | } 84 | 85 | numEdges *= 2; // Metis' edges are not directional 86 | 87 | std::cout << "Loading Metis file: " << fileName << ", numvertices: " << numVertices << 88 | ", numEdges: " << numEdges << std::endl; 89 | 90 | g = new Graph(numVertices, numEdges); 91 | 92 | GraphElem edgePos = 0; 93 | 94 | g->setEdgeStartForVertex(0, 0); 95 | 96 | switch (value) { 97 | case 10: 98 | std::cout << "Metis format vertex weights ignored" << std::endl; 99 | case 0: // No weights in Metis file 100 | for (GraphElem i = 0; i < numVertices; i++) { 101 | GraphElem j = 0L, neighbor, oldNeighbor=-1; 102 | 103 | std::getline(ifs, line); 104 | std::istringstream iss(line); 105 | 106 | while (!iss.eof()) { 107 | iss >> neighbor; 108 | if (!iss || iss.eof()) 109 | // if((oldNeighbor == neighbor)) 110 | break; 111 | GraphWeight weight; 112 | oldNeighbor = neighbor; 113 | j++; 114 | Edge &edge = g->getEdge(edgePos); 115 | edge.tail = neighbor - 1; 116 | 117 | if (wtype == ONE_WEIGHT) 118 | weight = 1.0; 119 | 120 | if (wtype == RND_WEIGHT) 121 | weight = genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 122 | 123 | edge.weight = weight; 124 | edgePos++; 125 | } 126 | g->setEdgeStartForVertex(i + 1L, edgePos); 127 | } 128 | break; 129 | case 11: 130 | std::cout << "Metis format vertex weights ignored" << std::endl; 131 | case 1: 132 | for (GraphElem i = 0; i < numVertices; i++) { 133 | GraphElem j = 0, neighbor; 134 | GraphWeight weight; 135 | 136 | std::getline(ifs, line); 137 | std::istringstream iss(line); 138 | 139 | while (!iss.eof()) { 140 | iss >> neighbor >> weight; 141 | if (!iss || iss.eof()) 142 | break; 143 | 144 | if (wtype == ONE_WEIGHT) 145 | weight = 1.0; 146 | 147 | if (wtype == RND_WEIGHT) 148 | weight = genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 149 | 150 | if (wtype == ABS_WEIGHT) 151 | weight = std::fabs(weight); 152 | 153 | j++; 154 | Edge &edge = g->getEdge(edgePos); 155 | edge.tail = neighbor - 1; 156 | edge.weight = weight; 157 | edgePos++; 158 | } 159 | 160 | g->setEdgeStartForVertex(i + 1, edgePos); 161 | } 162 | break; 163 | default: 164 | std::cerr << "Inconsistent value for weight flag in Metis format: " << value << std::endl; 165 | exit(EXIT_FAILURE); 166 | } 167 | 168 | ifs.close(); 169 | } // loadMetisFile 170 | -------------------------------------------------------------------------------- /converters/metis.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #ifndef __METIS_H 51 | #define __METIS_H 52 | 53 | #include 54 | 55 | #include "../utils.hpp" 56 | #include "../graph.hpp" 57 | 58 | // This function will most likely be purged from future versions of Vite 59 | 60 | void loadMetisFile(Graph *&g, const std::string &fileName, Weight_t wtype = ABS_WEIGHT); 61 | 62 | #endif // __METIS_H 63 | -------------------------------------------------------------------------------- /converters/shards.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | 59 | #include "shards.hpp" 60 | 61 | /// Pairwise read files between fileStartIndex and fileEndIndex and store 62 | /// it into a single file, also selectively convert to binary 63 | 64 | /// fileInShardsPath expects the input directory where the shards are ('/' is not allowed at the end) 65 | 66 | /// the input file is expected to contain 4 data points in every line: 67 | /// ----ai, aj, common, jaccard---- (common is not processed at present) 68 | 69 | /// We expect the shards to contain the 'upper triangle' of the adjacency...during binary 70 | /// conversion, we consider the data to be an undirected graph, and store both combinations of edge pairs 71 | 72 | void loadFileShards(Graph *&g, const std::string &fileInShardsPath, 73 | const int fileStartIndex, const int fileEndIndex, bool indexOneBased, 74 | Weight_t wtype, GraphElem shardCount) 75 | { 76 | double t0, t1; 77 | t0 = mytimer(); 78 | 79 | assert(fileStartIndex >= 0); 80 | assert(fileEndIndex >= 0); 81 | assert(fileEndIndex >= fileStartIndex); 82 | 83 | GraphElem numFiles = 0, numEdges = 0, numVertices, v_idx = 0, maxVertex = -1; 84 | std::vector edgeList; 85 | 86 | /// Part 1: Read the file shards 87 | // start reading 88 | for (GraphElem ci = fileStartIndex; ci < fileEndIndex + 1; ci++) { 89 | for (GraphElem cj = fileStartIndex; cj < fileEndIndex + 1; cj++) { 90 | 91 | // construct file name for each shard 92 | std::string fileNameShard = std::to_string(ci) + "__" + std::to_string(cj) + ".csv"; 93 | std::string fileName = fileInShardsPath + "/" + fileNameShard; 94 | 95 | // open file shard 96 | std::ifstream ifs; 97 | ifs.open(fileName.c_str(), std::ifstream::in); 98 | 99 | if (ifs.fail()) 100 | continue; 101 | 102 | // actual chunk lo/hi range 103 | GraphElem v_lo = (ci - 1)*shardCount; 104 | GraphElem v_hi = (cj - 1)*shardCount; 105 | std::cout << "File processing: " << fileNameShard << "; Ranges: " << v_lo << ", " << v_hi << std::endl; 106 | numFiles++; 107 | 108 | // start reading shard 109 | std::string line, dummyline; 110 | std::getline(ifs, dummyline); 111 | 112 | while(std::getline(ifs, line)) { 113 | 114 | GraphElem v0, v1, info; 115 | GraphWeight w; 116 | char ch; 117 | 118 | std::istringstream iss(line); 119 | 120 | // read from current shard 121 | if (wtype == ORG_WEIGHT || wtype == ABS_WEIGHT) 122 | iss >> v0 >> ch >> v1 >> ch >> info >> ch >> w; 123 | else 124 | iss >> v0 >> ch >> v1 >> ch >> info; 125 | 126 | if (indexOneBased) { 127 | v0--; 128 | v1--; 129 | } 130 | 131 | // normalize v0/v1 by adding lo/hi shard ID 132 | v0 += v_lo; 133 | v1 += v_hi; 134 | 135 | if (v0 > maxVertex) 136 | maxVertex = v0; 137 | if (v1 > maxVertex) 138 | maxVertex = v1; 139 | 140 | if (wtype == ONE_WEIGHT) 141 | w = 1.0; 142 | 143 | if (wtype == RND_WEIGHT) 144 | w = (GraphWeight)genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 145 | 146 | if (wtype == ABS_WEIGHT) 147 | w = std::fabs(w); 148 | 149 | if (v0 != v1) { 150 | edgeList.emplace_back(v0, v1, w); 151 | edgeList.emplace_back(v1, v0, w); 152 | } 153 | else 154 | edgeList.emplace_back(v0, v1, w); 155 | } 156 | 157 | // close current shard 158 | ifs.close(); 159 | } 160 | } 161 | 162 | numEdges = edgeList.size(); 163 | numVertices = maxVertex + 1; 164 | double t2 = mytimer(); 165 | 166 | std::cout << "Read " << numFiles << " file shards (undirected data)." << std::endl; 167 | std::cout << "Unadjusted #vertices: " << numVertices << " and #edges: " << numEdges << std::endl; 168 | std::cout << "Time taken for serial file I/O: " << (t2 - t0) << " secs." << std::endl; 169 | 170 | /// Part 1.5: remap vertex IDs 171 | /// edgelist is unsorted, so two loops needed 172 | std::vector< GraphElem > vertexMap(numVertices, -1); 173 | for (GraphElem i = 0; i < numEdges; i++) { 174 | 175 | if (vertexMap[edgeList[i].i_] == -1) { 176 | vertexMap[edgeList[i].i_] = 1; 177 | } 178 | if (vertexMap[edgeList[i].j_] == -1) { 179 | vertexMap[edgeList[i].j_] = 1; 180 | } 181 | } 182 | 183 | for (GraphElem i = 0; i < numVertices; i++) { 184 | 185 | if (vertexMap[i] == 1) { 186 | vertexMap[i] = v_idx; 187 | v_idx += 1; 188 | } 189 | } 190 | 191 | #ifdef PRINT_VERTEX_MAP 192 | std::string outFileName = std::to_string(numVertices); 193 | outFileName += ".mappings"; 194 | 195 | std::ofstream ofs; 196 | ofs.open(outFileName.c_str(), std::ios::out); 197 | 198 | if (!ofs) { 199 | std::cerr << "Error creating community file: " << outFileName << std::endl; 200 | exit(EXIT_FAILURE); 201 | } 202 | 203 | // write the data 204 | for (GraphElem i = 0; i < numVertices; i++) { 205 | if (vertexMap[i] != -1) 206 | ofs << i << " " << vertexMap[i] << "\n"; 207 | } 208 | 209 | ofs.close(); 210 | 211 | std::cout << std::endl; 212 | std::cout << "-----------------------------------------------------" << std::endl; 213 | std::cout << "Saved vertex mapping information on file: " << outFileName << std::endl; 214 | std::cout << "-----------------------------------------------------" << std::endl; 215 | #endif 216 | 217 | /// Part 2: perform edge counts and generate the binary file 218 | 219 | numVertices = v_idx; 220 | std::cout << "Updated #vertices, after renumbering the indices consecutively to handle gaps: " << numVertices << std::endl; 221 | 222 | std::vector< GraphElem > edgeCount(numVertices + 1, 0); 223 | for (GraphElem i = 0; i < numEdges; i++) { 224 | 225 | edgeList[i].i_ = vertexMap[edgeList[i].i_]; 226 | edgeList[i].j_ = vertexMap[edgeList[i].j_]; 227 | 228 | assert(edgeList[i].i_ >= 0 && edgeList[i].i_ < numVertices); 229 | assert(edgeList[i].j_ >= 0 && edgeList[i].j_ < numVertices); 230 | 231 | edgeCount[edgeList[i].i_+1]++; 232 | //edgeCount[edgeList[i].j_+1]++; 233 | } 234 | std::cout << "Stored graph data as edgelist and counted number of edges per vertex for graph CSR preparation." << std::endl; 235 | g = new Graph(numVertices, numEdges); 236 | processGraphData(*g, edgeCount, edgeList, numVertices, numEdges); 237 | 238 | t1 = mytimer(); 239 | 240 | std::cout << "Total time (processing shards + binary generation): " << (t1 - t2) << " secs." << std::endl; 241 | } // loadFileShards 242 | -------------------------------------------------------------------------------- /converters/shards.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #ifndef __LOAD_SHARDS_H 52 | #define __LOAD_SHARDS_H 53 | 54 | #include 55 | 56 | #include "../utils.hpp" 57 | #include "../graph.hpp" 58 | 59 | /// Pairwise read files between fileStartIndex and fileEndIndex and store 60 | /// it into a single file, also convert to binary 61 | void loadFileShards(Graph *&g, const std::string &fileInShardsPath, 62 | const int fileStartIndex, const int fileEndIndex, bool indexOneBased, 63 | Weight_t wtype = ORG_WEIGHT, GraphElem shardCount = 1000000); 64 | 65 | #endif // __LOAD_SHARDS_H 66 | -------------------------------------------------------------------------------- /converters/simple.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | 58 | #include "simple.hpp" 59 | 60 | /// Assuming a file with just an edge list (directed) 61 | void loadSimpleFile(Graph *&g, const std::string &fileName, 62 | bool indexOneBased, Weight_t wtype) 63 | { 64 | std::ifstream ifs; 65 | 66 | double t0, t1; 67 | t0 = mytimer(); 68 | 69 | ifs.open(fileName.c_str(), std::ifstream::in); 70 | if (!ifs) { 71 | std::cerr << "Error opening Simple format file: " << fileName << std::endl; 72 | exit(EXIT_FAILURE); 73 | } 74 | 75 | std::string line; 76 | GraphElem maxVertex = -1, numEdges = 0, numVertices, skipLines = 0; 77 | 78 | do { 79 | GraphElem v0, v1; 80 | GraphWeight w = 0.0; 81 | 82 | std::getline(ifs, line); 83 | if(line[0] == '#' || line[0] == '%') { 84 | skipLines++; 85 | continue; 86 | } 87 | 88 | std::istringstream iss(line); 89 | if (wtype == ORG_WEIGHT || wtype == ABS_WEIGHT) 90 | iss >> v0 >> v1 >> w; 91 | else 92 | iss >> v0 >> v1; 93 | 94 | if (indexOneBased) { 95 | v0--; 96 | v1--; 97 | } 98 | 99 | if (v0 > maxVertex) 100 | maxVertex = v0; 101 | if (v1 > maxVertex) 102 | maxVertex = v1; 103 | 104 | numEdges++; 105 | 106 | } while (!ifs.eof()); 107 | 108 | numVertices = maxVertex + 1; 109 | numEdges -= 1; // last line 110 | 111 | std::cout << "Loading simple format file (directed edge-list): " 112 | << fileName << ", numvertices: " << numVertices << std::endl; 113 | 114 | ifs.close(); 115 | 116 | // read the data 117 | ifs.open(fileName.c_str(), std::ifstream::in); 118 | 119 | std::vector edgeCount(numVertices + 1); 120 | std::vector edgeList; 121 | 122 | for (GraphElem i = 0; i < numEdges+skipLines; i++) { 123 | 124 | GraphElem v0, v1; 125 | GraphWeight w = 0.0; 126 | 127 | std::getline(ifs, line); 128 | 129 | if(line[0] == '#' || line[0] == '%') 130 | continue; 131 | 132 | std::istringstream iss(line); 133 | if (wtype == ORG_WEIGHT || wtype == ABS_WEIGHT) 134 | iss >> v0 >> v1 >> w; 135 | else 136 | iss >> v0 >> v1; 137 | 138 | if (indexOneBased) { 139 | v0--; 140 | v1--; 141 | } 142 | 143 | if (wtype == ONE_WEIGHT) 144 | w = 1.0; 145 | 146 | if (wtype == RND_WEIGHT) 147 | w = genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 148 | 149 | if (wtype == ABS_WEIGHT) 150 | w = std::fabs(w); 151 | 152 | edgeList.push_back({v0, v1, w}); 153 | edgeList.push_back({v1, v0, w}); 154 | edgeCount[v0+1]++; 155 | edgeCount[v1+1]++; 156 | } 157 | 158 | numEdges = edgeList.size(); 159 | 160 | ifs.close(); 161 | std::cout << "Number of edges: " << numEdges << std::endl; 162 | 163 | g = new Graph(numVertices, numEdges); 164 | processGraphData(*g, edgeCount, edgeList, numVertices, numEdges); 165 | 166 | t1 = mytimer(); 167 | 168 | std::cout << "Total graph processing time: " << (t1 - t0) << std::endl; 169 | } // loadSimpleFile 170 | -------------------------------------------------------------------------------- /converters/simple.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #ifndef __LOAD_SIMPLE_H 52 | #define __LOAD_SIMPLE_H 53 | 54 | #include 55 | 56 | #include "../utils.hpp" 57 | #include "../graph.hpp" 58 | 59 | void loadSimpleFile(Graph *&g, const std::string &fileName, bool indexOneBased, Weight_t wtype = ABS_WEIGHT); 60 | void loadSimpleFileUn(Graph *&g, const std::string &fileName, bool indexOneBased, Weight_t wtype = ABS_WEIGHT); 61 | 62 | #endif // __LOAD_SIMPLE_H 63 | -------------------------------------------------------------------------------- /converters/simple2.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | 59 | #include "simple.hpp" 60 | 61 | void loadSimpleFileUn(Graph *&g, const std::string &fileName, bool indexOneBased, Weight_t wtype) 62 | { 63 | std::ifstream ifs; 64 | 65 | double t0, t1; 66 | 67 | t0 = mytimer(); 68 | 69 | ifs.open(fileName.c_str(), std::ifstream::in); 70 | if (!ifs) { 71 | std::cerr << "Error opening Simple format file: " << fileName << std::endl; 72 | exit(EXIT_FAILURE); 73 | } 74 | 75 | std::string line; 76 | GraphElem maxVertex = -1, numEdges = 0, numVertices, skipLines = 0; 77 | 78 | do { 79 | GraphElem v0, v1; 80 | GraphWeight w = 0.0; 81 | 82 | std::getline(ifs, line); 83 | 84 | if(line[0]=='#' || line[0]=='%') { 85 | skipLines++; 86 | continue; 87 | } 88 | 89 | std::istringstream iss(line); 90 | if (wtype == ORG_WEIGHT || wtype == ABS_WEIGHT) 91 | iss >> v0 >> v1 >> w; 92 | else 93 | iss >> v0 >> v1; 94 | 95 | if (indexOneBased) { 96 | v0--; 97 | v1--; 98 | } 99 | 100 | if (v0 > maxVertex) 101 | maxVertex = v0; 102 | if (v1 > maxVertex) 103 | maxVertex = v1; 104 | 105 | numEdges++; 106 | 107 | } while (!ifs.eof()); 108 | 109 | numVertices = maxVertex + 1; 110 | numEdges -= 1; // last line 111 | 112 | std::cout << "Loading Simple file: " << fileName << ", numvertices: " << numVertices << std::endl; 113 | 114 | ifs.close(); 115 | 116 | // file bounds known, start reading data 117 | ifs.open(fileName.c_str(), std::ifstream::in); 118 | 119 | std::vector edgeCount(numVertices + 1); 120 | std::vector edgeList; 121 | 122 | for (GraphElem i = 0; i < numEdges+skipLines; i++) { 123 | GraphElem v0, v1; 124 | GraphWeight w = 0.0; 125 | 126 | std::getline(ifs, line); 127 | if(line[0]=='#' || line[0]=='%') 128 | continue; 129 | 130 | std::istringstream iss(line); 131 | 132 | if (wtype == ORG_WEIGHT || wtype == ABS_WEIGHT) 133 | iss >> v0 >> v1 >> w; 134 | else 135 | iss >> v0 >> v1; 136 | 137 | if (indexOneBased) { 138 | v0--; 139 | v1--; 140 | } 141 | 142 | if (wtype == ONE_WEIGHT) 143 | w = 1.0; 144 | 145 | if (wtype == RND_WEIGHT) 146 | w = genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 147 | 148 | if (wtype == ABS_WEIGHT) 149 | w = std::fabs(w); 150 | 151 | edgeList.push_back({v0, v1, w}); 152 | edgeCount[v0+1]++; 153 | } 154 | 155 | ifs.close(); 156 | 157 | numEdges = edgeList.size(); 158 | g = new Graph(numVertices, numEdges); 159 | processGraphData(*g, edgeCount, edgeList, numVertices, numEdges); 160 | 161 | t1 = mytimer(); 162 | std::cout << "Total graph processing time: " << (t1 - t0) << std::endl; 163 | } // loadSimpleFile 164 | -------------------------------------------------------------------------------- /converters/snap.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | #include 60 | 61 | #include "snap.hpp" 62 | 63 | // Nodes have explicit (and arbitrary) node ids. There is no restriction for node 64 | // ids to be contiguous integers starting at 0. In TUNGraph and TNGraph edges have no 65 | // explicit ids -- edges are identified by a pair node ids. 66 | void loadSNAPFile(Graph *&g, const std::string &fileName, Weight_t wtype) 67 | { 68 | std::ifstream ifs; 69 | 70 | double t0, t1, t2, t3; 71 | 72 | t0 = mytimer(); 73 | 74 | ifs.open(fileName.c_str(), std::ifstream::in); 75 | if (!ifs) { 76 | std::cerr << "Error opening SNAP format file: " << fileName << std::endl; 77 | exit(EXIT_FAILURE); 78 | } 79 | 80 | std::string line; 81 | GraphElem numEdges = 0, numVertices; 82 | 83 | //Parse the comment lines for problem size 84 | size_t place = 0; 85 | while(1) { 86 | std::getline(ifs, line); 87 | //Check if this line has problem sizes 88 | if (line[0] == '#') { 89 | std::size_t found_nodes = line.find("Nodes"); 90 | if (found_nodes != std::string::npos) { // found our line 91 | std::istringstream iss(line); 92 | std::string key, val; 93 | int c = 0; 94 | while(std::getline(iss, key, ':') >> val) { 95 | // only interested in parsing nodes/edges 96 | if (c == 0) 97 | numVertices = std::stol(val); 98 | else 99 | numEdges = std::stol(val); 100 | c++; 101 | } 102 | } 103 | place = ifs.tellg(); 104 | } 105 | else { 106 | ifs.seekg(place); // move back 107 | break; 108 | } 109 | } 110 | 111 | t1 = mytimer(); 112 | 113 | std::cout << "Loading SNAP file: " << fileName << ", numvertices: " << numVertices << 114 | ", numEdges: " << numEdges << std::endl; 115 | std::cout << "Time taken to open file and parse number of vertices/edges: " 116 | << (t1 - t0) << std::endl; 117 | 118 | // start parsing the data in file 119 | t2 = mytimer(); 120 | 121 | std::vector edgeCount(numVertices+1); 122 | std::vector edgeList(numEdges*2); // each edge stored twice 123 | 124 | // complete reading the file 125 | // compute the node renumbering by pushing nodeid on 126 | // a map and updating a temp edgeList 127 | std::map clusterLocalMap; //Renumber vertices contiguously from zero 128 | std::map::iterator storedAlready; 129 | GraphElem numUniqueVertices = 0; 130 | 131 | // update numEdges, as the input graph is 132 | // assumed to be undirected and weights are 1 133 | GraphElem nnz = 0; 134 | for (GraphElem i = 0L; i < numEdges; i++) { 135 | GraphWeight weight; 136 | GraphElem v0, v1; 137 | ifs >> v0 >> v1; 138 | 139 | if (!ifs || ifs.fail()) { 140 | std::cerr << "Error parsing SNAP edge" << std::endl; 141 | exit(EXIT_FAILURE); 142 | } 143 | 144 | // for node_0 145 | storedAlready = clusterLocalMap.find(v0); //Check if it already exists 146 | if( storedAlready != clusterLocalMap.end() ) { //Already exists 147 | v0 = storedAlready->second; //Renumber the cluster id 148 | } else { 149 | clusterLocalMap[v0] = numUniqueVertices; //Does not exist, add to the map 150 | v0 = numUniqueVertices; //Renumber the vertex id 151 | numUniqueVertices++; //Increment the number 152 | } 153 | 154 | // for node_1 155 | storedAlready = clusterLocalMap.find(v1); //Check if it already exists 156 | if( storedAlready != clusterLocalMap.end() ) { //Already exists 157 | v1 = storedAlready->second; //Renumber the cluster id 158 | } else { 159 | clusterLocalMap[v1] = numUniqueVertices; //Does not exist, add to the map 160 | v1 = numUniqueVertices; //Renumber the vertex id 161 | numUniqueVertices++; //Increment the number 162 | } 163 | 164 | if (wtype == RND_WEIGHT) 165 | weight = genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 166 | 167 | if (wtype == ONE_WEIGHT) 168 | weight = 1.0; 169 | 170 | edgeList.emplace_back(v0, v1, weight); 171 | edgeList.emplace_back(v1, v0, weight); 172 | 173 | // edge counts 174 | edgeCount[v0 + 1]++; 175 | edgeCount[v1 + 1]++; 176 | nnz += 2; 177 | } 178 | 179 | ifs.close(); 180 | 181 | assert((numEdges * 2) == nnz); 182 | 183 | t3 = mytimer(); 184 | std::cout << "Time taken to allocate edgeList/edgeCount and read from file: " 185 | << (t3 - t2) << std::endl; 186 | 187 | numEdges = nnz; 188 | 189 | t2 = mytimer(); 190 | g = new Graph(numVertices, numEdges); 191 | processGraphData(*g, edgeCount, edgeList, numVertices, numEdges); 192 | 193 | t3 = mytimer(); 194 | 195 | std::cout << "Graph populate time: " << (t3 - t2) << std::endl; 196 | std::cout << "Total I/O time: " << (t3 - t0) << std::endl; 197 | } // loadSNAPFile 198 | -------------------------------------------------------------------------------- /converters/snap.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #ifndef __SNAP_H 52 | #define __SNAP_H 53 | 54 | #include 55 | 56 | #include "../utils.hpp" 57 | #include "../graph.hpp" 58 | 59 | void loadSNAPFile(Graph *&g, const std::string &fileName, Weight_t wtype = ONE_WEIGHT); 60 | 61 | #endif // __SNAP_H 62 | -------------------------------------------------------------------------------- /distgraph.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #ifndef __DISTGRAPH_H 52 | #define __DISTGRAPH_H 53 | /// 54 | #include 55 | #include 56 | 57 | #include 58 | 59 | #include "graph.hpp" 60 | 61 | // TODO FIXME purge this entire class 62 | // and just have one graph class like 63 | // miniVite 64 | 65 | #define PI (3.14159) 66 | #define SR_UP_TAG 100 67 | #define SR_DOWN_TAG 101 68 | #define SR_SIZES_UP_TAG 102 69 | #define SR_SIZES_DOWN_TAG 103 70 | #define SR_X_UP_TAG 104 71 | #define SR_X_DOWN_TAG 105 72 | #define SR_Y_UP_TAG 106 73 | #define SR_Y_DOWN_TAG 107 74 | 75 | typedef std::vector PartRanges; 76 | 77 | class DistGraph { 78 | protected: 79 | GraphElem totalNumVertices; 80 | GraphElem totalNumEdges; 81 | Graph *localGraph; 82 | 83 | public: 84 | DistGraph(const GraphElem tnv, const GraphElem tne); 85 | DistGraph(const DistGraph &othis); 86 | ~DistGraph(); 87 | 88 | GraphElem getTotalNumVertices() const; 89 | GraphElem getTotalNumEdges() const; 90 | 91 | void createLocalGraph(const GraphElem lnv, const GraphElem lne, 92 | const PartRanges *oparts = NULL); 93 | Graph &getLocalGraph(); 94 | const Graph &getLocalGraph() const; 95 | GraphElem getBase(const int me) const; 96 | GraphElem getBound(const int me) const; 97 | GraphElem localToGlobal(GraphElem idx, const int me) const; 98 | GraphElem globalToLocal(GraphElem idx, const int me) const; 99 | int getOwner(const GraphElem v) const; 100 | PartRanges *parts; 101 | void setNumEdges(GraphElem numEdges); 102 | void printStats(); 103 | GraphElem getNumGhosts(const int me) const; 104 | protected: 105 | DistGraph(); 106 | DistGraph &operator = (const DistGraph &othis); 107 | }; 108 | 109 | void balanceEdges(int nprocs, std::string& fileName, std::vector& mbins); 110 | void loadDistGraphMPIIO(int me, int nprocs, int ranks_per_node, 111 | DistGraph *&dg, std::string& fileName); 112 | void loadDistGraphMPIIOBalanced(int me, int nprocs, int ranks_per_node, 113 | DistGraph *&dg, std::string& fileName); 114 | 115 | // graph generation 116 | void generateInMemGraph(int rank, int nprocs, DistGraph *&dg, GraphElem nv, GraphWeight randomEdgePercent, std::string fileOut); 117 | DistGraph* generateRGG(int rank, int nprocs, GraphElem nv, GraphWeight rn, GraphWeight randomEdgePercent, std::string fileOut); 118 | 119 | void writeGraph(int me, int nprocs, DistGraph *&dg, std::vector& edgeCount, std::string &fileName); 120 | 121 | inline DistGraph::DistGraph() 122 | : totalNumVertices(0), totalNumEdges(0), localGraph(NULL), parts(NULL) 123 | { 124 | } // DistGraph 125 | 126 | inline DistGraph::DistGraph(const GraphElem tnv, const GraphElem tne) 127 | : totalNumVertices(tnv), totalNumEdges(tne), localGraph(NULL), parts(NULL) 128 | { 129 | } // DistGraph 130 | 131 | inline DistGraph::DistGraph(const DistGraph &othis) 132 | : totalNumVertices(othis.totalNumVertices), totalNumEdges(othis.totalNumEdges), 133 | localGraph(new Graph(*othis.localGraph)), parts(NULL) 134 | { parts = new PartRanges(*othis.parts); } // DistGraph 135 | 136 | inline DistGraph::~DistGraph() 137 | { 138 | if (localGraph) 139 | delete localGraph; 140 | delete parts; 141 | } // ~DistGraph 142 | 143 | inline GraphElem DistGraph::getTotalNumVertices() const 144 | { return totalNumVertices; } // getTotalNumVertices 145 | 146 | inline GraphElem DistGraph::getTotalNumEdges() const 147 | { return totalNumEdges; } // getTotalNumEdges 148 | 149 | // print statistics about edge distribution 150 | inline void DistGraph::printStats() 151 | { 152 | int me, size; 153 | MPI_Comm_size(MPI_COMM_WORLD, &size); 154 | MPI_Comm_rank(MPI_COMM_WORLD, &me); 155 | 156 | Graph &g = this->getLocalGraph(); // local graph 157 | long lne = (long)g.getNumEdges(); // local #edges 158 | long nv = (long)this->getTotalNumVertices(); // global #vertices 159 | // TODO FIXME currently totalNumEdges variable stores local 160 | // number of edges and not total, keep a separate variable 161 | //const GraphElem ne = this->getTotalNumEdges(); // global #edges 162 | long ne = 0; 163 | MPI_Allreduce(&lne, &ne, 1, MPI_LONG, MPI_SUM, MPI_COMM_WORLD); 164 | 165 | long sumdeg = 0, maxdeg = 0, mindeg = 0; 166 | MPI_Reduce(&lne, &sumdeg, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD); 167 | MPI_Reduce(&lne, &maxdeg, 1, MPI_LONG, MPI_MAX, 0, MPI_COMM_WORLD); 168 | MPI_Reduce(&lne, &mindeg, 1, MPI_LONG, MPI_MIN, 0, MPI_COMM_WORLD); 169 | 170 | long my_sq = lne*lne; 171 | long sum_sq = 0; 172 | MPI_Reduce(&my_sq, &sum_sq, 1, MPI_LONG, MPI_SUM, 0, MPI_COMM_WORLD); 173 | 174 | double average = (double) sumdeg / size; 175 | double avg_sq = (double) sum_sq / size; 176 | double var = avg_sq - (average*average); 177 | double stddev = sqrt(var); 178 | 179 | MPI_Barrier(MPI_COMM_WORLD); 180 | 181 | if (me == 0) 182 | { 183 | std::cout << std::endl; 184 | std::cout << "-------------------------------------------------------" << std::endl; 185 | std::cout << "Graph edge distribution characteristics" << std::endl; 186 | std::cout << "-------------------------------------------------------" << std::endl; 187 | std::cout << "Number of vertices: " << nv << std::endl; 188 | std::cout << "Number of edges: " << ne << std::endl; 189 | std::cout << "Maximum number of edges: " << maxdeg << std::endl; 190 | std::cout << "Average number of edges: " << average << std::endl; 191 | std::cout << "Minimum number of edges: " << mindeg << std::endl; 192 | std::cout << "Expected value of X^2: " << avg_sq << std::endl; 193 | std::cout << "Variance: " << var << std::endl; 194 | std::cout << "Standard deviation: " << stddev << std::endl; 195 | std::cout << "-------------------------------------------------------" << std::endl; 196 | 197 | } 198 | } 199 | 200 | inline void DistGraph::createLocalGraph(const GraphElem lnv, const GraphElem lne, 201 | const PartRanges *oparts) 202 | { 203 | #ifdef DEBUG_PRINTF 204 | assert(!localGraph); 205 | #endif 206 | 207 | localGraph = new Graph(lnv, lne); 208 | parts = new PartRanges(*oparts); 209 | } // createLocalGraph 210 | 211 | inline Graph &DistGraph::getLocalGraph() 212 | { 213 | #ifdef DEBUG_PRINTF 214 | assert(localGraph); 215 | #endif 216 | 217 | return *localGraph; 218 | } // getLocalGraph 219 | 220 | inline const Graph &DistGraph::getLocalGraph() const 221 | { 222 | #ifdef DEBUG_PRINTF 223 | assert(localGraph); 224 | #endif 225 | 226 | return *localGraph; 227 | } // getLocalGraph 228 | 229 | inline GraphElem DistGraph::getBase(const int me) const 230 | { 231 | #ifdef DEBUG_PRINTF 232 | return parts->at(me); 233 | #else 234 | return parts->operator[](me); 235 | #endif 236 | } // getBase 237 | 238 | inline GraphElem DistGraph::getBound(const int me) const 239 | { 240 | #ifdef DEBUG_PRINTF 241 | return parts->at(me + 1); 242 | #else 243 | return parts->operator[](me + 1); 244 | #endif 245 | } // getBound 246 | 247 | inline GraphElem DistGraph::localToGlobal(GraphElem idx, const int me) const 248 | { 249 | return (idx + getBase(me)); 250 | } // localToGlobal 251 | 252 | inline GraphElem DistGraph::globalToLocal(GraphElem idx, const int me) const 253 | { 254 | return (idx - getBase(me)); 255 | } // globalToLocal 256 | 257 | inline void DistGraph::setNumEdges(const GraphElem numEdges) 258 | { this->totalNumEdges=numEdges; } 259 | 260 | inline int DistGraph::getOwner(const GraphElem v) const 261 | { 262 | #ifdef DEBUG_PRINTF 263 | assert((v >= 0) && (v < totalNumVertices)); 264 | #endif 265 | const PartRanges::const_iterator iter = std::upper_bound(parts->begin(), parts->end(), v); 266 | 267 | #ifdef DEBUG_PRINTF 268 | assert(iter != parts->end()); 269 | #endif 270 | return (iter - parts->begin() -1); 271 | } // getOwner 272 | 273 | inline GraphElem DistGraph::getNumGhosts(const int me) const 274 | { 275 | const Graph &g = this->getLocalGraph(); 276 | GraphElem numGhosts = 0; 277 | for (GraphElem i = 0; i < g.getNumVertices(); i++) { 278 | const GraphElem lb = g.edgeListIndexes[i], ub = g.edgeListIndexes[i + 1]; 279 | for (GraphElem j = lb; j < ub; j++) { 280 | const Edge &edge = g.getEdge(j); 281 | if (this->getOwner(edge.tail) != me) 282 | numGhosts += 1; 283 | } 284 | } 285 | return numGhosts; 286 | } // return number of ghost vertices 287 | #endif // __DISTGRAPH_H 288 | -------------------------------------------------------------------------------- /edge.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 12 | // Hao Lu (luhowardmark@wsu.edu) 13 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #ifndef __EDGE_H 52 | #define __EDGE_H 53 | 54 | #include 55 | 56 | #include 57 | 58 | #include 59 | 60 | #ifdef USE_32_BIT_GRAPH 61 | typedef int32_t GraphElem; 62 | typedef float GraphWeight; 63 | const MPI_Datatype MPI_GRAPH_TYPE = MPI_INT32_T; 64 | const MPI_Datatype MPI_WEIGHT_TYPE = MPI_FLOAT; 65 | #else 66 | typedef int64_t GraphElem; 67 | typedef double GraphWeight; 68 | const MPI_Datatype MPI_GRAPH_TYPE = MPI_INT64_T; 69 | const MPI_Datatype MPI_WEIGHT_TYPE = MPI_DOUBLE; 70 | #endif 71 | 72 | struct Edge { 73 | GraphElem tail; 74 | GraphWeight weight; 75 | 76 | Edge(); 77 | Edge(GraphElem t, GraphWeight w): 78 | tail(t), weight(w) 79 | {} 80 | }; 81 | 82 | struct EdgeTuple 83 | { 84 | GraphElem ij_[2]; 85 | GraphWeight w_; 86 | 87 | EdgeTuple(GraphElem i, GraphElem j, GraphWeight w): 88 | ij_{i, j}, w_(w) 89 | {} 90 | EdgeTuple(GraphElem i, GraphElem j): 91 | ij_{i, j}, w_(1.0) 92 | {} 93 | EdgeTuple(): 94 | ij_{-1, -1}, w_(0.0) 95 | {} 96 | }; 97 | 98 | typedef std::vector EdgeIndexes; 99 | 100 | inline Edge::Edge() 101 | : tail(-1), weight(0.0) 102 | { 103 | } // Edge 104 | 105 | #endif // __EDGE_DECL_H 106 | -------------------------------------------------------------------------------- /generators/gen-io.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #include 51 | #include 52 | #include 53 | 54 | #include 55 | #include 56 | #include 57 | 58 | #include 59 | #include 60 | #include 61 | #include 62 | 63 | #include "../graph.hpp" 64 | #include "../utils.hpp" 65 | 66 | #include "nkit-er.hpp" 67 | #include "nkit-crg.hpp" 68 | #include "nkit-ba.hpp" 69 | #include "nkit-hg.hpp" 70 | 71 | static std::string outputFileName; 72 | 73 | // G = (N,p) for ER 74 | static unsigned int N = 0; 75 | static double p = 0.0; // probability 76 | // G = (N,k) for RGG 77 | static unsigned int k = 0; // nclusters 78 | static unsigned int m0 = 0; // initial attached vertices 79 | 80 | static bool erGen = false; // ER 81 | static bool crgGen = false; // RGG 82 | static bool baGen = false; // Barabasi-Albert 83 | static bool hgGen = false; // Hyperbolic 84 | 85 | // create random weights (can be used for matching) 86 | static bool randomEdgeWeight = false; 87 | 88 | static void parseCommandLine(const int argc, char * const argv[]); 89 | 90 | int main(int argc, char *argv[]) 91 | { 92 | parseCommandLine(argc, argv); 93 | 94 | // Only the following generators supported for now 95 | assert(erGen || crgGen || baGen || hgGen); 96 | 97 | Graph *g = NULL; 98 | rusage rus; 99 | 100 | if (erGen) 101 | generateER(g, N, p, randomEdgeWeight); 102 | else if (crgGen) 103 | generateCRG(g, N, k, randomEdgeWeight); 104 | else if (baGen) 105 | generateBA(g, N, m0, k, randomEdgeWeight); 106 | else if (hgGen) 107 | generateHG(g, N, randomEdgeWeight); 108 | else { 109 | std::cerr << "Generator not specified correctly!" << std::endl; 110 | if (g) 111 | delete g; 112 | return 1; 113 | } 114 | 115 | getrusage(RUSAGE_SELF, &rus); 116 | std::cout << "Generated file of size " << N << " x " << N << std::endl; 117 | std::cout << "Memory used: " << (static_cast(rus.ru_maxrss) * 1024.0) / 1048576.0 << 118 | std::endl; 119 | 120 | double t0, t1; 121 | 122 | t0 = mytimer(); 123 | 124 | std::ofstream ofs(outputFileName.c_str(), std::ofstream::out | std::ofstream::binary | 125 | std::ofstream::trunc); 126 | if (!ofs) { 127 | std::cerr << "Error opening output file: " << outputFileName << std::endl; 128 | exit(EXIT_FAILURE); 129 | } 130 | 131 | GraphElem nv, ne; 132 | 133 | nv = g->getNumVertices(); 134 | ne = g->getNumEdges(); 135 | 136 | ofs.write(reinterpret_cast(&nv), sizeof(GraphElem)); 137 | ofs.write(reinterpret_cast(&ne), sizeof(GraphElem)); 138 | 139 | ofs.write(reinterpret_cast(&g->edgeListIndexes[0]), (nv+1)*sizeof(GraphElem)); 140 | 141 | for (GraphElem v = 0; v < nv; v++) { 142 | GraphElem e0, e1; 143 | 144 | g->getEdgeRangeForVertex(v, e0, e1); 145 | 146 | for (GraphElem j = e0; j < e1; j++) { 147 | const Edge &edge = g->getEdge(j); 148 | 149 | ofs.write(reinterpret_cast(&edge), sizeof(Edge)); 150 | } 151 | } 152 | 153 | ofs.close(); 154 | delete g; 155 | 156 | t1 = mytimer(); 157 | 158 | std::cout << "Time writing binary file: " << (t1 - t0) << std::endl; 159 | 160 | return 0; 161 | } // main 162 | 163 | void parseCommandLine(const int argc, char * const argv[]) 164 | { 165 | int ret; 166 | 167 | while ((ret = getopt(argc, argv, "o:en:p:bhcm:k:r")) != -1) { 168 | switch (ret) { 169 | case 'o': 170 | outputFileName.assign(optarg); 171 | break; 172 | case 'e': 173 | erGen = true; 174 | break; 175 | case 'c': 176 | crgGen = true; 177 | break; 178 | case 'b': 179 | baGen = true; 180 | break; 181 | case 'h': 182 | hgGen = true; 183 | break; 184 | case 'n': 185 | N = atoi(optarg); 186 | break; 187 | case 'p': 188 | p = atof(optarg); 189 | break; 190 | case 'k': 191 | k = atoi(optarg); 192 | break; 193 | case 'r': 194 | randomEdgeWeight = true; 195 | break; 196 | case 'm': 197 | m0 = atoi(optarg); 198 | break; 199 | default: 200 | assert(0 && "Incorrect parameter!!"); 201 | break; 202 | } 203 | } 204 | 205 | if ((erGen || crgGen || baGen || hgGen) == false) { 206 | std::cerr << "Must select a generator for the input file!" << std::endl; 207 | exit(EXIT_FAILURE); 208 | } 209 | 210 | if (outputFileName.empty()) { 211 | std::cerr << "Must specify an output file name with -o" << std::endl; 212 | exit(EXIT_FAILURE); 213 | } 214 | } // parseCommandLine 215 | -------------------------------------------------------------------------------- /generators/nkit-ba.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | 60 | #include "nkit-er.hpp" 61 | #include "../utils.hpp" 62 | 63 | #include "networkit/Globals.hpp" 64 | #include "networkit/generators/BarabasiAlbertGenerator.hpp" 65 | 66 | void generateBA(Graph *&g, unsigned int Nmax, unsigned int m0, unsigned int ki, bool randomEdgeWeight=false) 67 | { 68 | double t1, t2; 69 | 70 | if (!Nmax || !ki) 71 | { 72 | std::cerr << "For Barabasi-Albert graph, Nmax and ki must be positive!" << std::endl; 73 | exit(EXIT_FAILURE); 74 | } 75 | 76 | NetworKit::Graph G(0); 77 | NetworKit::BarabasiAlbertGenerator BarabasiAlbert(ki, Nmax, m0, false); 78 | G = BarabasiAlbert.generate(); 79 | 80 | // expected edges: (m0-1) + ((Nmax - m0) * ki) 81 | 82 | /* Generate binary file containing edgelist */ 83 | GraphElem numEdges = G.numberOfEdges(); 84 | GraphElem numVertices = G.numberOfNodes(); 85 | 86 | std::vector edgeCount(numVertices + 1); 87 | std::vector edgeList; 88 | 89 | std::cout << "Generating Barbasi-Albert graph of numvertices: " << numVertices << 90 | ", numEdges: " << numEdges << std::endl; 91 | 92 | GraphWeight weight = 1.0; 93 | 94 | G.forEdges([&](NetworKit::node u, NetworKit::node v) { 95 | assert((u >= 0) && (u < numVertices)); 96 | assert((v >= 0) && (v < numVertices)); 97 | 98 | if (randomEdgeWeight) 99 | weight = (GraphWeight)genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 100 | 101 | edgeList.emplace_back(u, v, weight); 102 | edgeList.emplace_back(v, u, weight); 103 | 104 | edgeCount[u+1]++; 105 | edgeCount[v+1]++; 106 | }); 107 | 108 | numEdges *= 2; 109 | g = new Graph(numVertices, numEdges); 110 | 111 | processGraphData(*g, edgeCount, edgeList, numVertices, numEdges); 112 | t2 = mytimer(); 113 | 114 | std::cout << "Total time to generate Barabasi-Albert graph with Nmax = " << Nmax << 115 | ", m0 = " << m0 << " and ki = " << ki << " : " << (t2-t1) << " secs\n"; 116 | } 117 | -------------------------------------------------------------------------------- /generators/nkit-ba.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #ifndef __NETWORKIT_BA_H 51 | #define __NETWORKIT_BA_H 52 | 53 | #include "../graph.hpp" 54 | 55 | void generateBA(Graph *&g, unsigned int Nmax, unsigned int m0, unsigned int ki, bool randomEdgeWeight=false); 56 | 57 | #endif // __NETWORKIT_BA_H 58 | 59 | -------------------------------------------------------------------------------- /generators/nkit-crg.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | 57 | #include "nkit-crg.hpp" 58 | #include "../utils.hpp" 59 | 60 | #include "networkit/Globals.hpp" 61 | 62 | #include "networkit/generators/ClusteredRandomGraphGenerator.hpp" 63 | 64 | void generateCRG(Graph *&g, unsigned int N, unsigned int k, bool randomEdgeWeight) 65 | { 66 | if (!N || !k) 67 | { 68 | std::cerr << "For RGG graph, N and k must be specified!" << std::endl; 69 | exit(EXIT_FAILURE); 70 | } 71 | 72 | double t1, t2; 73 | 74 | t1 = mytimer(); 75 | 76 | NetworKit::ClusteredRandomGraphGenerator graphGen(N, k, 0.1, 0.005); 77 | NetworKit::Graph G = graphGen.generate(); 78 | 79 | /* Generate binary file containing edgelist */ 80 | GraphElem numEdges = G.numberOfEdges(); 81 | GraphElem numVertices = G.numberOfNodes(); 82 | 83 | std::vector edgeCount(numVertices + 1); 84 | std::vector edgeList; 85 | 86 | std::cout << "Generating RGG graph of numvertices: " << numVertices << 87 | ", numEdges: " << numEdges << std::endl; 88 | 89 | GraphWeight weight = 1.0; 90 | 91 | G.forEdges([&](NetworKit::node u, NetworKit::node v) { 92 | assert((u >= 0) && (u < numVertices)); 93 | assert((v >= 0) && (v < numVertices)); 94 | 95 | if (randomEdgeWeight) 96 | weight = (GraphWeight)genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 97 | 98 | edgeList.emplace_back(u, v, weight); 99 | edgeList.emplace_back(v, u, weight); 100 | 101 | edgeCount[u+1]++; 102 | edgeCount[v+1]++; 103 | }); 104 | 105 | numEdges *= 2; 106 | g = new Graph(numVertices, numEdges); 107 | 108 | processGraphData(*g, edgeCount, edgeList, numVertices, numEdges); 109 | t2 = mytimer(); 110 | 111 | std::cout << "Total time to generate RGG graph with N = " << N << 112 | " and k = " << k << ": " << (t2-t1) << " secs\n"; 113 | } 114 | -------------------------------------------------------------------------------- /generators/nkit-crg.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #ifndef __NETWORKIT_RGG_H 51 | #define __NETWORKIT_RGG_H 52 | 53 | #include "../graph.hpp" 54 | 55 | void generateCRG(Graph *&g, unsigned int N, unsigned int k, bool randomEdgeWeight=false); 56 | 57 | #endif // __NETWORKIT_RGG_H 58 | -------------------------------------------------------------------------------- /generators/nkit-er.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | 60 | #include "nkit-er.hpp" 61 | #include "../utils.hpp" 62 | 63 | #include "networkit/Globals.hpp" 64 | 65 | #include "networkit/graph/GraphBuilder.hpp" 66 | 67 | void generateER(Graph *&g, unsigned int N, double p, bool randomEdgeWeight) 68 | { 69 | if (!N || p <= 0.0) 70 | { 71 | std::cerr << "For ER graph, N and p must be specified!" << std::endl; 72 | exit(EXIT_FAILURE); 73 | } 74 | 75 | NetworKit::Graph G; 76 | NetworKit::GraphBuilder builder; 77 | 78 | // prepare a random generator for each possible thread 79 | int maxThreads = omp_get_max_threads(); 80 | std::vector< std::function > randomPerThread; 81 | std::random_device device; 82 | std::uniform_int_distribution intDist; 83 | for (int tid = 0; tid < maxThreads; tid++) { 84 | auto seed = intDist(device); 85 | std::mt19937_64 gen(seed); 86 | std::uniform_real_distribution dist{0.0, std::nexttoward(1.0, 2.0)}; 87 | auto rdn = std::bind(dist, gen); 88 | randomPerThread.push_back(rdn); 89 | } 90 | 91 | double t1, t2; 92 | 93 | t1 = mytimer(); 94 | builder = NetworKit::GraphBuilder(N); 95 | builder.parallelForNodePairs([&](NetworKit::node u, NetworKit::node v) { 96 | int tid = omp_get_thread_num(); 97 | double rdn = randomPerThread[tid](); 98 | if (rdn <= p) { 99 | builder.addHalfEdge(u, v); 100 | } 101 | }); 102 | G = builder.completeGraph(true); 103 | 104 | /* Generate binary file containing edgelist */ 105 | GraphElem numEdges = G.numberOfEdges(); 106 | GraphElem numVertices = G.numberOfNodes(); 107 | 108 | std::vector edgeCount(numVertices + 1); 109 | std::vector edgeList; 110 | 111 | std::cout << "Generating ER graph of numvertices: " << numVertices << 112 | ", numEdges: " << numEdges << std::endl; 113 | 114 | GraphWeight weight = 1.0; 115 | 116 | G.forEdges([&](NetworKit::node u, NetworKit::node v) { 117 | assert((u >= 0) && (u < numVertices)); 118 | assert((v >= 0) && (v < numVertices)); 119 | 120 | if (randomEdgeWeight) 121 | weight = (GraphWeight)genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 122 | 123 | edgeList.emplace_back(u, v, weight); 124 | edgeList.emplace_back(v, u, weight); 125 | 126 | edgeCount[u+1]++; 127 | edgeCount[v+1]++; 128 | }); 129 | 130 | numEdges *= 2; 131 | g = new Graph(numVertices, numEdges); 132 | 133 | processGraphData(*g, edgeCount, edgeList, numVertices, numEdges); 134 | t2 = mytimer(); 135 | 136 | std::cout << "Total time to generate ER graph with N = " << N << 137 | " and p = " << p << ": " << (t2-t1) << " secs\n"; 138 | } 139 | -------------------------------------------------------------------------------- /generators/nkit-er.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #ifndef __NETWORKIT_ER_H 51 | #define __NETWORKIT_ER_H 52 | 53 | #include "../graph.hpp" 54 | 55 | void generateER(Graph *&g, unsigned int N, double p, bool randomEdgeWeight=false); 56 | 57 | #endif // __NETWORKIT_ER_H 58 | -------------------------------------------------------------------------------- /generators/nkit-hg.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | 60 | #include "nkit-er.hpp" 61 | #include "../utils.hpp" 62 | 63 | #include "networkit/Globals.hpp" 64 | #include "networkit/generators/HyperbolicGenerator.hpp" 65 | 66 | void generateHG(Graph *&g, unsigned int N, bool randomEdgeWeight=false) 67 | { 68 | double t1, t2; 69 | 70 | if (!N) 71 | { 72 | std::cerr << "For hyperbolic graph, N must be specified!" << std::endl; 73 | exit(EXIT_FAILURE); 74 | } 75 | 76 | NetworKit::HyperbolicGenerator gen(N); 77 | NetworKit::Graph G = gen.generate(); 78 | 79 | /* Generate binary file containing edgelist */ 80 | GraphElem numEdges = G.numberOfEdges(); 81 | GraphElem numVertices = G.numberOfNodes(); 82 | 83 | std::vector edgeCount(numVertices + 1); 84 | std::vector edgeList; 85 | 86 | std::cout << "Generating hyperbolic graph of numvertices: " << numVertices << 87 | ", numEdges: " << numEdges << std::endl; 88 | 89 | GraphWeight weight = 1.0; 90 | 91 | G.forEdges([&](NetworKit::node u, NetworKit::node v) { 92 | assert((u >= 0) && (u < numVertices)); 93 | assert((v >= 0) && (v < numVertices)); 94 | 95 | if (randomEdgeWeight) 96 | weight = (GraphWeight)genRandom(RANDOM_MIN_WEIGHT, RANDOM_MAX_WEIGHT); 97 | 98 | edgeList.emplace_back(u, v, weight); 99 | edgeList.emplace_back(v, u, weight); 100 | 101 | edgeCount[u+1]++; 102 | edgeCount[v+1]++; 103 | }); 104 | 105 | numEdges *= 2; 106 | g = new Graph(numVertices, numEdges); 107 | 108 | processGraphData(*g, edgeCount, edgeList, numVertices, numEdges); 109 | t2 = mytimer(); 110 | 111 | std::cout << "Total time to generate hyperbolic graph with N = " << N 112 | << " : " << (t2-t1) << " secs\n"; 113 | } 114 | 115 | -------------------------------------------------------------------------------- /generators/nkit-hg.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #ifndef __NETWORKIT_HG_H 51 | #define __NETWORKIT_HG_H 52 | 53 | #include "../graph.hpp" 54 | 55 | void generateHG(Graph *&g, unsigned int N, bool randomEdgeWeight=false); 56 | 57 | #endif // __NETWORKIT_HG_H 58 | 59 | 60 | -------------------------------------------------------------------------------- /graph.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #ifndef __GRAPH_H 52 | #define __GRAPH_H 53 | 54 | #include 55 | 56 | #include 57 | #include 58 | #include 59 | 60 | #include "edge.hpp" 61 | 62 | struct Comm { 63 | GraphElem size; 64 | GraphWeight degree; 65 | 66 | Comm() : size(0), degree(0) { }; 67 | }; 68 | 69 | 70 | #if defined(__CRAY_MIC_KNL) && defined(USE_AUTOHBW_MEMALLOC) 71 | #include 72 | typedef std::vector > CommVector; 73 | #else 74 | typedef std::vector CommVector; 75 | #endif 76 | 77 | class Graph { 78 | protected: 79 | typedef std::vector EdgeList; 80 | 81 | GraphElem numVertices; 82 | GraphElem numEdges; 83 | 84 | public: 85 | EdgeIndexes edgeListIndexes; 86 | EdgeList edgeList; 87 | 88 | Graph(const GraphElem onv, const GraphElem one); 89 | Graph(const Graph &othis); 90 | ~Graph(); 91 | 92 | GraphElem getNumVertices() const; 93 | GraphElem getNumEdges() const; 94 | void setEdgeWeightstoOne(); 95 | void setNumEdges(GraphElem numEdges); 96 | void getEdgeRangeForVertex(const GraphElem vertex, GraphElem &e0, GraphElem &e1) const; 97 | const Edge &getEdge(const GraphElem edge) const; 98 | 99 | void setEdgeStartForVertex(const GraphElem vertex, const GraphElem e0); 100 | Edge &getEdge(const GraphElem edge); 101 | 102 | friend std::ostream &operator <<(std::ostream &os, const Graph &g); 103 | protected: 104 | Graph(); 105 | Graph &operator = (const Graph &othis); 106 | }; 107 | 108 | inline Graph::Graph() 109 | : numVertices(0), numEdges(0) 110 | { 111 | } // Graph 112 | 113 | inline Graph::Graph(const GraphElem onv, const GraphElem one) 114 | : numVertices(onv), numEdges(one) 115 | { 116 | edgeListIndexes.resize(numVertices + 1); 117 | edgeList.resize(numEdges); 118 | 119 | std::for_each(edgeListIndexes.begin(), edgeListIndexes.end(), 120 | [] (GraphElem &idx) { idx = 0; } ); 121 | 122 | } // Graph 123 | 124 | inline Graph::Graph(const Graph &othis) 125 | : numVertices(othis.numVertices), numEdges(othis.numEdges) 126 | { 127 | edgeListIndexes.resize(numVertices + 1); 128 | edgeList.resize(numEdges); 129 | 130 | std::copy(othis.edgeListIndexes.begin(), othis.edgeListIndexes.end(), 131 | edgeListIndexes.begin()); 132 | std::copy(othis.edgeList.begin(), othis.edgeList.end(), edgeList.begin()); 133 | } // Graph 134 | 135 | inline Graph::~Graph() 136 | { 137 | } // ~Graph 138 | 139 | inline GraphElem Graph::getNumVertices() const 140 | { 141 | return numVertices; 142 | } // getNumVertices 143 | 144 | inline GraphElem Graph::getNumEdges() const 145 | { 146 | return numEdges; 147 | } // getNumEdges 148 | 149 | inline void Graph::setNumEdges(GraphElem numEdges) { 150 | this->edgeList.resize(numEdges); 151 | this->numEdges=numEdges; 152 | } 153 | 154 | inline void Graph::setEdgeWeightstoOne() { 155 | for (GraphElem i = 0; i < this->numEdges; i++) 156 | this->edgeList[i].weight = 1.0; 157 | } 158 | 159 | inline void Graph::getEdgeRangeForVertex(const GraphElem vertex, GraphElem &e0, GraphElem &e1) const 160 | { 161 | assert((vertex >= 0) && (vertex < numVertices)); 162 | #if defined(DEBUG_BUILD) 163 | e0 = edgeListIndexes.at(vertex); 164 | e1 = edgeListIndexes.at(vertex + 1); 165 | #else 166 | e0 = edgeListIndexes[vertex]; 167 | e1 = edgeListIndexes[vertex + 1]; 168 | #endif 169 | } // getEdgeRangeForVertex 170 | 171 | inline const Edge &Graph::getEdge(const GraphElem edge) const 172 | { 173 | #if defined(DEBUG_BUILD) 174 | assert((edge >= 0) && (edge < numEdges)); 175 | return edgeList.at(edge); 176 | #else 177 | return edgeList[edge]; 178 | #endif 179 | } // getEdge 180 | 181 | inline void Graph::setEdgeStartForVertex(const GraphElem vertex, const GraphElem e0) 182 | { 183 | #if defined(DEBUG_BUILD) 184 | assert((vertex >= 0) && (vertex <= numVertices)); 185 | assert((e0 >= 0) && (e0 <= numEdges)); 186 | edgeListIndexes.at(vertex) = e0; 187 | #else 188 | edgeListIndexes[vertex] = e0; 189 | #endif 190 | } // setEdgeRangeForVertex 191 | 192 | inline Edge &Graph::getEdge(const GraphElem edge) 193 | { 194 | #if defined(DEBUG_BUILD) 195 | if ((edge < 0) || (edge >= numEdges)) 196 | std::cerr << "ERROR: out of bounds access: " << edge << ", max: " << numEdges << std::endl; 197 | assert((edge >= 0) && (edge < numEdges)); 198 | return edgeList.at(edge); 199 | #else 200 | return edgeList[edge]; 201 | #endif 202 | } // getEdge 203 | 204 | inline std::ostream &operator <<(std::ostream &os, const Graph &g) 205 | { 206 | #if defined(DEBUG_BUILD) 207 | os << "Number of vertices: " << g.numVertices << ", number of edges: " << g.numEdges << 208 | std::endl; 209 | #endif 210 | 211 | for (GraphElem i = 0; i < g.numVertices; i++) { 212 | #if defined(DEBUG_BUILD) 213 | const GraphElem lb = g.edgeListIndexes.at(i), ub = g.edgeListIndexes.at(i + 1); 214 | os << "Vertex: " << i << ", number of neighbors: " << ub - lb << std::endl; 215 | #else 216 | const GraphElem lb = g.edgeListIndexes[i], ub = g.edgeListIndexes[i + 1]; 217 | #endif 218 | 219 | for (GraphElem j = lb; j < ub; j++) { 220 | const Edge &edge = g.getEdge(j); 221 | 222 | #if defined(DEBUG_BUILD) 223 | os << "Edge to: " << edge.tail << ", weight: " << edge.weight << std::endl; 224 | #endif 225 | } 226 | } 227 | 228 | return os; 229 | } // operator << 230 | 231 | #endif // __GRAPH_H 232 | -------------------------------------------------------------------------------- /locks.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #ifndef __LOCKS_H 52 | #define __LOCKS_H 53 | 54 | #ifdef USE_OPENMP_LOCK 55 | #else 56 | #ifdef USE_SPINLOCK 57 | #include 58 | std::atomic_flag lkd_ = ATOMIC_FLAG_INIT; 59 | #else 60 | #include 61 | std::mutex mtx_; 62 | #endif 63 | void lock() { 64 | #ifdef USE_SPINLOCK 65 | while (lkd_.test_and_set(std::memory_order_acquire)) { ; } 66 | #else 67 | mtx_.lock(); 68 | #endif 69 | } 70 | void unlock() { 71 | #ifdef USE_SPINLOCK 72 | lkd_.clear(std::memory_order_release); 73 | #else 74 | mtx_.unlock(); 75 | #endif 76 | } 77 | #endif 78 | 79 | #endif // __LOCKS_H 80 | -------------------------------------------------------------------------------- /louvain.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __LOUVAIN_H 2 | #define __LOUVAIN_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | #include "distgraph.hpp" 19 | #include "coloring.hpp" 20 | 21 | #if defined(__CRAY_MIC_KNL) && defined(USE_AUTOHBW_MEMALLOC) 22 | #include 23 | 24 | typedef std::vector > CommunityVector; 25 | typedef std::vector > GraphWeightVector; 26 | typedef std::vector > GraphElemVector; 27 | 28 | typedef std::unordered_map, std::equal_to, 29 | hbw::allocator< std::pair< const GraphElem, GraphElem > > > VertexCommMap; 30 | 31 | typedef std::unordered_set, std::equal_to, 32 | hbw::allocator > RemoteVertexList; 33 | typedef std::vector > PartnerArray; 34 | 35 | typedef std::vector > CommVector; 36 | typedef std::map, 37 | hbw::allocator< std::pair< const GraphElem, Comm > > > CommMap; 38 | typedef std::unordered_map, std::equal_to, 39 | hbw::allocator< std::pair< const GraphElem, GraphElem > > > ClusterLocalMap; 40 | #else 41 | typedef std::vector CommunityVector; 42 | typedef std::vector GraphWeightVector; 43 | typedef std::vector GraphElemVector; 44 | 45 | typedef std::unordered_map VertexCommMap; 46 | 47 | typedef std::unordered_set RemoteVertexList; 48 | typedef std::vector PartnerArray; 49 | 50 | typedef std::vector CommVector; 51 | typedef std::map CommMap; 52 | 53 | typedef std::unordered_map ClusterLocalMap; 54 | #endif 55 | 56 | const int SizeTag = 1; 57 | const int VertexTag = 2; 58 | const int CommunityTag = 3; 59 | const int CommunitySizeTag = 4; 60 | const int CommunityDataTag = 5; 61 | 62 | struct CommInfo { 63 | GraphElem community; 64 | GraphElem size; 65 | GraphWeight degree; 66 | }; 67 | 68 | // early termination cutoff for frozen 69 | // vertices is 90% 70 | #define ET_CUTOFF 90 71 | // early termination cutoff for 72 | // probabilistically freezing vertices 73 | // is 20% 74 | #define P_CUTOFF 0.02 75 | 76 | typedef std::vector CommInfoVector; 77 | extern std::ofstream ofs; 78 | static MPI_Datatype commType; 79 | 80 | GraphWeight distLouvainMethod(const int me, const int nprocs, const DistGraph &dg, 81 | size_t &ssz, size_t &rsz, std::vector &ssizes, 82 | std::vector &rsizes, std::vector &svdata, 83 | std::vector &rvdata, CommunityVector &cvect, const GraphWeight lower, 84 | const GraphWeight thresh, int& iters); 85 | 86 | GraphWeight distLouvainMethod(const int me, const int nprocs, const DistGraph &dg, 87 | size_t &ssz, size_t &rsz, std::vector &ssizes, 88 | std::vector &rsizes, std::vector &svdata, 89 | std::vector &rvdata, CommunityVector &cvect, const GraphWeight lower, 90 | const GraphWeight thresh, int& iters, bool ETLocalOrRemote); 91 | 92 | GraphWeight distLouvainMethod(const int me, const int nprocs, const DistGraph &dg, 93 | size_t &ssz, size_t &rsz, std::vector &ssizes, 94 | std::vector &rsizes, std::vector &svdata, 95 | std::vector &rvdata, CommunityVector &cvect, const GraphWeight lower, 96 | const GraphWeight thresh, int& iters, GraphWeight ETDelta, bool ETLocalOrRemote); 97 | 98 | GraphWeight distLouvainMethodWithColoring(const int me, const int nprocs, const DistGraph &dg, 99 | const long numColor, const ColorVector &vertexColor, size_t &ssz, size_t &rsz, 100 | std::vector &ssizes, std::vector &rsizes, 101 | std::vector &svdata, std::vector &rvdata, 102 | CommunityVector &cvect, const GraphWeight lower, const GraphWeight thresh, int& iters); 103 | 104 | GraphWeight distLouvainMethodWithColoring(const int me, const int nprocs, const DistGraph &dg, 105 | const long numColor, const ColorVector &vertexColor, size_t &ssz, size_t &rsz, 106 | std::vector &ssizes, std::vector &rsizes, 107 | std::vector &svdata, std::vector &rvdata, 108 | CommunityVector &cvect, const GraphWeight lower, const GraphWeight thresh, 109 | int& iters, bool ETLocalOrRemote); 110 | 111 | GraphWeight distLouvainMethodWithColoring(const int me, const int nprocs, const DistGraph &dg, 112 | const long numColor, const ColorVector &vertexColor, 113 | size_t &ssz, size_t &rsz, std::vector &ssizes, 114 | std::vector &rsizes, std::vector &svdata, 115 | std::vector &rvdata, CommunityVector &cvect, 116 | const GraphWeight lower, const GraphWeight thresh, int& iters, GraphWeight ETDelta, 117 | bool ETLocalOrRemote); 118 | 119 | GraphWeight distLouvainMethodVertexOrder(const int me, const int nprocs, const DistGraph &dg, 120 | const long numColor, const ColorVector &vertexColor, size_t &ssz, size_t &rsz, 121 | std::vector &ssizes, std::vector &rsizes, 122 | std::vector &svdata, std::vector &rvdata, 123 | CommunityVector &cvect, const GraphWeight lower, const GraphWeight thresh, int& iters); 124 | 125 | GraphWeight distLouvainMethodVertexOrder(const int me, const int nprocs, const DistGraph &dg, 126 | const long numColor, const ColorVector &vertexColor, size_t &ssz, size_t &rsz, 127 | std::vector &ssizes, std::vector &rsizes, 128 | std::vector &svdata, std::vector &rvdata, 129 | CommunityVector &cvect, const GraphWeight lower, const GraphWeight thresh, int& iters, 130 | bool ETLocalOrRemote); 131 | 132 | GraphWeight distLouvainMethodVertexOrder(const int me, const int nprocs, const DistGraph &dg, 133 | const long numColor, const ColorVector &vertexColor, size_t &ssz, size_t &rsz, 134 | std::vector &ssizes, std::vector &rsizes, 135 | std::vector &svdata, std::vector &rvdata, 136 | CommunityVector &cvect, const GraphWeight lower, const GraphWeight thresh, int& iters, 137 | GraphWeight ETDelta, bool ETLocalOrRemote); 138 | 139 | static void distInitLouvain(const DistGraph &dg, CommunityVector &pastComm, 140 | CommunityVector &currComm, GraphWeightVector &vDegree, 141 | GraphWeightVector &clusterWeight, CommVector &localCinfo, 142 | CommVector &localCupdate, GraphWeight &constantForSecondTerm, 143 | const int me); 144 | 145 | static void distExecuteLouvainIteration(const GraphElem i, const DistGraph &dg, 146 | const CommunityVector &currComm, CommunityVector &targetComm, 147 | const GraphWeightVector &vDegree, CommVector &localCinfo, CommVector &localCupdate, 148 | const VertexCommMap &remoteComm, const CommMap &remoteCinfo, CommMap &remoteCupdate, 149 | const GraphWeight constantForSecondTerm, GraphWeightVector &clusterWeight, const int me); 150 | 151 | static void distSumVertexDegree(const Graph &g, GraphWeightVector &vDegree, CommVector &localCinfo); 152 | 153 | static GraphWeight distCalcConstantForSecondTerm(const GraphWeightVector &vDegree); 154 | 155 | static GraphElem distGetMaxIndex(const ClusterLocalMap &clmap, const GraphWeightVector &counter, 156 | const GraphWeight selfLoop, const CommVector &localCinfo, const CommMap &remoteCinfo, 157 | const GraphWeight vDegree, const GraphElem currSize, const GraphElem currDegree, 158 | const GraphElem currComm, const GraphElem base, const GraphElem bound, const GraphWeight constant); 159 | 160 | static GraphWeight distBuildLocalMapCounter(const GraphElem e0, const GraphElem e1, 161 | ClusterLocalMap &clmap, GraphWeightVector &counter, const Graph &g, const CommunityVector &currComm, 162 | const VertexCommMap &remoteComm, const GraphElem vertex, const GraphElem base, const GraphElem bound); 163 | 164 | static GraphWeight distComputeModularity(const Graph &g, CommVector &localCinfo, 165 | const GraphWeightVector &clusterWeight, 166 | const GraphWeight constantForSecondTerm, const int me); 167 | 168 | static void distUpdateLocalCinfo(CommVector &localCinfo, const CommVector &localCupdate); 169 | 170 | static void distCleanCWandCU(const GraphElem nv, GraphWeightVector &clusterWeight, 171 | CommVector &localCupdate); 172 | 173 | static void distInitComm(CommunityVector &pastComm, CommunityVector &currComm, 174 | const GraphElem base); 175 | 176 | static void updateRemoteCommunities(const DistGraph &dg, CommVector &localCinfo, 177 | const CommMap &remoteCupdate, 178 | const int me, const int nprocs); 179 | 180 | static void fillRemoteCommunities(const DistGraph &dg, const int me, 181 | const int nprocs, const size_t &ssz, const size_t &rsz, 182 | const std::vector &ssizes, const std::vector &rsizes, 183 | const std::vector &svdata, const std::vector &rvdata, 184 | const CommunityVector &currComm, const CommVector &localCinfo, 185 | CommMap &remoteCinfo, VertexCommMap &remoteComm, CommMap &remoteCupdate); 186 | 187 | static void exchangeVertexReqs(const DistGraph &dg, size_t &ssz, size_t &rsz, 188 | std::vector &ssizes, std::vector &rsizes, 189 | std::vector &svdata, std::vector &rvdata, 190 | const int me, const int nprocs); 191 | 192 | void createCommunityMPIType(); 193 | void destroyCommunityMPIType(); 194 | 195 | // load ground truth file (vertex id --- community id) into a vector 196 | 197 | // Ground Truth file is expected to be of the format generated 198 | // by LFR-gen by Fortunato, et al. 199 | // https://sites.google.com/site/santofortunato/inthepress2 200 | 201 | void loadGroundTruthFile(std::vector& commGroundTruth, 202 | std::string const& groundTruthFileName, 203 | bool isGroundTruthZeroBased = true); 204 | 205 | // gather current community info to root 206 | void gatherAllComm(int root, int me, int nprocs, 207 | std::vector& commAll, 208 | const CommunityVector& localComm); 209 | 210 | #endif 211 | -------------------------------------------------------------------------------- /parallel-converters/parallel-converter.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | #include 51 | #include 52 | #include 53 | 54 | #include 55 | #include 56 | #include 57 | 58 | #include 59 | #include 60 | #include 61 | #include 62 | 63 | #include "../graph.hpp" 64 | #include "../utils.hpp" 65 | 66 | #include "parallel-shards.hpp" 67 | 68 | static std::string inputFileName, outputFileName, shardedFileArgs; 69 | static bool indexOneBased = false; 70 | static int nAggrPEs = 1; 71 | 72 | // this option will override whatever 73 | // weights there are in the file, and 74 | // make weights 1.0 for ALL edges 75 | static bool makeWeightsOne = false; 76 | static bool randomEdgeWeight = false; 77 | static bool origEdgeWeight = false; 78 | 79 | static void parseCommandLine(const int argc, char * const argv[]); 80 | 81 | int main(int argc, char *argv[]) 82 | { 83 | int me, size; 84 | 85 | MPI_Init(&argc, &argv); 86 | 87 | MPI_Comm_rank(MPI_COMM_WORLD, &me); 88 | MPI_Comm_size(MPI_COMM_WORLD, &size); 89 | 90 | parseCommandLine(argc, argv); 91 | 92 | // fetch shard specific arguments, expecting four 93 | std::stringstream ss(shardedFileArgs); 94 | std::string s; 95 | std::vector args; 96 | 97 | while (std::getline(ss, s, ' ')) { 98 | args.push_back(s); 99 | } 100 | 101 | if (args.size() != 4) { 102 | std::cerr << "Expected arguments to '-x' (in this order): " 103 | << std::endl; 104 | MPI_Abort(MPI_COMM_WORLD, -99); 105 | } 106 | 107 | GraphElem numFiles = std::stol(args[0]); 108 | GraphElem startChunk = std::stol(args[1]); 109 | GraphElem endChunk = std::stol(args[2]); 110 | GraphElem shardCount = std::stol(args[3]); 111 | 112 | args.clear(); 113 | 114 | double t0, t1, t2, rt; 115 | t0 = mytimer(); 116 | 117 | if (me == 0) { 118 | std::cout << "Start reading " << numFiles << " files." << std::endl; 119 | } 120 | 121 | if (randomEdgeWeight) 122 | loadParallelFileShards(me, size, nAggrPEs, inputFileName, outputFileName, 123 | startChunk, endChunk, indexOneBased, RND_WEIGHT, shardCount); 124 | else if (makeWeightsOne) 125 | loadParallelFileShards(me, size, nAggrPEs, inputFileName, outputFileName, 126 | startChunk, endChunk, indexOneBased, ONE_WEIGHT, shardCount); 127 | else if (origEdgeWeight) 128 | loadParallelFileShards(me, size, nAggrPEs, inputFileName, outputFileName, 129 | startChunk, endChunk, indexOneBased, ORG_WEIGHT, shardCount); 130 | else 131 | loadParallelFileShards(me, size, nAggrPEs, inputFileName, outputFileName, 132 | startChunk, endChunk, indexOneBased, ABS_WEIGHT, shardCount); 133 | 134 | t1 = mytimer(); 135 | t2 = t1 - t0; 136 | MPI_Reduce(&t2, &rt, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); 137 | 138 | if (me == 0) { 139 | std::cout << "Average time reading " << numFiles << " sharded files and writing binary file (in secs): " << rt / size << std::endl; 140 | } 141 | 142 | return 0; 143 | } // main 144 | 145 | void parseCommandLine(const int argc, char * const argv[]) 146 | { 147 | int ret; 148 | 149 | while ((ret = getopt(argc, argv, "f:o:n:x:zw")) != -1) { 150 | switch (ret) { 151 | case 'f': 152 | inputFileName.assign(optarg); 153 | break; 154 | case 'o': 155 | outputFileName.assign(optarg); 156 | break; 157 | case 'x': 158 | shardedFileArgs.assign(optarg); 159 | break; 160 | case 'n': 161 | nAggrPEs = atoi(optarg); 162 | break; 163 | case 'z': 164 | indexOneBased = true; 165 | break; 166 | case 'w': 167 | makeWeightsOne = true; 168 | break; 169 | default: 170 | assert(0 && "Should not reach here!!"); 171 | break; 172 | } 173 | } 174 | 175 | if (inputFileName.empty()) { 176 | std::cerr << "Must specify an input file name with -f" << std::endl; 177 | exit(EXIT_FAILURE); 178 | } 179 | 180 | if (outputFileName.empty()) { 181 | std::cerr << "Must specify an output file name with -o" << std::endl; 182 | exit(EXIT_FAILURE); 183 | } 184 | } // parseCommandLine 185 | -------------------------------------------------------------------------------- /parallel-converters/parallel-shards.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #ifndef __LOAD_PARALLEL_SHARDS_H 52 | #define __LOAD_PARALLEL_SHARDS_H 53 | 54 | #include 55 | 56 | #include "../utils.hpp" 57 | #include "../graph.hpp" 58 | 59 | /// Pairwise read files between fileStartIndex and fileEndIndex and store 60 | /// it into a single file, also convert to binary 61 | void loadParallelFileShards(int rank, int nprocs, int naggr, 62 | const std::string &fileInShardsPath, const std::string &fileOutPath, 63 | const int fileStartIndex, const int fileEndIndex, bool indexOneBased, 64 | Weight_t wtype = ORG_WEIGHT, GraphElem shardCount = 1000000); 65 | 66 | #endif // __LOAD_PARALLEL_SHARDS_H 67 | -------------------------------------------------------------------------------- /rebuild.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #ifndef __BUILD_NEXT_PHASE_H 52 | #define __BUILD_NEXT_PHASE_H 53 | 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | 60 | #include 61 | #include 62 | 63 | #include 64 | 65 | #include "edge.hpp" 66 | #include "louvain.hpp" 67 | 68 | typedef struct edgeInfo{ 69 | GraphElem s; 70 | GraphElem t; 71 | GraphWeight w; 72 | }EdgeInfo; 73 | 74 | static MPI_Datatype edgeType; 75 | 76 | #if defined(__CRAY_MIC_KNL) && defined(USE_AUTOHBW_MEMALLOC) 77 | typedef std::vector > EdgeVector; 78 | typedef std::unordered_set, std::equal_to, 79 | hbw::allocator > RemoteCommList; 80 | typedef std::vector> PartArray; 81 | typedef std::map, 82 | hbw::allocator< std::pair< const GraphElem, GraphWeight > > > NewEdge; 83 | typedef std::unordered_map, std::equal_to, 84 | hbw::allocator< std::pair< const GraphElem, NewEdge > > > NewEdgesMap; 85 | #else 86 | typedef std::vector EdgeVector; 87 | typedef std::unordered_set RemoteCommList; 88 | typedef std::vector PartArray; 89 | typedef std::map NewEdge; 90 | typedef std::unordered_map NewEdgesMap; 91 | #endif 92 | 93 | void createEdgeMPIType(); 94 | void destroyEdgeMPIType(); 95 | 96 | static GraphElem distReNumber(int nprocs, ClusterLocalMap& lookUp, int me, 97 | DistGraph &dg, const size_t &ssz, const size_t &rsz, 98 | const std::vector &ssizes, const std::vector &rsizes, 99 | const std::vector &svdata, const std::vector &rvdata, 100 | CommunityVector &cvect, VertexCommMap &remoteComm); 101 | 102 | void fill_newEdgesMap(int me, NewEdgesMap &newEdgesMap, 103 | DistGraph& dg, CommunityVector &cvect, VertexCommMap &remoteComm, 104 | ClusterLocalMap &lookUp); 105 | 106 | void send_newEdges(int me, int nprocs, DistGraph* &dg, GraphElem newGlobalNumVertices, 107 | NewEdgesMap& newEdgesMap); 108 | 109 | void distbuildNextLevelGraph(int nprocs, int me, DistGraph*& dg, 110 | const size_t &ssz, const size_t &rsz, const std::vector &ssizes, 111 | const std::vector &rsizes, const std::vector &svdata, 112 | const std::vector &rvdata, CommunityVector &cvect); 113 | #endif 114 | -------------------------------------------------------------------------------- /utils.cpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 12 | // Hao Lu (luhowardmark@wsu.edu) 13 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #include 52 | #include 53 | 54 | #include 55 | #include 56 | #include 57 | 58 | #include "utils.hpp" 59 | 60 | double mytimer(void) 61 | { 62 | static long int start = 0L, startu; 63 | 64 | const double million = 1000000.0; 65 | 66 | timeval tp; 67 | 68 | if (start == 0L) { 69 | gettimeofday(&tp, NULL); 70 | 71 | start = tp.tv_sec; 72 | startu = tp.tv_usec; 73 | } 74 | 75 | gettimeofday(&tp, NULL); 76 | 77 | return (static_cast(tp.tv_sec - start) + (static_cast(tp.tv_usec - startu) / 78 | million)); 79 | } 80 | 81 | // Random number generator from B. Stroustrup: 82 | // http://www.stroustrup.com/C++11FAQ.html#std-random 83 | GraphWeight genRandom(GraphWeight low, GraphWeight high) 84 | { 85 | static std::default_random_engine re {}; 86 | using Dist = std::uniform_real_distribution; 87 | static Dist uid {}; 88 | return uid(re, Dist::param_type{low,high}); 89 | } 90 | 91 | void processGraphData(Graph &g, std::vector &edgeCount, 92 | std::vector &edgeList, 93 | const GraphElem nv, const GraphElem ne) 94 | { 95 | std::vector ecTmp(nv + 1); 96 | 97 | std::partial_sum(edgeCount.begin(), edgeCount.end(), ecTmp.begin()); 98 | edgeCount = ecTmp; 99 | 100 | g.setEdgeStartForVertex(0, 0); 101 | 102 | for (GraphElem i = 0; i < nv; i++) 103 | g.setEdgeStartForVertex(i + 1, edgeCount[i + 1]); 104 | 105 | edgeCount.clear(); 106 | 107 | auto ecmp = [] (GraphElemTuple const& e0, GraphElemTuple const& e1) 108 | { return ((e0.i_ < e1.i_) || ((e0.i_ == e1.i_) && (e0.j_ < e1.j_))); }; 109 | 110 | if (!std::is_sorted(edgeList.begin(), edgeList.end(), ecmp)) { 111 | #if defined(DEBUG_PRINTF) 112 | std::cout << "Edge list is not sorted" << std::endl; 113 | #endif 114 | std::sort(edgeList.begin(), edgeList.end(), ecmp); 115 | } 116 | else { 117 | #if defined(DEBUG_PRINTF) 118 | std::cout << "Edge list is sorted!" << std::endl; 119 | #endif 120 | } 121 | 122 | GraphElem ePos = 0; 123 | for (GraphElem i = 0; i < nv; i++) { 124 | GraphElem e0, e1; 125 | g.getEdgeRangeForVertex(i, e0, e1); 126 | 127 | if ((i % 100000) == 0) 128 | std::cout << "Processing edges for vertex: " << i << ", range(" << e0 << ", " << e1 << 129 | ")" << std::endl; 130 | 131 | for (GraphElem j = e0; j < e1; j++) { 132 | Edge &edge = g.getEdge(j); 133 | 134 | assert(ePos == j); 135 | assert(i == edgeList[ePos].i_); 136 | edge.tail = edgeList[ePos].j_; 137 | edge.weight = edgeList[ePos].w_; 138 | 139 | ePos++; 140 | } 141 | } 142 | } // processGraphData 143 | -------------------------------------------------------------------------------- /utils.hpp: -------------------------------------------------------------------------------- 1 | // *********************************************************************** 2 | // 3 | // Vite: A C++ library for distributed-memory graph clustering 4 | // using MPI+OpenMP 5 | // 6 | // Daniel Chavarria (daniel.chavarria@pnnl.gov) 7 | // Antonino Tumeo (antonino.tumeo@pnnl.gov) 8 | // Mahantesh Halappanavar (hala@pnnl.gov) 9 | // Pacific Northwest National Laboratory 10 | // 11 | // Hao Lu (luhowardmark@wsu.edu) 12 | // Sayan Ghosh (sayan.ghosh@wsu.edu) 13 | // Ananth Kalyanaraman (ananth@eecs.wsu.edu) 14 | // Washington State University 15 | // 16 | // *********************************************************************** 17 | // 18 | // Copyright (2017) Battelle Memorial Institute 19 | // All rights reserved. 20 | // 21 | // Redistribution and use in source and binary forms, with or without 22 | // modification, are permitted provided that the following conditions 23 | // are met: 24 | // 25 | // 1. Redistributions of source code must retain the above copyright 26 | // notice, this list of conditions and the following disclaimer. 27 | // 28 | // 2. Redistributions in binary form must reproduce the above copyright 29 | // notice, this list of conditions and the following disclaimer in the 30 | // documentation and/or other materials provided with the distribution. 31 | // 32 | // 3. Neither the name of the copyright holder nor the names of its 33 | // contributors may be used to endorse or promote products derived from 34 | // this software without specific prior written permission. 35 | // 36 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 37 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 38 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 39 | // FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 40 | // COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 41 | // INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 42 | // BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 43 | // LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 44 | // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 | // LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 46 | // ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 47 | // POSSIBILITY OF SUCH DAMAGE. 48 | // 49 | // ************************************************************************ 50 | 51 | #ifndef __UTILS_H 52 | #define __UTILS_H 53 | 54 | #include "graph.hpp" 55 | 56 | #include 57 | #include 58 | #include 59 | 60 | #ifndef RANDOM_MAX_WEIGHT 61 | #define RANDOM_MAX_WEIGHT (1.0) 62 | #endif 63 | #ifndef RANDOM_MIN_WEIGHT 64 | #define RANDOM_MIN_WEIGHT (0.01) 65 | #endif 66 | 67 | #ifndef TERMINATION_PHASE_COUNT 68 | #define TERMINATION_PHASE_COUNT (200) 69 | #endif 70 | 71 | // Read https://en.wikipedia.org/wiki/Linear_congruential_generator#Period_length 72 | // about choice of LCG parameters 73 | // From numerical recipes 74 | // TODO FIXME investigate larger periods 75 | #define MLCG (2147483647) // 2^31 - 1 76 | #define ALCG (16807) // 7^5 77 | #define BLCG (0) 78 | #define SR_LCG_TAG 108 79 | 80 | struct GraphElemTuple { 81 | 82 | GraphElem i_, j_; 83 | GraphWeight w_; 84 | 85 | GraphElemTuple(GraphElem i, GraphElem j, GraphWeight w): 86 | i_(i), j_(j), w_(w) 87 | {} 88 | GraphElemTuple(GraphElem i, GraphElem j): 89 | i_(i), j_(j), w_(1.0) 90 | {} 91 | GraphElemTuple(): 92 | i_(-1), j_(-1), w_(0.0) 93 | {} 94 | 95 | // compare 96 | bool operator <(GraphElemTuple const& tp) const 97 | { return (i_ < tp.i_) || ((!(tp.i_ < i_)) && (j_ < tp.j_)); } 98 | }; 99 | 100 | typedef enum 101 | { 102 | RND_WEIGHT, // random real weight, between 0-1 103 | ONE_WEIGHT, // weight = 1 104 | ORG_WEIGHT, // use original weights of graph 105 | ABS_WEIGHT // use absolute original weights of graph 106 | } Weight_t; 107 | 108 | extern unsigned seed; 109 | 110 | double mytimer(void); 111 | 112 | // uses a static random engine (seed) 113 | GraphWeight genRandom(GraphWeight low, GraphWeight high); 114 | 115 | // sort edge list 116 | void processGraphData(Graph &g, std::vector &edgeCount, 117 | std::vector &edgeList, 118 | const GraphElem nv, const GraphElem ne); 119 | 120 | // forward declare LCG class for 121 | // distributed random number 122 | // generation 123 | 124 | // Parallel Linear Congruential Generator 125 | // x[i] = (a*x[i-1] + b)%M 126 | class LCG 127 | { 128 | public: 129 | LCG(unsigned seed, GraphWeight* drand, 130 | GraphElem n, MPI_Comm comm = MPI_COMM_WORLD): 131 | seed_(seed), drand_(drand), n_(n) 132 | { 133 | comm_ = comm; 134 | MPI_Comm_size(comm_, &nprocs_); 135 | MPI_Comm_rank(comm_, &rank_); 136 | 137 | // allocate long random numbers 138 | rnums_.resize(n_); 139 | 140 | // init x0 141 | if (rank_ == 0) 142 | x0_ = reseeder(seed_); 143 | 144 | // step #1: bcast x0 from root 145 | MPI_Bcast(&x0_, 1, MPI_GRAPH_TYPE, 0, comm_); 146 | 147 | // step #2: parallel prefix to generate first random value per process 148 | parallel_prefix_op(); 149 | } 150 | 151 | ~LCG() { rnums_.clear(); } 152 | 153 | // return unint32_t seed 154 | GraphElem reseeder(unsigned initseed) 155 | { 156 | std::seed_seq seq({initseed}); 157 | std::vector seeds(1); 158 | seq.generate(seeds.begin(), seeds.end()); 159 | 160 | return (GraphElem)seeds[0]; 161 | } 162 | 163 | // matrix-matrix multiplication for 2x2 matrices 164 | void matmat_2x2(GraphElem c[], GraphElem a[], GraphElem b[]) 165 | { 166 | for (int i = 0; i < 2; i++) { 167 | for (int j = 0; j < 2; j++) { 168 | GraphElem sum = 0; 169 | for (int k = 0; k < 2; k++) { 170 | sum += a[i*2+k]*b[k*2+j]; 171 | } 172 | c[i*2+j] = sum; 173 | } 174 | } 175 | } 176 | 177 | // x *= y 178 | void matop_2x2(GraphElem x[], GraphElem y[]) 179 | { 180 | GraphElem tmp[4]; 181 | matmat_2x2(tmp, x, y); 182 | memcpy(x, tmp, sizeof(GraphElem[4])); 183 | } 184 | 185 | // find kth power of a 2x2 matrix 186 | void mat_power(GraphElem mat[], GraphElem k) 187 | { 188 | GraphElem tmp[4]; 189 | memcpy(tmp, mat, sizeof(GraphElem[4])); 190 | 191 | // mat-mat multiply k times 192 | for (GraphElem p = 0; p < k-1; p++) 193 | matop_2x2(mat, tmp); 194 | } 195 | 196 | // parallel prefix for matrix-matrix operation 197 | // `x0 is the very first random number in the series 198 | // `ab is a 2-length array which stores a and b 199 | // `n_ is (n/p) 200 | // `rnums is n_ length array which stores the random nums for a process 201 | void parallel_prefix_op() 202 | { 203 | GraphElem global_op[4]; 204 | global_op[0] = ALCG; 205 | global_op[1] = 0; 206 | global_op[2] = BLCG; 207 | global_op[3] = 1; 208 | 209 | mat_power(global_op, n_); // M^(n/p) 210 | GraphElem prefix_op[4] = {1,0,0,1}; // I in row-major 211 | 212 | GraphElem global_op_recv[4]; 213 | 214 | int steps = (int)(log2((double)nprocs_)); 215 | 216 | for (int s = 0; s < steps; s++) { 217 | 218 | int mate = rank_^(1 << s); // toggle the sth LSB to find my neighbor 219 | 220 | // send/recv global to/from mate 221 | MPI_Sendrecv(global_op, 4, MPI_GRAPH_TYPE, mate, SR_LCG_TAG, 222 | global_op_recv, 4, MPI_GRAPH_TYPE, mate, SR_LCG_TAG, 223 | comm_, MPI_STATUS_IGNORE); 224 | 225 | matop_2x2(global_op, global_op_recv); 226 | 227 | if (mate < rank_) 228 | matop_2x2(prefix_op, global_op_recv); 229 | 230 | MPI_Barrier(comm_); 231 | } 232 | 233 | // populate the first random number entry for each process 234 | // (x0*a + b)%P 235 | if (rank_ == 0) 236 | rnums_[0] = x0_; 237 | else 238 | rnums_[0] = (x0_*prefix_op[0] + prefix_op[2])%MLCG; 239 | } 240 | 241 | // generate random number based on the first 242 | // random number on a process 243 | // TODO check the 'quick'n dirty generators to 244 | // see if we can avoid the mod 245 | void generate() 246 | { 247 | #if defined(PRINT_LCG_LONG_RANDOM_NUMBERS) 248 | for (int k = 0; k < nprocs_; k++) { 249 | if (k == rank_) { 250 | std::cout << "------------" << std::endl; 251 | std::cout << "Process#" << rank_ << " :" << std::endl; 252 | std::cout << "------------" << std::endl; 253 | std::cout << rnums_[0] << std::endl; 254 | for (GraphElem i = 1; i < n_; i++) { 255 | rnums_[i] = (rnums_[i-1]*ALCG + BLCG)%MLCG; 256 | std::cout << rnums_[i] << std::endl; 257 | } 258 | } 259 | MPI_Barrier(comm_); 260 | } 261 | #else 262 | for (GraphElem i = 1; i < n_; i++) { 263 | rnums_[i] = (rnums_[i-1]*ALCG + BLCG)%MLCG; 264 | } 265 | #endif 266 | GraphWeight mult = 1.0 / (GraphWeight)(1.0 + (GraphWeight)(MLCG-1)); 267 | 268 | #if defined(PRINT_LCG_DOUBLE_RANDOM_NUMBERS) 269 | for (int k = 0; k < nprocs_; k++) { 270 | if (k == rank_) { 271 | std::cout << "------------" << std::endl; 272 | std::cout << "Process#" << rank_ << " :" << std::endl; 273 | std::cout << "------------" << std::endl; 274 | 275 | for (GraphElem i = 0; i < n_; i++) { 276 | drand_[i] = (GraphWeight)((GraphWeight)fabs(rnums_[i]) * mult ); // 0-1 277 | std::cout << drand_[i] << std::endl; 278 | } 279 | } 280 | MPI_Barrier(comm_); 281 | } 282 | #else 283 | for (GraphElem i = 0; i < n_; i++) 284 | drand_[i] = (GraphWeight)((GraphWeight)fabs(rnums_[i]) * mult); // 0-1 285 | #endif 286 | } 287 | 288 | // copy from drand_[idx_start] to new_drand, 289 | // rescale the random numbers between lo and hi 290 | void rescale(GraphWeight* new_drand, GraphElem idx_start, GraphWeight const& lo) 291 | { 292 | GraphWeight range = (1.0 / (GraphWeight)nprocs_); 293 | 294 | #if defined(PRINT_LCG_DOUBLE_LOHI_RANDOM_NUMBERS) 295 | for (int k = 0; k < nprocs_; k++) { 296 | if (k == rank_) { 297 | std::cout << "------------" << std::endl; 298 | std::cout << "Process#" << rank_ << " :" << std::endl; 299 | std::cout << "------------" << std::endl; 300 | 301 | for (GraphElem i = idx_start, j = 0; i < n_; i++, j++) { 302 | new_drand[j] = lo + (GraphWeight)(range * drand_[i]); 303 | std::cout << new_drand[j] << std::endl; 304 | } 305 | } 306 | MPI_Barrier(comm_); 307 | } 308 | #else 309 | for (GraphElem i = idx_start, j = 0; i < n_; i++, j++) 310 | new_drand[j] = lo + (GraphWeight)(range * drand_[i]); // lo-hi 311 | #endif 312 | } 313 | 314 | private: 315 | MPI_Comm comm_; 316 | int nprocs_, rank_; 317 | unsigned seed_; 318 | GraphElem n_, x0_; 319 | GraphWeight* drand_; 320 | std::vector rnums_; 321 | }; 322 | 323 | #endif // __UTILS_H 324 | --------------------------------------------------------------------------------