├── Makefile ├── README.md ├── bfs ├── batch128.cpp ├── batch256.cpp ├── batch64.cpp ├── naive.cpp ├── noqueue.cpp ├── parabfs.cpp ├── sc2012.cpp └── sse.cpp ├── compile.sh ├── graph.cpp ├── include ├── TraceStats.hpp ├── bench.hpp ├── bench.hpp.bak ├── bfs │ ├── base.hpp │ ├── batch128.hpp │ ├── batch256.hpp │ ├── batch64.hpp │ ├── batchdistance.hpp │ ├── batchhuge.hpp │ ├── batchhugedeferred.hpp │ ├── bitops.hpp │ ├── naive.hpp │ ├── noqueue.hpp │ ├── parabfs.hpp │ ├── sc2012.hpp │ ├── sse.hpp │ └── statistics.hpp ├── graph.hpp ├── hashtable.hpp ├── idschedulers.hpp ├── io.hpp ├── log.hpp ├── macros.hpp ├── pool.hpp ├── queue.hpp ├── scheduler.hpp ├── tokenizer.hpp ├── topklist.hpp └── worker.hpp ├── io.cpp ├── log.cpp ├── main.cpp ├── query4.cpp ├── query4.hpp ├── runBencher.cpp ├── scheduler.cpp ├── test_queries ├── data │ └── ldbc10k.csv └── ldbc10k.txt ├── vida ├── lib │ └── boost.tar.gz └── src │ ├── CMakeLists.txt │ ├── Makefile │ ├── README.md │ ├── compile.sh │ ├── data │ ├── ldbc10k.csv │ └── ldbc10k.txt │ ├── hsort.hpp │ ├── msbfs.hpp │ ├── naive.hpp │ ├── parabfs.hpp │ ├── person_graph.hpp │ └── query.cpp └── worker.cpp /Makefile: -------------------------------------------------------------------------------- 1 | # Compiler flags 2 | ifeq ($(CC),cc) 3 | CC=g++ 4 | endif 5 | 6 | BOOSTDIR=./boost 7 | BOOST_INC=$(BOOSTDIR) 8 | BOOST_LIB=$(BOOSTDIR)/stage/lib/libboost_ 9 | BOOST_LIBS=$(BOOST_LIB)iostreams.a $(BOOST_LIB)system.a $(BOOST_LIB)thread.a $(BOOST_LIB)timer.a $(BOOST_LIB)chrono.a 10 | LDFLAGS=$(BOOST_LIBS) 11 | UNAME := $(shell uname) 12 | ifeq ($(UNAME), Linux) 13 | LDFLAGS+=-lrt 14 | endif 15 | 16 | #Profiling CFLAGS 17 | MATH_FLAGS=-ffast-math -funsafe-math-optimizations -fassociative-math -ffinite-math-only -fno-signed-zeros 18 | ifeq ($(CC),g++) 19 | COMPILER_FLAGS=-fabi-version=6 20 | else 21 | COMPILER_FLAGS= 22 | endif 23 | BASE_FLAGS=-g -std=c++11 -W -Wall -Wextra -pedantic $(MATH_FLAGS) -funroll-all-loops -fno-omit-frame-pointer $(COMPILER_FLAGS) -I$(BOOST_INC) 24 | 25 | LOCAL_ARCH=-march=native 26 | RELEASE_ARCH=-march=native 27 | CFLAGS=$(LOCAL_ARCH) -O3 $(BASE_FLAGS) -DNDEBUG 28 | RELEASE_CFLAGS=$(RELEASE_ARCH) -O3 $(BASE_FLAGS) -DNDEBUG -DNDBGPRINT 29 | LD_FLAGS=-Wl,-O1 -pthread 30 | 31 | # Source / Executable Variables 32 | CORE_SOURCES=graph.cpp io.cpp log.cpp scheduler.cpp bfs/naive.cpp bfs/sc2012.cpp bfs/parabfs.cpp bfs/noqueue.cpp bfs/batch64.cpp bfs/batch128.cpp bfs/batch256.cpp bfs/sse.cpp worker.cpp query4.cpp 33 | ALL_SOURCES=main.cpp $(CORE_SOURCES) 34 | CORE_OBJECTS=$(addsuffix .o, $(basename $(CORE_SOURCES))) 35 | CORE_DEPS=$(addsuffix .depends, $(basename $(ALL_SOURCES))) 36 | 37 | EXECUTABLE_BENCHER=runBencher 38 | EXECUTABLE_BENCH=runBench 39 | EXECUTABLE_BENCH_PROFILE=runBenchProfile 40 | EXECUTABLE_FAST=runBfs 41 | EXECUTABLE_DEBUG=runBfsDebug 42 | ifndef DEBUG 43 | EXEC_EXECUTABLE=$(EXECUTABLE_FAST) 44 | else 45 | EXEC_EXECUTABLE=$(EXECUTABLE_DEBUG) 46 | endif 47 | 48 | RELEASE_OBJECTS=$(addsuffix .release.o, $(basename $(CORE_SOURCES))) 49 | 50 | # Testing related variables 51 | TEST_DATA_PATH=test_data 52 | 10K_DATASET_PATH=$(TEST_DATA_PATH)/data10k/person_knows_person.csv 53 | 1K_DATASET_PATH=$(TEST_DATA_PATH)/data1k/person_knows_person.csv 54 | 100K_QUERIES=test_queries/test_100k.txt 55 | 10K_QUERIES=test_queries/ldbc10k.txt 56 | 1K_QUERIES=test_queries/ldbc1k.txt 57 | 58 | # Program rules 59 | .PHONY: test_env test_all 60 | 61 | all: $(EXEC_EXECUTABLE) $(EXECUTABLE_BENCHER) 62 | @rm -f $(CORE_DEPS) 63 | 64 | test_all: test_1k test_10k 65 | 66 | test_10k: test_env $(EXEC_EXECUTABLE) 67 | @rm -f $(CORE_DEPS) 68 | $(TEST_PREF) ./$(EXEC_EXECUTABLE) $(10K_QUERIES) 3 1 69 | 70 | test_1k: test_env $(EXEC_EXECUTABLE) 71 | @rm -f $(CORE_DEPS) 72 | $(TEST_PREF) ./$(EXEC_EXECUTABLE) $(1K_QUERIES) 73 | 74 | clean: 75 | -rm $(EXECUTABLE_FAST) $(EXECUTABLE_DEBUG) $(EXECUTABLE_BENCH_PROFILE) $(EXECUTABLE_BENCH) $(EXECUTABLE_BENCHER) 76 | -rm $(CORE_OBJECTS) $(RELEASE_OBJECTS) *.o 77 | -rm $(CORE_DEPS) 78 | -rm *.gcda util/*.gdca 79 | 80 | $(EXECUTABLE_BENCH): $(ALL_SOURCES) 81 | $(CC) $(RELEASE_CFLAGS) $(CORE_SOURCES) -fprofile-generate $< -o $(EXECUTABLE_BENCH_PROFILE) $(LD_FLAGS) $(LIBS) 82 | ./$(EXECUTABLE_BENCH_PROFILE) $(10K_QUERIES) 1 1 83 | $(CC) $(RELEASE_CFLAGS) $(CORE_SOURCES) -fprofile-use $< -o $(EXECUTABLE_BENCH) $(LD_FLAGS) $(LIBS) 84 | -rm *.gcda util/*.gcda 85 | -rm $(EXECUTABLE_BENCH_PROFILE) 86 | 87 | $(EXECUTABLE_BENCH_THREADS): benchThreads.release.o $(RELEASE_OBJECTS) $(BOOST_LIBS) 88 | @rm -f $(CORE_DEPS) 89 | $(CC) benchThreads.release.o $(RELEASE_OBJECTS) $(LDFLAGS) -o $@ $(LD_FLAGS) $(LIBS) 90 | 91 | $(EXECUTABLE_BENCHER): runBencher.release.o $(RELEASE_OBJECTS) $(BOOST_LIBS) 92 | @rm -f $(CORE_DEPS) 93 | $(CC) runBencher.release.o $(RELEASE_OBJECTS) $(LDFLAGS) -o $@ $(LD_FLAGS) $(LIBS) 94 | 95 | $(EXECUTABLE_BENCH_VARIANTS): benchVariants.release.o $(RELEASE_OBJECTS) $(BOOST_LIBS) 96 | @rm -f $(CORE_DEPS) 97 | $(CC) benchVariants.release.o $(RELEASE_OBJECTS) $(LDFLAGS) -o $@ $(LD_FLAGS) $(LIBS) 98 | 99 | ifndef DEBUG 100 | $(EXEC_EXECUTABLE): main.release.o $(RELEASE_OBJECTS) $(BOOST_LIBS) 101 | @rm -f $(CORE_DEPS) 102 | $(CC) main.release.o $(RELEASE_OBJECTS) $(LDFLAGS) -o $@ $(LD_FLAGS) $(LIBS) 103 | else 104 | $(EXEC_EXECUTABLE): main.o $(CORE_OBJECTS) 105 | @rm -f $(CORE_DEPS) 106 | $(CC) main.o $(CORE_OBJECTS) -o $@ $(LD_FLAGS) $(LIBS) 107 | endif 108 | 109 | %.release.o: %.cpp 110 | $(CC) $(RELEASE_CFLAGS) $(LDFLAGS) -c $< -o $@ $(LIBS) 111 | 112 | .cpp.o: 113 | $(CC) $(CFLAGS) -DTRACE -c $< -o $@ $(LIBS) 114 | 115 | 116 | %.depends: %.cpp 117 | @$(CC) -M $(CFLAGS) -c $< > $@ $(LIBS) 118 | 119 | # Test data rules 120 | test_env: $(TEST_DATA_PATH) 121 | 122 | -include $(CORE_DEPS) 123 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # About 2 | 3 | This is the experimental framework used to evaluate the **MS-BFS** algorithm (proposed in the VLDB 2015 paper [The More the Merrier: Efficient Multi-Source Graph Traversal](http://www.vldb.org/pvldb/vol8/p449-then.pdf)) and its related work. 4 | 5 | The code computes the top-k closeness centrality values for the vertices in a given graph using different BFS algorithms. 6 | 7 | Following are the instructions to compile and run the source code. 8 | 9 | # Build 10 | `./compile` 11 | 12 | Tested on Ubuntu 14.04 using GCC 4.8.2. 13 | 14 | # Usage 15 | `./runBencher [nRun] [nThreads] [BFSType] [bWidth] (nSources) (force)` 16 | 17 | - `nRun`: Number of execution 18 | - `nThreads`: Number of threads 19 | - `BFSType`: Type of BFSs 20 | - MS-BFS variant values: 16, 32, 64, 128, 256 (e.g. 128 executes MS-BFS using SSE registers) 21 | - Related work values: naive (textbook BFS), noqueue (textbook BFS based on bit fields), scbfs (direction-optimized BFS), parabfs (parallel BFS) 22 | - `bWidth`: Number of registers that are used per vertex for MS-BFS, e.g. 4 with the BFSType 128 runs 512 concurrent BFSs 23 | - `nSources`: (optional) Number of source vertices for which the closeness centrality values are computed. If omitted, all vertices are used 24 | - `force`: (optional) Set to 'f' in order to force the execution even for high parallelism on few source vertices 25 | 26 | # Examples 27 | - `./runBencher test_queries/ldbc10k.txt 3 8 naive 1 20 f` 28 | - `./runBencher test_queries/ldbc10k.txt 1 32 256 2` (only works when compiled for the architecture core-avx2) 29 | 30 | # Team 31 | 32 | - [Manuel Then](http://www-db.in.tum.de/~then/) (Technische Universität München) 33 | - [Moritz Kaufmann](http://www-db.in.tum.de/~kaufmann/) (Technische Universität München) 34 | - [Fernando Chirigati](http://bigdata.poly.edu/~fchirigati/) (New York University) 35 | - [Tuan-Anh Hoang-Vu](http://bigdata.poly.edu/~tuananh/) (New York University) 36 | - [Kien Pham](http://bigdata.poly.edu/~kienpham/) (New York University) 37 | - [Alfons Kemper](http://www3.in.tum.de/~kemper/) (Technische Universität München) 38 | - [Thomas Neumann](http://www-db.in.tum.de/~neumann/) (Technische Universität München) 39 | - [Huy T. Vo](http://serv.cusp.nyu.edu/~hvo/) (New York University) -------------------------------------------------------------------------------- /bfs/batch128.cpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #include "../include/bfs/batch128.hpp" 5 | #include "../include/bfs/batchdistance.hpp" 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | using namespace std; 12 | 13 | namespace Query4 { 14 | 15 | void BatchBFSRunner128::runBatch(vector& bfsData, const PersonSubgraph& subgraph 16 | #ifdef STATISTICS 17 | , BatchStatistics& statistics 18 | #endif 19 | ) { 20 | const auto subgraphSize = subgraph.size(); 21 | 22 | array<__m128i*,2> toVisitLists; 23 | toVisitLists[0] = new __m128i[subgraphSize]; 24 | memset(toVisitLists[0],0,sizeof(__m128i)*subgraphSize); 25 | toVisitLists[1] = new __m128i[subgraphSize]; 26 | memset(toVisitLists[1],0,sizeof(__m128i)*subgraphSize); 27 | 28 | // const uint32_t numQueries = bfsData.size(); 29 | // assert(numQueries>0 && numQueries<=128); 30 | 31 | __m128i* seen = new __m128i[subgraphSize]; 32 | memset(seen, 0, sizeof(__m128i)*subgraphSize); 33 | 34 | runBatchRound(bfsData, subgraph, 9999999, toVisitLists, seen 35 | #ifdef STATISTICS 36 | , statistics 37 | #endif 38 | ); 39 | 40 | delete[] seen; 41 | delete[] toVisitLists[0]; 42 | delete[] toVisitLists[1]; 43 | 44 | #ifdef STATISTICS 45 | statistics.finishBatch(); 46 | #endif 47 | } 48 | 49 | void BatchBFSRunner128::runBatchRound(vector& bfsData, const PersonSubgraph& subgraph, PersonId minPerson, array<__m128i*,2>& toVisitLists, __m128i* __restrict__ seen 50 | #ifdef STATISTICS 51 | , BatchStatistics& statistics 52 | #endif 53 | ) { 54 | const auto subgraphSize = subgraph.size(); 55 | const uint32_t numQueries = bfsData.size(); 56 | 57 | const __m128i allOnes = _mm_set_epi32(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF); 58 | 59 | __m128i processQuery = allOnes; 60 | uint32_t queriesToProcess=numQueries; 61 | 62 | uint32_t numDistDiscovered[128] __attribute__((aligned(16))); 63 | memset(numDistDiscovered,0,sizeof(uint32_t)*numQueries); 64 | 65 | uint8_t curToVisitQueue = 0; 66 | 67 | bool nextToVisitEmpty = true; 68 | BatchDistance<__m128i, 1> batchDist(numDistDiscovered); 69 | 70 | 71 | map firstToVisit; 72 | for(size_t a=0; asecond = _mm_or_si128(vITer->second, sseMasks[a]); 85 | } 86 | 87 | minPerson = min(minPerson, *friendsBounds.first); 88 | ++friendsBounds.first; 89 | } 90 | } 91 | for(const auto& t : firstToVisit) { 92 | (toVisitLists[0])[t.first] = t.second; 93 | seen[t.first] = _mm_or_si128(seen[t.first], t.second); 94 | batchDist.updateDiscovered(t.second, 0); 95 | } 96 | for(uint32_t a=0; a 8 | 9 | using namespace std; 10 | 11 | namespace Query4 { 12 | 13 | #ifdef AVX2 14 | void BatchBFSRunner256::runBatch(vector& bfsData, const PersonSubgraph& subgraph) { 15 | const auto subgraphSize = subgraph.size(); 16 | 17 | __m256i** toVisitLists = new __m256i*[2]; 18 | auto ret=posix_memalign(reinterpret_cast(toVisitLists),64,sizeof(__m256i)*subgraphSize); 19 | if(unlikely(ret!=0)) { 20 | throw -1; 21 | } 22 | memset(toVisitLists[0],0,sizeof(__m256i)*subgraphSize); 23 | ret=posix_memalign(reinterpret_cast(toVisitLists+1),64,sizeof(__m256i)*subgraphSize); 24 | if(unlikely(ret!=0)) { 25 | throw -1; 26 | } 27 | memset(toVisitLists[1],0,sizeof(__m256i)*subgraphSize); 28 | 29 | const uint32_t numQueries = bfsData.size(); 30 | assert(numQueries>0 && numQueries<=256); 31 | 32 | PersonId minPerson = numeric_limits::max(); 33 | __m256i* seen; 34 | ret=posix_memalign(reinterpret_cast(&seen),64,sizeof(__m256i)*subgraphSize); 35 | if(unlikely(ret!=0)) { 36 | throw -1; 37 | } 38 | memset(seen, 0, sizeof(__m256i)*subgraphSize); 39 | for(size_t a=0; a& bfsData, const PersonSubgraph& subgraph, PersonId minPerson, __m256i** toVisitLists, __m256i* __restrict__ seen) { 55 | const auto subgraphSize = subgraph.size(); 56 | const uint32_t numQueries = bfsData.size(); 57 | 58 | const __m256i allZeros = _mm256_set_epi32(0,0,0,0,0,0,0,0); 59 | const __m256i allOnes = _mm256_set_epi32(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF); 60 | const __m256i byteOnes = _mm256_set_epi8(1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1); 61 | 62 | __m256i processQuery = allOnes; 63 | uint32_t queriesToProcess=numQueries; 64 | 65 | __m256i numDistDiscovered[8]; 66 | for(int a=0; a<8; a++) { 67 | numDistDiscovered[a] = allZeros; 68 | } 69 | uint32_t numDistDiscoveredIteration=0; 70 | 71 | uint8_t curToVisitQueue = 0; 72 | uint32_t nextDistance = 1; 73 | 74 | auto updateDiscoveredNodes = [&bfsData, &numDistDiscovered, &nextDistance, &allZeros]() { 75 | for(int a=0; a<8; a++) { 76 | const auto e0 = _mm256_extract_epi8(numDistDiscovered[a], 0); bfsData[a+8*0].totalReachable += e0; bfsData[a+8*0].totalDistances += e0*nextDistance; 77 | const auto e1 = _mm256_extract_epi8(numDistDiscovered[a], 1); bfsData[a+8*1].totalReachable += e1; bfsData[a+8*1].totalDistances += e1*nextDistance; 78 | const auto e2 = _mm256_extract_epi8(numDistDiscovered[a], 2); bfsData[a+8*2].totalReachable += e2; bfsData[a+8*2].totalDistances += e2*nextDistance; 79 | const auto e3 = _mm256_extract_epi8(numDistDiscovered[a], 3); bfsData[a+8*3].totalReachable += e3; bfsData[a+8*3].totalDistances += e3*nextDistance; 80 | const auto e4 = _mm256_extract_epi8(numDistDiscovered[a], 4); bfsData[a+8*4].totalReachable += e4; bfsData[a+8*4].totalDistances += e4*nextDistance; 81 | const auto e5 = _mm256_extract_epi8(numDistDiscovered[a], 5); bfsData[a+8*5].totalReachable += e5; bfsData[a+8*5].totalDistances += e5*nextDistance; 82 | const auto e6 = _mm256_extract_epi8(numDistDiscovered[a], 6); bfsData[a+8*6].totalReachable += e6; bfsData[a+8*6].totalDistances += e6*nextDistance; 83 | const auto e7 = _mm256_extract_epi8(numDistDiscovered[a], 7); bfsData[a+8*7].totalReachable += e7; bfsData[a+8*7].totalDistances += e7*nextDistance; 84 | const auto e8 = _mm256_extract_epi8(numDistDiscovered[a], 8); bfsData[a+8*8].totalReachable += e8; bfsData[a+8*8].totalDistances += e8*nextDistance; 85 | const auto e9 = _mm256_extract_epi8(numDistDiscovered[a], 9); bfsData[a+8*9].totalReachable += e9; bfsData[a+8*9].totalDistances += e9*nextDistance; 86 | const auto e10 = _mm256_extract_epi8(numDistDiscovered[a], 10); bfsData[a+8*10].totalReachable += e10; bfsData[a+8*10].totalDistances += e10*nextDistance; 87 | const auto e11 = _mm256_extract_epi8(numDistDiscovered[a], 11); bfsData[a+8*11].totalReachable += e11; bfsData[a+8*11].totalDistances += e11*nextDistance; 88 | const auto e12 = _mm256_extract_epi8(numDistDiscovered[a], 12); bfsData[a+8*12].totalReachable += e12; bfsData[a+8*12].totalDistances += e12*nextDistance; 89 | const auto e13 = _mm256_extract_epi8(numDistDiscovered[a], 13); bfsData[a+8*13].totalReachable += e13; bfsData[a+8*13].totalDistances += e13*nextDistance; 90 | const auto e14 = _mm256_extract_epi8(numDistDiscovered[a], 14); bfsData[a+8*14].totalReachable += e14; bfsData[a+8*14].totalDistances += e14*nextDistance; 91 | const auto e15 = _mm256_extract_epi8(numDistDiscovered[a], 15); bfsData[a+8*15].totalReachable += e15; bfsData[a+8*15].totalDistances += e15*nextDistance; 92 | const auto e16 = _mm256_extract_epi8(numDistDiscovered[a], 16); bfsData[a+8*16].totalReachable += e16; bfsData[a+8*16].totalDistances += e16*nextDistance; 93 | const auto e17 = _mm256_extract_epi8(numDistDiscovered[a], 17); bfsData[a+8*17].totalReachable += e17; bfsData[a+8*17].totalDistances += e17*nextDistance; 94 | const auto e18 = _mm256_extract_epi8(numDistDiscovered[a], 18); bfsData[a+8*18].totalReachable += e18; bfsData[a+8*18].totalDistances += e18*nextDistance; 95 | const auto e19 = _mm256_extract_epi8(numDistDiscovered[a], 19); bfsData[a+8*19].totalReachable += e19; bfsData[a+8*19].totalDistances += e19*nextDistance; 96 | const auto e20 = _mm256_extract_epi8(numDistDiscovered[a], 20); bfsData[a+8*20].totalReachable += e20; bfsData[a+8*20].totalDistances += e20*nextDistance; 97 | const auto e21 = _mm256_extract_epi8(numDistDiscovered[a], 21); bfsData[a+8*21].totalReachable += e21; bfsData[a+8*21].totalDistances += e21*nextDistance; 98 | const auto e22 = _mm256_extract_epi8(numDistDiscovered[a], 22); bfsData[a+8*22].totalReachable += e22; bfsData[a+8*22].totalDistances += e22*nextDistance; 99 | const auto e23 = _mm256_extract_epi8(numDistDiscovered[a], 23); bfsData[a+8*23].totalReachable += e23; bfsData[a+8*23].totalDistances += e23*nextDistance; 100 | const auto e24 = _mm256_extract_epi8(numDistDiscovered[a], 24); bfsData[a+8*24].totalReachable += e24; bfsData[a+8*24].totalDistances += e24*nextDistance; 101 | const auto e25 = _mm256_extract_epi8(numDistDiscovered[a], 25); bfsData[a+8*25].totalReachable += e25; bfsData[a+8*25].totalDistances += e25*nextDistance; 102 | const auto e26 = _mm256_extract_epi8(numDistDiscovered[a], 26); bfsData[a+8*26].totalReachable += e26; bfsData[a+8*26].totalDistances += e26*nextDistance; 103 | const auto e27 = _mm256_extract_epi8(numDistDiscovered[a], 27); bfsData[a+8*27].totalReachable += e27; bfsData[a+8*27].totalDistances += e27*nextDistance; 104 | const auto e28 = _mm256_extract_epi8(numDistDiscovered[a], 28); bfsData[a+8*28].totalReachable += e28; bfsData[a+8*28].totalDistances += e28*nextDistance; 105 | const auto e29 = _mm256_extract_epi8(numDistDiscovered[a], 29); bfsData[a+8*29].totalReachable += e29; bfsData[a+8*29].totalDistances += e29*nextDistance; 106 | const auto e30 = _mm256_extract_epi8(numDistDiscovered[a], 30); bfsData[a+8*30].totalReachable += e30; bfsData[a+8*30].totalDistances += e30*nextDistance; 107 | const auto e31 = _mm256_extract_epi8(numDistDiscovered[a], 31); bfsData[a+8*31].totalReachable += e31; bfsData[a+8*31].totalDistances += e31*nextDistance; 108 | } 109 | for(int a=0; a<8; a++) { 110 | numDistDiscovered[a] = allZeros; 111 | } 112 | }; 113 | 114 | #ifdef STATISTICS 115 | BatchStatistics statistics; 116 | #endif 117 | 118 | PersonId curPerson=minPerson; 119 | bool nextToVisitEmpty = true; 120 | do { 121 | const __m256i* const toVisit = toVisitLists[curToVisitQueue]; 122 | __m256i* const nextToVisit = toVisitLists[1-curToVisitQueue]; 123 | 124 | for(; curPerson 8 | 9 | using namespace std; 10 | 11 | namespace Query4 { 12 | 13 | void BatchBFSRunner::runBatch(vector& bfsData, const PersonSubgraph& subgraph 14 | #ifdef STATISTICS 15 | , BatchStatistics&/* statistics */ 16 | #endif 17 | ) { 18 | const auto subgraphSize = subgraph.size(); 19 | 20 | array toVisitLists; 21 | toVisitLists[0] = new uint64_t[subgraphSize]; 22 | memset(toVisitLists[0],0,sizeof(uint64_t)*subgraphSize); 23 | toVisitLists[1] = new uint64_t[subgraphSize]; 24 | memset(toVisitLists[1],0,sizeof(uint64_t)*subgraphSize); 25 | 26 | const uint32_t numQueries = bfsData.size(); 27 | assert(numQueries>0 && numQueries<=64); 28 | 29 | PersonId minPerson = numeric_limits::max(); 30 | uint64_t* seen = new uint64_t[subgraphSize]; 31 | memset(seen, 0, sizeof(uint64_t)*subgraphSize); 32 | for(size_t a=0; a& bfsData, const PersonSubgraph& subgraph, PersonId minPerson, array& toVisitLists, uint64_t* __restrict__ seen) { 48 | const auto subgraphSize = subgraph.size(); 49 | const uint32_t numQueries = bfsData.size(); 50 | 51 | uint64_t processQuery = (~0UL); 52 | uint32_t queriesToProcess=numQueries; 53 | 54 | uint32_t numDistDiscovered[64] __attribute__((aligned(16))); 55 | memset(numDistDiscovered,0,sizeof(uint32_t)*numQueries); 56 | 57 | uint8_t curToVisitQueue = 0; 58 | uint32_t nextDistance = 1; 59 | 60 | PersonId curPerson=minPerson; 61 | bool nextToVisitEmpty = true; 62 | BatchDistance batchDist(numDistDiscovered); 63 | do { 64 | const uint64_t* const toVisit = toVisitLists[curToVisitQueue]; 65 | uint64_t* const nextToVisit = toVisitLists[1-curToVisitQueue]; 66 | 67 | for(; curPerson>(firstQueryId+1)) == 0) { 76 | // //Only single person in this entry 77 | 78 | // auto friendsBounds = curFriends.bounds(); 79 | // while(friendsBounds.first != friendsBounds.second) { 80 | // if(toVisitProcess & (~seen[*friendsBounds.first])) { 81 | // seen[*friendsBounds.first] |= toVisitEntry; 82 | // nextToVisit[*friendsBounds.first] |= toVisitEntry; 83 | // nextToVisitEmpty = false; 84 | // numDistDiscovered[firstQueryId]++; 85 | // } 86 | // ++friendsBounds.first; 87 | // } 88 | // } else { 89 | //More than one person 90 | auto friendsBounds = curFriends.bounds(); 91 | while(friendsBounds.first != friendsBounds.second) { 92 | 93 | //XXX: TODO processQuery needed here? 94 | uint64_t newToVisit = toVisitProcess & (~seen[*friendsBounds.first]); //!seen & toVisit 95 | if(newToVisit == 0) { 96 | ++friendsBounds.first; 97 | continue; 98 | } 99 | 100 | seen[*friendsBounds.first] |= toVisitEntry; 101 | nextToVisit[*friendsBounds.first] |= newToVisit; 102 | 103 | //TODO: Profile-based approach, use shift-based counting once many bits are set 104 | 105 | batchDist.updateDiscovered(newToVisit, 0); 106 | ++friendsBounds.first; 107 | } 108 | // } 109 | 110 | //Go to next person 111 | curPerson++; 112 | } else { 113 | batchDist.finalize(); 114 | //Swap queues 115 | for(uint32_t a=0; a::max()); 30 | distance++; 31 | 32 | // Adjust result 33 | bfsData.totalReachable+=numDiscovered; 34 | bfsData.totalDistances+=numDiscovered*distance; 35 | 36 | TraceStats& stats = TraceStats::getStats(); 37 | stats.traceRound(distance); 38 | stats.addRoundDuration(distance, (tschrono::now()-startTime)); 39 | 40 | // Exit criteria for full BFS 41 | if((bfsData.componentSize-1)==bfsData.totalReachable) { 42 | break; 43 | } 44 | } while(true); 45 | 46 | delete[] seen; 47 | } 48 | 49 | // XXX: __attribute__((optimize("align-loops"))) this crashed the compiler 50 | Persons __attribute__((hot)) BFSRunner::runRound(const PersonSubgraph& subgraph, Distance* __restrict__ seen, BFSQueue& toVisit, const Persons numToVisit, const Persons numUnseen) { 51 | Persons numRemainingToVisit=numToVisit; 52 | Persons numRemainingUnseen=numUnseen; 53 | 54 | do { 55 | assert(!toVisit.empty()); 56 | 57 | const PersonId person = toVisit.front(); 58 | toVisit.pop_front(); 59 | 60 | // Iterate over friends 61 | auto friendsBounds = subgraph.retrieve(person)->bounds(); 62 | while(friendsBounds.first != friendsBounds.second) { 63 | if (seen[*friendsBounds.first]) { 64 | ++friendsBounds.first; 65 | continue; 66 | } 67 | toVisit.push_back_pos() = *friendsBounds.first; 68 | 69 | seen[*friendsBounds.first] = true; 70 | ++friendsBounds.first; 71 | numRemainingUnseen--; 72 | } 73 | 74 | numRemainingToVisit--; 75 | } while(numRemainingToVisit>0 && numRemainingUnseen>0); 76 | 77 | return numUnseen-numRemainingUnseen; 78 | } 79 | 80 | } -------------------------------------------------------------------------------- /bfs/noqueue.cpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #include "../include/bfs/noqueue.hpp" 5 | #include "../include/TraceStats.hpp" 6 | 7 | #include 8 | 9 | namespace Query4 { 10 | 11 | void __attribute__ ((noinline)) NoQueueBFSRunner::run(const PersonId start, const PersonSubgraph& subgraph, BatchBFSdata& bfsData) { 12 | // Initialize BFS 13 | Distance* seen = new Distance[subgraph.size()](); 14 | seen[start] = true; // Level = distance + 1, Level 0 = not seen 15 | 16 | Distance* currentVisit = new Distance[subgraph.size()](); 17 | Distance* nextVisit = new Distance[subgraph.size()](); 18 | currentVisit[start] = true; 19 | 20 | // Run rounds until we can either early exit or have reached all nodes 21 | Distance distance=0; 22 | do { 23 | size_t startTime = tschrono::now(); 24 | 25 | // const Persons personsRemaining=(bfsData.componentSize-1)-bfsData.totalReachable; 26 | Persons numDiscovered = runRound(subgraph, seen, currentVisit, nextVisit); 27 | assert(distance::max()); 28 | distance++; 29 | 30 | // Adjust result 31 | bfsData.totalReachable+=numDiscovered; 32 | bfsData.totalDistances+=numDiscovered*distance; 33 | 34 | TraceStats& stats = TraceStats::getStats(); 35 | stats.traceRound(distance); 36 | stats.addRoundDuration(distance, (tschrono::now()-startTime)); 37 | 38 | // Exit criteria for full BFS 39 | if((bfsData.componentSize-1)==bfsData.totalReachable) { 40 | break; 41 | } 42 | 43 | // Swap queues 44 | Distance* tmpVisit = currentVisit; 45 | currentVisit = nextVisit; 46 | nextVisit = tmpVisit; 47 | memset(nextVisit, 0, sizeof(Distance)*subgraph.size()); 48 | } while(true); 49 | 50 | delete[] seen; 51 | delete[] currentVisit; 52 | delete[] nextVisit; 53 | } 54 | 55 | Persons __attribute__((hot)) NoQueueBFSRunner::runRound(const PersonSubgraph& subgraph, Distance* __restrict__ seen, const Distance* __restrict__ currentVisit, Distance* __restrict__ nextVisit) { 56 | // XXX: Search set bytes with memchr? 57 | // XXX: Prefetch 58 | // XXX: Track remaining unseen 59 | const auto numPersons = subgraph.size(); 60 | Query4::Persons numDiscovered = 0; 61 | for(Persons person=0; personbounds(); 67 | while(friendsBounds.first != friendsBounds.second) { 68 | if (seen[*friendsBounds.first]) { 69 | ++friendsBounds.first; 70 | continue; 71 | } 72 | 73 | seen[*friendsBounds.first] = true; 74 | nextVisit[*friendsBounds.first] = true; 75 | ++friendsBounds.first; 76 | numDiscovered++; 77 | } 78 | } 79 | 80 | return numDiscovered; 81 | } 82 | 83 | } -------------------------------------------------------------------------------- /bfs/parabfs.cpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #include "../include/bfs/parabfs.hpp" 5 | #include "../include/TraceStats.hpp" 6 | 7 | namespace Query4 { 8 | 9 | PARABFSRunner::KData* PARABFSRunner::kdata = new PARABFSRunner::KData(); 10 | bool PARABFSRunner::isInitialized = false; 11 | 12 | void PARABFSRunner::run(const PersonId s, const PersonSubgraph& subgraph, BatchBFSdata& bfsData 13 | #ifdef STATISTICS 14 | , BatchStatistics&/* statistics */ 15 | #endif 16 | ) { 17 | memset(PARABFSRunner::kdata->visited, 0, PARABFSRunner::kdata->personSize*sizeof(bool)); 18 | PARABFSRunner::kdata->visited[s] = 1; 19 | uint32_t * cqueue = &PARABFSRunner::kdata->queue[kdata->numThreads*(PARABFSRunner::kdata->personSize)]; 20 | cqueue[0] = s; //Initialize current queue 21 | uint32_t cq = 1; //size of the current-queue 22 | int level = 0; 23 | for(int i=0; inumThreads; i++) { 24 | PARABFSRunner::kdata->params[i].sp = 0; 25 | PARABFSRunner::kdata->params[i].rp = 0; 26 | PARABFSRunner::kdata->params[i].level = 0; 27 | } 28 | Distance distance=0; 29 | while(cq > 0) { 30 | for(int i=0; inumThreads; i++) 31 | PARABFSRunner::kdata->params[i].cq_end = cq; 32 | size_t startTime = tschrono::now(); 33 | dispatchToWorkers(); 34 | waitForWorkers(); 35 | assert(distance::max()); 36 | distance++; 37 | TraceStats& stats = TraceStats::getStats(); 38 | stats.traceRound(distance); 39 | stats.addRoundDuration(distance, (tschrono::now()-startTime)); 40 | 41 | cq = 0; 42 | for(int i=0; inumThreads; ++i) { 43 | #if USE_ATOMIC 44 | memcpy(cqueue+cq, PARABFSRunner::kdata->params[i].nqueue, sizeof(uint32_t)*PARABFSRunner::kdata->params[i].nq); 45 | cq += PARABFSRunner::kdata->params[i].nq; 46 | #else 47 | uint32_t *nQ = PARABFSRunner::kdata->params[i].nqueue; 48 | for (int j=0; jparams[i].nq; ++j) { 49 | if (PARABFSRunner::kdata->visited2[nQ[j]]) { 50 | PARABFSRunner::kdata->visited2[nQ[j]] = false; 51 | cqueue[cq++] = nQ[j]; 52 | PARABFSRunner::kdata->params[i].sp += PARABFSRunner::kdata->params[i].level; 53 | ++PARABFSRunner::kdata->params[i].rp; 54 | } 55 | } 56 | #endif 57 | PARABFSRunner::kdata->params[i].nq = 0; 58 | } 59 | 60 | //bfsData.totalReachable+=numDiscovered; 61 | bfsData.totalReachable+=cq; 62 | //bfsData.totalDistances+=numDiscovered*distance; 63 | bfsData.totalDistances+=cq*distance; 64 | if((bfsData.componentSize-1)==bfsData.totalReachable) { 65 | break; 66 | } 67 | } 68 | 69 | 70 | } 71 | 72 | void PARABFSRunner::traverse(TraverseParams *tp, const PersonSubgraph& subgraph){ 73 | uint32_t *cqueue = tp->cqueue; 74 | uint32_t *nqueue = tp->nqueue; 75 | uint32_t cq = tp->cq_start; 76 | uint32_t cq_end = tp->cq_end; 77 | uint32_t st = tp->stride; 78 | uint32_t nq = tp->nq; 79 | uint64_t sp = tp->sp; 80 | uint64_t rp = tp->rp; 81 | uint32_t level = ++tp->level; 82 | for(; cqbounds(); 86 | while(friendsBounds.first != friendsBounds.second) { 87 | uint32_t v = *friendsBounds.first; 88 | { 89 | if (!PARABFSRunner::kdata->visited[v]) //This command would reduces executing atomic operations 90 | { 91 | #if USE_ATOMIC 92 | bool b = __sync_fetch_and_or(PARABFSRunner::kdata->visited+v, true); 93 | // for GCC >= 4.7, use the below instead 94 | // bool b = __atomic_fetch_or(visited+v, true, __ATOMIC_RELAXED); 95 | if (!b) 96 | { 97 | nqueue[nq++] = v; 98 | sp += level; 99 | rp += 1; 100 | } 101 | #else 102 | PARABFSRunner::kdata->visited[v] = true; 103 | PARABFSRunner::kdata->visited2[v] = true; 104 | nqueue[nq++] = v; 105 | #endif 106 | } 107 | } 108 | ++friendsBounds.first; 109 | } 110 | } 111 | tp->set(nq, sp, rp); 112 | } 113 | 114 | void PARABFSRunner::start(std::vector ¶ms, const PersonSubgraph& subgraph) { 115 | if (!PARABFSRunner::kdata->shutdown) 116 | return; 117 | PARABFSRunner::kdata->shutdown = false; 118 | PARABFSRunner::kdata->workerBusy = kdata->numThreads; 119 | for (int i=0; inumThreads; i++) { 120 | PARABFSRunner::kdata->workers.create_thread(boost::bind(&PARABFSRunner::worker, ¶ms[i], boost::cref(subgraph))); 121 | } 122 | waitForWorkers(); 123 | } 124 | 125 | void PARABFSRunner::stop() { 126 | PARABFSRunner::kdata->shutdown = true; 127 | PARABFSRunner::kdata->condWork.notify_all(); 128 | } 129 | 130 | void PARABFSRunner::dispatchToWorkers() 131 | { 132 | PARABFSRunner::kdata->workerBusy = kdata->numThreads; 133 | PARABFSRunner::kdata->condWork.notify_all(); 134 | } 135 | 136 | void PARABFSRunner::pfree(){ 137 | free(PARABFSRunner::kdata->queue); 138 | free(PARABFSRunner::kdata->visited); 139 | free(PARABFSRunner::kdata->cc); 140 | } 141 | 142 | void PARABFSRunner::waitForWorkers() { 143 | boost::mutex::scoped_lock lock(PARABFSRunner::kdata->mutex); 144 | if (PARABFSRunner::kdata->workerBusy>0) 145 | PARABFSRunner::kdata->condDone.wait(lock); 146 | } 147 | 148 | void PARABFSRunner::worker(TraverseParams *param, const PersonSubgraph& subgraph) { 149 | while (1) { 150 | { 151 | boost::mutex::scoped_lock lock(PARABFSRunner::kdata->mutex); 152 | if (--PARABFSRunner::kdata->workerBusy==0) 153 | PARABFSRunner::kdata->condDone.notify_all(); 154 | PARABFSRunner::kdata->condWork.wait(lock); 155 | } 156 | if (PARABFSRunner::kdata->shutdown) break; 157 | traverse(param, subgraph); 158 | } 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /bfs/sc2012.cpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #include "../include/bfs/sc2012.hpp" 5 | #include "../include/TraceStats.hpp" 6 | 7 | 8 | namespace Query4 { 9 | 10 | void __attribute__ ((noinline)) SCBFSRunner::run(const PersonId start, const PersonSubgraph& subgraph, BatchBFSdata& bfsData 11 | #ifdef STATISTICS 12 | , BatchStatistics&/* statistics */ 13 | #endif 14 | ) { 15 | const auto subgraphSize = subgraph.size(); 16 | 17 | // Initialize BFS 18 | std::array visitLists; 19 | int curVisitList=0; 20 | CustomBFSData seen(subgraphSize); 21 | visitLists[0].assign(subgraphSize, 0); 22 | visitLists[1].assign(subgraphSize, 0); 23 | seen[start] = 1; 24 | (visitLists[0])[start] = 1; 25 | 26 | // Initialize switching condition 27 | uint32_t m_frontier, last_m_frontier, m_unexplored, n_frontier, top_down, bottom_up; 28 | bool is_top_down = true; 29 | m_frontier = 0; 30 | last_m_frontier = 0; 31 | m_unexplored = subgraph.numEdges; 32 | 33 | // Terminating condition 34 | size_t frontierSize = 1; 35 | 36 | //fprintf(stderr, "Starting new BFS\n"); 37 | 38 | Distance distance=0; 39 | while(frontierSize > 0){ 40 | size_t startTime = tschrono::now(); 41 | 42 | SCBFSRunner::CustomBFSData& frontier = visitLists[curVisitList]; 43 | SCBFSRunner::CustomBFSData& next = visitLists[1-curVisitList]; 44 | 45 | 46 | // Switch between top-down and bottom-up 47 | last_m_frontier = m_frontier; 48 | m_frontier = 0; 49 | for (size_t i = 0; i < subgraphSize; ++i){ 50 | if (frontier[i] == 0) continue; 51 | const auto friends = subgraph.retrieve(i); 52 | m_frontier += friends->size(); 53 | } 54 | n_frontier = frontierSize; 55 | m_unexplored -= last_m_frontier; 56 | 57 | top_down = m_unexplored / SCBFSRunner::alpha; 58 | bottom_up = subgraph.size() / SCBFSRunner::beta; 59 | 60 | //fprintf(stderr, "m_frontier = %d, n_frontier = %d, m_unexplored = %d\n", m_frontier, n_frontier, m_unexplored); 61 | //fprintf(stderr, "top_down = %d, bottom_up = %d\n", top_down, bottom_up); 62 | 63 | if (is_top_down){ 64 | if (m_frontier <= top_down){ 65 | frontierSize = runRoundTopDown(subgraph, frontier, next, seen); 66 | is_top_down = true; 67 | } else { 68 | frontierSize = runRoundBottomUp(subgraph, frontier, next, seen); 69 | is_top_down = false; 70 | } 71 | } else { 72 | if (n_frontier >= bottom_up){ 73 | frontierSize = runRoundBottomUp(subgraph, frontier, next, seen); 74 | is_top_down = false; 75 | } else { 76 | frontierSize = runRoundTopDown(subgraph, frontier, next, seen); 77 | is_top_down = true; 78 | } 79 | } 80 | 81 | frontier.assign(subgraphSize, 0); 82 | curVisitList = 1-curVisitList; 83 | 84 | //Closeness centrality 85 | Persons numDiscovered = frontierSize; 86 | assert(distance::max()); 87 | distance++; 88 | 89 | // Adjust result 90 | bfsData.totalReachable+=numDiscovered; 91 | bfsData.totalDistances+=numDiscovered*distance; 92 | 93 | TraceStats& stats = TraceStats::getStats(); 94 | stats.traceRound(distance); 95 | stats.addRoundDuration(distance, (tschrono::now()-startTime)); 96 | } 97 | } 98 | 99 | size_t __attribute__((hot))SCBFSRunner::runRoundTopDown(const PersonSubgraph& subgraph,SCBFSRunner::CustomBFSData& frontier,SCBFSRunner::CustomBFSData& next,SCBFSRunner::CustomBFSData& seen){ 100 | //fprintf(stderr, "Top-Down\n"); 101 | size_t frontierSize = 0; 102 | const size_t subgraphSize = frontier.size(); 103 | for (size_t i = 0; i < subgraphSize; ++i){ 104 | if (frontier[i] == 0) continue; 105 | auto friendsBounds = subgraph.retrieve(i)->bounds(); 106 | while(friendsBounds.first != friendsBounds.second) { 107 | if (seen[*friendsBounds.first] == 0){ 108 | seen[*friendsBounds.first] = 1; 109 | next[*friendsBounds.first] = 1; 110 | frontierSize++; 111 | } 112 | ++friendsBounds.first; 113 | } 114 | } 115 | return frontierSize; 116 | } 117 | 118 | size_t __attribute__((hot))SCBFSRunner::runRoundBottomUp(const PersonSubgraph& subgraph,SCBFSRunner::CustomBFSData& frontier,SCBFSRunner::CustomBFSData& next,SCBFSRunner::CustomBFSData& seen){ 119 | //fprintf(stderr, "Bottom-Up\n"); 120 | size_t frontierSize = 0; 121 | const size_t subgraphSize = frontier.size(); 122 | for (size_t i = 0; i < subgraphSize; ++i){ 123 | if (seen[i] == 0){ 124 | auto friendsBounds = subgraph.retrieve(i)->bounds(); 125 | while(friendsBounds.first != friendsBounds.second) { 126 | PersonId n = *friendsBounds.first; 127 | if (frontier[n] != 0){ 128 | seen[i] = 1; 129 | next[i] = 1; 130 | frontierSize++; 131 | break; 132 | } 133 | ++friendsBounds.first; 134 | } 135 | } 136 | } 137 | return frontierSize; 138 | } 139 | 140 | } -------------------------------------------------------------------------------- /compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | HOMEDIR=$PWD/$(dirname $0) #this is actually src dir 3 | echo "Extracting boost..." 4 | tar xzf ${HOMEDIR}/vida/lib/boost.tar.gz -C ${HOMEDIR}/. 5 | cd ${HOMEDIR}/boost 6 | echo "Bootstrapping..." 7 | ./bootstrap.sh --with-libraries=chrono,filesystem,iostreams,system,thread,timer 8 | echo "Compiling boost..." 9 | ./b2 -s"NO_BZIP2=1" -j8 threading=multi variant=release link=static 10 | cd ${HOMEDIR} 11 | make -j8 12 | -------------------------------------------------------------------------------- /graph.cpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #include "include/tokenizer.hpp" 5 | #include "include/graph.hpp" 6 | #include "include/io.hpp" 7 | 8 | #include 9 | 10 | using namespace std; 11 | 12 | GraphData GraphData::loadFromPath(const std::string& edgesFile) { 13 | // Count number of persons (excluding header line) 14 | size_t numEdges = io::fileLines(edgesFile)-1; 15 | LOG_PRINT("[LOADING] Number of edges: "< edges; 26 | edges.reserve(numEdges); 27 | 28 | std::unordered_map nodeRenaming; 29 | std::unordered_map revNodeRenaming; 30 | uint64_t nextNodeId=0; 31 | 32 | auto mapExternalNodeId = [&nodeRenaming, &revNodeRenaming, &nextNodeId](uint64_t id) -> uint64_t { 33 | if(nodeRenaming.find(id)==nodeRenaming.end()) { 34 | nodeRenaming[id] = nextNodeId; 35 | revNodeRenaming[nextNodeId] = id; 36 | return nextNodeId++; 37 | } else { 38 | return nodeRenaming[id]; 39 | } 40 | }; 41 | 42 | while(!tokenizer.isFinished()) { 43 | assert(edges.size() uniqueEdges(edges.size()); 64 | size_t e=0; 65 | NodePair last(numeric_limits::max(), numeric_limits::max()); 66 | for(const NodePair& a : edges) { 67 | if(last.idA!=a.idA || last.idB!=a.idB) { 68 | last = a; 69 | uniqueEdges[e++] = NodePair(mapExternalNodeId(a.idA), a.idB); 70 | } 71 | } 72 | uniqueEdges.resize(e); 73 | for(NodePair& a : uniqueEdges) { 74 | a.idB = mapExternalNodeId(a.idB); 75 | } 76 | LOG_PRINT("[LOADING] Sorted edges"); 77 | 78 | LOG_PRINT("[LOADING] Number of nodes: "< 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | template 15 | class TraceStats { 16 | size_t numVertices; 17 | size_t numEdges; 18 | size_t numTraversedEdges; 19 | size_t batchSize; 20 | size_t runnerType; 21 | size_t typeSize; 22 | size_t width; 23 | size_t numThreads; 24 | size_t maxBfs; 25 | std::string bfsType; 26 | 27 | std::array, maxRounds> numRounds; 28 | std::array, maxRounds> roundDurations; 29 | std::array, numBits>, maxRounds> roundVisitBits; 30 | std::array, numBits>, maxRounds> roundFriendBits; 31 | 32 | TraceStats() : numRounds(), roundDurations(), roundVisitBits(), roundFriendBits() { 33 | 34 | } 35 | public: 36 | static TraceStats& getStats(bool reset=false) { 37 | static TraceStats* globalStats; 38 | static std::mutex m; 39 | std::lock_guard lock(m); 40 | if(reset && globalStats != nullptr) { 41 | globalStats = nullptr; 42 | } 43 | if(globalStats == nullptr) { 44 | globalStats = new TraceStats(); 45 | } 46 | return *globalStats; 47 | } 48 | 49 | void init(size_t numVertices, size_t numEdges, size_t batchSize, size_t runnerType, size_t typeSize, size_t width, size_t numThreads, size_t maxBfs, std::string bfsType) { 50 | this->numVertices = numVertices; 51 | this->numEdges = numEdges; 52 | this->numTraversedEdges = 0; 53 | this->batchSize = batchSize; 54 | this->runnerType = runnerType; 55 | this->typeSize = typeSize; 56 | this->width = width; 57 | this->numThreads = numThreads; 58 | this->maxBfs = maxBfs; 59 | this->bfsType = bfsType; 60 | } 61 | 62 | void traceRound(size_t round) { 63 | numRounds[round]++; 64 | } 65 | 66 | void addRoundDuration(size_t round, size_t duration) { 67 | roundDurations[round] += duration; 68 | } 69 | 70 | void addRoundVisitBits(size_t round, size_t visitBits, size_t count) { 71 | roundVisitBits[round][visitBits]+=count; 72 | } 73 | 74 | void addRoundFriendBits(size_t round, size_t friendBits, size_t count) { 75 | roundFriendBits[round][friendBits]+=count; 76 | } 77 | 78 | void setNumTraversedEdges(size_t c) { 79 | numTraversedEdges = c; 80 | } 81 | 82 | std::string prelude(std::string metric, size_t totalDuration) { 83 | return "["+metric+"]\t"+std::to_string(numVertices)+"\t"+std::to_string(numEdges)+"\t"+std::to_string(numTraversedEdges)+"\t"+std::to_string(batchSize)+"\t"+std::to_string(numThreads) 84 | +"\t"+std::to_string(runnerType)+"\t"+std::to_string(typeSize)+"\t"+std::to_string(width)+"\t"+std::to_string(totalDuration)+"\t"+std::to_string(maxBfs)+"\t"+bfsType; 85 | } 86 | 87 | std::string print(size_t totalDuration) { 88 | size_t lastRound = 0; 89 | for (int i = 1; i < maxRounds; ++i) { 90 | if(numRounds[i]>0) { lastRound = i; } 91 | } 92 | 93 | std::stringstream ss; 94 | ss< queries; 22 | 23 | static Queries loadFromFile(const std::string& queryFile) { 24 | io::MmapedFile file(queryFile, O_RDONLY); 25 | 26 | Queries queries; 27 | 28 | Tokenizer tokenizer(file.mapping, file.size); 29 | // Read queries 30 | while(!tokenizer.isFinished()) { 31 | Query query; 32 | query.numNodes = tokenizer.readId(' '); 33 | query.dataset = tokenizer.readStr(' '); 34 | query.reference = tokenizer.readStr('\n'); 35 | if(query.reference.size()==0) { 36 | FATAL_ERROR("[Queries] Could not load reference result, is it specified?"); 37 | } 38 | queries.queries.push_back(query); 39 | } 40 | 41 | LOG_PRINT("[Queries] Loaded "<< queries.queries.size()<< " queries."); 42 | return queries; 43 | } 44 | }; 45 | 46 | struct BFSBenchmark { 47 | const std::string name; 48 | std::vector runtimes; 49 | 50 | BFSBenchmark(std::string name) 51 | : name(name) 52 | { } 53 | virtual ~BFSBenchmark() { } 54 | 55 | uint32_t lastRuntime() const { 56 | return runtimes.back(); 57 | } 58 | 59 | uint32_t avgRuntime() const { 60 | uint32_t sum=0; 61 | for(auto t : runtimes) { 62 | sum += t; 63 | } 64 | return sum / runtimes.size(); 65 | } 66 | 67 | virtual void run(const uint32_t k, const Query4::PersonSubgraph& subgraph, const string& referenceResult, Workers& workers, uint64_t maxBfs) = 0; 68 | 69 | virtual size_t batchSize() = 0; 70 | 71 | virtual void initTrace(size_t numVertices, size_t numEdges, size_t numThreads, size_t maxBfs, std::string bfsType) = 0; 72 | 73 | virtual std::string getMinTrace() = 0; 74 | }; 75 | 76 | template 77 | struct SpecializedBFSBenchmark : public BFSBenchmark { 78 | #ifdef STATISTICS 79 | Query4::BatchStatistics statistics; 80 | #endif 81 | 82 | std::vector traces; 83 | 84 | typedef TraceStats RunnerTraceStats; 85 | 86 | SpecializedBFSBenchmark(std::string name) 87 | : BFSBenchmark(name) 88 | { } 89 | virtual void run(const uint32_t k, const Query4::PersonSubgraph& subgraph, const string& referenceResult, Workers& workers, uint64_t maxBfs) override { 90 | uint64_t runtime; 91 | std::string result = runBFS(k, subgraph, workers, maxBfs, runtime 92 | #ifdef STATISTICS 93 | ,statistics 94 | #endif 95 | ); 96 | if(maxBfs == std::numeric_limits::max() && result != referenceResult) { 97 | cout< queries; 17 | 18 | static Queries loadFromFile(const std::string& queryFile) { 19 | io::MmapedFile file(queryFile, O_RDONLY); 20 | 21 | Queries queries; 22 | 23 | Tokenizer tokenizer(file.mapping, file.size); 24 | // Read queries 25 | while(!tokenizer.isFinished()) { 26 | Query query; 27 | query.numNodes = tokenizer.readId(' '); 28 | query.dataset = tokenizer.readStr(' '); 29 | query.reference = tokenizer.readStr('\n'); 30 | if(query.reference.size()==0) { 31 | FATAL_ERROR("[Queries] Could not load reference result, is it specified?"); 32 | } 33 | queries.queries.push_back(query); 34 | } 35 | 36 | LOG_PRINT("[Queries] Loaded "<< queries.queries.size()<< " queries."); 37 | return queries; 38 | } 39 | }; 40 | 41 | struct BFSBenchmark { 42 | const std::string name; 43 | std::vector runtimes; 44 | 45 | BFSBenchmark(std::string name) 46 | : name(name) 47 | { } 48 | virtual ~BFSBenchmark() { } 49 | 50 | uint32_t lastRuntime() const { 51 | return runtimes.back(); 52 | } 53 | 54 | uint32_t avgRuntime() const { 55 | uint32_t sum=0; 56 | for(auto t : runtimes) { 57 | sum += t; 58 | } 59 | return sum / runtimes.size(); 60 | } 61 | 62 | virtual void run(const uint32_t k, const Query4::PersonSubgraph& subgraph, const string& referenceResult, Workers& workers, uint64_t maxBfs) = 0; 63 | 64 | virtual size_t batchSize() = 0; 65 | 66 | virtual void initTrace(size_t numVertices, size_t numEdges, size_t numThreads, size_t maxBfs, std::string bfsType) = 0; 67 | 68 | virtual std::string getMinTrace() = 0; 69 | }; 70 | 71 | template 72 | struct SpecializedBFSBenchmark : public BFSBenchmark { 73 | #ifdef STATISTICS 74 | Query4::BatchStatistics statistics; 75 | #endif 76 | 77 | std::vector traces; 78 | 79 | typedef TraceStats RunnerTraceStats; 80 | 81 | SpecializedBFSBenchmark(std::string name) 82 | : BFSBenchmark(name) 83 | { } 84 | virtual void run(const uint32_t k, const Query4::PersonSubgraph& subgraph, const string& referenceResult, Workers& workers, uint64_t maxBfs) override { 85 | const auto start = tschrono::now(); 86 | 87 | std::string result = runBFS(k, subgraph, workers, maxBfs 88 | #ifdef STATISTICS 89 | ,statistics 90 | #endif 91 | ); 92 | printf("%s and %s \n",result.c_str(), referenceResult.c_str()); 93 | printf("%d, and %d \n",maxBfs, std::numeric_limits::max()); 94 | if(maxBfs == std::numeric_limits::max() && result != referenceResult) { 95 | cout< 9 | 10 | namespace Query4 { 11 | typedef uint64_t Distances; // Type for the sum of distances 12 | typedef uint8_t Distance; // Type for a distance between two points 13 | typedef uint32_t PersonId; // Type for person ids 14 | typedef PersonId Persons; // Type for counting persons 15 | 16 | typedef Graph PersonSubgraph; 17 | 18 | inline awfy::FixedSizeQueue& getThreadLocalPersonVisitQueue(size_t queueSize) { 19 | static __thread awfy::FixedSizeQueue* toVisitPtr=nullptr; 20 | if(toVisitPtr != nullptr) { 21 | awfy::FixedSizeQueue& q = *toVisitPtr; 22 | q.reset(queueSize); 23 | return q; 24 | } else { 25 | toVisitPtr = new awfy::FixedSizeQueue(queueSize); 26 | return *toVisitPtr; 27 | 28 | } 29 | } 30 | 31 | struct BatchBFSdata { 32 | const PersonId person; 33 | const Persons componentSize; 34 | 35 | Distances totalDistances; 36 | Persons totalReachable; 37 | 38 | BatchBFSdata(PersonId person, Persons componentSize) 39 | : person(person), componentSize(componentSize), 40 | totalDistances(0),totalReachable(0) 41 | { } 42 | 43 | BatchBFSdata(const BatchBFSdata&) = delete; 44 | BatchBFSdata& operator=(const BatchBFSdata&) = delete; 45 | 46 | BatchBFSdata(BatchBFSdata&& other) 47 | : person(other.person), componentSize(other.componentSize), 48 | totalDistances(other.totalDistances), totalReachable(other.totalReachable) 49 | { } 50 | }; 51 | } -------------------------------------------------------------------------------- /include/bfs/batch128.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include "base.hpp" 7 | #include "sse.hpp" 8 | #include "statistics.hpp" 9 | #include 10 | 11 | namespace Query4 { 12 | 13 | struct BatchBFSRunner128 { 14 | 15 | static constexpr uint64_t batchSize() { 16 | return 128; 17 | } 18 | 19 | static void runBatch(std::vector& bfsData, const PersonSubgraph& subgraph 20 | #ifdef STATISTICS 21 | , BatchStatistics& statistics 22 | #endif 23 | ); 24 | 25 | private: 26 | static void runBatchRound(std::vector& bfsData, const PersonSubgraph& subgraph, PersonId minPerson, 27 | std::array<__m128i*,2>& toVisitLists, __m128i* __restrict__ seen 28 | #ifdef STATISTICS 29 | , BatchStatistics& statistics 30 | #endif 31 | ); 32 | }; 33 | 34 | } -------------------------------------------------------------------------------- /include/bfs/batch256.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include "base.hpp" 7 | #include "sse.hpp" 8 | 9 | namespace Query4 { 10 | 11 | #ifdef AVX2 12 | 13 | struct BatchBFSRunner256 { 14 | static constexpr uint64_t batchSize() { 15 | return 256; 16 | } 17 | 18 | static void runBatch(std::vector& bfsData, const PersonSubgraph& subgraph); 19 | 20 | private: 21 | static void runBatchRound(std::vector& bfsData, const PersonSubgraph& subgraph, PersonId minPerson, __m256i** toVisitLists, __m256i* __restrict__ seen); 22 | }; 23 | #endif 24 | 25 | } -------------------------------------------------------------------------------- /include/bfs/batch64.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include "base.hpp" 7 | #include "statistics.hpp" 8 | #include 9 | 10 | namespace Query4 { 11 | 12 | struct BatchBFSRunner { 13 | 14 | static constexpr uint64_t batchSize() { 15 | return sizeof(uint64_t)*8; 16 | } 17 | 18 | static void runBatch(std::vector& bfsData, const PersonSubgraph& subgraph 19 | #ifdef STATISTICS 20 | , BatchStatistics& statistics 21 | #endif 22 | ); 23 | 24 | private: 25 | static void runBatchRound(std::vector& bfsData, const PersonSubgraph& subgraph, PersonId minPerson, std::array& toVisitLists, 26 | uint64_t* __restrict__ seen); 27 | }; 28 | 29 | } -------------------------------------------------------------------------------- /include/bfs/batchhugedeferred.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | %%========================= 5 | %% Assistenten(PersNr, Name, Fachgebiet, Boss) 6 | assistenten(3002,platon,ideenlehre,2125). 7 | assistenten(3003,aristoteles,syllogistik,2125). 8 | assistenten(3004,wittgenstein,sprachtheorie,2126). 9 | assistenten(3005,rhetikus,planetenbewegung,2127). 10 | assistenten(3006,newton,keplersche_gesetze,2127). 11 | assistenten(3007,spinoza,gott_und_natur,2134). 12 | 13 | %% hoeren(MatrNr, VorlNr) 14 | hoeren(26120,5001). 15 | hoeren(27550,5001). 16 | hoeren(27550,4052). 17 | hoeren(28106,5041). 18 | hoeren(28106,5052). 19 | hoeren(28106,5216). 20 | hoeren(28106,5259). 21 | hoeren(29120,5001). 22 | hoeren(29120,5041). 23 | hoeren(29120,5049). 24 | hoeren(29555,5022). 25 | hoeren(25403,5022). 26 | hoeren(29555,5001). 27 | 28 | %% pruefen(MatrNr,VorlNr, PersNr, Note) 29 | pruefen(28106,5001,2126,1). 30 | pruefen(25403,5041,2125,2). 31 | pruefen(27550,4630,2137,2). 32 | 33 | %%Vorlesungen(VorlNr, Titel, SWS, gelesenVon) 34 | vorlesungen(5001,grundzuege,4,2137). 35 | vorlesungen(5041,ethik,4,2125). 36 | vorlesungen(5043,erkenntnistheorie,3,2126). 37 | vorlesungen(5049,maeeutik,2,2125). 38 | vorlesungen(4052,logik,4,2125). 39 | vorlesungen(5052,wissenschaftstheorie,3,2126). 40 | vorlesungen(5216,bioethik,2,2126). 41 | vorlesungen(5259,der_wiener_kreis,2,2133). 42 | vorlesungen(5022,glaube_und_wissen,2,2134). 43 | vorlesungen(4630,die_3_kritiken,4,2137). 44 | 45 | %%Professoren(PersNr,Name,Rang, Raum) 46 | professoren(2125,sokrates,c4,226). 47 | professoren(2126,russel,c4,232). 48 | professoren(2127,kopernikus,c3,310). 49 | professoren(2133,popper,c3,52). 50 | professoren(2134,augustinus,c3,309). 51 | professoren(2136,curie,c4,36). 52 | professoren(2137,kant,c4,7). 53 | 54 | %%voraussetzen(Vorg,Nachf) 55 | voraussetzen(5001,5041). 56 | voraussetzen(5001,5043). 57 | voraussetzen(5001,5049). 58 | voraussetzen(5041,5216). 59 | voraussetzen(5043,5052). 60 | voraussetzen(5041,5052). 61 | voraussetzen(5052,5259). 62 | 63 | %%Studenten(MatrNr, Name, Semester) 64 | studenten(24002,xenokrates,18). 65 | studenten(25403,jonas,12). 66 | studenten(26120,fichte,10). 67 | studenten(26830,aristoxenos,8). 68 | studenten(27550,schopenhauer,6). 69 | studenten(28106,carnap,3). 70 | studenten(29120,theophrastos,2). 71 | studenten(29555,feuerbach,2). -------------------------------------------------------------------------------- /include/bfs/bitops.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include "sse.hpp" 7 | #include 8 | #include 9 | 10 | template 11 | struct CtzlOp { 12 | static size_t ctzl(bit_t value) { 13 | // Only works for uint64_t 14 | static_assert(sizeof(bit_t)==8, "Only works for 64bit values"); 15 | return __builtin_ctzl(value); 16 | } 17 | }; 18 | 19 | template<> 20 | struct CtzlOp { 21 | static size_t ctzl(uint32_t value) { 22 | return __builtin_ctz(value); 23 | } 24 | }; 25 | template<> 26 | struct CtzlOp { 27 | static size_t ctzl(uint16_t value) { 28 | return __builtin_ctz(value); 29 | } 30 | }; 31 | 32 | template<> 33 | struct CtzlOp { 34 | static size_t ctzl(uint8_t value) { 35 | return __builtin_ctz(value); 36 | } 37 | }; 38 | 39 | template 40 | struct BitBaseOp { 41 | private: 42 | static const size_t TYPE_BITS_COUNT = sizeof(bit_t)*8; 43 | bit_t set[TYPE_BITS_COUNT]; 44 | 45 | BitBaseOp() { 46 | for (unsigned i = 0; i < sizeof(bit_t)*8; ++i) { 47 | set[i] = pow(((bit_t)2),i); 48 | } 49 | } 50 | static const BitBaseOp masks; 51 | 52 | public: 53 | static bit_t getSetMask(const size_t bitPos) { 54 | //return ((bit_t)1) << pos; 55 | return masks.set[bitPos]; 56 | } 57 | 58 | static bit_t andNot(const bit_t a, const bit_t b) { 59 | return a & ~b; 60 | } 61 | 62 | static bool isZero(const bit_t value) { 63 | return value == 0; 64 | } 65 | 66 | static bool notZero(const bit_t value) { 67 | return value != 0; 68 | } 69 | 70 | static bool notAllOnes(const bit_t value) { 71 | return value != std::numeric_limits::max(); 72 | } 73 | 74 | //Checks if bitwise and of two values is nonzero 75 | static bool andNotZero(const bit_t a, const bit_t b) { 76 | return (a&b)!=0; 77 | } 78 | 79 | static unsigned popCount(const bit_t a) { 80 | return std::bitset(a).count(); 81 | } 82 | 83 | static inline bit_t zero() { 84 | return 0; 85 | } 86 | 87 | static bool equals(const bit_t a, const bit_t b) { 88 | return a == b; 89 | } 90 | 91 | static bool notEquals(const bit_t a, const bit_t b) { 92 | return a != b; 93 | } 94 | }; 95 | 96 | // Normal logic again 97 | template const BitBaseOp BitBaseOp::masks = BitBaseOp(); 98 | 99 | // SSE specialization 100 | template<> 101 | struct CtzlOp<__m128i> { 102 | static size_t ctzl(__m128i value) { 103 | const uint64_t a = _mm_extract_epi64(value, 0); 104 | const uint64_t b = _mm_extract_epi64(value, 1); 105 | //assert(a!=0||b!=0); 106 | return a==0?__builtin_ctzl(b)+64:__builtin_ctzl(a); 107 | } 108 | }; 109 | 110 | template<> 111 | struct BitBaseOp<__m128i> { 112 | static __m128i getSetMask(const size_t bitPos) { 113 | return sseMasks[bitPos]; 114 | } 115 | 116 | static __m128i andNot(const __m128i a, const __m128i b) { 117 | return _mm_andnot_si128(b, a); 118 | } 119 | 120 | //Checks if bitwise and of two values is nonzero 121 | static bool andNotZero(const __m128i a, const __m128i b) { 122 | return _mm_testz_si128(a,b)==0; 123 | } 124 | 125 | static bool isZero(const __m128i value) { 126 | return _mm_testc_si128(_mm_setzero_si128(),value)!=0; 127 | } 128 | 129 | static bool notAllOnes(const __m128i value) { 130 | return _mm_test_all_ones(value)==0; 131 | } 132 | 133 | static bool notZero(const __m128i value) { 134 | return _mm_testc_si128(_mm_setzero_si128(),value)==0; 135 | } 136 | 137 | static unsigned popCount(const __m128i value) { 138 | return __builtin_popcountl(_mm_extract_epi64(value,0)) + __builtin_popcountl(_mm_extract_epi64(value,1)); 139 | } 140 | 141 | static inline __m128i zero() { 142 | return _mm_setzero_si128(); 143 | } 144 | 145 | static bool equals(const __m128i a, const __m128i b) { 146 | const __m128i cmp = _mm_cmpeq_epi64(a, b); 147 | return _mm_extract_epi64(cmp,0)!=0 && _mm_extract_epi64(cmp,1)!=0; 148 | } 149 | 150 | static bool notEquals(const __m128i a, const __m128i b) { 151 | const __m128i cmp = _mm_cmpeq_epi64(a, b); 152 | return _mm_extract_epi64(cmp,0)==0 || _mm_extract_epi64(cmp,1)==0; 153 | } 154 | }; 155 | 156 | #ifdef AVX2 157 | template<> 158 | struct CtzlOp<__m256i> { 159 | static size_t ctzl(__m256i value) { 160 | const uint64_t a = _mm256_extract_epi64(value, 0); 161 | const uint64_t b = _mm256_extract_epi64(value, 1); 162 | const uint64_t c = _mm256_extract_epi64(value, 2); 163 | const uint64_t d = _mm256_extract_epi64(value, 3); 164 | if(a==0) { 165 | if(b==0) { 166 | return c==0?__builtin_ctzl(d)+192:__builtin_ctzl(c)+128; 167 | } else { 168 | return __builtin_ctzl(b)+64; 169 | } 170 | } 171 | return __builtin_ctzl(a); 172 | } 173 | }; 174 | 175 | template<> 176 | struct BitBaseOp<__m256i> { 177 | static __m256i getSetMask(const size_t bitPos) { 178 | return sseMasks256[bitPos]; 179 | } 180 | 181 | static __m256i andNot(const __m256i a, const __m256i b) { 182 | return _mm256_andnot_si256(b, a); 183 | } 184 | 185 | static bool isZero(const __m256i value) { 186 | return _mm256_testc_si256(_mm256_setzero_si256(), value)!=0; 187 | } 188 | 189 | static bool notZero(const __m256i value) { 190 | return _mm256_testc_si256(_mm256_setzero_si256(), value)==0; 191 | } 192 | 193 | static bool notAllOnes(const __m256i value) { 194 | static const __m256i ones=_mm256_set_epi64x(std::numeric_limits::max(),std::numeric_limits::max(),std::numeric_limits::max(),std::numeric_limits::max()); 195 | return _mm256_testc_si256(value, ones)==0; 196 | } 197 | 198 | static unsigned popCount(const __m256i value) { 199 | return __builtin_popcountl(_mm256_extract_epi64(value,0)) 200 | + __builtin_popcountl(_mm256_extract_epi64(value,1)) 201 | + __builtin_popcountl(_mm256_extract_epi64(value,2)) 202 | + __builtin_popcountl(_mm256_extract_epi64(value,3)); 203 | } 204 | 205 | static inline __m256i zero() { 206 | return _mm256_setzero_si256(); 207 | } 208 | }; 209 | #endif -------------------------------------------------------------------------------- /include/bfs/naive.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include "base.hpp" 7 | #include "statistics.hpp" 8 | 9 | namespace Query4 { 10 | 11 | struct BFSRunner { 12 | static const size_t TYPE=1; 13 | static const size_t WIDTH=1; 14 | static const size_t TYPE_BITS=8; 15 | 16 | typedef awfy::FixedSizeQueue BFSQueue; 17 | 18 | static constexpr size_t batchSize() { 19 | return 1; 20 | } 21 | 22 | static inline void runBatch(std::vector& bfsData, const PersonSubgraph& subgraph 23 | #ifdef STATISTICS 24 | , BatchStatistics& statistics 25 | #endif 26 | ) { 27 | for(size_t i=0; i& bfsData, const PersonSubgraph& subgraph 22 | #ifdef STATISTICS 23 | , BatchStatistics&/* statistics*/ 24 | #endif 25 | ) { 26 | for(size_t i=0; i 9 | 10 | #define USE_ATOMIC 1 11 | #define TOPK 7 12 | namespace Query4 { 13 | 14 | struct PARABFSRunner { 15 | static const size_t TYPE=1; 16 | static const size_t WIDTH=1; 17 | static const size_t TYPE_BITS=8; 18 | 19 | struct TraverseParams { 20 | TraverseParams(uint32_t *cQ=0, uint32_t *nQ=0, uint32_t cqs=0, uint32_t st=0): cqueue(cQ), nqueue(nQ), cq_start(cqs), cq_end(0), stride(st), nq(0), sp(0), rp(0), level(0) {} 21 | 22 | void set(uint32_t q, uint64_t s, uint64_t r) { 23 | nq = q; 24 | sp = s; 25 | rp = r; 26 | } 27 | uint32_t *cqueue; 28 | uint32_t *nqueue; 29 | uint32_t cq_start; 30 | uint32_t cq_end; 31 | uint32_t stride; 32 | uint32_t nq; 33 | uint64_t sp; 34 | uint64_t rp; 35 | uint32_t level; 36 | }; 37 | 38 | typedef std::pair CC; 39 | struct CompareCC{ 40 | bool operator()(const CC &lhs, const CC &rhs) const{ 41 | if (lhs.second == rhs.second) 42 | return (lhs.first < rhs.first); 43 | return (lhs.second > rhs.second); 44 | } 45 | }; 46 | 47 | struct KData{ 48 | uint32_t personSize; 49 | uint32_t* queue; 50 | 51 | bool * visited; 52 | #if !USE_ATOMIC 53 | bool * visited2; 54 | #endif 55 | 56 | boost::mutex mutex; 57 | boost::condition_variable condDone, condWork; 58 | boost::thread_group workers; 59 | int workerBusy; 60 | bool shutdown; 61 | CC* cc;//closeness centrality 62 | std::vector params; 63 | int numThreads; 64 | }; 65 | 66 | static KData *kdata; 67 | static bool isInitialized; 68 | 69 | static void init(uint32_t personSize){ 70 | kdata->personSize = personSize; 71 | 72 | kdata->queue = (uint32_t*)malloc(sizeof(uint32_t)*kdata->personSize*(kdata->numThreads + 1));//Allocate n+1 queues 73 | kdata->visited = (bool*)malloc(sizeof(bool)*kdata->personSize*2); 74 | #if !USE_ATOMIC 75 | kdata->visited2 = kdata->visited + kdata->personSize; 76 | #endif 77 | kdata->cc = (CC*)malloc(sizeof(CC)*kdata->personSize); 78 | kdata->shutdown = true; 79 | isInitialized = true; 80 | } 81 | 82 | static void setThreadNum(int nThreads) 83 | { 84 | kdata->numThreads = nThreads; 85 | } 86 | 87 | static inline void runBatch(std::vector& bfsData, const PersonSubgraph& subgraph 88 | #ifdef STATISTICS 89 | , BatchStatistics& statistics 90 | #endif 91 | ) { 92 | if (!isInitialized) 93 | { 94 | init(subgraph.size()); 95 | uint32_t* cqueue = &kdata->queue[kdata->numThreads*(kdata->personSize)]; 96 | for (int i=0; inumThreads; ++i) 97 | kdata->params.push_back(TraverseParams(cqueue, kdata->queue+i*(kdata->personSize), i, kdata->numThreads)); 98 | start(kdata->params, subgraph); 99 | } 100 | for(size_t i=0; icc, kdata->cc+TOPK, kdata->cc+kdata->personSize, CompareCC()); 118 | // for(int i=0; icc[i].first); 120 | } 121 | 122 | static void waitForWorkers(); 123 | static void worker(TraverseParams *param, const PersonSubgraph& subgraph); 124 | static void stop(); 125 | static void pfree(); 126 | static void dispatchToWorkers(); 127 | static void start(std::vector ¶ms, const PersonSubgraph& subgraph); 128 | static void traverse(TraverseParams *tp, const PersonSubgraph& subgraph); 129 | static void run(const PersonId start, const PersonSubgraph& subgraph, BatchBFSdata& bfsData 130 | #ifdef STATISTICS 131 | , BatchStatistics& statistics 132 | #endif 133 | ); 134 | 135 | }; 136 | } 137 | -------------------------------------------------------------------------------- /include/bfs/sc2012.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include 7 | #include "base.hpp" 8 | #include "statistics.hpp" 9 | 10 | namespace Query4 { 11 | 12 | struct SCBFSRunner { 13 | static const size_t TYPE=1; 14 | static const size_t WIDTH=1; 15 | static const size_t TYPE_BITS=8; 16 | 17 | static const size_t alpha = 14; 18 | static const size_t beta = 24; 19 | 20 | typedef awfy::FixedSizeQueue BFSQueue; 21 | typedef std::vector CustomBFSData; 22 | 23 | static constexpr size_t batchSize() { 24 | return 1; 25 | } 26 | 27 | static inline void runBatch(std::vector& bfsData, const PersonSubgraph& subgraph 28 | #ifdef STATISTICS 29 | , BatchStatistics& statistics 30 | #endif 31 | ) { 32 | for(size_t i=0; i 7 | 8 | #ifdef __AVX2__ 9 | #define AVX2 1 10 | #endif 11 | 12 | extern __m128i sseMasks[128]; 13 | 14 | #ifdef AVX2 15 | extern __m256i sseMasks256[256]; 16 | #endif -------------------------------------------------------------------------------- /include/bfs/statistics.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include "base.hpp" 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | using namespace std; 13 | 14 | namespace Query4 { 15 | 16 | struct BatchStatistics { 17 | #ifdef STATISTICS 18 | size_t step; 19 | 20 | uint32_t ABC[512]; //How many of each? 21 | std::array,512> EFG; //In which step? 22 | // std::vector> numTouchedPersonInDistance; //Person -> Distance -> Number 23 | uint32_t maxStatisticsDistance = 10; //Max distance for statistics 24 | #endif 25 | 26 | #ifdef OVER_TIME_STATISTICS 27 | std::vector> iterationUsages; 28 | std::vector curIterationUse; 29 | #endif 30 | 31 | BatchStatistics() 32 | #ifdef STATISTICS 33 | : step(0) 34 | #endif 35 | { } 36 | 37 | #ifdef STATISTICS 38 | void inline log(PersonId/* person*/, uint32_t/* nextDistance */, size_t count) { 39 | // (numTouchedPersonInDistance[person])[nextDistance-1]++; 40 | 41 | 42 | 43 | #ifdef OVER_TIME_STATISTICS 44 | curIterationUse.push_back(count); 45 | #endif 46 | 47 | ABC[count-1]++; 48 | step++; 49 | 50 | cout<& b : iterationUsages) { 77 | uint64_t sum=0; 78 | for(uint32_t i : b) { 79 | sum+=i; 80 | } 81 | cout<& b : iterationUsages) { 86 | for(uint32_t i : b) { 87 | cout<subgraph.size(); p++) { 99 | // for(int a=0; a 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | struct NodePair { 19 | uint64_t idA; 20 | uint64_t idB; 21 | 22 | NodePair() { } 23 | NodePair(uint64_t idA, uint64_t idB) 24 | : idA(idA), idB(idB) 25 | { } 26 | }; 27 | 28 | struct GraphData { 29 | size_t numNodes; 30 | std::vector edges; 31 | std::unordered_map revNodeRenaming; 32 | 33 | GraphData(const size_t numNodes, std::vector edges, std::unordered_map revNodeRenaming) 34 | : numNodes(numNodes), edges(move(edges)), revNodeRenaming(std::move(revNodeRenaming)) { 35 | } 36 | 37 | GraphData(GraphData& other) = delete; 38 | GraphData(GraphData&& other) = default; 39 | 40 | static GraphData loadFromPath(const std::string& edgesFile); 41 | }; 42 | 43 | template 44 | class SizedList { 45 | public: 46 | typedef EntryType Size; 47 | typedef EntryType Entry; 48 | 49 | private: 50 | Size count; 51 | 52 | public: 53 | SizedList() { 54 | } 55 | 56 | /// Reads the size for the list at the pointer position 57 | inline const Size& size() const { 58 | return count; 59 | } 60 | 61 | inline Size& size() { 62 | return count; 63 | } 64 | 65 | /// Sets the size for the list at the pointer position 66 | void setSize(const Size count) { 67 | this->count = count; 68 | } 69 | 70 | const Entry* getPtr(const size_t index) const __attribute__ ((pure)) { 71 | const auto offsetPtr = reinterpret_cast(this) + sizeof(Size) + sizeof(Entry)*index; 72 | return reinterpret_cast(offsetPtr); 73 | } 74 | 75 | Entry* getPtr(const size_t index) __attribute__ ((pure)) { 76 | const auto offsetPtr = reinterpret_cast(this) + sizeof(Size) + sizeof(Entry)*index; 77 | return reinterpret_cast(offsetPtr); 78 | } 79 | 80 | std::pair bounds() const __attribute__ ((pure)) { 81 | const Entry* i=getPtr(0); 82 | return std::make_pair(i,i+count); 83 | } 84 | 85 | SizedList* nextList(Size count) __attribute__ ((pure)) { 86 | auto offsetPtr = reinterpret_cast(this)+sizeof(Size)+sizeof(Entry)*count; 87 | return reinterpret_cast*>(offsetPtr); 88 | } 89 | }; 90 | 91 | template 92 | class Graph { 93 | public: 94 | typedef IdType Id; 95 | typedef SizedList* Content; 96 | 97 | typedef uint32_t ComponentId; 98 | typedef uint64_t ComponentSize; 99 | 100 | const size_t numVertices; 101 | size_t numEdges; 102 | 103 | std::vector personComponents; 104 | std::vector componentSizes; 105 | std::vector componentEdgeCount; 106 | ComponentSize maxComponentSize; 107 | 108 | private: 109 | Content* table; 110 | 111 | std::unordered_map revNodeRenaming; 112 | 113 | public: 114 | uint8_t* data; 115 | 116 | Graph(size_t numVertices) : numVertices(numVertices), personComponents(numVertices), componentSizes(), componentEdgeCount(), maxComponentSize(), table(nullptr), data(nullptr) { 117 | // XXX: Maybe posix_memalign helps? 118 | table = new Content[numVertices](); 119 | } 120 | 121 | Graph(Graph& other) = delete; 122 | 123 | Graph(Graph&& other) : numVertices(other.numVertices), numEdges(other.numEdges), personComponents(other.personComponents), componentSizes(other.componentSizes), componentEdgeCount(other.componentEdgeCount), maxComponentSize(other.maxComponentSize), table(other.table), revNodeRenaming(std::move(other.revNodeRenaming)), data(other.data) { 124 | other.table=nullptr; 125 | other.data=nullptr; 126 | } 127 | 128 | ~Graph() { 129 | if(table) { 130 | delete[] table; 131 | table = nullptr; 132 | } 133 | 134 | if(data) { 135 | delete[] data; 136 | data = nullptr; 137 | } 138 | } 139 | 140 | IdType mapInternalNodeId(IdType id) const { 141 | const auto iter = revNodeRenaming.find(id); 142 | if(iter!=revNodeRenaming.cend()) { 143 | return iter->second; 144 | } else { 145 | throw -1; 146 | } 147 | } 148 | 149 | /// Inserts the data for the specified id into the index 150 | void insert(Id id, Content content) { 151 | assert(table!=nullptr); 152 | assert(id& edges = graphData.edges; 175 | 176 | // Build graph 177 | Graph personGraph(numPersons); 178 | personGraph.revNodeRenaming = std::move(graphData.revNodeRenaming); 179 | const size_t dataSize = (numPersons+edges.size())*sizeof(IdType); 180 | uint8_t* data = new uint8_t[dataSize](); 181 | { 182 | SizedList* neighbours = reinterpret_cast*>(data); 183 | size_t ix=0; 184 | for(IdType person=0; person(neighbours)getPtr(0); 187 | IdType count=0; 188 | while(ixsetSize(count); 196 | personGraph.insert(person, neighbours); 197 | neighbours = neighbours->nextList(count); 198 | } 199 | } 200 | 201 | personGraph.data = data; 202 | personGraph.numEdges = edges.size(); 203 | 204 | LOG_PRINT("[LOADING] Created person graph of size: "<< dataSize/1024<<" kb"); 205 | 206 | #ifdef DEBUG 207 | uint64_t retrievableFriends = 0; 208 | for(IdType person=0; personsize(); 210 | } 211 | assert(retrievableFriends==edges.size()); 212 | #endif 213 | 214 | personGraph.analyzeGraph(); 215 | 216 | return std::move(personGraph); 217 | } 218 | 219 | private: 220 | void analyzeGraph() { 221 | const auto graphSize = size(); 222 | 223 | componentSizes.push_back(std::numeric_limits::max()); // Component 0 is invalid 224 | componentEdgeCount.push_back(std::numeric_limits::max()); // Component 0 is invalid 225 | 226 | // Identify connected components by running bfs 227 | awfy::FixedSizeQueue toVisit = awfy::FixedSizeQueue(graphSize); 228 | 229 | size_t trivialComponents=0; 230 | ComponentId componentId=1; 231 | for(IdType node=0; nodesize(); 247 | 248 | auto neighbourBounds = curNeighbours->bounds(); 249 | while(neighbourBounds.first != neighbourBounds.second) { 250 | const IdType curNeighbour=*neighbourBounds.first; 251 | ++neighbourBounds.first; 252 | if (personComponents[curNeighbour]!=0) { continue; } 253 | personComponents[curNeighbour]=componentId; 254 | componentSize++; 255 | toVisit.push_back_pos() = curNeighbour; 256 | } 257 | } while(!toVisit.empty()); 258 | 259 | componentEdgeCount.push_back(componentNeighborCount); 260 | componentSizes.push_back(componentSize); 261 | if(componentSize>maxComponentSize) { 262 | maxComponentSize = componentSize; 263 | } 264 | if(componentSize<5) { 265 | trivialComponents++; 266 | } else { 267 | std::cout<<"# C "<0); 274 | } 275 | 276 | LOG_PRINT("[Query4] Max component size "<< maxComponentSize); 277 | std::cout<<"# Found number components "<< componentId-1<<" ("<. 21 | */ 22 | 23 | /* 24 | COMMENT SUMMARY FOR THIS FILE: 25 | 26 | This file contains an implementation of an optimized chaining hashset and hashmap. 27 | Depending on the use case, these hashes perform up to 4x better than std::unordered_map. 28 | */ 29 | 30 | #pragma once 31 | 32 | #include "hash.hpp" 33 | #include "pool.hpp" 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | namespace campers { 40 | 41 | /// A fast, chaining HashMap 42 | template 43 | class HashMap { 44 | /// An entry inside the hash map 45 | public: 46 | struct Entry { 47 | /// The key 48 | T word; 49 | /// The value for this key 50 | Value value; 51 | /// The hash value 52 | uint32_t hashValue; 53 | /// Pointer to chain of entries for collisions 54 | Entry* next; 55 | 56 | /// Constructor 57 | Entry () : hashValue(0),next(nullptr) {} 58 | /// Constructor 59 | Entry (const T& word,uint32_t hashValue) : word(word),hashValue(hashValue),next(nullptr) {} 60 | }; 61 | 62 | /// The hashmap of pointers to actual entries 63 | std::vector entries; 64 | 65 | private: 66 | /// An instance of the hash funtion functor 67 | HashFunc hashFunc; 68 | /// The pool allocator. 69 | Pool entryAllocator; 70 | /// The exponent, see HashSet and summary for details 71 | uint64_t exp; 72 | /// The mask, see HashSet and summary for details 73 | uint64_t mask; 74 | /// The number of elements inside the hash, 75 | uint32_t size_; 76 | /// The size this hash map was allocated with 77 | uint64_t allocSize; 78 | 79 | public: 80 | /// Constructor 81 | HashMap() : HashMap(64) { } 82 | HashMap(uint32_t size) { hintSize(size); } 83 | /// Inserts an element into the hashtable. Returns the value for that key, regardless if an insertion happened or not. Hash value supplied externally 84 | Value* tryInsert(const T& word,uint32_t hashValue); 85 | /// Inserts an element into the hashtable. Returns the value for that key, regardless if an insertion happened or not. 86 | Value* tryInsert(const T& word) { return tryInsert(word,hashFunc(word)); } 87 | /// Finds an element inside the hashmap, returns the value if found or nullptr otherwise. Hash value supplied externally 88 | Value* find(const T& word,uint32_t hashValue); 89 | /// Finds an element inside the hashmap, returns the value if found or nullptr otherwise 90 | Value* find(const T& word) { return find(word,hashFunc(word)); } 91 | /// Alternative find: uses a type that can differ from T, e.g. T=string, T2=string_ref 92 | template Value* altFind(const T2& word); 93 | /// Pre allocates the hash to be able to contain size elements efficiently. Also, see summary and HashSet 94 | void hintSize(uint32_t size); 95 | /// Returns the current number of elements inside the hash 96 | uint32_t size() const { return size_; } 97 | /// Grows the hash to twice its capacity 98 | // void grow() { grow(2*capacity()); }; 99 | // /// Grows the hash to be able to efficiently contain 2* capacity elements 100 | // void grow(uint32_t newSize); 101 | /// Clears all entries in the HashMap 102 | // void clear(); 103 | /// Returns the current capacity of the hash 104 | size_t capacity() const { return entries.size(); }; 105 | 106 | /// Returns 1 if the value is contained 107 | size_t count(const T& word) { return find(word)!=nullptr?1:0; } 108 | /// Clears all entries in the map 109 | void clear(); 110 | }; 111 | 112 | /// Inserts an element into the hashtable. Returns the value for that key, regardless if an insertion happened or not. 113 | template 114 | Value* HashMap::tryInsert(const T& word,uint32_t hashValue) { 115 | hashValue>>=8; 116 | Entry*& entry=entries[hashValue&mask]; 117 | // Unroll empty slot branch 118 | if (entry==nullptr) { 119 | ++size_; 120 | entry=entryAllocator.allocate(); 121 | new (entry) Entry(word,hashValue); 122 | return &entry->value; 123 | } else { 124 | // Unroll immediate hit branch 125 | if (entry->hashValue==hashValue&&entry->word==word) return &entry->value; 126 | 127 | // Iterate to the end of the chain 128 | Entry* entryIter; 129 | for (entryIter=entry;entryIter->next!=nullptr;entryIter=entryIter->next) if (entryIter->next->hashValue==hashValue&&entryIter->next->word==word) return &entryIter->next->value; 130 | 131 | // Word is not present, insert and return reference for backpatching 132 | ++size_; 133 | entryIter->next=entryAllocator.allocate(); 134 | new (entryIter->next) Entry(word,hashValue); 135 | return &entryIter->next->value; 136 | } 137 | } 138 | 139 | /// Finds an element inside the hashmap, returns the value if found or nullptr otherwise. Hash value supplied externally 140 | template 141 | Value* HashMap::find(const T& word,uint32_t hashValue) { 142 | hashValue>>=8; 143 | Entry* entry=entries[hashValue&mask]; 144 | // Unroll empty slot branch 145 | if (entry==nullptr) { 146 | return nullptr; 147 | } else { 148 | // Unroll immediate hit branch 149 | if (entry->hashValue==hashValue&&entry->word==word) return &entry->value; 150 | 151 | // Iterate to the end of the chain 152 | for (;entry->next!=nullptr;entry=entry->next) if (entry->next->hashValue==hashValue&&entry->next->word==word) return &entry->next->value; 153 | 154 | return nullptr; 155 | } 156 | } 157 | 158 | /// Alternative find: uses a type that can differ from T, e.g. T=string, T2=string_ref 159 | template 160 | template 161 | Value* HashMap::altFind(const T2& word) { 162 | uint32_t hashValue=hashFunc(word); 163 | hashValue>>=8; 164 | Entry* entry=entries[hashValue&mask]; 165 | // Unroll empty slot branch 166 | if (entry==nullptr) { 167 | return nullptr; 168 | } else { 169 | // Unroll immediate hit branch 170 | if (entry->hashValue==hashValue&&entry->word==word) return &entry->value; 171 | 172 | // Iterate to the end of the chain 173 | for (;entry->next!=nullptr;entry=entry->next) if (entry->next->hashValue==hashValue&&entry->next->word==word) return &entry->next->value; 174 | 175 | return nullptr; 176 | } 177 | } 178 | 179 | /// Pre allocates the hash to be able to contain size elements efficiently. Also, see summary and HashSet 180 | template 181 | void HashMap::hintSize(uint32_t size) { 182 | this->allocSize=size; 183 | for (exp=1;(uint32_t)(1< 191 | void HashMap::clear() { 192 | memset(entries.data(),0,entries.size()*sizeof(Entry*)); 193 | entryAllocator.reset(); 194 | size_=0; 195 | } 196 | 197 | } 198 | -------------------------------------------------------------------------------- /include/idschedulers.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include "bfs/base.hpp" 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | struct RandomNodeOrdering { 15 | static void order(std::vector& ids, size_t /*batchSize*/, const Query4::PersonSubgraph& /*subgraph*/) { 16 | // Deterministic shuffeling 17 | std::random_device rd; 18 | std::mt19937 g(rd()); 19 | g.seed(1987); 20 | std::shuffle(ids.begin(), ids.end(), g); 21 | } 22 | }; 23 | 24 | struct ComponentOrdering { 25 | static void order(std::vector& ids, size_t maxBfs, size_t /*batchSize*/, const Query4::PersonSubgraph& subgraph) { 26 | if(maxBfs>ids.size()) { 27 | maxBfs=ids.size(); 28 | } 29 | // Sort by degree 30 | std::sort(ids.begin(), ids.begin()+maxBfs, [&subgraph](const Query4::PersonId a, const Query4::PersonId b) { 31 | return subgraph.componentSizes[subgraph.personComponents[a]]>subgraph.componentSizes[subgraph.personComponents[b]] || (subgraph.componentSizes[subgraph.personComponents[a]]==subgraph.componentSizes[subgraph.personComponents[b]] && subgraph.personComponents[a]& ids, size_t maxBfs, size_t /*batchSize*/, const Query4::PersonSubgraph& subgraph) { 38 | if(maxBfs>ids.size()) { 39 | maxBfs=ids.size(); 40 | } 41 | // Sort by degree 42 | std::sort(ids.begin(), ids.begin()+maxBfs, [&subgraph](const Query4::PersonId a, const Query4::PersonId b) { 43 | return subgraph.personComponents[a]size() > subgraph.retrieve(b)->size()); 44 | }); 45 | } 46 | }; 47 | 48 | struct TwoHopDegreeOrdering { 49 | static uint32_t countTwoHopNeighbors(Query4::PersonId p, const Query4::PersonSubgraph& subgraph) { 50 | uint32_t n = 0; 51 | auto bounds = subgraph.retrieve(p)->bounds(); 52 | while(bounds.first != bounds.second) { 53 | n += subgraph.retrieve(*bounds.first)->size(); 54 | bounds.first++; 55 | } 56 | return n; 57 | } 58 | 59 | static void order(std::vector& ids, size_t maxBfs, size_t /*batchSize*/, const Query4::PersonSubgraph& subgraph) { 60 | if(maxBfs>ids.size()) { 61 | maxBfs=ids.size(); 62 | } 63 | // Sort by degree 64 | std::sort(ids.begin(), ids.begin()+maxBfs, [&subgraph](const Query4::PersonId a, const Query4::PersonId b) { 65 | return subgraph.personComponents[a] countTwoHopNeighbors(b,subgraph)); 66 | }); 67 | } 68 | }; 69 | 70 | struct NeighourDegreeOrdering { 71 | 72 | static void order(std::vector& ids, size_t /*batchSize*/, const Query4::PersonSubgraph& subgraph) { 73 | // Sort by degree 74 | std::sort(ids.begin(), ids.end(), [&subgraph](const Query4::PersonId a, const Query4::PersonId b) { 75 | return subgraph.retrieve(a)->size() > subgraph.retrieve(b)->size(); 76 | }); 77 | 78 | std::vector new_ids(ids.size()); 79 | // Assign rounds by batching neighbours 80 | std::vector assigned(subgraph.size()); 81 | uint64_t addedFriends=0; 82 | size_t currentIx=0; 83 | for(Query4::Persons i=0; ibounds(); 96 | while(bounds.first != bounds.second) { 97 | if(!assigned[*bounds.first]) { 98 | new_ids[currentIx] = *bounds.first; 99 | assigned[*bounds.first] = true; 100 | addedFriends++; 101 | currentIx++; 102 | } 103 | bounds.first++; 104 | } 105 | } 106 | 107 | LOG_PRINT("[Query4] Added friends "<& ids, size_t batchSize, const Query4::PersonSubgraph& subgraph) { 120 | using namespace std; 121 | using NodeDegreePair = pair; 122 | 123 | auto degreePairSorter = [](const NodeDegreePair& p1, const NodeDegreePair& p2) -> bool { 124 | return p1.second>p2.second; 125 | }; 126 | 127 | std::vector newIds(subgraph.size()); 128 | std::vector assigned(subgraph.size()); 129 | 130 | priority_queue, vector, decltype(degreePairSorter)> degreeQueue(degreePairSorter); 131 | for(Query4::PersonId p=0; psize())); 133 | } 134 | 135 | size_t currentIx=0; 136 | uint32_t numAssignedPersons=0; 137 | while(!degreeQueue.empty()) { 138 | const auto topElement = degreeQueue.top(); 139 | degreeQueue.pop(); 140 | if(assigned[topElement.first]) { continue; } 141 | 142 | uint32_t numUnprocessedFriends = 0; 143 | auto bounds = subgraph.retrieve(topElement.first)->bounds(); 144 | while(bounds.first!=bounds.second) { 145 | if(!assigned[*bounds.first]) { 146 | numUnprocessedFriends++; 147 | } 148 | bounds.first++; 149 | } 150 | 151 | if(numUnprocessedFriends == topElement.second) { 152 | //We already have the correct number of unprocessed friends for this node 153 | queue assignedPersons; 154 | auto bounds = subgraph.retrieve(topElement.first)->bounds(); 155 | while(bounds.first != bounds.second && numAssignedPersons < batchSize) { 156 | if(!assigned[*bounds.first]) { 157 | assert(currentIxbounds(); 182 | while(bounds.first != bounds.second && numAssignedPersons < batchSize) { 183 | if(!assigned[*bounds.first]) { 184 | assert(currentIx 7 | #include 8 | #include 9 | #include 10 | 11 | namespace io { 12 | /// Owner of a mapped file 13 | class MmapedFile { 14 | int fd; 15 | public: 16 | size_t size; 17 | void* mapping; 18 | 19 | MmapedFile(const std::string& path, int flags); 20 | MmapedFile(const MmapedFile&) = delete; 21 | MmapedFile(MmapedFile&&); 22 | ~MmapedFile(); 23 | }; 24 | 25 | size_t fileLines(const std::string& path); 26 | } -------------------------------------------------------------------------------- /include/log.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include 7 | 8 | namespace tschrono { 9 | typedef uint64_t Time; 10 | 11 | Time now(); 12 | 13 | struct TimeFrame { 14 | Time startTime; 15 | Time endTime; 16 | Time duration; 17 | 18 | void start(); 19 | void end(); 20 | }; 21 | } 22 | 23 | #ifdef DEBUG 24 | #ifdef NDBGPRINT 25 | #define LOG_PRINT(X) 26 | #else 27 | #define LOG_PRINT(X) std::cerr<. 21 | */ 22 | 23 | /* 24 | COMMENT SUMMARY FOR THIS FILE: 25 | 26 | A same size pool allocator which allows for extremely fast allocation of memory for same sized items. 27 | */ 28 | 29 | #pragma once 30 | 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | /// A fixed size pool allocator 40 | template 41 | class Pool { 42 | /// Size of the first chunk used for allocations 43 | static const uint64_t initChunkSize=4096; 44 | /// Static assertions for correctness 45 | static_assert(sizeof(T)<=initChunkSize,"T too big for Pool allocator"); 46 | static_assert(sizeof(T)>=sizeof(void*),"T must be bigger than pointer size for my free list"); 47 | 48 | /// Free element type used to manage the free list inside the free elements 49 | struct FreeElement { FreeElement* next; }; 50 | 51 | /// A chunk of memory which is later used to fulfill allocations 52 | struct Chunk { 53 | /// Size of this chunk 54 | uint64_t size; 55 | /// Number of bytes inside this chunk which are already used 56 | uint64_t used; 57 | /// Next chunk (inline linked list between chunks) 58 | Chunk* next; 59 | /// Used such that memory is always 16 aligned. 60 | uint64_t alignmentSpacer; 61 | 62 | /// Pointer to the memory, we overallocate the memory for the Chunk such that Chunk is just a header for a stretch of memory 63 | char* getMem() const { return reinterpret_cast(((char*)this)+sizeof(Chunk)); } 64 | 65 | /// Allocate a new chunk 66 | static Chunk* allocate(uint64_t size) { 67 | assert("Chunk allocation size must be bigger than chunk metadata size"&&size>sizeof(Chunk)); 68 | Chunk* chunk=reinterpret_cast(malloc(size)); 69 | chunk->used=0; 70 | chunk->size=size-sizeof(Chunk); 71 | chunk->next=0; 72 | return chunk; 73 | } 74 | 75 | /// Swap chunks 76 | void swap(Chunk& other) { 77 | swap(size,other.size); 78 | swap(used,other.used); 79 | swap(next,other.next); 80 | } 81 | }; 82 | 83 | /// First chunk (for deallocation) 84 | Chunk* firstChunk; 85 | /// Last chunk 86 | Chunk* lastChunk; 87 | /// List of free elements 88 | FreeElement* freeElements; 89 | 90 | public: 91 | /// Iterator for this memory pool. Iterates over all elements, NOT all allocated elements. 92 | struct PoolIterator { 93 | /// Current chunk for iteration 94 | Chunk* currentChunk; 95 | /// Position inside this chunk 96 | uint64_t pos; 97 | 98 | /// Constructor 99 | PoolIterator() : currentChunk(0),pos(0) {} 100 | /// Constructor 101 | PoolIterator(Chunk* c,uint64_t p) : currentChunk(c),pos(p) {} 102 | /// Moves to the next element 103 | PoolIterator& operator++() { 104 | // Can not increment further 105 | if (!currentChunk) return *this; 106 | 107 | if (pos+sizeof(T) < currentChunk->used) 108 | // next in chunk 109 | pos+=sizeof(T); 110 | else if (currentChunk->next && currentChunk->next->used>0) { 111 | // next in next chunk 112 | currentChunk=currentChunk->next; pos=0; 113 | } else { 114 | // end 115 | currentChunk=0; 116 | pos=0; 117 | } 118 | return *this; 119 | } 120 | /// Equal for the iterator 121 | bool operator==(const PoolIterator& other) const { return other.currentChunk==currentChunk && (!other.currentChunk || other.pos==pos); } 122 | /// Not equal for the iterator 123 | bool operator!=(const PoolIterator& other) const { return other.currentChunk!=currentChunk || other.pos!=pos; } 124 | /// Dereferenciation 125 | T& operator*() { return *(T*)¤tChunk->getMem()[pos]; } 126 | }; 127 | 128 | /// Create an iterator for the first element inside the pool 129 | PoolIterator begin() { return (firstChunk&&firstChunk->used) ? PoolIterator{firstChunk,0} : end(); } 130 | /// Create an iterator for the element past the last element inside this pool 131 | PoolIterator end() { return PoolIterator{0,0}; } 132 | 133 | /// Constructor, creates first chunk 134 | Pool() : firstChunk(Chunk::allocate(initChunkSize)),lastChunk(firstChunk),freeElements(0) {} 135 | /// Destructor, deallocates all memory 136 | ~Pool() { 137 | for (Chunk* c=firstChunk;c!=0;) { 138 | Chunk* toFree=c; 139 | c=c->next; 140 | free(toFree); 141 | } 142 | } 143 | /// Allocate a fixed-size stretch of memory using the pool allocator 144 | T* allocate() { 145 | assert(lastChunk!=nullptr); 146 | if (freeElements!=nullptr) { 147 | T* ptr=reinterpret_cast(freeElements); 148 | freeElements=freeElements->next; 149 | return ptr; 150 | } else if (lastChunk->used+sizeof(T)>=lastChunk->size) { 151 | // If there was a reset before we can avoid the allocation 152 | if(lastChunk->next!=nullptr) { 153 | lastChunk=lastChunk->next; 154 | } else { 155 | // Add a new chunk if space isn't sufficient 156 | lastChunk->next=Chunk::allocate(lastChunk->size<<1); 157 | lastChunk=lastChunk->next; 158 | lastChunk->next=nullptr; 159 | } 160 | } 161 | 162 | T* ptr=(T*)&lastChunk->getMem()[lastChunk->used]; 163 | lastChunk->used+=sizeof(T); 164 | return ptr; 165 | } 166 | /// Return a piece of memory from the allocator to this allocator 167 | void deallocate(T* ptr) { 168 | FreeElement* f = reinterpret_cast(ptr); 169 | f->next = freeElements; 170 | freeElements=f; 171 | } 172 | /// Swap two pools 173 | void swap(Pool& other) { 174 | std::swap(firstChunk,other.firstChunk); 175 | std::swap(lastChunk,other.lastChunk); 176 | std::swap(freeElements,other.freeElements); 177 | } 178 | 179 | void reset() { 180 | for (Chunk* c=firstChunk;c!=0;) { 181 | c->used=0; 182 | c=c->next; 183 | } 184 | assert(lastChunk->size>0); 185 | lastChunk = firstChunk; 186 | assert(lastChunk->size>0); 187 | freeElements = nullptr; 188 | } 189 | }; 190 | 191 | /// Swap two pools 192 | template void swap(Pool& a,Pool& b) { 193 | a.swap(b); 194 | } 195 | -------------------------------------------------------------------------------- /include/queue.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include 7 | #include 8 | #include 9 | #include "macros.hpp" 10 | 11 | namespace awfy { 12 | 13 | template 14 | class FixedSizeQueue { 15 | private: 16 | T* elems; 17 | T* startPtr; 18 | T* endPtr; 19 | size_t size_; 20 | 21 | public: 22 | FixedSizeQueue(size_t size) : elems(new T[size]),startPtr(elems),endPtr(elems),size_(size) 23 | { } 24 | 25 | ~FixedSizeQueue() { 26 | if(elems!=nullptr) { 27 | delete[] elems; 28 | } 29 | } 30 | 31 | inline bool empty() const { 32 | return startPtr==endPtr; 33 | } 34 | 35 | inline size_t size() const { 36 | return endPtr-startPtr; 37 | } 38 | 39 | inline const T& front() const { 40 | assert(!empty()); 41 | return *startPtr; 42 | } 43 | 44 | inline void pop_front() { 45 | assert(!empty()); 46 | startPtr++; 47 | } 48 | 49 | inline T& push_back_pos() { 50 | assert(endPtrsize_) { 56 | delete[] elems; 57 | elems = new T[newSize]; 58 | size_ = newSize; 59 | } 60 | startPtr=elems; 61 | endPtr=elems; 62 | } 63 | 64 | inline std::pair bounds() { 65 | return make_pair(startPtr,endPtr); 66 | } 67 | }; 68 | 69 | } 70 | -------------------------------------------------------------------------------- /include/scheduler.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | class Task { 15 | // XXX: Everything should be const 16 | private: 17 | void (*functionPtr)(void*); 18 | void* arguments; 19 | 20 | public: 21 | unsigned groupId; 22 | 23 | Task(void (*functionPtr)(void*),void* arguments, unsigned groupId=999) 24 | : functionPtr(functionPtr),arguments(arguments),groupId(groupId) { 25 | } 26 | 27 | ~Task() { 28 | } 29 | 30 | void execute() { 31 | (*functionPtr)(arguments); 32 | } 33 | }; 34 | 35 | struct Priorities { 36 | enum Priority { 37 | LOW=10, 38 | DEFAULT=11, 39 | NORMAL=30, 40 | URGENT=50, 41 | CRITICAL=70, 42 | HYPER_CRITICAL=80 43 | }; 44 | }; 45 | 46 | struct TaskOrder { 47 | Priorities::Priority priority; 48 | unsigned insertion; 49 | 50 | TaskOrder(const Priorities::Priority priority,const unsigned insertion) 51 | : priority(priority), insertion(insertion) { } 52 | }; 53 | 54 | typedef std::pair OrderedTask; 55 | 56 | struct TaskOrderCmp { 57 | bool operator() (OrderedTask const &a, OrderedTask const &b) { 58 | // XXX: insertion order should be the other way round but leads to performance regression 59 | return a.first.priority < b.first.priority || ((a.first.priority == b.first.priority) && a.first.insertion > b.first.insertion); 60 | } 61 | }; 62 | 63 | // Priority ordered scheduler 64 | class Scheduler { 65 | std::priority_queue, TaskOrderCmp> ioTasks; 66 | std::priority_queue, TaskOrderCmp> workTasks; 67 | std::mutex taskMutex; 68 | std::condition_variable taskCondition; 69 | std::mutex threadsMutex; 70 | std::condition_variable threadsCondition; 71 | std::atomic numThreads; 72 | volatile bool closeOnEmpty; 73 | volatile bool currentlyEmpty; 74 | volatile unsigned nextTaskId; 75 | 76 | public: 77 | Scheduler(); 78 | ~Scheduler(); 79 | 80 | Scheduler(const Scheduler&) = delete; 81 | Scheduler(Scheduler&&) = delete; 82 | 83 | void schedule(const std::vector& funcs, Priorities::Priority priority=Priorities::DEFAULT, bool isIO=true); 84 | void schedule(const Task& task, Priorities::Priority priority=Priorities::DEFAULT, bool isIO=true); 85 | Task* getTask(bool preferIO=true); 86 | size_t size(); 87 | void setCloseOnEmpty(); 88 | void registerThread(); 89 | void unregisterThread(); 90 | void waitAllFinished(); 91 | }; 92 | 93 | // Simple executor that will run the tasks until no more exist 94 | struct Executor { 95 | const bool preferIO; 96 | const uint32_t coreId; 97 | Scheduler& scheduler; 98 | 99 | Executor(Scheduler& scheduler, uint32_t coreId, bool preferIO) 100 | : preferIO(preferIO), coreId(coreId), scheduler(scheduler) { 101 | } 102 | 103 | void run(); 104 | 105 | static void* start(void* argument); 106 | }; 107 | 108 | class TaskGroup { 109 | std::vector tasks; 110 | 111 | public: 112 | void schedule(Task task); 113 | void join(Task joinTask); 114 | std::vector close(); 115 | }; 116 | 117 | /// Utility class to make conversion from lambda to task easier 118 | class LambdaRunner { 119 | std::function fn; 120 | 121 | LambdaRunner(std::function&& fn); 122 | public: 123 | static Task createLambdaTask(std::function&& fn); 124 | static void* run(LambdaRunner* runner); 125 | }; 126 | -------------------------------------------------------------------------------- /include/tokenizer.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | struct Tokenizer { 11 | const uint8_t* end; 12 | 13 | uint8_t* iter; 14 | 15 | Tokenizer(void* data, uint64_t size) 16 | : end(reinterpret_cast(data)+size), iter(reinterpret_cast(data)) { 17 | } 18 | 19 | inline uint64_t readId(const char delim) { 20 | uint64_t id=0; 21 | while(iter!=end && *iter != delim) { 22 | assert(*iter>='0' && *iter<='9'); 23 | id=id*10+(*iter-'0'); 24 | iter++; 25 | } 26 | 27 | if(*iter==delim) { 28 | iter++; 29 | } 30 | 31 | return id; 32 | } 33 | 34 | inline std::string readStr(const char delim) { 35 | const unsigned char* start = iter; 36 | size_t length = 0; 37 | while(iter!=end && *iter != delim) { 38 | length++; 39 | iter++; 40 | } 41 | 42 | std::string result((char*)start, length); 43 | 44 | if(*iter==delim) { 45 | iter++; 46 | } 47 | 48 | return result; 49 | } 50 | 51 | inline void skipLine() { 52 | while(iter!=end && *iter != '\n') { 53 | iter++; 54 | } 55 | 56 | if(*iter == '\n') { 57 | iter++; 58 | } 59 | } 60 | 61 | inline bool isFinished() { 62 | return iter==end; 63 | } 64 | }; -------------------------------------------------------------------------------- /include/topklist.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace awfy { 12 | 13 | template 14 | class TopKComparer; 15 | 16 | template 17 | bool compare(const T& a, const T& b) { return TopKComparer::compare(a,b); } 18 | 19 | template 20 | class TopKComparer { 21 | // Returns true if first param is larger or equal 22 | static bool compare(const T& a, const T& b); 23 | }; 24 | 25 | 26 | /// Class managing a list of top k values 27 | template 28 | class TopKList { 29 | public: 30 | typedef std::pair EntryPair; 31 | typedef typename std::vector::iterator EntryIter; 32 | 33 | private: 34 | const EntryPair initialBound; 35 | std::vector topMatches; 36 | size_t k; 37 | 38 | // Find insert position of element 39 | EntryIter getInsertPos(const EntryPair& pair) { 40 | //TODO: Search from end 41 | EntryIter iter=topMatches.begin(); 42 | while(iter!=topMatches.end() && compare(*iter, pair)) { 43 | iter++; 44 | } 45 | return iter; 46 | } 47 | 48 | public: 49 | TopKList(EntryPair initialBound) : initialBound(initialBound) { 50 | } 51 | 52 | TopKList(const TopKList&) = delete; 53 | TopKList& operator=(const TopKList&) = delete; 54 | TopKList(TopKList&&) = delete; 55 | TopKList& operator=(TopKList&&) = delete; 56 | 57 | /// Resets the top k list and initializes it with the k value 58 | void init(const size_t k) { 59 | this->k=k; 60 | topMatches.clear(); 61 | topMatches.reserve(k); 62 | topMatches.push_back(initialBound); 63 | } 64 | 65 | /// Try to insert a value into top k list, will not update if value is too small 66 | /// Returns the new bound 67 | void insert(const Key& key, const Value& value) { 68 | EntryPair pair = std::make_pair(key, value); 69 | assert(compare(pair, initialBound)); //Everything we insert must be smaller than the initial bound 70 | // Insert entries in order until list is full 71 | if (topMatches.size() < k){ 72 | auto insertPos=getInsertPos(pair); 73 | topMatches.insert(insertPos, pair); 74 | } 75 | // Check if new entry is larger than existing entry 76 | else if(!compare(topMatches.back(), pair)) { 77 | topMatches.pop_back(); // Remove last entry 78 | auto insertPos=getInsertPos(pair); // Insert new entry 79 | topMatches.insert(insertPos, pair); 80 | } 81 | } 82 | 83 | inline const EntryPair& getBound() { 84 | return topMatches.back(); 85 | } 86 | 87 | const std::vector& getEntries() { 88 | //Remove dummy element if necesary 89 | if(topMatches.size()>0 && topMatches.back() == initialBound) { 90 | topMatches.pop_back(); 91 | } 92 | //Return matches 93 | return topMatches; 94 | } 95 | }; 96 | } -------------------------------------------------------------------------------- /include/worker.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | #include "scheduler.hpp" 7 | 8 | struct Workers { 9 | 10 | Scheduler scheduler; 11 | std::vector threads; 12 | 13 | Workers(uint32_t numWorkers); 14 | 15 | void assist(Scheduler& scheduler); 16 | void close(); 17 | }; 18 | -------------------------------------------------------------------------------- /io.cpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #include "include/io.hpp" 5 | #include "include/log.hpp" 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | using namespace io; 15 | 16 | MmapedFile::MmapedFile(const std::string& path, int flags) { 17 | fd = ::open(path.c_str(),flags); 18 | if (fd<0) { FATAL_ERROR("Could not open file. Missing \"/\" at end of path? " << path); } 19 | size = lseek(fd,0,SEEK_END); 20 | if (!(~size)) { ::close(fd); FATAL_ERROR("Could not get file size. " << path); } 21 | mapping = mmap(0,size,PROT_READ,MAP_PRIVATE,fd,0); 22 | if (!mapping) { ::close(fd); FATAL_ERROR("Could not memory map file. " << path); } 23 | if (mapping == (void*)(~((size_t)0))) { ::close(fd); FATAL_ERROR("Could not memory map file. (Maybe directory): " << path); } 24 | } 25 | 26 | MmapedFile::MmapedFile(MmapedFile&& other) : fd(other.fd), size(other.size), mapping(other.mapping) { 27 | other.fd = -1; 28 | other.size = 0; 29 | other.mapping = nullptr; 30 | } 31 | 32 | MmapedFile::~MmapedFile() { 33 | if(fd != -1) { 34 | munmap(mapping, size); 35 | ::close(fd); 36 | } 37 | } 38 | 39 | size_t io::fileLines(const std::string& path) { 40 | io::MmapedFile file(path, O_RDONLY); 41 | madvise(reinterpret_cast(file.mapping),file.size,MADV_SEQUENTIAL|MADV_WILLNEED); 42 | 43 | char* data = reinterpret_cast(file.mapping); 44 | size_t numLines=0; 45 | for(size_t i=0; i 7 | 8 | tschrono::Time tschrono::now() { 9 | static const std::chrono::high_resolution_clock::time_point startTime = std::chrono::high_resolution_clock::now(); 10 | auto current=std::chrono::high_resolution_clock::now(); 11 | return std::chrono::duration_cast(current-startTime).count(); 12 | } 13 | 14 | void tschrono::TimeFrame::start() { 15 | startTime=tschrono::now(); 16 | } 17 | 18 | void tschrono::TimeFrame::end() { 19 | endTime=tschrono::now(); 20 | duration=endTime-startTime; 21 | } -------------------------------------------------------------------------------- /main.cpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #include "include/bench.hpp" 5 | #include "include/TraceStats.hpp" 6 | 7 | int main(int argc, char** argv) { 8 | if(argc<2) { 9 | FATAL_ERROR("Not enough parameters"); 10 | } 11 | 12 | Queries queries = Queries::loadFromFile(std::string(argv[1])); 13 | const int numRuns = argc==2 ? 1 : std::stoi(std::string(argv[2])); 14 | 15 | size_t numThreads = std::thread::hardware_concurrency()/2; 16 | if(argc>3) { 17 | numThreads = std::stoi(std::string(argv[3])); 18 | } 19 | LOG_PRINT("[Main] Using "<< numThreads <<" threads"); 20 | 21 | size_t bfsLimit = std::numeric_limits::max(); 22 | if(argc>4) { 23 | bfsLimit = std::stoi(std::string(argv[4])); 24 | LOG_PRINT("[Main] Limit to "<< bfsLimit <<" bfs"); 25 | } 26 | 27 | vector> benchmarks; 28 | #ifdef AVX2 29 | benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 256 SP (width 1)"))); 30 | benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 256 NoSP (width 1)"))); 31 | benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 256 SP (width 2)"))); 32 | benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 256 NoSp (width 2)"))); 33 | benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 256 SP (width 4)"))); 34 | benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 256 NoSp (width 4)"))); 35 | // benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark("Batch BFS Runner 256"))); 36 | #endif 37 | //benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 128 (width 1)"))); 38 | //benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 64 (width 1)"))); 39 | benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 128 (width 4)"))); 40 | // benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 64 (width 8)"))); 41 | //benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 32 (width 16)"))); 42 | //benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 16 (width 32)"))); 43 | //benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 8 (width 64)"))); 44 | //benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 8 (width 1)"))); 45 | //benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark("BatchBFSRunner128"))); 46 | //benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 64 (width 2)"))); 47 | //benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark>("Huge Batch BFS Runner 64 (width 4)"))); 48 | 49 | // benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark("BatchBFSRunner64"))); 50 | // benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark("NoQueueBFSRunner"))); 51 | // benchmarks.push_back(std::unique_ptr(new SpecializedBFSBenchmark("BFSRunner"))); 52 | 53 | size_t maxBatchSize=0; 54 | for(auto& benchmark : benchmarks) { 55 | if(benchmark->batchSize()>maxBatchSize) { 56 | maxBatchSize = benchmark->batchSize(); 57 | } 58 | } 59 | LOG_PRINT("[Main] Max batch size: "<::max(); 66 | for(unsigned i=0; i::loadFromPath(query.dataset); 70 | { 71 | auto ranges = generateTasks(bfsLimit, personGraph.size(), maxBatchSize); 72 | auto desiredTasks=numThreads*4; 73 | if(ranges.size()name<<" ... "; 80 | cout.flush(); 81 | for(int a=0; ainitTrace(personGraph.numVertices, personGraph.numEdges, numThreads, bfsLimit, "NotWorkingHere"); 83 | b->run(7, personGraph, query.reference, workers, bfsLimit); 84 | cout<lastRuntime()<<"ms "; 85 | cout.flush(); 86 | } 87 | 88 | if(b->avgRuntime()avgRuntime(); 90 | cout<<" new best => "<avgRuntime()<<" rel: "<< b->avgRuntime()/(double)minAvgRuntime<<"x"<avgRuntime()<<"\t"<<(b->avgRuntime()/(double)minAvgRuntime)<<" # (ms, factor) "<name< 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "query4.hpp" 10 | #include 11 | 12 | using namespace std; 13 | 14 | std::vector> generateTasks(const uint64_t maxBfs, const Query4::PersonId graphSize, const size_t batchSize) { 15 | // Determine number of persons 16 | Query4::PersonId numBfs = graphSizeQuery4::maxMorselTasks) { 23 | taskSize+=batchSize; 24 | } 25 | 26 | LOG_PRINT("[TaskGen] Task size "<< taskSize); 27 | 28 | // Create tasks 29 | std::vector> ranges; 30 | for (unsigned i = 0; i < numBfs; i+=taskSize) { 31 | Query4::PersonId start = i; 32 | Query4::PersonId limit = (start+taskSize)>numBfs?numBfs:(start+taskSize); 33 | ranges.push_back(make_pair(start,limit)); 34 | } 35 | 36 | return ranges; 37 | } 38 | // Comperator to create correct ordering of results 39 | namespace Query4 { 40 | 41 | size_t getMaxMorselBatchSize() { 42 | char* userBatchSizeStr; 43 | userBatchSizeStr = getenv("MAX_BATCH_SIZE"); 44 | if(userBatchSizeStr!=nullptr) { 45 | return atoi(userBatchSizeStr); 46 | } else { 47 | return std::numeric_limits::max(); 48 | } 49 | } 50 | 51 | double getCloseness(uint32_t totalPersons,uint64_t totalDistances,uint32_t totalReachable) { 52 | return (totalDistances>0 && totalReachable>0 && totalPersons>0) 53 | ? static_cast((totalReachable-1)*(totalReachable-1)) / (static_cast((totalPersons-1))*totalDistances) 54 | : 0.0; 55 | } 56 | 57 | 58 | ResultConcatenator::ResultConcatenator(QueryState* state, const char*& resultOut 59 | #ifdef STATISTICS 60 | , BatchStatistics& statistics 61 | #endif 62 | ) 63 | : state(state), resultOut(resultOut) 64 | #ifdef STATISTICS 65 | , statistics(statistics) 66 | #endif 67 | { 68 | } 69 | 70 | void ResultConcatenator::operator()() { 71 | #ifdef STATISTICS 72 | statistics.print(); 73 | #endif 74 | 75 | ostringstream output; 76 | auto& topEntries=state->topResults.getEntries(); 77 | assert(topEntries.size()<=state->k); 78 | const uint32_t resNum = min(state->k, (uint32_t)topEntries.size()); 79 | for (uint32_t i=0; i0) { 81 | output<<"|"; 82 | } 83 | 84 | output<subgraph.mapInternalNodeId(topEntries[i].first); 85 | } 86 | const auto& outStr = output.str(); 87 | auto resultBuffer = new char[outStr.size()+1]; 88 | outStr.copy(resultBuffer, outStr.size()); 89 | resultBuffer[outStr.size()]=0; 90 | resultOut = resultBuffer; 91 | delete state; 92 | } 93 | } -------------------------------------------------------------------------------- /query4.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #pragma once 5 | 6 | // #define STATISTICS 7 | // #define FULL_STATISTICS 8 | 9 | #include "include/idschedulers.hpp" 10 | 11 | #include "include/topklist.hpp" 12 | #include "include/log.hpp" 13 | #include "include/bfs/base.hpp" 14 | 15 | #include "include/bfs/batchhuge.hpp" 16 | #include "include/scheduler.hpp" 17 | #include "include/worker.hpp" 18 | #include "include/bfs/naive.hpp" 19 | #include "include/bfs/batch64.hpp" 20 | #include "include/bfs/batch128.hpp" 21 | #include "include/bfs/batch256.hpp" 22 | #include "include/bfs/statistics.hpp" 23 | 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | 30 | // #define OUTPUT_PROGRESS 31 | 32 | using namespace std; 33 | 34 | namespace Query4 { 35 | 36 | static const uint32_t maxMorselTasks = 256000; 37 | static const uint32_t minMorselSize = 1; 38 | 39 | struct CentralityResult { 40 | PersonId person; 41 | uint64_t distances; 42 | uint32_t numReachable; 43 | double centrality; 44 | 45 | CentralityResult(PersonId person, uint64_t distances, uint32_t numReachable, double centrality) 46 | : person(person), distances(distances), numReachable(numReachable), centrality(centrality) { 47 | } 48 | 49 | bool operator==(const CentralityResult& other) const { 50 | return person==other.person&¢rality==other.centrality; 51 | } 52 | 53 | friend std::ostream& operator<<(std::ostream &out, const CentralityResult& v) { 54 | out< CentralityEntry; 60 | } 61 | 62 | namespace awfy { 63 | static const double EPSILON = 0.000000000001; 64 | 65 | template<> 66 | class TopKComparer { 67 | public: 68 | // Returns true if first param is larger or equal 69 | static bool compare(const Query4::CentralityEntry& a, const Query4::CentralityEntry& b) 70 | { 71 | auto delta = a.second.centrality - b.second.centrality; 72 | return ((delta >0) || (fabs(delta)< EPSILON && a.second.person < b.second.person)); 73 | } 74 | }; 75 | } 76 | 77 | namespace Query4 { 78 | typedef awfy::TopKComparer CentralityCmp; 79 | 80 | class QueryState { 81 | public: 82 | const uint32_t k; 83 | const PersonSubgraph& subgraph; 84 | const uint64_t startTime; 85 | 86 | vector personChecked; 87 | 88 | mutex topResultsMutex; 89 | awfy::TopKList topResults; 90 | 91 | QueryState(const uint32_t k, const PersonSubgraph& subgraph) 92 | : k(k), subgraph(move(subgraph)), startTime(tschrono::now()), personChecked(subgraph.size()), topResultsMutex(), 93 | topResults(make_pair(std::numeric_limits::max(),CentralityResult(std::numeric_limits::max(), 0, 0, 0.0))) { 94 | topResults.init(k); 95 | } 96 | }; 97 | 98 | double getCloseness(uint32_t totalPersons,uint64_t totalDistances,uint32_t totalReachable); 99 | 100 | struct ResultConcatenator { 101 | QueryState* state; 102 | const char*& resultOut; 103 | #ifdef STATISTICS 104 | BatchStatistics& statistics; 105 | #endif 106 | 107 | ResultConcatenator(QueryState* state, const char*& resultOut 108 | #ifdef STATISTICS 109 | , BatchStatistics& statistics 110 | #endif 111 | ); 112 | void operator()(); 113 | 114 | // ResultConcatenator(ResultConcatenator&&) = default; 115 | // ResultConcatenator& operator=(ResultConcatenator&&) = default; 116 | }; 117 | 118 | size_t getMaxMorselBatchSize(); 119 | 120 | template 121 | struct MorselTask { 122 | private: 123 | const uint32_t rangeStart; 124 | const uint32_t rangeEnd; 125 | 126 | size_t batchSize; 127 | 128 | QueryState& state; 129 | const PersonSubgraph& subgraph; 130 | vector& ids; 131 | 132 | const uint64_t startTime; 133 | 134 | 135 | #ifdef STATISTICS 136 | BatchStatistics& statistics; 137 | #endif 138 | 139 | public: 140 | MorselTask(QueryState& state, PersonId rangeStart, PersonId rangeEnd, const PersonSubgraph& subgraph, vector& ids, uint64_t startTime 141 | #ifdef STATISTICS 142 | , BatchStatistics& statistics 143 | #endif 144 | ) 145 | : rangeStart(rangeStart), rangeEnd(rangeEnd), state(state), subgraph(subgraph), ids(ids), startTime(startTime) 146 | #ifdef STATISTICS 147 | , statistics(statistics) 148 | #endif 149 | { 150 | batchSize = BFSRunnerT::batchSize(); 151 | if(batchSize>getMaxMorselBatchSize()) { 152 | batchSize=getMaxMorselBatchSize(); 153 | } 154 | } 155 | 156 | //Returns pair of processed persons and whether the bound was updated 157 | pair processPersonBatch(PersonId begin, PersonId end) { 158 | // Build batch with the desired size 159 | vector batchData; 160 | batchData.reserve(batchSize); 161 | 162 | PersonId index=begin; 163 | for(; batchData.size()0) { 180 | //Run BFS 181 | BFSRunnerT::runBatch(batchData, state.subgraph 182 | #ifdef STATISTICS 183 | , statistics 184 | #endif 185 | ); 186 | 187 | for(auto bIter=batchData.begin(); bIter!=batchData.end(); bIter++) { 188 | const auto closeness = getCloseness(bIter->componentSize, bIter->totalDistances, bIter->totalReachable); 189 | const PersonId externalPersonId = bIter->person; 190 | CentralityResult resultCentrality(externalPersonId, bIter->totalDistances, bIter->totalReachable, closeness); 191 | 192 | // Check if person qualifies as new top k value 193 | if(CentralityCmp::compare(make_pair(resultCentrality.person, resultCentrality), state.topResults.getBound())) { 194 | // Add improved value to top k list 195 | lock_guard lock(state.topResultsMutex); 196 | state.topResults.insert(resultCentrality.person, resultCentrality); 197 | } 198 | } 199 | } 200 | 201 | return make_pair(last-begin+1, boundUpdated); 202 | } 203 | 204 | void operator()() { 205 | if(rangeEnd lock(m); 213 | std::cout<<"#"<> generateTasks(const uint64_t maxBfs, const Query4::PersonId graphSize, const size_t batchSize); 227 | 228 | template 229 | std::string runBFS(const uint32_t k, const Query4::PersonSubgraph& subgraph, Workers& workers, const uint64_t maxBfs, uint64_t& runtimeOut 230 | #ifdef STATISTICS 231 | , Query4::BatchStatistics& statistics 232 | #endif 233 | ) { 234 | // #ifdef STATISTICS 235 | // numTouchedPersonInDistance.clear(); 236 | // numTouchedPersonInDistance.resize(subgraph.size()); 237 | // for(uint32_t a=1; a ids(subgraph.size()); 247 | for (unsigned i = 0; i < subgraph.size(); ++i) { 248 | ids[i] = i; 249 | } 250 | 251 | // Deterministic random shuffeling 252 | RandomNodeOrdering::order(ids, BFSRunnerT::batchSize(), subgraph); 253 | // Do final ordering on specified subset 254 | LOG_PRINT("[Query4] Starting sort "< bfsTask(*queryState, range.first, range.second, subgraph, ids, start 271 | #ifdef STATISTICS 272 | , statistics 273 | #endif 274 | ); 275 | tasks.schedule(LambdaRunner::createLambdaTask(bfsTask)); 276 | 277 | for(Query4::PersonId i=range.first; i& stats = TraceStats::getStats(); 282 | stats.setNumTraversedEdges(numTraversedEdges); 283 | 284 | //std::cout << "# TaskStats "<>("BatchBFS "+std::to_string(sizeof(CTYPE)*8)+" ("+std::to_string(WIDTH)+")"); \ 10 | maxBatchSize = sizeof(CTYPE)*8*WIDTH; \ 11 | bfsType = std::to_string(sizeof(CTYPE)*8)+"_"+std::to_string(WIDTH); \ 12 | } \ 13 | 14 | 15 | int main(int argc, char** argv) { 16 | if(argc!=6 && argc!=7 && argc!=8) { 17 | FATAL_ERROR("Not enough parameters"); 18 | } 19 | 20 | Queries queries = Queries::loadFromFile(std::string(argv[1])); 21 | const int numRuns = std::stoi(std::string(argv[2])); 22 | 23 | //size_t bfsLimit = std::numeric_limits::max(); 24 | size_t bfsLimit = argc>=7?std::stoi(std::string(argv[6])):std::numeric_limits::max(); 25 | bool checkNumTasks = argc>=8&&argv[7][0]=='f'?false:true; 26 | size_t numThreads = std::thread::hardware_concurrency()/2; 27 | if(argc>3) { 28 | numThreads = std::stoi(std::string(argv[3])); 29 | } 30 | LOG_PRINT("[Main] Using "<< numThreads <<" threads"); 31 | 32 | size_t maxBatchSize; 33 | BFSBenchmark* bencher; 34 | std::string bfsType; 35 | Query4::PARABFSRunner::setThreadNum(numThreads);//Using threads inside BFS 36 | if(std::string(argv[4])=="naive") { 37 | bencher = new SpecializedBFSBenchmark("BFSRunner"); 38 | maxBatchSize = 1; 39 | bfsType = "naive"; 40 | } else if(std::string(argv[4])=="noqueue") { 41 | bencher = new SpecializedBFSBenchmark("NoQueueBFSRunner"); 42 | maxBatchSize = 1; 43 | bfsType = "noqueue"; 44 | } else if(std::string(argv[4])=="scbfs") { 45 | bencher = new SpecializedBFSBenchmark("SCBFSRunner"); 46 | maxBatchSize = 1; 47 | bfsType = "scbfs"; 48 | } else if(std::string(argv[4])=="parabfs") { 49 | bencher = new SpecializedBFSBenchmark("PARABFSRunner"); 50 | maxBatchSize = 1; 51 | bfsType = "parabfs"; 52 | numThreads = 1; 53 | } else { 54 | const int batchType = std::stoi(std::string(argv[4])); 55 | const int batchWidth = std::stoi(std::string(argv[5])); 56 | GEN_BENCH_BRANCH(if,__m128i,8) 57 | GEN_BENCH_BRANCH(else if,__m128i,4) 58 | GEN_BENCH_BRANCH(else if,__m128i,1) 59 | #ifdef AVX2 60 | GEN_BENCH_BRANCH(else if,__m256i,2) 61 | GEN_BENCH_BRANCH(else if,__m256i,1) 62 | #endif 63 | GEN_BENCH_BRANCH(else if,uint64_t,8) 64 | GEN_BENCH_BRANCH(else if,uint64_t,1) 65 | GEN_BENCH_BRANCH(else if,uint32_t,16) 66 | GEN_BENCH_BRANCH(else if,uint32_t,1) 67 | GEN_BENCH_BRANCH(else if,uint16_t,32) 68 | GEN_BENCH_BRANCH(else if,uint16_t,1) 69 | GEN_BENCH_BRANCH(else if,uint8_t,64) 70 | GEN_BENCH_BRANCH(else if,uint8_t,1) 71 | else { 72 | exit(-1); 73 | } 74 | } 75 | 76 | // Allocate additional worker threads 77 | Workers workers(numThreads-1); 78 | 79 | for(unsigned i=0; i::loadFromPath(query.dataset); 83 | if(bfsLimit>personGraph.size()) { 84 | bfsLimit=personGraph.size(); 85 | } 86 | if(checkNumTasks) 87 | { 88 | auto ranges = generateTasks(bfsLimit, personGraph.size(), maxBatchSize); 89 | auto desiredTasks=numThreads*3; 90 | if(ranges.size()name<<" ... "<initTrace(personGraph.numVertices, personGraph.numEdges, numThreads, bfsLimit, bfsType); 99 | bencher->run(7, personGraph, query.reference, workers, bfsLimit); 100 | std::cout<lastRuntime()<<"ms "; 101 | std::cout.flush(); 102 | } 103 | std::cout<getMinTrace()< 8 | #include 9 | 10 | ///--- Scheduler related methods 11 | Scheduler::Scheduler() 12 | : ioTasks(), workTasks(), numThreads(0), closeOnEmpty(false), currentlyEmpty(false), nextTaskId(0) { 13 | } 14 | 15 | Scheduler::~Scheduler() { 16 | assert(ioTasks.size()==0); 17 | assert(workTasks.size()==0); 18 | } 19 | 20 | void Scheduler::schedule(const std::vector& funcs, Priorities::Priority priority, bool isIO) { 21 | { 22 | std::lock_guard lock(taskMutex); 23 | for (unsigned i = 0; i < funcs.size(); ++i) { 24 | Task* task = new Task(funcs[i]); 25 | 26 | if(isIO) { 27 | ioTasks.push(std::make_pair(TaskOrder(priority, nextTaskId++), task)); 28 | } else { 29 | workTasks.push(std::make_pair(TaskOrder(priority, nextTaskId++), task)); 30 | } 31 | } 32 | 33 | if(currentlyEmpty) { 34 | currentlyEmpty=false; 35 | } 36 | } 37 | 38 | taskCondition.notify_all(); 39 | } 40 | 41 | void Scheduler::schedule(const Task& scheduleTask, Priorities::Priority priority, bool isIO) { 42 | { 43 | std::lock_guard lock(taskMutex); 44 | Task* task = new Task(scheduleTask); 45 | 46 | if(isIO) { 47 | ioTasks.push(std::make_pair(TaskOrder(priority, nextTaskId++), task)); 48 | } else { 49 | workTasks.push(std::make_pair(TaskOrder(priority, nextTaskId++), task)); 50 | } 51 | 52 | if(currentlyEmpty) { 53 | currentlyEmpty=false; 54 | } 55 | } 56 | 57 | taskCondition.notify_one(); 58 | } 59 | 60 | Task* Scheduler::getTask(bool preferIO) { 61 | std::unique_lock lck(taskMutex); 62 | while(true) { 63 | // Try to acquire task 64 | if(!ioTasks.empty()||!workTasks.empty()) { 65 | Task* task; 66 | if((preferIO && !ioTasks.empty()) || workTasks.empty()) { 67 | task = ioTasks.top().second; 68 | ioTasks.pop(); 69 | } else { 70 | task = workTasks.top().second; 71 | workTasks.pop(); 72 | } 73 | assert(task!=nullptr); 74 | auto numTasks=ioTasks.size()+workTasks.size(); 75 | if(numTasks==0&&!closeOnEmpty) { 76 | currentlyEmpty=true; 77 | } 78 | 79 | if(numTasks>0) { lck.unlock(); taskCondition.notify_one(); } 80 | else { lck.unlock(); } 81 | return task; 82 | } else { 83 | // Wait if no task is available 84 | if(closeOnEmpty) { 85 | break; 86 | } else { 87 | taskCondition.wait(lck); 88 | } 89 | } 90 | } 91 | 92 | return nullptr; 93 | } 94 | 95 | void Scheduler::setCloseOnEmpty() { 96 | closeOnEmpty=true; 97 | taskCondition.notify_all(); 98 | } 99 | 100 | size_t Scheduler::size() { 101 | return ioTasks.size()+workTasks.size(); 102 | } 103 | 104 | void Scheduler::registerThread() { 105 | numThreads++; 106 | } 107 | 108 | void Scheduler::unregisterThread() { 109 | numThreads--; 110 | threadsCondition.notify_all(); 111 | } 112 | 113 | void Scheduler::waitAllFinished() { 114 | std::unique_lock lck(threadsMutex); 115 | while(true) { 116 | if(closeOnEmpty && numThreads==0) { 117 | return; 118 | } else { 119 | threadsCondition.wait(lck); 120 | } 121 | } 122 | } 123 | 124 | // Executor related implementation 125 | void Executor::run() { 126 | cpu_set_t cpuset; 127 | CPU_ZERO(&cpuset); 128 | CPU_SET(coreId, &cpuset); 129 | pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); 130 | 131 | LOG_PRINT("[Executor] Starting"); 132 | scheduler.registerThread(); 133 | while(true) { 134 | auto task = scheduler.getTask(preferIO); 135 | if(task==nullptr) { break; } 136 | 137 | task->execute(); 138 | delete task; 139 | } 140 | scheduler.unregisterThread(); 141 | LOG_PRINT("[Executor] Stopping"); 142 | } 143 | 144 | void* Executor::start(void* argument) { 145 | Executor* executor = static_cast(argument); 146 | executor->run(); 147 | 148 | delete executor; 149 | return nullptr; 150 | } 151 | 152 | // Taskgroup related implementations 153 | void TaskGroup::schedule(Task task) { 154 | tasks.push_back(task); 155 | } 156 | 157 | struct JoinRunner { 158 | Task fn; 159 | Task* joinTask; 160 | std::atomic* semaphore; 161 | 162 | JoinRunner(Task fn, Task* joinTask, std::atomic* semaphore) : fn(fn), joinTask(joinTask), semaphore(semaphore) { 163 | } 164 | 165 | static void* run(JoinRunner* runner) { 166 | runner->fn.execute(); 167 | auto status=runner->semaphore->fetch_add(-1)-1; 168 | if(status==0) { 169 | runner->joinTask->execute(); 170 | delete runner->semaphore; 171 | delete runner->joinTask; 172 | delete runner; 173 | } 174 | return nullptr; 175 | } 176 | }; 177 | 178 | void TaskGroup::join(Task joinTask) { 179 | if(tasks.size()==0) { 180 | schedule(joinTask); 181 | return; 182 | } 183 | 184 | std::vector wrappedTasks; 185 | 186 | auto joinTaskPtr = new Task(joinTask); 187 | auto joinCounter = new std::atomic(tasks.size()); 188 | 189 | // Wrap existing functions to implement the join 190 | for (unsigned i = 0; i < tasks.size(); ++i) { 191 | JoinRunner* runner = new JoinRunner(tasks[i], joinTaskPtr, joinCounter); 192 | Task wrappedFn((void (*)(void*)) &(JoinRunner::run), runner, tasks[i].groupId); 193 | wrappedTasks.push_back(wrappedFn); 194 | } 195 | tasks = std::move(wrappedTasks); 196 | 197 | } 198 | 199 | std::vector TaskGroup::close() { 200 | return std::move(tasks); 201 | } 202 | 203 | ///--- LambdaRunner related methods 204 | LambdaRunner::LambdaRunner(std::function&& fn) : fn(std::move(fn)) { 205 | } 206 | 207 | Task LambdaRunner::createLambdaTask(std::function&& fn) { 208 | LambdaRunner* runner=new LambdaRunner(std::move(fn)); 209 | return Task((void (*)(void*)) &(LambdaRunner::run), runner); 210 | } 211 | 212 | void* LambdaRunner::run(LambdaRunner* runner) { 213 | runner->fn(); 214 | delete runner; 215 | return nullptr; 216 | } 217 | -------------------------------------------------------------------------------- /test_queries/ldbc10k.txt: -------------------------------------------------------------------------------- 1 | 10000 test_queries/data/ldbc10k.csv 9202|2616|5537|6233|810|434|3560 -------------------------------------------------------------------------------- /vida/lib/boost.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mtodat/ms-bfs/32b44f0a5c06ac8b3831c7b29834563d73dcd5b1/vida/lib/boost.tar.gz -------------------------------------------------------------------------------- /vida/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | set(CMAKE_BUILD_TYPE Release CACHE STRING "Set project build type.") 3 | project(SIGMOD14Contest) 4 | 5 | find_path(BOOST_ROOT boost) 6 | set(Boost_USE_STATIC_LIBS ON) 7 | find_package(Boost 1.42 COMPONENTS iostreams system thread timer chrono REQUIRED) 8 | #find_package(OpenMP) 9 | #if (OPENMP_FOUND) 10 | # set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") 11 | # set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") 12 | #endif() 13 | set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse3 -msse4.1 -march=native") 14 | set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse3 -msse4.1 -march=native") 15 | 16 | include_directories( 17 | ${Boost_INCLUDE_DIR} 18 | ) 19 | 20 | set(SOURCES_COMMON 21 | ## Common source files that are used by all queries 22 | query4.hpp 23 | ) 24 | 25 | if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") 26 | set(LIBS_COMMON 27 | ## Common libraries that are used by all queries 28 | rt 29 | ) 30 | endif (${CMAKE_SYSTEM_NAME} MATCHES "Linux") 31 | 32 | ## add_executable(query4 query4.hpp ${SOURCES_COMMON}) 33 | ## target_link_libraries(query4 ${Boost_LIBRARIES} ${LIBS_COMMON}) 34 | 35 | add_executable(query query.cpp ${SOURCES_COMMON}) 36 | target_link_libraries(query ${Boost_LIBRARIES} ${LIBS_COMMON}) 37 | -------------------------------------------------------------------------------- /vida/src/Makefile: -------------------------------------------------------------------------------- 1 | # C++ FLAGS 2 | BOOSTDIR=./boost 3 | BOOST_INC=$(BOOSTDIR) 4 | BOOST_LIB=$(BOOSTDIR)/stage/lib/libboost_ 5 | 6 | CC=g++ 7 | CXXFLAGS=-O3 -D_FILE_OFSET_BITS=64 -I$(BOOST_INC) -I. 8 | # -msse3 -march=native 9 | CFLAGS=$(CXXFLAGS) 10 | LDFLAGS=$(BOOST_LIB)iostreams.a $(BOOST_LIB)system.a $(BOOST_LIB)thread.a $(BOOST_LIB)timer.a $(BOOST_LIB)chrono.a 11 | 12 | UNAME := $(shell uname) 13 | ifeq ($(UNAME), Linux) 14 | LDFLAGS+=-lrt 15 | endif 16 | 17 | SOURCES=query.cpp 18 | HEADERS=person_graph.hpp msbfs.hpp naive.hpp 19 | EXECUTABLE=query 20 | OBJECTS=$(SOURCES:.cpp=.o) 21 | DEPS=$(OBJECTS:.o=.d) 22 | 23 | all: $(SOURCES) $(EXECUTABLE) 24 | 25 | clean: 26 | @rm -f $(EXECUTABLE) 27 | @rm -f *.{o,d} 28 | @rm -f *.d.* 29 | 30 | .cpp.o: 31 | $(CC) $(CFLAGS) -c -o $@ $< 32 | 33 | $(EXECUTABLE): $(OBJECTS) $(HEADERS) 34 | $(CC) $(OBJECTS) $(LDFLAGS) -o $(EXECUTABLE) 35 | 36 | define create-deps 37 | @$(CC) -x c++ -MM $(CFLAGS) $< | sed 's,\($*\)\.o[ :]*,\1.o $@ : ,g' > $@ 38 | endef 39 | 40 | $(DEPS): $(SOURCES) $(HEADERS) 41 | $(create-deps) 42 | 43 | sinclude $(DEPS) 44 | 45 | 46 | -------------------------------------------------------------------------------- /vida/src/README.md: -------------------------------------------------------------------------------- 1 | # Compile: 2 | ./compile.sh 3 | 4 | # Usage: 5 | ./query x y z t 6 | x: path to input file (ldbci1k.txt, ldbc10k.txt, ldbc100k.txt, etc) 7 | y: number of threads 8 | z: Number of runs 9 | t: type of bfs: 10 | naive: naive BFS 11 | para: parallel BFS, single socket implementation from “Scalable Graph Exploration on Multicore Processors, Agarwal et. al., SC'10" 12 | w: number of sources (if not provided, run for all sources). Not applicable for msbfs. 13 | 14 | # Example: 15 | ./query data/ldbc10k.txt 8 3 para 50000 16 | ./query data/ldbc10k.txt 8 3 para 17 | ./query data/ldbc10k.txt 8 3 naive 50000 18 | ./query data/ldbc10k.txt 8 3 msbfs 19 | -------------------------------------------------------------------------------- /vida/src/compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | HOMEDIR=$PWD/$(dirname $0) #this is actually src dir 3 | echo "Extracting boost..." 4 | tar xzf ${HOMEDIR}/../lib/boost.tar.gz -C ${HOMEDIR}/. 5 | cd ${HOMEDIR}/boost 6 | echo "Bootstrapping..." 7 | ./bootstrap.sh --with-libraries=chrono,filesystem,iostreams,system,thread,timer 8 | echo "Compiling boost..." 9 | ./b2 -s"NO_BZIP2=1" -j8 threading=multi variant=release link=static 10 | cd ${HOMEDIR} 11 | make -j8 12 | -------------------------------------------------------------------------------- /vida/src/data/ldbc10k.txt: -------------------------------------------------------------------------------- 1 | 10000 data/ldbc10k.csv 9202|2616|5537|6233|810|434|3560 2 | -------------------------------------------------------------------------------- /vida/src/hsort.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #ifndef HSORT_HPP 5 | #define HSORT_HPP 6 | #include 7 | 8 | #define HSORT_SWAP(TYPE, __x, __y) { register TYPE __tmp(__x); __x = __y; __y = __tmp; } 9 | #define HSORT_MIN_FOR_RADIX 64 10 | namespace hsort { 11 | 12 | template 13 | static inline void inplaceSelectionSort(Key *A, const unsigned &n) { 14 | register unsigned i, j; 15 | for (i=0; i 26 | static inline void inplaceSelectionSortPair(void *P, const unsigned &n) { 27 | typedef std::pair StdPair; 28 | StdPair *A = (StdPair*)P; 29 | register unsigned i, j; 30 | for (i=0; i::*Field> 41 | static inline void inplaceSelectionSortField(void *P, const unsigned &n) { 42 | typedef std::pair StdPair; 43 | StdPair *A = (StdPair*)P; 44 | register unsigned i, j; 45 | for (i=0; i 56 | static inline void inplaceSelectionSortValue(void *P, const unsigned &n) { 57 | typedef std::pair StdPair; 58 | StdPair *A = (StdPair*)P; 59 | register unsigned i, j; 60 | for (i=0; i 71 | inline void inplaceRadixSort(const int &curByte, Key *A, const unsigned &n) { 72 | uint8_t *B = (uint8_t*)A + curByte; 73 | unsigned i, j, end; 74 | unsigned count[256] = {}; 75 | 76 | for (i=0; i0) { 86 | end += count[i]; 87 | for (j=bucket[i]; j0) { 100 | if (count[i](A+end-count[i], count[i]); 102 | else 103 | hsort::inplaceRadixSort(curByte-1, A+end-count[i], count[i]); 104 | } 105 | } 106 | } 107 | } 108 | 109 | template 110 | inline void inplaceRadixSortKey(const int &curByte, std::pair *A, const unsigned &n) { 111 | typedef std::pair StdPair; 112 | uint8_t *B = (uint8_t*)A + curByte; 113 | unsigned i, j, end; 114 | unsigned count[256] = {}; 115 | 116 | for (i=0; i0) { 126 | end += count[i]; 127 | for (j=bucket[i]; j0) { 140 | if (count[i](A+end-count[i], count[i]); 142 | else 143 | hsort::inplaceRadixSortKey(curByte-1, A+end-count[i], count[i]); 144 | } 145 | } 146 | } 147 | } 148 | 149 | template 150 | inline void inplaceRadixSortValue(const int &curByte, std::pair *A, const unsigned &n) { 151 | typedef std::pair StdPair; 152 | uint8_t *B = (uint8_t*)A + curByte + sizeof(Key); 153 | unsigned i, j, end; 154 | unsigned count[256] = {}; 155 | 156 | for (i=0; i0) { 166 | end += count[i]; 167 | for (j=bucket[i]; j0) { 180 | if (count[i](A+end-count[i], count[i]); 182 | else 183 | hsort::inplaceRadixSortValue(curByte-1, A+end-count[i], count[i]); 184 | } 185 | } 186 | } 187 | } 188 | 189 | template 190 | inline void inplaceRadixSortPair(const int &curByte, std::pair *A, const unsigned &n) { 191 | typedef std::pair StdPair; 192 | uint8_t *B = (uint8_t*)A + curByte; 193 | unsigned i, j, end; 194 | unsigned count[256] = {}; 195 | 196 | for (i=0; i0) { 206 | end += count[i]; 207 | for (j=bucket[i]; j(A+end-count[i], count[i]); 221 | else { 222 | if (curByte>0) 223 | hsort::inplaceRadixSortPair(curByte-1, A+end-count[i], count[i]); 224 | else 225 | hsort::inplaceRadixSortValue(sizeof(Value)-1, A+end-count[i], count[i]); 226 | } 227 | } 228 | } 229 | } 230 | 231 | template 232 | static inline void sort(Key *A, size_t n) { 233 | hsort::inplaceRadixSort(sizeof(Key)-1, A, n); 234 | } 235 | 236 | template 237 | static inline void sort(std::pair *A, size_t n) { 238 | hsort::inplaceRadixSortPair(sizeof(Key)-1, A, n); 239 | } 240 | 241 | template 242 | static inline void sort(RandomAccessIterator A, RandomAccessIterator B) { 243 | hsort::sort(&(*A), &(*B)-&(*A)); 244 | } 245 | 246 | template 247 | static inline void sortByKey(std::pair *A, size_t n) { 248 | hsort::inplaceRadixSortKey(sizeof(Key)-1, A, n); 249 | } 250 | 251 | template 252 | static inline void sortByKey(RandomAccessIterator A, RandomAccessIterator B) { 253 | hsort::sortByKey(&(*A), &(*B)-&(*A)); 254 | } 255 | 256 | } 257 | #undef HSORT_SWAP 258 | #undef HSORT_MIN_FOR_RADIX 259 | 260 | #endif 261 | -------------------------------------------------------------------------------- /vida/src/msbfs.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #include "person_graph.hpp" 5 | #include 6 | #include 7 | #include 8 | 9 | typedef uint64_t BitMask; 10 | 11 | struct Query4 12 | { 13 | typedef std::pair IdPair; 14 | typedef std::vector U32Vector; 15 | typedef std::vector IdPairVector; 16 | typedef boost::unordered_map IdPairMap; 17 | typedef boost::unordered_map U64Map; 18 | typedef std::pair TopK; 19 | typedef std::vector TopKVector; 20 | 21 | PersonGraph *pg; 22 | uint32_t personSize; 23 | U32Vector QR; 24 | uint32_t *R; 25 | uint32_t *D; 26 | uint32_t *E; 27 | 28 | boost::mutex pMutex; 29 | uint32_t pCurrent; 30 | uint32_t pMax; 31 | 32 | const char * dataPath; 33 | void * data; 34 | int k;//TOP K 35 | 36 | void perform() 37 | { 38 | std::string result = this->compute(k); 39 | printf("Result: %s\n", result.c_str()); 40 | } 41 | 42 | Query4(PersonGraph *G, int topk): data(0) 43 | { 44 | k = topk; 45 | this->pg = G; 46 | this->personSize = this->pg->personSize; 47 | this->QR.resize(3*this->personSize+1 + this->pg->adjacencyList.size() + 8*this->personSize); 48 | this->R = &this->QR[0]; 49 | this->D = this->R + personSize; 50 | this->E = this->D + personSize; 51 | } 52 | 53 | struct topk_comparator { 54 | bool operator()(const TopK &lhs, const TopK &rhs) const { 55 | if (lhs.first == rhs.first) 56 | return (lhs.second > rhs.second); 57 | return (lhs.first < rhs.first); 58 | } 59 | }; 60 | typedef std::set TopKSet; 61 | 62 | template 63 | void createAdjacencyList(uint32_t vertexCount, uint32_t *vMap, BitSet &personMap, uint32_t *E, uint32_t *A) 64 | { 65 | uint32_t edgeCount = 0; 66 | IdType *edges = (IdType*)(A); 67 | for (int nu=0; nuD[nu]; 69 | E[nu] = edgeCount; 70 | for (int k=this->pg->edgeIndex[u]; kpg->edgeIndex[u+1]; k++) { 71 | uint32_t v = this->pg->adjacencyList[k]; 72 | if (personMap[v]) { 73 | edges[edgeCount++] = vMap[v]; 74 | } 75 | } 76 | } 77 | E[vertexCount] = edgeCount; 78 | } 79 | 80 | std::string compute(int k) 81 | { 82 | BitSet personMap(this->personSize); 83 | uint32_t n = this->personSize; 84 | if (n==0) return ""; 85 | //for (int i=0; iD = this->R + n; 90 | this->E = this->D + n; 91 | uint32_t *A = this->E + n+1; 92 | uint32_t *Q = A + this->pg->adjacencyList.size(); 93 | 94 | bool use16bit = n<65536; 95 | { 96 | for (int u=0,cnt=0; upersonSize; u++) { 97 | if (personMap[u]) { 98 | this->D[cnt] = u; 99 | Q[u] = cnt++; 100 | } 101 | } 102 | } 103 | if (use16bit) 104 | this->createAdjacencyList(n, Q, personMap, E, A); 105 | else 106 | this->createAdjacencyList(n, Q, personMap, E, A); 107 | Q = A + E[n]; 108 | 109 | IdPairVector rank(n); 110 | { 111 | personMap.resize(n); 112 | personMap.reset(); 113 | for (int i=0; icompute_r_p(i, personMap, n, E, (uint16_t*)A, Q); 117 | else 118 | this->compute_r_p(i, personMap, n, E, A, Q); 119 | } 120 | uint32_t deg = E[i+1]-E[i]; 121 | rank[i] = IdPair(~deg, i); 122 | } 123 | hsort::sort(rank.begin(), rank.end()); 124 | } 125 | uint32_t topPos = k + (n-k) % NBITS; 126 | double top_min = -1; 127 | const int nthread = 8; 128 | TopKSet pc[nthread]; 129 | boost::thread_group workers; 130 | for (int i=0; i, this, IdPair(i, topPos), nthread, top_min, 133 | IdPair(n, k), boost::cref(rank), (uint16_t*)A, Q + i*n, boost::ref(pc[i]))); 134 | else 135 | workers.create_thread(boost::bind(&Query4::call, this, IdPair(i, topPos), nthread, top_min, 136 | IdPair(n, k), boost::cref(rank), A, Q + i*n, boost::ref(pc[i]))); 137 | workers.join_all(); 138 | 139 | for (int i=0; i0 && (top_min<0 || top_min>pc[i].begin()->first)) 141 | top_min = pc[i].begin()->first; 142 | 143 | { 144 | if (use16bit) 145 | this->closeness_centrality(rank[topPos+8].second, top_min, n, this->E, (uint16_t*)A, Q); 146 | else 147 | this->closeness_centrality(rank[topPos+8].second, top_min, n, this->E, A, Q); 148 | } 149 | 150 | this->pCurrent = topPos; 151 | this->pMax = n; 152 | for (int i=0; i, this, nthread, top_min, 155 | IdPair(n, k), boost::cref(rank), (uint16_t*)A, boost::ref(pc[i]))); 156 | else 157 | workers.create_thread(boost::bind(&Query4::call64, this, nthread, top_min, 158 | IdPair(n, k), boost::cref(rank), A, boost::ref(pc[i]))); 159 | workers.join_all(); 160 | 161 | TopKVector personCentrality; 162 | for (int i=0; i0 && it!=personCentrality.rend(); it++, --k) { 170 | sprintf(result, "%llu ", it->second); 171 | results += result; 172 | } 173 | 174 | if (!results.empty()) 175 | results.resize(results.size()-1); 176 | return results; 177 | } 178 | 179 | 180 | #define EXPAND(u) { \ 181 | if (!curF[u]) continue; \ 182 | register BitMask bm = curF[u] & active; \ 183 | curF[u] = 0; \ 184 | if (!bm) continue; \ 185 | for(int k=this->E[u],e=this->E[u+1]; ks_p_max[j]) { \ 207 | s_p[j] = 0; \ 208 | active &= nshift; \ 209 | } \ 210 | } \ 211 | } \ 212 | } 213 | 214 | template 215 | void call64(int stride, double current_min, IdPair nk, const IdPairVector &rank, IdType *adjacencyList, TopKSet &personCentrality) 216 | { 217 | BitMask *F = (BitMask*)malloc(3*nk.first*sizeof(BitMask)); 218 | BitMask *FF[2] = {F, F + nk.first}; 219 | BitMask *VV = F + 2*nk.first; 220 | 221 | uint32_t i=0; 222 | while (1) { 223 | { 224 | this->pMutex.lock(); 225 | i = this->pCurrent; 226 | this->pCurrent += NBITS; 227 | this->pMutex.unlock(); 228 | } 229 | if (i>=this->pMax) break; 230 | 231 | // initialize 232 | memset(F, 0, 3*nk.first*sizeof(BitMask)); 233 | IdType r_p[NBITS]; 234 | uint32_t s_p[NBITS]; 235 | uint32_t s_p_max[NBITS]; 236 | double value[NBITS]; 237 | BitMask *curF=FF[0], *nextF=FF[0]; 238 | BitMask active = -1; 239 | for (BitMask j=0,shift=1; jR[p]-1; 242 | value[j] = (nk.first<2)?0:(double)r_p[j]*r_p[j]/(double)(nk.first-1); 243 | s_p[j] = r_p[j]; 244 | s_p_max[j] = current_min==0?INT_MAX:value[j]/current_min; 245 | if (r_p[j]<2) { 246 | active &= ~shift; 247 | s_p[j] = 0; 248 | } 249 | VV[p] |= shift; 250 | nextF[p] |= shift; 251 | } 252 | 253 | // performing bfs for hop=1 254 | for (int hop=1; hop<2 && active; hop++, curF=nextF) { 255 | nextF = FF[hop%2]; 256 | for (uint32_t j=0; j=current_min) { 276 | // personCentrality.insert(std::make_pair(cc, SharedData::instance()->person->denormalized(this->D[p]))); 277 | personCentrality.insert(std::make_pair(cc, this->D[p])); 278 | if (personCentrality.size()>nk.second) { 279 | personCentrality.erase(personCentrality.begin()); 280 | current_min = personCentrality.begin()->first; 281 | } 282 | } 283 | } 284 | } 285 | free(F); 286 | } 287 | 288 | template 289 | void compute_r_p(uint32_t pid, BitSet &personMap, uint32_t n, uint32_t *edgeIndex, IdType *adjacencyList, uint32_t *Q) 290 | { 291 | int front=0, count=0; 292 | personMap[pid] = 1; 293 | Q[count++] = pid; 294 | while (frontR[Q[i]] = count; 306 | } 307 | 308 | template 309 | double closeness_centrality(uint32_t pid, double kmin, uint32_t n, uint32_t *edgeIndex, IdType *adjacencyList, uint32_t *Q) 310 | { 311 | uint32_t r_p = this->R[pid]; 312 | if (r_p==1) return 0.0; 313 | uint32_t s_p = 0; 314 | double value = (n<1)?0:(double)(r_p-1)*(r_p-1)/(double)(n-1); 315 | uint32_t s_p_max = kmin==0?INT_MAX:value/kmin; 316 | int front=0, count=0; 317 | BitSet V(n); 318 | V[pid] = 1; 319 | Q[count++] = pid; 320 | for (int hop=1; fronts_p_max) 322 | return 0.0; 323 | for (int end=count; front 341 | void call(IdPair range, int stride, double current_min, IdPair nk, const IdPairVector &rank, IdType *adjacencyList, uint32_t *Q, TopKSet &personCentrality) 342 | { 343 | for (uint32_t i=range.first; icloseness_centrality(p, current_min, nk.first, this->E, adjacencyList, Q); 346 | if (cc>=current_min) { 347 | //personCentrality.insert(std::make_pair(cc, SharedData::instance()->person->denormalized(this->D[p]))); 348 | personCentrality.insert(std::make_pair(cc, this->D[p])); 349 | if (personCentrality.size()>nk.second) { 350 | personCentrality.erase(personCentrality.begin()); 351 | current_min = personCentrality.begin()->first; 352 | } 353 | } 354 | } 355 | } 356 | }; 357 | -------------------------------------------------------------------------------- /vida/src/naive.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #include "person_graph.hpp" 5 | #include 6 | 7 | struct NaiveBFS 8 | { 9 | typedef std::pair CC; 10 | PersonGraph *pg; 11 | uint32_t personSize; 12 | int k; 13 | uint32_t* queue; 14 | bool* visited; 15 | CC* cc;//closeness centrality 16 | int nThread; 17 | uint32_t nVertex; 18 | int* diameter; 19 | 20 | NaiveBFS(PersonGraph *g, int topk, int numThread, uint32_t numVertex) 21 | { 22 | pg = g; 23 | k = topk; 24 | nThread = numThread; 25 | this->personSize = g->personSize; 26 | if (numVertex != 0 && numVertex < this->personSize) 27 | nVertex = numVertex; 28 | else nVertex = this->personSize; 29 | 30 | queue = (uint32_t*)malloc(sizeof(uint32_t)*this->personSize*this->nThread); 31 | visited = (bool*)malloc(sizeof(bool)*this->personSize*this->nThread); 32 | cc = (CC*)malloc(sizeof(CC)*this->personSize); 33 | diameter = (int*)malloc(sizeof(int)*this->personSize); 34 | memset(diameter, 0, this->personSize); 35 | } 36 | 37 | void run(uint32_t s, bool *t_visited, uint32_t *t_queue) 38 | { 39 | memset(t_visited, 0, this->personSize); 40 | t_visited[s] = 1; 41 | uint64_t sp = 0; 42 | uint64_t rp = 0; 43 | 44 | uint32_t cq = 0; //index to the current queue 45 | t_queue[cq] = s; 46 | uint32_t nq = 1; //index to the next queue 47 | uint32_t nq_base = 1; 48 | int level = 0; 49 | while(cq < nq) 50 | { 51 | level++; 52 | while(cq < nq_base)//Parallize here for each level 53 | {//Enqueue all nodes in this level 54 | uint32_t u = t_queue[cq++]; 55 | for (int i=this->pg->edgeIndex[u]; ipg->edgeIndex[u+1]; i++) 56 | {//For each v adjacent to u 57 | uint32_t v = this->pg->adjacencyList[i]; 58 | if (t_visited[v] == 0) 59 | { 60 | t_queue[nq++] = v; 61 | t_visited[v] = 1; 62 | sp += level; 63 | rp += 1; 64 | } 65 | } 66 | } 67 | nq_base = nq; 68 | } 69 | cc[s] = std::make_pair(s, (rp*rp)/(double)(sp*(this->personSize-1))); 70 | //if (diameter < level) 71 | // diameter = level; 72 | diameter[s] = level; 73 | } 74 | 75 | struct CompareCC 76 | { 77 | bool operator()(const CC &lhs, const CC &rhs) const 78 | { 79 | if (lhs.second == rhs.second) 80 | return (lhs.first < rhs.first); 81 | return (lhs.second > rhs.second); 82 | } 83 | }; 84 | 85 | void call(uint32_t start, uint32_t source, int stride) 86 | { 87 | bool *t_visited = visited + start*this->personSize; 88 | uint32_t *t_queue = queue + start*this->personSize; 89 | for(int s=start; snThread; i++) 97 | workers.create_thread(boost::bind(&NaiveBFS::call, this, i, this->nVertex, this->nThread)); 98 | // for(int n=0; npersonSize; n++) 99 | // run(n); 100 | workers.join_all(); 101 | std::partial_sort(cc, cc+k, cc+this->personSize, CompareCC()); 102 | for(int i=0; ipersonSize; i++) 107 | if (max_diameter < diameter[i]) 108 | max_diameter = diameter[i]; 109 | printf ("Diameter = %d\n", max_diameter); 110 | } 111 | }; 112 | -------------------------------------------------------------------------------- /vida/src/parabfs.hpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #include "person_graph.hpp" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #define USE_ATOMIC 1 11 | 12 | struct ParaBFS 13 | { 14 | typedef std::pair CC; 15 | PersonGraph *pg; 16 | uint32_t personSize; 17 | int nThread; 18 | int k; 19 | uint32_t* queue; 20 | 21 | bool * visited; 22 | #if !USE_ATOMIC 23 | bool * visited2; 24 | #endif 25 | CC* cc;//closeness centrality 26 | uint32_t nVertex; 27 | 28 | boost::mutex mutex; 29 | boost::condition_variable condDone, condWork; 30 | boost::thread_group workers; 31 | int workerBusy; 32 | bool shutdown; 33 | 34 | struct TraverseParams { 35 | TraverseParams(uint32_t *cQ=0, uint32_t *nQ=0, uint32_t cqs=0, uint32_t st=0): cqueue(cQ), nqueue(nQ), cq_start(cqs), cq_end(0), stride(st), nq(0), sp(0), rp(0), level(0) {} 36 | 37 | void set(uint32_t q, uint64_t s, uint64_t r) { 38 | this->nq = q; 39 | this->sp = s; 40 | this->rp = r; 41 | } 42 | 43 | uint32_t *cqueue; 44 | uint32_t *nqueue; 45 | uint32_t cq_start; 46 | uint32_t cq_end; 47 | uint32_t stride; 48 | uint32_t nq; 49 | uint64_t sp; 50 | uint64_t rp; 51 | uint32_t level; 52 | }; 53 | 54 | std::vector params; 55 | 56 | ParaBFS(PersonGraph *g, int topk, int numThread, uint32_t numVertex): shutdown(true) 57 | { 58 | pg = g; 59 | k = topk; 60 | this->nThread = numThread; 61 | //nThread = 4;//Test 62 | this->personSize = g->personSize; 63 | if (numVertex != 0 && numVertex < this->personSize) 64 | nVertex = numVertex; 65 | else nVertex = this->personSize; 66 | 67 | this->queue = (uint32_t*)malloc(sizeof(uint32_t)*this->personSize*(this->nThread + 1));//Allocate n+1 queues 68 | this->visited = (bool*)malloc(sizeof(bool)*this->personSize*2); 69 | #if !USE_ATOMIC 70 | this->visited2 = this->visited + this->personSize; 71 | #endif 72 | this->cc = (CC*)malloc(sizeof(CC)*this->personSize); 73 | } 74 | 75 | ~ParaBFS() 76 | { 77 | free(this->queue); 78 | free(this->visited); 79 | free(this->cc); 80 | } 81 | 82 | void traverse(TraverseParams *tp) 83 | { 84 | uint32_t *cqueue = tp->cqueue; 85 | uint32_t *nqueue = tp->nqueue; 86 | uint32_t cq = tp->cq_start; 87 | uint32_t cq_end = tp->cq_end; 88 | uint32_t st = tp->stride; 89 | uint32_t nq = tp->nq; 90 | uint64_t sp = tp->sp; 91 | uint64_t rp = tp->rp; 92 | uint32_t level = ++tp->level; 93 | for(; cqpg->edgeIndex[u]; ipg->edgeIndex[u+1]; i++) 97 | { 98 | uint32_t v = this->pg->adjacencyList[i]; 99 | { 100 | if (!this->visited[v]) //This command would reduces executing atomic operations 101 | { 102 | #if USE_ATOMIC 103 | bool b = __sync_fetch_and_or(visited+v, true); 104 | // for GCC >= 4.7, use the below instead 105 | // bool b = __atomic_fetch_or(visited+v, true, __ATOMIC_RELAXED); 106 | if (!b) 107 | { 108 | nqueue[nq++] = v; 109 | sp += level; 110 | rp += 1; 111 | } 112 | #else 113 | this->visited[v] = true; 114 | this->visited2[v] = true; 115 | nqueue[nq++] = v; 116 | #endif 117 | } 118 | } 119 | } 120 | } 121 | tp->set(nq, sp, rp); 122 | } 123 | 124 | void waitForWorkers() { 125 | boost::mutex::scoped_lock lock(this->mutex); 126 | if (this->workerBusy>0) 127 | this->condDone.wait(lock); 128 | } 129 | 130 | void worker(TraverseParams *param) { 131 | while (1) { 132 | { 133 | boost::mutex::scoped_lock lock(this->mutex); 134 | if (--this->workerBusy==0) 135 | this->condDone.notify_all(); 136 | this->condWork.wait(lock); 137 | } 138 | if (this->shutdown) break; 139 | this->traverse(param); 140 | } 141 | } 142 | 143 | void stop() { 144 | this->shutdown = true; 145 | this->condWork.notify_all(); 146 | } 147 | 148 | void dispatchToWorkers() 149 | { 150 | this->workerBusy = this->nThread; 151 | this->condWork.notify_all(); 152 | } 153 | 154 | void start(std::vector ¶ms) { 155 | if (!this->shutdown) 156 | return; 157 | this->shutdown = false; 158 | this->workerBusy = this->nThread; 159 | for (int i=0; inThread; i++) { 160 | this->workers.create_thread(boost::bind(&ParaBFS::worker, this, ¶ms[i])); 161 | } 162 | this->waitForWorkers(); 163 | } 164 | 165 | void run(uint32_t s) 166 | { 167 | memset(this->visited, 0, this->personSize*sizeof(bool)); 168 | this->visited[s] = 1; 169 | uint32_t * cqueue = &queue[this->nThread*(this->personSize)]; 170 | cqueue[0] = s; //Initialize current queue 171 | uint32_t cq = 1; //size of the current-queue 172 | int level = 0; 173 | for(int i=0; inThread; i++) { 174 | params[i].sp = 0; 175 | params[i].rp = 0; 176 | params[i].level = 0; 177 | } 178 | while(cq > 0) { 179 | for(int i=0; idispatchToWorkers(); 182 | this->waitForWorkers(); 183 | cq = 0; 184 | for(int i=0; inThread; ++i) { 185 | #if USE_ATOMIC 186 | memcpy(cqueue+cq, params[i].nqueue, sizeof(uint32_t)*params[i].nq); 187 | cq += params[i].nq; 188 | #else 189 | uint32_t *nQ = params[i].nqueue; 190 | for (int j=0; jvisited2[nQ[j]]) { 192 | this->visited2[nQ[j]] = false; 193 | cqueue[cq++] = nQ[j]; 194 | params[i].sp += params[i].level; 195 | ++params[i].rp; 196 | } 197 | } 198 | #endif 199 | params[i].nq = 0; 200 | } 201 | } 202 | uint64_t accum_sp=0, accum_rp=0; 203 | for(int i=0; inThread; i++) { 204 | accum_sp += params[i].sp; 205 | accum_rp += params[i].rp; 206 | } 207 | if (accum_sp) 208 | cc[s] = std::make_pair(s, (accum_rp*accum_rp)/(double)(accum_sp*(this->personSize-1))); 209 | } 210 | 211 | struct CompareCC 212 | { 213 | bool operator()(const CC &lhs, const CC &rhs) const 214 | { 215 | if (lhs.second == rhs.second) 216 | return (lhs.first < rhs.first); 217 | return (lhs.second > rhs.second); 218 | } 219 | }; 220 | 221 | void perform() 222 | { 223 | uint32_t* cqueue = &queue[this->nThread*(this->personSize)]; 224 | for (int i=0; ipersonSize), i, this->nThread)); 226 | this->start(params); 227 | for(int n=0; nnVertex; n++) 228 | run(n); 229 | this->stop(); 230 | 231 | std::partial_sort(cc, cc+k, cc+this->personSize, CompareCC()); 232 | for(int i=0; i 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "hsort.hpp" 21 | 22 | #define NBITS 64 23 | typedef boost::dynamic_bitset BitSet; 24 | 25 | struct PersonGraph 26 | { 27 | typedef std::pair IdPair; 28 | typedef std::vector IdPairVector; 29 | 30 | std::string graphFile; //full path to file that contains edge list 31 | std::string refResult; 32 | const char * queryFile; 33 | size_t personSize; 34 | 35 | std::vector edgeIndex; 36 | std::vector adjacencyList; 37 | std::vector adjacencyWeight; 38 | // BitSet personMap; 39 | 40 | PersonGraph(const char *queryFile) 41 | { 42 | if (queryFile) { 43 | this->readInputFile(queryFile); 44 | this->readPersonKnowsPerson(this->graphFile); 45 | } 46 | } 47 | 48 | void readInputFile(const char *queryFile) 49 | { 50 | std::string f = std::string(queryFile); 51 | boost::iostreams::mapped_file_source file(f); 52 | const char *cur = file.data(); 53 | const char *end = cur + file.size(); 54 | 55 | size_t pSize; 56 | for (pSize=0; *cur>='0' && *cur<='9'; cur++) 57 | pSize = pSize*10 + (*cur-'0'); 58 | this->personSize = pSize; 59 | //personMap.resize(pSize); 60 | //for (int i=0; igraphFile = std::string(start, length); 73 | 74 | while(*cur==' ') { 75 | cur++; 76 | } 77 | 78 | length = 0; 79 | start = cur; 80 | while(cur!=end && *cur != ' ') { 81 | length++; 82 | cur++; 83 | } 84 | this->refResult = std::string(start, length); 85 | } 86 | 87 | void readPersonKnowsPerson(std::string graphfile) 88 | { 89 | IdPairVector adjacencyPair; 90 | { 91 | boost::iostreams::mapped_file_source file(graphFile); 92 | const char *cur = file.data(); 93 | const char *end = cur + file.size(); 94 | while (*cur!='\n') cur++; 95 | IdPair ip; 96 | while (++cur='0' && *cur<='9'; cur++) 99 | p = p*10 + (*cur-'0'); 100 | ip.first = p; 101 | // personMap[p] = 1; 102 | for (p=0,cur++; *cur>='0' && *cur<='9'; cur++) 103 | p = p*10 + (*cur-'0'); 104 | ip.second = p; 105 | // personMap[p] = 1; 106 | while (*cur!='\n') cur++; 107 | adjacencyPair.push_back(ip); 108 | } 109 | } 110 | this->adjacencyList.resize(adjacencyPair.size()); 111 | this->adjacencyWeight.resize(adjacencyPair.size()); 112 | 113 | { 114 | hsort::sort(adjacencyPair.begin(), adjacencyPair.end()); 115 | 116 | this->edgeIndex.clear(); 117 | this->edgeIndex.resize(this->personSize+1, 0); 118 | 119 | unsigned last = 0; 120 | for (unsigned i=0; iedgeIndex[adjacencyPair[i].first+1] = i+1-last; 123 | last = i+1; 124 | } 125 | this->adjacencyList[i] = adjacencyPair[i].second; 126 | this->adjacencyWeight[i] = 0; 127 | } 128 | for (unsigned i=0; ipersonSize; i++) 129 | this->edgeIndex[i+1] += this->edgeIndex[i]; 130 | } 131 | } 132 | }; 133 | 134 | #endif 135 | -------------------------------------------------------------------------------- /vida/src/query.cpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #include 5 | #include "person_graph.hpp" 6 | #include "msbfs.hpp" 7 | #include "parabfs.hpp" 8 | #include "naive.hpp" 9 | #define DEBUG 0 10 | 11 | 12 | int main(int argc, char**argv) 13 | { 14 | const char *queryFile = argv[1]; 15 | int topk = 7; 16 | if (argc < 4) 17 | { 18 | printf("Wrong arguments!\n"); 19 | return 0; 20 | } 21 | 22 | int nThread = atoi(argv[2]); 23 | int nRun = atoi(argv[3]); 24 | uint32_t nVertex = 0; 25 | if (argc == 6) 26 | nVertex = atoi(argv[5]); 27 | 28 | if (strcmp(argv[4], "para") == 0) 29 | { 30 | PersonGraph * pg = new PersonGraph(queryFile); 31 | ParaBFS *parabfs = new ParaBFS(pg, topk, nThread, nVertex); 32 | for(int i=0; iperform(); 36 | } 37 | } 38 | 39 | if (strcmp(argv[4], "msbfs") == 0) 40 | { 41 | PersonGraph * pg = new PersonGraph(queryFile); 42 | //Query4 *query4 = new Query4(pg, topk, nVertex); 43 | Query4 *query4 = new Query4(pg, topk); 44 | for(int i=0; iperform(); 48 | } 49 | } 50 | 51 | if (strcmp(argv[4], "naive") == 0) 52 | { 53 | PersonGraph * pg = new PersonGraph(queryFile); 54 | NaiveBFS *naive = new NaiveBFS(pg, topk, nThread, nVertex); 55 | for(int i=0; iperform(); 59 | } 60 | } 61 | 62 | return 0; 63 | } 64 | -------------------------------------------------------------------------------- /worker.cpp: -------------------------------------------------------------------------------- 1 | //Copyright (C) 2014 by Manuel Then, Moritz Kaufmann, Fernando Chirigati, Tuan-Anh Hoang-Vu, Kien Pham, Alfons Kemper, Huy T. Vo 2 | // 3 | //Code must not be used, distributed, without written consent by the authors 4 | #include "include/worker.hpp" 5 | #include "include/log.hpp" 6 | 7 | Workers::Workers(uint32_t numWorkers) : scheduler() { 8 | LOG_PRINT("[Workers] Allocating worker pool with "<< numWorkers << " workers."); 9 | for (unsigned i = 0; i < numWorkers; ++i) { 10 | Executor* executor = new Executor(scheduler,i+1, false); 11 | threads.emplace_back(&Executor::start, executor); 12 | } 13 | } 14 | 15 | void Workers::assist(Scheduler& tasks) { 16 | for (unsigned i = 0; i < threads.size(); ++i) { 17 | scheduler.schedule(LambdaRunner::createLambdaTask([&tasks, i] { 18 | Executor(tasks,i+1, false).run(); 19 | })); 20 | } 21 | } 22 | void Workers::close() { 23 | scheduler.setCloseOnEmpty(); 24 | for(auto& thread : threads) { 25 | thread.join(); 26 | } 27 | } 28 | --------------------------------------------------------------------------------