├── .gitignore
├── src
    ├── sort.hpp
    ├── partitioner.hpp
    ├── random_partitioner.hpp
    ├── dbh_partitioner.hpp
    ├── util.cpp
    ├── shuffler.hpp
    ├── graph.cpp
    ├── graph2edgelist.cpp
    ├── graph.hpp
    ├── hsfc_partitioner.hpp
    ├── main.cpp
    ├── conversions.hpp
    ├── edgepart.hpp
    ├── random_partitioner.cpp
    ├── min_heap.hpp
    ├── dbh_partitioner.cpp
    ├── util.hpp
    ├── shuffler.cpp
    ├── conversions.cpp
    ├── hsfc_partitioner.cpp
    ├── sort.cpp
    ├── ne_partitioner.hpp
    ├── sne_partitioner.hpp
    ├── ne_partitioner.cpp
    ├── sne_partitioner.cpp
    └── dense_bitset.hpp
├── .clang-format
├── threadpool11
    ├── CMakeLists.txt
    ├── include
    │   └── threadpool11
    │   │   ├── worker.hpp
    │   │   ├── threadpool11.hpp
    │   │   ├── utility.hpp
    │   │   └── pool.hpp
    └── src
    │   ├── worker.cpp
    │   └── pool.cpp
├── LICENSE
├── CMakeLists.txt
├── README.md
└── .ycm_extra_conf.py


/.gitignore:
--------------------------------------------------------------------------------
1 | release/
2 | debug/
3 | *.pyc
4 | .DS_Store
5 | 


--------------------------------------------------------------------------------
/src/sort.hpp:
--------------------------------------------------------------------------------
1 | #pragma once
2 | 
3 | #include <cstdint>
4 | 
5 | void externalSort(int fdInput, uint64_t size, int fdOutput, uint64_t memSize);
6 | 


--------------------------------------------------------------------------------
/src/partitioner.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "util.hpp"
 4 | 
 5 | class Partitioner
 6 | {
 7 |   protected:
 8 |     Timer total_time;
 9 | 
10 |   public:
11 |     virtual void split() = 0;
12 | };
13 | 


--------------------------------------------------------------------------------
/.clang-format:
--------------------------------------------------------------------------------
1 | BasedOnStyle: LLVM
2 | IndentWidth: 4
3 | UseTab: Never
4 | BreakBeforeBraces: Linux
5 | AllowShortIfStatementsOnASingleLine: false
6 | IndentCaseLabels: false
7 | ForEachMacros: ['RANGES_FOR', 'FOREACH', 'rep', 'repv']
8 | 


--------------------------------------------------------------------------------
/threadpool11/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | add_definitions(-Dthreadpool11_EXPORTING)
 2 | 
 3 | include_directories(include)
 4 | 
 5 | add_library(threadpool11 STATIC src/pool.cpp src/worker.cpp include/threadpool11/pool.hpp include/threadpool11/worker.hpp include/threadpool11/utility.hpp)
 6 | 
 7 | if (CMAKE_COMPILER_IS_GNUCXX)
 8 |     target_link_libraries(threadpool11 pthread)
 9 | endif()
10 | 
11 | if (UNIX)
12 |     install(FILES include/threadpool11/threadpool11.hpp DESTINATION include/threadpool11)
13 |     install(FILES include/threadpool11/worker.hpp DESTINATION include/threadpool11)
14 |     install(FILES include/threadpool11/pool.hpp DESTINATION include/threadpool11)
15 |     install(FILES include/threadpool11/utility.hpp DESTINATION include/threadpool11)
16 |     install(TARGETS threadpool11 DESTINATION lib)
17 | endif()
18 | 


--------------------------------------------------------------------------------
/src/random_partitioner.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <string>
 4 | #include <vector>
 5 | #include <iostream>
 6 | #include <fstream>
 7 | #include <random>
 8 | #include <fcntl.h>
 9 | #include <sys/stat.h>
10 | #include <sys/mman.h>
11 | #include <unistd.h>
12 | #include <parallel/algorithm>
13 | 
14 | #include "util.hpp"
15 | #include "dense_bitset.hpp"
16 | #include "edgepart.hpp"
17 | #include "partitioner.hpp"
18 | 
19 | class RandomPartitioner : public Partitioner
20 | {
21 |   private:
22 |     const size_t BUFFER_SIZE = 64 * 1024 / sizeof(edge_t);
23 |     std::string basefilename;
24 | 
25 |     vid_t num_vertices;
26 |     size_t num_edges;
27 |     int p;
28 | 
29 |     // use mmap for file input
30 |     int fin;
31 |     off_t filesize;
32 |     char *fin_map, *fin_ptr, *fin_end;
33 | 
34 |   public:
35 |     RandomPartitioner(std::string basefilename);
36 |     void split();
37 | };
38 | 


--------------------------------------------------------------------------------
/src/dbh_partitioner.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <string>
 4 | #include <vector>
 5 | #include <iostream>
 6 | #include <fstream>
 7 | #include <random>
 8 | #include <fcntl.h>
 9 | #include <sys/stat.h>
10 | #include <sys/mman.h>
11 | #include <unistd.h>
12 | #include <parallel/algorithm>
13 | 
14 | #include "util.hpp"
15 | #include "dense_bitset.hpp"
16 | #include "edgepart.hpp"
17 | #include "partitioner.hpp"
18 | 
19 | class DbhPartitioner : public Partitioner
20 | {
21 |   private:
22 |     const size_t BUFFER_SIZE = 64 * 1024 / sizeof(edge_t);
23 |     std::string basefilename;
24 | 
25 |     vid_t num_vertices;
26 |     size_t num_edges;
27 |     int p;
28 | 
29 |     // use mmap for file input
30 |     int fin;
31 |     off_t filesize;
32 |     char *fin_map, *fin_ptr, *fin_end;
33 | 
34 |     std::vector<vid_t> degrees;
35 | 
36 |     std::random_device rd;
37 |     std::mt19937 gen;
38 |     std::uniform_int_distribution<int> dis;
39 | 
40 |   public:
41 |     DbhPartitioner(std::string basefilename);
42 |     void split();
43 | };
44 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2016-2018 Qin Liu
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/src/util.cpp:
--------------------------------------------------------------------------------
 1 | #include <iterator>
 2 | #include <algorithm>
 3 | #include <functional>
 4 | 
 5 | #include "util.hpp"
 6 | 
 7 | threadpool11::Pool pool;
 8 | 
 9 | void preada(int f, char *buf, size_t nbytes, size_t off)
10 | {
11 |     size_t nread = 0;
12 |     while (nread < nbytes) {
13 |         ssize_t a = pread(f, buf, nbytes - nread, off + nread);
14 |         PCHECK(a != ssize_t(-1)) << "Could not read " << (nbytes - nread)
15 |                                  << " bytes!";
16 |         buf += a;
17 |         nread += a;
18 |     }
19 | }
20 | 
21 | void reada(int f, char *buf, size_t nbytes)
22 | {
23 |     size_t nread = 0;
24 |     while (nread < nbytes) {
25 |         ssize_t a = read(f, buf, nbytes - nread);
26 |         PCHECK(a != ssize_t(-1)) << "Could not read " << (nbytes - nread)
27 |                                  << " bytes!";
28 |         buf += a;
29 |         nread += a;
30 |     }
31 | }
32 | 
33 | void writea(int f, char *buf, size_t nbytes)
34 | {
35 |     size_t nwritten = 0;
36 |     while (nwritten < nbytes) {
37 |         ssize_t a = write(f, buf, nbytes - nwritten);
38 |         PCHECK(a != ssize_t(-1)) << "Could not write " << (nbytes - nwritten)
39 |                                  << " bytes!";
40 |         buf += a;
41 |         nwritten += a;
42 |     }
43 | }
44 | 


--------------------------------------------------------------------------------
/src/shuffler.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <string>
 4 | 
 5 | #include <sys/types.h>
 6 | #include <sys/stat.h>
 7 | #include <fcntl.h>
 8 | 
 9 | #include "util.hpp"
10 | #include "conversions.hpp"
11 | 
12 | class Shuffler : public Converter
13 | {
14 |   private:
15 |     struct work_t {
16 |         Shuffler *shuffler;
17 |         int nchunks;
18 |         std::vector<edge_t> chunk_buf;
19 | 
20 |         void operator()() {
21 |             std::random_shuffle(chunk_buf.begin(), chunk_buf.end());
22 |             int file =
23 |                 open(shuffler->chunk_filename(nchunks).c_str(),
24 |                      O_WRONLY | O_CREAT, S_IROTH | S_IWOTH | S_IWUSR | S_IRUSR);
25 |             size_t chunk_size = chunk_buf.size() * sizeof(edge_t);
26 |             writea(file, (char *)&chunk_buf[0], chunk_size);
27 |             close(file);
28 |             chunk_buf.clear();
29 |         }
30 |     };
31 | 
32 |     size_t chunk_bufsize;
33 |     int nchunks, old_nchunks;
34 |     std::vector<edge_t> chunk_buf, old_chunk_buf;
35 | 
36 |     std::string chunk_filename(int chunk);
37 |     void chunk_clean();
38 |     void cwrite(edge_t e, bool flush = false);
39 | 
40 |   public:
41 |     Shuffler(std::string basefilename) : Converter(basefilename) {}
42 |     bool done() { return is_exists(shuffled_binedgelist_name(basefilename)); }
43 |     void init();
44 |     void finalize();
45 |     void add_edge(vid_t source, vid_t target);
46 | };
47 | 


--------------------------------------------------------------------------------
/CMakeLists.txt:
--------------------------------------------------------------------------------
 1 | cmake_minimum_required (VERSION 2.8)
 2 | project (edgepart)
 3 | 
 4 | set(CMAKE_VERBOSE_MAKEFILE off)
 5 | 
 6 | # find_package (glog REQUIRED)
 7 | # find_package (gflags REQUIRED)
 8 | # find_package (Boost REQUIRED)
 9 | 
10 | # SET a default build type if none was specified
11 | if(NOT CMAKE_BUILD_TYPE)
12 |   message(STATUS "Setting build type to 'Release' as none was specified.")
13 |   set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
14 |   # SET the possible values of build type for cmake-gui
15 | endif()
16 | 
17 | set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Release" "Debug")
18 | 
19 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g -std=c++11 -pthread -fopenmp")
20 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_EXE_LINK_FLAGS_RELEASE} -Ofast -DNDEBUG")
21 | 
22 | add_subdirectory(threadpool11)
23 | include_directories(./threadpool11/include)
24 | 
25 | add_executable (main
26 |     src/main.cpp
27 |     src/util.cpp
28 |     src/sort.cpp
29 |     src/graph.cpp
30 |     src/ne_partitioner.cpp
31 |     src/sne_partitioner.cpp
32 |     src/random_partitioner.cpp
33 |     src/hsfc_partitioner.cpp
34 |     src/dbh_partitioner.cpp
35 |     src/conversions.cpp
36 |     src/shuffler.cpp)
37 | add_executable (graph2edgelist
38 |     src/graph2edgelist.cpp
39 |     src/util.cpp
40 |     src/conversions.cpp)
41 | 
42 | target_link_libraries (main glog gflags threadpool11)
43 | target_link_libraries (graph2edgelist glog gflags threadpool11)
44 | 


--------------------------------------------------------------------------------
/src/graph.cpp:
--------------------------------------------------------------------------------
 1 | #include "graph.hpp"
 2 | 
 3 | void graph_t::build(const std::vector<edge_t> &edges)
 4 | {
 5 |     if (edges.size() > nedges)
 6 |         neighbors = (uint40_t *)realloc(neighbors, sizeof(uint40_t) * edges.size());
 7 |     CHECK(neighbors) << "allocation failed";
 8 |     nedges = edges.size();
 9 | 
10 |     std::vector<size_t> count(num_vertices, 0);
11 |     for (size_t i = 0; i < nedges; i++)
12 |         count[edges[i].first]++;
13 | 
14 |     vdata[0] = adjlist_t(neighbors);
15 |     for (vid_t v = 1; v < num_vertices; v++) {
16 |         count[v] += count[v-1];
17 |         vdata[v] = adjlist_t(neighbors + count[v-1]);
18 |     }
19 |     for (size_t i = 0; i < edges.size(); i++)
20 |         vdata[edges[i].first].push_back(i);
21 | }
22 | 
23 | void graph_t::build_reverse(const std::vector<edge_t> &edges)
24 | {
25 |     if (edges.size() > nedges)
26 |         neighbors = (uint40_t *)realloc(neighbors, sizeof(uint40_t) * edges.size());
27 |     CHECK(neighbors) << "allocation failed";
28 |     nedges = edges.size();
29 | 
30 |     std::vector<size_t> count(num_vertices, 0);
31 |     for (size_t i = 0; i < nedges; i++)
32 |         count[edges[i].second]++;
33 | 
34 |     vdata[0] = adjlist_t(neighbors);
35 |     for (vid_t v = 1; v < num_vertices; v++) {
36 |         count[v] += count[v-1];
37 |         vdata[v] = adjlist_t(neighbors + count[v-1]);
38 |     }
39 |     for (size_t i = 0; i < edges.size(); i++)
40 |         vdata[edges[i].second].push_back(i);
41 | }
42 | 


--------------------------------------------------------------------------------
/src/graph2edgelist.cpp:
--------------------------------------------------------------------------------
 1 | #include <string>
 2 | #include <fstream>
 3 | 
 4 | #include "util.hpp"
 5 | #include "conversions.hpp"
 6 | 
 7 | DECLARE_bool(help);
 8 | DECLARE_bool(helpshort);
 9 | 
10 | DEFINE_uint64(memsize, 4096, "memory size in megabytes");
11 | DEFINE_string(filename, "", "the file name of the input graph");
12 | DEFINE_string(filetype, "edgelist",
13 |               "the type of input file (supports 'edgelist' and 'adjlist')");
14 | 
15 | class Graph2Edgelist : public Converter
16 | {
17 |   private:
18 |     std::ofstream fout;
19 | 
20 |   public:
21 |     Graph2Edgelist(std::string basefilename)
22 |         : Converter(basefilename)
23 |     {
24 |     }
25 |     bool done() { return is_exists(basefilename + ".edgelist"); }
26 |     void init() { fout.open(basefilename + ".edgelist"); }
27 |     void add_edge(vid_t from, vid_t to) { fout << from << ' ' << to << '\n'; }
28 |     void finalize() { fout.close(); }
29 | };
30 | 
31 | int main(int argc, char *argv[])
32 | {
33 |     std::string usage = "-filename <path to the input graph> "
34 |                         "[-filetype <edgelist|adjlist>]";
35 |     google::SetUsageMessage(usage);
36 |     google::ParseCommandLineNonHelpFlags(&argc, &argv, true);
37 |     google::InitGoogleLogging(argv[0]);
38 |     FLAGS_logtostderr = 1; // output log to stderr
39 |     if (FLAGS_help) {
40 |         FLAGS_help = false;
41 |         FLAGS_helpshort = true;
42 |     }
43 |     google::HandleCommandLineHelpFlags();
44 | 
45 |     Timer timer;
46 |     timer.start();
47 | 
48 |     convert(FLAGS_filename, new Graph2Edgelist(FLAGS_filename));
49 |     timer.stop();
50 |     LOG(INFO) << "total time: " << timer.get_time();
51 | }
52 | 


--------------------------------------------------------------------------------
/threadpool11/include/threadpool11/worker.hpp:
--------------------------------------------------------------------------------
 1 | /*!
 2 | Copyright (c) 2013, 2014, 2015 Tolga HOŞGÖR
 3 | All rights reserved.
 4 | 
 5 | This file is part of threadpool11.
 6 | 
 7 |     threadpool11 is free software: you can redistribute it and/or modify
 8 |     it under the terms of the GNU General Public License as published by
 9 |     the Free Software Foundation, either version 3 of the License, or
10 |     (at your option) any later version.
11 | 
12 |     threadpool11 is distributed in the hope that it will be useful,
13 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
14 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 |     GNU General Public License for more details.
16 | 
17 |     You should have received a copy of the GNU General Public License
18 |     along with threadpool11.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #pragma once
22 | 
23 | #include "utility.hpp"
24 | 
25 | #include <condition_variable>
26 | #include <functional>
27 | #include <list>
28 | #include <mutex>
29 | #include <thread>
30 | 
31 | namespace threadpool11 {
32 | 
33 | class Pool;
34 | 
35 | class Worker {
36 |   friend class Pool;
37 | 
38 | public:
39 |   Worker(Pool& pool);
40 |   ~Worker() = default;
41 | 
42 | private:
43 |   Worker(Worker&&) = delete;
44 |   Worker(Worker const&) = delete;
45 |   Worker& operator=(Worker&&) = delete;
46 |   Worker& operator=(Worker const&) = delete;
47 | 
48 |   void execute(Pool& pool);
49 | 
50 | private:
51 |   /*!
52 |   * This should always stay at bottom so that it is called at the most end.
53 |   */
54 |   std::thread thread_;
55 | };
56 | }
57 | 


--------------------------------------------------------------------------------
/threadpool11/include/threadpool11/threadpool11.hpp:
--------------------------------------------------------------------------------
 1 | /*
 2 | Copyright (c) 2013, Tolga HOŞGÖR
 3 | All rights reserved.
 4 | 
 5 | Redistribution and use in source and binary forms, with or without
 6 | modification, are permitted provided that the following conditions are met:
 7 | 
 8 | 1. Redistributions of source code must retain the above copyright notice, this
 9 |    list of conditions and the following disclaimer.
10 | 2. Redistributions in binary form must reproduce the above copyright notice,
11 |    this list of conditions and the following disclaimer in the documentation
12 |    and/or other materials provided with the distribution.
13 | 
14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
18 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
19 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
20 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
21 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
23 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 | 
25 | The views and conclusions contained in the software and documentation are those
26 | of the authors and should not be interpreted as representing official policies,
27 | either expressed or implied, of the FreeBSD Project.
28 | */
29 | 
30 | #pragma once
31 | 
32 | #include "pool.hpp"
33 | #include "worker.hpp"
34 | 


--------------------------------------------------------------------------------
/src/graph.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <vector>
 4 | #include <parallel/algorithm>
 5 | 
 6 | #include "util.hpp"
 7 | 
 8 | struct uint40_t {
 9 |         uint64_t v:40;
10 | } __attribute__((packed));
11 | 
12 | class adjlist_t
13 | {
14 |   private:
15 |     uint40_t *adj;
16 |     vid_t len;
17 | 
18 |   public:
19 |     adjlist_t() : adj(NULL), len(0) {}
20 |     adjlist_t(uint40_t *adj, vid_t len = 0) : adj(adj), len(len) {}
21 |     uint40_t *begin() { return adj; }
22 |     uint40_t *end() { return adj + len; }
23 |     void increment() { len++; }
24 |     void push_back(size_t data) { adj[len++].v = data; }
25 |     size_t size() const { return len; }
26 |     uint40_t &operator[](size_t idx) { return adj[idx]; };
27 |     const uint40_t &operator[](size_t idx) const { return adj[idx]; };
28 |     uint40_t &back() { return adj[len - 1]; };
29 |     const uint40_t &back() const { return adj[len - 1]; };
30 |     void pop_back() { len--; }
31 |     void clear() { len = 0; }
32 | };
33 | 
34 | class graph_t
35 | {
36 |   private:
37 |     vid_t num_vertices;
38 |     size_t nedges;
39 |     uint40_t *neighbors;
40 |     std::vector<adjlist_t> vdata;
41 | 
42 |   public:
43 |     graph_t() : num_vertices(0), nedges(0), neighbors(NULL) {}
44 | 
45 |     ~graph_t()
46 |     {
47 |         if (neighbors)
48 |             free(neighbors);
49 |     }
50 | 
51 |     void resize(vid_t _num_vertices)
52 |     {
53 |         num_vertices = _num_vertices;
54 |         vdata.resize(num_vertices);
55 |     }
56 | 
57 |     size_t num_edges() const { return nedges; }
58 | 
59 |     void build(const std::vector<edge_t> &edges);
60 | 
61 |     void build_reverse(const std::vector<edge_t> &edges);
62 | 
63 |     adjlist_t &operator[](size_t idx) { return vdata[idx]; };
64 |     const adjlist_t &operator[](size_t idx) const { return vdata[idx]; };
65 | };
66 | 


--------------------------------------------------------------------------------
/threadpool11/include/threadpool11/utility.hpp:
--------------------------------------------------------------------------------
 1 | /*!
 2 | Copyright (c) 2013, 2014, 2015 Tolga HOŞGÖR
 3 | All rights reserved.
 4 | 
 5 | This file is part of threadpool11.
 6 | 
 7 |     threadpool11 is free software: you can redistribute it and/or modify
 8 |     it under the terms of the GNU General Public License as published by
 9 |     the Free Software Foundation, either version 3 of the License, or
10 |     (at your option) any later version.
11 | 
12 |     threadpool11 is distributed in the hope that it will be useful,
13 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
14 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 |     GNU General Public License for more details.
16 | 
17 |     You should have received a copy of the GNU General Public License
18 |     along with threadpool11.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #pragma once
22 | 
23 | #include <functional>
24 | 
25 | namespace threadpool11 {
26 | 
27 | namespace Work {
28 | enum class Type { STD, TERMINAL };
29 | enum class Prio { DEFERRED, IMMIDIATE };
30 | 
31 | typedef std::function<Work::Type()> Callable;
32 | }
33 | 
34 | template <typename T>
35 | class move_on_copy {
36 | public:
37 |   move_on_copy(T&& aValue)
38 |       : value_(std::move(aValue)) {}
39 |   move_on_copy(const move_on_copy& other)
40 |       : value_(std::move(other.value_)) {}
41 | 
42 |   move_on_copy& operator=(move_on_copy&& aValue) = delete;      // not needed here
43 |   move_on_copy& operator=(const move_on_copy& aValue) = delete; // not needed here
44 | 
45 |   T& value() { return value_; }
46 |   const T& value() const { return value_; }
47 | 
48 | private:
49 |   mutable T value_;
50 | };
51 | 
52 | template <typename T>
53 | inline move_on_copy<T> make_move_on_copy(T&& aValue) {
54 |   return move_on_copy<T>(std::move(aValue));
55 | }
56 | }
57 | 


--------------------------------------------------------------------------------
/src/hsfc_partitioner.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <string>
 4 | #include <vector>
 5 | #include <iostream>
 6 | #include <fstream>
 7 | #include <random>
 8 | #include <fcntl.h>
 9 | #include <sys/stat.h>
10 | #include <sys/mman.h>
11 | #include <unistd.h>
12 | #include <parallel/algorithm>
13 | 
14 | #include "util.hpp"
15 | #include "dense_bitset.hpp"
16 | #include "edgepart.hpp"
17 | #include "partitioner.hpp"
18 | 
19 | class HsfcPartitioner : public Partitioner
20 | {
21 |   private:
22 |     const size_t BUFFER_SIZE = 64 * 1024 / sizeof(edge_t);
23 |     std::string basefilename;
24 | 
25 |     vid_t num_vertices;
26 |     size_t num_edges;
27 |     int p;
28 | 
29 |     // use mmap for file input
30 |     int fin;
31 |     off_t filesize;
32 |     char *fin_map, *fin_ptr, *fin_end;
33 | 
34 |     uint64_t n;
35 | 
36 |     // convert (x,y) to d
37 |     uint64_t xy2d(uint32_t x, uint32_t y)
38 |     {
39 |         uint64_t rx, ry, s, d = 0;
40 |         for (s = n / 2; s > 0; s /= 2) {
41 |             rx = (x & s) > 0;
42 |             ry = (y & s) > 0;
43 |             d += s * s * ((3 * rx) ^ ry);
44 |             rot(s, &x, &y, rx, ry);
45 |         }
46 |         return d;
47 |     }
48 | 
49 |     // convert d to (x,y)
50 |     void d2xy(uint64_t d, uint32_t *x, uint32_t *y)
51 |     {
52 |         uint64_t rx, ry, s, t = d;
53 |         *x = *y = 0;
54 |         for (s = 1; s < n; s *= 2) {
55 |             rx = 1 & (t / 2);
56 |             ry = 1 & (t ^ rx);
57 |             rot(s, x, y, rx, ry);
58 |             *x += s * rx;
59 |             *y += s * ry;
60 |             t /= 4;
61 |         }
62 |     }
63 | 
64 |     // rotate/flip a quadrant appropriately
65 |     void rot(uint64_t n, uint32_t *x, uint32_t *y, uint64_t rx, uint64_t ry)
66 |     {
67 |         if (ry == 0) {
68 |             if (rx == 1) {
69 |                 *x = n - 1 - *x;
70 |                 *y = n - 1 - *y;
71 |             }
72 | 
73 |             // Swap x and y
74 |             uint32_t t = *x;
75 |             *x = *y;
76 |             *y = t;
77 |         }
78 |     }
79 | 
80 |     void generate_hilber();
81 | 
82 |   public:
83 |     HsfcPartitioner(std::string basefilename);
84 |     void split();
85 | };
86 | 


--------------------------------------------------------------------------------
/src/main.cpp:
--------------------------------------------------------------------------------
 1 | #include <string>
 2 | 
 3 | #include "util.hpp"
 4 | #include "ne_partitioner.hpp"
 5 | #include "sne_partitioner.hpp"
 6 | #include "random_partitioner.hpp"
 7 | #include "hsfc_partitioner.hpp"
 8 | #include "dbh_partitioner.hpp"
 9 | 
10 | DECLARE_bool(help);
11 | DECLARE_bool(helpshort);
12 | 
13 | DEFINE_int32(p, 10, "number of parititions");
14 | DEFINE_uint64(memsize, 4096, "memory size in megabytes");
15 | DEFINE_string(filename, "", "the file name of the input graph");
16 | DEFINE_string(filetype, "edgelist",
17 |               "the type of input file (supports 'edgelist' and 'adjlist')");
18 | DEFINE_bool(inmem, false, "in-memory mode");
19 | DEFINE_double(sample_ratio, 2, "the sample size divided by num_vertices");
20 | DEFINE_string(method, "sne",
21 |               "partition method: ne, sne, random, and dbh");
22 | 
23 | int main(int argc, char *argv[])
24 | {
25 |     std::string usage = "-filename <path to the input graph> "
26 |                         "[-filetype <edgelist|adjlist>] "
27 |                         "[-p <number of partitions>] "
28 |                         "[-memsize <memory budget in MB>]";
29 |     google::SetUsageMessage(usage);
30 |     google::ParseCommandLineNonHelpFlags(&argc, &argv, true);
31 |     google::InitGoogleLogging(argv[0]);
32 |     FLAGS_logtostderr = 1; // output log to stderr
33 |     if (FLAGS_help) {
34 |         FLAGS_help = false;
35 |         FLAGS_helpshort = true;
36 |     }
37 |     google::HandleCommandLineHelpFlags();
38 | 
39 |     Timer timer;
40 |     timer.start();
41 | 
42 |     Partitioner *partitioner = NULL;
43 |     if (FLAGS_method == "ne")
44 |         partitioner = new NePartitioner(FLAGS_filename);
45 |     else if (FLAGS_method == "sne")
46 |         partitioner = new SnePartitioner(FLAGS_filename);
47 |     else if (FLAGS_method == "random")
48 |         partitioner = new RandomPartitioner(FLAGS_filename);
49 |     else if (FLAGS_method == "dbh")
50 |         partitioner = new DbhPartitioner(FLAGS_filename);
51 |     else if (FLAGS_method == "hsfc")
52 |         partitioner = new HsfcPartitioner(FLAGS_filename);
53 |     else
54 |         LOG(ERROR) << "unkown method: " << FLAGS_method;
55 |     LOG(INFO) << "partition method: " << FLAGS_method;
56 |     partitioner->split();
57 | 
58 |     timer.stop();
59 |     LOG(INFO) << "total time: " << timer.get_time();
60 | }
61 | 


--------------------------------------------------------------------------------
/src/conversions.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <string>
 4 | #include <fstream>
 5 | 
 6 | #include <boost/unordered_map.hpp>
 7 | 
 8 | #include "util.hpp"
 9 | 
10 | DECLARE_string(filetype);
11 | 
12 | class Converter
13 | {
14 |   protected:
15 |     std::string basefilename;
16 |     vid_t num_vertices;
17 |     size_t num_edges;
18 |     std::vector<vid_t> degrees;
19 |     std::ofstream fout;
20 |     boost::unordered_map<vid_t, vid_t> name2vid;
21 | 
22 |     vid_t get_vid(vid_t v)
23 |     {
24 |         auto it = name2vid.find(v);
25 |         if (it == name2vid.end()) {
26 |             name2vid[v] = num_vertices;
27 |             degrees.resize(num_vertices + 1);
28 |             return num_vertices++;
29 |         }
30 |         return name2vid[v];
31 |     }
32 | 
33 |   public:
34 |     Converter(std::string basefilename) : basefilename(basefilename) {}
35 |     virtual ~Converter() {}
36 |     virtual bool done() { return is_exists(binedgelist_name(basefilename)); }
37 | 
38 |     virtual void init()
39 |     {
40 |         num_vertices = 0;
41 |         num_edges = 0;
42 |         degrees.reserve(1<<20);
43 |         fout.open(binedgelist_name(basefilename), std::ios::binary);
44 |         fout.write((char *)&num_vertices, sizeof(num_vertices));
45 |         fout.write((char *)&num_edges, sizeof(num_edges));
46 |     }
47 | 
48 |     virtual void add_edge(vid_t from, vid_t to)
49 |     {
50 |         if (to == from) {
51 |             LOG(WARNING) << "Tried to add self-edge " << from << "->" << to
52 |                          << std::endl;
53 |             return;
54 |         }
55 | 
56 |         num_edges++;
57 |         from = get_vid(from);
58 |         to = get_vid(to);
59 |         degrees[from]++;
60 |         degrees[to]++;
61 | 
62 |         fout.write((char *)&from, sizeof(vid_t));
63 |         fout.write((char *)&to, sizeof(vid_t));
64 |     }
65 | 
66 |     virtual void finalize() {
67 |         fout.seekp(0);
68 |         fout.write((char *)&num_vertices, sizeof(num_vertices));
69 |         fout.write((char *)&num_edges, sizeof(num_edges));
70 |         fout.close();
71 | 
72 |         fout.open(degree_name(basefilename), std::ios::binary);
73 |         fout.write((char *)&degrees[0], num_vertices * sizeof(vid_t));
74 |         fout.close();
75 |     }
76 | };
77 | 
78 | void convert(std::string basefilename, Converter *converter);
79 | 


--------------------------------------------------------------------------------
/src/edgepart.hpp:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <string>
 4 | #include <sstream>
 5 | #include <iostream>
 6 | #include <fstream>
 7 | 
 8 | #include "util.hpp"
 9 | 
10 | template <typename vertex_type, typename proc_type>
11 | struct edgepart_writer {
12 |     char *buffer;
13 |     std::ofstream fout;
14 | 
15 |     edgepart_writer(const std::string &basefilename)
16 |         : fout(partitioned_name(basefilename))
17 |     {
18 |         size_t s = std::max(sizeof(vertex_type) + sizeof(proc_type),
19 |                             sizeof(vertex_type) + sizeof(vertex_type) +
20 |                                 sizeof(proc_type));
21 |         buffer = new char[sizeof(char) + s];
22 |     }
23 | 
24 |     ~edgepart_writer() { delete[] buffer; }
25 | 
26 |     /**
27 |      * Replaces \255 with \255\1
28 |      * Replaces \\n with \255\0
29 |      */
30 |     static std::string escape_newline(char *ptr, size_t strmlen)
31 |     {
32 |         size_t ctr = 0;
33 |         for (size_t i = 0; i < strmlen; ++i) {
34 |             ctr += (ptr[i] == (char)255 || ptr[i] == '\n');
35 |         }
36 | 
37 |         std::string ret(ctr + strmlen, 0);
38 | 
39 |         size_t target = 0;
40 |         for (size_t i = 0; i < strmlen; ++i, ++ptr) {
41 |             if ((*ptr) == (char)255) {
42 |                 ret[target] = (char)255;
43 |                 ret[target + 1] = 1;
44 |                 target += 2;
45 |             } else if ((*ptr) == '\n') {
46 |                 ret[target] = (char)255;
47 |                 ret[target + 1] = 0;
48 |                 target += 2;
49 |             } else {
50 |                 ret[target] = (*ptr);
51 |                 ++target;
52 |             }
53 |         }
54 |         CHECK_EQ(target, ctr + strmlen);
55 |         return ret;
56 |     }
57 | 
58 |     void save_vertex(vertex_type v, proc_type proc)
59 |     {
60 |         buffer[0] = 0;
61 |         memcpy(buffer + 1, &v, sizeof(vertex_type));
62 |         memcpy(buffer + 1 + sizeof(vertex_type), &proc,
63 |                sizeof(proc_type));
64 |         std::string result = escape_newline(
65 |             buffer, 1 + sizeof(vertex_type) + sizeof(proc_type));
66 |         fout << result << '\n';
67 |     }
68 | 
69 |     void save_edge(vertex_type from, vertex_type to, proc_type proc)
70 |     {
71 |         buffer[0] = 1;
72 |         memcpy(buffer + 1, &from, sizeof(vertex_type));
73 |         memcpy(buffer + 1 + sizeof(vertex_type), &to, sizeof(vertex_type));
74 |         memcpy(buffer + 1 + sizeof(vertex_type) + sizeof(vertex_type), &proc,
75 |                sizeof(proc_type));
76 |         std::string result = escape_newline(buffer, 1 + sizeof(vertex_type) +
77 |                                                         sizeof(vertex_type) +
78 |                                                         sizeof(proc_type));
79 |         fout << result << '\n';
80 |     }
81 | };
82 | 


--------------------------------------------------------------------------------
/src/random_partitioner.cpp:
--------------------------------------------------------------------------------
 1 | #include <utility>
 2 | #include <functional>
 3 | 
 4 | #include "util.hpp"
 5 | #include "random_partitioner.hpp"
 6 | #include "conversions.hpp"
 7 | 
 8 | RandomPartitioner::RandomPartitioner(std::string basefilename)
 9 | {
10 |     Timer convert_timer;
11 |     convert_timer.start();
12 |     convert(basefilename, new Converter(basefilename));
13 |     convert_timer.stop();
14 |     LOG(INFO) << "convert time: " << convert_timer.get_time();
15 | 
16 |     total_time.start();
17 |     LOG(INFO) << "initializing partitioner";
18 | 
19 |     fin = open(binedgelist_name(basefilename).c_str(), O_RDONLY, (mode_t)0600);
20 |     PCHECK(fin != -1) << "Error opening file for read";
21 |     struct stat fileInfo = {0};
22 |     PCHECK(fstat(fin, &fileInfo) != -1) << "Error getting the file size";
23 |     PCHECK(fileInfo.st_size != 0) << "Error: file is empty";
24 |     LOG(INFO) << "file size: " << fileInfo.st_size;
25 | 
26 |     fin_map = (char *)mmap(0, fileInfo.st_size, PROT_READ, MAP_SHARED, fin, 0);
27 |     if (fin_map == MAP_FAILED) {
28 |         close(fin);
29 |         PLOG(FATAL) << "error mapping the file";
30 |     }
31 | 
32 |     filesize = fileInfo.st_size;
33 |     fin_ptr = fin_map;
34 |     fin_end = fin_map + filesize;
35 | 
36 |     num_vertices = *(vid_t *)fin_ptr;
37 |     fin_ptr += sizeof(vid_t);
38 |     num_edges = *(size_t *)fin_ptr;
39 |     fin_ptr += sizeof(size_t);
40 | 
41 |     LOG(INFO) << "num_vertices: " << num_vertices
42 |               << ", num_edges: " << num_edges;
43 | 
44 |     p = FLAGS_p;
45 | }
46 | 
47 | void RandomPartitioner::split()
48 | {
49 |     std::vector<dense_bitset> is_mirrors(p, dense_bitset(num_vertices));
50 |     std::vector<size_t> counter(p, 0);
51 |     auto hash = std::hash<vid_t>();
52 |     while (fin_ptr < fin_end) {
53 |         edge_t *e = (edge_t *)fin_ptr;
54 |         fin_ptr += sizeof(edge_t);
55 |         vid_t u = e->first, v = e->second;
56 |         if (u > v) std::swap(u, v);
57 |         int bucket = (hash(u) ^ (hash(v) << 1)) % p;
58 |         counter[bucket]++;
59 |         is_mirrors[bucket].set_bit_unsync(u);
60 |         is_mirrors[bucket].set_bit_unsync(v);
61 |     }
62 | 
63 |     if (munmap(fin_map, filesize) == -1) {
64 |         close(fin);
65 |         PLOG(FATAL) << "Error un-mmapping the file";
66 |     }
67 |     close(fin);
68 | 
69 | 
70 |     size_t max_occupied = *std::max_element(counter.begin(), counter.end());
71 |     LOG(INFO) << "balance: " << (double)max_occupied / ((double)num_edges / p);
72 |     size_t total_mirrors = 0;
73 |     rep (i, p)
74 |         total_mirrors += is_mirrors[i].popcount();
75 |     LOG(INFO) << "total mirrors: " << total_mirrors;
76 |     LOG(INFO) << "replication factor: " << (double)total_mirrors / num_vertices;
77 | 
78 |     total_time.stop();
79 |     LOG(INFO) << "total partition time: " << total_time.get_time();
80 | }
81 | 


--------------------------------------------------------------------------------
/threadpool11/src/worker.cpp:
--------------------------------------------------------------------------------
 1 | /*!
 2 | Copyright (c) 2013, 2014, 2015 Tolga HOŞGÖR
 3 | All rights reserved.
 4 | 
 5 | This file is part of threadpool11.
 6 | 
 7 |     threadpool11 is free software: you can redistribute it and/or modify
 8 |     it under the terms of the GNU General Public License as published by
 9 |     the Free Software Foundation, either version 3 of the License, or
10 |     (at your option) any later version.
11 | 
12 |     threadpool11 is distributed in the hope that it will be useful,
13 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
14 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 |     GNU General Public License for more details.
16 | 
17 |     You should have received a copy of the GNU General Public License
18 |     along with threadpool11.  If not, see <http://www.gnu.org/licenses/>.
19 | */
20 | 
21 | #include "threadpool11/pool.hpp"
22 | #include "threadpool11/worker.hpp"
23 | 
24 | #include "concurrentqueue.h"
25 | 
26 | namespace threadpool11 {
27 | 
28 | Worker::Worker(Pool& pool)
29 |     : thread_(std::bind(&Worker::execute, this, std::ref(pool))) {
30 |   // std::cout << std::this_thread::get_id() << " Worker created" << std::endl;
31 |   thread_.detach();
32 | }
33 | 
34 | void Worker::execute(Pool& pool) {
35 |   const std::unique_ptr<Worker> self(this); //! auto de-allocation when thread is terminated
36 | 
37 |   while (true) {
38 |     Work::Callable* work_ptr;
39 |     Work::Type work_type = Work::Type::STD;
40 | 
41 |     ++pool.active_worker_count_;
42 | 
43 |     while (work_type != Work::Type::TERMINAL
44 |            && pool.work_queue_->try_dequeue(work_ptr)) {
45 |       --pool.work_queue_size_;
46 | 
47 |       const std::unique_ptr<Work::Callable> work(work_ptr);
48 | 
49 |       work_type = (*work)();
50 |     }
51 | 
52 |     std::unique_lock<std::mutex> work_signal_lock(pool.work_signal_mutex_);
53 | 
54 |     // work_queue_size is increased when work_signal_mutex locked
55 |     // active_worker_count_ is only decreased here so it is also when mutex is locked
56 | 
57 |     // as far as I can see when there is a work, it is certain that either work_queue_size or active_worker counter
58 |     // is greater than zero so I don't see any race condition
59 |     if (--pool.active_worker_count_ == 0 && pool.work_queue_size_ == 0) {
60 |       std::unique_lock<std::mutex> notify_all_finished_lock(pool.notify_all_finished_mutex_);
61 | 
62 |       pool.are_all_really_finished_ = true;
63 |       pool.notify_all_finished_signal_.notify_all();
64 |     }
65 | 
66 |     if (work_type == Work::Type::TERMINAL) {
67 |       --pool.worker_count_;
68 | 
69 |       return;
70 |     }
71 | 
72 |     // block here until signalled
73 |     pool.work_signal_.wait(work_signal_lock, [&pool]() { return (pool.work_queue_size_ > 0); });
74 |   }
75 | }
76 | 
77 | }
78 | 


--------------------------------------------------------------------------------
/src/min_heap.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <vector>
  4 | 
  5 | #include "util.hpp"
  6 | 
  7 | template<typename ValueType, typename KeyType, typename IdxType = vid_t>
  8 | class MinHeap {
  9 | private:
 10 |     IdxType n;
 11 |     std::vector<std::pair<ValueType, KeyType>> heap;
 12 |     std::vector<IdxType> key2idx;
 13 | 
 14 | public:
 15 |     MinHeap() : n(0), heap(), key2idx() { }
 16 | 
 17 |     IdxType shift_up(IdxType cur) {
 18 |         if (cur == 0) return 0;
 19 |         IdxType p = (cur-1) / 2;
 20 | 
 21 |         if (heap[cur].first < heap[p].first) {
 22 |             std::swap(heap[cur], heap[p]);
 23 |             std::swap(key2idx[heap[cur].second], key2idx[heap[p].second]);
 24 |             return shift_up(p);
 25 |         }
 26 |         return cur;
 27 |     }
 28 | 
 29 |     void shift_down(IdxType cur) {
 30 |         IdxType l = cur*2 + 1;
 31 |         IdxType r = cur*2 + 2;
 32 | 
 33 |         if (l >= n)
 34 |             return;
 35 | 
 36 |         IdxType m = cur;
 37 |         if (heap[l].first < heap[cur].first)
 38 |             m = l;
 39 |         if (r < n && heap[r].first < heap[m].first)
 40 |             m = r;
 41 | 
 42 |         if (m != cur) {
 43 |             std::swap(heap[cur], heap[m]);
 44 |             std::swap(key2idx[heap[cur].second], key2idx[heap[m].second]);
 45 |             shift_down(m);
 46 |         }
 47 |     }
 48 | 
 49 |     void insert(ValueType value, KeyType key) {
 50 |         heap[n] = std::make_pair(value, key);
 51 |         key2idx[key] = n++;
 52 |         IdxType cur = shift_up(n-1);
 53 |         shift_down(cur);
 54 |     }
 55 | 
 56 |     bool contains(KeyType key) {
 57 |         return key2idx[key] < n && heap[key2idx[key]].second == key;
 58 |     }
 59 | 
 60 |     void decrease_key(KeyType key, ValueType d = 1) {
 61 |         if (d == 0) return;
 62 |         IdxType cur = key2idx[key];
 63 |         CHECK(cur < n && heap[cur].second == key) << "key not found";
 64 | 
 65 |         CHECK_GE(heap[cur].first, d) << "value cannot be negative";
 66 |         heap[cur].first -= d;
 67 |         shift_up(cur);
 68 |     }
 69 | 
 70 |     bool remove(KeyType key) {
 71 |         IdxType cur = key2idx[key];
 72 |         if (cur >= n || heap[cur].second != key)
 73 |             return false;
 74 | 
 75 |         n--;
 76 |         if (n > 0) {
 77 |             heap[cur] = heap[n];
 78 |             key2idx[heap[cur].second] = cur;
 79 |             cur = shift_up(cur);
 80 |             shift_down(cur);
 81 |         }
 82 |         return true;
 83 |     }
 84 | 
 85 |     bool get_min(ValueType& value, KeyType& key) {
 86 |         if (n > 0) {
 87 |             value = heap[0].first;
 88 |             key = heap[0].second;
 89 |             return true;
 90 |         } else
 91 |             return false;
 92 |     }
 93 | 
 94 |     void reserve(IdxType nelements) {
 95 |         n = 0;
 96 |         heap.resize(nelements);
 97 |         key2idx.resize(nelements);
 98 |     }
 99 | 
100 |     void clear() {
101 |         n = 0;
102 |     }
103 | };
104 | 


--------------------------------------------------------------------------------
/src/dbh_partitioner.cpp:
--------------------------------------------------------------------------------
 1 | #include <utility>
 2 | 
 3 | #include "util.hpp"
 4 | #include "dbh_partitioner.hpp"
 5 | #include "conversions.hpp"
 6 | 
 7 | DbhPartitioner::DbhPartitioner(std::string basefilename)
 8 | {
 9 |     Timer convert_timer;
10 |     convert_timer.start();
11 |     convert(basefilename, new Converter(basefilename));
12 |     convert_timer.stop();
13 |     LOG(INFO) << "convert time: " << convert_timer.get_time();
14 | 
15 |     total_time.start();
16 |     LOG(INFO) << "initializing partitioner";
17 | 
18 |     fin = open(binedgelist_name(basefilename).c_str(), O_RDONLY, (mode_t)0600);
19 |     PCHECK(fin != -1) << "Error opening file for read";
20 |     struct stat fileInfo = {0};
21 |     PCHECK(fstat(fin, &fileInfo) != -1) << "Error getting the file size";
22 |     PCHECK(fileInfo.st_size != 0) << "Error: file is empty";
23 |     LOG(INFO) << "file size: " << fileInfo.st_size;
24 | 
25 |     fin_map = (char *)mmap(0, fileInfo.st_size, PROT_READ, MAP_SHARED, fin, 0);
26 |     if (fin_map == MAP_FAILED) {
27 |         close(fin);
28 |         PLOG(FATAL) << "error mapping the file";
29 |     }
30 | 
31 |     filesize = fileInfo.st_size;
32 |     fin_ptr = fin_map;
33 |     fin_end = fin_map + filesize;
34 | 
35 |     num_vertices = *(vid_t *)fin_ptr;
36 |     fin_ptr += sizeof(vid_t);
37 |     num_edges = *(size_t *)fin_ptr;
38 |     fin_ptr += sizeof(size_t);
39 | 
40 |     LOG(INFO) << "num_vertices: " << num_vertices
41 |               << ", num_edges: " << num_edges;
42 | 
43 |     p = FLAGS_p;
44 |     dis.param(std::uniform_int_distribution<int>::param_type(0, p - 1));
45 | 
46 |     degrees.resize(num_vertices);
47 |     std::ifstream degree_file(degree_name(basefilename), std::ios::binary);
48 |     degree_file.read((char *)&degrees[0], num_vertices * sizeof(vid_t));
49 |     degree_file.close();
50 | }
51 | 
52 | void DbhPartitioner::split()
53 | {
54 |     std::vector<dense_bitset> is_mirrors(p, dense_bitset(num_vertices));
55 |     std::vector<size_t> counter(p, 0);
56 |     while (fin_ptr < fin_end) {
57 |         edge_t *e = (edge_t *)fin_ptr;
58 |         fin_ptr += sizeof(edge_t);
59 |         vid_t w = degrees[e->first] <= degrees[e->second] ? e->first : e->second;
60 |         int bucket = w % p;
61 |         counter[bucket]++;
62 |         is_mirrors[bucket].set_bit_unsync(e->first);
63 |         is_mirrors[bucket].set_bit_unsync(e->second);
64 |     }
65 | 
66 |     if (munmap(fin_map, filesize) == -1) {
67 |         close(fin);
68 |         PLOG(FATAL) << "Error un-mmapping the file";
69 |     }
70 |     close(fin);
71 | 
72 | 
73 |     size_t max_occupied = *std::max_element(counter.begin(), counter.end());
74 |     LOG(INFO) << "balance: " << (double)max_occupied / ((double)num_edges / p);
75 |     size_t total_mirrors = 0;
76 |     rep (i, p)
77 |         total_mirrors += is_mirrors[i].popcount();
78 |     LOG(INFO) << "total mirrors: " << total_mirrors;
79 |     LOG(INFO) << "replication factor: " << (double)total_mirrors / num_vertices;
80 | 
81 |     total_time.stop();
82 |     LOG(INFO) << "total partition time: " << total_time.get_time();
83 | }
84 | 


--------------------------------------------------------------------------------
/src/util.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <utility>
  4 | #include <chrono>
  5 | #include <stdint.h>
  6 | #include <sys/stat.h>
  7 | 
  8 | #include <gflags/gflags.h>
  9 | #include <glog/logging.h>
 10 | 
 11 | #include "threadpool11/threadpool11.hpp"
 12 | 
 13 | #define rep(i, n) for (int i = 0; i < (int)(n); ++i)
 14 | #define repv(i, n) for (vid_t i = 0; i < n; ++i)
 15 | 
 16 | DECLARE_int32(p);
 17 | DECLARE_uint64(memsize);
 18 | DECLARE_string(filename);
 19 | DECLARE_string(filetype);
 20 | DECLARE_bool(inmem);
 21 | DECLARE_double(sample_ratio);
 22 | 
 23 | typedef uint32_t vid_t;
 24 | const vid_t INVALID_VID = -1;
 25 | struct edge_t {
 26 |     vid_t first, second;
 27 |     edge_t() : first(0), second(0) {}
 28 |     edge_t(vid_t first, vid_t second) : first(first), second(second) {}
 29 |     const bool valid() { return first != INVALID_VID; }
 30 |     void remove() { first = INVALID_VID; }
 31 | };
 32 | 
 33 | extern threadpool11::Pool pool;
 34 | 
 35 | void preada(int f, char *buf, size_t nbytes, size_t off);
 36 | void reada(int f, char *buf, size_t nbytes);
 37 | void writea(int f, char *buf, size_t nbytes);
 38 | 
 39 | inline std::string binedgelist_name(const std::string &basefilename)
 40 | {
 41 |     std::stringstream ss;
 42 |     ss << basefilename << ".binedgelist";
 43 |     return ss.str();
 44 | }
 45 | inline std::string shuffled_binedgelist_name(const std::string &basefilename)
 46 | {
 47 |     std::stringstream ss;
 48 |     ss << basefilename << ".shuffled.binedgelist";
 49 |     return ss.str();
 50 | }
 51 | inline std::string degree_name(const std::string &basefilename)
 52 | {
 53 |     std::stringstream ss;
 54 |     ss << basefilename << ".degree";
 55 |     return ss.str();
 56 | }
 57 | 
 58 | inline std::string partitioned_name(const std::string &basefilename)
 59 | {
 60 |     std::stringstream ss;
 61 |     ss << basefilename << ".edgepart." << FLAGS_p; // chenzi: add partition number to the output file
 62 |     return ss.str();
 63 | }
 64 | 
 65 | inline std::string hilbert_name(const std::string &basefilename)
 66 | {
 67 |     std::stringstream ss;
 68 |     ss << basefilename << ".hilbert.bin";
 69 |     return ss.str();
 70 | }
 71 | inline std::string sorted_hilbert_name(const std::string &basefilename)
 72 | {
 73 |     std::stringstream ss;
 74 |     ss << basefilename << ".sorted_hilbert.bin";
 75 |     return ss.str();
 76 | }
 77 | 
 78 | 
 79 | inline bool is_exists(const std::string &name)
 80 | {
 81 |     struct stat buffer;
 82 |     return (stat(name.c_str(), &buffer) == 0);
 83 | }
 84 | 
 85 | class Timer
 86 | {
 87 |   private:
 88 |     std::chrono::system_clock::time_point t1, t2;
 89 |     double total;
 90 | 
 91 |   public:
 92 |     Timer() : total(0) {}
 93 |     void reset() { total = 0; }
 94 |     void start() { t1 = std::chrono::system_clock::now(); }
 95 |     void stop()
 96 |     {
 97 |         t2 = std::chrono::system_clock::now();
 98 |         std::chrono::duration<double> diff = t2 - t1;
 99 |         total += diff.count();
100 |     }
101 |     double get_time() { return total; }
102 | };
103 | 


--------------------------------------------------------------------------------
/src/shuffler.cpp:
--------------------------------------------------------------------------------
  1 | #include <utility>
  2 | #include <algorithm>
  3 | #include <iostream>
  4 | #include <fstream>
  5 | 
  6 | #include <stdlib.h>
  7 | 
  8 | #include "util.hpp"
  9 | #include "shuffler.hpp"
 10 | 
 11 | void Shuffler::init()
 12 | {
 13 |     num_vertices = 0;
 14 |     num_edges = 0;
 15 |     nchunks = 0;
 16 |     degrees.reserve(1<<20);
 17 |     pool.setWorkerCount(2, threadpool11::Pool::Method::SYNC);
 18 |     chunk_bufsize =
 19 |         FLAGS_memsize * 1024 * 1024 / pool.getWorkerCount() / sizeof(edge_t);
 20 |     chunk_buf.reserve(chunk_bufsize);
 21 | }
 22 | 
 23 | void Shuffler::finalize()
 24 | {
 25 |     cwrite(edge_t(0, 0), true);
 26 |     pool.waitAll();
 27 |     LOG(INFO) << "num_vertices: " << num_vertices
 28 |               << ", num_edges: " << num_edges;
 29 |     LOG(INFO) << "number of chunks: " << nchunks;
 30 | 
 31 |     std::vector<std::ifstream> fin(nchunks);
 32 |     rep (i, nchunks) {
 33 |         fin[i].open(chunk_filename(i), std::ios::binary);
 34 |         CHECK(fin[i]) << "open chunk " << i << " failed";
 35 |     }
 36 |     std::vector<bool> finished(nchunks, false);
 37 |     std::ofstream fout(shuffled_binedgelist_name(basefilename), std::ios::binary);
 38 |     int count = 0;
 39 |     fout.write((char *)&num_vertices, sizeof(num_vertices));
 40 |     fout.write((char *)&num_edges, sizeof(num_edges));
 41 |     while (true) {
 42 |         int i = rand() % nchunks;
 43 |         if (!fin[i].eof()) {
 44 |             edge_t e;
 45 |             fin[i].read((char *)&e, sizeof(edge_t));
 46 |             if (fin[i].eof()) {
 47 |                 finished[i] = true;
 48 |                 count++;
 49 |                 if (count == nchunks)
 50 |                     break;
 51 |             } else
 52 |                 fout.write((char *)&e, sizeof(edge_t));
 53 |         }
 54 |     }
 55 |     rep (i, nchunks)
 56 |         fin[i].close();
 57 |     fout.close();
 58 |     chunk_clean();
 59 | 
 60 |     fout.open(degree_name(basefilename), std::ios::binary);
 61 |     fout.write((char *)&degrees[0], num_vertices * sizeof(vid_t));
 62 |     fout.close();
 63 | 
 64 |     LOG(INFO) << "finished shuffle";
 65 | }
 66 | 
 67 | void Shuffler::add_edge(vid_t from, vid_t to)
 68 | {
 69 |     if (to == from) {
 70 |         LOG(WARNING) << "Tried to add self-edge " << from << "->" << to
 71 |                   << std::endl;
 72 |         return;
 73 |     }
 74 | 
 75 |     num_edges++;
 76 |     from = get_vid(from);
 77 |     to = get_vid(to);
 78 |     degrees[from]++;
 79 |     degrees[to]++;
 80 | 
 81 |     edge_t e(from, to);
 82 |     cwrite(e);
 83 | }
 84 | 
 85 | std::string Shuffler::chunk_filename(int chunk)
 86 | {
 87 |     std::stringstream ss;
 88 |     ss << basefilename << "." << chunk << ".chunk";
 89 |     return ss.str();
 90 | }
 91 | 
 92 | void Shuffler::chunk_clean()
 93 | {
 94 |     rep (i, nchunks)
 95 |         remove(chunk_filename(i).c_str());
 96 | }
 97 | 
 98 | void Shuffler::cwrite(edge_t e, bool flush)
 99 | {
100 |     if (!flush)
101 |         chunk_buf.push_back(e);
102 |     if (flush || chunk_buf.size() >= chunk_bufsize) {
103 |         work_t work;
104 |         work.shuffler = this;
105 |         work.nchunks = nchunks;
106 |         chunk_buf.swap(work.chunk_buf);
107 |         pool.postWork<void>(work);
108 |         nchunks++;
109 |     }
110 | }
111 | 


--------------------------------------------------------------------------------
/threadpool11/src/pool.cpp:
--------------------------------------------------------------------------------
  1 | /*!
  2 | Copyright (c) 2013, 2014, 2015 Tolga HOŞGÖR
  3 | All rights reserved.
  4 | 
  5 | This file is part of threadpool11.
  6 | 
  7 |     threadpool11 is free software: you can redistribute it and/or modify
  8 |     it under the terms of the GNU General Public License as published by
  9 |     the Free Software Foundation, either version 3 of the License, or
 10 |     (at your option) any later version.
 11 | 
 12 |     threadpool11 is distributed in the hope that it will be useful,
 13 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
 14 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 15 |     GNU General Public License for more details.
 16 | 
 17 |     You should have received a copy of the GNU General Public License
 18 |     along with threadpool11.  If not, see <http://www.gnu.org/licenses/>.
 19 | */
 20 | 
 21 | #include "threadpool11/pool.hpp"
 22 | 
 23 | #include <boost/lockfree/queue.hpp>
 24 | 
 25 | #include <algorithm>
 26 | #include <vector>
 27 | 
 28 | namespace threadpool11 {
 29 | 
 30 | Pool::Pool(std::size_t worker_count)
 31 |     : worker_count_(0)
 32 |     , active_worker_count_(0)
 33 |     , are_all_really_finished_{true}
 34 |     , work_queue_(new moodycamel::ConcurrentQueue<Work::Callable*>(10000U))
 35 |     , work_queue_size_(0) {
 36 |   spawnWorkers(worker_count);
 37 | }
 38 | 
 39 | Pool::~Pool() { joinAll(); }
 40 | 
 41 | void Pool::waitAll() {
 42 |   std::unique_lock<std::mutex> notify_all_finished_lock(notify_all_finished_mutex_);
 43 | 
 44 |   notify_all_finished_signal_.wait(notify_all_finished_lock, [this]() { return are_all_really_finished_.load(); });
 45 | }
 46 | 
 47 | void Pool::joinAll() { decWorkerCountBy(std::numeric_limits<size_t>::max(), Method::SYNC); }
 48 | 
 49 | size_t Pool::getWorkerCount() const { return worker_count_.load(); }
 50 | 
 51 | void Pool::setWorkerCount(std::size_t n, Method method) {
 52 |   if (getWorkerCount() < n)
 53 |     incWorkerCountBy(n - getWorkerCount());
 54 |   else
 55 |     decWorkerCountBy(getWorkerCount() - n, method);
 56 | }
 57 | 
 58 | size_t Pool::getWorkQueueSize() const { return work_queue_size_.load(); }
 59 | 
 60 | size_t Pool::getActiveWorkerCount() const { return active_worker_count_.load(); }
 61 | 
 62 | size_t Pool::getInactiveWorkerCount() const { return worker_count_.load() - active_worker_count_.load(); }
 63 | 
 64 | void Pool::incWorkerCountBy(std::size_t n) { spawnWorkers(n); }
 65 | 
 66 | void Pool::decWorkerCountBy(size_t n, Method method) {
 67 |   n = std::min(n, getWorkerCount());
 68 |   if (method == Method::SYNC) {
 69 |     std::vector<std::future<void>> futures;
 70 |     futures.reserve(n);
 71 |     while (n-- > 0)
 72 |       futures.emplace_back(postWork<void>([]() {}, Work::Type::TERMINAL));
 73 |     for (auto& it : futures)
 74 |       it.get();
 75 |   } else {
 76 |     while (n-- > 0)
 77 |       postWork<void>([]() {}, Work::Type::TERMINAL);
 78 |   }
 79 | }
 80 | 
 81 | void Pool::spawnWorkers(std::size_t n) {
 82 |   //'OR' makes sure the case where one of the expressions is zero, is valid.
 83 |   assert(static_cast<size_t>(worker_count_ + n) > n ||
 84 |          static_cast<size_t>(worker_count_ + n) > worker_count_);
 85 |   while (n-- > 0) {
 86 |     new Worker(*this); //! Worker class takes care of its de-allocation itself after here
 87 |     ++worker_count_;
 88 |   }
 89 | }
 90 | 
 91 | void Pool::push(Work::Callable* workFunc) {
 92 |   are_all_really_finished_ = false;
 93 | 
 94 |   std::unique_lock<std::mutex> work_signal_lock(work_signal_mutex_);
 95 |   ++work_queue_size_;
 96 |   work_queue_->try_enqueue(workFunc);
 97 |   work_signal_.notify_one();
 98 | }
 99 | }
100 | 


--------------------------------------------------------------------------------
/src/conversions.cpp:
--------------------------------------------------------------------------------
  1 | #include <string.h>
  2 | 
  3 | #include "conversions.hpp"
  4 | 
  5 | // Removes \n from the end of line
  6 | void FIXLINE(char *s)
  7 | {
  8 |     int len = (int)strlen(s) - 1;
  9 |     if (s[len] == '\n')
 10 |         s[len] = 0;
 11 | }
 12 | 
 13 | void convert_edgelist(std::string inputfile, Converter *converter)
 14 | {
 15 |     FILE *inf = fopen(inputfile.c_str(), "r");
 16 |     size_t bytesread = 0;
 17 |     size_t linenum = 0;
 18 |     if (inf == NULL) {
 19 |         LOG(FATAL) << "Could not load:" << inputfile
 20 |                    << ", error: " << strerror(errno) << std::endl;
 21 |     }
 22 | 
 23 |     LOG(INFO) << "Reading in edge list format!" << std::endl;
 24 |     char s[1024];
 25 |     while (fgets(s, 1024, inf) != NULL) {
 26 |         linenum++;
 27 |         if (linenum % 10000000 == 0) {
 28 |             LOG(INFO) << "Read " << linenum << " lines, "
 29 |                       << bytesread / 1024 / 1024. << " MB" << std::endl;
 30 |         }
 31 |         FIXLINE(s);
 32 |         bytesread += strlen(s);
 33 |         if (s[0] == '#')
 34 |             continue; // Comment
 35 |         if (s[0] == '%')
 36 |             continue; // Comment
 37 | 
 38 |         char delims[] = "\t, ";
 39 |         char *t;
 40 |         t = strtok(s, delims);
 41 |         if (t == NULL) {
 42 |             LOG(FATAL) << "Input file is not in right format. "
 43 |                        << "Expecting \"<from>\t<to>\". "
 44 |                        << "Current line: \"" << s << "\"\n";
 45 |         }
 46 |         vid_t from = atoi(t);
 47 |         t = strtok(NULL, delims);
 48 |         if (t == NULL) {
 49 |             LOG(FATAL) << "Input file is not in right format. "
 50 |                        << "Expecting \"<from>\t<to>\". "
 51 |                        << "Current line: \"" << s << "\"\n";
 52 |         }
 53 |         vid_t to = atoi(t);
 54 | 
 55 |         if (from != to) {
 56 |             converter->add_edge(from, to);
 57 |         }
 58 |     }
 59 |     fclose(inf);
 60 | }
 61 | 
 62 | void convert_adjlist(std::string inputfile, Converter *converter)
 63 | {
 64 |     FILE *inf = fopen(inputfile.c_str(), "r");
 65 |     if (inf == NULL) {
 66 |         LOG(FATAL) << "Could not load:" << inputfile
 67 |                    << " error: " << strerror(errno) << std::endl;
 68 |     }
 69 |     LOG(INFO) << "Reading in adjacency list format!" << std::endl;
 70 | 
 71 |     int maxlen = 1000000000;
 72 |     char *s = (char *)malloc(maxlen);
 73 | 
 74 |     size_t bytesread = 0;
 75 | 
 76 |     char delims[] = " \t";
 77 |     size_t linenum = 0;
 78 |     size_t lastlog = 0;
 79 | 
 80 |     while (fgets(s, maxlen, inf) != NULL) {
 81 |         linenum++;
 82 |         if (bytesread - lastlog >= 500000000) {
 83 |             LOG(INFO) << "Read " << linenum << " lines, "
 84 |                       << bytesread / 1024 / 1024. << " MB" << std::endl;
 85 |             lastlog = bytesread;
 86 |         }
 87 |         FIXLINE(s);
 88 |         bytesread += strlen(s);
 89 | 
 90 |         if (s[0] == '#')
 91 |             continue; // Comment
 92 |         if (s[0] == '%')
 93 |             continue; // Comment
 94 |         char *t = strtok(s, delims);
 95 |         vid_t from = atoi(t);
 96 |         t = strtok(NULL, delims);
 97 |         if (t != NULL) {
 98 |             vid_t num = atoi(t);
 99 |             vid_t i = 0;
100 |             while ((t = strtok(NULL, delims)) != NULL) {
101 |                 vid_t to = atoi(t);
102 |                 if (from != to) {
103 |                     converter->add_edge(from, to);
104 |                 }
105 |                 i++;
106 |             }
107 |             if (num != i)
108 |                 LOG(FATAL) << "Mismatch when reading adjacency list: " << num
109 |                            << " != " << i << " s: " << std::string(s)
110 |                            << " on line: " << linenum << std::endl;
111 |         }
112 |     }
113 |     free(s);
114 |     fclose(inf);
115 | }
116 | 
117 | void convert(std::string basefilename, Converter *converter)
118 | {
119 |     LOG(INFO) << "converting `" << basefilename << "'";
120 |     if (basefilename.empty())
121 |         LOG(FATAL) << "empty file name";
122 |     if (converter->done()) {
123 |         LOG(INFO) << "skip";
124 |         return;
125 |     }
126 |     converter->init();
127 |     if (FLAGS_filetype == "adjlist") {
128 |         convert_adjlist(basefilename, converter);
129 |     } else if (FLAGS_filetype == "edgelist") {
130 |         convert_edgelist(basefilename, converter);
131 |     } else {
132 |         LOG(FATAL) << "unknown filetype";
133 |     }
134 |     converter->finalize();
135 | }
136 | 
137 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | Edge Partitioning Algorithms for Large Graphs
 2 | =============================================
 3 | 
 4 | > These algorithms are implemented by Qin Liu during his study at CUHK.
 5 | 
 6 | In this repo, we implement several edge partitioning algorithms and compute
 7 | their replication factors for comparison:
 8 | 
 9 | *   Random partitioning (random)
10 | *   Degree-based hashing (DBH): a
11 |     [paper](http://papers.nips.cc/paper/5396-distributed-power-law-graph-computing-theoretical-and-empirical-analysis.pdf)
12 |     on NIPS'14
13 | *   A method based on
14 |     [Hilber space-filling curve](https://en.wikipedia.org/wiki/Hilbert_curve) (HSFC):
15 |     this one is inspired by Frank McSherry's [post](https://github.com/frankmcsherry/blog/blob/master/posts/2015-01-15.md)
16 | *   Our algorithms described in our paper **[Graph Edge Partitioning via Neighborhood Heuristic](http://www.kdd.org/kdd2017/papers/view/graph-edge-partitioning-via-neighborhood-heuristic)** (published in KDD'17)
17 |     -   Neighbor expansion (NE): described in Section 3 of our paper
18 |     -   Streaming neighbor expansion (SNE): described in Appendix B of our paper
19 | 
20 | Compilation and Usage
21 | ---------------------
22 | 
23 | We tested our program on Ubuntu 14.04/16.04, and it requires the following
24 | packages: `cmake`, `glog`, `gflags`, `boost`:
25 | ```
26 | sudo apt-get install libgoogle-glog-dev libgflags-dev libboost-all-dev
27 | ```
28 | 
29 | Compilation:
30 | ```
31 | git clone https://github.com/ansrlab/edgepart.git
32 | cd edgepart
33 | mkdir release && cd release
34 | cmake ..
35 | make -j8
36 | ```
37 | 
38 | Usage:
39 | ```
40 | $ ./main --help
41 | main: -filename <path to the input graph> [-filetype <edgelist|adjlist>] [-p <number of partitions>] [-memsize <memory budget in MB>]
42 | 
43 |   Flags from /home/qliu/workspace/edgepart/src/main.cpp:
44 |     -filename (the file name of the input graph) type: string default: ""
45 |     -filetype (the type of input file (supports 'edgelist' and 'adjlist'))
46 |       type: string default: "edgelist"
47 |     -inmem (in-memory mode) type: bool default: false
48 |     -memsize (memory size in megabytes) type: uint64 default: 4096
49 |     -method (partition method: ne, sne, random, and dbh) type: string
50 |       default: "sne"
51 |     -p (number of parititions) type: int32 default: 10
52 |     -sample_ratio (the sample size divided by num_vertices) type: double
53 |       default: 2
54 | ```
55 | 
56 | **Example.** Partition the Orkut graph into 30 parts using our NE algorithm:
57 | ```
58 | $ ./main -p 30 -method ne -filename /path/to/com-orkut.ungraph.txt
59 | ```
60 | 
61 | **Example.** Partition the LiveJournal graph into 30 parts using our SNE
62 | algorithm (`CacheSize = 2|V|`, see our paper for detailed description):
63 | ```
64 | $ ./main -p 30 -method sne -filename /path/to/com-lj.ungraph.txt -sample_ratio 2
65 | ```
66 | 
67 | Evaluation
68 | ----------
69 | 
70 | The experiments are conducted on a PC with 16 GB RAM. MLE means "memory limit
71 | exceeded".
72 | 
73 | Algorithms that are listed but not contained in this repo:
74 | 
75 | *   [METIS](http://glaros.dtc.umn.edu/gkhome/metis/metis/overview)
76 | *   [Sheep](https://github.com/dmargo/sheep): published on VLDB'15
77 | *   Algorithms that has been integrated in to
78 |     [PowerGraph](https://github.com/jegonzal/PowerGraph)
79 |     -   Oblivious
80 |     -   High-Degree (are) Replicated First (HDRF): published on CIKM'15
81 | 
82 | Algorithm  |  wiki-Vote  |  email-Enron  |  web-Google  |  com-LiveJournal  |  com-Orkut  |  twitter-2010  |  com-Friendster  |  uk-union
83 | ---------  |  ---------  |  -----------  |  ----------  |  ---------------  |  ---------  |  ------------  |  --------------  |  --------
84 | METIS      |  5.25       |  2.40         |  1.05        |  2.13             |  MLE        |  MLE           |  MLE             |  MLE
85 | NE         |  2.35       |  1.34         |  1.12        |  1.55             |  2.48       |  1.88          |  1.98            |  1.04
86 | Random     |  9.28       |  5.10         |  6.54        |  8.27             |  19.48      |  11.68         |  11.84           |  15.99
87 | DBH        |  5.43       |  3.32         |  4.09        |  5.18             |  11.97      |  3.67          |  6.88            |  5.14
88 | Oblivious  |  3.85       |  2.30         |  2.28        |  3.43             |  6.94       |  8.60          |  8.82            |  2.03
89 | HDRF       |  3.90       |  2.12         |  2.18        |  3.33             |  7.27       |  7.90          |  8.87            |  1.62
90 | Sheep      |  4.20       |  1.78         |  1.71        |  3.33             |  7.94       |  2.34          |  4.45            |  1.29
91 | SNE        |  3.05       |  1.44         |  1.17        |  1.88             |  4.49       |  2.83          |  3.00            |  1.65
92 | HSFC       |  3.80       |  3.75         |  2.66        |  3.78             |  6.31       |  5.82          |  4.80            |  1.96
93 | 


--------------------------------------------------------------------------------
/src/hsfc_partitioner.cpp:
--------------------------------------------------------------------------------
  1 | #include <utility>
  2 | #include <functional>
  3 | #include <fcntl.h>
  4 | 
  5 | #include "util.hpp"
  6 | #include "hsfc_partitioner.hpp"
  7 | #include "sort.hpp"
  8 | #include "conversions.hpp"
  9 | 
 10 | HsfcPartitioner::HsfcPartitioner(std::string basefilename)
 11 |     : basefilename(basefilename)
 12 | {
 13 |     Timer convert_timer;
 14 |     convert_timer.start();
 15 |     convert(basefilename, new Converter(basefilename));
 16 |     convert_timer.stop();
 17 |     LOG(INFO) << "convert time: " << convert_timer.get_time();
 18 | 
 19 |     total_time.start();
 20 |     LOG(INFO) << "initializing partitioner";
 21 | 
 22 |     fin = open(binedgelist_name(basefilename).c_str(), O_RDONLY, (mode_t)0600);
 23 |     PCHECK(fin != -1) << "Error opening file for read";
 24 |     struct stat fileInfo = {0};
 25 |     PCHECK(fstat(fin, &fileInfo) != -1) << "Error getting the file size";
 26 |     PCHECK(fileInfo.st_size != 0) << "Error: file is empty";
 27 |     LOG(INFO) << "file size: " << fileInfo.st_size;
 28 | 
 29 |     fin_map = (char *)mmap(0, fileInfo.st_size, PROT_READ, MAP_SHARED, fin, 0);
 30 |     if (fin_map == MAP_FAILED) {
 31 |         close(fin);
 32 |         PLOG(FATAL) << "error mapping the file";
 33 |     }
 34 | 
 35 |     filesize = fileInfo.st_size;
 36 |     fin_ptr = fin_map;
 37 |     fin_end = fin_map + filesize;
 38 | 
 39 |     num_vertices = *(vid_t *)fin_ptr;
 40 |     fin_ptr += sizeof(vid_t);
 41 |     num_edges = *(size_t *)fin_ptr;
 42 |     fin_ptr += sizeof(size_t);
 43 | 
 44 |     LOG(INFO) << "num_vertices: " << num_vertices
 45 |               << ", num_edges: " << num_edges;
 46 | 
 47 |     n = 1;
 48 |     while (n < num_vertices)
 49 |         n = n << 1;
 50 | 
 51 |     p = FLAGS_p;
 52 | }
 53 | 
 54 | void HsfcPartitioner::generate_hilber()
 55 | {
 56 |     Timer timer;
 57 |     timer.start();
 58 |     LOG(INFO) << "generating hilber distance file...";
 59 |     std::ofstream fout(hilbert_name(basefilename), std::ios::binary);
 60 |     int gb = 0;
 61 |     size_t bytes = 0;
 62 |     while (fin_ptr < fin_end) {
 63 |         edge_t *e = (edge_t *)fin_ptr;
 64 |         fin_ptr += sizeof(edge_t);
 65 |         bytes += sizeof(edge_t);
 66 |         vid_t u = e->first, v = e->second;
 67 |         uint64_t d = xy2d(u, v);
 68 |         fout.write((char *)&d, sizeof(d));
 69 |         if (bytes == 1024*1024*1024) {
 70 |             gb++;
 71 |             bytes = 0;
 72 |         }
 73 |     }
 74 |     fout.close();
 75 |     timer.stop();
 76 |     LOG(INFO) << "load time: " << timer.get_time();
 77 | }
 78 | 
 79 | void HsfcPartitioner::split()
 80 | {
 81 |     std::vector<dense_bitset> is_mirrors(p, dense_bitset(num_vertices));
 82 |     std::vector<size_t> counter(p, 0);
 83 | 
 84 |     if (!is_exists(hilbert_name(basefilename)))
 85 |         generate_hilber();
 86 |     else
 87 |         LOG(INFO) << "skip generating hilbert distance file";
 88 |     if (munmap(fin_map, filesize) == -1) {
 89 |         close(fin);
 90 |         PLOG(FATAL) << "Error un-mmapping the file";
 91 |     }
 92 |     close(fin);
 93 | 
 94 |     Timer timer;
 95 |     if (!is_exists(sorted_hilbert_name(basefilename))) {
 96 |         timer.start();
 97 |         LOG(INFO) << "sorting...";
 98 |         int fdInput = open(hilbert_name(basefilename).c_str(), O_RDONLY);
 99 |         PCHECK(fdInput != -1) << "Error opening file for read";
100 |         int fdOutput = open(sorted_hilbert_name(basefilename).c_str(),
101 |                             O_WRONLY | O_CREAT | O_TRUNC,
102 |                             S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
103 |         PCHECK(fdOutput != -1) << "Error opening file `"
104 |                                << sorted_hilbert_name(basefilename)
105 |                                << "` for write";
106 |         externalSort(fdInput, num_edges, fdOutput, FLAGS_memsize * 1024 * 1024);
107 |         close(fdInput);
108 |         close(fdOutput);
109 |         timer.stop();
110 |         LOG(INFO) << "sort time: " << timer.get_time();
111 |     } else
112 |         LOG(INFO) << "skip sorting";
113 | 
114 |     timer.reset();
115 |     timer.start();
116 |     LOG(INFO) << "partitioning...";
117 |     std::ifstream hilbert_file(sorted_hilbert_name(basefilename), std::ios::binary);
118 |     size_t range = num_edges / p + 1;
119 |     for (size_t i = 0; i < num_edges; i++) {
120 |         int bucket = i / range;
121 |         counter[bucket]++;
122 |         vid_t u, v;
123 |         uint64_t d;
124 |         hilbert_file.read((char *)&d, sizeof(d));
125 |         d2xy(d, &u, &v);
126 |         is_mirrors[bucket].set_bit_unsync(u);
127 |         is_mirrors[bucket].set_bit_unsync(v);
128 |     }
129 |     timer.stop();
130 |     LOG(INFO) << "partition time: " << timer.get_time();
131 | 
132 |     size_t max_occupied = *std::max_element(counter.begin(), counter.end());
133 |     LOG(INFO) << "balance: " << (double)max_occupied / ((double)num_edges / p);
134 |     size_t total_mirrors = 0;
135 |     rep (i, p)
136 |         total_mirrors += is_mirrors[i].popcount();
137 |     LOG(INFO) << "total mirrors: " << total_mirrors;
138 |     LOG(INFO) << "replication factor: " << (double)total_mirrors / num_vertices;
139 | 
140 |     total_time.stop();
141 |     LOG(INFO) << "total partition time: " << total_time.get_time();
142 | }
143 | 


--------------------------------------------------------------------------------
/src/sort.cpp:
--------------------------------------------------------------------------------
  1 | #include <algorithm>
  2 | #include <fcntl.h>
  3 | #include <iostream>
  4 | #include <queue>
  5 | #include <stdio.h>
  6 | #include <unistd.h>
  7 | 
  8 | #include "sort.hpp"
  9 | #include "util.hpp"
 10 | 
 11 | using namespace std;
 12 | 
 13 | struct mergeItem {
 14 |     uint64_t value;
 15 |     uint64_t srcIndex;
 16 | 
 17 |     bool operator<(const mergeItem &a) const { return a.value < value; }
 18 | };
 19 | 
 20 | template <typename T> struct fileBuffer {
 21 |     FILE *srcFile;
 22 |     T *buffer;
 23 |     uint64_t length;
 24 |     uint64_t index;
 25 | };
 26 | 
 27 | void externalSort(int fdInput, uint64_t size, int fdOutput, uint64_t memSize)
 28 | {
 29 |     if (size == 0) {
 30 |         // nothing to do here
 31 |         return;
 32 |     }
 33 | 
 34 |     CHECK_GT(memSize, sizeof(uint64_t)) << "Not enough memory for sorting";
 35 | 
 36 |     /*** STEP 1: Chunking ***/
 37 | 
 38 |     // how many values fit into one chunk
 39 |     const uint64_t chunkSize = min(memSize / sizeof(uint64_t), size);
 40 | 
 41 |     // number of chunks we must split the input into
 42 |     const uint64_t numChunks = (size + chunkSize - 1) / chunkSize;
 43 | 
 44 |     DLOG(INFO) << "filesize: " << (size * sizeof(uint64_t))
 45 |                << " bytes, memSize: " << memSize
 46 |                << " bytes, chunkSize: " << chunkSize
 47 |                << ", numChunks: " << numChunks;
 48 | 
 49 |     vector<FILE *> chunkFiles;
 50 |     chunkFiles.reserve(numChunks);
 51 | 
 52 |     vector<uint64_t> memBuf;
 53 |     memBuf.resize(chunkSize);
 54 | 
 55 |     for (uint64_t i = 0; i < numChunks; i++) {
 56 |         // read one chunk of input into memory
 57 |         size_t valuesToRead = chunkSize;
 58 |         if (size - i * chunkSize < valuesToRead) {
 59 |             valuesToRead = size - i * chunkSize;
 60 |             memBuf.resize(valuesToRead);
 61 |         }
 62 |         size_t bytesToRead = valuesToRead * sizeof(uint64_t);
 63 | 
 64 |         DLOG(INFO) << "chunk #" << i << " bytesToRead: " << bytesToRead;
 65 | 
 66 |         reada(fdInput, (char *)&memBuf[0], bytesToRead);
 67 | 
 68 |         // sort the values in the chunk
 69 |         sort(memBuf.begin(), memBuf.end());
 70 | 
 71 |         // open a new temporary file and save the chunk externally
 72 |         FILE *tmpf = tmpfile();
 73 |         PCHECK(tmpf != NULL) << "Creating a temporary file failed!";
 74 |         chunkFiles.push_back(tmpf);
 75 | 
 76 |         fwrite(&memBuf[0], sizeof(uint64_t), valuesToRead, tmpf);
 77 |         PCHECK(!ferror(tmpf)) << "Writing chunk to file failed!";
 78 |     }
 79 | 
 80 |     /*** STEP 2: k-way merge chunks ***/
 81 | 
 82 |     // priority queue used for n-way merging values from n chunks
 83 |     priority_queue<mergeItem> queue;
 84 | 
 85 |     // We reuse the already allocated memory from the memBuf and split the
 86 |     // available memory between numChunks chunk buffers and 1 output buffer.
 87 |     const size_t bufSize = chunkSize / (numChunks + 1);
 88 | 
 89 |     // input buffers (from chunks)
 90 |     vector<fileBuffer<uint64_t>> inBufs;
 91 |     inBufs.reserve(numChunks);
 92 | 
 93 |     for (uint64_t i = 0; i < numChunks; i++) {
 94 |         fileBuffer<uint64_t> buf = {
 95 |             chunkFiles[i],        /* srcFile */
 96 |             &memBuf[i * bufSize], /* buffer  */
 97 |             bufSize,              /* length  */
 98 |             0                     /* index   */
 99 |         };
100 | 
101 |         rewind(buf.srcFile);
102 |         PCHECK(fread(buf.buffer, sizeof(uint64_t), bufSize, buf.srcFile) ==
103 |                bufSize)
104 |             << "Reading values from tmp chunk file #" << i << " failed";
105 |         queue.push(mergeItem{buf.buffer[0], i});
106 |         buf.index++;
107 |         buf.length--;
108 | 
109 |         inBufs.push_back(buf);
110 |     }
111 | 
112 |     // output buffer
113 |     uint64_t *outBuf = &memBuf[numChunks * bufSize];
114 |     uint64_t outLength = 0;
115 | 
116 |     while (!queue.empty()) {
117 |         // put min item in buffer
118 |         mergeItem top = queue.top();
119 |         queue.pop();
120 |         outBuf[outLength] = top.value;
121 |         outLength++;
122 | 
123 |         // flush buffer to file, if buffer is full
124 |         if (outLength == bufSize - 1) {
125 |             writea(fdOutput, (char *)outBuf, outLength * sizeof(uint64_t));
126 |             outLength = 0;
127 |         }
128 | 
129 |         // refill input buffer, from which the value was taken
130 |         // if length == 0 then the chunk was completely read
131 |         fileBuffer<uint64_t> &srcBuf = inBufs[top.srcIndex];
132 |         if (srcBuf.length > 0) {
133 |             queue.push(mergeItem{srcBuf.buffer[srcBuf.index], top.srcIndex});
134 |             srcBuf.index++;
135 |             srcBuf.length--;
136 |             if (srcBuf.length == 0 && srcBuf.srcFile) {
137 |                 DLOG(INFO) << "refilling inBuf #" << top.srcIndex;
138 | 
139 |                 srcBuf.index = 0;
140 |                 srcBuf.length = fread(srcBuf.buffer, sizeof(uint64_t), bufSize,
141 |                                       srcBuf.srcFile);
142 | 
143 |                 // Close the file stream when it is completely read
144 |                 if (srcBuf.length < bufSize) {
145 |                     fclose(srcBuf.srcFile);
146 |                     srcBuf.srcFile = NULL;
147 |                     DLOG(INFO) << "merged all data from chunk #"
148 |                                << top.srcIndex;
149 |                 }
150 |             }
151 |         }
152 |     }
153 | 
154 |     // flush rest, if output buffer is not already empty
155 |     if (outLength > 0)
156 |         writea(fdOutput, (char *)outBuf, outLength * sizeof(uint64_t));
157 | }
158 | 


--------------------------------------------------------------------------------
/src/ne_partitioner.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <string>
  4 | #include <vector>
  5 | #include <iostream>
  6 | #include <fstream>
  7 | #include <random>
  8 | #include <fcntl.h>
  9 | #include <sys/stat.h>
 10 | #include <sys/mman.h>
 11 | #include <unistd.h>
 12 | 
 13 | #include "util.hpp"
 14 | #include "min_heap.hpp"
 15 | #include "dense_bitset.hpp"
 16 | #include "edgepart.hpp"
 17 | #include "partitioner.hpp"
 18 | #include "graph.hpp"
 19 | 
 20 | /* Neighbor Expansion (NE) */
 21 | class NePartitioner : public Partitioner
 22 | {
 23 |   private:
 24 |     const double BALANCE_RATIO = 1.00;
 25 | 
 26 |     std::string basefilename;
 27 | 
 28 |     vid_t num_vertices;
 29 |     size_t num_edges, assigned_edges;
 30 |     int p, bucket;
 31 |     double average_degree;
 32 |     size_t capacity;
 33 | 
 34 |     std::vector<edge_t> edges;
 35 |     graph_t adj_out, adj_in;
 36 |     MinHeap<vid_t, vid_t> min_heap;
 37 |     std::vector<size_t> occupied;
 38 |     std::vector<vid_t> degrees;
 39 |     std::vector<int8_t> master;
 40 |     std::vector<dense_bitset> is_cores, is_boundarys;
 41 | 
 42 |     std::random_device rd;
 43 |     std::mt19937 gen;
 44 |     std::uniform_int_distribution<vid_t> dis;
 45 | 
 46 |     edgepart_writer<vid_t, uint16_t> writer;
 47 | 
 48 |     int check_edge(const edge_t *e)
 49 |     {
 50 |         rep (i, bucket) {
 51 |             auto &is_boundary = is_boundarys[i];
 52 |             if (is_boundary.get(e->first) && is_boundary.get(e->second) &&
 53 |                 occupied[i] < capacity) {
 54 |                 return i;
 55 |             }
 56 |         }
 57 | 
 58 |         rep (i, bucket) {
 59 |             auto &is_core = is_cores[i], &is_boundary = is_boundarys[i];
 60 |             if ((is_core.get(e->first) || is_core.get(e->second)) &&
 61 |                 occupied[i] < capacity) {
 62 |                 if (is_core.get(e->first) && degrees[e->second] > average_degree)
 63 |                     continue;
 64 |                 if (is_core.get(e->second) && degrees[e->first] > average_degree)
 65 |                     continue;
 66 |                 is_boundary.set_bit(e->first);
 67 |                 is_boundary.set_bit(e->second);
 68 |                 return i;
 69 |             }
 70 |         }
 71 | 
 72 |         return p;
 73 |     }
 74 | 
 75 |     void assign_edge(int bucket, vid_t from, vid_t to)
 76 |     {
 77 |         writer.save_edge(from, to, bucket);
 78 |         assigned_edges++;
 79 |         occupied[bucket]++;
 80 |         degrees[from]--;
 81 |         degrees[to]--;
 82 |     }
 83 | 
 84 |     void add_boundary(vid_t vid)
 85 |     {
 86 |         auto &is_core = is_cores[bucket], &is_boundary = is_boundarys[bucket];
 87 | 
 88 |         if (is_boundary.get(vid))
 89 |             return;
 90 |         is_boundary.set_bit_unsync(vid);
 91 | 
 92 |         if (!is_core.get(vid)) {
 93 |             min_heap.insert(adj_out[vid].size() + adj_in[vid].size(), vid);
 94 |         }
 95 | 
 96 |         rep (direction, 2) {
 97 |             adjlist_t &neighbors = direction ? adj_out[vid] : adj_in[vid];
 98 |             for (size_t i = 0; i < neighbors.size();) {
 99 |                 if (edges[neighbors[i].v].valid()) {
100 |                     vid_t &u = direction ? edges[neighbors[i].v].second : edges[neighbors[i].v].first;
101 |                     if (is_core.get(u)) {
102 |                         assign_edge(bucket, direction ? vid : u,
103 |                                     direction ? u : vid);
104 |                         min_heap.decrease_key(vid);
105 |                         edges[neighbors[i].v].remove();
106 |                         std::swap(neighbors[i], neighbors.back());
107 |                         neighbors.pop_back();
108 |                     } else if (is_boundary.get(u) &&
109 |                                occupied[bucket] < capacity) {
110 |                         assign_edge(bucket, direction ? vid : u,
111 |                                     direction ? u : vid);
112 |                         min_heap.decrease_key(vid);
113 |                         min_heap.decrease_key(u);
114 |                         edges[neighbors[i].v].remove();
115 |                         std::swap(neighbors[i], neighbors.back());
116 |                         neighbors.pop_back();
117 |                     } else
118 |                         i++;
119 |                 } else {
120 |                     std::swap(neighbors[i], neighbors.back());
121 |                     neighbors.pop_back();
122 |                 }
123 |             }
124 |         }
125 |     }
126 | 
127 |     void occupy_vertex(vid_t vid, vid_t d)
128 |     {
129 |         CHECK(!is_cores[bucket].get(vid)) << "add " << vid << " to core again";
130 |         is_cores[bucket].set_bit_unsync(vid);
131 | 
132 |         if (d == 0)
133 |             return;
134 | 
135 |         add_boundary(vid);
136 | 
137 |         for (auto &i : adj_out[vid])
138 |             if (edges[i.v].valid())
139 |                 add_boundary(edges[i.v].second);
140 |         adj_out[vid].clear();
141 | 
142 |         for (auto &i : adj_in[vid])
143 |             if (edges[i.v].valid())
144 |                 add_boundary(edges[i.v].first);
145 |         adj_in[vid].clear();
146 |     }
147 | 
148 |     bool get_free_vertex(vid_t &vid)
149 |     {
150 |         vid = dis(gen);
151 |         vid_t count = 0;
152 |         while (count < num_vertices &&
153 |                (adj_out[vid].size() + adj_in[vid].size() == 0 ||
154 |                 adj_out[vid].size() + adj_in[vid].size() >
155 |                     2 * average_degree ||
156 |                 is_cores[bucket].get(vid))) {
157 |             vid = (vid + ++count) % num_vertices;
158 |         }
159 |         if (count == num_vertices)
160 |             return false;
161 |         return true;
162 |     }
163 | 
164 |     void assign_remaining();
165 |     void assign_master();
166 |     size_t count_mirrors();
167 | 
168 |   public:
169 |     NePartitioner(std::string basefilename);
170 |     void split();
171 | };
172 | 


--------------------------------------------------------------------------------
/src/sne_partitioner.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <string>
  4 | #include <vector>
  5 | #include <iostream>
  6 | #include <fstream>
  7 | #include <random>
  8 | #include <fcntl.h>
  9 | #include <sys/stat.h>
 10 | #include <sys/mman.h>
 11 | #include <unistd.h>
 12 | 
 13 | #include "util.hpp"
 14 | #include "min_heap.hpp"
 15 | #include "dense_bitset.hpp"
 16 | #include "edgepart.hpp"
 17 | #include "partitioner.hpp"
 18 | #include "graph.hpp"
 19 | 
 20 | /* Streaming Neighbor Expansion (SNE) */
 21 | class SnePartitioner : public Partitioner
 22 | {
 23 |   private:
 24 |     const double BALANCE_RATIO = 1.05;
 25 |     size_t BUFFER_SIZE;
 26 | 
 27 |     std::string basefilename;
 28 | 
 29 |     vid_t num_vertices;
 30 |     size_t num_edges, assigned_edges;
 31 |     int p, bucket;
 32 |     double average_degree, local_average_degree;
 33 |     size_t max_sample_size;
 34 |     size_t capacity, local_capacity;
 35 | 
 36 |     // use mmap for file input
 37 |     int fin;
 38 |     off_t filesize;
 39 |     char *fin_map, *fin_ptr, *fin_end;
 40 | 
 41 |     std::vector<edge_t> buffer;
 42 |     std::vector<edge_t> sample_edges;
 43 |     graph_t adj_out, adj_in;
 44 |     MinHeap<vid_t, vid_t> min_heap;
 45 |     std::vector<size_t> occupied;
 46 |     std::vector<vid_t> degrees;
 47 |     std::vector<int8_t> master;
 48 |     std::vector<dense_bitset> is_cores, is_boundarys;
 49 |     std::vector<int8_t> results;
 50 | 
 51 |     std::random_device rd;
 52 |     std::mt19937 gen;
 53 |     std::uniform_int_distribution<vid_t> dis;
 54 | 
 55 |     edgepart_writer<vid_t, uint16_t> writer;
 56 | 
 57 |     int check_edge(const edge_t *e)
 58 |     {
 59 |         rep (i, bucket) {
 60 |             auto &is_boundary = is_boundarys[i];
 61 |             if (is_boundary.get(e->first) && is_boundary.get(e->second) &&
 62 |                 occupied[i] < capacity) {
 63 |                 return i;
 64 |             }
 65 |         }
 66 | 
 67 |         rep (i, bucket) {
 68 |             auto &is_core = is_cores[i], &is_boundary = is_boundarys[i];
 69 |             if ((is_core.get(e->first) || is_core.get(e->second)) &&
 70 |                 occupied[i] < capacity) {
 71 |                 if (is_core.get(e->first) && degrees[e->second] > average_degree)
 72 |                     continue;
 73 |                 if (is_core.get(e->second) && degrees[e->first] > average_degree)
 74 |                     continue;
 75 |                 is_boundary.set_bit(e->first);
 76 |                 is_boundary.set_bit(e->second);
 77 |                 return i;
 78 |             }
 79 |         }
 80 | 
 81 |         return p;
 82 |     }
 83 | 
 84 |     void assign_edge(int bucket, vid_t from, vid_t to)
 85 |     {
 86 |         writer.save_edge(from, to, bucket);
 87 |         assigned_edges++;
 88 |         occupied[bucket]++;
 89 |         degrees[from]--;
 90 |         degrees[to]--;
 91 |     }
 92 | 
 93 |     void add_boundary(vid_t vid)
 94 |     {
 95 |         auto &is_core = is_cores[bucket], &is_boundary = is_boundarys[bucket];
 96 | 
 97 |         if (is_boundary.get(vid))
 98 |             return;
 99 |         is_boundary.set_bit_unsync(vid);
100 | 
101 |         if (!is_core.get(vid)) {
102 |             min_heap.insert(adj_out[vid].size() + adj_in[vid].size(), vid);
103 |         }
104 | 
105 |         rep (direction, 2) {
106 |             adjlist_t &neighbors = direction ? adj_out[vid] : adj_in[vid];
107 |             for (size_t i = 0; i < neighbors.size();) {
108 |                 if (sample_edges[neighbors[i].v].valid()) {
109 |                     vid_t &u = direction ? sample_edges[neighbors[i].v].second : sample_edges[neighbors[i].v].first;
110 |                     if (is_core.get(u)) {
111 |                         assign_edge(bucket, direction ? vid : u,
112 |                                     direction ? u : vid);
113 |                         min_heap.decrease_key(vid);
114 |                         sample_edges[neighbors[i].v].remove();
115 |                         std::swap(neighbors[i], neighbors.back());
116 |                         neighbors.pop_back();
117 |                     } else if (is_boundary.get(u) &&
118 |                                occupied[bucket] < capacity) {
119 |                         assign_edge(bucket, direction ? vid : u,
120 |                                     direction ? u : vid);
121 |                         min_heap.decrease_key(vid);
122 |                         min_heap.decrease_key(u);
123 |                         sample_edges[neighbors[i].v].remove();
124 |                         std::swap(neighbors[i], neighbors.back());
125 |                         neighbors.pop_back();
126 |                     } else
127 |                         i++;
128 |                 } else {
129 |                     std::swap(neighbors[i], neighbors.back());
130 |                     neighbors.pop_back();
131 |                 }
132 |             }
133 |         }
134 |     }
135 | 
136 |     void occupy_vertex(vid_t vid, vid_t d)
137 |     {
138 |         CHECK(!is_cores[bucket].get(vid)) << "add " << vid << " to core again";
139 |         is_cores[bucket].set_bit_unsync(vid);
140 | 
141 |         if (d == 0)
142 |             return;
143 | 
144 |         add_boundary(vid);
145 | 
146 |         for (auto &i : adj_out[vid])
147 |             if (sample_edges[i.v].valid())
148 |                 add_boundary(sample_edges[i.v].second);
149 |         adj_out[vid].clear();
150 | 
151 |         for (auto &i : adj_in[vid])
152 |             if (sample_edges[i.v].valid())
153 |                 add_boundary(sample_edges[i.v].first);
154 |         adj_in[vid].clear();
155 |     }
156 | 
157 |     bool get_free_vertex(vid_t &vid)
158 |     {
159 |         vid = dis(gen);
160 |         vid_t count = 0;
161 |         while (count < num_vertices &&
162 |                (adj_out[vid].size() + adj_in[vid].size() == 0 ||
163 |                 adj_out[vid].size() + adj_in[vid].size() >
164 |                     2 * local_average_degree ||
165 |                 is_cores[bucket].get(vid))) {
166 |             vid = (vid + ++count) % num_vertices;
167 |         }
168 |         if (count == num_vertices)
169 |             return false;
170 |         return true;
171 |     }
172 | 
173 |     void read_more();
174 |     void read_remaining();
175 |     void clean_samples();
176 |     void assign_master();
177 |     size_t count_mirrors();
178 | 
179 |   public:
180 |     SnePartitioner(std::string basefilename);
181 |     void split();
182 | };
183 | 


--------------------------------------------------------------------------------
/.ycm_extra_conf.py:
--------------------------------------------------------------------------------
  1 | # This file is NOT licensed under the GPLv3, which is the license for the rest
  2 | # of YouCompleteMe.
  3 | #
  4 | # Here's the license text for this file:
  5 | #
  6 | # This is free and unencumbered software released into the public domain.
  7 | #
  8 | # Anyone is free to copy, modify, publish, use, compile, sell, or
  9 | # distribute this software, either in source code form or as a compiled
 10 | # binary, for any purpose, commercial or non-commercial, and by any
 11 | # means.
 12 | #
 13 | # In jurisdictions that recognize copyright laws, the author or authors
 14 | # of this software dedicate any and all copyright interest in the
 15 | # software to the public domain. We make this dedication for the benefit
 16 | # of the public at large and to the detriment of our heirs and
 17 | # successors. We intend this dedication to be an overt act of
 18 | # relinquishment in perpetuity of all present and future rights to this
 19 | # software under copyright law.
 20 | #
 21 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 22 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 23 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 24 | # IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 25 | # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 26 | # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 27 | # OTHER DEALINGS IN THE SOFTWARE.
 28 | #
 29 | # For more information, please refer to <http://unlicense.org/>
 30 | 
 31 | import os
 32 | import ycm_core
 33 | 
 34 | # These are the compilation flags that will be used in case there's no
 35 | # compilation database set (by default, one is not set).
 36 | # CHANGE THIS LIST OF FLAGS. YES, THIS IS THE DROID YOU HAVE BEEN LOOKING FOR.
 37 | flags = [
 38 | '-Wall',
 39 | # THIS IS IMPORTANT! Without a "-std=<something>" flag, clang won't know which
 40 | # language to use when compiling headers. So it will guess. Badly. So C++
 41 | # headers will be compiled as C headers. You don't want that so ALWAYS specify
 42 | # a "-std=<something>".
 43 | # For a C project, you would set this to something like 'c99' instead of
 44 | # 'c++11'.
 45 | '-std=c++11',
 46 | # ...and the same thing goes for the magic -x option which specifies the
 47 | # language that the files to be compiled are written in. This is mostly
 48 | # relevant for c++ headers.
 49 | # For a C project, you would set this to 'c' instead of 'c++'.
 50 | '-x',
 51 | 'c++',
 52 | '-I',
 53 | '.',
 54 | '-I',
 55 | './threadpool11/include',
 56 | '-isystem',
 57 | '/usr/include',
 58 | ]
 59 | 
 60 | 
 61 | # Set this to the absolute path to the folder (NOT the file!) containing the
 62 | # compile_commands.json file to use that instead of 'flags'. See here for
 63 | # more details: http://clang.llvm.org/docs/JSONCompilationDatabase.html
 64 | #
 65 | # You can get CMake to generate this file for you by adding:
 66 | #   set( CMAKE_EXPORT_COMPILE_COMMANDS 1 )
 67 | # to your CMakeLists.txt file.
 68 | #
 69 | # Most projects will NOT need to set this to anything; you can just change the
 70 | # 'flags' list of compilation flags. Notice that YCM itself uses that approach.
 71 | compilation_database_folder = ''
 72 | 
 73 | if os.path.exists( compilation_database_folder ):
 74 |   database = ycm_core.CompilationDatabase( compilation_database_folder )
 75 | else:
 76 |   database = None
 77 | 
 78 | SOURCE_EXTENSIONS = [ '.cpp', '.cxx', '.cc', '.c', '.m', '.mm' ]
 79 | 
 80 | def DirectoryOfThisScript():
 81 |   return os.path.dirname( os.path.abspath( __file__ ) )
 82 | 
 83 | 
 84 | def MakeRelativePathsInFlagsAbsolute( flags, working_directory ):
 85 |   if not working_directory:
 86 |     return list( flags )
 87 |   new_flags = []
 88 |   make_next_absolute = False
 89 |   path_flags = [ '-isystem', '-I', '-iquote', '--sysroot=' ]
 90 |   for flag in flags:
 91 |     new_flag = flag
 92 | 
 93 |     if make_next_absolute:
 94 |       make_next_absolute = False
 95 |       if not flag.startswith( '/' ):
 96 |         new_flag = os.path.join( working_directory, flag )
 97 | 
 98 |     for path_flag in path_flags:
 99 |       if flag == path_flag:
100 |         make_next_absolute = True
101 |         break
102 | 
103 |       if flag.startswith( path_flag ):
104 |         path = flag[ len( path_flag ): ]
105 |         new_flag = path_flag + os.path.join( working_directory, path )
106 |         break
107 | 
108 |     if new_flag:
109 |       new_flags.append( new_flag )
110 |   return new_flags
111 | 
112 | 
113 | def IsHeaderFile( filename ):
114 |   extension = os.path.splitext( filename )[ 1 ]
115 |   return extension in [ '.h', '.hxx', '.hpp', '.hh' ]
116 | 
117 | 
118 | def GetCompilationInfoForFile( filename ):
119 |   # The compilation_commands.json file generated by CMake does not have entries
120 |   # for header files. So we do our best by asking the db for flags for a
121 |   # corresponding source file, if any. If one exists, the flags for that file
122 |   # should be good enough.
123 |   if IsHeaderFile( filename ):
124 |     basename = os.path.splitext( filename )[ 0 ]
125 |     for extension in SOURCE_EXTENSIONS:
126 |       replacement_file = basename + extension
127 |       if os.path.exists( replacement_file ):
128 |         compilation_info = database.GetCompilationInfoForFile(
129 |           replacement_file )
130 |         if compilation_info.compiler_flags_:
131 |           return compilation_info
132 |     return None
133 |   return database.GetCompilationInfoForFile( filename )
134 | 
135 | 
136 | def FlagsForFile( filename, **kwargs ):
137 |   if database:
138 |     # Bear in mind that compilation_info.compiler_flags_ does NOT return a
139 |     # python list, but a "list-like" StringVec object
140 |     compilation_info = GetCompilationInfoForFile( filename )
141 |     if not compilation_info:
142 |       return None
143 | 
144 |     final_flags = MakeRelativePathsInFlagsAbsolute(
145 |       compilation_info.compiler_flags_,
146 |       compilation_info.compiler_working_dir_ )
147 | 
148 |     # NOTE: This is just for YouCompleteMe; it's highly likely that your project
149 |     # does NOT need to remove the stdlib flag. DO NOT USE THIS IN YOUR
150 |     # ycm_extra_conf IF YOU'RE NOT 100% SURE YOU NEED IT.
151 |     try:
152 |       final_flags.remove( '-stdlib=libc++' )
153 |     except ValueError:
154 |       pass
155 |   else:
156 |     relative_to = DirectoryOfThisScript()
157 |     final_flags = MakeRelativePathsInFlagsAbsolute( flags, relative_to )
158 | 
159 |   return {
160 |     'flags': final_flags,
161 |     'do_cache': True
162 |   }
163 | 


--------------------------------------------------------------------------------
/src/ne_partitioner.cpp:
--------------------------------------------------------------------------------
  1 | #include "ne_partitioner.hpp"
  2 | #include "conversions.hpp"
  3 | 
  4 | NePartitioner::NePartitioner(std::string basefilename)
  5 |     : basefilename(basefilename), rd(), gen(rd()), writer(basefilename)
  6 | {
  7 |     Timer convert_timer;
  8 |     convert_timer.start();
  9 |     Converter *converter = new Converter(basefilename);
 10 |     convert(basefilename, converter);
 11 |     delete converter;
 12 |     convert_timer.stop();
 13 |     LOG(INFO) << "convert time: " << convert_timer.get_time();
 14 | 
 15 |     total_time.start();
 16 |     LOG(INFO) << "initializing partitioner";
 17 | 
 18 |     std::ifstream fin(binedgelist_name(basefilename),
 19 |                       std::ios::binary | std::ios::ate);
 20 |     auto filesize = fin.tellg();
 21 |     LOG(INFO) << "file size: " << filesize;
 22 |     fin.seekg(0, std::ios::beg);
 23 | 
 24 |     fin.read((char *)&num_vertices, sizeof(num_vertices));
 25 |     fin.read((char *)&num_edges, sizeof(num_edges));
 26 | 
 27 |     LOG(INFO) << "num_vertices: " << num_vertices
 28 |               << ", num_edges: " << num_edges;
 29 |     CHECK_EQ(sizeof(vid_t) + sizeof(size_t) + num_edges * sizeof(edge_t), filesize);
 30 | 
 31 |     p = FLAGS_p;
 32 |     average_degree = (double)num_edges * 2 / num_vertices;
 33 |     assigned_edges = 0;
 34 |     capacity = (double)num_edges * BALANCE_RATIO / p + 1;
 35 |     occupied.assign(p, 0);
 36 |     adj_out.resize(num_vertices);
 37 |     adj_in.resize(num_vertices);
 38 |     is_cores.assign(p, dense_bitset(num_vertices));
 39 |     is_boundarys.assign(p, dense_bitset(num_vertices));
 40 |     master.assign(num_vertices, -1);
 41 |     dis.param(std::uniform_int_distribution<vid_t>::param_type(0, num_vertices - 1));
 42 | 
 43 |     Timer read_timer;
 44 |     read_timer.start();
 45 |     LOG(INFO) << "loading...";
 46 |     edges.resize(num_edges);
 47 |     fin.read((char *)&edges[0], sizeof(edge_t) * num_edges);
 48 | 
 49 |     LOG(INFO) << "constructing...";
 50 |     adj_out.build(edges);
 51 |     adj_in.build_reverse(edges);
 52 | 
 53 |     degrees.resize(num_vertices);
 54 |     std::ifstream degree_file(degree_name(basefilename), std::ios::binary);
 55 |     degree_file.read((char *)&degrees[0], num_vertices * sizeof(vid_t));
 56 |     degree_file.close();
 57 |     read_timer.stop();
 58 |     LOG(INFO) << "time used for graph input and construction: " << read_timer.get_time();
 59 | };
 60 | 
 61 | void NePartitioner::assign_remaining()
 62 | {
 63 |     auto &is_boundary = is_boundarys[p - 1], &is_core = is_cores[p - 1];
 64 |     repv (u, num_vertices)
 65 |         for (auto &i : adj_out[u])
 66 |             if (edges[i.v].valid()) {
 67 |                 assign_edge(p - 1, u, edges[i.v].second);
 68 |                 is_boundary.set_bit_unsync(u);
 69 |                 is_boundary.set_bit_unsync(edges[i.v].second);
 70 |             }
 71 | 
 72 |     repv (i, num_vertices) {
 73 |         if (is_boundary.get(i)) {
 74 |             is_core.set_bit_unsync(i);
 75 |             rep (j, p - 1)
 76 |                 if (is_cores[j].get(i)) {
 77 |                     is_core.set_unsync(i, false);
 78 |                     break;
 79 |                 }
 80 |         }
 81 |     }
 82 | }
 83 | 
 84 | void NePartitioner::assign_master()
 85 | {
 86 |     std::vector<vid_t> count_master(p, 0);
 87 |     std::vector<vid_t> quota(p, num_vertices);
 88 |     long long sum = p * num_vertices;
 89 |     std::uniform_real_distribution<double> distribution(0.0, 1.0);
 90 |     std::vector<dense_bitset::iterator> pos(p);
 91 |     rep (b, p)
 92 |         pos[b] = is_boundarys[b].begin();
 93 |     vid_t count = 0;
 94 |     while (count < num_vertices) {
 95 |         long long r = distribution(gen) * sum;
 96 |         int k;
 97 |         for (k = 0; k < p; k++) {
 98 |             if (r < quota[k])
 99 |                 break;
100 |             r -= quota[k];
101 |         }
102 |         while (pos[k] != is_boundarys[k].end() && master[*pos[k]] != -1)
103 |             pos[k]++;
104 |         if (pos[k] != is_boundarys[k].end()) {
105 |             count++;
106 |             master[*pos[k]] = k;
107 |             writer.save_vertex(*pos[k], k);
108 |             count_master[k]++;
109 |             quota[k]--;
110 |             sum--;
111 |         }
112 |     }
113 |     int max_masters =
114 |         *std::max_element(count_master.begin(), count_master.end());
115 |     LOG(INFO) << "master balance: "
116 |               << (double)max_masters / ((double)num_vertices / p);
117 | }
118 | 
119 | size_t NePartitioner::count_mirrors()
120 | {
121 |     size_t result = 0;
122 |     rep (i, p)
123 |         result += is_boundarys[i].popcount();
124 |     return result;
125 | }
126 | 
127 | void NePartitioner::split()
128 | {
129 |     LOG(INFO) << "partition `" << basefilename << "'";
130 |     LOG(INFO) << "number of partitions: " << p;
131 | 
132 |     Timer compute_timer;
133 | 
134 |     min_heap.reserve(num_vertices);
135 | 
136 |     LOG(INFO) << "partitioning...";
137 |     compute_timer.start();
138 |     for (bucket = 0; bucket < p - 1; bucket++) {
139 |         std::cerr << bucket << ", ";
140 |         DLOG(INFO) << "sample size: " << adj_out.num_edges();
141 |         while (occupied[bucket] < capacity) {
142 |             vid_t d, vid;
143 |             if (!min_heap.get_min(d, vid)) {
144 |                 if (!get_free_vertex(vid)) {
145 |                     DLOG(INFO) << "partition " << bucket
146 |                                << " stop: no free vertices";
147 |                     break;
148 |                 }
149 |                 d = adj_out[vid].size() + adj_in[vid].size();
150 |             } else {
151 |                 min_heap.remove(vid);
152 |                 /* CHECK_EQ(d, adj_out[vid].size() + adj_in[vid].size()); */
153 |             }
154 | 
155 |             occupy_vertex(vid, d);
156 |         }
157 |         min_heap.clear();
158 |         rep (direction, 2)
159 |             repv (vid, num_vertices) {
160 |                 adjlist_t &neighbors = direction ? adj_out[vid] : adj_in[vid];
161 |                 for (size_t i = 0; i < neighbors.size();) {
162 |                     if (edges[neighbors[i].v].valid()) {
163 |                         i++;
164 |                     } else {
165 |                         std::swap(neighbors[i], neighbors.back());
166 |                         neighbors.pop_back();
167 |                     }
168 |                 }
169 |             }
170 |     }
171 |     bucket = p - 1;
172 |     std::cerr << bucket << std::endl;
173 |     assign_remaining();
174 |     assign_master();
175 |     compute_timer.stop();
176 |     LOG(INFO) << "expected edges in each partition: " << num_edges / p;
177 |     rep (i, p)
178 |         DLOG(INFO) << "edges in partition " << i << ": " << occupied[i];
179 |     size_t max_occupied = *std::max_element(occupied.begin(), occupied.end());
180 |     LOG(INFO) << "balance: " << (double)max_occupied / ((double)num_edges / p);
181 |     size_t total_mirrors = count_mirrors();
182 |     LOG(INFO) << "total mirrors: " << total_mirrors;
183 |     LOG(INFO) << "replication factor: " << (double)total_mirrors / num_vertices;
184 |     LOG(INFO) << "time used for partitioning: " << compute_timer.get_time();
185 | 
186 |     CHECK_EQ(assigned_edges, num_edges);
187 | 
188 |     total_time.stop();
189 |     LOG(INFO) << "total partition time: " << total_time.get_time();
190 | }
191 | 


--------------------------------------------------------------------------------
/threadpool11/include/threadpool11/pool.hpp:
--------------------------------------------------------------------------------
  1 | /*!
  2 | Copyright (c) 2013, 2014, 2015 Tolga HOŞGÖR
  3 | All rights reserved.
  4 | 
  5 | This file is part of threadpool11.
  6 | 
  7 |     threadpool11 is free software: you can redistribute it and/or modify
  8 |     it under the terms of the GNU General Public License as published by
  9 |     the Free Software Foundation, either version 3 of the License, or
 10 |     (at your option) any later version.
 11 | 
 12 |     threadpool11 is distributed in the hope that it will be useful,
 13 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
 14 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 15 |     GNU General Public License for more details.
 16 | 
 17 |     You should have received a copy of the GNU General Public License
 18 |     along with threadpool11.  If not, see <http://www.gnu.org/licenses/>.
 19 | */
 20 | 
 21 | #pragma once
 22 | 
 23 | #include "worker.hpp"
 24 | #include "utility.hpp"
 25 | #include "concurrentqueue.h"
 26 | 
 27 | #include <atomic>
 28 | #include <cassert>
 29 | #include <condition_variable>
 30 | #include <future>
 31 | #include <memory>
 32 | #include <mutex>
 33 | 
 34 | #if defined(WIN32) && defined(threadpool11_DLL)
 35 | #ifdef threadpool11_EXPORTING
 36 | #define threadpool11_EXPORT __declspec(dllexport)
 37 | #else
 38 | #define threadpool11_EXPORT __declspec(dllimport)
 39 | #endif
 40 | #else
 41 | #define threadpool11_EXPORT
 42 | #endif
 43 | 
 44 | namespace threadpool11 {
 45 | 
 46 | class Pool {
 47 |   friend class Worker;
 48 | 
 49 | public:
 50 |   enum class Method { SYNC, ASYNC };
 51 | 
 52 | public:
 53 |   threadpool11_EXPORT Pool(std::size_t worker_count = std::thread::hardware_concurrency());
 54 |   ~Pool();
 55 | 
 56 |   /*!
 57 |    * \brief postWork Posts a work to the pool for getting processed.
 58 |    *
 59 |    * If there are no threads left (i.e. you called Pool::joinAll(); prior to
 60 |    * this function) all the works you post gets enqueued. If you spawn new threads in
 61 |    * future, they will be executed then.
 62 |    *
 63 |    * Properties: thread-safe.
 64 |    */
 65 |   template <typename T>
 66 |   threadpool11_EXPORT std::future<T> postWork(std::function<T()> callable, Work::Type type = Work::Type::STD);
 67 |   // TODO: convert 'type' above to const& when MSVC fixes that bug.
 68 | 
 69 |   /**
 70 |    * \brief waitAll Blocks the calling thread until all posted works are finished.
 71 |    *
 72 |    * This function suspends the calling thread until all posted works are finished and, therefore, all worker
 73 |    * threads are free. It guarantees you that before the function returns, all queued works are finished.
 74 |    */
 75 |   threadpool11_EXPORT void waitAll();
 76 | 
 77 |   /*!
 78 |    * \brief joinAll Joins the worker threads.
 79 |    *
 80 |    * This function joins all the threads in the thread pool as fast as possible.
 81 |    * All the posted works are NOT GUARANTEED to be finished before the worker threads
 82 |    * are destroyed and this function returns.
 83 |    *
 84 |    * However, ongoing works in the threads in the pool are guaranteed
 85 |    * to finish before that threads are terminated.
 86 |    *
 87 |    * Properties: NOT thread-safe.
 88 |    */
 89 |   threadpool11_EXPORT void joinAll();
 90 | 
 91 |   /*!
 92 |    * \brief getWorkerCount
 93 |    *
 94 |    * Properties: thread-safe.
 95 |    *
 96 |    * \return The number of worker threads.
 97 |    */
 98 |   threadpool11_EXPORT size_t getWorkerCount() const;
 99 | 
100 |   /*!
101 |    * \brief setWorkerCount
102 |    * \param n
103 |    *
104 |    * WARNING: This function behaves different based on second parameter. (Only if decreasing)
105 |    *
106 |    * Method::ASYNC: It will return before the threads are joined. It will just post
107 |    *  'n' requests for termination. This means that if you call this function multiple times,
108 |    *  worker termination requests will pile up. It can even kill the newly
109 |    *  created workers if all workers are removed before all requests are processed.
110 |    *
111 |    * Method::SYNC: It won't return until the specified number of workers are actually destroyed.
112 |    *  There still may be a few milliseconds delay before value returned by Pool::getWorkerCount is updated.
113 |    *  But it will be more accurate compared to ASYNC one.
114 |    */
115 |   threadpool11_EXPORT void setWorkerCount(std::size_t n, Method method = Method::ASYNC);
116 | 
117 |   /*!
118 |    * \brief getWorkQueueSize
119 |    *
120 |    * Properties: thread-safe.
121 |    *
122 |    * \return The number of work items that has not been acquired by workers.
123 |    */
124 |   threadpool11_EXPORT size_t getWorkQueueSize() const;
125 | 
126 |   /*!
127 |     * \brief getActiveWorkerCount Gets the number of active workers when the function is called.
128 |     *
129 |     * The information this function returns does not really mean much. The worker may be starting to execute a work from queue,
130 |     * it may be executing a work or it may have just executed a work.
131 |     *
132 |     * \return The number of active workers.
133 |     */
134 |   threadpool11_EXPORT size_t getActiveWorkerCount() const;
135 | 
136 |   /**
137 |    * \brief getInactiveWorkerCount Gets the number of the inactive worker count.
138 |    *
139 |     * The information this function returns does not really mean much. The worker may be starting to execute a work from queue,
140 |     * it may be executing a work or it may have just executed a work.
141 |     *
142 |     * \return The number of active workers.
143 |    */
144 |   threadpool11_EXPORT size_t getInactiveWorkerCount() const;
145 | 
146 |   /*!
147 |    * \brief incWorkerCountBy Increases the number of threads in the pool by n.
148 |    *
149 |    * Properties: thread-safe.
150 |    */
151 |   threadpool11_EXPORT void incWorkerCountBy(std::size_t n);
152 | 
153 |   /*!
154 |    * \brief decWorkerCountBy Tries to decrease the number of threads in the pool by n.
155 |    *
156 |    * Setting 'n' higher than the number of workers has no effect.
157 |    * Calling without arguments asynchronously terminates all workers.
158 |    *
159 |    * \warning This function behaves different based on second parameter.
160 |    *
161 |    * Method::ASYNC: It will return before the threads are joined. It will just post
162 |    *  'n' requests for termination. This means that if you call this function multiple times,
163 |    *  worker termination requests will pile up. It can even kill the newly
164 |    *  created workers if all workers are removed before all requests are processed.
165 |    *
166 |    * Method::SYNC: It won't return until the specified number of workers are actually destroyed.
167 |    *  There still may be a few milliseconds delay before value returned by Pool::getWorkerCount is updated.
168 |    *  But it will be more accurate compared to ASYNC one.
169 |    *
170 |    * Properties: thread-safe.
171 |    */
172 |   threadpool11_EXPORT void decWorkerCountBy(std::size_t n = std::numeric_limits<size_t>::max(),
173 |                                             Method method = Method::ASYNC);
174 | 
175 | private:
176 |   Pool(Pool&&) = delete;
177 |   Pool(Pool const&) = delete;
178 |   Pool& operator=(Pool&&) = delete;
179 |   Pool& operator=(Pool const&) = delete;
180 | 
181 |   void spawnWorkers(std::size_t n);
182 | 
183 |   /*!
184 |    * \brief executor
185 |    * This is run by different threads to do necessary operations for queue processing.
186 |    */
187 |   void executor(std::unique_ptr<std::thread> self);
188 | 
189 |   /**
190 |    * @brief push Internal usage.
191 |    * @param workFunc
192 |    */
193 |   void push(Work::Callable* workFunc);
194 | 
195 | private:
196 |   std::atomic<size_t> worker_count_;
197 |   std::atomic<size_t> active_worker_count_;
198 | 
199 |   mutable std::mutex notify_all_finished_mutex_;
200 |   std::condition_variable notify_all_finished_signal_;
201 |   std::atomic<bool> are_all_really_finished_;
202 | 
203 |   mutable std::mutex work_signal_mutex_;
204 |   // bool work_signal_prot; //! wake up protection // <- work_queue_size is used instead of this
205 |   std::condition_variable work_signal_;
206 | 
207 |   std::unique_ptr<moodycamel::ConcurrentQueue<Work::Callable*>> work_queue_;
208 |   std::atomic<size_t> work_queue_size_;
209 | };
210 | 
211 | template <typename T>
212 | threadpool11_EXPORT inline std::future<T> Pool::postWork(std::function<T()> callable, Work::Type type) {
213 |   std::promise<T> promise;
214 |   auto future = promise.get_future();
215 | 
216 |   /* TODO: how to avoid copy of callable into this lambda and the ones below? In a decent way... */
217 |   /* evil move hack */
218 |   auto move_callable = make_move_on_copy(std::move(callable));
219 |   /* evil move hack */
220 |   auto move_promise = make_move_on_copy(std::move(promise));
221 | 
222 |   auto workFunc = new Work::Callable([move_callable, move_promise, type]() mutable {
223 |     move_promise.value().set_value((move_callable.value())());
224 |     return type;
225 |   });
226 | 
227 |   push(workFunc);
228 | 
229 |   return future;
230 | }
231 | 
232 | template <>
233 | threadpool11_EXPORT inline std::future<void> Pool::postWork(std::function<void()> callable,
234 |                                                             Work::Type const type) {
235 |   std::promise<void> promise;
236 |   auto future = promise.get_future();
237 | 
238 |   /* evil move hack */
239 |   auto move_callable = make_move_on_copy(std::move(callable));
240 |   /* evil move hack */
241 |   auto move_promise = make_move_on_copy(std::move(promise));
242 | 
243 |   auto workFunc = new Work::Callable([move_callable, move_promise, type]() mutable {
244 |     (move_callable.value())();
245 |     move_promise.value().set_value();
246 |     return type;
247 |   });
248 | 
249 |   push(workFunc);
250 | 
251 |   return future;
252 | }
253 | 
254 | #undef threadpool11_EXPORT
255 | #undef threadpool11_EXPORTING
256 | }
257 | 


--------------------------------------------------------------------------------
/src/sne_partitioner.cpp:
--------------------------------------------------------------------------------
  1 | #include "sne_partitioner.hpp"
  2 | #include "conversions.hpp"
  3 | #include "shuffler.hpp"
  4 | 
  5 | SnePartitioner::SnePartitioner(std::string basefilename)
  6 |     : basefilename(basefilename), rd(), gen(rd()), writer(basefilename)
  7 | {
  8 |     Timer shuffle_timer;
  9 |     shuffle_timer.start();
 10 |     convert(basefilename, new Shuffler(basefilename));
 11 |     shuffle_timer.stop();
 12 |     LOG(INFO) << "shuffle time: " << shuffle_timer.get_time();
 13 | 
 14 |     total_time.start();
 15 |     LOG(INFO) << "initializing partitioner";
 16 | 
 17 |     fin = open(shuffled_binedgelist_name(basefilename).c_str(), O_RDONLY, (mode_t)0600);
 18 |     PCHECK(fin != -1) << "Error opening file for read";
 19 |     struct stat fileInfo = {0};
 20 |     PCHECK(fstat(fin, &fileInfo) != -1) << "Error getting the file size";
 21 |     PCHECK(fileInfo.st_size != 0) << "Error: file is empty";
 22 |     LOG(INFO) << "file size: " << fileInfo.st_size;
 23 | 
 24 |     fin_map = (char *)mmap(0, fileInfo.st_size, PROT_READ, MAP_SHARED, fin, 0);
 25 |     if (fin_map == MAP_FAILED) {
 26 |         close(fin);
 27 |         PLOG(FATAL) << "error mapping the file";
 28 |     }
 29 | 
 30 |     filesize = fileInfo.st_size;
 31 |     fin_ptr = fin_map;
 32 |     fin_end = fin_map + filesize;
 33 | 
 34 |     num_vertices = *(vid_t *)fin_ptr;
 35 |     fin_ptr += sizeof(vid_t);
 36 |     num_edges = *(size_t *)fin_ptr;
 37 |     fin_ptr += sizeof(size_t);
 38 | 
 39 |     LOG(INFO) << "num_vertices: " << num_vertices
 40 |               << ", num_edges: " << num_edges;
 41 |     CHECK_EQ(sizeof(vid_t) + sizeof(size_t) + num_edges * sizeof(edge_t), filesize);
 42 | 
 43 |     p = FLAGS_p;
 44 |     average_degree = (double)num_edges * 2 / num_vertices;
 45 |     assigned_edges = 0;
 46 |     LOG(INFO) << "inmem: " << FLAGS_inmem;
 47 |     if (!FLAGS_inmem) {
 48 |         LOG(INFO) << "sample_ratio: " << FLAGS_sample_ratio;
 49 |         max_sample_size = num_vertices * FLAGS_sample_ratio;
 50 |     } else
 51 |         max_sample_size = num_edges;
 52 |     local_average_degree = 2 * (double)max_sample_size / num_vertices;
 53 |     capacity = (double)num_edges * BALANCE_RATIO / p + 1;
 54 |     BUFFER_SIZE = std::min(64 * 1024 / sizeof(edge_t),
 55 |                            std::max((size_t)1, (size_t)(num_edges * 0.05 / p + 1)));
 56 |     LOG(INFO) << "buffer size: " << BUFFER_SIZE;
 57 |     occupied.assign(p, 0);
 58 |     adj_out.resize(num_vertices);
 59 |     adj_in.resize(num_vertices);
 60 |     is_cores.assign(p, dense_bitset(num_vertices));
 61 |     is_boundarys.assign(p, dense_bitset(num_vertices));
 62 |     master.assign(num_vertices, -1);
 63 |     dis.param(std::uniform_int_distribution<vid_t>::param_type(0, num_vertices - 1));
 64 | 
 65 |     degrees.resize(num_vertices);
 66 |     std::ifstream degree_file(degree_name(basefilename), std::ios::binary);
 67 |     degree_file.read((char *)&degrees[0], num_vertices * sizeof(vid_t));
 68 |     degree_file.close();
 69 | };
 70 | 
 71 | void SnePartitioner::read_more()
 72 | {
 73 |     while (sample_edges.size() < max_sample_size && fin_ptr < fin_end) {
 74 |         edge_t *fin_buffer_end = std::min((edge_t *)fin_ptr + BUFFER_SIZE, (edge_t *)fin_end);
 75 |         size_t n = fin_buffer_end - (edge_t *)fin_ptr;
 76 |         results.resize(n);
 77 | 
 78 | #pragma omp parallel for
 79 |         for (size_t i = 0; i < n; i++)
 80 |             results[i] = check_edge((edge_t *)fin_ptr + i);
 81 | 
 82 |         for (size_t i = 0; i < n; i++) {
 83 |             edge_t *e = (edge_t *)fin_ptr + i;
 84 |             if (results[i] == p)
 85 |                 sample_edges.push_back(*e);
 86 |             else
 87 |                 assign_edge(results[i], e->first, e->second);
 88 |         }
 89 |         fin_ptr = (char *)fin_buffer_end;
 90 |     }
 91 | 
 92 |     adj_out.build(sample_edges);
 93 | 
 94 |     adj_in.build_reverse(sample_edges);
 95 | }
 96 | 
 97 | void SnePartitioner::read_remaining()
 98 | {
 99 |     auto &is_boundary = is_boundarys[p - 1], &is_core = is_cores[p - 1];
100 | 
101 |     for (auto &e : sample_edges)
102 |         if (e.valid()) {
103 |             is_boundary.set_bit_unsync(e.first);
104 |             is_boundary.set_bit_unsync(e.second);
105 |             assign_edge(p - 1, e.first, e.second);
106 |         }
107 | 
108 |     while (fin_ptr < fin_end) {
109 |         edge_t *fin_buffer_end = std::min((edge_t *)fin_ptr + BUFFER_SIZE, (edge_t *)fin_end);
110 |         size_t n = fin_buffer_end - (edge_t *)fin_ptr;
111 |         results.resize(n);
112 | 
113 | #pragma omp parallel for
114 |         for (size_t i = 0; i < n; i++)
115 |             results[i] = check_edge((edge_t *)fin_ptr + i);
116 | 
117 |         for (size_t i = 0; i < n; i++) {
118 |             edge_t *e = (edge_t *)fin_ptr + i;
119 |             if (results[i] == p) {
120 |                 is_boundary.set_bit_unsync(e->first);
121 |                 is_boundary.set_bit_unsync(e->second);
122 |                 assign_edge(p - 1, e->first, e->second);
123 |             } else
124 |                 assign_edge(results[i], e->first, e->second);
125 |         }
126 |         fin_ptr = (char *)fin_buffer_end;
127 |     }
128 | 
129 |     repv (i, num_vertices) {
130 |         if (is_boundary.get(i)) {
131 |             is_core.set_bit_unsync(i);
132 |             rep (j, p - 1)
133 |                 if (is_cores[j].get(i)) {
134 |                     is_core.set_unsync(i, false);
135 |                     break;
136 |                 }
137 |         }
138 |     }
139 | }
140 | 
141 | void SnePartitioner::clean_samples()
142 | {
143 |     for (size_t i = 0; i < sample_edges.size();) {
144 |         if (sample_edges[i].valid()) {
145 |             int bucket = check_edge(&sample_edges[i]);
146 |             if (bucket < p) {
147 |                 assign_edge(bucket, sample_edges[i].first, sample_edges[i].second);
148 |                 std::swap(sample_edges[i], sample_edges.back());
149 |                 sample_edges.pop_back();
150 |             } else
151 |                 i++;
152 |         } else {
153 |             std::swap(sample_edges[i], sample_edges.back());
154 |             sample_edges.pop_back();
155 |         }
156 |     }
157 | }
158 | 
159 | void SnePartitioner::assign_master()
160 | {
161 |     std::vector<vid_t> count_master(p, 0);
162 |     std::vector<vid_t> quota(p, num_vertices);
163 |     long long sum = p * num_vertices;
164 |     std::uniform_real_distribution<double> distribution(0.0, 1.0);
165 |     std::vector<dense_bitset::iterator> pos(p);
166 |     rep (b, p)
167 |         pos[b] = is_boundarys[b].begin();
168 |     vid_t count = 0;
169 |     while (count < num_vertices) {
170 |         long long r = distribution(gen) * sum;
171 |         int k;
172 |         for (k = 0; k < p; k++) {
173 |             if (r < quota[k])
174 |                 break;
175 |             r -= quota[k];
176 |         }
177 |         while (pos[k] != is_boundarys[k].end() && master[*pos[k]] != -1)
178 |             pos[k]++;
179 |         if (pos[k] != is_boundarys[k].end()) {
180 |             count++;
181 |             master[*pos[k]] = k;
182 |             writer.save_vertex(*pos[k], k);
183 |             count_master[k]++;
184 |             quota[k]--;
185 |             sum--;
186 |         }
187 |     }
188 |     int max_masters =
189 |         *std::max_element(count_master.begin(), count_master.end());
190 |     LOG(INFO) << "master balance: "
191 |               << (double)max_masters / ((double)num_vertices / p);
192 | }
193 | 
194 | size_t SnePartitioner::count_mirrors()
195 | {
196 |     size_t result = 0;
197 |     rep (i, p)
198 |         result += is_boundarys[i].popcount();
199 |     return result;
200 | }
201 | 
202 | void SnePartitioner::split()
203 | {
204 |     LOG(INFO) << "partition `" << basefilename << "'";
205 |     LOG(INFO) << "number of partitions: " << p;
206 | 
207 |     Timer read_timer, compute_timer;
208 | 
209 |     min_heap.reserve(num_vertices);
210 |     sample_edges.reserve(max_sample_size);
211 |     LOG(INFO) << "partitioning...";
212 |     for (bucket = 0; bucket < p - 1; bucket++) {
213 |         std::cerr << bucket << ", ";
214 |         read_timer.start();
215 |         read_more();
216 |         read_timer.stop();
217 |         DLOG(INFO) << "sample size: " << adj_out.num_edges();
218 |         compute_timer.start();
219 |         local_capacity =
220 |             FLAGS_inmem ? capacity : adj_out.num_edges() / (p - bucket);
221 |         while (occupied[bucket] < local_capacity) {
222 |             vid_t d, vid;
223 |             if (!min_heap.get_min(d, vid)) {
224 |                 if (!get_free_vertex(vid)) {
225 |                     DLOG(INFO) << "partition " << bucket
226 |                                << " stop: no free vertices";
227 |                     break;
228 |                 }
229 |                 d = adj_out[vid].size() + adj_in[vid].size();
230 |             } else {
231 |                 min_heap.remove(vid);
232 |                 /* CHECK_EQ(d, adj_out[vid].size() + adj_in[vid].size()); */
233 |             }
234 | 
235 |             occupy_vertex(vid, d);
236 |         }
237 |         min_heap.clear();
238 |         clean_samples();
239 |         compute_timer.stop();
240 |         LOG(INFO) << "finished part: " << bucket;
241 |     }
242 |     bucket = p - 1;
243 |     std::cerr << bucket << std::endl;
244 |     read_timer.start();
245 |     read_remaining();
246 |     read_timer.stop();
247 |     LOG(INFO) << "finished part: " << bucket;
248 | 
249 |     // assign_master();
250 |     LOG(INFO) << "expected edges in each partition: " << num_edges / p;
251 |     rep (i, p)
252 |         DLOG(INFO) << "edges in partition " << i << ": " << occupied[i];
253 |     size_t max_occupied = *std::max_element(occupied.begin(), occupied.end());
254 |     LOG(INFO) << "balance: " << (double)max_occupied / ((double)num_edges / p);
255 |     size_t total_mirrors = count_mirrors();
256 |     LOG(INFO) << "total mirrors: " << total_mirrors;
257 |     LOG(INFO) << "replication factor: " << (double)total_mirrors / num_vertices;
258 |     LOG(INFO) << "time used for graph input and construction: " << read_timer.get_time();
259 |     LOG(INFO) << "time used for partitioning: " << compute_timer.get_time();
260 | 
261 |     LOG(INFO) << "delayed master assignment: ";
262 |     assign_master();
263 | 
264 |     if (munmap(fin_map, filesize) == -1) {
265 |         close(fin);
266 |         PLOG(FATAL) << "Error un-mmapping the file";
267 |     }
268 |     close(fin);
269 | 
270 |     CHECK_EQ(assigned_edges, num_edges);
271 | 
272 |     total_time.stop();
273 |     LOG(INFO) << "total partition time: " << total_time.get_time();
274 | }
275 | 


--------------------------------------------------------------------------------
/src/dense_bitset.hpp:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include <cstdio>
  4 | #include <cstdlib>
  5 | #include <cstring>
  6 | #include <stdint.h>
  7 | 
  8 | #include "util.hpp"
  9 | 
 10 | class dense_bitset
 11 | {
 12 |   public:
 13 |     /// Constructs a bitset of 0 length
 14 |     dense_bitset() : array(NULL), len(0), arrlen(0) {}
 15 | 
 16 |     /// Constructs a bitset with 'size' bits. All bits will be cleared.
 17 |     explicit dense_bitset(size_t size) : array(NULL), len(0), arrlen(0)
 18 |     {
 19 |         resize(size);
 20 |         clear();
 21 |     }
 22 | 
 23 |     /// Make a copy of the bitset db
 24 |     dense_bitset(const dense_bitset &db)
 25 |     {
 26 |         array = NULL;
 27 |         len = 0;
 28 |         arrlen = 0;
 29 |         *this = db;
 30 |     }
 31 | 
 32 |     /// destructor
 33 |     ~dense_bitset() { free(array); }
 34 | 
 35 |     /// Make a copy of the bitset db
 36 |     inline dense_bitset &operator=(const dense_bitset &db)
 37 |     {
 38 |         resize(db.size());
 39 |         len = db.len;
 40 |         arrlen = db.arrlen;
 41 |         memcpy(array, db.array, sizeof(size_t) * arrlen);
 42 |         return *this;
 43 |     }
 44 | 
 45 |     /** Resizes the current bitset to hold n bits.
 46 |     Existing bits will not be changed. If the array size is increased,
 47 |     the value of the new bits are undefined.
 48 | 
 49 |     \Warning When shirnking, the current implementation may still leave the
 50 |     "deleted" bits in place which will mess up the popcount.
 51 |     */
 52 |     inline void resize(size_t n)
 53 |     {
 54 |         len = n;
 55 |         // need len bits
 56 |         size_t prev_arrlen = arrlen;
 57 |         arrlen = (n / (sizeof(size_t) * 8)) + (n % (sizeof(size_t) * 8) > 0);
 58 |         array = (size_t *)realloc(array, sizeof(size_t) * arrlen);
 59 |         // this zeros the remainder of the block after the last bit
 60 |         fix_trailing_bits();
 61 |         // if we grew, we need to zero all new blocks
 62 |         if (arrlen > prev_arrlen) {
 63 |             for (size_t i = prev_arrlen; i < arrlen; ++i) {
 64 |                 array[i] = 0;
 65 |             }
 66 |         }
 67 |     }
 68 | 
 69 |     /// Sets all bits to 0
 70 |     inline void clear()
 71 |     {
 72 |         for (size_t i = 0; i < arrlen; ++i)
 73 |             array[i] = 0;
 74 |     }
 75 | 
 76 |     inline bool empty() const
 77 |     {
 78 |         for (size_t i = 0; i < arrlen; ++i)
 79 |             if (array[i])
 80 |                 return false;
 81 |         return true;
 82 |     }
 83 | 
 84 |     /// Sets all bits to 1
 85 |     inline void fill()
 86 |     {
 87 |         for (size_t i = 0; i < arrlen; ++i)
 88 |             array[i] = (size_t)-1;
 89 |         fix_trailing_bits();
 90 |     }
 91 | 
 92 |     /// Prefetches the word containing the bit b
 93 |     inline void prefetch(size_t b) const
 94 |     {
 95 |         __builtin_prefetch(&(array[b / (8 * sizeof(size_t))]));
 96 |     }
 97 | 
 98 |     /// Returns the value of the bit b
 99 |     inline bool get(size_t b) const
100 |     {
101 |         size_t arrpos, bitpos;
102 |         bit_to_pos(b, arrpos, bitpos);
103 |         return array[arrpos] & (size_t(1) << size_t(bitpos));
104 |     }
105 | 
106 |     //! Atomically sets the bit at position b to true returning the old value
107 |     inline bool set_bit(size_t b)
108 |     {
109 |         // use CAS to set the bit
110 |         size_t arrpos, bitpos;
111 |         bit_to_pos(b, arrpos, bitpos);
112 |         const size_t mask(size_t(1) << size_t(bitpos));
113 |         return __sync_fetch_and_or(array + arrpos, mask) & mask;
114 |     }
115 | 
116 |     //! Atomically xors a bit with 1
117 |     inline bool xor_bit(size_t b)
118 |     {
119 |         // use CAS to set the bit
120 |         size_t arrpos, bitpos;
121 |         bit_to_pos(b, arrpos, bitpos);
122 |         const size_t mask(size_t(1) << size_t(bitpos));
123 |         return __sync_fetch_and_xor(array + arrpos, mask) & mask;
124 |     }
125 | 
126 |     //! Returns the value of the word containing the bit b
127 |     inline size_t containing_word(size_t b)
128 |     {
129 |         size_t arrpos, bitpos;
130 |         bit_to_pos(b, arrpos, bitpos);
131 |         return array[arrpos];
132 |     }
133 | 
134 |     //! Returns the value of the word containing the bit b
135 |     inline size_t get_containing_word_and_zero(size_t b)
136 |     {
137 |         size_t arrpos, bitpos;
138 |         bit_to_pos(b, arrpos, bitpos);
139 |         return __sync_lock_test_and_set(&array[arrpos], size_t(0));
140 |     }
141 | 
142 |     /**
143 |      * \brief Transfers approximately b bits from another bitset to this bitset
144 |      *
145 |      * "Moves" at least b bits from the other bitset to this bitset
146 |      * starting from the given position.
147 |      * At return, b will contain the actual number of bits moved,
148 |      * and start will point to the end of the transfered region.
149 |      *
150 |      * Semantically what this accomplishes is something like:
151 |      *
152 |      * \code
153 |      * idx = start;
154 |      * if other.get_bit(idx) == false {
155 |      *    idx = next true bit after idx in other (with loop around)
156 |      * }
157 |      * for(transferred = 0; transferred < b; transferred++) {
158 |      *    other.clear_bit(idx);
159 |      *    this->set_bit(idx);
160 |      *    idx = next true bit after idx in other.
161 |      *    if no more bits, return
162 |      * }
163 |      * \endcode
164 |      * However, the implementation here may transfer more than b bits.
165 |      * ( up to b + 2 * wordsize_in_bits )
166 |      */
167 |     inline void transfer_approximate_unsafe(dense_bitset &other, size_t &start,
168 |                                             size_t &b)
169 |     {
170 |         // must be identical in length
171 |         CHECK_EQ(other.len, len);
172 |         CHECK_EQ(other.arrlen, arrlen);
173 |         size_t arrpos, bitpos;
174 |         bit_to_pos(start, arrpos, bitpos);
175 |         size_t initial_arrpos = arrpos;
176 |         if (arrpos >= arrlen)
177 |             arrpos = 0;
178 |         // ok. we will only look at arrpos
179 |         size_t transferred = 0;
180 |         while (transferred < b) {
181 |             if (other.array[arrpos] > 0) {
182 |                 transferred += __builtin_popcountl(other.array[arrpos]);
183 |                 array[arrpos] |= other.array[arrpos];
184 |                 other.array[arrpos] = 0;
185 |             }
186 |             ++arrpos;
187 |             if (arrpos >= other.arrlen)
188 |                 arrpos = 0;
189 |             else if (arrpos == initial_arrpos)
190 |                 break;
191 |         }
192 |         start = 8 * sizeof(size_t) * arrpos;
193 |         b = transferred;
194 |     }
195 | 
196 |     /** Set the bit at position b to true returning the old value.
197 |         Unlike set_bit(), this uses a non-atomic set which is faster,
198 |         but is unsafe if accessed by multiple threads.
199 |     */
200 |     inline bool set_bit_unsync(size_t b)
201 |     {
202 |         // use CAS to set the bit
203 |         size_t arrpos, bitpos;
204 |         bit_to_pos(b, arrpos, bitpos);
205 |         const size_t mask(size_t(1) << size_t(bitpos));
206 |         bool ret = array[arrpos] & mask;
207 |         array[arrpos] |= mask;
208 |         return ret;
209 |     }
210 | 
211 |     //! Atomically sets the state of the bit to the new value returning the old
212 |     //value
213 |     inline bool set(size_t b, bool value)
214 |     {
215 |         if (value)
216 |             return set_bit(b);
217 |         else
218 |             return clear_bit(b);
219 |     }
220 | 
221 |     /** Set the state of the bit returning the old value.
222 |       This version uses a non-atomic set which is faster, but
223 |       is unsafe if accessed by multiple threads.
224 |     */
225 |     inline bool set_unsync(size_t b, bool value)
226 |     {
227 |         if (value)
228 |             return set_bit_unsync(b);
229 |         else
230 |             return clear_bit_unsync(b);
231 |     }
232 | 
233 |     //! Atomically set the bit at b to false returning the old value
234 |     inline bool clear_bit(size_t b)
235 |     {
236 |         // use CAS to set the bit
237 |         size_t arrpos, bitpos;
238 |         bit_to_pos(b, arrpos, bitpos);
239 |         const size_t test_mask(size_t(1) << size_t(bitpos));
240 |         const size_t clear_mask(~test_mask);
241 |         return __sync_fetch_and_and(array + arrpos, clear_mask) & test_mask;
242 |     }
243 | 
244 |     /** Clears the state of the bit returning the old value.
245 |       This version uses a non-atomic set which is faster, but
246 |       is unsafe if accessed by multiple threads.
247 |     */
248 |     inline bool clear_bit_unsync(size_t b)
249 |     {
250 |         // use CAS to set the bit
251 |         size_t arrpos, bitpos;
252 |         bit_to_pos(b, arrpos, bitpos);
253 |         const size_t test_mask(size_t(1) << size_t(bitpos));
254 |         const size_t clear_mask(~test_mask);
255 |         bool ret = array[arrpos] & test_mask;
256 |         array[arrpos] &= clear_mask;
257 |         return ret;
258 |     }
259 | 
260 |     struct bit_pos_iterator {
261 |         typedef std::input_iterator_tag iterator_category;
262 |         typedef size_t value_type;
263 |         typedef size_t difference_type;
264 |         typedef const size_t reference;
265 |         typedef const size_t *pointer;
266 |         size_t pos;
267 |         const dense_bitset *db;
268 |         bit_pos_iterator() : pos(-1), db(NULL) {}
269 |         bit_pos_iterator(const dense_bitset *const db, size_t pos)
270 |             : pos(pos), db(db)
271 |         {
272 |         }
273 | 
274 |         size_t operator*() const { return pos; }
275 |         size_t operator++()
276 |         {
277 |             if (db->next_bit(pos) == false)
278 |                 pos = (size_t)(-1);
279 |             return pos;
280 |         }
281 |         size_t operator++(int)
282 |         {
283 |             size_t prevpos = pos;
284 |             if (db->next_bit(pos) == false)
285 |                 pos = (size_t)(-1);
286 |             return prevpos;
287 |         }
288 |         bool operator==(const bit_pos_iterator &other) const
289 |         {
290 |             CHECK_EQ(db, other.db);
291 |             return other.pos == pos;
292 |         }
293 |         bool operator!=(const bit_pos_iterator &other) const
294 |         {
295 |             CHECK_EQ(db, other.db);
296 |             return other.pos != pos;
297 |         }
298 |     };
299 | 
300 |     typedef bit_pos_iterator iterator;
301 |     typedef bit_pos_iterator const_iterator;
302 | 
303 |     bit_pos_iterator begin() const
304 |     {
305 |         size_t pos;
306 |         if (first_bit(pos) == false)
307 |             pos = size_t(-1);
308 |         return bit_pos_iterator(this, pos);
309 |     }
310 | 
311 |     bit_pos_iterator end() const
312 |     {
313 |         return bit_pos_iterator(this, (size_t)(-1));
314 |     }
315 | 
316 |     /** Returns true with b containing the position of the
317 |         first bit set to true.
318 |         If such a bit does not exist, this function returns false.
319 |     */
320 |     inline bool first_bit(size_t &b) const
321 |     {
322 |         for (size_t i = 0; i < arrlen; ++i) {
323 |             if (array[i]) {
324 |                 b = (size_t)(i * (sizeof(size_t) * 8)) +
325 |                     first_bit_in_block(array[i]);
326 |                 return true;
327 |             }
328 |         }
329 |         return false;
330 |     }
331 | 
332 |     /** Returns true with b containing the position of the
333 |         first bit set to false.
334 |         If such a bit does not exist, this function returns false.
335 |     */
336 |     inline bool first_zero_bit(size_t &b) const
337 |     {
338 |         for (size_t i = 0; i < arrlen; ++i) {
339 |             if (~array[i]) {
340 |                 b = (size_t)(i * (sizeof(size_t) * 8)) +
341 |                     first_bit_in_block(~array[i]);
342 |                 return true;
343 |             }
344 |         }
345 |         return false;
346 |     }
347 | 
348 |     /** Where b is a bit index, this function will return in b,
349 |         the position of the next bit set to true, and return true.
350 |         If all bits after b are false, this function returns false.
351 |     */
352 |     inline bool next_bit(size_t &b) const
353 |     {
354 |         size_t arrpos, bitpos;
355 |         bit_to_pos(b, arrpos, bitpos);
356 |         // try to find the next bit in this block
357 |         bitpos = next_bit_in_block(bitpos, array[arrpos]);
358 |         if (bitpos != 0) {
359 |             b = (size_t)(arrpos * (sizeof(size_t) * 8)) + bitpos;
360 |             return true;
361 |         } else {
362 |             // we have to loop through the rest of the array
363 |             for (size_t i = arrpos + 1; i < arrlen; ++i) {
364 |                 if (array[i]) {
365 |                     b = (size_t)(i * (sizeof(size_t) * 8)) +
366 |                         first_bit_in_block(array[i]);
367 |                     return true;
368 |                 }
369 |             }
370 |         }
371 |         return false;
372 |     }
373 | 
374 |     ///  Returns the number of bits in this bitset
375 |     inline size_t size() const { return len; }
376 | 
377 |     size_t popcount() const
378 |     {
379 |         size_t ret = 0;
380 |         for (size_t i = 0; i < arrlen; ++i) {
381 |             ret += __builtin_popcountl(array[i]);
382 |         }
383 |         return ret;
384 |     }
385 | 
386 |     dense_bitset operator&(const dense_bitset &other) const
387 |     {
388 |         CHECK_EQ(size(), other.size());
389 |         dense_bitset ret(size());
390 |         for (size_t i = 0; i < arrlen; ++i) {
391 |             ret.array[i] = array[i] & other.array[i];
392 |         }
393 |         return ret;
394 |     }
395 | 
396 |     dense_bitset operator|(const dense_bitset &other) const
397 |     {
398 |         CHECK_EQ(size(), other.size());
399 |         dense_bitset ret(size());
400 |         for (size_t i = 0; i < arrlen; ++i) {
401 |             ret.array[i] = array[i] | other.array[i];
402 |         }
403 |         return ret;
404 |     }
405 | 
406 |     dense_bitset operator-(const dense_bitset &other) const
407 |     {
408 |         CHECK_EQ(size(), other.size());
409 |         dense_bitset ret(size());
410 |         for (size_t i = 0; i < arrlen; ++i) {
411 |             ret.array[i] = array[i] - (array[i] & other.array[i]);
412 |         }
413 |         return ret;
414 |     }
415 | 
416 |     dense_bitset &operator&=(const dense_bitset &other)
417 |     {
418 |         CHECK_EQ(size(), other.size());
419 |         for (size_t i = 0; i < arrlen; ++i) {
420 |             array[i] &= other.array[i];
421 |         }
422 |         return *this;
423 |     }
424 | 
425 |     dense_bitset &operator|=(const dense_bitset &other)
426 |     {
427 |         CHECK_EQ(size(), other.size());
428 |         for (size_t i = 0; i < arrlen; ++i) {
429 |             array[i] |= other.array[i];
430 |         }
431 |         return *this;
432 |     }
433 | 
434 |     dense_bitset &operator-=(const dense_bitset &other)
435 |     {
436 |         CHECK_EQ(size(), other.size());
437 |         for (size_t i = 0; i < arrlen; ++i) {
438 |             array[i] = array[i] - (array[i] & other.array[i]);
439 |         }
440 |         return *this;
441 |     }
442 | 
443 |     void invert()
444 |     {
445 |         for (size_t i = 0; i < arrlen; ++i) {
446 |             array[i] = ~array[i];
447 |         }
448 |         fix_trailing_bits();
449 |     }
450 | 
451 |   private:
452 |     inline static void bit_to_pos(size_t b, size_t &arrpos, size_t &bitpos)
453 |     {
454 |         // the compiler better optimize this...
455 |         arrpos = b / (8 * sizeof(size_t));
456 |         bitpos = b & (8 * sizeof(size_t) - 1);
457 |     }
458 | 
459 |     // returns 0 on failure
460 |     inline size_t next_bit_in_block(const size_t &b, const size_t &block) const
461 |     {
462 |         size_t belowselectedbit =
463 |             size_t(-1) - (((size_t(1) << b) - 1) | (size_t(1) << b));
464 |         size_t x = block & belowselectedbit;
465 |         if (x == 0)
466 |             return 0;
467 |         else
468 |             return (size_t)__builtin_ctzl(x);
469 |     }
470 | 
471 |     // returns 0 on failure
472 |     inline size_t first_bit_in_block(const size_t &block) const
473 |     {
474 |         if (block == 0)
475 |             return 0;
476 |         else
477 |             return (size_t)__builtin_ctzl(block);
478 |     }
479 | 
480 |     void fix_trailing_bits()
481 |     {
482 |         // how many bits are in the last block
483 |         size_t lastbits = len % (8 * sizeof(size_t));
484 |         if (lastbits == 0)
485 |             return;
486 |         array[arrlen - 1] &= ((size_t(1) << lastbits) - 1);
487 |     }
488 | 
489 |     size_t *array;
490 |     size_t len;
491 |     size_t arrlen;
492 | 
493 |     template <int len> friend class fixed_dense_bitset;
494 | };
495 | 


--------------------------------------------------------------------------------