├── INSTALL.md ├── LICENSE.md ├── Makefile ├── README.md ├── algorithms ├── als │ ├── als.hpp │ ├── als_async.hpp │ ├── als_graphchi.hpp │ └── checkbip.hpp ├── bc │ └── bc.hpp ├── belief_propagation │ ├── bp.hpp │ ├── bp_graphchi.hpp │ └── bp_graphchi_new.hpp ├── bfs │ ├── bfs.hpp │ ├── bfs.py │ ├── bfs_async.hpp │ ├── bfs_filtered.hpp │ ├── bfs_forest.hpp │ ├── bfs_forest_traced.hpp │ ├── bfs_yahoo.hpp │ └── bfs_yahoo_count.hpp ├── cc │ ├── cc.hpp │ └── cc_online.hpp ├── hyper-anf │ └── hyper-anf.hpp ├── kcores │ └── kcore.hpp ├── mcst │ └── mcst.hpp ├── mis │ ├── mis.hpp │ └── proof ├── pagerank │ ├── pagerank.hpp │ ├── pagerank.py │ └── pagerank_ddf.hpp ├── scan │ ├── conductance.hpp │ ├── degree_cnt.hpp │ ├── degree_cnt_max.hpp │ ├── noop.hpp │ ├── range_check.hpp │ └── reverse.hpp ├── scc │ └── scc.hpp ├── spmv │ └── spmv.hpp ├── sssp │ ├── sssp.hpp │ └── sssp_forest.hpp ├── template.hpp └── triangles │ └── triangle_counting.hpp ├── benchmarks └── driver.cpp ├── core ├── autotuner.hpp ├── core.cpp ├── sg_driver.hpp ├── sg_driver_async.hpp ├── sg_driver_python.hpp ├── types.h ├── x-ingest.hpp ├── x-lib.hpp └── x-splitter.hpp ├── format ├── TYPES ├── tools │ ├── matrix_market2type_1.py │ ├── netflix2type_1.py │ ├── snap2type_1.py │ ├── twitter2type_1.py │ ├── type1_to_matrix_market.rb │ ├── xstream2pegasus.py │ └── yahooweb2type_1.py ├── type_1.hpp └── type_2.hpp ├── generators ├── Makefile ├── defs.h ├── options.cpp ├── options.h ├── output.cpp ├── output.h ├── prng │ ├── README │ ├── mod_arith.h │ ├── mod_arith_32bit.h │ ├── mod_arith_64bit.h │ ├── mod_arith_xmt.h │ ├── mrg_transitions.c │ ├── splittable_mrg.c │ ├── splittable_mrg.h │ ├── utils.c │ └── utils.h ├── random.cpp ├── rmat.cpp ├── scramble.h └── util.h ├── slipstore ├── async_direct_io.hpp ├── checkpoints.hpp ├── dio_buffer.hpp ├── dummy_io.hpp ├── generic_io.hpp ├── interface.hpp ├── io.hpp ├── memory_buffer.hpp └── sync_pagecache_io.hpp └── utils ├── barrier.h ├── block_mem.cpp ├── boost_log_wrapper.h ├── clock.hpp ├── clock_utils.h ├── desc_utils.h ├── diskspeed.sh ├── diskspeed_mix.sh ├── diskspeed_rand.sh ├── fanout_test.cpp ├── feeder.cpp ├── hyperloglog.h ├── mem_speed_random.cpp ├── mem_speed_sequential.cpp ├── memory_utils.h ├── memspeed.sh ├── options_utils.h ├── per_cpu_data.hpp ├── slipbench.cpp ├── sort_edges.cpp ├── utils.cpp └── zpipe.c /INSTALL.md: -------------------------------------------------------------------------------- 1 | Chaos Installation 2 | ================== 3 | 4 | Installation 5 | ------------ 6 | Chaos requires boost and zmq to work. To build and install, just run: 7 | 8 | $ make 9 | $ make install 10 | 11 | The default installation prefix is `/usr/local`. You can change this to prefix PREFIX by running: 12 | 13 | $ make install prefix=PREFIX 14 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Chaos is released under the Apache License 2.0 2 | ============================================= 3 | 4 | **Copyright 2015 Operating Systems Laboratory EPFL** 5 | 6 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 11 | 12 | Chaos contains code written by third parties. Such code will have its own individual license file in the directory where it appears. That file will describe the copyrights, license, and restrictions which apply to that code. 13 | 14 | Code in the following directories has alternate licenses: *generators/prng* 15 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | OBJECT_DIR= object_files 2 | UTILS_SRC= utils.o 3 | LIBS_SRC= core.o $(UTILS_SRC) 4 | LIBS= $(addprefix $(OBJECT_DIR)/, $(LIBS_SRC)) 5 | UTILS= $(addprefix $(OBJECT_DIR)/, $(UTILS_SRC)) 6 | PROGS_SRC= driver.o file_splitter.o mem_speed_sequential.o sort_edges.o\ 7 | mem_speed_random.o block_mem.o feeder.o 8 | PROGS= $(addprefix $(OBJECT_DIR)/, $(PROGS_SRC)) 9 | TARGETS= bin/benchmark_driver bin/mem_speed_sequential bin/sort_edges\ 10 | bin/mem_speed_random bin/block_mem bin/zpipe bin/feeder bin/slipbench\ 11 | bin/fanout_test 12 | 13 | 14 | all: $(LIBS) $(TARGETS) generators 15 | 16 | CXX?= g++ 17 | 18 | CXXFLAGS?= -O3 -DNDEBUG -Wall -Wno-unused-function -L/usr/local/lib 19 | #CXXFLAGS?= -O3 -g -Wall -Wno-unused-function 20 | #CXXFLAGS?= -g 21 | CXXFLAGS += -Wfatal-errors 22 | EXTRA_INCLUDES=-include core/types.h 23 | EXTRA_INCLUDES+=-I/usr/local/include/boost-numeric-bindings 24 | 25 | #Graph size limits 26 | CXXFLAGS += -DCOMPACT_GRAPH 27 | 28 | #ZLIB SPEEDS 29 | #CXXFLAGS += -DZLIB_COMPRESSION_LEVEL=Z_DEFAULT_COMPRESSION 30 | CXXFLAGS += -DZLIB_COMPRESSION_LEVEL=Z_BEST_SPEED 31 | #CXXFLAGS += -DZLIB_COMPRESSION_LEVEL=Z_BEST_COMPRESSION 32 | 33 | #ISA extensions 34 | CXXFLAGS += -msse4.2 35 | #Note avx2 is untested 36 | #CXXFLAGS += -mavx2 37 | 38 | #System libraries 39 | SYSLIBS = -lboost_system -lboost_program_options -lboost_thread -lpthread -lz -lrt -lzmq 40 | # Uncomment for ALS 41 | SYSLIBS += -llapack 42 | 43 | #Uncomment if you want low latency control messages 44 | #CXXFLAGS += -DLOW_LATENCY_CONTROL 45 | 46 | #Uncomment if you want periodic flushes 47 | #CXXFLAGS += -DFLUSH_PERIODIC 48 | 49 | #Python support (uncomment following lines) 50 | #CXXFLAGS += -DPYTHON_SUPPORT 51 | #SYSLIBS += -lpython3.2mu 52 | #EXTRA_INCLUDES+=-L/usr/lib/python3.2/config 53 | #EXTRA_INCLUDES+=-I/usr/include/python3.2 54 | 55 | 56 | 57 | 58 | -include $(LIBS:.o=.d) $(PROGS:.o=.d) 59 | 60 | $(OBJECT_DIR)/core.o:core/core.cpp 61 | $(CXX) $(CXXFLAGS) $(EXTRA_INCLUDES) -c -o $@ $< 62 | $(CXX) -MM -MT '$(OBJECT_DIR)/core.o' $< > $(@:.o=.d) 63 | 64 | $(OBJECT_DIR)/utils.o:utils/utils.cpp 65 | $(CXX) $(CXXFLAGS) $(EXTRA_INCLUDES) -c -o $@ $< 66 | $(CXX) -MM -MT '$(OBJECT_DIR)/utils.o' $< > $(@:.o=.d) 67 | 68 | $(OBJECT_DIR)/driver.o:benchmarks/driver.cpp 69 | $(CXX) $(CXXFLAGS) $(EXTRA_INCLUDES) -c -o $@ $< 70 | $(CXX) -MM -MT '$(OBJECT_DIR)/driver.o' $< > $(@:.o=.d) 71 | 72 | $(OBJECT_DIR)/sort_edges.o:utils/sort_edges.cpp 73 | $(CXX) $(CXXFLAGS) $(EXTRA_INCLUDES) -c -o $@ $< 74 | $(CXX) -MM -MT '$(OBJECT_DIR)/sort_edges.o' $< > $(@:.o=.d) 75 | 76 | $(OBJECT_DIR)/feeder.o:utils/feeder.cpp 77 | $(CXX) $(CXXFLAGS) $(EXTRA_INCLUDES) -c -o $@ $< 78 | $(CXX) -MM -MT '$(OBJECT_DIR)/feeder.o' $< > $(@:.o=.d) 79 | 80 | $(OBJECT_DIR)/slipbench.o:utils/slipbench.cpp 81 | $(CXX) $(CXXFLAGS) $(EXTRA_INCLUDES) -c -o $@ $< 82 | $(CXX) -MM -MT '$(OBJECT_DIR)/slipbench.o' $< > $(@:.o=.d) 83 | 84 | $(OBJECT_DIR)/fanout_test.o:utils/fanout_test.cpp 85 | $(CXX) $(CXXFLAGS) $(EXTRA_INCLUDES) -c -o $@ $< 86 | $(CXX) -MM -MT '$(OBJECT_DIR)/fanout_test.o' $< > $(@:.o=.d) 87 | 88 | $(OBJECT_DIR)/block_mem.o:utils/block_mem.cpp 89 | $(CXX) $(CXXFLAGS) $(EXTRA_INCLUDES) -c -o $@ $< 90 | $(CXX) -MM -MT '$(OBJECT_DIR)/block_mem.o' $< > $(@:.o=.d) 91 | 92 | $(OBJECT_DIR)/mem_speed_sequential.o:utils/mem_speed_sequential.cpp 93 | $(CXX) $(CXXFLAGS) $(EXTRA_INCLUDES) -c -o $@ $< 94 | $(CXX) -MM -MT '$(OBJECT_DIR)/mem_speed_sequential.o' $< > $(@:.o=.d) 95 | 96 | $(OBJECT_DIR)/mem_speed_random.o:utils/mem_speed_random.cpp 97 | $(CXX) $(CXXFLAGS) $(EXTRA_INCLUDES) -c -o $@ $< 98 | $(CXX) -MM -MT '$(OBJECT_DIR)/mem_speed_random.o' $< > $(@:.o=.d) 99 | 100 | $(OBJECT_DIR)/zpipe.o:utils/zpipe.c 101 | $(CXX) $(CXXFLAGS) $(EXTRA_INCLUDES) -c -o $@ $< 102 | $(CXX) -MM -MT '$(OBJECT_DIR)/zpipe.o' $< > $(@:.o=.d) 103 | 104 | bin/benchmark_driver:$(OBJECT_DIR)/driver.o $(LIBS) 105 | $(CXX) $(CXXFLAGS) -o $@ $< $(LIBS) $(SYSLIBS) 106 | 107 | bin/mem_speed_sequential:$(OBJECT_DIR)/mem_speed_sequential.o $(LIBS) 108 | $(CXX) $(CXXFLAGS) -o $@ $< $(LIBS) $(SYSLIBS) 109 | 110 | bin/mem_speed_random:$(OBJECT_DIR)/mem_speed_random.o $(LIBS) 111 | $(CXX) $(CXXFLAGS) -o $@ $< $(LIBS) $(SYSLIBS) 112 | 113 | bin/zpipe:$(OBJECT_DIR)/zpipe.o $(LIBS) 114 | $(CXX) $(CXXFLAGS) -o $@ $< $(LIBS) $(SYSLIBS) 115 | 116 | bin/sort_edges:$(OBJECT_DIR)/sort_edges.o $(LIBS) 117 | $(CXX) $(CXXFLAGS) -o $@ $< $(LIBS) $(SYSLIBS) 118 | 119 | bin/feeder:$(OBJECT_DIR)/feeder.o $(LIBS) 120 | $(CXX) $(CXXFLAGS) -o $@ $< $(LIBS) $(SYSLIBS) 121 | 122 | bin/slipbench:$(OBJECT_DIR)/slipbench.o 123 | $(CXX) $(CXXFLAGS) -o $@ $< $(LIBS) $(SYSLIBS) 124 | 125 | bin/fanout_test:$(OBJECT_DIR)/fanout_test.o 126 | $(CXX) $(CXXFLAGS) -o $@ $< $(LIBS) $(SYSLIBS) 127 | 128 | bin/block_mem:$(OBJECT_DIR)/block_mem.o $(LIBS) 129 | $(CXX) $(CXXFLAGS) -o $@ $< $(LIBS) $(SYSLIBS) 130 | 131 | .PHONY: generators 132 | generators: 133 | $(MAKE) -C generators 134 | 135 | clean: 136 | rm -f $(TARGETS) $(LIBS) $(OBJECT_DIR)/* 137 | $(MAKE) -C generators clean 138 | 139 | prefix = ~ 140 | bindir = $(prefix)/bin 141 | .PHONY: install uninstall 142 | install: all 143 | cp bin/benchmark_driver $(bindir) 144 | cp generators/rmat $(bindir) 145 | cp generators/erdos-renyi $(bindir) 146 | 147 | uninstall: 148 | rm -f $(bindir)/benchmark_driver 149 | rm -f $(bindir)/rmat 150 | rm -f $(bindir)/erdos-renyi 151 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Chaos 2 | ===== 3 | 4 | What is it? 5 | ----------- 6 | 7 | Chaos is a graph processing system for analytics on big graphs using small clusters. Chaos builds on the [X-Stream](https://github.com/epfl-labos/x-stream) single-machine graph processing system, but scales out to multiple machines. Chaos treats the aggregate storage of all machines as a single flat disk and uses work stealing to balance the load across nodes in the cluster. It exposes the familiar scatter-gather-apply programming model. 8 | 9 | Installation 10 | ------------ 11 | 12 | Please see the file called [INSTALL.md](INSTALL.md). 13 | 14 | Getting started 15 | --------------- 16 | 17 | Like X-Stream, Chaos expects the input graphs in the binary edge list format. For details see the file called TYPES in the format directory. In addition to the edge list file, Chaos expects an ini file that specifies the graph type, the number of vertices and the number of edges. 18 | 19 | When running on multiple machines, Chaos supports loading input graphs in parallel. To do so, each instance of Chaos must be provided with a part of the input. We recommend splitting the input in equal-sized parts and store one such part on each machine. 20 | 21 | Provided with Chaos are two synthetic graph generators - a random graph generator and an RMAT graph generator. They also generate the needed ini files. For convenience, the generator can directly produce a part of a synthetic graph instead of the whole graph. 22 | 23 | For example, to create the 3rd part of a scale-free graph with 1M vertices and 16M edges which is intended to run on 4 machines, run: 24 | 25 | $ rmat --name test --scale 20 --edges 16777216 --xscale_interval 4 --xscale_node 2 26 | 27 | This creates a directed graph. To create an undirected graph instead, use `--symmetric`. Note that some algorithms require directed and some undirected graphs to work properly. 28 | 29 | Directory format/tools contains a number of scripts to convert graphs from different formats into the format that Chaos expects. `split` is your friend when it comes to splitting the input for distribution to multiple machines (be careful to only split at edge boundaries!). 30 | 31 | In order to provide context information to a Chaos instance (e.g., which other Chaos instances are running in the cluster, which ports to use for communication, etc.), `benchmark_driver` expected a file called `slipstore.ini` in the current directory. The structure of `slipstore.ini` is as follows: 32 | 33 | [machines] 34 | count=4 # number of machines 35 | me=2 # machine id (starts at 0) 36 | name0=192.168.1.30 # hostname or IP address of first machine 37 | base_port0=5555 # base port of first machine 38 | iface0=eth1 # network interface of first machine 39 | name1=192.168.1.31 # hostname or IP address of second machine 40 | base_port1=5555 # base port of second machine 41 | iface1=eth1 # network interface of second machine 42 | ... # repeat for all other machines in the cluster 43 | 44 | To run any of the currently implemented algorithms in Chaos, you can use the provided example program called benchmark_driver. To see supported options run it with `-h`. For example, to run 10 iterations of pagerank on the test graph generated above, invoke benchmark_driver as: 45 | 46 | $ benchmark_driver -g test -b pagerank --pagerank::niters 10 -a -p 16 --physical_memory 268435456 47 | 48 | This will run a Chaos instance with 16 worker threads and with 256MB of physical memory allotted to it. The -a flag tells Chaos to automatically tune the number of partitions and other system parameters. 49 | 50 | When the run consists of multiple machines, you must run the above command for each instance! We recommend using `clush` or similar tools to make this easy. 51 | 52 | Orchestrator 53 | ------------ 54 | 55 | Deploying, running and coordinating Chaos runs on many machines is arduous. We're planning to release our orchestration tool shortly, which should help with these tasks. Keep an eye out for the release. 56 | 57 | 58 | Licensing 59 | --------- 60 | 61 | Please see the file called LICENSE. 62 | 63 | Contact 64 | ------- 65 | - Laurent Bindschaedler 66 | - Jasmina Malicevic 67 | - Amitabha Roy 68 | -------------------------------------------------------------------------------- /algorithms/als/checkbip.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _CHECKBIP_ 20 | #define _CHECKBIP_ 21 | 22 | #include "../../utils/boost_log_wrapper.h" 23 | #include "../../core/x-lib.hpp" 24 | 25 | // Tests if graph is bipartite 26 | 27 | namespace algorithm { 28 | namespace sg_simple { 29 | template 30 | class check_if_bipartite { 31 | private: 32 | 33 | struct vertex { 34 | int side; // -1 is left, 1 is right, 0 is not set 35 | } __attribute__((__packed__)); 36 | 37 | struct update { 38 | vertex_t target; 39 | int side; 40 | } __attribute__((__packed__)); 41 | 42 | static bool is_bip; 43 | 44 | public: 45 | 46 | static int open_vertex_format() { return -1; } 47 | 48 | static unsigned long vertex_format_bytes() { return 0; } 49 | 50 | static unsigned long vertex_state_bytes() { 51 | return sizeof(struct vertex); 52 | } 53 | 54 | static unsigned long split_size_bytes() { 55 | return sizeof(struct update); 56 | } 57 | 58 | static unsigned long split_key(unsigned char *buffer, unsigned long jump) { 59 | struct update *u = (struct update *) buffer; 60 | vertex_t key = u->target; 61 | key = key >> jump; 62 | return key; 63 | } 64 | 65 | static bool init(unsigned char *vertex_state, 66 | unsigned long vertex_index, 67 | unsigned long bsp_phase, 68 | per_processor_data *cpu_state) { 69 | struct vertex *vertices = (struct vertex *) vertex_state; 70 | vertices->side = 0; 71 | return true; 72 | } 73 | 74 | static bool need_init(unsigned long bsp_phase) { 75 | return (bsp_phase == 0); 76 | } 77 | 78 | static bool apply_one_update(unsigned char *vertex_state, 79 | unsigned char *update_stream, 80 | per_processor_data *per_cpu_data, 81 | unsigned long bsp_phase) { 82 | struct update *u = (struct update *) update_stream; 83 | struct vertex *vertices = (struct vertex *) vertex_state; 84 | struct vertex *v = &vertices[x_lib::configuration::map_offset(u->target)]; 85 | 86 | if (v->side == 0) { 87 | v->side = -u->side; 88 | } 89 | 90 | if (v->side != -u->side) { 91 | is_bip = false; 92 | } 93 | 94 | return false; 95 | } 96 | 97 | static bool generate_update(unsigned char *vertex_state, 98 | unsigned char *edge_format, 99 | unsigned char *update_stream, 100 | per_processor_data *per_cpu_data, 101 | unsigned long bsp_phase) { 102 | vertex_t src, dst; 103 | F::read_edge(edge_format, src, dst); 104 | 105 | struct vertex *vertices = (struct vertex *) vertex_state; 106 | struct vertex *v = &vertices[x_lib::configuration::map_offset(src)]; 107 | 108 | if (src < dst) { 109 | v->side = -1; 110 | } else { 111 | v->side = 1; 112 | } 113 | 114 | struct update *u = (struct update *) update_stream; 115 | u->target = dst; 116 | u->side = v->side; 117 | 118 | return true; 119 | } 120 | 121 | 122 | static void preprocessing() { 123 | } 124 | 125 | static void postprocessing() { 126 | if (!is_bip) { 127 | BOOST_LOG_TRIVIAL(info) << "ALGORITHM::CHECKBIP::NOT_BIPARTITE"; 128 | } 129 | } 130 | 131 | static per_processor_data *create_per_processor_data(unsigned long processor_id) { return NULL; } 132 | 133 | static unsigned long min_super_phases() { 134 | return 1; 135 | } 136 | 137 | }; 138 | 139 | template 140 | bool check_if_bipartite::is_bip = true; 141 | } 142 | } 143 | #endif 144 | -------------------------------------------------------------------------------- /algorithms/bfs/bfs.py: -------------------------------------------------------------------------------- 1 | from ctypes import * 2 | import struct 3 | import numpy as np 4 | import sys 5 | import traceback 6 | import time 7 | 8 | class bfs_update(Structure): 9 | _fields_ = [ 10 | ("parent", c_uint), 11 | ("child", c_uint), 12 | ] 13 | 14 | class bfs_vertex(Structure): 15 | _fields_ = [ 16 | ("bfs_parent", c_uint), 17 | ("bsp_phase", c_uint), 18 | ] 19 | 20 | class AlgorithmBFS: 21 | #Common parameters 22 | partition_shift = 0 23 | super_partition_shift = 0 24 | edge_size = 0 25 | total_time_python = 0 26 | #BFS parameters 27 | vertices_discovered = 0 28 | edges_explored = 0 29 | 30 | def initialization_bfs(self, partition_shift, super_partition_shift, edge_size): 31 | AlgorithmBFS.partition_shift = partition_shift 32 | AlgorithmBFS.super_partition_shift = super_partition_shift 33 | AlgorithmBFS.edge_size = edge_size 34 | 35 | def do_scatter_bfs(self, mem_vertices, mem_edges, mem_output, params_stream): 36 | 37 | try: 38 | #unpacking parameters 39 | start_scatter_time = time.clock() 40 | offset_params = 0 41 | offset_edges = 0 42 | offset_updates = 0 43 | local_edges_explored = 0 44 | bytes_out = c_ulong(0) 45 | bsp_phase = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 46 | offset_params += sizeof(c_ulong) 47 | bytes_in = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 48 | offset_params += sizeof(c_ulong) 49 | start_edges = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 50 | offset_params += sizeof(c_ulong) 51 | state_offset = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 52 | offset_params += sizeof(c_ulong) 53 | bytes_out_max = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 54 | offset_params += sizeof(c_ulong) 55 | edges_stream = mem_edges[start_edges:start_edges+bytes_in] 56 | counter = 0 57 | while bytes_in > 0: 58 | if bytes_out.value + sizeof(bfs_update) > bytes_out_max: 59 | break 60 | update_generated = self.generate_update_bfs(edges_stream, offset_edges, state_offset, offset_updates, mem_vertices, mem_output, bsp_phase) 61 | offset_edges += AlgorithmBFS.edge_size 62 | bytes_in -= AlgorithmBFS.edge_size 63 | if update_generated: 64 | local_edges_explored += 1 65 | bytes_out.value += sizeof(bfs_update) 66 | offset_updates += sizeof(bfs_update) 67 | counter += 1 68 | AlgorithmBFS.edges_explored += local_edges_explored 69 | end_scatter_time = time.clock() 70 | AlgorithmBFS.total_time_python += (end_scatter_time - start_scatter_time) 71 | return (bytes_out.value, bytes_in) 72 | 73 | except Exception as inst: 74 | exc_type, exc_value, exc_traceback = sys.exc_info() 75 | traceback.print_tb(exc_traceback, limit=1, file=sys.stdout) 76 | print(type(inst)) 77 | print(inst) 78 | raise 79 | 80 | def do_gather_bfs(self, mem_vertices, mem_updates, params_stream): 81 | 82 | try: 83 | start_gather_time = time.clock() 84 | offset_params = 0 85 | offset_updates = 0 86 | local_vertices_discovered = 0 87 | bsp_phase = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 88 | offset_params += sizeof(c_ulong) 89 | bytes_in = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 90 | offset_params += sizeof(c_ulong) 91 | start_updates = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 92 | offset_params += sizeof(c_ulong) 93 | state_offset = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 94 | counter = 0 95 | activate_OR = False 96 | while(bytes_in > 0): 97 | activate = self.apply_one_update_bfs(mem_updates, offset_updates, 98 | mem_vertices, bsp_phase, start_updates, state_offset) 99 | if activate: 100 | local_vertices_discovered += 1 101 | offset_updates += sizeof(bfs_update) 102 | bytes_in -= sizeof(bfs_update) 103 | activate_OR = activate_OR or activate 104 | counter += 1 105 | AlgorithmBFS.vertices_discovered += local_vertices_discovered 106 | end_gather_time = time.clock() 107 | AlgorithmBFS.total_time_python += (end_gather_time - start_gather_time) 108 | return (activate_OR, mem_vertices) 109 | except Exception as inst: 110 | print(type(inst)) 111 | print(inst) 112 | raise 113 | 114 | def generate_update_bfs(self, edges_stream, offset_edges, state_offset, offset_updates, 115 | vertices_stream, mem_output, bsp_phase): 116 | #should be changed depending on the type of egde!! now it is (src, dst) type 117 | edge = struct.unpack('II', edges_stream[offset_edges:offset_edges+2*sizeof(c_uint)]) 118 | vindex = self.map_offset(edge[0], AlgorithmBFS.super_partition_shift, AlgorithmBFS.partition_shift) 119 | current_vertex = struct.unpack('II', vertices_stream[state_offset+vindex*sizeof(bfs_vertex):state_offset+vindex*sizeof(bfs_vertex)+sizeof(bfs_vertex)]) 120 | if current_vertex[0] != 4294967295 and current_vertex[1] == bsp_phase: 121 | single_update = bfs_update(edge[0], edge[1]) 122 | mem_output[offset_updates:offset_updates + sizeof(c_uint)] = struct.pack('I', single_update.parent) 123 | mem_output[offset_updates + sizeof(c_uint):offset_updates + 2*sizeof(c_uint)] = struct.pack('I', single_update.child) 124 | return True 125 | else: 126 | return False 127 | 128 | def apply_one_update_bfs(self, updates, updates_offset, vertices_bytes, 129 | bsp_phase, start_updates, state_offset): 130 | current_update = struct.unpack('II', updates[start_updates+updates_offset:start_updates+updates_offset+2*sizeof(c_uint)]) 131 | vindex = self.map_offset(current_update[1], AlgorithmBFS.super_partition_shift, AlgorithmBFS.partition_shift) 132 | child_vertex = self.read_vertex(vertices_bytes, vindex, state_offset) 133 | if child_vertex[0] == 4294967295: 134 | new_parent = current_update[0] 135 | new_phase = bsp_phase 136 | vertices_bytes[vindex*sizeof(bfs_vertex):vindex*sizeof(bfs_vertex)+sizeof(bfs_vertex)] = struct.pack('II', new_parent, new_phase) 137 | return True 138 | else: 139 | return False 140 | 141 | def map_offset(self, key, super_partition_shift, partition_shift): 142 | return key >> (super_partition_shift + partition_shift) 143 | 144 | def read_vertex(self, vertices_bytes, vindex, state_offset): 145 | tmp = vertices_bytes[state_offset + vindex*sizeof(bfs_vertex):state_offset + vindex*sizeof(bfs_vertex)+sizeof(bfs_vertex)] 146 | return struct.unpack('II', tmp) 147 | 148 | def postprocessing_bfs(self): 149 | print("PYTHON::BFS::VERTICES_DISCOVERED ", AlgorithmBFS.vertices_discovered); 150 | print("PYTHON::BFS::EDGES_EXPLORED ", AlgorithmBFS.edges_explored); 151 | print("Total time in python: ", AlgorithmBFS.total_time_python) 152 | 153 | -------------------------------------------------------------------------------- /algorithms/cc/cc.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _CC_ 20 | #define _CC_ 21 | 22 | #include "../../core/x-lib.hpp" 23 | 24 | // Note: CC is defined for undirected graphs 25 | 26 | namespace algorithm { 27 | namespace sg_simple { 28 | class cc_per_processor_data : public per_processor_data { 29 | public: 30 | static unsigned long cc_neg; 31 | double cc_neg_delta; 32 | 33 | cc_per_processor_data() 34 | : cc_neg_delta(0) { } 35 | 36 | bool reduce(per_processor_data **per_cpu_array, 37 | unsigned long processors) { 38 | for (unsigned long i = 0; i < processors; i++) { 39 | cc_per_processor_data *data = 40 | static_cast(per_cpu_array[i]); 41 | cc_neg += data->cc_neg_delta; 42 | data->cc_neg_delta = 0; 43 | } 44 | return false; 45 | } 46 | } __attribute__((__aligned__(64))); 47 | 48 | template 49 | class cc { 50 | private: 51 | 52 | struct vertex { 53 | vertex_t component; 54 | unsigned long active_phase; 55 | bool merged; 56 | } __attribute__((__packed__)); 57 | 58 | struct update { 59 | vertex_t target; 60 | vertex_t component; 61 | } __attribute__((__packed__)); 62 | 63 | public: 64 | static unsigned long vertex_state_bytes() { 65 | return sizeof(struct vertex); 66 | } 67 | 68 | static unsigned long split_size_bytes() { 69 | return sizeof(struct update); 70 | } 71 | 72 | static unsigned long split_key(unsigned char *buffer, unsigned long jump) { 73 | struct update *u = (struct update *) buffer; 74 | vertex_t key = u->target; 75 | key = key >> jump; 76 | return key; 77 | } 78 | 79 | static bool init(unsigned char *vertex_state, 80 | unsigned long vertex_index, 81 | unsigned long bsp_phase, 82 | per_processor_data *cpu_state) { 83 | struct vertex *vertices = (struct vertex *) vertex_state; 84 | vertices->component = vertex_index; 85 | vertices->active_phase = 0; 86 | vertices->merged = false; 87 | return true; 88 | } 89 | 90 | static bool need_init(unsigned long bsp_phase) { 91 | return (bsp_phase == 0); 92 | } 93 | 94 | static bool apply_one_update(unsigned char *vertex_state, 95 | unsigned char *update_stream, 96 | per_processor_data *per_cpu_data, 97 | unsigned long bsp_phase) { 98 | struct update *u = (struct update *) update_stream; 99 | struct vertex *vertices = (struct vertex *) vertex_state; 100 | struct vertex *v = &vertices[x_lib::configuration::map_offset(u->target)]; 101 | 102 | if (u->component < v->component) { 103 | v->component = u->component; 104 | v->active_phase = bsp_phase; 105 | if (!v->merged) { 106 | static_cast(per_cpu_data)->cc_neg_delta++; // One component merged 107 | v->merged = true; 108 | } 109 | return true; 110 | } else { 111 | return false; 112 | } 113 | } 114 | 115 | static bool generate_update(unsigned char *vertex_state, 116 | unsigned char *edge_format, 117 | unsigned char *update_stream, 118 | per_processor_data *per_cpu_data, 119 | unsigned long bsp_phase) { 120 | vertex_t src, dst; 121 | F::read_edge(edge_format, src, dst); 122 | 123 | struct vertex *vertices = (struct vertex *) vertex_state; 124 | struct vertex *v = &vertices[x_lib::configuration::map_offset(src)]; 125 | 126 | if (v->active_phase == bsp_phase) { 127 | struct update *u = (struct update *) update_stream; 128 | u->target = dst; 129 | u->component = v->component; 130 | return true; 131 | } else { 132 | return false; 133 | } 134 | } 135 | 136 | static void preprocessing() { 137 | } 138 | 139 | static void postprocessing() { 140 | BOOST_LOG_TRIVIAL(info) << "ALGORITHM::CC::CC_COUNT " << 141 | (pt.get < unsigned 142 | long > ("graph.vertices") - 143 | cc_per_processor_data::cc_neg); 144 | } 145 | 146 | static per_processor_data * 147 | create_per_processor_data(unsigned long processor_id) { 148 | return new cc_per_processor_data(); 149 | } 150 | 151 | static unsigned long min_super_phases() { 152 | return 1; 153 | } 154 | 155 | }; 156 | 157 | unsigned long cc_per_processor_data::cc_neg = 0; 158 | } 159 | } 160 | #endif 161 | -------------------------------------------------------------------------------- /algorithms/mis/proof: -------------------------------------------------------------------------------- 1 | A termination proof for the mis algorithm implemented for undirected graphs 2 | G=(V,E): 3 | 4 | Let V(i) be the MIS at the end of the ith superstep. 5 | 6 | Theorem 1: 7 | (1) V(2*n) is an independent set. 8 | (2) For any edge (u,v) with u in V(2*n) and v outside, u < v. 9 | (3) V(2*n + 1) 10 | 11 | Trivially true for V(0) as the lower numbered end votes out the higher numbered 12 | end. 13 | 14 | Assume that it is true till V(2*i). 15 | 16 | Since V(2*i) is independent no vertex can be voted out of it and hence 17 | V(2*i + 1) contains V(2*i). The vertices in V(2*i + 1) - V(2*i) cannot be 18 | connected to any vertex in V(2*i) as any such connection must be from a lower 19 | numbered vertex in V(2*i) and hence would have out-voted the other 20 | vertex. Therefore V(2*i + 1) consists of two disconnected partitions V(2*i) and 21 | V(2*i + 1) - V(2*i). 22 | 23 | At the next step therefore, not only does V(2*i) survive but in addition the 24 | edges connecting vertices in V(2*i + 1) - V(2*i) out-vote all the higher 25 | numbered ends. V(2*i + 2) hence satisfies both (1) and (2). 26 | 27 | Theorem 2: If V(2*n + 1) is strictly larger than V(2*n) then V(2*n + 2) is 28 | strictly larger than V(2*n). 29 | 30 | Proof: This means at least one vertex has been added to V(2*n + 1). Let u be the 31 | minimum numbered vertex in V(2*n + 1) - V(2*n). u cannot be out-voted from a 32 | vertex in V(2*n) as those are disconnected from the ones in V(2*n + 1). u being 33 | the minimum numbered vertex in V(2*n + 1) - V(2*n) cannot be out-voted by any 34 | other vertex in that partition either. Hence u must be in V(2*n + 2). 35 | 36 | 37 | Theorem: The algorithm terminates when the MIS is maximal and independent. Note 38 | that the algorithm terminates when the MIS does not change. 39 | 40 | Proof: Assuming that the size of the MIS does not change from 2*n to 2*n + 41 | 1. Therefore every vertex in V - V(2*n) must have at least one edge into V(2*n). 42 | Therefore V(2*n) is maximal and independent by Theorem (1). The termination is 43 | therefore correct. 44 | 45 | V(2*n) covers every edge in G as otherwise an uncovered 46 | edge would have added its lower numbered vertex to the MIS. The termination in 47 | this case is therefore correct as the MIS is independent by theorem 1. 48 | 49 | If the MIS does not change from 2*n + 1 to 2*n + 2 then this means that V(2n + 50 | 1) is independent as otherwise at least some vertex is voted out. Further V(2n + 51 | 1) must be maximal. Then there exists some vertices with no connection into V(2n 52 | + 1). In such a case however the minimum numbered vertex would have been added 53 | to V(2n + 1). 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /algorithms/pagerank/pagerank.py: -------------------------------------------------------------------------------- 1 | from ctypes import * 2 | import struct 3 | import numpy as np 4 | import sys 5 | import traceback 6 | import time 7 | 8 | class pgrnk_vertex(Structure): 9 | _fields_ = [ 10 | ("degree", c_uint), 11 | ("rank", c_float), 12 | ("sum", c_float), 13 | ] 14 | 15 | class pgrnk_update(Structure): 16 | _fields_ = [ 17 | ("target", c_uint), 18 | ("rank", c_float), 19 | ] 20 | 21 | class AlgorithmPagerank: 22 | #Common parameters 23 | partition_shift = 0 24 | super_partition_shift = 0 25 | edge_size = 0 26 | total_time_python = 0 27 | #Pagerank parameters 28 | DAMPING_FACTOR = 0.85 29 | niters = 4 30 | 31 | def initialization_pgrnk(self, partition_shift, super_partition_shift, edge_size): 32 | AlgorithmPagerank.partition_shift = partition_shift 33 | AlgorithmPagerank.super_partition_shift = super_partition_shift 34 | AlgorithmPagerank.edge_size = edge_size 35 | 36 | def do_scatter_pgrnk(self, mem_vertices, mem_edges, mem_output, params_stream): 37 | try: 38 | start_scatter_time = time.clock() 39 | offset_params = 0 40 | offset_edges = 0 41 | offset_updates = 0 42 | bytes_out = c_ulong(0) 43 | bsp_phase = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 44 | offset_params += sizeof(c_ulong) 45 | bytes_in = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 46 | offset_params += sizeof(c_ulong) 47 | start_edges = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 48 | offset_params += sizeof(c_ulong) 49 | state_offset = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 50 | offset_params += sizeof(c_ulong) 51 | bytes_out_max = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 52 | offset_params += sizeof(c_ulong) 53 | counter = 0 54 | 55 | while bytes_in > 0: 56 | if bytes_out.value + sizeof(pgrnk_update) > bytes_out_max: 57 | break 58 | update_generated = self.generate_update_pgrnk(mem_edges, offset_edges, state_offset, offset_updates, mem_vertices, mem_output, bsp_phase) 59 | offset_edges += AlgorithmPagerank.edge_size 60 | bytes_in -= AlgorithmPagerank.edge_size 61 | if update_generated: 62 | bytes_out.value += sizeof(pgrnk_update) 63 | offset_updates += sizeof(pgrnk_update) 64 | counter += 1 65 | end_scatter_time = time.clock() 66 | AlgorithmPagerank.total_time_python += (end_scatter_time - start_scatter_time) 67 | return (bytes_out.value, bytes_in) 68 | 69 | except Exception as inst: 70 | exc_type, exc_value, exc_traceback = sys.exc_info() 71 | traceback.print_tb(exc_traceback, limit=1, file=sys.stdout) 72 | print(type(inst)) 73 | print(inst) 74 | raise 75 | 76 | def do_gather_pgrnk(self, mem_vertices, mem_updates, params_stream): 77 | try: 78 | start_gather_time = time.clock() 79 | offset_params = 0 80 | offset_updates = 0 81 | local_vertices_discovered = 0 82 | bsp_phase = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 83 | offset_params += sizeof(c_ulong) 84 | bytes_in = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 85 | offset_params += sizeof(c_ulong) 86 | start_updates = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 87 | offset_params += sizeof(c_ulong) 88 | state_offset = struct.unpack('L', params_stream[offset_params:offset_params+sizeof(c_ulong)])[0] 89 | counter = 0 90 | activate_OR = False 91 | while(bytes_in > 0): 92 | activate = self.apply_one_update_pgrnk(mem_updates, offset_updates, mem_vertices, bsp_phase, start_updates, state_offset) 93 | offset_updates += sizeof(pgrnk_update) 94 | bytes_in -= sizeof(pgrnk_update) 95 | activate_OR = activate_OR or activate 96 | counter += 1 97 | end_gather_time = time.clock() 98 | AlgorithmPagerank.total_time_python += (end_gather_time - start_gather_time) 99 | return (activate_OR, mem_vertices) 100 | 101 | except Exception as inst: 102 | exc_type, exc_value, exc_traceback = sys.exc_info() 103 | traceback.print_tb(exc_traceback, limit=1, file=sys.stdout) 104 | print(type(inst)) 105 | print(inst) 106 | raise 107 | 108 | def generate_update_pgrnk(self, edges_stream, offset_edges, state_offset, offset_updates, 109 | vertices_stream, mem_output, bsp_phase): 110 | edge = struct.unpack('II', edges_stream[offset_edges:offset_edges+2*sizeof(c_uint)]) 111 | vindex = self.map_offset(edge[0], AlgorithmPagerank.super_partition_shift, AlgorithmPagerank.partition_shift) 112 | #print("State offset = ", state_offset) 113 | vertex = struct.unpack('Iff', vertices_stream[state_offset+vindex*sizeof(pgrnk_vertex):state_offset+vindex*sizeof(pgrnk_vertex)+sizeof(pgrnk_vertex)]) 114 | if bsp_phase == 0: 115 | v_degree = vertex[0]+1 116 | vertices_stream[state_offset+vindex*sizeof(pgrnk_vertex):state_offset+vindex*sizeof(pgrnk_vertex)+sizeof(c_uint)] = struct.pack('I', v_degree) 117 | return False 118 | else: 119 | mem_output[offset_updates:offset_updates + sizeof(c_uint)] = struct.pack('I', edge[1]) 120 | u_rank = vertex[1]/vertex[0] 121 | mem_output[offset_updates + sizeof(c_uint):offset_updates + sizeof(c_uint) + sizeof(c_float)] = struct.pack('f', u_rank) 122 | u_sum = 0 123 | vertices_stream[state_offset+vindex*sizeof(pgrnk_vertex)+sizeof(c_uint)+sizeof(c_float):state_offset+vindex*sizeof(pgrnk_vertex)+sizeof(c_uint)+2*sizeof(c_float)] = struct.pack('f',u_sum) 124 | return True 125 | 126 | def apply_one_update_pgrnk(self, updates, updates_offset, vertices_bytes, 127 | bsp_phase, start_updates, state_offset): 128 | update = struct.unpack('If', updates[start_updates+updates_offset:start_updates+updates_offset+sizeof(pgrnk_update)]) 129 | vindex = self.map_offset(update[0], AlgorithmPagerank.super_partition_shift, AlgorithmPagerank.partition_shift) 130 | vertex = struct.unpack('Iff', vertices_bytes[state_offset+vindex*sizeof(pgrnk_vertex):state_offset+vindex*sizeof(pgrnk_vertex)+sizeof(pgrnk_vertex)]) 131 | v_sum = vertex[2] + update[1] 132 | v_rank = 1 - AlgorithmPagerank.DAMPING_FACTOR + AlgorithmPagerank.DAMPING_FACTOR * v_sum 133 | vertices_bytes[state_offset+vindex*sizeof(pgrnk_vertex)+sizeof(c_uint):state_offset+vindex*sizeof(pgrnk_vertex)+sizeof(c_uint)+2*sizeof(c_float)]=struct.pack('ff', v_rank, v_sum) 134 | if bsp_phase==AlgorithmPagerank.niters: 135 | #print("RANK: ", v_rank) 136 | return False 137 | else: 138 | return True 139 | 140 | def map_offset(self, key, super_partition_shift, partition_shift): 141 | return key >> (super_partition_shift + partition_shift) 142 | 143 | def postprocessing_pagerank(self): 144 | print("Total time in python: ", AlgorithmPagerank.total_time_python) -------------------------------------------------------------------------------- /algorithms/pagerank/pagerank_ddf.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _PAGERANK_DDF_ 20 | #define _PAGERANK_DDF_ 21 | 22 | #include "../../core/x-lib.hpp" 23 | 24 | 25 | namespace algorithm { 26 | namespace sg_simple { 27 | template 28 | class pagerank_ddf { 29 | private: 30 | static const weight_t propagation_threshold = 0.5; 31 | struct vertex { 32 | vertex_t degree; 33 | weight_t rank; 34 | weight_t delta; 35 | unsigned long updates_pushed; 36 | } __attribute__((__packed__)); 37 | 38 | struct update { 39 | vertex_t target; 40 | weight_t rank; 41 | } __attribute__((__packed__)); 42 | 43 | static const weight_t DAMPING_FACTOR = 0.85; 44 | static unsigned long niters; 45 | 46 | public: 47 | 48 | static unsigned long vertex_state_bytes() { 49 | return sizeof(struct vertex); 50 | } 51 | 52 | static unsigned long split_size_bytes() { 53 | return sizeof(struct update); 54 | } 55 | 56 | static unsigned long split_key(unsigned char *buffer, unsigned long jump) { 57 | struct update *u = (struct update *) buffer; 58 | vertex_t key = u->target; 59 | key = key >> jump; 60 | return key; 61 | } 62 | 63 | static void preprocessing() { 64 | // The extra +1 is for the iteration 0 which is used just to compute the vertex degrees 65 | niters = 1 + vm["pagerank::niters"].as < unsigned 66 | long > (); 67 | } 68 | 69 | static bool init(unsigned char *vertex_state, 70 | unsigned long vertex_index, 71 | unsigned long bsp_phase, 72 | per_processor_data *cpu_state) { 73 | if (bsp_phase == 0) { 74 | struct vertex *vertices = (struct vertex *) vertex_state; 75 | vertices->degree = 0; 76 | vertices->updates_pushed = 0; 77 | vertices->rank = 1.0 - DAMPING_FACTOR; 78 | vertices->delta = 1.0; 79 | } 80 | return true; 81 | } 82 | 83 | static bool need_init(unsigned long bsp_phase) { 84 | return (bsp_phase < 2); 85 | } 86 | 87 | static bool apply_one_update(unsigned char *vertex_state, 88 | unsigned char *update_stream, 89 | per_processor_data *per_cpu_data, 90 | unsigned long bsp_phase) { 91 | struct update *u = (struct update *) update_stream; 92 | struct vertex *vertices = (struct vertex *) vertex_state; 93 | struct vertex *v = &vertices[x_lib::configuration::map_offset(u->target)]; 94 | 95 | v->delta += DAMPING_FACTOR * u->rank; 96 | if (bsp_phase == niters) { 97 | return false; 98 | } 99 | else if ((v->delta / v->degree) > propagation_threshold) { 100 | return true; 101 | } 102 | else { 103 | return false; 104 | } 105 | } 106 | 107 | static bool generate_update(unsigned char *vertex_state, 108 | unsigned char *edge_format, 109 | unsigned char *update_stream, 110 | per_processor_data *per_cpu_data, 111 | unsigned long bsp_phase) { 112 | vertex_t src, dst; 113 | F::read_edge(edge_format, src, dst); 114 | 115 | struct vertex *vertices = (struct vertex *) vertex_state; 116 | struct vertex *v = &vertices[x_lib::configuration::map_offset(src)]; 117 | 118 | if (bsp_phase == 0) { 119 | v->degree++; 120 | return false; 121 | } 122 | else { 123 | struct update *u = (struct update *) update_stream; 124 | weight_t out = v->delta / v->degree; 125 | if (out > propagation_threshold) { 126 | u->target = dst; 127 | u->rank = out; 128 | v->updates_pushed++; 129 | if (v->updates_pushed == v->degree) { 130 | v->rank = v->rank + v->delta; 131 | v->delta = 0; 132 | v->updates_pushed = 0; 133 | } 134 | return true; 135 | } 136 | else { 137 | return false; 138 | } 139 | } 140 | } 141 | 142 | static per_processor_data *create_per_processor_data(unsigned long processor_id) { 143 | return NULL; 144 | } 145 | 146 | static unsigned long min_super_phases() { 147 | return 2; 148 | } 149 | 150 | static void postprocessing() { } 151 | 152 | }; 153 | 154 | template 155 | unsigned long pagerank_ddf::niters; 156 | } 157 | } 158 | #endif 159 | -------------------------------------------------------------------------------- /algorithms/scan/degree_cnt.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _DEGREE_CNT_ 20 | #define _DEGREE_CNT_ 21 | 22 | #include "../../core/x-lib.hpp" 23 | #include "../../utils/options_utils.h" 24 | #include "../../utils/boost_log_wrapper.h" 25 | #include 26 | #include 27 | 28 | namespace algorithm { 29 | namespace sg_simple { 30 | template 31 | class degree_cnt { 32 | public: 33 | struct __attribute__((__packed__)) degree_cnts { 34 | unsigned long degree; 35 | }; 36 | 37 | static unsigned long split_size_bytes() { 38 | return 0; 39 | } 40 | 41 | static unsigned long split_key(unsigned char *buffer, 42 | unsigned long jump) { 43 | return 0; 44 | } 45 | 46 | static unsigned long vertex_state_bytes() { 47 | return sizeof(struct degree_cnts); 48 | } 49 | 50 | static bool apply_one_update(unsigned char *vertex_state, 51 | unsigned char *update_stream, 52 | per_processor_data *per_cpu_data, 53 | unsigned long bsp_phase) { 54 | BOOST_ASSERT_MSG(false, "Should not be called !"); 55 | return false; 56 | } 57 | 58 | static bool generate_update(unsigned char *vertex_state, 59 | unsigned char *edge_format, 60 | unsigned char *update_stream, 61 | per_processor_data *per_cpu_data, 62 | unsigned long bsp_phase) { 63 | vertex_t src, dst; 64 | F::read_edge(edge_format, src, dst); 65 | unsigned long vindex = x_lib::configuration::map_offset(src); 66 | struct degree_cnts *v = (struct degree_cnts *) vertex_state; 67 | v[vindex].degree++; 68 | return false; 69 | } 70 | 71 | static bool init(unsigned char *vertex_state, 72 | unsigned long vertex_index, 73 | unsigned long bsp_phase, 74 | per_processor_data *cpu_state) { 75 | struct degree_cnts *dc = (struct degree_cnts *) vertex_state; 76 | dc->degree = 0; 77 | return true; 78 | } 79 | 80 | static bool need_init(unsigned long bsp_phase) { 81 | return (bsp_phase == 0); 82 | } 83 | 84 | static void preprocessing() { } 85 | 86 | static void postprocessing() { } 87 | 88 | static per_processor_data * 89 | create_per_processor_data(unsigned long processor_id) { 90 | return NULL; 91 | } 92 | 93 | static unsigned long min_super_phases() { 94 | return 0; 95 | } 96 | 97 | }; 98 | } 99 | } 100 | 101 | #endif 102 | -------------------------------------------------------------------------------- /algorithms/scan/degree_cnt_max.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _DEGREE_CNT_MAX_ 20 | #define _DEGREE_CNT_MAX_ 21 | 22 | #include "../../core/x-lib.hpp" 23 | #include "../../utils/options_utils.h" 24 | #include "../../utils/boost_log_wrapper.h" 25 | #include 26 | #include 27 | 28 | namespace algorithm { 29 | namespace sg_simple { 30 | class degree_cnt_pcpu : public per_processor_data { 31 | public: 32 | static unsigned long max_degree; 33 | static vertex_t max_degree_vertex_id; 34 | unsigned long local_max_degree; 35 | vertex_t local_max_degree_vertex_id; 36 | 37 | degree_cnt_pcpu() 38 | : local_max_degree(0), 39 | local_max_degree_vertex_id((vertex_t) - 1) { } 40 | 41 | bool reduce(per_processor_data **per_cpu_array, 42 | unsigned long processors) { 43 | degree_cnt_pcpu *data = static_cast(per_cpu_array[0]); 44 | max_degree = data->local_max_degree; 45 | max_degree_vertex_id = data->local_max_degree_vertex_id; 46 | for (unsigned long i = 1; i < processors; i++) { 47 | data = static_cast(per_cpu_array[i]); 48 | if (data->local_max_degree > max_degree) { 49 | max_degree = data->local_max_degree; 50 | max_degree_vertex_id = data->local_max_degree_vertex_id; 51 | } 52 | } 53 | return false; 54 | } 55 | } __attribute__((__aligned__(64))); 56 | 57 | template 58 | class degree_cnt_max { 59 | public: 60 | struct __attribute__((__packed__)) degree_cnts { 61 | unsigned long degree; 62 | }; 63 | 64 | static unsigned long split_size_bytes() { 65 | return 0; 66 | } 67 | 68 | static unsigned long split_key(unsigned char *buffer, 69 | unsigned long jump) { 70 | return 0; 71 | } 72 | 73 | static unsigned long vertex_state_bytes() { 74 | return sizeof(struct degree_cnts); 75 | } 76 | 77 | static bool apply_one_update(unsigned char *vertex_state, 78 | unsigned char *update_stream, 79 | per_processor_data *per_cpu_data, 80 | unsigned long bsp_phase) { 81 | BOOST_ASSERT_MSG(false, "Should not be called !"); 82 | return false; 83 | } 84 | 85 | static bool generate_update(unsigned char *vertex_state, 86 | unsigned char *edge_format, 87 | unsigned char *update_stream, 88 | per_processor_data *per_cpu_data, 89 | unsigned long bsp_phase) { 90 | vertex_t src, dst; 91 | F::read_edge(edge_format, src, dst); 92 | unsigned long vindex = x_lib::configuration::map_offset(src); 93 | struct degree_cnts *v = (struct degree_cnts *) vertex_state; 94 | if (bsp_phase == 0) { 95 | v[vindex].degree++; 96 | } 97 | return false; 98 | } 99 | 100 | static bool init(unsigned char *vertex_state, 101 | unsigned long vertex_index, 102 | unsigned long bsp_phase, 103 | per_processor_data *cpu_state) { 104 | struct degree_cnts *dc = (struct degree_cnts *) vertex_state; 105 | if (bsp_phase == 0) { 106 | dc->degree = 0; 107 | return true; 108 | } else if (bsp_phase == 1) { 109 | degree_cnt_pcpu *pcpu = static_cast(cpu_state); 110 | if (dc->degree > pcpu->local_max_degree) { 111 | pcpu->local_max_degree = dc->degree; 112 | pcpu->local_max_degree_vertex_id = vertex_index; 113 | } 114 | } 115 | return false; 116 | } 117 | 118 | static bool need_init(unsigned long bsp_phase) { 119 | return (bsp_phase < 2); 120 | } 121 | 122 | static void preprocessing() { } 123 | 124 | static void postprocessing() { 125 | BOOST_LOG_TRIVIAL(info) << "ALGORITHM::DEGREE_CNT::MAX_DEGREE " 126 | << degree_cnt_pcpu::max_degree; 127 | BOOST_LOG_TRIVIAL(info) << "ALGORITHM::DEGREE_CNT::VERTEX_WITH_MAX_DEGREE " 128 | << degree_cnt_pcpu::max_degree_vertex_id; 129 | 130 | } 131 | 132 | static per_processor_data * 133 | create_per_processor_data(unsigned long processor_id) { 134 | return new degree_cnt_pcpu(); 135 | } 136 | 137 | static unsigned long min_super_phases() { 138 | return 2; 139 | } 140 | }; 141 | 142 | unsigned long degree_cnt_pcpu::max_degree = 0; 143 | vertex_t degree_cnt_pcpu::max_degree_vertex_id = (vertex_t) - 1; 144 | } 145 | } 146 | 147 | #endif 148 | -------------------------------------------------------------------------------- /algorithms/scan/noop.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _NOOP_ 20 | #define _NOOP_ 21 | 22 | #include "../../core/x-lib.hpp" 23 | #include "../../utils/options_utils.h" 24 | #include "../../utils/boost_log_wrapper.h" 25 | 26 | namespace algorithm { 27 | namespace sg_simple { 28 | 29 | class noop_pcpu : public per_processor_data { 30 | public: 31 | noop_pcpu() { } 32 | 33 | bool reduce(per_processor_data **per_cpu_array, unsigned long processors) { 34 | return false; 35 | } 36 | } __attribute__((__aligned__(64))); 37 | 38 | template 39 | class noop { 40 | private: 41 | struct vertex { 42 | } __attribute__((__packed__)); 43 | 44 | struct update { 45 | vertex_t target; 46 | } __attribute__((__packed__)); 47 | 48 | public: 49 | static unsigned long vertex_state_bytes() { 50 | return sizeof(struct vertex); 51 | } 52 | 53 | static unsigned long split_size_bytes() { 54 | return sizeof(struct update); 55 | } 56 | 57 | static unsigned long split_key(unsigned char *buffer, unsigned long jump) { 58 | struct update *u = (struct update *) buffer; 59 | vertex_t key = u->target; 60 | return key >> jump; 61 | } 62 | 63 | static unsigned long min_super_phases() { 64 | return 1; 65 | } 66 | 67 | static bool need_init(unsigned long bsp_phase) { 68 | return bsp_phase == 0; 69 | } 70 | 71 | static bool init(unsigned char *vertex_state, 72 | unsigned long vertex_index, 73 | unsigned long bsp_phase, 74 | per_processor_data *cpu_state) { 75 | struct vertex *vertices = (struct vertex *) vertex_state; 76 | (void) vertices; 77 | return true; 78 | } 79 | 80 | static bool apply_one_update(unsigned char *vertex_state, 81 | unsigned char *update_stream, 82 | per_processor_data *per_cpu_data, 83 | unsigned long bsp_phase) { 84 | struct update *u = (struct update *) update_stream; 85 | struct vertex *vertices = (struct vertex *) vertex_state; 86 | struct vertex *v = &vertices[x_lib::configuration::map_offset(u->target)]; 87 | 88 | (void) u; 89 | (void) vertices; 90 | (void) v; 91 | 92 | return false; 93 | } 94 | 95 | static bool generate_update(unsigned char *vertex_state, 96 | unsigned char *edge_stream, 97 | unsigned char *update_stream, 98 | per_processor_data *per_cpu_data, 99 | unsigned long bsp_phase) { 100 | vertex_t src, dst; 101 | F::read_edge(edge_stream, src, dst); 102 | 103 | struct vertex *vertices = (struct vertex *) vertex_state; 104 | struct vertex *v = &vertices[x_lib::configuration::map_offset(src)]; 105 | (void) vertices; 106 | (void) v; 107 | 108 | struct update *u = (struct update *) update_stream; 109 | u->target = dst; 110 | 111 | return false; 112 | } 113 | 114 | static void preprocessing() { } 115 | 116 | static void postprocessing() { } 117 | 118 | static per_processor_data *create_per_processor_data(unsigned long proc_id) { 119 | return NULL; 120 | } 121 | }; 122 | 123 | } // namespace sg_simple 124 | } // namespace algorithm 125 | 126 | #endif // _NOOP_ 127 | -------------------------------------------------------------------------------- /algorithms/scan/range_check.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _RANGE_CHECK_ 20 | #define _RANGE_CHECK_ 21 | 22 | #include "../../core/x-lib.hpp" 23 | #include "../../utils/options_utils.h" 24 | #include "../../utils/boost_log_wrapper.h" 25 | #include 26 | #include 27 | 28 | namespace algorithm { 29 | namespace sg_simple { 30 | // Assumes a partition between even numbered vertices (black) and 31 | // odd numbered vertices (red) 32 | template 33 | class range_check { 34 | static vertex_t max_vertex_id; 35 | public: 36 | static unsigned long split_size_bytes() { 37 | return 1; 38 | } 39 | 40 | static unsigned long split_key(unsigned char *buffer, 41 | unsigned long jump) { 42 | return 0; 43 | } 44 | 45 | static unsigned long vertex_state_bytes() { 46 | return 0; 47 | } 48 | 49 | static bool apply_one_update(unsigned char *vertex_state, 50 | unsigned char *update_stream, 51 | per_processor_data *per_cpu_data, 52 | unsigned long bsp_phase) { 53 | BOOST_ASSERT_MSG(false, "Should not be called !"); 54 | return false; 55 | } 56 | 57 | static bool generate_update(unsigned char *vertex_state, 58 | unsigned char *edge_format, 59 | unsigned char *update_stream, 60 | per_processor_data *per_cpu_data, 61 | unsigned long bsp_phase) { 62 | vertex_t src, dst; 63 | F::read_edge(edge_format, src, dst); 64 | if ((src > max_vertex_id) || (dst > max_vertex_id)) { 65 | BOOST_LOG_TRIVIAL(fatal) << "Range check failed for edge:" 66 | << src << "->" << dst; 67 | exit(-1); 68 | } 69 | return false; 70 | } 71 | 72 | static bool init(unsigned char *vertex_state, 73 | unsigned long vertex_id, 74 | unsigned long bsp_phase, 75 | per_processor_data *cpu_state) { 76 | return true; 77 | } 78 | 79 | static bool need_init(unsigned long bsp_phase) { 80 | return (bsp_phase == 0); 81 | } 82 | 83 | static void preprocessing() { 84 | max_vertex_id = pt.get("graph.vertices") - 1; 85 | } 86 | 87 | static void postprocessing() { 88 | } 89 | 90 | static per_processor_data * 91 | create_per_processor_data(unsigned long processor_id) { 92 | return NULL; 93 | } 94 | 95 | static unsigned long min_super_phases() { 96 | return 1; 97 | } 98 | }; 99 | 100 | template 101 | vertex_t range_check::max_vertex_id = 0; 102 | } 103 | } 104 | #endif 105 | -------------------------------------------------------------------------------- /algorithms/scan/reverse.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _REVERSE_ 20 | #define _REVERSE_ 21 | 22 | #include "../../core/x-lib.hpp" 23 | 24 | // Note: Don't forget to truncate the output graph to (edges * F::split_size_bytes()) 25 | 26 | namespace algorithm { 27 | namespace reverse { 28 | 29 | template 30 | class reverse { 31 | x_lib::streamIO *graph_storage; 32 | unsigned long in_stream; 33 | unsigned long out_stream; 34 | 35 | rtc_clock wall_clock; 36 | rtc_clock setup_time; 37 | 38 | public: 39 | reverse(); 40 | 41 | static void partition_pre_callback(unsigned long super_partition, 42 | unsigned long partition, 43 | per_processor_data *cpu_state) { } 44 | 45 | static void partition_callback(x_lib::stream_callback_state *state); 46 | 47 | static void partition_post_callback(unsigned long super_partition, 48 | unsigned long partition, 49 | per_processor_data *cpu_state) { } 50 | 51 | static void do_cpu_callback(per_processor_data *cpu_state) { } 52 | 53 | static void state_iter_callback(unsigned long superp, 54 | unsigned long partition, 55 | unsigned long index, 56 | unsigned char *vertex, 57 | per_processor_data *cpu_state) { } 58 | 59 | void operator()(); 60 | 61 | static unsigned long max_streams() { 62 | return 2; 63 | } 64 | 65 | static unsigned long max_buffers() { 66 | return 4; 67 | } 68 | 69 | static unsigned long vertex_state_bytes() { 70 | return 0; 71 | } 72 | 73 | static unsigned long vertex_stream_buffer_bytes() { 74 | return F::split_size_bytes(); 75 | } 76 | 77 | static per_processor_data *create_per_processor_data(unsigned long processor_id) { 78 | return NULL; 79 | } 80 | }; 81 | 82 | template 83 | reverse::reverse() { 84 | wall_clock.start(); 85 | setup_time.start(); 86 | graph_storage = new x_lib::streamIO(); 87 | if (graph_storage->get_config()->super_partitions > 1) { 88 | BOOST_LOG_TRIVIAL(fatal) << "ALGORITHM::REVERSE::WORKS_ONLY_WITH_1_SUPER_PARTITION"; 89 | exit(-1); 90 | } 91 | std::string infile = pt.get("graph.name"); 92 | std::string outfile = infile + "-rev"; 93 | in_stream = graph_storage->open_stream((const char *) infile.c_str(), false, 94 | vm["input_disk"].as < unsigned 95 | long > (), 96 | F::split_size_bytes(), 1); 97 | out_stream = graph_storage->open_stream((const char *) outfile.c_str(), true, 98 | vm["output_disk"].as < unsigned 99 | long > (), 100 | F::split_size_bytes()); 101 | setup_time.stop(); 102 | } 103 | 104 | template 105 | struct edge_type_wrapper { 106 | static unsigned long item_size() { 107 | return F::split_size_bytes(); 108 | } 109 | 110 | static unsigned long key(unsigned char *buffer) { 111 | return F::split_key(buffer, 0); 112 | } 113 | }; 114 | 115 | template 116 | void reverse::operator()() { 117 | x_lib::do_stream, 118 | edge_type_wrapper, 119 | edge_type_wrapper > 120 | (graph_storage, 0, in_stream, out_stream, NULL); 121 | graph_storage->rewind_stream(in_stream); 122 | graph_storage->rewind_stream(out_stream); 123 | 124 | setup_time.start(); 125 | graph_storage->terminate(); 126 | setup_time.stop(); 127 | wall_clock.stop(); 128 | 129 | setup_time.print("CORE::TIME::SETUP"); 130 | wall_clock.print("CORE::TIME::WALL"); 131 | } 132 | 133 | template 134 | void reverse::partition_callback(x_lib::stream_callback_state *callback) { 135 | while (callback->bytes_in) { 136 | if (callback->bytes_out + F::split_size_bytes() > callback->bytes_out_max) { 137 | break; 138 | } 139 | vertex_t src, dst; 140 | weight_t value; 141 | F::read_edge(callback->bufin, src, dst, value); 142 | callback->bufin += F::split_size_bytes(); 143 | callback->bytes_in -= F::split_size_bytes(); 144 | F::write_edge(callback->bufout, dst, src, value); 145 | callback->bufout += F::split_size_bytes(); 146 | callback->bytes_out += F::split_size_bytes(); 147 | } 148 | } 149 | } 150 | } 151 | 152 | #endif 153 | -------------------------------------------------------------------------------- /algorithms/spmv/spmv.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _SPMV_ 20 | #define _SPMV_ 21 | 22 | #include "../../core/x-lib.hpp" 23 | #include "../../utils/options_utils.h" 24 | #include "../../utils/boost_log_wrapper.h" 25 | #include 26 | #include 27 | // Compute Y=XA 28 | namespace algorithm { 29 | namespace sg_simple { 30 | template 31 | class spmv { 32 | public: 33 | static unsigned long checkpoint_size() { 34 | return 0; 35 | } 36 | 37 | static void take_checkpoint(unsigned char *buffer, 38 | per_processor_data **cpu_array, 39 | unsigned long num_processors) { 40 | } 41 | 42 | static void restore_checkpoint(unsigned char *buffer, 43 | per_processor_data **cpu_array, 44 | unsigned long num_processors) { 45 | } 46 | 47 | struct __attribute__((__packed__)) term { 48 | vertex_t column; 49 | weight_t value; 50 | }; 51 | 52 | struct __attribute__((__packed__)) vector_element { 53 | weight_t value_out; 54 | weight_t value_in; 55 | }; 56 | 57 | static unsigned long split_size_bytes() { 58 | return sizeof(struct term); 59 | } 60 | 61 | static unsigned long split_key(unsigned char *buffer, 62 | unsigned long jump) { 63 | struct term *t = (struct term *) buffer; 64 | vertex_t key = t->column; 65 | key = key >> jump; 66 | return key; 67 | } 68 | 69 | static unsigned long vertex_state_bytes() { 70 | return sizeof(struct vector_element); 71 | } 72 | 73 | static 74 | bool need_data_barrier() { 75 | return false; 76 | } 77 | 78 | class db_sync { 79 | public: 80 | void prep_db_data(per_processor_data **pcpu_array, 81 | unsigned long me, 82 | unsigned long processors) { } 83 | 84 | void finalize_db_data(per_processor_data **pcpu_array, 85 | unsigned long me, 86 | unsigned long processors) { } 87 | 88 | unsigned char *db_buffer() { return NULL; } 89 | 90 | unsigned long db_size() { return 0; } 91 | 92 | void db_generate() { } 93 | 94 | void db_merge() { } 95 | 96 | void db_absorb() { } 97 | }; 98 | 99 | static 100 | db_sync *get_db_sync() { return NULL; } 101 | 102 | static bool need_scatter_merge(unsigned long bsp_phase) { 103 | return false; 104 | } 105 | 106 | static void vertex_apply(unsigned char *v, 107 | unsigned char *copy, 108 | unsigned long copy_machine, 109 | per_processor_data *per_cpu_data, 110 | unsigned long bsp_phase) { 111 | struct vector_element *vtx = (struct vector_element *) v; 112 | struct vector_element *vtx_cpy = (struct vector_element *) copy; 113 | vtx->value_out += vtx_cpy->value_out; 114 | } 115 | 116 | static bool apply_one_update(unsigned char *vertex_state, 117 | unsigned char *update_stream, 118 | per_processor_data *per_cpu_data, 119 | bool local_tile, 120 | unsigned long bsp_phase) { 121 | 122 | struct term *t = (struct term *) update_stream; 123 | unsigned long vindex = x_lib::configuration::map_offset(t->column); 124 | struct vector_element *vect = (struct vector_element *) vertex_state; 125 | vect[vindex].value_out += t->value; 126 | return false; 127 | } 128 | 129 | static bool generate_update(unsigned char *vertex_state, 130 | unsigned char *edge_format, 131 | unsigned char *update_stream, 132 | per_processor_data *per_cpu_data, 133 | bool local_tile, 134 | unsigned long bsp_phase) { 135 | vertex_t row, column; 136 | weight_t matrix_element; 137 | F::read_matrix_element(edge_format, row, column, matrix_element); 138 | unsigned long vindex = x_lib::configuration::map_offset(row); 139 | struct vector_element *vector_input = 140 | (struct vector_element *) vertex_state; 141 | struct term *t = (struct term *) update_stream; 142 | t->column = column; 143 | t->value = matrix_element * vector_input[vindex].value_in; 144 | return true; 145 | } 146 | 147 | static bool init(unsigned char *vertex_state, 148 | unsigned long vertex_index, 149 | unsigned long bsp_phase, 150 | per_processor_data *cpu_state) { 151 | struct vector_element *vec = (struct vector_element *) vertex_state; 152 | vec->value_out = 0; 153 | vec->value_in = vertex_index; 154 | return true; 155 | } 156 | 157 | static bool need_init(unsigned long bsp_phase) { 158 | return (bsp_phase == 0); 159 | } 160 | 161 | static void postprocessing() { } 162 | 163 | static void preprocessing() { } 164 | 165 | static per_processor_data * 166 | create_per_processor_data(unsigned long processor_id, 167 | unsigned long machines) { 168 | return NULL; 169 | } 170 | 171 | static unsigned long min_super_phases() { 172 | return 1; 173 | } 174 | 175 | }; 176 | } 177 | } 178 | 179 | #endif 180 | -------------------------------------------------------------------------------- /algorithms/sssp/sssp.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _SSSP_ 20 | #define _SSSP_ 21 | 22 | #include "../../utils/options_utils.h" 23 | #include "../../core/x-lib.hpp" 24 | #include 25 | 26 | namespace algorithm { 27 | namespace sg_simple { 28 | template 29 | class sssp { 30 | static unsigned long sssp_source; 31 | public: 32 | 33 | struct vertex { 34 | vertex_t predecessor; 35 | weight_t distance; 36 | unsigned long active_phase; 37 | } __attribute__((__packed__)); 38 | 39 | struct update { 40 | vertex_t target; 41 | vertex_t predecessor; 42 | weight_t distance; 43 | } __attribute__((__packed__)); 44 | 45 | static unsigned long vertex_state_bytes() { 46 | return sizeof(struct vertex); 47 | } 48 | 49 | static unsigned long split_size_bytes() { 50 | return sizeof(struct update); 51 | } 52 | 53 | static unsigned long split_key(unsigned char *buffer, unsigned long jump) { 54 | struct update *u = (struct update *) buffer; 55 | vertex_t key = u->target; 56 | key = key >> jump; 57 | return key; 58 | } 59 | 60 | static void preprocessing() { 61 | sssp_source = vm["sssp::source"].as < unsigned 62 | long > (); 63 | } 64 | 65 | static bool init(unsigned char *vertex_state, 66 | unsigned long vertex_index, 67 | unsigned long bsp_phase, 68 | per_processor_data *cpu_state) { 69 | bool should_start = false; 70 | struct vertex *sssp_vertex = (struct vertex *) vertex_state; 71 | sssp_vertex->predecessor = (vertex_t) - 1; 72 | sssp_vertex->distance = (weight_t) std::numeric_limits::max(); 73 | sssp_vertex->active_phase = -1UL; 74 | if (vertex_index == sssp_source) { 75 | should_start = true; 76 | sssp_vertex->distance = 0; 77 | sssp_vertex->active_phase = 0; 78 | } 79 | return should_start; 80 | } 81 | 82 | static bool need_init(unsigned long bsp_phase) { 83 | return (bsp_phase == 0); 84 | } 85 | 86 | static bool apply_one_update(unsigned char *vertex_state, 87 | unsigned char *update_stream, 88 | per_processor_data *per_cpu_data, 89 | unsigned long bsp_phase) { 90 | struct update *u = (struct update *) update_stream; 91 | struct vertex *vertices = (struct vertex *) vertex_state; 92 | struct vertex *v = &vertices[x_lib::configuration::map_offset(u->target)]; 93 | if (u->distance < v->distance) { 94 | v->predecessor = u->predecessor; 95 | v->distance = u->distance; 96 | v->active_phase = bsp_phase; 97 | return true; 98 | } else { 99 | return false; 100 | } 101 | } 102 | 103 | static bool generate_update(unsigned char *vertex_state, 104 | unsigned char *edge_format, 105 | unsigned char *update_stream, 106 | per_processor_data *per_cpu_data, 107 | unsigned long bsp_phase) { 108 | vertex_t src, dst; 109 | weight_t edge_distance; 110 | F::read_edge(edge_format, src, dst, edge_distance); 111 | 112 | struct vertex *vertices = (struct vertex *) vertex_state; 113 | struct vertex *v = &vertices[x_lib::configuration::map_offset(src)]; 114 | if (v->active_phase == bsp_phase) { 115 | struct update *u = (struct update *) update_stream; 116 | u->target = dst; 117 | u->predecessor = src; 118 | u->distance = v->distance + edge_distance; 119 | return true; 120 | } else { 121 | return false; 122 | } 123 | } 124 | 125 | // not used 126 | static void postprocessing() { } 127 | 128 | static per_processor_data * 129 | create_per_processor_data(unsigned long processor_id) { 130 | return NULL; 131 | } 132 | 133 | static unsigned long min_super_phases() { 134 | return 1; 135 | } 136 | 137 | }; 138 | 139 | template 140 | unsigned long sssp::sssp_source = ULONG_MAX; 141 | } 142 | } 143 | #endif 144 | -------------------------------------------------------------------------------- /algorithms/template.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _XXX_ 20 | #define _XXX_ 21 | 22 | namespace algorithm { 23 | 24 | template 25 | class XXX { 26 | private: 27 | 28 | struct vertex { 29 | } __attribute__((__packed__)); 30 | 31 | struct update { 32 | vertex_t target; 33 | } __attribute__((__packed__)); 34 | 35 | public: 36 | 37 | static unsigned long vertex_state_bytes() { 38 | return sizeof(struct vertex); 39 | } 40 | 41 | static unsigned long split_size_bytes() { 42 | return sizeof(struct update); 43 | } 44 | 45 | static unsigned long split_key(unsigned char *buffer, unsigned long jump) { 46 | struct update *u = (struct update *) buffer; 47 | vertex_t key = u->target; 48 | key = key >> jump; 49 | return key; 50 | } 51 | 52 | static bool need_init(unsigned long bsp_phase) { 53 | return (bsp_phase == 0); 54 | } 55 | 56 | // Return true if the partition partition_id should be active in the first scatter phase 57 | static bool init(unsigned char *vertex_state, 58 | unsigned long global_partition_id, 59 | unsigned long vertex_cnt, 60 | unsigned long bsp_phase) { 61 | struct vertex *vertices = (struct vertex *) vertex_state; 62 | for (unsigned long i = 0; i < vertex_cnt; ++i) { 63 | vertices[i] 64 | } 65 | 66 | return true; 67 | } 68 | 69 | // Return true if the vertex should generate an update in the next phase 70 | // true here means that the vertex's partition will be active in the next scatter phase 71 | static bool apply_one_update(unsigned char *vertex_state, 72 | unsigned char *update_stream, 73 | per_processor_data *per_cpu_data, 74 | unsigned long bsp_phase) { 75 | struct update *u = (struct update *) update_stream; 76 | struct vertex *vertices = (struct vertex *) vertex_state; 77 | struct vertex *v = &vertices[ce_bsp::map_vertex_to_pindex(u->target)]; 78 | 79 | return true; 80 | } 81 | 82 | // Return true if update generated 83 | static bool generate_update(unsigned char *vertex_state, 84 | unsigned char *edge_format, 85 | unsigned char *update_stream, 86 | per_processor_data *per_cpu_data, 87 | unsigned long bsp_phase) { 88 | unsigned long src, dst; 89 | weight_t value; 90 | F::read_edge(edge_format, src, dst, value); 91 | 92 | struct vertex *vertices = (struct vertex *) vertex_state; 93 | struct vertex *v = &vertices[ce_bsp::map_vertex_to_pindex(src)]; 94 | 95 | struct update *u = (struct update *) update_stream; 96 | u->target = dst; 97 | 98 | return true; 99 | } 100 | 101 | static per_processor_data *create_per_processor_data(unsigned long processor_id) { 102 | return NULL; 103 | } 104 | 105 | static unsigned long min_super_phases() { 106 | return 1; 107 | } 108 | 109 | static void postprocessing() { } 110 | 111 | }; 112 | } 113 | 114 | #endif 115 | -------------------------------------------------------------------------------- /core/core.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include 20 | #include "x-lib.hpp" 21 | #include "sg_driver.hpp" 22 | //#include "sg_driver_async.hpp" 23 | namespace x_lib { 24 | unsigned long configuration::cached_partitions = 0; 25 | unsigned long configuration::partition_shift = 0; 26 | unsigned long configuration::tiles = 0; 27 | unsigned long configuration::tile_shift = 0; 28 | unsigned long configuration::super_partitions = 0; 29 | unsigned long configuration::super_partition_shift = 0; 30 | unsigned long configuration::ext_mem_bits = 0; 31 | unsigned long configuration::ext_fanout_bits = 0; 32 | unsigned long map_spshift_wrap::map_spshift = 100; 33 | 34 | x_barrier *x_thread::sync; 35 | volatile bool x_thread::terminate = false; 36 | struct work_base *volatile x_thread::work_to_do = NULL; 37 | unsigned long **memory_buffer::aux_index = NULL; 38 | unsigned char *memory_buffer::aux_buffer = NULL; 39 | memory_buffer *volatile memory_buffer::freelist = NULL; 40 | volatile unsigned long memory_buffer::freelist_lock = 0; 41 | #ifdef PYTHON_SUPPORT 42 | PyObject * memory_buffer::auxpBuffer = NULL; 43 | #endif 44 | bool memory_buffer::use_qsort = false; 45 | bool memory_buffer::do_batch_io = false; 46 | bool memory_buffer::gather_io_drain = false; 47 | unsigned long qsort_keys = 0; 48 | } 49 | 50 | namespace algorithm { 51 | unsigned long sg_pcpu::bsp_phase; 52 | unsigned long sg_pcpu::current_step; 53 | sg_pcpu **sg_pcpu::per_cpu_array; 54 | x_barrier *sg_pcpu::sync; 55 | x_lib::filter *sg_pcpu::scatter_filter; 56 | per_processor_data **sg_pcpu::algo_pcpu_array; 57 | bool sg_pcpu::do_algo_reduce; 58 | //unsigned long sg_async_pcpu::bsp_phase; 59 | //unsigned long sg_async_pcpu::current_step; 60 | //sg_async_pcpu ** sg_async_pcpu::per_cpu_array; 61 | //x_barrier *sg_async_pcpu::sync; 62 | //per_processor_data ** sg_async_pcpu::algo_pcpu_array; 63 | //bool sg_async_pcpu::do_algo_reduce; 64 | } 65 | 66 | namespace slipstore { 67 | slipstore::client_fill *slipstore_client_fill = NULL; 68 | slipstore::client_drain *slipstore_client_drain = NULL; 69 | slipstore::client_barrier *slipstore_client_barrier = NULL; 70 | slipstore::server *slipstore_server = NULL; 71 | slipstore::busy_counter *have_outstanding_request = NULL; 72 | boost::thread *server_thread = NULL; 73 | unsigned long baseport = 0; 74 | void *zmq_context = NULL; 75 | void *zmq_context_data_out = NULL; 76 | void *zmq_context_data_in = NULL; 77 | unsigned long quota = 0; 78 | unsigned long usage = 0; 79 | unsigned long max_quota_usage = 0; 80 | boost::asio::io_service ioService; 81 | boost::asio::io_service::work work(ioService); 82 | boost::thread_group threadpool; 83 | boost::asio::io_service dioService; 84 | boost::asio::io_service::work dio_work(dioService); 85 | boost::thread_group diopool; 86 | }; 87 | -------------------------------------------------------------------------------- /core/types.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _TYPES_ 20 | #define _TYPES_ 21 | #ifdef COMPACT_GRAPH 22 | typedef unsigned int vertex_t; 23 | typedef float weight_t; 24 | #else 25 | typedef unsigned long vertex_t; 26 | typedef double weight_t; 27 | #endif 28 | #endif 29 | -------------------------------------------------------------------------------- /core/x-ingest.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _X_INGEST_ 20 | #define _X_INGEST_ 21 | 22 | #include 23 | #include 24 | #include "../utils/boost_log_wrapper.h" 25 | 26 | struct ingest_t { 27 | unsigned long max; 28 | volatile unsigned long avail; 29 | volatile bool eof; 30 | unsigned char buffer[0]; 31 | }; 32 | 33 | static 34 | ingest_t *attach_ingest(unsigned long key) { 35 | void *data = shmat(shmget(key, 1, S_IRWXU), NULL, 0); 36 | if (data == MAP_FAILED) { 37 | perror("Failed to attach to ingest SHM:"); 38 | exit(-1); 39 | } 40 | return (ingest_t *) data; 41 | } 42 | 43 | static 44 | ingest_t *create_ingest_buffer(unsigned long key, 45 | unsigned long size) { 46 | int shmid = shmget(key, 47 | sizeof(ingest_t) + size, 48 | S_IRWXU | IPC_CREAT); 49 | if (shmid == -1) { 50 | perror("Failed to create delta segment:"); 51 | exit(-1); 52 | } 53 | ingest_t *ingest = attach_ingest(key); 54 | ingest->max = size; 55 | ingest->avail = 0; 56 | ingest->eof = false; 57 | BOOST_LOG_TRIVIAL(info) << "Ingest buffer ready"; 58 | return ingest; 59 | } 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /format/TYPES: -------------------------------------------------------------------------------- 1 | # | Short description 2 | -------------------------------------------------------------------------------- 3 | 1 | Only edge lists. Entries are triples of the form: 4 | | <8 byte src, 8 byte dst, 8 byte double typed weight> 5 | -------------------------------------------------------------------------------- 6 | 1 | Only edge lists. Entries are triples of the form: 7 | COMPACT | <4 byte src, 4 byte dst, 4 byte float typed weight> 8 | -------------------------------------------------------------------------------- 9 | 2 | Only edge lists. Entries are tuples of the form: 10 | | <8 byte src, 8 byte dst> 11 | -------------------------------------------------------------------------------- 12 | 2 | Only edge lists. Entries are tuples of the form: 13 | COMPACT | <4 byte src, 4 byte dst> 14 | -------------------------------------------------------------------------------- -------------------------------------------------------------------------------- /format/tools/matrix_market2type_1.py: -------------------------------------------------------------------------------- 1 | #Convert matrix market files into x-stream type 1 inputs 2 | #Arguments 3 | import sys 4 | import struct 5 | import random 6 | 7 | #Choose one 8 | add_rev_edges = False 9 | #add_rev_edges = True 10 | 11 | random.seed(0) 12 | 13 | infile=file(sys.argv[1], "r") 14 | outfile=file(sys.argv[2], "wtb") 15 | outfile_meta=file(sys.argv[2]+".ini", "wt") 16 | 17 | outfile_meta.write("[graph]\n") 18 | outfile_meta.write("type=1\n") 19 | outfile_meta.write("name="+sys.argv[2]+"\n") 20 | 21 | s = struct.Struct('@IIf') 22 | 23 | for line in infile: 24 | if line[0] == '%': 25 | pass 26 | else: 27 | vector = line.strip().split(" ") 28 | if len(vector) == 3: 29 | outfile_meta.write("vertices="+vector[0]+"\n") 30 | if add_rev_edges: 31 | vector[2] = str(2 * int(vector[2])) 32 | outfile_meta.write("edges="+vector[2]+"\n") 33 | else: 34 | outfile_meta.write("edges="+vector[2]+"\n") 35 | 36 | else: 37 | vector = list(map(int, vector)) 38 | vector[0] = vector[0] - 1 39 | vector[1] = vector[1] - 1 40 | vector.append(random.random()) # Edge weight 41 | outfile.write(s.pack(*vector)) 42 | if add_rev_edges: 43 | tmp = vector[0] 44 | vector[0] = vector[1] 45 | vector[1] = tmp 46 | outfile.write(s.pack(*vector)) 47 | 48 | infile.close() 49 | outfile.close() 50 | outfile_meta.close() 51 | -------------------------------------------------------------------------------- /format/tools/netflix2type_1.py: -------------------------------------------------------------------------------- 1 | #Convert netflix prize files into x-stream type 1 inputs 2 | #Arguments 3 | import sys 4 | import struct 5 | import random 6 | 7 | #Choose one 8 | add_rev_edges = False 9 | #add_rev_edges = True 10 | 11 | movies = 17770 12 | users = 480189 13 | user_ids = 2649429 14 | user_id_map = [] 15 | 16 | for i in range(user_ids): 17 | user_id_map.append(-1) 18 | 19 | outfile=file(sys.argv[1], "wtb") 20 | outfile_meta=file(sys.argv[1]+".ini", "wt") 21 | outfile_meta.write("[graph]\n") 22 | outfile_meta.write("type=1\n") 23 | outfile_meta.write("name="+sys.argv[1]+"\n") 24 | outfile_meta.write("vertices=" + str(users+movies) + "\n") 25 | s = struct.Struct('@IIf') 26 | 27 | edges = 0 28 | 29 | for i in range(movies): 30 | infile=file("mv_" + str(i+1).zfill(7) + ".txt", "r") 31 | infile.readline() 32 | for line in infile: 33 | vector = line.strip().split(",") 34 | vec_out = [0, 0, 0] 35 | vec_out[0] = i 36 | user_id = int(vector[0]) - 1 37 | if user_id_map[user_id] == -1: 38 | user_id_map[user_id] = users - 1 39 | users = users - 1 40 | vec_out[1] = user_id_map[user_id] + movies 41 | vec_out[2] = float(vector[1]) 42 | outfile.write(s.pack(*vec_out)) 43 | edges = edges + 1 44 | if add_rev_edges: 45 | tmp = vec_out[0] 46 | vec_out[0] = vec_out[1] 47 | vec_out[1] = tmp 48 | outfile.write(s.pack(*vec_out)) 49 | edges = edges + 1 50 | infile.close() 51 | 52 | outfile.close() 53 | outfile_meta.write("edges=" + str(edges) + "\n") 54 | outfile_meta.close() 55 | 56 | -------------------------------------------------------------------------------- /format/tools/snap2type_1.py: -------------------------------------------------------------------------------- 1 | #Convert snap files into x-stream type 1 inputs 2 | #Arguments 3 | import sys 4 | import struct 5 | import random 6 | 7 | #Choose one 8 | #add_rev_edges = False 9 | add_rev_edges = True 10 | 11 | random.seed(0) 12 | 13 | infile=file(sys.argv[1], "r") 14 | outfile=file(sys.argv[2], "wtb") 15 | outfile_meta=file(sys.argv[2]+".ini", "wt") 16 | 17 | outfile_meta.write("[graph]\n") 18 | outfile_meta.write("type=1\n") 19 | outfile_meta.write("name="+sys.argv[2]+"\n") 20 | 21 | s = struct.Struct('@IIf') 22 | edges=0 23 | vertices=0 24 | for line in infile: 25 | if line[0] == '#': 26 | pass 27 | else: 28 | vector = line.strip().split("\t") 29 | vector = list(map(int, vector)) 30 | vector.append(random.random()) # Edge weight 31 | if vector[0] > vertices: 32 | vertices = vector[0] 33 | if vector[1] > vertices: 34 | vertices = vector[1] 35 | edges = edges + 1 36 | outfile.write(s.pack(*vector)) 37 | if add_rev_edges: 38 | edges = edges + 1 39 | tmp = vector[0] 40 | vector[0] = vector[1] 41 | vector[1] = tmp 42 | outfile.write(s.pack(*vector)) 43 | 44 | vertices = vertices + 1 45 | outfile_meta.write("vertices="+str(vertices)+"\n") 46 | outfile_meta.write("edges="+str(edges)+"\n") 47 | infile.close() 48 | outfile.close() 49 | outfile_meta.close() 50 | -------------------------------------------------------------------------------- /format/tools/twitter2type_1.py: -------------------------------------------------------------------------------- 1 | #Convert twitter files into x-stream type 1 inputs 2 | #Arguments 3 | import sys 4 | import struct 5 | import random 6 | 7 | #Choose one 8 | add_rev_edges = False 9 | #add_rev_edges = True 10 | 11 | random.seed(0) 12 | 13 | infile=file(sys.argv[1], "r") 14 | outfile=file(sys.argv[2], "wtb") 15 | outfile_meta=file(sys.argv[2]+".ini", "wt") 16 | 17 | outfile_meta.write("[graph]\n") 18 | outfile_meta.write("type=1\n") 19 | outfile_meta.write("name="+sys.argv[2]+"\n") 20 | 21 | s = struct.Struct('@IIf') 22 | edges=0 23 | vertices=0 24 | for line in infile: 25 | if line[0] == '%': 26 | pass 27 | else: 28 | vector = line.strip().split(" ") 29 | vector = list(map(int, vector)) 30 | vector[0] = vector[0] - 1 31 | vector[1] = vector[1] - 1 32 | vector.append(random.random()) # Edge weight 33 | if vector[0] > vertices: 34 | vertices = vector[0] 35 | if vector[1] > vertices: 36 | vertices = vector[1] 37 | edges = edges + 1 38 | outfile.write(s.pack(*vector)) 39 | if add_rev_edges: 40 | edges = edges + 1 41 | tmp = vector[0] 42 | vector[0] = vector[1] 43 | vector[1] = tmp 44 | outfile.write(s.pack(*vector)) 45 | 46 | vertices = vertices + 1 47 | outfile_meta.write("vertices="+str(vertices)+"\n") 48 | outfile_meta.write("edges="+str(edges)+"\n") 49 | infile.close() 50 | outfile.close() 51 | outfile_meta.close() 52 | -------------------------------------------------------------------------------- /format/tools/type1_to_matrix_market.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | if (ARGV.size != 2) 4 | puts "Usage: #{$0} input_file output_file" 5 | exit 6 | end 7 | input_file = ARGV[0] 8 | output_file = ARGV[1] 9 | 10 | File.open(output_file, "w") do |outf| 11 | # Header 12 | ini_contents = File.read("#{input_file}.ini") 13 | vertices = ini_contents.scan(/vertices=(.*)/).to_s.to_i 14 | edges = ini_contents.scan(/edges=(.*)/).to_s.to_i 15 | outf.puts "%%MatrixMarket matrix coordinate real general" 16 | outf.puts "#{vertices} #{vertices} #{edges}" 17 | 18 | File.open(input_file, "r") do |inf| 19 | edge_struct = inf.read(12) 20 | while (edge_struct != nil) 21 | from = edge_struct[0..3].unpack("L").first + 1 22 | to = edge_struct[4..7].unpack("L").first + 1 23 | value = edge_struct[8..11].unpack("f").first 24 | outf.puts "#{from} #{to} #{value}" 25 | edge_struct = inf.read(12) 26 | end 27 | end 28 | end 29 | 30 | -------------------------------------------------------------------------------- /format/tools/xstream2pegasus.py: -------------------------------------------------------------------------------- 1 | # Converts X-Stream input into Pegasus-compatible input 2 | import sys 3 | import struct 4 | 5 | print 'Converting from ' + sys.argv[1] + ' to ' + sys.argv[2] 6 | 7 | with open(sys.argv[1],'rb') as infile: 8 | with open(sys.argv[2],'w') as outfile: 9 | for chunk in iter((lambda:infile.read(12)),''): 10 | source = struct.unpack('I', chunk[0:4])[0] 11 | target = struct.unpack('I', chunk[4:8])[0] 12 | outfile.write(str(source) + '\t' + str(target) + '\n') 13 | 14 | print 'Done!' 15 | -------------------------------------------------------------------------------- /format/tools/yahooweb2type_1.py: -------------------------------------------------------------------------------- 1 | #Convert yahoo web into x-stream type 1 inputs 2 | #Arguments 3 | import sys 4 | import struct 5 | import random 6 | import io 7 | 8 | #Choose one 9 | #add_rev_edges = False 10 | add_rev_edges = True 11 | 12 | random.seed(0) 13 | 14 | infile=file(sys.argv[1], "r") 15 | #outfile=file(sys.argv[2], "wtb") 16 | outfile_stream=io.open(sys.argv[2], "wb", 16777216) 17 | outfile_meta=file(sys.argv[2]+".ini", "wt") 18 | 19 | outfile_meta.write("[graph]\n") 20 | outfile_meta.write("type=1\n") 21 | outfile_meta.write("name="+sys.argv[2]+"\n") 22 | 23 | s = struct.Struct('@IIf') 24 | edges=0 25 | vertices=0 26 | for line in infile: 27 | if line[0] == '%': 28 | pass 29 | else: 30 | vector = line.strip().split(" ") 31 | vector = list(map(int, vector)) 32 | src=vector[0] 33 | if src > vertices: 34 | vertices = src 35 | for i in vector[2:]: 36 | weight = random.random() 37 | if i > vertices: 38 | vertices = i 39 | out = [src, i, weight] 40 | #print out 41 | outfile_stream.write(s.pack(*out)) 42 | edges = edges + 1 43 | if add_rev_edges: 44 | out = [i, src, weight] 45 | #print out 46 | outfile_stream.write(s.pack(*out)) 47 | edges = edges + 1 48 | 49 | vertices = vertices + 1 50 | outfile_meta.write("vertices="+str(vertices)+"\n") 51 | outfile_meta.write("edges="+str(edges)+"\n") 52 | infile.close() 53 | #outfile.close() 54 | outfile_meta.close() 55 | -------------------------------------------------------------------------------- /format/type_1.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _TYPE_1_FORMAT_ 20 | #define _TYPE_1_FORMAT_ 21 | 22 | #include 23 | #include 24 | 25 | namespace format { 26 | namespace type1 { 27 | class format_utils { 28 | public: 29 | static unsigned long split_size_bytes() { 30 | return 2 * sizeof(vertex_t) + sizeof(weight_t); 31 | } 32 | 33 | static void read_edge(const unsigned char *buffer, 34 | vertex_t &src, 35 | vertex_t &dst) { 36 | src = *(vertex_t *) buffer; 37 | dst = *(vertex_t * )(buffer + sizeof(vertex_t)); 38 | } 39 | 40 | static void read_edge(const unsigned char *buffer, 41 | vertex_t &src, 42 | vertex_t &dst, 43 | weight_t &value) { 44 | src = *(vertex_t *) buffer; 45 | dst = *(vertex_t * )(buffer + sizeof(vertex_t)); 46 | value = *(weight_t * )(buffer + 2 * sizeof(vertex_t)); 47 | } 48 | 49 | static void write_edge(unsigned char *buffer, 50 | vertex_t &src, 51 | vertex_t &dst, 52 | weight_t &value) { 53 | *(vertex_t *) buffer = src; 54 | *(vertex_t * )(buffer + sizeof(vertex_t)) = dst; 55 | *(weight_t * )(buffer + 2 * sizeof(vertex_t)) = value; 56 | } 57 | 58 | // Interpret as matrix 59 | static void read_matrix_element(const unsigned char *buffer, 60 | vertex_t &row, 61 | vertex_t &col, 62 | weight_t &value) { 63 | row = *(vertex_t *) buffer; 64 | col = *(vertex_t * )(buffer + sizeof(vertex_t)); 65 | value = *(weight_t * )(buffer + 2 * sizeof(vertex_t)); 66 | } 67 | 68 | static void vector_element(const unsigned char *buffer, 69 | weight_t &value) { 70 | value = *(weight_t *) buffer; 71 | } 72 | 73 | static void vector_name(std::stringstream &name, unsigned long partition) { 74 | name << pt.get("graph"); 75 | name << ".vec"; 76 | name << "." << partition; // suffix the partition 77 | } 78 | 79 | static unsigned long split_key(const unsigned char *buffer, unsigned long jump) { 80 | vertex_t key; 81 | memcpy(&key, buffer, sizeof(vertex_t)); 82 | key = key >> jump; 83 | return (unsigned long) key; 84 | } 85 | }; 86 | } 87 | } 88 | #endif 89 | -------------------------------------------------------------------------------- /format/type_2.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _TYPE_2_FORMAT_ 20 | #define _TYPE_2_FORMAT_ 21 | 22 | #include 23 | #include 24 | 25 | namespace format { 26 | namespace type2 { 27 | class format_utils { 28 | public: 29 | static unsigned long split_size_bytes() { 30 | return 2 * sizeof(vertex_t); 31 | } 32 | 33 | static void read_edge(const unsigned char *buffer, 34 | vertex_t &src, 35 | vertex_t &dst) { 36 | src = *(vertex_t *) buffer; 37 | dst = *(vertex_t * )(buffer + sizeof(vertex_t)); 38 | } 39 | 40 | static void read_edge(const unsigned char *buffer, 41 | vertex_t &src, 42 | vertex_t &dst, 43 | weight_t &value) { 44 | src = *(vertex_t *) buffer; 45 | dst = *(vertex_t * )(buffer + sizeof(vertex_t)); 46 | } 47 | 48 | static void write_edge(unsigned char *buffer, 49 | vertex_t &src, 50 | vertex_t &dst, 51 | weight_t &value) { 52 | *(vertex_t *) buffer = src; 53 | *(vertex_t * )(buffer + sizeof(vertex_t)) = dst; 54 | } 55 | 56 | static unsigned long split_key(const unsigned char *buffer, unsigned long jump) { 57 | vertex_t key; 58 | memcpy(&key, buffer, sizeof(vertex_t)); 59 | key = key >> jump; 60 | return (unsigned long) key; 61 | } 62 | }; 63 | } 64 | } 65 | #endif 66 | -------------------------------------------------------------------------------- /generators/Makefile: -------------------------------------------------------------------------------- 1 | CC = g++ 2 | CFLAGS = -std=c++0x -DNDEBUG -O3 -Wall -Drestrict=__restrict__ -ffast-math -L/usr/local/lib 3 | #CFLAGS = -std=c++0x -g -O3 -Wall -Drestrict=__restrict__ -ffast-math -L/usr/local/lib 4 | CFLAGS += -DVERTEX_TYPE_32 5 | CFLAGS += -DVALUE_TYPE_32 6 | #CFLAGS += -DNOISE 7 | CFLAGS += -DWEIGHT 8 | LDFLAGS = -lboost_program_options -lboost_thread -lboost_system 9 | 10 | PRNG_SOURCES = splittable_mrg.c utils.c 11 | GEN_SOURCES = options.cpp output.cpp 12 | 13 | .PHONY: all clean 14 | 15 | all: erdos-renyi rmat 16 | 17 | erdos-renyi: random.cpp $(addprefix prng/,$(PRNG_SOURCES)) $(GEN_SOURCES) 18 | $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS) 19 | 20 | rmat: rmat.cpp $(addprefix prng/,$(PRNG_SOURCES)) $(GEN_SOURCES) 21 | $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS) 22 | 23 | clean: 24 | rm -f erdos-renyi rmat 25 | -------------------------------------------------------------------------------- /generators/defs.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef HPGP_GENERATORS_DEFS_H 20 | #define HPGP_GENERATORS_DEFS_H 21 | 22 | #include 23 | #include 24 | 25 | #ifndef VERTEX_TYPE_32 26 | typedef uint64_t vertex_t; 27 | #define VT_MAX (UINT64_C(0xFFFFFFFFFFFFFFFF)) 28 | #define PRIvt PRIu64 29 | #else 30 | typedef uint32_t vertex_t; 31 | #define VT_MAX (UINT32_C(0xFFFFFFFF)) 32 | #define PRIvt PRIu32 33 | #endif 34 | 35 | typedef uint64_t edge_t; 36 | #define PRIet PRIu64 37 | 38 | #ifndef VALUE_TYPE_32 39 | typedef double value_t; 40 | #else 41 | typedef float value_t; 42 | #endif 43 | 44 | #ifdef __GNUC__ 45 | #define LIKELY(x) __builtin_expect((x),1) 46 | #define UNLIKELY(x) __builtin_expect((x),0) 47 | #else 48 | #define LIKELY(x) (x) 49 | #define UNLIKELY(x) (x) 50 | #endif 51 | 52 | #endif /* HPGP_GENERATORS_DEFS_H */ 53 | -------------------------------------------------------------------------------- /generators/options.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include 20 | #include 21 | #include "options.h" 22 | 23 | namespace po = boost::program_options; 24 | 25 | int process_options(int argc, char **argv, bool rmat_generator, struct options *options) { 26 | // Options common for both generators 27 | po::options_description common("Common options"); 28 | common.add_options() 29 | ("help", "print this message") 30 | ("name", po::value(&options->global.graphname), "graph name") 31 | ("threads", po::value(&options->global.nthreads)->default_value(DEFAULT_NTHREADS), "number of threads") 32 | ("buffer-size", po::value(&options->global.buffer_size), "buffer size in bytes") 33 | ("bpt", po::value(&options->global.buffers_per_thread)->default_value(DEFAULT_BUFFERS_PER_THREAD), 34 | "number of buffers per thread") 35 | ("symmetric", "emit also a reverse edge for each generated edge") 36 | ("seed1", po::value(&options->rng.userseed1)->default_value(DEFAULT_RNG_USERSEED1), 37 | "first 64b of seed for rng") 38 | ("seed2", po::value(&options->rng.userseed2)->default_value(DEFAULT_RNG_USERSEED2), 39 | "second 64b of seed for rng"); 40 | 41 | // Options specific to ER 42 | po::options_description er("Erdos-Renyi"); 43 | er.add_options() 44 | ("vertices", po::value(&options->erdos_renyi.vertices)->default_value(DEFAULT_ER_VERTICES), 45 | "number of vertices") 46 | ("edges", po::value(&options->erdos_renyi.edges)->default_value(DEFAULT_ER_EDGES), "number of edges") 47 | ("self-loops", "allow self loops") 48 | ("bipartite", po::value(&options->erdos_renyi.bipartite), 49 | "argument should specify the number of vertices on the left side"); 50 | 51 | // Options specific to RMAT 52 | po::options_description rmat("R-MAT"); 53 | rmat.add_options() 54 | ("scale", po::value(&options->rmat.scale)->default_value(DEFAULT_RMAT_SCALE), 55 | "log2 of the number of vertices") 56 | ("edges", po::value(&options->rmat.edges)->default_value(DEFAULT_RMAT_EDGES), "number of edges") 57 | ("a", po::value(&options->rmat.a)->default_value(DEFAULT_RMAT_A, "0.57"), 58 | "a, b, c, d are RMAT probabilities (usually a = 3b = 3c > d)") 59 | ("b", po::value(&options->rmat.b)->default_value(DEFAULT_RMAT_B, "0.19"), "") 60 | ("c", po::value(&options->rmat.c)->default_value(DEFAULT_RMAT_C, "0.19"), "") 61 | ("xscale_interval", po::value < unsigned 62 | int > (&options->rmat.xscale_interval)->default_value(1), "# xscale machines") 63 | ("xscale_node", po::value < unsigned 64 | int > (&options->rmat.xscale_node)->default_value(0), "xscale machine number"); 65 | 66 | po::options_description cmdline_options; 67 | if (rmat_generator) 68 | cmdline_options.add(common).add(rmat); 69 | else 70 | cmdline_options.add(common).add(er); 71 | 72 | // Store options 73 | bool err = false; 74 | po::variables_map vm; 75 | try { 76 | po::store(po::parse_command_line(argc, argv, cmdline_options), vm); 77 | po::notify(vm); 78 | } catch (boost::program_options::error) { 79 | err = true; 80 | } 81 | 82 | // Help 83 | if (vm.count("help") || !vm.count("name") || err) { 84 | std::cout << "Usage: " << (rmat_generator ? "rmat" : "erdor-renyi") << " --name graphname [options]\n"; 85 | std::cout << " " << (rmat_generator ? "rmat" : "erdor-renyi") << " --help\n"; 86 | std::cout << cmdline_options << "\n"; 87 | return 1; 88 | } 89 | 90 | // Process options (where needed) 91 | if (vm.count("buffer-size")) { 92 | options->global.buffer_size = vm["buffer-size"].as(); 93 | } else { 94 | options->global.buffer_size = 0; 95 | } 96 | if (vm.count("symmetric")) { 97 | options->global.symmetric = true; 98 | } else { 99 | options->global.symmetric = false; 100 | } 101 | if (vm.count("self-loops")) { 102 | options->erdos_renyi.self_loops = true; 103 | } else { 104 | options->erdos_renyi.self_loops = false; 105 | } 106 | if (!vm.count("bipartite")) { 107 | options->erdos_renyi.bipartite = 0; 108 | } 109 | 110 | return 0; 111 | } 112 | -------------------------------------------------------------------------------- /generators/options.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef HPGP_GENERATORS_OPTIONS_H 20 | #define HPGP_GENERATORS_OPTIONS_H 21 | 22 | #include 23 | #include "defs.h" 24 | 25 | struct global { 26 | std::string graphname; 27 | int nthreads; 28 | size_t buffer_size; 29 | int buffers_per_thread; 30 | bool symmetric; 31 | }; 32 | 33 | const int DEFAULT_NTHREADS = 32; 34 | const int DEFAULT_BUFFERS_PER_THREAD = 2; 35 | 36 | struct erdos_renyi { 37 | vertex_t vertices; 38 | edge_t edges; 39 | bool self_loops; 40 | vertex_t bipartite; 41 | }; 42 | 43 | const vertex_t DEFAULT_ER_VERTICES = (vertex_t) 1 << 16; 44 | const vertex_t DEFAULT_ER_EDGES = (edge_t) 1 << 20; 45 | 46 | struct rmat { 47 | int scale; 48 | edge_t edges; 49 | double a; 50 | double b; 51 | double c; 52 | unsigned int xscale_interval; 53 | unsigned int xscale_node; 54 | }; 55 | 56 | const int DEFAULT_RMAT_SCALE = 16; 57 | const edge_t DEFAULT_RMAT_EDGES = (edge_t) 1 << 20; 58 | const double DEFAULT_RMAT_A = 0.57; 59 | const double DEFAULT_RMAT_B = 0.19; 60 | const double DEFAULT_RMAT_C = 0.19; 61 | 62 | struct rng { 63 | uint64_t userseed1; 64 | uint64_t userseed2; 65 | }; 66 | 67 | const uint64_t DEFAULT_RNG_USERSEED1 = (uint64_t) 1; 68 | const uint64_t DEFAULT_RNG_USERSEED2 = (uint64_t) 2; 69 | 70 | struct options { 71 | struct global global; 72 | struct erdos_renyi erdos_renyi; 73 | struct rmat rmat; 74 | struct rng rng; 75 | }; 76 | 77 | int process_options(int argc, char **argv, bool rmat_generator, struct options *options); 78 | 79 | #endif /* HPGP_GENERATORS_OPTIONS_H */ 80 | -------------------------------------------------------------------------------- /generators/output.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef HPGP_GENERATORS_OUTPUT_H 20 | #define HPGP_GENERATORS_OUTPUT_H 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include "defs.h" 27 | 28 | const size_t DEFAULT_BLOCK_SIZE = 4096; 29 | 30 | struct edge_struct { 31 | vertex_t src; 32 | vertex_t dst; 33 | #ifdef WEIGHT 34 | value_t weight; 35 | #endif 36 | } __attribute__((packed)); 37 | 38 | class threadid_t { 39 | public: 40 | threadid_t(const int id) : m_id(id) { } 41 | 42 | bool operator<(const threadid_t &rhs) const { 43 | return m_id < rhs.m_id; 44 | } 45 | 46 | private: 47 | int m_id; 48 | }; 49 | 50 | class edge_buffer { 51 | public: 52 | edge_buffer(const size_t size, const threadid_t owner); 53 | 54 | ~edge_buffer(); 55 | 56 | struct edge_struct *m_data; 57 | size_t m_size; 58 | size_t m_pos; 59 | threadid_t m_owner; 60 | }; 61 | 62 | // This queue is designed to work properly only with 63 | // one consumer thread 64 | class buffer_queue { 65 | public: 66 | void enqueue(edge_buffer *pbuffer); 67 | 68 | edge_buffer *dequeue(); 69 | 70 | edge_buffer *try_dequeue(); 71 | 72 | edge_buffer *front(); 73 | 74 | edge_buffer *try_front(); 75 | 76 | void signal(); 77 | 78 | private: 79 | typedef std::queue storage_t; 80 | storage_t m_data; 81 | boost::condition_variable m_cond; 82 | boost::mutex m_mutex; 83 | }; 84 | 85 | class thread_buffer { 86 | public: 87 | thread_buffer(buffer_queue *pflushq); 88 | 89 | ~thread_buffer(); 90 | 91 | struct edge_struct *edge_struct(); 92 | 93 | void flush(); 94 | 95 | buffer_queue *get_queue(); 96 | 97 | private: 98 | buffer_queue m_queue; 99 | buffer_queue *m_pflushq; 100 | }; 101 | 102 | class buffer_manager { 103 | public: 104 | buffer_manager(buffer_queue *pflushq, const int queue_size, const size_t buffer_size); 105 | 106 | thread_buffer *register_thread(const threadid_t tid); 107 | 108 | void unregister_thread(const threadid_t tid); 109 | 110 | thread_buffer *get_buffer(const threadid_t tid); 111 | 112 | private: 113 | typedef std::map map_t; 114 | map_t m_allbuffers; 115 | buffer_queue *m_pflushq; 116 | int m_queue_size; 117 | size_t m_buffer_size; 118 | }; 119 | 120 | class io_thread_func { 121 | public: 122 | io_thread_func(const char *filename, const edge_t nedges, buffer_queue *pflushq, buffer_manager *pmanager, 123 | const size_t buffer_size); 124 | 125 | void operator()(); 126 | 127 | void stop(); 128 | 129 | private: 130 | void flush_buffer(edge_buffer *pbuffer); 131 | 132 | void save_for_later(edge_buffer *pbuffer); 133 | 134 | int m_fd; 135 | size_t m_filesize; 136 | buffer_queue *m_pflushq; 137 | buffer_manager *m_pmanager; 138 | edge_buffer m_remainder; 139 | bool m_stop; 140 | }; 141 | 142 | void make_ini_file(const char *graphname, const uint64_t vertices, const uint64_t edges); 143 | 144 | #endif /* HPGP_GENERATORS_OUTPUT_H */ 145 | -------------------------------------------------------------------------------- /generators/prng/README: -------------------------------------------------------------------------------- 1 | Multiple recursive generator from L'Ecuyer, P., Blouin, F., and 2 | Couture, R. 1993. A search for good multiple recursive random number 3 | generators. ACM Trans. Model. Comput. Simul. 3, 2 (Apr. 1993), 87-98. 4 | DOI= http://doi.acm.org/10.1145/169702.169698 -- particular generator 5 | used is from table 3, entry for m = 2^31 - 1, k = 5 (same generator 6 | is used in GNU Scientific Library). 7 | 8 | Authors: Jeremiah Willcock, Indiana University 9 | Andrew Lumsdaine, Indiana University 10 | -------------------------------------------------------------------------------- /generators/prng/mod_arith.h: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2010 The Trustees of Indiana University. */ 2 | /* */ 3 | /* Use, modification and distribution is subject to the Boost Software */ 4 | /* License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at */ 5 | /* http://www.boost.org/LICENSE_1_0.txt) */ 6 | /* */ 7 | /* Authors: Jeremiah Willcock */ 8 | /* Andrew Lumsdaine */ 9 | 10 | #ifndef MOD_ARITH_H 11 | #define MOD_ARITH_H 12 | 13 | #define FAST_64BIT_ARITHMETIC /* Use 64-bit arithmetic when possible. */ 14 | 15 | /* Various modular arithmetic operations for modulus 2^31-1 (0x7FFFFFFF). 16 | * These may need to be tweaked to get acceptable performance on some platforms 17 | * (especially ones without conditional moves). */ 18 | 19 | /* This code is now just a dispatcher that chooses the right header file to use 20 | * per-platform. */ 21 | 22 | #ifdef __MTA__ 23 | #include "mod_arith_xmt.h" 24 | #else 25 | #ifdef FAST_64BIT_ARITHMETIC 26 | #include "mod_arith_64bit.h" 27 | #else 28 | #include "mod_arith_32bit.h" 29 | #endif 30 | #endif 31 | 32 | #endif /* MOD_ARITH_H */ 33 | -------------------------------------------------------------------------------- /generators/prng/mod_arith_32bit.h: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2010 The Trustees of Indiana University. */ 2 | /* */ 3 | /* Use, modification and distribution is subject to the Boost Software */ 4 | /* License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at */ 5 | /* http://www.boost.org/LICENSE_1_0.txt) */ 6 | /* */ 7 | /* Authors: Jeremiah Willcock */ 8 | /* Andrew Lumsdaine */ 9 | 10 | #ifndef MOD_ARITH_32BIT_H 11 | #define MOD_ARITH_32BIT_H 12 | 13 | #include 14 | #include 15 | 16 | /* Various modular arithmetic operations for modulus 2^31-1 (0x7FFFFFFF). 17 | * These may need to be tweaked to get acceptable performance on some platforms 18 | * (especially ones without conditional moves). */ 19 | 20 | static inline uint_fast32_t mod_add(uint_fast32_t a, uint_fast32_t b) { 21 | uint_fast32_t x; 22 | assert (a <= 0x7FFFFFFE); 23 | assert (b <= 0x7FFFFFFE); 24 | #if 0 25 | return (a + b) % 0x7FFFFFFF; 26 | #else 27 | x = a + b; /* x <= 0xFFFFFFFC */ 28 | x = (x >= 0x7FFFFFFF) ? (x - 0x7FFFFFFF) : x; 29 | return x; 30 | #endif 31 | } 32 | 33 | static inline uint_fast32_t mod_mul(uint_fast32_t a, uint_fast32_t b) { 34 | uint_fast64_t temp; 35 | uint_fast32_t temp2; 36 | assert (a <= 0x7FFFFFFE); 37 | assert (b <= 0x7FFFFFFE); 38 | #if 0 39 | return (uint_fast32_t)((uint_fast64_t)a * b % 0x7FFFFFFF); 40 | #else 41 | temp = (uint_fast64_t)a * b; /* temp <= 0x3FFFFFFE00000004 */ 42 | temp2 = (uint_fast32_t)(temp & 0x7FFFFFFF) + (uint_fast32_t)(temp >> 31); /* temp2 <= 0xFFFFFFFB */ 43 | return (temp2 >= 0x7FFFFFFF) ? (temp2 - 0x7FFFFFFF) : temp2; 44 | #endif 45 | } 46 | 47 | static inline uint_fast32_t mod_mac(uint_fast32_t sum, uint_fast32_t a, uint_fast32_t b) { 48 | uint_fast64_t temp; 49 | uint_fast32_t temp2; 50 | assert (sum <= 0x7FFFFFFE); 51 | assert (a <= 0x7FFFFFFE); 52 | assert (b <= 0x7FFFFFFE); 53 | #if 0 54 | return (uint_fast32_t)(((uint_fast64_t)a * b + sum) % 0x7FFFFFFF); 55 | #else 56 | temp = (uint_fast64_t)a * b + sum; /* temp <= 0x3FFFFFFE80000002 */ 57 | temp2 = (uint_fast32_t)(temp & 0x7FFFFFFF) + (uint_fast32_t)(temp >> 31); /* temp2 <= 0xFFFFFFFC */ 58 | return (temp2 >= 0x7FFFFFFF) ? (temp2 - 0x7FFFFFFF) : temp2; 59 | #endif 60 | } 61 | 62 | static inline uint_fast32_t mod_mac2(uint_fast32_t sum, uint_fast32_t a, uint_fast32_t b, uint_fast32_t c, uint_fast32_t d) { 63 | assert (sum <= 0x7FFFFFFE); 64 | assert (a <= 0x7FFFFFFE); 65 | assert (b <= 0x7FFFFFFE); 66 | assert (c <= 0x7FFFFFFE); 67 | assert (d <= 0x7FFFFFFE); 68 | return mod_mac(mod_mac(sum, a, b), c, d); 69 | } 70 | 71 | static inline uint_fast32_t mod_mac3(uint_fast32_t sum, uint_fast32_t a, uint_fast32_t b, uint_fast32_t c, uint_fast32_t d, uint_fast32_t e, uint_fast32_t f) { 72 | assert (sum <= 0x7FFFFFFE); 73 | assert (a <= 0x7FFFFFFE); 74 | assert (b <= 0x7FFFFFFE); 75 | assert (c <= 0x7FFFFFFE); 76 | assert (d <= 0x7FFFFFFE); 77 | assert (e <= 0x7FFFFFFE); 78 | assert (f <= 0x7FFFFFFE); 79 | return mod_mac2(mod_mac(sum, a, b), c, d, e, f); 80 | } 81 | 82 | static inline uint_fast32_t mod_mac4(uint_fast32_t sum, uint_fast32_t a, uint_fast32_t b, uint_fast32_t c, uint_fast32_t d, uint_fast32_t e, uint_fast32_t f, uint_fast32_t g, uint_fast32_t h) { 83 | assert (sum <= 0x7FFFFFFE); 84 | assert (a <= 0x7FFFFFFE); 85 | assert (b <= 0x7FFFFFFE); 86 | assert (c <= 0x7FFFFFFE); 87 | assert (d <= 0x7FFFFFFE); 88 | assert (e <= 0x7FFFFFFE); 89 | assert (f <= 0x7FFFFFFE); 90 | assert (g <= 0x7FFFFFFE); 91 | assert (h <= 0x7FFFFFFE); 92 | return mod_mac2(mod_mac2(sum, a, b, c, d), e, f, g, h); 93 | } 94 | 95 | /* The two constants x and y are special cases because they are easier to 96 | * multiply by on 32-bit systems. They are used as multipliers in the random 97 | * number generator. The techniques for fast multiplication by these 98 | * particular values are from: 99 | * 100 | * Pierre L'Ecuyer, Francois Blouin, and Raymond Couture. 1993. A search 101 | * for good multiple recursive random number generators. ACM Trans. Model. 102 | * Comput. Simul. 3, 2 (April 1993), 87-98. DOI=10.1145/169702.169698 103 | * http://doi.acm.org/10.1145/169702.169698 104 | * 105 | * Pierre L'Ecuyer. 1990. Random numbers for simulation. Commun. ACM 33, 10 106 | * (October 1990), 85-97. DOI=10.1145/84537.84555 107 | * http://doi.acm.org/10.1145/84537.84555 108 | */ 109 | 110 | inline uint_fast32_t mod_mul_x(uint_fast32_t a) { 111 | static const int32_t q = 20 /* UINT32_C(0x7FFFFFFF) / 107374182 */; 112 | static const int32_t r = 7 /* UINT32_C(0x7FFFFFFF) % 107374182 */; 113 | int_fast32_t result = (int_fast32_t)(a) / q; 114 | result = 107374182 * ((int_fast32_t)(a) - result * q) - result * r; 115 | result += (result < 0 ? 0x7FFFFFFF : 0); 116 | assert ((uint_fast32_t)(result) == mod_mul(a, 107374182)); 117 | return (uint_fast32_t)result; 118 | } 119 | 120 | inline uint_fast32_t mod_mul_y(uint_fast32_t a) { 121 | static const int32_t q = 20554 /* UINT32_C(0x7FFFFFFF) / 104480 */; 122 | static const int32_t r = 1727 /* UINT32_C(0x7FFFFFFF) % 104480 */; 123 | int_fast32_t result = (int_fast32_t)(a) / q; 124 | result = 104480 * ((int_fast32_t)(a) - result * q) - result * r; 125 | result += (result < 0 ? 0x7FFFFFFF : 0); 126 | assert ((uint_fast32_t)(result) == mod_mul(a, 104480)); 127 | return (uint_fast32_t)result; 128 | } 129 | 130 | inline uint_fast32_t mod_mac_y(uint_fast32_t sum, uint_fast32_t a) { 131 | uint_fast32_t result = mod_add(sum, mod_mul_y(a)); 132 | assert (result == mod_mac(sum, a, 104480)); 133 | return result; 134 | } 135 | 136 | #endif /* MOD_ARITH_32BIT_H */ 137 | -------------------------------------------------------------------------------- /generators/prng/mod_arith_64bit.h: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2010 The Trustees of Indiana University. */ 2 | /* */ 3 | /* Use, modification and distribution is subject to the Boost Software */ 4 | /* License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at */ 5 | /* http://www.boost.org/LICENSE_1_0.txt) */ 6 | /* */ 7 | /* Authors: Jeremiah Willcock */ 8 | /* Andrew Lumsdaine */ 9 | 10 | #ifndef MOD_ARITH_64BIT_H 11 | #define MOD_ARITH_64BIT_H 12 | 13 | #include 14 | #include 15 | 16 | /* Various modular arithmetic operations for modulus 2^31-1 (0x7FFFFFFF). 17 | * These may need to be tweaked to get acceptable performance on some platforms 18 | * (especially ones without conditional moves). */ 19 | 20 | static inline uint_fast32_t mod_add(uint_fast32_t a, uint_fast32_t b) { 21 | assert (a <= 0x7FFFFFFE); 22 | assert (b <= 0x7FFFFFFE); 23 | return (a + b) % 0x7FFFFFFF; 24 | } 25 | 26 | static inline uint_fast32_t mod_mul(uint_fast32_t a, uint_fast32_t b) { 27 | assert (a <= 0x7FFFFFFE); 28 | assert (b <= 0x7FFFFFFE); 29 | return (uint_fast32_t)((uint_fast64_t)a * b % 0x7FFFFFFF); 30 | } 31 | 32 | static inline uint_fast32_t mod_mac(uint_fast32_t sum, uint_fast32_t a, uint_fast32_t b) { 33 | assert (sum <= 0x7FFFFFFE); 34 | assert (a <= 0x7FFFFFFE); 35 | assert (b <= 0x7FFFFFFE); 36 | return (uint_fast32_t)(((uint_fast64_t)a * b + sum) % 0x7FFFFFFF); 37 | } 38 | 39 | static inline uint_fast32_t mod_mac2(uint_fast32_t sum, uint_fast32_t a, uint_fast32_t b, uint_fast32_t c, uint_fast32_t d) { 40 | assert (sum <= 0x7FFFFFFE); 41 | assert (a <= 0x7FFFFFFE); 42 | assert (b <= 0x7FFFFFFE); 43 | assert (c <= 0x7FFFFFFE); 44 | assert (d <= 0x7FFFFFFE); 45 | return (uint_fast32_t)(((uint_fast64_t)a * b + (uint_fast64_t)c * d + sum) % 0x7FFFFFFF); 46 | } 47 | 48 | static inline uint_fast32_t mod_mac3(uint_fast32_t sum, uint_fast32_t a, uint_fast32_t b, uint_fast32_t c, uint_fast32_t d, uint_fast32_t e, uint_fast32_t f) { 49 | assert (sum <= 0x7FFFFFFE); 50 | assert (a <= 0x7FFFFFFE); 51 | assert (b <= 0x7FFFFFFE); 52 | assert (c <= 0x7FFFFFFE); 53 | assert (d <= 0x7FFFFFFE); 54 | assert (e <= 0x7FFFFFFE); 55 | assert (f <= 0x7FFFFFFE); 56 | return (uint_fast32_t)(((uint_fast64_t)a * b + (uint_fast64_t)c * d + (uint_fast64_t)e * f + sum) % 0x7FFFFFFF); 57 | } 58 | 59 | static inline uint_fast32_t mod_mac4(uint_fast32_t sum, uint_fast32_t a, uint_fast32_t b, uint_fast32_t c, uint_fast32_t d, uint_fast32_t e, uint_fast32_t f, uint_fast32_t g, uint_fast32_t h) { 60 | assert (sum <= 0x7FFFFFFE); 61 | assert (a <= 0x7FFFFFFE); 62 | assert (b <= 0x7FFFFFFE); 63 | assert (c <= 0x7FFFFFFE); 64 | assert (d <= 0x7FFFFFFE); 65 | assert (e <= 0x7FFFFFFE); 66 | assert (f <= 0x7FFFFFFE); 67 | assert (g <= 0x7FFFFFFE); 68 | assert (h <= 0x7FFFFFFE); 69 | return (uint_fast32_t)(((uint_fast64_t)a * b + (uint_fast64_t)c * d + (uint_fast64_t)e * f + (uint_fast64_t)g * h + sum) % 0x7FFFFFFF); 70 | } 71 | 72 | /* The two constants x and y are special cases because they are easier to 73 | * multiply by on 32-bit systems. They are used as multipliers in the random 74 | * number generator. The techniques for fast multiplication by these 75 | * particular values are in L'Ecuyer's papers; we don't use them yet. */ 76 | 77 | static inline uint_fast32_t mod_mul_x(uint_fast32_t a) { 78 | return mod_mul(a, 107374182); 79 | } 80 | 81 | static inline uint_fast32_t mod_mul_y(uint_fast32_t a) { 82 | return mod_mul(a, 104480); 83 | } 84 | 85 | static inline uint_fast32_t mod_mac_y(uint_fast32_t sum, uint_fast32_t a) { 86 | return mod_mac(sum, a, 104480); 87 | } 88 | 89 | #endif /* MOD_ARITH_64BIT_H */ 90 | -------------------------------------------------------------------------------- /generators/prng/mod_arith_xmt.h: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2010 The Trustees of Indiana University. */ 2 | /* */ 3 | /* Use, modification and distribution is subject to the Boost Software */ 4 | /* License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at */ 5 | /* http://www.boost.org/LICENSE_1_0.txt) */ 6 | /* */ 7 | /* Authors: Jeremiah Willcock */ 8 | /* Andrew Lumsdaine */ 9 | 10 | #ifndef MOD_ARITH_XMT_H 11 | #define MOD_ARITH_XMT_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | /* Various modular arithmetic operations for modulus 2^31-1 (0x7FFFFFFF). 18 | * These may need to be tweaked to get acceptable performance on some platforms 19 | * (especially ones without conditional moves). */ 20 | 21 | /* MTA code is based on description in 22 | * http://www.hackersdelight.org/divcMore.pdf and experimentation based on 23 | * that. */ 24 | 25 | #pragma mta inline 26 | static inline uint_fast32_t mod_add(uint_fast32_t a, uint_fast32_t b) { 27 | uint_fast32_t x; 28 | assert (a <= 0x7FFFFFFE); 29 | assert (b <= 0x7FFFFFFE); 30 | x = a + b; /* x <= 0xFFFFFFFC */ 31 | return (x + MTA_UNS_ADD_MUL_UPPER(0x0000000200000003, 0x0000000200000004, x)) & 0x7FFFFFFF; 32 | } 33 | 34 | #pragma mta inline 35 | static inline uint_fast32_t mod_mul(uint_fast32_t a, uint_fast32_t b) { 36 | uint_fast64_t temp; 37 | uint_fast32_t temp2; 38 | assert (a <= 0x7FFFFFFE); 39 | assert (b <= 0x7FFFFFFE); 40 | temp = MTA_INT_ADD_MUL(0, a, b); 41 | return (temp + MTA_UNS_ADD_MUL_UPPER(0x0000000200000003, 0x0000000200000004, temp)) & 0x7FFFFFFF; 42 | } 43 | 44 | #pragma mta inline 45 | static inline uint_fast32_t mod_mac(uint_fast32_t sum, uint_fast32_t a, uint_fast32_t b) { 46 | uint_fast64_t temp; 47 | uint_fast32_t temp2; 48 | assert (sum <= 0x7FFFFFFE); 49 | assert (a <= 0x7FFFFFFE); 50 | assert (b <= 0x7FFFFFFE); 51 | temp = MTA_INT_ADD_MUL(sum, a, b); 52 | return (temp + MTA_UNS_ADD_MUL_UPPER(0x0000000200000003, 0x0000000200000004, temp)) & 0x7FFFFFFF; 53 | } 54 | 55 | #pragma mta inline 56 | static inline uint_fast32_t mod_mac2(uint_fast32_t sum, uint_fast32_t a, uint_fast32_t b, uint_fast32_t c, uint_fast32_t d) { 57 | uint_fast64_t temp; 58 | assert (sum <= 0x7FFFFFFE); 59 | assert (a <= 0x7FFFFFFE); 60 | assert (b <= 0x7FFFFFFE); 61 | assert (c <= 0x7FFFFFFE); 62 | assert (d <= 0x7FFFFFFE); 63 | temp = MTA_INT_ADD_MUL(MTA_INT_ADD_MUL(sum, a, b), c, d); 64 | return (temp + MTA_UNS_ADD_MUL_UPPER(0x0000000200000003, 0x0000000200000004, temp)) & 0x7FFFFFFF; 65 | } 66 | 67 | #pragma mta inline 68 | static inline uint_fast32_t mod_mac3(uint_fast32_t sum, uint_fast32_t a, uint_fast32_t b, uint_fast32_t c, uint_fast32_t d, uint_fast32_t e, uint_fast32_t f) { 69 | uint_fast64_t temp; 70 | assert (sum <= 0x7FFFFFFE); 71 | assert (a <= 0x7FFFFFFE); 72 | assert (b <= 0x7FFFFFFE); 73 | assert (c <= 0x7FFFFFFE); 74 | assert (d <= 0x7FFFFFFE); 75 | assert (e <= 0x7FFFFFFE); 76 | assert (f <= 0x7FFFFFFE); 77 | temp = MTA_INT_ADD_MUL(MTA_INT_ADD_MUL(MTA_INT_ADD_MUL(sum, a, b), c, d), e, f); 78 | return (temp + MTA_UNS_ADD_MUL_UPPER(0x0000000200000003, 0x0000000200000004, temp)) & 0x7FFFFFFF; 79 | } 80 | 81 | #pragma mta inline 82 | static inline uint_fast32_t mod_mac4(uint_fast32_t sum, uint_fast32_t a, uint_fast32_t b, uint_fast32_t c, uint_fast32_t d, uint_fast32_t e, uint_fast32_t f, uint_fast32_t g, uint_fast32_t h) { 83 | uint_fast64_t temp; 84 | assert (sum <= 0x7FFFFFFE); 85 | assert (a <= 0x7FFFFFFE); 86 | assert (b <= 0x7FFFFFFE); 87 | assert (c <= 0x7FFFFFFE); 88 | assert (d <= 0x7FFFFFFE); 89 | assert (e <= 0x7FFFFFFE); 90 | assert (f <= 0x7FFFFFFE); 91 | assert (g <= 0x7FFFFFFE); 92 | assert (h <= 0x7FFFFFFE); 93 | temp = MTA_INT_ADD_MUL(MTA_INT_ADD_MUL(MTA_INT_ADD_MUL(MTA_INT_ADD_MUL(sum, a, b), c, d), e, f), g, h); 94 | return (temp + MTA_UNS_ADD_MUL_UPPER(0x0000000200000003, 0x0000000200000004, temp)) & 0x7FFFFFFF; 95 | } 96 | 97 | static inline uint_fast64_t mod_down(uint_fast64_t x); 98 | 99 | #pragma mta inline 100 | static inline uint_fast64_t mod_down_fast(uint_fast64_t x) { 101 | return (x >> 31) + (x & 0x7FFFFFFF); 102 | } 103 | 104 | #pragma mta inline 105 | static inline uint_fast64_t mod_down(uint_fast64_t x) { 106 | return ((x + MTA_UNS_ADD_MUL_UPPER(0x0000000200000003, 0x0000000200000004, x)) & 0x7FFFFFFF); 107 | // x = mod_down_fast(mod_down_fast(x)); 108 | // x = (x + (x >= 0x7FFFFFFF)) & 0x7FFFFFFF; 109 | // return x; 110 | } 111 | 112 | /* The two constants x and y are special cases because they are easier to 113 | * multiply by on 32-bit systems. They are used as multipliers in the random 114 | * number generator. The techniques for fast multiplication by these 115 | * particular values are in L'Ecuyer's papers; we don't use them yet. */ 116 | 117 | #pragma mta inline 118 | static inline uint_fast32_t mod_mul_x(uint_fast32_t a) { 119 | return mod_mul(a, 107374182); 120 | } 121 | 122 | #pragma mta inline 123 | static inline uint_fast32_t mod_mul_y(uint_fast32_t a) { 124 | return mod_mul(a, 104480); 125 | } 126 | 127 | #pragma mta inline 128 | static inline uint_fast32_t mod_mac_y(uint_fast32_t sum, uint_fast32_t a) { 129 | return mod_mac(sum, a, 104480); 130 | } 131 | 132 | #endif /* MOD_ARITH_XMT_H */ 133 | -------------------------------------------------------------------------------- /generators/prng/splittable_mrg.h: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2010 The Trustees of Indiana University. */ 2 | /* */ 3 | /* Use, modification and distribution is subject to the Boost Software */ 4 | /* License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at */ 5 | /* http://www.boost.org/LICENSE_1_0.txt) */ 6 | /* */ 7 | /* Authors: Jeremiah Willcock */ 8 | /* Andrew Lumsdaine */ 9 | 10 | #ifndef SPLITTABLE_MRG_H 11 | #define SPLITTABLE_MRG_H 12 | 13 | #include 14 | 15 | /* Multiple recursive generator from L'Ecuyer, P., Blouin, F., and */ 16 | /* Couture, R. 1993. A search for good multiple recursive random number */ 17 | /* generators. ACM Trans. Model. Comput. Simul. 3, 2 (Apr. 1993), 87-98. */ 18 | /* DOI= http://doi.acm.org/10.1145/169702.169698 -- particular generator */ 19 | /* used is from table 3, entry for m = 2^31 - 1, k = 5 (same generator */ 20 | /* is used in GNU Scientific Library). */ 21 | 22 | /* See notes at top of splittable_mrg.c for information on this */ 23 | /* implementation. */ 24 | 25 | #ifdef __cplusplus 26 | extern "C" { 27 | #endif 28 | 29 | typedef struct mrg_state { 30 | uint_fast32_t z1, z2, z3, z4, z5; 31 | } mrg_state; 32 | 33 | /* Returns integer value in [0, 2^31-1) using original transition matrix */ 34 | uint_fast32_t mrg_get_uint_orig(mrg_state* state); 35 | 36 | /* Returns real value in [0, 1) using original transition matrix */ 37 | double mrg_get_double_orig(mrg_state* state); 38 | 39 | /* Seed PRNG with a given array of five values in the range [0, 0x7FFFFFFE] and 40 | * not all zero. */ 41 | void mrg_seed(mrg_state* st, const uint_fast32_t seed[5]); 42 | 43 | /* Skip the PRNG ahead _exponent_ steps. This code treats the exponent as a 44 | * 192-bit word, even though the PRNG period is less than that. */ 45 | void mrg_skip(mrg_state* state, 46 | uint_least64_t exponent_high, 47 | uint_least64_t exponent_middle, 48 | uint_least64_t exponent_low); 49 | 50 | #ifdef __cplusplus 51 | } 52 | #endif 53 | 54 | #endif /* SPLITTABLE_MRG_H */ 55 | -------------------------------------------------------------------------------- /generators/prng/utils.c: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2010 The Trustees of Indiana University. */ 2 | /* */ 3 | /* Use, modification and distribution is subject to the Boost Software */ 4 | /* License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at */ 5 | /* http://www.boost.org/LICENSE_1_0.txt) */ 6 | /* */ 7 | /* Authors: Jeremiah Willcock */ 8 | /* Andrew Lumsdaine */ 9 | 10 | #ifndef __STDC_CONSTANT_MACROS 11 | #define __STDC_CONSTANT_MACROS 12 | #endif 13 | #include "splittable_mrg.h" 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #ifdef __MTA__ 20 | #include 21 | #endif 22 | #ifdef GRAPH_GENERATOR_MPI 23 | #include 24 | #endif 25 | #ifdef GRAPH_GENERATOR_OMP 26 | #include 27 | #endif 28 | #include "utils.h" 29 | 30 | /* Spread the two 64-bit numbers into five nonzero values in the correct 31 | * range. */ 32 | void make_mrg_seed(uint64_t userseed1, uint64_t userseed2, uint_fast32_t* seed) { 33 | seed[0] = (userseed1 & 0x3FFFFFFF) + 1; 34 | seed[1] = ((userseed1 >> 30) & 0x3FFFFFFF) + 1; 35 | seed[2] = (userseed2 & 0x3FFFFFFF) + 1; 36 | seed[3] = ((userseed2 >> 30) & 0x3FFFFFFF) + 1; 37 | seed[4] = ((userseed2 >> 60) << 4) + (userseed1 >> 60) + 1; 38 | } 39 | -------------------------------------------------------------------------------- /generators/prng/utils.h: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2010 The Trustees of Indiana University. */ 2 | /* */ 3 | /* Use, modification and distribution is subject to the Boost Software */ 4 | /* License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at */ 5 | /* http://www.boost.org/LICENSE_1_0.txt) */ 6 | /* */ 7 | /* Authors: Jeremiah Willcock */ 8 | /* Andrew Lumsdaine */ 9 | 10 | #ifndef UTILS_H 11 | #define UTILS_H 12 | 13 | #include 14 | #include 15 | #include "splittable_mrg.h" 16 | 17 | uint_fast64_t random_up_to(mrg_state* st, uint_fast64_t n); 18 | void make_mrg_seed(uint64_t userseed1, uint64_t userseed2, uint_fast32_t* seed); 19 | 20 | #endif /* UTILS_H */ 21 | -------------------------------------------------------------------------------- /generators/random.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include 20 | #include 21 | #include "defs.h" 22 | #include "options.h" 23 | #include "util.h" 24 | #include "output.h" 25 | #include "prng/splittable_mrg.h" 26 | #include "prng/utils.h" 27 | 28 | inline static vertex_t generate_vertex(mrg_state *pstate, const vertex_t nvertices) { 29 | /* Generate a pseudorandom number in the range [0, vertices) without modulo bias. */ 30 | vertex_t limit = VT_MAX % nvertices; 31 | vertex_t v; 32 | 33 | do { 34 | v = mrg_get_uint_orig(pstate); 35 | #ifndef VERTEX_TYPE_32 36 | v = (v << 32) + mrg_get_uint_orig(pstate); 37 | #endif 38 | } while (UNLIKELY(v < limit)); 39 | return v % nvertices; 40 | } 41 | 42 | static void generate(thread_buffer *buffer, const mrg_state &state, const vertex_t nvertices, 43 | const edge_t start, const edge_t end, const bool allow_self_loops, const bool symmetric) { 44 | for (edge_t ei = start; ei < end; ++ei) { 45 | mrg_state new_state = state; 46 | mrg_skip(&new_state, 0, ei, 0); 47 | struct edge_struct *edge = buffer->edge_struct(); 48 | edge->src = generate_vertex(&new_state, nvertices); 49 | do { 50 | edge->dst = generate_vertex(&new_state, nvertices); 51 | } while (UNLIKELY(edge->src == edge->dst && !allow_self_loops)); 52 | #ifdef WEIGHT 53 | edge->weight = (value_t)mrg_get_double_orig(&new_state); 54 | #endif 55 | if (symmetric) { 56 | struct edge_struct *reverse_edge = buffer->edge_struct(); 57 | reverse_edge->src = edge->dst; 58 | reverse_edge->dst = edge->src; 59 | #ifdef WEIGHT 60 | reverse_edge->weight = edge->weight; 61 | #endif 62 | } 63 | } 64 | buffer->flush(); 65 | } 66 | 67 | static void generate_bipartite(thread_buffer *buffer, const mrg_state &state, const vertex_t nleft, 68 | const vertex_t nvertices, 69 | const edge_t start, const edge_t end, const bool symmetric) { 70 | for (edge_t ei = start; ei < end; ++ei) { 71 | mrg_state new_state = state; 72 | mrg_skip(&new_state, 0, ei, 0); 73 | struct edge_struct *edge = buffer->edge_struct(); 74 | edge->src = generate_vertex(&new_state, nvertices); 75 | if (edge->src < nleft) { 76 | edge->dst = nleft + generate_vertex(&new_state, nvertices - nleft); 77 | } else { 78 | edge->dst = generate_vertex(&new_state, nleft); 79 | } 80 | #ifdef WEIGHT 81 | edge->weight = (value_t)mrg_get_double_orig(&new_state); 82 | #endif 83 | if (symmetric) { 84 | struct edge_struct *reverse_edge = buffer->edge_struct(); 85 | reverse_edge->src = edge->dst; 86 | reverse_edge->dst = edge->src; 87 | #ifdef WEIGHT 88 | reverse_edge->weight = edge->weight; 89 | #endif 90 | } 91 | } 92 | buffer->flush(); 93 | } 94 | 95 | int main(int argc, char **argv) { 96 | struct options options; 97 | if (process_options(argc, argv, false, &options) != 0) 98 | return 0; 99 | 100 | uint_fast32_t seed[5]; 101 | make_mrg_seed(options.rng.userseed1, options.rng.userseed2, seed); 102 | mrg_state state; 103 | mrg_seed(&state, seed); 104 | //mrg_skip(&new_state, 50, 7, 0); // Do an initial skip? 105 | 106 | edge_t total_edges = options.erdos_renyi.edges; 107 | if (options.global.symmetric) { 108 | total_edges *= 2; 109 | } 110 | 111 | printf("Generator type: Erdos-Renyi\n"); 112 | printf("Vertices: %" PRIvt "\n", options.erdos_renyi.vertices); 113 | printf("Edges: %" PRIet "\n", total_edges); 114 | printf("Self-loops:%s allowed\n", (options.erdos_renyi.self_loops ? "" : " not")); 115 | 116 | double start = get_time(); 117 | 118 | // io thread 119 | size_t buffer_size = calculate_buffer_size(options.global.buffer_size); 120 | buffer_queue flushq; 121 | buffer_manager manager(&flushq, options.global.buffers_per_thread, buffer_size); 122 | io_thread_func io_func(options.global.graphname.c_str(), total_edges, &flushq, &manager, buffer_size); 123 | boost::thread io_thread(boost::ref(io_func)); 124 | 125 | // worker threads 126 | int nthreads = options.global.nthreads; 127 | edge_t edges_per_thread = options.erdos_renyi.edges / nthreads; 128 | threadid_t *workers[nthreads]; 129 | boost::thread *worker_threads[nthreads]; 130 | for (int i = 0; i < nthreads; i++) { 131 | workers[i] = new threadid_t(i); 132 | thread_buffer *buffer = manager.register_thread(*workers[i]); 133 | // last thread gets the remainder (if any) 134 | edge_t start = i * edges_per_thread; 135 | edge_t end = (i == nthreads - 1) ? (options.erdos_renyi.edges) : ((i + 1) * edges_per_thread); 136 | if (options.erdos_renyi.bipartite) { 137 | worker_threads[i] = new boost::thread( 138 | generate_bipartite, buffer, 139 | state, options.erdos_renyi.bipartite, options.erdos_renyi.vertices, start, end, options.global.symmetric 140 | ); 141 | } else { 142 | worker_threads[i] = new boost::thread( 143 | generate, buffer, 144 | state, options.erdos_renyi.vertices, start, end, options.erdos_renyi.self_loops, options.global.symmetric 145 | ); 146 | } 147 | } 148 | 149 | // Wait until work completes 150 | for (int i = 0; i < nthreads; i++) { 151 | worker_threads[i]->join(); 152 | } 153 | io_func.stop(); 154 | io_thread.join(); 155 | 156 | // cleanup 157 | for (int i = 0; i < nthreads; i++) { 158 | manager.unregister_thread(*workers[i]); 159 | delete worker_threads[i]; 160 | delete workers[i]; 161 | } 162 | 163 | double elapsed = get_time() - start; 164 | printf("Generation time: %fs\n", elapsed); 165 | 166 | make_ini_file(options.global.graphname.c_str(), options.erdos_renyi.vertices, total_edges); 167 | 168 | return 0; 169 | } 170 | 171 | -------------------------------------------------------------------------------- /generators/rmat.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include 20 | #include 21 | #include "defs.h" 22 | #include "options.h" 23 | #include "util.h" 24 | #include "output.h" 25 | #include "prng/splittable_mrg.h" 26 | #include "prng/utils.h" 27 | 28 | static void generate_edge(mrg_state *pstate, const int scale, 29 | double a, double b, double c, double d, 30 | struct edge_struct *edge) { 31 | vertex_t i = 0, j = 0; 32 | vertex_t bit = (vertex_t) 1 << (scale - 1); 33 | 34 | while (true) { 35 | double r = mrg_get_double_orig(pstate); 36 | if (r > a) { /* outside quadrant 1 */ 37 | if (r <= a + b) /* in quadrant 2 */ 38 | j |= bit; 39 | else if (r <= a + b + c) /* in quadrant 3 */ 40 | i |= bit; 41 | else { /* in quadrant 4 */ 42 | j |= bit; 43 | i |= bit; 44 | } 45 | } 46 | 47 | if (1 == bit) break; 48 | 49 | /* 50 | * Noise is introduced by modifying the probabilites by +/- 5% 51 | * and normalising them. 52 | */ 53 | #ifdef NOISE 54 | r = mrg_get_double_orig(pstate); 55 | a *= 0.95 + r/10; 56 | r = mrg_get_double_orig(pstate); 57 | b *= 0.95 + r/10; 58 | r = mrg_get_double_orig(pstate); 59 | c *= 0.95 + r/10; 60 | r = mrg_get_double_orig(pstate); 61 | d *= 0.95 + r/10; 62 | 63 | double norm = 1.0 / (a + b + c + d); 64 | a *= norm; 65 | b *= norm; 66 | c *= norm; 67 | d = 1.0 - (a + b + c); 68 | #endif 69 | 70 | /* Iterates scale times. */ 71 | bit >>= 1; 72 | } 73 | 74 | edge->src = i; 75 | edge->dst = j; 76 | #ifdef WEIGHT 77 | edge->weight = (value_t)mrg_get_double_orig(pstate); 78 | #endif 79 | } 80 | 81 | // ugly, but max 10 args to functions allowed in boost::thread 82 | static unsigned int xscale_interval; 83 | static unsigned int xscale_node; 84 | 85 | static void generate(thread_buffer *buffer, const mrg_state &state, const int 86 | scale, const edge_t start, const edge_t end, 87 | const double a, const double b, const double c, /*const 88 | double d,*/ const bool symmetric) { 89 | const double d = 1 - (a + b + c); 90 | for (edge_t ei = start; ei < end; ++ei) { 91 | if (ei % xscale_interval != xscale_node) { continue; } 92 | mrg_state new_state = state; 93 | mrg_skip(&new_state, 0, ei, 0); 94 | struct edge_struct *edge = buffer->edge_struct(); 95 | generate_edge(&new_state, scale, a, b, c, d, edge); 96 | if (symmetric) { 97 | struct edge_struct *reverse_edge = buffer->edge_struct(); 98 | reverse_edge->src = edge->dst; 99 | reverse_edge->dst = edge->src; 100 | #ifdef WEIGHT 101 | reverse_edge->weight = edge->weight; 102 | #endif 103 | } 104 | } 105 | buffer->flush(); 106 | } 107 | 108 | int main(int argc, char **argv) { 109 | struct options options; 110 | if (process_options(argc, argv, true, &options) != 0) 111 | return 0; 112 | 113 | if (options.rmat.a + options.rmat.b + options.rmat.c >= 1) { 114 | printf("Error: The sum of probabilities must equal 1\n"); 115 | return 0; 116 | } 117 | double d = 1 - (options.rmat.a + options.rmat.b + options.rmat.c); 118 | xscale_node = options.rmat.xscale_node; 119 | xscale_interval = options.rmat.xscale_interval; 120 | uint_fast32_t seed[5]; 121 | make_mrg_seed(options.rng.userseed1, options.rng.userseed2, seed); 122 | mrg_state state; 123 | mrg_seed(&state, seed); 124 | //mrg_skip(&new_state, 50, 7, 0); // Do an initial skip? 125 | 126 | edge_t total_edges = options.rmat.edges; 127 | if ((total_edges % options.rmat.xscale_interval) > options.rmat.xscale_node) { 128 | total_edges /= options.rmat.xscale_interval; 129 | total_edges++; 130 | } 131 | else { 132 | total_edges /= options.rmat.xscale_interval; 133 | } 134 | 135 | if (options.global.symmetric) { 136 | total_edges *= 2; 137 | } 138 | 139 | printf("Generator type: R-MAT\n"); 140 | printf("Scale: %d (%" PRIu64 " vertices)\n", options.rmat.scale, ((uint64_t) 1 << options.rmat.scale)); 141 | printf("Edges: %" PRIet "\n", total_edges); 142 | printf("Probabilities: A=%4.2f, B=%4.2f, C=%4.2f, D=%4.2f\n", options.rmat.a, options.rmat.b, options.rmat.c, d); 143 | 144 | double start = get_time(); 145 | 146 | // io thread 147 | size_t buffer_size = calculate_buffer_size(options.global.buffer_size); 148 | buffer_queue flushq; 149 | buffer_manager manager(&flushq, options.global.buffers_per_thread, buffer_size); 150 | io_thread_func io_func(options.global.graphname.c_str(), total_edges, &flushq, &manager, buffer_size); 151 | boost::thread io_thread(boost::ref(io_func)); 152 | 153 | // worker threads 154 | int nthreads = options.global.nthreads; 155 | edge_t edges_per_thread = options.rmat.edges / nthreads; 156 | threadid_t *workers[nthreads]; 157 | boost::thread *worker_threads[nthreads]; 158 | for (int i = 0; i < nthreads; i++) { 159 | workers[i] = new threadid_t(i); 160 | thread_buffer *buffer = manager.register_thread(*workers[i]); 161 | // last thread gets the remainder (if any) 162 | edge_t start = i * edges_per_thread; 163 | edge_t end = (i == nthreads - 1) ? (options.rmat.edges) : ((i + 1) * edges_per_thread); 164 | worker_threads[i] = new boost::thread(generate, buffer, 165 | state, options.rmat.scale, start, end, 166 | options.rmat.a, options.rmat.b, options.rmat.c, /*d,*/ 167 | options.global.symmetric); 168 | } 169 | 170 | // Wait until work completes 171 | for (int i = 0; i < nthreads; i++) { 172 | worker_threads[i]->join(); 173 | } 174 | io_func.stop(); 175 | io_thread.join(); 176 | 177 | // cleanup 178 | for (int i = 0; i < nthreads; i++) { 179 | manager.unregister_thread(*workers[i]); 180 | delete worker_threads[i]; 181 | delete workers[i]; 182 | } 183 | 184 | double elapsed = get_time() - start; 185 | printf("Generation time: %fs\n", elapsed); 186 | 187 | make_ini_file(options.global.graphname.c_str(), (uint64_t) 1 << options.rmat.scale, total_edges); 188 | 189 | return 0; 190 | } 191 | 192 | -------------------------------------------------------------------------------- /generators/scramble.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef HPGP_GENERATORS_SCRAMBLE_H 20 | #define HPGP_GENERATORS_SCRAMBLE_H 21 | 22 | /* Reverse bits in a number; this should be optimized for performance 23 | * (including using bit- or byte-reverse intrinsics if your platform has them). 24 | */ 25 | static inline uint64_t bitreverse(uint64_t x) { 26 | #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3) 27 | #define USE_GCC_BYTESWAP /* __builtin_bswap* are in 4.3 but not 4.2 */ 28 | #endif 29 | 30 | #ifdef FAST_64BIT_ARITHMETIC 31 | 32 | /* 64-bit code */ 33 | #ifdef USE_GCC_BYTESWAP 34 | x = __builtin_bswap64(x); 35 | #else 36 | x = (x >> 32) | (x << 32); 37 | x = ((x >> 16) & UINT64_C(0x0000FFFF0000FFFF)) | ((x & UINT64_C(0x0000FFFF0000FFFF)) << 16); 38 | x = ((x >> 8) & UINT64_C(0x00FF00FF00FF00FF)) | ((x & UINT64_C(0x00FF00FF00FF00FF)) << 8); 39 | #endif 40 | x = ((x >> 4) & UINT64_C(0x0F0F0F0F0F0F0F0F)) | ((x & UINT64_C(0x0F0F0F0F0F0F0F0F)) << 4); 41 | x = ((x >> 2) & UINT64_C(0x3333333333333333)) | ((x & UINT64_C(0x3333333333333333)) << 2); 42 | x = ((x >> 1) & UINT64_C(0x5555555555555555)) | ((x & UINT64_C(0x5555555555555555)) << 1); 43 | return x; 44 | 45 | #else 46 | 47 | /* 32-bit code */ 48 | uint32_t h = (uint32_t)(x >> 32); 49 | uint32_t l = (uint32_t)(x & UINT32_MAX); 50 | #ifdef USE_GCC_BYTESWAP 51 | h = __builtin_bswap32(h); 52 | l = __builtin_bswap32(l); 53 | #else 54 | h = (h >> 16) | (h << 16); 55 | l = (l >> 16) | (l << 16); 56 | h = ((h >> 8) & UINT32_C(0x00FF00FF)) | ((h & UINT32_C(0x00FF00FF)) << 8); 57 | l = ((l >> 8) & UINT32_C(0x00FF00FF)) | ((l & UINT32_C(0x00FF00FF)) << 8); 58 | #endif 59 | h = ((h >> 4) & UINT32_C(0x0F0F0F0F)) | ((h & UINT32_C(0x0F0F0F0F)) << 4); 60 | l = ((l >> 4) & UINT32_C(0x0F0F0F0F)) | ((l & UINT32_C(0x0F0F0F0F)) << 4); 61 | h = ((h >> 2) & UINT32_C(0x33333333)) | ((h & UINT32_C(0x33333333)) << 2); 62 | l = ((l >> 2) & UINT32_C(0x33333333)) | ((l & UINT32_C(0x33333333)) << 2); 63 | h = ((h >> 1) & UINT32_C(0x55555555)) | ((h & UINT32_C(0x55555555)) << 1); 64 | l = ((l >> 1) & UINT32_C(0x55555555)) | ((l & UINT32_C(0x55555555)) << 1); 65 | return ((uint64_t) l << 32) | h; /* Swap halves */ 66 | 67 | #endif 68 | } 69 | 70 | /* Apply a permutation to scramble vertex numbers; a randomly generated 71 | * permutation is not used because applying it at scale is too expensive. 72 | */ 73 | inline int64_t scramble(int64_t v0, int lgN, uint64_t val0, uint64_t val1) { 74 | uint64_t v = (uint64_t) v0; 75 | v += val0 + val1; 76 | v *= (val0 | UINT64_C(0x4519840211493211)); 77 | v = (bitreverse(v) >> (64 - lgN)); 78 | assert((v >> lgN) == 0); 79 | v *= (val1 | UINT64_C(0x3050852102C843A5)); 80 | v = (bitreverse(v) >> (64 - lgN)); 81 | assert((v >> lgN) == 0); 82 | return (int64_t) v; 83 | } 84 | 85 | #endif 86 | -------------------------------------------------------------------------------- /generators/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef HPGP_GENERATORS_UTIL_H 20 | #define HPGP_GENERATORS_UTIL_H 21 | 22 | #include 23 | #include 24 | #include "defs.h" 25 | #include "output.h" 26 | 27 | inline double get_time() { 28 | struct timeval tv; 29 | gettimeofday(&tv, NULL); 30 | return tv.tv_sec + tv.tv_usec * 1.e-6; 31 | } 32 | 33 | inline size_t calculate_buffer_size(const size_t cmdline_value) { 34 | size_t buffer_size = cmdline_value; 35 | if (!buffer_size) { 36 | buffer_size = 256 * DEFAULT_BLOCK_SIZE * sizeof(struct edge_struct); 37 | } 38 | if (buffer_size % sizeof(struct edge_struct) != 0) { 39 | fprintf(stderr, "Error: Buffer size must be a multiple of edge_struct size\n"); 40 | exit(1); 41 | } 42 | if (buffer_size % DEFAULT_BLOCK_SIZE != 0) { 43 | fprintf(stderr, "Error: Buffer size must be a multiple of 4096 (disk sector size)\n"); 44 | exit(1); 45 | } 46 | return buffer_size; 47 | } 48 | 49 | #endif /* HPGP_GENERATORS_UTIL_H */ 50 | -------------------------------------------------------------------------------- /slipstore/checkpoints.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _CHECKPOINTS_ 20 | #define _CHECKPOINTS_ 21 | 22 | // Manage checkpoint metadata 23 | // File format: 24 | // 25 | // 26 | // MAGIC NUMBER 27 | // Checkpoint number 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include "../utils/memory_utils.h" 33 | 34 | /* Written out at the end of a stable checkpoint */ 35 | static const unsigned long CHK_MAGIC = 0xCAFECAFECAFECAFE; 36 | struct __attribute__((__packed__)) chkpnt_tail { 37 | unsigned long magic; 38 | unsigned long num; 39 | }; 40 | 41 | static unsigned long chkpnt_no; 42 | 43 | /* Set the next checkpoint number */ 44 | static void set_checkpoint_no(unsigned long chkpnt_no_in) { 45 | chkpnt_no = chkpnt_no_in; 46 | } 47 | 48 | /* Load checkpoint */ 49 | static void read_checkpoint(unsigned char *app_data, 50 | unsigned long app_data_size, 51 | unsigned char *runtime_data, 52 | unsigned long runtime_data_size, 53 | unsigned long no) { 54 | char fname[100]; 55 | sprintf(fname, "chkpnt_meta.%lu", no); 56 | int fd = open(fname, O_RDONLY); 57 | if (fd == -1) { 58 | BOOST_LOG_TRIVIAL(fatal) 59 | << "Unable to open checkpoint metadata file for reading: " 60 | << strerror(errno); 61 | exit(-1); 62 | } 63 | if (app_data_size > 0) { 64 | read_from_file(fd, app_data, app_data_size); 65 | } 66 | if (runtime_data_size > 0) { 67 | read_from_file(fd, runtime_data, runtime_data_size); 68 | } 69 | close(fd); 70 | } 71 | 72 | static unsigned long next_checkpoint() { 73 | return (chkpnt_no++); 74 | } 75 | 76 | /* Write out a new checkpoint */ 77 | static void write_checkpoint(unsigned long no, 78 | unsigned char *app_data, 79 | unsigned long app_data_size, 80 | unsigned char *runtime_data, 81 | unsigned long runtime_data_size) { 82 | char fname[100], fname_final[100]; 83 | chkpnt_tail tail; 84 | sprintf(fname, "chkpnt_meta_prep.%lu", no); 85 | sprintf(fname_final, "chkpnt_meta.%lu", no); 86 | // Open the file in sync mode for append guarantees 87 | int fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY | O_APPEND | O_SYNC, S_IRWXU); 88 | if (fd == -1) { 89 | BOOST_LOG_TRIVIAL(fatal) 90 | << "Unable to open checkpoint metadata file for writing: " 91 | << strerror(errno); 92 | exit(-1); 93 | } 94 | if (app_data_size > 0) { 95 | write_to_file(fd, app_data, app_data_size); 96 | } 97 | if (runtime_data_size > 0) { 98 | write_to_file(fd, runtime_data, runtime_data_size); 99 | } 100 | tail.magic = CHK_MAGIC; 101 | tail.num = no; 102 | write_to_file(fd, (unsigned char *) &tail, sizeof(chkpnt_tail)); 103 | fsync(fd);// Ensure file is created and on disk 104 | int ret = rename(fname, fname_final); 105 | if (ret < 0) { 106 | BOOST_LOG_TRIVIAL(fatal) 107 | << "Unable to save checkpoint " 108 | << strerror(errno); 109 | exit(-1); 110 | } 111 | fsync(fd);// Ensure all metadata changes are on disk 112 | close(fd); 113 | } 114 | 115 | /* Delete a checkpoint */ 116 | static void del_checkpoint(unsigned long no) { 117 | char fname[100]; 118 | sprintf(fname, "chkpnt_meta.%lu", no); 119 | unlink_file(fname); 120 | } 121 | 122 | /* Scan current working directory to locate the last stable checkpoint */ 123 | static unsigned long find_checkpoint(unsigned long app_data_size, 124 | unsigned long runtime_data_size) { 125 | unsigned long cmx = ULONG_MAX; 126 | DIR *dp; 127 | struct dirent *ep; 128 | chkpnt_tail tail; 129 | dp = opendir("./"); 130 | if (dp != NULL) { 131 | while ((ep = readdir(dp)) != NULL) { 132 | if (strncmp(ep->d_name, "chkpnt_meta.", 12) == 0) { 133 | unsigned long cno = atol(ep->d_name + 12); 134 | if (cno > cmx || cmx == ULONG_MAX) { 135 | cmx = cno; 136 | } 137 | } 138 | } 139 | (void) closedir(dp); 140 | } 141 | else { 142 | BOOST_LOG_TRIVIAL(fatal) << "Couldnt open current directory:" 143 | << strerror(errno); 144 | exit(-1); 145 | } 146 | if (cmx != ULONG_MAX) { 147 | /* Verify that the checkpoint is stable */ 148 | do { 149 | char fname[100]; 150 | sprintf(fname, "chkpnt_meta.%lu", cmx); 151 | int fd = open(fname, O_RDONLY); 152 | if (fd != -1) { 153 | // Make sure the checkpoint is complete 154 | if (get_file_size(fd) == 155 | (app_data_size + runtime_data_size + sizeof(chkpnt_tail))) { 156 | // Verify end of file 157 | set_filepos(fd, app_data_size + runtime_data_size); 158 | read_from_file(fd, (unsigned char *) &tail, sizeof(chkpnt_tail)); 159 | if (tail.magic != CHK_MAGIC || tail.num != cmx) { 160 | BOOST_LOG_TRIVIAL(warn) 161 | << "SLIPSTREAM::CHECKPOINT Skipping corrupt checkpoint " 162 | << cmx; 163 | } 164 | else { 165 | close(fd); 166 | break; 167 | } 168 | } 169 | else { 170 | BOOST_LOG_TRIVIAL(warn) 171 | << "SLIPSTREAM::CHECKPOINT Skipping incomplete checkpoint " 172 | << cmx; 173 | } 174 | close(fd); 175 | } 176 | } while ((cmx--) != 0); 177 | } 178 | if (cmx != ULONG_MAX) { 179 | BOOST_LOG_TRIVIAL(info) 180 | << "SLIPSTREAM::CHECKPOINT Found max checkpoint " 181 | << cmx; 182 | } 183 | return cmx; 184 | } 185 | 186 | #endif 187 | -------------------------------------------------------------------------------- /slipstore/dio_buffer.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _DIO_BUFFER_ 20 | #define _DIO_BUFFER_ 21 | 22 | #include "../utils/memory_utils.h" 23 | 24 | // Direct I/O buffer 25 | class dio_buffer { 26 | unsigned char *buffer; 27 | unsigned long bufsize; 28 | unsigned long bufcontents; 29 | volatile bool busy; 30 | bool dirty; 31 | int fd; 32 | 33 | void flush_buffer_internal() { 34 | if (dirty && bufcontents > 0) { 35 | write_to_file(fd, buffer, bufsize); 36 | } 37 | bufcontents = 0; 38 | dirty = false; 39 | } 40 | 41 | void fill_buffer_internal() { 42 | read_from_file(fd, buffer, bufsize); 43 | bufcontents = bufsize; 44 | } 45 | 46 | public: 47 | 48 | void set_busy() { 49 | busy = true; 50 | __sync_synchronize(); 51 | } 52 | 53 | bool get_busy() { 54 | return busy; 55 | } 56 | 57 | unsigned long alignment() { 58 | return 4096; 59 | } 60 | 61 | void seek(unsigned long offset) { 62 | unsigned char dummy[128]; 63 | unsigned long aligned_offset = (offset / 4096) * 4096; 64 | set_filepos(fd, aligned_offset); 65 | bufcontents = 0; 66 | unsigned long to_waste = offset - aligned_offset; 67 | while (to_waste) { 68 | unsigned long get_bytes = to_waste; 69 | if (get_bytes > 128) { 70 | get_bytes = 128; 71 | } 72 | dio_buffer::read(dummy, get_bytes); 73 | to_waste -= get_bytes; 74 | } 75 | __sync_synchronize(); 76 | busy = false; 77 | } 78 | 79 | dio_buffer(int fd_in, unsigned long bufsize_in) 80 | : bufsize(bufsize_in), 81 | bufcontents(0), 82 | busy(false), 83 | dirty(false), 84 | fd(fd_in) { 85 | // Adjust to 4K boundary 86 | bufsize = ((bufsize + 4095) / 4096) * 4096; 87 | // Allocate and mlock buffer 88 | buffer = (unsigned char *) map_anon_memory(bufsize, true, "dio buffer"); 89 | } 90 | 91 | void write(unsigned char *buf, unsigned long size) { 92 | while (size) { 93 | unsigned long space = bufsize - bufcontents; 94 | if (space == 0) { 95 | flush_buffer_internal(); 96 | space = bufsize; 97 | } 98 | if (space > size) { 99 | space = size; 100 | } 101 | memcpy(buffer + bufcontents, buf, space); 102 | dirty = true; 103 | buf += space; 104 | bufcontents += space; 105 | size -= space; 106 | } 107 | __sync_synchronize(); 108 | busy = false; 109 | } 110 | 111 | void read(unsigned char *buf, unsigned long size) { 112 | while (size) { 113 | if (bufcontents == 0) { 114 | fill_buffer_internal(); 115 | } 116 | unsigned long avail = bufcontents; 117 | if (avail > size) { 118 | avail = size; 119 | } 120 | memcpy(buf, buffer + (bufsize - bufcontents), avail); 121 | buf += avail; 122 | bufcontents -= avail; 123 | size -= avail; 124 | } 125 | __sync_synchronize(); 126 | busy = false; 127 | } 128 | 129 | void flush() { 130 | flush_buffer_internal(); 131 | __sync_synchronize(); 132 | busy = false; 133 | } 134 | 135 | ~dio_buffer() { 136 | flush_buffer_internal(); 137 | } 138 | }; 139 | 140 | #endif 141 | -------------------------------------------------------------------------------- /slipstore/dummy_io.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _DUMMY_IO_ 20 | #define _DUMMY_IO_ 21 | 22 | #include "generic_io.hpp" 23 | 24 | namespace slipstore { 25 | class dummy_io : public slipstore_io { 26 | unsigned long stream; 27 | unsigned long partition; 28 | unsigned long tile; 29 | unsigned long seen_bytes; 30 | unsigned long data_bytes; 31 | char name[100]; 32 | int fd; 33 | bool snapped; 34 | public: 35 | dummy_io(unsigned long stream_in, 36 | unsigned long partition_in, 37 | unsigned long tile_in, 38 | const char *name_in = NULL) 39 | : stream(stream_in), 40 | partition(partition_in), 41 | tile(tile_in), 42 | seen_bytes(0), 43 | snapped(false) { 44 | if (name_in != NULL) { 45 | strcpy(name, name_in); 46 | fd = open(name, O_RDWR | O_LARGEFILE, S_IRWXU); 47 | data_bytes = get_file_size(fd); 48 | } 49 | else { 50 | sprintf(name, "stream.%lu.%lu.%lu", stream, partition, tile); 51 | // This prevents us from over writing any snapshots 52 | unlink_file(name); 53 | fd = open(name, 54 | O_RDWR | O_LARGEFILE | O_CREAT | O_TRUNC, 55 | S_IRWXU); 56 | data_bytes = 0; 57 | } 58 | posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); 59 | } 60 | 61 | void force_file_size(unsigned long size) { 62 | data_bytes = size; 63 | } 64 | 65 | virtual unsigned long fill(unsigned char *buffer, 66 | unsigned long size) { 67 | unsigned long remaining_bytes = data_bytes - seen_bytes; 68 | if (size > remaining_bytes) { 69 | size = remaining_bytes; 70 | } 71 | seen_bytes += size; 72 | // Dont actually do the I/O 73 | //read_from_file(fd, buffer, size); 74 | return size; 75 | } 76 | 77 | virtual unsigned long seek_and_fill(unsigned char *buffer, 78 | unsigned long offset, 79 | unsigned long size) { 80 | unsigned long remaining_bytes; 81 | set_filepos(fd, offset); 82 | seen_bytes = offset; 83 | remaining_bytes = data_bytes - seen_bytes; 84 | if (size > remaining_bytes) { 85 | size = remaining_bytes; 86 | } 87 | seen_bytes += size; 88 | // Dont actually do the I/O 89 | //read_from_file(fd, buffer, size); 90 | return size; 91 | } 92 | 93 | virtual void drain(unsigned char *buffer, 94 | unsigned long size) { 95 | seen_bytes += size; 96 | // Don't actually do the IO 97 | //write_to_file(fd, buffer, size); 98 | #ifdef FLUSH_PERIODIC 99 | if(((data_bytes + size)/(4*1024*1024)) != 100 | (data_bytes/(4*1024*1024))) { 101 | fdatasync(fd); 102 | } 103 | #endif 104 | data_bytes += size; 105 | usage += size; 106 | if (usage > max_quota_usage) { 107 | max_quota_usage = usage; 108 | } 109 | } 110 | 111 | virtual bool eof() { 112 | return seen_bytes == data_bytes; 113 | } 114 | 115 | virtual unsigned long size() { 116 | return data_bytes; 117 | } 118 | 119 | virtual unsigned long left() { 120 | return data_bytes - seen_bytes; 121 | } 122 | 123 | virtual void rewind() { 124 | rewind_file(fd); 125 | seen_bytes = 0; 126 | } 127 | 128 | virtual void trunc() { 129 | if (!snapped) { 130 | rewind_file(fd); 131 | truncate_file(fd); 132 | } 133 | else { 134 | close(fd); 135 | unlink_file(name); 136 | fd = open(name, 137 | O_RDWR | O_LARGEFILE | O_CREAT | O_TRUNC, 138 | S_IRWXU); 139 | posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); 140 | snapped = false; 141 | } 142 | usage -= data_bytes; 143 | seen_bytes = 0; 144 | data_bytes = 0; 145 | } 146 | 147 | virtual void take_snap(unsigned long no, fsnap *meta) { 148 | char new_name[200]; 149 | fsync(fd); 150 | sprintf(new_name, "snap_%lu_", no); 151 | strcat(new_name, name); 152 | int ret = link(name, new_name); 153 | if (ret < 0) { 154 | BOOST_LOG_TRIVIAL(fatal) << "Unable to create snap link" 155 | << strerror(errno); 156 | exit(-1); 157 | } 158 | snapped = true; 159 | meta->pos = seen_bytes; 160 | meta->size = data_bytes; 161 | } 162 | 163 | virtual void restore_snap(unsigned long no, 164 | fsnap *meta) { 165 | char snap_name[200]; 166 | close(fd); 167 | unlink_file(name); 168 | sprintf(snap_name, "snap_%lu_", no); 169 | strcat(snap_name, name); 170 | int ret = link(snap_name, name); 171 | if (ret < 0) { 172 | BOOST_LOG_TRIVIAL(fatal) << "Unable to create snap link" 173 | << strerror(errno); 174 | exit(-1); 175 | } 176 | fd = open(name, O_RDWR | O_LARGEFILE); 177 | posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); 178 | truncate_file_to_size(fd, meta->size); 179 | data_bytes = meta->size; 180 | set_filepos(fd, meta->pos); 181 | seen_bytes = meta->pos; 182 | snapped = true; 183 | } 184 | 185 | virtual void delete_snap(unsigned long no) { 186 | char snap_name[200]; 187 | sprintf(snap_name, "snap_%lu_", no); 188 | strcat(snap_name, name); 189 | unlink_file(snap_name); 190 | } 191 | 192 | virtual ~dummy_io() { 193 | close(fd); 194 | } 195 | }; 196 | } 197 | #endif 198 | -------------------------------------------------------------------------------- /slipstore/generic_io.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _GENERIC_IO_ 20 | #define _GENERIC_IO_ 21 | namespace slipstore { 22 | 23 | struct fsnap { 24 | unsigned long pos; 25 | unsigned long size; 26 | }; 27 | 28 | extern unsigned long quota; 29 | extern unsigned long usage; 30 | extern unsigned long max_quota_usage; 31 | 32 | class slipstore_io { 33 | public: 34 | virtual unsigned long fill(unsigned char *buffer, 35 | unsigned long size) = 0; 36 | 37 | virtual unsigned long seek_and_fill(unsigned char *buffer, 38 | unsigned long offset, 39 | unsigned long size) = 0; 40 | 41 | virtual void drain(unsigned char *buffer, 42 | unsigned long size) = 0; 43 | 44 | virtual bool eof() = 0; 45 | 46 | virtual unsigned long size() = 0; 47 | 48 | virtual unsigned long left() = 0; 49 | 50 | virtual void rewind() = 0; 51 | 52 | virtual void trunc() = 0; 53 | 54 | virtual void take_snap(unsigned long no, fsnap *meta) = 0; 55 | 56 | virtual void restore_snap(unsigned long no, 57 | fsnap *meta) = 0; 58 | 59 | virtual void delete_snap(unsigned long no) = 0; 60 | 61 | virtual ~slipstore_io() { } 62 | }; 63 | } 64 | #endif 65 | -------------------------------------------------------------------------------- /slipstore/sync_pagecache_io.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _SYNC_PAGECACHE_IO_ 20 | #define _SYNC_PAGECACHE_IO_ 21 | 22 | #include "generic_io.hpp" 23 | 24 | namespace slipstore { 25 | class sync_pagecache_io : public slipstore_io { 26 | unsigned long stream; 27 | unsigned long partition; 28 | unsigned long tile; 29 | unsigned long seen_bytes; 30 | unsigned long data_bytes; 31 | char name[100]; 32 | int fd; 33 | bool snapped; 34 | public: 35 | sync_pagecache_io(unsigned long stream_in, 36 | unsigned long partition_in, 37 | unsigned long tile_in, 38 | const char *name_in = NULL) 39 | : stream(stream_in), 40 | partition(partition_in), 41 | tile(tile_in), 42 | seen_bytes(0), 43 | snapped(false) { 44 | if (name_in != NULL) { 45 | strcpy(name, name_in); 46 | fd = open(name, O_RDWR | O_LARGEFILE, S_IRWXU); 47 | data_bytes = get_file_size(fd); 48 | } 49 | else { 50 | sprintf(name, "stream.%lu.%lu.%lu", stream, partition, tile); 51 | // This prevents us from over writing any snapshots 52 | unlink_file(name); 53 | fd = open(name, 54 | O_RDWR | O_LARGEFILE | O_CREAT | O_TRUNC, 55 | S_IRWXU); 56 | data_bytes = 0; 57 | } 58 | posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); 59 | } 60 | 61 | virtual unsigned long fill(unsigned char *buffer, 62 | unsigned long size) { 63 | unsigned long remaining_bytes = data_bytes - seen_bytes; 64 | if (size > remaining_bytes) { 65 | size = remaining_bytes; 66 | } 67 | seen_bytes += size; 68 | read_from_file(fd, buffer, size); 69 | return size; 70 | } 71 | 72 | virtual unsigned long seek_and_fill(unsigned char *buffer, 73 | unsigned long offset, 74 | unsigned long size) { 75 | unsigned long remaining_bytes; 76 | set_filepos(fd, offset); 77 | seen_bytes = offset; 78 | remaining_bytes = data_bytes - seen_bytes; 79 | if (size > remaining_bytes) { 80 | size = remaining_bytes; 81 | } 82 | seen_bytes += size; 83 | read_from_file(fd, buffer, size); 84 | return size; 85 | } 86 | 87 | virtual void drain(unsigned char *buffer, 88 | unsigned long size) { 89 | seen_bytes += size; 90 | write_to_file(fd, buffer, size); 91 | #ifdef FLUSH_PERIODIC 92 | if(((data_bytes + size)/(4*1024*1024)) != 93 | (data_bytes/(4*1024*1024))) { 94 | fdatasync(fd); 95 | } 96 | #endif 97 | data_bytes += size; 98 | usage += size; 99 | if (usage > max_quota_usage) { 100 | max_quota_usage = usage; 101 | } 102 | } 103 | 104 | virtual bool eof() { 105 | return seen_bytes == data_bytes; 106 | } 107 | 108 | virtual unsigned long size() { 109 | return data_bytes; 110 | } 111 | 112 | virtual unsigned long left() { 113 | return data_bytes - seen_bytes; 114 | } 115 | 116 | virtual void rewind() { 117 | rewind_file(fd); 118 | seen_bytes = 0; 119 | } 120 | 121 | virtual void trunc() { 122 | if (!snapped) { 123 | rewind_file(fd); 124 | truncate_file(fd); 125 | } 126 | else { 127 | close(fd); 128 | unlink_file(name); 129 | fd = open(name, 130 | O_RDWR | O_LARGEFILE | O_CREAT | O_TRUNC, 131 | S_IRWXU); 132 | posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); 133 | snapped = false; 134 | } 135 | usage -= data_bytes; 136 | seen_bytes = 0; 137 | data_bytes = 0; 138 | } 139 | 140 | virtual void take_snap(unsigned long no, fsnap *meta) { 141 | char new_name[200]; 142 | fsync(fd); 143 | sprintf(new_name, "snap_%lu_", no); 144 | strcat(new_name, name); 145 | int ret = link(name, new_name); 146 | if (ret < 0) { 147 | BOOST_LOG_TRIVIAL(fatal) << "Unable to create snap link" 148 | << strerror(errno); 149 | exit(-1); 150 | } 151 | snapped = true; 152 | meta->pos = seen_bytes; 153 | meta->size = data_bytes; 154 | } 155 | 156 | virtual void restore_snap(unsigned long no, 157 | fsnap *meta) { 158 | char snap_name[200]; 159 | close(fd); 160 | unlink_file(name); 161 | sprintf(snap_name, "snap_%lu_", no); 162 | strcat(snap_name, name); 163 | int ret = link(snap_name, name); 164 | if (ret < 0) { 165 | BOOST_LOG_TRIVIAL(fatal) << "Unable to create snap link" 166 | << strerror(errno); 167 | exit(-1); 168 | } 169 | fd = open(name, O_RDWR | O_LARGEFILE); 170 | posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); 171 | truncate_file_to_size(fd, meta->size); 172 | data_bytes = meta->size; 173 | set_filepos(fd, meta->pos); 174 | seen_bytes = meta->pos; 175 | snapped = true; 176 | } 177 | 178 | virtual void delete_snap(unsigned long no) { 179 | char snap_name[200]; 180 | sprintf(snap_name, "snap_%lu_", no); 181 | strcat(snap_name, name); 182 | unlink_file(snap_name); 183 | } 184 | 185 | virtual ~sync_pagecache_io() { 186 | close(fd); 187 | } 188 | }; 189 | } 190 | #endif 191 | -------------------------------------------------------------------------------- /utils/barrier.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _BARRIER_ 20 | #define _BARRIER_ 21 | // Sense reversing barrier 22 | // without syscall overheads of pthread_barrier 23 | 24 | class x_barrier { 25 | volatile unsigned long count[2]; 26 | volatile unsigned long sense; 27 | unsigned long expected; 28 | public: 29 | x_barrier(unsigned long expected_in) 30 | : sense(0), expected(expected_in) { 31 | count[0] = 0; 32 | count[1] = 0; 33 | } 34 | 35 | void wait() { 36 | unsigned long sense_used = sense; 37 | unsigned long arrived = 38 | __sync_fetch_and_add(&count[sense_used], 1); 39 | if (arrived == (expected - 1)) { 40 | sense = 1 - sense_used; // Reverse sense 41 | count[sense_used] = 0; 42 | } 43 | while (count[sense_used] != 0); 44 | __sync_synchronize(); // Also clobber memory 45 | } 46 | }; 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /utils/block_mem.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include 20 | #include "memory_utils.h" 21 | #include "unistd.h" 22 | 23 | #define TOLOCK (1024*1024UL) 24 | 25 | int main() { 26 | fprintf(stderr, "Locking %lu bytes", TOLOCK); 27 | void *dummy = map_anon_memory(TOLOCK, true, "Dummy"); 28 | (void) dummy; 29 | while (1) { 30 | pause(); 31 | } 32 | fprintf(stderr, "Exiting .. this should not happen !"); 33 | return 0; 34 | } 35 | -------------------------------------------------------------------------------- /utils/boost_log_wrapper.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | // Until boost::log becomes mainstream use this substitute 20 | #ifndef _BOOST_LOG_WRAPPER_ 21 | #define _BOOST_LOG_WRAPPER_ 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | enum log_state { 29 | fatal = 0, 30 | error = 1, 31 | warn = 2, 32 | info = 3, 33 | total = 4 34 | }; 35 | 36 | static const char *log_headers[total] = 37 | {"FATAL", 38 | "ERROR", 39 | "WARNING", 40 | "INFO"}; 41 | 42 | class BOOST_LOG_TRIVIAL { 43 | enum log_state level; 44 | public: 45 | std::stringstream state; 46 | 47 | BOOST_LOG_TRIVIAL(enum log_state level_in) 48 | : level(level_in) { 49 | } 50 | 51 | template 52 | BOOST_LOG_TRIVIAL &operator<<(T value) { 53 | state << value; 54 | return *this; 55 | } 56 | 57 | ~BOOST_LOG_TRIVIAL() { 58 | std::stringstream final; 59 | final << "<" << log_headers[level] << " "; 60 | final << boost::this_thread::get_id() << "> "; 61 | final << state.str() << std::endl; 62 | std::cerr << final.str(); 63 | if (level == fatal) { 64 | BOOST_ASSERT_MSG(false, "Fatal stop."); 65 | } 66 | } 67 | }; 68 | 69 | #endif 70 | -------------------------------------------------------------------------------- /utils/clock.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _CLOCK_GENERIC_ 20 | #define _CLOCK_GENERIC_ 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | class clock { 28 | public: 29 | virtual void start() = 0; 30 | 31 | virtual void stop() = 0; 32 | 33 | virtual void reset() = 0; 34 | 35 | virtual void print(const char header[]) = 0; 36 | 37 | virtual unsigned long elapsed_time() = 0; 38 | 39 | static std::string timestamp() { 40 | struct timeval tv; 41 | struct tm *now; 42 | char buf[64]; 43 | 44 | gettimeofday(&tv, NULL); 45 | now = localtime(&tv.tv_sec); 46 | strftime(buf, sizeof buf, "%Y-%m-%d %H:%M:%S", now); 47 | std::string str(buf); 48 | return str; 49 | } 50 | }; 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /utils/clock_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _CLOCK_UTILS_ 20 | #define _CLOCK_UTILS_ 21 | 22 | #include "clock.hpp" 23 | #include 24 | #include "boost_log_wrapper.h" 25 | 26 | //! Real time clock 27 | class rtc_clock : public clock { 28 | unsigned long start_time; 29 | unsigned long elapsed_useconds; 30 | 31 | unsigned long get_current_rtc() { 32 | struct timeval tm; 33 | gettimeofday(&tm, NULL); 34 | return tm.tv_sec * 1000000 + tm.tv_usec; 35 | } 36 | 37 | public: 38 | rtc_clock() 39 | : elapsed_useconds(0) { 40 | } 41 | 42 | void start() { 43 | start_time = get_current_rtc(); 44 | } 45 | 46 | void stop() { 47 | elapsed_useconds += (get_current_rtc() - start_time); 48 | } 49 | 50 | void reset() { 51 | elapsed_useconds = 0; 52 | } 53 | 54 | unsigned long elapsed_time() { 55 | return elapsed_useconds; 56 | } 57 | 58 | void print(const char header[]) { 59 | double elapsed_seconds = (double) elapsed_useconds / 1000000; 60 | BOOST_LOG_TRIVIAL(info) << header << " " << elapsed_seconds << " seconds"; 61 | } 62 | }; 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /utils/desc_utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _DESC_UTILS_ 20 | #define _DESC_UTILS_ 21 | 22 | #include 23 | #include 24 | #include 25 | #include "boost_log_wrapper.h" 26 | #include 27 | 28 | extern boost::property_tree::ptree pt; 29 | extern boost::property_tree::ptree pt_slipstore; 30 | 31 | static void init_graph_desc(const std::string &graph_name) { 32 | try { 33 | boost::property_tree::ini_parser::read_ini(graph_name + ".ini", pt); 34 | } 35 | catch (...) { 36 | BOOST_LOG_TRIVIAL(fatal) << "Unable to read property file"; 37 | exit(-1); 38 | } 39 | } 40 | 41 | static void init_slipstore_desc() { 42 | try { 43 | boost::property_tree::ini_parser::read_ini("slipstore.ini", pt_slipstore); 44 | } 45 | catch (...) { 46 | BOOST_LOG_TRIVIAL(fatal) << "Unable to read slipstore property file"; 47 | exit(-1); 48 | } 49 | } 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /utils/diskspeed.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR=/tmp/ 3 | SIZE=4G 4 | start="4096" 5 | end="16777216" 6 | rm -f diskspeed.out 7 | echo "# blocksize read(KB/s) write(KB/s)" > diskspeed.out 8 | i=$start 9 | while [ $i -le $end ] 10 | do 11 | rm -f dump 12 | echo -n "$i " >> diskspeed.out 13 | fio --name=test --runtime=300 --ioengine=sync --direct=1 --rw=read --bs=${i} --size=${SIZE} --directory=${DIR} --minimal --output dump 14 | cat dump | grep ';' | cut -d ';' -f 6 | xargs echo -n >> diskspeed.out 15 | echo -n " " >> diskspeed.out 16 | cat dump | grep ';' | cut -d ';' -f 20 | xargs echo -n >> diskspeed.out 17 | echo -n " " >> diskspeed.out 18 | cat dump | grep ';' | cut -d ';' -f 21 | xargs echo -n >> diskspeed.out 19 | echo -n " " >> diskspeed.out 20 | cat dump | grep ';' | cut -d ';' -f 23 | xargs echo -n >> diskspeed.out 21 | echo -n " " >> diskspeed.out 22 | fio --name=test --runtime=300 --ioengine=sync --direct=1 --rw=write --bs=${i} --size=${SIZE} --directory=${DIR} --minimal --output dump 23 | cat dump | grep ';' | cut -d ';' -f 26 | xargs echo -n >> diskspeed.out 24 | echo -n " " >> diskspeed.out 25 | cat dump | grep ';' | cut -d ';' -f 40 | xargs echo -n >> diskspeed.out 26 | echo -n " " >> diskspeed.out 27 | cat dump | grep ';' | cut -d ';' -f 41 | xargs echo -n >> diskspeed.out 28 | echo -n " " >> diskspeed.out 29 | cat dump | grep ';' | cut -d ';' -f 43 | xargs echo -n >> diskspeed.out 30 | echo " " >> diskspeed.out 31 | let "i *= 2" 32 | done -------------------------------------------------------------------------------- /utils/diskspeed_mix.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR=/tmp/ 3 | SIZE=4G 4 | start="4096" 5 | end="16777216" 6 | rm -f diskspeed.out 7 | echo "# blocksize read(KB/s) write(KB/s)" > diskspeed.out 8 | i=$start 9 | while [ $i -le $end ] 10 | do 11 | rm -f dump 12 | echo -n "$i " >> diskspeed.out 13 | fio --name=test --runtime=300 --ioengine=sync --direct=1 --rw=rw --rwmixread=40 --bs=${i} --size=${SIZE} --directory=${DIR} --minimal --output dump 14 | cat dump | grep ';' | cut -d ';' -f 6 | xargs echo -n >> diskspeed.out 15 | echo -n " " >> diskspeed.out 16 | cat dump | grep ';' | cut -d ';' -f 20 | xargs echo -n >> diskspeed.out 17 | echo -n " " >> diskspeed.out 18 | cat dump | grep ';' | cut -d ';' -f 21 | xargs echo -n >> diskspeed.out 19 | echo -n " " >> diskspeed.out 20 | cat dump | grep ';' | cut -d ';' -f 23 | xargs echo -n >> diskspeed.out 21 | echo -n " " >> diskspeed.out 22 | cat dump | grep ';' | cut -d ';' -f 26 | xargs echo -n >> diskspeed.out 23 | echo -n " " >> diskspeed.out 24 | cat dump | grep ';' | cut -d ';' -f 40 | xargs echo -n >> diskspeed.out 25 | echo -n " " >> diskspeed.out 26 | cat dump | grep ';' | cut -d ';' -f 41 | xargs echo -n >> diskspeed.out 27 | echo -n " " >> diskspeed.out 28 | cat dump | grep ';' | cut -d ';' -f 43 | xargs echo -n >> diskspeed.out 29 | echo " " >> diskspeed.out 30 | let "i *= 2" 31 | done 32 | -------------------------------------------------------------------------------- /utils/diskspeed_rand.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DIR=/tmp/ 3 | SIZE=4G 4 | start="4096" 5 | end="16777216" 6 | rm -f diskspeed.out 7 | echo "# blocksize read(KB/s) write(KB/s)" > diskspeed.out 8 | i=$start 9 | while [ $i -le $end ] 10 | do 11 | rm -f dump 12 | echo -n "$i " >> diskspeed.out 13 | fio --name=test --runtime=300 --ioengine=sync --direct=1 --rw=randread --bs=${i} --size=${SIZE} --directory=${DIR} --minimal --output dump 14 | cat dump | grep ';' | cut -d ';' -f 6 | xargs echo -n >> diskspeed.out 15 | echo -n " " >> diskspeed.out 16 | cat dump | grep ';' | cut -d ';' -f 20 | xargs echo -n >> diskspeed.out 17 | echo -n " " >> diskspeed.out 18 | cat dump | grep ';' | cut -d ';' -f 21 | xargs echo -n >> diskspeed.out 19 | echo -n " " >> diskspeed.out 20 | cat dump | grep ';' | cut -d ';' -f 23 | xargs echo -n >> diskspeed.out 21 | echo -n " " >> diskspeed.out 22 | fio --name=test --runtime=300 --ioengine=sync --direct=1 --rw=randwrite --bs=${i} --size=${SIZE} --directory=${DIR} --minimal --output dump 23 | cat dump | grep ';' | cut -d ';' -f 26 | xargs echo -n >> diskspeed.out 24 | echo -n " " >> diskspeed.out 25 | cat dump | grep ';' | cut -d ';' -f 40 | xargs echo -n >> diskspeed.out 26 | echo -n " " >> diskspeed.out 27 | cat dump | grep ';' | cut -d ';' -f 41 | xargs echo -n >> diskspeed.out 28 | echo -n " " >> diskspeed.out 29 | cat dump | grep ';' | cut -d ';' -f 43 | xargs echo -n >> diskspeed.out 30 | echo " " >> diskspeed.out 31 | let "i *= 2" 32 | done 33 | -------------------------------------------------------------------------------- /utils/fanout_test.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include "../slipstore/interface.hpp" 20 | 21 | class null_barrier_work : public slipstore::barrier_work { 22 | public: 23 | virtual void operator()() { } 24 | }; 25 | 26 | slipstore::cyclic *choice; 27 | 28 | 29 | void control_barrier() { 30 | null_barrier_work null_obj; 31 | slipstore::slipstore_client_barrier->in_progress = true; 32 | slipstore::ioService.post 33 | (boost::bind(&slipstore::client_barrier::work_barrier, 34 | slipstore::slipstore_client_barrier, 35 | &null_obj)); 36 | while (slipstore::slipstore_client_barrier->in_progress); 37 | } 38 | 39 | unsigned long slipchunk; 40 | 41 | void drain_test(unsigned char *buffer, unsigned long amount) { 42 | slipstore::slipstore_req_t req; 43 | while (amount) { 44 | unsigned long bytes; 45 | bytes = (amount > slipchunk) ? 46 | slipchunk : amount; 47 | req.cmd = slipstore::CMD_DRAIN; 48 | req.stream = slipstore::STREAM_VERTEX_STATE; 49 | req.partition = 0; 50 | req.tile = choice->cyclic_next(); 51 | req.size = bytes; 52 | if (!slipstore::slipstore_client_drain->access_store(&req, buffer)) { 53 | BOOST_LOG_TRIVIAL(fatal) << "Unable to write to slipstore"; 54 | exit(-1); 55 | } 56 | amount -= bytes; 57 | } 58 | } 59 | 60 | int main(int argc, const char **argv) { 61 | slipstore::io *streams; 62 | rtc_clock prefill_clock; 63 | rtc_clock drain_clock; 64 | /* Get options */ 65 | setup_options(argc, argv); 66 | slipchunk = vm["slipchunk"].as < unsigned 67 | long > (); 68 | BOOST_LOG_TRIVIAL(info) << "SLIPBENCH::SLIPCHUNK " 69 | << slipchunk; 70 | unsigned long slipfile_size = vm["slipbench_fsize"].as < unsigned 71 | long > (); 72 | unsigned long blockmem = vm["blocked_memory"].as < unsigned 73 | long > (); 74 | unsigned long prefill = vm["test_prefill"].as < unsigned 75 | long > (); 76 | /* Read in slipstore information */ 77 | init_slipstore_desc(); 78 | unsigned long fanout = vm["ext_fanout"].as < unsigned 79 | long > (); 80 | BOOST_LOG_TRIVIAL(info) << "SLIPBENCH::FANOUT_UNDER_TEST " 81 | << fanout; 82 | streams = new slipstore::io(1, 1, fanout, NULL); 83 | slipstore::init(streams, 100, 1); // tilesize does not matter 84 | unsigned char *buffer = new unsigned char[slipchunk]; 85 | choice = new slipstore::cyclic(fanout, 86 | slipstore::slipstore_client_drain->get_me()); 87 | if (blockmem > 0) { 88 | BOOST_LOG_TRIVIAL(info) << "Blocking " << blockmem; 89 | (void) map_anon_memory(blockmem, true, "Blockmem"); 90 | BOOST_LOG_TRIVIAL(info) << "Done"; 91 | } 92 | unsigned long drain_bytes = slipfile_size; 93 | if (drain_bytes < 2 * fanout * slipchunk) { 94 | drain_bytes = 2 * fanout * slipchunk; 95 | } 96 | BOOST_LOG_TRIVIAL(info) << "SLIPBENCH::PREFILL " 97 | << prefill; 98 | BOOST_LOG_TRIVIAL(info) << "SLIPBENCH::DRAIN_AMOUNT " 99 | << drain_bytes; 100 | /////////// Prefill 101 | control_barrier(); // everyone started 102 | prefill_clock.start(); 103 | drain_test(buffer, prefill); 104 | prefill_clock.stop(); 105 | control_barrier(); // everyone done 106 | prefill_clock.print("SLIPBENCH::PREFILL_TIME "); 107 | ////////// Drain test 108 | control_barrier(); // everyone started 109 | drain_clock.start(); 110 | drain_test(buffer, drain_bytes); 111 | control_barrier(); // everyone done 112 | // Flush the remaining blocks 113 | for (unsigned long i = 0; i < fanout; i++) { 114 | streams->rewind(slipstore::STREAM_VERTEX_STATE, 0, i); 115 | } 116 | drain_clock.stop(); 117 | drain_clock.print("SLIPBENCH::DRAIN_TIME "); 118 | /////////////////////// 119 | slipstore::shutdown(); 120 | delete buffer; 121 | delete streams; 122 | 123 | } 124 | -------------------------------------------------------------------------------- /utils/feeder.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include "memory_utils.h" 20 | #include "clock_utils.h" 21 | #include "desc_utils.h" 22 | #include "options_utils.h" 23 | #include "boost_log_wrapper.h" 24 | #include "../core/x-ingest.hpp" 25 | #include "../format/type_1.hpp" 26 | #include "../format/type_2.hpp" 27 | #include 28 | #include 29 | #include 30 | 31 | #define MIN(_a, _b) ((_a) < (_b) ? (_a) : (_b)) 32 | 33 | void feed(int fd, 34 | unsigned long align, 35 | unsigned long bytes) { 36 | rtc_clock wall_clock; 37 | rtc_clock feed_clock; 38 | ingest_t *delta; 39 | delta = attach_ingest(0xf22); 40 | wall_clock.start(); 41 | feed_clock.start(); 42 | while (bytes) { 43 | unsigned long bytes_to_read = MIN(delta->max, bytes); 44 | bytes_to_read = (bytes_to_read / align) * align; 45 | read_from_file(fd, &delta->buffer[0], bytes_to_read); 46 | bytes -= bytes_to_read; 47 | delta->avail = bytes_to_read; 48 | __sync_synchronize(); 49 | while (delta->avail); 50 | feed_clock.stop(); 51 | wall_clock.stop(); 52 | wall_clock.start(); 53 | double wall_time_sec = 54 | ((double) wall_clock.elapsed_time()) / 1000000; 55 | double feed_time_sec = 56 | ((double) feed_clock.elapsed_time()) / 1000000; 57 | BOOST_LOG_TRIVIAL(info) << "[" << clock::timestamp() << "] " 58 | << "WALL_TIME:INGEST_INTERVAL " 59 | << wall_time_sec << " " 60 | << feed_time_sec; 61 | feed_clock.reset(); 62 | feed_clock.start(); 63 | } 64 | delta->eof = true; 65 | } 66 | 67 | int main(int argc, char *argv[]) { 68 | desc.add_options() 69 | ("help,h", "Produce help message") 70 | ("source,i", boost::program_options::value()->required(), 71 | "Name of the input to feed (need input.ini)"); 72 | try { 73 | boost::program_options::store(boost::program_options::parse_command_line(argc, 74 | argv, 75 | desc), 76 | vm); 77 | boost::program_options::notify(vm); 78 | } 79 | catch (boost::program_options::error &e) { 80 | if (vm.count("help") || argc == 1) { 81 | std::cerr << desc << "\n"; 82 | } 83 | std::cerr << "Error:" << e.what() << std::endl; 84 | std::cerr << "Try: " << argv[0] << " --help" << std::endl; 85 | exit(-1); 86 | } 87 | init_graph_desc(vm["source"].as()); 88 | unsigned long edges, record_size; 89 | std::stringstream fname; 90 | fname << pt.get("graph.name"); 91 | edges = pt.get < unsigned 92 | long > ("graph.edges"); 93 | int fd_edges = open(fname.str().c_str(), O_RDONLY | O_LARGEFILE); 94 | if (fd_edges == -1) { 95 | BOOST_LOG_TRIVIAL(fatal) << "Unable to open edges file " << 96 | fname.str() << ":" << strerror(errno); 97 | exit(-1); 98 | } 99 | if (pt.get < unsigned 100 | long > ("graph.type") == 1) { 101 | record_size = format::type1::format_utils::split_size_bytes(); 102 | } 103 | else if (pt.get < unsigned 104 | long > ("graph.type") == 2) { 105 | record_size = format::type2::format_utils::split_size_bytes(); 106 | } 107 | else { 108 | BOOST_LOG_TRIVIAL(fatal) << "Unsupported file type"; 109 | exit(-1); 110 | } 111 | feed(fd_edges, record_size, record_size * edges); 112 | return 0; 113 | } 114 | -------------------------------------------------------------------------------- /utils/mem_speed_random.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include "memory_utils.h" 20 | #include "clock_utils.h" 21 | #include "boost_log_wrapper.h" 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #define MIN_BUFFER_SIZE 256*1024*1024 29 | unsigned long buffer_items = 1; 30 | #define LCG_NEXT(_n, mask) ((1103515245*(_n) + 12345) & mask) 31 | 32 | class mem_speed { 33 | unsigned char *buffer1; 34 | unsigned char *buffer2; 35 | boost::barrier *thread_sync; 36 | 37 | double convert(double mbps) { 38 | double factor = (1 << 20) / ((double) 1000000.0); 39 | return mbps * factor; 40 | } 41 | 42 | unsigned long chunk_size; 43 | public: 44 | mem_speed(unsigned char *buffer1_in, 45 | unsigned char *buffer2_in, 46 | unsigned long chunk_size_in, 47 | boost::barrier *thread_sync_in) //! Barrier for threads 48 | : buffer1(buffer1_in), 49 | buffer2(buffer2_in), 50 | thread_sync(thread_sync_in), 51 | chunk_size(chunk_size_in) { 52 | } 53 | 54 | void operator()() { 55 | rtc_clock timer; 56 | unsigned char tmp[chunk_size]; 57 | const unsigned long mask = buffer_items - 1; 58 | thread_sync->wait(); 59 | timer.reset(); 60 | timer.start(); 61 | for (unsigned long i = 0, j = 0; i < buffer_items; i++) { 62 | memcpy(tmp, buffer1 + j * chunk_size, chunk_size); 63 | j = LCG_NEXT(j, mask); 64 | } 65 | timer.stop(); 66 | double mbps = ((double) buffer_items * chunk_size) / timer.elapsed_time(); 67 | BOOST_LOG_TRIVIAL(info) << "MEMREAD_SPEED " << convert(mbps) << "MB/s"; 68 | timer.reset(); 69 | timer.start(); 70 | for (unsigned long i = 0, j = 0; i < buffer_items; i++) { 71 | memcpy(buffer1 + j * chunk_size, tmp, chunk_size); 72 | j = LCG_NEXT(j, mask); 73 | } 74 | timer.stop(); 75 | mbps = ((double) buffer_items * chunk_size) / timer.elapsed_time(); 76 | BOOST_LOG_TRIVIAL(info) << "MEMWRITE_SPEED " << convert(mbps) << "MB/s"; 77 | timer.reset(); 78 | timer.start(); 79 | for (unsigned long i = 0, j = 0; i < buffer_items; i++) { 80 | memcpy(buffer2 + j * chunk_size, buffer1 + j * chunk_size, chunk_size); 81 | j = LCG_NEXT(j, mask); 82 | } 83 | timer.stop(); 84 | mbps = ((double) buffer_items * chunk_size) / timer.elapsed_time(); 85 | BOOST_LOG_TRIVIAL(info) << "MEMCPY_SPEED " << convert(mbps) << "MB/s"; 86 | } 87 | 88 | }; 89 | 90 | int main(int argc, char *argv[]) { 91 | unsigned long threads, chunk_size; 92 | if (argc < 3) { 93 | std::cerr << "Usage " << argv[0] << " chunk_size threads" << std::endl; 94 | exit(-1); 95 | } 96 | else { 97 | chunk_size = atol(argv[1]); 98 | threads = atol(argv[2]); 99 | while (buffer_items * chunk_size < MIN_BUFFER_SIZE) { 100 | buffer_items = buffer_items * 2; 101 | } 102 | } 103 | 104 | boost::thread **thread_array = new boost::thread *[threads]; 105 | mem_speed **obj = new mem_speed *[threads]; 106 | boost::barrier *thread_sync = new boost::barrier(threads); 107 | for (unsigned long i = 0; i < threads; i++) { 108 | unsigned char *buffer1 = (unsigned char *) 109 | map_anon_memory(buffer_items * chunk_size, true, "buffer"); 110 | unsigned char *buffer2 = (unsigned char *) 111 | map_anon_memory(buffer_items * chunk_size, true, "buffer"); 112 | obj[i] = new mem_speed(buffer1, buffer2, chunk_size, thread_sync); 113 | thread_array[i] = new boost::thread 114 | (boost::ref(*obj[i])); 115 | } 116 | 117 | for (unsigned long i = 0; i < threads; i++) { 118 | thread_array[i]->join(); 119 | } 120 | return 0; 121 | } 122 | -------------------------------------------------------------------------------- /utils/mem_speed_sequential.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include "memory_utils.h" 20 | #include "clock_utils.h" 21 | #include "boost_log_wrapper.h" 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #define BUFFER_SIZE (256*1024*1024 + 4096) 29 | 30 | 31 | class mem_speed { 32 | unsigned char *buffer1; 33 | unsigned char *buffer2; 34 | boost::barrier *thread_sync; 35 | 36 | double convert(double mbps) { 37 | double factor = (1 << 20) / ((double) 1000000.0); 38 | return mbps * factor; 39 | } 40 | 41 | unsigned long chunk_size; 42 | public: 43 | mem_speed(unsigned long chunk_size_in, 44 | boost::barrier *thread_sync_in, //! Barrier for threads 45 | unsigned char *buffer1_in, 46 | unsigned char *buffer2_in) 47 | : buffer1(buffer1_in), 48 | buffer2(buffer2_in), 49 | thread_sync(thread_sync_in), 50 | chunk_size(chunk_size_in) { 51 | } 52 | 53 | void operator()() { 54 | rtc_clock timer; 55 | unsigned char tmp[chunk_size]; 56 | thread_sync->wait(); 57 | timer.reset(); 58 | timer.start(); 59 | for (unsigned long i = 0; i < BUFFER_SIZE; i += chunk_size) { 60 | memcpy(tmp, buffer1 + i, chunk_size); 61 | } 62 | timer.stop(); 63 | double mbps = ((double) BUFFER_SIZE) / timer.elapsed_time(); 64 | BOOST_LOG_TRIVIAL(info) << "MEMREAD_SPEED " << convert(mbps) << "MB/s"; 65 | timer.reset(); 66 | timer.start(); 67 | for (unsigned long i = 0; i < BUFFER_SIZE; i += chunk_size) { 68 | memcpy(buffer1 + i, tmp, chunk_size); 69 | } 70 | timer.stop(); 71 | mbps = ((double) BUFFER_SIZE) / timer.elapsed_time(); 72 | BOOST_LOG_TRIVIAL(info) << "MEMWRITE_SPEED " << convert(mbps) << "MB/s"; 73 | timer.reset(); 74 | timer.start(); 75 | for (unsigned long i = 0; i < BUFFER_SIZE; i += chunk_size) { 76 | memcpy(buffer2 + i, buffer1 + i, chunk_size); 77 | } 78 | timer.stop(); 79 | mbps = ((double) BUFFER_SIZE) / timer.elapsed_time(); 80 | BOOST_LOG_TRIVIAL(info) << "MEMCPY_SPEED " << convert(mbps) << "MB/s"; 81 | } 82 | 83 | }; 84 | 85 | int main(int argc, char *argv[]) { 86 | unsigned long threads, chunk_size; 87 | if (argc < 3) { 88 | std::cerr << "Usage " << argv[0] << " chunk_size threads" << std::endl; 89 | exit(-1); 90 | } 91 | else { 92 | chunk_size = atol(argv[1]); 93 | threads = atol(argv[2]); 94 | } 95 | 96 | boost::thread **thread_array = new boost::thread *[threads]; 97 | mem_speed **obj = new mem_speed *[threads]; 98 | boost::barrier *thread_sync = new boost::barrier(threads); 99 | unsigned long alloc_bytes = 100 | ((BUFFER_SIZE + chunk_size - 1) / chunk_size) * chunk_size; 101 | for (unsigned long i = 0; i < threads; i++) { 102 | unsigned char *buffer1 = (unsigned char *) 103 | map_anon_memory(alloc_bytes, true, "buffer"); 104 | unsigned char *buffer2 = (unsigned char *) 105 | map_anon_memory(alloc_bytes, true, "buffer"); 106 | obj[i] = new mem_speed(chunk_size, thread_sync, buffer1, buffer2); 107 | thread_array[i] = new boost::thread 108 | (boost::ref(*obj[i])); 109 | } 110 | 111 | for (unsigned long i = 0; i < threads; i++) { 112 | thread_array[i]->join(); 113 | } 114 | return 0; 115 | } 116 | -------------------------------------------------------------------------------- /utils/memspeed.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | BIN=../x-stream/bin 3 | chunk_size=8 4 | start_thread_cnt=1 5 | end_thread_cnt=4 6 | #rm -f memspeed.seq.out 7 | #rm -f memspeed.rand.out 8 | printf "# %-15s%-15s%-15s%-15s%-15s\n" "threads" "chunk_size" "read(MB/s)" "write(MB/s)" "memcpy(MB/s)" >> memspeed.seq.out 9 | printf "# %-15s%-15s%-15s%-15s%-15s\n" "threads" "chunk_size" "read(MB/s)" "write(MB/s)" "memcpy(MB/s)" >> memspeed.rand.out 10 | for i in `seq $start_thread_cnt 1 $end_thread_cnt` 11 | do 12 | numactl --interleave=all ${BIN}/mem_speed_sequential ${chunk_size} ${i} 2> dump 13 | numactl --interleave=all ${BIN}/mem_speed_sequential ${chunk_size} ${i} 2>> dump 14 | numactl --interleave=all ${BIN}/mem_speed_sequential ${chunk_size} ${i} 2>> dump 15 | numactl --interleave=all ${BIN}/mem_speed_sequential ${chunk_size} ${i} 2>> dump 16 | memread=$(cat dump | grep MEMREAD | awk '{sum += $4} END {printf "%f ",sum/4}') 17 | memwrite=$(cat dump | grep MEMWRITE | awk '{sum += $4} END {printf "%f ",sum/4}') 18 | memcpy=$(cat dump | grep MEMCPY | awk '{sum += $4} END {printf "%f ",sum/4}') 19 | printf " %-15s%-15s%-15s%-15s%-15s\n" $i $chunk_size $memread $memwrite $memcpy >> memspeed.seq.out 20 | numactl --interleave=all ${BIN}/mem_speed_random ${chunk_size} ${i} 2> dump 21 | numactl --interleave=all ${BIN}/mem_speed_random ${chunk_size} ${i} 2>> dump 22 | numactl --interleave=all ${BIN}/mem_speed_random ${chunk_size} ${i} 2>> dump 23 | numactl --interleave=all ${BIN}/mem_speed_random ${chunk_size} ${i} 2>> dump 24 | memread=$(cat dump | grep MEMREAD | awk '{sum += $4} END {printf "%f ",sum/4}') 25 | memwrite=$(cat dump | grep MEMWRITE | awk '{sum += $4} END {printf "%f ",sum/4}') 26 | memcpy=$(cat dump | grep MEMCPY | awk '{sum += $4} END {printf "%f ",sum/4}') 27 | printf " %-15s%-15s%-15s%-15s%-15s\n" $i $chunk_size $memread $memwrite $memcpy >> memspeed.rand.out 28 | rm -f dump 29 | done 30 | -------------------------------------------------------------------------------- /utils/per_cpu_data.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #ifndef _PER_CPU_DATA_ 20 | #define _PER_CPU_DATA_ 21 | namespace algorithm { 22 | class per_processor_data { 23 | public: 24 | virtual bool reduce(per_processor_data **per_cpu_array, 25 | unsigned long processors) 26 | = 0; 27 | }; 28 | } 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /utils/sort_edges.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include "memory_utils.h" 20 | #include "clock_utils.h" 21 | #include "desc_utils.h" 22 | #include "options_utils.h" 23 | #include "boost_log_wrapper.h" 24 | #include "../format/type_1.hpp" 25 | #include 26 | #include 27 | #include 28 | 29 | template 30 | int compare(const void *a, const void *b) { 31 | unsigned long key_a, key_b; 32 | key_a = FormatUtil::split_key((const unsigned char *) a, 0); 33 | key_b = FormatUtil::split_key((const unsigned char *) b, 0); 34 | if (key_a < key_b) { 35 | return -1; 36 | } 37 | else if (key_a > key_b) { 38 | return 1; 39 | } 40 | else { 41 | return 0; 42 | } 43 | } 44 | 45 | template 46 | void counting_sort(unsigned char *buf_in, 47 | unsigned char *buf_out, 48 | unsigned long items, 49 | unsigned long *index, 50 | unsigned long keys) { 51 | // Init 52 | for (unsigned long i = 0; i < keys; i++) { 53 | index[i] = 0; 54 | } 55 | // Count 56 | for (unsigned long i = 0; i < items; i++) { 57 | unsigned long key = 58 | F::split_key(buf_in + i * F::split_size_bytes(), 0); 59 | index[key]++; 60 | } 61 | // Prefix sum 62 | unsigned long prev_keys = 0; 63 | for (unsigned long i = 0; i < keys; i++) { 64 | unsigned long tmp = index[i]; 65 | index[i] = prev_keys; 66 | prev_keys += tmp; 67 | } 68 | // Sanity check 69 | if (prev_keys != items) { 70 | BOOST_LOG_TRIVIAL(fatal) << "Counting sort failed !"; 71 | exit(-1); 72 | } 73 | // Copy 74 | for (unsigned long i = 0; i < items; i++) { 75 | unsigned char *src = buf_in + i * F::split_size_bytes(); 76 | unsigned long key = F::split_key(src, 0); 77 | unsigned char *dst = buf_out + index[key] * F::split_size_bytes(); 78 | index[key]++; 79 | memcpy(dst, src, F::split_size_bytes()); 80 | } 81 | } 82 | 83 | int main(int argc, char *argv[]) { 84 | rtc_clock wall_clock; 85 | desc.add_options() 86 | ("help,h", "Produce help message") 87 | ("algorithm,a", 88 | boost::program_options::value()->default_value("quick_sort"), 89 | "Sort algorithm: quick_sort (default), counting_sort") 90 | ("source,i", boost::program_options::value()->required(), 91 | "Name of the input to sort (need input.ini)"); 92 | try { 93 | boost::program_options::store(boost::program_options::parse_command_line(argc, 94 | argv, 95 | desc), 96 | vm); 97 | boost::program_options::notify(vm); 98 | } 99 | catch (boost::program_options::error &e) { 100 | if (vm.count("help") || argc == 1) { 101 | std::cerr << desc << "\n"; 102 | } 103 | std::cerr << "Error:" << e.what() << std::endl; 104 | std::cerr << "Try: " << argv[0] << " --help" << std::endl; 105 | exit(-1); 106 | } 107 | init_graph_desc(vm["source"].as()); 108 | unsigned char *stream; 109 | unsigned long items, record_size; 110 | std::stringstream fname; 111 | fname << pt.get("graph.name"); 112 | items = pt.get < unsigned 113 | long > ("graph.edges"); 114 | int fd_edges = open(fname.str().c_str(), O_RDONLY | O_LARGEFILE); 115 | if (fd_edges == -1) { 116 | BOOST_LOG_TRIVIAL(fatal) << "Unable to open edges file " << 117 | fname.str() << ":" << strerror(errno); 118 | exit(-1); 119 | } 120 | if (pt.get < unsigned 121 | long > ("graph.type") == 1) { 122 | record_size = format::type1::format_utils::split_size_bytes(); 123 | stream = (unsigned char *) 124 | map_anon_memory(items * record_size, 125 | true, "Sort buffer"); 126 | read_from_file(fd_edges, stream, items * record_size); 127 | close(fd_edges); 128 | } 129 | else { 130 | BOOST_LOG_TRIVIAL(fatal) << "Unknown file type"; 131 | exit(-1); 132 | } 133 | if (vm["algorithm"].as() == "quick_sort") { 134 | wall_clock.start(); 135 | qsort(stream, items, 136 | record_size, compare); 137 | wall_clock.stop(); 138 | } 139 | else if (vm["algorithm"].as() == "counting_sort") { 140 | wall_clock.start(); 141 | unsigned long keys = pt.get < unsigned 142 | long > ("graph.vertices"); 143 | counting_sort 144 | (stream, 145 | (unsigned char *) map_anon_memory(items * record_size, 146 | true, "Output buffer"), 147 | items, 148 | (unsigned long *) map_anon_memory(keys * sizeof(unsigned long), 149 | true, "Index"), 150 | keys); 151 | wall_clock.stop(); 152 | } 153 | else { 154 | BOOST_LOG_TRIVIAL(fatal) << "Don't know how to do " << 155 | vm["algorithm"].as(); 156 | exit(-1); 157 | } 158 | wall_clock.print("TIME WALL "); 159 | return 0; 160 | } 161 | -------------------------------------------------------------------------------- /utils/utils.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Chaos 3 | * 4 | * Copyright 2015 Operating Systems Laboratory EPFL 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include "options_utils.h" 20 | #include "desc_utils.h" 21 | 22 | boost::program_options::options_description desc("Supported options"); 23 | boost::program_options::variables_map vm; 24 | boost::property_tree::ptree pt; 25 | boost::property_tree::ptree pt_slipstore; 26 | 27 | unsigned long stat_bytes_read = 0; 28 | unsigned long stat_bytes_written = 0; 29 | --------------------------------------------------------------------------------