├── .gitignore ├── LICENSE ├── Makefile ├── README.txt ├── benchmark ├── Makefile ├── benchmark.cc ├── benchmarks │ ├── avg_teen_cnt.h │ ├── bc_adj.h │ ├── bc_random.h │ ├── benchmark.h │ ├── bfs.h │ ├── pagerank.h │ ├── sssp.h │ ├── tarjan_scc.h │ └── triangle_counting.h ├── tests │ ├── delete_edges.h │ └── delete_nodes.h └── tools │ ├── cross_validate.h │ ├── degree_distribution.h │ ├── dump.h │ ├── flatten.h │ ├── level_spread.h │ ├── print.h │ └── property_stats.h ├── examples ├── Makefile └── llama-pagerank.cc ├── llama └── include │ ├── llama.h │ └── llama │ ├── ll_bfs_template.h │ ├── ll_common.h │ ├── ll_config.h │ ├── ll_database.h │ ├── ll_dfs_template.h │ ├── ll_edge_table.h │ ├── ll_external_sort.h │ ├── ll_growable_array.h │ ├── ll_lock.h │ ├── ll_mem_array.h │ ├── ll_mem_helper.h │ ├── ll_mlcsr_graph.h │ ├── ll_mlcsr_helpers.h │ ├── ll_mlcsr_iterator.h │ ├── ll_mlcsr_properties.h │ ├── ll_mlcsr_sp.h │ ├── ll_page_manager.h │ ├── ll_persistent_storage.h │ ├── ll_seq.h │ ├── ll_slcsr.h │ ├── ll_streaming.h │ ├── ll_utils.h │ ├── ll_writable_array.h │ ├── ll_writable_elements.h │ ├── ll_writable_graph.h │ └── loaders │ ├── ll_gen_erdosrenyi.h │ ├── ll_gen_rmat.h │ ├── ll_load_async_writable.h │ ├── ll_load_fgf.h │ ├── ll_load_net.h │ ├── ll_load_utils.h │ ├── ll_load_xstream1.h │ └── ll_loaders.h ├── tests ├── generate-report.py └── run-tests.sh ├── tools ├── Makefile └── llama-load.cc └── utils ├── Makefile ├── memhog.cpp ├── snap-to-xs1.cpp ├── xs1-common.h ├── xs1-ini.cpp ├── xs1-max-degree.cpp ├── xs1-reorder.cpp ├── xs1-shuffle.cpp ├── xs1-split.cpp └── xs1-to-snap.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | # 2 | # Git Ignore List 3 | # 4 | 5 | # Targets 6 | *.o 7 | bin 8 | 9 | # Temporary files 10 | *~ 11 | *.swp 12 | 13 | # Databases 14 | db 15 | db-c 16 | db-m 17 | 18 | # Supporting files that should not be checked in 19 | data 20 | *.bin 21 | nbproject 22 | 23 | # Temporary analysis results 24 | asm* 25 | *.s 26 | 27 | # Profiling results 28 | cachegrind.out.* 29 | gmon.* 30 | massif.out.* 31 | 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2014 2 | The President and Fellows of Harvard College. 3 | 4 | Copyright 2014 5 | Oracle Labs. 6 | 7 | Redistribution and use in source and binary forms, with or without 8 | modification, are permitted provided that the following conditions 9 | are met: 10 | 1. Redistributions of source code must retain the above copyright 11 | notice, this list of conditions and the following disclaimer. 12 | 2. Redistributions in binary form must reproduce the above copyright 13 | notice, this list of conditions and the following disclaimer in the 14 | documentation and/or other materials provided with the distribution. 15 | 3. Neither the name of the University nor the names of its contributors 16 | may be used to endorse or promote products derived from this software 17 | without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 | OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 | OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 | SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # LLAMA's top level makefile 3 | # 4 | 5 | TARGETS := benchmark examples tools utils 6 | 7 | BENCHMARK_CORE_TARGETS := benchmark-memory benchmark-memory-wd \ 8 | benchmark-persistent benchmark-persistent-wd benchmark-slcsr \ 9 | benchmark-streaming 10 | BENCHMARK_CORE_DEBUG_TARGETS := $(patsubst %,%_debug,${BENCHMARK_CORE_TARGETS}) 11 | BENCHMARK_OTHER_TARGETS := benchmark-w-memory 12 | BENCHMARK_OTHER_DEBUG_TARGETS := $(patsubst %,%_debug,${BENCHMARK_OTHER_TARGETS}) 13 | BENCHMARK_TARGETS := ${BENCHMARK_CORE_TARGETS} ${BENCHMARK_CORE_DEBUG_TARGETS}\ 14 | ${BENCHMARK_OTHER_TARGETS} ${BENCHMARK_OTHER_DEBUG_TARGETS} 15 | 16 | ifdef TASK 17 | BENCHMARK_BASE := bench-${TASK} 18 | else 19 | BENCHMARK_BASE := benchmark 20 | endif 21 | 22 | ifdef ONE_VT 23 | BENCHMARK_BASE := ${BENCHMARK_BASE}-onevt 24 | MFLAGS := ${MFLAGS} ONE_VT=${ONE_VT} 25 | endif 26 | 27 | ifdef FLAT_VT 28 | BENCHMARK_BASE := ${BENCHMARK_BASE}-flatvt 29 | MFLAGS := ${MFLAGS} FLAT_VT=${FLAT_VT} 30 | endif 31 | 32 | MFLAGS := ${MFLAGS} TASK=${TASK} DEBUG_NODE=${DEBUG_NODE} 33 | 34 | .PHONY: all clean ${BENCHMARK_TARGETS} 35 | 36 | all clean: 37 | @for t in ${TARGETS}; do \ 38 | ${MAKE} ${MFLAGS} -C "$$t" $@ || exit 1; \ 39 | done 40 | 41 | ${BENCHMARK_TARGETS}: 42 | @${MAKE} ${MFLAGS} -C benchmark \ 43 | ../bin/`echo "$@" | sed "s/benchmark/${BENCHMARK_BASE}/g"` 44 | 45 | -------------------------------------------------------------------------------- /README.txt: -------------------------------------------------------------------------------- 1 | LLAMA 2 | ========= 3 | 4 | 5 | 0. Introduction 6 | ----------------- 7 | 8 | LLAMA is a graph storage and analysis system that supports mutability and 9 | out-of-memory execution built on top of the compressed sparse row (CSR) 10 | representation. Its goal is to perform comparably to immutable main-memory 11 | analysis systems for graphs that fit in memory and to match or outperform 12 | existing out-of-memory analysis systems for graphs that exceed main memory. 13 | 14 | Relevant publications: 15 | * Peter Macko, Virendra Marathe, Daniel Margo, and Margo Seltzer. "LLAMA: 16 | Efficient Graph Analytics Using Large Multiversioned Arrays." 31st IEEE 17 | International Conference on Data Engineering (ICDE '15), Seoul, Korea, 18 | April 2015. 19 | * Peter Macko. "LLAMA: A Persistent, Mutable Representation for Graphs." 20 | PhD Dissertation. Harvard University, Cambridge, MA, January 2015. 21 | 22 | 23 | 1. Configuring and Building LLAMA 24 | ----------------------------------- 25 | 26 | Prerequisites: 27 | - Linux or NetBSD 28 | - GCC 4.4 or newer (4.8 or newer recommended) 29 | - gperftools (recommended) 30 | 31 | LLAMA comes in four different flavors, which you can choose at the compile-time 32 | by defining an appropriate preprocessor directive: 33 | * LL_MEMORY_ONLY - an in-memory version of LLAMA 34 | * LL_PERSISTENCE - a persistent version of LLAMA 35 | * LL_SLCSR - traditional (single-snapshot) CSR 36 | 37 | You can also define the following directives: 38 | * LL_DELETIONS - enable deletion vector (required to support deletions in the 39 | write-optimized graph store and also required for multiversioned sliding 40 | windows) 41 | * LL_FLAT_VT - a multiversioned flat vertex table (disable COW) 42 | * LL_ONE_VT - a single shared flat vertex table 43 | 44 | The benchmark suite bypasses the write-optimized store by default, but you can 45 | change that by defining: 46 | * BENCHMARK_WRITABLE - use the write-optimized store in the benchmark 47 | 48 | The Makefile has several defined targets for these configurations; the most 49 | useful targets are: 50 | * benchmark-memory - uses an in-memory version of LLAMA 51 | * benchmark-persistent - uses a persistent version of LLAMA 52 | * benchmark-slcsr - uses the single-snapshot (single-level) version of LLAMA 53 | * benchmark-memory-wd - uses the in-memory version with a deletion vector 54 | * benchmark-persistent-wd - uses the persistent version with a deletion 55 | vector (currently broken) 56 | * benchmark-w-memory - in-memory version of LLAMA, use write-optimized store 57 | Most benchmark targets have corresponding %_debug targets useful for debugging 58 | LLAMA, such as benchmark-memory_debug or benchmark-persistent_debug. Typing 59 | "make all" builds benchmark-memory, benchmark-persistent, benchmark-slcsr, 60 | benchmark-memory-wd, and benchmark-persistent-wd. 61 | 62 | You can further customize the build by passing additional definitions to the 63 | Makefile. If you use the global Makefile, you can use the targets above, but if 64 | you use benchmark/Makefile directly, you might need to modify the targets. Here 65 | are examples of a few things that you can do: 66 | * make TASK=pagerank benchmark-memory - compile a PageRank-specific benchmark 67 | so that you do not need to use the -r argument to specify the task when 68 | running it; using this compile-time specialization usually results in a 69 | more optimized (and therefore faster) code 70 | * make ONE_VT=1 benchmark-memory - enable LL_ONE_VT, a single shared flat 71 | vertex table 72 | * make FLAT_VT=1 benchmark-memory - enable LL_FLAT_VT, a multiversioned flat 73 | vertex table 74 | * make NO_CONT=1 benchmark-memory - enable LL_NO_CONTINUATIONS, which 75 | disables explicit adjacency list linking 76 | You can use any other combination of these except combining ONE_VT and FLAT_VT. 77 | 78 | 79 | 2. Using LLAMA 80 | ---------------- 81 | 82 | If you would like to use LLAMA in your project, 83 | 1. Include llama.h in your code 84 | 2. Compile with -fopenmp -std=gnu++11 85 | 86 | Please refer to the following files for examples about how to use LLAMA: 87 | * tools/llama-load.cc - a stand-alone example that loads a LLAMA database 88 | * examples/llama-pagerank.cc - a stand-alone example that opens an existing 89 | LLAMA database and computes a few iterations of PageRank 90 | 91 | The code includes Doxygen comments, which documents the functionality, 92 | arguments, and return values of most data functions and describes most data 93 | structures and their fields. 94 | 95 | A lot of the streaming functionality is currently hard-coded in benchmark.cc, 96 | and it is in a desperate need for refactoring (see the corresponding issue in 97 | TODO.txt) so that it would be possible to easily build stand-alone streaming 98 | applications. If you would like to do that before that happens, implement your 99 | program as another benchmark in the benchmark suite -- or go ahead and do the 100 | refactoring yourself :) We will be grateful for the contribution! 101 | 102 | 103 | 3. Modifying LLAMA 104 | -------------------- 105 | 106 | If you would like to contribute to LLAMA, we would certainly appreciate your 107 | help! There is a long list of TODO items and known bugs in docs/TODO.txt, so 108 | feel free to tackle any of those, or if there is anything else that you would 109 | like to do, please feel free! 110 | 111 | All source code of LLAMA is in the header files to facilitate deep inlining, 112 | which is crucial for performance, and to enable the use of templates. 113 | 114 | Please refer to docs/FILES.txt for short descriptions of files. In short: 115 | * benchmark/ - contains the benchmark suite 116 | * examples/ - contains examples (currently just one PageRank program) 117 | * llama/ - contains the actual source code of LLAMA 118 | * tools/ - contains the database loading tool 119 | 120 | The files in llama/include/llama roughly break into the following categories: 121 | * Shared high-level functionality, definitions, and configuration: 122 | ll_common.h - Shared definitions 123 | ll_config.h - Configuration classes 124 | ll_database.h - A high-level database class 125 | * The read-optimized, multi-snapshot (multi-level) graph store: 126 | ll_edge_table.h - The edge table 127 | ll_mem_array.h - In-memory arrays, both COW and flat, used for 128 | the vertex table and properties 129 | ll_mlcsr_graph.h - The (multi-level CSR) read-only graph that 130 | consists of CSR graph stores, such as in- and 131 | out- edges, and node and edge properties 132 | ll_mlcsr_helpers.h - Various multi-level CSR helpers 133 | ll_mlcsr_iterator.h - Definitions of edge and node iterators 134 | ll_mlcsr_properties.h - Multi-level properties 135 | ll_mlcsr_sp.h - The actual multi-level CSR implementation 136 | ll_page_manager.h - The reference-counted page manager used by 137 | in-memory vertex tables and properties 138 | ll_persistent_storage.h - Almost everything related to persistence, 139 | including persistent COW arrays used for 140 | persistent vertex tables and properties 141 | * The write-optimized graph store: 142 | ll_growable_array.h - A growable array 143 | ll_writable_array.h - Arrays used by the write-optimized store 144 | ll_writable_elements.h - Writable node and edge objects 145 | ll_writable_graph.h - The actual write-optimized store 146 | * Miscellaneous functionality: 147 | ll_slcsr.h - Traditional CSR implementation 148 | * Miscellaneous utilities: 149 | ll_lock.h - Locks and atomic functions 150 | ll_bfs_template.h - A BFS template (adapted from GM) 151 | ll_dfs_template.h - A DFS template (adapted from GM) 152 | ll_external_sort.h - An external sort routine 153 | ll_mem_helper.h - Various types of memory pools 154 | ll_seq.h - The sequence container (from GM) 155 | ll_utils.h - Miscellaneous utilities 156 | * Graph loaders and generators, all inside loaders/ : 157 | ll_gen_erdosrenyi.h - Random graph generator 158 | ll_gen_rmat.h - R-MAT graph generator 159 | ll_load_async_writable.h - Helpers for loading the write-store 160 | ll_loaders.h - A collection of all loaders 161 | ll_load_fgf.h - FGF loader 162 | ll_load_net.h - SNAP edge-list loader 163 | ll_load_utils.h - Common loader functionality 164 | ll_load_xstream1.h - X-Stream Type 1 edge-list loader 165 | 166 | -------------------------------------------------------------------------------- /benchmark/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Benchmark for LLAMA 3 | # 4 | 5 | 6 | # 7 | # Compiler setup 8 | # 9 | 10 | CC := g++ 11 | CFLAGS := -Wall -fopenmp -std=c++0x -I. -I../llama/include 12 | LFLAGS := 13 | 14 | 15 | # 16 | # Task specialization 17 | # 18 | 19 | ifdef TASK 20 | BENCHMARK_TASK_ID := $(shell \ 21 | fgrep -A 100 'g_runnable_things[]' benchmark.cc \ 22 | | grep -E '^\W*{ \"ll_' | awk '{print NR-1, "\t", $$0}' \ 23 | | grep '"${TASK}"' | sed -E 's/^([0-9]+)\W.*$$/\1/') 24 | CFLAGS := -DBENCHMARK_TASK_ID=${BENCHMARK_TASK_ID} \ 25 | -DBENCHMARK_TASK_STR=${TASK} ${CFLAGS} 26 | endif 27 | 28 | 29 | # 30 | # Other configuration 31 | # 32 | 33 | ifdef ONE_VT 34 | CFLAGS := -DLL_ONE_VT ${CFLAGS} 35 | endif 36 | 37 | ifdef FLAT_VT 38 | CFLAGS := -DLL_FLAT_VT ${CFLAGS} 39 | endif 40 | 41 | 42 | # 43 | # Debug 44 | # 45 | 46 | ifdef DEBUG_NODE 47 | CFLAGS := ${CFLAGS} -DD_DEBUG_NODE=${DEBUG_NODE} 48 | endif 49 | 50 | 51 | # 52 | # Set up the flags 53 | # 54 | 55 | ifeq ($(VIMRUNTIME),) 56 | ifneq ($(wildcard /usr/bin/color-g++),) 57 | CC := color-g++ 58 | endif 59 | endif 60 | 61 | CFLAGS_D := -ggdb -O0 -D_DEBUG ${CFLAGS} 62 | CFLAGS := -g -O3 -DNDEBUG ${CFLAGS} 63 | 64 | LFLAGS_D := ${LFLAGS} 65 | LFLAGS := ${LFLAGS} 66 | #LFLAGS := ${LFLAGS} -L/usr/local/lib -ljemalloc 67 | 68 | CACHEGRIND := valgrind --tool=cachegrind --branch-sim=yes 69 | MEMCHECK := valgrind --tool=memcheck 70 | 71 | 72 | # 73 | # Sources 74 | # 75 | 76 | HEADERS := ../llama/include/llama/*.h benchmarks/*.h \ 77 | ../llama/include/llama/loaders/*.h tools/*.h 78 | 79 | COMMON_DEPENDENCIES := ${HEADERS} 80 | 81 | 82 | # 83 | # Setup targets 84 | # 85 | 86 | T_DIR := ../bin 87 | 88 | ifdef TASK 89 | T_BASE := ${T_DIR}/bench-${TASK} 90 | else 91 | T_BASE := ${T_DIR}/benchmark 92 | endif 93 | 94 | ifdef ONE_VT 95 | T_BASE := ${T_BASE}-onevt 96 | endif 97 | 98 | ifdef FLAT_VT 99 | T_BASE := ${T_BASE}-flatvt 100 | endif 101 | 102 | CORE_TARGETS := ${T_BASE}-memory ${T_BASE}-memory-wd ${T_BASE}-persistent \ 103 | ${T_BASE}-persistent-wd ${T_BASE}-slcsr ${T_BASE}-streaming 104 | DEBUG_TARGETS := $(patsubst %,%_debug,${CORE_TARGETS}) 105 | COMMON_TARGETS := ${CORE_TARGETS} 106 | 107 | .PHONY: all clean debug 108 | 109 | .SUFFIXES: 110 | 111 | 112 | 113 | # 114 | # Helper targers 115 | # 116 | 117 | all: ${COMMON_TARGETS} 118 | 119 | clean: 120 | @rm -rf ${T_DIR} 121 | 122 | debug: ${DEBUG_TARGETS} 123 | 124 | 125 | 126 | # 127 | # Core targets 128 | # 129 | 130 | ${T_BASE}-memory: benchmark.cc ${COMMON_DEPENDENCIES} 131 | @echo CC $@ >&2 132 | @mkdir -p ${T_DIR} 133 | @${CC} -DLL_MEMORY_ONLY ${CFLAGS} $< ${LFLAGS} -o $@ 134 | 135 | ${T_BASE}-persistent: benchmark.cc ${COMMON_DEPENDENCIES} 136 | @echo CC $@ >&2 137 | @mkdir -p ${T_DIR} 138 | @${CC} -DLL_PERSISTENCE ${CFLAGS} $< ${LFLAGS} -o $@ 139 | 140 | ${T_BASE}-slcsr: benchmark.cc ${COMMON_DEPENDENCIES} 141 | @echo CC $@ >&2 142 | @mkdir -p ${T_DIR} 143 | @${CC} -DLL_SLCSR ${CFLAGS} $< ${LFLAGS} -o $@ 144 | 145 | ${T_BASE}-streaming: benchmark.cc ${COMMON_DEPENDENCIES} 146 | @echo CC $@ >&2 147 | @mkdir -p ${T_DIR} 148 | @${CC} -DLL_STREAMING ${CFLAGS} $< ${LFLAGS} -o $@ 149 | 150 | ${T_BASE}-memory-wd: benchmark.cc ${COMMON_DEPENDENCIES} 151 | @echo CC $@ >&2 152 | @mkdir -p ${T_DIR} 153 | @${CC} -DLL_MEMORY_ONLY -DLL_DELETIONS ${CFLAGS} $< ${LFLAGS} -o $@ 154 | 155 | ${T_BASE}-persistent-wd: benchmark.cc ${COMMON_DEPENDENCIES} 156 | @echo CC $@ >&2 157 | @mkdir -p ${T_DIR} 158 | @${CC} -DLL_PERSISTENCE -DLL_DELETIONS ${CFLAGS} $< ${LFLAGS} -o $@ 159 | 160 | 161 | 162 | # 163 | # Core debug targets 164 | # 165 | 166 | ${T_BASE}-memory_debug: benchmark.cc ${COMMON_DEPENDENCIES} 167 | @echo CC $@ >&2 168 | @mkdir -p ${T_DIR} 169 | @${CC} -DLL_MEMORY_ONLY ${CFLAGS_D} $< ${LFLAGS_D} -o $@ 170 | 171 | ${T_BASE}-persistent_debug: benchmark.cc ${COMMON_DEPENDENCIES} 172 | @echo CC $@ >&2 173 | @mkdir -p ${T_DIR} 174 | @${CC} -DLL_PERSISTENCE ${CFLAGS_D} $< ${LFLAGS_D} -o $@ 175 | 176 | ${T_BASE}-slcsr_debug: benchmark.cc ${COMMON_DEPENDENCIES} 177 | @echo CC $@ >&2 178 | @mkdir -p ${T_DIR} 179 | @${CC} -DLL_SLCSR ${CFLAGS_D} $< ${LFLAGS_D} -o $@ 180 | 181 | ${T_BASE}-streaming_debug: benchmark.cc ${COMMON_DEPENDENCIES} 182 | @echo CC $@ >&2 183 | @mkdir -p ${T_DIR} 184 | @${CC} -DLL_STREAMING ${CFLAGS_D} $< ${LFLAGS_D} -o $@ 185 | 186 | ${T_BASE}-memory-wd_debug: benchmark.cc ${COMMON_DEPENDENCIES} 187 | @echo CC $@ >&2 188 | @mkdir -p ${T_DIR} 189 | @${CC} -DLL_MEMORY_ONLY -DLL_DELETIONS ${CFLAGS_D} $< ${LFLAGS_D} -o $@ 190 | 191 | ${T_BASE}-persistent-wd_debug: benchmark.cc ${COMMON_DEPENDENCIES} 192 | @echo CC $@ >&2 193 | @mkdir -p ${T_DIR} 194 | @${CC} -DLL_PERSISTENCE -DLL_DELETIONS ${CFLAGS_D} $< ${LFLAGS_D} -o $@ 195 | 196 | 197 | 198 | # 199 | # Profiling targets 200 | # 201 | 202 | ${T_BASE}-memory_gprof: benchmark.cc ${T_DIR} ${HEADERS} Makefile 203 | @echo CC $@ >&2 204 | @${CC} -DLL_MEMORY_ONLY -pg ${CFLAGS} $< ${LFLAGS} -o $@ 205 | 206 | ${T_BASE}-memory_gperf: benchmark.cc ${T_DIR} ${HEADERS} Makefile 207 | @echo CC $@ >&2 208 | @${CC} -DLL_MEMORY_ONLY -DGPERF -DGPERF_CPU ${CFLAGS} $< \ 209 | -lprofiler ${LFLAGS} -o $@ 210 | 211 | 212 | 213 | # 214 | # Additional debug benchmark targets 215 | # 216 | 217 | ${T_BASE}-w-memory_debug: benchmark.cc ${T_DIR} ${HEADERS} Makefile 218 | @echo CC $@ >&2 219 | @${CC} -DLL_MEMORY_ONLY -DBENCHMARK_WRITABLE \ 220 | -DLL_DELETIONS ${CFLAGS_D} $< ${LFLAGS_D} -o $@ 221 | 222 | 223 | 224 | # 225 | # Additional benchmark targets 226 | # 227 | 228 | ${T_BASE}-w-memory: benchmark.cc ${T_DIR} ${HEADERS} Makefile 229 | @echo CC $@ >&2 230 | @${CC} -DLL_MEMORY_ONLY -DBENCHMARK_WRITABLE \ 231 | -DLL_DELETIONS ${CFLAGS} $< ${LFLAGS} -o $@ 232 | 233 | 234 | 235 | # 236 | # Obsolete targets 237 | # 238 | 239 | oprofile-setup: 240 | sudo opcontrol --setup \ 241 | --event=LLC_MISSES:10000:0x41:0:1 \ 242 | --event=BR_MISS_PRED_RETIRED:1000:0:0:1 \ 243 | --no-vmlinux 244 | 245 | oprofile-benchmark: ${BENCHMARK_TARGETS} 246 | @for i in $^; do \ 247 | echo ============================== $$i ==============================; \ 248 | sudo opcontrol --reset; \ 249 | sudo opcontrol --start; \ 250 | echo $$i ${INPUT} >&2; \ 251 | $$i ${INPUT} || exit 1; \ 252 | sudo opcontrol --stop; \ 253 | opreport $$i; \ 254 | done 255 | 256 | oprofile-singlethreaded-benchmark: ${BENCHMARK_TARGETS} 257 | @for i in $^; do \ 258 | echo ============================== $$i ==============================; \ 259 | sudo opcontrol --reset; \ 260 | sudo opcontrol --start; \ 261 | echo $$i -GMNumThreads=1 ${INPUT} >&2; \ 262 | $$i -GMNumThreads=1 ${INPUT} || exit 1; \ 263 | sudo opcontrol --stop; \ 264 | opreport $$i; \ 265 | done 266 | 267 | counters: ${T_BASE}_counters 268 | $< -GMNumThreads=1 ${INPUT} 269 | 270 | gprof: ${T_BASE}_gprof 271 | $< -GMNumThreads=1 ${INPUT} 272 | 273 | objdump: ${T_BASE} 274 | @objdump -d $< \ 275 | | sed -En '/^[0-9a-f]+.._ZN22ll_b_triangle_countingI/,$$p' \ 276 | | sed -n '/^$$/q;p' \ 277 | | sed 's/_ZN22ll_b_triangle_countingI.*runEv/triangle_counting/' \ 278 | | sed 's/_ZN27ll_exp_common_neighbor_iter.*E8get_nextEv/cni::get_next/' 279 | 280 | objdump-test: ${T_BASE} 281 | @objdump -d $< \ 282 | | sed -En '/^[0-9a-f]+.._[A-Z][0-9]+TEST_FUNCTION/,$$p' \ 283 | | sed -n '/^$$/q;p' \ 284 | | sed 's/_Z13TEST_FUNCTION.*RT_/TEST_FUNCTION/' 285 | 286 | benchmark_swcow: benchmark.cc ${T_DIR} ${HEADERS} Makefile 287 | @rm -f $@_*.txt 288 | @for i in 1 2 3 4 5 6 7 8 9 10 24; do \ 289 | echo ============================== $$i ==============================; \ 290 | ${CC} -DLL_DEFAULT_ENTRIES_PER_PAGE_BITS=$$i -DLL_VT=ll_vt_swcow_array \ 291 | ${CFLAGS} $< ${LFLAGS} -o ${T_DIR}/benchmark_tmp || exit 1; \ 292 | echo $@ $$i > $@_$$i.txt; \ 293 | date >> $@_$$i.txt; \ 294 | echo >> $@_$$i.txt; \ 295 | ${T_DIR}/benchmark_tmp ${INPUT} | tee -a $@_$$i.txt; \ 296 | done 297 | 298 | benchmark_vt: benchmark.cc ${T_DIR} ${HEADERS} Makefile 299 | @rm -f $@_*.txt 300 | @for i in array cow_array swcow_array; do \ 301 | echo ============================== $$i ==============================; \ 302 | ${CC} -DLL_VT=ll_vt_$$i ${CFLAGS} $< ${LFLAGS} -o ${T_DIR}/benchmark_tmp || exit 1; \ 303 | echo $@ $$i > $@_$$i.txt; \ 304 | date >> $@_$$i.txt; \ 305 | echo >> $@_$$i.txt; \ 306 | ${T_DIR}/benchmark_tmp ${INPUT} | tee -a $@_$$i.txt; \ 307 | done 308 | 309 | -------------------------------------------------------------------------------- /benchmark/benchmarks/avg_teen_cnt.h: -------------------------------------------------------------------------------- 1 | /* 2 | * avg_teen_cnt.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_GENERATED_CPP_AVG_TEEN_CNT_H 38 | #define LL_GENERATED_CPP_AVG_TEEN_CNT_H 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | #include "benchmarks/benchmark.h" 50 | 51 | 52 | /** 53 | * The average teen counting benchmark 54 | */ 55 | template 56 | class ll_b_avg_teen_cnt : public ll_benchmark { 57 | 58 | int K; 59 | int32_t* G_teen_cnt; 60 | ll_mlcsr_node_property* G_age; 61 | 62 | 63 | public: 64 | 65 | /** 66 | * Create the benchmark 67 | * 68 | * @param k the age threshold 69 | */ 70 | ll_b_avg_teen_cnt(int k) : ll_benchmark("Average Teen Count") { 71 | 72 | K = k; 73 | 74 | this->create_auto_array_for_nodes(G_teen_cnt); 75 | this->create_auto_property(G_age, "age"); 76 | } 77 | 78 | 79 | /** 80 | * Destroy the benchmark 81 | */ 82 | virtual ~ll_b_avg_teen_cnt(void) { 83 | } 84 | 85 | 86 | /** 87 | * Run the benchmark 88 | * 89 | * @return the numerical result, if applicable 90 | */ 91 | virtual double run(void) { 92 | 93 | Graph& G = *this->_graph; 94 | 95 | float avg = 0.0 ; 96 | double _avg4 = 0.0 ; 97 | int64_t _cnt3 = 0 ; 98 | int32_t __S2 = 0 ; 99 | 100 | __S2 = 0 ; 101 | _cnt3 = 0 ; 102 | #pragma omp parallel 103 | { 104 | int32_t __S2_prv = 0 ; 105 | int64_t _cnt3_prv = 0 ; 106 | 107 | _cnt3_prv = 0 ; 108 | __S2_prv = 0 ; 109 | 110 | #pragma omp for nowait schedule(dynamic,4096) 111 | for (node_t n = 0; n < G.max_nodes(); n ++) 112 | { 113 | int32_t __S1 = 0 ; 114 | 115 | __S1 = 0 ; 116 | ll_edge_iterator iter; 117 | G.inm_iter_begin(iter, n); 118 | for (node_t t = G.inm_iter_next(iter); 119 | t != LL_NIL_NODE; 120 | t = G.inm_iter_next(iter)) { 121 | 122 | if ((((*G_age)[t] >= 10) && ((*G_age)[t] < 20))) 123 | { 124 | __S1 = __S1 + 1 ; 125 | } 126 | } 127 | G.set_node_prop(G_teen_cnt, n, __S1); 128 | if (((*G_age)[n] > K)) 129 | { 130 | __S2_prv = __S2_prv + G_teen_cnt[n] ; 131 | _cnt3_prv = _cnt3_prv + 1 ; 132 | } 133 | } 134 | ATOMIC_ADD(&_cnt3, _cnt3_prv); 135 | ATOMIC_ADD(&__S2, __S2_prv); 136 | } 137 | _avg4 = (0 == _cnt3)?0.000000:(__S2 / ((double)_cnt3)) ; 138 | avg = (float)_avg4 ; 139 | return avg; 140 | } 141 | }; 142 | 143 | #endif 144 | -------------------------------------------------------------------------------- /benchmark/benchmarks/bc_adj.h: -------------------------------------------------------------------------------- 1 | /* 2 | * bc_adj.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_GENERATED_CPP_BC_ADJ_H 38 | #define LL_GENERATED_CPP_BC_ADJ_H 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | #include "llama/ll_bfs_template.h" 50 | #include "llama/ll_writable_graph.h" 51 | #include "benchmarks/benchmark.h" 52 | 53 | 54 | // BFS/DFS definitions for the procedure 55 | template 56 | class comp_BC_adj_bfs : public ll_bfs_template 57 | 58 | { 59 | public: 60 | comp_BC_adj_bfs(Graph& _G, float*& _G_BC, node_t& _s, 61 | float*& _G_sigma, float*& _G_delta) 62 | : ll_bfs_template(_G), 63 | G(_G), G_BC(_G_BC), s(_s), G_sigma(_G_sigma), G_delta(_G_delta){} 64 | 65 | private: // list of varaibles 66 | Graph& G; 67 | float*& G_BC; 68 | node_t& s; 69 | float*& G_sigma; 70 | float*& G_delta; 71 | 72 | protected: 73 | virtual void visit_fw(node_t v) 74 | { 75 | { 76 | ll_edge_iterator iter; 77 | G.out_iter_begin(iter, v); 78 | for (edge_t w_idx = G.out_iter_next(iter); 79 | w_idx != LL_NIL_EDGE; 80 | w_idx = G.out_iter_next(iter)) { 81 | if (!this->is_down_edge(w_idx)) continue; 82 | node_t w = LL_ITER_OUT_NEXT_NODE(G, iter, w_idx); 83 | float sigma_w_prv = 0.0 ; 84 | 85 | sigma_w_prv = ((float)(0.000000)) ; 86 | sigma_w_prv = sigma_w_prv + G_sigma[v] ; 87 | ATOMIC_ADD(&G_sigma[w], sigma_w_prv); 88 | } 89 | } 90 | } 91 | 92 | virtual void visit_rv(node_t v) 93 | { 94 | if (v != s) 95 | { 96 | float __S3 = 0.0 ; 97 | 98 | __S3 = ((float)(0.000000)) ; 99 | ll_edge_iterator iter; 100 | G.out_iter_begin(iter, v); 101 | for (edge_t w_idx = G.out_iter_next(iter); 102 | w_idx != LL_NIL_EDGE; 103 | w_idx = G.out_iter_next(iter)) { 104 | if (!this->is_down_edge(w_idx)) continue; 105 | node_t w = LL_ITER_OUT_NEXT_NODE(G, iter, w_idx); 106 | __S3 = __S3 + G_sigma[v] / G_sigma[w] * (1 + G_delta[w]) ; 107 | } 108 | G.set_node_prop(G_delta, v, __S3); 109 | G.set_node_prop(G_BC, v, G_BC[v] + G_delta[v]); 110 | } 111 | } 112 | 113 | virtual bool check_navigator(node_t v, edge_t v_idx) {return true;} 114 | 115 | 116 | }; 117 | 118 | 119 | 120 | /** 121 | * Betweenness Centrality - Exact 122 | */ 123 | template 124 | class ll_b_bc_adj : public ll_benchmark { 125 | 126 | float* G_BC; 127 | 128 | 129 | public: 130 | 131 | /** 132 | * Create the benchmark 133 | */ 134 | ll_b_bc_adj() : ll_benchmark("Betweenness Centrality - Exact") { 135 | this->create_auto_array_for_nodes(G_BC); 136 | } 137 | 138 | 139 | /** 140 | * Destroy the benchmark 141 | */ 142 | virtual ~ll_b_bc_adj(void) { 143 | } 144 | 145 | 146 | /** 147 | * Run the benchmark 148 | * 149 | * @return the numerical result, if applicable 150 | */ 151 | virtual double run(void) { 152 | 153 | Graph& G = *this->_graph; 154 | ll_memory_helper m; 155 | 156 | float* G_sigma = m.allocate(G.max_nodes()); 157 | float* G_delta = m.allocate(G.max_nodes()); 158 | 159 | 160 | #pragma omp parallel for 161 | for (node_t t0 = 0; t0 < G.max_nodes(); t0 ++) 162 | G.set_node_prop(G_BC, t0, (float)0); 163 | 164 | for (node_t s = 0; s < G.max_nodes(); s ++) 165 | { 166 | 167 | #pragma omp parallel for 168 | for (node_t t1 = 0; t1 < G.max_nodes(); t1 ++) 169 | G.set_node_prop(G_sigma, t1, (float)0); 170 | 171 | G.set_node_prop(G_sigma, s, (float)1); 172 | 173 | comp_BC_adj_bfs _BFS(G, G_BC, s, G_sigma, G_delta); 174 | _BFS.prepare(s); 175 | _BFS.do_bfs_forward(); 176 | _BFS.do_bfs_reverse(); 177 | } 178 | 179 | return 0; 180 | } 181 | 182 | 183 | /** 184 | * Finalize the benchmark 185 | * 186 | * @return the updated numerical result, if applicable 187 | */ 188 | virtual double finalize(void) { 189 | float max = 0; 190 | 191 | for (node_t n = 0; n < this->_graph->max_nodes(); n++) { 192 | if (G_BC[n] > max) max = G_BC[n]; 193 | } 194 | 195 | return max; 196 | } 197 | 198 | 199 | /** 200 | * Print the results 201 | * 202 | * @param f the output file 203 | */ 204 | virtual void print_results(FILE* f) { 205 | print_results_part(f, this->_graph, G_BC); 206 | } 207 | }; 208 | 209 | #endif 210 | 211 | -------------------------------------------------------------------------------- /benchmark/benchmarks/bc_random.h: -------------------------------------------------------------------------------- 1 | /* 2 | * bc_random.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_GENERATED_CPP_BC_RANDOM_H 38 | #define LL_GENERATED_CPP_BC_RANDOM_H 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | #include "llama/ll_bfs_template.h" 50 | #include "llama/ll_writable_graph.h" 51 | #include "benchmarks/benchmark.h" 52 | 53 | 54 | // BFS/DFS definitions for the procedure 55 | template 56 | class bc_random_bfs : public ll_bfs_template 57 | 58 | { 59 | public: 60 | bc_random_bfs(Graph& _G, float*& _G_BC, node_t& _s, 61 | float*& _G_sigma, float*& _G_delta) 62 | : ll_bfs_template(_G), 63 | G(_G), G_BC(_G_BC), s(_s), G_sigma(_G_sigma), G_delta(_G_delta){} 64 | 65 | private: // list of varaibles 66 | Graph& G; 67 | float*& G_BC; 68 | node_t& s; 69 | float*& G_sigma; 70 | float*& G_delta; 71 | 72 | protected: 73 | virtual void visit_fw(node_t v) 74 | { 75 | { 76 | ll_edge_iterator iter; 77 | G.out_iter_begin(iter, v); 78 | for (edge_t w_idx = G.out_iter_next(iter); 79 | w_idx != LL_NIL_EDGE; 80 | w_idx = G.out_iter_next(iter)) { 81 | if (!this->is_down_edge(w_idx)) continue; 82 | node_t w = LL_ITER_OUT_NEXT_NODE(G, iter, w_idx); 83 | float sigma_w_prv = 0.0 ; 84 | 85 | sigma_w_prv = ((float)(0.000000)) ; 86 | sigma_w_prv = sigma_w_prv + G_sigma[v] ; 87 | ATOMIC_ADD(&G_sigma[w], sigma_w_prv); 88 | } 89 | } 90 | } 91 | 92 | virtual void visit_rv(node_t v) 93 | { 94 | if (v != s) 95 | { 96 | float __S3 = 0.0 ; 97 | 98 | __S3 = ((float)(0.000000)) ; 99 | ll_edge_iterator iter; 100 | G.out_iter_begin(iter, v); 101 | for (edge_t w_idx = G.out_iter_next(iter); 102 | w_idx != LL_NIL_EDGE; 103 | w_idx = G.out_iter_next(iter)) { 104 | if (!this->is_down_edge(w_idx)) continue; 105 | node_t w = LL_ITER_OUT_NEXT_NODE(G, iter, w_idx); 106 | __S3 = __S3 + G_sigma[v] / G_sigma[w] * (1 + G_delta[w]) ; 107 | } 108 | G.set_node_prop(G_delta, v, __S3); 109 | G.set_node_prop(G_BC, v, G_BC[v] + G_delta[v]); 110 | } 111 | } 112 | 113 | virtual bool check_navigator(node_t v, edge_t v_idx) {return true;} 114 | 115 | 116 | }; 117 | 118 | 119 | 120 | /** 121 | * Betweenness Centrality - Randomized Algorithm 122 | */ 123 | template 124 | class ll_b_bc_random : public ll_benchmark { 125 | 126 | int K; 127 | float* G_BC; 128 | 129 | 130 | public: 131 | 132 | /** 133 | * Create the benchmark 134 | * 135 | * @param k the number of seeds 136 | */ 137 | ll_b_bc_random(int k) 138 | : ll_benchmark("Betweenness Centrality - Randomized") { 139 | 140 | K = k; 141 | 142 | this->create_auto_array_for_nodes(G_BC); 143 | } 144 | 145 | 146 | /** 147 | * Destroy the benchmark 148 | */ 149 | virtual ~ll_b_bc_random(void) { 150 | } 151 | 152 | 153 | /** 154 | * Run the benchmark 155 | * 156 | * @return the numerical result, if applicable 157 | */ 158 | virtual double run(void) { 159 | 160 | Graph& G = *this->_graph; 161 | int32_t k = 0 ; 162 | ll_memory_helper m; 163 | 164 | float* G_sigma = m.allocate(G.max_nodes()); 165 | float* G_delta = m.allocate(G.max_nodes()); 166 | 167 | k = 0 ; 168 | 169 | #pragma omp parallel for 170 | for (node_t t0 = 0; t0 < G.max_nodes(); t0 ++) 171 | G.set_node_prop(G_BC, t0, (float)0); 172 | 173 | while (k < K) 174 | { 175 | node_t s; 176 | 177 | s = G.pick_random_node() ; 178 | 179 | #pragma omp parallel for 180 | for (node_t t1 = 0; t1 < G.max_nodes(); t1 ++) 181 | G.set_node_prop(G_sigma, t1, (float)0); 182 | 183 | G.set_node_prop(G_sigma, s, (float)1); 184 | 185 | bc_random_bfs _BFS(G, G_BC, s, G_sigma, G_delta); 186 | _BFS.prepare(s); 187 | _BFS.do_bfs_forward(); 188 | _BFS.do_bfs_reverse(); 189 | k = k + 1 ; 190 | } 191 | 192 | return 0; 193 | } 194 | 195 | 196 | /** 197 | * Finalize the benchmark 198 | * 199 | * @return the updated numerical result, if applicable 200 | */ 201 | virtual double finalize(void) { 202 | float max = 0; 203 | 204 | for (node_t n = 0; n < this->_graph->max_nodes(); n++) { 205 | if (G_BC[n] > max) max = G_BC[n]; 206 | } 207 | 208 | return max; 209 | } 210 | 211 | 212 | /** 213 | * Print the results 214 | * 215 | * @param f the output file 216 | */ 217 | virtual void print_results(FILE* f) { 218 | print_results_part(f, this->_graph, G_BC); 219 | } 220 | }; 221 | 222 | #endif 223 | -------------------------------------------------------------------------------- /benchmark/benchmarks/bfs.h: -------------------------------------------------------------------------------- 1 | /* 2 | * bfs.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_BFS_H 38 | #define LL_BFS_H 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | #include "llama/ll_bfs_template.h" 50 | #include "llama/ll_writable_graph.h" 51 | #include "benchmarks/benchmark.h" 52 | 53 | 54 | // BFS/DFS definitions for the procedure 55 | template 56 | class bfs_bfs : public ll_bfs_template 57 | 58 | { 59 | public: 60 | bfs_bfs(Graph& _G, node_t& _root, int32_t& _count) 61 | : ll_bfs_template(_G), 62 | G(_G), root(_root), count(_count){} 63 | 64 | private: // list of varaibles 65 | Graph& G; 66 | node_t& root; 67 | int32_t& count; 68 | 69 | protected: 70 | virtual void visit_fw(node_t v) 71 | { 72 | ATOMIC_ADD(&count, 1); 73 | } 74 | 75 | virtual void visit_rv(node_t v) {} 76 | virtual bool check_navigator(node_t v, edge_t v_idx) {return true;} 77 | 78 | 79 | }; 80 | 81 | 82 | /** 83 | * Count vertices in the given BFS traversal 84 | */ 85 | template 86 | class ll_b_bfs : public ll_benchmark { 87 | 88 | node_t root; 89 | 90 | 91 | public: 92 | 93 | /** 94 | * Create the benchmark 95 | * 96 | * @param r the root 97 | */ 98 | ll_b_bfs(node_t r) 99 | : ll_benchmark("BFS - Count") { 100 | 101 | root = r; 102 | } 103 | 104 | 105 | /** 106 | * Destroy the benchmark 107 | */ 108 | virtual ~ll_b_bfs(void) { 109 | } 110 | 111 | 112 | /** 113 | * Run the benchmark 114 | * 115 | * @return the numerical result, if applicable 116 | */ 117 | virtual double run(void) { 118 | 119 | Graph& G = *this->_graph; 120 | int32_t count = 0; 121 | 122 | bfs_bfs _BFS(G, root, count); 123 | _BFS.prepare(root); 124 | _BFS.do_bfs_forward(); 125 | 126 | return count; 127 | } 128 | }; 129 | 130 | #endif 131 | -------------------------------------------------------------------------------- /benchmark/benchmarks/tarjan_scc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * tarjan_scc.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_GENERATED_CPP_TARJAN_SCC_H 38 | #define LL_GENERATED_CPP_TARJAN_SCC_H 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | #include "llama/ll_dfs_template.h" 50 | #include "llama/ll_writable_graph.h" 51 | #include "llama/ll_seq.h" 52 | #include "benchmarks/benchmark.h" 53 | 54 | 55 | // BFS/DFS definitions for the procedure 56 | template 57 | class Tarjan_dfs : public ll_dfs_template 58 | 59 | { 60 | public: 61 | Tarjan_dfs(Graph& _G, node_t*& _G_SCC, bool*& _G_InStack, 62 | node_t*& _G_LowLink, ll_node_seq_vec& _Stack, node_t& _n) 63 | : ll_dfs_template(_G), 64 | G(_G), G_SCC(_G_SCC), G_InStack(_G_InStack), G_LowLink(_G_LowLink), 65 | Stack(_Stack), n(_n){} 66 | 67 | private: // list of varaibles 68 | Graph& G; 69 | node_t*& G_SCC; 70 | bool*& G_InStack; 71 | node_t*& G_LowLink; 72 | ll_node_seq_vec& Stack; 73 | node_t& n; 74 | 75 | protected: 76 | virtual void visit_pre(node_t t) 77 | { 78 | G.set_node_prop(G_InStack, t, true); 79 | Stack.push_back(t); 80 | G.set_node_prop(G_LowLink, t, t); 81 | } 82 | 83 | virtual void visit_post(node_t t) 84 | { 85 | ll_edge_iterator iter; 86 | G.out_iter_begin(iter, t); 87 | FOREACH_OUTEDGE_ITER(k_idx, G, iter) { 88 | node_t k = iter.last_node; 89 | if (G_LowLink[k] < G_LowLink[t]) 90 | G.set_node_prop(G_LowLink, t, G_LowLink[k]); 91 | } 92 | if (G_LowLink[t] == t) 93 | { 94 | node_t w; 95 | 96 | w = Stack.pop_back() ; 97 | while (w != t) 98 | { 99 | G.set_node_prop(G_InStack, w, false); 100 | G.set_node_prop(G_SCC, w, t); 101 | w = Stack.pop_back() ; 102 | } 103 | G.set_node_prop(G_InStack, w, false); 104 | G.set_node_prop(G_SCC, w, t); 105 | } 106 | } 107 | 108 | virtual bool check_navigator(node_t t, edge_t t_idx) 109 | { 110 | return ( !G_InStack[t]); 111 | } 112 | 113 | 114 | }; 115 | 116 | 117 | 118 | /** 119 | * Tarjan's SCC 120 | */ 121 | template 122 | class ll_b_tarjan_scc : public ll_benchmark { 123 | 124 | node_t* G_SCC; 125 | 126 | 127 | public: 128 | 129 | /** 130 | * Create the benchmark 131 | */ 132 | ll_b_tarjan_scc() : ll_benchmark("Tarjan's SCC") { 133 | this->create_auto_array_for_nodes(G_SCC); 134 | } 135 | 136 | 137 | /** 138 | * Destroy the benchmark 139 | */ 140 | virtual ~ll_b_tarjan_scc(void) { 141 | } 142 | 143 | 144 | /** 145 | * Run the benchmark 146 | * 147 | * @return the numerical result, if applicable 148 | */ 149 | virtual double run(void) { 150 | 151 | //Initializations 152 | Graph& G = *this->_graph; 153 | ll_memory_helper m; 154 | 155 | ll_node_seq_vec Stack(omp_get_max_threads()); 156 | bool* G_InStack = m.allocate(G.max_nodes()); 157 | node_t* G_LowLink = m.allocate(G.max_nodes()); 158 | 159 | 160 | #pragma omp parallel for 161 | for (node_t t0 = 0; t0 < G.max_nodes(); t0 ++) 162 | { 163 | G.set_node_prop(G_SCC, t0, LL_NIL_NODE); 164 | G.set_node_prop(G_InStack, t0, false); 165 | } 166 | for (node_t n = 0; n < G.max_nodes(); n ++) 167 | { 168 | if (G_SCC[n] == LL_NIL_NODE) 169 | { 170 | 171 | Tarjan_dfs _DFS(G, G_SCC, G_InStack, G_LowLink, 172 | Stack, n); 173 | _DFS.prepare(n); 174 | _DFS.do_dfs(); 175 | } 176 | } 177 | 178 | return 0; 179 | } 180 | 181 | 182 | /** 183 | * Finalize the benchmark 184 | * 185 | * @return the updated numerical result, if applicable 186 | */ 187 | virtual double finalize(void) { 188 | node_t max = 0; 189 | 190 | for (node_t n = 0; n < this->_graph->max_nodes(); n++) { 191 | if (G_SCC[n] > max) max = G_SCC[n]; 192 | } 193 | 194 | return max; 195 | } 196 | 197 | 198 | /** 199 | * Print the results 200 | * 201 | * @param f the output file 202 | */ 203 | virtual void print_results(FILE* f) { 204 | print_results_part(f, this->_graph, G_SCC); 205 | } 206 | }; 207 | 208 | #endif 209 | -------------------------------------------------------------------------------- /benchmark/tests/delete_edges.h: -------------------------------------------------------------------------------- 1 | /* 2 | * delete_edges.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_TEST_DELETE_EDGES_H 38 | #define LL_TEST_DELETE_EDGES_H 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | #include "llama/ll_writable_graph.h" 50 | #include "benchmarks/benchmark.h" 51 | 52 | 53 | /** 54 | * Test: Delete edges 55 | */ 56 | template 57 | class ll_t_delete_edges : public ll_benchmark { 58 | 59 | 60 | public: 61 | 62 | /** 63 | * Create the test 64 | */ 65 | ll_t_delete_edges() : ll_benchmark("[Test] Delete Edges") { 66 | } 67 | 68 | 69 | /** 70 | * Destroy the test 71 | */ 72 | virtual ~ll_t_delete_edges(void) { 73 | } 74 | 75 | 76 | /** 77 | * Run the benchmark 78 | * 79 | * @return the numerical result, if applicable 80 | */ 81 | virtual double run(void) { 82 | 83 | Graph& G = *this->_graph; 84 | 85 | printf("\nDELETE EDGES TEST START\n"); 86 | printf(" * Delete: "); fflush(stdout); 87 | 88 | int num_nodes = 0; 89 | int num_edges = 0; 90 | int numDeletedEdges = 0; 91 | 92 | G.tx_begin(); 93 | 94 | #pragma omp parallel 95 | { 96 | int nn = 0; 97 | int ne = 0; 98 | int nd = 0; 99 | 100 | #pragma omp for schedule(dynamic,4096) 101 | for (node_t n = 0; n < G.max_nodes(); n++) { 102 | nn++; 103 | 104 | ll_edge_iterator iter; 105 | G.out_iter_begin(iter, n); 106 | for (edge_t v_idx = G.out_iter_next(iter); v_idx != LL_NIL_EDGE; 107 | v_idx = G.out_iter_next(iter)) { 108 | node_t v = LL_ITER_OUT_NEXT_NODE(G, iter, v_idx); 109 | (void) v; 110 | ne++; 111 | 112 | if (v_idx % 10 == 0) { 113 | G.delete_edge(n, v_idx); 114 | //fprintf(stderr, "Delete %08lx %08ld --> %08ld\n", v_idx, n, v); 115 | nd++; 116 | } 117 | } 118 | } 119 | 120 | ATOMIC_ADD(&num_nodes, nn); 121 | ATOMIC_ADD(&num_edges, ne); 122 | ATOMIC_ADD(&numDeletedEdges, nd); 123 | } 124 | //printf("\n --> "); 125 | 126 | printf("%d edges originaly, %d deleted, %d left\n", 127 | num_edges, numDeletedEdges, num_edges - numDeletedEdges); fflush(stdout); 128 | 129 | G.tx_commit(); 130 | 131 | printf(" * Validate: "); fflush(stdout); 132 | 133 | int num_edges2_count = 0; 134 | int num_edges2_degree = 0; 135 | bool ok = true; 136 | bool ok_edge_sources = true; 137 | 138 | G.tx_begin(); 139 | 140 | #pragma omp parallel 141 | { 142 | int ne = 0; 143 | int ng = 0; 144 | 145 | #pragma omp for schedule(dynamic,4096) 146 | for (node_t n = 0; n < G.max_nodes(); n++) { 147 | 148 | ll_edge_iterator iter; 149 | G.out_iter_begin(iter, n); 150 | for (edge_t v_idx = G.out_iter_next(iter); v_idx != LL_NIL_EDGE; 151 | v_idx = G.out_iter_next(iter)) { 152 | 153 | ne++; 154 | 155 | if (v_idx % 10 == 0) { 156 | ok = false; 157 | } 158 | } 159 | 160 | assert(G.out_degree(n) >= 0); 161 | ng += G.out_degree(n); 162 | } 163 | 164 | ATOMIC_ADD(&num_edges2_count, ne); 165 | ATOMIC_ADD(&num_edges2_degree, ng); 166 | } 167 | //printf("\n --> "); 168 | 169 | printf("%d counted, %d by summing out_degree\n", 170 | num_edges2_count, num_edges2_degree); fflush(stdout); 171 | 172 | G.tx_commit(); 173 | 174 | if (!ok || num_edges2_count != num_edges2_degree || num_edges2_count != num_edges - numDeletedEdges) { 175 | printf(" --> failed"); 176 | if (!ok_edge_sources) printf(" [edge sources do not match]"); 177 | printf("\n"); 178 | return NAN; 179 | } 180 | 181 | printf(" * Checkpoint: "); fflush(stdout); 182 | 183 | G.checkpoint(); 184 | 185 | printf("done\n"); fflush(stdout); 186 | 187 | printf(" * Validate: "); fflush(stdout); 188 | 189 | int num_edges3_count = 0; 190 | int num_edges3_degree = 0; 191 | ok = true; 192 | 193 | G.tx_begin(); 194 | 195 | #pragma omp parallel 196 | { 197 | int ne = 0; 198 | int ng = 0; 199 | 200 | #pragma omp for schedule(dynamic,4096) 201 | for (node_t n = 0; n < G.max_nodes(); n++) { 202 | 203 | ll_edge_iterator iter; 204 | G.out_iter_begin(iter, n); 205 | for (edge_t v_idx = G.out_iter_next(iter); v_idx != LL_NIL_EDGE; 206 | v_idx = G.out_iter_next(iter)) { 207 | 208 | ne++; 209 | } 210 | 211 | ng += G.out_degree(n); 212 | } 213 | 214 | ATOMIC_ADD(&num_edges3_count, ne); 215 | ATOMIC_ADD(&num_edges3_degree, ng); 216 | } 217 | //printf("\n --> "); 218 | 219 | printf("%d counted, %d by summing out_degree\n", 220 | num_edges2_count, num_edges2_degree); fflush(stdout); 221 | 222 | G.tx_commit(); 223 | 224 | if (!ok || num_edges2_count != num_edges2_degree || num_edges2_count != num_edges - numDeletedEdges) { 225 | printf(" --> failed"); 226 | if (!ok_edge_sources) printf(" [edge sources do not match]"); 227 | printf("\n"); 228 | return NAN; 229 | } 230 | 231 | printf("DID NOT CRASH :)\n"); 232 | return NAN; 233 | } 234 | }; 235 | 236 | #endif 237 | -------------------------------------------------------------------------------- /benchmark/tools/degree_distribution.h: -------------------------------------------------------------------------------- 1 | /* 2 | * degree_distribution.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_DEGREE_DISTRIBUTION_H 38 | #define LL_DEGREE_DISTRIBUTION_H 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | #include "benchmarks/benchmark.h" 50 | 51 | 52 | /** 53 | * Tool: Degree distribution 54 | */ 55 | template 56 | class ll_t_degree_distribution : public ll_benchmark { 57 | 58 | std::vector log2degree_counts; 59 | std::vector log2degree_sums; 60 | size_t max_edges; 61 | 62 | 63 | public: 64 | 65 | /** 66 | * Create the tool 67 | */ 68 | ll_t_degree_distribution() 69 | : ll_benchmark("[Tool] Degree Distribution") { 70 | max_edges = 0; 71 | } 72 | 73 | 74 | /** 75 | * Destroy the tool 76 | */ 77 | virtual ~ll_t_degree_distribution(void) { 78 | } 79 | 80 | 81 | /** 82 | * Run the tool 83 | * 84 | * @return the numerical result, if applicable 85 | */ 86 | virtual double run(void) { 87 | Graph& G = *this->_graph; 88 | 89 | for (node_t n = 0; n < G.max_nodes(); n++) { 90 | size_t d = G.out_degree(n); 91 | 92 | size_t l = 0; 93 | size_t x = d; 94 | while (x > 1) { l++; x >>= 1; } 95 | 96 | while (log2degree_counts.size() <= l) log2degree_counts.push_back(0); 97 | log2degree_counts[l] += 1; 98 | 99 | while (log2degree_sums.size() <= l) log2degree_sums.push_back(0); 100 | log2degree_sums[l] += d; 101 | max_edges += d; 102 | } 103 | 104 | return NAN; 105 | } 106 | 107 | 108 | /** 109 | * Print the results 110 | * 111 | * @param f the output file 112 | */ 113 | virtual void print_results(FILE* f) { 114 | 115 | Graph& G = *this->_graph; 116 | size_t n = G.max_nodes(); 117 | size_t m = max_edges; 118 | 119 | fprintf(f, " Deg >= X | Deg < X | Count | Percent | Cummul. | Percent \n"); 120 | fprintf(f, "----------+----------+----------+----------+----------+----------\n"); 121 | 122 | size_t cummulative_count = 0; 123 | for (size_t i = 0; i < log2degree_counts.size(); i++) { 124 | cummulative_count += log2degree_counts[i]; 125 | fprintf(f, " %8lu | %8lu | %8lu | %8.2lf | %8lu | %8.2lf\n", 126 | i == 0 ? 0 : (1ul << i), (1ul << (i + 1)) - 1, 127 | log2degree_counts[i], 100.0 * log2degree_counts[i] / (double) n, 128 | cummulative_count, 100.0 * cummulative_count / (double) n); 129 | } 130 | fprintf(f, "\n"); 131 | 132 | fprintf(f, " Deg >= X | Deg < X | Deg. Sum | Percent | Cummul. | Percent \n"); 133 | fprintf(f, "----------+----------+----------+----------+----------+----------\n"); 134 | 135 | size_t cummulative_sum = 0; 136 | for (size_t i = 0; i < log2degree_sums.size(); i++) { 137 | cummulative_sum += log2degree_sums[i]; 138 | fprintf(f, " %8lu | %8lu | %8lu | %8.2lf | %8lu | %8.2lf\n", 139 | i == 0 ? 0 : (1ul << i), (1ul << (i + 1)) - 1, 140 | log2degree_sums[i], 100.0 * log2degree_sums[i] / (double) m, 141 | cummulative_sum, 100.0 * cummulative_sum / (double) m); 142 | } 143 | fprintf(f, "\n"); 144 | } 145 | }; 146 | 147 | #endif 148 | -------------------------------------------------------------------------------- /benchmark/tools/dump.h: -------------------------------------------------------------------------------- 1 | /* 2 | * dump.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_DUMP_H 38 | #define LL_DUMP_H 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | #include "benchmarks/benchmark.h" 50 | 51 | 52 | /** 53 | * Tool: Dump the graph 54 | */ 55 | template 56 | class ll_t_dump : public ll_benchmark { 57 | 58 | FILE* _out; 59 | 60 | 61 | public: 62 | 63 | /** 64 | * Create the tool 65 | */ 66 | ll_t_dump() : ll_benchmark("[Tool] Dump") { 67 | _out = stdout; 68 | } 69 | 70 | 71 | /** 72 | * Destroy the tool 73 | */ 74 | virtual ~ll_t_dump(void) { 75 | } 76 | 77 | 78 | /** 79 | * Run the tool 80 | * 81 | * @return the numerical result, if applicable 82 | */ 83 | virtual double run(void) { 84 | 85 | Graph& G = *this->_graph; 86 | size_t max_nodes = G.max_nodes(); 87 | bool reverse = G.has_reverse_edges(); 88 | 89 | printf("\n\n"); 90 | 91 | fprintf(_out, "Out-edges\n"); 92 | for (size_t n = 0; n < max_nodes; n++) { 93 | fprintf(_out, "%5ld:", n); 94 | 95 | ll_edge_iterator iter; 96 | G.out_iter_begin(iter, n); 97 | for (edge_t v_idx = G.out_iter_next(iter); 98 | v_idx != LL_NIL_EDGE; 99 | v_idx = G.out_iter_next(iter)) { 100 | fprintf(_out, "\t%ld", iter.last_node); 101 | } 102 | fprintf(_out, "\n"); 103 | } 104 | 105 | if (reverse) { 106 | fprintf(_out, "\nIn-edges\n"); 107 | for (size_t n = 0; n < max_nodes; n++) { 108 | fprintf(_out, "%5ld:", n); 109 | 110 | ll_edge_iterator iter; 111 | G.inm_iter_begin(iter, n); 112 | for (node_t v = G.inm_iter_next(iter); 113 | v != LL_NIL_NODE; 114 | v = G.inm_iter_next(iter)) { 115 | fprintf(_out, "\t%ld", v); 116 | } 117 | fprintf(_out, "\n"); 118 | } 119 | } 120 | 121 | fprintf(_out, "\n"); 122 | 123 | return NAN; 124 | } 125 | 126 | 127 | /** 128 | * Print the results 129 | */ 130 | virtual void print_results(void) { 131 | } 132 | }; 133 | 134 | #endif 135 | -------------------------------------------------------------------------------- /benchmark/tools/flatten.h: -------------------------------------------------------------------------------- 1 | /* 2 | * flatten.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_FLATTEN_H 38 | #define LL_FLATTEN_H 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | #include "benchmarks/benchmark.h" 50 | 51 | 52 | /** 53 | * Tool: Flatten the entire database into one level -- i.e. do a full merge 54 | */ 55 | template 56 | class ll_t_flatten : public ll_benchmark { 57 | 58 | std::string _out; 59 | 60 | 61 | public: 62 | 63 | /** 64 | * Create the tool 65 | * 66 | * @param out the output directory 67 | */ 68 | ll_t_flatten(const char* out) : ll_benchmark("[Tool] Flatten") { 69 | _out = out; 70 | } 71 | 72 | 73 | /** 74 | * Destroy the tool 75 | */ 76 | virtual ~ll_t_flatten(void) { 77 | } 78 | 79 | 80 | /** 81 | * Run the tool 82 | * 83 | * @return the numerical result, if applicable 84 | */ 85 | virtual double run(void) { 86 | 87 | Graph& G = *this->_graph; 88 | size_t max_nodes = G.max_nodes(); 89 | bool reverse = G.has_reverse_edges(); 90 | 91 | 92 | // Initialize 93 | 94 | degree_t* degrees_out 95 | = (degree_t*) malloc(sizeof(*degrees_out)*max_nodes); 96 | memset(degrees_out, 0, sizeof(*degrees_out) * max_nodes); 97 | 98 | degree_t* degrees_in = NULL; 99 | if (reverse) { 100 | degrees_in = (degree_t*) malloc(sizeof(*degrees_in) * max_nodes); 101 | memset(degrees_in, 0, sizeof(*degrees_in) * max_nodes); 102 | } 103 | 104 | 105 | // Create the new database 106 | 107 | #ifdef LL_PERSISTENCE 108 | ll_persistent_storage* storage 109 | = new ll_persistent_storage(_out.c_str()); 110 | #endif 111 | 112 | ll_writable_graph* p_new_w_graph = new ll_writable_graph 113 | (NULL /* XXX for now */, IF_LL_PERSISTENCE(storage,) max_nodes); 114 | ll_mlcsr_ro_graph& new_r_graph = p_new_w_graph->ro_graph(); 115 | 116 | 117 | // Determine the degree counts 118 | 119 | for (node_t n = 0; n < G.max_nodes(); n++) { 120 | degrees_out[n] = G.out_degree(n); 121 | } 122 | 123 | if (reverse) { 124 | for (node_t n = 0; n < G.max_nodes(); n++) { 125 | degrees_in[n] = G.in_degree(n); 126 | } 127 | } 128 | 129 | 130 | // Create the out-edges 131 | 132 | new_r_graph.out().init_level_from_degrees(max_nodes, degrees_out,NULL); 133 | auto* vt = new_r_graph.out().vertex_table(new_r_graph.num_levels()-1); 134 | LL_ET* et = new_r_graph.out().edge_table( 135 | new_r_graph.num_levels()-1); 136 | 137 | #pragma omp parallel 138 | { 139 | std::vector> v; 140 | 141 | #pragma omp for nowait schedule(dynamic,1) 142 | for (node_t n1 = 0; n1 < G.max_nodes(); n1 += 1024 * 256) { 143 | node_t n2 = n1 + 1024 * 256; 144 | if (n2 > G.max_nodes()) n2 = G.max_nodes(); 145 | 146 | edge_t e = LL_NIL_EDGE; 147 | while (n1 < n2 && (e = (*vt)[n1].adj_list_start) == LL_NIL_EDGE) n1++; 148 | if (e == LL_NIL_EDGE) continue; 149 | 150 | size_t index = LL_EDGE_INDEX(e); 151 | 152 | for (node_t n = n1; n < n2; n++) { 153 | 154 | v.clear(); 155 | ll_edge_iterator iter; 156 | G.out_iter_begin(iter, n); 157 | for (edge_t v_idx = G.out_iter_next(iter); 158 | v_idx != LL_NIL_EDGE; 159 | v_idx = G.out_iter_next(iter)) { 160 | v.push_back(std::pair(iter.last_node, 161 | v_idx)); 162 | } 163 | 164 | std::sort(v.begin(), v.end()); 165 | 166 | for (size_t i = 0; i < v.size(); i++) { 167 | (*et)[index] = LL_VALUE_CREATE((node_t) v[i].first); 168 | 169 | // TODO Edge properties 170 | /*edge_t edge = LL_EDGE_CREATE(0, index); 171 | prop_weight->cow_write(edge, e.weight);*/ 172 | 173 | index++; 174 | } 175 | } 176 | } 177 | } 178 | 179 | 180 | // Create the in-edges 181 | 182 | // TODO 183 | 184 | 185 | // Merge node properties 186 | 187 | // TODO 188 | 189 | 190 | // Cleanup 191 | 192 | new_r_graph.finish_level_edges(); 193 | 194 | delete p_new_w_graph; 195 | 196 | #ifdef LL_PERSISTENCE 197 | delete storage; 198 | #endif 199 | 200 | if (degrees_in != NULL) free(degrees_in); 201 | if (degrees_out != NULL) free(degrees_out); 202 | 203 | return NAN; 204 | } 205 | 206 | 207 | /** 208 | * Print the results 209 | */ 210 | virtual void print_results(void) { 211 | } 212 | }; 213 | 214 | #endif 215 | -------------------------------------------------------------------------------- /benchmark/tools/level_spread.h: -------------------------------------------------------------------------------- 1 | /* 2 | * level_spread.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_LEVEL_SPREAD_H 38 | #define LL_LEVEL_SPREAD_H 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | #include "benchmarks/benchmark.h" 50 | 51 | 52 | template 53 | void level_spread(EXP_GRAPH& G, std::vector& r, std::vector& ds) 54 | { 55 | r.clear(); 56 | int nl = G.num_levels(); 57 | for (int i = 0; i < nl+1; i++) r.push_back(0); 58 | for (int i = 0; i < nl+1; i++) ds.push_back(0); 59 | bool* b = (bool*) alloca(sizeof(bool) * (nl+1)); 60 | 61 | for (node_t n = 0; n < G.max_nodes(); n++) { 62 | bzero(b, sizeof(bool) * (nl+1)); 63 | 64 | ll_edge_iterator iter; 65 | G.out_iter_begin(iter, n); 66 | for (edge_t e = G.out_iter_next(iter); 67 | e != LL_NIL_EDGE; 68 | e = G.out_iter_next(iter)) { 69 | 70 | edge_t l = LL_EDGE_LEVEL(e); 71 | if (l == LL_WRITABLE_LEVEL) l = nl; 72 | b[l] = true; 73 | } 74 | 75 | int x = 0; 76 | for (int i = 0; i < nl+1; i++) { 77 | if (b[i]) x++; 78 | } 79 | 80 | r[x]++; 81 | ds[x] += G.out_degree(n); 82 | } 83 | } 84 | 85 | 86 | /** 87 | * Tool: Level spread 88 | */ 89 | template 90 | class ll_t_level_spread : public ll_benchmark { 91 | 92 | std::vector R_level_spread; 93 | std::vector R_degree_sum; 94 | 95 | 96 | public: 97 | 98 | /** 99 | * Create the tool 100 | */ 101 | ll_t_level_spread() : ll_benchmark("[Tool] Level Spread") { 102 | } 103 | 104 | 105 | /** 106 | * Destroy the tool 107 | */ 108 | virtual ~ll_t_level_spread(void) { 109 | } 110 | 111 | 112 | /** 113 | * Run the tool 114 | * 115 | * @return the numerical result, if applicable 116 | */ 117 | virtual double run(void) { 118 | 119 | level_spread(*this->_graph, R_level_spread, R_degree_sum); 120 | 121 | return NAN; 122 | } 123 | 124 | 125 | /** 126 | * Print the results 127 | * 128 | * @param f the output file 129 | */ 130 | virtual void print_results(FILE* f) { 131 | 132 | fprintf(f, " Num levels | Frequency | Fraction | NZ Fraction | Out Degree \n"); 133 | fprintf(f, "------------+-----------+----------+-------------+------------\n"); 134 | 135 | size_t sum = 0, sum1 = 0; 136 | for (size_t n = 0; n < R_level_spread.size(); n++) sum += R_level_spread[n]; 137 | for (size_t n = 1; n < R_level_spread.size(); n++) sum1 += R_level_spread[n]; 138 | 139 | for (size_t n = 0; n < R_level_spread.size(); n++) { 140 | fprintf(f, "%11lu | %9d | %8.2lf |", 141 | n, R_level_spread[n], 142 | R_level_spread[n] / (double) sum); 143 | if (n > 0) 144 | fprintf(f, " %11.2lf", R_level_spread[n] / (double) sum1); 145 | else 146 | fprintf(f, " %11s", "-"); 147 | if (R_level_spread[n] > 0) 148 | fprintf(f, " | %10.2lf", R_degree_sum[n] / (double) R_level_spread[n]); 149 | else 150 | fprintf(f, " | %10s", "-"); 151 | fprintf(f, "\n"); 152 | } 153 | fprintf(f, "\n"); 154 | } 155 | }; 156 | 157 | #endif 158 | -------------------------------------------------------------------------------- /benchmark/tools/print.h: -------------------------------------------------------------------------------- 1 | /* 2 | * print.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_PRINT_H 38 | #define LL_PRINT_H 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | 50 | template 51 | void print_exp_adj_out(EXP_GRAPH& G, node_t n) { 52 | if (!G.node_exists(n)) { 53 | printf(" %7s\n", "--"); 54 | return; 55 | } 56 | ll_edge_iterator iter; 57 | G.out_iter_begin(iter, n); 58 | FOREACH_OUTEDGE_ITER(v_idx, G, iter) { 59 | node_t v = LL_ITER_OUT_NEXT_NODE(G, iter, v_idx); 60 | printf(" %7u", (unsigned) v); 61 | //printf("[%07x]", (unsigned long) v_idx); 62 | } 63 | printf(" [Degree: %d]\n", (int) G.out_degree(n)); 64 | } 65 | 66 | 67 | template 68 | void print_exp_adj_out_sorted(EXP_GRAPH& G, node_t n) { 69 | if (!G.node_exists(n)) { 70 | printf(" %7s\n", "--"); 71 | return; 72 | } 73 | ll_edge_iterator iter; 74 | G.out_iter_begin(iter, n); 75 | std::vector nodes; 76 | FOREACH_OUTEDGE_ITER(v_idx, G, iter) { 77 | node_t v = LL_ITER_OUT_NEXT_NODE(G, iter, v_idx); 78 | nodes.push_back(v); 79 | } 80 | std::sort(nodes.begin(), nodes.end()); 81 | for (size_t i = 0; i < nodes.size(); i++) 82 | printf(" %7u", (unsigned) nodes[i]); 83 | printf(" [Degree: %d]\n", (int) G.out_degree(n)); 84 | } 85 | 86 | 87 | template 88 | void print_exp_adj_in(EXP_GRAPH& G, node_t n) { 89 | if (!G.node_exists(n)) { 90 | printf(" %7s\n", "--"); 91 | return; 92 | } 93 | ll_edge_iterator iter; 94 | G.inm_iter_begin(iter, n); 95 | FOREACH_INNODE_ITER(v, G, iter) { 96 | printf(" %7u", (unsigned) v); 97 | } 98 | printf(" [Degree: %d]\n", (int) G.in_degree(n)); 99 | } 100 | 101 | 102 | template 103 | void print_exp_adj_within_level(MLCSR& g, node_t n, int l) { 104 | if (!g.node_exists(n)) { 105 | printf(" %7s\n", "--"); 106 | return; 107 | } 108 | ll_edge_iterator iter; 109 | g.iter_begin_within_level(iter, n, l); 110 | FOREACH_ITER_WITHIN_LEVEL(v_idx, g, iter) { 111 | node_t v = LL_ITER_OUT_NEXT_NODE(g, iter, v_idx); 112 | printf(" %7u", (unsigned) v); 113 | } 114 | if (g.max_nodes(l) > n) { 115 | printf(" [Degree: %d]\n", (int) g.degree(n, l)); 116 | } 117 | else { 118 | printf("\n"); 119 | } 120 | } 121 | 122 | 123 | template 124 | void print_exp_graph_out(EXP_GRAPH& G) { 125 | 126 | for (node_t n = 0; n < G.max_nodes(); n++) { 127 | if (!G.node_exists(n)) continue; 128 | printf(" %7ld:", n); 129 | print_exp_adj_out(G, n); 130 | } 131 | } 132 | 133 | 134 | template 135 | void print_exp_graph_out_sorted(EXP_GRAPH& G) { 136 | 137 | for (node_t n = 0; n < G.max_nodes(); n++) { 138 | if (!G.node_exists(n) || G.out_degree(n) == 0) continue; 139 | printf(" %7ld:", n); 140 | print_exp_adj_out_sorted(G, n); 141 | } 142 | } 143 | 144 | #endif 145 | -------------------------------------------------------------------------------- /benchmark/tools/property_stats.h: -------------------------------------------------------------------------------- 1 | /* 2 | * property_stats.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_GENERATED_CPP_PROPERTY_STATS_H 38 | #define LL_GENERATED_CPP_PROPERTY_STATS_H 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | #include "benchmarks/benchmark.h" 50 | 51 | 52 | /** 53 | * Tool: Edge property stats 54 | */ 55 | template 56 | class ll_t_edge_property_stats : public ll_benchmark { 57 | 58 | value_t _min; 59 | value_t _max; 60 | value_t _sum; 61 | double _mean; 62 | size_t _count; 63 | 64 | ll_mlcsr_edge_property* _p; 65 | 66 | 67 | public: 68 | 69 | /** 70 | * Create the tool 71 | * 72 | * @param graph the graph 73 | * @param name the weight property name 74 | */ 75 | ll_t_edge_property_stats(const char* name) 76 | : ll_benchmark("Edge Property Stats") { 77 | 78 | this->create_auto_property(_p, name); 79 | } 80 | 81 | 82 | /** 83 | * Destroy the tool 84 | */ 85 | virtual ~ll_t_edge_property_stats(void) { 86 | } 87 | 88 | 89 | /** 90 | * Run the benchmark 91 | * 92 | * @return the numerical result, if applicable 93 | */ 94 | virtual double run(void) { 95 | 96 | assert(sizeof(value_t) >= 4); 97 | 98 | Graph& G = *this->_graph; 99 | ll_mlcsr_edge_property& p = *_p; 100 | 101 | _min = 0; 102 | _max = 0; 103 | _sum = 0; 104 | _mean = 0; 105 | _count = 0; 106 | 107 | bool first = true; 108 | 109 | //#pragma omp parallel for schedule(dynamic,4096) 110 | for (node_t n = 0; n < G.max_nodes(); n++) { 111 | ll_edge_iterator iter; 112 | G.out_iter_begin(iter, n); 113 | for (edge_t e = G.out_iter_next(iter); 114 | e != LL_NIL_EDGE; 115 | e = G.out_iter_next(iter)) { 116 | 117 | value_t v = p[e]; 118 | 119 | if (first) { 120 | _min = v; 121 | _max = v; 122 | first = false; 123 | } 124 | else { 125 | if (v < _min) _min = v; 126 | if (v > _max) _max = v; 127 | } 128 | 129 | _sum += v; 130 | _count++; 131 | 132 | //if (v > 1) LL_D_PRINT("%lx: %ld --> %ld [w=%d]\n", 133 | //e, n, iter.last_node, (int) v); 134 | } 135 | } 136 | 137 | _mean = _sum / (double) _count; 138 | 139 | return 0; 140 | } 141 | 142 | 143 | /** 144 | * Finalize the benchmark 145 | * 146 | * @return the updated numerical result, if applicable 147 | */ 148 | virtual double finalize(void) { 149 | return _mean; 150 | } 151 | 152 | 153 | /** 154 | * Print the results 155 | * 156 | * @param f the output file 157 | */ 158 | virtual void print_results(FILE* f) { 159 | 160 | bool floating = ll_is_type_floating_point(_p->type()); 161 | 162 | if (floating) { 163 | fprintf(f, "Min : %lf\n", (double) _min); 164 | fprintf(f, "Max : %lf\n", (double) _max); 165 | fprintf(f, "Sum : %lf\n", (double) _sum); 166 | } 167 | else { 168 | fprintf(f, "Min : %ld\n", (long) _min); 169 | fprintf(f, "Max : %ld\n", (long) _max); 170 | fprintf(f, "Sum : %ld\n", (long) _sum); 171 | } 172 | fprintf(f, "Count: %lu\n", _count); 173 | fprintf(f, "Mean : %lf\n", _mean); 174 | } 175 | }; 176 | 177 | #endif 178 | 179 | -------------------------------------------------------------------------------- /examples/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # LLAMA Examples 3 | # 4 | 5 | CC := g++ 6 | CFLAGS := -Wall -fopenmp -std=c++0x -DLL_PERSISTENCE -I. -I../llama/include 7 | LFLAGS := 8 | 9 | ifeq ($(VIMRUNTIME),) 10 | ifneq ($(wildcard /usr/bin/color-g++),) 11 | CC := color-g++ 12 | endif 13 | endif 14 | 15 | CFLAGS_D := -ggdb -O0 -D_DEBUG ${CFLAGS} 16 | CFLAGS := -g -O3 -DNDEBUG ${CFLAGS} 17 | 18 | DIR := ../bin 19 | SOURCES := $(wildcard *.cc) 20 | TARGETS := $(patsubst %.cc,$(DIR)/%,$(SOURCES)) 21 | 22 | .PHONY: all clean 23 | 24 | all: $(TARGETS) 25 | 26 | clean: 27 | rm -f $(TARGETS) 28 | 29 | $(DIR)/%: %.cc 30 | @echo CC $@ >&2 31 | $(CC) $(CFLAGS) -o $@ $< $(LFLAGS) 32 | 33 | -------------------------------------------------------------------------------- /examples/llama-pagerank.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * llama-pagerank.cc 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | /** 37 | * LLAMA Example: PageRank using persistent storage (LL_PERSISTENCE) 38 | */ 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | 46 | #include 47 | 48 | 49 | /** 50 | * Run PageRank 51 | * 52 | * @param G the graph 53 | * @param damping the damping factor 54 | * @param threshold the threshold 55 | * @param max_iterations the maximum number of iterations 56 | * @param verbose true for verbose 57 | * @return the PageRank of each node 58 | */ 59 | double* pagerank(ll_mlcsr_ro_graph& G, double damping, double threshold, 60 | size_t max_iterations, bool verbose) { 61 | 62 | double* pr = (double*) calloc(G.max_nodes(), sizeof(double)); 63 | double* pr_next = (double*) calloc(G.max_nodes(), sizeof(double)); 64 | 65 | double N = G.max_nodes(); 66 | ll_foreach_node_omp(n, G) pr[n] = 1.0 / N; 67 | 68 | double diff = 0; 69 | size_t iteration = 0; 70 | 71 | do { 72 | diff = 0; 73 | 74 | #pragma omp parallel 75 | { 76 | double diff_prv = 0; 77 | 78 | #pragma omp for nowait schedule(dynamic,4096) 79 | ll_foreach_node(n, G) { 80 | 81 | double t = 0; 82 | ll_foreach_in(w, G, n) { 83 | t += pr[w] / (double) G.out_degree(w); 84 | } 85 | 86 | double val = (1 - damping) / N + damping * t; 87 | pr_next[n] = val; 88 | 89 | diff_prv += std::abs(val - pr[n]); 90 | } 91 | 92 | ATOMIC_ADD(&diff, diff_prv); 93 | } 94 | 95 | ll_foreach_node_omp(n, G) pr[n] = pr_next[n]; 96 | iteration++; 97 | 98 | if (verbose) { 99 | fprintf(stderr, "Iteration %lu: Diff = %lf\n", iteration, 100 | diff); 101 | } 102 | } 103 | while ((diff > threshold) && (iteration < max_iterations)); 104 | 105 | free(pr_next); 106 | return pr; 107 | } 108 | 109 | 110 | //==========================================================================// 111 | // The Command-Line Arguments // 112 | //==========================================================================// 113 | 114 | static const char* SHORT_OPTIONS = "d:ht:v"; 115 | 116 | static struct option LONG_OPTIONS[] = 117 | { 118 | {"database" , required_argument, 0, 'd'}, 119 | {"help" , no_argument, 0, 'h'}, 120 | {"threads" , required_argument, 0, 't'}, 121 | {"verbose" , no_argument, 0, 'v'}, 122 | {0, 0, 0, 0} 123 | }; 124 | 125 | 126 | /** 127 | * Print the usage information 128 | * 129 | * @param arg0 the first element in the argv array 130 | */ 131 | static void usage(const char* arg0) { 132 | 133 | char* s = strdup(arg0); 134 | char* p = basename(s); 135 | fprintf(stderr, "Usage: %s [OPTIONS]\n\n", p); 136 | free(s); 137 | 138 | fprintf(stderr, "Options:\n"); 139 | fprintf(stderr, " -d, --database DIR Set the database directory\n"); 140 | fprintf(stderr, " -h, --help Show this usage information and exit\n"); 141 | fprintf(stderr, " -t, --threads N Set the number of threads\n"); 142 | fprintf(stderr, " -v, --verbose Enable verbose output\n"); 143 | } 144 | 145 | 146 | //==========================================================================// 147 | // Utilities // 148 | //==========================================================================// 149 | 150 | struct node_and_data { 151 | node_t node; 152 | double data; 153 | }; 154 | 155 | 156 | /** 157 | * Compare by the node data 158 | */ 159 | int cmp_by_data_desc(const void* a, const void* b) { 160 | 161 | const node_and_data* A = (const node_and_data*) a; 162 | const node_and_data* B = (const node_and_data*) b; 163 | 164 | if (A->data > B->data) return -1; 165 | if (A->data < B->data) return 1; 166 | return (int) A->node - (int) B->node; 167 | } 168 | 169 | 170 | //==========================================================================// 171 | // The Main Function // 172 | //==========================================================================// 173 | 174 | /** 175 | * The main function 176 | */ 177 | int main(int argc, char** argv) 178 | { 179 | char* database_directory = NULL; 180 | bool verbose = false; 181 | int num_threads = -1; 182 | 183 | double damping = 0.85; 184 | double threshold = 0.000001; 185 | size_t max_iterations = 10; 186 | 187 | 188 | // Pase the command-line arguments 189 | 190 | int option_index = 0; 191 | while (true) { 192 | int c = getopt_long(argc, argv, SHORT_OPTIONS, LONG_OPTIONS, &option_index); 193 | 194 | if (c == -1) break; 195 | 196 | switch (c) { 197 | 198 | case 'd': 199 | database_directory = optarg; 200 | break; 201 | 202 | case 'h': 203 | usage(argv[0]); 204 | return 0; 205 | 206 | case 't': 207 | num_threads = atoi(optarg); 208 | break; 209 | 210 | case 'v': 211 | verbose = true; 212 | break; 213 | 214 | case '?': 215 | case ':': 216 | return 1; 217 | 218 | default: 219 | abort(); 220 | } 221 | } 222 | 223 | if (optind != argc) { 224 | fprintf(stderr, "Error: Too many command line arguments\n"); 225 | return 1; 226 | } 227 | 228 | 229 | // Open the database 230 | 231 | ll_database database(database_directory); 232 | if (num_threads > 0) database.set_num_threads(num_threads); 233 | ll_writable_graph& graph = *database.graph(); 234 | 235 | if (!graph.ro_graph().has_reverse_edges()) { 236 | fprintf(stderr, "Error: The graph does not have reverse edges\n"); 237 | return 1; 238 | } 239 | 240 | 241 | // Run the computation 242 | 243 | double ts = ll_get_time_ms(); 244 | double* pr = pagerank(graph.ro_graph(), damping, threshold, 245 | max_iterations, verbose); 246 | double t = ll_get_time_ms() - ts; 247 | 248 | if (verbose) { 249 | fprintf(stderr, "Finished in %3.2lf seconds.\n\n", t/1000.0); 250 | } 251 | 252 | 253 | // Print the top nodes 254 | 255 | size_t n = graph.max_nodes(); 256 | node_and_data* a = (node_and_data*) malloc(sizeof(node_and_data) * n); 257 | for (size_t i = 0; i < n; i++) { 258 | a[i].node = i; 259 | a[i].data = pr[i]; 260 | } 261 | 262 | qsort(a, n, sizeof(node_and_data), cmp_by_data_desc); 263 | 264 | size_t max = 10; 265 | if (max > n) max = n; 266 | 267 | printf("\tNode\tPageRank\n"); 268 | for (size_t i = 0; i < max; i++) { 269 | printf("%lu\t%lu\t%0.6lf\n", i+1, a[i].node, a[i].data); 270 | } 271 | 272 | free(a); 273 | free(pr); 274 | 275 | return 0; 276 | } 277 | -------------------------------------------------------------------------------- /llama/include/llama.h: -------------------------------------------------------------------------------- 1 | /* 2 | * llama.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LLAMA_H_ 38 | #define LLAMA_H_ 39 | 40 | #include "llama/ll_common.h" 41 | #include "llama/ll_lock.h" 42 | #include "llama/ll_mem_helper.h" 43 | #include "llama/ll_utils.h" 44 | #include "llama/ll_config.h" 45 | #include "llama/ll_slcsr.h" 46 | #include "llama/ll_mlcsr_graph.h" 47 | #include "llama/ll_writable_graph.h" 48 | #include "llama/ll_database.h" 49 | 50 | #ifdef LL_PERSISTENCE 51 | #include "llama/ll_persistent_storage.h" 52 | #endif 53 | 54 | #include "llama/ll_streaming.h" 55 | #include "llama/loaders/ll_loaders.h" 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /llama/include/llama/ll_config.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ll_config.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_CONFIG_H_ 38 | #define LL_CONFIG_H_ 39 | 40 | #include 41 | #include 42 | #include 43 | 44 | 45 | /* 46 | * Loader config constants 47 | */ 48 | 49 | #define LL_L_DIRECTED 0 50 | #define LL_L_UNDIRECTED_DOUBLE 1 51 | #define LL_L_UNDIRECTED_ORDERED 2 52 | 53 | 54 | /* 55 | * Features for checking their support 56 | */ 57 | 58 | #define LL_L_FEATURE(feature) (std::string(#feature)) 59 | 60 | typedef std::vector feature_vector_t; 61 | 62 | inline feature_vector_t& operator<< (feature_vector_t& v, 63 | const std::string& value) { 64 | v.push_back(value); 65 | return v; 66 | } 67 | 68 | 69 | 70 | /** 71 | * The loader configuration 72 | */ 73 | class ll_loader_config { 74 | 75 | public: 76 | 77 | /// Graph directionality (one of the LL_L_* constants) 78 | int lc_direction; 79 | 80 | /// Whether to have the reverse edges 81 | bool lc_reverse_edges; 82 | 83 | /// Whether to have the out-in and in-out edge ID maps 84 | bool lc_reverse_maps; 85 | 86 | /// Whether to deduplicate the edges on ingest 87 | bool lc_deduplicate; 88 | 89 | /// Whether to ignore the properties (and load only the graph structure) 90 | bool lc_no_properties; 91 | 92 | /// The temporary directories 93 | std::vector lc_tmp_dirs; 94 | 95 | /// Whether to print progress while loading 96 | bool lc_print_progress; 97 | 98 | /// The buffer size in bytes for external sort (0 = auto-configure) 99 | size_t lc_xs_buffer_size; 100 | 101 | /// The max number of edges to load 102 | size_t lc_max_edges; 103 | 104 | /// The partial load - part number (1-based, not 0-based) 105 | size_t lc_partial_load_part; 106 | 107 | /// The partial load - the total number of parts 108 | size_t lc_partial_load_num_parts; 109 | 110 | 111 | public: 112 | 113 | /** 114 | * Create an instance of ll_loader_config with the default configuration 115 | */ 116 | ll_loader_config() { 117 | 118 | lc_direction = LL_L_DIRECTED; 119 | lc_deduplicate = false; 120 | lc_reverse_edges = false; 121 | lc_reverse_maps = false; 122 | lc_no_properties = false; 123 | 124 | lc_tmp_dirs.clear(); 125 | lc_print_progress = false; 126 | lc_xs_buffer_size = 0; 127 | 128 | lc_max_edges = 0; 129 | lc_partial_load_part = 0; 130 | lc_partial_load_num_parts = 0; 131 | } 132 | 133 | 134 | /** 135 | * Assert (or check) that all enabled features are indeed supported by the 136 | * loader 137 | * 138 | * @param direct true if the loader is a direct loader (false = incremental) 139 | * @param error true to throw an error, false to just return the result 140 | * @param features the vector of supported features (use LL_L_FEATURE()) 141 | * @return true if okay, or false if not (for error == false) 142 | */ 143 | bool assert_features(bool direct, bool error, 144 | const feature_vector_t& features) const { 145 | 146 | feature_vector_t enabled_features; 147 | std::vector enabled_feature_names; 148 | 149 | # define FEATURE(x) \ 150 | if ((ssize_t) (x) != 0) { \ 151 | enabled_features << LL_L_FEATURE(x); \ 152 | enabled_feature_names.push_back(std::string(#x)); \ 153 | } 154 | 155 | FEATURE(lc_direction); 156 | FEATURE(lc_deduplicate); 157 | FEATURE(lc_no_properties); 158 | 159 | if (direct) FEATURE(lc_reverse_edges); 160 | if (direct) FEATURE(lc_reverse_maps); 161 | 162 | FEATURE(lc_max_edges); 163 | FEATURE(lc_partial_load_part); 164 | FEATURE(lc_partial_load_num_parts); 165 | 166 | # undef FEATURE 167 | 168 | for (size_t i = 0; i < enabled_features.size(); i++) { 169 | bool ok = false; 170 | for (size_t k = 0; k < features.size(); k++) { 171 | if (enabled_features[i] == features[k]) { 172 | ok = true; 173 | break; 174 | } 175 | } 176 | if (!ok) { 177 | if (error) { 178 | LL_E_PRINT("Feature not supported: %s\n", 179 | enabled_feature_names[i].c_str()); 180 | abort(); 181 | } 182 | else { 183 | return false; 184 | } 185 | } 186 | } 187 | 188 | if (lc_partial_load_num_parts > 0) { 189 | if (lc_partial_load_part <= 0 190 | || lc_partial_load_part > lc_partial_load_num_parts) { 191 | LL_E_PRINT("The partial load part ID is out of bounds\n"); 192 | abort(); 193 | } 194 | } 195 | else { 196 | if (lc_partial_load_part != 0) { 197 | LL_E_PRINT("Partial load part ID without the number of parts\n"); 198 | abort(); 199 | } 200 | } 201 | 202 | return true; 203 | } 204 | }; 205 | 206 | #endif 207 | 208 | -------------------------------------------------------------------------------- /llama/include/llama/ll_database.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ll_database.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_DATABASE_H_ 38 | #define LL_DATABASE_H_ 39 | 40 | #include 41 | #include 42 | 43 | #include "llama/ll_common.h" 44 | #include "llama/ll_config.h" 45 | #include "llama/ll_persistent_storage.h" 46 | 47 | 48 | /** 49 | * The database 50 | */ 51 | class ll_database { 52 | 53 | 54 | public: 55 | 56 | /** 57 | * Create a new database instance, or load it if it exists 58 | * 59 | * @param dir the database directory (if it is a persistent database) 60 | */ 61 | ll_database(const char* dir = NULL) { 62 | 63 | omp_set_num_threads(omp_get_max_threads()); 64 | 65 | _dir = IFE_LL_PERSISTENCE(dir == NULL ? "db" : dir, ""); 66 | IF_LL_PERSISTENCE(_storage = new ll_persistent_storage(_dir.c_str())); 67 | 68 | _graph = new ll_writable_graph(this, IF_LL_PERSISTENCE(_storage,) 69 | 80 * 1000000 /* XXX */); 70 | } 71 | 72 | 73 | /** 74 | * Destroy the in-memory representation of the database instance 75 | */ 76 | virtual ~ll_database() { 77 | 78 | delete _graph; 79 | IF_LL_PERSISTENCE(delete _storage); 80 | } 81 | 82 | 83 | /** 84 | * Set the number of OpenMP threads 85 | * 86 | * @param n the number of threads 87 | */ 88 | void set_num_threads(int n) { 89 | // XXX This really belongs to some global runtime 90 | omp_set_num_threads(n); 91 | } 92 | 93 | 94 | /** 95 | * Get the database directory, if this is a persistent database 96 | * 97 | * @return the database directory, or NULL if not persistent 98 | */ 99 | inline const char* directory() { 100 | return IFE_LL_PERSISTENCE(_dir.c_str(), NULL); 101 | } 102 | 103 | 104 | #ifdef LL_PERSISTENCE 105 | 106 | /** 107 | * Get the persistent storage 108 | * 109 | * @return the persistent storage 110 | */ 111 | inline ll_persistent_storage* storage() { 112 | return _storage; 113 | } 114 | #endif 115 | 116 | 117 | /** 118 | * Get the graph 119 | * 120 | * @return the graph 121 | */ 122 | inline ll_writable_graph* graph() { 123 | return _graph; 124 | } 125 | 126 | 127 | /** 128 | * Get the loader config 129 | * 130 | * @return the loader config 131 | */ 132 | inline ll_loader_config* loader_config() { 133 | return &_loader_config; 134 | } 135 | 136 | 137 | private: 138 | 139 | /// The graph 140 | ll_writable_graph* _graph; 141 | 142 | /// The persistent storage 143 | IF_LL_PERSISTENCE(ll_persistent_storage* _storage); 144 | 145 | /// The database directory 146 | std::string _dir; 147 | 148 | /// The loader configuration 149 | ll_loader_config _loader_config; 150 | }; 151 | 152 | #endif 153 | 154 | -------------------------------------------------------------------------------- /llama/include/llama/ll_dfs_template.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ll_dfs_template.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | /* 37 | * This file was adapted from Green-Marl, which includes the following notice: 38 | * 39 | * Copyright (c) 2011-2012 Stanford University, unless otherwise specified. 40 | * All rights reserved. 41 | * 42 | * This software was developed by the Pervasive Parallelism Laboratory of 43 | * Stanford University, California, USA. 44 | * 45 | * Permission to use, copy, modify, and distribute this software in source or 46 | * binary form for any purpose with or without fee is hereby granted, provided 47 | * that the following conditions are met: 48 | * 1. Redistributions of source code must retain the above copyright 49 | * notice, this list of conditions and the following disclaimer. 50 | * 2. Redistributions in binary form must reproduce the above copyright 51 | * notice, this list of conditions and the following disclaimer in the 52 | * documentation and/or other materials provided with the distribution. 53 | * 3. Neither the name of Stanford University nor the names of its contributors 54 | * may be used to endorse or promote products derived from this software 55 | * without specific prior written permission. 56 | * 57 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 58 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 59 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 60 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 61 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 62 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 63 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 64 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 65 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 66 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 67 | * SUCH DAMAGE. 68 | */ 69 | 70 | 71 | #ifndef LL_DFS_TEMPLATE_H 72 | #define LL_DFS_TEMPLATE_H 73 | 74 | #include 75 | #include 76 | #include 77 | #include 78 | #include 79 | 80 | 81 | template 82 | class ll_dfs_template 83 | { 84 | protected: 85 | virtual void visit_pre(node_t t)=0; 86 | virtual void visit_post(node_t t)=0; 87 | virtual bool check_navigator(node_t t, edge_t idx)=0; 88 | 89 | public: 90 | ll_dfs_template(Graph& _G) : 91 | G(_G) { 92 | visited_bitmap = NULL; // bitmap 93 | } 94 | 95 | virtual ~ll_dfs_template() { 96 | delete visited_bitmap; 97 | } 98 | 99 | void prepare(node_t root_node) { 100 | root = root_node; 101 | cnt = 0; 102 | visited_small.clear(); 103 | 104 | is_small = true; 105 | iter.node = INVALID_NODE; 106 | THRESHOLD_LARGE = std::max((int)(G.max_nodes()*0.1), 4096); 107 | } 108 | 109 | void do_dfs() { 110 | enter_node(root); 111 | main_loop(); 112 | } 113 | 114 | private: 115 | void prepare_large() { 116 | delete[] visited_bitmap; 117 | 118 | visited_bitmap = new unsigned char[(G.max_nodes() + 7) / 8]; 119 | 120 | #pragma omp parallel for schedule(dynamic,16384) 121 | for (int i = 0; i < (G.max_nodes() + 7) / 8; i++) 122 | visited_bitmap[i] = 0; 123 | 124 | std::unordered_set::iterator I; 125 | for (I = visited_small.begin(); I != visited_small.end(); I++) { 126 | node_t u = *I; 127 | _ll_set_bit(visited_bitmap, u); 128 | } 129 | is_small = false; 130 | stack.reserve(G.max_nodes()); 131 | } 132 | 133 | void enter_node(node_t n) { 134 | // push current node 135 | stack.push_back(iter); 136 | 137 | if (use_reverse_edge) 138 | G.in_iter_begin_fast(iter, n); 139 | else 140 | G.out_iter_begin(iter, n); 141 | 142 | // mark visited 143 | add_visited(n); 144 | cnt++; 145 | if (cnt == THRESHOLD_LARGE) { 146 | prepare_large(); 147 | } 148 | 149 | if (has_pre_visit) visit_pre(n); 150 | } 151 | 152 | void exit_node(node_t n) { 153 | if (has_post_visit) visit_post(n); 154 | iter = stack.back(); 155 | stack.pop_back(); 156 | } 157 | 158 | void main_loop() { 159 | //---------------------------------- 160 | // Repeat until stack is empty 161 | //---------------------------------- 162 | while (iter.node != INVALID_NODE) { 163 | //---------------------------------- 164 | // Every neighbor has been visited 165 | //---------------------------------- 166 | if (iter.edge == LL_NIL_EDGE) { 167 | exit_node(iter.node); 168 | continue; 169 | } 170 | 171 | else { 172 | //---------------------------------- 173 | // check every non-visited neighbor 174 | //---------------------------------- 175 | node_t z; 176 | edge_t e; 177 | if (use_reverse_edge) { 178 | e = G.in_iter_next_fast(iter); 179 | } else { 180 | e = G.out_iter_next(iter); 181 | } 182 | assert(e != LL_NIL_EDGE); 183 | z = iter.last_node; 184 | 185 | if (has_visited(z)) { 186 | continue; 187 | } 188 | if (has_navigator) { 189 | if (check_navigator(z, e) == false) { 190 | continue; 191 | } 192 | } 193 | enter_node(z); 194 | continue; 195 | } 196 | } 197 | } 198 | 199 | void add_visited(node_t n) { 200 | if (is_small) 201 | visited_small.insert(n); 202 | else 203 | _ll_set_bit(visited_bitmap, n); 204 | } 205 | 206 | bool has_visited(node_t n) { 207 | if (is_small) { 208 | return (visited_small.find(n) != visited_small.end()); 209 | } else { 210 | return _ll_get_bit(visited_bitmap, n); 211 | } 212 | } 213 | 214 | protected: 215 | node_t root; 216 | Graph& G; 217 | 218 | // stack implementation 219 | node_t stack_ptr; 220 | std::vector stack; 221 | ll_edge_iterator iter; 222 | 223 | // visited set implementation 224 | node_t cnt; 225 | unsigned char* visited_bitmap; 226 | std::unordered_set visited_small; 227 | bool is_small; 228 | int THRESHOLD_LARGE; 229 | static const node_t INVALID_NODE = -1; 230 | 231 | }; 232 | 233 | #endif 234 | -------------------------------------------------------------------------------- /llama/include/llama/ll_edge_table.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ll_edge_table.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_EDGE_TABLE_H_ 38 | #define LL_EDGE_TABLE_H_ 39 | 40 | #include "llama/ll_common.h" 41 | #include "llama/ll_mlcsr_helpers.h" 42 | 43 | 44 | 45 | //==========================================================================// 46 | // Class: ll_et_array // 47 | //==========================================================================// 48 | 49 | /** 50 | * An array representation of the edge table 51 | * 52 | * @author Peter Macko 53 | */ 54 | template struct ll_et_array { 55 | 56 | /// The values 57 | T _values[1]; /* must be the ONLY member */ 58 | 59 | 60 | /** 61 | * Access an element based on the index 62 | * 63 | * @param index the element 64 | * @return the reference to the element 65 | */ 66 | inline T& operator[] (edge_t index) { 67 | return _values[index]; 68 | } 69 | 70 | 71 | /** 72 | * Access an element based on the index 73 | * 74 | * @param index the element 75 | * @return the reference to the element 76 | */ 77 | inline const T& operator[] (edge_t index) const { 78 | return _values[index]; 79 | } 80 | 81 | 82 | /** 83 | * Access an element based on the node and the edge index 84 | * 85 | * @param node the node 86 | * @param index the edge index 87 | * @return the reference to the element 88 | */ 89 | inline T& edge_value(node_t node, edge_t edge) { 90 | return _values[edge]; 91 | } 92 | 93 | 94 | /** 95 | * Access an element based on the node and the edge index 96 | * 97 | * @param node the node 98 | * @param index the edge index 99 | * @return the reference to the element 100 | */ 101 | inline const T& edge_value(node_t node, edge_t edge) const { 102 | return _values[edge]; 103 | } 104 | 105 | 106 | /** 107 | * Access an element based on the node and the edge index 108 | * 109 | * @param node the node 110 | * @param index the edge index 111 | * @return the pointer to the element 112 | */ 113 | inline T* edge_ptr(node_t node, edge_t edge) { 114 | return &_values[edge]; 115 | } 116 | 117 | 118 | /** 119 | * Access an element based on the node and the edge index 120 | * 121 | * @param node the node 122 | * @param index the edge index 123 | * @return the pointer to the element 124 | */ 125 | inline const T* edge_ptr(node_t node, edge_t edge) const { 126 | return &_values[edge]; 127 | } 128 | 129 | 130 | /** 131 | * Memset 132 | * 133 | * @param start the start 134 | * @param finish the finish (exclusive) 135 | * @param byte the byte 136 | */ 137 | void memset(edge_t start, edge_t finish, int byte) { 138 | ::memset(&_values[start], byte, (finish - start) * sizeof(T)); 139 | } 140 | 141 | 142 | /** 143 | * Copy a range of values 144 | * 145 | * @param to the destination 146 | * @param source the source 147 | * @param start the start 148 | * @param length the length 149 | */ 150 | void copy(edge_t to, const ll_et_array* source, edge_t start, size_t length) { 151 | memcpy(&_values[to], &source->_values[start], sizeof(T) * length); 152 | } 153 | 154 | 155 | /** 156 | * Advise 157 | * 158 | * @param from the start index (inclusive) 159 | * @param to the end index (exclusive) 160 | * @param advice the advice (LL_ADV_*) 161 | */ 162 | void advise(edge_t from, edge_t to, int advice = LL_ADV_WILLNEED) { 163 | 164 | assert(from <= to); 165 | 166 | #ifdef LL_PERSISTENCE 167 | 168 | // Assume that the start of the mapping is page-aligned 169 | 170 | size_t fi = from - from % (4096 / sizeof(T)); 171 | size_t ti = to + (4096 / sizeof(T) - to % (4096 / sizeof(T))); 172 | 173 | madvise(&_values[fi], sizeof(T) * (ti - fi), advice); 174 | #else 175 | 176 | if (advice == LL_ADV_WILLNEED) { 177 | __builtin_prefetch(&_values[from]); 178 | __builtin_prefetch(&_values[from+1]); 179 | } 180 | #endif 181 | } 182 | }; 183 | 184 | 185 | /** 186 | * Create a new instance of ll_et_array 187 | * 188 | * @param capacity the capacity 189 | * @param max_nodes the number of nodes 190 | * @return the new instance 191 | */ 192 | template 193 | ll_et_array* new_ll_et_array(size_t capacity, size_t max_nodes) { 194 | ll_et_array* et = (ll_et_array*) malloc(sizeof(ll_et_array) 195 | + capacity * sizeof(T)); 196 | return et; 197 | } 198 | 199 | 200 | /** 201 | * Destroy an instance of ll_et_array 202 | * 203 | * @param et the edge table 204 | */ 205 | template 206 | void delete_ll_et_array(ll_et_array* et) { 207 | free(et); 208 | } 209 | 210 | 211 | 212 | //==========================================================================// 213 | // Class: ll_et_mmaped_array // 214 | //==========================================================================// 215 | 216 | /** 217 | * An array representation of the edge table that can be mmap-ed 218 | */ 219 | template struct ll_et_mmaped_array : ll_et_array {}; 220 | 221 | 222 | /** 223 | * Create a new instance of ll_et_mmaped_array 224 | * 225 | * @param capacity the capacity 226 | * @param max_nodes the number of nodes 227 | * @return the new instance 228 | */ 229 | template 230 | ll_et_mmaped_array* new_ll_et_mmaped_array(size_t capacity, size_t max_nodes) { 231 | // Nothing to do - this will be taken care of by the persistence manager 232 | return NULL; 233 | } 234 | 235 | 236 | /** 237 | * Destroy an instance of ll_et_mmaped_array 238 | * 239 | * @param et the edge table 240 | */ 241 | template 242 | void delete_ll_et_mmaped_array(ll_et_mmaped_array* et) { 243 | // Nothing to do - this will be unmapped during the persistence destroy 244 | } 245 | 246 | #endif 247 | -------------------------------------------------------------------------------- /llama/include/llama/ll_growable_array.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ll_growable_array.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_GROWABLE_ARRAY_H_ 38 | #define LL_GROWABLE_ARRAY_H_ 39 | 40 | #include "llama/ll_common.h" 41 | 42 | 43 | /** 44 | * Malloc/Free Allocator 45 | */ 46 | template struct ll_malloc_allocator { 47 | inline T operator() (size_t bytes) { return (T) malloc(bytes); } 48 | }; 49 | 50 | 51 | /** 52 | * Malloc/Free Deallocator 53 | */ 54 | template struct ll_free_deallocator { 55 | inline void operator() (T buffer) { free(buffer); } 56 | }; 57 | 58 | 59 | /** 60 | * NOP Deallocator 61 | */ 62 | template struct ll_nop_deallocator { 63 | inline void operator() (T) {} 64 | }; 65 | 66 | 67 | /** 68 | * A growable block array 69 | */ 70 | template , 73 | class block_deallocator=ll_free_deallocator, 74 | bool use_block_deallocator=true> 75 | class ll_growable_array { 76 | 77 | private: 78 | 79 | int _blocks; 80 | int _size; 81 | T** _arrays; 82 | 83 | deallocator _deallocator; 84 | block_allocator _block_allocator; 85 | block_deallocator _block_deallocator; 86 | 87 | ll_spinlock_t _lock; 88 | 89 | 90 | public: 91 | 92 | /** 93 | * Create an empty growable array 94 | * 95 | * @param blocks the number of blocks to preallocate 96 | */ 97 | ll_growable_array(int blocks = 16) { 98 | _blocks = blocks; 99 | _size = 0; 100 | _lock = 0; 101 | _arrays = (T**) _block_allocator(sizeof(T*) * _blocks); 102 | memset(_arrays, 0, sizeof(T*) * _blocks); 103 | _arrays[0] = (T*) _block_allocator(sizeof(T) * (1 << _block_size2)); 104 | } 105 | 106 | 107 | /** 108 | * Destroy the array 109 | */ 110 | inline ~ll_growable_array() { 111 | 112 | if (use_deallocator || use_block_deallocator) { 113 | 114 | for (int i = 0; i < _blocks; i++) { 115 | if (_arrays[i] == NULL) break; 116 | if (use_deallocator) { 117 | for (int j = 0; j < (1 << _block_size2); j++) { 118 | if (_size-- <= 0) break; 119 | _deallocator(_arrays[i][j]); 120 | } 121 | } 122 | if (use_block_deallocator) 123 | _block_deallocator(_arrays[i]); 124 | } 125 | 126 | if (use_block_deallocator) 127 | _block_deallocator(_arrays); 128 | } 129 | } 130 | 131 | 132 | /** 133 | * Get the number of elements in the array 134 | * 135 | * @return the size of the array 136 | */ 137 | inline size_t size() const { 138 | return _size; 139 | } 140 | 141 | 142 | /** 143 | * Clear 144 | */ 145 | void clear() { 146 | 147 | if (use_deallocator) { 148 | for (int i = 0; i < _blocks; i++) { 149 | if (_arrays[i] == NULL) break; 150 | for (int j = 0; j < (1 << _block_size2); j++) { 151 | if (_size-- <= 0) break; 152 | _deallocator(_arrays[i][j]); 153 | } 154 | } 155 | } 156 | 157 | _size = 0; 158 | } 159 | 160 | 161 | /** 162 | * Grow by one and return a pointer to the new unitialized cell 163 | * 164 | * @return a pointer to the new uninitialized cell 165 | */ 166 | T* append(void) { 167 | ll_spinlock_acquire(&_lock); 168 | 169 | T* p = &_arrays[_size >> _block_size2][_size & ((1 << _block_size2) - 1)]; 170 | _size++; 171 | 172 | if ((_size & ((1 << _block_size2) - 1)) == 0) { 173 | int newBlock = _size >> _block_size2; 174 | if (newBlock == _blocks) { 175 | int n = _blocks * 2; 176 | T** a = (T**) _block_allocator(sizeof(T*) * n); 177 | memcpy(a, _arrays, sizeof(T*) * _blocks); 178 | memset(&a[_blocks], 0, sizeof(T*) * (n - _blocks)); 179 | if (use_block_deallocator) 180 | _block_deallocator(_arrays); 181 | _arrays = a; 182 | _blocks = n; 183 | } 184 | if (_arrays[newBlock] == NULL) { 185 | _arrays[newBlock] = (T*) _block_allocator(sizeof(T) * (1 << _block_size2)); 186 | } 187 | } 188 | 189 | ll_spinlock_release(&_lock); 190 | return p; 191 | } 192 | 193 | 194 | /** 195 | * Append a value 196 | * 197 | * @param value the value to append 198 | * @return the appended value 199 | */ 200 | T append(T value) { 201 | *(append()) = value; 202 | return value; 203 | } 204 | 205 | 206 | /** 207 | * Read from the array 208 | * 209 | * @return index the index to read 210 | * @return the element at that position 211 | */ 212 | inline T& operator[] (size_t index) const { 213 | assert(index < (size_t) _size); 214 | return _arrays[index >> _block_size2][index & ((1 << _block_size2) - 1)]; 215 | } 216 | 217 | 218 | /** 219 | * Get the number of blocks in the array 220 | * 221 | * @return the number of blocks 222 | */ 223 | inline size_t block_count() const { 224 | size_t b = _size >> _block_size2; 225 | if ((_size & ((1 << _block_size2) - 1)) != 0) b++; 226 | return b; 227 | } 228 | 229 | 230 | /** 231 | * Get a block 232 | * 233 | * @param b the block number 234 | * @return the block pointer 235 | */ 236 | inline T const * block(size_t b) const { 237 | assert(b <= (size_t) (_size >> _block_size2) && b <= (size_t) _blocks); 238 | return _arrays[b]; 239 | } 240 | 241 | 242 | /** 243 | * Get the block size 244 | * 245 | * @param b the block 246 | * @return the number of elements in the block 247 | */ 248 | inline size_t block_size(size_t b) const { 249 | assert(b <= (size_t) (_size >> _block_size2) && b <= (size_t) _blocks); 250 | return b < (size_t) (_size >> _block_size2) 251 | ? 1 << _block_size2 252 | : (_size & ((1 << _block_size2) - 1)); 253 | } 254 | }; 255 | 256 | #endif 257 | -------------------------------------------------------------------------------- /llama/include/llama/ll_lock.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ll_lock.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef _LL_LOCK_H 38 | #define _LL_LOCK_H 39 | 40 | #include 41 | 42 | /** 43 | * Spinlock 44 | */ 45 | typedef volatile int32_t ll_spinlock_t; 46 | 47 | 48 | /** 49 | * Try to acquire a spinlock 50 | * 51 | * @param ptr the spinlock 52 | * @return true if acquired 53 | */ 54 | inline bool ll_spinlock_try_acquire(ll_spinlock_t* ptr) { 55 | int ret = __sync_lock_test_and_set(ptr, 1); 56 | return ret == 0; // 0 means success 57 | } 58 | 59 | 60 | /** 61 | * Acquire a spinlock 62 | * 63 | * @param ptr the spinlock 64 | */ 65 | inline void ll_spinlock_acquire(ll_spinlock_t* ptr) { 66 | while (__sync_lock_test_and_set(ptr, 1)) { 67 | while (*ptr == 1) { 68 | asm volatile ("pause" ::: "memory"); 69 | } 70 | } 71 | } 72 | 73 | 74 | /** 75 | * Release a spinlock 76 | * 77 | * @param ptr the spinlock 78 | */ 79 | inline void ll_spinlock_release(ll_spinlock_t* ptr) { 80 | __sync_synchronize(); 81 | *ptr = 0; 82 | } 83 | 84 | 85 | #define LL_CACHELINE 8 86 | 87 | 88 | /** 89 | * Spinlock table 90 | */ 91 | template // Must be a power of 2 92 | class ll_spinlock_table_ext { 93 | 94 | ll_spinlock_t ll_spinlock_tab[size * LL_CACHELINE]; 95 | 96 | 97 | public: 98 | 99 | /** 100 | * Initialize 101 | */ 102 | ll_spinlock_table_ext() { 103 | for (int i = 0; i < size * LL_CACHELINE; i++) 104 | ll_spinlock_tab[i] = 0; 105 | } 106 | 107 | 108 | /** 109 | * Acquire 110 | * 111 | * @param x the value 112 | */ 113 | template 114 | void acquire_for(T x) { 115 | uint32_t entry_idx = (uint32_t)(x & (size - 1)); 116 | uint32_t tab_idx = entry_idx * LL_CACHELINE; 117 | ll_spinlock_acquire(&ll_spinlock_tab[tab_idx]); 118 | } 119 | 120 | 121 | /** 122 | * Release 123 | * 124 | * @param x the value 125 | */ 126 | template 127 | void release_for(T x) { 128 | uint32_t entry_idx = (uint32_t)(x & (size - 1)); 129 | uint32_t tab_idx = entry_idx * LL_CACHELINE; 130 | ll_spinlock_release(&ll_spinlock_tab[tab_idx]); 131 | } 132 | }; 133 | 134 | 135 | /** 136 | * The default spinlock table 137 | */ 138 | typedef ll_spinlock_table_ext<1024> ll_spinlock_table; 139 | 140 | 141 | #endif 142 | -------------------------------------------------------------------------------- /llama/include/llama/ll_mlcsr_helpers.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ll_mlcsr_helpers.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_MLCSR_HELPERS_H_ 38 | #define LL_MLCSR_HELPERS_H_ 39 | 40 | #include 41 | #include 42 | 43 | #include "llama/ll_writable_elements.h" 44 | 45 | 46 | 47 | //==========================================================================// 48 | // Helpers // 49 | //==========================================================================// 50 | 51 | /* 52 | * Sizes in bits 53 | */ 54 | 55 | #ifdef LL_NODE32 56 | # define LL_DATA_TYPE unsigned 57 | # define LL_ONE 1u 58 | # define LL_BITS_TOTAL 32 59 | # define LL_BITS_LEVEL 4 60 | #else 61 | # define LL_DATA_TYPE unsigned long long 62 | # define LL_ONE 1ull 63 | # define LL_BITS_TOTAL 64 64 | # define LL_BITS_LEVEL 26 65 | #endif 66 | 67 | #define LL_MASK(bits) ((LL_ONE << (bits)) - 1) 68 | #define LL_BITS_INDEX (LL_BITS_TOTAL - LL_BITS_LEVEL) 69 | 70 | 71 | /* 72 | * Edges 73 | */ 74 | 75 | #define LL_MAX_LEVEL (LL_MASK(LL_BITS_LEVEL) - 2) 76 | 77 | #define LL_EDGE_CREATE(level, index) ((index) | (((LL_DATA_TYPE) level) << LL_BITS_INDEX)) 78 | #define LL_EDGE_LEVEL(x) (((LL_DATA_TYPE) (x)) >> LL_BITS_INDEX) 79 | #define LL_EDGE_INDEX(x) ((x) & LL_MASK(LL_BITS_INDEX)) 80 | #define LL_EDGE_NEXT_INDEX(x) ((x) + LL_ONE) 81 | 82 | #define LL_WRITABLE_LEVEL (LL_MAX_LEVEL + 1) 83 | #define LL_EDGE_IS_WRITABLE(x) (LL_EDGE_LEVEL(x) == LL_WRITABLE_LEVEL) 84 | 85 | 86 | /* 87 | * Values 88 | */ 89 | 90 | #ifdef LL_DELETIONS 91 | # ifdef LL_NODE32 92 | # error "LL_NODE32 does not support LL_DELETIONS." 93 | # endif 94 | # define LL_CHECK_EXT_DELETION (LL_MAX_LEVEL + 1) 95 | 96 | // max_visible_level is exclusive 97 | 98 | # define LL_VALUE_CREATE_EXT(payload, \ 99 | max_visible_level) (((LL_DATA_TYPE) (payload)) \ 100 | | ((LL_DATA_TYPE) (max_visible_level) \ 101 | << (LL_BITS_TOTAL - LL_BITS_LEVEL))) 102 | # define LL_VALUE_CREATE(payload) LL_VALUE_CREATE_EXT(payload, LL_MAX_LEVEL + 2) 103 | 104 | # define LL_VALUE_PAYLOAD(x) ((x) & LL_MASK(LL_BITS_TOTAL - LL_BITS_LEVEL)) 105 | # define LL_VALUE_MAX_LEVEL(x) ((((LL_DATA_TYPE) x) >> (LL_BITS_TOTAL - LL_BITS_LEVEL)) \ 106 | & LL_MASK(LL_BITS_LEVEL)) 107 | # define LL_VALUE_IS_DELETED(x, level) (LL_VALUE_MAX_LEVEL(x) <= (level)) 108 | 109 | #else /* ! LL_DELETIONS */ 110 | 111 | # define LL_VALUE_CREATE(payload) ((LL_DATA_TYPE) (payload)) 112 | # define LL_VALUE_PAYLOAD(x) (x) 113 | 114 | #endif 115 | 116 | inline bool ll_payload_compare(long i, long j) { 117 | return LL_VALUE_PAYLOAD(i) < LL_VALUE_PAYLOAD(j); 118 | } 119 | 120 | inline bool ll_payload_pair_compare(const std::pair& i, 121 | const std::pair& j) { 122 | 123 | long ip = LL_VALUE_PAYLOAD(i.first); 124 | long jp = LL_VALUE_PAYLOAD(j.first); 125 | 126 | if (ip != jp) 127 | return ip < jp; 128 | else 129 | return i.second < j.second; 130 | } 131 | 132 | 133 | 134 | //==========================================================================// 135 | // ll_mlcsr_core__begin_t // 136 | //==========================================================================// 137 | 138 | /** 139 | * An element in the vertex table 140 | */ 141 | struct ll_mlcsr_core__begin_t { 142 | 143 | edge_t adj_list_start; 144 | length_t level_length; 145 | 146 | #ifdef LL_PRECOMPUTED_DEGREE 147 | degree_t degree; 148 | #endif 149 | }; 150 | 151 | 152 | /** 153 | * Compare two vertex table elements 154 | * 155 | * @param a the first element 156 | * @param b the second element 157 | * @return true if they are equal 158 | */ 159 | inline bool operator== (const ll_mlcsr_core__begin_t& a, 160 | const ll_mlcsr_core__begin_t& b) { 161 | 162 | #ifdef LL_PRECOMPUTED_DEGREE 163 | return a.adj_list_start == b.adj_list_start 164 | && a.level_length == b.level_length 165 | && a.degree == b.degree; 166 | #else 167 | return a.adj_list_start == b.adj_list_start 168 | && a.level_length == b.level_length; 169 | #endif 170 | } 171 | 172 | 173 | /** 174 | * Compare two vertex table elements 175 | * 176 | * @param a the first element 177 | * @param b the second element 178 | * @return true if they are not equal 179 | */ 180 | inline bool operator!= (const ll_mlcsr_core__begin_t& a, 181 | const ll_mlcsr_core__begin_t& b) { 182 | return !(a == b); 183 | } 184 | 185 | 186 | 187 | //==========================================================================// 188 | // Debugging routines // 189 | //==========================================================================// 190 | 191 | #if 0 192 | 193 | static long __d_maxrss_last = 0; 194 | static double __d_t_last = 0; 195 | 196 | /** 197 | * Print MAXRSS and the delta from last 198 | */ 199 | void __d_maxrss(const char* msg = NULL) { 200 | 201 | struct rusage r; 202 | getrusage(RUSAGE_SELF, &r); 203 | long m = r.ru_maxrss; 204 | 205 | fprintf(stderr, "[DEBUG] MaxRSS=%7.2lf MB, Delta=%7.2lf MB %s\n", 206 | m / 1024.0, (m - __d_maxrss_last) / 1024.0, msg == NULL ? "" : msg); 207 | 208 | __d_maxrss_last = m; 209 | } 210 | 211 | 212 | /** 213 | * Timing 214 | * 215 | * @param msg the message to print, use NULL to reset 216 | */ 217 | double __d_time(const char* msg) { 218 | 219 | struct timeval TV; 220 | gettimeofday(&TV, NULL); 221 | double t = TV.tv_sec*1000 + TV.tv_usec*0.001; 222 | 223 | if (msg != NULL) { 224 | fprintf(stderr, "[DEBUG] %s: %7.2lf ms\n", msg, t - __d_t_last); 225 | } 226 | 227 | gettimeofday(&TV, NULL); 228 | __d_t_last = TV.tv_sec*1000 + TV.tv_usec*0.001; 229 | 230 | return __d_t_last; 231 | } 232 | 233 | #endif 234 | 235 | 236 | 237 | //==========================================================================// 238 | // Destructors // 239 | //==========================================================================// 240 | 241 | template 242 | void destructor64(const uint64_t& value) { 243 | if (value != 0) delete (T*) value; 244 | } 245 | 246 | 247 | 248 | //==========================================================================// 249 | // Interface: ll_mlcsr_external_deletions // 250 | //==========================================================================// 251 | 252 | /** 253 | * The data source for external deletions 254 | */ 255 | class ll_mlcsr_external_deletions { 256 | 257 | public: 258 | 259 | ll_mlcsr_external_deletions() {} 260 | 261 | virtual ~ll_mlcsr_external_deletions() {} 262 | 263 | virtual bool is_edge_deleted(edge_t edge) = 0; 264 | }; 265 | 266 | 267 | 268 | //==========================================================================// 269 | // Other // 270 | //==========================================================================// 271 | 272 | // Writable vertex table 273 | typedef LL_W_VT, 274 | w_node_deallocator_ext> ll_w_vt_vertices_t; 275 | 276 | 277 | #endif 278 | 279 | -------------------------------------------------------------------------------- /llama/include/llama/ll_streaming.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ll_streaming.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_STREAMING_H_ 38 | #define LL_STREAMING_H_ 39 | 40 | #include "llama/ll_common.h" 41 | #include "llama/ll_mlcsr_graph.h" 42 | #include "llama/ll_writable_graph.h" 43 | 44 | #include 45 | 46 | 47 | /** 48 | * The pull-based data source 49 | */ 50 | class ll_data_source { 51 | 52 | public: 53 | 54 | /** 55 | * Create an instance of the data source wrapper 56 | */ 57 | inline ll_data_source() {} 58 | 59 | 60 | /** 61 | * Destroy the data source 62 | */ 63 | virtual ~ll_data_source() {} 64 | 65 | 66 | /** 67 | * Load the next batch of data 68 | * 69 | * @param graph the writable graph 70 | * @param max_edges the maximum number of edges 71 | * @return true if data was loaded, false if there are no more data 72 | */ 73 | virtual bool pull(ll_writable_graph* graph, size_t max_edges) = 0; 74 | }; 75 | 76 | 77 | /** 78 | * A serial concatenation of multiple data sources 79 | */ 80 | class ll_concat_data_source : public ll_data_source { 81 | 82 | std::queue _data_sources; 83 | ll_spinlock_t _lock; 84 | 85 | 86 | public: 87 | 88 | /** 89 | * Create an instance of the concatenated data source 90 | */ 91 | ll_concat_data_source() { 92 | 93 | _lock = 0; 94 | } 95 | 96 | 97 | /** 98 | * Destroy the data source 99 | */ 100 | virtual ~ll_concat_data_source() { 101 | 102 | while (!_data_sources.empty()) { 103 | delete _data_sources.front(); 104 | _data_sources.pop(); 105 | } 106 | } 107 | 108 | 109 | /** 110 | * Add a data source 111 | * 112 | * @param data_source the data source 113 | */ 114 | void add(ll_data_source* data_source) { 115 | 116 | ll_spinlock_acquire(&_lock); 117 | _data_sources.push(data_source); 118 | ll_spinlock_release(&_lock); 119 | } 120 | 121 | 122 | /** 123 | * Load the next batch of data 124 | * 125 | * @param graph the writable graph 126 | * @param max_edges the maximum number of edges 127 | * @return true if data was loaded, false if there are no more data 128 | */ 129 | virtual bool pull(ll_writable_graph* graph, size_t max_edges) { 130 | 131 | ll_spinlock_acquire(&_lock); 132 | 133 | if (_data_sources.empty()) { 134 | ll_spinlock_release(&_lock); 135 | return false; 136 | } 137 | 138 | ll_data_source* d = _data_sources.front(); 139 | ll_spinlock_release(&_lock); 140 | 141 | while (true) { 142 | 143 | bool r = d->pull(graph, max_edges); 144 | if (r) return r; 145 | 146 | ll_spinlock_acquire(&_lock); 147 | 148 | if (d != _data_sources.front()) { 149 | LL_E_PRINT("Race condition\n"); 150 | abort(); 151 | } 152 | 153 | delete d; 154 | _data_sources.pop(); 155 | 156 | if (_data_sources.empty()) { 157 | ll_spinlock_release(&_lock); 158 | return false; 159 | } 160 | 161 | d = _data_sources.front(); 162 | ll_spinlock_release(&_lock); 163 | } 164 | } 165 | }; 166 | 167 | #endif 168 | -------------------------------------------------------------------------------- /llama/include/llama/loaders/ll_gen_erdosrenyi.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ll_gen_erdosrenyi.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_GEN_ERDOSRENYI_H_ 38 | #define LL_GEN_ERDOSRENYI_H_ 39 | 40 | #include 41 | #include 42 | #include 43 | 44 | #include "llama/ll_mem_array.h" 45 | #include "llama/ll_writable_graph.h" 46 | 47 | #include "llama/loaders/ll_load_async_writable.h" 48 | #include "llama/loaders/ll_load_utils.h" 49 | 50 | 51 | /** 52 | * Erdos-Renyi graph generator 53 | */ 54 | class ll_generator_erdos_renyi : public ll_file_loader { 55 | 56 | public: 57 | 58 | /** 59 | * Create a new instance of ll_generator_erdos_renyi 60 | */ 61 | ll_generator_erdos_renyi() {} 62 | 63 | 64 | /** 65 | * Destroy the instance 66 | */ 67 | virtual ~ll_generator_erdos_renyi() {} 68 | 69 | 70 | /** 71 | * Determine if this "file" can be opened by this loader 72 | * 73 | * @param file the file 74 | * @return true if it can be opened 75 | */ 76 | virtual bool accepts(const char* file) { 77 | 78 | if (*file == '\0') return false; 79 | if (file[strlen(file)-1] != ')') return false; 80 | 81 | return strncasecmp(file, "erdosrenyi(", 11) == 0 82 | || strncasecmp(file, "er(", 3) == 0; 83 | } 84 | 85 | 86 | /** 87 | * Load directly into the read-only representation by creating a new 88 | * level 89 | * 90 | * @param graph the graph 91 | * @param file the file 92 | * @param config the loader configuration 93 | */ 94 | virtual void load_direct(ll_mlcsr_ro_graph* graph, const char* file, 95 | const ll_loader_config* config) { 96 | 97 | er_generator loader(file); 98 | bool r = loader.load_direct(graph, config); 99 | if (!r) abort(); 100 | } 101 | 102 | 103 | /** 104 | * Load incrementally into the writable representation 105 | * 106 | * @param graph the graph 107 | * @param file the file 108 | * @param config the loader configuration 109 | */ 110 | virtual void load_incremental(ll_writable_graph* graph, const char* file, 111 | const ll_loader_config* config) { 112 | 113 | er_generator loader(file); 114 | bool r = loader.load_incremental(graph, config); 115 | if (!r) abort(); 116 | } 117 | 118 | 119 | /** 120 | * Create a data source object for the given file 121 | * 122 | * @param file the file 123 | * @return the data source 124 | */ 125 | virtual ll_data_source* create_data_source(const char* file) { 126 | return new er_generator(file); 127 | } 128 | 129 | 130 | private: 131 | 132 | /** 133 | * The generator (no weights for now) 134 | */ 135 | class er_generator : public ll_edge_list_loader 136 | { 137 | 138 | std::string _command_string; 139 | 140 | std::string _command; 141 | size_t _nodes; 142 | size_t _edges; 143 | unsigned _seed; 144 | 145 | size_t _generated_edges; 146 | unsigned _state; 147 | 148 | 149 | public: 150 | 151 | /** 152 | * Create an instance of class er_generator 153 | * 154 | * @param file_name the file name 155 | */ 156 | er_generator(const char* file_name) 157 | : ll_edge_list_loader() { 158 | 159 | _command_string = file_name; 160 | 161 | 162 | // Parse the string: ER(nodes, edges [, seed]) 163 | 164 | std::vector args; 165 | 166 | char _s[_command_string.length()+1]; 167 | strcpy(_s, _command_string.c_str()); 168 | if (_s[strlen(_s)-1] == ')') _s[strlen(_s)-1] = '\0'; 169 | 170 | char* p = strchr(_s, '('); 171 | if (p == NULL) abort(); // It should not have been accepted 172 | *p = '\0'; p++; 173 | _command = _s; 174 | 175 | while (true) { 176 | char* e = strchr(p, ','); 177 | if (e != NULL) *e = '\0'; 178 | 179 | while (*p != '\0' && isspace(*p)) p++; 180 | while (*p != '\0' && isspace(p[strlen(p)-1])) 181 | p[strlen(p)-1] = '\0'; 182 | 183 | args.push_back(p); 184 | 185 | if (e != NULL) p = e + 1; else break; 186 | } 187 | 188 | 189 | // Arguments 190 | 191 | if (args.size() != 2 && args.size() != 3) { 192 | LL_E_PRINT("Invalid syntax, expected: ER(nodes, edges [, seed])\n"); 193 | abort(); 194 | } 195 | 196 | _nodes = atol(args[0].c_str()); 197 | _edges = atol(args[1].c_str()); 198 | 199 | if (args.size() > 2) 200 | _seed = atol(args[2].c_str()); 201 | else 202 | _seed = time(NULL); 203 | 204 | 205 | // Initialize the generator 206 | 207 | rewind(); 208 | } 209 | 210 | 211 | /** 212 | * Destroy the loader 213 | */ 214 | virtual ~er_generator() { 215 | } 216 | 217 | 218 | protected: 219 | 220 | /** 221 | * Read the next edge 222 | * 223 | * @param o_tail the output for tail 224 | * @param o_head the output for head 225 | * @param o_weight the output for weight (ignore if HasWeight is false) 226 | * @return true if the edge was loaded, false if EOF or error 227 | */ 228 | virtual bool next_edge(unsigned* o_tail, unsigned* o_head, 229 | float* o_weight) { 230 | 231 | if (_generated_edges >= _edges) return false; 232 | 233 | *o_tail = ll_rand64_positive_r(&_state) % _nodes; 234 | *o_head = ll_rand64_positive_r(&_state) % _nodes; 235 | 236 | _generated_edges++; 237 | return true; 238 | } 239 | 240 | 241 | /** 242 | * Rewind the input file 243 | */ 244 | virtual void rewind() { 245 | _state = _seed; 246 | _generated_edges = 0; 247 | } 248 | 249 | 250 | /** 251 | * Get graph stats if they are available 252 | * 253 | * @param o_nodes the output for the number of nodes (1 + max node ID) 254 | * @param o_edges the output for the number of edges 255 | * @return true if succeeded, or false if not or the info is not available 256 | */ 257 | bool stat(size_t* o_nodes, size_t* o_edges) { 258 | *o_nodes = _nodes; 259 | *o_edges = _edges; 260 | return true; 261 | } 262 | }; 263 | }; 264 | 265 | #endif 266 | -------------------------------------------------------------------------------- /llama/include/llama/loaders/ll_gen_rmat.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ll_gen_rmat.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_GEN_RMAT_H_ 38 | #define LL_GEN_RMAT_H_ 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | 45 | #include "llama/ll_mem_array.h" 46 | #include "llama/ll_writable_graph.h" 47 | 48 | #include "llama/loaders/ll_load_async_writable.h" 49 | #include "llama/loaders/ll_load_utils.h" 50 | 51 | 52 | /** 53 | * R-MAT graph generator 54 | */ 55 | class ll_generator_rmat : public ll_file_loader { 56 | 57 | public: 58 | 59 | /** 60 | * Create a new instance of ll_generator_rmat 61 | */ 62 | ll_generator_rmat() {} 63 | 64 | 65 | /** 66 | * Destroy the instance 67 | */ 68 | virtual ~ll_generator_rmat() {} 69 | 70 | 71 | /** 72 | * Determine if this "file" can be opened by this loader 73 | * 74 | * @param file the file 75 | * @return true if it can be opened 76 | */ 77 | virtual bool accepts(const char* file) { 78 | 79 | if (*file == '\0') return false; 80 | if (file[strlen(file)-1] != ')') return false; 81 | 82 | return strncasecmp(file, "r-mat(", 6) == 0 83 | || strncasecmp(file, "rmat(", 5) == 0; 84 | } 85 | 86 | 87 | /** 88 | * Load directly into the read-only representation by creating a new 89 | * level 90 | * 91 | * @param graph the graph 92 | * @param file the file 93 | * @param config the loader configuration 94 | */ 95 | virtual void load_direct(ll_mlcsr_ro_graph* graph, const char* file, 96 | const ll_loader_config* config) { 97 | 98 | rmat_generator loader(file); 99 | bool r = loader.load_direct(graph, config); 100 | if (!r) abort(); 101 | } 102 | 103 | 104 | /** 105 | * Load incrementally into the writable representation 106 | * 107 | * @param graph the graph 108 | * @param file the file 109 | * @param config the loader configuration 110 | */ 111 | virtual void load_incremental(ll_writable_graph* graph, const char* file, 112 | const ll_loader_config* config) { 113 | 114 | rmat_generator loader(file); 115 | bool r = loader.load_incremental(graph, config); 116 | if (!r) abort(); 117 | } 118 | 119 | 120 | /** 121 | * Create a data source object for the given file 122 | * 123 | * @param file the file 124 | * @return the data source 125 | */ 126 | virtual ll_data_source* create_data_source(const char* file) { 127 | return new rmat_generator(file); 128 | } 129 | 130 | 131 | private: 132 | 133 | /** 134 | * The generator (no weights for now) 135 | */ 136 | class rmat_generator : public ll_edge_list_loader 137 | { 138 | 139 | std::string _command_string; 140 | std::string _command; 141 | 142 | size_t _nodes; 143 | size_t _edges; 144 | 145 | size_t _scale; 146 | double _degree; 147 | double _a, _b, _c, _d; 148 | bool _noise; 149 | 150 | unsigned _seed; 151 | 152 | size_t _generated_edges; 153 | unsigned _state; 154 | 155 | 156 | public: 157 | 158 | /** 159 | * Create an instance of class rmat_generator 160 | * 161 | * @param file_name the file name 162 | */ 163 | rmat_generator(const char* file_name) 164 | : ll_edge_list_loader() { 165 | 166 | _command_string = file_name; 167 | 168 | 169 | // Parse the string: RMAT(scale, degree [, a, b, c [, seed]]) 170 | 171 | std::vector args; 172 | 173 | char _s[_command_string.length()+1]; 174 | strcpy(_s, _command_string.c_str()); 175 | if (_s[strlen(_s)-1] == ')') _s[strlen(_s)-1] = '\0'; 176 | 177 | char* p = strchr(_s, '('); 178 | if (p == NULL) abort(); // It should not have been accepted 179 | *p = '\0'; p++; 180 | _command = _s; 181 | 182 | while (true) { 183 | char* e = strchr(p, ','); 184 | if (e != NULL) *e = '\0'; 185 | 186 | while (*p != '\0' && isspace(*p)) p++; 187 | while (*p != '\0' && isspace(p[strlen(p)-1])) 188 | p[strlen(p)-1] = '\0'; 189 | 190 | args.push_back(p); 191 | 192 | if (e != NULL) p = e + 1; else break; 193 | } 194 | 195 | 196 | // Arguments 197 | 198 | if (args.size() != 2 && args.size() != 5 && args.size() != 6) { 199 | LL_E_PRINT("Invalid syntax, expected: " 200 | "RMAT(scale, degree [, a, b, c [, seed]])\n"); 201 | abort(); 202 | } 203 | 204 | _scale = atol(args[0].c_str()); 205 | _degree = atof(args[1].c_str()); 206 | 207 | if (args.size() > 2) { 208 | _a = atof(args[2].c_str()); 209 | _b = atof(args[3].c_str()); 210 | _c = atof(args[4].c_str()); 211 | } 212 | else { 213 | _a = 0.57; 214 | _b = 0.19; 215 | _c = 0.19; 216 | } 217 | 218 | if (args.size() > 5) 219 | _seed = atol(args[5].c_str()); 220 | else 221 | _seed = time(NULL); 222 | 223 | if (_scale <= 0 || _scale > 31) { 224 | LL_E_PRINT("Invalid scale\n"); 225 | abort(); 226 | } 227 | 228 | if (_degree <= 0) { 229 | LL_E_PRINT("Invalid degree\n"); 230 | abort(); 231 | } 232 | 233 | _d = 1.0 - (_a + _b + _c); 234 | 235 | if (_a < 0 || _b < 0 || _c < 0 || _a + _b + _c > 1) { 236 | LL_E_PRINT("Invalid R-MAT probabilities\n"); 237 | abort(); 238 | } 239 | 240 | _noise = true; 241 | 242 | _nodes = (size_t) (0.1 + pow(2, _scale)); 243 | _edges = (size_t) (0.1 + round(_degree * _nodes)); 244 | 245 | LL_D_PRINT("Nodes = %lu, edges = %lu\n", _nodes, _edges); 246 | 247 | 248 | // Initialize the generator 249 | 250 | rewind(); 251 | } 252 | 253 | 254 | /** 255 | * Destroy the loader 256 | */ 257 | virtual ~rmat_generator() { 258 | } 259 | 260 | 261 | protected: 262 | 263 | /** 264 | * Read the next edge 265 | * 266 | * @param o_tail the output for tail 267 | * @param o_head the output for head 268 | * @param o_weight the output for weight (ignore if HasWeight is false) 269 | * @return true if the edge was loaded, false if EOF or error 270 | */ 271 | virtual bool next_edge(unsigned* o_tail, unsigned* o_head, 272 | float* o_weight) { 273 | 274 | if (_generated_edges >= _edges) return false; 275 | 276 | double a = _a; 277 | double b = _b; 278 | double c = _c; 279 | double d = _d; 280 | 281 | size_t h = 0; 282 | size_t t = 0; 283 | size_t bit = 1ul << (_scale - 1); 284 | 285 | while (bit != 0) { 286 | 287 | double r = rand_r(&_state) / (double) RAND_MAX; 288 | if (r < a) { 289 | } 290 | else if (r < a + b) { 291 | h |= bit; 292 | } 293 | else if (r < a + b + c) { 294 | t |= bit; 295 | } 296 | else { 297 | t |= bit; 298 | h |= bit; 299 | } 300 | 301 | bit >>= 1; 302 | 303 | if (_noise) { 304 | 305 | a *= 0.95 + 0.1 * (rand_r(&_state) / (double) RAND_MAX); 306 | b *= 0.95 + 0.1 * (rand_r(&_state) / (double) RAND_MAX); 307 | c *= 0.95 + 0.1 * (rand_r(&_state) / (double) RAND_MAX); 308 | d *= 0.95 + 0.1 * (rand_r(&_state) / (double) RAND_MAX); 309 | 310 | double n = 1.0 / (a + b + c + d); 311 | a *= n; 312 | b *= n; 313 | c *= n; 314 | d = 1.0 - (a + b + c); 315 | } 316 | } 317 | 318 | assert((size_t) h < _nodes); 319 | assert((size_t) t < _nodes); 320 | 321 | *o_tail = h; 322 | *o_head = t; 323 | 324 | LL_D_NODE2_PRINT(t, h, "%lu --> %lu\n", t, h); 325 | 326 | _generated_edges++; 327 | return true; 328 | } 329 | 330 | 331 | /** 332 | * Rewind the input file 333 | */ 334 | virtual void rewind() { 335 | _state = _seed; 336 | _generated_edges = 0; 337 | } 338 | 339 | 340 | /** 341 | * Get graph stats if they are available 342 | * 343 | * @param o_nodes the output for the number of nodes (1 + max node ID) 344 | * @param o_edges the output for the number of edges 345 | * @return true if succeeded, or false if not or the info is not available 346 | */ 347 | bool stat(size_t* o_nodes, size_t* o_edges) { 348 | *o_nodes = _nodes; 349 | *o_edges = _edges; 350 | return true; 351 | } 352 | }; 353 | }; 354 | 355 | #endif 356 | -------------------------------------------------------------------------------- /llama/include/llama/loaders/ll_loaders.h: -------------------------------------------------------------------------------- 1 | /* 2 | * ll_loaders.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef LL_LOADERS_H_ 38 | #define LL_LOADERS_H_ 39 | 40 | #include 41 | 42 | #include "llama/loaders/ll_load_fgf.h" 43 | #include "llama/loaders/ll_load_net.h" 44 | #include "llama/loaders/ll_load_xstream1.h" 45 | 46 | #include "llama/loaders/ll_gen_erdosrenyi.h" 47 | #include "llama/loaders/ll_gen_rmat.h" 48 | 49 | 50 | /** 51 | * A collection of loaders 52 | */ 53 | class ll_file_loaders { 54 | 55 | std::vector _loaders; 56 | 57 | 58 | public: 59 | 60 | /** 61 | * Create an instace of ll_file_loaders 62 | */ 63 | ll_file_loaders() { 64 | 65 | _loaders.push_back(new ll_loader_fgf()); 66 | _loaders.push_back(new ll_loader_net()); 67 | _loaders.push_back(new ll_loader_xs1()); 68 | 69 | _loaders.push_back(new ll_generator_erdos_renyi()); 70 | _loaders.push_back(new ll_generator_rmat()); 71 | } 72 | 73 | 74 | /** 75 | * Destroy the loaders 76 | */ 77 | virtual ~ll_file_loaders() { 78 | 79 | for (size_t i = 0; i < _loaders.size(); i++) { 80 | delete _loaders[i]; 81 | } 82 | } 83 | 84 | 85 | /** 86 | * Get loaders for the given file 87 | * 88 | * @param file_name the file name 89 | * @return a collection of loaders 90 | */ 91 | std::vector loaders_for(const char* file_name) { 92 | 93 | std::vector v; 94 | 95 | for (size_t i = 0; i < _loaders.size(); i++) { 96 | if (_loaders[i]->accepts(file_name)) v.push_back(_loaders[i]); 97 | } 98 | 99 | return v; 100 | } 101 | 102 | 103 | /** 104 | * Get a loader for the given file. If multiple loaders apply, return just 105 | * one of them. 106 | * 107 | * @param file_name the file name 108 | * @return the loader, or NULL if none found 109 | */ 110 | ll_file_loader* loader_for(const char* file_name) { 111 | 112 | for (size_t i = 0; i < _loaders.size(); i++) { 113 | if (_loaders[i]->accepts(file_name)) return _loaders[i]; 114 | } 115 | 116 | return NULL; 117 | } 118 | 119 | }; 120 | 121 | #endif 122 | 123 | -------------------------------------------------------------------------------- /tests/generate-report.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # 4 | # generate-report.py 5 | # LLAMA Graph Analytics 6 | # 7 | # Copyright 2014 8 | # The President and Fellows of Harvard College. 9 | # 10 | # Copyright 2014 11 | # Oracle Labs. 12 | # 13 | # Redistribution and use in source and binary forms, with or without 14 | # modification, are permitted provided that the following conditions 15 | # are met: 16 | # 1. Redistributions of source code must retain the above copyright 17 | # notice, this list of conditions and the following disclaimer. 18 | # 2. Redistributions in binary form must reproduce the above copyright 19 | # notice, this list of conditions and the following disclaimer in the 20 | # documentation and/or other materials provided with the distribution. 21 | # 3. Neither the name of the University nor the names of its contributors 22 | # may be used to endorse or promote products derived from this software 23 | # without specific prior written permission. 24 | # 25 | # THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 26 | # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 | # ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 29 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 | # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 | # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 | # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 | # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 | # SUCH DAMAGE. 36 | # 37 | 38 | 39 | import csv 40 | import getopt 41 | import glob 42 | import os.path 43 | import sys 44 | 45 | 46 | # 47 | # Usage 48 | # 49 | 50 | def usage(): 51 | sys.stderr.write("Usage: generate-report.py [OPTIONS]\n\n") 52 | sys.stderr.write("Options:\n") 53 | sys.stderr.write(" -d, --dir DIR Set the results directory\n") 54 | sys.stderr.write(" -h, --help Print this help message and exit\n") 55 | 56 | 57 | 58 | # 59 | # Global variables, settings, and parsing of the command-line arguments 60 | # 61 | 62 | global results_dir 63 | results_dir = "/var/www/llama-tests" 64 | 65 | try: 66 | opts, args = getopt.getopt(sys.argv[1:],"d:h",["dir=","help"]) 67 | except getopt.GetoptError as e: 68 | sys.stderr.write("Error: " + str(e) + "\n") 69 | sys.exit(2) 70 | 71 | for opt, arg in opts: 72 | if opt in ("-h", "--help"): 73 | usage() 74 | sys.exit() 75 | elif opt in ("-d", "--dir"): 76 | results_dir = arg 77 | 78 | 79 | 80 | # 81 | # Utilities 82 | # 83 | 84 | def ife(b, t, f): 85 | if b: 86 | return t 87 | else: 88 | return f 89 | 90 | 91 | 92 | # 93 | # Basic CSV functionality 94 | # 95 | 96 | class csv_row_array: 97 | 98 | def __init__(self, file_name): 99 | """ 100 | Initialize the object from the given file name 101 | """ 102 | 103 | self.file_name = file_name; 104 | self.header = [] 105 | self.header_dict = dict() 106 | self.rows = [] 107 | 108 | first = True 109 | with open(file_name, "rb") as csv_file: 110 | csv_reader = csv.reader(csv_file, delimiter=',', quotechar='\"') 111 | for row in csv_reader: 112 | if first: 113 | self.header = row 114 | for i in xrange(0, len(row)): 115 | self.header_dict[row[i]] = i 116 | first = False; 117 | else: 118 | self.rows.append(row); 119 | 120 | 121 | 122 | # 123 | # Find all runs 124 | # 125 | 126 | report_file = results_dir + "/report.html" 127 | out = open(report_file, "w") 128 | 129 | out.write("\n") 130 | out.write("\n") 131 | out.write(" LLAMA Test Results Summary\n") 132 | out.write(" \n") 145 | out.write("\n") 146 | out.write("\n") 147 | out.write(" \n") 148 | out.write(" \n") 149 | out.write(" \n") 150 | out.write(" \n") 151 | out.write(" \n") 152 | out.write(" \n") 153 | out.write(" \n") 154 | out.write(" \n") 155 | out.write(" \n") 156 | 157 | run_dirs = [] 158 | for r in glob.glob(results_dir + "/20*"): 159 | if not os.path.isdir(r): continue 160 | run_dirs.append(r) 161 | 162 | run_dirs.sort() 163 | 164 | for run_dir in run_dirs: 165 | 166 | out.write(" \n") 167 | out.write(" \n" % 168 | (os.path.basename(run_dir) + "/report.txt", os.path.basename(run_dir))) 169 | 170 | fn_status_details = run_dir + "/status-detailed.csv" 171 | if not os.path.isfile(fn_status_details): 172 | sys.stderr.write("Error: No details file for " + run_dir + "\n"); 173 | out.write(" \n") 174 | out.write(" \n") 175 | continue 176 | status_details = csv_row_array(fn_status_details) 177 | 178 | fn_status_summary = run_dir + "/status-summary.csv" 179 | if not os.path.isfile(fn_status_summary): 180 | sys.stderr.write("Error: No summary file for " + run_dir + "\n"); 181 | out.write(" \n") 182 | out.write(" \n") 183 | continue 184 | status_summary = csv_row_array(fn_status_summary) 185 | 186 | s_sum_row = status_summary.rows[0] 187 | num_errors = int(s_sum_row[status_summary.header_dict["errors"]]) 188 | num_warnings = int(s_sum_row[status_summary.header_dict["warnings"]]) 189 | num_successes = int(s_sum_row[status_summary.header_dict["successes"]]) 190 | num_regressions = int(s_sum_row[status_summary.header_dict["regressions"]]) 191 | 192 | out.write(" \n" % (num_successes + num_warnings + num_errors + 193 | num_regressions)) 194 | out.write(" %d\n" % 195 | (ife(num_successes != 0, " class=\"success\"", ""), num_successes)) 196 | out.write(" %d\n" % 197 | (ife(num_warnings != 0, " class=\"warning\"", ""), num_warnings)) 198 | out.write(" %d\n" % 199 | (ife(num_errors != 0, " class=\"error\"", ""), num_errors)) 200 | out.write(" %d\n" % 201 | (ife(num_regressions != 0, " class=\"regression\"", ""), num_regressions)) 202 | out.write(" \n") 203 | 204 | out.write("
Test RunTotal RunsSuccessesWarningsErrorsRegressions
%sNo details file
No summary file
%d
\n") 205 | out.write("\n") 206 | 207 | out.close() 208 | print "Generated: " + report_file 209 | 210 | -------------------------------------------------------------------------------- /tools/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # LLAMA Tools 3 | # 4 | 5 | CC := g++ 6 | CFLAGS := -Wall -fopenmp -std=c++0x -DLL_PERSISTENCE -I. -I../llama/include 7 | LFLAGS := 8 | 9 | ifeq ($(VIMRUNTIME),) 10 | ifneq ($(wildcard /usr/bin/color-g++),) 11 | CC := color-g++ 12 | endif 13 | endif 14 | 15 | CFLAGS_D := -ggdb -O0 -D_DEBUG ${CFLAGS} 16 | CFLAGS := -g -O3 -DNDEBUG ${CFLAGS} 17 | 18 | DIR := ../bin 19 | SOURCES := $(wildcard *.cc) 20 | TARGETS := $(patsubst %.cc,$(DIR)/%,$(SOURCES)) 21 | 22 | .PHONY: all clean 23 | 24 | all: $(TARGETS) 25 | 26 | clean: 27 | rm -f $(TARGETS) 28 | 29 | $(DIR)/%: %.cc 30 | @echo CC $@ >&2 31 | $(CC) $(CFLAGS) -o $@ $< $(LFLAGS) 32 | 33 | -------------------------------------------------------------------------------- /tools/llama-load.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * llama-load.cc 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #include 38 | #include 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | 51 | #include 52 | 53 | 54 | //==========================================================================// 55 | // The Command-Line Arguments // 56 | //==========================================================================// 57 | 58 | static const char* SHORT_OPTIONS = "d:DhIT:t:UOvX:"; 59 | 60 | static struct option LONG_OPTIONS[] = 61 | { 62 | {"database" , required_argument, 0, 'd'}, 63 | {"deduplicate" , no_argument , 0, 'D'}, 64 | {"help" , no_argument, 0, 'h'}, 65 | {"in-edges" , no_argument, 0, 'I'}, 66 | {"temp-dir" , required_argument, 0, 'T'}, 67 | {"threads" , required_argument, 0, 't'}, 68 | {"undir-double" , no_argument, 0, 'U'}, 69 | {"undir-order" , no_argument, 0, 'O'}, 70 | {"verbose" , no_argument, 0, 'v'}, 71 | {"xs-buffer" , required_argument, 0, 'X'}, 72 | {0, 0, 0, 0} 73 | }; 74 | 75 | 76 | /** 77 | * Print the usage information 78 | * 79 | * @param arg0 the first element in the argv array 80 | */ 81 | static void usage(const char* arg0) { 82 | 83 | char* s = strdup(arg0); 84 | char* p = basename(s); 85 | fprintf(stderr, "Usage: %s [OPTIONS] INPUT_FILE [INPUT_FILE...]\n\n", p); 86 | free(s); 87 | 88 | fprintf(stderr, "Options:\n"); 89 | fprintf(stderr, " -d, --database DIR Set the database directory\n"); 90 | fprintf(stderr, " -D, --deduplicate Deduplicate edges within level while loading\n"); 91 | fprintf(stderr, " -h, --help Show this usage information and exit\n"); 92 | fprintf(stderr, " -I, --in-edges Load or generate in-edges\n"); 93 | fprintf(stderr, " -O, --undir-order Load undirected by ordering all edges\n"); 94 | fprintf(stderr, " -t, --threads N Set the number of threads\n"); 95 | fprintf(stderr, " -T, --temp DIR Add a temporary directory\n"); 96 | fprintf(stderr, " -U, --undir-double Load undirected by doubling all edges\n"); 97 | fprintf(stderr, " -v, --verbose Enable verbose output\n"); 98 | fprintf(stderr, " -X, --xs-buffer GB Set the external sort buffer size, in GB\n"); 99 | } 100 | 101 | 102 | //==========================================================================// 103 | // The Main Function // 104 | //==========================================================================// 105 | 106 | /** 107 | * The main function 108 | */ 109 | int main(int argc, char** argv) 110 | { 111 | char* database_directory = NULL; 112 | bool verbose = false; 113 | 114 | ll_loader_config loader_config; 115 | int num_threads = -1; 116 | 117 | 118 | // Pase the command-line arguments 119 | 120 | int option_index = 0; 121 | while (true) { 122 | int c = getopt_long(argc, argv, SHORT_OPTIONS, LONG_OPTIONS, &option_index); 123 | 124 | if (c == -1) break; 125 | 126 | switch (c) { 127 | 128 | case 'd': 129 | database_directory = optarg; 130 | break; 131 | 132 | case 'D': 133 | loader_config.lc_deduplicate = true; 134 | break; 135 | 136 | case 'h': 137 | usage(argv[0]); 138 | return 0; 139 | 140 | case 'I': 141 | loader_config.lc_reverse_edges = true; 142 | loader_config.lc_reverse_maps = true; 143 | break; 144 | 145 | case 'O': 146 | loader_config.lc_direction = LL_L_UNDIRECTED_ORDERED; 147 | break; 148 | 149 | case 't': 150 | num_threads = atoi(optarg); 151 | break; 152 | 153 | case 'T': 154 | loader_config.lc_tmp_dirs.push_back(optarg); 155 | break; 156 | 157 | case 'U': 158 | loader_config.lc_direction = LL_L_UNDIRECTED_DOUBLE; 159 | break; 160 | 161 | case 'v': 162 | verbose = true; 163 | loader_config.lc_print_progress = true; 164 | break; 165 | 166 | case 'X': 167 | loader_config.lc_xs_buffer_size = (size_t) (atof(optarg) 168 | * 1024ul*1048576ul); 169 | break; 170 | 171 | case '?': 172 | case ':': 173 | return 1; 174 | 175 | default: 176 | abort(); 177 | } 178 | } 179 | 180 | std::vector input_files; 181 | for (int i = optind; i < argc; i++) { 182 | input_files.push_back(std::string(argv[i])); 183 | } 184 | 185 | if (database_directory == NULL) { 186 | database_directory = (char*) alloca(16); 187 | strcpy(database_directory, "db"); 188 | } 189 | 190 | 191 | // Check the input files 192 | 193 | if (input_files.empty()) { 194 | fprintf(stderr, "Error: No input files are specified\n"); 195 | return 1; 196 | } 197 | 198 | const char* first_input = input_files[0].c_str(); 199 | const char* file_type = ll_file_extension(first_input); 200 | 201 | for (size_t i = 1; i < input_files.size(); i++) { 202 | if (strcmp(ll_file_extension(input_files[i].c_str()), file_type)!=0) { 203 | fprintf(stderr, "Error: All imput files must have the same " 204 | "file extension.\n"); 205 | return 1; 206 | } 207 | } 208 | 209 | 210 | // Open the database 211 | 212 | ll_database database(database_directory); 213 | if (num_threads > 0) database.set_num_threads(num_threads); 214 | ll_writable_graph& graph = *database.graph(); 215 | 216 | 217 | // Load the graph 218 | 219 | ll_file_loaders loaders; 220 | ll_file_loader* loader = loaders.loader_for(first_input); 221 | if (loader == NULL) { 222 | fprintf(stderr, "Error: Unsupported input file type\n"); 223 | return 1; 224 | } 225 | 226 | if (verbose) fprintf(stderr, "Loading:\n"); 227 | for (size_t i = 0; i < input_files.size(); i++) { 228 | if (verbose) fprintf(stderr, " %2lu: %s", i+1, input_files[i].c_str()); 229 | 230 | double ts = ll_get_time_ms(); 231 | loader->load_direct(&graph, input_files[i].c_str(), &loader_config); 232 | double t = ll_get_time_ms() - ts; 233 | 234 | if (verbose) { 235 | fprintf(stderr, " (Load: %3.2lf s, %7.2lf Kedges/s)\n", 236 | t/1000.0, graph.max_edges(graph.num_levels() - 2) / t); 237 | } 238 | } 239 | 240 | return 0; 241 | } 242 | -------------------------------------------------------------------------------- /utils/Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Miscellaneous LLAMA Tools 3 | # 4 | 5 | CC := g++ 6 | CFLAGS := -Wall -std=c++0x -fopenmp -I../llama/include 7 | LFLAGS := 8 | 9 | SOURCES := $(wildcard *.cpp) 10 | DIR := ../bin 11 | 12 | TARGETS := $(patsubst %.cpp,$(DIR)/%,$(SOURCES)) 13 | 14 | ifdef DEBUG 15 | CFLAGS := -O0 -D_DEBUG -ggdb $(CFLAGS) 16 | else 17 | CFLAGS := -O3 -DNDEBUG -g $(CFLAGS) 18 | endif 19 | 20 | .PHONY: all clean 21 | 22 | all: $(TARGETS) 23 | 24 | clean: 25 | rm -f $(TARGETS) 26 | 27 | $(DIR)/%: %.cpp 28 | @echo CC $@ >&2 29 | $(CC) $(CFLAGS) -o $@ $< $(LFLAGS) 30 | 31 | -------------------------------------------------------------------------------- /utils/memhog.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * memhog.cpp 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | /** 38 | * Decrease the effective memory size by mlock-ing memory 39 | */ 40 | 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | 49 | 50 | /** 51 | * The main function 52 | */ 53 | int main(int argc, char** argv) { 54 | 55 | if (argc != 2) { 56 | fprintf(stderr, "Usage: %s MEM_GB\n", argv[0]); 57 | return 1; 58 | } 59 | 60 | int gb = atoi(argv[1]); 61 | 62 | char buf[64]; 63 | sprintf(buf, "%d", gb); 64 | if (strcmp(buf, argv[1]) != 0) { 65 | fprintf(stderr, "Error: Cannot parse the memory size argument\n"); 66 | return 1; 67 | } 68 | if (gb <= 0) { 69 | fprintf(stderr, "Error: Must specify at least 1 GB to hog\n"); 70 | return 1; 71 | } 72 | 73 | size_t size = ((size_t) gb) * 1024 * 1024 * 1024; 74 | void* m = malloc(size); 75 | if (m == NULL) { 76 | fprintf(stderr, "Error: Not enough memory\n"); 77 | return 1; 78 | } 79 | 80 | if (mlock(m, size)) { 81 | if (errno == ENOMEM) { 82 | fprintf(stderr, "Error: Could not mlock -- insufficient permissions\n"); 83 | } 84 | else { 85 | fprintf(stderr, "Error: Could not mlock -- %s\n", strerror(errno)); 86 | } 87 | free(m); 88 | return 1; 89 | } 90 | 91 | memset(m, 0xcc, size); // Is this necessary? 92 | 93 | fprintf(stderr, "Hogging %d GB RAM. Use Ctrl+C to exit.\n", gb); 94 | while (1) sleep(60); 95 | 96 | return 0; 97 | } 98 | 99 | -------------------------------------------------------------------------------- /utils/snap-to-xs1.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * snap-to-xs1.cpp 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | /* 38 | * LLAMA Utilities -- X-Stream tools 39 | * 40 | * Convert a SNAP file to an X-Stream Type 1 file 41 | * 42 | * DISCLAIMER: This tool was developed separately from X-Stream and without 43 | * their endorsement. 44 | */ 45 | 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include "xs1-common.h" 56 | 57 | using namespace std; 58 | 59 | 60 | static const char* SHORT_OPTIONS = "ho:uS:"; 61 | 62 | static struct option LONG_OPTIONS[] = 63 | { 64 | {"help" , no_argument , 0, 'h'}, 65 | {"output" , required_argument, 0, 'o'}, 66 | {"undirected" , no_argument , 0, 'u'}, 67 | {"seed" , required_argument, 0, 'S'}, 68 | {0, 0, 0, 0} 69 | }; 70 | 71 | 72 | /** 73 | * Print the usage information 74 | * 75 | * @param arg0 the first element in the argv array 76 | */ 77 | static void usage(const char* arg0) { 78 | 79 | char* s = strdup(arg0); 80 | char* p = basename(s); 81 | fprintf(stderr, "Usage: %s [OPTIONS] INPUT_FILE\n\n", p); 82 | free(s); 83 | 84 | fprintf(stderr, "Options:\n"); 85 | fprintf(stderr, " -h, --help Show this usage information and exit\n"); 86 | fprintf(stderr, " -o, --output FILE Set the output file\n"); 87 | fprintf(stderr, " -S, --seed N Set the random number generator seed\n"); 88 | fprintf(stderr, " -u, --undirected Generate an undirected output file\n"); 89 | fprintf(stderr, "\n"); 90 | } 91 | 92 | 93 | /** 94 | * The main function 95 | * 96 | * @param argc the number of command-line arguments 97 | * @param argv the command-line arguments 98 | * @return the exit code 99 | */ 100 | int main(int argc, char** argv) { 101 | 102 | srand(time(NULL)); 103 | 104 | bool undirected = false; 105 | std::string output = ""; 106 | 107 | bool progress = true; 108 | size_t progress_step = 10000000; 109 | 110 | 111 | // Pase the command-line arguments 112 | 113 | int option_index = 0; 114 | while (true) { 115 | int c = getopt_long(argc, argv, SHORT_OPTIONS, LONG_OPTIONS, &option_index); 116 | 117 | if (c == -1) break; 118 | 119 | switch (c) { 120 | 121 | case 'h': 122 | usage(argv[0]); 123 | return 0; 124 | 125 | case 'o': 126 | output = optarg; 127 | break; 128 | 129 | case 'S': 130 | srand(atoi(optarg)); 131 | break; 132 | 133 | case 'u': 134 | undirected = true; 135 | break; 136 | 137 | case '?': 138 | case ':': 139 | return 1; 140 | 141 | default: 142 | abort(); 143 | } 144 | } 145 | 146 | if (optind + 1 != argc) { 147 | usage(argv[0]); 148 | return 1; 149 | } 150 | 151 | const char* src = argv[optind]; 152 | 153 | if (output == "") { 154 | const char* s = strrchr(src, '.'); 155 | if (s != NULL && (strcmp(s, ".net") == 0 || strcmp(s, ".txt") == 0 156 | || strcmp(s, ".snap") == 0)) { 157 | char* p = strdup(src); 158 | *(p + (s - src)) = '\0'; 159 | output = p; 160 | free(p); 161 | } 162 | else { 163 | output = src; 164 | } 165 | output += ".dat"; 166 | } 167 | 168 | 169 | // Do it! 170 | 171 | FILE* f_in = fopen(src, "rb"); 172 | if (f_in == NULL) { 173 | perror("fopen"); 174 | return 1; 175 | } 176 | 177 | FILE* f_out = output == "-" ? stdout : fopen(output.c_str(), "wb"); 178 | if (f_out == NULL) { 179 | perror("fopen"); 180 | fclose(f_in); 181 | return 1; 182 | } 183 | 184 | ssize_t read; 185 | size_t num_nodes = 0; 186 | size_t num_edges = 0; 187 | size_t num_lines = 0; 188 | 189 | size_t line_len = 64; 190 | char* line = (char*) malloc(line_len); 191 | 192 | while ((read = getline(&line, &line_len, f_in)) != -1) { 193 | 194 | num_lines++; 195 | 196 | if (*line == '\0' || *line == '#' 197 | || *line == '\n' || *line == '\r') continue; 198 | 199 | size_t ln = strlen(line)-1; 200 | if (line[ln] == '\n' || line[ln] == '\r') line[ln] = '\0'; 201 | 202 | if (!isdigit(*line)) { 203 | fprintf(stderr, "Invalid .net format on line \"%s\"\n", line); 204 | abort(); 205 | } 206 | 207 | char* l = line; 208 | while (isdigit(*l)) l++; 209 | if (*l == '\0') { 210 | fprintf(stderr, "Invalid .net format on line \"%s\"\n", line); 211 | abort(); 212 | } 213 | 214 | while (isspace(*l)) l++; 215 | if (*l == '\0' || !isdigit(*l)) { 216 | fprintf(stderr, "Invalid .net format on line \"%s\"\n", line); 217 | abort(); 218 | } 219 | 220 | num_edges++; 221 | 222 | xs1 x; 223 | x.tail = atoi(line); 224 | x.head = atoi(l); 225 | x.weight = rand() / (float) RAND_MAX; 226 | 227 | if (x.head >= num_nodes) num_nodes = x.head + 1; 228 | if (x.tail >= num_nodes) num_nodes = x.tail + 1; 229 | 230 | ssize_t r = fwrite(&x, sizeof(xs1), 1, f_out); 231 | if (r <= 0) { 232 | if (r < 0) { 233 | perror("fwrite"); 234 | } 235 | else { 236 | fprintf(stderr, "Out of space\n"); 237 | } 238 | fclose(f_in); 239 | fclose(f_out); 240 | return 1; 241 | } 242 | 243 | if (undirected) { 244 | 245 | unsigned t = x.tail; x.tail = x.head; x.head = t; 246 | num_edges++; 247 | 248 | ssize_t r = fwrite(&x, sizeof(xs1), 1, f_out); 249 | if (r <= 0) { 250 | if (r < 0) { 251 | perror("fwrite"); 252 | } 253 | else { 254 | fprintf(stderr, "Out of space\n"); 255 | } 256 | fclose(f_in); 257 | fclose(f_out); 258 | return 1; 259 | } 260 | } 261 | 262 | if (progress && (num_lines % progress_step) == 0) { 263 | fprintf(stderr, "."); 264 | if ((num_lines % (progress_step * 10)) == 0) { 265 | fprintf(stderr, "%lum", num_lines / 1000000); 266 | } 267 | if ((num_lines % (progress_step * 50)) == 0) { 268 | fprintf(stderr, "\n"); 269 | } 270 | } 271 | } 272 | 273 | if (progress) { 274 | if ((num_lines % (progress_step * 50)) >= progress_step) { 275 | fprintf(stderr, "\n"); 276 | } 277 | } 278 | 279 | if (f_out != stdout && f_out != stderr) { 280 | fclose(f_out); 281 | 282 | if (!write_ini(output.c_str(), num_nodes, num_edges)) { 283 | perror("write_ini"); 284 | return 1; 285 | } 286 | } 287 | 288 | fclose(f_in); 289 | free(line); 290 | 291 | return 0; 292 | } 293 | 294 | -------------------------------------------------------------------------------- /utils/xs1-common.h: -------------------------------------------------------------------------------- 1 | /* 2 | * xs1-common.h 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | #ifndef XS1_COMMON_H_ 38 | #define XS1_COMMON_H_ 39 | 40 | #include 41 | #include 42 | #include 43 | 44 | 45 | /* 46 | * Debug output 47 | */ 48 | 49 | #ifdef _DEBUG 50 | #define LL_D_PRINT(format, ...) { \ 51 | fprintf(stderr, "[DEBUG] %s" format, __FUNCTION__, ## __VA_ARGS__); } 52 | #else 53 | #define LL_D_PRINT(format, ...) 54 | #endif 55 | 56 | 57 | /* 58 | * Error handling 59 | */ 60 | 61 | #define LL_W_PRINT(format, ...) { \ 62 | fprintf(stderr, "[WARN ] %s" format, __FUNCTION__, ## __VA_ARGS__); } 63 | #define LL_E_PRINT(format, ...) { \ 64 | fprintf(stderr, "[ERROR] %s" format, __FUNCTION__, ## __VA_ARGS__); } 65 | 66 | 67 | /** 68 | * The X-Stream Type 1 entry 69 | */ 70 | struct xs1 { 71 | unsigned tail; 72 | unsigned head; 73 | float weight; 74 | }; 75 | 76 | 77 | /** 78 | * Read the next Type 1 entry 79 | * 80 | * @param f the file 81 | * @param out the output 82 | * @return true if anything was read 83 | */ 84 | bool next(FILE* f, xs1* out) { 85 | ssize_t r = fread(out, sizeof(xs1), 1, f); 86 | if (r < 0) { 87 | perror("fread"); 88 | abort(); 89 | } 90 | return r > 0; 91 | } 92 | 93 | 94 | /** 95 | * Write the ini file 96 | * 97 | * @param dat_name the original (.dat) file name 98 | * @param num_nodes the number of nodes 99 | * @param num_edges the number of edges 100 | * @return true if okay, false if not 101 | */ 102 | bool write_ini(const char* dat_name, size_t num_nodes, size_t num_edges) { 103 | 104 | std::string s_out = dat_name; s_out += ".ini"; 105 | FILE* f_out = fopen(s_out.c_str(), "w"); 106 | if (f_out == NULL) return false; 107 | 108 | char* name = (char*) alloca(strlen(dat_name) + 1); 109 | memcpy(name, dat_name, strlen(dat_name) + 1); 110 | name = basename(name); 111 | 112 | fprintf(f_out, "[graph]\n"); 113 | fprintf(f_out, "type=1\n"); 114 | fprintf(f_out, "name=%s\n", name); 115 | fprintf(f_out, "vertices=%lu\n", num_nodes); 116 | fprintf(f_out, "edges=%lu\n", num_edges); 117 | 118 | fclose(f_out); 119 | 120 | return true; 121 | } 122 | 123 | 124 | #endif 125 | -------------------------------------------------------------------------------- /utils/xs1-ini.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * xs1-ini.cpp 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | /* 38 | * LLAMA Utilities -- X-Stream tools 39 | * 40 | * Create an ini file 41 | * 42 | * DISCLAIMER: This tool was developed separately from X-Stream and without 43 | * their endorsement. 44 | */ 45 | 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include "xs1-common.h" 55 | 56 | using namespace std; 57 | 58 | 59 | struct node_and_degree { 60 | unsigned node; 61 | int degree; 62 | }; 63 | 64 | 65 | int cmp_by_degree_desc(const void* a, const void* b) { 66 | const node_and_degree* A = (const node_and_degree*) a; 67 | const node_and_degree* B = (const node_and_degree*) b; 68 | return B->degree - A->degree; 69 | } 70 | 71 | 72 | /** 73 | * The main function 74 | * 75 | * @param argc the number of command-line arguments 76 | * @param argv the command-line arguments 77 | * @return the exit code 78 | */ 79 | int main(int argc, char** argv) { 80 | 81 | bool progress = true; 82 | size_t progress_step = 10000000; 83 | 84 | if (argc != 2) { 85 | fprintf(stderr, "Usage: %s FILE\n", argv[0]); 86 | return 1; 87 | } 88 | 89 | string s_in = argv[1]; 90 | FILE* f_in = fopen(s_in.c_str(), "rb"); 91 | if (f_in == NULL) { 92 | perror("fopen"); 93 | return 1; 94 | } 95 | 96 | string s_out = argv[1]; s_out += ".ini"; 97 | FILE* f_out_test = fopen(s_out.c_str(), "r"); 98 | if (f_out_test != NULL) { 99 | fprintf(stderr, "Error: File %s exists\n", s_out.c_str()); 100 | fclose(f_out_test); 101 | fclose(f_in); 102 | return 1; 103 | } 104 | 105 | xs1 x; 106 | size_t num_nodes = 0; 107 | size_t num_edges = 0; 108 | 109 | while (next(f_in, &x)) { 110 | 111 | num_edges++; 112 | 113 | if (x.head >= num_nodes) num_nodes = x.head + 1; 114 | if (x.tail >= num_nodes) num_nodes = x.tail + 1; 115 | 116 | if (progress && (num_edges % progress_step) == 0) { 117 | fprintf(stderr, "."); 118 | if ((num_edges % (progress_step * 10)) == 0) { 119 | fprintf(stderr, "%lum", num_edges / 1000000); 120 | } 121 | if ((num_edges % (progress_step * 50)) == 0) { 122 | fprintf(stderr, "\n"); 123 | } 124 | } 125 | } 126 | 127 | if (progress) { 128 | if ((num_edges % (progress_step * 50)) >= progress_step) { 129 | fprintf(stderr, "\n"); 130 | } 131 | } 132 | 133 | fclose(f_in); 134 | 135 | if (!write_ini(s_in.c_str(), num_nodes, num_edges)) { 136 | perror("write_ini"); 137 | return 1; 138 | } 139 | 140 | return 0; 141 | } 142 | 143 | -------------------------------------------------------------------------------- /utils/xs1-max-degree.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * xs1-max-degree.cpp 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | /* 38 | * LLAMA Utilities -- X-Stream tools 39 | * 40 | * Find the node(s) with the highest degrees 41 | * 42 | * DISCLAIMER: This tool was developed separately from X-Stream and without 43 | * their endorsement. 44 | */ 45 | 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include "xs1-common.h" 54 | 55 | using namespace std; 56 | 57 | 58 | struct node_and_degree { 59 | unsigned node; 60 | int degree; 61 | }; 62 | 63 | 64 | int cmp_by_degree_desc(const void* a, const void* b) { 65 | const node_and_degree* A = (const node_and_degree*) a; 66 | const node_and_degree* B = (const node_and_degree*) b; 67 | return B->degree - A->degree; 68 | } 69 | 70 | 71 | /** 72 | * The main function 73 | * 74 | * @param argc the number of command-line arguments 75 | * @param argv the command-line arguments 76 | * @return the exit code 77 | */ 78 | int main(int argc, char** argv) { 79 | 80 | int max = 10; 81 | bool progress = true; 82 | unsigned progress_step = 10000000; 83 | 84 | if (argc != 2) { 85 | fprintf(stderr, "Usage: %s FILE\n", argv[0]); 86 | return 1; 87 | } 88 | 89 | string s_in = argv[1]; 90 | FILE* f_in = fopen(s_in.c_str(), "rb"); 91 | if (f_in == NULL) { 92 | perror("fopen"); 93 | return 1; 94 | } 95 | 96 | size_t capacity = 80 * 1000 * 1000; 97 | node_and_degree* degrees 98 | = (node_and_degree*) malloc(sizeof(node_and_degree) * capacity); 99 | memset(degrees, 0, sizeof(node_and_degree) * capacity); 100 | for (unsigned i = 0; i < capacity; i++) degrees[i].node = i; 101 | 102 | xs1 x; 103 | unsigned n = 0; 104 | unsigned l = 0; 105 | 106 | while (next(f_in, &x)) { 107 | l++; 108 | 109 | if (x.head >= n) n = x.head; 110 | if (x.tail >= n) n = x.tail; 111 | if (n >= capacity) { 112 | size_t c = capacity; 113 | while (c <= n + 64) c *= 2; 114 | degrees = (node_and_degree*) realloc(degrees, sizeof(node_and_degree) * c); 115 | memset(degrees + capacity, 0, sizeof(node_and_degree) * (c - capacity)); 116 | for (unsigned i = capacity; i < c; i++) degrees[i].node = i; 117 | capacity = c; 118 | } 119 | 120 | degrees[x.tail].degree++; 121 | 122 | if (progress && (l % progress_step) == 0) { 123 | fprintf(stderr, "."); 124 | if ((l % (progress_step * 10)) == 0) { 125 | fprintf(stderr, "%um", l / 1000000); 126 | } 127 | if ((l % (progress_step * 50)) == 0) { 128 | fprintf(stderr, "\n"); 129 | } 130 | } 131 | } 132 | if (progress) { 133 | if ((l % (progress_step * 50)) >= progress_step) fprintf(stderr, "\n"); 134 | } 135 | 136 | fclose(f_in); 137 | 138 | n++; 139 | qsort(degrees, n, sizeof(node_and_degree), cmp_by_degree_desc); 140 | 141 | if ((unsigned) max > n) max = (int) n; 142 | printf("Node\tDegree\tRank\n"); 143 | for (int i = 0; i < max; i++) { 144 | printf("%u\t%d\t%d\n", degrees[i].node, degrees[i].degree, i+1); 145 | } 146 | 147 | return 0; 148 | } 149 | 150 | -------------------------------------------------------------------------------- /utils/xs1-shuffle.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * xs1-shuffle.cpp 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | /* 38 | * LLAMA Utilities -- X-Stream tools 39 | * 40 | * Shuffle the order of edges in an X-Stream Type 1 file. 41 | * 42 | * DISCLAIMER: This tool was developed separately from X-Stream and without 43 | * their endorsement. 44 | */ 45 | 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | 56 | #include "xs1-common.h" 57 | #include "llama/ll_external_sort.h" 58 | 59 | using namespace std; 60 | 61 | 62 | // Comnmand-line arguments 63 | 64 | static const char* SHORT_OPTIONS = "hM:S:T:X:"; 65 | 66 | static struct option LONG_OPTIONS[] = 67 | { 68 | {"help" , no_argument , 0, 'h'}, 69 | {"max-edges" , required_argument, 0, 'M'}, 70 | {"seed" , required_argument, 0, 'S'}, 71 | {"temp-dir" , required_argument, 0, 'T'}, 72 | {"xs-buffer" , required_argument, 0, 'X'}, 73 | {0, 0, 0, 0} 74 | }; 75 | 76 | 77 | /** 78 | * xs1 + a random key 79 | */ 80 | struct xs1r { 81 | struct xs1 payload; 82 | int random1; 83 | int random2; 84 | }; 85 | 86 | 87 | /** 88 | * Comparator for edge 89 | */ 90 | struct xs1r_edge_comparator { 91 | bool operator() (const xs1r& a, const xs1r& b) { 92 | if (a.random1 != b.random1) return a.random1 < b.random1; 93 | if (a.random2 != b.random2) return a.random2 < b.random2; 94 | if (a.payload.tail != b.payload.tail) return a.payload.tail < b.payload.tail; 95 | return a.payload.head < b.payload.head; 96 | } 97 | }; 98 | 99 | 100 | /** 101 | * Print the usage information 102 | * 103 | * @param arg0 the first element in the argv array 104 | */ 105 | static void usage(const char* arg0) { 106 | 107 | char* s = strdup(arg0); 108 | char* p = basename(s); 109 | fprintf(stderr, "Usage: %s [OPTIONS] INPUT_FILE OUTPUT_FILE\n\n", p); 110 | free(s); 111 | 112 | fprintf(stderr, "Options:\n"); 113 | fprintf(stderr, " -h, --help Show this usage information and exit\n"); 114 | fprintf(stderr, " -M, --max-edges N Set maximum number of edges to read\n"); 115 | fprintf(stderr, " -S, --seed SEED Set the random number generator seed\n"); 116 | fprintf(stderr, " -T, --temp DIR Set the temporary directory\n"); 117 | fprintf(stderr, " -X, --xs-buffer GB Set the external sort buffer size, in GB\n"); 118 | fprintf(stderr, "\n"); 119 | } 120 | 121 | 122 | /** 123 | * The main function 124 | * 125 | * @param argc the number of command-line arguments 126 | * @param argv the command-line arguments 127 | * @return the exit code 128 | */ 129 | int main(int argc, char** argv) { 130 | 131 | bool progress = true; 132 | unsigned progress_step = 10000000; 133 | int seed = time(NULL); 134 | 135 | ll_loader_config loader_config; 136 | size_t max_edges = 0; 137 | 138 | 139 | // Pase the command-line arguments 140 | 141 | int option_index = 0; 142 | while (true) { 143 | int c = getopt_long(argc, argv, SHORT_OPTIONS, LONG_OPTIONS, &option_index); 144 | 145 | if (c == -1) break; 146 | 147 | switch (c) { 148 | 149 | case 'h': 150 | usage(argv[0]); 151 | return 0; 152 | 153 | case 'M': 154 | max_edges = atol(optarg); 155 | break; 156 | 157 | case 'S': 158 | seed = atoi(optarg); 159 | break; 160 | 161 | case 'T': 162 | if (loader_config.lc_tmp_dirs.empty()) 163 | loader_config.lc_tmp_dirs.push_back(optarg); 164 | else 165 | loader_config.lc_tmp_dirs[0] = optarg; 166 | break; 167 | 168 | case 'X': 169 | loader_config.lc_xs_buffer_size = (size_t) (atof(optarg) 170 | * 1024ul*1048576ul); 171 | break; 172 | 173 | case '?': 174 | case ':': 175 | return 1; 176 | 177 | default: 178 | abort(); 179 | } 180 | } 181 | 182 | if (optind + 2 != argc) { 183 | usage(argv[0]); 184 | return 1; 185 | } 186 | 187 | srand(seed); 188 | 189 | const char* src = argv[optind]; 190 | const char* dst = argv[optind+1]; 191 | 192 | 193 | // Do it! 194 | 195 | string s_in = src; 196 | FILE* f_in = fopen(s_in.c_str(), "rb"); 197 | if (f_in == NULL) { 198 | perror("fopen"); 199 | return 1; 200 | } 201 | 202 | 203 | // Get the number of edges 204 | 205 | struct stat st; 206 | int fr = fstat(fileno(f_in), &st); 207 | if (fr < 0) { 208 | perror("fstat"); 209 | return 1; 210 | } 211 | size_t num_edges = st.st_size / sizeof(xs1); 212 | 213 | if (num_edges == 0) { 214 | fprintf(stderr, "The input file is empty"); 215 | fclose(f_in); 216 | return 1; 217 | } 218 | 219 | 220 | // Load the edges into an external "sort" (i.e. a shuffler) 221 | 222 | if (progress) fprintf(stderr, "Loading the edges:\n "); 223 | 224 | ll_external_sort external_shuffle(&loader_config); 225 | 226 | unsigned l = 0; 227 | xs1r r; 228 | while (next(f_in, &r.payload)) { 229 | l++; 230 | 231 | if (max_edges > 0 && l > max_edges) break; 232 | 233 | r.random1 = rand(); 234 | r.random2 = rand(); 235 | external_shuffle << r; 236 | 237 | if (progress && (l % progress_step) == 0) { 238 | fprintf(stderr, "."); 239 | if ((l % (progress_step * 10)) == 0) { 240 | fprintf(stderr, "%um", l / 1000000); 241 | } 242 | if ((l % (progress_step * 50)) == 0) { 243 | fprintf(stderr, "\n "); 244 | } 245 | } 246 | } 247 | if (progress) { 248 | if ((l % (progress_step * 50)) >= progress_step) fprintf(stderr, "\n "); 249 | fprintf(stderr, "Done.\n"); 250 | } 251 | 252 | 253 | // Save 254 | 255 | external_shuffle.sort(); 256 | 257 | if (progress) fprintf(stderr, "Saving the edges:\n "); 258 | 259 | string s_out = dst; 260 | FILE* f_out = fopen(s_out.c_str(), "wb"); 261 | if (f_out == NULL) { 262 | perror("fopen"); 263 | return 1; 264 | } 265 | 266 | xs1r* buffer; 267 | size_t length; 268 | 269 | l = 0; 270 | while (external_shuffle.next_block(&buffer, &length)) { 271 | while (length --> 0) { 272 | 273 | ssize_t r = fwrite(&buffer->payload, sizeof(xs1), 1, f_out); 274 | if (r <= 0) { 275 | if (r < 0) { 276 | perror("fwrite"); 277 | } 278 | else { 279 | fprintf(stderr, "Out of space\n"); 280 | } 281 | fclose(f_in); 282 | fclose(f_out); 283 | return 1; 284 | } 285 | 286 | buffer++; 287 | l++; 288 | 289 | if (progress && (l % progress_step) == 0) { 290 | fprintf(stderr, "."); 291 | if ((l % (progress_step * 10)) == 0) { 292 | fprintf(stderr, "%um", l / 1000000); 293 | } 294 | if ((l % (progress_step * 50)) == 0) { 295 | fprintf(stderr, "\n "); 296 | } 297 | } 298 | } 299 | } 300 | if (progress) { 301 | if ((l % (progress_step * 50)) >= progress_step) fprintf(stderr, "\n "); 302 | fprintf(stderr, "Done.\n"); 303 | } 304 | 305 | 306 | // Finalize 307 | 308 | if (f_out) fclose(f_out); 309 | if (f_in) fclose(f_in); 310 | 311 | return 0; 312 | } 313 | 314 | -------------------------------------------------------------------------------- /utils/xs1-split.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * xs1-split.cpp 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | /* 38 | * LLAMA Utilities -- X-Stream tools 39 | * 40 | * Split an X-Stream Type 1 file into 11 files containing 80% and 10 x 2% of 41 | * the edges using a uniform distribution. 42 | * 43 | * DISCLAIMER: This tool was developed separately from X-Stream and without 44 | * their endorsement. 45 | */ 46 | 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include "xs1-common.h" 57 | 58 | using namespace std; 59 | 60 | 61 | static const char* SHORT_OPTIONS = "hs:u"; 62 | 63 | static struct option LONG_OPTIONS[] = 64 | { 65 | {"help" , no_argument , 0, 'h'}, 66 | {"small-files" , required_argument, 0, 's'}, 67 | {"undirected" , no_argument , 0, 'u'}, 68 | {0, 0, 0, 0} 69 | }; 70 | 71 | 72 | /** 73 | * Print the usage information 74 | * 75 | * @param arg0 the first element in the argv array 76 | */ 77 | static void usage(const char* arg0) { 78 | 79 | char* s = strdup(arg0); 80 | char* p = basename(s); 81 | fprintf(stderr, "Usage: %s [OPTIONS] INPUT_FILE\n\n", p); 82 | free(s); 83 | 84 | fprintf(stderr, "Options:\n"); 85 | fprintf(stderr, " -h, --help Show this usage information and exit\n"); 86 | fprintf(stderr, " -s, --small-files N Set the number of small files\n"); 87 | fprintf(stderr, " -u, --undirected Treat the input file as undirected\n"); 88 | fprintf(stderr, "\n"); 89 | } 90 | 91 | 92 | /** 93 | * The main function 94 | * 95 | * @param argc the number of command-line arguments 96 | * @param argv the command-line arguments 97 | * @return the exit code 98 | */ 99 | int main(int argc, char** argv) { 100 | 101 | srand(time(NULL)); 102 | 103 | int n_small_files = 10; 104 | bool undirected = false; 105 | double p_large = 0.80; 106 | 107 | 108 | // Pase the command-line arguments 109 | 110 | int option_index = 0; 111 | while (true) { 112 | int c = getopt_long(argc, argv, SHORT_OPTIONS, LONG_OPTIONS, &option_index); 113 | 114 | if (c == -1) break; 115 | 116 | switch (c) { 117 | 118 | case 'h': 119 | usage(argv[0]); 120 | return 0; 121 | 122 | case 's': 123 | n_small_files = atoi(optarg); 124 | if (n_small_files <= 0) { 125 | fprintf(stderr, "Error: Invalid number of small files\n"); 126 | return 1; 127 | } 128 | break; 129 | 130 | case 'u': 131 | undirected = true; 132 | break; 133 | 134 | case '?': 135 | case ':': 136 | return 1; 137 | 138 | default: 139 | abort(); 140 | } 141 | } 142 | 143 | double p_small = (1 - p_large) / n_small_files; 144 | 145 | if (optind + 1 != argc) { 146 | usage(argv[0]); 147 | return 1; 148 | } 149 | 150 | const char* src = argv[optind]; 151 | 152 | 153 | // Do it! 154 | 155 | string s_in = src; 156 | FILE* f_in = fopen(s_in.c_str(), "rb"); 157 | if (f_in == NULL) { 158 | perror("fopen"); 159 | return 1; 160 | } 161 | 162 | struct stat st; 163 | int fr = fstat(fileno(f_in), &st); 164 | if (fr < 0) { 165 | perror("fstat"); 166 | return 1; 167 | } 168 | size_t lines = st.st_size / sizeof(xs1); 169 | 170 | size_t expected_out_counts[1 + n_small_files]; 171 | size_t expected_out_counts_sum = 0; 172 | expected_out_counts[0] = (size_t) (p_large * lines); 173 | expected_out_counts_sum = expected_out_counts[0]; 174 | for (int i = 1; i <= n_small_files; i++) { 175 | expected_out_counts[i] = (size_t) (p_small * lines); 176 | expected_out_counts_sum += expected_out_counts[i]; 177 | } 178 | if (expected_out_counts_sum > lines) abort(); 179 | expected_out_counts[0] += lines - expected_out_counts_sum; 180 | 181 | size_t out_counts[1 + n_small_files]; 182 | for (int i = 0; i <= n_small_files; i++) out_counts[i] = 0; 183 | 184 | char s_out_base[s_in.length() + 4]; 185 | strcpy(s_out_base, s_in.c_str()); 186 | size_t l = strlen(s_out_base); 187 | if (l > 4 && strcmp(s_out_base + (l - 4), ".dat") == 0) { 188 | *(s_out_base + (l -4)) = '\0'; 189 | } 190 | 191 | FILE* out_files[1 + n_small_files]; 192 | for (int i = 0; i <= n_small_files; i++) { 193 | string s = s_out_base; 194 | char b[16]; sprintf(b, "-part%d.dat", i); 195 | s += b; 196 | out_files[i] = fopen(s.c_str(), "wb"); 197 | if (out_files[i] == NULL) { 198 | perror("fopen"); 199 | return 1; 200 | } 201 | } 202 | 203 | xs1 x, x2; 204 | l = 0; 205 | while (next(f_in, &x)) { 206 | l++; 207 | 208 | double r = rand() / (double) RAND_MAX; 209 | int t = r < p_large ? 0 : 1 + (int) ((r - p_large) / (p_small)); 210 | if (t >= 1 + n_small_files) t = n_small_files + 1; 211 | for (int i = 0; i <= n_small_files; i++) { 212 | if (out_counts[t] < expected_out_counts[t]) break; 213 | t = (t + 1) % (1 + n_small_files); 214 | } 215 | 216 | out_counts[t]++; 217 | ssize_t q = fwrite(&x, sizeof(xs1), 1, out_files[t]); 218 | if (q <= 0) { 219 | perror("fwrite"); 220 | return 1; 221 | } 222 | 223 | if (undirected) { 224 | l++; 225 | if (!next(f_in, &x2)) { 226 | fprintf(stderr, "Odd number of edges\n"); 227 | return 1; 228 | } 229 | if (x2.head != x.tail || x2.tail != x.head) { 230 | fprintf(stderr, "Not an undirected edge\n"); 231 | return 1; 232 | } 233 | out_counts[t]++; 234 | ssize_t q = fwrite(&x2, sizeof(xs1), 1, out_files[t]); 235 | if (q <= 0) { 236 | perror("fwrite"); 237 | return 1; 238 | } 239 | } 240 | 241 | if ((l % 10000000ul) == 0) { 242 | fprintf(stderr, "."); 243 | if ((l % 100000000ul) == 0) fprintf(stderr, "%lu", l / 1000000ul); 244 | } 245 | } 246 | fprintf(stderr, "\n"); 247 | 248 | for (int i = 0; i <= n_small_files; i++) { 249 | fclose(out_files[i]); 250 | } 251 | fclose(f_in); 252 | return 0; 253 | } 254 | 255 | -------------------------------------------------------------------------------- /utils/xs1-to-snap.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * xs1-to-snap.cpp 3 | * LLAMA Graph Analytics 4 | * 5 | * Copyright 2014 6 | * The President and Fellows of Harvard College. 7 | * 8 | * Copyright 2014 9 | * Oracle Labs. 10 | * 11 | * Redistribution and use in source and binary forms, with or without 12 | * modification, are permitted provided that the following conditions 13 | * are met: 14 | * 1. Redistributions of source code must retain the above copyright 15 | * notice, this list of conditions and the following disclaimer. 16 | * 2. Redistributions in binary form must reproduce the above copyright 17 | * notice, this list of conditions and the following disclaimer in the 18 | * documentation and/or other materials provided with the distribution. 19 | * 3. Neither the name of the University nor the names of its contributors 20 | * may be used to endorse or promote products derived from this software 21 | * without specific prior written permission. 22 | * 23 | * THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY AND CONTRIBUTORS ``AS IS'' AND 24 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 | * ARE DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE 27 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 | * SUCH DAMAGE. 34 | */ 35 | 36 | 37 | /* 38 | * LLAMA Utilities -- X-Stream tools 39 | * 40 | * Convert a Type 1 file to the SNAP format 41 | * 42 | * DISCLAIMER: This tool was developed separately from X-Stream and without 43 | * their endorsement. 44 | */ 45 | 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include "xs1-common.h" 54 | 55 | using namespace std; 56 | 57 | 58 | /** 59 | * The main function 60 | * 61 | * @param argc the number of command-line arguments 62 | * @param argv the command-line arguments 63 | * @return the exit code 64 | */ 65 | int main(int argc, char** argv) { 66 | 67 | FILE* f_out = stdout; 68 | 69 | if (argc != 2) { 70 | fprintf(stderr, "Usage: %s FILE\n", argv[0]); 71 | return 1; 72 | } 73 | 74 | string s_in = argv[1]; 75 | FILE* f_in = fopen(s_in.c_str(), "rb"); 76 | if (f_in == NULL) { 77 | perror("fopen"); 78 | return 1; 79 | } 80 | 81 | xs1 x; 82 | while (next(f_in, &x)) { 83 | fprintf(f_out, "%u\t%u\n", x.tail, x.head); 84 | } 85 | 86 | fclose(f_in); 87 | if (f_out != stdout && f_out != stderr) fclose(f_out); 88 | 89 | return 0; 90 | } 91 | 92 | --------------------------------------------------------------------------------