├── .cproject ├── .gitignore ├── .project ├── .settings ├── language.settings.xml ├── org.eclipse.cdt.codan.core.prefs ├── org.eclipse.cdt.core.prefs └── org.eclipse.cdt.ui.prefs ├── LICENSE.txt ├── Makefile ├── README.md ├── block_analysis ├── DENSITIES ├── banalysis.m ├── banalysis_graph.m ├── btheoricalanalysis.m ├── btheoricalanalysis2.m ├── btheoricalanalysis3.m ├── btheoricalanalysisDensity.m ├── counters ├── empiricalanalysis.m ├── fillMemory.m ├── getIOCost.m ├── lj1 ├── lj1000 ├── lj2 ├── lj20 ├── lj200 ├── lj4 ├── lj40 ├── ouput ├── test ├── test.m ├── twitter1 ├── twitter10 ├── twitter2 ├── twitter20 ├── twitter200 ├── twitter4 ├── twitter40 ├── twitter5 ├── yahoo1 ├── yahoo1000 ├── yahoo2 ├── yahoo20 ├── yahoo200 ├── yahoo4 └── yahoo40 ├── conf └── mflash.cnf ├── example_apps ├── lancsosso.cpp ├── pagerank.cpp └── wcc.cpp ├── src ├── algorithm │ ├── lanczosSO.hpp │ ├── pagerank.hpp │ └── wcc.hpp ├── core │ ├── array.hpp │ ├── blockiterator.hpp │ ├── conversions.hpp │ ├── edgeconversor.hpp │ ├── edgelistconversor.hpp │ ├── edgelistthread.hpp │ ├── edgesplittermanager.hpp │ ├── edgesplittermanagerextended.hpp │ ├── edgesplittermanagerwithblockcounting.hpp │ ├── eigenwrapper.hpp │ ├── genericedgesplittermanager.hpp │ ├── linearcombination.hpp │ ├── malgorithm.hpp │ ├── mapped_stream.hpp │ ├── matrix.hpp │ ├── matrixworker.hpp │ ├── operator.hpp │ ├── primitivematrix.hpp │ ├── primitivevector.hpp │ ├── splitterbuffer.hpp │ ├── type.hpp │ ├── util.hpp │ └── vector.hpp ├── mflash_basic_includes.hpp └── util │ ├── cmdopts.hpp │ ├── configfile.hpp │ ├── easylogging++.h │ ├── kwaymergesort.h │ └── mflashnames.hpp └── test ├── ArrayTest.cpp ├── BinarySplitterTest.cpp ├── BlockEdgeTest.cpp ├── BoolTest.cpp ├── MMapGraphFormatter.cpp ├── MMapGraphFormatter2.cpp ├── MMapGraphFormatter3.cpp ├── PreprocessorTest.cpp ├── PrintVectorTest.cpp ├── SortedTest.cpp ├── SplitterTest.cpp ├── TestMergeInt64.cpp ├── TestPlainBlockCounters.cpp ├── TurboGraphFormatter.cpp ├── XStreamGraphFormatter.cpp ├── graphchi_test ├── log_test.cpp ├── main.cpp ├── next ├── pagerank.py ├── pg ├── sparse-graph ├── sparse-graph-shuffle ├── sparse-graph-shuffle_1first_round ├── sparse-graph-shuffle_2_round ├── tiny-graph ├── xtream-lj.ini ├── xtream-powerlawgraph.ini ├── xtream-twiiter.ini ├── xtream-yahoo.ini └── xtream_test /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | # ignoring hidden folders and files 3 | /.metadata 4 | /.recommenders 5 | /RemoteSystemsTempFiles 6 | Debug/ 7 | */Debug/* 8 | */Debug/** 9 | bin/ 10 | */bin/* 11 | */bin/** 12 | logs/ 13 | */logs/* 14 | */logs/** 15 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | m-flash-cpp 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.cdt.managedbuilder.core.genmakebuilder 10 | clean,full,incremental, 11 | 12 | 13 | 14 | 15 | org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder 16 | full,incremental, 17 | 18 | 19 | 20 | 21 | 22 | org.eclipse.cdt.core.cnature 23 | org.eclipse.cdt.core.ccnature 24 | org.eclipse.cdt.managedbuilder.core.managedBuildNature 25 | org.eclipse.cdt.managedbuilder.core.ScannerConfigNature 26 | 27 | 28 | -------------------------------------------------------------------------------- /.settings/language.settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /.settings/org.eclipse.cdt.codan.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.cdt.codan.checkers.errnoreturn=Warning 3 | org.eclipse.cdt.codan.checkers.errnoreturn.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},implicit\=>false} 4 | org.eclipse.cdt.codan.checkers.errreturnvalue=Error 5 | org.eclipse.cdt.codan.checkers.errreturnvalue.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 6 | org.eclipse.cdt.codan.checkers.noreturn=Error 7 | org.eclipse.cdt.codan.checkers.noreturn.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},implicit\=>false} 8 | org.eclipse.cdt.codan.internal.checkers.AbstractClassCreation=Error 9 | org.eclipse.cdt.codan.internal.checkers.AbstractClassCreation.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 10 | org.eclipse.cdt.codan.internal.checkers.AmbiguousProblem=Error 11 | org.eclipse.cdt.codan.internal.checkers.AmbiguousProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 12 | org.eclipse.cdt.codan.internal.checkers.AssignmentInConditionProblem=Warning 13 | org.eclipse.cdt.codan.internal.checkers.AssignmentInConditionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 14 | org.eclipse.cdt.codan.internal.checkers.AssignmentToItselfProblem=Error 15 | org.eclipse.cdt.codan.internal.checkers.AssignmentToItselfProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 16 | org.eclipse.cdt.codan.internal.checkers.CaseBreakProblem=Warning 17 | org.eclipse.cdt.codan.internal.checkers.CaseBreakProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},no_break_comment\=>"no break",last_case_param\=>false,empty_case_param\=>false} 18 | org.eclipse.cdt.codan.internal.checkers.CatchByReference=Warning 19 | org.eclipse.cdt.codan.internal.checkers.CatchByReference.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},unknown\=>false,exceptions\=>()} 20 | org.eclipse.cdt.codan.internal.checkers.CircularReferenceProblem=Error 21 | org.eclipse.cdt.codan.internal.checkers.CircularReferenceProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 22 | org.eclipse.cdt.codan.internal.checkers.ClassMembersInitialization=Warning 23 | org.eclipse.cdt.codan.internal.checkers.ClassMembersInitialization.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},skip\=>true} 24 | org.eclipse.cdt.codan.internal.checkers.FieldResolutionProblem=Error 25 | org.eclipse.cdt.codan.internal.checkers.FieldResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 26 | org.eclipse.cdt.codan.internal.checkers.FunctionResolutionProblem=Error 27 | org.eclipse.cdt.codan.internal.checkers.FunctionResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 28 | org.eclipse.cdt.codan.internal.checkers.InvalidArguments=Error 29 | org.eclipse.cdt.codan.internal.checkers.InvalidArguments.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 30 | org.eclipse.cdt.codan.internal.checkers.InvalidTemplateArgumentsProblem=Error 31 | org.eclipse.cdt.codan.internal.checkers.InvalidTemplateArgumentsProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 32 | org.eclipse.cdt.codan.internal.checkers.LabelStatementNotFoundProblem=Error 33 | org.eclipse.cdt.codan.internal.checkers.LabelStatementNotFoundProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 34 | org.eclipse.cdt.codan.internal.checkers.MemberDeclarationNotFoundProblem=Error 35 | org.eclipse.cdt.codan.internal.checkers.MemberDeclarationNotFoundProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 36 | org.eclipse.cdt.codan.internal.checkers.MethodResolutionProblem=Error 37 | org.eclipse.cdt.codan.internal.checkers.MethodResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 38 | org.eclipse.cdt.codan.internal.checkers.NamingConventionFunctionChecker=-Info 39 | org.eclipse.cdt.codan.internal.checkers.NamingConventionFunctionChecker.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},pattern\=>"^[a-z]",macro\=>true,exceptions\=>()} 40 | org.eclipse.cdt.codan.internal.checkers.NonVirtualDestructorProblem=Warning 41 | org.eclipse.cdt.codan.internal.checkers.NonVirtualDestructorProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 42 | org.eclipse.cdt.codan.internal.checkers.OverloadProblem=Error 43 | org.eclipse.cdt.codan.internal.checkers.OverloadProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 44 | org.eclipse.cdt.codan.internal.checkers.RedeclarationProblem=Error 45 | org.eclipse.cdt.codan.internal.checkers.RedeclarationProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 46 | org.eclipse.cdt.codan.internal.checkers.RedefinitionProblem=Error 47 | org.eclipse.cdt.codan.internal.checkers.RedefinitionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 48 | org.eclipse.cdt.codan.internal.checkers.ReturnStyleProblem=-Warning 49 | org.eclipse.cdt.codan.internal.checkers.ReturnStyleProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 50 | org.eclipse.cdt.codan.internal.checkers.ScanfFormatStringSecurityProblem=-Warning 51 | org.eclipse.cdt.codan.internal.checkers.ScanfFormatStringSecurityProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 52 | org.eclipse.cdt.codan.internal.checkers.StatementHasNoEffectProblem=Warning 53 | org.eclipse.cdt.codan.internal.checkers.StatementHasNoEffectProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},macro\=>true,exceptions\=>()} 54 | org.eclipse.cdt.codan.internal.checkers.SuggestedParenthesisProblem=Warning 55 | org.eclipse.cdt.codan.internal.checkers.SuggestedParenthesisProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},paramNot\=>false} 56 | org.eclipse.cdt.codan.internal.checkers.SuspiciousSemicolonProblem=Warning 57 | org.eclipse.cdt.codan.internal.checkers.SuspiciousSemicolonProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},else\=>false,afterelse\=>false} 58 | org.eclipse.cdt.codan.internal.checkers.TypeResolutionProblem=Error 59 | org.eclipse.cdt.codan.internal.checkers.TypeResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 60 | org.eclipse.cdt.codan.internal.checkers.UnusedFunctionDeclarationProblem=Warning 61 | org.eclipse.cdt.codan.internal.checkers.UnusedFunctionDeclarationProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},macro\=>true} 62 | org.eclipse.cdt.codan.internal.checkers.UnusedStaticFunctionProblem=Warning 63 | org.eclipse.cdt.codan.internal.checkers.UnusedStaticFunctionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},macro\=>true} 64 | org.eclipse.cdt.codan.internal.checkers.UnusedVariableDeclarationProblem=Warning 65 | org.eclipse.cdt.codan.internal.checkers.UnusedVariableDeclarationProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true},macro\=>true,exceptions\=>("@(\#)","$Id")} 66 | org.eclipse.cdt.codan.internal.checkers.VariableResolutionProblem=Error 67 | org.eclipse.cdt.codan.internal.checkers.VariableResolutionProblem.params={launchModes\=>{RUN_ON_FULL_BUILD\=>true,RUN_ON_INC_BUILD\=>true,RUN_ON_FILE_OPEN\=>false,RUN_ON_FILE_SAVE\=>false,RUN_AS_YOU_TYPE\=>true,RUN_ON_DEMAND\=>true}} 68 | -------------------------------------------------------------------------------- /.settings/org.eclipse.cdt.ui.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | formatter_profile=_m-flash 3 | formatter_settings_version=1 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile 2 | # 3 | # Replace GRAPHCHI_DIR and BOOST_DIR with your proper library directories. 4 | 5 | MFLASH_DIR = -I. 6 | BOOST_DIR = -I/usr/lib64/ -I/usr/lib/x86_64-linux-gnu/ 7 | BOOST_LIBRARIES = -lboost_filesystem -lboost_system 8 | SO_LIBRARIES = 9 | 10 | INCLUDES = $(MFLASH_DIR) $(BOOST_DIR) 11 | 12 | CPP = g++ 13 | CPPFLAGS = -O3 -J4 $(INCLUDES) -Wall -Wno-strict-aliasing -std=c++11 14 | CPPFLAGS-DEBUG = -g -J4 $(INCLUDES) -Wall -Wno-strict-aliasing -std=c++11 15 | LINKERFLAGS = $(BOOST_LIBRARIES) $(SO_LIBRARIES) 16 | DEBUGFLAGS = -g -ggdb $(INCFLAGS) 17 | HEADERS=$(shell find . -name '*.hpp') 18 | 19 | FILE_PATH = $(basename $(shell echo $@ | sed -E 's/^(debug|build)\///g')) 20 | FILENAME = $(notdir $(FILE_PATH)) 21 | SOURCE_FILE = $(addsuffix .cpp,$(FILE_PATH)) 22 | 23 | 24 | all: apps 25 | apps: build/example_apps/lancsosso.cpp build/example_apps/pagerank.cpp build/example_apps/wcc.cpp 26 | 27 | echo/%: 28 | echo bin/$(@F) 29 | 30 | clean: 31 | @rm -rf bin/* 32 | 33 | debug/%: $(SOURCE_FILE) $(HEADERS) 34 | @mkdir -p bin/$(dir $(FILE_PATH)) 35 | $(CPP) $(CPPFLAGS-DEBUG) -I. $(FILE_PATH).cpp -o bin/$(FILE_PATH) $(LINKERFLAGS) 36 | 37 | build/%: $(SOURCE_FILE) $(HEADERS) 38 | @mkdir -p bin/$(dir $(FILE_PATH)) 39 | $(CPP) $(CPPFLAGS) -I. $(FILE_PATH).cpp -o bin/$(FILE_PATH) $(LINKERFLAGS) 40 | 41 | example_apps/%: $(SOURCE_FILE) $(HEADERS) 42 | @mkdir -p bin/$(dir $(FILE_PATH)) 43 | $(CPP) $(CPPFLAGS) -I. $(FILE_PATH).cpp -o bin/$(FILE_PATH) $(LINKERFLAGS) 44 | 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # M-Flash: Fast billion-scale graph computation using a bimodal block processing model 2 | 3 | ## Dependencies 4 | * [EasyLogging++](http://easylogging.muflihun.com). It is included in M-Flash 5 | * [Boost](http://www.boost.org) (CentOS yum install boost boost-devel) 6 | * [Eigen3](http://eigen.tuxfamily.org/). It is required for Lanczos Selective Orthogonalization 7 | 8 | ## Examples 9 | 10 | ### Pagerank 11 | ``` 12 | cd [M-FLASH_ROOT]; 13 | make example_apps/pagerank 14 | bin/example_apps/pagerank file [FILE_GRAPH] niters 4 15 | ``` 16 | 17 | ### Weakly Connected Components (Iterative) 18 | ``` 19 | cd [M-FLASH_ROOT]; 20 | make example_apps/wcc 21 | bin/example_apps/wcc file [FILE_GRAPH] iterative 1 niters 4 22 | ``` 23 | Iterative until convergency 24 | ``` 25 | cd [M-FLASH_ROOT]; 26 | make example_apps/wcc 27 | bin/example_apps/wcc file [FILE_GRAPH] iterative 1 28 | ``` 29 | 30 | ### Weakly Connected Components (Union-Find) 31 | ``` 32 | cd [M-FLASH_ROOT]; 33 | make example_apps/wcc 34 | bin/example_apps/wcc file [FILE_GRAPH] iterative 0 35 | ``` 36 | 37 | 38 | ## License 39 | M-Flash is licensed under the Mozilla Public License, Version 2.0. 40 | -------------------------------------------------------------------------------- /block_analysis/banalysis.m: -------------------------------------------------------------------------------- 1 | function [ output_args ] = banalysis( partitions_counters, vertices, vertices_partition) 2 | %BANALYSIS Summary of this function goes here 3 | % Detailed explanation goes here 4 | vertex_size = 1; 5 | edge_size = 2; %* vertex_size; 6 | cost = sum(partitions_counters(:))*edge_size + 2*vertices*vertex_size; 7 | threads = 1; 8 | 9 | partitions = length(partitions_counters); 10 | 11 | nbpCost = 0; 12 | sbpCost = 0; 13 | mCost = 0; 14 | sparse_blocks = 0; 15 | dense_blocks = 0; 16 | for i=1:partitions 17 | for j=1:partitions 18 | block_vertices = vertices_partition; 19 | block_edges = partitions_counters(i,j); 20 | [nbp, sbp] = getIOCost(block_vertices, block_edges, vertex_size, edge_size, partitions); 21 | if nbp < sbp 22 | dense_blocks = dense_blocks + 1; 23 | else 24 | sparse_blocks = sparse_blocks + 1; 25 | end 26 | mCost = mCost + min(nbp, sbp); 27 | nbpCost = nbpCost + nbp; 28 | sbpCost = sbpCost + sbp; 29 | end 30 | end 31 | 32 | bar([nbpCost sbpCost mCost]/cost ); 33 | set(gca,'xticklabel', {'NBP' 'SBP' 'MFlash'}) 34 | ylabel('COST'); 35 | figure, 36 | bar([dense_blocks sparse_blocks]); 37 | set(gca,'xticklabel', {'Dense Blocks' 'Sparse Blocks'} ) 38 | end 39 | 40 | -------------------------------------------------------------------------------- /block_analysis/banalysis_graph.m: -------------------------------------------------------------------------------- 1 | function [ output_args ] = banalysis_graph(path, graph, vertices) 2 | vertex_size = 1; 3 | edge_size = 2; %* vertex_size; 4 | threads = 1; 5 | 6 | index = 1; 7 | pvalues = fliplr([2 4 20 40 200]); 8 | counters = zeros(length(pvalues), 3); 9 | blockCounters = zeros(length(pvalues), 2); 10 | vertex_percentage = 1./pvalues* 200; 11 | for p = pvalues 12 | partitions_counters = load(strcat(path, graph, num2str(p))); 13 | cost = sum(partitions_counters(:))*edge_size + 2*vertices*vertex_size; 14 | partitions = length(partitions_counters); 15 | vertices_partition = floor(vertices/p); 16 | mCost = 0; 17 | nbpCost = 0; 18 | sbpCost = 0; 19 | sparse_blocks = 0; 20 | dense_blocks = 0; 21 | for i=1:partitions 22 | for j=1:partitions 23 | block_vertices = vertices_partition; 24 | block_edges = partitions_counters(i,j); 25 | [nbp, sbp] = getIOCost(block_vertices, block_edges, vertex_size, edge_size, partitions); 26 | if nbp < sbp 27 | dense_blocks = dense_blocks + 1; 28 | else 29 | sparse_blocks = sparse_blocks + 1; 30 | end 31 | mCost = mCost + min(nbp, sbp); 32 | nbpCost = nbpCost + nbp; 33 | sbpCost = sbpCost + sbp; 34 | end 35 | end 36 | counters(index, :) = [nbpCost, sbpCost, mCost]/cost; 37 | blockCounters(index, :) = [dense_blocks, sparse_blocks]; 38 | blockCounters(index, :) = [dense_blocks, sparse_blocks]; 39 | index = index + 1; 40 | end 41 | figure; 42 | bar(counters, 'grouped'); 43 | title(sprintf('I/O cost for %s graph using different memory sizes', graph)); 44 | set(gca,'xticklabel', num2str(vertex_percentage', '%0.0f%%')); 45 | ylabel('t-cost'); 46 | xlabel('Percentage of Vertices in RAM'); 47 | legend({'DBP' 'SPP' 'BBP'}); 48 | saveas(gcf, strcat(path, graph, '.fig')); 49 | saveas(gcf, strcat(path, graph, '.pdf')); 50 | saveas(gcf, strcat(path, graph, '.png')); 51 | figure, 52 | bar(blockCounters, 'grouped'); 53 | title(sprintf('Blocks for %s graph using different memory sizes', graph)); 54 | set(gca,'xticklabel', num2str(vertex_percentage', '%0.0f%%')); 55 | legend ({'Dense Blocks' 'Sparse Blocks'} ) 56 | ylabel('COST'); 57 | saveas(gcf, strcat(path, graph, '-blocks', '.fig')); 58 | saveas(gcf, strcat(path, graph, '-blocks', '.pdf')); 59 | close all; 60 | end 61 | 62 | -------------------------------------------------------------------------------- /block_analysis/btheoricalanalysis.m: -------------------------------------------------------------------------------- 1 | function [ r ] = btheoricalanalysis( graph_density) 2 | %BANALYSIS Summary of this function goes here 3 | % Detailed explanation goes here 4 | ram_size = [1:250]; 5 | vertex_size = 1; 6 | edge_size = 2; %* vertex_size; 7 | vertices = 1; 8 | edges = vertices * graph_density; 9 | values = zeros(length(ram_size), 4); 10 | cost = edges*edge_size + 2*vertices*vertex_size; 11 | index = 1; 12 | threads = 1; 13 | for ram = ram_size 14 | partitions = max(threads * 2 * 100 / ram, 1); 15 | block_edges = edges / (partitions^2); 16 | block_vertices = vertices/ partitions; 17 | [nbp, sbp] = getIOCost(block_vertices, block_edges, vertex_size, edge_size, partitions); 18 | nbp = nbp * partitions^2; 19 | sbp = sbp * partitions^2; 20 | values(index, :) = [nbp, sbp, min(nbp, sbp), partitions]; 21 | index = index +1; 22 | end 23 | values = [values(:,1:3)/cost values(:,4)]; 24 | %logvalues = log(values); 25 | plot(values(:,4), values(:,1:3), '-'); 26 | r = [ram_size' values]; 27 | xlabel('Vertex partitions \Beta') 28 | ylabel('COST Units') 29 | 30 | end 31 | 32 | -------------------------------------------------------------------------------- /block_analysis/btheoricalanalysis2.m: -------------------------------------------------------------------------------- 1 | function [ r ] = btheoricalanalysis2( graph_density) 2 | %BANALYSIS Summary of this function goes here 3 | % Detailed explanation goes here 4 | ram_size = [0.1:0.1:1 1:250]; 5 | vertex_size = 1; 6 | edge_size = 2; %* vertex_size; 7 | vertices = 1; 8 | edges = vertices * graph_density; 9 | index = 1; 10 | threads = 1; 11 | 12 | densities = [0.01 0.05 0.1 0.5 1 5 10 50 100]; 13 | values = zeros(length(densities), 3); 14 | 15 | index = 1; 16 | threads = 1; 17 | for ram = ram_size 18 | partitions = max(threads * 2 * 100 / ram, 1); 19 | block_edges = edges / (partitions^2); 20 | block_vertices = vertices/ partitions; 21 | [nbp, sbp] = getIOCost(block_vertices, block_edges, vertex_size, edge_size, partitions); 22 | nbp = nbp * partitions^2; 23 | sbp = sbp * partitions^2; 24 | values(index, :) = [nbp, sbp, min(nbp, sbp), partitions]; 25 | index = index +1; 26 | end 27 | logvalues = values; 28 | logvalues = log(values); 29 | plot(ram_size, logvalues(:,1:3), '-'); 30 | r = [ram_size' values]; 31 | 32 | end 33 | 34 | -------------------------------------------------------------------------------- /block_analysis/btheoricalanalysis3.m: -------------------------------------------------------------------------------- 1 | function [ r ] = btheoricalanalysis3( graph_density) 2 | %BANALYSIS Summary of this function goes here 3 | % Detailed explanation goes here 4 | ram_size = fliplr([0.01:0.01:0.99 1:250]); 5 | vertex_size = 1; 6 | edge_size = 2; %* vertex_size; 7 | vertices = 1; 8 | index = 1; 9 | threads = 1; 10 | 11 | values = zeros(length(ram_size), 3); 12 | %densities = [1 2.5 5 10 25 50 100] 13 | 14 | 15 | for ram = ram_size 16 | partitions = max(threads * 2 * 100 / ram, 1); 17 | ppartitions = partitions^2; 18 | block_vertices = vertices/ partitions; 19 | threshold_edges = ( (partitions-1) * block_vertices )/ (2 * partitions * edge_size) 20 | cost = ppartitions* threshold_edges * edge_size + 2*vertices*vertex_size; 21 | [nbp, sbp] = getIOCost(block_vertices, threshold_edges, vertex_size, edge_size, partitions); 22 | values(index,:) = [nbp*ppartitions/cost threshold_edges*ppartitions/vertices threshold_edges/block_vertices]; 23 | index = index +1; 24 | end 25 | %logvalues = log(values); 26 | plot(ram_size, values(:,1), '-'); 27 | r = [ram_size' values]; 28 | title('Marginal I/O Cost for 1 iteration on M-Flash') 29 | ylabel('COST Units') 30 | xlabel('Percentage of vertices in RAM '); 31 | ylim([ 0 4]); 32 | 33 | end 34 | 35 | -------------------------------------------------------------------------------- /block_analysis/btheoricalanalysisDensity.m: -------------------------------------------------------------------------------- 1 | function [ r ] = btheoricalanalysisDensity( graph_density) 2 | %BANALYSIS Summary of this function goes here 3 | % Detailed explanation goes here 4 | ram_size = [1:100]; 5 | vertex_size = 1; 6 | edge_size = 2; %* vertex_size; 7 | vertices = 1; 8 | edges = vertices * graph_density; 9 | values = zeros(length(ram_size), 4); 10 | cost = edges*edge_size + 2*vertices*vertex_size; 11 | index = 1; 12 | threads = 1; 13 | 14 | %densities = [1 2.5 5 10 25 50 100] 15 | 16 | for ram = ram_size 17 | partitions = max(threads * 2 * 100 / ram, 1); 18 | block_edges = edges / (partitions^2); 19 | block_vertices = vertices/ partitions; 20 | [nbp, sbp] = getIOCost(block_vertices, block_edges, vertex_size, edge_size, partitions); 21 | nbp = nbp * (partitions^2); 22 | sbp = sbp * (partitions^2); 23 | values(index, :) = [nbp, sbp, min(nbp, sbp), partitions]; 24 | index = index +1; 25 | end 26 | values = [values(:,1:3)/cost values(:,4)]; 27 | %logvalues = log(values); 28 | plot(ram_size, values(:,1:3), '-'); 29 | r = [ram_size' values]; 30 | title(sprintf('I/O Cost for a graph with density %d', graph_density)); 31 | xlabel('Percentage of vertices in RAM ') 32 | ylabel('t-cost units') 33 | ylim([ 0 4]); 34 | legend('DBP', 'SPP', 'BBP') 35 | 36 | end 37 | 38 | -------------------------------------------------------------------------------- /block_analysis/counters: -------------------------------------------------------------------------------- 1 | 2 | Vertex percentage 3 | 1 5 10 50 100 4 | 5 | Twitter 6 | 3.7446 2.9448 1.8618 7 | 1.5379 2.9448 1.4097 8 | 1.2620 2.9448 1.2431 9 | 1.0414 2.9448 1.0414 10 | 1.0138 2.9448 1.0138 11 | 12 | 13 | Yahoo 14 | 3.7446 2.9448 1.8618 15 | 1.5379 2.9448 1.4097 16 | 1.2620 2.9448 1.2431 17 | 1.0414 2.9448 1.0414 18 | 1.0138 2.9448 1.0138 19 | 20 | 21 | live journal 22 | 23 | 7.5780 2.8678 2.1240 24 | 2.2892 2.8678 1.6403 25 | 1.6281 2.8678 1.4295 26 | 1.0992 2.8678 1.0992 27 | 1.0331 2.8678 1.0331 28 | -------------------------------------------------------------------------------- /block_analysis/empiricalanalysis.m: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /block_analysis/fillMemory.m: -------------------------------------------------------------------------------- 1 | function [] = fillMemory(current_size, expected_size) 2 | vector_size = (current_size - expected_size)/8; 3 | vector = ones(vector_size, 1); 4 | pause; 5 | end 6 | -------------------------------------------------------------------------------- /block_analysis/getIOCost.m: -------------------------------------------------------------------------------- 1 | function [nbp sbp] = getIOCost(vertices, edges, vertex_size, edge_size, partitions) 2 | nbp = vertices * vertex_size + (vertices * vertex_size) /partitions + edges * edge_size; 3 | sbp = edges * edge_size + (2 * vertex_size* vertices) / partitions + 2 * edges * edge_size * vertex_size; 4 | end -------------------------------------------------------------------------------- /block_analysis/lj1: -------------------------------------------------------------------------------- 1 | 68475391 2 | -------------------------------------------------------------------------------- /block_analysis/lj2: -------------------------------------------------------------------------------- 1 | 52901968 5171189 2 | 5112894 5289340 3 | -------------------------------------------------------------------------------- /block_analysis/lj20: -------------------------------------------------------------------------------- 1 | 5534180 1963428 2069316 1056027 892813 569696 357390 353303 260307 198245 199689 267489 145578 2934 4567 4359 4293 5120 3768 6449 2 | 2164196 2465354 915548 715235 484241 386943 325862 234215 196435 133331 129197 78929 149634 204897 1878 1738 2134 2000 2262 3764 3 | 2427165 952591 2539341 580006 678461 405424 290625 284914 220045 161344 109418 78261 57934 93798 188255 2649 2448 3569 2038 4277 4 | 1179998 742124 596581 1218632 425429 352012 235429 197021 168682 134015 123039 79541 77776 55714 90801 125618 1289 1640 1604 2050 5 | 1102658 514571 724206 439099 1052111 312750 192636 196511 155025 121907 99895 73382 57317 51510 38800 129969 30960 1838 1153 2005 6 | 670114 416514 433659 367543 302539 722366 253066 177737 141886 112436 103927 76138 67446 53467 41030 36119 131806 1172 1093 2109 7 | 404908 343147 292753 240059 189614 221714 1472855 141363 140719 98474 149399 73454 64598 89168 44901 34949 105813 43466 3063 3474 8 | 417895 248579 301923 203964 201498 177457 138891 469850 115339 93797 85352 64632 54534 51664 40287 30550 29117 103847 977 1630 9 | 304131 194925 230242 174321 157970 142188 139391 114914 512848 82891 78578 59171 48575 48431 39147 30512 27864 95941 544 1114 10 | 228009 139842 168626 138987 122669 112022 94957 92363 83079 277581 59392 47781 41356 37919 29249 24294 22550 37917 41557 880 11 | 221268 135992 113087 128317 99249 103959 140069 85418 77837 59374 439994 68437 50146 51896 37135 32615 31073 25805 75474 998 12 | 275638 81452 79303 82433 70972 74123 73218 63176 58726 46463 64221 203277 45918 46849 31054 27566 26272 21772 67255 651 13 | 146786 151052 58777 81700 58004 67752 65366 53666 47781 41037 48542 46674 260275 44106 30727 28619 24095 21141 62770 764 14 | 3764 200540 90656 56005 49641 52828 81066 51054 49015 37083 50022 46532 46821 332413 33957 27342 28460 26812 25990 41054 15 | 5734 1788 177549 90453 38212 39296 43004 39319 37873 27683 34434 31353 31024 32918 212548 21983 22824 23656 18764 39718 16 | 5316 1720 2523 119790 120978 36200 34264 29458 29322 23238 31108 27130 27750 25713 21551 146015 19980 17460 16297 36997 17 | 5787 2409 3053 1269 29167 120721 99698 27816 25752 21411 28561 25637 23804 27250 21713 19818 200608 22483 32763 40857 18 | 6478 2229 3689 1531 1742 997 36003 89992 84242 33550 23407 20368 20491 24922 21121 16767 20987 226219 22190 43644 19 | 4465 2633 2431 1642 1158 1198 6787 1398 558 34398 64316 60000 56968 22352 17665 15325 31771 20157 205173 43564 20 | 8844 4387 5726 2410 2549 3684 8315 2741 1501 1162 2641 755 926 35758 35057 31741 35526 37117 38437 235721 21 | -------------------------------------------------------------------------------- /block_analysis/lj4: -------------------------------------------------------------------------------- 1 | 33433303 6444065 2460237 348996 2 | 6894431 6130173 1549602 812354 3 | 2498370 1516191 2321271 723478 4 | 343932 754402 666969 1577617 5 | -------------------------------------------------------------------------------- /block_analysis/lj40: -------------------------------------------------------------------------------- 1 | 2710576 697127 497151 499097 871599 584286 278757 277931 436756 156742 142769 206183 102421 84862 86588 137602 72359 87606 58529 61933 41302 86030 119695 64695 340 427 917 1022 2386 922 1098 2141 1680 1441 2116 1859 1208 1253 1799 2682 2 | 763235 1363259 574756 392420 259408 354039 292803 206530 129013 170296 128211 92531 94804 75299 70008 59122 56886 43441 45868 31916 39858 32514 24445 58648 116357 28442 696 299 640 619 511 609 601 573 576 567 579 728 699 1269 3 | 570869 606516 969204 316390 168730 274482 233076 163477 96218 151906 113965 80393 83393 62882 64107 51432 50167 36553 41901 27126 34623 27546 22136 15639 19022 95163 54790 246 543 521 473 453 553 460 459 454 448 737 677 1013 4 | 569854 416960 324734 855029 255417 216943 167420 151255 120654 115465 104246 88343 95308 84276 64879 53811 66755 42943 35588 28712 39283 27746 24276 16879 18281 17176 72714 77137 417 397 357 455 518 606 642 442 454 623 768 1306 5 | 1022044 275544 176445 260772 1052374 355475 116330 151979 298436 85199 81690 137041 81790 67794 71775 97756 57094 67447 43425 48785 36178 32269 30122 21223 17301 15738 17288 54159 93618 327 523 989 773 557 1124 1003 508 554 1074 1383 6 | 738957 390644 289185 226218 367751 763732 177026 134691 193135 101689 74662 112042 57781 83306 47753 67610 46858 48633 39260 29882 23377 17587 12996 13926 12836 12064 10024 12687 35472 58485 402 735 611 507 702 740 482 494 906 914 7 | 304071 305265 239890 170808 116469 179446 530925 141026 89144 129610 116005 69198 70431 45622 46788 50635 47731 37160 40209 27493 35216 26021 22102 16221 19002 16911 15009 11250 8589 61065 34566 315 373 271 386 308 393 401 427 482 8 | 347340 223307 173169 158245 161189 139487 144197 402466 107439 99242 97323 69483 71142 48223 47047 52561 45292 38489 36393 29928 35068 26729 25285 15934 19400 22462 16107 13339 10134 11023 82488 8244 383 262 435 511 349 461 480 661 9 | 588635 144731 102990 128874 324785 205682 92247 112542 583735 71942 54836 114262 42861 37024 37955 74641 34183 49853 33760 35232 22284 18888 21019 14295 11328 10158 10779 12339 11372 7230 9536 63472 514 350 561 748 361 291 751 558 10 | 183638 185654 160774 121933 89008 104729 133264 101057 68100 328334 80329 63306 61732 51019 42354 41560 39952 31035 30770 22149 32980 25733 22254 15814 17902 17932 16381 12004 8747 11450 10259 46709 29932 151 326 203 251 250 336 360 11 | 161842 139145 122996 109787 85639 77645 121037 102027 55603 81195 282638 51734 63921 47120 44284 40233 40055 32298 29759 21982 32769 27151 26598 17393 18946 21174 17164 14405 9773 12460 11462 8225 67640 165 155 279 281 265 304 345 12 | 263592 105532 87320 96422 149050 121335 71899 72579 102238 63487 52254 335744 46210 95811 37136 56084 33884 35653 31467 29236 24162 19836 18598 13553 13235 14091 11547 10352 10159 8642 7690 8740 22751 41248 331 407 336 211 885 575 13 | 118479 101521 87912 100325 84524 59206 72521 72059 43135 61691 64050 45861 286639 48743 42285 35397 38353 31284 27015 22767 30821 25898 25433 15089 17712 19724 21087 21994 11074 10801 10289 8794 9124 65629 132 167 312 204 284 520 14 | 101964 82936 67258 87640 70128 78935 47306 48164 37529 47254 43560 68239 48365 1089231 37631 26245 32196 38891 27886 20800 42107 50573 20657 12272 12913 14252 13856 32275 12690 10751 7739 8127 7701 23377 42986 170 1722 827 1345 1346 15 | 104510 78615 68081 68896 75339 49688 48891 48501 39062 43132 43056 37133 42776 34352 214651 30057 30729 25259 22013 19807 26615 21174 21466 13419 15044 16573 15427 14846 10685 12098 8123 8033 7726 7805 59684 131 191 147 318 366 16 | 170151 64642 54246 57372 104930 71941 52471 54107 76780 42524 40918 56351 35930 25929 30063 194661 27688 31666 24858 27121 21622 15935 15769 13980 11703 11213 10440 10899 9810 7279 7136 7259 6519 7068 28525 15498 471 166 248 677 17 | 85708 62350 53192 57432 60106 47223 49253 46298 34474 40246 40394 33939 39164 30373 30374 27586 286258 23484 23746 19261 21739 21470 20253 12180 14795 13418 14550 13953 10127 10243 9033 8424 7677 8272 7822 46598 122 163 205 410 18 | 108696 47356 38620 45661 71935 50969 39170 39600 51878 31367 32496 35362 31459 38402 25372 31586 22983 180168 20575 19309 20305 15054 14682 12053 10535 9831 10230 9695 9711 9067 6742 6314 6248 5670 5526 35992 138 121 207 292 19 | 70193 50402 43883 37764 45742 40599 41248 38758 33671 31200 30508 30867 27319 25331 21790 24492 23940 20730 145073 16202 18212 14605 13192 10149 11243 12598 9849 9594 7419 7605 7074 5813 5767 5603 5655 22593 11653 120 163 308 20 | 72519 34896 27927 30264 51019 31271 28541 30446 35249 22550 22396 28260 23043 19257 19604 26479 19260 19151 15855 100419 13169 13416 13869 10573 9146 8369 8770 9705 8821 5403 6078 5332 4954 6226 5226 4457 29677 94 177 232 21 | 46794 42703 36810 40641 36956 23917 36367 35537 22255 33499 33147 23672 31264 38499 26696 21470 21613 19204 18770 12881 220440 17512 16068 10706 11640 11561 12924 13123 9133 8843 8245 6618 7974 6602 6398 5684 36663 132 167 385 22 | 96091 35698 29065 29477 34167 18042 27526 28882 16807 26680 27463 19669 26841 43466 21021 16226 21935 15077 14493 13235 17846 184182 25805 15858 13358 13587 13123 12720 9669 9492 9363 8386 8163 8338 7583 6134 38583 92 120 326 23 | 125250 25590 23134 24966 30166 13347 23073 26458 18008 22679 27016 16569 25517 20057 20898 15634 20503 14408 12897 13560 16087 22368 114213 11765 13378 15313 12646 14470 9922 8293 9003 7305 7194 7963 6755 5775 9148 29411 122 201 24 | 64729 60060 16053 17298 21334 14461 16603 16301 14307 15978 17099 13440 15231 12412 12894 13752 12078 11734 9942 10065 10702 15062 11696 65612 8877 8350 10452 9280 7104 5740 5502 5754 5448 5666 5046 4197 4368 24331 122 206 25 | 1005 117166 19651 18684 17360 12915 19464 20663 10979 17963 19371 12918 17860 13086 14666 11678 14124 10372 10937 9183 11458 12778 13216 8929 87008 12009 10221 9093 6897 8502 7199 5640 6090 6226 5801 4508 5433 26297 129 268 26 | 920 27684 94923 17805 16001 12505 17214 24361 10232 18828 21570 13892 20265 14157 16303 11017 13590 9701 12236 8683 11823 12486 16248 8279 12120 149138 14645 10159 7415 7898 9108 6681 6100 5680 6059 4772 4317 26724 98 269 27 | 1248 688 53364 71940 17041 9921 15193 16082 10979 17102 17066 11359 22014 12527 15326 10447 14568 10191 10008 8853 13073 12720 12334 10594 10250 18244 112980 13059 8586 6716 6660 6465 6312 6212 5451 4992 4360 10619 16106 158 28 | 1584 244 184 75041 53818 10096 11402 13322 9579 11974 14432 9972 25954 20602 14582 10664 14464 9788 9234 8987 12044 12179 14214 9390 9103 9225 11177 195265 11705 6970 7615 6595 6820 9112 9901 6468 5897 5127 24364 410 29 | 3298 576 506 362 87991 32852 8617 10009 10959 8635 9440 9452 10699 12163 10007 9441 9927 9977 7267 7644 8510 9217 10044 7379 7076 7846 8014 11536 100052 4985 4596 5632 5098 5246 8409 4850 4366 3988 19420 234 30 | 1182 678 511 409 262 56228 60504 11340 7183 11434 12283 8126 10344 10185 12439 7046 9331 8639 7411 5360 7860 8849 8133 5802 8347 7753 6665 6968 4892 102259 6077 5677 7820 4659 5910 4487 7059 3350 19759 305 31 | 1289 523 461 283 452 341 32628 78930 7156 9933 11361 7314 9895 7397 7824 6800 8761 6445 6701 5790 7661 9004 9032 5389 6934 8964 6478 7150 4469 5859 64606 5185 5180 4332 4757 4027 4170 3744 18983 124 32 | 2851 653 503 473 1023 707 311 7912 59890 44003 8261 9262 8692 8280 7798 7037 8096 6021 5542 5206 6352 8089 7289 5418 5363 6497 5833 6246 5975 5248 5182 71044 4433 6037 4841 3835 4822 3561 17815 77 33 | 2644 655 551 604 1046 784 368 414 675 28007 62366 20743 8786 7051 7265 6683 7073 5903 5486 4733 7282 7402 7070 5147 5825 5987 5718 7086 4728 7442 5108 4310 66742 5187 5317 4034 18665 3575 8123 11862 34 | 1879 612 532 728 644 579 249 238 334 138 127 37484 59386 24487 7460 6411 7354 5427 5301 5891 6463 7417 8043 5377 6465 5529 5750 8693 5129 4414 4309 6093 5222 123460 6917 6216 5420 5106 3870 17002 35 | 2644 588 487 705 1043 792 329 368 579 284 134 318 121 35346 50464 25983 7279 4891 5415 4415 5839 6750 6076 4980 5368 6023 5128 9201 7062 5131 4530 4688 5222 6404 123935 6640 5933 5599 5235 17654 36 | 2602 641 441 590 1109 745 336 498 701 178 201 344 194 340 119 13410 40611 31460 20144 3587 5270 5544 5311 4002 4386 4711 4642 5950 4721 4208 3855 3694 3614 5745 6461 89175 5067 5591 4194 16557 37 | 1656 499 425 512 575 517 362 330 350 255 266 408 378 4282 188 820 90 116 10180 23888 30720 33024 8052 4076 5006 4384 3986 4209 3805 7049 3757 4698 19769 4696 5354 4620 112191 4458 4619 17186 38 | 1477 833 870 826 660 679 430 520 287 266 285 239 381 1752 223 161 205 147 193 122 511 59 25946 21928 23507 24072 9121 5049 3631 3182 3520 3351 3089 4217 5034 5149 3959 84576 5331 16428 39 | 2820 865 687 960 1576 1260 542 576 949 423 325 1778 395 2741 438 374 307 246 274 199 316 137 115 155 117 119 13817 20517 16109 17785 16400 15083 7019 3438 4188 3572 4275 4697 77003 20459 40 | 3813 1346 1195 1545 1650 1240 539 753 758 419 415 1166 1047 4201 548 1312 540 408 413 276 1711 477 231 254 240 450 233 1175 524 641 120 134 10145 14923 14978 14380 15076 14382 17152 121100 41 | -------------------------------------------------------------------------------- /block_analysis/ouput: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/poloclub/m-flash-cpp/41f238cff821f7502fde2f19202ff213f42850d9/block_analysis/ouput -------------------------------------------------------------------------------- /block_analysis/test: -------------------------------------------------------------------------------- 1 | block_vertices = 1073741824; 2 | counters = [48345979280 635523016; 3816360040; 3816360040] / 8; 3 | n = length(counters); 4 | 5 | for i = 1:n 6 | for j = 1:n 7 | [dbp spp] = getIOCost(block_vertices, counters(i,j), 4, 8, n); 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /block_analysis/test.m: -------------------------------------------------------------------------------- 1 | block_vertices = 1073741824; 2 | counters = [48345979280 635523016; 3816360040 294943896 ] / 8; 3 | n = length(counters); 4 | sparse = logical(n); 5 | for i = 1:n 6 | for j = 1:n 7 | [dbp spp] = getIOCost(block_vertices, counters(i,j), 4, 8, 3); 8 | sparse(i,j) = spp < dbp; 9 | end 10 | end 11 | 12 | sparse 13 | -------------------------------------------------------------------------------- /block_analysis/twitter1: -------------------------------------------------------------------------------- 1 | 1468364884 2 | -------------------------------------------------------------------------------- /block_analysis/twitter10: -------------------------------------------------------------------------------- 1 | 250876446 147512010 50147018 37784019 20318153 7805527 3892514 2808460 5268667 3127591 2 | 141080899 127875522 41354365 30184660 19080260 8153710 3875043 2898946 4486189 2297057 3 | 39674844 42802338 23559525 18504551 13127876 5650388 3110510 2589635 4970567 2594496 4 | 28056600 31086287 17804078 17939875 10835341 4786780 2564414 2367422 4658164 2963740 5 | 18482335 23413992 13156863 10598160 10187526 4270242 2459000 2269314 4297432 2363247 6 | 10425975 13932754 8827630 7095588 5765861 4600047 1969216 2122428 4616129 2238785 7 | 6154594 8418438 5384713 4438667 3787726 2266488 2314040 1929783 4774087 2540949 8 | 4687379 5874016 3903893 3002951 2473569 1721364 1204720 3407078 4699327 2439503 9 | 2155663 2965279 1829648 1574150 1375264 1076552 757321 1833650 10474236 2113125 10 | 938616 2915375 1952455 1249577 864195 591059 421277 1705151 4542535 8039459 11 | 12 | -------------------------------------------------------------------------------- /block_analysis/twitter2: -------------------------------------------------------------------------------- 1 | 1185443543 96529055 2 | 111993976 74398309 3 | -------------------------------------------------------------------------------- /block_analysis/twitter20: -------------------------------------------------------------------------------- 1 | 67344785 62833658 50707078 19128899 16530499 13884804 12822426 10120196 7047070 4378617 2725417 2043131 1523425 907195 875075 808103 1188829 1305024 844118 461003 2 | 51464792 69233227 62824041 14851988 10905446 8826274 8714872 6126520 5614091 3278375 1915897 1121083 895001 566886 557692 567591 983713 1791100 1177392 645077 3 | 49193812 65906484 78758925 21560863 15543030 13599737 12019649 7854634 7184991 4791782 2960313 2278849 1409700 993683 925133 682706 787795 958211 484676 235530 4 | 11711559 14269072 19433667 8122087 6167673 6043926 6168062 4142322 4322091 2781403 1747998 1166548 856256 615402 575939 715170 1167452 1572732 1050332 526517 5 | 9747301 10576368 15159041 5968785 5860215 5134759 5180891 3646428 3841480 2423394 1533446 1026888 812628 539447 583268 678593 1049352 1498997 906873 452863 6 | 9838887 9512272 15068378 6606127 5832064 6732475 5688930 3988294 4088784 2774220 1813039 1277012 1022544 735892 625466 702307 1070887 1351336 829424 405335 7 | 7496639 7311565 12161723 5378781 4723424 4721804 6292220 3303142 3376034 2213969 1480955 981663 742413 535267 541772 621296 941337 1369795 983289 534488 8 | 7037955 6210454 8920435 4625361 4198131 4160718 4184616 4159909 3111515 2133828 1351606 972549 727200 559537 576905 627448 959690 1387348 932411 513549 9 | 4834973 4804937 9030355 3799841 3395529 3359657 3059696 2358463 3550747 1792418 1198976 851890 628094 489128 521433 571072 893230 1237323 788783 407230 10 | 4779848 4062579 7121751 3462040 3181988 3219688 2959675 2220324 2246787 2597588 1226218 993147 748881 592900 580653 596153 915010 1251867 780673 386559 11 | 3487398 2848837 5360588 2824226 2468521 2704142 2374234 1702958 1810469 1470255 1853011 787538 622104 475603 476711 607313 946409 1338248 778843 352359 12 | 2210145 1879588 3810233 1937695 1747741 1907220 1762214 1256179 1382828 1102300 818069 1141427 484215 387296 451289 587116 958356 1373118 774531 333050 13 | 1807971 1526815 3036386 1586511 1445148 1563751 1468358 1075275 1218276 974424 721160 545785 905175 329970 428815 557688 987884 1422885 880270 397031 14 | 1544239 1275599 2497175 1298394 1159943 1215878 1096677 798357 886554 708479 557221 442322 371329 707568 416012 527270 957044 1406271 865295 398353 15 | 1875650 1337613 2221175 1272386 1161417 1157203 1042615 781085 756790 631074 505411 416526 342107 292890 1028671 543915 959779 1422927 861953 370304 16 | 810046 664053 1534761 845679 747218 838054 682634 496618 564994 520705 434829 364596 304826 264901 391306 1443193 961020 1355610 837096 370148 17 | 730281 624436 1036153 617701 527699 530967 500340 406998 409872 333802 325569 244980 211030 186107 392868 550606 3481824 1183050 687666 314600 18 | 370722 430214 845310 466116 391859 379114 372006 294807 349912 281679 296911 209088 189870 170308 264360 625823 789630 5019746 780116 330747 19 | 235539 328729 807385 361196 324837 301098 281757 206572 238712 187745 221026 134013 129654 112812 515911 504083 1307146 1705742 4732074 332289 20 | 138987 235359 812642 934141 781136 545382 495158 266087 262334 175405 130596 105422 92497 86314 224494 460649 468734 1060880 656303 2318788 21 | -------------------------------------------------------------------------------- /block_analysis/twitter4: -------------------------------------------------------------------------------- 1 | 763803276 169948778 31156306 22539752 2 | 163396007 88295487 21775155 21057842 3 | 53621409 32847603 15508227 20608211 4 | 15602200 9922761 6004294 32277576 5 | -------------------------------------------------------------------------------- /block_analysis/twitter5: -------------------------------------------------------------------------------- 1 | 763803266 169948785 31156305 22539752 0 2 | 163396003 88295487 21775153 21057846 0 3 | 53621410 32847607 15508226 20608210 0 4 | 15602200 9922758 6004295 32277578 0 5 | 0 3 0 0 0 6 | 7 | -------------------------------------------------------------------------------- /block_analysis/yahoo1: -------------------------------------------------------------------------------- 1 | 6636600779 2 | -------------------------------------------------------------------------------- /block_analysis/yahoo2: -------------------------------------------------------------------------------- 1 | 5445971804 150265423 2 | 947003070 93360482 3 | -------------------------------------------------------------------------------- /block_analysis/yahoo20: -------------------------------------------------------------------------------- 1 | 2518416686 168138837 14456191 10563234 15269470 13035101 10820831 9764863 8246067 8767141 9419931 4651264 3939630 4679984 5121392 4931709 4911668 4887083 4990379 5808010 2 | 1409967774 342635585 2784702 3421645 3822602 3278126 4164602 3245376 2942955 3091648 4834347 8489355 6537591 7763265 8372928 7688131 7196762 7266576 8058611 7540802 3 | 106678342 5767096 7201780 567675 680536 419872 310444 412851 319589 385228 356322 198430 159075 186741 204043 194651 175523 192966 226505 291421 4 | 102660576 5825114 555824 5674015 530566 373591 216507 187813 208891 196669 388063 360073 303121 290213 231135 214393 178648 216854 222673 301302 5 | 113493641 5907536 688641 570411 6871138 391240 213733 211096 218851 183332 245383 185184 173631 232586 354037 347402 319083 265713 240490 296648 6 | 107444889 6086611 478362 435075 420676 5860945 286051 230021 243670 250876 248729 235048 233307 229222 266260 230961 204803 371885 369395 397757 7 | 82805357 5501522 220266 188434 200302 230960 6281465 383925 232796 335143 366340 347070 234476 312420 256609 354618 293937 267550 280063 297575 8 | 86332294 5621640 265950 171712 173297 216529 416064 7713183 343913 321144 789392 344300 282144 292935 385848 286528 421210 280221 371955 376119 9 | 83899090 5360275 238311 173051 183883 212026 292811 374049 5670890 290892 655296 351788 229696 240765 287874 351310 247375 342983 292638 229937 10 | 91045860 5402067 270014 193593 191737 231496 453958 329097 248641 6460990 288660 312140 262145 247564 279934 299651 247931 274418 290878 262015 11 | 89592621 8294021 303810 281579 252850 212045 399402 814169 587072 258520 7732111 460927 298958 314942 274124 395861 329389 334575 333462 306907 12 | 76004021 14336385 172110 292189 263648 192097 382634 303129 338118 299491 489503 6502746 261980 299580 231844 323168 338587 327248 300828 291389 13 | 64339655 13210423 145168 284458 218003 206219 315134 299033 280610 265478 316343 287066 5690437 263245 261732 277929 241215 294908 268443 238924 14 | 76081499 14020415 188188 239442 249340 214420 378311 291004 244545 250480 349152 343447 255236 7699941 288986 297582 260704 260277 255870 261914 15 | 78728090 14783875 188076 217132 305289 235225 303174 392134 254316 279974 296788 238784 242479 274902 6078401 318761 312617 280678 261797 250594 16 | 73761187 14370120 187571 192662 277108 214219 422111 282274 309557 300532 425976 315392 271320 281688 307555 6437707 316868 379665 392036 258651 17 | 82249039 14149902 172088 195437 302823 197706 350487 355282 232990 235886 356115 295312 234065 294499 275099 363870 6119474 261635 326668 278747 18 | 80868183 15194548 172962 215526 221426 288507 365216 312546 367630 295923 364338 348205 321160 280674 309010 414226 268355 6455647 440475 294756 19 | 87536919 14863928 169194 193086 181907 303981 398735 374973 251433 307793 342575 323288 246167 269636 265792 401618 314356 424965 6923802 244959 20 | 77500301 14359233 278370 314460 303581 365170 357233 336427 257037 298824 343996 308669 265572 286387 282400 349636 289576 294598 281374 6094637 21 | -------------------------------------------------------------------------------- /block_analysis/yahoo4: -------------------------------------------------------------------------------- 1 | 4853149585 71606417 67677724 66964024 2 | 483304279 37911535 7979962 7643713 3 | 452992280 7996734 39753655 7363633 4 | 478231571 7782473 7614897 38628297 5 | -------------------------------------------------------------------------------- /block_analysis/yahoo40: -------------------------------------------------------------------------------- 1 | 1429204274 67006032 57690507 41452066 7655839 4536673 5373100 3351811 4681042 8474997 5496511 2173191 1874438 1484471 1765589 1409138 1416375 1373573 1341619 1507195 1587048 1805501 1370218 1576470 1303845 1211215 1515739 1276627 1595145 1566494 1417068 1721078 1647001 1405540 1614076 1357583 1540205 1804993 1554718 2258637 2 | 817770918 204435468 41836862 27159411 1250713 1012966 1282102 556221 935062 1178367 1061708 4303692 4489664 2972259 3373672 3216464 2613087 2843032 2843054 3075273 3554659 2472723 850160 854416 796457 628113 1149274 738336 885398 1074363 852375 941188 791890 1067237 801132 1114291 830379 814801 1060720 933934 3 | 821376550 38801244 181420979 26029970 787337 823614 1383589 674673 1181201 1030497 815672 855954 1466759 925850 948265 902786 889835 864451 822328 926207 1112798 2222444 4060626 3158818 2729404 2625222 3923351 2635170 3272725 3572166 2773323 3548328 2638464 1431896 917110 863485 943478 888473 983460 972994 4 | 521766809 28023151 30689790 104494857 659232 514519 957677 405706 741793 869108 796833 809670 1053374 718619 763458 630867 604168 584501 629416 713697 722238 776867 650269 619641 658914 524052 638253 566488 823445 704593 627802 738680 594143 2532242 2787783 2698215 2876648 3350012 2895238 2689108 5 | 47580125 2079864 1447722 972625 3894686 138244 147456 117712 187667 167380 150197 76475 100329 88262 162232 158915 132653 66935 46954 54503 41788 53692 46931 53693 42257 36515 48714 47436 50217 58520 48768 48405 44787 43234 47708 48333 56035 56959 59860 76127 6 | 54467220 2551139 2042604 1304145 161478 3007372 142350 160157 159759 165730 117765 75435 62838 59015 40805 50899 61595 58406 120440 163331 196201 64641 46267 51539 41445 38858 45093 45498 45406 49899 49517 47962 44400 43102 50088 46837 57458 56053 63119 92315 7 | 42581340 1420869 1877767 1000436 138169 138070 3074878 112718 134334 131984 121956 80665 57500 50432 47197 44873 40863 49767 47584 50437 46859 192440 103903 156362 57893 47985 47527 56137 55453 59359 58677 47878 46144 42376 53501 49721 54432 61735 51258 108076 8 | 57347179 1311183 1966332 980579 125229 154356 127091 2359328 153584 110664 103164 67806 61414 47161 41141 54602 62288 55973 43989 54659 88330 60434 42427 57381 87422 109821 132370 54179 57162 59161 54606 53232 47810 42318 67186 46446 47162 59344 59587 82376 9 | 52991705 1494118 1969441 1033411 189923 173519 153955 131818 3007474 160670 109456 70583 63494 55201 45476 53631 54120 44393 46348 50987 93733 61423 43536 51501 46314 45511 46267 105093 123390 108878 83513 56422 54697 46729 55520 50751 56933 70799 60388 93640 10 | 57575252 1432567 1774393 1130292 163319 161880 156727 127911 169119 3533872 123340 87864 52560 42478 38403 73586 72501 47837 39250 46747 40586 49641 40785 49362 42408 39398 42554 38672 51000 70768 99192 108276 102568 115088 106328 53114 57028 55730 51898 90722 11 | 54858209 1773932 1601582 1583739 154931 137215 150754 117025 138335 126995 2695486 74961 66138 48468 41251 53289 58062 46011 45927 52646 49909 55533 47498 53571 47415 76095 46881 45537 57925 59799 47168 47626 55701 45911 100343 146845 98954 137326 71888 79827 12 | 44516875 6295873 1685289 1216001 81052 105164 97159 70137 75991 79354 69658 3020841 98378 73062 63480 72001 68700 70897 80937 71366 67703 75584 62479 71500 58620 51177 75951 60853 81170 67366 59794 76373 49978 53214 61437 63260 65678 67437 143837 102205 13 | 34753758 6202017 1690758 988055 47309 46198 64314 32804 61706 45917 45846 76391 3287492 78041 93870 73308 53998 73687 68833 86563 138214 74223 135886 70086 52999 71964 63613 131074 77497 60647 75800 104046 52808 101982 68955 59112 68046 59884 76356 88617 14 | 35673937 6175646 1790975 1031734 81849 44910 60148 31168 52590 40089 45575 63143 96712 2819230 158281 58466 61967 43144 62968 116779 71297 82606 83461 57637 61313 48200 55482 62251 54484 63981 94941 79831 57114 82033 68422 71061 94766 57367 64933 67669 15 | 32487010 6416609 1778488 1035211 110182 34956 58274 28390 48650 39823 40548 56649 112804 156906 4658826 69336 63690 147818 78477 100157 513643 137957 143576 74815 111614 67760 92585 85277 133130 84096 51943 88699 116657 165081 99270 64546 82473 136047 153149 90951 16 | 41248587 6180014 1779564 1028377 82743 38069 54509 30539 45741 39083 43222 76110 82996 63358 64885 2920136 73114 59290 70673 71837 68257 69535 55737 70172 61302 41468 52662 62411 86421 82201 65061 80825 76329 63143 57803 58602 90736 62698 75976 56044 17 | 38140773 5841890 1795292 1099174 73388 38952 52660 27473 48101 38293 40732 56995 75224 63155 60435 82078 2571966 61657 92630 60511 78214 47235 46392 72067 52429 39976 43008 59682 48561 110925 84723 91601 51940 58855 103675 59493 96182 61881 47970 50638 18 | 34192983 5723497 1550848 914961 65122 60849 60895 32023 51251 46238 44682 69617 108239 46193 163989 67547 112851 2924417 79474 58277 448437 81410 102761 130568 67401 69890 53548 84527 68960 59428 77205 97781 51910 84670 85366 94449 75105 59470 80498 50831 19 | 40229203 5892048 1741051 1009290 44903 87723 62533 35353 51250 43727 41310 86800 122102 76134 84103 77742 69172 68749 3009577 85432 77388 66207 71783 94081 60766 63535 62011 54255 87569 54745 96454 61067 56956 58346 74323 82732 116500 52361 74383 55956 20 | 38651110 6273520 1661168 990558 55596 81792 59344 36363 50604 46156 45789 57597 124325 131397 99313 67939 58130 52590 80180 3285801 70972 74093 63065 83211 65649 72195 67461 63836 68555 69066 61667 80463 48650 83979 62217 55146 61396 60621 79167 52509 21 | 36829283 6396273 1865254 1082161 41954 161424 60964 75138 96647 42896 38755 63672 146275 69438 526224 77442 65575 387083 63750 63959 3980171 134359 173177 121571 64727 95682 45292 149509 91903 51975 50612 162266 70960 137742 74513 87552 82720 87284 101708 61136 22 | 42224434 4142631 4232690 1113905 48043 52389 101685 43792 67716 45591 49193 60425 96172 87517 142698 67805 55582 78832 72164 58647 154702 3462879 93959 72220 76121 62428 61878 58263 59970 70276 109350 73633 54280 66407 69787 102723 101401 62057 71757 72306 23 | 35510895 1687524 6314656 1008877 41115 44437 102588 31686 49462 40802 37678 50688 144478 80051 125328 57765 50143 100723 65108 60978 195987 97483 3354740 78380 71510 77732 47850 136775 57631 64006 90717 86548 52463 146165 89226 115966 109347 57046 89696 66326 24 | 37215256 1590346 6039907 972955 40190 46368 121856 36059 49413 123971 46935 56796 93288 64817 66122 53914 68643 118609 97215 76190 131211 64822 72185 2997441 55045 57693 55911 59044 54723 55483 68707 77197 60107 79851 69072 52984 74669 59766 79262 56105 25 | 33178963 1450296 5980351 949926 40241 36793 67436 69730 68724 57611 50836 54122 100624 66141 88132 58667 105229 56772 59652 69305 56473 77983 74238 68373 2694257 62661 49885 83148 65886 69584 97790 63770 69664 61821 69213 91381 100933 56793 57084 62778 26 | 28359500 1350792 5440161 839986 35554 32580 66341 80951 48190 43478 50926 50335 94892 53477 74509 77725 49563 69046 68007 68514 111042 70845 81582 62873 64749 2868770 72285 57927 64979 61281 57002 59369 51973 57758 62291 72023 48708 62009 54535 64527 27 | 38644527 1605376 6442865 980712 56683 44877 65425 76663 52681 54891 58463 59129 99855 63619 88014 64072 51825 47403 70140 72247 57905 55191 62077 68984 53114 73060 5021134 72077 72686 62803 59491 64685 55086 58378 73134 56032 55859 65461 73967 54187 28 | 34346905 1484796 5686627 910212 40204 46424 60673 36681 101619 40149 42443 54385 147238 67599 79443 59475 56847 88470 49411 58682 174705 61351 140230 72156 70332 58730 84605 2522125 73364 80132 70185 103221 69301 77939 81341 49770 78503 56047 74947 58812 29 | 38835098 1568904 6289487 989484 57119 36828 67991 42322 110198 43290 44593 70193 103099 56355 145638 82145 48220 53901 79709 77845 94542 73281 59547 60544 59395 61878 67765 72198 3041395 62878 57985 67476 72508 83431 60449 61602 84188 58360 71313 62874 30 | 36731772 1592314 6512231 992673 49735 44394 67430 39389 89030 62771 56989 63450 78199 65521 79820 84531 98369 53826 60005 62415 52725 76240 60518 58175 61049 60157 64063 70875 57483 2916647 86673 106627 56194 100484 74357 84270 63476 55773 54658 61750 31 | 32748627 1577055 5922365 1039979 57392 48977 73376 34062 80671 69953 45841 64729 100853 108331 57026 68702 88895 65836 103202 67240 60908 103991 88254 76123 90300 58073 63198 70828 68561 87944 2913917 100163 79940 78348 117598 117461 168297 85615 71885 54710 32 | 37828620 1606885 6397079 1010681 39576 41626 55746 29478 44451 82033 42837 60812 133504 79423 83166 73380 67315 87511 59529 70561 182380 78697 80765 70250 48701 74246 60200 87462 57002 94048 85917 3337710 55575 103005 78593 66013 73661 64463 77894 54162 33 | 40184591 1492738 5768254 946257 37025 36497 60551 28130 54997 102276 44098 50655 71886 59722 84896 73386 53675 41621 50565 53767 70629 53402 55870 64845 68610 52638 65060 81325 71033 58691 80294 74566 2682798 75024 56468 49051 97259 70150 55780 51420 34 | 38883883 1687795 2944579 4490777 52695 45871 71097 35659 55331 90219 50970 51983 131868 87011 136139 60861 56713 80981 55157 76397 163079 69005 86815 87782 62567 50250 57847 90267 74390 70985 83042 125968 79605 3282015 92504 63612 85468 73791 114965 56581 35 | 37384928 1714471 1908349 6008558 44167 41761 68781 41023 57081 60727 77313 52539 100524 81318 97835 65255 109717 86290 80821 60035 91690 74349 82665 76563 82872 65037 71653 91005 58054 80861 102239 101250 57382 86719 3215881 116895 119988 69708 77022 56037 36 | 40118081 1650512 1806170 5471522 50264 36770 69075 36647 57670 45946 93218 65439 102900 80474 83294 66162 75367 96256 91306 63761 89922 108377 119218 69759 99430 73821 57371 60645 73198 96896 125511 85227 50736 73518 133162 2989741 188076 62703 92629 69066 37 | 38230045 1550564 1676493 5560387 40925 34137 63686 30725 49088 38143 85737 68222 119537 117951 84011 93283 85636 71794 138762 56934 102229 96454 108862 100271 105282 46703 72977 80924 87864 66036 170832 88601 90464 81185 134603 178972 3415872 64389 71743 54760 38 | 46298522 1458004 1524715 6102333 52118 42014 63655 35020 49685 44991 85248 64774 100197 61050 133098 64581 48807 45196 50910 61187 83132 60760 56119 58036 50480 43702 63120 52615 58460 53432 76600 65585 71581 71126 61951 49439 61878 3381663 62802 55654 39 | 36970324 1853106 1852013 5811732 61604 57471 73823 33624 62490 53050 71633 110401 104030 84035 139726 75747 55801 77069 75551 70637 118289 75380 82830 92115 62339 60579 87875 81785 80910 67137 66226 98419 64579 106810 85932 84181 81836 69882 3076654 60078 40 | 37150688 1526182 1652448 5043037 85759 73536 123383 83630 96488 91553 81204 101932 86613 82547 61005 59949 72339 51828 84230 68406 54470 95857 71323 62401 79662 62992 56210 60517 67906 66447 119203 65788 56644 61543 62274 62211 60672 68984 69581 2888276 41 | -------------------------------------------------------------------------------- /conf/mflash.cnf: -------------------------------------------------------------------------------- 1 | # M-Flash configuration. 2 | # Commandline parameters override values in the configuration file. 3 | # execthreads=2 4 | # configuration for 4GB 5 | memorysize = 4294967296 6 | #memorysize = 8589934592 7 | #memorysize = 17179869184 8 | #memory size for twitter 9 | #memorysize = 33554432 10 | cachesize = 262144 11 | #cachesize = 32768 12 | #filetype = adjlist 13 | edgeseparator = 14 | #file = /data/hugo-data/hugo-others/datasets/yahoo 15 | iterative = 0 16 | niters = 1 17 | #check_ids = 1 18 | -------------------------------------------------------------------------------- /example_apps/lancsosso.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #include "../src/mflash_basic_includes.hpp" 9 | #include "../src/algorithm/lanczosSO.hpp" 10 | 11 | using namespace std; 12 | using namespace mflash; 13 | 14 | int main(int argc, char ** argv){ 15 | mflash_init(argc, argv); 16 | //Matrix matrix2 ("/run/media/hugo/data/datasets/twitter"); 17 | PrimitiveMatrix matrix ("/run/media/hugo/data/datasets/twitter"); 18 | //PrimitiveVector pvector("/run/media/hugo/data/datasets/lancsos"); 19 | //matrix.multiply(pvector, pvector); 20 | //pvector.multiply(3.3); 21 | LanczosSO lanczos (matrix, 20, 6); 22 | lanczos.run(); 23 | return 0; 24 | } 25 | -------------------------------------------------------------------------------- /example_apps/pagerank.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #include "../src/mflash_basic_includes.hpp" 9 | #include "../src/algorithm/pagerank.hpp" 10 | 11 | using namespace std; 12 | using namespace mflash; 13 | 14 | int main(int argc, char ** argv){ 15 | mflash_init(argc, argv); 16 | 17 | /* Parameters */ 18 | std::string filename = get_option_string("file"); // Base filename 19 | int niters = get_option_int("niters", 4); 20 | 21 | std::string pg_ranks = get_parent_directory(filename) + "pg"; 22 | 23 | 24 | Matrix matrix (filename); 25 | PrimitiveVector pvector(pg_ranks); 26 | PageRank::run(matrix, pvector, niters); 27 | return 0; 28 | } 29 | -------------------------------------------------------------------------------- /example_apps/wcc.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #include "../src/mflash_basic_includes.hpp" 9 | #include "../src/algorithm/wcc.hpp" 10 | 11 | using namespace std; 12 | using namespace mflash; 13 | 14 | int main(int argc, char ** argv){ 15 | mflash_init(argc, argv); 16 | /* Parameters */ 17 | std::string filename = get_option_string("file"); // Base filename 18 | int iterative = get_option_int("iterative", 0); 19 | int niters = get_option_int("niters", -1); 20 | 21 | niters = (niters == 0? 1: niters); 22 | 23 | std::string wcc = get_parent_directory(filename) + "wcc"; 24 | 25 | Matrix matrix (filename); 26 | matrix = matrix.transpose(); 27 | PrimitiveVector pvector(wcc); 28 | WCC::run(matrix, pvector, iterative != 0, niters); 29 | return 0; 30 | } 31 | -------------------------------------------------------------------------------- /src/algorithm/lanczosSO.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef ALGORITHM_LANCZOSSO_HPP_ 9 | #define ALGORITHM_LANCZOSSO_HPP_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | 17 | #include "../core/eigenwrapper.hpp" 18 | #include "../core/matrix.hpp" 19 | #include "../core/primitivematrix.hpp" 20 | #include "../core/primitivevector.hpp" 21 | #include "../core/type.hpp" 22 | #include "../core/util.hpp" 23 | #include "../util/easylogging++.h" 24 | 25 | 26 | using namespace std; 27 | 28 | namespace mflash{ 29 | 30 | template< class V, class E, class IdType> 31 | class LanczosSO{ 32 | PrimitiveMatrix *matrix; 33 | int iterations; 34 | int k; 35 | 36 | mat build_tridiagonal_matrix(int m, V alpha [], V beta[]); 37 | 38 | public: 39 | LanczosSO(PrimitiveMatrix &matrix, int iterations, int k); 40 | void create_ritz_vectors(PrimitiveVector *vectors[], mat &q); 41 | void run(); 42 | }; 43 | 44 | template< class V, class E, class IdType> 45 | LanczosSO::LanczosSO(PrimitiveMatrix &matrix, int iterations, int k){ 46 | this->matrix = &matrix; 47 | this->iterations = iterations; 48 | this->k = k; 49 | } 50 | 51 | template< class V, class E, class IdType> 52 | void LanczosSO::run(){ 53 | string path = mflash::get_parent_directory(matrix->get_file()); 54 | cout<load(); 62 | 63 | const int64 block_size = matrix->get_elements_by_block(); 64 | const int64 node_count = matrix->size(); 65 | 66 | V epsilon = sqrt(1E-18); 67 | V *beta = new double[iterations]; 68 | V *alpha = new double[iterations]; 69 | 70 | //orthogonal vectors v 71 | PrimitiveVector *vectors[iterations];// = new PrimitiveVector*[iterations]; 72 | PrimitiveVector v (v_file, node_count, block_size); 73 | PrimitiveVector r (r_file, node_count, block_size); 74 | 75 | LOG (INFO) << "1: initial values"; 76 | vectors[0] = new PrimitiveVector ( path + "v0", node_count, block_size ); 77 | vectors[0]->fill_random(); 78 | vectors[0]->multiply(((double)1/vectors[0]->pnorm(2))); 79 | 80 | for (int i = 0; i < iterations; i++){ 81 | LOG (INFO) << "3: Find a new basis vector"; 82 | matrix->multiply( *(vectors[i]), v); 83 | 84 | LOG (INFO) << "4:"; 85 | alpha[i] = vectors[i]->transpose().multiply(v); 86 | 87 | LOG (INFO) << "5: v = v - beta[i-1]V[i-1] - alpha[i]V[i]"; 88 | if(i>0){ 89 | v.linear_combination(3, new V[3]{1, -1*beta[i-1], -1*alpha[i]}, new PrimitiveVector*[3]{&v, vectors[i-1], vectors[i]}); 90 | }else{ 91 | v.linear_combination(2, new V[2]{1, -1*alpha[i]}, new PrimitiveVector*[2]{&v, vectors[i]}); 92 | } 93 | 94 | LOG (INFO) << "6: beta[i] = ||v||"; 95 | beta[i] = v.pnorm(2); 96 | 97 | LOG (INFO) << "7: build tri-diagonal matrix from alpha and beta"; 98 | mat ti = build_tridiagonal_matrix(i+1, alpha, beta); 99 | 100 | LOG (INFO) << "8: "; 101 | Eigen::SelfAdjointEigenSolver eigensolver(ti); 102 | mat evalues = eigensolver.eigenvalues(); 103 | mat q = eigensolver.eigenvectors(); 104 | 105 | LOG (INFO) << "Iteration " << i << ", EigenValues: "; 106 | LOG (INFO) << evalues; 107 | 108 | mat mtmp(i+1,1);// = alpha; 109 | array2mat(i+1, alpha, mtmp); 110 | LOG (INFO) << "Alphas: "; 111 | LOG (INFO) << mtmp; 112 | array2mat(i+1, beta, mtmp); 113 | LOG (INFO) << "Betas: "; 114 | LOG (INFO) << mtmp; 115 | 116 | //Max singular value 117 | V max_sv = abs((V)evalues(0)); 118 | for (int j = 1; j <= i; j++){ 119 | max_sv = max(max_sv, abs((V)evalues(j))); 120 | } 121 | LOG (INFO) << "Max Singular Value = " << max_sv; 122 | 123 | bool so = false; 124 | 125 | LOG (INFO) << "9: Reorthogonalization"; 126 | for (int j = 0; j <= i; j++){ 127 | if (beta[i] * abs((V) q(i,j)) <= epsilon * max_sv){ 128 | LOG (INFO) << "Reorthogonalization for ritz vector = "<< j; 129 | 130 | V *constants = new V[i+1]; 131 | for ( int k = 0; k < i+1; k++ ){ 132 | constants[k] = (V)q(k,j); 133 | } 134 | r.linear_combination(i+1, constants, vectors); 135 | //-(r*v) 136 | V constant = -1 * r.transpose().multiply(v); 137 | //v=v-(r*v)r 138 | v.linear_combination(2, new V[2]{1, constant}, new PrimitiveVector*[2]{&v, &r}); 139 | so = true; 140 | delete constants; 141 | } 142 | } 143 | LOG (INFO) << "15:"; 144 | if ( so ){ 145 | LOG (INFO) << "16: Recompute normalization constant beta["<< i <<"]"; 146 | beta[i] = v.pnorm(2); 147 | } 148 | 149 | LOG (INFO) << "18:"; 150 | if ( beta[i] == 0){ 151 | iterations = i+1; 152 | break; 153 | } 154 | 155 | if ( i < iterations-1 ){ 156 | LOG (INFO) << "21: V[i+1] = v/beta[i]"; 157 | stringstream filename; 158 | filename << path << "v" << (i+1); 159 | vectors[i+1] = (PrimitiveVector *) new PrimitiveVector(filename.str(), node_count, block_size); 160 | v.multiply(1/beta[i], *vectors[i+1]); 161 | } 162 | } 163 | LOG (INFO) << "Creating EigenValues"; 164 | mat ti = build_tridiagonal_matrix(iterations, alpha, beta); 165 | Eigen::SelfAdjointEigenSolver eigensolver(ti); 166 | mat evalues = eigensolver.eigenvalues(); 167 | mat q = eigensolver.eigenvectors(); 168 | 169 | /*double *eigenValues = new double[iterations]; 170 | mat2array(evalues, eigenValues);*/ 171 | 172 | int64 *ids = sort_and_get_indexes(iterations, evalues.data(), false); 173 | swap_cols(q, ids, k); 174 | //reducing q to k columns 175 | q.resize(k,k); 176 | 177 | //storing eigen values 178 | PrimitiveVector eigens(eigens_file, k); 179 | eigens.store_region(0, k, evalues.data()); 180 | 181 | 182 | LOG (INFO) << "Creating RitzVectors"; 183 | create_ritz_vectors(vectors, q); 184 | 185 | delete ids; 186 | //delete [] vectors; 187 | delete [] alpha; 188 | delete [] beta; 189 | //delete eigenValues; 190 | } 191 | 192 | template< class V, class E, class IdType> 193 | inline void LanczosSO::create_ritz_vectors(PrimitiveVector *vectors[], mat &q){ 194 | string path = get_parent_directory(vectors[0]->get_file()) + FILE_SEPARATOR; 195 | int64 node_count = matrix->size(); 196 | int64 block_size = matrix->get_elements_by_block(); 197 | int nRitzVectors = q.cols(); 198 | int iterations = q.rows(); 199 | 200 | PrimitiveVector *ritz; 201 | 202 | for (int i = 0; i < nRitzVectors; i++){ 203 | stringstream ritz_file; 204 | ritz_file << path << "RIT" << i; 205 | V *constants = new V[iterations]; 206 | for ( int idx = 0; idx < iterations; idx++ ){ 207 | constants[idx] = (V)q(idx,i); 208 | } 209 | ritz = new PrimitiveVector(ritz_file.str(), node_count , block_size); 210 | ritz->linear_combination(iterations, constants, vectors); 211 | delete constants; 212 | delete ritz; 213 | } 214 | 215 | } 216 | 217 | template< class V, class E, class IdType> 218 | mat LanczosSO::build_tridiagonal_matrix(int m, V alpha [], V beta[]){ 219 | 220 | mat matrix(m, m); 221 | matrix.fill(0); 222 | 223 | matrix(0,0) = alpha[0]; 224 | 225 | if(m==1) return matrix; 226 | 227 | matrix(0,1) = beta[0]; 228 | matrix(m-1,m-2) = beta[m-2]; 229 | matrix(m-1,m-1) = alpha[m-1]; 230 | 231 | for(int i=1; i, Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #include "../mflash_basic_includes.hpp" 9 | 10 | namespace mflash{ 11 | 12 | 13 | #define RESISTANCE 0.85 14 | 15 | template 16 | class DegreeOperator {//: public MAlgorithm{ 17 | public: 18 | inline void initialize(MatrixWorker &worker, Element &out_element){ 19 | *(out_element.value) = 0; 20 | } 21 | inline void gather(MatrixWorker &worker, Element &in_element, Element &out_element, E &edge_data){ 22 | *(out_element.value) += 1; 23 | } 24 | inline void process(Element &accumulator1, Element &accumulator2, Element &out_accumulator){ 25 | *(out_accumulator.value) = *(accumulator1.value) + *(accumulator2.value); 26 | } 27 | inline void apply(MatrixWorker &worker, Element &out_element){ 28 | if(*(out_element.value) != 0){ 29 | *(out_element.value)= (V)1/ *(out_element.value) ; 30 | } 31 | } 32 | inline bool is_initialized(){return true;} 33 | inline bool is_applied(){return true;} 34 | 35 | inline bool is_destination_loaded(){return true;} 36 | inline bool is_source_loaded(){return true;} 37 | inline bool is_destination_stored(){return true;} 38 | 39 | inline void before_iteration(int iteration, MatrixWorker &worker) {} 40 | inline void after_iteration(int iteration, MatrixWorker &worker) {} 41 | }; 42 | 43 | 44 | template 45 | class PageRankOperator {//: public MAlgorithm{ 46 | public: 47 | IdType n; 48 | inline void initialize(MatrixWorker &worker, Element &out_element){ 49 | *(out_element.value) = 0; 50 | } 51 | inline void gather(MatrixWorker &worker, Element &in_element, Element &out_element, E &edge_data){ 52 | *(out_element.value) += *(in_element.value); 53 | } 54 | inline void process(Element &accumulator1, Element &accumulator2, Element &out_accumulator){ 55 | *(out_accumulator.value) = *(accumulator1.value) + *(accumulator2.value); 56 | } 57 | inline void apply(MatrixWorker &worker, Element &out_element){ 58 | *(out_element.value) = RESISTANCE * *(out_element.value) + ( (V)1 - RESISTANCE)/n; 59 | } 60 | inline bool is_initialized(){return true;} 61 | inline bool is_applied(){return true;} 62 | 63 | inline bool is_destination_loaded(){return true;} 64 | inline bool is_source_loaded(){return true;} 65 | inline bool is_destination_stored(){return true;} 66 | 67 | inline void before_iteration(int iteration, MatrixWorker &worker) {} 68 | inline void after_iteration(int iteration, MatrixWorker &worker) {} 69 | 70 | }; 71 | 72 | class PageRank { 73 | public: 74 | template 75 | static inline void run(Matrix &matrix, PrimitiveVector &destination, int iterations){ 76 | 77 | set_conf("elementsize", to_string(2 * sizeof(V))); 78 | set_conf("vertexsize", to_string(sizeof(V))); 79 | matrix.load(); 80 | 81 | destination.resize(matrix.size()); 82 | 83 | string path = get_parent_directory(matrix.get_file()); 84 | string degree_file = get_out_degree_file(matrix.get_file()); 85 | string next_file = path + "next"; 86 | 87 | PrimitiveVector degree_vec(degree_file, matrix.size()); 88 | PrimitiveVector *current_vec = &destination; 89 | PrimitiveVector *next_vec = new PrimitiveVector (next_file, matrix.size()); 90 | PrimitiveVector* tmp; 91 | 92 | DegreeOperator degree_operator; 93 | PageRankOperator pg_operator; 94 | 95 | //creating degree file 96 | if(!exist_file(degree_file)){ 97 | LOG (INFO) << "OUT-DEGREE MEASUREMENT STARTED"; 98 | matrix.operate(degree_operator, degree_vec, degree_vec); 99 | LOG (INFO) << "OUT-DEGREE MEASUREMENT FINISHED"; 100 | } 101 | Matrix m = matrix.transpose(); 102 | LOG (INFO) << "PR = 1/N"; 103 | LOG (INFO) << "PR = 1/"<< matrix.size(); 104 | 105 | pg_operator.n = matrix.size(); 106 | destination.fill((V)1/matrix.size()); 107 | 108 | for (int iteration = 0; iteration < iterations; iteration++){ 109 | LOG (INFO) << endl<< "========== ITERATION "<< iteration <<" =========="; 110 | LOG (INFO) << "PR = PR ./ OUTDEGREE"; 111 | current_vec->multiply(degree_vec); 112 | LOG (INFO) << "NEXT_PR = 0.15/N + 0.85 * M*PR "; 113 | 114 | m.operate(pg_operator, *current_vec, *next_vec); 115 | tmp = current_vec; 116 | current_vec = next_vec; 117 | next_vec = tmp; 118 | } 119 | if (current_vec == &destination){ 120 | delete_file(next_file); 121 | }else{ 122 | delete_file(destination.get_file()); 123 | rename_file(next_file, destination.get_file()); 124 | } 125 | } 126 | 127 | }; 128 | 129 | } 130 | -------------------------------------------------------------------------------- /src/algorithm/wcc.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef ALGORITHM_WCC_HPP_ 9 | #define ALGORITHM_WCC_HPP_ 10 | 11 | #include "../core/malgorithm.hpp" 12 | #include "../core/type.hpp" 13 | #include "../core/primitivevector.hpp" 14 | 15 | 16 | namespace mflash{ 17 | 18 | template 19 | class PrimitiveUnionFindOperator{ 20 | public: 21 | V*counts; 22 | V *sets; 23 | //int64 counter =0; 24 | 25 | /* Find operator of Union-Find with path compression */ 26 | inline V find(V x) { 27 | while (sets[x] != x) { 28 | x = sets[x] = sets[sets[x]]; 29 | } 30 | return sets[x]; 31 | } 32 | 33 | 34 | inline void before_iteration(Matrix &matrix) { 35 | V vertex_count = (V) matrix.size(); 36 | counts = new V[vertex_count]; 37 | sets = new V[vertex_count]; 38 | 39 | for(V i = 0; i < vertex_count; i++){ 40 | counts[i] = 1; 41 | sets[i] = i; 42 | } 43 | } 44 | 45 | inline void after_iteration(Matrix &matrix){ 46 | V vertex_count = (V)matrix.size(); 47 | for(V i = 0; i < vertex_count; i++){ 48 | sets[i] = find(i); 49 | } 50 | } 51 | inline void run(Matrix &matrix, PrimitiveVector &vector){ 52 | before_iteration(matrix); 53 | MappedStream stream(get_block_file(matrix.get_file(),0,0)); 54 | char * ptr = stream.current_ptr; 55 | char * last_ptr = stream.last_ptr; 56 | while(ptr < last_ptr){ 57 | IdType setSrc = find(*( (IdType*)ptr)); 58 | IdType setDst = find(*( (IdType*)ptr+sizeof(IdType))); 59 | 60 | // If in same component, nothing to do, otherwise, Unite 61 | if (setSrc != setDst) { 62 | if (counts[setSrc] > counts[setDst]) { 63 | // A is bigger set, merge with A 64 | sets[setDst] = setSrc; 65 | counts[setSrc] += counts[setDst]; 66 | } else { 67 | // or vice versa 68 | sets[setSrc] = setDst; 69 | counts[setDst] += counts[setSrc]; 70 | } 71 | } 72 | ptr+=(sizeof(IdType)<<1); 73 | } 74 | stream.close_stream(); 75 | 76 | after_iteration(matrix); 77 | //LOG(INFO)< 84 | class WCCAlgorithmUnionFindOperator {//: public MAlgorithm{ 85 | public: 86 | V*counts; 87 | V *sets; 88 | //int64 counter =0; 89 | 90 | /* Find operator of Union-Find with path compression */ 91 | inline V find(V x) { 92 | while (sets[x] != x) { 93 | x = sets[x] = sets[sets[x]]; 94 | } 95 | return sets[x]; 96 | } 97 | inline void before_iteration(int iteration, MatrixWorker &worker) { 98 | counts = (V*)worker.get_source_array()->address(); 99 | sets = (V*)worker.get_destination_array()->address(); 100 | 101 | int64 vertex_count = worker.get_matrix().size(); 102 | for(int64 i = 0; i < vertex_count; i++){ 103 | counts[i] = 1; 104 | sets[i] = i; 105 | } 106 | } 107 | inline void initialize(MatrixWorker &worker,Element &destination){} 108 | inline void gather(MatrixWorker &worker, Element &in_element, Element &out_element, E &edge_data){ 109 | V setDst = find(out_element.id); 110 | V setSrc = find(in_element.id); 111 | // If in same component, nothing to do, otherwise, Unite 112 | if (setSrc != setDst) { 113 | if (counts[setSrc] > counts[setDst]) { 114 | // A is bigger set, merge with A 115 | sets[setDst] = setSrc; 116 | counts[setSrc] += counts[setDst]; 117 | } else { 118 | // or vice versa 119 | sets[setSrc] = setDst; 120 | counts[setDst] += counts[setSrc]; 121 | } 122 | } 123 | } 124 | inline void process(Element &accumulator1, Element &accumulator2, Element &out_accumulator){} 125 | inline void apply(MatrixWorker &worker, Element &out_element){} 126 | inline bool is_initialized(){return false;} 127 | inline bool is_source_loaded(){return false;} 128 | inline bool is_destination_loaded(){return false;} 129 | inline bool is_destination_stored(){return true;} 130 | inline bool is_applied(){return false;} 131 | 132 | inline void after_iteration(int iteration, MatrixWorker &worker) { 133 | V vertex_count = (V)worker.get_matrix().size(); 134 | for(V i = 0; i < vertex_count; i++){ 135 | sets[i] = find(i); 136 | } 137 | } 138 | 139 | }; 140 | 141 | template 142 | class WCCAlgorithmIterativeOperator{//: public MAlgorithm{ 143 | public: 144 | bool state = false; 145 | bool initialize_ = true; 146 | int64 counter =0; 147 | 148 | inline void initialize(MatrixWorker &worker, Element &out_element){ 149 | *(out_element.value) = out_element.id; 150 | } 151 | inline void gather(MatrixWorker &worker, Element &source,Element &destination, E &edge_data){ 152 | //counter++; 153 | if(*(destination.value) > *(source.value)){ 154 | counter ++; 155 | state = true; 156 | *(destination.value) = *(source.value); 157 | } 158 | } 159 | inline void process(Element &accumulator1, Element &accumulator2, Element &out_accumulator){ 160 | *(out_accumulator.value) = min(*(accumulator1.value), *(accumulator2.value)); 161 | } 162 | inline void apply(MatrixWorker &worker,Element &out_element){} 163 | inline bool is_initialized(){return initialize_;} 164 | inline bool is_applied(){return false;} 165 | 166 | inline void before_iteration(int iteration, MatrixWorker &worker) { 167 | counter = 0; 168 | } 169 | inline void after_iteration(int iteration, MatrixWorker &worker) { 170 | cout<< "CHANGES = "< 181 | inline static void run(Matrix &matrix, PrimitiveVector &vector, bool iterative=false, int niters=-1){ 182 | int iteration = 0; 183 | set_conf("elementsize", to_string(2 * sizeof(V))); 184 | matrix.load(); 185 | //LOG(INFO) << "WEAK CONNECTED COMPONENT STARTED"< m = matrix.transpose(); 188 | LOG(INFO) << "ITERATIVE WEAK CONNECTED COMPONENT STARTED"; 189 | WCCAlgorithmIterativeOperator algorithm; 190 | algorithm.initialize_ = true; 191 | do{ 192 | algorithm.state = false; 193 | m.operate(algorithm, vector, vector); 194 | LOG(INFO) << "ITERATION "<< iteration++; 195 | m = m.transpose(); 196 | algorithm.initialize_ = false; 197 | niters--; 198 | }while(algorithm.state && niters != 0); 199 | LOG(INFO) << "WEAK CONNECTED COMPONENT FINISHED"; 200 | }else{ 201 | LOG(INFO) << "UNION-FIND CONNECTED COMPONENT STARTED"; 202 | PrimitiveUnionFindOperator algorithm; 203 | algorithm.run(matrix, vector); 204 | //WCCAlgorithmUnionFindOperator algorithm; 205 | //matrix.operate(algorithm, vector, vector/*, false,false*/); 206 | LOG(INFO) << "UNION-FIND CONNECTED COMPONENT FINISHED"; 207 | } 208 | } 209 | }; 210 | } 211 | 212 | 213 | #endif /* ALGORITHM_WCC_HPP_ */ 214 | -------------------------------------------------------------------------------- /src/core/blockiterator.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef MFLASH_CPP_CORE_BLOCKITERATOR_HPP_ 9 | #define MFLASH_CPP_CORE_BLOCKITERATOR_HPP_ 10 | 11 | #include "../mflash_basic_includes.hpp" 12 | 13 | using namespace std; 14 | 15 | namespace mflash{ 16 | template 17 | class Matrix; 18 | 19 | class Block{ 20 | string block_file; 21 | int col; 22 | int row; 23 | BlockType type; 24 | public: 25 | Block(string block_file, int col, int row, BlockType type){ 26 | this->block_file = block_file; 27 | this->row = row; 28 | this->col = col; 29 | this->type = type; 30 | } 31 | bool exist(){return exist_file(block_file);}; 32 | int64 size(){return file_size(block_file);}; 33 | string get_file(){return block_file;} 34 | int get_row(){return row;} 35 | int get_col(){return col;} 36 | BlockType getBlockType(){return type;} 37 | bool isSparse(){return BlockType::SPARSE == type;} 38 | bool isDense(){return BlockType::DENSE == type;} 39 | }; 40 | 41 | 42 | template 43 | class BlockIterator{ 44 | const static int DIRECTIONS = 4; 45 | string file; 46 | Matrix *matrix; 47 | MatrixProperties *matrix_properties; 48 | 49 | int partitions; 50 | int direction; 51 | 52 | int i = 0; 53 | int j = -1; 54 | //FileStream current; 55 | 56 | int row_offset; 57 | int col_offset; 58 | int row_sign; 59 | int col_sign; 60 | 61 | int count = 0; 62 | 63 | int get_row(); 64 | int get_col(); 65 | 66 | public: 67 | BlockIterator(Matrix *matrix, int direction = 0); 68 | bool has_next(); 69 | Block next(); 70 | 71 | 72 | }; 73 | 74 | template 75 | BlockIterator::BlockIterator(Matrix *matrix, int direction){ 76 | this->matrix = matrix; 77 | this->matrix_properties = &matrix->get_matrix_properties(); 78 | this->file = matrix->get_file(); 79 | this->partitions = matrix_properties->partitions; 80 | this->direction = direction; 81 | 82 | if(direction<0 || direction>=DIRECTIONS){ 83 | LOG(ERROR) << "The direction value must be between 0 and " << DIRECTIONS-1; 84 | assert(false); 85 | } 86 | 87 | row_offset = 0; 88 | col_offset = 0; 89 | row_sign = -1; 90 | col_sign = -1; 91 | 92 | if((direction==2||direction==3)){ 93 | row_offset = partitions-1; 94 | row_sign = 1; 95 | } 96 | 97 | if((direction==2||direction==3) && partitions%2!=0){ 98 | col_offset = partitions-1; 99 | col_sign = 1; 100 | } 101 | 102 | } 103 | 104 | template 105 | int BlockIterator::get_row(){ 106 | return row_offset-(row_sign)*i; 107 | } 108 | 109 | template 110 | int BlockIterator::get_col(){ 111 | return col_offset-(col_sign)*j; 112 | } 113 | 114 | template 115 | bool BlockIterator::has_next(){ 116 | return count!=partitions*partitions; 117 | } 118 | 119 | template 120 | Block BlockIterator::next(){ 121 | if(i%2==0){ 122 | if(j0){ 130 | j--; 131 | }else{ 132 | i++; 133 | j=0; 134 | } 135 | } 136 | if(countfile, row, col); 149 | 150 | Block block(block_file, get_col(), get_row(), getBlockType(partitions, matrix_properties->vertices_partition, matrix_properties->getEdgesBlock(row, col), matrix->source_size())); 151 | return (block); 152 | } 153 | 154 | LOG(ERROR) << "Iterator is after the last position"; 155 | assert(false); 156 | } 157 | } 158 | #endif /* MFLASH_CPP_CORE_BLOCKITERATOR_HPP_ */ 159 | -------------------------------------------------------------------------------- /src/core/conversions.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef SRC_CORE_CONVERSIONS_HPP_ 9 | #define SRC_CORE_CONVERSIONS_HPP_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "../util/easylogging++.h" 17 | #include "../util/cmdopts.hpp" 18 | #include "edgeconversor.hpp" 19 | #include "edgelistconversor.hpp" 20 | #include "mapped_stream.hpp" 21 | //#include "splitter_buffer_block_counting" 22 | //#include "splitter_buffer_extended" 23 | //#include "splitterbuffer.hpp" 24 | #include "splitterbuffer.hpp" 25 | #include "genericedgesplittermanager.hpp" 26 | #include "edgesplittermanagerextended.hpp" 27 | #include "edgesplittermanagerwithblockcounting.hpp" 28 | #include "type.hpp" 29 | #include "util.hpp" 30 | 31 | namespace mflash{ 32 | 33 | struct MatrixProperties; 34 | 35 | /** 36 | * El 37 | * @param graph_file 38 | * @param element_size is used to calculate the vertices by partition. This value is calculated using the number of threads, source size and destinations size. 39 | * @param vertex_size is the size used to calculate the block type 40 | * @param file_type_str 41 | * @return 42 | */ 43 | template 44 | MatrixProperties convert(const std::string graph_file, int64 element_size, int64 vertex_size, std::string file_type_str = "null"){ 45 | 46 | LOG(INFO) << "==== PREPROCESSING ==== "; 47 | if(file_type_str == "null"){ 48 | file_type_str = get_config_option_string("filetype", "null"); 49 | } 50 | 51 | int64 buffer_size = get_config_option_long("memorysize", DEFAULT_MEMORY_SIZE); 52 | int64 vertices_by_partition = getVeticesByPartition(element_size); 53 | int64 vertices_by_cache = getVeticesCache(element_size); 54 | int64 edge_data_size = mflash::getEdgeDataSize(); 55 | 56 | 57 | if(file_type_str == "null"){ 58 | file_type_str = get_option_string_interactive("filetype", "edgelist, adjlist, binedgelist"); 59 | } 60 | 61 | if (file_type_str != "adjlist" && file_type_str != "edgelist" && file_type_str != "adjlist2" && file_type_str != "binedgelist") { 62 | LOG(ERROR) << "You need to specify filetype: 'edgelist', 'adjlist', 'adjlist2', or 'binedgelist'." << std::endl; 63 | assert(false); 64 | } 65 | LOG(INFO) << "GRAPH PREPROCESSING USING THE FORMAT: "<< file_type_str; 66 | LOG(INFO) << "==== DIVIDING IN PARTITIONS ==== "; 67 | clean_mflash_directory(graph_file); 68 | EdgeSplitterManagerWithBlockCounting *emanager = new EdgeSplitterManagerWithBlockCounting(vertices_by_partition, true, 0, 0); 69 | 70 | SplitterBuffer > *bsplitter = new SplitterBuffer >(emanager, graph_file, edge_data_size ,buffer_size, "tmp", false); 71 | 72 | 73 | if (file_type_str == "adjlist" || file_type_str == "edgelist"){ 74 | EdgeConversor::process(graph_file, ' ', file_type_str == "edgelist" , *bsplitter ); 75 | } else if (file_type_str == "adjlist2" ){ 76 | EdgeListConversor::process(graph_file, ' ', *bsplitter ); 77 | }else { 78 | LOG(ERROR) << "Support for "<< file_type_str << " not implemented."; 79 | assert(false); 80 | } 81 | 82 | 83 | std::vector counters = emanager->getCounters(); 84 | int64 partitions = std::sqrt(counters.size()); 85 | int64 max_id = emanager->getMaxId(); 86 | delete emanager; 87 | delete bsplitter; 88 | /* 89 | if(partitions == 1){ 90 | LOG(INFO) << "==== DIVIDING IN BLOCKS OMITTED BECAUSE GRAPH HAS DIVIDED IN A SINGLE PARTITION ==== "; 91 | rename_file(get_partition_file(graph_file, 0, "tmp"), get_block_file(graph_file,0, 0)); 92 | }else{ 93 | */LOG(INFO) << "==== DIVIDING IN BLOCKS ==== "; 94 | for (int64 i = 0; i < partitions; i++){ 95 | LOG(INFO) << "==== DIVIDING PARTITION "<< i << "==== "; 96 | std::string tmp_partition = get_partition_file(graph_file, i, "tmp"); 97 | if(!mflash::exist_file(tmp_partition)) 98 | continue; 99 | MappedStream stream (tmp_partition); 100 | IdType from, to; 101 | 102 | std::vector block_types(partitions); 103 | //checking block density 104 | //int64 edge_size = 2 * sizeof(IdType) + edge_data_size; 105 | for (int64 j = 0; j < partitions; j++){ 106 | //ratio = ((double)1)/partitions + ( ((double)2) * counters[i * partitions + j] * edge_size / vertices_by_partition); 107 | block_types[j] = getBlockType(partitions, vertices_by_partition, counters[i * partitions + j], vertex_size); //ratio<1? BlockType::SPARSE: BlockType::DENSE; 108 | //block_types[j] = (j>1?BlockType::SPARSE: BlockType::DENSE);//getBlockType(partitions, vertices_by_partition, counters[i * partitions + j], vertex_size); //ratio<1? BlockType::SPARSE: BlockType::DENSE; 109 | } 110 | 111 | EdgeSplitterManagerExtended *emanagerext = new EdgeSplitterManagerExtended(vertices_by_partition, false,i, block_types, vertices_by_cache); 112 | SplitterBuffer > *psplitter = new SplitterBuffer >(emanagerext, graph_file, edge_data_size, buffer_size, "", true); 113 | 114 | EmptyField edge_data_type; 115 | while(stream.has_remain()){ 116 | from = stream.next(); 117 | to = stream.next(); 118 | psplitter->add(from, to, &edge_data_type); 119 | } 120 | psplitter->flush(); 121 | stream.close_stream(); 122 | //remove tmp file 123 | delete_file(tmp_partition); 124 | delete psplitter; 125 | delete emanagerext; 126 | } 127 | LOG(INFO) << "==== CREATING TRANPOSE AND SPARSE PARTITIONS ==== "; 128 | GenericEdgeSplitterManager *emanagerext = new GenericEdgeSplitterManager(vertices_by_partition, false, partitions); 129 | SplitterBuffer > *psplitter = new SplitterBuffer >(emanagerext, graph_file, edge_data_size, buffer_size, get_transpose_prefix()); 130 | 131 | for (int64 i = 0; i < partitions; i++){ 132 | std::string partition = get_partition_file(graph_file, i, ""); 133 | if(!mflash::exist_file(partition)) 134 | continue; 135 | MappedStream stream (partition); 136 | IdType from, to; 137 | EmptyField edge_data_type; 138 | 139 | while(stream.has_remain()){ 140 | from = stream.next(); 141 | to = stream.next(); 142 | psplitter->add(from, to, &edge_data_type); 143 | } 144 | stream.close_stream(); 145 | } 146 | psplitter->flush(); 147 | delete psplitter; 148 | 149 | //} 150 | LOG(INFO) << "==== PREPROCESSING FINISHED ==== "; 151 | MatrixProperties properties(max_id+1, sizeof(IdType) ,partitions, vertices_by_partition, counters.data()); 152 | update_matrix_properties(graph_file, properties); 153 | 154 | 155 | return properties; 156 | 157 | } 158 | 159 | 160 | } 161 | 162 | #endif /* SRC_CORE_CONVERSIONS_HPP_ */ 163 | -------------------------------------------------------------------------------- /src/core/edgeconversor.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef SRC_PREPROCESSOR_HPP_ 9 | #define SRC_PREPROCESSOR_HPP_ 10 | 11 | #include 12 | #include 13 | 14 | #include "../util/easylogging++.h" 15 | #include "mapped_stream.hpp" 16 | #include "splitterbuffer.hpp" 17 | 18 | namespace mflash{ 19 | 20 | 21 | template 22 | class EdgeConversor{ 23 | public: 24 | template 25 | static void process (const std::string file_graph, const char separator, const bool edgelist, Splitter &splitter); 26 | }; 27 | 28 | template 29 | template 30 | void EdgeConversor::process(const std::string file_graph, const char separator, const bool edgelist, Splitter &splitter){ 31 | MappedStream in(file_graph); 32 | 33 | IdType v1 = 0; 34 | IdType v2 = 0; 35 | bool isQuantity = !edgelist; 36 | bool isInVertice = true; 37 | 38 | const char comment = '#'; 39 | const char comment2 = '%'; 40 | const char comment3 = ' '; 41 | const char end_line = '\n'; 42 | const char end_line2 = '\r'; 43 | //char separator = ' '; 44 | const char separator2 = '\t'; 45 | const char separator3 = ' '; 46 | 47 | EdgeDataType *value = new EdgeDataType(); 48 | 49 | /* 50 | 51 | const int64 MEGABYTE = 1024 * 1024; 52 | const int64 STEP_INFO = 500 * MEGABYTE; 53 | */ 54 | LOG(INFO)<<"Graph Binarization started"; 55 | bool newline = true; 56 | while(in.has_remain()){ 57 | char b = in.next_char(); 58 | /*if( (in.current_ptr-in.ptr) % STEP_INFO ==0){ 59 | LOG(INFO)<<"Processed: " << (in.current_ptr-in.ptr) / MEGABYTE << "MB"; 60 | }*/ 61 | //removing comment line 62 | if(newline && (b == comment || b == comment2 || b == comment3)){ 63 | while(b != end_line){ 64 | b = in.next_char(); 65 | } 66 | b = in.next_char(); 67 | //when the line content \n\r 68 | while( b == end_line || b==end_line2){ 69 | b = in.next_char(); 70 | } 71 | in.set_position(in.position()- sizeof(char)); 72 | continue; 73 | } 74 | newline = false; 75 | if (b == separator || b == separator2 || b == separator3){ 76 | if(!isInVertice && !isQuantity){ // 77 | splitter.add(v1, v2, value); 78 | }else{ 79 | if(isInVertice) isInVertice = false; 80 | else if(isQuantity) isQuantity = false; 81 | } 82 | v2 = 0; 83 | continue; 84 | } 85 | if (b == end_line || b == end_line2){ 86 | newline = true; 87 | b = in.next_char(); 88 | //when the line content \r\n 89 | while( b == end_line || b==end_line2){ 90 | b = in.next_char(); 91 | } 92 | in.set_position(in.position()- sizeof(char)); 93 | 94 | if(!isQuantity || edgelist){ 95 | splitter.add(v1, v2, value); 96 | } 97 | v1 = 0; 98 | v2 = 0; 99 | isInVertice = true; 100 | isQuantity = !edgelist; 101 | continue; 102 | } 103 | if(b<48 || b > 57){ 104 | LOG(ERROR) << "The character '" << b << "' was not recognized."; 105 | assert(false); 106 | } 107 | 108 | if(isInVertice){ 109 | v1 = (v1<< 3) + (v1 << 1) + (0xF & b); 110 | }else{ 111 | v2 = (v2<< 3) + (v2 << 1) + (0xF & b); 112 | } 113 | } 114 | in.close_stream(); 115 | splitter.flush(); 116 | delete value; 117 | LOG(INFO)<<"Graph Binarization was succesfully"; 118 | } 119 | 120 | 121 | } 122 | 123 | 124 | #endif /* SRC_PREPROCESSOR_HPP_ */ 125 | -------------------------------------------------------------------------------- /src/core/edgelistconversor.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef SRC_EDGELISTCONVERSOR_HPP_ 9 | #define SRC_EDGELISTCONVERSOR_HPP_ 10 | 11 | #include 12 | #include 13 | 14 | #include "../util/easylogging++.h" 15 | #include "mapped_stream.hpp" 16 | #include "splitterbuffer.hpp" 17 | 18 | namespace mflash{ 19 | 20 | 21 | 22 | /** 23 | * Conversor for processing file graphs with special edge list format. First line with number of nodes. 24 | * The first line in the file is the total number of nodes. 25 | * Following is one line of outlinks (target nodes) for each node ID, starting with node 0. 26 | * Any target node IDs will be space-separated on the line for that node. 27 | * If a node does not have any outlinks associated with it, the line will be blank. 28 | * See http://www.lemurproject.org/clueweb12/webgraph.php/ 29 | * 30 | */ 31 | template 32 | class EdgeListConversor{ 33 | public: 34 | template 35 | static void process (const std::string file_graph, const char separator, Splitter &splitter); 36 | }; 37 | 38 | template 39 | template 40 | void EdgeListConversor::process(const std::string file_graph, const char separator, Splitter &splitter){ 41 | MappedStream in(file_graph); 42 | 43 | IdType v1 = 0; 44 | IdType v2 = 0; 45 | 46 | const char comment = '#'; 47 | const char comment2 = '%'; 48 | const char end_line = '\n'; 49 | const char end_line2 = '\r'; 50 | //char separator = ' '; 51 | const char separator2 = '\t'; 52 | const char separator3 = ' '; 53 | 54 | bool first_line = true; 55 | EdgeDataType *value = new EdgeDataType() ; 56 | 57 | while(in.has_remain()){ 58 | char b = in.next_char(); 59 | /*if( (in.current_ptr-in.ptr) % STEP_INFO ==0){ 60 | LOG(INFO)<<"Processed: " << (in.current_ptr-in.ptr) / MEGABYTE << "MB"; 61 | }*/ 62 | //removing comment line or first line with edges 63 | if(b == comment || b == comment2 || first_line){ 64 | if(first_line){ 65 | first_line = false; 66 | } 67 | while(b != end_line){ 68 | b = in.next_char(); 69 | } 70 | b = in.next_char(); 71 | //when the line content \n\r 72 | while( b==end_line2){ 73 | b = in.next_char(); 74 | } 75 | in.set_position(in.position()- sizeof(char)); 76 | continue; 77 | } 78 | if (b == separator || b == separator2 || b == separator3){ 79 | splitter.add(v1, v2, value); 80 | v2 = 0; 81 | continue; 82 | } 83 | if (b == end_line || b == end_line2){ 84 | b = in.next_char(); 85 | //when the line content \r\n 86 | while( b==end_line2){ 87 | b = in.next_char(); 88 | } 89 | in.set_position(in.position()- sizeof(char)); 90 | 91 | if(v2 != 0){ 92 | splitter.add(v1, v2, value); 93 | } 94 | v1++; 95 | v2 = 0; 96 | continue; 97 | } 98 | if(b<48 || b > 57){ 99 | LOG(ERROR) << "The character '" << b << "' was not recognized."; 100 | assert(false); 101 | } 102 | v2 = (v2<< 3) + (v2 << 1) + (0xF & b); 103 | 104 | } 105 | in.close_stream(); 106 | splitter.flush(); 107 | delete value; 108 | LOG(INFO)<<"Graph Binarization was succesfully"; 109 | } 110 | 111 | } 112 | 113 | 114 | #endif /* SRC_PREPROCESSOR_HPP_ */ 115 | -------------------------------------------------------------------------------- /src/core/edgelistthread.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef MFLASH_CPP_CORE_EDGELISTTHREAD_HPP_ 9 | #define MFLASH_CPP_CORE_EDGELISTTHREAD_HPP_ 10 | 11 | #include "array.hpp" 12 | #include "blockiterator.hpp" 13 | #include "mapped_stream.hpp" 14 | #include "matrixworker.hpp" 15 | #include "type.hpp" 16 | 17 | 18 | namespace mflash{ 19 | 20 | 21 | 22 | /* 23 | * WITHOUT MULTITHREAD SUPPORT AND AUXILIAR FIELDS FOR SOURCE AND DESTINATION 24 | */ 25 | template 26 | class EdgeListThread { 27 | private: 28 | 29 | MatrixWorker *worker; 30 | int thread_id; 31 | Block *block; 32 | MappedStream *stream; 33 | 34 | template 35 | bool check_ids(MALGORITHM &algorithm, int64 step); 36 | 37 | 38 | template 39 | void dense_transpose(MALGORITHM &algorithm, int64 step); 40 | 41 | template 42 | void dense_normal(MALGORITHM &algorithm, int64 step); 43 | 44 | template 45 | void sparse_transpose(MALGORITHM &algorithm, int64 step); 46 | 47 | template 48 | void sparse_normal(MALGORITHM &algorithm, int64 step); 49 | 50 | public: 51 | EdgeListThread(MatrixWorker &worker, int thread_id, Block &block); 52 | 53 | template 54 | void call(MALGORITHM &algorithm); 55 | 56 | }; 57 | 58 | template 59 | EdgeListThread::EdgeListThread(MatrixWorker &worker, int thread_id, Block &block){ 60 | this->worker = &worker; 61 | this->thread_id = thread_id; 62 | this->block = █ 63 | this->stream = 0; 64 | } 65 | 66 | template 67 | template 68 | void EdgeListThread::call(MALGORITHM &algorithm){ 69 | 70 | stream = new MappedStream(block->get_file()); 71 | 72 | int step = 0; 73 | 74 | if (get_option_int("check_ids", 0) == 1){ 75 | check_ids(algorithm, step); 76 | }else{ 77 | 78 | if(block->isDense()){ 79 | //this->stream->set_position (this->properties->offset + bytes * this->id); 80 | if(this->worker->matrix->is_transpose()){ 81 | LOG (INFO) << "--- PROCESSING DENSE BLOCK IN TRANSPOSE MODE"; 82 | dense_transpose(algorithm, step); 83 | }else{ 84 | LOG (INFO) << "--- PROCESSING DENSE BLOCK IN NORMAL MODE"; 85 | dense_normal(algorithm, step); 86 | } 87 | }else{ 88 | if(this->worker->matrix->is_transpose()){ 89 | LOG (INFO) << "--- PROCESSING SPARSE BLOCK IN TRANSPOSE MODE"; 90 | sparse_transpose(algorithm, step); 91 | }else{ 92 | LOG (INFO) << "--- PROCESSING SPARSE BLOCK IN NORMAL MODE"; 93 | sparse_normal(algorithm, step); 94 | } 95 | } 96 | } 97 | this->stream->close_stream(); 98 | delete stream; 99 | } 100 | 101 | template 102 | template 103 | bool EdgeListThread::check_ids(MALGORITHM &algorithm, int64 step) { 104 | const int64 elements = worker->matrix->size(); 105 | const int64 vertices_by_partition = worker->matrix->get_elements_by_block(); 106 | 107 | int64 source_offset; 108 | int64 destination_offset; 109 | int64 source_limit; 110 | int64 destination_limit; 111 | int64 next; 112 | 113 | 114 | if(block->isDense()){ 115 | next = getEdgeSize(); 116 | }else{ 117 | next = getEdgeSize() + worker->source_size(); 118 | } 119 | 120 | source_offset = block->get_col() * vertices_by_partition; 121 | destination_offset = block->get_row()*vertices_by_partition; 122 | source_limit =source_offset + min(source_offset + vertices_by_partition, elements - source_offset)- 1; 123 | destination_limit = destination_offset + min(destination_offset + vertices_by_partition, elements-destination_offset) -1; 124 | 125 | IdType in_vertex_id; 126 | IdType out_vertex_id; 127 | char * ptr; 128 | char * last_ptr; 129 | LOG (INFO) << "VERIFING IDS FOR ALL EDGES WITHIN BLOCKS"; 130 | ptr = stream->current_ptr; 131 | last_ptr = stream->last_ptr; 132 | bool transpose = this->worker->matrix->is_transpose(); 133 | while (ptr < last_ptr) { 134 | if(!transpose){ 135 | in_vertex_id = *((IdType*) ptr); 136 | out_vertex_id = *((IdType*) (ptr + sizeof(IdType))); 137 | }else{ 138 | out_vertex_id = *((IdType*) ptr); 139 | in_vertex_id = *((IdType*) (ptr + sizeof(IdType))); 140 | } 141 | 142 | if(in_vertex_id < source_offset || in_vertex_id > source_limit || 143 | out_vertex_id < destination_offset || out_vertex_id > destination_limit ){ 144 | LOG (ERROR) << "Edge ("<< in_vertex_id << "," << out_vertex_id <<") out of bounds within the block" << block->get_file() <<". " 145 | << "Input interval("< 155 | template 156 | void EdgeListThread::dense_transpose(MALGORITHM &algorithm, int64 step) { 157 | E edge_data; 158 | 159 | IdType in_vertex_id; 160 | IdType out_vertex_id; 161 | 162 | Array *in = (Array*)this->worker->source_pointer; 163 | Array *out = (Array *)this->worker->destination_pointer; 164 | 165 | Element in_vertex; 166 | Element out_vertex_accumulator; 167 | 168 | int64 next = getEdgeSize(); 169 | 170 | 171 | 172 | char * ptr; 173 | char * last_ptr; 174 | //LOG (INFO) << "Testing code"; 175 | ptr = stream->current_ptr; 176 | last_ptr = stream->last_ptr; 177 | while (ptr < last_ptr) { 178 | out_vertex_accumulator.id = out_vertex_id = *((IdType*) ptr); 179 | in_vertex.id = in_vertex_id = *((IdType*) (ptr + sizeof(IdType))); 180 | /*if(this->worker->load_vertex_data)*/ 181 | in_vertex.value = in->get_element(in_vertex_id); 182 | /*if(this->worker->load_dest_data) */ 183 | out_vertex_accumulator.value = out->get_element(out_vertex_id); 184 | algorithm.gather(*worker, in_vertex, out_vertex_accumulator, 185 | edge_data); 186 | ptr += next; 187 | } 188 | } 189 | template 190 | template 191 | void EdgeListThread::dense_normal(MALGORITHM &algorithm, int64 step){ 192 | E edge_data; 193 | 194 | MappedStream *stream = this->stream; 195 | 196 | IdType in_vertex_id; 197 | IdType out_vertex_id; 198 | 199 | Array *in = (Array*)this->worker->source_pointer; 200 | Array *out = (Array *)this->worker->destination_pointer; 201 | 202 | Element in_vertex; 203 | Element out_vertex_accumulator; 204 | 205 | int64 next = getEdgeSize(); 206 | 207 | char * ptr; 208 | char * last_ptr; 209 | //LOG (INFO) << "Testing code"; 210 | 211 | ptr = stream->current_ptr; 212 | last_ptr = stream->last_ptr; 213 | while (ptr < last_ptr) { 214 | in_vertex.id = in_vertex_id = *((IdType*) ptr); 215 | out_vertex_accumulator.id = out_vertex_id = *((IdType*) (ptr + sizeof(IdType))); 216 | /*if(this->worker->load_vertex_data)*/ 217 | in_vertex.value = in->get_element(in_vertex_id); 218 | /*if(this->worker->load_dest_data) */ 219 | out_vertex_accumulator.value = out->get_element(out_vertex_id); 220 | algorithm.gather(*worker, in_vertex, out_vertex_accumulator,edge_data); 221 | ptr += next; 222 | } 223 | } 224 | 225 | template 226 | template 227 | void EdgeListThread::sparse_transpose(MALGORITHM &algorithm, int64 step){ 228 | E edge_data; 229 | 230 | IdType in_vertex_id; 231 | IdType out_vertex_id; 232 | 233 | //Array *in = (Array*)this->worker->source_pointer; 234 | Array *out = (Array *)this->worker->destination_pointer; 235 | 236 | Element in_vertex; 237 | Element out_vertex_accumulator; 238 | 239 | int64 next = getEdgeSize() + worker->source_size(); 240 | 241 | char * ptr; 242 | char * last_ptr; 243 | 244 | ptr = stream->current_ptr; 245 | last_ptr = stream->last_ptr; 246 | while (ptr < last_ptr) { 247 | out_vertex_accumulator.id = out_vertex_id = *((IdType*) ptr); 248 | in_vertex.id = in_vertex_id = *((IdType*) (ptr + sizeof(IdType))); 249 | in_vertex.value = (VSource*) (ptr + (sizeof(IdType) <<1)); 250 | out_vertex_accumulator.value = out->get_element(out_vertex_id); 251 | algorithm.gather(*worker, in_vertex, out_vertex_accumulator, edge_data); 252 | ptr += next; 253 | } 254 | } 255 | 256 | template 257 | template 258 | void EdgeListThread::sparse_normal(MALGORITHM &algorithm, int64 step){ 259 | E edge_data; 260 | 261 | IdType in_vertex_id; 262 | IdType out_vertex_id; 263 | 264 | //Array *in = (Array*)this->worker->source_pointer; 265 | Array *out = (Array *)this->worker->destination_pointer; 266 | 267 | Element in_vertex; 268 | Element out_vertex_accumulator; 269 | 270 | int64 next = getEdgeSize() + worker->source_size(); 271 | 272 | char * ptr; 273 | char * last_ptr; 274 | 275 | ptr = stream->current_ptr; 276 | last_ptr = stream->last_ptr; 277 | while (ptr < last_ptr) { 278 | in_vertex.id = in_vertex_id = *((IdType*) ptr); 279 | out_vertex_accumulator.id = out_vertex_id = *((IdType*)(ptr + sizeof(IdType))); 280 | in_vertex.value = (VSource*) (ptr + (sizeof(IdType) <<1)); 281 | out_vertex_accumulator.value = out->get_element(out_vertex_id); 282 | algorithm.gather(*worker, in_vertex, out_vertex_accumulator, edge_data); 283 | ptr += next; 284 | } 285 | 286 | } 287 | 288 | } 289 | 290 | #endif /* MFLASH_CPP_CORE_EDGELISTTHREAD_HPP_ */ 291 | -------------------------------------------------------------------------------- /src/core/edgesplittermanager.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef CORE_EdgeSplitterManager_HPP_ 9 | #define CORE_EdgeSplitterManager_HPP_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "../util/easylogging++.h" 17 | #include "type.hpp" 18 | #include "util.hpp" 19 | 20 | namespace mflash{ 21 | 22 | template 23 | class EdgeSplitterManager{ 24 | public: 25 | virtual IdType getPartitionId(IdType in_id, IdType out_id); 26 | virtual IdType countEdge(IdType in_id, IdType out_id); 27 | virtual std::string getPartitionFile(IdType id); 28 | virtual std::vector& getPartitionCounters(); 29 | virtual IdType getPartitions(); 30 | virtual IdType getIdsByPartition(); 31 | virtual bool isInSplit(); 32 | 33 | virtual ~EdgeSplitterManager(); 34 | }; 35 | 36 | } 37 | #endif 38 | -------------------------------------------------------------------------------- /src/core/edgesplittermanagerextended.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef CORE_GenericEdgeEdgeSplitterManagerExtended_HPP_ 9 | #define CORE_GenericEdgeEdgeSplitterManagerExtended_HPP_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "../util/easylogging++.h" 18 | #include "type.hpp" 19 | #include "util.hpp" 20 | #include "edgesplittermanager.hpp" 21 | 22 | namespace mflash{ 23 | 24 | template 25 | class EdgeSplitterManagerExtended{//: public EdgeSplitterManager< IdType > { 26 | 27 | public: 28 | IdType getPartitionId(IdType in_id, IdType out_id); 29 | IdType countEdge(IdType in_id, IdType out_id); 30 | std::string getPartitionFile(IdType id); 31 | std::vector& getPartitionCounters(); 32 | IdType getPartitions(); 33 | IdType getIdsByPartition(); 34 | bool isInSplit(); 35 | void setInSplit(bool); 36 | 37 | EdgeSplitterManagerExtended( int64 ids_by_partitions, bool in_split, int64 partition_id, std::vector block_types, int64 ids_cache = 0); 38 | ~EdgeSplitterManagerExtended(){} 39 | 40 | private: 41 | IdType getPartitionId(IdType in_id, IdType out_id, int32 shift); 42 | 43 | bool in_split; 44 | IdType ids_by_partition; 45 | IdType partitions; 46 | IdType partitionshift = 0; 47 | IdType cache_shift = 0; 48 | IdType shift = 0; 49 | std::vector partition_counters; 50 | bool *sparse_block; 51 | int64 partition_id; 52 | IdType ids_cache; 53 | bool cache_partitioning; 54 | 55 | 56 | }; 57 | 58 | 59 | template EdgeSplitterManagerExtended::EdgeSplitterManagerExtended(int64 ids_by_partition, bool in_split, int64 partition_id, std::vector block_types, int64 ids_cache){ 60 | cache_partitioning = (ids_cache != 0); 61 | 62 | if(cache_partitioning && ids_by_partition % ids_cache != 0){ 63 | LOG (ERROR)<< "ids by partititon must be multiple of ids in cache"; 64 | assert(false); 65 | } 66 | if(!is2nNumber(ids_by_partition)){ 67 | LOG (ERROR)<< "ids by partititon must be multiple of 2^n"; 68 | assert(false); 69 | } 70 | 71 | 72 | cache_shift = (ids_cache != 0?log2(ids_cache) : 0); 73 | partitionshift = log2(ids_by_partition); 74 | shift = (cache_partitioning? cache_shift: partitionshift); 75 | 76 | this->ids_by_partition = ids_by_partition; 77 | this->ids_cache = ids_cache; 78 | this->partitions = block_types.size()+1; 79 | this->in_split = in_split; 80 | 81 | this->partition_id = partition_id; 82 | 83 | sparse_block = new bool[this->partitions]; 84 | 85 | for(int32 i = 0; i < this->partitions -1; i++){ 86 | sparse_block[i] = block_types[i] == BlockType::SPARSE; 87 | } 88 | sparse_block[this->partitions-1] = true; 89 | 90 | //updating the number of partitions considering the cache size 91 | if(cache_partitioning) 92 | this->partitions = (ids_by_partition * block_types.size() / ids_cache) + 1; 93 | 94 | this->partition_counters.resize(partitions);// = new std::vector(partitions); 95 | 96 | } 97 | 98 | template inline 99 | IdType EdgeSplitterManagerExtended::getPartitionId(IdType in_id, IdType out_id){ 100 | if (!in_split) 101 | in_id = out_id; 102 | if (sparse_block[in_id >>partitionshift]) 103 | return this->partitions-1; 104 | return (in_id>>shift); 105 | } 106 | 107 | template inline 108 | IdType EdgeSplitterManagerExtended::countEdge(IdType in_id, IdType out_id){ 109 | IdType partition_id = getPartitionId(in_id, out_id); 110 | //checking partition by source 111 | if ( partition_id > partitions-1){ 112 | partition_counters.resize(partition_id + 2); 113 | partition_counters[partition_id + 1] = partition_counters[partitions-1]; 114 | partition_counters[partitions-1] = 0; 115 | partitions = partition_id + 2; 116 | 117 | } 118 | partition_counters[partition_id] ++; 119 | return partition_id ; 120 | } 121 | 122 | template inline 123 | IdType EdgeSplitterManagerExtended::getPartitions(){ 124 | return partitions; 125 | } 126 | 127 | template inline 128 | IdType EdgeSplitterManagerExtended::getIdsByPartition(){ 129 | return ids_by_partition; 130 | } 131 | 132 | template inline 133 | bool EdgeSplitterManagerExtended::isInSplit(){ 134 | return in_split; 135 | } 136 | 137 | template inline 138 | void EdgeSplitterManagerExtended:: setInSplit(bool in_split){ 139 | this->in_split = in_split; 140 | } 141 | 142 | 143 | 144 | template inline 145 | std::vector& EdgeSplitterManagerExtended::getPartitionCounters(){ 146 | return partition_counters; 147 | } 148 | 149 | 150 | template inline 151 | std::string EdgeSplitterManagerExtended::getPartitionFile(IdType id){ 152 | if(id == this->partitions -1){ 153 | return get_partition_file("", partition_id); 154 | } 155 | 156 | if(cache_shift){ 157 | id = (int)(id/(ids_by_partition/ids_cache)); 158 | } 159 | 160 | return get_block_file("", partition_id, id); 161 | } 162 | 163 | } 164 | #endif 165 | -------------------------------------------------------------------------------- /src/core/edgesplittermanagerwithblockcounting.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef CORE_GenericEdgeEdgeSplitterManagerWithBlockCounting_HPP_ 9 | #define CORE_GenericEdgeEdgeSplitterManagerWithBlockCounting_HPP_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "../util/easylogging++.h" 18 | #include "type.hpp" 19 | #include "util.hpp" 20 | #include "edgesplittermanager.hpp" 21 | 22 | namespace mflash{ 23 | 24 | template 25 | class EdgeSplitterManagerWithBlockCounting{//: public EdgeSplitterManager< IdType > { 26 | 27 | public: 28 | IdType getPartitionId(IdType in_id, IdType out_id); 29 | IdType countEdge(IdType in_id, IdType out_id); 30 | std::string getPartitionFile(IdType id); 31 | std::vector& getPartitionCounters(); 32 | IdType getPartitions(); 33 | IdType getIdsByPartition(); 34 | bool isInSplit(); 35 | void setInSplit(bool in_split); 36 | 37 | //int64 getBlockPartitions(); 38 | IdType getMaxId(); 39 | std::vector getCounters(); 40 | 41 | 42 | EdgeSplitterManagerWithBlockCounting( int64 ids_by_partitions, bool in_split = true, int64 partitions = 0, int64 ids_cache = 0); 43 | ~EdgeSplitterManagerWithBlockCounting(){} 44 | 45 | private: 46 | IdType getPartitionId(IdType in_id, IdType out_id, int32 shift); 47 | 48 | bool in_split; 49 | IdType ids_by_partition; 50 | IdType partitions; 51 | IdType partitionshift = 0; 52 | IdType cache_shift = 0; 53 | IdType shift = 0; 54 | std::vector partition_counters; 55 | std::vector block_counters; 56 | IdType bpartitions; 57 | IdType max_id; 58 | IdType ids_cache; 59 | bool cache_partitioning; 60 | 61 | }; 62 | 63 | 64 | template 65 | EdgeSplitterManagerWithBlockCounting::EdgeSplitterManagerWithBlockCounting(int64 ids_by_partition, bool in_split, int64 partitions, int64 ids_cache){ 66 | 67 | cache_partitioning = (ids_cache != 0); 68 | 69 | if(cache_partitioning && ids_by_partition % ids_cache != 0){ 70 | LOG (ERROR)<< "ids by partititon must be multiple of ids in cache"; 71 | assert(false); 72 | } 73 | if(!is2nNumber(ids_by_partition)){ 74 | LOG (ERROR)<< "ids by partititon must be multiple of 2^n"; 75 | assert(false); 76 | } 77 | 78 | cache_shift = (ids_cache != 0?log2(ids_cache) : 0); 79 | partitionshift = log2(ids_by_partition); 80 | shift = (cache_partitioning? cache_shift: partitionshift); 81 | 82 | this->ids_by_partition = ids_by_partition; 83 | this->ids_cache = ids_cache; 84 | this->partitions = partitions; 85 | this->in_split = in_split; 86 | this->partition_counters.resize(partitions);// = new std::vector(partitions); 87 | 88 | this->bpartitions = partitions; 89 | this->partition_counters.resize(partitions * partitions); 90 | this->max_id = 0; 91 | 92 | } 93 | 94 | template inline 95 | IdType EdgeSplitterManagerWithBlockCounting::getPartitionId(IdType in_id, IdType out_id){ 96 | if (in_split) 97 | return in_id>>shift; 98 | return out_id>>shift; 99 | } 100 | 101 | template inline 102 | IdType EdgeSplitterManagerWithBlockCounting::getPartitionId(IdType in_id, IdType out_id, int32 shift){ 103 | if (in_split) 104 | return in_id>>shift; 105 | return out_id>>shift; 106 | } 107 | 108 | 109 | 110 | template inline 111 | IdType EdgeSplitterManagerWithBlockCounting::countEdge(IdType in_id, IdType out_id){ 112 | IdType partition_id = getPartitionId(in_id, out_id); 113 | //checking partition by source 114 | if ( partition_id >= partitions){ 115 | //LOG(INFO) << "Increasing partitions to "<< partition_id+1; 116 | partitions = partition_id + 1; 117 | partition_counters.resize(partition_id + 1); 118 | } 119 | partition_counters[partition_id] ++; 120 | //return partition_id ; 121 | 122 | IdType partition2_id; 123 | if(cache_shift){ 124 | partition_id = getPartitionId(in_id, out_id, partitionshift); 125 | } 126 | partition2_id= getPartitionId(out_id, in_id, partitionshift); 127 | 128 | int64 newPartitions = max(partition2_id, partition_id) + 1; 129 | max_id = max(max_id, max(in_id, out_id)); 130 | //checking block_counters 131 | if (newPartitions > bpartitions) { 132 | //printCounters(); 133 | block_counters.resize(newPartitions * newPartitions); 134 | 135 | //reallocating counters 136 | for (int i = bpartitions - 1; i > 0; i--) { 137 | memcpy(block_counters.data() + i * newPartitions, 138 | block_counters.data() + i * bpartitions, sizeof(int64) * bpartitions); 139 | memset(block_counters.data() + i * bpartitions, 0, 140 | sizeof(int64) * (newPartitions - bpartitions)); 141 | } 142 | bpartitions = newPartitions; 143 | } 144 | IdType id = partition_id * bpartitions + partition2_id; 145 | block_counters[id]++; 146 | return partition_id; 147 | } 148 | 149 | template inline 150 | IdType EdgeSplitterManagerWithBlockCounting::getPartitions(){ 151 | return partitions; 152 | //return std::max(partitions,bpartitions); 153 | } 154 | 155 | template inline 156 | IdType EdgeSplitterManagerWithBlockCounting::getIdsByPartition(){ 157 | return ids_by_partition; 158 | } 159 | 160 | template inline 161 | bool EdgeSplitterManagerWithBlockCounting::isInSplit(){ 162 | return in_split; 163 | } 164 | 165 | template inline 166 | void EdgeSplitterManagerWithBlockCounting:: setInSplit(bool in_split){ 167 | this->in_split = in_split; 168 | } 169 | 170 | 171 | template 172 | std::vector EdgeSplitterManagerWithBlockCounting::getCounters() { 173 | return block_counters; 174 | } 175 | 176 | 177 | template 178 | IdType EdgeSplitterManagerWithBlockCounting::getMaxId() { 179 | return max_id; 180 | } 181 | 182 | template inline 183 | std::vector& EdgeSplitterManagerWithBlockCounting::getPartitionCounters(){ 184 | return partition_counters; 185 | } 186 | 187 | template inline 188 | std::string EdgeSplitterManagerWithBlockCounting::getPartitionFile(IdType id){ 189 | if(cache_shift){ 190 | return get_partition_file("", (int)(id/(ids_by_partition/ids_cache)),""); 191 | } 192 | return get_partition_file("", id, ""); 193 | } 194 | 195 | 196 | } 197 | #endif 198 | -------------------------------------------------------------------------------- /src/core/eigenwrapper.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef CORE_EIGENWRAPPER_HPP_ 9 | #define CORE_EIGENWRAPPER_HPP_ 10 | 11 | #include 12 | 13 | typedef Eigen::MatrixXd mat; 14 | 15 | inline void mat2array(mat &m, int64 n, double * array){ 16 | memcpy (array, m.data(), n * sizeof(double)); 17 | } 18 | 19 | inline void array2mat(int64 n, double * array, mat &m){ 20 | if(n<=0 || n>m.cols() * m.rows()) n = m.cols() * m.rows(); 21 | memcpy (m.data(), array, n * sizeof(double)); 22 | } 23 | 24 | inline void array2mat(int64 n, double* array, mat *m){ 25 | array2mat(n, array, *m); 26 | } 27 | 28 | inline void swap_cols(mat &m , int64 col_ids[], int n_cols=0){ 29 | if (n_cols == 0) n_cols = m.cols(); 30 | mat tmp (m); 31 | 32 | for ( int64 i = 0; i < n_cols; i++){ 33 | tmp.col(col_ids[i]) = m.col(i); 34 | } 35 | m = tmp; 36 | } 37 | 38 | inline void swap_cols(mat *m , int64 *col_ids, int n_cols=0){ 39 | swap_cols(*m, col_ids, n_cols); 40 | } 41 | 42 | #endif /* CORE_EIGENWRAPPER_HPP_ */ 43 | -------------------------------------------------------------------------------- /src/core/genericedgesplittermanager.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef CORE_GenericEdgeGenericEdgeSplitterManager_HPP_ 9 | #define CORE_GenericEdgeGenericEdgeSplitterManager_HPP_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "../util/easylogging++.h" 18 | #include "type.hpp" 19 | #include "util.hpp" 20 | #include "edgesplittermanager.hpp" 21 | 22 | namespace mflash{ 23 | 24 | template 25 | class GenericEdgeSplitterManager{//: public EdgeSplitterManager{ 26 | 27 | public: 28 | IdType getPartitionId(IdType in_id, IdType out_id); 29 | IdType countEdge(IdType in_id, IdType out_id); 30 | std::string getPartitionFile(IdType id); 31 | std::vector& getPartitionCounters(); 32 | IdType getPartitions(); 33 | IdType getIdsByPartition(); 34 | bool isInSplit(); 35 | void setInSplit(bool in_split); 36 | 37 | GenericEdgeSplitterManager( int64 ids_by_partitions, bool in_split = true, int64 partitions = 0); 38 | ~GenericEdgeSplitterManager(){} 39 | 40 | private: 41 | bool in_split; 42 | IdType ids_by_partition; 43 | IdType partitions; 44 | IdType partitionshift = 0; 45 | std::vector partition_counters; 46 | 47 | 48 | }; 49 | 50 | 51 | template 52 | GenericEdgeSplitterManager::GenericEdgeSplitterManager(int64 ids_by_partition, bool in_split, int64 partitions){ 53 | 54 | if(!is2nNumber(ids_by_partition)){ 55 | LOG (ERROR)<< "ids by partititon must be multiple of 2^n"; 56 | assert(false); 57 | } 58 | partitionshift = log2(ids_by_partition); 59 | 60 | this->ids_by_partition = ids_by_partition; 61 | this->partitions = partitions; 62 | this->in_split = in_split; 63 | this->partition_counters.resize(partitions);// = new std::vector(partitions); 64 | 65 | } 66 | 67 | template inline 68 | IdType GenericEdgeSplitterManager::getPartitionId(IdType in_id, IdType out_id){ 69 | in_id = (in_split? in_id: out_id); 70 | #if IdType == int32 71 | if(partitionshift >=32) 72 | return 0; 73 | #endif 74 | return in_id >>=partitionshift; 75 | } 76 | 77 | 78 | template inline 79 | IdType GenericEdgeSplitterManager::countEdge(IdType in_id, IdType out_id){ 80 | IdType partition_id = getPartitionId(in_id, out_id); 81 | //checking partition by source 82 | if ( partition_id >= partitions){ 83 | //LOG(INFO) << "Increasing partitions to "<< partition_id+1; 84 | partitions = partition_id + 1; 85 | partition_counters.resize(partition_id + 1); 86 | } 87 | partition_counters[partition_id] ++; 88 | return partition_id ; 89 | } 90 | 91 | template inline 92 | IdType GenericEdgeSplitterManager::getIdsByPartition(){ 93 | return ids_by_partition; 94 | } 95 | 96 | template inline 97 | IdType GenericEdgeSplitterManager::getPartitions(){ 98 | return partitions; 99 | } 100 | 101 | template inline 102 | bool GenericEdgeSplitterManager::isInSplit(){ 103 | return in_split; 104 | } 105 | template inline 106 | void GenericEdgeSplitterManager:: setInSplit(bool in_split){ 107 | this->in_split = in_split; 108 | } 109 | 110 | 111 | 112 | template inline 113 | std::vector& GenericEdgeSplitterManager::getPartitionCounters(){ 114 | return partition_counters; 115 | } 116 | 117 | 118 | template inline 119 | std::string GenericEdgeSplitterManager::getPartitionFile(IdType id){ 120 | return get_partition_file("", id, ""); 121 | } 122 | 123 | } 124 | #endif 125 | -------------------------------------------------------------------------------- /src/core/linearcombination.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef MFLASH_CPP_CORE_LINEARCOMBINATION_HPP_ 9 | #define MFLASH_CPP_CORE_LINEARCOMBINATION_HPP_ 10 | 11 | #include "operator.hpp" 12 | #include "type.hpp" 13 | #include "vector.hpp" 14 | 15 | namespace mflash{ 16 | 17 | class OperationEvent{ 18 | int vector_id; 19 | public: 20 | OperationEvent(int vectorId) { 21 | this->vector_id = vectorId; 22 | } 23 | 24 | int get_vector_id(){ 25 | return vector_id; 26 | } 27 | }; 28 | 29 | class OperationListener { 30 | public: 31 | virtual void on_change(OperationEvent &e) = 0; 32 | virtual ~OperationListener(){} 33 | }; 34 | 35 | 36 | 37 | /** 38 | * 39 | * Make the linear combination between vectors. It uses two binary operators soperator (+) and moperator (*). 40 | * Y = A*V1 + B*V2 + ... + N*Vn 41 | * 42 | * @author Hugo Gualdron 43 | * 44 | * @param 45 | */ 46 | template 47 | class LinearOperator : public OperationListener, public BinaryOperator{ 48 | 49 | V * constants; 50 | BinaryOperator *soperator; 51 | BinaryOperator *moperator; 52 | 53 | int currentVectorId; 54 | Element currentConstant; 55 | Element lastConstant; 56 | 57 | /** 58 | * Attribute used to determined when apply the two constants. 59 | */ 60 | bool combined; 61 | 62 | public: 63 | LinearOperator(V constants[], BinaryOperator *soperator, BinaryOperator *moperator) { 64 | this->constants = constants; 65 | this->soperator = soperator; 66 | this->moperator = moperator; 67 | this->currentVectorId = -1; 68 | combined = false; 69 | } 70 | 71 | 72 | void on_change(OperationEvent &e) { 73 | combined = false; 74 | if(e.get_vector_id() == 1){ 75 | combined = true; 76 | lastConstant.value = &constants[0]; 77 | lastConstant.id = 0; 78 | } 79 | 80 | currentVectorId = e.get_vector_id(); 81 | 82 | currentConstant.value = &constants[currentVectorId]; 83 | currentConstant.id = currentVectorId; 84 | 85 | } 86 | 87 | void apply(Element &left, Element &right, Element &out) { 88 | if(combined){ 89 | moperator->apply(lastConstant, left ,left); 90 | } 91 | moperator->apply(currentConstant, right, right); 92 | soperator->apply(left, right, out); 93 | } 94 | 95 | }; 96 | } 97 | #endif /* MFLASH_CPP_CORE_LINEARCOMBINATION_HPP_ */ 98 | -------------------------------------------------------------------------------- /src/core/malgorithm.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef MFLASH_CPP_CORE_MALGORITHM_HPP_ 9 | #define MFLASH_CPP_CORE_MALGORITHM_HPP_ 10 | 11 | #include "../core/matrixworker.hpp" 12 | #include "../core/type.hpp" 13 | 14 | namespace mflash { 15 | 16 | template 17 | class MAlgorithm { 18 | public: 19 | 20 | /** 21 | * Called before start the processing of each edge to initialize the values to transfer using edges. 22 | * 23 | */ 24 | virtual void initialize_source(MatrixWorker &worker, 25 | Element &destination) { 26 | } 27 | 28 | /** 29 | * Called before start the processing of each each to initialize the values of the output or the accumulators. 30 | */ 31 | virtual void initialize(MatrixWorker &worker, 32 | Element &destination) = 0; 33 | 34 | /** 35 | * Called for each edge to transfers data since source to destination. On the destination is accumulated temporal state for the destination vertex, 36 | * if it is used multi-thread each thread maintain its own accumulator. 37 | */ 38 | virtual void gather(MatrixWorker &worker, Element &source, 39 | Element &destination, E &edge_data) = 0; 40 | 41 | /** 42 | * Called to combine accumulators between threads. 43 | */ 44 | virtual void process(Element &accumulator1, 45 | Element &accumulator2, 46 | Element &out_accumulator) = 0; 47 | 48 | /** 49 | * Called after combine the accumulators for each vertex to update the vertex value. 50 | */ 51 | virtual void apply(MatrixWorker &worker, 52 | Element &out_element) = 0; 53 | 54 | /** 55 | * Check is the initialized method is executed. By default is true. 56 | */ 57 | virtual bool is_initialized() { 58 | return true; 59 | } 60 | 61 | /** 62 | * Check is the apply method is executed. By default is true. 63 | */ 64 | virtual bool is_applied() { 65 | return true; 66 | } 67 | 68 | /** 69 | * Called before an iteration starts. 70 | */ 71 | virtual void before_iteration(int iteration, MatrixWorker &worker) { 72 | 73 | } 74 | 75 | /** 76 | * Called after an iteration has finished. 77 | */ 78 | virtual void after_iteration(int iteration, MatrixWorker &worker) { 79 | } 80 | 81 | 82 | /** 83 | * 84 | */ 85 | virtual bool is_source_loaded() { 86 | return true; 87 | } 88 | 89 | /** 90 | * 91 | */ 92 | virtual bool is_destination_loaded() { 93 | return true; 94 | } 95 | 96 | /** 97 | * Check is the initialized method is executed. By default is false and MFlash uses the current value stored. 98 | * When it is true, it is executed before iteration starts and after before_iteration method. 99 | * 100 | */ 101 | virtual bool is_destination_stored() { 102 | return true; 103 | } 104 | 105 | 106 | 107 | virtual ~MAlgorithm() { 108 | } 109 | }; 110 | 111 | } 112 | 113 | #endif /* MFLASH_CPP_CORE_MALGORITHM_HPP_ */ 114 | -------------------------------------------------------------------------------- /src/core/mapped_stream.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef MFLASH_CPP_CORE_MAPPED_STREAM_HPP_ 9 | #define MFLASH_CPP_CORE_MAPPED_STREAM_HPP_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "../core/type.hpp" 22 | #include "../core/util.hpp" 23 | 24 | 25 | using namespace std; 26 | 27 | namespace mflash{ 28 | 29 | /** 30 | * It creates a stream using memory mapping for reading files in sequential mode. 31 | * The first implementation suppose that we can map whole file. 32 | */ 33 | class MappedStream{ 34 | const static int64 CHAR_SIZE = (int64)sizeof(char); 35 | const static int64 INT_SIZE = (int64)sizeof(int32); 36 | const static int64 INT64_SIZE = (int64)sizeof(int64); 37 | const static int64 FLOAT_SIZE = (int64)sizeof(float); 38 | const static int64 DOUBLE_SIZE = (int64)sizeof(double); 39 | int64 PAGE_SIZE = boost::interprocess::mapped_region::get_page_size(); 40 | public: 41 | string file; 42 | int64 offset; 43 | int64 size; 44 | 45 | char *ptr; 46 | char *current_ptr; 47 | char *last_ptr; 48 | 49 | int file_id; 50 | 51 | bool reverse; 52 | 53 | MappedStream(string file, int64 offset, int64 size); 54 | MappedStream(string file): MappedStream(file, 0,0){}; 55 | 56 | int64 position(); 57 | void set_position(int64 position); 58 | 59 | char next_char(); 60 | int next_int(); 61 | int next_int(int64 step); 62 | char* next(int64 bytes, int64 step = 0); 63 | 64 | template 65 | Type next(int64 step = 0); 66 | 67 | bool has_remain(); 68 | 69 | void close_stream(); 70 | }; 71 | 72 | 73 | inline MappedStream::MappedStream(string file, int64 offset, int64 size){ 74 | this->file = file; 75 | this->offset = offset; 76 | //this->opposite_direction = opposite_direction; 77 | 78 | if(size <=0){ 79 | size = file_size(file); 80 | } 81 | 82 | //PAGE ALIGNMENT 83 | int64 bytes_align = offset %PAGE_SIZE; 84 | offset -= bytes_align; 85 | size += bytes_align; 86 | 87 | 88 | 89 | this->size = size; 90 | reverse = false; 91 | 92 | file_id = open(this->file.c_str(), O_RDONLY); 93 | if (file_id == -1) { 94 | perror("Error opening file for reading"); 95 | exit(EXIT_FAILURE); 96 | } 97 | //cout<< file << endl; 98 | //cout<< sizeof(size_t)< last_ptr : */current_ptr < last_ptr; 120 | } 121 | 122 | 123 | inline char* MappedStream::next(int64 bytes, int64 step){ 124 | char* ptr; 125 | //if(!opposite_direction){ 126 | ptr = current_ptr; 127 | current_ptr += bytes + step; 128 | /*}else{ 129 | ptr = current_ptr - bytes; 130 | current_ptr -= (bytes + step); 131 | }*/ 132 | return ptr; 133 | } 134 | 135 | template 136 | inline Type MappedStream::next(int64 step){ 137 | Type ptr = *( (Type*)current_ptr); 138 | current_ptr += sizeof(Type) + step; 139 | return ptr; 140 | } 141 | 142 | 143 | /*inline char* MappedStream::next(int64 bytes, int64 step){ 144 | char* ptr; 145 | ptr = current_ptr; 146 | current_ptr += bytes + step; 147 | return ptr; 148 | }*/ 149 | 150 | inline int MappedStream::next_int(){ 151 | return next_int(0); 152 | } 153 | 154 | inline char MappedStream::next_char(){ 155 | return next(); 156 | } 157 | 158 | 159 | inline int MappedStream::next_int(int64 step){ 160 | return *( (int*) next(INT_SIZE, step) ); 161 | //int v = *( (int*) next(INT_SIZE, step) ); 162 | /*if(reverse) 163 | return ((v >> 24) & 0xFF) 164 | +(((v >> 16) & 0xFF) << 8) 165 | +(((v >> 8) & 0xFF) << 16) 166 | + ((v & 0xFF) << 24);*/ 167 | // return v; 168 | } 169 | /* 170 | inline int64 Stream::next_int64(){ 171 | return (int)*( next(INT64_SIZE) ); 172 | }*/ 173 | 174 | /*inline float Stream::next_float(){ 175 | return (float)*( next(FLOAT_SIZE) ); 176 | } 177 | 178 | inline double Stream::next_double(){ 179 | return (double)*( next(DOUBLE_SIZE) ); 180 | }*/ 181 | 182 | inline int64 MappedStream::position(){ 183 | return current_ptr - ptr; 184 | } 185 | 186 | inline void MappedStream::set_position(int64 position){ 187 | char* p = position + ptr; 188 | if(p> ptr && p< last_ptr){ 189 | current_ptr = p; 190 | } 191 | } 192 | 193 | inline void MappedStream::close_stream(){ 194 | if (munmap(ptr, size) == -1) { 195 | perror("Error un-mmapping the file"); 196 | } 197 | close(file_id); 198 | } 199 | 200 | } 201 | #endif /* MFLASH_CPP_CORE_MAPPED_STREAM_HPP_ */ 202 | -------------------------------------------------------------------------------- /src/core/matrix.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef MFLASH_CPP_CORE_MATRIX_HPP_ 9 | #define MFLASH_CPP_CORE_MATRIX_HPP_ 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include "../core/array.hpp" 16 | #include "../core/operator.hpp" 17 | #include "../core/type.hpp" 18 | #include "../core/util.hpp" 19 | #include "../core/vector.hpp" 20 | #include "../core/blockiterator.hpp" 21 | #include "../util/easylogging++.h" 22 | 23 | namespace mflash { 24 | 25 | template 26 | class MAlgorithm; 27 | 28 | template 29 | class MatrixWorker; 30 | 31 | /* 32 | 33 | class MatrixInterface { 34 | virtual int64 size(); 35 | virtual string get_file(); 36 | virtual bool is_transpose(); 37 | virtual int id_size(); 38 | virtual int64 get_elements_by_block(); 39 | virtual int size_edge(); 40 | virtual int size_edge_data(); 41 | virtual ~MatrixInterface(); 42 | 43 | };*/ 44 | 45 | template 46 | class Matrix { //: MatrixInterface{ 47 | 48 | std::string file; 49 | //int64 n; 50 | //int64 m; 51 | bool transpose_; 52 | 53 | Mode mode; 54 | MatrixWorker *worker; 55 | 56 | MatrixProperties *properties; 57 | 58 | void removingDynamicVectors(); 59 | 60 | //BlockType get_block_type(int block_count, int64 vertex_size_bytes,int64 edge_size_bytes); 61 | public: 62 | Matrix(std::string file, bool transpose = false, Mode mode = Mode::VECTOR_REPLICATION/*, int64 size, int64 element_by_block*/); 63 | //Matrix(string file, int64 size, bool transpose, int64 element_by_block, Mode mode); 64 | int64 size() { 65 | return properties != NULL? properties->vertices : 0;//max(n, m); 66 | } 67 | 68 | ~Matrix(); 69 | 70 | 71 | Matrix transpose(); 72 | 73 | Matrix & operator=(const Matrix & from); 74 | 75 | std::string get_file() { 76 | return file; 77 | } 78 | bool is_transpose() { 79 | return transpose_; 80 | } 81 | int id_size() { 82 | return sizeof(IdType); 83 | } 84 | 85 | MatrixProperties& get_matrix_properties() { 86 | return *this->properties; 87 | } 88 | 89 | MatrixProperties has_sparse_blocks() { 90 | 91 | if(properties == NULL) return false; 92 | 93 | int n = properties->partitions; 94 | for(int i = 0 ; i < n; i++){ 95 | for(int j = 0 ; j < n; j++){ 96 | if(BlockType::SPARSE == getBlockType(n, properties->vertices_partition, properties->getEdgesBlock(i, j), source_size())){ 97 | return true; 98 | } 99 | } 100 | } 101 | return false; 102 | } 103 | 104 | void set_matrix_properties(MatrixProperties &newProperties){ 105 | if(properties != NULL) delete properties; 106 | 107 | if(&newProperties == NULL) return; 108 | 109 | properties = new MatrixProperties(newProperties); 110 | } 111 | 112 | 113 | int64 get_elements_by_block() { 114 | if(properties != NULL){ 115 | return properties->vertices_partition; 116 | } 117 | return 0; 118 | } 119 | 120 | int size_edge(); 121 | int size_edge_data(); 122 | 123 | bool add_field(FieldType type, std::string fieldname, AbstractVector &vector) { 124 | return worker->add_field(type, fieldname, vector); 125 | } 126 | //bool add_field(FieldType type, string fieldname, AbstractVector *vector){return worker->add_field(type, fieldname, *vector);} 127 | 128 | bool remove_field(FieldType type, std::string fieldname) { 129 | return worker->remove_field(type, fieldname); 130 | } 131 | 132 | int64 source_size(){ 133 | return worker->source_size(); 134 | } 135 | 136 | int64 destination_size(){ 137 | return worker->destination_size(); 138 | } 139 | 140 | template 141 | void operate(MALGORITHM &algorithm, Vector &inVector, Vector &outVector); 142 | 143 | template 144 | void operate(MALGORITHM &algorithm, Vector &outVector); 145 | 146 | template 147 | void operate(MALGORITHM &algorithm); 148 | /* 149 | 150 | template 151 | void operate(MALGORITHM &algorithm, PrimitiveVector &inVector, PrimitiveVector &outVector); 152 | */ 153 | 154 | template 155 | void operate(MALGORITHM &algorithm, 156 | Vector *inVector, Vector *outVector); 157 | 158 | void load(); 159 | 160 | }; 161 | 162 | //Matrix::Matrix(string filebool transpose, int64 size, int64 element_by_block, Mode mode){ 163 | template 164 | Matrix::Matrix(std::string graph, bool transpose, Mode mode) { 165 | this->file = graph; 166 | this->transpose_ = !transpose; 167 | this->mode = mode; 168 | this->properties = NULL; 169 | //check if 170 | if(exist_file(get_properties_file(graph))){ 171 | MatrixProperties properties = load_matrix_properties(graph); 172 | set_matrix_properties(properties); 173 | } 174 | 175 | this->worker = new MatrixWorker(*this); 176 | //this->n = 0; 177 | //this->m = 0; 178 | 179 | } 180 | 181 | template 182 | Matrix::~Matrix() { 183 | removingDynamicVectors(); 184 | } 185 | 186 | template void 187 | Matrix::removingDynamicVectors() { 188 | if(worker != NULL){ 189 | delete worker; 190 | worker = NULL; 191 | } 192 | if(properties != NULL){ 193 | delete properties; 194 | properties = NULL; 195 | } 196 | } 197 | 198 | template 199 | Matrix Matrix::transpose() { 200 | Matrix t(*this); 201 | t.transpose_ = !t.transpose_; 202 | t.worker = new MatrixWorker(t); 203 | if(properties != NULL){ 204 | t.properties = new MatrixProperties(*properties); 205 | } 206 | return (t); 207 | } 208 | 209 | template 210 | Matrix& Matrix::operator=(const Matrix& from) { 211 | removingDynamicVectors(); 212 | this->file = from.file; 213 | this->transpose_ = from.transpose_; 214 | this->mode = from.mode; 215 | set_matrix_properties(*(from.properties)); 216 | 217 | this->worker = new MatrixWorker(*this); 218 | 219 | return *this; 220 | } 221 | 222 | template 223 | int Matrix::size_edge_data() { 224 | int64 size = 0; 225 | #if E != EmptyType 226 | size = sizeof(E); 227 | #endif 228 | return size; 229 | } 230 | 231 | template 232 | int Matrix::size_edge() { 233 | //We are considering always the format (source_id, destination_id, edge_data) 234 | int64 size = 0; 235 | size += 2 * sizeof(IdType); 236 | return size; 237 | } 238 | 239 | //first implementation without replicates :) 240 | 241 | template 242 | template 243 | void Matrix::operate(MALGORITHM &algorithm, 244 | Vector *inVector, Vector *outVector) { 245 | 246 | worker->set_default_destination_field(0); 247 | worker->set_default_source_field(0); 248 | 249 | worker->set_default_source_field(inVector); 250 | worker->set_default_destination_field(outVector); 251 | 252 | load(); 253 | 254 | //resizing vectors 255 | if(outVector != 0){ 256 | outVector->resize(size()); 257 | } 258 | if(inVector != 0){ 259 | inVector->resize(size()); 260 | } 261 | 262 | worker->operate(algorithm); 263 | 264 | worker->set_default_destination_field(0); 265 | worker->set_default_source_field(0); 266 | } 267 | 268 | template 269 | template 270 | void Matrix::operate(MALGORITHM &algorithm, 271 | Vector &inVector, Vector &outVector) { 272 | operate(algorithm, &inVector, &outVector); 273 | } 274 | 275 | template 276 | template 277 | void Matrix::operate( 278 | MALGORITHM &algorithm, 279 | Vector &outVector) { 280 | operate(algorithm, 0, &outVector); 281 | } 282 | 283 | /* 284 | template 285 | template 286 | void Matrix::operate( 287 | MALGORITHM &algorithm, 288 | PrimitiveVector &outVector) { 289 | Vector *destination = &outVector; 290 | operate(algorithm, 0, &destination); 291 | } 292 | */ 293 | 294 | 295 | template 296 | template 297 | void Matrix::operate(MALGORITHM &algorithm) { 298 | operate(algorithm, 0, 0); 299 | } 300 | 301 | 302 | /* 303 | template 304 | template 305 | void Matrix::operate(MALGORITHM &algorithm, 306 | PrimitiveVector &inVector, PrimitiveVector &outVector) { 307 | operate(algorithm, &inVector, &outVector); 308 | } 309 | */ 310 | 311 | 312 | 313 | template 314 | void Matrix::load(){ 315 | int64 element_size = worker->element_size(); 316 | if(properties != NULL && properties->vertices_partition == getVeticesByPartition(element_size)){ 317 | LOG(INFO) << "Graph formatting ommitted because current partitioning is compatible."; 318 | }else{ 319 | MatrixProperties properties = convert(file, validateElementSize(element_size), validateVertexSize(max(source_size(), destination_size()))); 320 | set_matrix_properties(properties); 321 | } 322 | //convert(file, worker->element_size()); 323 | } 324 | 325 | 326 | 327 | 328 | } 329 | #endif /* MFLASH_CPP_CORE_MATRIX_HPP_ */ 330 | -------------------------------------------------------------------------------- /src/core/operator.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef MFLASH_CPP_CORE_OPERATOR_HPP_ 9 | #define MFLASH_CPP_CORE_OPERATOR_HPP_ 10 | 11 | #include "type.hpp" 12 | 13 | namespace mflash { 14 | 15 | template 16 | class Operator { 17 | public: 18 | virtual void dummy() {} 19 | virtual ~Operator(){} 20 | }; 21 | 22 | template 23 | class ZeroOperator: public Operator { 24 | public: 25 | virtual void apply(Element &element) = 0; 26 | virtual ~ZeroOperator(){} 27 | }; 28 | 29 | template 30 | class ZeroOperatorExtended: public Operator { 31 | public: 32 | virtual V apply() = 0; 33 | virtual ~ZeroOperatorExtended(){}; 34 | }; 35 | 36 | template 37 | class UnaryOperator: public Operator { 38 | public: 39 | virtual void apply(Element & element, Element & out) = 0; 40 | virtual ~UnaryOperator(){}; 41 | }; 42 | 43 | template 44 | class BinaryOperator: public Operator { 45 | public: 46 | virtual void apply(Element &left, Element &right, 47 | Element &out) = 0; 48 | virtual ~BinaryOperator(){} 49 | }; 50 | 51 | template 52 | class Reducer { 53 | public: 54 | virtual void initialize(V &out) = 0; 55 | virtual void sum(V &left, V &right, V &out) = 0; 56 | virtual ~Reducer(){} 57 | }; 58 | 59 | template 60 | class UnaryReducer: public Reducer, public UnaryOperator { 61 | }; 62 | 63 | template 64 | class BinaryReducer: public Reducer, public BinaryOperator { 65 | }; 66 | 67 | } 68 | #endif /* MFLASH_CPP_CORE_OPERATOR_HPP_ */ 69 | -------------------------------------------------------------------------------- /src/core/primitivematrix.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef CORE_PRIMITIVEMATRIX_HPP_ 9 | #define CORE_PRIMITIVEMATRIX_HPP_ 10 | 11 | #include 12 | 13 | #include "malgorithm.hpp" 14 | #include "matrix.hpp" 15 | #include "matrixworker.hpp" 16 | #include "primitivevector.hpp" 17 | #include "type.hpp" 18 | 19 | 20 | namespace mflash { 21 | 22 | template 23 | class SpMVMAlgorithmPrimitiveMatrix; 24 | 25 | template 26 | class MatrixWorker; 27 | 28 | 29 | template 30 | class PrimitiveMatrix: public Matrix { 31 | 32 | public: 33 | PrimitiveMatrix(string file, bool transpose = false, Mode mode = Mode::VECTOR_REPLICATION) : 34 | Matrix(file, transpose, mode) { 35 | } 36 | void multiply(PrimitiveVector &inVector,PrimitiveVector &outVector); 37 | }; 38 | 39 | template 40 | inline void PrimitiveMatrix::multiply( 41 | PrimitiveVector &inVector, PrimitiveVector &outVector) { 42 | SpMVMAlgorithmPrimitiveMatrix < V, E, IdType > spvm; 43 | Matrix::operate(spvm, inVector, outVector); 44 | } 45 | 46 | template 47 | class SpMVMAlgorithmPrimitiveMatrix { //: public MAlgorithm { 48 | public: 49 | inline void initialize(MatrixWorker &worker, Element &out_element) { 50 | *(out_element.value) = 0; 51 | } 52 | inline void gather(MatrixWorker &worker, Element &in_element, Element &out_element, E &edge_data) { 53 | #if (E == EmptyType) 54 | *(out_element.value) += *(in_element.value); 55 | #else 56 | *(out_element.value) += *(in_element.value) * edge_data; 57 | #endif 58 | } 59 | inline void process(Element &accumulator1, Element &accumulator2, Element &out_accumulator) { 60 | *(out_accumulator.value) = *(accumulator1.value) + *(accumulator2.value); 61 | } 62 | inline void apply(MatrixWorker &worker, Element &out_element) {} 63 | inline bool is_initialized() {return true;} 64 | inline bool is_applied() {return false;} 65 | 66 | inline bool is_destination_loaded(){return true;} 67 | inline bool is_source_loaded(){return true;} 68 | inline bool is_destination_stored(){return true;} 69 | 70 | inline void before_iteration(int iteration, MatrixWorker &worker) {} 71 | inline void after_iteration(int iteration, MatrixWorker &worker) {} 72 | 73 | }; 74 | } 75 | 76 | #endif /* CORE_PRIMITIVEMATRIX_HPP_ */ 77 | -------------------------------------------------------------------------------- /src/core/primitivevector.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef MFLASH_CPP_CORE_PRIMITIVEVECTOR_HPP_ 9 | #define MFLASH_CPP_CORE_PRIMITIVEVECTOR_HPP_ 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include "../core/operator.hpp" 16 | #include "../core/type.hpp" 17 | #include "../core/vector.hpp" 18 | 19 | using namespace std; 20 | 21 | namespace mflash{ 22 | 23 | template 24 | class PrimitiveVectorBinarySum : public BinaryOperator{ 25 | public: 26 | inline void apply(Element &left, Element &right, Element &out){ 27 | (*out.value) = (*left.value) + (*right.value); 28 | } 29 | }; 30 | 31 | template 32 | class PrimitiveVectorBinaryMul : public BinaryOperator{ 33 | public: 34 | inline void apply(Element &left, Element &right, Element &out){ 35 | (*out.value) = (*left.value) * (*right.value); 36 | } 37 | }; 38 | 39 | template 40 | class PrimitiveVectorProductPoint: public BinaryReducer{ 41 | public: 42 | inline void initialize(V &out) { 43 | out = 0; 44 | } 45 | inline void sum(V &left, V &right, V &out){ 46 | out = right + left; 47 | } 48 | inline void apply(Element &left, Element &right, Element &out){ 49 | (*out.value) = (*left.value) * (*right.value); 50 | } 51 | }; 52 | 53 | template 54 | class PrimitiveVectorPNorm: public UnaryReducer{ 55 | int p; 56 | bool even; 57 | 58 | public: 59 | PrimitiveVectorPNorm(int p){ 60 | this->p = p; 61 | even = !(p%2); 62 | } 63 | 64 | inline void initialize(V &out) { 65 | out = 0; 66 | } 67 | inline void sum(V &left, V &right, V &out){ 68 | out = right + left; 69 | } 70 | inline void apply(Element &left, Element &out){ 71 | if(even){ 72 | (*out.value) = pow((*left.value), p); 73 | }else{ 74 | (*out.value) = pow(abs((*left.value)), p); 75 | } 76 | } 77 | }; 78 | 79 | template 80 | class PrimitiveVectorFillRandom: public ZeroOperator{ 81 | public: 82 | PrimitiveVectorFillRandom(){srand (time(NULL));} 83 | inline void apply(Element &out){ 84 | (*out.value) = (V)rand()/ RAND_MAX; 85 | } 86 | }; 87 | 88 | template 89 | class PrimitiveVectorFill: public ZeroOperator{ 90 | V value; 91 | public: 92 | PrimitiveVectorFill(V value){this->value = value;} 93 | inline void apply(Element &out){ 94 | (*out.value) = value; 95 | } 96 | }; 97 | 98 | template 99 | class PrimitiveVectorMultiplyConstant: public UnaryOperator{ 100 | V value; 101 | public: 102 | PrimitiveVectorMultiplyConstant(V value){this->value = value;} 103 | inline void apply(Element &element, Element &out){ 104 | (*out.value) = *(element.value) * value; 105 | } 106 | }; 107 | 108 | template 109 | class PrimitiveVectorAddConstant: public UnaryOperator{ 110 | V value; 111 | public: 112 | PrimitiveVectorAddConstant(V value){this->value = value;} 113 | inline void apply(Element &element, Element &out){ 114 | (*out.value) = *(element.value) + value; 115 | } 116 | }; 117 | 118 | 119 | 120 | template 121 | class PrimitiveVector : public Vector{ 122 | bool transpose_; 123 | 124 | public: 125 | PrimitiveVector(string file, int64 size = 0, int64 elements_by_block = 0, bool transpose = false) : 126 | Vector::Vector(file, size, elements_by_block){ 127 | this->transpose_ = transpose; 128 | } 129 | 130 | void add(V value); 131 | void add(V value, PrimitiveVector &output); 132 | void add(PrimitiveVector &vector); 133 | void add(PrimitiveVector &vector, PrimitiveVector &output); 134 | 135 | void multiply(V value); 136 | void multiply(V value, PrimitiveVector &output); 137 | V multiply(PrimitiveVector &vector); 138 | V multiply(PrimitiveVector &vector, PrimitiveVector &output); 139 | 140 | V pnorm(int p); 141 | 142 | void fill(V p); 143 | void fill_random(); 144 | 145 | bool is_transpose(){return transpose_;} 146 | PrimitiveVector transpose(){ 147 | PrimitiveVector v(this->file, this->size, this->elements_by_block, !transpose_); 148 | return (v); 149 | }; 150 | 151 | void linear_combination(int n, V constants[], PrimitiveVector *vectors[]); 152 | //static V operate(Operator &operator_, PrimitiveVector &output, int n, PrimitiveVector vectors[]); 153 | }; 154 | 155 | /** 156 | * output = this + vector2 157 | */ 158 | template 159 | void PrimitiveVector::add(PrimitiveVector &vector2, PrimitiveVector &output){ 160 | PrimitiveVectorBinarySum operator_; 161 | Vector::operate(operator_, output, 2, new Vector[2]{*this, vector2}); 162 | } 163 | 164 | /** 165 | * output += vector2 166 | */ 167 | template 168 | void PrimitiveVector::add(PrimitiveVector &vector2){ 169 | add(vector2, *this); 170 | } 171 | 172 | /** 173 | * this += value 174 | */ 175 | template 176 | void PrimitiveVector::add(V value){ 177 | PrimitiveVectorAddConstant operator_(value); 178 | Vector::operate(operator_, *this, 1, new Vector[1]{*this}); 179 | } 180 | 181 | /** 182 | * this += value 183 | */ 184 | template 185 | void PrimitiveVector::add(V value, PrimitiveVector &out){ 186 | PrimitiveVectorAddConstant operator_(value); 187 | Vector::operate(operator_, out, 1, new Vector[1]{*this}); 188 | } 189 | 190 | /** 191 | * this *= value 192 | */ 193 | template 194 | void PrimitiveVector::multiply(V value){ 195 | PrimitiveVectorMultiplyConstant operator_(value); 196 | Vector::operate(operator_, *this, 1, new Vector *[1]{this}); 197 | } 198 | 199 | /** 200 | * this *= value 201 | */ 202 | template 203 | void PrimitiveVector::multiply(V value, PrimitiveVector &output){ 204 | PrimitiveVectorMultiplyConstant operator_(value); 205 | Vector::operate(operator_, output, 1, new Vector *[1]{this}); 206 | } 207 | 208 | /** 209 | * output = this * vector2 or this (.) vector where (.) is product point. 210 | */ 211 | template 212 | V PrimitiveVector::multiply(PrimitiveVector &vector2, PrimitiveVector &output){ 213 | //product point 214 | if(this->is_transpose() != vector2.is_transpose()){ 215 | PrimitiveVectorProductPoint operator_; 216 | return Vector::operate(operator_, *this, 2, new Vector*[2]{this, &vector2}); 217 | }else{ 218 | //multiplication element to element 219 | PrimitiveVectorBinaryMul operator_; 220 | Vector::operate(operator_, output, 2, new Vector *[2]{this, &vector2}); 221 | V v = 0; 222 | return v; 223 | } 224 | } 225 | 226 | /** 227 | * this *= vector2 or product point. 228 | */ 229 | template 230 | V PrimitiveVector::multiply(PrimitiveVector &vector2){ 231 | return multiply(vector2, *this); 232 | } 233 | 234 | 235 | /** 236 | * output = this * vector2 or this (.) vector where (.) is product point. 237 | */ 238 | template 239 | V PrimitiveVector::pnorm(int p){ 240 | PrimitiveVectorPNorm operator_(p); 241 | V value = Vector::operate(operator_, *this, 1, new Vector*[1]{this}); 242 | return pow(value, (double)1/p); 243 | } 244 | 245 | /** 246 | * output = this * vector2 or this (.) vector where (.) is product point. 247 | */ 248 | template 249 | void PrimitiveVector::fill(V value){ 250 | PrimitiveVectorFill operator_(value); 251 | Vector::operate(operator_, *this, 1, new Vector*[1]{this}); 252 | } 253 | 254 | /** 255 | * output = this * vector2 or this (.) vector where (.) is product point. 256 | */ 257 | template 258 | void PrimitiveVector::fill_random(){ 259 | PrimitiveVectorFillRandom operator_; 260 | Vector::operate(operator_, *this, 1, new Vector*[1]{this}); 261 | } 262 | 263 | template 264 | void PrimitiveVector::linear_combination(int n, V constants[], PrimitiveVector *vectors[]){ 265 | PrimitiveVectorBinarySum soperator; 266 | PrimitiveVectorBinaryMul moperator; 267 | 268 | if (n < 0){ 269 | return; 270 | } 271 | 272 | if (n == 1){ 273 | vectors[0]->multiply(constants[0], *this); 274 | return; 275 | } 276 | 277 | Vector *vvectors [n]; 278 | for(int i = 0; i < n; i++){ 279 | vvectors[i] = vectors[i]; 280 | } 281 | 282 | Vector::operate(moperator, soperator, *this,n, constants, vvectors); 283 | 284 | /* 285 | LinearOperator loperator (constants, &soperator, &moperator); 286 | OperationListener * listener = &loperator; 287 | this->add_listener(listener); 288 | Vector::operate(loperator, *this, n, vvectors); 289 | this->remove_listener(listener);*/ 290 | } 291 | 292 | } 293 | #endif /* MFLASH_CPP_CORE_PRIMITIVEVECTOR_HPP_ */ 294 | -------------------------------------------------------------------------------- /src/core/type.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef MFLASH_CPP_CORE_TYPE_HPP_ 9 | #define MFLASH_CPP_CORE_TYPE_HPP_ 10 | 11 | typedef unsigned int int32; 12 | typedef unsigned long long int64; 13 | 14 | namespace mflash { 15 | 16 | template 17 | class Element { 18 | public: 19 | IdType id; 20 | V* value; 21 | 22 | void set_value(V value) { 23 | *(this->value) = value; 24 | } 25 | }; 26 | 27 | class EmptyField { 28 | }; 29 | 30 | enum ElementIdType { 31 | SIMPLE, DOUBLE, 32 | }; 33 | 34 | enum Mode { 35 | UNSAFE, VECTOR_REPLICATION, 36 | //ATOMIC 37 | }; 38 | 39 | enum GraphFormatterMode { 40 | ADJACENCY_LIST, EDGE_LIST 41 | }; 42 | 43 | enum FieldType { 44 | SOURCE, DESTINATION, BOTH 45 | }; 46 | 47 | enum BlockType { 48 | DENSE, SPARSE 49 | }; 50 | 51 | enum MatrixProperty{ 52 | VERTICES, 53 | PARTITIONS, // Beta 54 | VERTICES_PARTITION, 55 | EDGES_BLOCKS 56 | }; 57 | 58 | struct MatrixProperties{ 59 | int64 vertices; 60 | int64 idSize; 61 | int64 partitions; 62 | int64 vertices_partition; 63 | int64 *edges_by_block; 64 | 65 | MatrixProperties(){ 66 | vertices = 0; 67 | idSize = 0; 68 | partitions = 0; 69 | vertices_partition = 0; 70 | edges_by_block = NULL; 71 | } 72 | 73 | MatrixProperties(int64 vertices, int64 idSize, int64 partitions, int64 vertices_partition, int64 edges_by_block[]){ 74 | this->vertices = vertices; 75 | this->idSize = idSize; 76 | this->partitions = partitions; 77 | this->vertices_partition = vertices_partition; 78 | this->edges_by_block = new int64[partitions * partitions]; 79 | memcpy(this->edges_by_block, edges_by_block, sizeof(int64) * partitions * partitions); 80 | } 81 | MatrixProperties(const MatrixProperties &p): MatrixProperties(p.vertices, p.idSize, p.partitions, p.vertices_partition, p.edges_by_block){ 82 | } 83 | 84 | MatrixProperties& operator= (const MatrixProperties &p){ 85 | vertices = p.vertices; 86 | idSize = p.idSize; 87 | partitions = p.partitions; 88 | vertices_partition = p.vertices_partition; 89 | if(edges_by_block != NULL){ 90 | delete [] edges_by_block; 91 | } 92 | this->edges_by_block = new int64[partitions * partitions]; 93 | memcpy(this->edges_by_block, p.edges_by_block, sizeof(int64) * partitions * partitions); 94 | return *this; 95 | } 96 | 97 | ~MatrixProperties(){ 98 | if(edges_by_block != NULL){ 99 | delete [] edges_by_block; 100 | edges_by_block = NULL; 101 | } 102 | 103 | } 104 | 105 | int64 getEdgesBlock(int32 row, int32 col){ 106 | if(row >= 0 && row = 0 && col type = type; 120 | this->offset = offset; 121 | this->size = size; 122 | } 123 | }; 124 | } 125 | #endif /* MFLASH_CPP_CORE_TYPE_HPP_ */ 126 | -------------------------------------------------------------------------------- /src/mflash_basic_includes.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | #ifndef MFLASH_BASIC_INCLUDES_HPP_ 9 | #define MFLASH_BASIC_INCLUDES_HPP_ 10 | 11 | #include 12 | 13 | #include "util/easylogging++.h" 14 | #include "core/type.hpp" 15 | #include "util/configfile.hpp" 16 | #include "util/cmdopts.hpp" 17 | #include "core/util.hpp" 18 | #include "core/conversions.hpp" 19 | #include "core/mapped_stream.hpp" 20 | #include "core/vector.hpp" 21 | #include "core/primitivevector.hpp" 22 | #include "core/matrix.hpp" 23 | #include "core/primitivematrix.hpp" 24 | //#include "core/splitter_buffer_block_counting" 25 | 26 | INITIALIZE_EASYLOGGINGPP 27 | 28 | #endif /* MFLASH_BASIC_INCLUDES_HPP_ */ 29 | -------------------------------------------------------------------------------- /src/util/cmdopts.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | // 9 | // NOTICE: 10 | // This file was adapted from graphchi framework https://github.com/GraphChi/graphchi-cpp/blob/master/src/util/cmdopts.hpp 11 | // This file incorporates work covered by the following copyright and permission notice: 12 | // Copyright [2012] [Aapo Kyrola, Guy Blelloch, Carlos Guestrin / Carnegie Mellon University] 13 | // 14 | // Licensed under the Apache License, Version 2.0 (the "License"); 15 | // you may not use this file except in compliance with the License. 16 | // You may obtain a copy of the License at 17 | // 18 | // http://www.apache.org/licenses/LICENSE-2.0 19 | // 20 | // Unless required by applicable law or agreed to in writing, software 21 | // distributed under the License is distributed on an "AS IS" BASIS, 22 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 23 | // See the License for the specific language governing permissions and 24 | // limitations under the License. 25 | #ifndef MFLASH_CMDOPTS_DEF 26 | #define MFLASH_CMDOPTS_DEF 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | #include "configfile.hpp" 34 | #include "mflashnames.hpp" 35 | 36 | namespace mflash { 37 | 38 | bool _cmd_configured = false; 39 | 40 | int _argc; 41 | char **_argv; 42 | std::map conf; 43 | 44 | 45 | void set_conf(std::string key, std::string value) { 46 | conf[key] = value; 47 | } 48 | 49 | // Config file 50 | std::string get_config_option_string(const char *option_name) { 51 | if (conf.find(option_name) != conf.end()) { 52 | return conf[option_name]; 53 | } else { 54 | std::cout << "ERROR: could not find option " << option_name << " from config."; 55 | assert(false); 56 | } 57 | } 58 | 59 | std::string get_config_option_string(const char *option_name, 60 | std::string default_value) { 61 | if (conf.find(option_name) != conf.end()) { 62 | return conf[option_name]; 63 | } else { 64 | return default_value; 65 | } 66 | 67 | } 68 | int get_config_option_int(const char *option_name, int default_value) { 69 | if (conf.find(option_name) != conf.end()) { 70 | return atoi(conf[option_name].c_str()); 71 | } else { 72 | return default_value; 73 | } 74 | } 75 | 76 | int get_config_option_int(const char *option_name) { 77 | if (conf.find(option_name) != conf.end()) { 78 | return atoi(conf[option_name].c_str()); 79 | } else { 80 | std::cout << "ERROR: could not find option " << option_name << " from config."; 81 | assert(false); 82 | } 83 | } 84 | 85 | uint64_t get_config_option_long(const char *option_name, uint64_t default_value) { 86 | if (conf.find(option_name) != conf.end()) { 87 | return atol(conf[option_name].c_str()); 88 | } else { 89 | return default_value; 90 | } 91 | } 92 | double get_config_option_double(const char *option_name, double default_value) { 93 | if (conf.find(option_name) != conf.end()) { 94 | return atof(conf[option_name].c_str()); 95 | } else { 96 | return default_value; 97 | } 98 | } 99 | 100 | void set_argc(int argc, const char ** argv); 101 | void set_argc(int argc, const char ** argv) { 102 | _argc = argc; 103 | _argv = (char**)argv; 104 | _cmd_configured = true; 105 | conf = loadconfig(filename_config_local(), filename_config()); 106 | 107 | /* Load --key=value type arguments into the conf map */ 108 | std::string prefix = "--"; 109 | for (int i = 1; i < argc; i++) { 110 | std::string arg = std::string(_argv[i]); 111 | 112 | if (arg.substr(0, prefix.size()) == prefix) { 113 | arg = arg.substr(prefix.size()); 114 | size_t a = arg.find_first_of("=", 0); 115 | if (a != arg.npos) { 116 | std::string key = arg.substr(0, a); 117 | std::string val = arg.substr(a + 1); 118 | 119 | std::cout << "[" << key << "]" << " => " << "[" << val << "]" << std::endl; 120 | conf[key] = val; 121 | } 122 | } 123 | } 124 | 125 | } 126 | 127 | void mflash_init(int argc, const char ** argv); 128 | void mflash_init(int argc, const char ** argv) { 129 | set_argc(argc, argv); 130 | } 131 | 132 | void mflash_init(int argc, char ** argv){ 133 | mflash_init(argc, const_cast(argv)); 134 | } 135 | 136 | void check_cmd_init() { 137 | if (!_cmd_configured) { 138 | std::cout << "ERROR: command line options not initialized." << std::endl; 139 | std::cout << " You need to call set_argc() in the beginning of the program." << std::endl; 140 | } 141 | } 142 | 143 | 144 | 145 | 146 | std::string get_option_string(const char *option_name, 147 | std::string default_value) 148 | { 149 | check_cmd_init(); 150 | int i; 151 | 152 | for (i = _argc - 2; i >= 0; i -= 1) 153 | if (strcmp(_argv[i], option_name) == 0) 154 | return std::string(_argv[i + 1]); 155 | return get_config_option_string(option_name, default_value); 156 | } 157 | 158 | std::string get_option_string(const char *option_name) 159 | { 160 | int i; 161 | check_cmd_init(); 162 | 163 | for (i = _argc - 2; i >= 0; i -= 1) 164 | if (strcmp(_argv[i], option_name) == 0) 165 | return std::string(_argv[i + 1]); 166 | return get_config_option_string(option_name); 167 | } 168 | 169 | std::string get_option_string_interactive(const char *option_name, std::string options) 170 | { 171 | int i; 172 | check_cmd_init(); 173 | 174 | for (i = _argc - 2; i >= 0; i -= 1) 175 | if (strcmp(_argv[i], option_name) == 0) 176 | return std::string(_argv[i + 1]); 177 | if (conf.find(option_name) != conf.end()) { 178 | return conf[option_name]; 179 | } 180 | 181 | std::cout << "Please enter value for command-line argument [" << std::string(option_name) << "]"<< std::endl; 182 | std::cout << " (Options are: " << options << ")" << std::endl; 183 | 184 | std::string val; 185 | std::cin >> val; 186 | 187 | return val; 188 | } 189 | 190 | 191 | 192 | 193 | 194 | int get_option_int(const char *option_name, int default_value) 195 | { 196 | int i; 197 | check_cmd_init(); 198 | 199 | for (i = _argc - 2; i >= 0; i -= 1) 200 | if (strcmp(_argv[i], option_name) == 0) 201 | return atoi(_argv[i + 1]); 202 | 203 | return get_config_option_int(option_name, default_value); 204 | } 205 | 206 | int get_option_int(const char *option_name) 207 | { 208 | int i; 209 | check_cmd_init(); 210 | 211 | for (i = _argc - 2; i >= 0; i -= 1) 212 | if (strcmp(_argv[i], option_name) == 0) 213 | return atoi(_argv[i + 1]); 214 | 215 | return get_config_option_int(option_name); 216 | 217 | } 218 | 219 | 220 | 221 | uint64_t get_option_long(const char *option_name, uint64_t default_value) 222 | { 223 | int i; 224 | check_cmd_init(); 225 | 226 | for (i = _argc - 2; i >= 0; i -= 1) 227 | if (strcmp(_argv[i], option_name) == 0) 228 | return atol(_argv[i + 1]); 229 | return get_config_option_long(option_name, default_value); 230 | } 231 | 232 | float get_option_float(const char *option_name, float default_value) 233 | { 234 | int i; 235 | check_cmd_init(); 236 | 237 | for (i = _argc - 2; i >= 0; i -= 1) 238 | if (strcmp(_argv[i], option_name) == 0) 239 | return (float)atof(_argv[i + 1]); 240 | return (float) get_config_option_double(option_name, default_value); 241 | } 242 | 243 | } // End namespace 244 | 245 | 246 | #endif 247 | 248 | 249 | -------------------------------------------------------------------------------- /src/util/configfile.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | // 9 | // NOTICE: 10 | // This file was adapted from graphchi framework https://github.com/GraphChi/graphchi-cpp/blob/master/src/util/configfile.hpp 11 | // This file incorporates work covered by the following copyright and permission notice: 12 | // Copyright [2012] [Aapo Kyrola, Guy Blelloch, Carlos Guestrin / Carnegie Mellon University] 13 | // 14 | // Licensed under the Apache License, Version 2.0 (the "License"); 15 | // you may not use this file except in compliance with the License. 16 | // You may obtain a copy of the License at 17 | // 18 | // http://www.apache.org/licenses/LICENSE-2.0 19 | // 20 | // Unless required by applicable law or agreed to in writing, software 21 | // distributed under the License is distributed on an "AS IS" BASIS, 22 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 23 | // See the License for the specific language governing permissions and 24 | // limitations under the License. 25 | #ifndef MFLASH_CONFIGFILE_DEF 26 | #define MFLASH_CONFIGFILE_DEF 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | namespace mflash { 35 | 36 | // Code for trimming strings copied from + modified 37 | // http://stackoverflow.com/questions/479080/trim-is-not-part-of-the-standard-c-c-library 38 | const std::string whiteSpaces( " \f\n\r\t\v" ); 39 | 40 | 41 | static void trimRight( std::string &str, 42 | const std::string& trimChars ) 43 | { 44 | std::string::size_type pos = str.find_last_not_of( trimChars ); 45 | str.erase( pos + 1 ); 46 | } 47 | 48 | 49 | static void trimLeft( std::string &str, 50 | const std::string& trimChars ) 51 | { 52 | std::string::size_type pos = str.find_first_not_of( trimChars ); 53 | str.erase( 0, pos ); 54 | } 55 | 56 | 57 | static std::string trim( std::string str) 58 | { 59 | std::string trimChars = " \f\n\r\t\v"; 60 | trimRight( str, trimChars ); 61 | trimLeft( str, trimChars ); 62 | return str; 63 | } 64 | 65 | 66 | // Removes \n from the end of line 67 | static void _FIXLINE(char * s) { 68 | int len = (int)strlen(s)-1; 69 | if(s[len] == '\n') s[len] = 0; 70 | } 71 | 72 | /** 73 | * Returns a key-value map of a configuration file key-values. 74 | * If file is not found, fails with an assertion. 75 | * @param filename filename of the configuration file 76 | * @param secondary_filename secondary filename if the first version is not found. 77 | */ 78 | static std::map loadconfig(std::string filename, std::string secondary_filename) { 79 | FILE * f = fopen(filename.c_str(), "r"); 80 | if (f == NULL) { 81 | f = fopen(secondary_filename.c_str(), "r"); 82 | if (f == NULL) { 83 | std::cout << "ERROR: Could not read configuration file: " << filename << std::endl; 84 | std::cout << "Please define environment variable MFLASH_ROOT or run the program from that directory." << std::endl; 85 | } 86 | assert(f != NULL); 87 | } 88 | 89 | char s[4096]; 90 | std::map conf; 91 | 92 | // I like C parsing more than C++, that is why this is such a mess 93 | while(fgets(s, 4096, f) != NULL) { 94 | _FIXLINE(s); 95 | if (s[0] == '#') continue; // Comment 96 | if (s[0] == '%') continue; // Comment 97 | 98 | char delims[] = "="; 99 | char * t; 100 | t = strtok(s, delims); 101 | const char * ckey = t; 102 | t = strtok(NULL, delims); 103 | const char * cval = t; 104 | 105 | if (ckey != NULL && cval != NULL) { 106 | std::string key = trim(std::string(ckey)); 107 | std::string val = trim(std::string(cval)); 108 | conf[key] = val; 109 | } 110 | } 111 | 112 | 113 | return conf; 114 | } 115 | 116 | }; 117 | 118 | 119 | #endif 120 | 121 | -------------------------------------------------------------------------------- /src/util/mflashnames.hpp: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2014 Hugo Gualdron , Sao Paulo University 2 | // Copyright (C) 2014 Jose Fernando Rodrigues Junior, Sao Paulo University 3 | // Copyright (C) 2014 Duen Horng Chau, Georgia Tech University 4 | // 5 | // This Source Code Form is subject to the terms of the Mozilla Public 6 | // License, v. 2.0. If a copy of the MPL was not distributed with this 7 | // file, You can obtain one at http://mozilla.org/MPL/2.0/. 8 | // 9 | // NOTICE: 10 | // This file was adapted from graphchi framework https://github.com/GraphChi/graphchi-cpp/blob/master/src/api/chifilenames.hpp 11 | // This file incorporates work covered by the following copyright and permission notice: 12 | // Copyright [2012] [Aapo Kyrola, Guy Blelloch, Carlos Guestrin / Carnegie Mellon University] 13 | // 14 | // Licensed under the Apache License, Version 2.0 (the "License"); 15 | // you may not use this file except in compliance with the License. 16 | // You may obtain a copy of the License at 17 | // 18 | // http://www.apache.org/licenses/LICENSE-2.0 19 | // 20 | // Unless required by applicable law or agreed to in writing, software 21 | // distributed under the License is distributed on an "AS IS" BASIS, 22 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 23 | // See the License for the specific language governing permissions and 24 | // limitations under the License. 25 | #ifndef MFLASH_FILENAMES_DEF 26 | #define MFLASH_FILENAMES_DEF 27 | 28 | #include 29 | #include 30 | 31 | 32 | 33 | namespace mflash{ 34 | /** 35 | * Configuration file name 36 | */ 37 | static std::string filename_config(); 38 | static std::string filename_config() { 39 | char * chi_root = getenv("MFLASH_ROOT"); 40 | if (chi_root != NULL) { 41 | return std::string(chi_root) + "/conf/mflash.cnf"; 42 | } else { 43 | return "conf/mflash.cnf"; 44 | } 45 | } 46 | 47 | /** 48 | * Configuration file name - local version which can 49 | * override the version in the version control. 50 | */ 51 | static std::string filename_config_local(); 52 | static std::string filename_config_local() { 53 | char * chi_root = getenv("MFLASH_ROOT"); 54 | if (chi_root != NULL) { 55 | return std::string(chi_root) + "/conf/mflash.local.cnf"; 56 | } else { 57 | return "conf/mflash.local.cnf"; 58 | } 59 | } 60 | 61 | 62 | 63 | } 64 | 65 | #endif 66 | 67 | -------------------------------------------------------------------------------- /test/ArrayTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * ArrayTest.cpp 3 | * 4 | * Created on: Feb 28, 2015 5 | * Author: hugo 6 | */ 7 | 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include "../core2/malgorithm.hpp" 14 | #include "../core2/primitivematrix.hpp" 15 | #include "../core2/primitivevector.hpp" 16 | #include "../log/easylogging++.h" 17 | 18 | INITIALIZE_EASYLOGGINGPP 19 | 20 | 21 | using namespace std; 22 | using namespace mflash; 23 | 24 | int main(){ 25 | 26 | PrimitiveVector pvector("/hugo/datasets/undirected-1GB-double/v1", size, bsize); 27 | 28 | Matrix matrix ("/hugo/datasets/undirected-1GB-double/yahoo", size, false, bsize, Mode::UNSAFE); 29 | //matrix.add_field(FieldType::SOURCE, "degree", aux); 30 | matrix.operate(algorithm, pvector, pout); 31 | return 0; 32 | } 33 | 34 | 35 | 36 | 37 | //cout<< sizeof(size_t)<< endl; 38 | //MappedStream stw ("/hugo/datasets/undirected-1GB-double/.G-FLASH/0_0.block", 2927861104, 19558831); 39 | /* 40 | 41 | PrimitiveVector pvector("/tmp/v31", size, bsize); 42 | PrimitiveVector pvector2("/tmp/v42", size, bsize); 43 | PrimitiveVector pout("/tmp/ou1t", size, bsize); 44 | 45 | pvector.fill(3); 46 | pvector.operate(prindst); 47 | //Vector::operate(prindst, pvector, 0, new Vector*[0]{}); 48 | 49 | pvector2.fill(2); 50 | pvector2.operate(prindst); 51 | //Vector::operate(prindst, pvector2, 0, new Vector*[0]{}); 52 | 53 | //PrimitiveVector *pvectors ;//new PrimitiveVector[2]{pvector, pvector2}; 54 | 55 | float constants [] = {5,6}; 56 | pout.linear_combination(2, constants, new PrimitiveVector*[3]{&pvector, &pvector2}); 57 | //Vector::operate(prindst, pout, 0, new Vector*[0]{}); 58 | pout.operate(prindst); 59 | 60 | cout<< "Linear combination = "<< pout.pnorm(1) << endl; 61 | 62 | pvector.fill_random(); 63 | cout<< pvector.pnorm(2) << endl; 64 | PrimitiveVector t = pvector.transpose(); 65 | cout<< pvector.multiply(t) << endl; 66 | cout<< pow(pvector.pnorm(2),2) << endl; 67 | 68 | pvector.multiply((float)(1/pvector.pnorm(2))); 69 | cout<< pvector.pnorm(2) << endl; 70 | */ 71 | /* 72 | 73 | size = 1413511394; 74 | bsize = 67108864; 75 | //SpMV spmv; 76 | 77 | PrimitiveVector *vector = new PrimitiveVector("/hugo/datasets/undirected-1GB-double/v1", size, bsize); 78 | PrimitiveVector *out = new PrimitiveVector("/hugo/datasets/undirected-1GB-double/v2", size, bsize); 79 | // vector->fill(1); 80 | */ 81 | 82 | 83 | //vector->fill(1); 84 | //value = vector->operate(op, *vector, 0, new Vector[0]{/**vector*/}); 85 | //exit(0); 86 | // value = vector->operate(op, *vector, 0, new Vector[0]{/**vector*/}); 87 | //value = vector->operate(op3, *vector, 0, new Vector[0]{/**vector*/}); 88 | //cout << value << endl; 89 | 90 | 91 | /* time_t timer1; 92 | time_t timer2; 93 | time(&timer1); 94 | PrimitiveMatrix matrix ("/hugo/datasets/undirected-1GB-double/yahoo", size, false, bsize, Mode::UNSAFE);*/ 95 | //matrix.multiply(*vector, *out); 96 | 97 | /* 98 | LanczosSO lanczos (matrix, 20, 6); 99 | lanczos.run(); 100 | */ 101 | 102 | 103 | //matrix.multiply(*vector, *out); 104 | //matrix.operate(spmv, *vector, *out); 105 | /* time(&timer2); 106 | time_t final = timer2 - timer1; 107 | cout<<"Time: "<< final << " seconds"; 108 | time(&timer1); 109 | matrix.multiply(*vector, *out); 110 | //matrix.operate(spmv, *vector, *out); 111 | time(&timer2); 112 | final = timer2 - timer1; 113 | cout<<"Time: "<< final << " seconds";*/ 114 | /* 115 | 116 | 117 | 118 | Array *array = new Array(size); 119 | Array::operate(op, *array); 120 | value = Array::operate(op3, *array); 121 | cout << value << endl; 122 | value = Array::operate(op4, *array, *array); 123 | cout << value << endl; 124 | 125 | Array::operate(op2, *array, *array, *array); 126 | 127 | cout<< *(array->get_element(size-1)) << endl;*/ 128 | -------------------------------------------------------------------------------- /test/BinarySplitterTest.cpp: -------------------------------------------------------------------------------- 1 | #include "../src/core/graphbinarization.hpp" 2 | #include "../src/core/mapped_stream.hpp" 3 | #include "../log/easylogging++.h" 4 | #include "../src/core/splitterbuffer.hpp" 5 | 6 | 7 | INITIALIZE_EASYLOGGINGPP 8 | 9 | using namespace mflash; 10 | int main(){ 11 | /* 12 | LOG(INFO)<<"Starting Binary splitter!!"; 13 | std::string graph_file = "/hugo/datasets/yahoo1/0.partition"; 14 | SplitterBuffer splitter(graph_file, 0, 268435456, 8589934592, 0, true); 15 | 16 | MappedStream stream (graph_file); 17 | 18 | int from, to; 19 | EmptyField field; 20 | stream.set_position(0 ); 21 | while(stream.has_remain()){ 22 | from = stream.next_int(); 23 | to = stream.next_int(); 24 | splitter.add(from, to, &field); 25 | } 26 | splitter.flush(); 27 | LOG(INFO)<<"Binary splitter ready!!";*/ 28 | 29 | } 30 | -------------------------------------------------------------------------------- /test/BlockEdgeTest.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "../src/mflash_basic_includes.hpp" 7 | #include "../src/core/mapped_stream.hpp" 8 | 9 | //INITIALIZE_EASYLOGGINGPP 10 | 11 | using namespace mflash; 12 | int main(){ 13 | std::string graph_file; 14 | graph_file = "/data/hugo-data/hugo-others/datasets/.M-FLASH/lj/0.partition.output"; 15 | MappedStream stream (graph_file); 16 | 17 | int32 from, to; 18 | //float value; 19 | int32 mfrom = 100000, mto = 100000; 20 | 21 | while(stream.has_remain()){ 22 | from = stream.next_int(); 23 | to = stream.next_int(); 24 | //value = *((float*)stream.next(4, 0)); 25 | if(from <= mfrom){ 26 | mfrom = from; 27 | if(to < mto){ 28 | mto = to; 29 | } 30 | } 31 | } 32 | stream.close_stream(); 33 | std::cout< 2 | #include 3 | #include 4 | #include 5 | 6 | 7 | template 8 | class Splitter{ 9 | public: 10 | IdType getId(IdType in, IdType out); 11 | }; 12 | 13 | template 14 | IdType Splitter::getId(IdType in, IdType out){ 15 | #if inSplit 16 | return in>>2; 17 | #else 18 | return out>>2; 19 | #endif 20 | }; 21 | 22 | int main(){ 23 | Splitter splitter1; 24 | Splitter splitter2; 25 | std::cout << splitter1.getId(4, 8) < 3 | #include "../src/mflash_basic_includes.hpp" 4 | #include "../src/core/genericedgesplittermanager.hpp" 5 | 6 | //INITIALIZE_EASYLOGGINGPP 7 | 8 | using namespace mflash; 9 | 10 | int main(){ 11 | 12 | std::string graph_file = "/data/hugo-data/hugo-others/datasets/lj"; 13 | GenericEdgeSplitterManager emanager( 4294967296, true, 0); 14 | SplitterBuffer , EmptyField> bsplitter (&emanager, graph_file, 0 ,mflash::DEFAULT_MEMORY_SIZE, "", false); 15 | EdgeConversor::process(graph_file, ' ', true, bsplitter ); 16 | return 0; 17 | } 18 | -------------------------------------------------------------------------------- /test/MMapGraphFormatter2.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include "../src/mflash_basic_includes.hpp" 4 | #include "../src/core/genericedgesplittermanager.hpp" 5 | #include "../src/util/kwaymergesort.h" 6 | 7 | //INITIALIZE_EASYLOGGINGPP 8 | 9 | using namespace mflash; 10 | 11 | 12 | 13 | 14 | // a basic struct for a Edge entry. 15 | struct Edge { 16 | int32 from; 17 | int32 to; 18 | 19 | bool operator < (const Edge &b) const 20 | { 21 | 22 | if( (((int64)from) <<32) + to < (((int64)b.from) <<32) + b.to){ 23 | return true; 24 | } 25 | 26 | return false; 27 | } 28 | 29 | 30 | }; 31 | 32 | // comparison functions for the ext. merge sort 33 | bool comparator(const Edge &a, const Edge &b) { return a < b; } 34 | 35 | 36 | int main(){ 37 | 38 | /* 39 | std::string graph_file = "/data/hugo-data/hugo-others/datasets/powerlawgraph"; 40 | GenericEdgeSplitterManager emanager( 4294967296, true, 0); 41 | SplitterBuffer , EmptyField> bsplitter (&emanager, graph_file, 0 ,mflash::DEFAULT_MEMORY_SIZE, "", false); 42 | EdgeConversor::process(graph_file, ' ', true, bsplitter ); 43 | */ 44 | 45 | 46 | string inFile = "/data/hugo-data/hugo-others/datasets/.M-FLASH/twitter/0_0.block";//argv[1]; 47 | string outFile = "/data/hugo-data/hugo-others/datasets/.M-FLASH/twitter/0_0.block.output";//argv[1]; 48 | int32 bufferSize = 1073741824; // allow the sorter to use 100Kb (base 10) of memory for sorting. 49 | // once full, it will dump to a temp file and grab another chunk. 50 | bool compressOutput = false; // not yet supported 51 | string tempPath = "/data/hugo-data/hugo-others/datasets/.M-FLASH/twitter/"; // allows you to write the intermediate files anywhere you want. 52 | 53 | // sort the lines of a file lexicographically in ascending order (akin to UNIX sort, "sort FILE") 54 | KwayMergeSort *sorter = new KwayMergeSort (inFile, 55 | outFile, 56 | comparator, 57 | bufferSize, 58 | compressOutput, 59 | tempPath); 60 | sorter->Sort(); 61 | return 0; 62 | } 63 | -------------------------------------------------------------------------------- /test/MMapGraphFormatter3.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include "../src/mflash_basic_includes.hpp" 4 | #include "../src/core/genericedgesplittermanager.hpp" 5 | #include "../src/util/kwaymergesort.h" 6 | 7 | //INITIALIZE_EASYLOGGINGPP 8 | 9 | using namespace mflash; 10 | 11 | 12 | 13 | 14 | // a basic struct for a Edge entry. 15 | struct Edge { 16 | int32 from; 17 | int32 to; 18 | 19 | bool operator < (const Edge &b) const 20 | { 21 | 22 | if( (((int64)from) <<32) + to < (((int64)b.from) <<32) + b.to){ 23 | return true; 24 | } 25 | 26 | return false; 27 | } 28 | 29 | 30 | }; 31 | 32 | // comparison functions for the ext. merge sort 33 | bool comparator(const Edge &a, const Edge &b) { return a < b; } 34 | 35 | 36 | int main(){ 37 | std::ofstream output("/data/hugo-data/hugo-others/datasets/.M-FLASH/twitter_plain_sorted"); 38 | MappedStream stream ("/data/hugo-data/hugo-others/datasets/.M-FLASH/twitter/0_0.block.output"); 39 | int32 from ,to; 40 | while(stream.has_remain()){ 41 | from = stream.next(); 42 | to = stream.next(); 43 | output <(graph_file, sizeof(float), sizeof(float), "edgelist"); 38 | 39 | 40 | //LiveJournal 41 | /* std::string graph_file = "/data/hugo-data/hugo-others/datasets/test"; 42 | std::string format = "adjlist2"; 43 | 44 | 45 | mflash::convert(graph_file, sizeof(float), format);*/ 46 | return 0; 47 | } 48 | -------------------------------------------------------------------------------- /test/PrintVectorTest.cpp: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include "../src/mflash_basic_includes.hpp" 4 | 5 | using namespace mflash; 6 | 7 | template 8 | class PrintVector: public ZeroOperator{ 9 | public: 10 | inline void apply(Element &out){ 11 | std::cout<< out.id << " = "<< *(out.value) << std::endl; 12 | } 13 | }; 14 | 15 | int main(int argc, char* argv[]){ 16 | mflash_init(argc, argv); 17 | 18 | /* Parameters */ 19 | std::string filename = get_option_string("file"); // Base filename 20 | 21 | PrintVector printOperator; 22 | 23 | PrimitiveVector pvector(filename, 12); 24 | pvector.operate(printOperator); 25 | 26 | return 0; 27 | } 28 | -------------------------------------------------------------------------------- /test/SortedTest.cpp: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include "../src/mflash_basic_includes.hpp" 4 | 5 | using namespace mflash; 6 | int main(int argc, char* argv[]){ 7 | 8 | std::string graph_file = "/data/hugo-data/hugo-others/datasets/.M-FLASH/powerlawgraph/powerlawgraph.bin"; 9 | 10 | MappedStream stream (graph_file); 11 | 12 | int64 value = -1; 13 | int64 tmp = 0; 14 | int32 from, to; 15 | int64 edges = 0; 16 | while(stream.has_remain()){ 17 | from = stream.next(); 18 | to = stream.next(); 19 | value = (((int64)from) <<32) + to; 20 | edges++; 21 | if(tmp >= value){ 22 | LOG(ERROR)<<"Sorting error in edge "<< edges<< " : " << tmp << " > " << value << " (" < splitter(graph_file, 0, 11, 24); 15 | SplitterBuffer splitter(graph_file, 0, 4847573, 8589934592); 16 | convert_edgelist(graph_file, splitter ); 17 | return 0; 18 | } 19 | -------------------------------------------------------------------------------- /test/TestMergeInt64.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | using namespace std; 8 | 9 | // local includes 10 | #include "../src/mflash_basic_includes.hpp" 11 | #include "../src/util/kwaymergesort.h" 12 | 13 | 14 | 15 | // a basic struct for a Edge entry. 16 | struct Edge { 17 | int32 from; 18 | int32 to; 19 | 20 | bool operator < (const Edge &b) const 21 | { 22 | if( (((int64)from) <<32) + to < (((int64)b.from) <<32) + b.to){ 23 | return true; 24 | } 25 | 26 | return false; 27 | } 28 | 29 | 30 | }; 31 | 32 | // comparison functions for the ext. merge sort 33 | bool comparator(const Edge &a, const Edge &b) { return a < b; } 34 | 35 | 36 | int main(int argc, char* argv[]) { 37 | 38 | string inFile = "/data/hugo-data/hugo-others/datasets/.M-FLASH/twitter/0.partition";//argv[1]; 39 | string outFile = "/data/hugo-data/hugo-others/datasets/.M-FLASH/twitter/0.partition.output";//argv[1]; 40 | int32 bufferSize = 1073741824; // allow the sorter to use 100Kb (base 10) of memory for sorting. 41 | // once full, it will dump to a temp file and grab another chunk. 42 | bool compressOutput = false; // not yet supported 43 | string tempPath = "/data/hugo-data/hugo-others/datasets/.M-FLASH/twitter/"; // allows you to write the intermediate files anywhere you want. 44 | 45 | // sort the lines of a file lexicographically in ascending order (akin to UNIX sort, "sort FILE") 46 | KwayMergeSort *sorter = new KwayMergeSort (inFile, 47 | outFile, 48 | comparator, 49 | bufferSize, 50 | compressOutput, 51 | tempPath); 52 | sorter->Sort(); 53 | } 54 | -------------------------------------------------------------------------------- /test/TestPlainBlockCounters.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "../src/mflash_basic_includes.hpp" 7 | #include "../src/core/mapped_stream.hpp" 8 | 9 | //INITIALIZE_EASYLOGGINGPP 10 | 11 | using namespace mflash; 12 | int main(){ 13 | std::string graph_file; 14 | 15 | 16 | int64 vertices_by_partition; 17 | int64 vertices_graph; 18 | int64 partitions = 200; 19 | //YAHOO GRAPH 20 | graph_file = "/data/hugo-data/hugo-others/datasets/.M-FLASH/yahoo/1.partition"; 21 | vertices_graph = 1413511393; 22 | vertices_by_partition = 268435456; 23 | partitions = vertices_graph /vertices_by_partition+ (vertices_graph %vertices_by_partition != 0? 1:0); 24 | 25 | //TWITTER 26 | /* 27 | 28 | graph_file = "/hugo/datasets/twitter/.M-FLASH/0.partition"; 29 | vertices_graph = 41652231; 30 | 31 | */ 32 | 33 | //LJ 34 | /* 35 | graph_file = "/hugo/datasets/lj/.M-FLASH/0.partition"; 36 | vertices_graph = 4847573; 37 | */ 38 | 39 | 40 | //vertices_by_partition = vertices_graph /partitions + (vertices_graph %partitions != 0? 1:0); 41 | 42 | 43 | MappedStream stream (graph_file); 44 | 45 | /* std::vector counters (partitions * partitions); 46 | 47 | 48 | memset(counters.data(), 0, sizeof(int64) * partitions * partitions);*/ 49 | int max_in = 0; 50 | int max_out = 0; 51 | int from, to, idx; 52 | while(stream.has_remain()){ 53 | from = stream.next_int(); 54 | to = stream.next_int(); 55 | //idx = ((int64)( from/vertices_by_partition)) * partitions+ (int64)(to/ vertices_by_partition); 56 | //counters[idx]++; 57 | max_in = max(from, max_in); 58 | max_out = max(to, max_out); 59 | } 60 | 61 | std::cout<<"MAX IN = "< 6 | 7 | //INITIALIZE_EASYLOGGINGPP 8 | 9 | using namespace mflash; 10 | int main(){ 11 | //std::string graph_file = "/hugo/datasets/tiny-graph/tiny-graph"; 12 | 13 | //std::string graph_file = "/hugo/datasets/twitter/twitter"; 14 | 15 | 16 | //LJ 17 | /*std::string graph_file = "/run/media/hugo/hugo/datasets/lj/.M-FLASH/lj/0_0.block"; 18 | int64 edges = 68993773; 19 | int64 max_vertex_id = 4847573; 20 | */ 21 | 22 | //YAHOO 23 | /* 24 | std::string graph_file = "/run/media/hugo/hugo/datasets/yahoo1/0.partition"; 25 | int64 edges = 6636600779; 26 | int64 max_vertex_id = 1413511393; 27 | */ 28 | 29 | 30 | //TWITTER 31 | 32 | std::string graph_file = "/run/media/hugo/hugo/datasets/twitter/.M-FLASH/0_0.block"; 33 | int64 edges = 1468365182; 34 | int64 max_vertex_id = 41652231; 35 | 36 | 37 | FILE* fout = fopen("/run/media/hugo/data/frameworks/turbograph/Turbograph/KDDResults/data/twitter.txt", "w"); 38 | 39 | MappedStream stream (graph_file); 40 | 41 | fprintf(fout, "%lu %lu\n", max_vertex_id, edges); 42 | int from, to, idx; 43 | while(stream.has_remain()){ 44 | from = stream.next_int(); 45 | to = stream.next_int(); 46 | fprintf(fout, "%d %d\n", from, to); 47 | } 48 | fclose(fout); 49 | stream.close_stream(); 50 | return 0; 51 | } 52 | -------------------------------------------------------------------------------- /test/XStreamGraphFormatter.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include "../src/mflash_basic_includes.hpp" 4 | #include "../src/core/genericedgesplittermanager.hpp" 5 | 6 | //INITIALIZE_EASYLOGGINGPP 7 | 8 | using namespace mflash; 9 | 10 | int main(){ 11 | 12 | std::string graph_file = "/data/hugo-data/hugo-others/datasets/powerlawgraph"; 13 | GenericEdgeSplitterManager emanager( 4294967296, true, 0); 14 | SplitterBuffer , float> bsplitter (&emanager, graph_file, sizeof(float) ,mflash::DEFAULT_MEMORY_SIZE, "", false); 15 | EdgeConversor::process(graph_file, ' ', true, bsplitter ); 16 | return 0; 17 | } 18 | -------------------------------------------------------------------------------- /test/graphchi_test: -------------------------------------------------------------------------------- 1 | #live journal pgrank 2 | cd /data/hugo-data/hugo-others/frameworks/graphchi-cpp/ 3 | sudo bin/example_apps/pagerank_functional mode sync file /data/hugo-data/hugo-others/datasets/lj filetype edgelist niters 10 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/graphchi-lj3 4 | 5 | 6 | #live journal cc 7 | $/data/hugo-data/hugo-others/frameworks/graphchi-cpp/example_apps/unionfind_connectedcomps.cpp 8 | 9 | cd /data/hugo-data/hugo-others/frameworks/graphchi-cpp/ 10 | make example_apps/unionfind_connectedcomps 11 | sudo bin/example_apps/unionfind_connectedcomps file /data/hugo-data/hugo-others/datasets/lj filetype edgelist 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/graphchi-cc-lj 12 | 13 | #twitter pgrank 14 | cd /data/hugo-data/hugo-others/frameworks/graphchi-cpp/ 15 | make example_apps/pagerank_functional 16 | sudo bin/example_apps/pagerank_functional mode sync file /data/hugo-data/hugo-others/datasets/twitter filetype edgelist niters 10 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/graphchi-twitter-pg1 17 | 18 | #twitter cc 19 | sudo bin/example_apps/unionfind_connectedcomps file /data/hugo-data/hugo-others/datasets/twitter filetype edgelist 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/graphchi-twitter-uf1 20 | 21 | #yahoo pgrank 22 | cd /data/hugo-data/hugo-others/frameworks/graphchi-cpp/ 23 | make example_apps/pagerank_functional 24 | sudo bin/example_apps/pagerank_functional mode sync file /data/hugo-data/hugo-others/datasets/yahoo filetype adjlist niters 1 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/graphchi-yahoo-pg1 25 | 26 | #yahoo cc 27 | cd /data/hugo-data/hugo-others/frameworks/graphchi-cpp/ 28 | make example_apps/connectedcomponents 29 | sudo bin/example_apps/connectedcomponents file /data/hugo-data/hugo-others/datasets/yahoo filetype adjlist niters 1 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/graphchi-yahoo-cc1 30 | 31 | 32 | #powerlaw pg 33 | cd /data/hugo-data/hugo-others/frameworks/graphchi-cpp/ 34 | make example_apps/pagerank_functional 35 | sudo bin/example_apps/pagerank_functional mode sync file /data/hugo-data/hugo-others/datasets/powerlawgraph filetype edgelist niters 1 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/graphchi-powerlawgraph-pg1 36 | 37 | #powerlaw cc 38 | cd /data/hugo-data/hugo-others/frameworks/graphchi-cpp/ 39 | make example_apps/connectedcomponents 40 | sudo bin/example_apps/connectedcomponents file /data/hugo-data/hugo-others/datasets/powerlawgraph filetype edgelist niters 1 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/graphchi-powerlawgraph-cc1 41 | -------------------------------------------------------------------------------- /test/log_test.cpp: -------------------------------------------------------------------------------- 1 | #include "../src/mflash_basic_includes.hpp" 2 | 3 | int main(int, char*[]) 4 | { 5 | LOG(INFO) << "INFO LOG"; 6 | LOG(DEBUG) << "DEBUG LOG"; 7 | LOG(ERROR) << "ERROR LOG"; 8 | return 0; 9 | } 10 | -------------------------------------------------------------------------------- /test/main.cpp: -------------------------------------------------------------------------------- 1 | #include "../src/core/graphbinarization.hpp" 2 | #include "../src/core/mapped_stream.hpp" 3 | #include "../log/easylogging++.h" 4 | #include "../src/core/splitterbuffer.hpp" 5 | 6 | 7 | INITIALIZE_EASYLOGGINGPP 8 | 9 | using namespace mflash; 10 | int main(){ 11 | 12 | //:SplitterBuffer(std::string graph, int64 edge_data_size, int64 ids_by_partitions, int64 buffer_size, int64 partitions, bool in_split){ 13 | 14 | 15 | std::string graph_file = "/hugo/datasets/yahoo.txt"; 16 | graph_file = "/hugo/datasets/yahoo.txt"; 17 | SplitterBuffer splitter(graph_file, 0, 268435456, 4294967296); 18 | convert_adjlist(graph_file, splitter ); 19 | 20 | 21 | 22 | 23 | 24 | /** 25 | * SPEED TEST EXAMPLE 26 | 27 | LOG (INFO) << sizeof(char); 28 | MappedStream stream("/hugo/datasets/yahoo.txt"); 29 | 30 | long sum = 0; 31 | 32 | while(stream.has_remain()){ 33 | sum += stream.next_int(); 34 | } 35 | LOG (INFO) << sum; 36 | stream.close_stream(); 37 | */ 38 | 39 | 40 | 41 | return 0; 42 | } 43 | -------------------------------------------------------------------------------- /test/next: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/poloclub/m-flash-cpp/41f238cff821f7502fde2f19202ff213f42850d9/test/next -------------------------------------------------------------------------------- /test/pagerank.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | n = 12 3 | a = np.zeros( [n,n]) 4 | a[0,4] = 1 5 | a[0,7] = 1 6 | a[0,8] = 1 7 | a[1,7] = 1 8 | a[2,7] = 1 9 | a[3,5] = 1 10 | a[3,6] = 1 11 | a[4,0] = 1 12 | a[4,5] = 1 13 | a[4,7] = 1 14 | a[5,3] = 1 15 | a[5,4] = 1 16 | a[5,6] = 1 17 | a[6,3] = 1 18 | a[6,5] = 1 19 | a[7,0] = 1 20 | a[7,1] = 1 21 | a[7,2] = 1 22 | a[7,4] = 1 23 | a[8,0] = 1 24 | a[8,9] = 1 25 | a[8,10] = 1 26 | a[8,11] = 1 27 | a[9,8] = 1 28 | a[10,8] = 1 29 | a[11,8] = 1 30 | 31 | 32 | at = a.transpose() 33 | out_degree = np.ones([n,1]) 34 | out_degree = np.matmul(at,out_degree) 35 | out_degree = 1/out_degree 36 | 37 | destination = np.ones([n,1]) /n 38 | for i in range(1,4): 39 | destination *= out_degree 40 | destination = np.matmul(a,destination) * 0.85 + (0.15/n) 41 | 42 | -------------------------------------------------------------------------------- /test/pg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/poloclub/m-flash-cpp/41f238cff821f7502fde2f19202ff213f42850d9/test/pg -------------------------------------------------------------------------------- /test/sparse-graph: -------------------------------------------------------------------------------- 1 | 0 1 2 | 0 4 3 | 0 8 4 | 1 0 5 | 1 5 6 | 1 10 7 | 3 1 8 | 3 9 9 | 4 1 10 | 5 4 11 | 6 0 12 | 7 2 13 | 7 5 14 | 7 10 15 | 9 1 16 | 9 7 17 | 10 3 18 | 10 5 19 | -------------------------------------------------------------------------------- /test/sparse-graph-shuffle: -------------------------------------------------------------------------------- 1 | 3 1 2 | 0 1 3 | 1 0 4 | 0 8 5 | 1 5 6 | 1 10 7 | 0 4 8 | 3 9 9 | 4 1 10 | 5 4 11 | 6 0 12 | 7 2 13 | 7 5 14 | 7 10 15 | 9 1 16 | 9 7 17 | 10 3 18 | 10 5 19 | -------------------------------------------------------------------------------- /test/sparse-graph-shuffle_1first_round: -------------------------------------------------------------------------------- 1 | 0 4 2 | 0 1 3 | 1 0 4 | 0 8 5 | 1 5 6 | 1 10 7 | 3 1 8 | 3 9 9 | 4 1 10 | 5 4 11 | 6 0 12 | 7 2 13 | 7 5 14 | 7 10 15 | 9 1 16 | 9 7 17 | 10 3 18 | 10 5 19 | 20 | -------------------------------------------------------------------------------- /test/sparse-graph-shuffle_2_round: -------------------------------------------------------------------------------- 1 | 4 1 2 | 0 1 3 | 1 0 4 | 9 1 5 | 6 0 6 | 3 1 7 | 10 3 8 | 7 2 9 | 0 4 10 | 1 5 11 | 5 4 12 | 10 5 13 | 7 5 14 | 9 7 15 | 0 8 16 | 3 9 17 | 1 10 18 | 7 10 19 | 20 | -------------------------------------------------------------------------------- /test/tiny-graph: -------------------------------------------------------------------------------- 1 | 0 1 2 2 | 1 4 0 4 5 10 3 | 2 1 1 4 | 3 1 7 5 | 4 2 5 7 6 | 5 2 7 8 7 | 6 1 9 8 | 7 2 4 10 9 | 8 0 10 | 9 1 8 11 | 10 1 7 -------------------------------------------------------------------------------- /test/xtream-lj.ini: -------------------------------------------------------------------------------- 1 | [graph] 2 | type=1 3 | name=twitter 4 | vertices=41652231 5 | edges= -------------------------------------------------------------------------------- /test/xtream-powerlawgraph.ini: -------------------------------------------------------------------------------- 1 | [graph] 2 | type=1 3 | name=powerlawgraph 4 | vertices=4000000000 5 | edges=12000000000 -------------------------------------------------------------------------------- /test/xtream-twiiter.ini: -------------------------------------------------------------------------------- 1 | [graph] 2 | type=1 3 | name=twitter 4 | vertices=4847571 5 | edges=1468365182 -------------------------------------------------------------------------------- /test/xtream-yahoo.ini: -------------------------------------------------------------------------------- 1 | [graph] 2 | type=1 3 | name=yahoo 4 | vertices=1413511392 5 | edges=6636600779 -------------------------------------------------------------------------------- /test/xtream_test: -------------------------------------------------------------------------------- 1 | #live journal 2 | cd /data/hugo-data/hugo-others/datasets/.M-FLASH/lj/ 3 | /data/hugo-data/hugo-others/frameworks/x-stream-0.9/bin/benchmark_driver -p 4 -a -b pagerank --pagerank::niters 10 -g lj --physical_memory 8589934592 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/xtream-lj 4 | 5 | sudo /data/hugo-data/hugo-others/frameworks/x-stream-0.9/bin/benchmark_driver -p 4 -a -b pagerank --pagerank::niters 10 -g lj --physical_memory 8589934592 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/xtream-lj3 6 | 7 | #connected components lj 8 | cd /data/hugo-data/hugo-others/datasets/.M-FLASH/lj/ 9 | sudo /data/hugo-data/hugo-others/frameworks/x-stream-0.9/bin/benchmark_driver -p 4 -a -b cc -g lj --physical_memory 8589934592 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/xtream-cc-lj1 10 | 11 | sudo /data/hugo-data/hugo-others/frameworks/x-stream-0.9/bin/benchmark_driver -p 4 -a -b pagerank --pagerank::niters 10 -g lj --physical_memory 8589934592 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/xtream-lj3 12 | 13 | 14 | #pagarank twitter 15 | cd /data/hugo-data/hugo-others/datasets/.M-FLASH/twitter 16 | sudo /data/hugo-data/hugo-others/frameworks/x-stream-0.9/bin/benchmark_driver -p 4 -a -b pagerank --pagerank::niters 10 -g twitter --physical_memory 8589934592 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/xtream-twitter_pagerank1 17 | 18 | #cc twitter 19 | cd /data/hugo-data/hugo-others/datasets/.M-FLASH/twitter 20 | sudo /data/hugo-data/hugo-others/frameworks/x-stream-0.9/bin/benchmark_driver -p 4 -a -b cc -g twitter --physical_memory 8589934592 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/xtream-cc-twitter3 21 | 22 | #yahoo pg 23 | cd /data/hugo-data/hugo-others/datasets/.M-FLASH/yahoo 24 | sudo /data/hugo-data/hugo-others/frameworks/x-stream-0.9/bin/benchmark_driver -p 4 -a -b pagerank --pagerank::niters 1 -g yahoo --physical_memory 8589934592 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/xtream-yahoo_pagerank1 25 | 26 | #yahoo cc 27 | cd /data/hugo-data/hugo-others/datasets/.M-FLASH/yahoo 28 | sudo /data/hugo-data/hugo-others/frameworks/x-stream-0.9/bin/benchmark_driver -p 4 -a -b cc -g yahoo --physical_memory 8589934592 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/xtream-cc-yahooo1 29 | 30 | 31 | #powerlawgraph pg 32 | cd /data/hugo-data/hugo-others/datasets/.M-FLASH/powerlawgraph 33 | sudo /data/hugo-data/hugo-others/frameworks/x-stream-0.9/bin/benchmark_driver -p 4 -a -b pagerank --pagerank::niters 1 -g powerlawgraph --physical_memory 8589934592 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/xtream-powerlawgraph_pagerank1 34 | 35 | #powerlawgraph cc 36 | cd /data/hugo-data/hugo-others/datasets/.M-FLASH/powerlawgraph 37 | sudo /data/hugo-data/hugo-others/frameworks/x-stream-0.9/bin/benchmark_driver -p 4 -a -b cc -g powerlawgraph --physical_memory 8589934592 2>&1 | tee /data/hugo-data/hugo-drive/doutorado/experiments/updated-experiments/xtream-cc-powerlawgraph2 38 | --------------------------------------------------------------------------------