├── AboutGraspan.md ├── graspan-java ├── executables │ ├── rules_np │ ├── graspan.jar │ ├── rules_pt │ └── graspan-java-run.sh ├── .gitignore ├── grammar ├── graph.grammar ├── test │ └── edu │ │ └── uci │ │ └── ics │ │ └── cs │ │ └── graspan │ │ ├── edgecomputationMerge │ │ ├── Class2.java │ │ ├── ArrayCopy.java │ │ ├── ArrayFunction.java │ │ ├── Setcopy.java │ │ ├── MinSet.java │ │ └── SortedArrMerger.java │ │ ├── edgecomputation │ │ ├── removeSet.java │ │ ├── ArrayPointerMadness.java │ │ ├── InitializeTwoDimArrTest.java │ │ └── TestPartitions.java │ │ ├── Time.java │ │ ├── MapTest.java │ │ ├── artificalgraphgenerator │ │ ├── EdgeValueAdder.java │ │ └── ArtificialGraphGenerator.java │ │ └── PartitionFileChecker.java ├── .classpath ├── .project ├── EdgeComputer-config.txt ├── src │ └── edu │ │ └── uci │ │ └── ics │ │ ├── cs │ │ └── graspan │ │ │ ├── scheduler │ │ │ ├── IScheduler.java │ │ │ ├── PartitionEdgeInfo.java │ │ │ ├── SchedulerInfo.java │ │ │ ├── BasicScheduler.java │ │ │ └── Scheduler.java │ │ │ ├── experiments │ │ │ ├── GPCppResultExtractorClient.java │ │ │ └── Extractor.java │ │ │ ├── datastructures │ │ │ ├── AllPartitions.java │ │ │ ├── RepartitioningData.java │ │ │ ├── LoadedVertexInterval.java │ │ │ ├── Vertex.java │ │ │ ├── ComputationSet.java │ │ │ ├── PartitionQuerier.java │ │ │ ├── NewEdgesList.java │ │ │ └── LoadedPartitions.java │ │ │ ├── dispatcher │ │ │ ├── PartERuleAdderAndSorterClient.java │ │ │ ├── PartGenClient.java │ │ │ ├── ComputationClient.java │ │ │ ├── GlobalParams.java │ │ │ └── PreprocessorClient.java │ │ │ ├── support │ │ │ ├── MemUsageCheckThread.java │ │ │ ├── Utilities.java │ │ │ └── GraspanLogger.java │ │ │ ├── computationM │ │ │ ├── MinSet.java │ │ │ ├── GrammarChecker.java │ │ │ └── EdgeComputerM.java │ │ │ └── computationEL │ │ │ ├── EdgeComputerEL.java │ │ │ └── EngineEL.java │ │ └── graspan │ │ └── oracle │ │ ├── GraphExtractor.java │ │ ├── NaiveComputer.java │ │ ├── GraphPartComparator.java │ │ └── WholeGraphComputer.java ├── .settings │ └── org.eclipse.jdt.core.prefs └── graph ├── Readme.txt └── README.md /AboutGraspan.md: -------------------------------------------------------------------------------- 1 | # About Graspan 2 | -------------------------------------------------------------------------------- /graspan-java/executables/rules_np: -------------------------------------------------------------------------------- 1 | n n e 2 | -------------------------------------------------------------------------------- /graspan-java/.gitignore: -------------------------------------------------------------------------------- 1 | /bin/ 2 | EdgeComputer-config.txt 3 | Preprocessor-config.txt 4 | -------------------------------------------------------------------------------- /graspan-java/executables/graspan.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Graspan/graspan-java/HEAD/graspan-java/executables/graspan.jar -------------------------------------------------------------------------------- /graspan-java/grammar: -------------------------------------------------------------------------------- 1 | DV=0 2 | d=1 3 | M=2 4 | -d=3 5 | V=4 6 | MAM=5 7 | AMs=6 8 | MAs=7 9 | Mq=8 10 | MA=9 11 | -a=10 12 | AM=11 13 | a=12 14 | -------------------------------------------------------------------------------- /graspan-java/graph.grammar: -------------------------------------------------------------------------------- 1 | M DV d 2 | DV -d V 3 | V MAM AMs 4 | MAM MAs Mq 5 | Mq 6 | Mq M 7 | MAs 8 | MAs MAs MA 9 | MA Mq -a 10 | AMs 11 | AMs AMs AM 12 | AM a Mq -------------------------------------------------------------------------------- /graspan-java/executables/rules_pt: -------------------------------------------------------------------------------- 1 | M DV d 2 | DV -d V 3 | V MAM AMs 4 | MAM MAs Mq 5 | Mq 6 | Mq M 7 | MAs 8 | MAs MAs MA 9 | MA Mq -a 10 | AMs 11 | AMs AMs AM 12 | AM a Mq 13 | -------------------------------------------------------------------------------- /Readme.txt: -------------------------------------------------------------------------------- 1 | This file shall contain a how-to-use guide to Graspan 2 | 3 | 4 | 5 | ----------------------------------------------------------------------- 6 | To be completed 7 | ----------------------------------------------------------------------- 8 | -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/edgecomputationMerge/Class2.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.edgecomputationMerge; 2 | 3 | public class Class2 { 4 | int a[]; 5 | 6 | public void abc() { 7 | a = new int[2]; 8 | a[0] = 1; 9 | } 10 | 11 | public int[] get_a() { 12 | return a; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /graspan-java/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/edgecomputation/removeSet.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.edgecomputation; 2 | 3 | import java.util.HashSet; 4 | 5 | public class removeSet { 6 | 7 | public static void main(String args[]) { 8 | 9 | HashSet myset = new HashSet(); 10 | 11 | myset.remove(8); 12 | System.out.println(myset.remove(8)); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /graspan-java/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | graspan-java 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | 15 | org.eclipse.jdt.core.javanature 16 | 17 | 18 | -------------------------------------------------------------------------------- /graspan-java/EdgeComputer-config.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | INPUT_GRAPH_FILEPATH = /home/aftab/workspace/graphs/POST_OSDI_TEST_CENTER/kernel_afterInline.txt.eRulesAdded.extracted 4 | TOTAL_NUM_PARTS = 5 5 | RELOAD_PLAN = RELOAD_PLAN_2 #(OPTIONS: RELOAD_PLAN_2) 6 | EDC_SIZE = 500 7 | NEW_EDGE_NODE_SIZE = 100 #DOES NOT APPLY FOR MERGE COMPUTATION LOGIC 8 | MAX_PART_SIZE_POST_NEW_EDGES = 10 9 | COMPUTATION_LOGIC = SMART_MERGE #(OPTIONS: SMART_MERGE, LINEAR_SCAN_OF_LLISTS) 10 | NUM_OF_THREADS = 8 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/edgecomputationMerge/ArrayCopy.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.edgecomputationMerge; 2 | 3 | import java.util.Arrays; 4 | 5 | public class ArrayCopy { 6 | 7 | public static void main(String args[]) { 8 | int[] arr = new int[] { 1, 2, 3, 4, 5 }; 9 | int[] arr2 = new int[2]; 10 | arr2[0] = arr[0]; 11 | arr2[1] = arr[1]; 12 | System.out.println("arr2 " + Arrays.toString(arr2)); 13 | // arr[1] = 46; 14 | arr = arr2; 15 | System.out.println("arr " + Arrays.toString(arr)); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/edgecomputationMerge/ArrayFunction.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.edgecomputationMerge; 2 | 3 | import java.util.Arrays; 4 | 5 | public class ArrayFunction { 6 | 7 | public static void main(String args[]) { 8 | int[] arr = new int[5]; 9 | function(arr); 10 | System.out.println(Arrays.toString(arr)); 11 | Class2 cls2 = new Class2(); 12 | cls2.abc(); 13 | System.out.println(Arrays.toString(cls2.get_a())); 14 | } 15 | 16 | public static void function(int arr[]) { 17 | arr[0] = 46; 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/scheduler/IScheduler.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.scheduler; 2 | 3 | /** 4 | * 5 | * @author Aftab 6 | * 7 | */ 8 | public interface IScheduler { 9 | 10 | /** 11 | * Initializes the Scheduler with the initial number of input partitions. 12 | * 13 | */ 14 | public void initScheduler(); 15 | 16 | /** 17 | * 18 | * @return 19 | */ 20 | public int[] getPartstoLoad(); 21 | 22 | /** 23 | * Update Scheduler map 24 | */ 25 | public void updateSchedulerInfo(); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/edgecomputationMerge/Setcopy.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.edgecomputationMerge; 2 | 3 | import java.util.HashSet; 4 | 5 | public class Setcopy { 6 | 7 | public static void main (String args[]){ 8 | HashSet set1=new HashSet(); 9 | set1.add(1); 10 | set1.add(2); 11 | 12 | System.out.println(set1); 13 | 14 | HashSet set2=new HashSet(set1); 15 | 16 | set2.remove(2); 17 | 18 | System.out.println("set1"+set1); 19 | System.out.println("set2"+set2); 20 | } 21 | 22 | 23 | } 24 | -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/edgecomputation/ArrayPointerMadness.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.edgecomputation; 2 | 3 | public class ArrayPointerMadness { 4 | public static void main(String args[]) { 5 | int edges[] = new int[2]; 6 | edges[0] = 0; 7 | edges[1] = 1; 8 | 9 | int newtargets[] = edges; 10 | int op[] = new int[4]; 11 | op[0] = -1; 12 | op[1] = -2; 13 | op[2] = -3; 14 | op[3] = -4; 15 | 16 | int oldtargets[] = newtargets; 17 | 18 | newtargets = op; 19 | edges = null; 20 | 21 | System.out 22 | .println(newtargets[0] + " " + oldtargets[0] + " " + edges[0]); 23 | 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/Time.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan; 2 | 3 | import java.util.concurrent.TimeUnit; 4 | 5 | public class Time { 6 | 7 | public static void main(String args[]) { 8 | long millis = 1000; 9 | String hms = String.format("%02d:%02d:%02d", TimeUnit.MILLISECONDS.toHours(millis), 10 | TimeUnit.MILLISECONDS.toMinutes(millis) - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(millis)), 11 | TimeUnit.MILLISECONDS.toSeconds(millis) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(millis))); 12 | System.out.println(hms); 13 | 14 | 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /graspan-java/.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled 3 | org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate 4 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 5 | org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve 6 | org.eclipse.jdt.core.compiler.compliance=1.6 7 | org.eclipse.jdt.core.compiler.debug.lineNumber=generate 8 | org.eclipse.jdt.core.compiler.debug.localVariable=generate 9 | org.eclipse.jdt.core.compiler.debug.sourceFile=generate 10 | org.eclipse.jdt.core.compiler.problem.assertIdentifier=error 11 | org.eclipse.jdt.core.compiler.problem.enumIdentifier=error 12 | org.eclipse.jdt.core.compiler.source=1.6 13 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/experiments/GPCppResultExtractorClient.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.experiments; 2 | 3 | import java.io.IOException; 4 | 5 | public class GPCppResultExtractorClient { 6 | 7 | // args[0] - io file 8 | // args[1] - total time 9 | // args[2] - total preprocessing time 10 | // args[3] - cpp.output 11 | public static void main(String args[]) throws IOException{ 12 | 13 | double total_time = Double.parseDouble(args[1]); 14 | double pp_time=Double.parseDouble(args[2]); 15 | 16 | Extractor ex = new Extractor(); 17 | double total_IO_time = ex.scan_IO_times(args[0], total_time, pp_time); 18 | 19 | System.out.println("Total IO time during computation= "+total_IO_time); 20 | 21 | ex.cpp_outputScanner(args[3]); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/edgecomputation/InitializeTwoDimArrTest.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.edgecomputation; 2 | 3 | public class InitializeTwoDimArrTest { 4 | 5 | public static void main(String args[]) { 6 | 7 | int[][] arr = new int[5][]; 8 | int degs[] = { 5,3,2,1,10}; 9 | initialize2DPartArray(arr, 5, degs); 10 | 11 | for (int i=0;i<5;i++){ 12 | for (int j=0;j evals; 13 | 14 | public int getMinSetId() { 15 | return minSetId; 16 | } 17 | 18 | public void setMinSetId(int id) { 19 | minSetId = id; 20 | } 21 | 22 | public MinSet() { 23 | ptr = -1; 24 | evals = new HashSet(); 25 | } 26 | 27 | public int getCurrentVId() { 28 | return currentVId; 29 | } 30 | 31 | public void setCurrentVId(int id) { 32 | currentVId = id; 33 | } 34 | 35 | public int getPtr() { 36 | return ptr; 37 | } 38 | 39 | public void incrementPtr() { 40 | ptr++; 41 | } 42 | 43 | public void addEval(int eval) { 44 | evals.add(eval); 45 | } 46 | 47 | public void clearEvalSet() { 48 | evals.clear(); 49 | } 50 | 51 | public HashSet getEvals() { 52 | return evals; 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | String s = ""; 58 | s = this.minSetId + ": " 59 | + this.getCurrentVId() + " " + this.getEvals(); 60 | return s; 61 | 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/dispatcher/PartERuleAdderAndSorterClient.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.dispatcher; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.util.logging.Logger; 6 | 7 | import edu.uci.ics.cs.graspan.computationM.GrammarChecker; 8 | import edu.uci.ics.cs.graspan.preproc.GraphERuleEdgeAdderAndSorter; 9 | import edu.uci.ics.cs.graspan.preproc.PartERuleAdderAndSorter; 10 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 11 | import edu.uci.ics.cs.graspan.support.Utilities; 12 | 13 | public class PartERuleAdderAndSorterClient { 14 | 15 | private static final Logger logger = GraspanLogger.getLogger("PartERuleAdderClient"); 16 | 17 | public static void main(String args[]) throws IOException { 18 | 19 | GlobalParams.setBasefilename(args[0]); 20 | GlobalParams.setNumParts(Integer.parseInt(args[1])); 21 | GlobalParams.setHasEdgeVals(args[2].trim()); 22 | GlobalParams.setFirstVertexID(Integer.parseInt(args[3])); 23 | 24 | GrammarChecker.loadGrammars(new File(GlobalParams.getBasefilename() + ".grammar")); 25 | 26 | logger.info("PREPROCESSING: Start computing and adding edges from eRules..."); 27 | long eAdd_start = System.currentTimeMillis(); 28 | 29 | for (int partId = 0; partId < GlobalParams.getNumParts(); partId++) { 30 | PartERuleAdderAndSorter partERadder = new PartERuleAdderAndSorter(); 31 | partERadder.loadAndProcessParts(partId); 32 | } 33 | 34 | logger.info("PREPROCESSING: Finished computing and adding edges from eRules."); 35 | 36 | logger.info("Edge Adding from Erules took: " + Utilities.getDurationInHMS(System.currentTimeMillis() - eAdd_start)); 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /graspan-java/graph: -------------------------------------------------------------------------------- 1 | 1 3 7 2 | 1 5 100 3 | 1 6 127 4 | 1 10 1 5 | 1 12 5 6 | 2 2 9 7 | 2 7 11 8 | 2 80 1 9 | 2 10 10 10 | 2 33 2 11 | 4 12 1 12 | 4 112 3 13 | 5 1 7 14 | 5 2 7 15 | 5 4 7 16 | 5 5 7 17 | 6 38 1 18 | 6 39 1 19 | 6 42 5 20 | 6 43 9 21 | 7 8 1 22 | 7 18 2 23 | 7 12 5 24 | 7 15 3 25 | 7 152 2 26 | 9 1 1 27 | 9 2 1 28 | 9 3 2 29 | 9 9 5 30 | 9 10 7 31 | 9 33 3 32 | 10 1 1 33 | 10 2 1 34 | 10 3 2 35 | 10 5 7 36 | 10 17 11 37 | 10 19 11 38 | 10 110 12 39 | 10 33 2 40 | 10 34 3 41 | 12 1 1 42 | 12 2 3 43 | 12 9 4 44 | 12 10 5 45 | 12 33 7 46 | 12 34 8 47 | 12 42 9 48 | 12 43 13 49 | 14 37 14 50 | 14 39 1 51 | 15 1 5 52 | 15 2 5 53 | 15 3 5 54 | 15 4 5 55 | 15 5 8 56 | 15 6 9 57 | 15 7 9 58 | 33 12 5 59 | 33 40 6 60 | 33 41 7 61 | 34 4 111 62 | 34 10 123 63 | 34 12 90 64 | 35 12 10 65 | 35 15 6 66 | 36 1 5 67 | 36 2 5 68 | 36 4 5 69 | 36 7 3 70 | 36 10 2 71 | 36 11 10 72 | 36 12 11 73 | 36 15 8 74 | 36 36 1 75 | 36 41 2 76 | 37 1 1 77 | 37 2 2 78 | 37 3 3 79 | 37 10 4 80 | 37 13 5 81 | 37 14 6 82 | 37 15 7 83 | 37 33 8 84 | 37 36 9 85 | 37 36 10 86 | 37 37 11 87 | 37 38 10 88 | 38 11 10 89 | 38 12 11 90 | 38 13 12 91 | 38 36 13 92 | 38 37 14 93 | 38 38 15 94 | 38 39 16 95 | 38 40 17 96 | 38 42 18 97 | 39 6 8 98 | 39 7 8 99 | 39 9 8 100 | 39 10 8 101 | 39 12 1 102 | 39 15 2 103 | 39 37 3 104 | 39 38 4 105 | 41 4 1 106 | 41 5 2 107 | 41 6 3 108 | 41 7 13 109 | 41 10 6 110 | 41 11 10 111 | 42 7 10 112 | 42 15 20 113 | 42 39 30 114 | 42 40 3 115 | 42 41 5 116 | 42 42 6 117 | -------------------------------------------------------------------------------- /graspan-java/executables/graspan-java-run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | JARFILE="graspan.jar" 3 | partGenClassPath="edu.uci.ics.cs.graspan.dispatcher.PartGenClient" 4 | erAdderAndSorterClassPath="edu.uci.ics.cs.graspan.dispatcher.PartERuleAdderAndSorterClient" 5 | graph=$1 6 | numParts=$2 7 | hasEdgeVals=$3 8 | firstVID=0 9 | #firstVID=$4 10 | grammar=$4 11 | 12 | echo "Preprocessing..." 13 | 14 | echo "Preparing grammar files..." 15 | cp $grammar $graph.grammar 16 | cp $grammar $graph.eRulesAdded.grammar 17 | 18 | echo "Generating initial partitions..." 19 | java -Xmx6G -cp $JARFILE $partGenClassPath $graph $numParts $hasEdgeVals $firstVID -ea 2> pp.pgen1.output 20 | #java -Xloggc:addEedges.gctimes.output -XX:+PrintGCTimeStamps -Xmx6G -cp $JARFILE $classPath ppconfig -ea 2> pp.eadd.output 21 | 22 | echo "# of Partitions Created:" 23 | numPartsActual=$(ls *.partition.*.degrees | wc -l) 24 | echo $numPartsActual 25 | 26 | echo "Adding Erules to partitions and sorting them..." 27 | java -Xmx6G -cp $JARFILE $erAdderAndSorterClassPath $graph $numPartsActual $hasEdgeVals $firstVID -ea 2> pp.erAndSort.output 28 | 29 | echo "Generating final partitions..." 30 | java -Xmx6G -cp $JARFILE $partGenClassPath $graph".eRulesAdded" $numPartsActual $hasEdgeVals $firstVID -ea 2> pp.pgen2.output 31 | 32 | echo "# of Partitions Created:" 33 | numPartsActual=$(ls *.partition.*.degrees | wc -l) 34 | echo $(( numPartsActual / 2 )) 35 | 36 | echo "Computing..." 37 | 38 | CompPath="edu.uci.ics.cs.graspan.dispatcher.ComputationClient" 39 | graph=$graph".eRulesAdded" 40 | numThreads=8 41 | partSizeAfterNewEdges=50 #millions of edges 42 | 43 | java -Xloggc:comp.gctimes.output -XX:+PrintGCTimeStamps -Xmx6G -cp $JARFILE $CompPath $graph $numPartsActual $numThreads $partSizeAfterNewEdges -ea 2> comp.output 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/dispatcher/PartGenClient.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.dispatcher; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.util.logging.Logger; 6 | 7 | import edu.uci.ics.cs.graspan.computationM.GrammarChecker; 8 | import edu.uci.ics.cs.graspan.preproc.Preprocessor; 9 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 10 | import edu.uci.ics.cs.graspan.support.Utilities; 11 | 12 | /** 13 | * This program performs preprocessing of the input graph to generate partitions 14 | * 15 | * @author Aftab 16 | * 17 | */ 18 | public class PartGenClient { 19 | 20 | private static final Logger logger = GraspanLogger.getLogger("PartGenClient"); 21 | 22 | public static void main(String[] args) throws IOException { 23 | 24 | // for (int i = 0; i < args.length; i++) { 25 | // logger.info(" "+i+" "+args[i].trim()); 26 | // } 27 | 28 | GlobalParams.setBasefilename(args[0]); 29 | GlobalParams.setNumParts(Integer.parseInt(args[1])); 30 | GlobalParams.setHasEdgeVals(args[2].trim()); 31 | GlobalParams.setFirstVertexID(Integer.parseInt(args[3])); 32 | 33 | logger.info("Input graph: " + GlobalParams.getBasefilename()); 34 | logger.info("Requested # partitions to generate: " + GlobalParams.getNumParts()); 35 | 36 | GrammarChecker.loadGrammars(new File(GlobalParams.getBasefilename() + ".grammar")); 37 | 38 | logger.info("PREPROCESSING: Start generating partitions..."); 39 | long pp_start = System.currentTimeMillis(); 40 | 41 | // initialize Partition Generator Program 42 | Preprocessor partgenerator = new Preprocessor(GlobalParams.getBasefilename(), GlobalParams.getNumParts()); 43 | partgenerator.run(); 44 | 45 | logger.info("PREPROCESSING: Finished generating partitions."); 46 | 47 | logger.info("Generating partitions took: " + Utilities.getDurationInHMS(System.currentTimeMillis() - pp_start)); 48 | 49 | } 50 | 51 | } -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/MapTest.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan; 2 | 3 | import java.util.HashMap; 4 | import java.util.HashSet; 5 | import java.util.Map; 6 | import java.util.Set; 7 | 8 | public class MapTest { 9 | 10 | Map> diffMap = new HashMap>(); 11 | 12 | public MapTest() { 13 | } 14 | 15 | private void run() { 16 | HashSet st = new HashSet(); 17 | st.add(new MyIntObj(5)); 18 | st.add(new MyIntObj(7)); 19 | diffMap.put(1, st); 20 | 21 | HashSet stextract2=new HashSet(); 22 | for (int id : diffMap.keySet()) { 23 | HashSet stextract = diffMap.get(id); 24 | for (MyIntObj intObj: stextract){ 25 | stextract2.add(intObj); 26 | } 27 | } 28 | 29 | System.out.println("stextract2.size() "+stextract2.size()); 30 | MyIntObj mio=new MyIntObj(5); 31 | stextract2.remove(mio); 32 | System.out.println("stextract2.size() after removing object: "+stextract2.size()); 33 | 34 | System.out.println(diffMap.keySet().size()); 35 | for (int id : diffMap.keySet()) { 36 | HashSet stextract = diffMap.get(id); 37 | for (MyIntObj num: stextract){ 38 | System.out.println("id="+id+" num="+num.getInt()); 39 | } 40 | } 41 | 42 | 43 | 44 | } 45 | 46 | public static void main(String args[]) { 47 | MapTest mpt = new MapTest(); 48 | mpt.run(); 49 | } 50 | 51 | 52 | private class MyIntObj { 53 | private int n; 54 | 55 | public MyIntObj(int n) { 56 | this.n = n; 57 | } 58 | 59 | public int getInt(){ 60 | return n; 61 | } 62 | 63 | public int hashCode(){ 64 | int r = 1; 65 | r = r * 31 + this.n; 66 | // r = r * 31 + this.evalue; 67 | return r; 68 | } 69 | 70 | public boolean equals(Object o){ 71 | return (o instanceof MyIntObj) && (((MyIntObj) o).n == this.n) ; 72 | } 73 | 74 | 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/support/MemUsageCheckThread.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.support; 2 | 3 | import java.io.BufferedWriter; 4 | import java.io.FileWriter; 5 | import java.io.IOException; 6 | import java.io.PrintWriter; 7 | import java.text.NumberFormat; 8 | import java.util.logging.Logger; 9 | 10 | import edu.uci.ics.cs.graspan.computationM.EngineM; 11 | import edu.uci.ics.cs.graspan.dispatcher.GlobalParams; 12 | 13 | public class MemUsageCheckThread extends Thread { 14 | private static final Logger logger = GraspanLogger.getLogger("MemUsageCheckThread"); 15 | public static PrintWriter memoryUsageOutput; 16 | 17 | public void run() { 18 | Runtime runtime = Runtime.getRuntime(); 19 | // NumberFormat format = NumberFormat.getInstance(); 20 | 21 | // round output info 22 | try { 23 | memoryUsageOutput = new PrintWriter(new BufferedWriter(new FileWriter(GlobalParams.getBasefilename() + ".output.memUsage.csv", true))); 24 | } catch (IOException e1) { 25 | e1.printStackTrace(); 26 | } 27 | memoryUsageOutput.println("Free_memory, Allocated_memory, Max_memory, Total_free_memory"); 28 | 29 | while (true) { 30 | long maxMemory = runtime.maxMemory(); 31 | long allocatedMemory = runtime.totalMemory(); 32 | long freeMemory = runtime.freeMemory(); 33 | 34 | // logger.info("Free memory (MB): " + format.format(freeMemory / 1048576)); 35 | // logger.info("Allocated memory (MB): " + format.format(allocatedMemory / 1048576)); 36 | // logger.info("Max memory (MB): " + format.format(maxMemory / 1048576)); 37 | // logger.info("Total free memory (MB): " + format.format((freeMemory + (maxMemory - allocatedMemory)) / 1048576)); 38 | 39 | memoryUsageOutput.println((freeMemory / 1048576) + "," 40 | + (allocatedMemory / 1048576) + "," + (maxMemory / 1048576) + "," 41 | + ((freeMemory + (maxMemory - allocatedMemory)) / 1048576)); 42 | 43 | if (((freeMemory + (maxMemory - allocatedMemory)) / 1048576) > 0) { 44 | EngineM.memFull = true; 45 | } 46 | 47 | try { 48 | this.sleep(3000); 49 | } catch (InterruptedException e) { 50 | e.printStackTrace(); 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/support/Utilities.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.support; 2 | 3 | import java.util.concurrent.TimeUnit; 4 | 5 | /** 6 | * This class consists of sorting methods. 7 | * 8 | * @author Aftab 9 | * 10 | */ 11 | public class Utilities { 12 | 13 | /** 14 | * (Called by loadPartitions method()) Rendered from: 15 | * http://www.programcreek.com/2012/11/quicksort-array-in-java/ accessed at 16 | * 9 October 2015 17 | * 18 | * For sorting each source vertex adjacency list in the loaded partitions 19 | * 20 | * @param arr 21 | * @param low 22 | * @param high 23 | */ 24 | public static void quickSort(int[] edgeArr, byte[] edgeValArr, int low, int high) { 25 | if (edgeArr == null || edgeArr.length == 0) 26 | return; 27 | 28 | if (low >= high) 29 | return; 30 | 31 | // pick the pivot 32 | int middle = low + (high - low) / 2; 33 | int pivot = edgeArr[middle]; 34 | 35 | // make left < pivot and right > pivot 36 | int i = low, j = high; 37 | while (i <= j) { 38 | while (edgeArr[i] < pivot) { 39 | i++; 40 | } 41 | 42 | while (edgeArr[j] > pivot) { 43 | j--; 44 | } 45 | 46 | if (i <= j) { 47 | 48 | int temp = edgeArr[i]; 49 | edgeArr[i] = edgeArr[j]; 50 | edgeArr[j] = temp; 51 | 52 | byte tempVal = edgeValArr[i]; 53 | edgeValArr[i] = edgeValArr[j]; 54 | edgeValArr[j] = tempVal; 55 | 56 | i++; 57 | j--; 58 | 59 | } 60 | } 61 | 62 | // recursively sort two sub parts 63 | if (low < j) 64 | quickSort(edgeArr, edgeValArr, low, j); 65 | 66 | if (high > i) 67 | quickSort(edgeArr, edgeValArr, i, high); 68 | } 69 | 70 | public static String getDurationInHMS(long duration){ 71 | String hms = String.format("%02d,%02d,%02d", TimeUnit.MILLISECONDS.toHours(duration), 72 | TimeUnit.MILLISECONDS.toMinutes(duration) - TimeUnit.HOURS.toMinutes(TimeUnit.MILLISECONDS.toHours(duration)), 73 | TimeUnit.MILLISECONDS.toSeconds(duration) - TimeUnit.MINUTES.toSeconds(TimeUnit.MILLISECONDS.toMinutes(duration))); 74 | return hms; 75 | } 76 | 77 | } -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/scheduler/PartitionEdgeInfo.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.scheduler; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | /** 7 | * @author Kai Wang 8 | * 9 | * Created by Oct 30, 2015 10 | */ 11 | public class PartitionEdgeInfo { 12 | private int partitionId; 13 | private long partSize; 14 | private List pEdgeInfo = new ArrayList(); 15 | // private List priorityInfo = new ArrayList(); 16 | private List terminationInfo = new ArrayList(); 17 | 18 | public PartitionEdgeInfo(int partitionId) { 19 | this.partitionId = partitionId; 20 | } 21 | 22 | /** 23 | * For initialization 24 | * @param partitionId 25 | * @param edgeInfo 26 | */ 27 | public PartitionEdgeInfo(int partitionId, long[] edgeInfo) { 28 | if (edgeInfo == null) 29 | throw new IllegalArgumentException("Null parameter in PartitionEdgeInfo!"); 30 | 31 | this.partitionId = partitionId; 32 | for (long info : edgeInfo) { 33 | pEdgeInfo.add(Long.valueOf(info)); 34 | // priorityInfo.add(0L); 35 | terminationInfo.add(Boolean.FALSE); 36 | } 37 | } 38 | 39 | /** 40 | * Constructor (termination map only) 41 | * 42 | * @param partitionId 43 | * @param numParts 44 | */ 45 | public PartitionEdgeInfo(int partitionId, int numParts) { 46 | this.partitionId = partitionId; 47 | 48 | // init termination info for each partition 49 | for (int i = 0; i < numParts; i++) 50 | terminationInfo.add(Boolean.FALSE); 51 | } 52 | 53 | public List getPartitionEdgeInfo() { 54 | return pEdgeInfo; 55 | } 56 | 57 | public int getPartitionId() { 58 | return partitionId; 59 | } 60 | 61 | // public List getPriorityInfo() { 62 | // return priorityInfo; 63 | // } 64 | 65 | public List getTerminationInfo() { 66 | return terminationInfo; 67 | } 68 | 69 | @Override 70 | public String toString() { 71 | StringBuilder result = new StringBuilder(); 72 | String NEW_LINE = System.getProperty("line.separator"); 73 | result.append(NEW_LINE + "partition id : " + partitionId); 74 | result.append(NEW_LINE + "termination info : " + terminationInfo); 75 | 76 | return result.toString(); 77 | 78 | } 79 | } -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/artificalgraphgenerator/EdgeValueAdder.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.artificalgraphgenerator; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.BufferedWriter; 5 | import java.io.File; 6 | import java.io.FileInputStream; 7 | import java.io.FileWriter; 8 | import java.io.IOException; 9 | import java.io.InputStream; 10 | import java.io.InputStreamReader; 11 | import java.io.PrintWriter; 12 | import java.util.ArrayList; 13 | import java.util.Random; 14 | 15 | /** 16 | * This program adds random edge values to a given input graph 17 | * 18 | * @author Aftab 19 | * 20 | */ 21 | public class EdgeValueAdder { 22 | 23 | static final int BUFFER = 1000000000; 24 | static int BUFFER_FREE_SPACE = BUFFER; 25 | static ArrayList EDGE_BUFFER; 26 | 27 | public static void main(String args[]) throws IOException { 28 | 29 | EDGE_BUFFER = new ArrayList(); 30 | long linecount = 0; 31 | 32 | Random rand = new Random(); 33 | String baseFilename = args[0]; 34 | InputStream a = new FileInputStream(new File(baseFilename)); 35 | BufferedReader ins = new BufferedReader(new InputStreamReader(a)); 36 | String ln; 37 | String modifiedInput; 38 | 39 | PrintWriter modifedEdgefile = new PrintWriter( 40 | new BufferedWriter(new FileWriter(baseFilename + ".edgevaladded.", true))); 41 | while ((ln = ins.readLine()) != null) { 42 | 43 | linecount++; 44 | if (linecount % 10000000 == 0) { 45 | System.out.println("Reading Line " + linecount); 46 | } 47 | modifiedInput = ln + "\t" + rand.nextInt(256) + "\n"; 48 | addEdgetoBuffer(modifiedInput); 49 | if (BUFFER_FREE_SPACE == 0) { 50 | writeToDisk(modifedEdgefile, baseFilename); 51 | } 52 | // add remaining edges in buffer to disk 53 | writeToDisk(modifedEdgefile, baseFilename); 54 | } 55 | ins.close(); 56 | modifedEdgefile.close(); 57 | } 58 | 59 | private static void addEdgetoBuffer(String modifiedInput) throws IOException { 60 | EDGE_BUFFER.add(modifiedInput); 61 | BUFFER_FREE_SPACE--; 62 | } 63 | 64 | public static void writeToDisk(PrintWriter modifedEdgefile, String baseFilename) throws IOException { 65 | 66 | for (int i = 0; i < EDGE_BUFFER.size(); i++) { 67 | modifedEdgefile.write(EDGE_BUFFER.get(i)); 68 | } 69 | EDGE_BUFFER.clear(); 70 | BUFFER_FREE_SPACE = BUFFER; 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/scheduler/SchedulerInfo.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.scheduler; 2 | 3 | import java.util.LinkedHashMap; 4 | 5 | public class SchedulerInfo { 6 | 7 | private static long[][] partSizes; 8 | private static long[][] edgeDestCount; 9 | private static long[][] edgeDestCountTwoWay; 10 | private int[][] terminationMap; 11 | private static double[][] edcPercentage = new double[50][50]; 12 | 13 | public static LinkedHashMap> edgeDestCountMP = 14 | new LinkedHashMap>(); 15 | 16 | /** 17 | * 18 | * @param arr 19 | */ 20 | public static void setPartSizes(long[][] arr) { 21 | partSizes = arr; 22 | } 23 | 24 | /** 25 | * 26 | * @return long[] partSizes 27 | */ 28 | public static long[][] getPartSizes() { 29 | return partSizes; 30 | } 31 | 32 | /** 33 | * 34 | * @param arr 35 | */ 36 | public static void setEdgeDestCount(long[][] arr) { 37 | edgeDestCount = arr; 38 | } 39 | 40 | /** 41 | * 42 | * @return long[][] edgeDestCount 43 | */ 44 | public static long[][] getEdgeDestCount() { 45 | return edgeDestCount; 46 | } 47 | 48 | /** 49 | * 50 | * @return 51 | */ 52 | public static double[][] getEdcPercentage() { 53 | return edcPercentage; 54 | } 55 | 56 | /** 57 | * 58 | * @param arr 59 | */ 60 | public static void setEdcPercentage(double[][] arr) { 61 | edcPercentage = arr; 62 | } 63 | 64 | /** 65 | * Prints the partition sizes and the partition edge counts 66 | */ 67 | public static void printData() { 68 | System.out.println("Printing Preliminary Scheduling Info..."); 69 | System.out.println("Partition sizes: "); 70 | for (int i = 0; i < partSizes.length; i++) { 71 | System.out.println(partSizes[i]); 72 | } 73 | 74 | System.out.println("Partition edge destination counts: "); 75 | for (int i = 0; i < partSizes.length; i++) { 76 | for (int j = 0; j < partSizes.length; j++) { 77 | System.out.println(i + " " + j + " " + edgeDestCount[i][j]); 78 | } 79 | } 80 | } 81 | 82 | public static void setEdcTwoWay(long[][] edcTwoWay) { 83 | edgeDestCountTwoWay=edcTwoWay; 84 | } 85 | 86 | public static long[][] getEdcTwoWay(){ 87 | return edgeDestCountTwoWay; 88 | } 89 | 90 | // @Override 91 | // public String toString() { 92 | // StringBuilder result = new StringBuilder(); 93 | // String NEW_LINE = System.getProperty("line.separator"); 94 | // 95 | // for (int i = 0; i < 50; i++) { 96 | // result.append(NEW_LINE + i + " : "); 97 | // for (int j = 0; j < 50; j++) { 98 | // result.append("(" + j + "," + edgeDestCount[i][j] + ") "); 99 | // } 100 | // } 101 | // return result.toString(); 102 | // } 103 | } 104 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/datastructures/RepartitioningData.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.datastructures; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashSet; 5 | import java.util.TreeSet; 6 | 7 | /** 8 | * 9 | * @author Aftab 9 Nov 2015 10 | * 11 | */ 12 | public class RepartitioningData { 13 | // splitVertices 14 | private static ArrayList splitVertices; 15 | 16 | // set of new intervals (the last source vertex Ids of each partition) 17 | private static TreeSet newPartLimits; 18 | 19 | // set of partitions that have been repartitioned 20 | private static HashSet repartitionedParts; 21 | 22 | // set of partitions that have been newly generated from 23 | // repartitioning 24 | private static HashSet newPartsFrmRepartitioning; 25 | 26 | // set of partitions that have new edges but have not been repartitioned 27 | private static HashSet modifiedParts; 28 | 29 | // set of partitions to which no new edges were added 30 | private static HashSet unModifiedParts; 31 | 32 | // set of all partitions in the memory after repartitioning 33 | private static HashSet loadedPartsPostProcessing; 34 | 35 | // set of partitions that are to be saved (depends on partition reload 36 | // strategy) 37 | private static HashSet partsToSave; 38 | 39 | public static void initRepartioningVars() { 40 | splitVertices = new ArrayList(); 41 | newPartLimits = new TreeSet(); 42 | repartitionedParts = new HashSet(); 43 | newPartsFrmRepartitioning = new HashSet(); 44 | modifiedParts = new HashSet(); 45 | unModifiedParts = new HashSet(); 46 | loadedPartsPostProcessing = new HashSet(); 47 | partsToSave = new HashSet(); 48 | } 49 | 50 | public static HashSet getModifiedParts() { 51 | return modifiedParts; 52 | } 53 | 54 | public static HashSet getUnModifiedParts() { 55 | return unModifiedParts; 56 | } 57 | 58 | public static ArrayList getSplitVertices() { 59 | return splitVertices; 60 | } 61 | 62 | public static TreeSet getNewPartLimits() { 63 | return newPartLimits; 64 | } 65 | 66 | public static HashSet getRepartitionedParts() { 67 | return repartitionedParts; 68 | } 69 | 70 | public static HashSet getNewPartsFrmRepartitioning() { 71 | return newPartsFrmRepartitioning; 72 | } 73 | 74 | public static void clearRepartitioningVars() { 75 | splitVertices.clear(); 76 | newPartLimits.clear(); 77 | repartitionedParts.clear(); 78 | newPartsFrmRepartitioning.clear(); 79 | modifiedParts.clear(); 80 | unModifiedParts.clear(); 81 | loadedPartsPostProcessing.clear(); 82 | partsToSave.clear(); 83 | } 84 | 85 | public static HashSet getPartsToSave() { 86 | return partsToSave; 87 | } 88 | 89 | public static HashSet getLoadedPartsPostProcessing() { 90 | return loadedPartsPostProcessing; 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/datastructures/LoadedVertexInterval.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.datastructures; 2 | 3 | /** 4 | * @author Kai Wang 5 | * 6 | * Created Oct 28, 2015 7 | */ 8 | public class LoadedVertexInterval { 9 | private int firstVertex; 10 | private int lastVertex; 11 | private int indexStart; 12 | private int indexEnd; 13 | private int partitionId; 14 | 15 | //THIS FLAG STORES WHETHER A NEW EDGE HAS BEEN ADDED FOR A PARTITION AFTER IT HAS BEEN LOADED, BEFORE IT HAS BEEN SAVED 16 | private boolean isNewEdgeAdded; 17 | 18 | //THIS FLAG STORES WHETHER A NEW EDGE HAS BEEN ADDED FOR A PARTITION IN THE CURRENT ROUND 19 | private boolean isNewEdgeAddedinCurrentRound; 20 | 21 | public LoadedVertexInterval(int firstVertex, int lastVertex, int partitionId) { 22 | this.firstVertex = firstVertex; 23 | this.lastVertex = lastVertex; 24 | this.partitionId = partitionId; 25 | } 26 | 27 | public void setFirstVertex(int firstVertex) { 28 | this.firstVertex = firstVertex; 29 | } 30 | 31 | /** 32 | * 33 | * @return int firstVertex 34 | */ 35 | public int getFirstVertex() { 36 | return firstVertex; 37 | } 38 | 39 | public void setLastVertex(int lastVertex) { 40 | this.lastVertex = lastVertex; 41 | } 42 | 43 | /** 44 | * 45 | * @return int lastVertex 46 | */ 47 | public int getLastVertex() { 48 | return lastVertex; 49 | } 50 | 51 | public void setIndexStart(int indexStart) { 52 | this.indexStart = indexStart; 53 | } 54 | 55 | /** 56 | * 57 | * @return int indexStart 58 | */ 59 | public int getIndexStart() { 60 | return indexStart; 61 | } 62 | 63 | public void setIndexEnd(int indexEnd) { 64 | this.indexEnd = indexEnd; 65 | } 66 | 67 | /** 68 | * 69 | * @return int indexEnd 70 | */ 71 | public int getIndexEnd() { 72 | return indexEnd; 73 | } 74 | 75 | public void setPartitionId(int partitionId) { 76 | this.partitionId = partitionId; 77 | } 78 | 79 | /** 80 | * 81 | * @return int partitionId 82 | */ 83 | public int getPartitionId() { 84 | return partitionId; 85 | } 86 | 87 | public void setIsNewEdgeAdded(boolean isNewEdgeAdded) { 88 | this.isNewEdgeAdded = isNewEdgeAdded; 89 | } 90 | 91 | public boolean hasNewEdges() { 92 | return isNewEdgeAdded; 93 | } 94 | 95 | public void setHasNewEdgesInCurrentRound(boolean isNewEdgeAddedinCurrentRound) { 96 | this.isNewEdgeAddedinCurrentRound = isNewEdgeAddedinCurrentRound; 97 | } 98 | 99 | public boolean hasNewEdgesInCurrentRound() { 100 | return isNewEdgeAddedinCurrentRound; 101 | } 102 | 103 | @Override 104 | public String toString() { 105 | StringBuilder result = new StringBuilder(); 106 | String NEW_LINE = System.getProperty("line.separator"); 107 | 108 | result.append(NEW_LINE + "(PartId: " + partitionId + ", VRange: " + firstVertex + "-" + lastVertex 109 | + ", IdRange: " + indexStart + "-" + indexEnd + ")"); 110 | 111 | return result.toString(); 112 | } 113 | 114 | } -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/computationM/MinSet.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.computationM; 2 | 3 | import java.util.HashSet; 4 | 5 | public class MinSet { 6 | 7 | // always points to the minimum element 8 | private int ptr; 9 | 10 | private final int minSetId; 11 | 12 | private int currentVId; 13 | 14 | private HashSet evals; 15 | 16 | private byte[] evals_arr; 17 | 18 | public int getMinSetId() { 19 | return minSetId; 20 | } 21 | 22 | public MinSet(int minSetId) { //TODO: NEED TO FIX. IDENTIFIED AS GC-EXPENSIVE BY YOURKIT. :DONE 23 | ptr = 0; 24 | this.minSetId = minSetId; 25 | 26 | //evals init - old hashset implementation 27 | // evals = new HashSet(); //TODO: NEED TO FIX. IDENTIFIED AS GC-EXPENSIVE BY YOURKIT. :DONE 28 | 29 | //evals init - new array implementation 30 | evals_arr = new byte[GrammarChecker.getNumOfGrammarSymbols()]; 31 | for (int i = 0; i < evals_arr.length; i++) { 32 | evals_arr[i] = -1; 33 | } 34 | } 35 | 36 | public int getCurrentVId() { 37 | return currentVId; 38 | } 39 | 40 | public void setCurrentVId(int id) { 41 | currentVId = id; 42 | } 43 | 44 | public int getPtr() { 45 | return ptr; 46 | } 47 | 48 | public void incrementPtr() { 49 | ptr++; 50 | } 51 | 52 | //adding edge val - old hashset implementation 53 | public void addEval(byte eval) { //TODO: NEED TO UPDATE FOR EVALS AS ARRAY :DONE 54 | evals.add(eval); 55 | } 56 | 57 | //adding edge val - new array implementation 58 | public void addToEvalArr(byte eval) { 59 | 60 | //ensure it does not already exist 61 | for (int i = 0; i < evals_arr.length; i++) { 62 | if (evals_arr[i] == eval) { 63 | return; 64 | } 65 | if (evals_arr[i] == -1) { 66 | evals_arr[i] = eval; 67 | break; 68 | } 69 | } 70 | 71 | // for (int i = 0; i < evals_arr.length; i++) { 72 | // if (evals_arr[i] == -1) { 73 | // break; 74 | // } 75 | // } 76 | } 77 | 78 | //clearing edge vals - old hashset implementation 79 | public void clearEvalSet() { //TODO: NEED TO UPDATE FOR EVALS AS ARRAY :DONE 80 | evals.clear(); 81 | } 82 | 83 | //clearing edge vals - new array implementation 84 | public void clearEvalArr() { 85 | for (int i = 0; i < evals_arr.length; i++) { 86 | evals_arr[i] = -1; 87 | } 88 | } 89 | 90 | //return edge vals - old hashset implementation 91 | public HashSet getEvals() { //TODO: NEED TO UPDATE FOR EVALS AS ARRAY :DONE 92 | return evals; 93 | } 94 | 95 | //return edge vals - new array implementation 96 | public byte[] getEvalsArr(){ 97 | return evals_arr; 98 | } 99 | 100 | @Override 101 | public String toString() { //TODO: NEED TO UPDATE FOR EVALS AS ARRAY :DONE 102 | 103 | String evals=""; 104 | for (int i = 0; i < evals_arr.length; i++) { 105 | if (evals_arr[i] == -1) 106 | break; 107 | evals += evals_arr[i] + " "; 108 | } 109 | evals.trim(); 110 | 111 | String s = ""; 112 | s = "MinSet no. " + this.minSetId + ": " + this.getCurrentVId() + " " + "Evals : <"+evals+">"; 113 | // s = "MinSet no. " + this.minSetId + ": " + this.getCurrentVId() + " " + this.getEvals(); 114 | return s; 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/scheduler/BasicScheduler.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.scheduler; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashSet; 5 | import java.util.List; 6 | import java.util.Set; 7 | 8 | import edu.uci.ics.cs.graspan.dispatcher.GlobalParams; 9 | 10 | /** 11 | * 12 | * CURRENTLY WORKS FOR ONLY TWO PARTITIONS LOADED IN THE MEMORY - Schedules the 13 | * selection of partitions on which memory computations is to be done, also sets 14 | * a limit of the maximum allowable number of new partitions that can be 15 | * generated from repartitioning 16 | * 17 | * @author Aftab 18 | * 19 | */ 20 | public class BasicScheduler implements IScheduler { 21 | 22 | // // We are assuming at most 50 - numOfOriginalParts will be created after 23 | // the 24 | // // complete process 25 | // private static final int SizeOfPartScheduleMap = 50; 26 | 27 | private int[][] partTerminationMap = new int[GlobalParams.getNumParts()][GlobalParams.getNumParts()]; 28 | 29 | /** 30 | * Initializes the scheduler. An entry of -1 in {@code partTerminationMap} 31 | * shows no active partition is represented by this row/column. An entry of 32 | * 0 shows this partition pair has not been computed. An entry of 1 shows 33 | * this partition pair has been computed. 34 | */ 35 | public void initScheduler() { 36 | int totalNumParts = GlobalParams.getNumParts(); 37 | // initialize partTerminationMap 38 | for (int i = 0; i < totalNumParts; i++) { 39 | for (int j = 0; j < i; j++) { 40 | partTerminationMap[i][j] = 0; 41 | } 42 | } 43 | } 44 | 45 | public void updateSchedulerInfo() { 46 | 47 | } 48 | 49 | /** 50 | * Returns the next set of partitions (ids) to be computed TODO currently 51 | * designed to load two partitions only. 52 | */ 53 | public int[] getPartstoLoad() { 54 | 55 | int numPartsPerComputation = GlobalParams.getNumPartsPerComputation(); 56 | for (int i = 0; i < partTerminationMap.length; i++) { 57 | for (int j = 0; j < i; j++) { 58 | if (!isComputed(i, j)) { 59 | int[] partsToLoad = new int[numPartsPerComputation]; 60 | partsToLoad[0] = i; 61 | partsToLoad[1] = j; 62 | partTerminationMap[i][j] = 1; 63 | return partsToLoad; 64 | } 65 | } 66 | } 67 | return null; 68 | } 69 | 70 | public int[][] getPartScheduleMap() { 71 | return partTerminationMap; 72 | } 73 | 74 | /** 75 | * Checks whether the partition-pair has been computed. Called by 76 | * getPartstoLoad() 77 | * 78 | * @param part1 79 | * @param part2 80 | * @return 81 | */ 82 | private boolean isComputed(int part1mapId, int part2mapId) { 83 | if (partTerminationMap[part1mapId][part2mapId] == 1) 84 | return true; 85 | else 86 | return false; 87 | } 88 | 89 | public void generateComputationPairs() { 90 | List lst = new ArrayList(); 91 | lst.add(1); 92 | lst.add(2); 93 | lst.add(3); 94 | lst.add(4); 95 | lst.add(5); 96 | 97 | 98 | Set> pairSet = new HashSet>(); 99 | Set pair = new HashSet(); 100 | 101 | for (int i = 0; i < lst.size(); i++) { 102 | for (int j = i + 1; j < lst.size(); j++) { 103 | pair.add(lst.get(i)); 104 | pair.add(lst.get(j)); 105 | } 106 | } 107 | 108 | } 109 | 110 | } 111 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/datastructures/Vertex.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.datastructures; 2 | 3 | import java.util.Arrays; 4 | 5 | /** 6 | * @author Kai Wang 7 | * 8 | * Created by Oct 8, 2015 9 | */ 10 | public class Vertex { 11 | 12 | // index of the vertex in Vertex[] Vertices array 13 | private int idx; 14 | 15 | private int id; 16 | private int numOutEdges; 17 | 18 | //in order to deal with arrays with empty elements. 19 | private int[] numOutEdgesActual; 20 | 21 | private int[] outEdges; 22 | private byte[] outEdgeValues; 23 | 24 | // the degree of the vertex counting original out edges and newly computed out edges 25 | private int combinedDeg; 26 | 27 | // Unused 28 | public Vertex(int id, int outDegree) { 29 | this.id = id; 30 | numOutEdges = outDegree; 31 | 32 | if (outDegree != 0) { 33 | outEdges = new int[outDegree]; 34 | outEdgeValues = new byte[outDegree]; 35 | } 36 | } 37 | 38 | public Vertex(int idx, int id, int[] outEdges, byte[] outEdgeValues) { 39 | this.idx = idx; 40 | this.id = id; 41 | this.numOutEdges = 0; 42 | numOutEdgesActual = new int[1]; 43 | this.outEdges = outEdges; 44 | this.outEdgeValues = outEdgeValues; 45 | if(outEdges != null){ 46 | this.numOutEdges = outEdges.length; 47 | // assert(this.outEdges.length == this.outEdgeValues.length); 48 | } 49 | } 50 | 51 | public int getNumOutEdges() { 52 | this.numOutEdges = 0; 53 | 54 | if(this.outEdges != null){ 55 | numOutEdges = outEdges.length; //TODO: IN PLACE OF REMOVE REDUNDANT ARRAYSPACE APPROACH // USE ACTUAL LENGTH INSTEAD OF CAPACITY 56 | } 57 | return numOutEdges; 58 | } 59 | 60 | public void setNumOutEdges(int i) { 61 | this.numOutEdgesActual[0] = i; 62 | } 63 | 64 | /** 65 | * Used in EdgeList Style Computation 66 | * 67 | * @return 68 | */ 69 | public int getCombinedDeg() { 70 | return combinedDeg; 71 | } 72 | 73 | /** 74 | * Used in EdgeList Style Computation 75 | * 76 | * @return 77 | */ 78 | public void setCombinedDeg(int num) { 79 | combinedDeg = num; 80 | } 81 | 82 | public int[] getOutEdges() { 83 | return outEdges; 84 | } 85 | 86 | public byte[] getOutEdgeValues() { 87 | return outEdgeValues; 88 | } 89 | 90 | public void setOutEdges(int outEdges[]) { 91 | this.outEdges = outEdges; 92 | } 93 | 94 | public void setOutEdgeValues(byte[] outEdgeValues) { 95 | this.outEdgeValues = outEdgeValues; 96 | } 97 | 98 | public int getOutEdge(int i) { 99 | return outEdges[i]; 100 | } 101 | 102 | public byte getOutEdgeValue(int i) { 103 | return outEdgeValues[i]; 104 | } 105 | 106 | public int getVertexId() { 107 | return id; 108 | } 109 | 110 | public int getVertexIdx() { 111 | return idx; 112 | } 113 | 114 | @Override 115 | public String toString() { 116 | StringBuilder result = new StringBuilder(); 117 | String NEW_LINE = System.getProperty("line.separator"); 118 | 119 | result.append(NEW_LINE + "vertex id : " + id + NEW_LINE); 120 | result.append("out edges: " + Arrays.toString(outEdges) + NEW_LINE); 121 | // result.append("edge value: " + Arrays.toString(outEdgeValues) + NEW_LINE); 122 | //TODO UNCOMMENT THIS LATER 123 | result.append("degree: " + combinedDeg + NEW_LINE); 124 | 125 | return result.toString(); 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/datastructures/ComputationSet.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.datastructures; 2 | 3 | /** 4 | * 5 | * @author aftab 6 | * 7 | * Created Mar 20, 2016 8 | * 9 | */ 10 | public class ComputationSet { 11 | 12 | // INPUT FOR EACH ITERATION 13 | 14 | // old 15 | 16 | private int oldEdgs[]; 17 | private byte oldVals[]; 18 | private int oldNumEdgs[]; 19 | 20 | public int[] getOldEdgs() { 21 | return oldEdgs; 22 | } 23 | 24 | public void setOldEdgs(int[] oldEdgs) { 25 | this.oldEdgs = oldEdgs; 26 | } 27 | 28 | public byte[] getOldVals() { 29 | return oldVals; 30 | } 31 | 32 | public void setOldVals(byte[] oldVals) { 33 | this.oldVals = oldVals; 34 | } 35 | 36 | // new 37 | 38 | private int newEdgs[]; 39 | private byte newVals[]; 40 | private int newNumEdgs[]; 41 | 42 | public int[] getNewEdgs() { 43 | return newEdgs; 44 | } 45 | 46 | public void setNewEdgs(int[] newEdgs) { 47 | this.newEdgs = newEdgs; 48 | } 49 | 50 | public byte[] getNewVals() { 51 | return newVals; 52 | } 53 | 54 | public void setNewVals(byte[] newVals) { 55 | this.newVals = newVals; 56 | } 57 | 58 | // oldUnew 59 | 60 | private int oldUnewEdgs[]; 61 | private byte oldUnewVals[]; 62 | private int oldUnewNumEdgs[]; 63 | 64 | public int[] getOldUnewEdgs() { 65 | return oldUnewEdgs; 66 | } 67 | 68 | public void setOldUnewEdgs(int[] oldUnewEdgs) { 69 | this.oldUnewEdgs = oldUnewEdgs; 70 | } 71 | 72 | public byte[] getOldUnewVals() { 73 | return oldUnewVals; 74 | } 75 | 76 | public void setOldUnewVals(byte[] oldUnewVals) { 77 | this.oldUnewVals = oldUnewVals; 78 | } 79 | 80 | // OUPUT FOR EACH ITERATION 81 | 82 | // delta 83 | 84 | private int deltaEdgs[]; 85 | private byte deltaVals[]; 86 | private int deltaNumEdgs[]; 87 | 88 | public int[] getDeltaEdgs() { 89 | return deltaEdgs; 90 | } 91 | 92 | public void setDeltaEdges(int[] deltaEdgs) { 93 | this.deltaEdgs = deltaEdgs; 94 | } 95 | 96 | public byte[] getDeltaVals() { 97 | return deltaVals; 98 | } 99 | 100 | public void setDeltaVals(byte[] deltaVals) { 101 | this.deltaVals = deltaVals; 102 | } 103 | 104 | // oldUnewUdelta 105 | 106 | private int oldUnewUdeltaEdgs[]; 107 | private byte oldUnewUdeltaVals[]; 108 | private int oldUnewUdeltaNumEdgs[]; 109 | 110 | public int[] getOldUnewUdeltaEdgs() { 111 | return oldUnewUdeltaEdgs; 112 | } 113 | 114 | public void setOldUnewUdeltaEdgs(int[] oldUnewUdeltaEdgs) { 115 | this.oldUnewUdeltaEdgs = oldUnewUdeltaEdgs; 116 | } 117 | 118 | public byte[] getOldUnewUdeltaVals() { 119 | return oldUnewUdeltaVals; 120 | } 121 | 122 | public void setOldUnewUdeltaVals(byte[] oldUnewUdeltaVals) { 123 | this.oldUnewUdeltaVals = oldUnewUdeltaVals; 124 | } 125 | 126 | public int[] getOldUnewUdeltaNumEdgs() { 127 | return this.oldUnewNumEdgs; 128 | } 129 | 130 | public void setOldUnewUdeltaNumEdgs(byte[] oldUnewUdeltaVals) { 131 | this.oldUnewUdeltaVals = oldUnewUdeltaVals; 132 | } 133 | 134 | public ComputationSet(){ 135 | this.oldEdgs = new int[0]; 136 | this.oldVals = new byte[0]; 137 | this.oldNumEdgs = new int[1]; 138 | 139 | this.newEdgs = new int[0]; 140 | this.newVals = new byte[0]; 141 | this.newNumEdgs = new int[1]; 142 | 143 | this.oldUnewEdgs = new int[0]; 144 | this.oldUnewVals = new byte[0]; 145 | this.oldUnewNumEdgs = new int[1]; 146 | 147 | this.deltaEdgs = new int[0]; 148 | this.deltaVals = new byte[0]; 149 | this.deltaNumEdgs = new int[1]; 150 | 151 | this.oldUnewUdeltaEdgs = new int[0]; 152 | this.oldUnewUdeltaVals = new byte[0]; 153 | this.oldUnewUdeltaNumEdgs = new int[1]; 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/graspan/oracle/GraphExtractor.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.graspan.oracle; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.BufferedWriter; 5 | import java.io.File; 6 | import java.io.FileInputStream; 7 | import java.io.FileNotFoundException; 8 | import java.io.FileWriter; 9 | import java.io.IOException; 10 | import java.io.InputStreamReader; 11 | import java.io.PrintWriter; 12 | import java.util.logging.Logger; 13 | 14 | import edu.uci.ics.cs.graspan.computationM.GrammarChecker; 15 | import edu.uci.ics.cs.graspan.dispatcher.GlobalParams; 16 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 17 | 18 | /** 19 | * Array-based Graph Extractor 20 | * @author aftab 21 | * 22 | */ 23 | public class GraphExtractor { 24 | 25 | private static final Logger logger = GraspanLogger.getLogger("GraphExtractor"); 26 | 27 | //NOTE: MAKE SURE RANGE IS BETWEEN THE LIMITS OF THE GRAPH 28 | private static final int RANGE_START =15000; 29 | private static final int RANGE_END =15050; 30 | private static final int EDGE_BUFFER_SIZE = RANGE_END - RANGE_START + 1; 31 | 32 | private static String basefilename; 33 | 34 | public static int gph[][] = new int[EDGE_BUFFER_SIZE][2]; 35 | public static byte vals[]=new byte[EDGE_BUFFER_SIZE]; 36 | 37 | public static void main(String args[]) throws NumberFormatException, IOException { 38 | 39 | init(args); 40 | GrammarChecker.loadGrammars(new File(basefilename + ".extracted.grammar")); 41 | 42 | // read in the original graph 43 | readGraph(); 44 | logger.info("Read graph"); 45 | storeGraph(); 46 | logger.info("Stored graph"); 47 | 48 | } 49 | 50 | private static String init(String[] args) { 51 | basefilename = args[0]; 52 | logger.info("Input graph: " + args[0]); 53 | 54 | // initialize the graph data structures with -1 55 | for (int i = 0; i < gph.length; i++) { 56 | gph[i][0] = -1; 57 | gph[i][1] = -1; 58 | vals[i]=-1; 59 | } 60 | logger.info("Completed initialization of graph data structures."); 61 | return basefilename; 62 | } 63 | 64 | 65 | private static void readGraph() throws FileNotFoundException, IOException { 66 | BufferedReader ins = new BufferedReader(new InputStreamReader(new FileInputStream(new File(basefilename)))); 67 | String ln; 68 | int gphPtr = 0, linePtr = 0; 69 | while ((ln = ins.readLine()) != null) { 70 | if (!ln.isEmpty()) { 71 | linePtr++; 72 | if (linePtr >= RANGE_START & linePtr <= RANGE_END) { 73 | // logger.info(linePtr+""); 74 | String[] tok = ln.split("\t");// NOTE: MAKE SURE INPUT FILE IS TAB DELIMITED 75 | gph[gphPtr][0] = Integer.parseInt(tok[0]); 76 | gph[gphPtr][1] = Integer.parseInt(tok[1]); 77 | vals[gphPtr] = GrammarChecker.getValue(tok[2].trim()); 78 | System.out.println("grammarcheckergetval: "+GrammarChecker.getValue(tok[2].trim())); 79 | gphPtr++; 80 | } 81 | } 82 | } 83 | ins.close(); 84 | } 85 | 86 | private static void storeGraph() throws IOException { 87 | 88 | PrintWriter outStrm = new PrintWriter(new BufferedWriter(new FileWriter(basefilename + ".extracted", true))); 89 | 90 | for (int i = 0; i < gph.length; i++) { 91 | if (gph[i][0] == -1) 92 | break; 93 | outStrm.println(gph[i][0] + "\t" + gph[i][1]+ "\t" + GrammarChecker.getValue((byte)vals[i])); 94 | } 95 | outStrm.close(); 96 | 97 | PrintWriter outStrm2 = new PrintWriter(new BufferedWriter(new FileWriter(basefilename + ".extracted_numericaVals", true))); 98 | 99 | for (int i = 0; i < gph.length; i++) { 100 | if (gph[i][0] == -1) 101 | break; 102 | outStrm2.println(gph[i][0] + "\t" + gph[i][1]+ "\t" + (byte)vals[i]); 103 | } 104 | outStrm2.close(); 105 | } 106 | 107 | 108 | } 109 | -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/PartitionFileChecker.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan; 2 | 3 | import java.io.BufferedInputStream; 4 | import java.io.DataInputStream; 5 | import java.io.FileInputStream; 6 | import java.io.FileNotFoundException; 7 | import java.io.IOException; 8 | import java.util.logging.Logger; 9 | 10 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 11 | 12 | /** 13 | * 14 | * @author Aftab 15 | * 16 | */ 17 | public class PartitionFileChecker { 18 | 19 | private static final Logger logger = GraspanLogger.getLogger("PartitionFileChecker"); 20 | String baseFilename; 21 | int reqVertexId; 22 | int numParts; 23 | 24 | public PartitionFileChecker(String basefilename){ 25 | // public PartitionFileChecker(String basefilename, int reqVertexId, int numParts){ 26 | this.baseFilename=basefilename; 27 | // this.reqVertexId=reqVertexId; 28 | // this.numParts=numParts; 29 | } 30 | 31 | /** 32 | * 33 | * @param args 34 | * @throws IOException 35 | */ 36 | public static void main(String args[]) throws IOException { 37 | PartitionFileChecker pfchecker = new PartitionFileChecker(args[0]); 38 | // PartitionFileChecker pfchecker = new PartitionFileChecker(args[0],Integer.parseInt(args[1]),Integer.parseInt(args[2])); 39 | pfchecker.run(Integer.parseInt(args[1])); 40 | } 41 | 42 | 43 | public void run(int i) throws IOException { 44 | // readPartitionFile(baseFilename); 45 | 46 | //read all partition files 47 | // for (int i = 0; i < numParts; i++) { 48 | // int i=1; 49 | // logger.info("partition " + i); 50 | // System.out.println("partition " +partId); 51 | readAndPrint(i); 52 | // } 53 | 54 | } 55 | 56 | /** 57 | * Reads and prints the contents of the partition files 58 | * 59 | * @param baseFilename 60 | * @throws IOException 61 | */ 62 | private void readAndPrint(int partId) throws IOException { 63 | DataInputStream dataIn = new DataInputStream(new BufferedInputStream(new FileInputStream(baseFilename + ".partition." + partId))); 64 | int srcVId, destVId; 65 | byte edgVal; 66 | String edgValStr=""; 67 | while (true) { 68 | try { 69 | 70 | srcVId = dataIn.readInt(); 71 | 72 | // if (srcVId==reqVertexId) 73 | { 74 | // srcVId 75 | // System.out.println("src " + srcVId);//Adjacency List Print Style 76 | // System.out.println("-----");//Adjacency List Print Style 77 | 78 | // count (number of destVs from srcV in the current list) 79 | int count = dataIn.readInt(); 80 | // System.out.println("# " + count);//Adjacency List Print Style 81 | // System.out.println("-----");//Adjacency List Print Style 82 | 83 | for (int i = 0; i < count; i++) { 84 | // dstVId 85 | destVId=dataIn.readInt(); 86 | // System.out.println("dst " + destVId);//Adjacency List Print Style 87 | 88 | // edge value 89 | edgVal = dataIn.readByte(); 90 | // System.out.println("val " + edgVal);//Adjacency List Print Style 91 | 92 | //edgeval to number conversion: 93 | 94 | // if (edgVal==0){ 95 | // edgValStr="n"; 96 | // } 97 | // else if (edgVal==1){ 98 | // edgValStr="e"; 99 | // } 100 | // else { 101 | edgValStr="" + edgVal; 102 | // } 103 | 104 | 105 | System.out.println(srcVId+"\t"+destVId+"\t"+edgValStr);//Edge List Print Style 106 | 107 | 108 | } 109 | // System.out.println("=====");//Adjacency List Print Style 110 | } 111 | 112 | } catch (Exception exception) { 113 | break; 114 | } 115 | } 116 | dataIn.close(); 117 | } 118 | 119 | } 120 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Graspan Java 2 | 3 | Welcome to the home repository of Graspan Java. 4 | 5 | Graspan is a **disk-based parallel graph system** that uses an edge-pair centric computation model to compute dynamic transitive closures on large program graphs. Graspan has been implemented in two languages: Java and C++. This repository provides the Java implementation of Graspan. 6 | 7 | This Readme provides a how-to-use guide for Graspan Java. To see how to use the **C++ version** of Graspan, see [here](https://github.com/Graspan/graspan-cpp). 8 | 9 | See [here](https://github.com/Graspan/graspan-java/blob/master/AboutGraspan.md) to understand **how Graspan works** and how it performs in comparison to other systems (under revision). 10 | 11 | For a detailed description of our system, please see the **preliminary version of our paper** [here](http://www.ics.uci.edu/~guoqingx/papers/wang-asplos17.pdf), which has been accepted in ASPLOS 2017. In addition, a tutorial of Graspan is scheduled to be presented in ASPLOS 2017. If you are interested, please visit our **ASPLOS 2017 tutorial** [page](http://www.ics.uci.edu/~guoqingx/asplos-tutorial/main.html). 12 | 13 | ## Getting Started 14 | 15 | Using Graspan Java is very simple; no compilation is necessary. 16 | 17 | First, ensure you have **JDK 1.6** (or a later version of JDK) installed in your machine. 18 | 19 | Then, copy the **executables** folder in the **src** folder of the **graspan-java** repository, into any location in your machine. The folder contains: 20 | 21 | * *graspan.jar* - executable of the graspan-java project 22 | * *graspan-java-run.sh* - script for running graspan-java 23 | * *rules_pt* - sample grammar file containing rules for points-to analysis 24 | * *rules_np* - sample grammar file containing rules for dataflow analysis 25 | 26 | Graspan needs two input files: (1) a graph on which Graspan can perform computations and (2) a grammar file which describes how the computations (edge additions) are to be performed. 27 | 28 | You may copy any graph and grammar file from our sample datasets [here](https://drive.google.com/drive/folders/1M4WxwykUd-jX8jBA50pSNf2R-1IJ49PJ?usp=drive_link) inside the **executables** folder in your machine. 29 | 30 | *Note that Graspan supports graph files in edge-list format as shown below,* 31 | 32 | ``` 33 | [EDGE SOURCE] [EDGE DESTINATION] [EDGE VALUE] 34 | ``` 35 | 36 | *The grammar file consists of production rules in CNF form, where a line like the following,* 37 | ``` 38 | A B C 39 | ``` 40 | *represents a rule such that A is the left-side of the rule and BC is the right-side of the rule.* 41 | 42 | After getting the graph and grammar file into the **executables** folder, run the **graspan-java-run.sh** script in your command line specifying, 43 | 44 | 1. the graph file, 45 | 2. the number of partitions user would like to generate from the graph during preprocessing, prior to computation, 46 | 3. whether or not the graph has edge values (yes or no), 47 | 4. the grammar file, 48 | 49 | as shown below, 50 | ``` 51 | ./graspan-java-run.sh 52 | ``` 53 | 54 | Here is an **example**, 55 | ``` 56 | ./graspan-java-run.sh mygraph 5 yes rules_pt 57 | ``` 58 | 59 | After running the above command, you can monitor the progress of the computation by viewing the generated **comp.output** file. After computation ends, **comp.output** will show the number of edges generated and the total computation time. The **.partition.** output files will contain the partitioned graph with new edges. 60 | 61 | ## Questions or Comments? 62 | Participate in our discussion group, [GraspanMeet](https://groups.google.com/d/forum/graspanmeet). 63 | 64 | ## Project Contributors 65 | 66 | * [**Kai Wang**](http://www.ics.uci.edu/~wangk7/) - *PhD Student, UCI* 67 | * [**Aftab Hussain**](http://www.ics.uci.edu/~aftabh/) - *PhD Student, UCI* 68 | * [**Zhiqiang Zuo**](http://zuozhiqiang.bitbucket.io/) - *Postdoc Scholar, UCI* 69 | * [**Harry Xu**](http://www.ics.uci.edu/~guoqingx/) - *Assistant Professor, UCI* 70 | * [**Ardalan Sani**](http://www.ics.uci.edu/~ardalan/) - *Assistant Professor, UCI* 71 | * **John Thorpe** - *Undergraduate Student, UCI* 72 | * **Sung-Soo Son** - *Visiting Undergraduate Student, UCI* 73 | * [**Khanh Ngyuen**](http://www.ics.uci.edu/~khanhtn1/) - *PhD Student, UCI* 74 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/dispatcher/ComputationClient.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.dispatcher; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.File; 5 | import java.io.FileInputStream; 6 | import java.io.IOException; 7 | import java.io.InputStreamReader; 8 | import java.util.logging.Logger; 9 | 10 | import edu.uci.ics.cs.graspan.computationEL.EngineEL; 11 | import edu.uci.ics.cs.graspan.computationM.EngineM; 12 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 13 | import edu.uci.ics.cs.graspan.support.Utilities; 14 | 15 | public class ComputationClient { 16 | 17 | private static final Logger logger = GraspanLogger.getLogger("ComputationClient"); 18 | 19 | public static void main(String args[]) throws IOException { 20 | 21 | // String computationConfigFilename = args[0]; 22 | GlobalParams.setBasefilename(args[0]); 23 | GlobalParams.setNumParts(Integer.parseInt(args[1])); 24 | GlobalParams.setNumThreads(Integer.parseInt(args[2])); 25 | GlobalParams.setPartMaxPostNewEdges(Integer.parseInt(args[3])*1000000); 26 | 27 | GlobalParams.setReloadPlan("RELOAD_PLAN_2"); 28 | GlobalParams.setEdcSize(1000); 29 | GlobalParams.setComputationLogic("SMART_MERGE"); 30 | 31 | if (GlobalParams.getNumParts()==2){ 32 | GlobalParams.setInMemComp(true); 33 | } 34 | else{ 35 | GlobalParams.setInMemComp(false); 36 | } 37 | 38 | 39 | /* 40 | * Scan the Computer-client config file 41 | */ 42 | 43 | // BufferedReader computationConfigStream = new BufferedReader(new InputStreamReader(new FileInputStream(new File(computationConfigFilename)))); 44 | /* String ln; 45 | 46 | String[] tok; 47 | while ((ln = computationConfigStream.readLine()) != null) { 48 | tok = ln.split(" "); 49 | if (tok[0].compareTo("INPUT_GRAPH_FILEPATH") == 0) { 50 | GlobalParams.setBasefilename(tok[2]); 51 | } 52 | if (tok[0].compareTo("TOTAL_NUM_PARTS") == 0) { 53 | GlobalParams.setNumParts(Integer.parseInt(tok[2])); 54 | } 55 | if (tok[0].compareTo("OP_EDGE_TRACKER_INTERVAL") == 0) { 56 | GlobalParams.setOpEdgeTrackerInterval(Integer.parseInt(tok[2])); 57 | } 58 | if (tok[0].compareTo("MAX_PART_SIZE_POST_NEW_EDGES") == 0) { 59 | GlobalParams.setPartMaxPostNewEdges(Integer.parseInt(tok[2])); 60 | } 61 | if (tok[0].compareTo("NEW_EDGE_NODE_SIZE") == 0) { 62 | GlobalParams.setNewEdgesNodeSize(Integer.parseInt(tok[2])); 63 | } 64 | if (tok[0].compareTo("NUM_OF_THREADS") == 0) { 65 | GlobalParams.setNumThreads(Integer.parseInt(tok[2])); 66 | } 67 | if (tok[0].compareTo("") == 0) { 68 | break; 69 | } 70 | } 71 | 72 | computationConfigStream.close();*/ 73 | 74 | // MemUsageCheckThread memUsage = new MemUsageCheckThread(); 75 | // memUsage.start(); 76 | 77 | logger.info("Starting computation."); 78 | logger.info("Total number of partitions: " + GlobalParams.getNumParts()); 79 | logger.info("Number of parts per computation: " + GlobalParams.getNumPartsPerComputation()); 80 | logger.info("Reload plan: " + GlobalParams.getReloadPlan()); 81 | 82 | if (GlobalParams.getComputationLogic().compareTo("LINEAR_SCAN_OF_LLISTS") == 0) { 83 | EngineEL engine = new EngineEL(); 84 | engine.run(); 85 | logger.info("Total number of new edges created: " + engine.get_totalNewEdgs()); 86 | } else if (GlobalParams.getComputationLogic().compareTo("SMART_MERGE") == 0) { 87 | 88 | logger.info("Starting Smart Merge Computation"); 89 | // GraspanTimer sm_comp = new GraspanTimer(System.currentTimeMillis()); 90 | long smart_merge_comp_start = System.currentTimeMillis(); 91 | 92 | EngineM engine = new EngineM(); 93 | engine.run(); 94 | logger.info("Total number of new edges created: " + engine.get_totalNewEdgs()); 95 | 96 | logger.info("Finished Smart Merge Computation"); 97 | 98 | // sm_comp.calculateDuration(System.currentTimeMillis()); 99 | 100 | // logger.info("Smart Merge Computation took " + Utilities.getDurationInHMS(sm_comp.getDuration())); 101 | logger.info("Smart Merge Computation took " + Utilities.getDurationInHMS(System.currentTimeMillis() - smart_merge_comp_start)); 102 | } 103 | 104 | // MemUsageCheckThread.memoryUsageOutput.close(); 105 | // memUsage.stop(); 106 | } 107 | } -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/artificalgraphgenerator/ArtificialGraphGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * HAS ERROR: EDGE VALUES OF GREATER THAN 256 IS OUTPUT 3 | */ 4 | 5 | package edu.uci.ics.cs.graspan.artificalgraphgenerator; 6 | 7 | import java.io.BufferedWriter; 8 | import java.io.FileWriter; 9 | import java.io.IOException; 10 | import java.io.PrintWriter; 11 | import java.util.HashMap; 12 | import java.util.Iterator; 13 | import java.util.Map; 14 | import java.util.Random; 15 | 16 | /** 17 | * This program generates a random set of distinct edges (srcId,dstId,edgeVal) 18 | * 19 | * @author Aftab 20 | * 21 | */ 22 | public class ArtificialGraphGenerator { 23 | 24 | static final int MAX_VERTEX_VALUE = 10000000; 25 | static final int MAX_EDGE_VALUE = 256; 26 | static final int MAX_NUM_OF_EDGES = 10000000; 27 | 28 | public static void main(String args[]) throws IOException { 29 | 30 | Random rand = new Random(); 31 | int numbers[] = new int[3];// = { 1000000, 1000000, 256 }; 32 | Integer decPlaces[] = new Integer[2]; 33 | HashMap edgeMap = new HashMap(); 34 | 35 | // writing to map 36 | for (int iteration = 0; iteration < MAX_NUM_OF_EDGES; iteration++) { 37 | 38 | numbers[0] = rand.nextInt(MAX_VERTEX_VALUE + 1) + 1; 39 | numbers[1] = rand.nextInt(MAX_VERTEX_VALUE + 1) + 1; 40 | numbers[2] = rand.nextInt(MAX_EDGE_VALUE + 1) + 1; 41 | 42 | // storing the number of decimal places of 2nd and 3rd numbers 43 | for (int i = 1; i < 3; i++) { 44 | int decPlaceCount = 0; 45 | int a1 = numbers[i]; 46 | while (a1 > 0) { 47 | a1 = a1 / 10; 48 | decPlaceCount++; 49 | } 50 | decPlaces[i - 1] = decPlaceCount; 51 | } 52 | // for (int i = 0; i < 2; i++) { 53 | // System.out.print(decPlaces[i] + " "); 54 | // } 55 | 56 | // combining the numbers 57 | long a = numbers[0] * 10 * (decPlaces[0] + decPlaces[1]) + numbers[1] * 10 * (decPlaces[1]) + numbers[2]; 58 | long component1 = numbers[0], component2 = numbers[1], component3 = numbers[2], combinedNum; 59 | for (int i = 0; i < decPlaces[0] + decPlaces[1]; i++) { 60 | component1 = component1 * 10; 61 | } 62 | for (int i = 0; i < decPlaces[1]; i++) { 63 | component2 = component2 * 10; 64 | } 65 | combinedNum = component1 + component2 + component3; 66 | // System.out.println(combinedNum); 67 | 68 | // adding the numbers to map (to ensure non-duplication) 69 | edgeMap.put(combinedNum, decPlaces); 70 | } 71 | 72 | /* 73 | * Retrieving the numbers and writing them to file 74 | */ 75 | PrintWriter edgeOutputStream = new PrintWriter(new BufferedWriter(new FileWriter(args[0], true))); 76 | 77 | Iterator> it = edgeMap.entrySet().iterator(); 78 | 79 | while (it.hasNext()) { 80 | 81 | Map.Entry pair = it.next(); 82 | 83 | // retrieve the combinedNumber and the decimalPlaces array from 84 | // the 85 | // map 86 | long combinedNum = pair.getKey(); 87 | decPlaces = pair.getValue(); 88 | 89 | // retrieving srcVId 90 | long temp = 0; 91 | temp = combinedNum; 92 | for (int i = 0; i < decPlaces[0] + decPlaces[1]; i++) { 93 | temp = temp / 10; 94 | } 95 | long srcVId = temp; 96 | // System.out.println("Retrieved srcVId: " + srcVId); 97 | 98 | // retrieving destVId 99 | for (int i = 0; i < decPlaces[0] + decPlaces[1]; i++) { 100 | temp = temp * 10; 101 | } 102 | long lastTwoCombinedNum = combinedNum - temp; 103 | // System.out.println(lastTwoCombinedNum); 104 | temp = lastTwoCombinedNum; 105 | for (int i = 0; i < decPlaces[1]; i++) { 106 | temp = temp / 10; 107 | } 108 | long destVId = temp; 109 | // System.out.println("Retrieved destVId: " + destVId); 110 | 111 | // retrieving edgeVal 112 | for (int i = 0; i < decPlaces[1]; i++) { 113 | temp = temp * 10; 114 | } 115 | long edgeVal = lastTwoCombinedNum - temp; 116 | // System.out.println("Retrieved edgeVal: " + edgeVal); 117 | 118 | // writing the edge to file 119 | edgeOutputStream.print(srcVId); 120 | edgeOutputStream.print("\t"); 121 | edgeOutputStream.print(destVId); 122 | edgeOutputStream.print("\t"); 123 | edgeOutputStream.print(edgeVal); 124 | edgeOutputStream.println(); 125 | 126 | } 127 | edgeOutputStream.close(); 128 | 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/computationM/GrammarChecker.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.computationM; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.BufferedWriter; 5 | import java.io.File; 6 | import java.io.FileReader; 7 | import java.io.FileWriter; 8 | import java.io.IOException; 9 | import java.io.PrintWriter; 10 | import java.util.ArrayList; 11 | import java.util.Collection; 12 | import java.util.HashMap; 13 | import java.util.LinkedHashMap; 14 | import java.util.LinkedHashSet; 15 | import java.util.Map; 16 | import java.util.Set; 17 | 18 | public class GrammarChecker { 19 | //sort the mapping info of inputed grammar with used grammar 20 | private static Map map = new LinkedHashMap(); 21 | 22 | private static Map reverse_map =new LinkedHashMap(); 23 | 24 | private static Map> dRules = new LinkedHashMap>(); 25 | 26 | private static Map sRules = new LinkedHashMap(); 27 | 28 | public static Set eRules = new LinkedHashSet(); 29 | 30 | 31 | public static void loadGrammars(File grammar_input) throws IOException { 32 | // initialize edgeDestCount and partSizes variables 33 | //Scan the grammar file 34 | BufferedReader inGrammarStrm = new BufferedReader(new FileReader(grammar_input)); 35 | String ln; 36 | 37 | String[] tok; 38 | while ((ln = inGrammarStrm.readLine()) != null) { 39 | tok = ln.split("\t"); 40 | if (tok.length == 1) { // production with 1 symbol (self) 41 | eRules.add(getValue(tok[0])); 42 | } 43 | else if (tok.length == 2) { // production with 2 symbols 44 | sRules.put(getValue(tok[1]), getValue(tok[0])); 45 | } 46 | else if (tok.length == 3) { // production with 3 symbols 47 | // consider production form : A->BC (Map will store them as (B,(C,A))) 48 | byte src1 = getValue(tok[1]); 49 | byte src2 = getValue(tok[2]); 50 | byte dst = getValue(tok[0]); 51 | 52 | if(dRules.containsKey(src1)){ 53 | HashMap mapR = dRules.get(src1); 54 | assert(!mapR.containsKey(src2)); 55 | mapR.put(src2, dst); 56 | } 57 | else{ 58 | HashMap mapR = new HashMap(); 59 | mapR.put(src2, dst); 60 | dRules.put(src1, mapR); 61 | } 62 | } 63 | else{ 64 | throw new RuntimeException("Wrong length in the grammar file!!!"); 65 | } 66 | } 67 | 68 | inGrammarStrm.close(); 69 | 70 | writeCollection(new ArrayList>(map.entrySet()), new File(grammar_input.getParentFile(), "grammar")); 71 | // logger.info("Loaded " + ".grammar"); 72 | } 73 | 74 | 75 | public static Byte getValue(String string) { 76 | string = string.trim(); 77 | if(map.containsKey(string)){ 78 | return map.get(string); 79 | } 80 | else{ 81 | byte e = (byte) map.size(); 82 | map.put(string, e); 83 | reverse_map.put(e, string); 84 | return e; 85 | } 86 | } 87 | 88 | public static String getValue(Byte e){ 89 | return reverse_map.get(e); 90 | } 91 | 92 | public static int getNumOfGrammarSymbols(){ 93 | return map.size(); 94 | } 95 | 96 | 97 | public static byte checkL2Rules(byte srcEval, byte destEval){ 98 | // BC ----- > A : > : > 99 | byte OPEval = -1; 100 | 101 | if(dRules.containsKey(srcEval)){ 102 | HashMap map = dRules.get(srcEval); 103 | if(map.containsKey(destEval)){ 104 | OPEval = map.get(destEval); 105 | } 106 | } 107 | 108 | return OPEval; 109 | } 110 | 111 | 112 | public static byte checkL1Rules(byte srcVal){ 113 | byte OPEval = -1; 114 | if(GrammarChecker.sRules.containsKey(srcVal)){ 115 | OPEval = GrammarChecker.sRules.get(srcVal); 116 | } 117 | return OPEval; 118 | } 119 | 120 | public static void writeCollection(Collection collection, File file){ 121 | PrintWriter out = null; 122 | try{ 123 | // if (!file.getParentFile().exists()) { 124 | // file.getParentFile().mkdirs(); 125 | // } 126 | //write the passing inputs 127 | out = new PrintWriter(new BufferedWriter(new FileWriter(file))); 128 | for(T element: collection){ 129 | out.println(element); 130 | } 131 | out.close(); 132 | } 133 | catch(IOException e){ 134 | e.printStackTrace(); 135 | } 136 | finally{ 137 | if(out != null) 138 | out.close(); 139 | } 140 | } 141 | 142 | public static void main(String[] args) throws IOException { 143 | GrammarChecker.loadGrammars(new File("graph.grammar")); 144 | System.out.println(dRules); 145 | System.out.println(sRules); 146 | System.out.println(eRules); 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/datastructures/PartitionQuerier.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.datastructures; 2 | 3 | import java.util.logging.Logger; 4 | 5 | import edu.uci.ics.cs.graspan.dispatcher.GlobalParams; 6 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 7 | 8 | /** 9 | * Contains of methods for retrieving miscellaneous info of loaded partitions 10 | * (used by all phases) 11 | * 12 | * @author Aftab 13 | */ 14 | 15 | // TODO NEED TO UPDATE THE LOGIC OF ALL FUNCTIONS HERE, IF PARTITION ALLOCATION 16 | // TABLE IS NOT CONTIGUOUS BY PARTID// u cannot operate on partition id anymore 17 | public class PartitionQuerier { 18 | 19 | private static final Logger logger = GraspanLogger.getLogger("PartitionQuerier"); 20 | 21 | /** 22 | * Returns the number of unique sources of in partition partId. IMP: we 23 | * consider the vertex numbering of the input graph to start from 1 NOT 0. 24 | * 25 | * @param partId 26 | */ 27 | public static int getNumUniqueSrcs(int partId) { 28 | int[][] partAllocTable = AllPartitions.getPartAllocTab(); 29 | for (int i = 0; i < AllPartitions.getPartAllocTab().length; i++) { 30 | if (partId == partAllocTable[i][0]) { 31 | if (i == 0) { 32 | return partAllocTable[i][1]; 33 | } else { 34 | return partAllocTable[i][1] - partAllocTable[i - 1][1]; 35 | } 36 | } 37 | } 38 | return -1; 39 | } 40 | 41 | /** 42 | * Returns the smallest source vertex Id in partition partId. 43 | * 44 | * @param partId 45 | * @return 46 | */ 47 | public static int getFirstSrc(int partId) { 48 | int[][] partAllocTable = AllPartitions.getPartAllocTab(); 49 | for (int i = 0; i < partAllocTable.length; i++) { 50 | if (partId == partAllocTable[i][0]) { 51 | if (i == 0) { 52 | return 1; 53 | } else { 54 | return partAllocTable[i - 1][1] + 1; 55 | } 56 | } 57 | } 58 | return -1; 59 | } 60 | 61 | /** 62 | * Returns the largest source vertex Id in partition partId. 63 | * 64 | * @param partId 65 | * @return 66 | */ 67 | public static int getLastSrc(int partId) { 68 | int[][] partAllocTable = AllPartitions.getPartAllocTab(); 69 | for (int i = 0; i < partAllocTable.length; i++) { 70 | if (partId == partAllocTable[i][0]) { 71 | return partAllocTable[i][1]; 72 | } 73 | } 74 | logger.info("ERROR: Last source is -1 for partition " + partId); 75 | return -1; 76 | } 77 | 78 | /** 79 | * Finds whether the given vertex belongs to a partition as a source vertex. 80 | * 81 | * @param srcVId 82 | * @param partId 83 | * @return 84 | */ 85 | public static boolean inPartition(int srcVId, int partId) { 86 | if (srcVId >= getFirstSrc(partId) && srcVId <= getLastSrc(partId)) 87 | return true; 88 | else 89 | return false; 90 | } 91 | 92 | /** 93 | * 94 | * Returns the actual Id of the source vertex from the Id of the vertex in 95 | * the loaded partition array. 96 | * 97 | * @param vertexPartArrIdx 98 | * @param partId 99 | * @return 100 | */ 101 | public static int getActualIdFrmPartArrIdx(int vertexPartArrIdx, int partId) { 102 | if (findPartition(vertexPartArrIdx + getFirstSrc(partId)) != partId) { 103 | logger.info("ERROR: The " + vertexPartArrIdx + "th element of partition " + partId + "does not exist"); 104 | return -1; 105 | } 106 | return vertexPartArrIdx + getFirstSrc(partId); 107 | } 108 | 109 | /** 110 | * Returns the Id of the source vertex in the loaded partition array from 111 | * the actual Id of the source vertex. 112 | * 113 | * @param src 114 | * - the actual Id of the source vertex. 115 | * @param partId 116 | * - the partition Id. 117 | * @return 118 | */ 119 | public static int getPartArrIdxFrmActualId(int src, int partId) { 120 | if (findPartition(src) != partId) { 121 | logger.info("ERROR: Source " + src + " does not exist in partition " + partId); 122 | return -1; 123 | } 124 | return src - getFirstSrc(partId); 125 | } 126 | 127 | /** 128 | * Returns the partition id of a given source vertex. Returns -1 if vertex 129 | * does not exist in any partition as a source vertex. 130 | * 131 | * @param src 132 | * - the actual Id of the source vertex. 133 | */ 134 | public static int findPartition(int src) { 135 | int[][] partAllocTable = AllPartitions.getPartAllocTab(); 136 | for (int i = 0; i < partAllocTable.length; i++) { 137 | if (src <= partAllocTable[i][1]) {//this is because pat is is sorted in asc. order by vertex id, not part id 138 | return partAllocTable[i][0]; 139 | } 140 | } 141 | return -1; 142 | } 143 | 144 | } -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/edgecomputation/TestPartitions.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.edgecomputation; 2 | 3 | /** 4 | * This program creates the data structures for 3 artificial partitions. 5 | * 6 | * @author Aftab 7 | * 8 | */ 9 | public class TestPartitions { 10 | 11 | public int partAllocTable[] = { 7, 8, 15, 20, 32, 43 }; 12 | 13 | // Contains the ids of the partitions to be loaded in the memory 14 | public int loadedParts[] = { 0, 2, 5 }; 15 | 16 | // Data structures for storing the partitions to load: 17 | // Dimension 1 indicates partition number, Dimension 2 indicates list for a 18 | // source vertex, Dimension 3 indicates an out-edge from each source vertex 19 | public int partEdgeArrays[][][] = new int[3][][]; 20 | public int partEdgeValArrays[][][] = new int[3][][]; 21 | 22 | // Stores the out degrees of each source vertex of each partition. 23 | // Dimension 1 indicates partition number, Dimension 2 indicates out degree 24 | // of a source vertex in the partition indicated by Index 1 25 | public int partOutDegrees[][] = new int[3][]; 26 | 27 | public void createPartitions() { 28 | 29 | // create outDegrees 30 | partOutDegrees[0] = new int[] { 6, 5, 0, 6, 4, 4, 5 };// 7 srcs 31 | partOutDegrees[1] = new int[] { 6, 9, 0, 15, 0, 2, 10 };// 7 srcs 32 | partOutDegrees[2] = new int[] { 3, 3, 2, 10, 12, 9, 8, 0, 7, 7, 0 };// 11 33 | // srcs 34 | 35 | // create Partition 0 Edges 36 | partEdgeArrays[0][0] = new int[] { 3, 5, 6, 10, 12 }; 37 | partEdgeArrays[0][1] = new int[] { 2, 7, 80, 10, 33, 33 }; 38 | partEdgeArrays[0][2] = new int[] {}; 39 | partEdgeArrays[0][3] = new int[] { 12, 12, 112, 12, 12, 12 }; 40 | partEdgeArrays[0][4] = new int[] { 1, 2, 4, 5 }; 41 | partEdgeArrays[0][5] = new int[] { 38, 39, 42, 43 }; 42 | partEdgeArrays[0][6] = new int[] { 8, 18, 12, 15, 152 }; 43 | 44 | // create Partition 0 Edge Values 45 | partEdgeValArrays[0][0] = new int[] { 7, 100, 101, 1, 5 }; 46 | partEdgeValArrays[0][1] = new int[] { 9, 11, 1, 10, 2, 8 }; 47 | partEdgeValArrays[0][2] = new int[] {}; 48 | partEdgeValArrays[0][3] = new int[] { 1, 2, 3, 4, 5, 6 }; 49 | partEdgeValArrays[0][4] = new int[] { 7, 7, 7, 7 }; 50 | partEdgeValArrays[0][5] = new int[] { 1, 1, 5, 9 }; 51 | partEdgeValArrays[0][6] = new int[] { 1, 2, 5, 3, 2 }; 52 | 53 | // create Partition 2 Edges 54 | partEdgeArrays[1][0] = new int[] { 1, 2, 3, 9, 10, 33 }; 55 | partEdgeArrays[1][1] = new int[] { 1, 2, 3, 5, 17, 19, 110, 33, 34 }; 56 | partEdgeArrays[1][2] = new int[] {}; 57 | partEdgeArrays[1][3] = new int[] { 1, 1, 2, 9, 10, 10, 33, 34, 42, 42, 42, 42, 43, 43, 43 }; 58 | partEdgeArrays[1][4] = new int[] {}; 59 | partEdgeArrays[1][5] = new int[] { 37, 39 }; 60 | partEdgeArrays[1][6] = new int[] { 1, 2, 3, 4, 5, 5, 5, 5, 6, 7 }; 61 | 62 | // create Partition 2 Edge Values 63 | partEdgeValArrays[1][0] = new int[] { 1, 1, 2, 5, 7, 3 }; 64 | partEdgeValArrays[1][1] = new int[] { 1, 1, 2, 7, 11, 11, 12, 2, 3 }; 65 | partEdgeValArrays[1][2] = new int[] {}; 66 | partEdgeValArrays[1][3] = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; 67 | partEdgeValArrays[1][4] = new int[] {}; 68 | partEdgeValArrays[1][5] = new int[] { 14, 1 }; 69 | partEdgeValArrays[1][6] = new int[] { 5, 5, 5, 5, 5, 6, 7, 8, 9, 9 }; 70 | 71 | // create Partition 5 Edges 72 | partEdgeArrays[2][0] = new int[] { 12, 40, 41 }; 73 | partEdgeArrays[2][1] = new int[] { 4, 10, 12 }; 74 | partEdgeArrays[2][2] = new int[] { 12, 15 }; 75 | partEdgeArrays[2][3] = new int[] { 1, 2, 4, 7, 10, 11, 12, 15, 36, 41 }; 76 | partEdgeArrays[2][4] = new int[] { 1, 2, 3, 10, 13, 14, 15, 33, 36, 36, 37, 38 }; 77 | partEdgeArrays[2][5] = new int[] { 11, 12, 13, 36, 37, 38, 39, 40, 42 }; 78 | partEdgeArrays[2][6] = new int[] { 6, 7, 9, 10, 12, 15, 37, 38 }; 79 | partEdgeArrays[2][7] = new int[] {}; 80 | partEdgeArrays[2][8] = new int[] { 4, 5, 6, 7, 7, 10, 11 }; 81 | partEdgeArrays[2][9] = new int[] { 7, 15, 39, 40, 41, 42, 42 }; 82 | partEdgeArrays[2][10] = new int[] {}; 83 | 84 | // create Partition 5 Edge Values 85 | partEdgeValArrays[2][0] = new int[] { 5, 6, 7 }; 86 | partEdgeValArrays[2][1] = new int[] { 100, 100, 50 }; 87 | partEdgeValArrays[2][2] = new int[] { 10, 6 }; 88 | partEdgeValArrays[2][3] = new int[] { 5, 5, 5, 3, 2, 10, 11, 8, 1, 2 }; 89 | partEdgeValArrays[2][4] = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 10 }; 90 | partEdgeValArrays[2][5] = new int[] { 10, 11, 12, 13, 14, 15, 16, 17, 18 }; 91 | partEdgeValArrays[2][6] = new int[] { 8, 8, 8, 8, 1, 2, 3, 4 }; 92 | partEdgeValArrays[2][7] = new int[] {}; 93 | partEdgeValArrays[2][8] = new int[] { 1, 2, 3, 13, 2, 6, 10 }; 94 | partEdgeValArrays[2][9] = new int[] { 10, 20, 30, 3, 5, 6, 7 }; 95 | partEdgeValArrays[2][10] = new int[] {}; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/support/GraspanLogger.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.support; 2 | 3 | import java.io.File; 4 | import java.io.FileInputStream; 5 | import java.io.PrintWriter; 6 | import java.io.StringWriter; 7 | import java.text.MessageFormat; 8 | import java.util.Date; 9 | import java.util.logging.ConsoleHandler; 10 | import java.util.logging.Formatter; 11 | import java.util.logging.LogManager; 12 | import java.util.logging.LogRecord; 13 | import java.util.logging.Logger; 14 | 15 | 16 | /** 17 | * 18 | * Wrapper for Java logging. 19 | * Use ChiLogger.getLogger("object-name") to get a logger object. 20 | */ 21 | 22 | public class GraspanLogger { 23 | 24 | 25 | 26 | public static void init() { 27 | try { 28 | File logProperties = new File("conf/logging.properties"); 29 | if (logProperties.exists()) 30 | LogManager.getLogManager().readConfiguration(new FileInputStream(logProperties.getAbsolutePath())); 31 | else System.err.println("Could not find cond/logging.properties!"); 32 | } catch (Exception err) { 33 | err.printStackTrace(); 34 | } 35 | } 36 | 37 | public static Logger getLogger(String name) { 38 | Logger log = Logger.getLogger(name); 39 | if (log.getHandlers().length == 0) { 40 | ConsoleHandler handler = new ConsoleHandler(); 41 | handler.setFormatter(new SingleLineFormatter()); 42 | log.setUseParentHandlers(false); 43 | log.addHandler(handler); 44 | } 45 | return log; 46 | } 47 | 48 | // http://stackoverflow.com/questions/194765/how-do-i-get-java-logging-output-to-appear-on-a-single-line 49 | public static class SingleLineFormatter extends Formatter { 50 | 51 | Date dat = new Date(); 52 | private final static String format = "{0,time}"; 53 | private MessageFormat formatter; 54 | private Object args[] = new Object[1]; 55 | 56 | // Line separator string. This is the value of the line.separator 57 | // property at the moment that the SimpleFormatter was created. 58 | //private String lineSeparator = (String) java.security.AccessController.doPrivileged( 59 | // new sun.security.action.GetPropertyAction("line.separator")); 60 | private String lineSeparator = "\n"; 61 | 62 | /** 63 | * Format the given LogRecord. 64 | * @param record the log record to be formatted. 65 | * @return a formatted log record 66 | */ 67 | public synchronized String format(LogRecord record) { 68 | 69 | StringBuilder sb = new StringBuilder(); 70 | 71 | // Minimize memory allocations here. 72 | dat.setTime(record.getMillis()); 73 | args[0] = dat; 74 | 75 | 76 | // Date and time 77 | StringBuffer text = new StringBuffer(); 78 | if (formatter == null) { 79 | formatter = new MessageFormat(format); 80 | } 81 | formatter.format(args, text, null); 82 | sb.append(text); 83 | sb.append(" "); 84 | 85 | 86 | // Logger name 87 | sb.append(record.getLoggerName()); 88 | 89 | // Method name 90 | if (record.getSourceMethodName() != null) { 91 | sb.append(" "); 92 | sb.append(record.getSourceMethodName()); 93 | } 94 | sb.append(" - t:"); // lineSeparator 95 | 96 | // Thread name 97 | sb.append(Thread.currentThread().getId()); 98 | sb.append(" "); 99 | 100 | String message = formatMessage(record); 101 | 102 | // Level 103 | sb.append(record.getLevel().getLocalizedName()); 104 | sb.append(": "); 105 | 106 | // Indent - the more serious, the more indented. 107 | //sb.append( String.format("% ""s") ); 108 | int iOffset = (1000 - record.getLevel().intValue()) / 100; 109 | for( int i = 0; i < iOffset; i++ ){ 110 | sb.append(" "); 111 | } 112 | 113 | 114 | sb.append(message); 115 | sb.append(lineSeparator); 116 | if (record.getThrown() != null) { 117 | try { 118 | StringWriter sw = new StringWriter(); 119 | PrintWriter pw = new PrintWriter(sw); 120 | record.getThrown().printStackTrace(pw); 121 | pw.close(); 122 | sb.append(sw.toString()); 123 | } catch (Exception ex) { 124 | } 125 | } 126 | return sb.toString(); 127 | } 128 | } 129 | 130 | } 131 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/datastructures/NewEdgesList.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.datastructures; 2 | 3 | import java.util.Arrays; 4 | import java.util.logging.Logger; 5 | 6 | import edu.uci.ics.cs.graspan.dispatcher.GlobalParams; 7 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 8 | 9 | /** 10 | * @author Kai Wang 11 | * 12 | * Created by Oct 8, 2015 13 | */ 14 | public class NewEdgesList { 15 | private static final Logger logger = GraspanLogger 16 | .getLogger("NewEdgesList"); 17 | public static final int NODE_SIZE = GlobalParams.getNewEdgesNodeSize(); 18 | 19 | private NewEdgesNode first; 20 | private NewEdgesNode last; 21 | private NewEdgesNode readableLast; 22 | private int size; 23 | private int readableSize; 24 | private int readableIndex; 25 | 26 | public class NewEdgesNode { 27 | private int[] dstVertices = null; 28 | private byte[] edgeValues = null; 29 | private int index; 30 | private NewEdgesNode next = null; 31 | 32 | NewEdgesNode() { 33 | dstVertices = new int[NODE_SIZE]; 34 | edgeValues = new byte[NODE_SIZE]; 35 | 36 | for (int i = 0; i < NODE_SIZE; i++) { 37 | dstVertices[i] = -1; 38 | } 39 | 40 | } 41 | 42 | public int[] getDstVertices() { 43 | return dstVertices; 44 | } 45 | 46 | public byte[] getEdgeValues() { 47 | return edgeValues; 48 | } 49 | 50 | public int getIndex() { 51 | return index; 52 | } 53 | 54 | public void setIndex(int index) { 55 | this.index = index; 56 | } 57 | 58 | public void add(int vertexId, byte edgeValue) { 59 | dstVertices[index] = vertexId; 60 | edgeValues[index] = edgeValue; 61 | index++; 62 | } 63 | 64 | public int getNewOutEdge(int i) { 65 | return dstVertices[i]; 66 | } 67 | 68 | public byte getNewOutEdgeValue(int i) { 69 | return edgeValues[i]; 70 | } 71 | } 72 | 73 | public NewEdgesList() { 74 | last = first = null; 75 | size = 0; 76 | readableIndex = 0; 77 | readableSize = 0; 78 | } 79 | 80 | public int getSize() { 81 | return size; 82 | } 83 | 84 | public int getReadableSize() { 85 | return readableSize; 86 | } 87 | 88 | public int getReadableIndex() { 89 | return readableIndex; 90 | } 91 | 92 | public void setReadableSize(int readableSize) { 93 | this.readableSize = readableSize; 94 | } 95 | 96 | public void setReadableIndex(int readableIndex) { 97 | this.readableIndex = readableIndex; 98 | } 99 | 100 | public NewEdgesNode getFirst() { 101 | return first; 102 | } 103 | 104 | public NewEdgesNode getLast() { 105 | return last; 106 | } 107 | 108 | public NewEdgesNode getReadableLast() { 109 | return readableLast; 110 | } 111 | 112 | public void setReadableLast(NewEdgesNode readableLast) { 113 | this.readableLast = readableLast; 114 | } 115 | 116 | public int getIndex() { 117 | return last.getIndex(); 118 | } 119 | 120 | public void add(int vertexId, byte edgeValue) { 121 | if (size == 0 && first == null) { // empty linklist 122 | first = new NewEdgesNode(); 123 | size++; 124 | last = first; 125 | } else if (size == 0) { // nodes already allocated, data invalid, reuse 126 | // nodes 127 | size++; 128 | 129 | } else { 130 | int index = last.getIndex(); 131 | if (index >= NODE_SIZE) { 132 | if (last.next == null) 133 | last.next = new NewEdgesNode(); // modify for nodes reuse 134 | last = last.next; 135 | size++; 136 | } 137 | 138 | } 139 | 140 | last.add(vertexId, edgeValue); 141 | } 142 | 143 | public NewEdgesNode getNode(int index) { 144 | if (index == 0) 145 | return first; 146 | if (index > size) 147 | return null; 148 | 149 | NewEdgesNode node = first; 150 | for (int i = 0; i < index; i++) { 151 | node = node.next; 152 | } 153 | 154 | return node; 155 | } 156 | 157 | public void clear() { 158 | NewEdgesNode currentNode = first; 159 | if (first == null) 160 | return; 161 | 162 | for (int i = 0; i < size; i++) { 163 | currentNode.setIndex(0); 164 | currentNode = currentNode.next; 165 | } 166 | 167 | size = 0; 168 | first = last; 169 | } 170 | 171 | @Override 172 | public String toString() { 173 | if (size == 0) 174 | return "[]"; 175 | 176 | StringBuilder result = new StringBuilder(); 177 | String NEW_LINE = System.getProperty("line.separator"); 178 | NewEdgesNode temp = first; 179 | for (int i = 0; i < size; i++) { 180 | result.append(NEW_LINE + "edge list size : " + size); 181 | result.append(NEW_LINE + "dst vertex id : " 182 | + Arrays.toString(temp.getDstVertices())); 183 | // result.append(NEW_LINE + "edge value : " + 184 | // Arrays.toString(temp.getEdgeValues()));//TODO UNCOMMENT THIS 185 | // LATER 186 | temp = temp.next; 187 | } 188 | 189 | return result.toString(); 190 | } 191 | } 192 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/datastructures/LoadedPartitions.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.datastructures; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashSet; 5 | import java.util.LinkedHashSet; 6 | import java.util.List; 7 | 8 | /** 9 | * This class maintains all the variables used for loaded partitions. 10 | * 11 | * @author Aftab 12 | * 13 | */ 14 | public class LoadedPartitions { 15 | 16 | // Contains the ids of the partitions loaded in the memory 17 | private static int loadedParts[]; 18 | 19 | // Partitions to save to disk 20 | private static HashSet savePartsSet; 21 | 22 | // Loaded partitions that have been repartitioned 23 | private static HashSet repartitionedParts; 24 | 25 | // Contains the ids of the partitions that are to be loaded in the memory 26 | private static int newParts[]; 27 | 28 | // Stores the out degrees of each source vertex of each partition. 29 | // Dimension 1 indicates partition number, Dimension 2 indicates out degree 30 | // of a source vertex in the partition indicated by Index 1 31 | private static int loadedPartOutDegs[][]; 32 | 33 | // Data structures for storing the partitions to load: 34 | // Dimension 1 indicates partition number, Dimension 2 indicates list for a 35 | // source vertex, Dimension 3 indicates an out-edge from each source vertex 36 | private static int loadedPartEdges[][][]; 37 | private static byte loadedPartEdgeVals[][][]; 38 | 39 | //Loaded partition intervals 40 | public static List intervals = new ArrayList(); 41 | 42 | /** 43 | * 44 | * @return 45 | */ 46 | public static int[][][] getLoadedPartEdges() { 47 | return loadedPartEdges; 48 | } 49 | 50 | /** 51 | * 52 | * @param arr3d 53 | */ 54 | public static void setLoadedPartEdgeVals(byte[][][] arr3d) { 55 | loadedPartEdgeVals = arr3d; 56 | } 57 | 58 | /** 59 | * 60 | * @return byte[][][] loadedPartEdgeVals 61 | */ 62 | public static byte[][][] getLoadedPartEdgeVals() { 63 | return loadedPartEdgeVals; 64 | } 65 | 66 | /** 67 | * Saves the loaded partitions in 3D arrays. 68 | * 69 | * @param arr3d 70 | */ 71 | public static void setLoadedPartEdges(int[][][] arr3d) { 72 | loadedPartEdges = arr3d; 73 | } 74 | 75 | /** 76 | * 77 | * @return int[][] loadedPartOutDegs 78 | */ 79 | public static int[][] getLoadedPartOutDegs() { 80 | return loadedPartOutDegs; 81 | } 82 | 83 | /** 84 | * 85 | */ 86 | public static void printLoadedPartOutDegs() { 87 | System.out.println("*****"); 88 | for (int i = 0; i < loadedParts.length; i++) { 89 | System.out.println("Source vertex degrees for partition " + loadedParts[i]); 90 | for (int j = 0; j < PartitionQuerier.getNumUniqueSrcs(loadedParts[i]); j++) { 91 | System.out.println(loadedPartOutDegs[i][j]); 92 | } 93 | System.out.println(); 94 | } 95 | System.out.println("*****"); 96 | } 97 | 98 | /** 99 | * 100 | * @param arr2d 101 | */ 102 | public static void setLoadedPartOutDegs(int[][] arr2d) { 103 | loadedPartOutDegs = arr2d; 104 | } 105 | 106 | /** 107 | * 108 | * @return int[] loadedParts 109 | */ 110 | public static int[] getLoadedParts() { 111 | return loadedParts; 112 | } 113 | 114 | /** 115 | * 116 | * @param arr 117 | */ 118 | public static void setLoadedParts(int[] arr) { 119 | loadedParts = arr; 120 | } 121 | 122 | /** 123 | * 124 | * @return int[] newParts 125 | */ 126 | public static int[] getNewParts() { 127 | return newParts; 128 | } 129 | 130 | /** 131 | * 132 | * @param arr 133 | */ 134 | public static void setNewParts(int[] arr) { 135 | newParts = arr; 136 | } 137 | 138 | /** 139 | * Prints the set of partitions to load 140 | */ 141 | public static void printNewParts() { 142 | System.out.println("*****"); 143 | System.out.println("New partitions to load:"); 144 | for (int i = 0; i < newParts.length; i++) { 145 | System.out.println(newParts[i]); 146 | } 147 | System.out.println("*****"); 148 | } 149 | 150 | /** 151 | * Print the loaded partitions 152 | */ 153 | public static void printLoadedPartitions() { 154 | System.out.println("*****"); 155 | System.out.println("Loaded partitions:"); 156 | for (int i = 0; i < loadedParts.length; i++) { 157 | System.out.println("--- Partition: " + loadedParts[i] + " ---"); 158 | for (int j = 0; j < PartitionQuerier.getNumUniqueSrcs(loadedParts[i]); j++) { 159 | int srcv = j + PartitionQuerier.getFirstSrc(loadedParts[i]); 160 | System.out.println("SourceV: " + srcv); 161 | System.out.println("Dest Vs: "); 162 | for (int k = 0; k < loadedPartOutDegs[i][j]; k++) { 163 | System.out.print(loadedPartEdges[i][j][k] + " "); 164 | } 165 | System.out.println(); 166 | System.out.println("Edge Vals: "); 167 | for (int k = 0; k < loadedPartOutDegs[i][j]; k++) { 168 | System.out.print(loadedPartEdgeVals[i][j][k] + " "); 169 | } 170 | System.out.println(); 171 | } 172 | System.out.println(); 173 | } 174 | System.out.println("*****"); 175 | } 176 | 177 | public static void setPartsToSave(HashSet set) { 178 | savePartsSet = set; 179 | 180 | } 181 | 182 | public static HashSet getPartsToSave() { 183 | return savePartsSet; 184 | } 185 | 186 | public static void setRepartitionedParts(HashSet set) { 187 | repartitionedParts = set; 188 | 189 | } 190 | 191 | public static HashSet getRepartitionedParts() { 192 | return repartitionedParts; 193 | } 194 | 195 | 196 | 197 | } -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/graspan/oracle/NaiveComputer.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.graspan.oracle; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.BufferedWriter; 5 | import java.io.File; 6 | import java.io.FileReader; 7 | import java.io.FileWriter; 8 | import java.io.IOException; 9 | import java.io.PrintWriter; 10 | import java.util.HashMap; 11 | import java.util.HashSet; 12 | import java.util.Map; 13 | import java.util.Set; 14 | 15 | import edu.uci.ics.cs.graspan.computationM.GrammarChecker; 16 | 17 | /* 18 | * Naive Computer class using map 19 | */ 20 | public class NaiveComputer { 21 | 22 | private String fileName; 23 | 24 | private Map> graph; 25 | 26 | private long totalEdgesLoaded; 27 | 28 | private long totalNewEdgesAdded; 29 | 30 | 31 | public NaiveComputer(String fileName){ 32 | this.fileName = fileName; 33 | this.graph = new HashMap>(); 34 | } 35 | 36 | public static void main(String[] args) throws IOException { 37 | NaiveComputer c = new NaiveComputer(args[0]); 38 | c.run(); 39 | } 40 | 41 | public void run() throws IOException{ 42 | //load grammars 43 | GrammarChecker.loadGrammars(new File(this.fileName + ".grammar")); 44 | 45 | //load input graph 46 | loadGraph(); 47 | 48 | //add edges 49 | doComputation(); 50 | 51 | //write the final graph to disk 52 | export(); 53 | 54 | System.out.println("Total number of edges loaded:\t" + this.totalEdgesLoaded); 55 | System.out.println("Total number of edges added:\t" + this.totalNewEdgesAdded); 56 | } 57 | 58 | 59 | /** 60 | * write the final graph to disk 61 | */ 62 | private void export() { 63 | PrintWriter out = null; 64 | try { 65 | out = new PrintWriter(new BufferedWriter(new FileWriter(new File(this.fileName + ".final")))); 66 | for(int srcId: this.graph.keySet()){ 67 | for(Pair p: this.graph.get(srcId)){ 68 | out.println(srcId + "\t" + p.target + "\t" + GrammarChecker.getValue(p.evalue)); 69 | } 70 | } 71 | 72 | out.close(); 73 | } catch (IOException e) { 74 | // TODO Auto-generated catch block 75 | e.printStackTrace(); 76 | } 77 | 78 | } 79 | 80 | /** 81 | * 82 | */ 83 | private void doComputation() { 84 | boolean newAdded_flag = false; 85 | int iteration = 0; 86 | 87 | do{ 88 | //one iteration computation 89 | newAdded_flag = doComputationForOneIteration(iteration); 90 | iteration++; 91 | } 92 | while(newAdded_flag); 93 | 94 | //last iteration to guarantee real termination 95 | newAdded_flag = doComputationForOneIteration(iteration); 96 | assert(!newAdded_flag); 97 | } 98 | 99 | /** 100 | * add edges for one iteration 101 | * @param iteration number 102 | * @return 103 | */ 104 | private boolean doComputationForOneIteration(int iteration) { 105 | boolean newAdded = false; 106 | long number_added = 0; 107 | 108 | for(int srcId: this.graph.keySet()){ 109 | Set newAddedSet = new HashSet(); 110 | HashSet first_set = this.graph.get(srcId); 111 | 112 | for(Pair first_p: first_set){ 113 | int first_tgt = first_p.target; 114 | byte first_eval = first_p.evalue; 115 | 116 | //for length 1 rule checking 117 | byte new_eval_1 = GrammarChecker.checkL1Rules(first_eval); 118 | if(new_eval_1 != -1){ 119 | Pair newPair_1 = new Pair(first_tgt, new_eval_1); 120 | newAddedSet.add(newPair_1); 121 | // if(first_set.add(newPair_1)){ 122 | // newAdded = true; 123 | // number_added++; 124 | // } 125 | } 126 | 127 | //for length 2 rule checking 128 | if(this.graph.containsKey(first_tgt)){ 129 | HashSet second_set = this.graph.get(first_tgt); 130 | 131 | for(Pair second_p: second_set){ 132 | int second_tgt = second_p.target; 133 | byte second_eval = second_p.evalue; 134 | 135 | byte new_eval_2 = GrammarChecker.checkL2Rules(first_eval, second_eval); 136 | if(new_eval_2 != -1){ 137 | Pair newPair_2 = new Pair(second_tgt, new_eval_2); 138 | newAddedSet.add(newPair_2); 139 | // if(first_set.add(newPair)){ 140 | // newAdded = true; 141 | // number_added++; 142 | // } 143 | } 144 | } 145 | } 146 | } 147 | 148 | for(Pair np: newAddedSet){ 149 | if(first_set.add(np)){ 150 | newAdded = true; 151 | number_added++; 152 | } 153 | } 154 | } 155 | 156 | System.out.println("Number of edges added in Iteration " + iteration + ":\t" + number_added); 157 | this.totalNewEdgesAdded += number_added; 158 | 159 | return newAdded; 160 | } 161 | 162 | /** 163 | * load input graph into memory for further computation 164 | */ 165 | private void loadGraph() { 166 | // TODO Auto-generated method stub 167 | BufferedReader reader; 168 | String line; 169 | try { 170 | reader = new BufferedReader(new FileReader(new File(this.fileName))); 171 | while((line = reader.readLine()) != null){ 172 | if (!line.isEmpty()) { 173 | String[] tok = line.split("\t");//NOTE: MAKE SURE INPUT FILE IS TAB DELIMITED 174 | int srcId = Integer.parseInt(tok[0]); 175 | int tgtId = Integer.parseInt(tok[1]); 176 | byte eval = GrammarChecker.getValue(tok[2].trim()); 177 | 178 | this.totalEdgesLoaded++; 179 | if(this.graph.containsKey(srcId)){ 180 | boolean flag = this.graph.get(srcId).add(new Pair(tgtId, eval)); 181 | // assert(flag); 182 | } 183 | else{ 184 | HashSet set = new HashSet(); 185 | set.add(new Pair(tgtId, eval)); 186 | this.graph.put(srcId, set); 187 | } 188 | } 189 | } 190 | 191 | reader.close(); 192 | } catch (IOException e) { 193 | // TODO Auto-generated catch block 194 | e.printStackTrace(); 195 | } 196 | } 197 | 198 | 199 | static class Pair{ 200 | final int target; 201 | final byte evalue; 202 | 203 | public Pair(int t, byte eval){ 204 | this.target = t; 205 | this.evalue = eval; 206 | } 207 | 208 | public int hashCode(){ 209 | int r = 1; 210 | r = r * 31 + this.target; 211 | r = r * 31 + this.evalue; 212 | return r; 213 | } 214 | 215 | public boolean equals(Object o){ 216 | return (o instanceof Pair) && (((Pair) o).target == this.target) && (((Pair) o).evalue == this.evalue); 217 | } 218 | 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/dispatcher/GlobalParams.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.dispatcher; 2 | 3 | /** 4 | * 5 | * @author Aftab 1 November 2015 6 | */ 7 | public class GlobalParams { 8 | 9 | // input graph full file name and path 10 | public static String baseFilename = ""; 11 | 12 | // total number of partitions 13 | static int numParts; 14 | 15 | // in-memory computation 16 | public static boolean inMemComp; 17 | 18 | // number of partitions during each computation 19 | static final int NUM_PARTS_PER_COMPUTATION = 2; 20 | 21 | // The strategy for reloading partitions; 22 | // RELOAD_PLAN_1 - Reload all the requested partitions everytime, 23 | // regardless of which are already in the memory 24 | // RELOAD_PLAN_2 - Reload only the requested partitions that are not in 25 | // the memory. If a partition has been repartitioned, we consider it not to 26 | // be in the memory. 27 | // RELOAD_PLAN_3 - Reload only the requested partitions that are not in 28 | // the memory, however, if a partition has been repartitioned and a 29 | // requested partition is one of its child partitions, we keep the child 30 | // partition 31 | static String reloadPlan = ""; 32 | 33 | static String computation_logic = ""; 34 | 35 | private static final int GRAMMAR_SIZE = 200; 36 | 37 | // The grammar file 38 | private static byte[][] grammarTab = new byte[GRAMMAR_SIZE][3]; 39 | 40 | // The size of the Edge Destination Count Table 41 | private static int EdcSize; 42 | 43 | // Output edge tracker interval 44 | private static int OpEdgeTrackerInterval; 45 | 46 | // Size of each new edges node 47 | private static int NewEdgeNodeSize; 48 | 49 | // Maximum size of a partition after adding new edges 50 | private static long PartMaxPostNewEdges; 51 | 52 | // The type of input graph: DATAFLOW OR POINTS-TO (used to indicate whether 53 | // or not edge values exist) 54 | private static String hasEdgeVals; 55 | 56 | // The id of the first vertex of the graph 57 | private static int firstVId; 58 | 59 | // The number of threads 60 | private static int num_Threads; 61 | 62 | // The type of preprocessing operation: +eRULE or GenParts 63 | private static String ppOperation; 64 | 65 | // DATAFLOW or POINTSTO: if dataflow, then erules adding process is skipped 66 | private static String analysisType; 67 | 68 | private static int heapSize; 69 | 70 | private static double pSizeConst, repartConst; 71 | 72 | public static void setEdcSize(int num) { 73 | EdcSize = num; 74 | } 75 | 76 | public static int getEdcSize() { 77 | return EdcSize; 78 | } 79 | 80 | public static boolean inMemComp(){ 81 | return inMemComp; 82 | } 83 | 84 | public static void setInMemComp(boolean val){ 85 | inMemComp=val; 86 | } 87 | 88 | 89 | public static void setOpEdgeTrackerInterval(int num) { 90 | OpEdgeTrackerInterval = num; 91 | } 92 | 93 | public static int getOpEdgeTrackerInterval() { 94 | return OpEdgeTrackerInterval; 95 | } 96 | 97 | public static void setNewEdgesNodeSize(int num) { 98 | NewEdgeNodeSize = num; 99 | } 100 | 101 | public static int getNewEdgesNodeSize() { 102 | return NewEdgeNodeSize; 103 | } 104 | 105 | public static void setPartMaxPostNewEdges(long num) { 106 | PartMaxPostNewEdges = num; 107 | } 108 | 109 | public static long getPartMaxPostNewEdges() { 110 | return PartMaxPostNewEdges; 111 | } 112 | 113 | /** 114 | * 115 | * @return String baseFilename 116 | */ 117 | public static String getBasefilename() { 118 | return baseFilename; 119 | } 120 | 121 | /** 122 | * 123 | * @param str 124 | */ 125 | public static void setBasefilename(String str) { 126 | baseFilename = str; 127 | } 128 | 129 | /** 130 | * 131 | * @return int numParts 132 | */ 133 | public static int getNumParts() { 134 | return numParts; 135 | } 136 | 137 | /** 138 | * 139 | * @param n 140 | */ 141 | public static void setNumParts(int n) { 142 | numParts = n; 143 | } 144 | 145 | /** 146 | * 147 | * @return int numPartsPerComputation 148 | */ 149 | public static int getNumPartsPerComputation() { 150 | return NUM_PARTS_PER_COMPUTATION; 151 | } 152 | 153 | /** 154 | * 155 | * @return 156 | */ 157 | public static String getReloadPlan() { 158 | return reloadPlan; 159 | } 160 | 161 | /** 162 | * 163 | * @param str 164 | */ 165 | public static void setReloadPlan(String str) { 166 | reloadPlan = str; 167 | } 168 | 169 | /** 170 | * 171 | * @return 172 | */ 173 | public static byte[][] getGrammarTab() { 174 | return grammarTab; 175 | } 176 | 177 | /** 178 | * 179 | * @param str 180 | */ 181 | public static void setGrammarTab(byte[][] arr) { 182 | grammarTab = arr; 183 | } 184 | 185 | public static String hasEdgeVals() { 186 | return hasEdgeVals; 187 | } 188 | 189 | 190 | public static void setHasEdgeVals(String hasedgvals) { 191 | hasEdgeVals = hasedgvals; 192 | 193 | } 194 | 195 | public static int getFirstVertexID() { 196 | return firstVId; 197 | } 198 | 199 | public static void setFirstVertexID(int id) { 200 | firstVId = id; 201 | } 202 | 203 | public static void setComputationLogic(String compLogic) { 204 | computation_logic = compLogic; 205 | } 206 | 207 | public static String getComputationLogic() { 208 | return computation_logic; 209 | } 210 | 211 | public static void setNumThreads(int numThreads) { 212 | num_Threads=numThreads; 213 | } 214 | 215 | public static int getNumThreads() { 216 | return num_Threads; 217 | } 218 | 219 | public static void setPPOperation(String ppOP) { 220 | ppOperation = ppOP; 221 | } 222 | 223 | public static String getPPOperation(){ 224 | return ppOperation; 225 | } 226 | 227 | public static void setHeapSize(int hs) { 228 | heapSize=hs; 229 | } 230 | 231 | public static int getHeapSize(){ 232 | return heapSize; 233 | } 234 | 235 | public static void setRepartConst(double rc){ 236 | repartConst=rc; 237 | } 238 | 239 | public static double getRepartConst(){ 240 | return repartConst; 241 | } 242 | 243 | public static void setPSizeConst(double pc){ 244 | pSizeConst=pc; 245 | } 246 | 247 | public static double getPSizeConst(){ 248 | return pSizeConst; 249 | } 250 | 251 | } 252 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/graspan/oracle/GraphPartComparator.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.graspan.oracle; 2 | 3 | import java.io.BufferedInputStream; 4 | import java.io.BufferedReader; 5 | import java.io.DataInputStream; 6 | import java.io.File; 7 | import java.io.FileInputStream; 8 | import java.io.FileReader; 9 | import java.io.IOException; 10 | import java.util.HashMap; 11 | import java.util.HashSet; 12 | import java.util.Map; 13 | 14 | public class GraphPartComparator { 15 | 16 | int numParts; 17 | String basefilename; 18 | 19 | private Map> gph_NaiveCompOP; 20 | private Map> gph_EngineOP; 21 | 22 | private Map> gph_NaiveDiffEngine; 23 | private Map> gph_EngineDiffNaive; 24 | 25 | private long totalEdgesLoaded_EngineOutput; 26 | private long totalEdgesLoaded_NaiveCompGraph; 27 | private long totalEdges_gph_NaiveDiffEngine; 28 | private long totalEdges_gph_EngineDiffNaive; 29 | 30 | public GraphPartComparator(String basefilename, int numParts) { 31 | this.basefilename = basefilename; 32 | this.numParts = numParts; 33 | this.gph_NaiveCompOP = new HashMap>(); 34 | this.gph_EngineOP = new HashMap>(); 35 | 36 | } 37 | 38 | public static void main(String args[]) throws IOException { 39 | 40 | GraphPartComparator gpc = new GraphPartComparator(args[0], Integer.parseInt(args[1])); 41 | gpc.run(); 42 | 43 | } 44 | 45 | private void run() throws IOException { 46 | 47 | loadNaiveCompGraphOP(); 48 | System.out.println("loaded naive computer graph"); 49 | 50 | for (int i = 0; i < numParts; i++) { 51 | loadPartsOPfromEngine(i); 52 | } 53 | System.out.println("loaded engine computer graph"); 54 | 55 | 56 | gph_EngineDiffNaive = genGraphDiff(gph_EngineOP, gph_NaiveCompOP, 57 | "engine"); 58 | System.out.println("found gph_EngineDiffNaive"); 59 | 60 | gph_NaiveDiffEngine = genGraphDiff(gph_NaiveCompOP, gph_EngineOP, 61 | "naive"); 62 | System.out.println("found gph_NaiveDiffEngine"); 63 | 64 | System.out.println("totalEdges_EngineOutput: " + totalEdgesLoaded_EngineOutput); 65 | System.out.println("totalEdges_NaiveCompGraph: " + totalEdgesLoaded_NaiveCompGraph); 66 | System.out.println("totalEdges_gph_NaiveDiffEngine: " + totalEdges_gph_NaiveDiffEngine); 67 | System.out.println("totalEdges_gph_EngineDiffNaive: " + totalEdges_gph_EngineDiffNaive); 68 | 69 | System.out.println("gph_EngineDiffNaive :-"); 70 | 71 | for (Integer src : gph_EngineDiffNaive.keySet()) { 72 | HashSet dstValPairs = gph_EngineDiffNaive.get(src); 73 | for (Pair pr : dstValPairs){ 74 | System.out.println(src + "\t" + pr.target + "\t" + pr.evalue); 75 | } 76 | } 77 | 78 | System.out.println("gph_NaiveDiffEngine :-"); 79 | 80 | for (Integer src : gph_NaiveDiffEngine.keySet()) { 81 | HashSet dstValPairs = gph_NaiveDiffEngine.get(src); 82 | for (Pair pr : dstValPairs) { 83 | System.out.println(src + "\t" + pr.target + "\t" + pr.evalue); 84 | } 85 | } 86 | 87 | } 88 | 89 | private Map> genGraphDiff(Map> mapA, Map> mapB, 90 | String firstgraph) { 91 | 92 | Map> diffMap = new HashMap>(); 93 | 94 | for (int srcA : mapA.keySet()) { 95 | HashSet dstValPairsA = mapA.get(srcA); 96 | HashSet dstValPairsB = mapB.get(srcA); 97 | 98 | HashSet dstValPairsA_copy = new HashSet(); 99 | HashSet dstValPairsB_copy = new HashSet(); 100 | 101 | for (Pair pr:dstValPairsA){ 102 | dstValPairsA_copy.add(pr); 103 | } 104 | 105 | if (dstValPairsB!=null){ 106 | for (Pair pr:dstValPairsB){ 107 | dstValPairsB_copy.add(pr); 108 | } 109 | 110 | dstValPairsA_copy.removeAll(dstValPairsB_copy); 111 | } 112 | diffMap.put(srcA, dstValPairsA_copy); 113 | 114 | if (firstgraph.compareTo("naive") == 0) { 115 | this.totalEdges_gph_NaiveDiffEngine += dstValPairsA_copy.size(); 116 | } 117 | if (firstgraph.compareTo("engine") == 0) { 118 | this.totalEdges_gph_EngineDiffNaive += dstValPairsA_copy.size(); 119 | 120 | } 121 | } 122 | return diffMap; 123 | 124 | 125 | 126 | 127 | } 128 | 129 | /** 130 | * load input graph into memory for further computation 131 | */ 132 | private void loadNaiveCompGraphOP() { 133 | BufferedReader reader; 134 | String line; 135 | try { 136 | reader = new BufferedReader(new FileReader(new File(this.basefilename + ".finalOracleOutput"))); 137 | while ((line = reader.readLine()) != null) { 138 | if (!line.isEmpty()) { 139 | String[] tok = line.split("\t");// NOTE: MAKE SURE INPUT 140 | // FILE IS TAB DELIMITED 141 | int srcId = Integer.parseInt(tok[0]); 142 | int tgtId = Integer.parseInt(tok[1]); 143 | byte eval = Byte.parseByte(tok[2]); 144 | 145 | this.totalEdgesLoaded_NaiveCompGraph++; 146 | if (this.gph_NaiveCompOP.containsKey(srcId)) { 147 | boolean flag = this.gph_NaiveCompOP.get(srcId).add(new Pair(tgtId, eval)); 148 | assert (flag); 149 | } else { 150 | HashSet set = new HashSet(); 151 | set.add(new Pair(tgtId, eval)); 152 | this.gph_NaiveCompOP.put(srcId, set); 153 | } 154 | } 155 | } 156 | 157 | reader.close(); 158 | } catch (IOException e) { 159 | e.printStackTrace(); 160 | } 161 | } 162 | 163 | /** 164 | * load input partition into memory for further computation 165 | * 166 | * @throws IOException 167 | */ 168 | private void loadPartsOPfromEngine(int partId) throws IOException { 169 | DataInputStream dataIn = new DataInputStream( 170 | new BufferedInputStream(new FileInputStream(basefilename + ".partition." + partId))); 171 | while (true) { 172 | try { 173 | 174 | int srcVId = dataIn.readInt(); 175 | 176 | // count (number of destVs from srcV in the current list) 177 | int count = dataIn.readInt(); 178 | 179 | for (int i = 0; i < count; i++) { 180 | int dstVId = dataIn.readInt(); 181 | byte eval = dataIn.readByte(); 182 | 183 | this.totalEdgesLoaded_EngineOutput++; 184 | if (this.gph_EngineOP.containsKey(srcVId)) { 185 | boolean flag = this.gph_EngineOP.get(srcVId).add(new Pair(dstVId, eval)); 186 | assert (flag); 187 | } else { 188 | HashSet set = new HashSet(); 189 | set.add(new Pair(dstVId, eval)); 190 | this.gph_EngineOP.put(srcVId, set); 191 | } 192 | 193 | } 194 | 195 | } catch (Exception exception) { 196 | break; 197 | } 198 | } 199 | dataIn.close(); 200 | 201 | } 202 | 203 | static class Pair { 204 | final int target; 205 | final byte evalue; 206 | 207 | public Pair(int t, byte eval) { 208 | this.target = t; 209 | this.evalue = eval; 210 | } 211 | 212 | public int hashCode() { 213 | int r = 1; 214 | r = r * 31 + this.target; 215 | r = r * 31 + this.evalue; 216 | return r; 217 | } 218 | 219 | public boolean equals(Object o) { 220 | return (o instanceof Pair) && (((Pair) o).target == this.target) && (((Pair) o).evalue == this.evalue); 221 | } 222 | 223 | } 224 | 225 | } 226 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/scheduler/Scheduler.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.scheduler; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.logging.Logger; 6 | 7 | import edu.uci.ics.cs.graspan.datastructures.LoadedVertexInterval; 8 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 9 | 10 | /** 11 | * @author Kai Wang 12 | * 13 | * Created by Oct 30, 2015 14 | */ 15 | public class Scheduler { 16 | private List allEdgeInfo = new ArrayList(); 17 | private List partitionNumEdges = new ArrayList(); 18 | private List intervals = null; 19 | private static final Logger logger = GraspanLogger.getLogger("Scheduler"); 20 | private boolean premature_terminate_status; 21 | 22 | // for temp use 23 | public static int counterOne = 0; 24 | public static int counterTwo = 1; 25 | 26 | /** 27 | * Constructor 1. 28 | * 29 | * @param numParts 30 | */ 31 | public Scheduler(int numParts) { 32 | for (int i = 0; i < numParts; i++) { 33 | PartitionEdgeInfo edgeInfo = new PartitionEdgeInfo(i, numParts); 34 | allEdgeInfo.add(edgeInfo); 35 | } 36 | 37 | // // 38 | // Scheduler Output 39 | // StringBuilder result = new StringBuilder(); 40 | // String NEW_LINE = System.getProperty("line.separator"); 41 | // for (int i = 0; i < 50; i++) { 42 | // result.append(NEW_LINE + i + " : "); 43 | // for (int j = 0; j < 50; j++) { 44 | // result.append("(" + j + "," + SchedulerInfo.getEdgeDestCount()[i][j] 45 | // + ") "); 46 | // } 47 | // } 48 | // logger.info("Edge Dest Count" + result); 49 | // // 50 | } 51 | 52 | /** 53 | * Constructor 2. (unused) 54 | * 55 | * @param partSizes 56 | * @param edgeDstCount 57 | */ 58 | // public Scheduler(long[][] edgeDstCount) { 59 | // if (SchedulerInfo.getPartSizes() == null || edgeDstCount == null) 60 | // throw new IllegalArgumentException("Null parameter in scheduler!"); 61 | // 62 | // for (int i = 0; i < SchedulerInfo.getPartSizes().length; i++) { 63 | // partitionNumEdges.add(i, SchedulerInfo.getPartSizes()[i][1]); 64 | // } 65 | // 66 | // for (int i = 0; i < edgeDstCount.length; i++) { 67 | // PartitionEdgeInfo edgeInfo = new PartitionEdgeInfo(i, 68 | // edgeDstCount[i]); 69 | // allEdgeInfo.add(edgeInfo); 70 | // } 71 | // } 72 | 73 | public void setLoadedIntervals(List intervals) { 74 | this.intervals = intervals; 75 | } 76 | 77 | public List getLoadedIntervals() { 78 | return intervals; 79 | } 80 | 81 | 82 | /** 83 | * Sequential Scheduler (Basic) 84 | * 85 | * @param numOfPartitions 86 | * @return 87 | */ 88 | public int[] schedulePartitionSimple(int numOfPartitions) { 89 | logger.info("Scheduling using Simple Scheduler."); 90 | 91 | int[] scheduled = new int[2]; 92 | 93 | // schedule two partitions every time 94 | scheduled[0] = counterOne; 95 | scheduled[1] = counterTwo++; 96 | 97 | if (counterTwo >= numOfPartitions) { 98 | counterOne++; 99 | counterTwo = counterOne + 1; 100 | } 101 | 102 | if (counterOne >= numOfPartitions - 1) { 103 | counterOne = 0; 104 | counterTwo = counterOne + 1; 105 | } 106 | return scheduled; 107 | } 108 | 109 | /** 110 | * EDC-Based Scheduler 111 | * 112 | * @return 113 | */ 114 | public int[] schedulePartitionEDC(int numOfPartitions) { 115 | logger.info("Scheduling using EDC-based Scheduler"); 116 | // int partA = 0, partB = 0; 117 | // long partA_Size = 0; 118 | // long partB_Size = 0; 119 | // double[][] edcPercentage = SchedulerInfo.getEdcPercentage(); 120 | 121 | // for (int i = 0; i < numOfPartitions; i++) { 122 | // partA = i; 123 | 124 | // find size of partA 125 | // for (int j = 0; j < SchedulerInfo.getPartSizes().length; j++) { 126 | // if (SchedulerInfo.getPartSizes()[j][0] == partA) { 127 | // partA_Size = SchedulerInfo.getPartSizes()[j][1]; 128 | // break; 129 | // } 130 | // } 131 | 132 | // for (int j = 0; j < numOfPartitions; j++) { 133 | 134 | // OPTION 1----- 135 | // edcPercentage[i][j] = (double) SchedulerInfo.getEdgeDestCount()[i][j] / partA_Size; 136 | // the remaining implementation needs to be changed if you want 137 | // to 138 | // implement this one 139 | // ------------- 140 | 141 | // OPTION 2----- 142 | //<> 143 | // find size of partB 144 | // partB = j; 145 | // for (int k = 0; k < SchedulerInfo.getPartSizes().length; k++) 146 | // { 147 | // if (SchedulerInfo.getPartSizes()[k][0] == partB) { 148 | // partB_Size = SchedulerInfo.getPartSizes()[k][1]; 149 | // break; 150 | // } 151 | // } 152 | // 153 | // edcPercentage[i][j] = (double) 154 | // SchedulerInfo.getEdgeDestCount()[i][j] 155 | // / partA_Size 156 | // + (double) SchedulerInfo.getEdgeDestCount()[j][i] 157 | // / partB_Size; 158 | // ------------- 159 | // } 160 | // } 161 | 162 | // find max value in edctwoWay 163 | double max = -1; 164 | int maxPartA = -1, maxPartB = -1; 165 | long[][] edcTwoWay = SchedulerInfo.getEdcTwoWay(); 166 | for (int i = 0; i < numOfPartitions; i++) { 167 | for (int j = 0; j < numOfPartitions; j++) { 168 | // logger.info("hello world" + edcPercentage[i][j] + " " + max); 169 | // if (edcPercentage[i][j] > max && i != j) { 170 | if (edcTwoWay[i][j]>max && i!=j) { 171 | List terminationInfoForOne = allEdgeInfo.get(i).getTerminationInfo(); 172 | if (terminationInfoForOne.get(j) == false) { 173 | // max = edcPercentage[i][j]; 174 | max = edcTwoWay[i][j]; 175 | maxPartA = i; 176 | maxPartB = j; 177 | } 178 | } 179 | } 180 | } 181 | 182 | // schedule two partitions every time 183 | int[] scheduled = new int[2]; 184 | scheduled[0] = maxPartA; 185 | scheduled[1] = maxPartB; 186 | 187 | return scheduled; 188 | } 189 | 190 | /** 191 | * 192 | * Description:check whether to be terminated. call it before every 193 | * iteration. 194 | * 195 | * @param: 196 | * @return:boolean 197 | */ 198 | public boolean shouldTerminate() { 199 | int size = allEdgeInfo.size(); 200 | 201 | // logger.info(this+" Scheduler Termination Map"); 202 | 203 | for (int i = 0; i < allEdgeInfo.size(); i++) { 204 | int partitionId = allEdgeInfo.get(i).getPartitionId(); 205 | List terminationInfo = allEdgeInfo.get(i).getTerminationInfo(); 206 | for (int j = (partitionId + 1); j < size; j++) { 207 | if (!terminationInfo.get(j)) 208 | return false; 209 | } 210 | } 211 | 212 | // // 213 | // Scheduler Output 214 | // StringBuilder result = new StringBuilder(); 215 | // String NEW_LINE = System.getProperty("line.separator"); 216 | // for (int i = 0; i < 50; i++) { 217 | // result.append(NEW_LINE + i + " : "); 218 | // for (int j = 0; j < 50; j++) { 219 | // result.append("(" + j + "," + SchedulerInfo.getEdgeDestCount()[i][j] 220 | // + ") "); 221 | // } 222 | // } 223 | // logger.info("Edge Dest Count" + result); 224 | // // 225 | 226 | logger.info("=================Terminate!================="); 227 | return true; 228 | } 229 | 230 | /** 231 | * 232 | * Description: set termination status every time after computation. 233 | * 234 | * @param: 235 | * @return:void 236 | */ 237 | public void setTerminationStatus() { 238 | logger.info("\nintervals : " + intervals); 239 | assert (intervals.size() == 2); 240 | 241 | int loadedPartitionOne = intervals.get(0).getPartitionId(); 242 | // boolean isNewEdgeAddedForOne = intervals.get(0).hasNewEdges();//ORIGINAL 243 | boolean isNewEdgeAddedForOne = intervals.get(0).hasNewEdgesInCurrentRound(); 244 | int loadedPartitionTwo = intervals.get(1).getPartitionId(); 245 | // boolean isNewEdgeAddedForTwo = intervals.get(1).hasNewEdges();//ORIGINAL 246 | boolean isNewEdgeAddedForTwo = intervals.get(1).hasNewEdgesInCurrentRound();; 247 | 248 | logger.info("partitionId 1: " + loadedPartitionOne + ". New edges : "+ isNewEdgeAddedForOne); 249 | logger.info("partitionId 2: " + loadedPartitionTwo + ". New edges : "+ isNewEdgeAddedForTwo); 250 | 251 | List terminationInfoForOne = allEdgeInfo.get(loadedPartitionOne).getTerminationInfo(); 252 | List terminationInfoForTwo = allEdgeInfo.get(loadedPartitionTwo).getTerminationInfo(); 253 | 254 | if (isNewEdgeAddedForOne) { 255 | // set the row to false 256 | for (int i = 0; i < terminationInfoForOne.size(); i++) 257 | terminationInfoForOne.set(i, false); 258 | 259 | // set the column to false 260 | for (int i = 0; i < allEdgeInfo.size(); i++) 261 | { 262 | // allEdgeInfo.get(i).getTerminationInfo().set(loadedPartitionTwo, false); ORIGINAL 263 | allEdgeInfo.get(i).getTerminationInfo().set(loadedPartitionOne, false); 264 | } 265 | } 266 | 267 | if (isNewEdgeAddedForTwo) { 268 | // set the row to false 269 | for (int i = 0; i < terminationInfoForTwo.size(); i++) 270 | terminationInfoForTwo.set(i, false); 271 | 272 | // set the column to false 273 | for (int i = 0; i < allEdgeInfo.size(); i++) 274 | { 275 | // allEdgeInfo.get(i).getTerminationInfo().set(loadedPartitionOne, false); ORIGINAL 276 | allEdgeInfo.get(i).getTerminationInfo().set(loadedPartitionTwo, false); 277 | } 278 | } 279 | 280 | if (this.getPrematureTerminationStatus()==false){ 281 | terminationInfoForOne.set(loadedPartitionTwo, true); 282 | terminationInfoForTwo.set(loadedPartitionOne, true); 283 | } 284 | 285 | } 286 | 287 | /** 288 | * update termination status after repartitioning 289 | * 290 | * @param numOfNewPartitions 291 | * @param totalPartitions 292 | */ 293 | public void updateSchedInfoPostRepart(int numOfNewPartitions, 294 | int totalPartitions) { 295 | 296 | // for each of the old partitions, add "false" for each new partition 297 | for (PartitionEdgeInfo info : allEdgeInfo) { 298 | for (int i = 0; i < numOfNewPartitions; i++) { 299 | info.getTerminationInfo().add(Boolean.FALSE); 300 | } 301 | } 302 | 303 | // create records for each new partition created 304 | int numOfOldPartitions = totalPartitions - numOfNewPartitions; 305 | for (int i = 0; i < numOfNewPartitions; i++) { 306 | PartitionEdgeInfo newInfo = new PartitionEdgeInfo(numOfOldPartitions + i, totalPartitions); 307 | allEdgeInfo.add(newInfo); 308 | } 309 | } 310 | 311 | @Override 312 | public String toString() { 313 | StringBuilder result = new StringBuilder(); 314 | String NEW_LINE = System.getProperty("line.separator"); 315 | result.append(NEW_LINE + "number of partitions : " + allEdgeInfo.size()); 316 | for (PartitionEdgeInfo info : allEdgeInfo) 317 | result.append(NEW_LINE + info.toString()); 318 | 319 | return result.toString(); 320 | } 321 | 322 | public void setPrematureTerminationStatus(boolean status){ 323 | premature_terminate_status=status; 324 | } 325 | public boolean getPrematureTerminationStatus(){ 326 | return premature_terminate_status; 327 | } 328 | } -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/computationM/EdgeComputerM.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.computationM; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | import java.util.HashSet; 6 | import java.util.List; 7 | import java.util.logging.Logger; 8 | 9 | import edu.uci.ics.cs.graspan.datastructures.ComputationSet; 10 | import edu.uci.ics.cs.graspan.datastructures.LoadedVertexInterval; 11 | import edu.uci.ics.cs.graspan.datastructures.Vertex; 12 | import edu.uci.ics.cs.graspan.dispatcher.GlobalParams; 13 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 14 | 15 | /** 16 | * @author Aftab Hussain 17 | * 18 | * Created by Mar 18, 2016 19 | */ 20 | public class EdgeComputerM { 21 | 22 | private static final Logger logger = GraspanLogger.getLogger("EdgeComputer"); 23 | 24 | /** 25 | * 26 | * @param i 27 | * @param compSets 28 | * @param intervals 29 | * @return 30 | */ 31 | public static long execUpdate(int i, ComputationSet[] compSets, List intervals) { 32 | 33 | 34 | ComputationSet compSet = compSets[i]; 35 | 36 | // 1. get the compSet components 37 | // 1.1. we shall scan these to get the targets 38 | int[] oldEdgs = compSet.getOldEdgs(); 39 | byte[] oldVals = compSet.getOldVals(); 40 | 41 | int[] newEdgs = compSet.getNewEdgs(); 42 | byte[] newVals = compSet.getNewVals(); 43 | 44 | boolean oldEdgs_empty = true, newEdgs_empty = true; 45 | if (oldEdgs.length != 0) { 46 | oldEdgs_empty = false; 47 | } 48 | if (newEdgs.length != 0) { 49 | newEdgs_empty = false; 50 | } 51 | 52 | // 1.2. if there is nothing to merge, return 53 | if (oldEdgs_empty && newEdgs_empty) 54 | return 0; 55 | 56 | // if (vertices[i].getVertexId() == 4017) { 57 | // logger.info("oldEdgs" + Arrays.toString(oldEdgs)); 58 | // logger.info("oldVals" + Arrays.toString(oldVals)); 59 | // logger.info("newEdgs" + Arrays.toString(newEdgs)); 60 | // logger.info("newVals" + Arrays.toString(newVals)); 61 | // } 62 | 63 | // 2. get the rows to merge 64 | HashSet newRowIndicesToMerge = new HashSet(); 65 | HashSet oldUnewRowIndicesToMerge = new HashSet(); 66 | 67 | // 2.1. get the indices of the new_components to merge 68 | getRowIndicesToMerge(compSets, intervals, oldEdgs, oldVals, oldEdgs_empty, newRowIndicesToMerge, "old"); 69 | 70 | // 2.2. get the indices of the oldUnew_components to merge 71 | getRowIndicesToMerge(compSets, intervals, newEdgs, newVals, newEdgs_empty, oldUnewRowIndicesToMerge, "new"); 72 | 73 | // if (vertices[i].getVertexId() == 4017) { 74 | // for (IdValuePair ivp : newRowIndicesToMerge) { 75 | // logger.info("newRowIndicesToMerge - " + ivp.id + " " + ivp.value);//+ " The vertex"+vertices[ivp.id]); 76 | // } 77 | // for (IdValuePair ivp : oldUnewRowIndicesToMerge) { 78 | // logger.info("oldUnewRowIndicesToMerge - " + ivp.id + " " + ivp.value);//+" The vertex"+vertices[ivp.id]); 79 | // } 80 | // } 81 | 82 | int num_of_rows_to_merge = 2 + oldUnewRowIndicesToMerge.size() + newRowIndicesToMerge.size(); 83 | // 3. store the refs to rows in edgArrstoMerge & valArrstoMerge 84 | int[][] edgArrstoMerge = new int[num_of_rows_to_merge][]; 85 | byte[][] valArrstoMerge = new byte[num_of_rows_to_merge][]; 86 | // 3.1. first store the source row 87 | int rows_to_merge_id = 0; 88 | edgArrstoMerge[0] = compSet.getOldUnewEdgs(); 89 | valArrstoMerge[0] = compSet.getOldUnewVals(); 90 | rows_to_merge_id++; 91 | 92 | //for singleton rule 93 | rows_to_merge_id = genEdgesToMergeForSRule(newEdgs, newVals, edgArrstoMerge, valArrstoMerge, rows_to_merge_id); //TODO: NEED TO FIX. IDENTIFIED AS GC-EXPENSIVE BY YOURKIT. 94 | 95 | //for length 2 rule 96 | rows_to_merge_id = genEdgesToMerge(compSets, newRowIndicesToMerge, edgArrstoMerge, valArrstoMerge, rows_to_merge_id, "old"); 97 | rows_to_merge_id = genEdgesToMerge(compSets, oldUnewRowIndicesToMerge, edgArrstoMerge, valArrstoMerge, rows_to_merge_id, "new"); 98 | 99 | 100 | //------------------------------------------------------------------------------- 101 | // 4. call the SortedArrMerger merge function 102 | SortedArrMerger sortedArrMerger = new SortedArrMerger(); 103 | // logger.info("Vertex ID: " + this.vertex.getVertexId()); 104 | int srcRowId = 0; 105 | sortedArrMerger.mergeTgtstoSrc(edgArrstoMerge, valArrstoMerge, srcRowId); //TODO: NEED TO FIX. IDENTIFIED AS GC-EXPENSIVE BY YOURKIT. 106 | 107 | //------------------------------------------------------------------------------- 108 | compSet.setDeltaEdges(sortedArrMerger.get_src_delta_edgs()); 109 | compSet.setDeltaVals(sortedArrMerger.get_src_delta_vals()); 110 | compSet.setOldUnewUdeltaEdgs(sortedArrMerger.get_src_oldUnewUdelta_edgs()); 111 | compSet.setOldUnewUdeltaVals(sortedArrMerger.get_src_oldUnewUdelta_vals()); 112 | 113 | // get number of new edges 114 | return sortedArrMerger.get_num_new_edges(); 115 | 116 | } 117 | 118 | 119 | /** 120 | * 121 | * @param newEdgs 122 | * @param newVals 123 | * @param edgArrstoMerge 124 | * @param valArrstoMerge 125 | * @param rows_to_merge_id 126 | * @return 127 | */ 128 | private static int genEdgesToMergeForSRule(int[] newEdgs, byte[] newVals, int[][] edgArrstoMerge, byte[][] valArrstoMerge, int rows_to_merge_id) { 129 | //get the resulting edge and value array 130 | List list = new ArrayList();//TODO: NEED TO FIX. IDENTIFIED AS GC-EXPENSIVE BY YOURKIT. 131 | 132 | int dstId; 133 | byte dstVal,newVal; 134 | 135 | for(int i = 0; i < newEdgs.length; i++){ 136 | // int dstId = newEdgs[i]; 137 | dstId = newEdgs[i]; 138 | 139 | // byte dstVal = newVals[i]; 140 | dstVal = newVals[i]; 141 | 142 | // byte newVal = GrammarChecker.checkL1Rules(dstVal); 143 | newVal = GrammarChecker.checkL1Rules(dstVal); 144 | 145 | if(newVal != -1){ 146 | list.add(new IdValuePair(dstId, newVal)); 147 | } 148 | } 149 | 150 | // if(!list.isEmpty()){ 151 | int[] newEdgeArray = new int[list.size()]; 152 | byte[] newValArray = new byte[list.size()]; 153 | IdValuePair ele; 154 | for(int i = 0; i < list.size(); i++){ 155 | // IdValuePair ele = list.get(i); 156 | ele = list.get(i); 157 | newEdgeArray[i] = ele.id; 158 | newValArray[i] = ele.value; 159 | } 160 | 161 | edgArrstoMerge[rows_to_merge_id] = newEdgeArray; 162 | valArrstoMerge[rows_to_merge_id] = newValArray; 163 | rows_to_merge_id++; 164 | // } 165 | 166 | return rows_to_merge_id; 167 | } 168 | 169 | 170 | /** 171 | * 172 | * @param compSets 173 | * @param idsToMerge 174 | * @param edgArrstoMerge 175 | * @param valArrstoMerge 176 | * @param rows_to_merge_id 177 | * @param flag 178 | * @return 179 | */ 180 | private static int genEdgesToMerge(ComputationSet[] compSets, HashSet idsToMerge, 181 | int[][] edgArrstoMerge, byte[][] valArrstoMerge, int rows_to_merge_id, String flag) { 182 | 183 | int index; 184 | int[] edges; 185 | 186 | byte srcVal; 187 | byte[] vals; 188 | 189 | for(IdValuePair pair: idsToMerge){ 190 | // int index = pair.id; 191 | index = pair.id; 192 | // byte srcVal = pair.value; 193 | srcVal = pair.value; 194 | 195 | // 196 | // int[] edges = null; 197 | edges = null; 198 | // byte[] vals = null; 199 | vals = null; 200 | 201 | if(flag.equals("old")){ 202 | edges = compSets[index].getNewEdgs(); 203 | vals = compSets[index].getNewVals(); 204 | } 205 | else if(flag.equals("new")){ 206 | edges = compSets[index].getOldUnewEdgs(); 207 | vals = compSets[index].getOldUnewVals(); 208 | } 209 | 210 | //get the resulting edge and value array 211 | List list = new ArrayList(); 212 | for(int i = 0; i < edges.length; i++){ 213 | int dstId = edges[i]; 214 | byte dstVal = vals[i]; 215 | 216 | byte newVal = GrammarChecker.checkL2Rules(srcVal, dstVal);// use this for general TC 217 | // byte newVal = 0; //use this for pure TC 218 | 219 | 220 | if(newVal != -1){ 221 | list.add(new IdValuePair(dstId, newVal)); 222 | } 223 | } 224 | 225 | // if(!list.isEmpty()){ 226 | int[] newEdgeArray = new int[list.size()]; 227 | byte[] newValArray = new byte[list.size()]; 228 | IdValuePair ele; 229 | for(int i = 0; i < list.size(); i++){ 230 | // IdValuePair ele = list.get(i); 231 | ele = list.get(i); 232 | newEdgeArray[i] = ele.id; 233 | newValArray[i] = ele.value; 234 | } 235 | 236 | edgArrstoMerge[rows_to_merge_id] = newEdgeArray; 237 | valArrstoMerge[rows_to_merge_id] = newValArray; 238 | rows_to_merge_id++; 239 | // } 240 | } 241 | 242 | return rows_to_merge_id; 243 | } 244 | 245 | // private static byte checkGrammarAndGetNewEdgeVal(byte edgeVal1, byte edgeVal2) { 246 | // 247 | // byte[][] grammarTab = GlobalParams.getGrammarTab(); 248 | // byte edgeVal3 = -1; 249 | // for (int i = 0; i < grammarTab.length; i++) { 250 | // if (grammarTab[i][0] == edgeVal1 && grammarTab[i][1] == edgeVal2) { 251 | // edgeVal3 = grammarTab[i][2]; 252 | // break; 253 | // } 254 | // } 255 | // return edgeVal3; 256 | // } 257 | 258 | /** 259 | * 260 | * @param compSets 261 | * @param intervals 262 | * @param edgs 263 | * @param vals 264 | * @param edgs_empty 265 | * @param newIdsToMerge 266 | * @param flag 267 | */ 268 | private static void getRowIndicesToMerge(ComputationSet[] compSets, List intervals, int[] edgs, byte[] vals, boolean edgs_empty, HashSet newIdsToMerge, String flag) { 269 | int targetRowIndex = -1; 270 | LoadedVertexInterval interval; 271 | 272 | int newTgt = 0; 273 | byte val = 0; 274 | 275 | if (!edgs_empty) { 276 | for (int i = 0; i < edgs.length; i++) { 277 | newTgt = edgs[i]; 278 | val = vals[i]; 279 | 280 | for (int j = 0; j < intervals.size(); j++) { 281 | targetRowIndex = -1; 282 | interval = intervals.get(j); 283 | if (newTgt >= interval.getFirstVertex() && newTgt <= interval.getLastVertex()) { // chk if newTgt is among loaded src vertices 284 | targetRowIndex = newTgt - interval.getFirstVertex() + interval.getIndexStart(); 285 | assert (targetRowIndex != -1); 286 | break; 287 | } 288 | } 289 | if (targetRowIndex == -1) 290 | continue; 291 | 292 | if((flag.equals("old") && compSets[targetRowIndex].getNewEdgs().length > 0) 293 | || (flag.equals("new") && compSets[targetRowIndex].getOldUnewEdgs().length > 0)){ 294 | newIdsToMerge.add(new IdValuePair(targetRowIndex, val)); 295 | } 296 | // if (edgs[i] != this.vertex.getVertexId()) { 297 | // } 298 | 299 | } 300 | } 301 | } 302 | 303 | static class IdValuePair { 304 | private final int id; 305 | private final byte value; 306 | 307 | IdValuePair(int id, byte value) { 308 | this.id = id; 309 | this.value = value; 310 | } 311 | 312 | public int hashCode() { 313 | int r = 1; 314 | r = r * 31 + this.id; 315 | r = r * 31 + this.value; 316 | return r; 317 | } 318 | 319 | public boolean equals(Object o) { 320 | return (o instanceof IdValuePair) && (((IdValuePair) o).id == id) && (((IdValuePair) o).value == value); 321 | } 322 | } 323 | 324 | } -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/dispatcher/PreprocessorClient.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.dispatcher; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.BufferedWriter; 5 | import java.io.File; 6 | import java.io.FileInputStream; 7 | import java.io.FileWriter; 8 | import java.io.IOException; 9 | import java.io.InputStreamReader; 10 | import java.io.PrintWriter; 11 | import java.util.logging.Logger; 12 | 13 | import edu.uci.ics.cs.graspan.computationM.GrammarChecker; 14 | import edu.uci.ics.cs.graspan.datastructures.AllPartitions; 15 | import edu.uci.ics.cs.graspan.preproc.GraphERuleEdgeAdderAndSorter; 16 | import edu.uci.ics.cs.graspan.preproc.PartERuleAdderAndSorter; 17 | import edu.uci.ics.cs.graspan.preproc.Preprocessor; 18 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 19 | import edu.uci.ics.cs.graspan.support.Utilities; 20 | 21 | /** 22 | * This program performs preprocessing of the input graph to generate partitions 23 | * 24 | * @author Aftab 25 | * 26 | */ 27 | public class PreprocessorClient { 28 | 29 | private static final Logger logger = GraspanLogger.getLogger("PreprocessorClient"); 30 | 31 | public static void main(String[] args) throws IOException { 32 | 33 | String preprocessorConfigFilename = args[0]; 34 | 35 | /* 36 | * for scanning main.config and pp.pgen.config 37 | */ 38 | BufferedReader preprocessorConfigStream = new BufferedReader(new InputStreamReader(new FileInputStream(new File(preprocessorConfigFilename)))); 39 | String ln; 40 | 41 | GlobalParams.setReloadPlan("RELOAD_PLAN_2"); 42 | GlobalParams.setComputationLogic("SMART_MERGE"); 43 | GlobalParams.setEdcSize(1000); 44 | GlobalParams.setBasefilename(args[1]); 45 | 46 | String[] tok; 47 | while ((ln = preprocessorConfigStream.readLine()) != null) { 48 | tok = ln.split(" "); 49 | // if (tok[0].compareTo("GRAPH_PATH") == 0) { 50 | // GlobalParams.setBasefilename(tok[2].trim());//use when running through EclipseIDE TODO 51 | // } 52 | if (tok[0].compareTo("TOTAL_NUM_PARTS") == 0) { 53 | GlobalParams.setNumParts(Integer.parseInt(tok[2])); 54 | } 55 | if (tok[0].compareTo("HAS_EDGE_VALS") == 0) { 56 | GlobalParams.setHasEdgeVals(tok[2].trim()); 57 | } 58 | // if (tok[0].compareTo("RELOAD_PLAN") == 0) { 59 | // GlobalParams.setReloadPlan(tok[2]); 60 | // } 61 | // if (tok[0].compareTo("EDC_SIZE") == 0) { 62 | // GlobalParams.setEdcSize(Integer.parseInt(tok[2])); 63 | // } 64 | // NEED TO ENSURE INPUT GRAPH NUMBERING STARTS FROM 1 OR 0 65 | if (tok[0].compareTo("1ST_VTX_ID") == 0) { 66 | GlobalParams.setFirstVertexID(Integer.parseInt(tok[2])); 67 | } 68 | if (tok[0].compareTo("PREPROCESSING_OPERATION")==0){ 69 | GlobalParams.setPPOperation(tok[2].trim()); 70 | } 71 | 72 | // THE FOLLOWING ARE NOT USED FOR PREPROCESSING, BUT USED TO GENERATE THE CONFIG FILE 73 | // FOR THE COMPUTATION 74 | if (tok[0].compareTo("HEAP_SIZE(GB)")==0){ 75 | GlobalParams.setHeapSize(Integer.parseInt(tok[2].trim())); 76 | } 77 | if (tok[0].compareTo("PART_SIZE_CONST")==0){ 78 | GlobalParams.setPSizeConst(Double.parseDouble(tok[2].trim())); 79 | } 80 | if (tok[0].compareTo("REPART_SIZE_CONST")==0){ 81 | GlobalParams.setRepartConst(Double.parseDouble(tok[2].trim())); 82 | } 83 | // if (tok[0].compareTo("COMPUTATION_LOGIC") == 0) { 84 | // GlobalParams.setComputationLogic(tok[2]); 85 | // } 86 | if (tok[0].compareTo("NUM_OF_THREADS") == 0) { 87 | GlobalParams.setNumThreads(Integer.parseInt(tok[2])); 88 | } 89 | if (tok[0].compareTo("MAX_PART_SIZE_POST_NEW_EDGES") == 0) { 90 | GlobalParams.setPartMaxPostNewEdges(Integer.parseInt(tok[2])); 91 | } 92 | // if (tok[0].compareTo("") == 0) { 93 | // break; 94 | // } 95 | } 96 | 97 | preprocessorConfigStream.close(); 98 | 99 | // MemUsageCheckThread memUsage = new MemUsageCheckThread(); 100 | // memUsage.start(); 101 | 102 | logger.info("Input graph: " + GlobalParams.getBasefilename()); 103 | logger.info("Requested # partitions to generate: " + GlobalParams.getNumParts()); 104 | 105 | GrammarChecker.loadGrammars(new File(GlobalParams.getBasefilename() + ".grammar")); 106 | 107 | if (GlobalParams.getPPOperation().compareTo("erule") == 0) 108 | // adding edges to the graph based on eRules 109 | { 110 | logger.info("PREPROCESSING: Start computing and adding edges from eRules..."); 111 | // GraspanTimer ppERedgeAdding = new GraspanTimer(System.currentTimeMillis()); 112 | long eAdd_start = System.currentTimeMillis(); 113 | 114 | 115 | GraphERuleEdgeAdderAndSorter edgeAdder = new GraphERuleEdgeAdderAndSorter(); 116 | edgeAdder.run(); 117 | 118 | logger.info("PREPROCESSING: Finished computing and adding edges from eRules."); 119 | 120 | // ppERedgeAdding.calculateDuration(System.currentTimeMillis()); 121 | // logger.info("Edge Adding from Erules took: "+ GraspanTimer.getDurationInHMS(ppERedgeAdding.getDuration())); 122 | logger.info("Edge Adding from Erules took: " + Utilities.getDurationInHMS(System.currentTimeMillis() - eAdd_start)); 123 | 124 | 125 | /* 126 | * Generating pp.pgen.config file for partition generation process 127 | */ 128 | 129 | double gSize = getSizeOfGraphWithEREdgs_MB(); 130 | logger.info("sizeOfGraphWithEREdgs(MB)"+gSize); 131 | 132 | double pSize_expected = getExpectedPartSize_MB(); 133 | logger.info("getExpectedPartSize_MB " + pSize_expected); 134 | 135 | double gSize_to_pSizeExpected = (double) gSize/pSize_expected; 136 | logger.info("gSize_to_pSizeExpected " + gSize_to_pSizeExpected); 137 | 138 | int numparts_torequest = getNumOfPartsToRequest(gSize_to_pSizeExpected); 139 | logger.info("numparts_torequest " + numparts_torequest); 140 | 141 | double pSize_asPerNumparts_torequest = (double)gSize/numparts_torequest; 142 | logger.info("pSize_asPerNumparts_torequest " + pSize_asPerNumparts_torequest); 143 | 144 | double pSize_postNewEdges_MB = getPSize_postNewEdges_MB(pSize_asPerNumparts_torequest); 145 | logger.info("pSize_postNewEdges_MB " + pSize_postNewEdges_MB); 146 | 147 | double pSize_postNewEdges_inNumOfEdges = pSize_postNewEdges_MB * 50000;//1MB of Graph roughly equal to 50000 edges on disk 148 | logger.info("pSize_postNewEdges_inNumOfEdges "+pSize_postNewEdges_inNumOfEdges); 149 | 150 | printValstoPgenconfFile(numparts_torequest, pSize_postNewEdges_inNumOfEdges); 151 | 152 | logger.info("Generated pp.pgen.config"); 153 | 154 | } 155 | 156 | else if (GlobalParams.getPPOperation().compareTo("genparts") == 0) 157 | // generate pp parts 158 | { 159 | logger.info("PREPROCESSING: Start generating partitions..."); 160 | // GraspanTimer ppPartGen = new GraspanTimer(System.currentTimeMillis()); 161 | long pp_start = System.currentTimeMillis(); 162 | 163 | // initialize Partition Generator Program 164 | Preprocessor partgenerator = new Preprocessor(GlobalParams.getBasefilename(), GlobalParams.getNumParts()); 165 | partgenerator.run(); 166 | 167 | logger.info("PREPROCESSING: Finished generating partitions."); 168 | 169 | // ppPartGen.calculateDuration(System.currentTimeMillis()); 170 | // logger.info("Generating partitions took: "+ GraspanTimer.getDurationInHMS(ppPartGen.getDuration())); 171 | logger.info("Generating partitions took: " + Utilities.getDurationInHMS(System.currentTimeMillis()-pp_start)); 172 | 173 | /* 174 | * Generating comp.config file for computation process 175 | */ 176 | 177 | printValstoCompconfFile(); 178 | 179 | logger.info("Generated comp.config"); 180 | 181 | //TODO: TEMPORARY CODE: TO REMOVE: 182 | // PartitionPreprocessor partProc = new PartitionPreprocessor(); 183 | // for (int i=0;i (AUTOGENERATED)"); 197 | compconfStrm.println(); 198 | compconfStrm.println("INPUT_GRAPH_FILEPATH = " + GlobalParams.getBasefilename()); 199 | compconfStrm.println("TOTAL_NUM_PARTS = " + AllPartitions.getPartAllocTab().length); 200 | compconfStrm.println("MAX_PART_SIZE_POST_NEW_EDGES = " + GlobalParams.getPartMaxPostNewEdges()); 201 | compconfStrm.println("RELOAD_PLAN = "+GlobalParams.getReloadPlan()); 202 | compconfStrm.println("EDC_SIZE = " + GlobalParams.getEdcSize()); 203 | compconfStrm.println("COMPUTATION_LOGIC = " + GlobalParams.getComputationLogic()); 204 | compconfStrm.println("NUM_OF_THREADS = " + GlobalParams.getNumThreads()); 205 | compconfStrm.println(); 206 | compconfStrm.println(""); 207 | 208 | compconfStrm.close(); 209 | } 210 | 211 | private static void printValstoPgenconfFile(int numparts_torequest, double pSize_postNewEdges_inNumOfEdges) throws IOException { 212 | PrintWriter pgenconfStrm; 213 | pgenconfStrm = new PrintWriter(new BufferedWriter(new FileWriter("pp.pgen.config", false))); 214 | 215 | pgenconfStrm.println(" (AUTOGENERATED)"); 216 | pgenconfStrm.println(); 217 | 218 | pgenconfStrm.println("INPUT_GRAPH_FILEPATH = " + GlobalParams.getBasefilename() + ".eRulesAdded"); 219 | pgenconfStrm.println("TOTAL_NUM_PARTS = " + numparts_torequest); 220 | pgenconfStrm.println("HAS_EDGE_VALS = " + GlobalParams.hasEdgeVals()); 221 | pgenconfStrm.println("INPUT_GRAPH_NUMBERING_STARTS_FROM = " + GlobalParams.getFirstVertexID()); 222 | pgenconfStrm.println("MAX_PART_SIZE_POST_NEW_EDGES = " + Math.round(pSize_postNewEdges_inNumOfEdges)); 223 | pgenconfStrm.println("PREPROCESSING_OPERATION = genparts"); 224 | pgenconfStrm.println("RELOAD_PLAN = "+GlobalParams.getReloadPlan()); 225 | pgenconfStrm.println("EDC_SIZE = " + GlobalParams.getEdcSize()); 226 | pgenconfStrm.println("COMPUTATION_LOGIC = " + GlobalParams.getComputationLogic()); 227 | pgenconfStrm.println("NUM_OF_THREADS = " + GlobalParams.getNumThreads()); 228 | pgenconfStrm.println(); 229 | pgenconfStrm.println(""); 230 | 231 | pgenconfStrm.close(); 232 | } 233 | 234 | private static double getPSize_postNewEdges_MB(double pSize_asPerNumparts_torequest) { 235 | double pSize_postNewEdges_MB = pSize_asPerNumparts_torequest + 236 | ((double)pSize_asPerNumparts_torequest * GlobalParams.getRepartConst()); 237 | return pSize_postNewEdges_MB; 238 | } 239 | 240 | private static int getNumOfPartsToRequest(double gSize_to_pSizeExpected) { 241 | int numparts_torequest=0; 242 | if (gSize_to_pSizeExpected<=2){ 243 | numparts_torequest = 2; 244 | } 245 | else { 246 | numparts_torequest = (int) Math.ceil(gSize_to_pSizeExpected); 247 | } 248 | return numparts_torequest; 249 | } 250 | 251 | private static double getExpectedPartSize_MB() { 252 | double pSize_expected = (GlobalParams.getPSizeConst()*GlobalParams.getHeapSize()*1000)/2; 253 | return pSize_expected; 254 | } 255 | 256 | private static double getSizeOfGraphWithEREdgs_MB() { 257 | File graph = new File(GlobalParams.getBasefilename()+".eRulesAdded"); 258 | double gSize = (double)graph.length()/1000000; 259 | return gSize; 260 | } 261 | 262 | } -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/graspan/oracle/WholeGraphComputer.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.graspan.oracle; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.BufferedWriter; 5 | import java.io.File; 6 | import java.io.FileInputStream; 7 | import java.io.FileWriter; 8 | import java.io.IOException; 9 | import java.io.InputStreamReader; 10 | import java.io.PrintWriter; 11 | import java.util.logging.Logger; 12 | 13 | import edu.uci.ics.cs.graspan.computationM.GrammarChecker; 14 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 15 | import edu.uci.ics.cs.graspan.support.Utilities; 16 | 17 | 18 | /* 19 | * Naive Computer Class using arrays 20 | */ 21 | public class WholeGraphComputer { 22 | 23 | private static final Logger logger = GraspanLogger.getLogger("DTC-Oracle"); 24 | private static final int EDGE_BUFFER_SIZE = 5000; 25 | private static int FIRST_VERTEX_ID = 0; 26 | 27 | public static int gph[][] = new int[EDGE_BUFFER_SIZE][2]; 28 | public static byte vals[]=new byte[EDGE_BUFFER_SIZE]; 29 | public static int newEdgesMain[][] = new int[EDGE_BUFFER_SIZE][2]; 30 | public static byte newValsMain[] = new byte[EDGE_BUFFER_SIZE]; 31 | 32 | // static int iterationNewEdges[][] = new int[EDGE_BUFFER_SIZE][2]; 33 | // static byte iterationNewVals[] = new byte[EDGE_BUFFER_SIZE]; 34 | 35 | public static void main(String args[]) throws IOException { 36 | 37 | String baseFilename = init(args); 38 | 39 | //load grammar 40 | GrammarChecker.loadGrammars(new File(baseFilename + ".grammar")); 41 | 42 | // read in the original graph 43 | BufferedReader ins = new BufferedReader(new InputStreamReader(new FileInputStream(new File(baseFilename)))); 44 | String ln; 45 | int i = 0; 46 | while ((ln = ins.readLine()) != null) { 47 | if (!ln.isEmpty()) { 48 | String[] tok = ln.split("\t");//NOTE: MAKE SURE INPUT FILE IS TAB DELIMITED 49 | 50 | if (FIRST_VERTEX_ID == 1) { 51 | gph[i][0] = Integer.parseInt(tok[0]); 52 | gph[i][1] = Integer.parseInt(tok[1]); 53 | vals[i] = GrammarChecker.getValue(tok[2].trim()); 54 | i++; 55 | } else if (FIRST_VERTEX_ID == 0) { 56 | gph[i][0] = Integer.parseInt(tok[0])+1; 57 | gph[i][1] = Integer.parseInt(tok[1])+1; 58 | vals[i] = GrammarChecker.getValue(tok[2].trim()); 59 | i++; 60 | } 61 | 62 | } 63 | } 64 | 65 | logger.info("Finished reading original graph into memory."); 66 | 67 | int nextGphPos = i; 68 | logger.info("# edges in original graph: " + nextGphPos); 69 | 70 | //--------------------------------------------------------------------------------------------------------------- 71 | //BEGINNING COMPUTATION 72 | 73 | 74 | int iterationNo = 1; 75 | 76 | int[][] graphToCompute = gph; 77 | byte[] valsToCompute = vals; 78 | 79 | boolean isNewEdgeAdded = false; 80 | do { 81 | // printGraph(); 82 | logger.info("Computation iteration #: " + iterationNo); 83 | isNewEdgeAdded = performComputation(graphToCompute, valsToCompute); 84 | logger.info("New edges added in this iteration?: " + isNewEdgeAdded); 85 | nextGphPos = transferNewEdgestoOriginal(nextGphPos); 86 | iterationNo++; 87 | } while (isNewEdgeAdded); 88 | 89 | // printGraph(); 90 | storeActualEdges(baseFilename); 91 | 92 | } 93 | 94 | private static int transferNewEdgestoOriginal(int nextGphPos) { 95 | for (int j = 0; j < newEdgesMain.length; j++) { 96 | if (newEdgesMain[j][0] == 0 | newEdgesMain[j][1] == 0) 97 | break; 98 | gph[nextGphPos][0] = newEdgesMain[j][0]; 99 | gph[nextGphPos][1] = newEdgesMain[j][1]; 100 | vals[nextGphPos]=newValsMain[j]; 101 | 102 | newEdgesMain[j][0] = 0; 103 | newEdgesMain[j][1] = 0; 104 | newValsMain[j] = 0; 105 | 106 | nextGphPos++; 107 | } 108 | return nextGphPos; 109 | } 110 | 111 | private static String init(String[] args) { 112 | String baseFilename = args[0]; 113 | logger.info("Input graph: " + args[0]); 114 | 115 | // initialize the graph data structures with -1 116 | // for (int i = 0; i < gph.length; i++) { 117 | // gph[i][0] = -1; 118 | // gph[i][1] = -1; 119 | // vals[i]=-1; 120 | // } 121 | // for (int i = 0; i < newEdgesMain.length; i++) { 122 | // newEdgesMain[i][0] = -1; 123 | // newEdgesMain[i][1] = -1; 124 | // newValsMain[i]=-1; 125 | // } 126 | logger.info("Completed initialization of graph data structures."); 127 | return baseFilename; 128 | } 129 | 130 | // public static void performComputationL1Rule(int[][] gph, byte[] vals) { 131 | // int newEdges[][] = new int[EDGE_BUFFER_SIZE][2]; 132 | // byte newVals[] = new byte[EDGE_BUFFER_SIZE]; 133 | // int newEdgesMarker = 0; 134 | // 135 | // // initializing newEdges 136 | // for (int i = 0; i < newEdges.length; i++) { 137 | // newEdges[i][0] = -1; 138 | // newEdges[i][1] = -1; 139 | // newVals[i] = -1; 140 | // } 141 | // 142 | // int candidateEdgeV1, candidateEdgeV2; 143 | // byte edgeVal, OPEval; 144 | // 145 | // for (int j = 0; j < gph.length; j++) { 146 | // 147 | // candidateEdgeV1 = gph[j][0]; 148 | // candidateEdgeV2 = gph[j][1]; 149 | // edgeVal = vals[j]; 150 | // 151 | // OPEval = GrammarChecker.checkL1Rules(edgeVal); 152 | // 153 | // if (OPEval==-1) continue; 154 | // 155 | // boolean edgeExists = false; 156 | // 157 | // // check whether this edge already exists in the original graph 158 | // edgeExists = isInGraph(gph, vals, candidateEdgeV1, candidateEdgeV2, OPEval, edgeExists); 159 | // 160 | // // check whether edge exists in the new edges 161 | // edgeExists = isInGraph(newEdges, newVals, candidateEdgeV1, candidateEdgeV2, OPEval, edgeExists); 162 | // 163 | // if (!edgeExists) { 164 | // 165 | // // logger.info("New edge found: " + candidateEdgeV1 + "---->" + candidateEdgeV2 + "("+ OPEval+")"); 166 | // 167 | // // add the edge 168 | // newEdges[newEdgesMarker][0] = candidateEdgeV1; 169 | // newEdges[newEdgesMarker][1] = candidateEdgeV2; 170 | // newVals[newEdgesMarker]=OPEval; 171 | // newEdgesMarker++; 172 | // } 173 | // 174 | // } 175 | // newEdgesMain = newEdges; 176 | // newValsMain = newVals; 177 | // } 178 | 179 | public static boolean performComputation(int[][] gph, byte[] vals) { 180 | // int iterationNewEdges[][] = new int[EDGE_BUFFER_SIZE][2]; 181 | // byte iterationNewVals[] = new byte[EDGE_BUFFER_SIZE]; 182 | int newEdgesMarker = 0; 183 | 184 | // resetting iterationNewEdges 185 | // for (int i = 0; i < newEdgesMain.length; i++) { 186 | // newEdgesMain[i][0] = -1; 187 | // newEdgesMain[i][1] = -1; 188 | // newValsMain[i] = -1; 189 | // } 190 | 191 | 192 | 193 | boolean isNewEdgeAdded = false; 194 | int candidateEdgeV1, candidateEdgeV2; 195 | byte srcEval, destEval, OPEval; 196 | for (int j = 0; j < gph.length; j++) { 197 | 198 | if (gph[j][0] == 0) 199 | break; 200 | 201 | candidateEdgeV1 = gph[j][0]; 202 | candidateEdgeV2 = gph[j][1]; 203 | srcEval = vals[j]; 204 | 205 | 206 | OPEval = GrammarChecker.checkL1Rules(srcEval); 207 | logger.info("checked L1 Rules for edge " + candidateEdgeV1+" ----> "+candidateEdgeV2+" : "+srcEval); 208 | 209 | if (OPEval!=-1) { 210 | 211 | boolean edgeExists = false; 212 | 213 | // check whether this edge already exists in the original graph 214 | edgeExists = isInGraph(gph, vals, candidateEdgeV1, candidateEdgeV2, OPEval, edgeExists); 215 | 216 | // check whether edge exists in the new edges 217 | edgeExists = isInGraph(newEdgesMain, newValsMain, candidateEdgeV1, candidateEdgeV2, OPEval, edgeExists); 218 | 219 | if (!edgeExists) { 220 | 221 | // logger.info("New edge found: " + candidateEdgeV1 + "---->" + candidateEdgeV2 + "("+ OPEval+")"); 222 | 223 | // add the edge 224 | newEdgesMain[newEdgesMarker][0] = candidateEdgeV1; 225 | newEdgesMain[newEdgesMarker][1] = candidateEdgeV2; 226 | newValsMain[newEdgesMarker]=OPEval; 227 | newEdgesMarker++; 228 | isNewEdgeAdded = true; 229 | } 230 | 231 | } 232 | 233 | for (int k = 0; k < gph.length; k++) { 234 | 235 | if (gph[k][0] == 0) 236 | break; 237 | 238 | if (gph[j][1] == gph[k][0] && gph[j][1] != 0 && gph[k][0] != 0) { 239 | candidateEdgeV1 = gph[j][0]; 240 | candidateEdgeV2 = gph[k][1]; 241 | 242 | srcEval=vals[j]; 243 | destEval=vals[k]; 244 | 245 | 246 | // check whether pair-wise grammar rule (d-rule) is satisfied 247 | OPEval = GrammarChecker.checkL2Rules(srcEval, destEval); 248 | 249 | // logger.info("L2 Rule Check "+ OPEval+" "+srcEval+" "+destEval); 250 | 251 | if (OPEval==-1) continue; 252 | 253 | boolean edgeExists = false; 254 | 255 | // check whether this edge already exists in the original graph 256 | edgeExists = isInGraph(gph, vals, candidateEdgeV1, candidateEdgeV2, OPEval, edgeExists); 257 | 258 | // check whether edge exists in the new edges 259 | edgeExists = isInGraph(newEdgesMain, newValsMain, candidateEdgeV1, candidateEdgeV2, OPEval, edgeExists); 260 | 261 | if (!edgeExists) { 262 | // logger.info("New edge found: " + candidateEdgeV1 + "---->" + candidateEdgeV2 + "("+ OPEval+")"); 263 | 264 | // add the edge 265 | newEdgesMain[newEdgesMarker][0] = candidateEdgeV1; 266 | newEdgesMain[newEdgesMarker][1] = candidateEdgeV2; 267 | newValsMain[newEdgesMarker]=OPEval; 268 | newEdgesMarker++; 269 | isNewEdgeAdded = true; 270 | } 271 | } 272 | } 273 | } 274 | 275 | logger.info("# new edges added in this iteration: " + newEdgesMarker); 276 | // newEdgesMain = iterationNewEdges; 277 | // newValsMain = iterationNewVals; 278 | // if (newEdgesMarker != 0) // if new edges have been added 279 | // { 280 | // printNewEdges(); 281 | // } 282 | return isNewEdgeAdded; 283 | } 284 | 285 | private static void storeActualEdges(String basefilename) throws IOException { 286 | // clear current graph file 287 | // PrintWriter partOutStrm = new PrintWriter(new BufferedWriter( new FileWriter(basefilename+".finaloutput" , false))); 288 | // partOutStrm.close(); 289 | 290 | PrintWriter partOutStrm = new PrintWriter(new BufferedWriter(new FileWriter(basefilename+".finalOracleOutput" , true))); 291 | 292 | for (int i = 0; i < gph.length; i++) { 293 | if (gph[i][0] == 0) 294 | break; 295 | 296 | //use for storing edge string vals 297 | // partOutStrm.println(gph[i][0] + "\t" + gph[i][1]+ "\t" + GrammarChecker.getValue((byte)vals[i])); 298 | 299 | //use for storing edge byte vals 300 | partOutStrm.println(gph[i][0] + "\t" + gph[i][1]+ "\t" + vals[i]); 301 | } 302 | partOutStrm.close(); 303 | } 304 | 305 | 306 | /** 307 | * Checks whether an edge exists in a set of edges 308 | * @param gph 309 | * @param vals 310 | * @param candidateEdgeV1 311 | * @param candidateEdgeV2 312 | * @param OPEval 313 | * @param edgeExists 314 | * @return 315 | */ 316 | private static boolean isInGraph(int[][] gph, byte[] vals, int candidateEdgeV1, int candidateEdgeV2, byte OPEval,boolean edgeExists) { 317 | if (edgeExists) { 318 | return true; 319 | } 320 | for (int m = 0; m < gph.length; m++) { 321 | if (gph[m][0] == 0) { 322 | break; 323 | } 324 | if (candidateEdgeV1 == gph[m][0] && candidateEdgeV2 == gph[m][1] && OPEval==vals[m]) { 325 | // logger.info("Edge already exists: " + 326 | // candidateEdgeV1 + "---->" + candidateEdgeV2); 327 | edgeExists = true; 328 | break; 329 | } 330 | } 331 | return edgeExists; 332 | } 333 | 334 | /** 335 | * Prints the whole graph 336 | */ 337 | public static void printGraph() { 338 | // input vertex 339 | int v = 8; 340 | 341 | String s = ""; 342 | int numOfEdges = 0; 343 | for (int i = 0; i < gph.length; i++) { 344 | if (gph[i][0] == 0) 345 | break; 346 | // if (gph[i][0]==v){//only prints the graph for the input vertex 347 | s = s + gph[i][0] + "\t" + gph[i][1] + "\n"; 348 | // numOfEdges++; 349 | } 350 | // } 351 | // logger.info("The complete graph showing only edges for vertex " + v 352 | // + " is (" + numOfEdges + " edges): " + "\n" + s); 353 | } 354 | 355 | /** 356 | * Prints the new edges 357 | */ 358 | public static void printNewEdges() { 359 | String s = ""; 360 | for (int i = 0; i < newEdgesMain.length; i++) { 361 | if (newEdgesMain[i][0] == 0) 362 | break; 363 | s = s + newEdgesMain[i][0] + "\t" + newEdgesMain[i][1] + "\n"; 364 | } 365 | logger.info("New edges added:" + "\n" + s); 366 | } 367 | 368 | } 369 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/computationEL/EdgeComputerEL.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.computationEL; 2 | 3 | import java.util.List; 4 | import java.util.logging.Logger; 5 | 6 | import edu.uci.ics.cs.graspan.datastructures.LoadedVertexInterval; 7 | import edu.uci.ics.cs.graspan.datastructures.NewEdgesList; 8 | import edu.uci.ics.cs.graspan.datastructures.Vertex; 9 | import edu.uci.ics.cs.graspan.dispatcher.GlobalParams; 10 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 11 | 12 | /** 13 | * @author Kai Wang 14 | * 15 | * Created by Oct 8, 2015 16 | */ 17 | public class EdgeComputerEL { 18 | 19 | private static final Logger logger = GraspanLogger.getLogger("EdgeComputer"); 20 | 21 | // test flag for viewing new edges of a source 22 | public static int flag = 0; 23 | String s = ""; 24 | 25 | private Vertex vertex = null; 26 | private NewEdgesList edgeList = null; 27 | private int nNewEdges, nDupEdges; 28 | private boolean terminateStatus; 29 | private static NewEdgesList[] edgesLists = null; 30 | private static Vertex[] vertices = null; 31 | private static List intervals = null; 32 | 33 | public EdgeComputerEL(Vertex vertex, NewEdgesList edgeList) { 34 | this.vertex = vertex; 35 | this.edgeList = edgeList; 36 | } 37 | 38 | public int getNumNewEdges() { 39 | return nNewEdges; 40 | } 41 | 42 | public void setNumNewEdges(int nNewEdges) { 43 | this.nNewEdges = nNewEdges; 44 | } 45 | 46 | public int getNumDupEdges() { 47 | return nDupEdges; 48 | } 49 | 50 | public void setNumDupEdges(int nDupEdges) { 51 | this.nDupEdges = nDupEdges; 52 | } 53 | 54 | public boolean getTerminateStatus() { 55 | return terminateStatus; 56 | } 57 | 58 | public void setTerminateStatus(boolean terminateStatus) { 59 | this.terminateStatus = terminateStatus; 60 | } 61 | 62 | public static void setEdgesLists(NewEdgesList[] lists) { 63 | edgesLists = lists; 64 | } 65 | 66 | public static void setVertices(Vertex[] v) { 67 | vertices = v; 68 | } 69 | 70 | public static void setIntervals(List vertexIntervals) { 71 | intervals = vertexIntervals; 72 | } 73 | 74 | public void execUpdate() { 75 | if (vertex == null || vertices == null) 76 | return; 77 | 78 | int nOutEdges = vertex.getNumOutEdges(); 79 | if (nOutEdges == 0) 80 | return; 81 | 82 | // get edges of vertex 2 / test code 1 83 | if (vertex.getVertexId() == 2) 84 | flag = 1; 85 | 86 | // 1. check edges which are stored in vertex outEdges array 87 | for (int i = 0; i < nOutEdges; i++) { 88 | 89 | // get dst vertex id and edge value from array 90 | int vertexId = vertex.getOutEdge(i); 91 | byte edgeValue = vertex.getOutEdgeValue(i); 92 | 93 | // get edges of vertex 2 / test code 2 94 | // if (flag == 1) { 95 | // s = 2 + " ---> " + vertexId; 96 | // System.out.println(s); 97 | // } 98 | 99 | // check vertex range and scan edges 100 | checkRangeAndScanEdges(vertexId, edgeValue); 101 | 102 | } 103 | 104 | // 2. check new edges linked array 105 | if (edgeList == null) 106 | return; 107 | int readableSize = edgeList.getReadableSize(); 108 | if (readableSize == 0) 109 | return; 110 | 111 | int[] ids = null; 112 | byte[] values = null; 113 | 114 | // 2.1 check (readableSize - 1) node, each node is full of NODE_SIZE 115 | // elements 116 | for (int j = 0; j < readableSize - 1; j++) { 117 | ids = edgeList.getNode(j).getDstVertices(); 118 | values = edgeList.getNode(j).getEdgeValues(); 119 | assert (ids.length == NewEdgesList.NODE_SIZE); 120 | 121 | for (int k = 0; k < ids.length; k++) { 122 | 123 | // check vertex range and scan edges 124 | checkRangeAndScanEdges(ids[k], values[k]); 125 | 126 | } 127 | } 128 | 129 | // 2.2 check the last node, the num of elements is index 130 | ids = edgeList.getReadableLast().getDstVertices(); 131 | values = edgeList.getReadableLast().getEdgeValues(); 132 | int readableIndex = edgeList.getReadableIndex(); 133 | for (int m = 0; m < readableIndex; m++) { 134 | 135 | // check vertex range and scan edges 136 | checkRangeAndScanEdges(ids[m], values[m]); 137 | 138 | } 139 | } 140 | 141 | /** 142 | * Description: 143 | * 144 | * @param: 145 | * @return: boolean 146 | */ 147 | // TODO: to be optimized 148 | private boolean isInRange(int vertexId) { 149 | for (LoadedVertexInterval interval : intervals) { 150 | int intervalSt = interval.getFirstVertex(); 151 | int intervalEnd = interval.getLastVertex(); 152 | if (vertexId >= intervalSt && vertexId <= intervalEnd) 153 | return true; 154 | } 155 | 156 | return false; 157 | } 158 | 159 | /** 160 | * Description: 161 | * 162 | * @param: 163 | * @return: 164 | */ 165 | private void checkRangeAndScanEdges(int vertexId, byte edgeValue) { 166 | if (isInRange(vertexId)) 167 | scanEdges(vertexId, edgeValue); 168 | } 169 | 170 | /** 171 | * Description: 172 | * 173 | * @param vertex 174 | * @param edgeValue 175 | * @param vertexId 176 | * @param: 177 | * @return: 178 | */ 179 | private boolean isDuplicationEdge(int vertexId, byte edgeValue) { 180 | if (vertex == null || vertex.getNumOutEdges() == 0) 181 | return false; 182 | 183 | // 1. check fixed size array 184 | for (int i = 0; i < vertex.getNumOutEdges(); i++) { 185 | int id = vertex.getOutEdge(i); 186 | byte value = vertex.getOutEdgeValue(i); 187 | // if((vertexId == id) && (edgeValue == value)) 188 | // TODO: JUST for testing, change back soon!!! 189 | if (vertexId == id) 190 | return true; 191 | } 192 | 193 | // 2. check new edges linked array 194 | if (edgeList == null) 195 | return false; 196 | 197 | int size = edgeList.getSize(); 198 | if (size == 0) 199 | return false; 200 | 201 | int[] ids = null; 202 | byte[] values = null; 203 | 204 | // 2.1 check (size - 1) node, each node is full of NODE_SIZE elements 205 | for (int j = 0; j < size - 1; j++) { 206 | ids = edgeList.getNode(j).getDstVertices(); 207 | values = edgeList.getNode(j).getEdgeValues(); 208 | assert (ids.length == NewEdgesList.NODE_SIZE); 209 | 210 | for (int k = 0; k < ids.length; k++) { 211 | // TODO: JUST for testing, change back soon!!! 212 | // if((ids[k] == vertexId) && (values[k] == edgeValue)) 213 | if (ids[k] == vertexId) 214 | return true; 215 | } 216 | } 217 | 218 | // 2.2 check the last node, the num of elements is index 219 | ids = edgeList.getLast().getDstVertices(); 220 | values = edgeList.getLast().getEdgeValues(); 221 | int index = edgeList.getIndex(); 222 | for (int m = 0; m < index; m++) { 223 | // TODO: JUST for testing, change back soon!!! 224 | // if((ids[m] == vertexId) && (values[m] == edgeValue)) 225 | if (ids[m] == vertexId) 226 | return true; 227 | } 228 | 229 | return false; 230 | } 231 | 232 | private int getDstVertexIndex(int vertexId) { 233 | for (LoadedVertexInterval interval : intervals) { 234 | int intervalSt = interval.getFirstVertex(); 235 | int intervalEnd = interval.getLastVertex(); 236 | if ((vertexId >= intervalSt) && (vertexId <= intervalEnd)) { 237 | int indexSt = interval.getIndexStart(); 238 | return (indexSt + vertexId - intervalSt); 239 | } 240 | } 241 | return -1; 242 | } 243 | 244 | /** 245 | * Description: 246 | * 247 | * @param vertices 248 | * @param vertexId 249 | * @param: 250 | * @return: 251 | */ 252 | private void scanEdges(int vertexId, byte edgeValue1) { 253 | // logger.info("scanning edges for vertex " + vertexId); 254 | if (vertices == null || vertices.length == 0) 255 | return; 256 | 257 | int index = getDstVertexIndex(vertexId); 258 | if (index == -1) 259 | return; 260 | 261 | // logger.info("candidate!"); 262 | 263 | assert index >= 0 && index < vertices.length; 264 | Vertex v = vertices[index]; 265 | if (v == null || v.getNumOutEdges() == 0) 266 | return; 267 | 268 | // 1. scan original fixed size array 269 | for (int i = 0; i < v.getNumOutEdges(); i++) { 270 | int dstId = v.getOutEdge(i); 271 | byte dstEdgeValue = v.getOutEdgeValue(i); 272 | 273 | byte edgeValue2 = dstEdgeValue; 274 | 275 | // get edges of vertex 2 / test code 3 276 | // if (flag == 1) { 277 | // String olds = s; 278 | // s = s + " ---> " + dstId; 279 | // System.out.println(s); 280 | // s = olds; 281 | // } 282 | 283 | // check grammar, check duplication and add edges 284 | // TODO: dstEdgeValue to be fixed based on grammar!! 285 | checkGrammarAndAddEdge(dstId, edgeValue2); 286 | } 287 | 288 | // 2. scan new edges linked array 289 | if (edgeList == null) 290 | return; 291 | 292 | NewEdgesList dstNewEdgeList = edgesLists[index]; 293 | if (dstNewEdgeList == null) 294 | return; 295 | 296 | int readableSize = dstNewEdgeList.getReadableSize(); 297 | if (readableSize == 0) 298 | return; 299 | 300 | int[] ids = null; 301 | byte[] values = null; 302 | 303 | // 2.1 check (readableSize - 1) node, each node is full of NODE_SIZE 304 | // elements 305 | for (int j = 0; j < readableSize - 1; j++) { 306 | ids = edgesLists[index].getNode(j).getDstVertices(); 307 | values = edgesLists[index].getNode(j).getEdgeValues(); 308 | assert (ids.length == NewEdgesList.NODE_SIZE); 309 | 310 | for (int k = 0; k < ids.length; k++) { 311 | // test 312 | // if (flag == 1) { 313 | // String olds = s; 314 | // s = s + " ---> " + ids[k]; 315 | // System.out.println(s); 316 | // s = olds; 317 | // } 318 | // check grammar, check duplication and add edges 319 | // TODO: values[k] to be fixed based on grammar!! 320 | checkGrammarAndAddEdge(ids[k], values[k]); 321 | 322 | } 323 | } 324 | 325 | // 2.2 check the last node, the num of elements is index 326 | ids = edgesLists[index].getReadableLast().getDstVertices(); 327 | values = edgesLists[index].getReadableLast().getEdgeValues(); 328 | int readableIndex = edgesLists[index].getReadableIndex(); 329 | for (int m = 0; m < readableIndex; m++) { 330 | // check grammar, check duplication and add edges 331 | // TODO: values[m] to be fixed based on grammar!! 332 | checkGrammarAndAddEdge(ids[m], values[m]); 333 | } 334 | // logger.info("Completed Scanning edges for " + vertexId); 335 | } 336 | 337 | /** 338 | * Description: 339 | * 340 | * @param: 341 | * @return: 342 | */ 343 | private void checkGrammarAndAddEdge(int vertexId, byte edgeValue) { 344 | 345 | // TODO: add grammar check 346 | if (checkGrammar()) { 347 | // TODO: dstEdgeValue to be fixed based on grammar!! 348 | if (!isDuplicationEdge(vertexId, edgeValue)) { 349 | // no duplication, needs to add edge to linked array 350 | // TODO: assume happens-before relationship is guaranteed 351 | // by main thread waiting for all threads finish 352 | edgeList.add(vertexId, edgeValue); 353 | 354 | // get edges of vertex 2 / test code 4 355 | // if (flag == 1) { 356 | // System.out.println("edge added " + vertexId); 357 | // } 358 | 359 | nNewEdges++; 360 | } else { 361 | nDupEdges++; 362 | } 363 | } 364 | } 365 | 366 | /** 367 | * Description: 368 | * 369 | * @param: 370 | * @return: 371 | */ 372 | private static boolean checkGrammar() { 373 | return true; 374 | } 375 | 376 | /** 377 | * Description: TODO: CALL THIS METHOD INSTEAD OF checkGrammarAndAddEdge() 378 | * 379 | * @param: 380 | * @return: 381 | */ 382 | private void checkGrammarAndAddEdgeComplete(int vertexId, byte edgeValue1, 383 | byte edgeValue2) { 384 | byte edgeValue3 = checkGrammarAndGetNewEdgeVal(edgeValue1, edgeValue2); 385 | if (edgeValue3 != -1) { 386 | if (!isDuplicationEdge(vertexId, edgeValue3)) { 387 | edgeList.add(vertexId, edgeValue3); 388 | nNewEdges++; 389 | } 390 | } 391 | } 392 | 393 | /** 394 | * 395 | * @return 396 | */ 397 | private static byte checkGrammarAndGetNewEdgeVal(byte edgeVal1, 398 | byte edgeVal2) { 399 | 400 | byte[][] grammarTab = GlobalParams.getGrammarTab(); 401 | byte edgeVal3 = -1; 402 | for (int i = 0; i < grammarTab.length; i++) { 403 | if (grammarTab[i][0] == edgeVal1 && grammarTab[i][1] == edgeVal2) { 404 | edgeVal3 = grammarTab[i][2]; 405 | break; 406 | } 407 | } 408 | return edgeVal3; 409 | } 410 | } -------------------------------------------------------------------------------- /graspan-java/test/edu/uci/ics/cs/graspan/edgecomputationMerge/SortedArrMerger.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.edgecomputationMerge; 2 | 3 | import java.util.Arrays; 4 | import java.util.HashSet; 5 | 6 | public class SortedArrMerger { 7 | 8 | static int delta_ptr = -1; 9 | static int oldUnewUdelta_ptr = -1; 10 | static int currentId; 11 | static HashSet currentEvals = new HashSet(); 12 | 13 | public static void main(String args[]) { 14 | 15 | // GIVEN 16 | int[][] new_edgs = new int[5][]; 17 | int[][] new_vals = new int[5][]; 18 | 19 | new_edgs[0] = new int[] { 100, 100, 200, 200, 300, 1700, 1700 }; 20 | new_vals[0] = new int[] { 7, 2, 1, 3, 4, 6, 8 }; 21 | 22 | new_edgs[1] = new int[] { 2, 17, 17, 17, 25 }; 23 | new_vals[1] = new int[] { 3, 4, 8, 6, 1 }; 24 | 25 | new_edgs[2] = new int[] { 1, 3, 3, 6, 10 }; 26 | new_vals[2] = new int[] { 2, 5, 1, 17, 18 }; 27 | 28 | new_edgs[3] = new int[] { 11, 12, 13, 14, 15, 15 }; 29 | new_vals[3] = new int[] { 10, 1, 2, 3, 4, 5 }; 30 | 31 | new_edgs[4] = new int[] { 1, 4, 11, 21 }; 32 | new_vals[4] = new int[] { 6, 5, 10, 1 }; 33 | 34 | // ALLOCATE DELTA 35 | int[][] delta_edgs = new int[5][]; 36 | int[][] delta_vals = new int[5][]; 37 | int[][] oldUnewUdelta_edgs = new int[5][]; 38 | int[][] oldUnewUdelta_vals = new int[5][]; 39 | 40 | // NO. OF ROWS IN ARRAY SET TO MERGE 41 | int num = 3; 42 | 43 | int[][] edgArrstoMerge = new int[num][]; 44 | int[][] valArrstoMerge = new int[num][]; 45 | 46 | // THE ROWS TO MERGE 47 | edgArrstoMerge[0] = new_edgs[0]; 48 | valArrstoMerge[0] = new_vals[0]; 49 | 50 | edgArrstoMerge[1] = new_edgs[1]; 51 | valArrstoMerge[1] = new_vals[1]; 52 | 53 | edgArrstoMerge[2] = new_edgs[2]; 54 | valArrstoMerge[2] = new_vals[2]; 55 | 56 | // ID OF SOURCE ROW (IN edgArrstoMerge) 57 | int srcRowId = 0; 58 | 59 | System.out.println("Source Array:"); 60 | System.out.println(Arrays.toString(edgArrstoMerge[0])); 61 | System.out.println("Arrays to merge:"); 62 | System.out.println(Arrays.deepToString(edgArrstoMerge)); 63 | 64 | mergeTgtstoSrc(edgArrstoMerge, valArrstoMerge, srcRowId, 65 | delta_edgs[srcRowId], delta_vals[srcRowId], 66 | oldUnewUdelta_edgs[srcRowId], oldUnewUdelta_vals[srcRowId]); 67 | 68 | // System.out.println("delta:"); 69 | // System.out.println(Arrays.toString(delta_edgs[srcRowId])); 70 | // System.out.println(Arrays.deepToString(delta_vals)); 71 | // 72 | // System.out.println("oldUnewUdelta:"); 73 | // System.out.println(Arrays.deepToString(oldUnewUdelta_edgs)); 74 | // System.out.println(Arrays.deepToString(oldUnewUdelta_vals)); 75 | // System.out.println("delta:"); 76 | // System.out.println(Arrays.toString(src_delta_edgs)); 77 | // System.out.println(Arrays.toString(src_delta_vals)); 78 | 79 | } 80 | 81 | /** 82 | * 83 | * @param edgArrstoMerge 84 | * @param valArrstoMerge 85 | * @param srcRowId 86 | * @param delta_edgs 87 | * @param delta_vals 88 | * @param oldUnewUdelta_edgs 89 | * @param oldUnewUdelta_vals 90 | */ 91 | public static void mergeTgtstoSrc(int[][] edgArrstoMerge, 92 | int[][] valArrstoMerge, int srcRowId, int[] src_delta_edgs, 93 | int[] src_delta_vals, int[] src_oldUnewUdelta_edgs, 94 | int[] src_oldUnewUdelta_vals) { 95 | 96 | MinSet minSetFrmTgtRows = new MinSet(); 97 | MinSet minSetFromSrcRow = new MinSet(); 98 | 99 | // MIN_SETS ARRAY 100 | MinSet[] minSets = new MinSet[edgArrstoMerge.length]; 101 | 102 | // INITIALIZE MIN SET FOR ROW i & track sizes to declare src_delta and 103 | // src_oldUnewUdelta 104 | int cumTgtRowsSize = 0; 105 | for (int i = 0; i < edgArrstoMerge.length; i++) { 106 | minSets[i] = new MinSet(); 107 | minSets[i].setMinSetId(i); 108 | if (i != srcRowId) 109 | cumTgtRowsSize += edgArrstoMerge[i].length; 110 | } 111 | 112 | // declare & initialize src_delta and src_oldUnewUdelta 113 | 114 | src_delta_edgs = new int[cumTgtRowsSize]; 115 | src_delta_vals = new int[cumTgtRowsSize]; 116 | for (int i = 0; i < cumTgtRowsSize; i++) { 117 | src_delta_edgs[i] = -1; 118 | src_delta_vals[i] = -1; 119 | } 120 | 121 | src_oldUnewUdelta_edgs = new int[edgArrstoMerge[srcRowId].length 122 | + cumTgtRowsSize]; 123 | src_oldUnewUdelta_vals = new int[edgArrstoMerge[srcRowId].length 124 | + cumTgtRowsSize]; 125 | for (int i = 0; i < edgArrstoMerge[srcRowId].length + cumTgtRowsSize; i++) { 126 | src_oldUnewUdelta_edgs[i] = -1; 127 | src_oldUnewUdelta_vals[i] = -1; 128 | } 129 | 130 | // generate the minSets for all rows to merge 131 | int i = 0; 132 | for (MinSet minSet : minSets) { 133 | if (edgArrstoMerge[i].length > 0) { 134 | if (minSet.getPtr() == -1) { 135 | minSet.incrementPtr(); 136 | } 137 | createNextMinSet(minSet, edgArrstoMerge[i], valArrstoMerge[i]); 138 | } 139 | i++; 140 | } 141 | 142 | // MinSet Test 143 | // System.out.println("Minsets:"); 144 | // for (MinSet minSet : minSets) { 145 | // System.out.println(minSet); 146 | // } 147 | 148 | System.out.println("Starting processing of Minsets"); 149 | 150 | while (true) { 151 | // pick the min set from source row and min set from target rows 152 | minSetFromSrcRow = minSets[srcRowId]; 153 | minSetFrmTgtRows = getNextMinSetFrmTgtRows(minSets, srcRowId); 154 | if (minSetFromSrcRow.getCurrentVId() == Integer.MAX_VALUE 155 | && minSetFrmTgtRows.getCurrentVId() == Integer.MAX_VALUE) { 156 | break; 157 | } 158 | System.out.println("minSetFrmTgtRows: " + minSetFrmTgtRows); 159 | int rowIdOfTgtMinSet = minSetFrmTgtRows.getMinSetId(); 160 | 161 | processMinSets(minSetFromSrcRow, minSetFrmTgtRows, src_delta_edgs, 162 | src_delta_vals, src_oldUnewUdelta_edgs, 163 | src_oldUnewUdelta_vals, edgArrstoMerge[srcRowId], 164 | valArrstoMerge[srcRowId], edgArrstoMerge[rowIdOfTgtMinSet], 165 | valArrstoMerge[rowIdOfTgtMinSet]); 166 | 167 | // MinSet Test 168 | // System.out.println("MinSets"); 169 | // for (MinSet minSet : minSets) { 170 | // System.out.println(minSet); 171 | // } 172 | 173 | // System.out.println("delta:"); 174 | // System.out.println(Arrays.toString(src_delta_edgs)); 175 | // System.out.println(Arrays.toString(src_delta_vals)); 176 | // 177 | // System.out.println("oldUnewUdelta:"); 178 | // System.out.println(Arrays.toString(src_oldUnewUdelta_edgs)); 179 | // System.out.println(Arrays.toString(src_oldUnewUdelta_vals)); 180 | } 181 | } 182 | 183 | /** 184 | * Generates the next minSet of a Row 185 | * 186 | * @param minSet 187 | * - stores the minSet elements 188 | * @param edgRow 189 | * - provides the vertex Id 190 | * @param valRow 191 | * - provides the edge values 192 | */ 193 | public static void createNextMinSet(MinSet minSet, int[] edgRow, 194 | int[] valRow) { 195 | System.out.println("Updating MinSet no. " + minSet.getMinSetId()); 196 | minSet.setCurrentVId(Integer.MAX_VALUE); 197 | minSet.clearEvalSet(); 198 | for (int i = minSet.getPtr(); i < edgRow.length; i++) { 199 | if (edgRow[i] <= minSet.getCurrentVId()) { 200 | minSet.setCurrentVId(edgRow[i]); 201 | minSet.addEval(valRow[i]); 202 | minSet.incrementPtr(); 203 | } else 204 | break; 205 | } 206 | } 207 | 208 | public static MinSet getNextMinSetFrmTgtRows(MinSet[] minSets, int srcRowId) { 209 | int min = Integer.MAX_VALUE; 210 | MinSet minset = null; 211 | for (int i = 0; i < minSets.length; i++) { 212 | if (i != srcRowId) { 213 | if (minSets[i].getCurrentVId() <= min) { 214 | min = minSets[i].getCurrentVId(); 215 | minset = minSets[i]; 216 | } 217 | } 218 | } 219 | return minset; 220 | } 221 | 222 | public static void processMinSets(MinSet minSetFrmSrcRow, 223 | MinSet minSetFrmTgtRows, int[] src_delta_edgs, 224 | int[] src_delta_vals, int[] src_oldUnewUdelta_edgs, 225 | int[] src_oldUnewUdelta_vals, int[] srcEdgRow, int[] srcValRow, 226 | int[] tgtEdgRow, int[] tgtValRow) { 227 | 228 | System.out.println("Processing SrcMinSet # " 229 | + minSetFrmSrcRow.getMinSetId() + " and TgtMinSet # " 230 | + minSetFrmTgtRows.getMinSetId()); 231 | 232 | // System.out.println("minset from target rows:"); 233 | // System.out.println(minSetFrmTgtRows.getCurrentVId()); 234 | // System.out.println(minSetFrmTgtRows.getEvals()); 235 | // System.out.println("minset from source row:"); 236 | // System.out.println(minSetFrmSrcRow.getCurrentVId()); 237 | // System.out.println(minSetFrmSrcRow.getEvals()); 238 | 239 | // case 1 240 | if (minSetFrmTgtRows.getCurrentVId() < minSetFrmSrcRow.getCurrentVId()) { 241 | if (currentId != minSetFrmTgtRows.getCurrentVId()) { 242 | currentId = minSetFrmTgtRows.getCurrentVId(); 243 | currentEvals.clear(); 244 | } 245 | 246 | HashSet evals_tgt; 247 | evals_tgt = minSetFrmTgtRows.getEvals(); 248 | for (Integer tgt_eval : evals_tgt) { 249 | if (!currentEvals.contains(tgt_eval)) { 250 | 251 | // add the target row minSet to src_oldUnewUdelta 252 | oldUnewUdelta_ptr++; 253 | if (oldUnewUdelta_ptr < src_oldUnewUdelta_edgs.length) { 254 | src_oldUnewUdelta_edgs[oldUnewUdelta_ptr] = minSetFrmTgtRows 255 | .getCurrentVId(); 256 | src_oldUnewUdelta_vals[oldUnewUdelta_ptr] = tgt_eval; 257 | } else { 258 | System.out 259 | .println("Error, oldUnewUdelta_ptr has gone past the size the array!"); 260 | } 261 | 262 | // add the target row minSet to delta 263 | delta_ptr++; 264 | if (delta_ptr < src_delta_edgs.length) { 265 | src_delta_edgs[delta_ptr] = minSetFrmTgtRows 266 | .getCurrentVId(); 267 | src_delta_vals[delta_ptr] = tgt_eval; 268 | currentEvals.add(tgt_eval); 269 | } else { 270 | System.out 271 | .println("Error, delta_ptr has gone past the size the array!"); 272 | } 273 | } 274 | } 275 | // increment the pointers for this minset 276 | if (tgtEdgRow.length > 0) { 277 | createNextMinSet(minSetFrmTgtRows, tgtEdgRow, tgtValRow); 278 | } 279 | return; 280 | 281 | } 282 | 283 | // case 2 284 | if (minSetFrmTgtRows.getCurrentVId() == minSetFrmSrcRow.getCurrentVId()) { 285 | if (currentId != minSetFrmTgtRows.getCurrentVId()) { 286 | currentId = minSetFrmTgtRows.getCurrentVId(); 287 | currentEvals.clear(); 288 | } 289 | HashSet evals_src, evals_tgt; 290 | evals_tgt = minSetFrmTgtRows.getEvals(); 291 | evals_src = minSetFrmSrcRow.getEvals(); 292 | 293 | for (Integer tgt_eval : evals_tgt) { 294 | // compare the src evals and the tgt evals 295 | if (!evals_src.contains(tgt_eval)) { 296 | 297 | if (!currentEvals.contains(tgt_eval)) { 298 | 299 | // add the target row minSet to src_oldUnewUdelta 300 | oldUnewUdelta_ptr++; 301 | if (oldUnewUdelta_ptr < src_oldUnewUdelta_edgs.length) { 302 | src_oldUnewUdelta_edgs[oldUnewUdelta_ptr] = minSetFrmTgtRows 303 | .getCurrentVId(); 304 | src_oldUnewUdelta_vals[oldUnewUdelta_ptr] = tgt_eval; 305 | } else { 306 | System.out 307 | .println("Error, oldUnewUdelta_ptr has gone past the size the array!"); 308 | } 309 | 310 | // add the target row minSet to delta 311 | delta_ptr++; 312 | if (delta_ptr < src_delta_edgs.length) { 313 | src_delta_edgs[delta_ptr] = minSetFrmTgtRows 314 | .getCurrentVId(); 315 | src_delta_vals[delta_ptr] = tgt_eval; 316 | currentEvals.add(tgt_eval); 317 | } else { 318 | System.out 319 | .println("Error, delta_ptr has gone past the size the array!"); 320 | } 321 | } 322 | } 323 | } 324 | 325 | // increment the pointers for this minset 326 | createNextMinSet(minSetFrmTgtRows, tgtEdgRow, tgtValRow); 327 | return; 328 | } 329 | 330 | // case 3 331 | if (minSetFrmTgtRows.getCurrentVId() > minSetFrmSrcRow.getCurrentVId()) { 332 | 333 | if (currentId != minSetFrmSrcRow.getCurrentVId()) { 334 | currentId = minSetFrmSrcRow.getCurrentVId(); 335 | currentEvals.clear(); 336 | } 337 | 338 | HashSet evals_src, evals_tgt; 339 | 340 | evals_tgt = minSetFrmTgtRows.getEvals(); 341 | evals_src = minSetFrmSrcRow.getEvals(); 342 | 343 | // add the source row minSet to src_oldUnewUdelta 344 | for (Integer src_eval : evals_src) { 345 | if (!currentEvals.contains(src_eval)) { 346 | 347 | oldUnewUdelta_ptr++; 348 | if (oldUnewUdelta_ptr < src_oldUnewUdelta_edgs.length) { 349 | src_oldUnewUdelta_edgs[oldUnewUdelta_ptr] = minSetFrmSrcRow 350 | .getCurrentVId(); 351 | src_oldUnewUdelta_vals[oldUnewUdelta_ptr] = src_eval; 352 | currentEvals.add(src_eval); 353 | } else { 354 | System.out 355 | .println("Error, oldUnewUdelta_ptr has gone past the size the array!"); 356 | } 357 | } 358 | } 359 | 360 | // increment the pointers for this minset 361 | createNextMinSet(minSetFrmSrcRow, srcEdgRow, srcValRow); 362 | return; 363 | } 364 | 365 | } 366 | 367 | public static boolean tgt_eval_exists(int a, int b) { 368 | if (a == b) 369 | return true; 370 | else 371 | return false; 372 | } 373 | 374 | public static void trial(int[] arr) { 375 | arr = new int[5]; 376 | } 377 | 378 | } 379 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/experiments/Extractor.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.experiments; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.BufferedWriter; 5 | import java.io.File; 6 | import java.io.FileInputStream; 7 | import java.io.FileNotFoundException; 8 | import java.io.FileWriter; 9 | import java.io.IOException; 10 | import java.io.InputStreamReader; 11 | import java.io.PrintWriter; 12 | import java.util.logging.Logger; 13 | 14 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 15 | import edu.uci.ics.cs.graspan.support.Utilities; 16 | 17 | public class Extractor { 18 | 19 | private long numEdges1, numVs1, numEdges2, numVs2, numEdgesEnd, numVsEnd, numOfNewEdges; 20 | private double gcDuration_s; 21 | private String pgen1Time_hms, eRandStime_hms, pgen2Time_hms, compTime_hms, gcTime_hms; 22 | 23 | private static final Logger logger = GraspanLogger.getLogger("Extractor"); 24 | private String finishTime_hms; 25 | private long finishTime_s, totalMemUsage, maxMemUsage; 26 | private long maxNumIters; // the maximum number of iterations for any round 27 | 28 | private long writeDuration, readDuration; 29 | 30 | private double ioTime_s; 31 | 32 | private int numRounds; 33 | private int numRoundsWithRepartitioning; 34 | 35 | // VERY IMPORTANT 36 | // args[0] -- pp.pgen1.output 37 | // args[1] -- pp.erAndSort.output 38 | // args[2] -- pp.pgen2.output 39 | // args[3] -- comp.output 40 | // args[4] -- comp.gctimes.output 41 | // args[5] -- basefilename 42 | // args[6] -- io.output 43 | // not used -- comp.memusage.output 44 | public static void main(String args[]) throws IOException { 45 | Extractor extractor = new Extractor(); 46 | extractor.run(args); 47 | } 48 | 49 | public void cpp_outputScanner(String cpp_op) throws IOException { 50 | BufferedReader cpp_opStrm = new BufferedReader(new InputStreamReader(new FileInputStream(new File(cpp_op)))); 51 | 52 | PrintWriter cpp_result_Strm; 53 | cpp_result_Strm = new PrintWriter(new BufferedWriter(new FileWriter("cppOP.csv", true))); 54 | 55 | String ln; 56 | int round = 0, iteration = 0; 57 | 58 | while ((ln = cpp_opStrm.readLine()) != null) { 59 | if (ln.contains("STARTING ROUND")) { 60 | round++; 61 | iteration = 0; 62 | } 63 | if (ln.contains("STARTING ITERATION")) { 64 | iteration++; 65 | } 66 | if (ln.contains("EDGES THIS ITER:")) { 67 | String[] tok = ln.split("\\s+"); 68 | cpp_result_Strm.println(round + "," + iteration + "," + tok[3]); 69 | } 70 | } 71 | cpp_opStrm.close(); 72 | cpp_result_Strm.close(); 73 | } 74 | 75 | public void run(String args[]) throws FileNotFoundException, IOException { 76 | scan_pp_pgen1(args[0]); 77 | scan_pp_eadd(args[1]); 78 | scan_pp_pgen2(args[2]); 79 | 80 | scan_comp(args); 81 | 82 | this.finishTime_s = (long) Integer.parseInt(finishTime_hms.split(",")[0]) * 60 * 60 // hour 83 | // --> 84 | // sec 85 | + Integer.parseInt(finishTime_hms.split(",")[1]) * 60 // min --> 86 | // sec 87 | + Integer.parseInt(finishTime_hms.split(",")[2]); 88 | 89 | numEdgesEnd = numEdges1 + numOfNewEdges; 90 | numVsEnd = this.numVs2; 91 | 92 | scan_gctimes(args[4]); 93 | gcTime_hms = Utilities.getDurationInHMS((long) gcDuration_s * 1000); 94 | 95 | // scan_pmap(args); 96 | // scan_IO_times(args[6]); 97 | 98 | // write the file 99 | printData(args); 100 | 101 | } 102 | 103 | public double scan_IO_times(String iofile, double total_time, double pp_time) throws IOException { 104 | String ln; 105 | String[] tok; 106 | double IO_Duration = 0; 107 | double iostat_time = 0; 108 | // double total_time= 109 | // Double.parseDouble(this.compTime_hms.split(",")[0])*60*60 110 | // +Double.parseDouble(this.compTime_hms.split(",")[1])*60 111 | // +Double.parseDouble(this.compTime_hms.split(",")[2]); 112 | 113 | BufferedReader iostat_Strm = new BufferedReader(new InputStreamReader(new FileInputStream(new File(iofile)))); 114 | 115 | while ((ln = iostat_Strm.readLine()) != null) { 116 | if (ln.contains("avg-cpu")) { 117 | 118 | iostat_time += 5; 119 | if (iostat_time < pp_time) {// for excluding IOtime during 120 | // preprocessing 121 | continue; 122 | } 123 | 124 | ln = iostat_Strm.readLine();// read the next line after line 125 | // beginning with avg-cpu 126 | tok = ln.split("\\s+"); 127 | // System.out.println(ln);//TODO: TEST CODE 128 | IO_Duration += (double) ((double) Double.parseDouble(tok[3]) / 100) * 5;// IO 129 | // time 130 | // off 131 | // 5 132 | // seconds 133 | if (iostat_time > total_time) { 134 | break; 135 | } 136 | } 137 | } 138 | iostat_Strm.close(); 139 | this.ioTime_s = IO_Duration; 140 | return IO_Duration; 141 | } 142 | 143 | private void printData(String[] args) throws IOException { 144 | PrintWriter gptabStrm; 145 | gptabStrm = new PrintWriter(new BufferedWriter(new FileWriter("graspanPerformance.Table.csv", true))); 146 | 147 | String head = ""; 148 | String val = ""; 149 | 150 | head += "graph,"; 151 | val += args[5] + ","; 152 | 153 | head += "numOfEdges1,"; 154 | val += this.numEdges1 + ","; 155 | 156 | head += "numVs1,"; 157 | val += this.numVs1 + ","; 158 | 159 | head += "numOfEdges2,"; 160 | val += this.numEdges2 + ","; 161 | 162 | head += "numVs2,"; 163 | val += this.numVs2 + ","; 164 | 165 | head += "numEdgesEnd,"; 166 | val += this.numEdgesEnd + ","; 167 | 168 | head += "numVsEnd,"; 169 | val += this.numVsEnd + ","; 170 | 171 | head += "pgen1Time_h,pgen1Time_m,pgen1Time_s,"; 172 | val += this.pgen1Time_hms + ","; 173 | 174 | head += "eRandStime_h,eRandStime_m,eRandStime_s,"; 175 | val += this.eRandStime_hms + ","; 176 | 177 | head += "pgen2Time_h,pgen2Time_m,pgen2Time_s,"; 178 | val += this.pgen2Time_hms + ","; 179 | 180 | head += "numRounds,"; 181 | val += this.numRounds + ","; 182 | 183 | head += "numRoundsWithRepartitioning,"; 184 | val += this.numRoundsWithRepartitioning + ","; 185 | 186 | head += "compTime_h,compTime_m,compTime_s,"; 187 | val += this.compTime_hms + ","; 188 | 189 | head += "gcTime_h,gcTime_m,gcTime_s,"; 190 | val += this.gcTime_hms + ","; 191 | 192 | head += "ioTime_s,"; 193 | val += this.ioTime_s + ","; 194 | 195 | gptabStrm.println(head); 196 | gptabStrm.println(val); 197 | 198 | // gptabStrm.println(args[5] + "," + this.numEdges1 + "," + this.numVs1 199 | // + "," + this.numEdgesEnd + "," 200 | // + this.numVs1 + "," + this.eRandStime_hms + "," + this.pgen1Time_hms 201 | // + "," + this.numRounds 202 | // + "," + this.numRoundsWithRepartitioning + "," + this.compTime_hms + 203 | // "," + this.gcTime_hms 204 | // + "," + this.maxMemUsage + "," + this.ioTime_s + "," + 205 | // this.maxNumIters); 206 | 207 | gptabStrm.close(); 208 | } 209 | 210 | private void scan_pmap(String[] args) throws FileNotFoundException, IOException { 211 | String ln, time; 212 | String[] tok, timeparts; 213 | long currentTime_s = 0, totalMemUsage = 0, maxMemUsage = 0; 214 | 215 | BufferedReader pmapStrm = new BufferedReader(new InputStreamReader(new FileInputStream(new File(args[4])))); 216 | 217 | while ((ln = pmapStrm.readLine()) != null) { 218 | if (!ln.contains("data")) { 219 | tok = ln.split("\\s+"); 220 | time = tok[0]; 221 | timeparts = time.split("-"); 222 | 223 | currentTime_s = (long) Integer.parseInt(timeparts[3]) * 60 * 60 // hour 224 | // --> 225 | // sec 226 | + Integer.parseInt(timeparts[4]) * 60 // min --> sec 227 | + Integer.parseInt(timeparts[5]); 228 | 229 | if (currentTime_s > this.finishTime_s) { 230 | break; 231 | } 232 | 233 | if (tok.length > 1) { 234 | if (Long.parseLong(tok[2]) > maxMemUsage) { 235 | maxMemUsage = Long.parseLong(tok[2]); 236 | } 237 | // totalMemUsage += Long.parseLong(tok[2]); 238 | } 239 | } 240 | } 241 | // this.totalMemUsage = totalMemUsage; 242 | this.maxMemUsage = maxMemUsage; 243 | pmapStrm.close(); 244 | } 245 | 246 | private void scan_gctimes(String gcfile) throws FileNotFoundException, IOException { 247 | String ln; 248 | String[] tok; 249 | double gcDuration_s = 0; 250 | BufferedReader gcTimeStrm = new BufferedReader(new InputStreamReader(new FileInputStream(new File(gcfile)))); 251 | 252 | while ((ln = gcTimeStrm.readLine()) != null) { 253 | if (ln.contains("secs")) { 254 | tok = ln.split(" "); 255 | // logger.info(args[5]); 256 | // logger.info(tok[tok.length-2]); 257 | gcDuration_s += Double.parseDouble(tok[tok.length - 2]); 258 | } 259 | } 260 | this.gcDuration_s = gcDuration_s; 261 | gcTimeStrm.close(); 262 | } 263 | 264 | private void scan_comp(String[] args) throws FileNotFoundException, IOException { 265 | String ln; 266 | String[] tok; 267 | int numRounds = 0; 268 | int numRoundsWithoutRepart = 0; 269 | long numOfNewEdges = 0; 270 | long numIters = 0, maxNumIters = 0; 271 | long readDuration = 0, writeDuration = 0; 272 | String totalDuration_hms = "", finishTime_hms = ""; 273 | String[] roundOutputComponent = null; // round#,h,m,s,#edges 274 | String[] iterationOutputComponent = null; // iteration#, round#, h,m,s, 275 | // #edges 276 | BufferedReader compStrm = new BufferedReader(new InputStreamReader(new FileInputStream(new File(args[3])))); 277 | 278 | // PrintWriter edgesPRound; 279 | // edgesPRound = new PrintWriter(new BufferedWriter(new 280 | // FileWriter("edgesPerRound.Table.csv", true))); 281 | // edgesPRound.println(args[5]); 282 | 283 | while ((ln = compStrm.readLine()) != null) { 284 | 285 | if (ln.contains("output.iteration")) { 286 | tok = ln.split("\\|\\|"); 287 | iterationOutputComponent = tok[tok.length - 1].split(","); 288 | numIters = Long.parseLong(iterationOutputComponent[1]); 289 | if (numIters > maxNumIters) { 290 | maxNumIters = numIters; 291 | } 292 | } 293 | 294 | if (ln.contains("output.round")) { 295 | tok = ln.split("\\|\\|"); 296 | // logger.info(Arrays.deepToString(tok)); 297 | roundOutputComponent = tok[tok.length - 1].split(","); 298 | numRounds = Integer.parseInt(roundOutputComponent[0]); 299 | // numRounds = 300 | // Integer.parseInt(tok[tok.length-1].split(",")[0]); 301 | // edgesPRound.println(numRounds + "," + 302 | // roundOutputComponent[roundOutputComponent.length-1] ); 303 | } 304 | if (ln.contains("No Parts Repartitioned.")) { 305 | tok = ln.split("\\|\\|"); 306 | // logger.info(Arrays.deepToString(tok)); 307 | numRoundsWithoutRepart++; 308 | } 309 | if (ln.contains("Total Num of New Edges:")) { 310 | tok = ln.split(" "); 311 | numOfNewEdges = Long.parseLong(tok[tok.length - 1]); 312 | } 313 | if (ln.contains("Computation took")) { 314 | tok = ln.split(" "); 315 | totalDuration_hms = tok[tok.length - 1]; 316 | 317 | // finding finish time 318 | String finishTime[] = tok[0].split(":"); 319 | int hour = Integer.parseInt(finishTime[0]); 320 | if (tok[1].compareTo("PM") == 0) { 321 | hour += 12; 322 | } 323 | finishTime_hms = hour + "," + finishTime[1] + "," + finishTime[2]; 324 | } 325 | if (ln.contains("output.IO") & ln.contains("write")) { 326 | tok = ln.split("\\s+"); 327 | String[] outputIO = tok[tok.length - 1].split(","); 328 | writeDuration += Long.parseLong(outputIO[outputIO.length - 1]); 329 | } 330 | if (ln.contains("output.IO") & ln.contains("read")) { 331 | tok = ln.split("\\s+"); 332 | String[] outputIO = tok[tok.length - 1].split(","); 333 | readDuration += Long.parseLong(outputIO[outputIO.length - 1]); 334 | } 335 | } 336 | 337 | this.maxNumIters = maxNumIters; 338 | this.numRounds = numRounds; 339 | this.numRoundsWithRepartitioning = numRounds - numRoundsWithoutRepart; 340 | this.numOfNewEdges = numOfNewEdges; 341 | this.compTime_hms = totalDuration_hms; 342 | this.finishTime_hms = finishTime_hms; 343 | this.writeDuration = writeDuration; 344 | this.readDuration = readDuration; 345 | // edgesPRound.close(); 346 | compStrm.close(); 347 | } 348 | 349 | private void scan_pp_pgen1(String ppoutput) throws FileNotFoundException, IOException { 350 | String ln; 351 | String[] tok; 352 | long numOfEdgesStart = 0, numOfVs = 0; 353 | String pgenDuration_hms = ""; 354 | BufferedReader pp_pgenStrm = new BufferedReader(new InputStreamReader(new FileInputStream(new File(ppoutput)))); 355 | 356 | while ((ln = pp_pgenStrm.readLine()) != null) { 357 | if (ln.contains("The total number of edges in graph")) { 358 | tok = ln.split(" "); 359 | numOfEdgesStart = Long.parseLong(tok[tok.length - 1]); 360 | } 361 | if (ln.contains("Total number of vertices in input graph:")) { 362 | tok = ln.split(" "); 363 | numOfVs = Long.parseLong(tok[tok.length - 1]); 364 | } 365 | if (ln.contains("Generating partitions took:")) { 366 | tok = ln.split(" "); 367 | pgenDuration_hms = tok[tok.length - 1]; 368 | } 369 | } 370 | this.numEdges1 = numOfEdgesStart; 371 | this.numVs1 = numOfVs; 372 | this.pgen1Time_hms = pgenDuration_hms; 373 | pp_pgenStrm.close(); 374 | } 375 | 376 | private void scan_pp_pgen2(String ppoutput) throws FileNotFoundException, IOException { 377 | String ln; 378 | String[] tok; 379 | long numOfEdgesStart = 0, numOfVs = 0; 380 | String pgenDuration_hms = ""; 381 | BufferedReader pp_pgenStrm = new BufferedReader(new InputStreamReader(new FileInputStream(new File(ppoutput)))); 382 | 383 | while ((ln = pp_pgenStrm.readLine()) != null) { 384 | if (ln.contains("The total number of edges in graph")) { 385 | tok = ln.split(" "); 386 | numOfEdgesStart = Long.parseLong(tok[tok.length - 1]); 387 | } 388 | if (ln.contains("Total number of vertices in input graph:")) { 389 | tok = ln.split(" "); 390 | numOfVs = Long.parseLong(tok[tok.length - 1]); 391 | } 392 | if (ln.contains("Generating partitions took:")) { 393 | tok = ln.split(" "); 394 | pgenDuration_hms = tok[tok.length - 1]; 395 | } 396 | } 397 | this.numEdges2 = numOfEdgesStart; 398 | this.numVs2 = numOfVs; 399 | this.pgen2Time_hms = pgenDuration_hms; 400 | pp_pgenStrm.close(); 401 | } 402 | 403 | private void scan_pp_eadd(String eaddoutput) throws FileNotFoundException, IOException { 404 | String ln; 405 | String tok[], eredgsDuration_hms = ""; 406 | BufferedReader pp_eredgs = new BufferedReader(new InputStreamReader(new FileInputStream(new File(eaddoutput)))); 407 | 408 | while ((ln = pp_eredgs.readLine()) != null) { 409 | if (ln.contains("Edge Adding from Erules took:")) { 410 | tok = ln.split(" "); 411 | eredgsDuration_hms = tok[tok.length - 1]; 412 | } 413 | } 414 | this.eRandStime_hms = eredgsDuration_hms; 415 | pp_eredgs.close(); 416 | } 417 | } 418 | -------------------------------------------------------------------------------- /graspan-java/src/edu/uci/ics/cs/graspan/computationEL/EngineEL.java: -------------------------------------------------------------------------------- 1 | package edu.uci.ics.cs.graspan.computationEL; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.Arrays; 6 | import java.util.List; 7 | import java.util.concurrent.ExecutorService; 8 | import java.util.concurrent.Executors; 9 | import java.util.concurrent.atomic.AtomicInteger; 10 | import java.util.logging.Logger; 11 | 12 | import edu.uci.ics.cs.graspan.datastructures.AllPartitions; 13 | import edu.uci.ics.cs.graspan.datastructures.LoadedVertexInterval; 14 | import edu.uci.ics.cs.graspan.datastructures.NewEdgesList; 15 | import edu.uci.ics.cs.graspan.datastructures.RepartitioningData; 16 | import edu.uci.ics.cs.graspan.datastructures.Vertex; 17 | import edu.uci.ics.cs.graspan.scheduler.IScheduler; 18 | import edu.uci.ics.cs.graspan.scheduler.Scheduler; 19 | import edu.uci.ics.cs.graspan.scheduler.SchedulerInfo; 20 | import edu.uci.ics.cs.graspan.support.GraspanLogger; 21 | import edu.uci.ics.cs.graspan.support.MemUsageCheckThread; 22 | 23 | /** 24 | * @author Kai Wang 25 | * 26 | * Created by Oct 8, 2015 27 | */ 28 | public class EngineEL { 29 | private static final Logger logger = GraspanLogger.getLogger("Engine"); 30 | private ExecutorService computationExecutor; 31 | private long totalNewEdgs; 32 | private long totalNewEdgsForIteratn; 33 | private long totalDupEdges; 34 | private long newEdgesInOne; 35 | private long newEdgesInTwo; 36 | // private String baseFileName; 37 | private int[] partsToLoad; 38 | private IScheduler scheduler; 39 | public static boolean memFull; 40 | 41 | public static boolean premature_Terminate; 42 | 43 | public static Vertex[] vertices_prevIt = null; 44 | public static NewEdgesList[] newEdgeLists_prevIt = null; 45 | public List intervals_prevIt = null; 46 | 47 | // public Engine(int[] partitionsToLoad) { 48 | // // this.baseFileName = baseFileName; 49 | // this.partsToLoad = partitionsToLoad; 50 | // } 51 | // 52 | // public Engine(IScheduler scheduler) { 53 | // this.scheduler = scheduler; 54 | // } 55 | 56 | /** 57 | * Description: 58 | * 59 | * @param: 60 | * @return: 61 | * @throws IOException 62 | */ 63 | public void run() throws IOException { 64 | 65 | // get the num of processors 66 | int nThreads = 8; 67 | if (Runtime.getRuntime().availableProcessors() > nThreads) { 68 | nThreads = Runtime.getRuntime().availableProcessors(); 69 | } 70 | // TODO: REMOVE THIS LATER 71 | nThreads = 8; 72 | 73 | computationExecutor = Executors.newFixedThreadPool(nThreads); 74 | // logger.info("Executing partition loader."); 75 | long t = System.currentTimeMillis(); 76 | 77 | // 1. load partitions into memory 78 | LoaderEL loader = new LoaderEL(); 79 | 80 | Scheduler scheduler = new Scheduler(AllPartitions.partAllocTable.length); 81 | 82 | // TODO: AVOID USING THIS SECOND SCHEDULER CONSTRUCTOR 83 | // Scheduler scheduler = new 84 | // Scheduler(SchedulerInfo.getEdgeDestCount()); 85 | 86 | int roundNo = 0; 87 | 88 | while (!scheduler.shouldTerminate()) { 89 | 90 | roundNo++; 91 | logger.info("STARTING ROUND NO #" + roundNo); 92 | // partsToLoad = 93 | // scheduler.schedulePartitionSimple(AllPartitions.partAllocTable.length); 94 | // TODO: USE partitionEDC later 95 | // partsToLoad = scheduler 96 | // .schedulePartitionEDC(AllPartitions.partAllocTable.length); 97 | partsToLoad = scheduler 98 | .schedulePartitionSimple(AllPartitions.partAllocTable.length); 99 | logger.info("Scheduling Partitions : " 100 | + Arrays.toString(partsToLoad)); 101 | logger.info("Start loading partitions..."); 102 | loader.loadParts(partsToLoad); 103 | // logger.info("Total time for loading partitions : " + 104 | // (System.currentTimeMillis() - t) + " ms"); 105 | 106 | Vertex[] vertices; 107 | NewEdgesList[] edgesLists; 108 | vertices = loader.getVertices(); 109 | List intervals = null; 110 | EdgeComputerEL[] edgeComputers = new EdgeComputerEL[vertices.length]; 111 | intervals = loader.getIntervals(); 112 | 113 | // this only does a shallow copy 114 | List intervalsForScheduler = new ArrayList(intervals); 115 | scheduler.setLoadedIntervals(intervalsForScheduler); 116 | logger.info("\nLVI after loading : " + intervals); 117 | assert (vertices != null && vertices.length > 0); 118 | assert (intervals != null && intervals.size() > 0); 119 | 120 | edgesLists = loader.getNewEdgeLists(); 121 | 122 | // if (vertices_prevIt == null) {// if there was no previous 123 | // iteration 124 | // edgesLists = loader.getNewEdgeLists(); 125 | // } else { 126 | // //use intervals_prevIt to set up edge lists; 127 | // for (LoadedVertexInterval intvNew:intervals){ 128 | // for (LoadedVertexInterval intvOld:intervals){ 129 | // if (intvNew.getPartitionId()==intvOld.getPartitionId()){ 130 | // 131 | // // newEdgeLists 132 | // } 133 | // } 134 | // } 135 | // edgesLists = newEdgeLists_prevIt; 136 | // } 137 | // logger.info("VERTEX LENGTH: " + vertices.length); 138 | // logger.info("The vertices before setting degree after new 139 | // edges:"); 140 | // logger.info(vertices[i]+""); 141 | // logger.info("New Edges: "+newEdgesLL[i]+""); 142 | 143 | // logger.info("Loading complete."); 144 | 145 | logger.info("Start computation and edge addition..."); 146 | t = System.currentTimeMillis(); 147 | 148 | // MemUsageCheckThread job1 = new MemUsageCheckThread(); 149 | // job1.start(); 150 | 151 | // 2. do computation and add edges 152 | EdgeComputerEL.setEdgesLists(edgesLists); 153 | EdgeComputerEL.setVertices(vertices); 154 | EdgeComputerEL.setIntervals(intervals); 155 | 156 | doComputation(vertices, edgesLists, edgeComputers, intervals); 157 | 158 | logger.info("Finish computation..."); 159 | logger.info("Computation and edge addition took: " 160 | + (System.currentTimeMillis() - t) + " ms"); 161 | // logger.info("VERTEX LENGTH: " + vertices.length); 162 | // for(int i = 0; i < vertices.length; i++) { 163 | // logger.info("" + vertices[i]); 164 | // logger.info("" + edgesLists[i]); 165 | // } 166 | 167 | logger.info("Start storing partitions..."); 168 | // 3. process computed partitions 169 | int numPartsStart = AllPartitions.getPartAllocTab().length; 170 | RepartitioningData.initRepartioningVars(); 171 | ComputedPartProcessorEL.initRepartitionConstraints(); 172 | ComputedPartProcessorEL.processParts(vertices, edgesLists, intervals); 173 | int numPartsFinal = AllPartitions.getPartAllocTab().length; 174 | // logger.info("termination map before: " + scheduler.toString()); 175 | // for (int i = 0; i < vertices.length; i++) { 176 | // logger.info("" + vertices[i]); 177 | // logger.info("" + edgesLists[i]); 178 | // } 179 | 180 | vertices_prevIt = vertices; 181 | newEdgeLists_prevIt = edgesLists; 182 | intervals_prevIt = intervals; 183 | logger.info("\nLVI after computedPartProcessor saves partitions : " 184 | + intervals); 185 | logger.info("\nLVI (scheduler) after computedPartProcessor saves partitions : " 186 | + intervalsForScheduler); 187 | scheduler.setTerminationStatus(); 188 | // logger.info("termination map after: " + scheduler.toString()); 189 | scheduler.updateSchedInfoPostRepart(numPartsFinal - numPartsStart, 190 | numPartsFinal); 191 | // logger.info("termination map after: " + scheduler.toString()); 192 | } 193 | 194 | computationExecutor.shutdown(); 195 | } 196 | 197 | /** 198 | * Description: 199 | * 200 | * @param: 201 | * @return: 202 | */ 203 | private void doComputation(final Vertex[] vertices, 204 | final NewEdgesList[] edgesLists, 205 | final EdgeComputerEL[] edgeComputers, 206 | List intervals) { 207 | if (vertices == null || vertices.length == 0) 208 | return; 209 | 210 | final Object termationLock = new Object(); 211 | final Object countLock = new Object(); 212 | final int chunkSize = 1 + vertices.length / 64; 213 | // final int chunkSize = 1 + vertices.length / 4; 214 | 215 | final int nWorkers = vertices.length / chunkSize + 1; 216 | final AtomicInteger countDown = new AtomicInteger(nWorkers); 217 | 218 | newEdgesInOne = 0; 219 | newEdgesInTwo = 0; 220 | assert (intervals.size() == 2); 221 | final int indexStartForOne = intervals.get(0).getIndexStart(); 222 | final int indexEndForOne = intervals.get(0).getIndexEnd(); 223 | final int indexStartForTwo = intervals.get(1).getIndexStart(); 224 | final int indexEndForTwo = intervals.get(1).getIndexEnd(); 225 | int iterationNo = 0; 226 | 227 | do { 228 | // set readable index, for read and write concurrency 229 | // for current iteration, readable index points to the last new edge 230 | // in the previous iteration 231 | // which is readable for the current iteration 232 | setReadableIndex(edgesLists); 233 | iterationNo++; 234 | long t = System.currentTimeMillis(); 235 | logger.info("Entered iteration no. " + iterationNo); 236 | totalNewEdgsForIteratn = 0; 237 | totalDupEdges = 0; 238 | countDown.set(nWorkers); 239 | // Parallel updates 240 | for (int id = 0; id < nWorkers; id++) { 241 | final int currentId = id; 242 | final int chunkStart = currentId * chunkSize; 243 | final int chunkEnd = chunkStart + chunkSize; 244 | 245 | computationExecutor.submit(new Runnable() { 246 | 247 | public void run() { 248 | int threadUpdates = 0; 249 | int dups = 0; 250 | 251 | try { 252 | int end = chunkEnd; 253 | if (end > vertices.length) 254 | end = vertices.length; 255 | 256 | // logger.info(vertices.length + " vertex length"); 257 | 258 | for (int i = chunkStart; i < end; i++) { 259 | // each vertex is associated with an edgeList 260 | Vertex vertex = vertices[i]; 261 | NewEdgesList edgeList = edgesLists[i]; 262 | EdgeComputerEL edgeComputer = edgeComputers[i]; 263 | 264 | if (vertex != null 265 | && vertex.getNumOutEdges() != 0) { 266 | if (edgeList == null) { 267 | edgeList = new NewEdgesList(); 268 | edgesLists[i] = edgeList; 269 | } 270 | 271 | if (edgeComputer == null) { 272 | edgeComputer = new EdgeComputerEL( 273 | vertex, edgeList); 274 | edgeComputers[i] = edgeComputer; 275 | } 276 | 277 | // get termination status for each vertex 278 | if (edgeComputer.getTerminateStatus()) 279 | continue; 280 | 281 | edgeComputer.execUpdate(); 282 | threadUpdates = edgeComputer 283 | .getNumNewEdges(); 284 | dups = edgeComputer.getNumDupEdges(); 285 | 286 | // if (edgeComputer.getNumNewEdges() >= 287 | // 1000) { 288 | // logger.info("No. of New edges added for vertex " 289 | // + vertices[i].getVertexId() 290 | // + " is " 291 | // + edgeComputer.getNumNewEdges()); 292 | // } 293 | 294 | // if (edgeComputer.getNumDupEdges()>=1000){ 295 | // logger.info("No. of Duplicate edges for vertex " 296 | // + vertices[i].getVertexId() 297 | // + " is " 298 | // + edgeComputer.getNumDupEdges());} 299 | 300 | // check if there are new edges added in 301 | // partition one and two 302 | synchronized(countLock) { 303 | totalNewEdgsForIteratn += threadUpdates; 304 | if (i >= indexStartForOne && i <= indexEndForOne) 305 | newEdgesInOne += threadUpdates; 306 | else if (i >= indexStartForTwo && i <= indexEndForTwo) 307 | newEdgesInTwo += threadUpdates; 308 | } 309 | 310 | // set termination status if nNewEdges == 0 311 | // for each vertex 312 | if (threadUpdates == 0) 313 | edgeComputer.setTerminateStatus(true); 314 | edgeComputer.setNumNewEdges(0); 315 | edgeComputer.setNumDupEdges(0); 316 | } 317 | } 318 | 319 | } catch (Exception e) { 320 | e.printStackTrace(); 321 | } finally { 322 | int pending = countDown.decrementAndGet(); 323 | synchronized (termationLock) { 324 | // totalNewEdgsForIteratn += threadUpdates; 325 | // totalDupEdges += dups; 326 | if (pending == 0) { 327 | termationLock.notifyAll(); 328 | } 329 | } 330 | } 331 | } 332 | 333 | }); 334 | } 335 | 336 | synchronized (termationLock) { 337 | while (countDown.get() > 0) { 338 | try { 339 | termationLock.wait(1500); 340 | } catch (InterruptedException e) { 341 | e.printStackTrace(); 342 | } 343 | 344 | // if (countDown.get() > 0) 345 | // logger.info("Waiting for execution to finish: countDown:" 346 | // + countDown.get()); 347 | } 348 | } 349 | 350 | // TODO: PRINTING NEW EDGES (COMMENT OUT LATER) 351 | // for (int i = 0; i < vertices.length; i++) { 352 | // logger.info("Vertex Id#" + vertices[i].getVertexId() + " " 353 | // + Arrays.toString(vertices[i].getOutEdges()) 354 | // + " New Edges: " + edgesLists[i]); 355 | // } 356 | 357 | // TODO: PRINTING VERTEX DEGREES (COMMENT THIS OUT LATER:) 358 | // logger.info("PRINTING DEGREES OF PARTITION AT THE END OF ITERATION"); 359 | // for (int i = 0; i < vertices.length; i++) { 360 | // logger.info(vertices[i].getVertexId() + " | " 361 | // + vertices[i].getNumOutEdges()); 362 | // } 363 | 364 | this.totalNewEdgs += totalNewEdgsForIteratn; 365 | logger.info("========total # new edges for iteration #" 366 | + iterationNo + " is " + totalNewEdgsForIteratn); 367 | // logger.info("========total # dup edges for this iteration: " 368 | // + totalDupEdges); 369 | logger.info("Finshed iteration no. " + iterationNo + " took " + (System.currentTimeMillis() - t) / 1000 + " s"); 370 | } while (totalNewEdgsForIteratn > 0); 371 | 372 | // set new edge added flag for scheduler 373 | if (newEdgesInOne > 0) 374 | intervals.get(0).setIsNewEdgeAdded(true); 375 | if (newEdgesInTwo > 0) 376 | intervals.get(1).setIsNewEdgeAdded(true); 377 | } 378 | 379 | public long get_totalNewEdgs() { 380 | return totalNewEdgs; 381 | } 382 | 383 | /** 384 | * Description: 385 | * 386 | * @param: 387 | * @return: 388 | */ 389 | private void setReadableIndex(NewEdgesList[] edgesList) { 390 | if (edgesList == null || edgesList.length == 0) 391 | return; 392 | 393 | int size; 394 | for (int i = 0; i < edgesList.length; i++) { 395 | NewEdgesList list = edgesList[i]; 396 | if (list == null) 397 | continue; 398 | size = list.getSize(); 399 | if (size == 0) 400 | continue; 401 | list.setReadableSize(size); 402 | list.setReadableIndex(list.getIndex()); 403 | list.setReadableLast(list.getLast()); 404 | } 405 | } 406 | 407 | } --------------------------------------------------------------------------------